Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
cb8cec0
examples/camera_livekit_bridge: add the CUDA IPC client and a probe
gareth-morgan-nv Nov 11, 2026
57aebc3
examples/camera_livekit_bridge: encode the IPC frames and publish the…
gareth-morgan-nv Nov 11, 2026
b9a8e5e
examples/camera_livekit_bridge: shrink a stream on the GPU before enc…
gareth-morgan-nv Nov 11, 2026
a452c44
examples/camera_livekit_bridge: document the bridge
gareth-morgan-nv Nov 11, 2026
4f02b24
examples/camera_livekit_bridge: add tests for the bridge modules and …
gareth-morgan-nv Nov 11, 2026
2a27d5b
examples/camera_livekit_bridge: choose what the frame timestamp means
gareth-morgan-nv Nov 11, 2026
1ba974d
examples/camera_livekit_bridge: stamp frame-ready before encoding, ga…
gareth-morgan-nv Nov 11, 2026
deac99a
examples/camera_livekit_bridge: follow the transport's target bitrate
gareth-morgan-nv Nov 11, 2026
34c7134
examples/camera_livekit_bridge: report capture-to-ready latency, keep…
gareth-morgan-nv Nov 11, 2026
caa3203
examples/camera_livekit_bridge: drop a picture whose IPC slot may hav…
gareth-morgan-nv Nov 11, 2026
61b1ac3
examples/camera_livekit_bridge: signal full range for full-range sour…
gareth-morgan-nv Nov 12, 2026
4c820bf
camera_livekit_bridge: describe the bitrate behaviour as current beha…
gareth-morgan-nv Nov 13, 2026
012423b
camera_livekit_bridge: state the integration boundary
gareth-morgan-nv Nov 13, 2026
bfccaea
examples/camera_livekit_bridge: drop a picture whose copy ran past th…
gareth-morgan-nv Nov 18, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 70 additions & 0 deletions examples/camera_livekit_bridge/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

cmake_minimum_required(VERSION 3.24)
project(camera_livekit_bridge LANGUAGES CXX CUDA)

set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_EXTENSIONS OFF)
if(NOT CMAKE_CUDA_ARCHITECTURES)
set(CMAKE_CUDA_ARCHITECTURES native)
endif()
if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE Release)
endif()

find_package(CUDAToolkit REQUIRED)
find_package(nlohmann_json REQUIRED)
find_package(Threads REQUIRED)

include(FetchContent)

# ==============================================================================
# NVENC headers (ffnvcodec): headers only, loaded dynamically at run time
# ==============================================================================
# Point FETCHCONTENT_SOURCE_DIR_NV_CODEC_HEADERS at an unpacked copy to build offline.
FetchContent_Declare(
nv_codec_headers
URL https://github.com/FFmpeg/nv-codec-headers/archive/refs/tags/n12.2.72.0.tar.gz
URL_HASH SHA256=dbeaec433d93b850714760282f1d0992b1254fc3b5a6cb7d76fc1340a1e47563
SOURCE_SUBDIR no-cmake-here
)
FetchContent_MakeAvailable(nv_codec_headers)
add_library(nv_codec_headers INTERFACE)
target_include_directories(nv_codec_headers SYSTEM INTERFACE "${nv_codec_headers_SOURCE_DIR}/include")

# ==============================================================================
# LiveKit C++ SDK: a prebuilt package (headers and shared libraries)
# ==============================================================================
# Point LIVEKIT_SDK_DIR at an unpacked copy to use another, or to build offline.
if(NOT LIVEKIT_SDK_DIR)
set(LIVEKIT_SDK_UBUNTU "24.04" CACHE STRING "Ubuntu release the prebuilt SDK was built for")
if(LIVEKIT_SDK_UBUNTU STREQUAL "22.04")
set(_livekit_hash "9e568ee6b2bb3c7b0d07faba694930767096a14d0e09422ba01f34077ffb03cb")
else()
set(_livekit_hash "5e0feb2e519c6ff131dd2d1c45149115057ae4e21a324fdb249e1789d5614e7b")
endif()
FetchContent_Declare(
livekit_sdk
URL https://github.com/livekit/client-sdk-cpp/releases/download/v1.12.4/livekit-sdk-ubuntu-${LIVEKIT_SDK_UBUNTU}-x64-1.12.4.tar.gz
URL_HASH SHA256=${_livekit_hash}
)
FetchContent_MakeAvailable(livekit_sdk)
set(LIVEKIT_SDK_DIR "${livekit_sdk_SOURCE_DIR}")
endif()
find_package(LiveKit CONFIG REQUIRED PATHS "${LIVEKIT_SDK_DIR}" NO_DEFAULT_PATH)
message(STATUS "LiveKit SDK=${LIVEKIT_SDK_DIR}")

add_subdirectory(video_types)
add_subdirectory(ipc_client)
add_subdirectory(gpu_ops)
add_subdirectory(nvenc)
add_subdirectory(livekit_publisher)
add_subdirectory(probe)
add_subdirectory(bridge)

include(CTest)
if(BUILD_TESTING)
add_subdirectory(tests)
endif()
127 changes: 127 additions & 0 deletions examples/camera_livekit_bridge/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
<!--
SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
SPDX-License-Identifier: Apache-2.0
-->

# camera_livekit_bridge

A downstream consumer of `camera_streamer`'s CUDA IPC output that publishes the
camera streams to LiveKit as pre-encoded H.264 tracks. It talks to
`camera_receiver` only through its IPC socket and imports nothing from egocap or
the Isaac Capture runtime; egocap joins the LiveKit room on its own.

```text
camera_receiver ──CUDA IPC──> bridge ──H.264──> LiveKit room ──> egocap page / recorder
(GPU NV12) GPU copy or shrink, NVENC
```

Each stream runs on its own thread. A frame is copied (or box-averaged down) on
the GPU from the IPC slot into an NVENC input surface, encoded, and submitted as
a pre-encoded access unit. Pixels never reach host memory. The frame's source
sequence is the track's `frame_id`, the sensor start time (when the receiver
derived one) is the `user_timestamp` in Unix microseconds, and the sensor
counter is the 8-byte `user_data`. The newest picture is encoded and older
queued ones are skipped. The receiver overwrites a slot without notice once its ring wraps, so after the GPU copy the
bridge checks for a newer notice for the same slot, or as many newer notices as the ring has other slots, and drops the
picture instead of encoding it (counted as `slot overruns`). The session also answers clock-sync requests that
arrive as LiveKit data messages.

## Integration boundary

This example publishes receiver pictures to a browser through LiveKit. The LiveKit session, publication and feedback
wiring (`livekit_publisher/` and the session handling in `bridge/`) is temporary integration. It is intended to move
behind transport adapters: a layer that carries camera video and application messages over a selectable network
transport and owns sessions, admission limits and feedback. Camera buffer ownership and encode and decode
responsibilities belong to the camera interface. A replacement must preserve browser delivery, bounded latency and
safe GPU access. Until receiver slots have read leases, the residual risk described under "Reading the receiver's
slots" applies.

| Current path | Intended owner or replacement | Reuse, adapt or retire | Follow-up and landing condition |
| --- | --- | --- | --- |
| `livekit_publisher/` and the session, publication and feedback wiring in `bridge/` | Transport adapters | Adapt the useful implementation; retire the hard-coded transport glue | Map sessions, publications, encoder feedback and reconnect behaviour to the adapter boundary. Lands when real web clients receive streams, and feedback and recovery work, through an adapter. |
| `ipc_client/` and the slot read in `bridge/stream_worker.cpp` | Camera reader views and leases | Retire | Read frames through a lease held until the final GPU read. Lands with the receiver's lease support. |

## Build

```bash
cmake -S . -B build && cmake --build build -j
```

Needs the CUDA toolkit and nlohmann_json. The NVENC headers and the LiveKit C++
SDK are fetched at configure time; pass `-DLIVEKIT_SDK_DIR=<unpacked SDK>` and
`-DFETCHCONTENT_SOURCE_DIR_NV_CODEC_HEADERS=<unpacked headers>` to build
offline. The build is standalone and is not part of the repository's top-level
build.

## Run

With `camera_receiver` running (for example fed by `camera_streamer/mock_sender`)
and a LiveKit server reachable:

```bash
build/bridge/camera_livekit_bridge \
--livekit-url ws://127.0.0.1:7880 --livekit-token-file publisher-token.txt \
--socket /tmp/camera_receiver_avs1_cuda_ipc.sock \
--stream head=sensing-head:2 --stream wrist1=sensing-wrist1 \
--fps 30 --bitrate-mbps 8
```

`--stream NAME=TRACK[:DIV]` publishes the receiver's stream `NAME` as the
LiveKit track `TRACK`, shrunk by the integer `DIV` on the GPU (the size must
divide to even dimensions). Track names, the join token's identity and the
published sizes must match the streams file egocap is given. The token needs
publish permission. Every 5 s it prints each track's frame rate, keyframes,
skipped, rejected and slot-overrun frames and mean encode time.

`--timestamp-basis` sets what each frame's `user_timestamp` means, and the consumer's streams file must name
the same basis:

| Basis | Value |
| --- | --- |
| `capture_unix_us` (default) | The sensor start-of-frame as Unix time on the sender's clock. |
| `monotonic_us` | This host's monotonic clock at the instant the picture is ready to encode (before encoding): frame-ready time, later than the sensor capture. |

`capture_unix_us` is used only when the receiver reports a realtime sender clock; a frame with no usable sensor
start-of-frame (missing, in the future, older than 5 s, or on another clock domain) carries no `user_timestamp`, so
a consumer that needs a capture time says so instead of using a guess. `capture_monotonic_us` is refused: it would
need a measured sender-to-host clock mapping with a stated uncertainty, which does not exist yet.

`--source-full-range` says the receiver's pictures use the full 0-255 range, as SIPL sources do; the encoded stream
then signals full range, so players and recordings keep the black and white levels. Without it the stream signals
limited range (16-235). The receiver's IPC catalog does not carry the range, so it is set here and must match the
source: a full-range picture published as limited range has crushed blacks and clipped whites, and the reverse
looks washed out. The encoder copies levels as they are and does no range conversion. The stream signals no colour
primaries, transfer or matrix, so a player assumes its default (BT.709 for HD).

`--bitrate-mbps` is the maximum. The encoder starts at 2 Mbps and follows the target bitrate that LiveKit's
congestion control reports for each track (90% of the target, never above the maximum or below 0.5 Mbps), so a
fresh connection does not fill the sender's pacing queue. The `[stats]` line shows the current rate, the transport's
target and how many times the rate changed.

Measured with the mock sender on an RTX PRO 5000 against egocap's receiver and recording-feed checks (30 fps, head
and one wrist, local LiveKit server): frames are 60-150 ms old from the first frame and settle near 80 ms while the
rate climbs to the maximum within roughly 20 s. The head at 2560x1984 and 8 Mbps and at 1280x992 both ran with no
missing frames and a median capture-to-receive time of 65-85 ms. A 20 Mbps maximum on the head built a standing
backlog of about 1.2 s, so keep the maximum near 8 Mbps.

## Probe

```bash
build/probe/camera_ipc_probe --socket /tmp/camera_receiver_avs1_cuda_ipc.sock \
--stream head --frames 30
```

It prints each stream's plane layout, then per frame the slot, generation,
stream sequence, session id, sensor start time and a checksum of the first
pixels. Slots are overwritten without notice when the publisher's ring wraps.

## Reading the receiver's slots

The receiver rewrites its GPU ring slots in rotation and never waits for a reader: a slot is overwritten without
notice about `ring_depth` frames after it was announced, and the only signal is a later frame notice. A consumer
therefore has to finish with a slot within that window or skip ahead. The bridge follows this: it uses only the
newest notice, starts the GPU copy at once on its own stream, waits for it to complete, and drops (never encodes)
the picture when later notices show the ring may have wrapped; those frames count as slot overruns in the `[stats]`
line. The notices arrive asynchronously, so this check cannot prove a copy was clean: a stall of most of the ring's
window between the notice and the end of the copy can still publish a picture that mixes two frames under the earlier
frame's metadata. Only a read guard in the receiver, which skips slots a reader holds, would remove that case.
20 changes: 20 additions & 0 deletions examples/camera_livekit_bridge/bridge/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

add_executable(camera_livekit_bridge
main.cpp
stream_worker.cpp
)

target_link_libraries(camera_livekit_bridge
PRIVATE
camera_livekit_bridge::ipc_client
camera_livekit_bridge::gpu_ops
camera_livekit_bridge::nvenc
camera_livekit_bridge::livekit_publisher
camera_livekit_bridge::video_types
CUDA::cudart
Threads::Threads
)

target_compile_options(camera_livekit_bridge PRIVATE -Wall -Wextra)
Loading
Loading