feat(observability): supervisor OTLP telemetry relay

Cargo.toml

@@ -66,6 +66,7 @@ tracing-appender = "0.2"
 opentelemetry = "0.32"
 opentelemetry_sdk = { version = "0.32", features = ["rt-tokio"] }
 opentelemetry-otlp = { version = "0.32", default-features = false, features = ["grpc-tonic", "trace"] }
+opentelemetry-proto = { version = "0.32", default-features = false, features = ["gen-tonic", "trace", "with-serde"] }
 tracing-opentelemetry = { version = "0.33", default-features = false, features = ["tracing-log"] }
 
 # Metrics

architecture/sandbox.md

@@ -462,6 +462,84 @@ Sandbox logs are emitted locally and can also be pushed back to the gateway.
 Security-relevant sandbox behavior uses OCSF structured events; internal
 diagnostics use ordinary tracing.
 
+## Telemetry Relay
+
+The supervisor can relay OpenTelemetry trace data and OCSF events from agent
+processes to the gateway over the session protocol. This gives OTel-instrumented
+agents (LangChain, CrewAI, etc.) a zero-configuration path to an external
+collector without requiring direct egress from the sandbox.
+
+### Data Flow
+
+```
+Agent process --> OTLP HTTP (127.0.0.1:4318) --> Supervisor receiver
+  --> Enrichment (sandbox resource attributes)
+  --> Bounded buffer (4096 slots, shared traces + OCSF)
+  --> Forwarder --> Session channel (OtelExportData message)
+  --> Gateway --> Dedicated SpanExporter --> External OTLP collector
+```
+
+### Receiver Binding
+
+The OTLP HTTP receiver binds to `127.0.0.1:4318` only when the relay is
+active (the gateway has `[openshell.gateway.otlp]` configured and confirms
+the `otel_export` capability). When OTLP is not configured, no port is
+bound and no receiver runs.
+
+The bind address depends on the supervisor topology. In all current
+topologies, the process supervisor runs co-located with the agent workload,
+so `127.0.0.1` is reachable from the agent. For Docker/Podman drivers
+where the supervisor creates a network namespace, the bind happens inside
+the namespace via `bind_tcp_in_netns()`. For Kubernetes and VM drivers,
+the supervisor and agent share the same network namespace, so a direct
+bind suffices. The receiver accepts both `application/x-protobuf` and
+`application/json` content types.
+
+Future topologies where the process supervisor moves out of the workload
+pod would require the bind address and the `OTEL_EXPORTER_OTLP_ENDPOINT`
+env var to reflect the supervisor's reachable address from the agent's
+perspective (e.g., a service IP or pod IP).
+
+### Span Enrichment
+
+Forwarded spans are enriched with sandbox resource attributes:
+`openshell.sandbox.id`, `openshell.workspace.id`, `openshell.sandbox.policy`,
+`openshell.sandbox.user`, `openshell.sandbox.image`, `openshell.sandbox.driver`.
+The `openshell.telemetry.source` attribute (fixed value `"agent"`) is always
+injected regardless of the enrichment toggle so collectors can filter agent
+spans from gateway infrastructure spans. Enrichment can be disabled for
+pass-through forwarding.
+
+### Activation and Capability Negotiation
+
+The relay is opt-in. It starts only when the gateway has `[openshell.gateway.otlp]`
+configured. The supervisor advertises `"otel_export"` in
+`SupervisorHello.capabilities`. The gateway confirms via `SessionAccepted.capabilities`.
+The supervisor gates `OtelExportData` sending on this confirmation. When the
+relay is active, the supervisor sets `OTEL_EXPORTER_OTLP_ENDPOINT` and
+`OTEL_EXPORTER_OTLP_PROTOCOL` in agent child processes via `child_env.rs`.
+
+### Non-Interference
+
+The buffer uses `try_send` (non-blocking) on the session channel. When the
+buffer reaches capacity, the oldest entries are dropped and a counter records
+each drop. A queue depth gauge tracks buffer pressure. This ensures telemetry
+cannot block or degrade sandbox control operations.
+
+### OCSF Event Relay
+
+OCSF events generated inside the sandbox (e.g., network deny events) can also
+be forwarded through the same transport. A per-sandbox token bucket rate limiter
+controls the OCSF event rate, with configurable rate and drop counter.
+
+### Gateway-Side Handling
+
+The gateway receives `OtelExportData` messages and exports trace data through a
+dedicated `OtelRelayExporter` that connects directly to the configured OTLP
+collector. This bypasses the gateway's own `SdkTracerProvider` to preserve the
+supervisor-enriched resource attributes. OCSF events are emitted via
+`tracing::info!` on the `ocsf_relay` target.
+
 ## Policy Proposals
 
 When an L4 CONNECT is denied, the proxy emits a `DenialEvent`. The denial

crates/openshell-core/src/sandbox_env.rs

@@ -235,6 +235,29 @@ pub const SANDBOX_GID: &str = "OPENSHELL_SANDBOX_GID";
 /// OCI only for the former contract.
 pub const OCI_IMAGE_USER: &str = "OPENSHELL_OCI_IMAGE_USER";
 
+/// Standard OpenTelemetry environment variable for the OTLP exporter endpoint.
+///
+/// Set conditionally by the telemetry relay when the gateway has an OTLP
+/// endpoint configured. Points agent SDKs at the supervisor's local OTLP
+/// HTTP receiver.
+pub const OTEL_EXPORTER_OTLP_ENDPOINT: &str = "OTEL_EXPORTER_OTLP_ENDPOINT";
+
+/// Standard OpenTelemetry environment variable for the OTLP exporter protocol.
+///
+/// Set to `http/protobuf` when the telemetry relay is active.
+pub const OTEL_EXPORTER_OTLP_PROTOCOL: &str = "OTEL_EXPORTER_OTLP_PROTOCOL";
+
+/// Default OTLP receiver bind address and port.
+///
+/// All current topologies keep the process supervisor co-located with the
+/// agent, so localhost is correct. Future topologies that move the supervisor
+/// out of the agent's network namespace would derive the address from the
+/// topology (e.g., pod IP via downward API).
+pub const OTLP_RECEIVER_ADDR: &str = "127.0.0.1:4318";
+
+/// Default OTLP receiver endpoint URL for agent env var injection.
+pub const OTLP_RECEIVER_ENDPOINT: &str = "http://127.0.0.1:4318";
+
 // The corporate upstream-proxy configuration deliberately has no reserved
 // environment variables: it travels on the supervisor's argv
 // (`--upstream-proxy` and friends), which a sandbox image cannot forge the

crates/openshell-driver-kubernetes/src/driver.rs

@@ -4341,6 +4341,16 @@ fn build_env_list(
         tls_enabled,
         provider_spiffe_socket_path,
     );
+    upsert_env(
+        &mut env,
+        openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_ENDPOINT,
+        openshell_core::sandbox_env::OTLP_RECEIVER_ENDPOINT,
+    );
+    upsert_env(
+        &mut env,
+        openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_PROTOCOL,
+        "http/protobuf",
+    );
     env
 }
 

crates/openshell-ocsf/src/lib.rs

@@ -64,5 +64,6 @@ pub use builders::{
 
 // --- Tracing layers ---
 pub use tracing_layers::{
-    OCSF_TARGET, OcsfJsonlLayer, OcsfShorthandLayer, clone_current_event, emit_ocsf_event,
+    OCSF_TARGET, OcsfJsonlLayer, OcsfRelayLayer, OcsfRelaySink, OcsfShorthandLayer,
+    clone_current_event, emit_ocsf_event,
 };

crates/openshell-ocsf/src/tracing_layers/mod.rs

@@ -9,8 +9,10 @@
 
 pub(crate) mod event_bridge;
 mod jsonl_layer;
+mod relay_layer;
 mod shorthand_layer;
 
 pub use event_bridge::{OCSF_TARGET, clone_current_event, emit_ocsf_event};
 pub use jsonl_layer::OcsfJsonlLayer;
+pub use relay_layer::{OcsfRelayLayer, OcsfRelaySink};
 pub use shorthand_layer::OcsfShorthandLayer;

crates/openshell-ocsf/src/tracing_layers/relay_layer.rs

@@ -0,0 +1,46 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Tracing layer that captures OCSF events and forwards them as JSON bytes
+//! through a telemetry buffer sender for relay to the gateway.
+
+use std::sync::Arc;
+
+use tracing::Subscriber;
+use tracing_subscriber::Layer;
+use tracing_subscriber::layer::Context;
+
+use super::event_bridge::{OCSF_TARGET, clone_current_event};
+
+/// Callback trait for delivering serialized OCSF events.
+pub trait OcsfRelaySink: Send + Sync + 'static {
+    fn send(&self, json_bytes: Vec<u8>);
+}
+
+/// A tracing layer that captures OCSF events and serializes them to JSON
+/// for relay through the telemetry transport.
+pub struct OcsfRelayLayer {
+    sink: Arc<dyn OcsfRelaySink>,
+}
+
+impl OcsfRelayLayer {
+    pub fn new(sink: Arc<dyn OcsfRelaySink>) -> Self {
+        Self { sink }
+    }
+}
+
+impl<S: Subscriber> Layer<S> for OcsfRelayLayer {
+    fn on_event(&self, event: &tracing::Event<'_>, _ctx: Context<'_, S>) {
+        if event.metadata().target() != OCSF_TARGET {
+            return;
+        }
+
+        let Some(ocsf_event) = clone_current_event() else {
+            return;
+        };
+
+        if let Ok(json) = serde_json::to_vec(&ocsf_event) {
+            self.sink.send(json);
+        }
+    }
+}

crates/openshell-otel/src/lib.rs

@@ -5,7 +5,7 @@
 
 mod driver;
 mod grpc;
-mod propagation;
+pub mod propagation;
 
 pub use driver::{
     BoxGrpcStream, ComputeDriverTracing, DriverTracingConfig, DriverTracingHandle,

crates/openshell-otel/src/propagation.rs

@@ -54,6 +54,22 @@ impl Injector for MetadataMapInjector<'_> {
     }
 }
 
+/// Writes OpenTelemetry propagation fields to HTTP headers.
+#[derive(Debug)]
+pub struct HeaderMapInjector<'a>(pub &'a mut HeaderMap);
+
+impl Injector for HeaderMapInjector<'_> {
+    fn set(&mut self, key: &str, value: String) {
+        let Ok(key) = key.parse::<http::header::HeaderName>() else {
+            return;
+        };
+        let Ok(value) = value.parse() else {
+            return;
+        };
+        self.0.insert(key, value);
+    }
+}
+
 #[derive(Debug)]
 struct TraceContextMapInjector<'a>(&'a mut BTreeMap<String, String>);
 
@@ -75,6 +91,15 @@ pub fn current_trace_context_carrier() -> Option<BTreeMap<String, String>> {
     carrier.contains_key("traceparent").then_some(carrier)
 }
 
+/// Inject W3C traceparent into HTTP headers if not already present.
+pub fn inject_traceparent_if_missing(headers: &mut HeaderMap) {
+    if headers.contains_key("traceparent") {
+        return;
+    }
+    let context = tracing::Span::current().context();
+    TraceContextPropagator::new().inject_context(&context, &mut HeaderMapInjector(headers));
+}
+
 /// Injects the active W3C trace context into an outbound tonic request.
 #[derive(Debug, Clone, Copy)]
 pub struct TraceContextInterceptor;

crates/openshell-sandbox/src/lib.rs

@@ -852,6 +852,9 @@ pub async fn run_sandbox(
     };
     tokio::pin!(proxy_exited);
 
+    #[cfg_attr(not(target_os = "linux"), allow(unused_mut))]
+    let mut otel_relay_handle: Option<openshell_supervisor_network::otlp::RelayHandle> = None;
+
     let exit_code = if process_enabled {
         let ca_file_paths = networking
             .as_ref()
@@ -952,6 +955,71 @@ pub async fn run_sandbox(
             None
         };
 
+        // OTEL relay: bind OTLP receiver for all Linux topologies.
+        // All current topologies keep the process supervisor co-located with
+        // the agent, so 127.0.0.1 is reachable from agent processes. Future
+        // topologies that move the supervisor out of the workload pod would
+        // need to derive the bind address from the topology (e.g., pod IP
+        // via downward API) and update OTEL_EXPORTER_OTLP_ENDPOINT to match.
+        let otel_rx = {
+            #[cfg(target_os = "linux")]
+            {
+                let otlp_addr = openshell_core::sandbox_env::OTLP_RECEIVER_ADDR;
+
+                let (otel_session_tx, otel_session_rx) =
+                    tokio::sync::mpsc::channel::<openshell_core::proto::SupervisorMessage>(64);
+
+                let relay_config = openshell_supervisor_network::otlp::RelayConfig::default();
+                let metadata = openshell_supervisor_network::otlp::SandboxMetadata {
+                    sandbox_id: sandbox_id.clone().unwrap_or_default(),
+                    workspace_id: workspace_rx.borrow().clone(),
+                    policy: sandbox_name_for_agg.clone().unwrap_or_default(),
+                    user: resolved_process_identity
+                        .uid()
+                        .map_or_else(String::new, |uid| uid.to_string()),
+                    image: std::env::var("OPENSHELL_CONTAINER_IMAGE").unwrap_or_default(),
+                    driver: std::env::var(openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY)
+                        .unwrap_or_else(|_| "container".to_string()),
+                };
+                let relay = openshell_supervisor_network::otlp::OtelRelay::new(
+                    relay_config,
+                    metadata,
+                    otel_session_tx,
+                );
+
+                let bind_addr: std::net::SocketAddr = otlp_addr.parse().unwrap();
+
+                if let Some(ns) = netns.as_ref() {
+                    match ns.bind_tcp_in_netns(otlp_addr).await {
+                        Ok(listener) => {
+                            let handle = relay.start_with_listener(listener);
+                            tracing::info!(bind = %bind_addr, "OTEL relay started (netns)");
+                            otel_relay_handle = Some(handle);
+                        }
+                        Err(e) => {
+                            tracing::warn!(error = %e, "OTEL relay failed to bind in netns; continuing without relay");
+                        }
+                    }
+                } else {
+                    match relay.start(bind_addr).await {
+                        Ok(handle) => {
+                            tracing::info!(bind = %bind_addr, "OTEL relay started");
+                            otel_relay_handle = Some(handle);
+                        }
+                        Err(e) => {
+                            tracing::warn!(error = %e, "OTEL relay failed to start; continuing without relay");
+                        }
+                    }
+                }
+                Some(otel_session_rx)
+            }
+            #[cfg(not(target_os = "linux"))]
+            {
+                debug!("OTEL relay not available on this platform");
+                None
+            }
+        };
+
         let process = openshell_supervisor_process::run::run_process(
             program,
             args,
@@ -980,6 +1048,7 @@ pub async fn run_sandbox(
             bypass_denial_tx,
             #[cfg(target_os = "linux")]
             bypass_activity_tx,
+            otel_rx,
         );
 
         if let Some(control_closed) = process_control_closed.as_mut() {
@@ -1139,6 +1208,12 @@ pub async fn run_sandbox(
         }
     };
 
+    // Drain OTEL relay before tearing down networking so short-lived
+    // agents don't lose their final spans.
+    if let Some(handle) = otel_relay_handle {
+        handle.shutdown().await;
+    }
+
     // Drop networking explicitly so the proxy + bypass monitor RAII
     // handles tear down before we return.
     drop(networking);
@@ -1456,6 +1531,7 @@ fn spawn_sidecar_entrypoint_handler(
                     Some(supervisor_pid),
                     Arc::clone(&terminating),
                     started.instance_id.clone(),
+                    None,
                 ));
                 session_started = true;
                 info!("sidecar supervisor session task spawned");

crates/openshell-server/Cargo.toml

@@ -73,6 +73,7 @@ tracing-subscriber = { workspace = true }
 # OpenTelemetry (OTLP trace export, opt-in via [openshell.gateway.otlp])
 opentelemetry = { workspace = true }
 opentelemetry_sdk = { workspace = true }
+opentelemetry-proto = { workspace = true }
 tracing-opentelemetry = { workspace = true }
 
 # Metrics