diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 33d71d867c..9349886d72 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -25,6 +25,7 @@ use crate::l7::rest::WebSocketExtensionMode; use crate::l7::{EndpointObserver, EnforcementMode, L7EndpointConfig, L7Protocol, L7RequestInfo}; use crate::opa::{PolicyGenerationGuard, TunnelPolicyEngine}; use crate::telemetry::egress_emit; +use futures::FutureExt; use miette::{IntoDiagnostic, Result, miette}; use openshell_core::activity::{ActivitySender, try_record_activity}; use openshell_core::endpoint_status::{EndpointObservationSender, EndpointResult}; @@ -37,13 +38,15 @@ use openshell_ocsf::{ use std::collections::BTreeMap; use std::collections::HashSet; use std::sync::Arc; -use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use tracing::{debug, warn}; #[cfg(test)] pub(crate) mod correlation_dispatch_tests; #[cfg(test)] mod token_grant_ownership_tests; +#[cfg(test)] +mod upstream_close_tests; const CONNECTION_READ_AHEAD_BYTES: usize = 8 * 1024; @@ -888,6 +891,7 @@ where client, upstream, crate::l7::rest::RelayRequestOptions { + request_id: Some(request_id), resolver: ctx.secret_resolver.as_deref(), body_classifier: ctx.body_classifier.as_deref(), mcp_request_validation: (config.protocol == L7Protocol::Mcp).then_some( @@ -956,6 +960,62 @@ where .await } +fn build_upstream_http_failure_event( + ctx: &L7EvalContext, + method: &str, + request_id: Option<&str>, + response_code: Option, +) -> openshell_ocsf::OcsfEvent { + let mut event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::for_http_method(method)) + .http_request(method_only_request(method)) + .action(ActionId::Allowed) + .disposition(DispositionId::Allowed) + .severity(SeverityId::Low) + .status(StatusId::Failure) + .dst_endpoint(Endpoint::from_domain(&ctx.host, ctx.port)) + .message("HTTP upstream connection closed or failed") + .status_detail("upstream_http_transport_failure"); + if let Some(request_id) = request_id { + event = event.unmapped("request_id", request_id); + } + if let Some(code) = response_code { + event = event.http_response(openshell_ocsf::HttpResponse { code }); + } + event.build() +} + +pub(crate) async fn handle_upstream_http_error( + client: &mut C, + upstream: &mut U, + ctx: &L7EvalContext, + method: &str, + request_id: Option<&str>, + error: &crate::l7::rest::UpstreamHttpError, +) -> Result<()> +where + C: AsyncWrite + Unpin, + U: AsyncWrite + Unpin, +{ + debug!(error = ?error, "HTTP upstream transport failure"); + let _ = upstream.shutdown().await; + // An unanswered request may already have executed. Never replay it or + // append a new status after the client stream accepted response bytes. + let delivery = if error.response_started { + Ok(()) + } else { + write_bad_gateway_response(client, request_id).await + }; + let response_code = (!error.response_started && delivery.is_ok()).then_some(502); + egress_emit!(build_upstream_http_failure_event( + ctx, + method, + request_id, + response_code + )); + delivery +} + #[derive(Clone, Copy, Default)] struct RelayHttpInstrumentation<'a> { observer: Option<&'a EndpointObserver>, @@ -1011,20 +1071,44 @@ where observer, correlation, } = instrumentation; - match Box::pin( - crate::l7::rest::relay_http_request_with_body_guarded_observed_correlated( - request, - client, - upstream, - options, - live_body, - response_middleware, - observer, - correlation, - ), - ) - .await + // The ready request has passed policy evaluation, but data sent before + // forwarding it cannot be treated as its response. + let mut byte = [0_u8; 1]; + let probe_error = match upstream + .read(&mut byte) + .now_or_never() + .map(IntoDiagnostic::into_diagnostic) { + Some(Ok(1..)) => Some(miette!( + "HTTP upstream sent unsolicited data before the request" + )), + Some(Err(error)) => Some(error), + _ => None, + }; + let result = if let Some(source) = probe_error { + if let Some(observer) = observer { + observer.observe(EndpointResult::TransportFailed); + } + Err(miette::Report::new(crate::l7::rest::UpstreamHttpError { + response_started: false, + source, + })) + } else { + Box::pin( + crate::l7::rest::relay_http_request_with_body_guarded_observed_correlated( + request, + client, + upstream, + options, + live_body, + response_middleware, + observer, + correlation, + ), + ) + .await + }; + match result { Ok(outcome) => Ok(Some(outcome)), Err(report) => { if let Some(failure) = report.downcast_ref::() { @@ -1033,7 +1117,30 @@ where .await?; return Ok(None); } - if let Some(error) = report.downcast_ref::() { + if let Some(error) = report.downcast_ref::() { + if let Some(correlation) = correlation { + correlation.record_failure( + HttpOutcome::UpstreamError, + "upstream_http_transport_failure", + ); + } + handle_upstream_http_error( + client, + upstream, + ctx, + &request.action, + options.request_id, + error, + ) + .await?; + if !error.response_started + && let Some(correlation) = correlation + { + correlation.record_response_status(502); + } + Ok(Some(RelayOutcome::Consumed)) + } else if let Some(error) = report.downcast_ref::() + { // Body classification includes credentials for other destinations, // so denial does not establish that this endpoint lacks credentials. // Record the local decision before cleanup or client I/O can fail. @@ -1540,25 +1647,26 @@ where .endpoint_observation_tx .as_ref() .and_then(EndpointObservationSender::capture); - let mut req = match provider.parse_request(client).await { - Ok(Some(req)) => req, - Ok(None) => return Ok(()), - Err(e) => { - if is_benign_connection_error(&e) { - debug!( - host = %ctx.host, - port = ctx.port, - error = %e, - "L7 route-selected connection closed" - ); - } else { - let detail = - parse_rejection_detail(&e.to_string(), ParseRejectionMode::L7Endpoint); - emit_parse_rejection(ctx, &detail, "l7"); + let mut req = + match parse_next_http_request(provider.parse_request(client), upstream, ctx).await { + Ok(Some(req)) => req, + Ok(None) => return Ok(()), + Err(e) => { + if is_benign_connection_error(&e) { + debug!( + host = %ctx.host, + port = ctx.port, + error = %e, + "L7 route-selected connection closed" + ); + } else { + let detail = + parse_rejection_detail(&e.to_string(), ParseRejectionMode::L7Endpoint); + emit_parse_rejection(ctx, &detail, "l7"); + } + return Ok(()); } - return Ok(()); - } - }; + }; let correlation = RequestCorrelation::new(&req, ctx); let capture_error = |error| correlation.capture_error(error); if !request_authority_matches_endpoint(&req, ctx) { @@ -2563,9 +2671,10 @@ where .filter(|_| ctx.token_grant_resolver.is_some()) .and_then(EndpointObservationSender::capture); // Parse one HTTP request from client - let req = match provider.parse_request(client).await { + let req = match parse_next_http_request(provider.parse_request(client), upstream, ctx).await + { Ok(Some(req)) => req, - Ok(None) => return Ok(()), // Client closed connection + Ok(None) => return Ok(()), // The HTTP connection is no longer reusable. Err(e) => { if is_benign_connection_error(&e) { debug!( @@ -3106,14 +3215,18 @@ where .endpoint_observation_tx .as_ref() .and_then(EndpointObservationSender::capture); - let parsed = match crate::l7::jsonrpc::parse_jsonrpc_http_request( - client, - config.json_rpc_max_body_bytes, - crate::l7::path::CanonicalizeOptions { - allow_encoded_slash: config.allow_encoded_slash, - ..Default::default() - }, - crate::l7::jsonrpc::JsonRpcInspectionOptions::for_config(config), + let parsed = match parse_next_http_request( + Box::pin(crate::l7::jsonrpc::parse_jsonrpc_http_request( + client, + config.json_rpc_max_body_bytes, + crate::l7::path::CanonicalizeOptions { + allow_encoded_slash: config.allow_encoded_slash, + ..Default::default() + }, + crate::l7::jsonrpc::JsonRpcInspectionOptions::for_config(config), + )), + upstream, + ctx, ) .await { @@ -3491,25 +3604,26 @@ where // Validate the head, including body framing, before deciding whether // this endpoint can inspect the requested protocol. Upgrade refusal // must not wait for a body or depend on its inspection size limit. - let mut req = match provider.parse_request(client).await { - Ok(Some(req)) => req, - Ok(None) => return Ok(()), - Err(e) => { - if is_benign_connection_error(&e) { - debug!( - host = %ctx.host, - port = ctx.port, - error = %e, - "GraphQL L7 connection closed" - ); - } else { - let detail = - parse_rejection_detail(&e.to_string(), ParseRejectionMode::L7Endpoint); - emit_parse_rejection(ctx, &detail, "l7-graphql"); + let mut req = + match parse_next_http_request(provider.parse_request(client), upstream, ctx).await { + Ok(Some(req)) => req, + Ok(None) => return Ok(()), + Err(e) => { + if is_benign_connection_error(&e) { + debug!( + host = %ctx.host, + port = ctx.port, + error = %e, + "GraphQL L7 connection closed" + ); + } else { + let detail = + parse_rejection_detail(&e.to_string(), ParseRejectionMode::L7Endpoint); + emit_parse_rejection(ctx, &detail, "l7-graphql"); + } + return Ok(()); } - return Ok(()); - } - }; + }; let correlation = RequestCorrelation::new(&req, ctx); let capture_error = |error| correlation.capture_error(error); @@ -4350,6 +4464,62 @@ fn evaluate_l7_request_once( Ok((allowed, reason)) } +/// Read a request while watching the otherwise idle upstream connection. +/// +/// A closed upstream must not leave a live socket in the client's pool. If +/// both sides are ready, finish the request first so a forwarding failure can +/// produce an explicit HTTP error. If the upstream closes during partial +/// headers, close without a reply: no complete request is available to answer. +/// Ready requests are checked for unsolicited upstream data before forwarding. +async fn parse_next_http_request( + request: F, + upstream: &mut U, + ctx: &L7EvalContext, +) -> Result> +where + F: Future>>, + U: AsyncRead + Unpin, +{ + let mut byte = [0u8; 1]; + tokio::select! { + biased; + request = request => request, + result = upstream.read(&mut byte) => { + emit_idle_upstream_close(ctx, result); + Ok(None) + } + } +} + +fn emit_idle_upstream_close(ctx: &L7EvalContext, result: std::io::Result) { + let (detail, status, severity) = match result { + Ok(0) => { + debug!(host = %ctx.host, port = ctx.port, "HTTP upstream closed while idle"); + return; + } + Ok(_) => ( + "HTTP upstream sent unsolicited data while idle", + StatusId::Failure, + SeverityId::Low, + ), + Err(_) => ( + "HTTP upstream failed while idle", + StatusId::Failure, + SeverityId::Low, + ), + }; + let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Close) + .action(ActionId::Allowed) + .disposition(DispositionId::Allowed) + .severity(severity) + .status(status) + .dst_endpoint(Endpoint::from_domain(&ctx.host, ctx.port)) + .message(detail) + .build(); + egress_emit!(event); +} + /// Relay HTTP traffic with credential injection only (no L7 OPA evaluation). /// /// Used when TLS is auto-terminated but no L7 policy (`protocol` + `access`/`rules`) @@ -4412,14 +4582,16 @@ where Some(Ok((chain, _))) if !chain.is_empty() ); // Read next request from client. - let parsed = if inspected { - provider.parse_request(client).await - } else { - provider.parse_uninspected_request(client).await + let parsed = async { + if inspected { + provider.parse_request(client).await + } else { + provider.parse_uninspected_request(client).await + } }; - let req = match parsed { + let req = match parse_next_http_request(parsed, upstream, ctx).await { Ok(Some(req)) => req, - Ok(None) => break, // Client closed connection. + Ok(None) => break, // The HTTP connection is no longer reusable. Err(e) => { if is_benign_connection_error(&e) { break; @@ -4639,6 +4811,7 @@ where client, upstream, crate::l7::rest::RelayRequestOptions { + request_id: Some(&request_id), resolver, credential_generation: credential_generation_guard(ctx), generation_guard: Some(generation_guard), diff --git a/crates/openshell-supervisor-network/src/l7/relay/upstream_close_tests.rs b/crates/openshell-supervisor-network/src/l7/relay/upstream_close_tests.rs new file mode 100644 index 0000000000..6e085bf59d --- /dev/null +++ b/crates/openshell-supervisor-network/src/l7/relay/upstream_close_tests.rs @@ -0,0 +1,441 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::*; +use crate::opa::{NetworkInput, OpaEngine}; +use std::path::PathBuf; +use std::time::Duration; +use tokio::net::{TcpListener, TcpStream}; +use tokio::time::timeout; + +const REQUEST: &[u8] = + b"POST /mcp HTTP/1.1\r\nHost: api.example.test\r\nContent-Length: 2\r\n\r\n{}"; +const RESPONSE: &[u8] = b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nOK"; + +fn context() -> L7EvalContext { + L7EvalContext { + host: "api.example.test".into(), + port: 80, + request_default_port: Some(80), + policy_name: "rest_api".into(), + binary_path: "/usr/bin/curl".into(), + ..Default::default() + } +} + +fn without_request_id(bytes: &[u8]) -> Vec { + let Some(end) = bytes.windows(4).position(|part| part == b"\r\n\r\n") else { + return bytes.to_vec(); + }; + let (head, body) = bytes.split_at(end + 4); + let mut result = Vec::new(); + for line in head.split_inclusive(|byte| *byte == b'\n') { + if line.starts_with(b"x-openshell-request-id:") { + let id = std::str::from_utf8(line) + .unwrap() + .split_once(':') + .unwrap() + .1 + .trim(); + uuid::Uuid::parse_str(id).unwrap(); + } else { + result.extend_from_slice(line); + } + } + result.extend_from_slice(body); + result +} + +async fn read_exact_message(stream: &mut S, expected: &[u8]) { + let mut bytes = Vec::new(); + while !bytes.ends_with(b"\r\n\r\n") { + bytes.push(stream.read_u8().await.unwrap()); + } + let expected_head_end = expected + .windows(4) + .position(|part| part == b"\r\n\r\n") + .unwrap() + + 4; + let mut body = vec![0; expected.len() - expected_head_end]; + stream.read_exact(&mut body).await.unwrap(); + bytes.extend_from_slice(&body); + assert_eq!(without_request_id(&bytes), expected); +} + +async fn tcp_pair() -> (TcpStream, TcpStream) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let client = + openshell_core::net::connect_tcp_nodelay_best_effort(listener.local_addr().unwrap()) + .await + .unwrap(); + let (server, _) = listener.accept().await.unwrap(); + openshell_core::net::set_tcp_nodelay_best_effort(&client); + openshell_core::net::set_tcp_nodelay_best_effort(&server); + (client, server) +} + +async fn relay_fixture(protocol: &str, mut client: C, mut upstream: U) -> Result<()> +where + C: AsyncRead + AsyncWrite + Unpin + Send, + U: AsyncRead + AsyncWrite + Unpin + Send, +{ + let ctx = context(); + let policy_protocol = if protocol == "passthrough" { + "rest" + } else { + protocol + }; + let rule = if policy_protocol == "rest" { + serde_json::json!({"method":"POST", "path":"/mcp"}) + } else { + serde_json::json!({"method":"initialize"}) + }; + let data = serde_json::json!({"network_policies": {"rest_api": { + "name":"rest_api", "endpoints":[{"host":ctx.host, "port":ctx.port, + "path":"/mcp", "protocol":policy_protocol, "enforcement":"enforce", "rules":[{"allow":rule}]}], + "binaries":[{"path":ctx.binary_path}] + }}}); + let engine = OpaEngine::from_strings( + include_str!("../../../data/sandbox-policy.rego"), + &data.to_string(), + ) + .unwrap(); + if protocol == "passthrough" { + let guard = engine + .generation_guard(engine.current_generation()) + .unwrap(); + relay_passthrough_with_credentials(&mut client, &mut upstream, &ctx, &guard, Some(&engine)) + .await + } else { + let input = NetworkInput { + host: ctx.host.clone(), + port: ctx.port, + binary_path: PathBuf::from(&ctx.binary_path), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + let (config, generation) = engine + .query_endpoint_config_with_generation(&input) + .unwrap(); + let config = crate::l7::parse_l7_config(&config.unwrap()).unwrap(); + relay_with_inspection( + &config, + engine.clone_engine_for_tunnel(generation).unwrap(), + &mut client, + &mut upstream, + &ctx, + ) + .await + } +} + +fn assert_bad_gateway(bytes: &[u8]) { + let response = std::str::from_utf8(bytes).unwrap(); + assert!( + response.starts_with("HTTP/1.1 502 Bad Gateway\r\n"), + "{response:?}" + ); + assert!(response.contains("Content-Length: 0\r\n")); + assert!(response.ends_with("Connection: close\r\n\r\n")); + let prefix = format!( + "{}: ", + openshell_core::token_grant_failure::REQUEST_ID_HEADER + ); + let id = response + .lines() + .find_map(|line| line.strip_prefix(&prefix)) + .expect("correlation header"); + uuid::Uuid::parse_str(id).unwrap(); +} + +#[tokio::test] +async fn keep_alive_propagates_upstream_idle_close_over_tcp() { + timeout(Duration::from_secs(5), async { + for protocol in ["passthrough", "rest", "json-rpc", "mcp"] { + let body = if matches!(protocol, "json-rpc" | "mcp") { + r#"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"test","version":"1"}}}"# + } else { "{}" }; + let request = format!("POST /mcp HTTP/1.1\r\nHost: api.example.test\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", body.len()); + let (mut app, client) = tcp_pair().await; + let (relay_upstream, mut upstream) = tcp_pair().await; + let relay = tokio::spawn(relay_fixture(protocol, client, relay_upstream)); + for _ in 0..2 { + app.write_all(request.as_bytes()).await.unwrap(); + read_exact_message(&mut upstream, request.as_bytes()).await; + upstream.write_all(RESPONSE).await.unwrap(); + read_exact_message(&mut app, RESPONSE).await; + } + upstream.shutdown().await.unwrap(); + let mut rest = Vec::new(); + app.read_to_end(&mut rest).await.unwrap(); + assert!(rest.is_empty(), "idle close must not invent an HTTP response"); + relay.await.unwrap().unwrap(); + } + }).await.expect("idle upstream close must reach the client promptly"); +} + +#[tokio::test] +async fn queued_request_handles_upstream_close_without_replay() { + timeout(Duration::from_secs(5), async { + // None closes before request two is forwarded (BrokenPipe). The other + // cases accept it, then close before or during the response headers. + for protocol in ["passthrough", "rest"] { + for (response, expect_gateway) in [ + (None, true), + (Some(&b""[..]), true), + (Some(&b"HTTP/1.1 100 Continue\r\n\r\n"[..]), true), + (Some(&b"HTTP/1.1 103 Early Hints\r\n\r\nHTTP/1.1 100 Continue\r\n\r\nHTTP/1.1 200"[..]), true), + (Some(&b"HTTP/1.1 200"[..]), true), + ( + Some(&b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nOK"[..]), + false, + ), + ( + Some(&b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n2\r\nOK\r\n"[..]), + false, + ), + ] { + let (mut app, client) = tokio::io::duplex(8192); + let (relay_upstream, mut upstream) = tokio::io::duplex(8192); + // Buffer both requests so request two and the upstream close can + // become ready together after the first successful exchange. + app.write_all(&[REQUEST, REQUEST].concat()).await.unwrap(); + let relay = tokio::spawn(relay_fixture(protocol, client, relay_upstream)); + read_exact_message(&mut upstream, REQUEST).await; + upstream.write_all(RESPONSE).await.unwrap(); + let server = response.map(|response| { + tokio::spawn(async move { + read_exact_message(&mut upstream, REQUEST).await; + upstream.write_all(response).await.unwrap(); + upstream.shutdown().await.unwrap(); + let mut extra = Vec::new(); + upstream.read_to_end(&mut extra).await.unwrap(); + assert!(extra.is_empty(), "POST must not be replayed"); + }) + }); + read_exact_message(&mut app, RESPONSE).await; + let mut received = Vec::new(); + app.read_to_end(&mut received).await.unwrap(); + if expect_gateway { + let interim_end = response.and_then(|bytes| bytes.windows(4).rposition(|part| part == b"\r\n\r\n")).map_or(0, |end| end + 4); + assert_bad_gateway(&received[interim_end..]); + } else { + assert_eq!(without_request_id(&received), response.unwrap()); + } + relay.await.unwrap().unwrap(); + if let Some(server) = server { + server.await.unwrap(); + } + } + } + }) + .await + .expect("queued request must finish with a response or a clean close"); +} + +#[tokio::test] +async fn client_body_eof_is_not_an_upstream_failure() { + timeout(Duration::from_secs(5), async { + let engine = OpaEngine::from_strings( + include_str!("../../../data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .unwrap(); + let guard = engine + .generation_guard(engine.current_generation()) + .unwrap(); + let (mut app, mut client) = tokio::io::duplex(8192); + let (mut relay_upstream, _upstream) = tokio::io::duplex(8192); + app.write_all(&REQUEST[..REQUEST.len() - 2]).await.unwrap(); + app.shutdown().await.unwrap(); + relay_passthrough_with_credentials( + &mut client, + &mut relay_upstream, + &context(), + &guard, + Some(&engine), + ) + .await + .expect_err("client did not provide its declared body"); + drop(client); + let mut response = Vec::new(); + app.read_to_end(&mut response).await.unwrap(); + assert!( + response.is_empty(), + "client error must not be reported as a 502" + ); + }) + .await + .expect("client EOF must terminate the exchange"); +} + +#[test] +fn upstream_failure_event_reports_502_without_request_secrets() { + let request_id = "9b8a7f6e-5d4c-4b3a-9f8e-7d6c5b4a3f2e"; + for (response_started, request_id) in [ + (false, Some(request_id)), + (true, Some(request_id)), + (false, None), + ] { + let event = build_upstream_http_failure_event( + &context(), + "POST", + request_id, + (!response_started).then_some(502), + ); + let json = event.to_json().unwrap(); + assert_eq!(json["class_uid"], 4002); + assert_eq!(json["status_detail"], "upstream_http_transport_failure"); + assert_eq!(json["http_request"]["http_method"], "POST"); + assert!(json["http_request"].get("url").is_none()); + assert_eq!( + json["unmapped"].get("request_id"), + request_id.map(serde_json::Value::from).as_ref() + ); + if response_started { + assert!(json.get("http_response").is_none()); + } else { + assert_eq!(json["http_response"]["code"], 502); + } + openshell_ocsf_validate::validate_event(&json).unwrap(); + } +} + +// A reset after a scripted prefix covers the error path separately from FIN. +struct ResetAfterPrefix(&'static [u8], bool, usize); +impl AsyncRead for ResetAfterPrefix { + fn poll_read( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + let this = self.get_mut(); + if this.0.is_empty() { + std::task::Poll::Ready(Err(std::io::ErrorKind::ConnectionReset.into())) + } else { + let n = this.0.len().min(buf.remaining()); + buf.put_slice(&this.0[..n]); + this.0 = &this.0[n..]; + std::task::Poll::Ready(Ok(())) + } + } +} +impl AsyncWrite for ResetAfterPrefix { + fn poll_write( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + bytes: &[u8], + ) -> std::task::Poll> { + self.get_mut().2 += bytes.len(); + std::task::Poll::Ready(Ok(bytes.len())) + } + fn poll_flush( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + fn poll_shutdown( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.get_mut().1 = true; + std::task::Poll::Ready(Ok(())) + } +} + +#[tokio::test] +async fn upstream_reset_uses_client_commitment_and_preserves_cause() { + timeout(Duration::from_secs(5), async { + let (_, resolver) = SecretResolver::from_provider_env([ + ("AWS_ACCESS_KEY_ID".into(), "synthetic-access-key".into()), + ("AWS_SECRET_ACCESS_KEY".into(), "synthetic-signing-secret".into()), + ].into_iter().collect()); + for local_continue in [false, true] { + for (prefix, committed) in [ + (&b""[..], false), + (&b"HTTP/1.1 200"[..], false), + (&b"HTTP/1.1 100 Continue\r\n\r\n"[..], false), + (&b"HTTP/1.1 103 Early Hints\r\n\r\nHTTP/1.1 200"[..], false), + (&b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nOK"[..], true), + ] { + let (mut app, mut client) = tokio::io::duplex(8192); + let mut upstream = ResetAfterPrefix(prefix, false, 0); + let raw = if local_continue { + b"POST /mcp HTTP/1.1\r\nHost: api.example.test\r\nExpect: 100-continue\r\nContent-Length: 2\r\n\r\n{}".to_vec() + } else { + REQUEST.to_vec() + }; + let request = crate::l7::rest::request_from_buffered_http("POST", "/mcp", "/mcp", raw).unwrap(); + let report = crate::l7::rest::relay_http_request_with_response_middleware_guarded_observed( + &request, &mut client, &mut upstream, + crate::l7::rest::RelayRequestOptions { + resolver: resolver.as_ref(), + credential_signing: if local_continue { crate::l7::CredentialSigning::SigV4NoBody } else { crate::l7::CredentialSigning::None }, + signing_service: "execute-api", signing_region: "us-east-1", host: "api.example.test", port: 443, + ..Default::default() + }, None, None, None, + ).await.expect_err("scripted upstream reset"); + assert!(report.chain().any(|cause| cause.to_string().to_lowercase().contains("reset")), + "original I/O cause must remain available: {report:?}"); + let error = report.downcast_ref::().unwrap(); + assert_eq!(error.response_started, committed); + let request_id = "9b8a7f6e-5d4c-4b3a-9f8e-7d6c5b4a3f2e"; + handle_upstream_http_error(&mut client, &mut upstream, &context(), "POST", Some(request_id), error).await.unwrap(); + assert!(upstream.1, "failed upstream must be shut down"); + drop(client); + let mut response = Vec::new(); + app.read_to_end(&mut response).await.unwrap(); + let local_interim = if local_continue { &b"HTTP/1.1 100 Continue\r\n\r\n"[..] } else { &b""[..] }; + assert!(response.starts_with(local_interim)); + let response = &response[local_interim.len()..]; + if committed { + assert_eq!(response, prefix); + } else { + let interim_end = prefix.windows(4).rposition(|part| part == b"\r\n\r\n").map_or(0, |end| end + 4); + assert_eq!(&response[..interim_end], &prefix[..interim_end]); + assert_bad_gateway(&response[interim_end..]); + assert!(std::str::from_utf8(response).unwrap().contains(request_id)); + } + } + } + }).await.expect("reset must terminate the exchange"); +} + +#[tokio::test] +async fn ready_request_does_not_consume_unsolicited_upstream_response() { + timeout(Duration::from_secs(5), async { + for protocol in ["passthrough", "rest"] { + let (mut app, client) = tokio::io::duplex(8192); + let (relay_upstream, mut server) = tokio::io::duplex(8192); + app.write_all(REQUEST).await.unwrap(); + server.write_all(RESPONSE).await.unwrap(); + let relay = tokio::spawn(relay_fixture(protocol, client, relay_upstream)); + let mut response = Vec::new(); + app.read_to_end(&mut response).await.unwrap(); + assert_bad_gateway(&response); + let mut forwarded = Vec::new(); + server.read_to_end(&mut forwarded).await.unwrap(); + assert!( + forwarded.is_empty(), + "request must not reach the invalid upstream" + ); + relay.await.unwrap().unwrap(); + + let (mut app, mut client) = tokio::io::duplex(8192); + app.write_all(REQUEST).await.unwrap(); + let mut reset = ResetAfterPrefix(b"", false, 0); + relay_fixture(protocol, &mut client, &mut reset) + .await + .unwrap(); + assert_eq!(reset.2, 0, "a ready upstream reset must prevent forwarding"); + drop(client); + let mut response = Vec::new(); + app.read_to_end(&mut response).await.unwrap(); + assert_bad_gateway(&response); + } + }) + .await + .expect("queued upstream failures must produce a final error"); +} diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index caec858eaf..bdcb406000 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -797,6 +797,7 @@ where client, upstream, RelayRequestOptions { + request_id: None, resolver, body_classifier: None, mcp_request_validation: None, @@ -825,6 +826,7 @@ pub(crate) enum WebSocketExtensionMode { #[derive(Clone, Copy, Default)] pub(crate) struct RelayRequestOptions<'a> { + pub(crate) request_id: Option<&'a str>, pub(crate) resolver: Option<&'a SecretResolver>, pub(crate) body_classifier: Option<&'a openshell_core::secrets::body::BodyCredentialClassifier>, /// Revalidate buffered MCP requests after header transformations. @@ -886,6 +888,33 @@ impl std::error::Error for CredentialUnavailableError {} impl miette::Diagnostic for CredentialUnavailableError {} +/// A body ended before its declared framing was complete. The upstream +/// adapter identifies the origin when the same readers handle request bodies. +#[derive(Debug, thiserror::Error)] +#[error("{0}")] +struct IncompleteHttpBody(String); +impl miette::Diagnostic for IncompleteHttpBody {} + +/// An upstream transport failure, distinct from client I/O and credential refusals. +#[derive(Debug)] +pub(crate) struct UpstreamHttpError { + pub(crate) response_started: bool, + pub(super) source: miette::Report, +} + +impl fmt::Display for UpstreamHttpError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("HTTP upstream connection closed or failed") + } +} + +impl std::error::Error for UpstreamHttpError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&*self.source) + } +} +impl miette::Diagnostic for UpstreamHttpError {} + impl<'a> CredentialGenerationGuard<'a> { pub(crate) fn new( state: &'a openshell_core::provider_credentials::ProviderCredentialState, @@ -994,8 +1023,63 @@ where correlation, request_span: None, track_request_span: live_body.is_none(), + transport_failed: false, + read_eof: false, }; - let upstream = &mut observed_upstream; + let mut client = ResponseProgress { + client, + response_started: false, + informational_remaining: 0, + }; + let result = Box::pin(relay_http_exchange( + req, + &mut client, + &mut observed_upstream, + options, + live_body, + response_middleware, + observer, + correlation, + )) + .await; + // Only writes accepted by the client commit the final response. Normal + // close-delimited body EOF must not reclassify a later middleware error. + match result { + Err(mut report) => { + if let Some(error) = report.downcast_mut::() { + error.response_started = client.response_started; + Err(report) + } else if observed_upstream.transport_failed + || (observed_upstream.read_eof + && report.downcast_ref::().is_some()) + { + Err(miette::Report::new(UpstreamHttpError { + response_started: client.response_started, + source: report, + })) + } else { + Err(report) + } + } + result => result, + } +} + +#[allow(clippy::too_many_arguments)] +async fn relay_http_exchange( + req: &L7Request, + client: &mut C, + upstream: &mut ObservedUpstream<'_, U>, + options: RelayRequestOptions<'_>, + live_body: Option<&mut crate::l7::middleware::RequestBodyStream>, + response_middleware: Option>, + observer: Option<&EndpointObserver>, + correlation: Option<&RequestCorrelation>, +) -> Result +where + C: AsyncRead + AsyncWrite + Unpin, + U: AsyncRead + AsyncWrite + Unpin, +{ let mut options = options; if let Some(middleware) = response_middleware.as_ref() { options.identity_response_coding = middleware.inspects_bodies().await; @@ -1494,6 +1578,69 @@ fn ensure_body_generation_current(options: RelayRequestOptions<'_>) -> Result<() Ok(()) } +/// Track final response bytes accepted by the client, excluding interim heads. +struct ResponseProgress<'a, C> { + client: &'a mut C, + response_started: bool, + informational_remaining: usize, +} + +impl AsyncRead for ResponseProgress<'_, C> { + fn poll_read( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + let this = self.get_mut(); + std::pin::Pin::new(&mut *this.client).poll_read(cx, buf) + } +} + +impl AsyncWrite for ResponseProgress<'_, C> { + fn poll_write( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &[u8], + ) -> std::task::Poll> { + let this = self.get_mut(); + // Interim heads are written separately, both by the response relay + // and by local Expect handling. Retain the remaining count when the + // client accepts only part of a head. A 101 commits the upgrade. + if !this.response_started + && this.informational_remaining == 0 + && let Some(end) = buf.windows(4).position(|bytes| bytes == b"\r\n\r\n") + && matches!( + http_response::parse_observed_http_status_code(&String::from_utf8_lossy( + &buf[..end + 4] + )), + Some(100 | 102..=199) + ) + { + this.informational_remaining = end + 4; + } + let result = std::pin::Pin::new(&mut *this.client).poll_write(cx, buf); + if let std::task::Poll::Ready(Ok(n)) = result { + this.response_started |= n > this.informational_remaining; + this.informational_remaining = this.informational_remaining.saturating_sub(n); + } + result + } + fn poll_flush( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + let this = self.get_mut(); + std::pin::Pin::new(&mut *this.client).poll_flush(cx) + } + fn poll_shutdown( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + let this = self.get_mut(); + std::pin::Pin::new(&mut *this.client).poll_shutdown(cx) + } +} + /// Upstream-only I/O adapter that cannot accidentally classify client errors. /// /// Once a valid response status consumes the observation token, later stream @@ -1501,6 +1648,8 @@ fn ensure_body_generation_current(options: RelayRequestOptions<'_>) -> Result<() struct ObservedUpstream<'a, U> { upstream: &'a mut U, observer: Option<&'a EndpointObserver>, + transport_failed: bool, + read_eof: bool, correlation: Option<&'a RequestCorrelation>, request_span: Option, track_request_span: bool, @@ -1516,6 +1665,14 @@ impl ObservedUpstream<'_, U> { correlation.record_failure(HttpOutcome::UpstreamError, "upstream_send_failed"); } } + + fn transport_error(&mut self, error: std::io::Error) -> std::io::Error { + self.transport_failed = true; + if let Some(observer) = self.observer { + observer.observe(EndpointResult::TransportFailed); + } + error + } } impl AsyncRead for ObservedUpstream<'_, U> { @@ -1525,13 +1682,18 @@ impl AsyncRead for ObservedUpstream<'_, U> { buf: &mut tokio::io::ReadBuf<'_>, ) -> std::task::Poll> { let this = self.get_mut(); + let before = buf.filled().len(); + let can_read = buf.remaining() > 0; let result = std::pin::Pin::new(&mut *this.upstream).poll_read(cx, buf); - if matches!(result, std::task::Poll::Ready(Err(_))) - && let Some(observer) = this.observer - { - observer.observe(EndpointResult::TransportFailed); + this.read_eof |= matches!(result, std::task::Poll::Ready(Ok(()))) + && can_read + && buf.filled().len() == before; + match result { + std::task::Poll::Ready(Err(error)) => { + std::task::Poll::Ready(Err(this.transport_error(error))) + } + result => result, } - result } } @@ -1553,13 +1715,19 @@ impl AsyncWrite for ObservedUpstream<'_, U> { } else { std::pin::Pin::new(&mut *this.upstream).poll_write(cx, buf) }; - if matches!(result, std::task::Poll::Ready(Err(_))) { - this.record_write_failure(); - if let Some(observer) = this.observer { - observer.observe(EndpointResult::TransportFailed); + let result = match result { + std::task::Poll::Ready(Ok(0)) if !buf.is_empty() => { + std::task::Poll::Ready(Err(std::io::ErrorKind::WriteZero.into())) + } + result => result, + }; + match result { + std::task::Poll::Ready(Err(error)) => { + this.record_write_failure(); + std::task::Poll::Ready(Err(this.transport_error(error))) } + result => result, } - result } fn poll_flush( @@ -1572,17 +1740,20 @@ impl AsyncWrite for ObservedUpstream<'_, U> { } else { std::pin::Pin::new(&mut *this.upstream).poll_flush(cx) }; - if matches!(result, std::task::Poll::Ready(Err(_))) { - this.record_write_failure(); - if let Some(observer) = this.observer { - observer.observe(EndpointResult::TransportFailed); + match result { + std::task::Poll::Ready(Err(error)) => { + this.record_write_failure(); + std::task::Poll::Ready(Err(this.transport_error(error))) + } + result => { + if matches!(result, std::task::Poll::Ready(Ok(()))) + && let Some(span) = &this.request_span + { + span.record("otel.status_code", "OK"); + } + result } - } else if matches!(result, std::task::Poll::Ready(Ok(()))) - && let Some(span) = &this.request_span - { - span.record("otel.status_code", "OK"); } - result } fn poll_shutdown( @@ -3703,9 +3874,9 @@ where .min(buf.len()); let n = reader.read(&mut buf[..to_read]).await.into_diagnostic()?; if n == 0 { - return Err(miette!( + return Err(miette::Report::new(IncompleteHttpBody(format!( "Connection closed with {remaining} bytes remaining" - )); + )))); } if let Some(guard) = generation_guard { guard.ensure_current()?; @@ -3903,7 +4074,9 @@ impl<'a> ChunkedRelayInput<'a> { .into_diagnostic()?; if read == 0 { self.pending.truncate(start); - return Err(miette!(eof_message)); + return Err(miette::Report::new(IncompleteHttpBody( + eof_message.to_string(), + ))); } self.pending.truncate(start + read); if let Some(guard) = generation_guard { @@ -3927,7 +4100,9 @@ impl<'a> ChunkedRelayInput<'a> { let byte = match reader.read_u8().await { Ok(byte) => byte, Err(error) if error.kind() == std::io::ErrorKind::UnexpectedEof => { - return Err(miette!(eof_message)); + return Err(miette::Report::new(IncompleteHttpBody( + eof_message.to_string(), + ))); } Err(error) => return Err(error).into_diagnostic(), }; @@ -6899,7 +7074,10 @@ mod tests { } } - async fn assert_observed_response_result(response: &'static [u8], expected: EndpointResult) { + async fn assert_observed_response_result( + response: &'static [u8], + expected: EndpointResult, + ) -> Result { let (observer, mut receiver) = test_endpoint_observer().await; let (mut upstream, mut upstream_peer) = tokio::io::duplex(4096); let writer = tokio::spawn(async move { @@ -6909,7 +7087,7 @@ mod tests { .expect("write response"); }); let mut client = tokio::io::sink(); - relay_response( + let result = relay_response( "POST", &mut upstream, &mut client, @@ -6919,14 +7097,14 @@ mod tests { }, None, ) - .await - .expect("relay observed response"); + .await; writer.await.expect("join upstream writer"); let command = receiver.recv().await.expect("endpoint observation"); assert!(matches!( command, EndpointStatusCommand::Observe { result, .. } if result == expected )); + result } #[tokio::test] @@ -6935,7 +7113,8 @@ mod tests { b"HTTP/1.1 204 No Content\r\n\r\n", EndpointResult::HttpResponseReceived, ) - .await; + .await + .unwrap(); } #[tokio::test] @@ -6944,7 +7123,9 @@ mod tests { &b"HTTP/1.1 401 Unauthorized\r\nContent-Length: 0\r\n\r\n"[..], &b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\n\r\n"[..], ] { - assert_observed_response_result(response, EndpointResult::UpstreamRejected).await; + assert_observed_response_result(response, EndpointResult::UpstreamRejected) + .await + .unwrap(); } } @@ -7023,7 +7204,15 @@ mod tests { &b"HTTP/1.1 100 Continue\r\n\r\n"[..], &b"HTTP/1.1 103 Early Hints\r\n\r\n"[..], ] { - assert_observed_response_result(response, EndpointResult::TransportFailed).await; + let error = assert_observed_response_result(response, EndpointResult::TransportFailed) + .await + .expect_err("interim response without a final response"); + assert!( + error + .downcast_ref::() + .unwrap() + .response_started + ); } } @@ -7095,6 +7284,8 @@ mod tests { correlation: None, request_span: None, track_request_span: true, + transport_failed: false, + read_eof: false, }; let mut client = Vec::new(); relay_response( @@ -7127,7 +7318,8 @@ mod tests { b"not-http 200 nope\r\nContent-Length: 0\r\n\r\n", EndpointResult::TransportFailed, ) - .await; + .await + .unwrap(); } #[tokio::test] @@ -7967,6 +8159,7 @@ mod tests { async fn run_response_relay_across_policy_reload( script: ResponseRelayScript, + close_delimited: bool, ) -> (Result, Vec) { let entered = Arc::new(tokio::sync::Notify::new()); let release = Arc::new(tokio::sync::Notify::new()); @@ -7993,14 +8186,29 @@ mod tests { .expect("initial generation guard"); let mut middleware = response_middleware_context(&runner, &chain, "GET"); middleware.generation_guard = Some(&guard); - let mut upstream = b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello".as_slice(); - let mut delivered = Vec::new(); - let relay = relay_response( - "GET", + let (mut upstream, mut server) = tokio::io::duplex(8192); + let (mut app, mut client) = tokio::io::duplex(8192); + let raw = b"GET / HTTP/1.1\r\nHost: example.test\r\n\r\n"; + let request = request_from_buffered_http("GET", "/", "/", raw.to_vec()).unwrap(); + let response = if close_delimited { + &b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nhello"[..] + } else { + &b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello"[..] + }; + let writer = tokio::spawn(async move { + let mut received = vec![0; raw.len()]; + server.read_exact(&mut received).await.unwrap(); + assert_eq!(received, raw); + server.write_all(response).await.unwrap(); + }); + let relay = relay_http_request_with_response_middleware_guarded_observed( + &request, + &mut client, &mut upstream, - &mut delivered, - RelayResponseOptions::default(), + RelayRequestOptions::default(), Some(middleware), + None, + None, ); let reload = async { entered.notified().await; @@ -8010,28 +8218,50 @@ mod tests { release.notify_one(); }; let (outcome, ()) = tokio::join!(relay, reload); + writer.await.unwrap(); + drop(client); + let mut delivered = Vec::new(); + app.read_to_end(&mut delivered).await.unwrap(); (outcome, delivered) } #[tokio::test] async fn response_middleware_rechecks_generation_after_stream_exchange() { - let (outcome, delivered) = - run_response_relay_across_policy_reload(ResponseRelayScript::Stream).await; + for close_delimited in [false, true] { + let (outcome, delivered) = run_response_relay_across_policy_reload( + ResponseRelayScript::Stream, + close_delimited, + ) + .await; - let error = outcome.expect_err("stale stream output must not be delivered"); - assert!(error.to_string().contains("policy generation is stale")); - assert!(delivered.ends_with(b"\r\n\r\n")); - assert!(!delivered.windows(5).any(|window| window == b"HELLO")); + let error = outcome.expect_err("stale stream output must not be delivered"); + assert!( + error.downcast_ref::().is_none(), + "normal EOF then policy reload is not an upstream failure" + ); + assert!(error.to_string().contains("policy generation is stale")); + assert!(delivered.ends_with(b"\r\n\r\n")); + assert!(!delivered.windows(5).any(|window| window == b"HELLO")); + } } #[tokio::test] async fn response_middleware_rechecks_generation_after_whole_body_finish() { - let (outcome, delivered) = - run_response_relay_across_policy_reload(ResponseRelayScript::WholeBody).await; + for close_delimited in [false, true] { + let (outcome, delivered) = run_response_relay_across_policy_reload( + ResponseRelayScript::WholeBody, + close_delimited, + ) + .await; - let error = outcome.expect_err("stale whole-body output must not be delivered"); - assert!(error.to_string().contains("policy generation is stale")); - assert!(delivered.is_empty()); + let error = outcome.expect_err("stale whole-body output must not be delivered"); + assert!( + error.downcast_ref::().is_none(), + "normal EOF then policy reload is not an upstream failure" + ); + assert!(error.to_string().contains("policy generation is stale")); + assert!(delivered.is_empty()); + } } /// HTTP hook response stage that streams its input back and diff --git a/crates/openshell-supervisor-network/src/l7/rest/http_response.rs b/crates/openshell-supervisor-network/src/l7/rest/http_response.rs index 85e3a602b1..f118ae6daf 100644 --- a/crates/openshell-supervisor-network/src/l7/rest/http_response.rs +++ b/crates/openshell-supervisor-network/src/l7/rest/http_response.rs @@ -128,10 +128,10 @@ where observer.observe(EndpointResult::TransportFailed); } record_upstream_response_error(options.correlation, "upstream_response_incomplete"); - if !buf.is_empty() { - client.write_all(&buf).await.into_diagnostic()?; - } - return Ok(RelayOutcome::Consumed); + return Err(miette::Report::new(UpstreamHttpError { + response_started: informational_header_bytes > 0, + source: miette!("HTTP upstream closed before a complete response header"), + })); } buf.extend_from_slice(&tmp[..n]); }; @@ -1855,7 +1855,9 @@ impl<'a, R: AsyncRead + Unpin> BufferedResponseReader<'a, R> { while self.exact_buffer.len() < length { let remaining = length - self.exact_buffer.len(); let Some(data) = self.read_some(remaining).await? else { - return Err(miette!("HTTP response body ended unexpectedly")); + return Err(miette::Report::new(IncompleteHttpBody( + "HTTP response body ended unexpectedly".into(), + ))); }; self.exact_buffer.extend_from_slice(&data); } @@ -1866,7 +1868,9 @@ impl<'a, R: AsyncRead + Unpin> BufferedResponseReader<'a, R> { async fn read_line(&mut self) -> Result> { loop { let Some(byte) = self.read_some(1).await? else { - return Err(miette!("HTTP response ended before line terminator")); + return Err(miette::Report::new(IncompleteHttpBody( + "HTTP response ended before line terminator".into(), + ))); }; self.line_buffer.push(byte[0]); if self.line_buffer.len() > MAX_HEADER_BYTES { @@ -2230,7 +2234,11 @@ where } else { read.await }; - let data = result?.ok_or_else(|| miette!("HTTP response body ended unexpectedly"))?; + let data = result?.ok_or_else(|| { + miette::Report::new(IncompleteHttpBody( + "HTTP response body ended unexpectedly".into(), + )) + })?; payload.extend_from_slice(&data); if payload.len() == limit { return Ok(payload); @@ -2991,10 +2999,11 @@ mod tests { let _subscriber = capture.install(); let correlation = test_correlation("eof-request"); let (outcome, client) = relay_bytes(b"", Some(&correlation)).await; - assert!(matches!( - outcome.expect("upstream close"), - RelayOutcome::Consumed - )); + let error = outcome.expect_err("upstream EOF must reach the failure handler"); + let error = error + .downcast_ref::() + .expect("typed upstream failure"); + assert!(!error.response_started); assert!( client.is_empty(), "EOF must not become a synthetic HTTP response" diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 9742249119..8a76eb1550 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -5488,6 +5488,7 @@ where client, upstream, crate::l7::rest::RelayRequestOptions { + request_id: None, resolver: options.secret_resolver, body_classifier: options.body_classifier, mcp_request_validation: None, @@ -7254,39 +7255,64 @@ async fn handle_forward_proxy_inner( .await; let outcome_result = match outcome_result { Err(report) => { - if let Some(error) = report.downcast_ref::() { - if let Some(session) = middleware_session.take() { - session - .end(openshell_core::proto::MiddlewareSessionEndReason::Cancellation) - .await; - } - let _ = upstream.shutdown().await; - crate::l7::relay::reject_body_credential( - client, - *error, - &l7_ctx, - request_correlation, - ) - .await?; - return Ok(()); - } - if let Some(error) = report.downcast_ref::() { + if let Some(error) = report.downcast_ref::() { + correlation.record_failure( + HttpOutcome::UpstreamError, + "upstream_http_transport_failure", + ); if let Some(session) = middleware_session.take() { session - .end(openshell_core::proto::MiddlewareSessionEndReason::Cancellation) + .end(openshell_core::proto::MiddlewareSessionEndReason::UpstreamFailure) .await; } - crate::l7::relay::reject_credential_resolution( + crate::l7::relay::handle_upstream_http_error( client, + &mut upstream, &l7_ctx, method, + Some(&request_id), error, - request_correlation, ) .await?; - return Ok(()); + if !error.response_started { + correlation.record_response_status(502); + } + Ok(crate::l7::provider::RelayOutcome::Consumed) + } else { + if let Some(error) = report.downcast_ref::() { + if let Some(session) = middleware_session.take() { + session + .end(openshell_core::proto::MiddlewareSessionEndReason::Cancellation) + .await; + } + let _ = upstream.shutdown().await; + crate::l7::relay::reject_body_credential( + client, + *error, + &l7_ctx, + request_correlation, + ) + .await?; + return Ok(()); + } + if let Some(error) = report.downcast_ref::() { + if let Some(session) = middleware_session.take() { + session + .end(openshell_core::proto::MiddlewareSessionEndReason::Cancellation) + .await; + } + crate::l7::relay::reject_credential_resolution( + client, + &l7_ctx, + method, + error, + request_correlation, + ) + .await?; + return Ok(()); + } + Err(report) } - Err(report) } outcome => outcome, }; @@ -10346,6 +10372,92 @@ network_policies: assert!(!record.to_string().contains("not-logged")); } + #[tokio::test] + async fn forward_upstream_eof_returns_502_and_preserves_allow_activity() { + let Some(ip) = non_loopback_test_ipv4() else { + eprintln!("skipping: no non-loopback IPv4 address"); + return; + }; + tokio::time::timeout(std::time::Duration::from_secs(5), async { + let listener = TcpListener::bind((ip, 0)).await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let data = serde_json::json!({"network_policies":{"upstream":{ + "name":"upstream", "endpoints":[{"host":ip.to_string(),"port":port,"allowed_ips":[format!("{ip}/32")]}], + "binaries":[{"path":"/usr/bin/python3"}] + }}}); + let engine = Arc::new( + native_identity_engine( + include_str!("../data/sandbox-policy.rego"), + &data.to_string(), + ) + .unwrap(), + ); + let identity = Ok(ContractBinaryIdentity { + executable: ContractExecutableIdentity { + path: executable_path("/usr/bin/python3"), + digest: Some("00".repeat(32).parse().unwrap()), + }, + ancestors: vec![], + cmdline_paths: vec![], + }); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut request = Vec::new(); + let mut byte = [0u8; 1]; + while !request.ends_with(b"\r\n\r\n") { + stream.read_exact(&mut byte).await.unwrap(); + request.push(byte[0]); + } + assert!(request.starts_with(b"GET /mcp HTTP/1.1")); + // Accept the allowed request, then close before its response. + }); + let target = format!("http://{ip}:{port}/mcp"); + let request = format!("GET {target} HTTP/1.1\r\nHost: {ip}:{port}\r\n\r\n"); + let (mut app, client) = tokio::io::duplex(8192); + let mut client: ProxyClient = tokio::io::BufReader::new(Box::new(client)); + let (activity, mut events) = mpsc::channel(4); + Box::pin(handle_forward_proxy( + "GET", + &target, + request.as_bytes(), + request.len(), + &mut client, + Some(&identity), + None, + engine, + Arc::new(BinaryIdentityCache::new()), + Arc::new(AtomicU32::new(0)), + None, + AgentProposals::default(), + Arc::new(None), + Arc::new(None), + None, + None, + None, + Some(&activity), + None, + )) + .await + .unwrap(); + drop(client); + let mut response = String::new(); + app.read_to_string(&mut response).await.unwrap(); + assert!( + response.starts_with("HTTP/1.1 502 Bad Gateway\r\n"), + "{response}" + ); + assert!(response.contains("x-openshell-request-id:")); + assert!(response.contains("Connection: close\r\n")); + assert!( + !events.try_recv().unwrap().denied, + "allowed decision must survive upstream EOF" + ); + server.await.unwrap(); + }) + .await + .expect("forward EOF must yield a response"); + } + #[tokio::test] async fn plaintext_mcp_forwarding_preserves_initialization_and_selected_revision() { if !cfg!(target_os = "linux") { diff --git a/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs b/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs index 86aa709cec..dad2d67145 100644 --- a/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs +++ b/crates/openshell-supervisor-network/src/proxy/tests/request_trace.rs @@ -188,14 +188,14 @@ network_policies: let request = read_http_headers(&mut upstream).await; assert!(String::from_utf8_lossy(&request).starts_with("GET /allowed HTTP/1.1\r\n")); upstream - .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\n") + .write_all(b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n") .await .unwrap(); upstream.flush().await.unwrap(); released.await.unwrap(); - upstream.write_all(b"ok").await.unwrap(); - // Clean TLS EOF too: the HTTP declared-length failure must - // reach the non-benign outer relay-error callsite. + upstream.write_all(b"invalid\r\n").await.unwrap(); + // Invalid chunk framing reaches the non-benign outer error + // callsite; transport EOF is handled by the relay itself. upstream.shutdown().await.unwrap(); }; let exchange = async { @@ -244,7 +244,10 @@ network_policies: // The downstream TLS stream may omit close_notify when its // failed HTTP relay drops it. The captured event is the oracle. let _ = client.read_to_end(&mut body).await; - assert_eq!(body, b"ok"); + assert!( + body.is_empty(), + "malformed chunk framing must not be forwarded" + ); receipt }; let ((), receipt) = tokio::join!(serve, exchange); @@ -294,7 +297,7 @@ network_policies: terminal["message"] .as_str() .unwrap() - .contains("2 bytes remaining") + .contains("Invalid HTTP chunk size token") ); assert_eq!(terminal["class_uid"], 4001); assert_eq!(terminal["unmapped"]["http_event_kind"], "connection"); @@ -477,17 +480,22 @@ network_policies: .unwrap(); result.unwrap(); let spans = capture.spans(); - let expected = (!matches!(outcome, Outcome::UpstreamClosed)).then_some(if policy_denied { + let upstream_failure = matches!(outcome, Outcome::UpstreamClosed); + let expected = if policy_denied { 403 } else if credential_failure { 500 + } else if upstream_failure { + 502 } else { 200 - }); + }; let response = String::from_utf8(response).unwrap(); let records = capture.events.json_events(); let expected_records = if policy_denied || credential_failure { 1 + } else if upstream_failure { + 3 } else { 2 }; @@ -508,12 +516,20 @@ network_policies: assert_eq!(record["firewall_rule"]["type"], "l7"); assert!(record.get("status_id").is_none()); assert!(record.get("status_detail").is_none()); - for control in &records { - assert_eq!(control["http_request"]["url"]["path"], path); - assert!(control.get("http_response").is_none()); + for (index, control) in records.iter().enumerate() { + if upstream_failure && index == 1 { + assert_eq!(control["status_detail"], "upstream_http_transport_failure"); + assert_eq!(control["status_id"], 2); + assert_eq!(control["http_request"]["http_method"], "GET"); + assert!(control["http_request"].get("url").is_none()); + assert_eq!(control["http_response"]["code"], 502); + } else { + assert_eq!(control["http_request"]["url"]["path"], path); + assert!(control.get("http_response").is_none()); + } } if !policy_denied && !credential_failure { - let forwarded = &records[1]; + let forwarded = records.last().unwrap(); assert!( forwarded["message"] .as_str() @@ -526,33 +542,30 @@ network_policies: } let receipt = record["metadata"]["correlation_uid"].as_str().unwrap(); assert!(uuid::Uuid::parse_str(receipt).is_ok()); + if upstream_failure { + assert_eq!(records[1]["unmapped"]["request_id"], receipt); + assert!(response.contains("Connection: close\r\n")); + } assert_request_record_sequence(&records, receipt); assert!(!format!("{records:?}").contains("SENTINEL")); assert!(!format!("{records:?}").contains("forged-receipt")); assert!(!format!("{records:?}").contains("%ff")); - if let Some(expected) = expected { - assert!( - response.starts_with(&format!("HTTP/1.1 {expected} ")), - "{response}" - ); - assert_eq!(response_request_id(response.as_bytes()), receipt); - if policy_denied { - let (_, body) = response.split_once("\r\n\r\n").unwrap(); - let body: serde_json::Value = serde_json::from_str(body).unwrap(); - assert_eq!(body["error"], "policy_denied"); - assert!( - body["detail"] - .as_str() - .unwrap() - .starts_with(&format!("GET {ip}:{port}{path} denied by L7 policy: ")) - ); - assert_eq!(body.as_object().unwrap().len(), 2); - } - } else { + assert!( + response.starts_with(&format!("HTTP/1.1 {expected} ")), + "{response}" + ); + assert_eq!(response_request_id(response.as_bytes()), receipt); + if policy_denied { + let (_, body) = response.split_once("\r\n\r\n").unwrap(); + let body: serde_json::Value = serde_json::from_str(body).unwrap(); + assert_eq!(body["error"], "policy_denied"); assert!( - response.is_empty(), - "upstream EOF must not synthesize a response" + body["detail"] + .as_str() + .unwrap() + .starts_with(&format!("GET {ip}:{port}{path} denied by L7 policy: ")) ); + assert_eq!(body.as_object().unwrap().len(), 2); } assert_eq!(record["action_id"], if policy_denied { 2 } else { 1 }); let created = capture.created_span_names(); @@ -606,18 +619,10 @@ network_policies: root.span_context.trace_id().to_string() ); } - if let Some(expected) = expected { - assert!(root.attributes.iter().any(|attribute| { - attribute.key.as_str() == "http.response.status_code" - && attribute.value == opentelemetry::Value::I64(expected) - })); - } else { - assert!( - root.attributes - .iter() - .all(|attribute| attribute.key.as_str() != "http.response.status_code") - ); - } + assert!(root.attributes.iter().any(|attribute| { + attribute.key.as_str() == "http.response.status_code" + && attribute.value == opentelemetry::Value::I64(expected) + })); assert!(root.attributes.iter().any(|attribute| { attribute.key.as_str() == "openshell.request.id" && attribute.value.as_str() == receipt })); diff --git a/docs/how-it-works/policies/network-rules.mdx b/docs/how-it-works/policies/network-rules.mdx index e2383ef9bb..1c92a3c809 100644 --- a/docs/how-it-works/policies/network-rules.mdx +++ b/docs/how-it-works/policies/network-rules.mdx @@ -121,6 +121,21 @@ protocol combine. Rules using another protocol cannot authorize the request; for example, a broader REST rule cannot allow a method excluded by a JSON-RPC route. Matching deny rules still take precedence over allows. +### HTTP Connection Failures + +When an HTTP upstream closes an idle connection, OpenShell closes the client +connection so clients can retire it from their pools. If an allowed request +encounters an upstream failure before its final response starts, OpenShell +returns `502 Bad Gateway` with `Connection: close` and a request ID. A completed +informational response, such as `100 Continue`, still permits a final `502`. +After any final response bytes reach the client, OpenShell closes the connection +without appending another status. + +This applies to every HTTP method, including `GET`. Clients that automatically +retry a closed connection may treat the `502` as a response and leave retry +decisions to the application. OpenShell does not reconnect or replay requests; +an upstream may have processed a request even when its response was lost. + ### Enforcement The `enforcement` field on an inspected endpoint decides what happens when a