diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index 212cdc5366..4a220672bb 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -3090,6 +3090,49 @@ async fn reload_gateway_policy_runtime( .await } +fn validate_live_filesystem_policy( + installed: Option<&openshell_core::proto::SandboxPolicy>, + candidate: Option<&openshell_core::proto::SandboxPolicy>, +) -> Result<()> { + let (Some(applied), Some(candidate)) = (installed, candidate) else { + return Ok(()); + }; + let mut candidate = candidate.clone(); + if !applied.network_policies.is_empty() { + let (ro, rw) = proxy_baseline_paths(); + // Global policies retain their raw gateway form. Restore only the + // proxy prerequisites that startup actually installed. + enrich_proto_baseline_paths_with(&mut candidate, &ro, &rw, |path| { + applied.filesystem.as_ref().is_some_and(|filesystem| { + filesystem + .read_only + .iter() + .chain(&filesystem.read_write) + .any(|installed| installed == path) + }) + }); + } + let same_paths = |left: &[String], right: &[String]| { + left.iter().collect::>() == right.iter().collect() + }; + let unchanged = match (&applied.filesystem, &candidate.filesystem) { + (Some(applied), Some(candidate)) => { + applied.include_workdir == candidate.include_workdir + && same_paths(&applied.read_only, &candidate.read_only) + && same_paths(&applied.read_write, &candidate.read_write) + } + (None, None) => true, + _ => false, + }; + if !unchanged { + // Reloading the network engine cannot change installed filesystem permissions. + return Err(miette::miette!( + "filesystem policy cannot be changed on a running sandbox; recreate the sandbox to apply filesystem changes" + )); + } + Ok(()) +} + async fn reload_gateway_configuration_runtime( engine: &OpaEngine, policy: Option<&openshell_core::proto::SandboxPolicy>, @@ -3799,7 +3842,9 @@ struct PolicyPollLoopContext { endpoint_observation_tx: Option, /// Single FIFO consumed by the endpoint status reporter. endpoint_status_rx: Option, - /// Gateway policy currently installed in OPA, absent for local overrides. + /// Policy installed at startup, including its filesystem prerequisites. + /// Kept unchanged when gateway updates replace the policy in OPA. + /// Absent for local overrides. endpoint_policy: Option, /// Session accepted by `ConnectSupervisor`; `None` suspends endpoint reports. supervisor_session_id: tokio::sync::watch::Receiver>, @@ -4681,28 +4726,36 @@ async fn run_policy_poll_loop_with_client( if policy_runtime_changed { let pid = ctx.entrypoint_pid.load(Ordering::Acquire); - let runtime_result = reload_gateway_configuration_runtime( - &ctx.opa_engine, + let runtime_result = match validate_live_filesystem_policy( + ctx.endpoint_policy.as_ref(), result.policy.as_ref(), - pid, - MiddlewareReloadContext { - desired_services: &result.supervisor_middleware_services, - authentication: &MiddlewareAuthentication { - credentials: middleware_credentials.clone(), - enabled: result.extension_authentication_enabled, - }, - registry_changed: middleware_registry_changed, - connector: &ctx.middleware_connector, - }, - ctx.transparent_tcp, - ctx.vm_identity, - || { - if let Some(prepared) = prepared_provider.as_ref() { - ctx.provider_credentials.install_prepared(prepared); - } - }, - ) - .await; + ) { + Ok(()) => { + reload_gateway_configuration_runtime( + &ctx.opa_engine, + result.policy.as_ref(), + pid, + MiddlewareReloadContext { + desired_services: &result.supervisor_middleware_services, + authentication: &MiddlewareAuthentication { + credentials: middleware_credentials.clone(), + enabled: result.extension_authentication_enabled, + }, + registry_changed: middleware_registry_changed, + connector: &ctx.middleware_connector, + }, + ctx.transparent_tcp, + ctx.vm_identity, + || { + if let Some(prepared) = prepared_provider.as_ref() { + ctx.provider_credentials.install_prepared(prepared); + } + }, + ) + .await + } + Err(error) => Err(GatewayRuntimeReloadError::PolicyValidation(error)), + }; match runtime_result { Ok(generation) => { @@ -8345,8 +8398,11 @@ network_policies: 1, openshell_core::proto::PolicySource::Sandbox, ); + // Keep the applied filesystem grants so only the TCP expansion differs. + let mut tcp_policy = proto_policy_fixture(); + tcp_policy.network_policies = proto_tcp_policy_fixture().network_policies; let v2 = settings_poll_result( - Some(proto_tcp_policy_fixture()), + Some(tcp_policy), 2, openshell_core::proto::PolicySource::Sandbox, ); @@ -8386,6 +8442,241 @@ network_policies: handle.abort(); } + #[tokio::test] + async fn live_filesystem_changes_reject_polled_policy_without_advancing_active_revision() { + use openshell_core::proto::{FilesystemPolicy, PolicySource}; + + for source in [PolicySource::Sandbox, PolicySource::Global] { + for change in ["read_only", "read_write", "from_none"] { + let mut policy = proto_policy_fixture(); + if change == "from_none" { + policy.filesystem = None; + } else { + // Startup may already have added the proxy's filesystem prerequisites. + policy + .filesystem + .as_mut() + .unwrap() + .read_only + .push("/proxy-baseline".into()); + } + let initial = settings_poll_result(Some(policy.clone()), 1, source); + let engine = Arc::new(OpaEngine::from_proto(&policy).expect("build OPA engine")); + let mut ctx = policy_poll_test_context( + engine.clone(), + LoadedPolicyOrigin::Gateway { + revision: Some(LoadedPolicyRevision::from_snapshot(&initial)), + has_last_valid_policy: true, + }, + default_middleware_connector(), + ); + ctx.endpoint_policy = Some(policy.clone()); + let (client, polls, mut reports) = scripted_policy_gateway(); + let observed_polls = client.polled_sandboxes.clone(); + let wait_for_polls = |count: usize| { + let observed_polls = observed_polls.clone(); + async move { + timeout(Duration::from_secs(1), async { + while observed_polls.lock().await.len() < count { + tokio::task::yield_now().await; + } + }) + .await + .expect("policy poll loop stalled"); + } + }; + polls.send(initial).unwrap(); + let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); + if source == PolicySource::Sandbox { + expect_policy_report(&mut reports, 1).await; + } + wait_for_polls(2).await; + let original_generation = engine.current_generation(); + + let filesystem = policy + .filesystem + .get_or_insert_with(FilesystemPolicy::default); + if change == "read_write" { + filesystem.read_write.push("/extra-readwrite".into()); + } else { + filesystem.read_only.push("/extra-readonly".into()); + } + let mut candidate = settings_poll_result(Some(policy), 2, source); + let fail_closed = source == PolicySource::Global && change == "read_only"; + candidate.policy_validation_failure_mode = if fail_closed { + PolicyValidationFailureMode::FailClosed + } else { + PolicyValidationFailureMode::RetainLastValid + }; + polls.send(candidate).unwrap(); + wait_for_polls(3).await; + + if source == PolicySource::Sandbox { + let (version, loaded, error) = timeout(Duration::from_secs(1), reports.recv()) + .await + .expect("filesystem rejection report timed out") + .expect("policy reporter stopped"); + assert_eq!(version, 2, "{source:?}: {change}"); + assert!(!loaded, "{source:?}: {change}"); + assert!( + error.contains("filesystem"), + "{source:?}: {change}: {error}" + ); + } else { + expect_no_policy_report(&mut reports).await; + } + assert_eq!( + engine.fail_closed_reason().is_some(), + fail_closed, + "{source:?}: {change}" + ); + assert_eq!( + engine.current_generation(), + original_generation + u64::from(fail_closed), + "{source:?}: {change}" + ); + handle.abort(); + } + } + } + + #[tokio::test] + async fn global_policy_retry_uses_installed_filesystem_after_missing_initial_acknowledgement() { + use openshell_core::proto::{FilesystemPolicy, PolicySource}; + + let mut policy = proto_tcp_policy_fixture(); + policy.filesystem = Some(FilesystemPolicy { + include_workdir: true, + read_only: vec!["/usr".into(), "/tmp".into()], + ..Default::default() + }); + let initial = settings_poll_result(Some(policy.clone()), 1, PolicySource::Global); + let mut installed = policy.clone(); + let (ro, rw) = proxy_baseline_paths(); + assert!(enrich_proto_baseline_paths_with( + &mut installed, + &ro, + &rw, + |path| matches!(path, "/proc" | "/dev/null"), + )); + let engine = Arc::new(OpaEngine::from_proto(&installed).expect("build OPA engine")); + let mut ctx = policy_poll_test_context( + engine.clone(), + LoadedPolicyOrigin::Gateway { + revision: Some(LoadedPolicyRevision::from_snapshot(&initial)), + has_last_valid_policy: true, + }, + default_middleware_connector(), + ); + ctx.endpoint_policy = Some(installed); + ctx.transparent_tcp = TransparentTcpReloadState { + capable: true, + substrate_ready: true, + }; + let (client, polls, mut reports) = scripted_policy_gateway(); + let observed_polls = client.polled_sandboxes.clone(); + let wait_for_polls = |count: usize| { + let observed_polls = observed_polls.clone(); + async move { + timeout(Duration::from_secs(1), async { + while observed_polls.lock().await.len() < count { + tokio::task::yield_now().await; + } + }) + .await + .expect("policy poll loop stalled"); + } + }; + // A global policy is never acknowledged by the first poll, so its raw + // gateway form is reconciled against the policy startup installed. + polls.send(initial).unwrap(); + let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); + wait_for_polls(2).await; + assert!(engine.fail_closed_reason().is_none()); + assert_eq!(engine.current_generation(), 1); + + let mut network_only = policy; + network_only + .network_policies + .values_mut() + .next() + .expect("network rule") + .endpoints[0] + .port = 6380; + let mut changed = network_only.clone(); + let network_only = settings_poll_result(Some(network_only), 2, PolicySource::Global); + polls.send(network_only).unwrap(); + wait_for_polls(3).await; + assert!(engine.fail_closed_reason().is_none()); + assert_eq!(engine.current_generation(), 2); + + changed + .filesystem + .as_mut() + .expect("filesystem policy") + .read_only + .push("/app".into()); + let mut changed = settings_poll_result(Some(changed), 3, PolicySource::Global); + changed.policy_validation_failure_mode = PolicyValidationFailureMode::FailClosed; + polls.send(changed).unwrap(); + wait_for_polls(4).await; + assert!(engine.fail_closed_reason().is_some()); + expect_no_policy_report(&mut reports).await; + handle.abort(); + } + + #[tokio::test] + async fn live_filesystem_change_poll_reports_failure_then_accepts_network_only_repair() { + use openshell_core::proto::PolicySource; + + let policy = proto_policy_fixture(); + let initial = settings_poll_result(Some(policy.clone()), 1, PolicySource::Sandbox); + let mut rejected = policy.clone(); + rejected + .filesystem + .as_mut() + .unwrap() + .read_only + .push("/extra-readonly".into()); + let mut rejected = settings_poll_result(Some(rejected), 2, PolicySource::Sandbox); + rejected.policy_validation_failure_mode = PolicyValidationFailureMode::RetainLastValid; + let mut repaired = policy.clone(); + repaired.network_policies = proto_tcp_policy_fixture().network_policies; + repaired.filesystem.as_mut().unwrap().read_only.reverse(); + let repaired = settings_poll_result(Some(repaired), 3, PolicySource::Sandbox); + let engine = Arc::new(OpaEngine::from_proto(&policy).expect("build OPA engine")); + let mut ctx = policy_poll_test_context( + engine.clone(), + LoadedPolicyOrigin::Gateway { + revision: Some(LoadedPolicyRevision::from_snapshot(&initial)), + has_last_valid_policy: true, + }, + default_middleware_connector(), + ); + ctx.endpoint_policy = Some(policy); + ctx.transparent_tcp = TransparentTcpReloadState { + capable: true, + substrate_ready: true, + }; + let (client, polls, mut reports) = scripted_policy_gateway(); + polls.send(initial).unwrap(); + let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); + expect_policy_report(&mut reports, 1).await; + polls.send(rejected).unwrap(); + let (version, loaded, error) = timeout(Duration::from_secs(1), reports.recv()) + .await + .expect("filesystem rejection report timed out") + .expect("policy reporter stopped"); + assert_eq!(version, 2); + assert!(!loaded); + assert!(error.contains("filesystem")); + assert_eq!(engine.current_generation(), 0); + polls.send(repaired).unwrap(); + expect_policy_report(&mut reports, 3).await; + assert_eq!(engine.current_generation(), 1); + handle.abort(); + } + #[tokio::test] async fn same_hash_ack_waits_for_failed_middleware_reconciliation_and_retries_once() { let mut v1 = settings_poll_result( diff --git a/docs/how-it-works/policies/manage-policies.mdx b/docs/how-it-works/policies/manage-policies.mdx index c20af7b67b..6c5e04ce2e 100644 --- a/docs/how-it-works/policies/manage-policies.mdx +++ b/docs/how-it-works/policies/manage-policies.mdx @@ -191,7 +191,7 @@ running sandbox. The startup sections do not: | Network rules | The sandbox loads the new rules. Connections opened under the previous rules close. | Apply the change, then retry the request. | | Middleware in `network_middlewares` | You can add, remove, or reconfigure built-in middleware and external services already registered with the gateway. | Apply the change with `openshell policy set`. | | External middleware registration | Policy changes cannot register a service or change its gateway connection settings. | Update the gateway configuration and restart the gateway. | -| Added filesystem paths | The saved policy can change, but the running workload keeps its existing filesystem permissions. | Recreate the sandbox. | +| Added filesystem paths | The sandbox rejects the entire revision, including network changes. With the default `fail_closed` failure mode, it blocks network traffic until a valid revision loads. | Recreate the sandbox. | | Removed filesystem paths, or changed `include_workdir`, Landlock, or process settings | OpenShell rejects the change after the workload starts. | Recreate the sandbox. | When new network rules take effect, OpenShell closes connections that were diff --git a/nix/pkgs/vm-runtime.nix b/nix/pkgs/vm-runtime.nix index eb3819c09f..bf3d2a50a7 100644 --- a/nix/pkgs/vm-runtime.nix +++ b/nix/pkgs/vm-runtime.nix @@ -13,7 +13,7 @@ let { x86_64-linux = { platform = "linux-x86_64"; - hash = "sha256-kA6ZQz53geBp2zHx8tv+D8VcGkSrnC74Op/60H4fmck="; + hash = "sha256-mZwSUhs9IZ0pPLxzVs4AbRLdYWkUlHwP4lmfsbeiUGw="; artifacts = [ "libkrun.so" "libkrunfw.so.5" @@ -22,7 +22,7 @@ let }; aarch64-linux = { platform = "linux-aarch64"; - hash = "sha256-7FNqUtC6ixfw202EyoYCgZOLdyxJYfg/YzdlkTT/+sM="; + hash = "sha256-Yk5Rxn6JWxMu82Vkyx3eSAxh5/mnfUwcNDNeDIQJz4c="; artifacts = [ "libkrun.so" "libkrunfw.so.5" @@ -31,7 +31,7 @@ let }; aarch64-darwin = { platform = "darwin-aarch64"; - hash = "sha256-cPfogd7QiPFk76kR1JJeKLaWwFksAZ9BeW/LsmDgsjo="; + hash = "sha256-Sm0zwSkctPETKd46zqOqDO02jLapQ3ST1hafqIVkVjU="; artifacts = [ "libkrun.dylib" "libkrunfw.5.dylib" diff --git a/tests/ansible/playbooks/k3s.yaml b/tests/ansible/playbooks/k3s.yaml index e1a2aa3397..957dba612a 100644 --- a/tests/ansible/playbooks/k3s.yaml +++ b/tests/ansible/playbooks/k3s.yaml @@ -21,10 +21,12 @@ state: present update_cache: true + # get.k3s.io serves the latest script, so pin the one from the k3s_version + # release tag. Update the checksum whenever k3s_version changes. - name: Download K3s installer ansible.builtin.get_url: - url: https://get.k3s.io - checksum: sha256:e5cc3b3d9dfc1662c2d9be6da5abc9a4cd317d6abc3a5ffc02e3dd3248207fee + url: "https://raw.githubusercontent.com/k3s-io/k3s/v{{ k3s_version | urlencode }}/install.sh" + checksum: sha256:46177d4c99440b4c0311b67233823a8e8a2fc09693f6c89af1a7161e152fbfad dest: /tmp/install-k3s.sh mode: "0755"