From 1c14276a257c91176a5b606bf11ffc0de8e06121 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Thu, 22 Oct 2026 02:14:15 -0400 Subject: [PATCH 1/3] test(runtime): isolate fork-exposed host checks and make scratch names unique HACK-1347: in the shared parallel test process, an unrelated test's pre-exec fork (process::clean_command) holds a copy of every open descriptor until it execs. Endpoint discovery then sees the fork as a second holder of the test's listener (or finds a listener the test already dropped), and the lsof release checks see disk handles the fixture had just written, so they correctly refuse before the result under test. Run every such fixture alone in a child test process with the shared run_isolated helper: - host_endpoint: exclusive and executable-pinned discovery, plus the two close/rebind fixtures that still shared the process; the module's own runner now delegates to run_isolated (120 s hang guard, not 5 s). - admission_pool: every release-check test that expects a pass or a non-active refusal (7 more), behind one isolated() helper that keeps serial scans. - short_home, host_filesystem, interrupted: recovery, migration and stop tests that need the lsof proof to pass. HACK-1348: fixtures named scratch directories from the PID and the wall clock and created them exclusively. macOS reports the clock in whole microseconds, so parallel tests can read the same value and the second create fails with "File exists (os error 17)". Add crate::next_test_scratch() and append it to every such name in the lib tests, with a regression test. No product code changes. Refs HACK-1347 Refs HACK-1348 Co-Authored-By: Claude Opus 5.5 --- packages/runtime-core/src/lib.rs | 4 + packages/runtime-core/src/project/snapshot.rs | 4 +- .../runtime-core/src/provider/config_audit.rs | 5 +- .../src/provider/dependency_socket.rs | 5 +- .../runtime-core/src/provider/disk_audit.rs | 5 +- .../src/provider/disk_template.rs | 5 +- packages/runtime-core/src/provider/engine.rs | 5 +- .../enrollment/witnessed/tests.rs | 5 +- .../runtime-core/src/provider/guest_tool.rs | 5 +- .../src/provider/host_endpoint/tests.rs | 88 ++++++++----------- .../provider/hostname_authority/managed.rs | 5 +- .../runtime-core/src/provider/lifecycle.rs | 25 +++--- .../lifecycle/admission_pool/tests.rs | 73 ++++++++++++--- .../lifecycle/host_filesystem/tests.rs | 58 ++++++++++++ .../src/provider/lifecycle/interrupted.rs | 19 +++- .../provider/lifecycle/prepared_boot/tests.rs | 5 +- .../src/provider/lifecycle/pressure_tests.rs | 5 +- .../provider/lifecycle/short_home/tests.rs | 40 +++++++++ .../provider/relay_loop/admission_tests.rs | 5 +- .../src/provider/source_transfer.rs | 5 +- .../src/provider/test_executable.rs | 4 + packages/runtime-core/src/test_scratch.rs | 62 +++++++++++++ 22 files changed, 338 insertions(+), 99 deletions(-) create mode 100644 packages/runtime-core/src/test_scratch.rs diff --git a/packages/runtime-core/src/lib.rs b/packages/runtime-core/src/lib.rs index 46e848a85..1ea639d3d 100644 --- a/packages/runtime-core/src/lib.rs +++ b/packages/runtime-core/src/lib.rs @@ -15,6 +15,10 @@ pub const CANDIDATE_VERSION: &str = "5.0.0-dev.6"; mod error; mod error_stop_diagnostics; pub use error::CandidateError; +#[cfg(test)] +mod test_scratch; +#[cfg(test)] +pub(crate) use test_scratch::next_test_scratch; #[derive(Debug, Serialize)] pub struct Candidate { diff --git a/packages/runtime-core/src/project/snapshot.rs b/packages/runtime-core/src/project/snapshot.rs index 21921b285..db1ec5ce0 100644 --- a/packages/runtime-core/src/project/snapshot.rs +++ b/packages/runtime-core/src/project/snapshot.rs @@ -525,12 +525,12 @@ mod tests { #[test] fn source_replacement_after_inventory_refuses_capture() { let root = Fixture(std::env::temp_dir().canonicalize().unwrap().join(format!( - "hack-snapshot-replacement-{}-{}", + "hack-snapshot-replacement-{}-{}-{}", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() - .as_nanos() + .as_nanos(), crate::next_test_scratch() ))); fs::create_dir(&root.0).unwrap(); let project = root.0.join("project"); diff --git a/packages/runtime-core/src/provider/config_audit.rs b/packages/runtime-core/src/provider/config_audit.rs index f3a096dd9..3124c1120 100644 --- a/packages/runtime-core/src/provider/config_audit.rs +++ b/packages/runtime-core/src/provider/config_audit.rs @@ -271,12 +271,13 @@ mod tests { #[test] fn an_interrupted_create_is_recognized_only_by_this_pools_own_record() { let checkout = std::env::temp_dir().canonicalize().unwrap().join(format!( - "hack-config-audit-recorded-{}-{}", + "hack-config-audit-recorded-{}-{}-{}", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() - .as_nanos() + .as_nanos(), + crate::next_test_scratch() )); fs::create_dir(&checkout).unwrap(); let candidate = Candidate::discover(&checkout).unwrap(); diff --git a/packages/runtime-core/src/provider/dependency_socket.rs b/packages/runtime-core/src/provider/dependency_socket.rs index 071cac20f..93b86b15f 100644 --- a/packages/runtime-core/src/provider/dependency_socket.rs +++ b/packages/runtime-core/src/provider/dependency_socket.rs @@ -223,12 +223,13 @@ mod tests { impl Fixture { fn new() -> Self { let root = Path::new("/tmp").canonicalize().unwrap().join(format!( - "hkd-{}-{}", + "hkd-{}-{}-{}", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() - .as_nanos() + .as_nanos(), + crate::next_test_scratch() )); state::private_directory(&root).unwrap(); Self(root) diff --git a/packages/runtime-core/src/provider/disk_audit.rs b/packages/runtime-core/src/provider/disk_audit.rs index 875a68e4a..8ffd78f1b 100644 --- a/packages/runtime-core/src/provider/disk_audit.rs +++ b/packages/runtime-core/src/provider/disk_audit.rs @@ -270,12 +270,13 @@ mod tests { } } let root = std::env::temp_dir().join(format!( - "hack-audit-{}-{}", + "hack-audit-{}-{}-{}", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() - .as_nanos() + .as_nanos(), + crate::next_test_scratch() )); std::fs::create_dir(&root).unwrap(); let root = root.canonicalize().unwrap(); diff --git a/packages/runtime-core/src/provider/disk_template.rs b/packages/runtime-core/src/provider/disk_template.rs index e20c4913f..2e6822628 100644 --- a/packages/runtime-core/src/provider/disk_template.rs +++ b/packages/runtime-core/src/provider/disk_template.rs @@ -295,12 +295,13 @@ mod tests { impl Fixture { fn new(label: &str) -> Self { let root = std::env::temp_dir().join(format!( - "hack-disk-template-{label}-{}-{}", + "hack-disk-template-{label}-{}-{}-{}", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() - .as_nanos() + .as_nanos(), + crate::next_test_scratch() )); fs::create_dir(&root).unwrap(); fs::set_permissions(&root, fs::Permissions::from_mode(0o700)).unwrap(); diff --git a/packages/runtime-core/src/provider/engine.rs b/packages/runtime-core/src/provider/engine.rs index 0664eecf2..e0299cd9c 100644 --- a/packages/runtime-core/src/provider/engine.rs +++ b/packages/runtime-core/src/provider/engine.rs @@ -715,12 +715,13 @@ mod tests { fn bounded_acquisition_cancellation_and_expiry_do_not_initialize_state() { use std::os::unix::fs::DirBuilderExt; let path = std::env::temp_dir().join(format!( - "hack-engine-acquire-{}-{}", + "hack-engine-acquire-{}-{}-{}", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() - .as_nanos() + .as_nanos(), + crate::next_test_scratch() )); std::fs::DirBuilder::new() .mode(0o700) diff --git a/packages/runtime-core/src/provider/graph/native/persistent_data/enrollment/witnessed/tests.rs b/packages/runtime-core/src/provider/graph/native/persistent_data/enrollment/witnessed/tests.rs index 08d20d5c0..331381ce1 100644 --- a/packages/runtime-core/src/provider/graph/native/persistent_data/enrollment/witnessed/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/persistent_data/enrollment/witnessed/tests.rs @@ -12,12 +12,13 @@ impl Fixture { let path = fs::canonicalize(std::env::temp_dir()) .unwrap() .join(format!( - "witness-enroll-{}-{}", + "witness-enroll-{}-{}-{}", std::process::id(), SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() - .as_nanos() + .as_nanos(), + crate::next_test_scratch() )); fs::DirBuilder::new().mode(0o700).create(&path).unwrap(); Self(path) diff --git a/packages/runtime-core/src/provider/guest_tool.rs b/packages/runtime-core/src/provider/guest_tool.rs index 056bd3404..df1e38c47 100644 --- a/packages/runtime-core/src/provider/guest_tool.rs +++ b/packages/runtime-core/src/provider/guest_tool.rs @@ -196,12 +196,13 @@ mod tests { let path = fs::canonicalize(std::env::temp_dir()) .unwrap() .join(format!( - "guest-artifact-{}-{}", + "guest-artifact-{}-{}-{}", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() - .as_nanos() + .as_nanos(), + crate::next_test_scratch() )); fs::DirBuilder::new().mode(0o700).create(&path).unwrap(); let fixture = Fixture(path); diff --git a/packages/runtime-core/src/provider/host_endpoint/tests.rs b/packages/runtime-core/src/provider/host_endpoint/tests.rs index 2041accfc..8776eb135 100644 --- a/packages/runtime-core/src/provider/host_endpoint/tests.rs +++ b/packages/runtime-core/src/provider/host_endpoint/tests.rs @@ -73,6 +73,13 @@ fn peer_inspection_requires_one_exact_open_accepted_socket() { #[cfg(target_os = "macos")] #[test] fn discovery_requires_the_exact_executable_and_live_exclusive_listener() { + isolated_endpoint_fixture("provider::host_endpoint::tests::exclusive_discovery_child"); +} + +#[cfg(target_os = "macos")] +#[test] +#[ignore = "isolated discovery fixture launched by discovery_requires_the_exact_executable_and_live_exclusive_listener"] +fn exclusive_discovery_child() { let (listener, endpoint) = fixture(); let port = listener.local_addr().unwrap().port(); let (pid, fingerprint) = HostEndpoint::discover(&endpoint.process.executable, port).unwrap(); @@ -87,6 +94,15 @@ fn discovery_requires_the_exact_executable_and_live_exclusive_listener() { #[cfg(target_os = "macos")] #[test] fn executable_pinned_discovery_follows_a_new_listener_process() { + // Discovery scans every process running this executable, and the listener child must + // rebind a port this process reserved and released; keep both away from parallel forks. + isolated_endpoint_fixture("provider::host_endpoint::tests::pinned_discovery_child"); +} + +#[cfg(target_os = "macos")] +#[test] +#[ignore = "isolated discovery fixture launched by executable_pinned_discovery_follows_a_new_listener_process"] +fn pinned_discovery_child() { use std::process::{Child, Command, Stdio}; struct OwnedChild(Child); impl Drop for OwnedChild { @@ -270,7 +286,7 @@ fn pending_accepted_identity_notification_child() { #[cfg(target_os = "macos")] #[test] -#[ignore = "owned subprocess fixture for executable_pinned_discovery_follows_a_new_listener_process"] +#[ignore = "owned subprocess fixture for pinned_discovery_child"] fn discovery_listener_child() { let port: u16 = std::env::var("HACK_DISCOVERY_LISTENER_PORT") .unwrap() @@ -426,6 +442,13 @@ fn unsupported_host_refuses_identity_capture() { #[cfg(target_os = "macos")] #[test] fn replacement_between_precheck_and_connect_receives_no_payload() { + isolated_endpoint_fixture("provider::host_endpoint::tests::precheck_replacement_child"); +} + +#[cfg(target_os = "macos")] +#[test] +#[ignore = "isolated close/rebind fixture launched by replacement_between_precheck_and_connect_receives_no_payload"] +fn precheck_replacement_child() { let (listener, endpoint) = fixture(); let address = listener.local_addr().unwrap(); let mut replacement = None; @@ -590,56 +613,16 @@ fn pending_timeout_and_drop_release_the_socket_without_data() { fn pending_replacement_cannot_release_stream() { isolated_endpoint_fixture("provider::host_endpoint::tests::pending_replacement_child"); } + +/// Run one ignored fixture alone in a child test process and require its full assertion +/// path. Unrelated parallel tests' pre-exec forks (`process::clean_command`) hold copies of +/// every descriptor in this process until they exec. A copied listener keeps a closed port +/// bound and, because the fork still runs this executable, is a second exclusive holder for +/// [`HostEndpoint::discover`]. Close/rebind and discovery fixtures therefore run isolated. #[cfg(target_os = "macos")] fn isolated_endpoint_fixture(test: &str) { - // Close/rebind fixtures must not be inherited by unrelated parallel forks. - // Keep its sockets in an isolated child and require the complete assertion path. - use std::process::{Child, Command, Stdio}; - struct OwnedChild(Child); - impl Drop for OwnedChild { - fn drop(&mut self) { - let _ = self.0.kill(); - let _ = self.0.wait(); - } - } - let mut child = OwnedChild( - Command::new(std::env::current_exe().unwrap()) - .args(["--ignored", "--exact", test]) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::inherit()) - .spawn() - .unwrap(), - ); - let deadline = Instant::now() + Duration::from_secs(5); - let status = loop { - if let Some(status) = child.0.try_wait().unwrap() { - break status; - } - assert!( - Instant::now() < deadline, - "endpoint fixture child exceeded its watchdog" - ); - std::thread::sleep(Duration::from_millis(1)); - }; - let mut output = String::new(); - child - .0 - .stdout - .take() - .unwrap() - .take(4097) - .read_to_string(&mut output) - .unwrap(); - assert!(output.len() <= 4096); - assert!( - status.success(), - "owned endpoint fixture {test} failed: {output}" - ); - assert!( - output.contains("test result: ok. 1 passed"), - "isolated assertion did not run" - ); + use crate::provider::test_executable::{ISOLATED_HANG_GUARD, run_isolated}; + run_isolated(test, ISOLATED_HANG_GUARD); } #[cfg(target_os = "macos")] #[test] @@ -843,6 +826,13 @@ fn pending_accept_after_backend_fin_preserves_request_direction() { #[cfg(target_os = "macos")] #[test] fn backend_fin_does_not_bypass_listener_replacement() { + isolated_endpoint_fixture("provider::host_endpoint::tests::backend_fin_replacement_child"); +} + +#[cfg(target_os = "macos")] +#[test] +#[ignore = "isolated close/rebind fixture launched by backend_fin_does_not_bypass_listener_replacement"] +fn backend_fin_replacement_child() { let (listener, endpoint) = fixture(); let (_authority, session) = authorization(); let mut pending = endpoint diff --git a/packages/runtime-core/src/provider/hostname_authority/managed.rs b/packages/runtime-core/src/provider/hostname_authority/managed.rs index 8f8ebdc26..81da03a12 100644 --- a/packages/runtime-core/src/provider/hostname_authority/managed.rs +++ b/packages/runtime-core/src/provider/hostname_authority/managed.rs @@ -83,12 +83,13 @@ mod tests { let root = fs::canonicalize(std::env::temp_dir()) .unwrap() .join(format!( - "hkl-authority-admission-{}-{}", + "hkl-authority-admission-{}-{}-{}", std::process::id(), SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() - .as_nanos() + .as_nanos(), + crate::next_test_scratch() )); state::private_directory(&root).unwrap(); Self(root) diff --git a/packages/runtime-core/src/provider/lifecycle.rs b/packages/runtime-core/src/provider/lifecycle.rs index 9f8220318..41458e9a3 100644 --- a/packages/runtime-core/src/provider/lifecycle.rs +++ b/packages/runtime-core/src/provider/lifecycle.rs @@ -2184,12 +2184,13 @@ mod tests { #[test] fn source_preflight_is_read_only_and_preserves_unowned_state() { let directory = std::env::temp_dir().canonicalize().unwrap().join(format!( - "hack-source-preflight-{}-{}", + "hack-source-preflight-{}-{}-{}", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() - .as_nanos() + .as_nanos(), + crate::next_test_scratch() )); state::private_directory(&directory).unwrap(); struct Remove(std::path::PathBuf); @@ -2267,12 +2268,13 @@ mod tests { let root = std::fs::canonicalize(std::env::temp_dir()) .unwrap() .join(format!( - "hkl-lease-{}-{}", + "hkl-lease-{}-{}-{}", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() - .as_nanos() + .as_nanos(), + crate::next_test_scratch() )); struct Remove(std::path::PathBuf); impl Drop for Remove { @@ -2338,12 +2340,13 @@ mod tests { let root = std::fs::canonicalize(std::env::temp_dir()) .unwrap() .join(format!( - "hkl-startup-lease-{}-{}", + "hkl-startup-lease-{}-{}-{}", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() - .as_nanos() + .as_nanos(), + crate::next_test_scratch() )); struct Remove(std::path::PathBuf); impl Drop for Remove { @@ -2722,12 +2725,13 @@ printf 'verified-cache\n' #[test] fn a_prepared_start_refuses_ambiguous_capacity_before_any_pool_state() { let checkout = std::env::temp_dir().canonicalize().unwrap().join(format!( - "hack-prepared-sockets-{}-{}", + "hack-prepared-sockets-{}-{}-{}", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() - .as_nanos() + .as_nanos(), + crate::next_test_scratch() )); fs::create_dir(&checkout).unwrap(); let candidate = Candidate::discover(&checkout).unwrap(); @@ -2763,12 +2767,13 @@ printf 'verified-cache\n' // disks and expands nothing until the first start, so only the compressed templates // exist. The removed after-create check refused exactly this state. let checkout = std::env::temp_dir().canonicalize().unwrap().join(format!( - "hack-template-sequence-{}-{}", + "hack-template-sequence-{}-{}-{}", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() - .as_nanos() + .as_nanos(), + crate::next_test_scratch() )); fs::create_dir(&checkout).unwrap(); let candidate = Candidate::discover(&checkout).unwrap(); diff --git a/packages/runtime-core/src/provider/lifecycle/admission_pool/tests.rs b/packages/runtime-core/src/provider/lifecycle/admission_pool/tests.rs index f161797bb..2a1fe219a 100644 --- a/packages/runtime-core/src/provider/lifecycle/admission_pool/tests.rs +++ b/packages/runtime-core/src/provider/lifecycle/admission_pool/tests.rs @@ -304,6 +304,24 @@ fn serial() -> std::sync::MutexGuard<'static, ()> { .unwrap_or_else(std::sync::PoisonError::into_inner) } +/// Run one ignored release-check fixture of this module alone in a child test process. +/// +/// A release check must not see this test's provider resources after it let them go. In the +/// shared parallel test process, an unrelated test's pre-exec fork (`process::clean_command`) +/// holds a copy of every descriptor open at the fork until it execs: a disk the fixture just +/// wrote or a handle the test just dropped then shows up in the host-wide `lsof` scan, which +/// correctly refuses as `retained_reserve_active_owner` before the result under test. So every +/// test that expects a release check to pass, or to refuse for another reason, runs isolated. +/// Holding `serial` here keeps the module's whole-host scans one at a time. +fn isolated(child: &str) { + use crate::provider::test_executable::{ISOLATED_HANG_GUARD, run_isolated}; + let _serial = serial(); + run_isolated( + &format!("provider::lifecycle::admission_pool::tests::{child}"), + ISOLATED_HANG_GUARD, + ); +} + /// A pool `down` left exactly clean: provider proven dead after a quiesced engine stop of /// its last boot, clean-stop marker recorded, retained disks identified, provider /// configuration audited, VM lock free and no listener or disk handle. @@ -438,6 +456,12 @@ fn exclusive_lock(path: &Path) -> HeldLock { #[test] fn only_a_clean_stopped_owned_development_pool_selects_the_retained_reserve() { + isolated("clean_stopped_selection_child"); +} + +#[test] +#[ignore = "isolated release-check fixture launched by only_a_clean_stopped_owned_development_pool_selects_the_retained_reserve"] +fn clean_stopped_selection_child() { let _serial = serial(); let pool = stopped_pool(); let receipt = fs::read(root(&pool.candidate).join("owner.json")).unwrap(); @@ -513,6 +537,12 @@ fn receipt_checks_take_no_provider_lock() { #[test] fn foreign_stopped_owner_refuses_the_retained_reserve() { + isolated("foreign_stopped_owner_child"); +} + +#[test] +#[ignore = "isolated release-check fixture launched by foreign_stopped_owner_refuses_the_retained_reserve"] +fn foreign_stopped_owner_child() { let _serial = serial(); let pool = stopped_pool(); fs::write(pool.data().join("name"), b"hack-foreign0000").unwrap(); @@ -659,6 +689,12 @@ fn dirty_stopped_owner_refuses_the_retained_reserve() { /// match, but the stop was never completed, so the pool never qualifies. #[test] fn an_uncertain_stop_rewritten_to_stopped_never_qualifies() { + isolated("uncertain_stop_rewrite_child"); +} + +#[test] +#[ignore = "isolated release-check fixture launched by an_uncertain_stop_rewritten_to_stopped_never_qualifies"] +fn uncertain_stop_rewrite_child() { let _serial = serial(); const LATER: &str = "22222222-2222-2222-2222-222222222222"; let pool = stopped_pool(); @@ -717,6 +753,12 @@ fn a_start_refused_before_boot_keeps_a_clean_stop_clean() { #[test] fn only_a_completed_quiesced_down_records_a_clean_stop() { + isolated("quiesced_down_clean_stop_child"); +} + +#[test] +#[ignore = "isolated release-check fixture launched by only_a_completed_quiesced_down_records_a_clean_stop"] +fn quiesced_down_clean_stop_child() { let _serial = serial(); // `down` reached `stopping` for the clean boot; the provider has exited. let unfinished = || { @@ -810,6 +852,12 @@ fn stale_or_unknown_stopped_owner_refuses_the_retained_reserve() { #[test] fn selected_stopped_owner_drift_refuses_reverification() { + isolated("stopped_owner_drift_child"); +} + +#[test] +#[ignore = "isolated release-check fixture launched by selected_stopped_owner_drift_refuses_reverification"] +fn stopped_owner_drift_child() { let _serial = serial(); let mut pool = stopped_pool(); let selection = select(&pool.candidate, Profile::Development) @@ -1019,6 +1067,12 @@ fn the_boot_gate_rechecks_the_selection_around_its_sample() { #[test] fn machine_start_needs_a_final_leased_recheck_of_the_selection() { + isolated("machine_start_recheck_child"); +} + +#[test] +#[ignore = "isolated release-check fixture launched by machine_start_needs_a_final_leased_recheck_of_the_selection"] +fn machine_start_recheck_child() { let _serial = serial(); let pool = stopped_pool(); let choice = select(&pool.candidate, Profile::Development).unwrap(); @@ -1059,6 +1113,12 @@ fn machine_start_needs_a_final_leased_recheck_of_the_selection() { #[test] fn a_reserve_selection_only_restarts_and_never_creates() { + isolated("reserve_restart_only_child"); +} + +#[test] +#[ignore = "isolated release-check fixture launched by a_reserve_selection_only_restarts_and_never_creates"] +fn reserve_restart_only_child() { let _serial = serial(); let pool = stopped_pool(); let choice = select(&pool.candidate, Profile::Development).unwrap(); @@ -1138,20 +1198,9 @@ fn window(owned: admission::OwnedPool, free: &[u64]) -> Vec Self { let checkout = std::env::temp_dir().canonicalize().unwrap().join(format!( - "hack-interrupted-{label}-{}-{}", + "hack-interrupted-{label}-{}-{}-{}", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() - .as_nanos() + .as_nanos(), + crate::next_test_scratch() )); fs::create_dir(&checkout).unwrap(); let candidate = Candidate::discover(&checkout).unwrap(); @@ -277,6 +278,20 @@ mod tests { #[test] fn an_unclean_stop_after_recovery_leaves_unadopted_disks_unadopted() { + // The stop must find the disks this test just wrote released. In the shared parallel + // test process an unrelated test's pre-exec fork (`process::clean_command`) can hold a + // copy of such a handle until it execs, which the host-wide `lsof` scan correctly + // reports as `stop_uncertain`. Run the fixture alone in a child test process. + use crate::provider::test_executable::{ISOLATED_HANG_GUARD, run_isolated}; + run_isolated( + "provider::lifecycle::interrupted::tests::unadopted_unclean_stop_child", + ISOLATED_HANG_GUARD, + ); + } + + #[test] + #[ignore = "isolated release-check fixture launched by an_unclean_stop_after_recovery_leaves_unadopted_disks_unadopted"] + fn unadopted_unclean_stop_child() { let pool = Pool::new("unadopted"); let mut owner = pool.owner("booting", true, false); pool.disks(&owner, &["storage.raw", "overlay.raw", "vm.lock"]); diff --git a/packages/runtime-core/src/provider/lifecycle/prepared_boot/tests.rs b/packages/runtime-core/src/provider/lifecycle/prepared_boot/tests.rs index 3613737d0..76a5c2ed3 100644 --- a/packages/runtime-core/src/provider/lifecycle/prepared_boot/tests.rs +++ b/packages/runtime-core/src/provider/lifecycle/prepared_boot/tests.rs @@ -10,12 +10,13 @@ struct Fixture { impl Fixture { fn new(label: &str) -> Self { let root = std::env::temp_dir().canonicalize().unwrap().join(format!( - "hack-prepared-work-{label}-{}-{}", + "hack-prepared-work-{label}-{}-{}-{}", std::process::id(), SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() - .as_nanos() + .as_nanos(), + crate::next_test_scratch() )); fs::create_dir(&root).unwrap(); let checkout = root.join("checkout"); diff --git a/packages/runtime-core/src/provider/lifecycle/pressure_tests.rs b/packages/runtime-core/src/provider/lifecycle/pressure_tests.rs index d7280223e..5c4f1b89e 100644 --- a/packages/runtime-core/src/provider/lifecycle/pressure_tests.rs +++ b/packages/runtime-core/src/provider/lifecycle/pressure_tests.rs @@ -35,12 +35,13 @@ fn lease_wait_pressure_change_refuses_owner_commit_and_releases_the_mutation_lea let path = std::fs::canonicalize(std::env::temp_dir()) .unwrap() .join(format!( - "hkl-pressure-lease-{}-{}", + "hkl-pressure-lease-{}-{}-{}", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() - .as_nanos() + .as_nanos(), + crate::next_test_scratch() )); struct Remove(std::path::PathBuf); impl Drop for Remove { diff --git a/packages/runtime-core/src/provider/lifecycle/short_home/tests.rs b/packages/runtime-core/src/provider/lifecycle/short_home/tests.rs index 2654463e9..29555947b 100644 --- a/packages/runtime-core/src/provider/lifecycle/short_home/tests.rs +++ b/packages/runtime-core/src/provider/lifecycle/short_home/tests.rs @@ -142,8 +142,30 @@ impl Drop for Pool { } } +/// Run one ignored recovery fixture of this module alone in a child test process. +/// +/// Recovery must find the pool's disks, VM lock and sockets released before it changes the +/// receipt. In the shared parallel test process, an unrelated test's pre-exec fork +/// (`process::clean_command`) holds a copy of every descriptor open at the fork until it +/// execs, so a disk this test just read or a listener it just dropped can still show up to +/// the host-wide `lsof` scan or accept a connection, and recovery correctly refuses as +/// `stop_uncertain`. Every test that expects those release checks to pass runs isolated. +fn isolated(child: &str) { + use crate::provider::test_executable::{ISOLATED_HANG_GUARD, run_isolated}; + run_isolated( + &format!("provider::lifecycle::short_home::tests::{child}"), + ISOLATED_HANG_GUARD, + ); +} + #[test] fn explicit_recovery_restores_only_missing_alias_and_preserves_disk_bytes() { + isolated("explicit_recovery_child"); +} + +#[test] +#[ignore = "isolated release-check fixture launched by explicit_recovery_restores_only_missing_alias_and_preserves_disk_bytes"] +fn explicit_recovery_child() { let pool = Pool::new(); let storage = fs::read(pool.data().join("storage.raw")).unwrap(); let overlay = fs::read(pool.data().join("overlay.raw")).unwrap(); @@ -222,6 +244,12 @@ fn changed_disk_held_vm_lock_and_open_disk_refuse_before_alias_creation() { #[test] fn existing_file_directory_and_foreign_symlink_are_never_replaced() { + isolated("foreign_alias_child"); +} + +#[test] +#[ignore = "isolated release-check fixture launched by existing_file_directory_and_foreign_symlink_are_never_replaced"] +fn foreign_alias_child() { for case in 0..3 { let pool = Pool::new(); pool.remove_alias(); @@ -253,6 +281,12 @@ fn existing_file_directory_and_foreign_symlink_are_never_replaced() { #[test] fn active_socket_leaves_explicit_incomplete_recovery_then_retry_finishes() { + isolated("active_socket_retry_child"); +} + +#[test] +#[ignore = "isolated release-check fixture launched by active_socket_leaves_explicit_incomplete_recovery_then_retry_finishes"] +fn active_socket_retry_child() { let pool = Pool::new(); let listener = UnixListener::bind(pool.owner.data_dir().join("agent.sock")).unwrap(); pool.remove_alias(); @@ -321,6 +355,12 @@ fn pending_owner_update_is_preserved_and_refuses_alias_repair() { #[test] fn vm_lock_releases_on_success_and_error_even_with_an_inherited_descriptor() { + isolated("vm_lock_release_child"); +} + +#[test] +#[ignore = "isolated release-check fixture launched by vm_lock_releases_on_success_and_error_even_with_an_inherited_descriptor"] +fn vm_lock_release_child() { for fail in [false, true] { let pool = Pool::new(); let listener = diff --git a/packages/runtime-core/src/provider/relay_loop/admission_tests.rs b/packages/runtime-core/src/provider/relay_loop/admission_tests.rs index 46b5ba69b..6b6b3060c 100644 --- a/packages/runtime-core/src/provider/relay_loop/admission_tests.rs +++ b/packages/runtime-core/src/provider/relay_loop/admission_tests.rs @@ -196,12 +196,13 @@ fn listener_accepts_one_and_preserves_owned_path() { } } let root = Dir(std::env::temp_dir().join(format!( - "hka-{}-{}", + "hka-{}-{}-{}", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() - .as_nanos() + .as_nanos(), + crate::next_test_scratch() ))); std::fs::create_dir(&root.0).unwrap(); let path = root.0.join("relay.sock"); diff --git a/packages/runtime-core/src/provider/source_transfer.rs b/packages/runtime-core/src/provider/source_transfer.rs index 005f2a6eb..218c0b7e5 100644 --- a/packages/runtime-core/src/provider/source_transfer.rs +++ b/packages/runtime-core/src/provider/source_transfer.rs @@ -555,12 +555,13 @@ mod tests { #[test] fn publication_load_reads_a_regular_receipt_and_refuses_aliases() { let root = std::env::temp_dir().canonicalize().unwrap().join(format!( - "hack-publication-{}-{}", + "hack-publication-{}-{}-{}", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() - .as_nanos() + .as_nanos(), + crate::next_test_scratch() )); state::private_directory(&root).unwrap(); let source = root.join("source"); diff --git a/packages/runtime-core/src/provider/test_executable.rs b/packages/runtime-core/src/provider/test_executable.rs index b563ea5fc..e00ad93d1 100644 --- a/packages/runtime-core/src/provider/test_executable.rs +++ b/packages/runtime-core/src/provider/test_executable.rs @@ -42,6 +42,10 @@ pub(super) fn sleeping_executable(binary: &Path) { fs::set_permissions(binary, fs::Permissions::from_mode(0o700)).unwrap(); } +/// Hang guard for [`run_isolated`]: the child launches the whole test binary on a host that +/// may be heavily loaded, so this bounds a hang rather than the fixture's own timing. +pub(super) const ISOLATED_HANG_GUARD: Duration = Duration::from_secs(120); + /// Run one ignored test alone in a child process of this test binary and require its /// complete assertion path. Use it when a fixture's descriptors must not be shared with /// unrelated parallel tests: their pre-exec forks (`process::clean_command`) hold copies diff --git a/packages/runtime-core/src/test_scratch.rs b/packages/runtime-core/src/test_scratch.rs new file mode 100644 index 000000000..b1521e377 --- /dev/null +++ b/packages/runtime-core/src/test_scratch.rs @@ -0,0 +1,62 @@ +//! Unique names for test scratch directories. + +use std::sync::atomic::{AtomicU64, Ordering}; + +/// A sequence number unique within this test process, for scratch directory names. +/// +/// Fixtures name a private temporary directory from the process ID and the wall clock, then +/// create it exclusively. The clock alone does not make the name unique: macOS reports it in +/// whole microseconds, so two parallel tests in one process can read the same value and the +/// second exclusive create fails with `File exists`. Appending this sequence keeps names apart. +pub(crate) fn next_test_scratch() -> u64 { + static NEXT: AtomicU64 = AtomicU64::new(0); + NEXT.fetch_add(1, Ordering::Relaxed) +} + +mod tests { + use std::{ + fs, + os::unix::fs::DirBuilderExt, + sync::{Arc, Barrier}, + thread, + time::{SystemTime, UNIX_EPOCH}, + }; + + #[test] + fn parallel_fixtures_reading_one_clock_tick_create_distinct_directories() { + const FIXTURES: usize = 32; + let base = fs::canonicalize(std::env::temp_dir()).unwrap(); + let start = Arc::new(Barrier::new(FIXTURES)); + // Release every fixture at once so they read the clock together, then create the + // directory exclusively as the module fixtures do. + let fixtures: Vec<_> = (0..FIXTURES) + .map(|_| { + let base = base.clone(); + let start = Arc::clone(&start); + thread::spawn(move || { + start.wait(); + let path = base.join(format!( + "hack-test-scratch-{}-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(), + super::next_test_scratch() + )); + let result = fs::DirBuilder::new().mode(0o700).create(&path); + (path, result) + }) + }) + .collect(); + let mut failures = Vec::new(); + for fixture in fixtures { + let (path, result) = fixture.join().unwrap(); + match result { + Ok(()) => fs::remove_dir(&path).unwrap(), + Err(error) => failures.push(format!("{}: {error}", path.display())), + } + } + assert!(failures.is_empty(), "{failures:?}"); + } +} From f6bc8ed79f26a78c7478c9ced09d24ebf4c4d326 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Thu, 22 Oct 2026 06:28:24 -0400 Subject: [PATCH 2/3] test(runtime): isolate the remaining dropped-listener recovery tests HACK-1347 audit, second part: more runtime-core tests drop a listener (or a publication that owns one) in the shared parallel test process and then require a release check to pass. An unrelated test's pre-exec fork (process::clean_command) holds a copy of every open descriptor until it execs, so the dropped listener still accepts and its flock stays held; the check correctly refuses and the test fails. Under in-process fork pressure with a 20 ms pre-exec hold the full all-features lib suite failed 3/3 runs on 9 to 13 such tests. - Make provider::test_executable available on every Unix target (the quiescent-recovery tests also run on Linux) and isolate, with run_isolated: dependency_slots socket recovery (2), quiescent_recovery dropped-listener tests (2), foreground::transport abandoned-publisher tests that expect retirement to pass (7), acknowledged_publisher::missing_lock (5, all use the dead-publisher fixture), native foreground abandoned publication (1) and the https_recovery full commands (2, their fixture drops the provider flock that recovery must then acquire). - Fixtures that only need a stale socket inode now bind a socket that never listens (new provider::test_socket::bound_unlistened, shared with the quiescent-recovery fixture) instead of dropping a listener: https_recovery and dependency_slots::fixture_prior_boot. Regression test included. - relay_exec framing test: shut the peer down instead of only dropping it, so the truncated frame always reaches end of stream. No product code changes. Refs HACK-1347 Co-Authored-By: Claude Opus 5.5 --- .../quiescent_recovery_tests.rs | 60 +++++++------- .../src/provider/engine/relay_exec.rs | 8 ++ .../acknowledged_publisher/missing_lock.rs | 44 ++++++++++ .../src/provider/graph/dependency_slots.rs | 23 +++++ .../dependency_slots/fixture_prior_boot.rs | 6 +- .../graph/foreground/transport_tests.rs | 65 +++++++++++++++ .../provider/graph/native/foreground/tests.rs | 19 +++++ .../src/provider/https_recovery/tests.rs | 33 +++++++- packages/runtime-core/src/provider/mod.rs | 4 +- .../src/provider/test_executable.rs | 2 + .../runtime-core/src/provider/test_socket.rs | 83 +++++++++++++++++++ 11 files changed, 311 insertions(+), 36 deletions(-) create mode 100644 packages/runtime-core/src/provider/test_socket.rs diff --git a/packages/runtime-core/src/provider/dependency_socket/quiescent_recovery_tests.rs b/packages/runtime-core/src/provider/dependency_socket/quiescent_recovery_tests.rs index d5459bf56..4820cf8d5 100644 --- a/packages/runtime-core/src/provider/dependency_socket/quiescent_recovery_tests.rs +++ b/packages/runtime-core/src/provider/dependency_socket/quiescent_recovery_tests.rs @@ -2,8 +2,8 @@ use super::*; use std::{ cell::Cell, os::{ - fd::{AsRawFd, FromRawFd, OwnedFd}, - unix::{ffi::OsStrExt, fs::PermissionsExt, net::UnixListener}, + fd::OwnedFd, + unix::{fs::PermissionsExt, net::UnixListener}, }, sync::atomic::{AtomicU64, Ordering}, }; @@ -48,36 +48,8 @@ impl Fixture { fn unlistened(&self, slot: u8) -> OwnedFd { // A stale fixture needs a bound stream inode, never a transient listener. // It stays unlistened even if a concurrent child inherits a descriptor. - let fd = unsafe { libc::socket(libc::AF_UNIX, libc::SOCK_STREAM, 0) }; - assert!(fd >= 0); - let socket = unsafe { OwnedFd::from_raw_fd(fd) }; - assert_eq!( - unsafe { libc::fcntl(socket.as_raw_fd(), libc::F_SETFD, libc::FD_CLOEXEC) }, - 0 - ); let target = path(&self.0, slot); - let bytes = target.as_os_str().as_bytes(); - let mut address: libc::sockaddr_un = unsafe { std::mem::zeroed() }; - address.sun_family = libc::AF_UNIX as libc::sa_family_t; - assert!(!bytes.contains(&0) && bytes.len() < address.sun_path.len()); - for (destination, byte) in address.sun_path.iter_mut().zip(bytes) { - *destination = *byte as libc::c_char; - } - let length = std::mem::offset_of!(libc::sockaddr_un, sun_path) + bytes.len() + 1; - #[cfg(target_vendor = "apple")] - { - address.sun_len = length.try_into().unwrap(); - } - assert_eq!( - unsafe { - libc::bind( - socket.as_raw_fd(), - (&raw const address).cast(), - length.try_into().unwrap(), - ) - }, - 0 - ); + let socket = crate::provider::test_socket::bound_unlistened(&target); fs::set_permissions(target, fs::Permissions::from_mode(0o600)).unwrap(); socket } @@ -115,8 +87,28 @@ fn stale_stream_inode_never_listens_even_while_an_owned_duplicate_survives() { assert_eq!(observed(&fixture.0, 0).unwrap(), Some(selected)); } +/// Run one ignored fixture of this module alone in a child test process. +/// +/// The selection must find a listener this test just dropped closed. In the shared parallel +/// test process an unrelated test's pre-exec fork (`process::clean_command`) holds a copy of +/// every descriptor open at the fork until it execs, so the dropped listener can still accept +/// and the selection correctly refuses it as live. +fn isolated(child: &str) { + use crate::provider::test_executable::{ISOLATED_HANG_GUARD, run_isolated}; + run_isolated( + &format!("provider::dependency_socket::quiescent_recovery::tests::{child}"), + ISOLATED_HANG_GUARD, + ); +} + #[test] fn first_socket_refusal_records_closed_live_listener_facts_without_removal() { + isolated("closed_live_listener_facts_child"); +} + +#[test] +#[ignore = "isolated dropped-listener fixture launched by first_socket_refusal_records_closed_live_listener_facts_without_removal"] +fn closed_live_listener_facts_child() { use super::super::recovery::observation_diagnostic as diagnostic; let fixture = Fixture::new(); let listener = fixture.bind(0); @@ -248,6 +240,12 @@ fn owner_graph_boot_or_volume_proof_drift_refuses_before_any_effect() { #[test] fn live_listener_and_foreign_replacement_are_never_removed() { + isolated("live_listener_and_foreign_replacement_child"); +} + +#[test] +#[ignore = "isolated dropped-listener fixture launched by live_listener_and_foreign_replacement_are_never_removed"] +fn live_listener_and_foreign_replacement_child() { let fixture = Fixture::new(); let listener = fixture.bind(0); assert!(current(fixture.scope(), &fixture.0).is_err()); diff --git a/packages/runtime-core/src/provider/engine/relay_exec.rs b/packages/runtime-core/src/provider/engine/relay_exec.rs index f91520551..0910f4f0c 100644 --- a/packages/runtime-core/src/provider/engine/relay_exec.rs +++ b/packages/runtime-core/src/provider/engine/relay_exec.rs @@ -320,6 +320,13 @@ mod tests { } let (client, mut server) = UnixStream::pair().unwrap(); server.write_all(&[1, 0, 0, 0, 0, 0, 0, 5, b'x']).unwrap(); + // Shut the peer down instead of only dropping it: an unrelated parallel test's + // pre-exec fork (`process::clean_command`) can hold a copy of this descriptor until + // it execs, and the client then sees no end of stream yet. A shutdown ends the + // stream for every copy, so the truncated frame is always observed at its close. + // `inherited` models such a copy, so this holds whether or not a fork is running. + let inherited = server.try_clone().unwrap(); + server.shutdown(std::net::Shutdown::Write).unwrap(); drop(server); client.set_nonblocking(true).unwrap(); let mut transport = RelayExec { @@ -329,6 +336,7 @@ mod tests { received: 0, }; assert!(transport.poll(&mut vec![], &mut vec![]).is_err()); + drop(inherited); let mut out = vec![0; 4096]; assert!(decode(&mut frame(2, b"x"), &mut out, &mut vec![]).is_err()); } diff --git a/packages/runtime-core/src/provider/graph/acknowledged_publisher/missing_lock.rs b/packages/runtime-core/src/provider/graph/acknowledged_publisher/missing_lock.rs index ec1589484..1cac52be4 100644 --- a/packages/runtime-core/src/provider/graph/acknowledged_publisher/missing_lock.rs +++ b/packages/runtime-core/src/provider/graph/acknowledged_publisher/missing_lock.rs @@ -816,6 +816,20 @@ pub fn retire(candidate: &Candidate, run: &str, expected: &str) -> Result ( super::super::super::tests::Fixture, Candidate, @@ -854,6 +868,12 @@ mod tests { #[test] fn missing_lock_intent_blocks_ordinary_publication_until_exact_archival_completion() { + isolated("intent_blocks_publication_child"); + } + + #[test] + #[ignore = "isolated dead-publisher fixture launched by missing_lock_intent_blocks_ordinary_publication_until_exact_archival_completion"] + fn intent_blocks_publication_child() { let (_fixture, candidate, run, root, intent) = abandoned(); state::write(&intent_path(&root), &intent).unwrap(); assert!(require_no_pending(&candidate, &run).is_err()); @@ -918,6 +938,12 @@ mod tests { #[test] fn replacement_lock_recovers_exact_temp_and_hardlink_crashes() { + isolated("temp_and_hardlink_crashes_child"); + } + + #[test] + #[ignore = "isolated dead-publisher fixture launched by replacement_lock_recovers_exact_temp_and_hardlink_crashes"] + fn temp_and_hardlink_crashes_child() { for stage in ["temp", "linked", "final"] { let (_fixture, candidate, run, root, intent) = abandoned(); state::write(&intent_path(&root), &intent).unwrap(); @@ -962,6 +988,12 @@ mod tests { #[test] fn replacement_lock_preserves_foreign_names_and_malformed_intent() { + isolated("foreign_names_child"); + } + + #[test] + #[ignore = "isolated dead-publisher fixture launched by replacement_lock_preserves_foreign_names_and_malformed_intent"] + fn foreign_names_child() { let (_fixture, candidate, run, root, mut intent) = abandoned(); let temp = root.join(temp_name(&intent.selection_sha256)); std::os::unix::fs::symlink("/tmp/foreign", &temp).unwrap(); @@ -994,6 +1026,12 @@ mod tests { #[test] fn initial_selection_refuses_foreign_completion_pending_without_effects() { + isolated("foreign_completion_pending_child"); + } + + #[test] + #[ignore = "isolated dead-publisher fixture launched by initial_selection_refuses_foreign_completion_pending_without_effects"] + fn foreign_completion_pending_child() { let (_fixture, _candidate, _run, root, _intent) = abandoned(); let pending = completion_path(&root).with_extension("pending"); let original_owner = fs::read(root.join("owner.json")).unwrap(); @@ -1019,6 +1057,12 @@ mod tests { #[test] fn replacement_lock_path_substitution_refuses_with_both_inodes_preserved() { + isolated("lock_path_substitution_child"); + } + + #[test] + #[ignore = "isolated dead-publisher fixture launched by replacement_lock_path_substitution_refuses_with_both_inodes_preserved"] + fn lock_path_substitution_child() { let (_fixture, _candidate, _run, root, intent) = abandoned(); state::write(&intent_path(&root), &intent).unwrap(); let held = replacement_lock(&root, &intent).unwrap(); diff --git a/packages/runtime-core/src/provider/graph/dependency_slots.rs b/packages/runtime-core/src/provider/graph/dependency_slots.rs index 95b684afa..7d31c5b22 100644 --- a/packages/runtime-core/src/provider/graph/dependency_slots.rs +++ b/packages/runtime-core/src/provider/graph/dependency_slots.rs @@ -702,8 +702,26 @@ mod tests { record.sockets.insert(slot, (m.dev(), m.ino())); listener } + /// Run one ignored socket-recovery fixture of this module alone in a child test process. + /// + /// Recovery must find a listener this test just dropped closed. In the shared parallel + /// test process an unrelated test's pre-exec fork (`process::clean_command`) holds a copy + /// of every descriptor open at the fork until it execs, so the dropped listener can still + /// accept and recovery correctly refuses it as live. + fn isolated(child: &str) { + use crate::provider::test_executable::{ISOLATED_HANG_GUARD, run_isolated}; + run_isolated( + &format!("provider::graph::dependency_slots::tests::{child}"), + ISOLATED_HANG_GUARD, + ); + } #[test] fn recovery_refuses_live_and_replaced_sockets_without_touching_siblings() { + isolated("live_and_replaced_sockets_child"); + } + #[test] + #[ignore = "isolated socket-recovery fixture launched by recovery_refuses_live_and_replaced_sockets_without_touching_siblings"] + fn live_and_replaced_sockets_child() { let fixture = ShortFixture::new(); let mut a = fixture_record('1', 0); let mut b = fixture_record('2', 1); @@ -723,6 +741,11 @@ mod tests { } #[test] fn recovery_preflights_entire_assignment_before_unlinking() { + isolated("preflight_assignment_child"); + } + #[test] + #[ignore = "isolated socket-recovery fixture launched by recovery_preflights_entire_assignment_before_unlinking"] + fn preflight_assignment_child() { let fixture = ShortFixture::new(); let mut a = fixture_record('1', 0); a.slots.insert(1, 1); diff --git a/packages/runtime-core/src/provider/graph/dependency_slots/fixture_prior_boot.rs b/packages/runtime-core/src/provider/graph/dependency_slots/fixture_prior_boot.rs index 0216db7d6..6345971c2 100644 --- a/packages/runtime-core/src/provider/graph/dependency_slots/fixture_prior_boot.rs +++ b/packages/runtime-core/src/provider/graph/dependency_slots/fixture_prior_boot.rs @@ -149,14 +149,16 @@ mod tests { sockets: BTreeMap::new(), }; for slot in 0..3 { + // Stale sockets that never listened: a dropped listener could still accept + // through an unrelated parallel test's pre-exec fork copy. let path = socket(&short_home, slot); - let listener = UnixListener::bind(&path).unwrap(); + let stale = crate::provider::test_socket::bound_unlistened(&path); fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); let m = fs::symlink_metadata(&path).unwrap(); if slot < 2 { record.sockets.insert(slot, (m.dev(), m.ino())); } - drop(listener); + drop(stale); } state::private_directory(&super::super::root(&candidate)).unwrap(); state::write( diff --git a/packages/runtime-core/src/provider/graph/foreground/transport_tests.rs b/packages/runtime-core/src/provider/graph/foreground/transport_tests.rs index d9b362781..6768b4981 100644 --- a/packages/runtime-core/src/provider/graph/foreground/transport_tests.rs +++ b/packages/runtime-core/src/provider/graph/foreground/transport_tests.rs @@ -62,6 +62,22 @@ fn pool_gate_refuses_a_replaced_lock_path() { assert!(directory.join("original.lock").exists()); } +/// Run one ignored abandoned-publisher fixture of this module alone in a child test process. +/// +/// [`abandoned_publisher`] drops a live publication, and retirement must then find its +/// listener closed and its operation lock free. In the shared parallel test process an +/// unrelated test's pre-exec fork (`process::clean_command`) holds a copy of every descriptor +/// open at the fork until it execs, so the dropped listener can still accept and the `flock` +/// stays held, and retirement correctly refuses. Every test that expects it to pass runs here. +#[cfg(target_os = "macos")] +fn isolated(child: &str) { + use crate::provider::test_executable::{ISOLATED_HANG_GUARD, run_isolated}; + run_isolated( + &format!("provider::graph::foreground::transport::tests::{child}"), + ISOLATED_HANG_GUARD, + ); +} + #[cfg(target_os = "macos")] fn abandoned_publisher() -> ( super::super::super::tests::Fixture, @@ -87,6 +103,13 @@ fn abandoned_publisher() -> ( #[cfg(target_os = "macos")] #[test] fn recovered_publisher_retirement_is_exact_and_idempotent() { + isolated("recovered_retirement_child"); +} + +#[cfg(target_os = "macos")] +#[test] +#[ignore = "isolated abandoned-publisher fixture launched by recovered_publisher_retirement_is_exact_and_idempotent"] +fn recovered_retirement_child() { let (_fixture, candidate, run, owner, root) = abandoned_publisher(); let receipt = "f".repeat(64); assert!(Retired::acquire(&candidate, &run).unwrap().is_none()); @@ -134,6 +157,13 @@ fn recovered_publisher_retirement_is_exact_and_idempotent() { #[cfg(target_os = "macos")] #[test] fn cleanup_fence_refuses_effects_and_resumes_each_retirement_rename() { + isolated("cleanup_fence_child"); +} + +#[cfg(target_os = "macos")] +#[test] +#[ignore = "isolated abandoned-publisher fixture launched by cleanup_fence_refuses_effects_and_resumes_each_retirement_rename"] +fn cleanup_fence_child() { for fail_at in [1, 3, 4, 5, 6] { let (_fixture, candidate, run, owner, directory) = abandoned_publisher(); let receipt = "f".repeat(64); @@ -255,6 +285,13 @@ fn cleanup_fence_refuses_effects_and_resumes_each_retirement_rename() { #[cfg(target_os = "macos")] #[test] fn retired_recovery_refuses_pending_missing_and_partial_proof_without_mutation() { + isolated("retired_pending_proof_child"); +} + +#[cfg(target_os = "macos")] +#[test] +#[ignore = "isolated abandoned-publisher fixture launched by retired_recovery_refuses_pending_missing_and_partial_proof_without_mutation"] +fn retired_pending_proof_child() { let (_fixture, candidate, run, owner, root) = abandoned_publisher(); let receipt = "f".repeat(64); retire_recovered_publisher(&candidate, &run, &owner, &receipt).unwrap(); @@ -292,6 +329,13 @@ fn retired_recovery_refuses_pending_missing_and_partial_proof_without_mutation() #[cfg(target_os = "macos")] #[test] fn retired_recovery_rechecks_archived_record_socket_and_lock_identity() { + isolated("retired_identity_recheck_child"); +} + +#[cfg(target_os = "macos")] +#[test] +#[ignore = "isolated abandoned-publisher fixture launched by retired_recovery_rechecks_archived_record_socket_and_lock_identity"] +fn retired_identity_recheck_child() { for changed in ["record", "socket", "lock"] { let (_fixture, candidate, run, owner, root) = abandoned_publisher(); let receipt = "f".repeat(64); @@ -334,6 +378,13 @@ fn retired_recovery_rechecks_archived_record_socket_and_lock_identity() { #[cfg(target_os = "macos")] #[test] fn interrupted_socket_move_resumes_but_replaced_owner_refuses() { + isolated("interrupted_socket_move_child"); +} + +#[cfg(target_os = "macos")] +#[test] +#[ignore = "isolated abandoned-publisher fixture launched by interrupted_socket_move_resumes_but_replaced_owner_refuses"] +fn interrupted_socket_move_child() { let (_fixture, candidate, run, owner, root) = abandoned_publisher(); let receipt = "f".repeat(64); let lock = state::Lock::acquire_existing(&root).unwrap(); @@ -386,6 +437,13 @@ fn interrupted_socket_move_resumes_but_replaced_owner_refuses() { #[cfg(target_os = "macos")] #[test] fn selected_legacy_device_socket_move_resumes_only_exact_retirement() { + isolated("legacy_device_socket_move_child"); +} + +#[cfg(target_os = "macos")] +#[test] +#[ignore = "isolated abandoned-publisher fixture launched by selected_legacy_device_socket_move_resumes_only_exact_retirement"] +fn legacy_device_socket_move_child() { let (_fixture, candidate, run, _original_owner, root) = abandoned_publisher(); let receipt = "f".repeat(64); let current = id(&fs::symlink_metadata(&root).unwrap()).0; @@ -437,6 +495,13 @@ fn selected_legacy_device_socket_move_resumes_only_exact_retirement() { #[cfg(target_os = "macos")] #[test] fn dead_owner_guard_refuses_replaced_foreground_lock_path() { + isolated("dead_owner_guard_child"); +} + +#[cfg(target_os = "macos")] +#[test] +#[ignore = "isolated abandoned-publisher fixture launched by dead_owner_guard_refuses_replaced_foreground_lock_path"] +fn dead_owner_guard_child() { let (_fixture, candidate, run, _owner, root) = abandoned_publisher(); let dead = DeadOwner::acquire(&candidate, &run).unwrap(); dead.verify().unwrap(); diff --git a/packages/runtime-core/src/provider/graph/native/foreground/tests.rs b/packages/runtime-core/src/provider/graph/native/foreground/tests.rs index 165e4a529..3c7061a1c 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/tests.rs @@ -572,8 +572,27 @@ fn publication_respects_shared_gate_and_existing_native_reservation() { assert!(!fixture.owner_root().exists()); } +/// Run one ignored dropped-publication fixture of this module alone in a child test process. +/// +/// The test drops a live publication and requires its listener closed. In the shared parallel +/// test process an unrelated test's pre-exec fork (`process::clean_command`) holds a copy of +/// every descriptor open at the fork until it execs, so the dropped listener can still accept. +fn isolated(child: &str) { + use crate::provider::test_executable::{ISOLATED_HANG_GUARD, run_isolated}; + run_isolated( + &format!("provider::graph::native::foreground::tests::{child}"), + ISOLATED_HANG_GUARD, + ); +} + #[test] fn abandoned_publication_is_evidence_and_never_implicit_recovery_authority() { + isolated("abandoned_publication_evidence_child"); +} + +#[test] +#[ignore = "isolated dropped-publication fixture launched by abandoned_publication_is_evidence_and_never_implicit_recovery_authority"] +fn abandoned_publication_evidence_child() { let fixture = Fixture::new(); let review = fixture.receipt().review; let publication = owner::Publication::bind(&fixture.candidate, &review).unwrap(); diff --git a/packages/runtime-core/src/provider/https_recovery/tests.rs b/packages/runtime-core/src/provider/https_recovery/tests.rs index 7ec05e81f..55c6788ff 100644 --- a/packages/runtime-core/src/provider/https_recovery/tests.rs +++ b/packages/runtime-core/src/provider/https_recovery/tests.rs @@ -45,9 +45,12 @@ fn fixture() -> Fixture { directory(&root.join("owner.lock")); directory(&root.join("shared-owner")); directory(&root.join("shared-owner/leases")); - let listener = UnixListener::bind(root.join("owner.sock")).unwrap(); + // A stale owner socket that never listened: a dropped listener could still accept through + // an unrelated parallel test's pre-exec fork copy, and recovery would then refuse it as live. + drop(crate::provider::test_socket::bound_unlistened( + &root.join("owner.sock"), + )); fs::set_permissions(root.join("owner.sock"), fs::Permissions::from_mode(0o600)).unwrap(); - drop(listener); let socket = inode(&root.join("owner.sock")).unwrap(); let lock = inode(&root.join("owner.lock")).unwrap(); let receipt = serde_json::json!({"version":1,"listener":{"pid":999999,"start_micros":1,"uid":unsafe{libc::geteuid()},"executable":"/unused/caddy","port":18443,"fingerprint":"1".repeat(64)},"caddySha256":"2".repeat(64),"caSha256":"3".repeat(64),"authority":{"pid":999998,"socket":"/private/tmp/unused/route.sock","sha256":"4".repeat(64)},"lock":lock,"owner":{"publicKey":"public-key","dev":socket.dev,"ino":socket.ino}}); @@ -104,6 +107,20 @@ fn suffix(f: &Fixture) -> String { fn verify_socket(_: &Receipt, _: &Configuration, socket: &Path) -> Result<(), CandidateError> { socket_absent(socket) } +/// Run one ignored full-recovery fixture of this module alone in a child test process. +/// +/// [`complete_fixture`] creates the provider lock with an `flock` it drops at once, and the +/// full command must then acquire that lock. In the shared parallel test process an unrelated +/// test's pre-exec fork (`process::clean_command`) holds a copy of every descriptor open at +/// the fork until it execs, so the dropped `flock` can still be held and recovery correctly +/// refuses. Every full-command test that expects recovery to pass runs here. +fn isolated(child: &str) { + use crate::provider::test_executable::{ISOLATED_HANG_GUARD, run_isolated}; + run_isolated( + &format!("provider::https_recovery::tests::{child}"), + ISOLATED_HANG_GUARD, + ); +} fn all_original(f: &Fixture) { for entry in &f.journal.entries { assert_eq!(inode(&f.root.join(&entry.name)).unwrap(), entry.id); @@ -418,6 +435,12 @@ fn hashes(f: &Fixture) -> (String, String) { } #[test] fn full_command_preserves_large_public_mode_ca_and_exact_retry() { + isolated("full_command_retry_child"); +} + +#[test] +#[ignore = "isolated full-recovery fixture launched by full_command_preserves_large_public_mode_ca_and_exact_retry"] +fn full_command_retry_child() { let (f, candidate, _alias) = complete_fixture(); let (owner, config) = hashes(&f); let ca = f.root.join("data/caddy/pki/authorities/local/root.crt"); @@ -637,6 +660,12 @@ fn full_command_refuses_unconfirmed_legacy_witness_before_publishing_journal() { #[test] fn full_legacy_command_archives_with_real_current_witness_and_exact_retry() { + isolated("full_legacy_command_child"); +} + +#[test] +#[ignore = "isolated full-recovery fixture launched by full_legacy_command_archives_with_real_current_witness_and_exact_retry"] +fn full_legacy_command_child() { let (f, candidate, _alias) = complete_fixture(); fs::write(f.home.join("compose.yaml"), "services: {}\n").unwrap(); let share = crate::provider::ProjectShareIntent::approve(&f.home, true).unwrap(); diff --git a/packages/runtime-core/src/provider/mod.rs b/packages/runtime-core/src/provider/mod.rs index 410efb0cf..e9ff94e64 100644 --- a/packages/runtime-core/src/provider/mod.rs +++ b/packages/runtime-core/src/provider/mod.rs @@ -69,8 +69,10 @@ pub fn storage_root_witness_tool() -> std::process::ExitCode { storage_root_witness::entry() } pub mod storage_usage; -#[cfg(all(test, target_os = "macos"))] +#[cfg(all(test, unix))] mod test_executable; +#[cfg(all(test, unix))] +mod test_socket; pub use image_load::load as load_image; mod lifecycle; pub use lifecycle::host_filesystem; diff --git a/packages/runtime-core/src/provider/test_executable.rs b/packages/runtime-core/src/provider/test_executable.rs index e00ad93d1..6acd16d09 100644 --- a/packages/runtime-core/src/provider/test_executable.rs +++ b/packages/runtime-core/src/provider/test_executable.rs @@ -10,6 +10,8 @@ use std::{ /// Build a finite owned test process instead of copying an Apple platform binary, /// which can die before a process-identity or quiescence observation. Callers keep /// their original duration arguments and exact child ownership/cleanup checks. +// Only the macOS-only process-identity tests use it; `run_isolated` is used on every Unix. +#[cfg_attr(not(target_os = "macos"), allow(dead_code))] pub(super) fn sleeping_executable(binary: &Path) { assert!(!binary.try_exists().unwrap()); let source = binary.with_extension("c"); diff --git a/packages/runtime-core/src/provider/test_socket.rs b/packages/runtime-core/src/provider/test_socket.rs new file mode 100644 index 000000000..0aadc6530 --- /dev/null +++ b/packages/runtime-core/src/provider/test_socket.rs @@ -0,0 +1,83 @@ +use std::{ + os::{ + fd::{AsRawFd, FromRawFd, OwnedFd}, + unix::ffi::OsStrExt, + }, + path::Path, +}; + +/// Bind a stream socket inode at `path` without ever listening on it, for fixtures that need +/// a stale (closed) socket pathname. +/// +/// Do not make a stale socket by binding a `UnixListener` and dropping it: in the shared +/// parallel test process an unrelated test's pre-exec fork (`process::clean_command`) holds a +/// copy of every descriptor open at the fork until it execs, and its copy of the listener keeps +/// accepting, so a liveness check correctly reports the socket as live. A socket that never +/// listens refuses every connection, whichever process holds a copy. The descriptor is +/// close-on-exec; drop it when the fixture no longer needs it. +pub(super) fn bound_unlistened(path: &Path) -> OwnedFd { + // SAFETY: plain socket creation; the result is checked before ownership is taken. + let fd = unsafe { libc::socket(libc::AF_UNIX, libc::SOCK_STREAM, 0) }; + assert!(fd >= 0, "socket: {}", std::io::Error::last_os_error()); + // SAFETY: `fd` is a fresh descriptor owned by nothing else. + let socket = unsafe { OwnedFd::from_raw_fd(fd) }; + // SAFETY: `socket` is an open descriptor owned by this function. + assert_eq!( + unsafe { libc::fcntl(socket.as_raw_fd(), libc::F_SETFD, libc::FD_CLOEXEC) }, + 0 + ); + let bytes = path.as_os_str().as_bytes(); + // SAFETY: `sockaddr_un` is plain old data; all-zero is a valid initial value. + let mut address: libc::sockaddr_un = unsafe { std::mem::zeroed() }; + address.sun_family = libc::AF_UNIX as libc::sa_family_t; + assert!(!bytes.contains(&0) && bytes.len() < address.sun_path.len()); + for (destination, byte) in address.sun_path.iter_mut().zip(bytes) { + *destination = *byte as libc::c_char; + } + let length = std::mem::offset_of!(libc::sockaddr_un, sun_path) + bytes.len() + 1; + #[cfg(target_vendor = "apple")] + { + address.sun_len = length.try_into().unwrap(); + } + // SAFETY: `address` is a valid NUL-terminated `sockaddr_un` of `length` bytes. + let bound = unsafe { + libc::bind( + socket.as_raw_fd(), + (&raw const address).cast(), + length.try_into().unwrap(), + ) + }; + assert_eq!(bound, 0, "bind: {}", std::io::Error::last_os_error()); + socket +} + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::{fs::FileTypeExt, net::UnixStream}; + + #[test] + fn an_inherited_copy_of_an_unlistened_socket_still_refuses_connections() { + let root = std::env::temp_dir().join(format!( + "hack-test-socket-{}-{}", + std::process::id(), + crate::next_test_scratch() + )); + std::fs::create_dir(&root).unwrap(); + let path = root.join("stale.sock"); + let socket = bound_unlistened(&path); + // Model an unrelated fork's copy: it outlives the fixture's own descriptor. + let copy = socket.try_clone().unwrap(); + drop(socket); + let error = UnixStream::connect(&path).unwrap_err(); + assert_eq!(error.raw_os_error(), Some(libc::ECONNREFUSED)); + assert!( + std::fs::symlink_metadata(&path) + .unwrap() + .file_type() + .is_socket() + ); + drop(copy); + std::fs::remove_dir_all(root).unwrap(); + } +} From cb6661fd5301e57288f16703072b0e1173da2f49 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Thu, 22 Oct 2026 06:31:14 -0400 Subject: [PATCH 3/3] test(runtime): let the publisher stop fixture bind its own port provider::publisher::tests::Fixture picked a port by binding 127.0.0.1:0, reading the port and dropping the probe, then started stream-relay-host on that explicit port. In the shared parallel test process an unrelated test's pre-exec fork (process::clean_command) can still hold the probe listener, or another test can take the port, and the publisher exits 73 without readiness ("publisher readiness missing", 2 of 5 full-suite runs under fork pressure with a 20 ms pre-exec hold). Start the publisher with --publish 0 (from #382): it binds a kernel-chosen port and reports it as "ready ", which the fixture now requires. Refs HACK-1347 Co-Authored-By: Claude Opus 5.5 --- .../runtime-core/src/provider/publisher.rs | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/packages/runtime-core/src/provider/publisher.rs b/packages/runtime-core/src/provider/publisher.rs index 8965df0f9..de7e682da 100644 --- a/packages/runtime-core/src/provider/publisher.rs +++ b/packages/runtime-core/src/provider/publisher.rs @@ -165,7 +165,7 @@ mod tests { use std::{ fs, io::{BufRead, BufReader}, - net::{TcpListener, TcpStream}, + net::TcpStream, os::{ fd::AsRawFd, unix::{fs::PermissionsExt, net::UnixListener}, @@ -197,14 +197,13 @@ mod tests { let path = root.join("upstream"); let upstream = UnixListener::bind(&path).unwrap(); fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap(); - let port = TcpListener::bind("127.0.0.1:0") - .unwrap() - .local_addr() - .unwrap() - .port(); let control = root.join("control"); + // Port 0: the publisher binds a kernel-chosen port and reports it, so no released + // probe port is handed to it. A probe listener dropped here could still be held by + // an unrelated parallel test's pre-exec fork (`process::clean_command`), or taken + // by another test, and the publisher would exit 73 without readiness. let mut child = Command::new(concat!(env!("OUT_DIR"), "/stream-relay-host")) - .args(["--publish", &port.to_string()]) + .args(["--publish", "0"]) .arg(path) .args([TOKEN, "10000", "--control"]) .arg(&control) @@ -227,14 +226,20 @@ mod tests { } let mut line = String::new(); let read = BufReader::new(output).read_line(&mut line); - if read.is_err() || line != "ready\n" { + // "ready \n" is the port-0 readiness contract. + let port = line + .strip_prefix("ready ") + .and_then(|value| value.strip_suffix('\n')) + .filter(|value| !value.starts_with('0')) + .and_then(|value| value.parse::().ok()); + let Some(port) = port.filter(|_| read.is_ok()) else { // Preserve the natural failure status before forced cleanup; never retry. let observed = child.try_wait().ok().flatten(); let _ = child.kill(); let reaped = child.wait().ok(); let _ = fs::remove_dir_all(&root); panic!("publisher readiness missing: observed={observed:?}, reaped={reaped:?}"); - } + }; let process = identity::observe(child.id() as i32).unwrap(); Self { child,