From cfefde93822e02633b7ae6c11a65e150ca7e86c4 Mon Sep 17 00:00:00 2001 From: Ralf Biedert Date: Mon, 9 Nov 2026 15:03:54 +0100 Subject: [PATCH 01/12] feat(arty): integrate demand-driven runtime diagnostics Register runtime workers and expose native async and blocking task snapshots through Seismograph. Observe task polls only when collection or recording needs them, and preserve task identity, timestamps, and terminal outcomes. Store async diagnostic metadata in pooled executor tasks and reduce it to 56 bytes. Resolve recorded spawn descriptors lazily and reuse the event timestamp. Add monitoring integration tests and spawn benchmarks. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- Cargo.lock | 3 + crates/arty/Cargo.toml | 30 +- crates/arty/README.md | 15 +- crates/arty/benches/arty_inspection.rs | 261 ++++++ crates/arty/benches/arty_spawn_allocations.rs | 85 ++ crates/arty/src/lib.rs | 12 +- crates/arty/src/runtime/bootstrap/startup.rs | 21 +- crates/arty/src/runtime/builder.rs | 6 +- .../src/runtime/config/monitor_options.rs | 18 +- .../arty/src/runtime/config/monitor_policy.rs | 57 +- crates/arty/src/runtime/dispatch/client.rs | 10 + crates/arty/src/runtime/dispatch/core.rs | 35 + crates/arty/src/runtime/handle.rs | 41 +- crates/arty/src/runtime/mod.rs | 29 +- .../src/runtime/monitoring/asynchronous.rs | 102 +++ crates/arty/src/runtime/monitoring/mod.rs | 12 + crates/arty/src/runtime/monitoring/runtime.rs | 46 + crates/arty/src/runtime/monitoring/task.rs | 258 ++++++ .../runtime/worker/asynchronous/protocol.rs | 6 + .../src/runtime/worker/asynchronous/thread.rs | 7 +- .../src/runtime/worker/asynchronous/worker.rs | 40 +- .../arty/src/runtime/worker/blocking/mod.rs | 1 + .../arty/src/runtime/worker/blocking/queue.rs | 239 +++++ .../src/runtime/worker/blocking/worker.rs | 118 ++- crates/arty/src/task/preparation/blocking.rs | 50 + crates/arty/src/task/preparation/mod.rs | 3 + crates/arty/src/task/preparation/remote.rs | 3 + crates/arty/src/task/runtime_scheduler.rs | 4 +- crates/arty/src/task/scheduler.rs | 7 +- crates/arty/tests/monitoring.rs | 858 ++++++++++++++++++ crates/arty_executor/src/executor_shared.rs | 2 +- crates/arty_executor/src/inspection.rs | 16 +- crates/arty_executor/src/task.rs | 19 +- crates/arty_executor/src/task_set.rs | 50 + crates/arty_executor/tests/inspection.rs | 129 +++ crates/seismograph_runtime/src/task.rs | 43 +- crates/seismograph_runtime/tests/on_demand.rs | 84 ++ 37 files changed, 2643 insertions(+), 77 deletions(-) create mode 100644 crates/arty/benches/arty_inspection.rs create mode 100644 crates/arty/benches/arty_spawn_allocations.rs create mode 100644 crates/arty/src/runtime/monitoring/asynchronous.rs create mode 100644 crates/arty/src/runtime/monitoring/mod.rs create mode 100644 crates/arty/src/runtime/monitoring/runtime.rs create mode 100644 crates/arty/src/runtime/monitoring/task.rs create mode 100644 crates/arty/src/runtime/worker/blocking/queue.rs create mode 100644 crates/arty/tests/monitoring.rs diff --git a/Cargo.lock b/Cargo.lock index 89f724ea26..e718fd7334 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -238,6 +238,7 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" name = "arty" version = "0.6.0" dependencies = [ + "alloc_tracker", "arty_executor", "arty_io_core", "arty_io_host", @@ -260,6 +261,8 @@ dependencies = [ "plurality", "scopeguard", "seismograph", + "seismograph_protocol", + "seismograph_runtime", "static_assertions", "testing_aids", "thread_aware", diff --git a/crates/arty/Cargo.toml b/crates/arty/Cargo.toml index 547bf81d8f..239cd7eb7f 100644 --- a/crates/arty/Cargo.toml +++ b/crates/arty/Cargo.toml @@ -21,6 +21,8 @@ allowed_external_types = [ "arty_io_core::io_context::IoContext", "arty_macros::main", "arty_macros::test", + "seismograph_protocol::io::MonitorIo", + "seismograph_runtime::RuntimeHandle", "thread_aware_core::thread::NumaNode", "thread_aware_core::thread::Owner", "thread_aware_core::thread::Thread", @@ -36,7 +38,14 @@ all-features = true [features] default = ["rt", "macros", "monitor"] macros = ["rt", "dep:arty_macros"] -monitor = ["dep:seismograph", "seismograph/monitor", "performables?/seismograph"] +monitor = [ + "dep:seismograph", + "dep:seismograph_protocol", + "dep:seismograph_runtime", + "arty_executor?/inspection", + "seismograph/monitor", + "performables?/seismograph", +] rt = [ "dep:arty_executor", "dep:arty_io_host", @@ -53,7 +62,7 @@ rt = [ "dep:plurality", "dep:thread_aware", ] -test-util = ["tick/test-util", "many_cpus?/test-util"] +test-util = ["tick/test-util", "many_cpus?/test-util", "seismograph_protocol?/test-util"] [dependencies] arty_executor = { workspace = true, optional = true } @@ -72,12 +81,15 @@ performables = { workspace = true, optional = true } pin-project = { workspace = true, optional = true } plurality = { workspace = true, features = ["std"], optional = true } seismograph = { workspace = true, optional = true } +seismograph_protocol = { workspace = true, optional = true } +seismograph_runtime = { workspace = true, optional = true } thread_aware = { workspace = true, features = ["std", "derive"], optional = true } thread_aware_core = { workspace = true, features = ["std"] } tick = { workspace = true } [dev-dependencies] -arty_executor = { path = "../arty_executor" } +alloc_tracker = { workspace = true } +arty_executor = { path = "../arty_executor", features = ["inspection"] } arty_io_host = { path = "../arty_io_host" } arty_macros = { path = "../arty_macros" } bolero = { workspace = true, features = ["std"] } @@ -99,6 +111,8 @@ pin-project = { workspace = true } plurality = { path = "../plurality", features = ["std"] } scopeguard = { workspace = true } seismograph = { path = "../seismograph", features = ["monitor"] } +seismograph_protocol = { path = "../seismograph_protocol", features = ["test-util"] } +seismograph_runtime = { path = "../seismograph_runtime" } static_assertions = { workspace = true } testing_aids = { path = "../testing_aids" } thread_aware = { path = "../thread_aware", features = ["std", "derive", "test-utils"] } @@ -123,6 +137,16 @@ name = "arty_contention" harness = false required-features = ["rt"] +[[bench]] +name = "arty_inspection" +harness = false +required-features = ["rt"] + +[[bench]] +name = "arty_spawn_allocations" +harness = false +required-features = ["rt"] + [[example]] name = "arty_advanced" required-features = ["rt"] diff --git a/crates/arty/README.md b/crates/arty/README.md index 8b2fde1406..a0ca77efc9 100644 --- a/crates/arty/README.md +++ b/crates/arty/README.md @@ -121,11 +121,13 @@ The default feature set enables `rt`, `macros`, and `monitor`. * `rt`: Enables the runtime, task, and I/O APIs. * `macros`: Enables [`main`][__link15] and [`test`][__link16]. It also enables `rt`. -* `monitor`: Enables the Seismograph monitor dependency. It forwards - monitoring support for `performables` when `performables` is enabled. It does - not start monitoring or emit new telemetry. +* `monitor`: Connects runtime workers and tasks to Seismograph. An enabled + [`MonitorPolicy`][__link17] starts a local connection for + the Seismograph CLI. Task collection runs on request. Event recording stays + off until an application or the CLI enables it. This feature also forwards + monitoring support for `performables` when `performables` is enabled. * `test-util`: Enables testing utilities, including - [`ClockControl`][__link17]. + [`ClockControl`][__link18]. Enable it in dev-dependencies, not production dependencies. @@ -134,7 +136,7 @@ The default feature set enables `rt`, `macros`, and `monitor`. This crate was developed as part of The Oxidizer Project. Browse this crate's source code. - [__cargo_doc2readme_dependencies_info]: ggGmYW0CYXZlMC43LjNhdIQborR2_k_xJd4bTcf2krrNPIcbP72Pw1UdRjkbim_eMDe2BBthYvRhcoQbVOOkiwtK_84bGyYJldq1SSMbSs5R1-8kbtkbwouPZ79D401hZIKCZGFydHllMC42LjCCa2FydHlfbWFjcm9zZTAuNi4w + [__cargo_doc2readme_dependencies_info]: ggGmYW0CYXZlMC43LjNhdIQborR2_k_xJd4bTcf2krrNPIcbP72Pw1UdRjkbim_eMDe2BBthYvRhcoQbzAp7t6NKScAb3WDVIfzt6sUbEShFs-Ipr28b7AssFBf93fFhZIKCZGFydHllMC42LjCCa2FydHlfbWFjcm9zZTAuNi4w [__link0]: https://doc.rust-lang.org/stable/std/marker/trait.Send.html [__link1]: https://docs.rs/arty/0.6.0/arty/?search=core::ThreadAware [__link10]: https://docs.rs/arty/0.6.0/arty/time/index.html @@ -144,7 +146,8 @@ This crate was developed as part of &'static str { + match self { + Self::Disabled => "disabled", + #[cfg(feature = "monitor")] + Self::Idle => "idle", + #[cfg(feature = "monitor")] + Self::Collecting => "collecting", + #[cfg(feature = "monitor")] + Self::Recording => "recording", + } + } +} + +#[derive(Clone, Copy)] +enum Workload { + LocalSpawn, + RepeatedPoll, + RemoteBatch, + BlockingBatch, +} + +impl Workload { + const ALL: [Self; 4] = [Self::LocalSpawn, Self::RepeatedPoll, Self::RemoteBatch, Self::BlockingBatch]; + + fn name(self) -> &'static str { + match self { + Self::LocalSpawn => "local_spawn_w1", + Self::RepeatedPoll => "repeated_poll_w1", + Self::RemoteBatch => "remote_batch_w4", + Self::BlockingBatch => "blocking_batch_w4", + } + } +} + +struct Case { + runtime: Runtime, + handles: Vec>, + collector: Option>, + stop: Arc, + captures: Arc, +} + +impl Case { + fn new(mode: Mode, workload: Workload) -> Self { + #[cfg(feature = "monitor")] + seismograph::recorder(Configuration::default()); + let workers = if matches!(workload, Workload::RemoteBatch | Workload::BlockingBatch) { + 4 + } else { + 1 + }; + let policy = if matches!(mode, Mode::Disabled) { + MonitorPolicy::disabled() + } else { + MonitorPolicy::enabled() + }; + let runtime = Runtime::builder() + .workers(WorkersPolicy::exactly(workers)) + .blocking_pool(BlockingPoolPolicy::shared().max(4)) + .monitor(policy) + .build() + .expect("benchmark runtime configuration is valid"); + let stop = Arc::new(AtomicBool::new(false)); + let captures = Arc::new(AtomicU64::new(0)); + #[cfg(feature = "monitor")] + let collector = matches!(mode, Mode::Collecting).then(|| { + let stop = Arc::clone(&stop); + let captures = Arc::clone(&captures); + let (ready, started) = mpsc::sync_channel(1); + let collector = thread::spawn(move || { + black_box(seismograph::snapshot(SnapshotOptions::default()).expect("benchmark collection succeeds")); + captures.fetch_add(1, Ordering::Relaxed); + ready.send(()).expect("benchmark caller waits for collection"); + while !stop.load(Ordering::Acquire) { + thread::park_timeout(Duration::from_millis(10)); + black_box(seismograph::snapshot(SnapshotOptions::default()).expect("benchmark collection succeeds")); + captures.fetch_add(1, Ordering::Relaxed); + } + }); + started + .recv_timeout(Duration::from_secs(5)) + .expect("benchmark collector must complete its first snapshot before measurement"); + collector + }); + #[cfg(not(feature = "monitor"))] + let collector = None; + #[cfg(feature = "monitor")] + if matches!(mode, Mode::Recording) { + seismograph::recorder(Configuration { + runtime_tasks: Policy::all(false), + ..Configuration::default() + }); + } + let mut case = Self { + runtime, + handles: Vec::with_capacity(usize::try_from(OPERATIONS).expect("64 operations fit in usize")), + collector, + stop, + captures, + }; + black_box(case.run(workload, 4)); + case + } + + fn run(&mut self, workload: Workload, iterations: u64) -> Duration { + let start = Instant::now(); + match workload { + Workload::LocalSpawn => { + let task = self + .runtime + .scheduler() + .spawn_anywhere(iterations * OPERATIONS, |cx, operations| async move { + for _ in 0..operations { + cx.scheduler() + .spawn(async |_| black_box(())) + .await + .expect("benchmark child succeeds"); + } + }); + futures::executor::block_on(task).expect("benchmark parent succeeds"); + } + Workload::RepeatedPoll => { + let task = self + .runtime + .scheduler() + .spawn_anywhere(iterations * OPERATIONS, |_, operations| RepeatedPoll(operations)); + futures::executor::block_on(task).expect("benchmark task succeeds"); + } + Workload::RemoteBatch | Workload::BlockingBatch => { + for _ in 0..iterations { + self.handles.clear(); + for _ in 0..OPERATIONS { + let task = if matches!(workload, Workload::RemoteBatch) { + self.runtime.scheduler().spawn_anywhere((), |_, ()| async { black_box(()) }) + } else { + self.runtime.scheduler().spawn_blocking(|| black_box(())) + }; + self.handles.push(task); + } + for task in &mut self.handles { + futures::executor::block_on(black_box(task)).expect("benchmark task succeeds"); + } + } + } + } + start.elapsed() + } +} + +impl Drop for Case { + fn drop(&mut self) { + self.stop.store(true, Ordering::Release); + if let Some(collector) = self.collector.take() { + collector.thread().unpark(); + collector.join().expect("benchmark diagnostic caller succeeds"); + eprintln!("Completed process snapshots: {}", self.captures.load(Ordering::Relaxed)); + } + #[cfg(feature = "monitor")] + seismograph::recorder(Configuration::default()); + } +} + +struct RepeatedPoll(u64); + +impl Future for RepeatedPoll { + type Output = (); + + fn poll(mut self: Pin<&mut Self>, cx: &mut PollContext<'_>) -> Poll<()> { + if self.0 == 0 { + Poll::Ready(()) + } else { + self.0 = black_box(self.0 - 1); + cx.waker().wake_by_ref(); + Poll::Pending + } + } +} + +fn benchmarks(criterion: &mut Criterion) { + let mut group = criterion.benchmark_group("arty_inspection/tasks"); + group.throughput(Throughput::Elements(OPERATIONS)); + for workload in Workload::ALL { + for &mode in Mode::ALL { + let mut case = None; + group.bench_function(BenchmarkId::new(workload.name(), mode.name()), |bencher| { + let case = case.get_or_insert_with(|| Case::new(mode, workload)); + bencher.iter_custom(|iterations| case.run(workload, iterations)); + }); + } + } + group.finish(); +} + +criterion_group! { + name = inspection; + config = Criterion::default() + .sample_size(50) + .warm_up_time(Duration::from_secs(1)) + .measurement_time(Duration::from_secs(3)); + targets = benchmarks +} +criterion_main!(inspection); diff --git a/crates/arty/benches/arty_spawn_allocations.rs b/crates/arty/benches/arty_spawn_allocations.rs new file mode 100644 index 0000000000..25caf2a360 --- /dev/null +++ b/crates/arty/benches/arty_spawn_allocations.rs @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Counts process-wide allocations for warmed async spawning. +//! +//! Run each case in a separate process. Select `local` or `remote`, followed by +//! `idle` or `disabled`. To exclude monitoring dependencies, build this source +//! in a separate package with Arty as a normal, runtime-only dependency. +//! Arty's own dev-dependencies can enable monitoring through feature unification. +//! Runtime setup and warm-up are excluded. +//! Unrelated process threads can contribute allocations. The local case measures +//! 4096 sequential children in one parent. The remote case measures 64 batches +//! of 64 tasks on four workers, including dispatch and joins. + +use std::alloc::System; +use std::hint::black_box; + +use alloc_tracker::{Allocator, Session}; +use arty::runtime::{MonitorPolicy, Runtime, WorkersPolicy}; + +#[global_allocator] +static ALLOCATOR: Allocator = Allocator::system(); + +fn main() -> Result<(), &'static str> { + let arguments = std::env::args().skip(1).collect::>(); + let [workload, mode] = arguments.as_slice() else { + return Err("pass a workload (local or remote) and a monitoring mode (idle or disabled)"); + }; + let local = match workload.as_str() { + "local" => true, + "remote" => false, + _ => return Err("workload must be local or remote"), + }; + let monitor = match mode.as_str() { + "idle" => MonitorPolicy::enabled(), + "disabled" => MonitorPolicy::disabled(), + _ => return Err("monitoring mode must be idle or disabled"), + }; + let runtime = Runtime::builder() + .workers(WorkersPolicy::exactly(if local { 1 } else { 4 })) + .monitor(monitor) + .build() + .expect("benchmark requires the configured worker count"); + let session = Session::new().no_stdout().no_file(); + let operation = session.operation("spawn"); + let mut handles = Vec::with_capacity(64); + for measured in [false, true] { + let measurement = measured.then(|| operation.measure_process().iterations(4096)); + if local { + let parent = runtime.scheduler().spawn_anywhere((), |cx, ()| async move { + for _ in 0..4096 { + cx.scheduler() + .spawn(async |_| black_box(())) + .await + .expect("benchmark child succeeds"); + } + }); + futures::executor::block_on(parent).expect("benchmark parent succeeds"); + } else { + for _ in 0..64 { + handles.clear(); + for _ in 0..64 { + handles.push(runtime.scheduler().spawn_anywhere((), |_, ()| async { black_box(()) })); + } + for task in &mut handles { + futures::executor::block_on(task).expect("benchmark remote task succeeds"); + } + } + } + drop(measurement); + } + let report = session.to_report(); + let (_, operation) = report + .operations() + .find(|(name, _)| *name == "spawn") + .expect("the measured operation exists"); + println!( + "{workload},{mode},{},{},{}", + operation.total_iterations(), + operation.total_allocations_count(), + operation.total_bytes_allocated() + ); + runtime.stop().expect("benchmark runtime stops"); + Ok(()) +} diff --git a/crates/arty/src/lib.rs b/crates/arty/src/lib.rs index c23a29894e..328216193a 100644 --- a/crates/arty/src/lib.rs +++ b/crates/arty/src/lib.rs @@ -132,9 +132,11 @@ //! //! - `rt`: Enables the runtime, task, and I/O APIs. //! - `macros`: Enables [`main`] and [`test`]. It also enables `rt`. -//! - `monitor`: Enables the Seismograph monitor dependency. It forwards -//! monitoring support for `performables` when `performables` is enabled. It does -//! not start monitoring or emit new telemetry. +//! - `monitor`: Connects runtime workers and tasks to Seismograph. An enabled +//! [`MonitorPolicy`](crate::runtime::MonitorPolicy) starts a local connection for +//! the Seismograph CLI. Task collection runs on request. Event recording stays +//! off until an application or the CLI enables it. This feature also forwards +//! monitoring support for `performables` when `performables` is enabled. //! - `test-util`: Enables testing utilities, including //! [`ClockControl`](crate::time::ClockControl). //! Enable it in dev-dependencies, not production dependencies. @@ -165,7 +167,8 @@ use arty_io_core as _; /// causes runtime construction to fail. /// - `workers = all` starts one async worker per available processor. /// - `monitor = enabled` or `monitor = disabled` selects whether the runtime -/// is prepared for monitoring. The default is `enabled`. +/// starts a local Seismograph monitor when the `monitor` feature is present. +/// The default is `enabled`. /// - `panic = shutdown` or `panic = ignore` selects whether the runtime stops /// after Arty catches a task panic. The default is `shutdown`. /// @@ -268,7 +271,6 @@ pub use arty_macros::main; /// `monitor`, and `panic` options also apply. /// By default, a test uses one worker, disables monitoring, and stops its /// runtime after Arty catches a task panic. -/// Ordinary tests therefore keep monitoring disabled if Arty adds automatic monitoring later. /// To test monitoring, use `monitor = enabled`. /// /// # Simulated time diff --git a/crates/arty/src/runtime/bootstrap/startup.rs b/crates/arty/src/runtime/bootstrap/startup.rs index 84ae637742..a87a5a78a6 100644 --- a/crates/arty/src/runtime/bootstrap/startup.rs +++ b/crates/arty/src/runtime/bootstrap/startup.rs @@ -27,7 +27,8 @@ fn build_with_processors( ) -> Result { let processors = processor_config.workers_policy.select(available)?; - let mut workers = Vec::with_capacity(processors.len()); + let worker_count = processors.len(); + let mut workers = Vec::with_capacity(worker_count); // Respect the RUST_MIN_STACK environment variable if set and larger than the configured stack size. let stack_size = processor_config @@ -40,6 +41,13 @@ fn build_with_processors( let shutdown = Shutdown::default(); let io_runtime = IoRuntime::new(observed::Sink::noop()); let blocking_pools = processor_config.blocking_pool_policy.into_pools()?; + #[cfg(feature = "monitor")] + let (diagnostics, monitor_options) = { + let diagnostics = crate::runtime::monitoring::RuntimeDiagnostics::start(monitor_options.policy(), worker_count)?; + let mut monitor_options = monitor_options; + monitor_options.set_runtime(diagnostics.as_ref().map(crate::runtime::monitoring::RuntimeDiagnostics::handle)); + (diagnostics, monitor_options) + }; for (worker_index, processor) in processors.into_iter().enumerate() { workers.push(asynchronous::start(asynchronous::StartOptions { @@ -63,6 +71,8 @@ fn build_with_processors( ); let dispatcher_client = DispatcherClient::new(dispatcher); + #[cfg(feature = "monitor")] + dispatcher_client.set_diagnostics(monitor_options.runtime().cloned()); let mut join_handles = Vec::with_capacity(workers.len()); for worker in workers { @@ -70,7 +80,14 @@ fn build_with_processors( } let worker_joiner = WorkerJoiner::new(join_handles); - Ok(Runtime::with_dispatcher(dispatcher_client, worker_joiner)) + let runtime = Runtime::with_dispatcher(dispatcher_client, worker_joiner); + #[cfg(feature = "monitor")] + let runtime = { + let mut runtime = runtime; + runtime.set_diagnostics(diagnostics); + runtime + }; + Ok(runtime) } fn available_processors() -> ProcessorSet { diff --git a/crates/arty/src/runtime/builder.rs b/crates/arty/src/runtime/builder.rs index 41569008a1..b3ba0043ca 100644 --- a/crates/arty/src/runtime/builder.rs +++ b/crates/arty/src/runtime/builder.rs @@ -64,8 +64,9 @@ impl RuntimeBuilder { /// /// The default is [`MonitorPolicy::enabled`]. The `monitor` Cargo feature /// enables the Seismograph dependencies that monitoring needs. - /// This choice does not start a monitor, add task measurements, or change - /// which tasks run. + /// Building an enabled runtime starts a local connection for the Seismograph CLI. + /// Task collection runs on request. This does not enable event recording or + /// change which tasks run. See the [monitoring guide](crate::runtime#monitoring). /// /// # Examples /// @@ -193,6 +194,7 @@ impl RuntimeBuilder { /// # Errors /// /// Returns [`Error`] if the worker count or blocking-pool limit is zero. + /// With monitoring enabled, also returns an error if the local monitor cannot start. /// /// # Panics /// diff --git a/crates/arty/src/runtime/config/monitor_options.rs b/crates/arty/src/runtime/config/monitor_options.rs index c8513751b0..91a70b2dd4 100644 --- a/crates/arty/src/runtime/config/monitor_options.rs +++ b/crates/arty/src/runtime/config/monitor_options.rs @@ -7,6 +7,8 @@ use crate::runtime::config::MonitorPolicy; #[derive(Clone, Debug)] pub(crate) struct MonitorOptions { policy: MonitorPolicy, + #[cfg(feature = "monitor")] + runtime: Option, } impl MonitorOptions { @@ -19,6 +21,16 @@ impl MonitorOptions { pub(crate) const fn policy(&self) -> &MonitorPolicy { &self.policy } + + #[cfg(feature = "monitor")] + pub(crate) fn set_runtime(&mut self, runtime: Option) { + self.runtime = runtime; + } + + #[cfg(feature = "monitor")] + pub(crate) fn runtime(&self) -> Option<&seismograph_runtime::RuntimeHandle> { + self.runtime.as_ref() + } } impl Default for MonitorOptions { @@ -29,7 +41,11 @@ impl Default for MonitorOptions { impl From for MonitorOptions { fn from(policy: MonitorPolicy) -> Self { - Self { policy } + Self { + policy, + #[cfg(feature = "monitor")] + runtime: None, + } } } diff --git a/crates/arty/src/runtime/config/monitor_policy.rs b/crates/arty/src/runtime/config/monitor_policy.rs index 1794d42d19..bcd1aa77ed 100644 --- a/crates/arty/src/runtime/config/monitor_policy.rs +++ b/crates/arty/src/runtime/config/monitor_policy.rs @@ -8,8 +8,9 @@ /// The enabled policy is the default. /// /// The `monitor` Cargo feature enables the Seismograph dependencies that -/// monitoring needs. This choice does not start a monitor, add task measurements, -/// or change which tasks run. +/// monitoring needs. Building an enabled runtime starts a local monitor. +/// Task collection and poll timing run only when requested. This policy does +/// not enable event recording or install an allocator. /// /// # Examples /// @@ -22,29 +23,73 @@ /// runtime.stop()?; /// # Ok::<(), Box>(()) /// ``` -#[repr(transparent)] #[derive(Debug, Clone)] pub struct MonitorPolicy { enabled: bool, + name: Option, + #[cfg(feature = "monitor")] + io: Option, } impl MonitorPolicy { /// Creates a policy with monitoring enabled in the runtime configuration. #[must_use] pub const fn enabled() -> Self { - Self { enabled: true } + Self { + enabled: true, + name: None, + #[cfg(feature = "monitor")] + io: None, + } } /// Creates a policy with monitoring disabled in the runtime configuration. #[must_use] pub const fn disabled() -> Self { - Self { enabled: false } + Self { + enabled: false, + name: None, + #[cfg(feature = "monitor")] + io: None, + } + } + + /// Sets the runtime name shown by the Seismograph CLI. + /// + /// The default name is `arty`. Each runtime has a separate local endpoint. + #[must_use] + pub fn name(mut self, name: impl Into) -> Self { + self.name = Some(name.into()); + self + } + + /// Selects isolated monitor I/O for tests. + /// + /// This does not change the process-wide recorder or runtime source. + #[cfg(all(feature = "monitor", any(test, feature = "test-util")))] + #[must_use] + pub fn io(mut self, io: seismograph_protocol::io::MonitorIo) -> Self { + self.io = Some(io); + self } - #[cfg(test)] pub(crate) const fn is_enabled(&self) -> bool { self.enabled } + + #[cfg(feature = "monitor")] + pub(crate) fn runtime_name(&self) -> &str { + self.name.as_deref().unwrap_or("arty") + } + + #[cfg(feature = "monitor")] + pub(crate) fn monitor_io(&self) -> Option { + #[cfg(all(miri, any(test, feature = "test-util")))] + if self.io.is_none() { + return Some(seismograph_protocol::io::MonitorIo::new_mocked().0); + } + self.io.clone() + } } impl Default for MonitorPolicy { diff --git a/crates/arty/src/runtime/dispatch/client.rs b/crates/arty/src/runtime/dispatch/client.rs index 40ad6c1787..949eb99921 100644 --- a/crates/arty/src/runtime/dispatch/client.rs +++ b/crates/arty/src/runtime/dispatch/client.rs @@ -25,6 +25,16 @@ pub(crate) struct DispatcherClient { } impl DispatcherClient { + #[cfg(feature = "monitor")] + pub(crate) fn set_diagnostics(&self, runtime: Option) { + self.core.set_diagnostics(runtime); + } + + #[cfg(feature = "monitor")] + pub(crate) fn register_task_provider(&self, thread: ThreadId, handle: &seismograph_runtime::worker::WorkerHandle) { + self.core.register_task_provider(thread, handle); + } + pub(crate) const fn new(core: Arc) -> Self { Self { core } } diff --git a/crates/arty/src/runtime/dispatch/core.rs b/crates/arty/src/runtime/dispatch/core.rs index 16395236f4..2484e0692b 100644 --- a/crates/arty/src/runtime/dispatch/core.rs +++ b/crates/arty/src/runtime/dispatch/core.rs @@ -68,6 +68,8 @@ pub(crate) struct DispatcherCore { next_async_worker_index: AtomicUsize, remote_results: JoinHandlePool, + #[cfg(feature = "monitor")] + diagnostics: std::sync::OnceLock, } impl DispatcherCore { @@ -101,12 +103,45 @@ impl DispatcherCore { worker_indices, next_async_worker_index: AtomicUsize::new(0), remote_results: JoinHandlePool::new(shutdown), + #[cfg(feature = "monitor")] + diagnostics: std::sync::OnceLock::new(), }) } + #[cfg(feature = "monitor")] + pub(crate) fn set_diagnostics(&self, runtime: Option) { + if let Some(runtime) = runtime { + self.diagnostics + .set(runtime) + .expect("bootstrap sets runtime diagnostics exactly once"); + } + } + + #[cfg(feature = "monitor")] + pub(crate) fn register_task_provider(&self, thread: ThreadId, handle: &seismograph_runtime::worker::WorkerHandle) { + let index = self + .worker_index(thread) + .expect("worker diagnostics use a registered runtime thread"); + let endpoint = &self.worker_endpoints[usize::from(index)]; + let commands = endpoint.command_tx.clone(); + let waker = endpoint.waker.clone(); + handle + .register_task_provider(move |request| { + if commands.send(Command::Inspect { request }).is_ok() { + waker.wake_by_ref(); + } + // A failed send drops the request and reports collection failure. + }) + .expect("bootstrap registers providers before their workers can stop"); + } + #[cfg_attr(test, mutants::skip)] // Tests hang if mutation removes the shutdown signal or worker notifications. pub(crate) fn stop_inner(&self, cause: ShutdownCause) -> ShutdownCause { self.shutdown().begin(cause); + #[cfg(feature = "monitor")] + if let Some(runtime) = self.diagnostics.get() { + runtime.stopping(); + } let cause = self.shutdown().cause().unwrap_or_default(); // Shutdown stores the cause independently. This atomic only elects the // broadcaster. Observing `true` does not mean notification is complete. diff --git a/crates/arty/src/runtime/handle.rs b/crates/arty/src/runtime/handle.rs index 005e815806..0f0113d3f2 100644 --- a/crates/arty/src/runtime/handle.rs +++ b/crates/arty/src/runtime/handle.rs @@ -56,6 +56,8 @@ pub struct Runtime { scheduler: RuntimeScheduler, worker_joiner: WorkerJoiner, shutdown_on_drop: bool, + #[cfg(feature = "monitor")] + diagnostics: Option, } impl Runtime { @@ -134,6 +136,19 @@ impl Runtime { &self.scheduler } + /// Returns the runtime's diagnostic handle for collection tests. + /// + /// Returns `None` when monitoring is disabled. Capture from a diagnostic + /// thread, not from an Arty worker that must answer the collection request. + /// This does not change the process-wide recorder. + #[cfg(all(feature = "monitor", any(test, feature = "test-util")))] + #[must_use] + pub fn diagnostics(&self) -> Option { + self.diagnostics + .as_ref() + .map(crate::runtime::monitoring::RuntimeDiagnostics::handle) + } + /// Takes ownership of this runtime, requests shutdown, and waits for its workers to stop. /// /// Cancels pending async tasks and prevents queued blocking functions from @@ -149,6 +164,8 @@ impl Runtime { /// Returns an error if a worker panicked or an I/O driver failed during /// startup or normal operation. It does not return I/O driver shutdown /// failures. It waits for all workers to stop before it reports a failure. + /// With monitoring enabled, also reports a stored monitor failure unless a + /// worker or runtime failure takes precedence. /// To receive a task failure, await its [`JoinHandle`](crate::task::JoinHandle). /// /// # Panics @@ -186,19 +203,41 @@ impl Runtime { self.wait() } - fn wait(&self) -> Result<(), Error> { + #[cfg_attr( + not(feature = "monitor"), + expect( + clippy::needless_pass_by_ref_mut, + reason = "monitoring consumes the diagnostic registration after workers stop" + ) + )] + fn wait(&mut self) -> Result<(), Error> { let wait_error = self.worker_joiner.join().err(); + #[cfg(feature = "monitor")] + let monitor_error = self.diagnostics.take().and_then(|diagnostics| diagnostics.finish().err()); if let Some(error) = self.scheduler.dispatcher.shutdown().error() { return Err(Error::shutdown_failure(error)); } + #[cfg(feature = "monitor")] + if let Some(error) = monitor_error + && wait_error.is_none() + { + return Err(error); + } wait_error.map_or(Ok(()), Err) } + #[cfg(feature = "monitor")] + pub(crate) fn set_diagnostics(&mut self, diagnostics: Option) { + self.diagnostics = diagnostics; + } + pub(crate) const fn with_dispatcher(dispatcher: DispatcherClient, worker_joiner: WorkerJoiner) -> Self { Self { scheduler: RuntimeScheduler::new(dispatcher), worker_joiner, shutdown_on_drop: true, + #[cfg(feature = "monitor")] + diagnostics: None, } } } diff --git a/crates/arty/src/runtime/mod.rs b/crates/arty/src/runtime/mod.rs index c451b677dd..fec5743959 100644 --- a/crates/arty/src/runtime/mod.rs +++ b/crates/arty/src/runtime/mod.rs @@ -43,14 +43,37 @@ //! default thread limits can change. Use [`BlockingPoolPolicy::max`] to set //! a maximum. //! - The clock follows real time. -//! - [`MonitorPolicy::default`] enables monitoring in the runtime configuration. -//! This choice does not start a monitor or add task measurements. +//! - [`MonitorPolicy::default`] starts a local Seismograph monitor when the +//! `monitor` feature is present. Task collection runs on request. This does +//! not enable event recording. //! - [`PanicPolicy::default`] stops the runtime after Arty catches a panic //! while it creates or polls a task, or runs a blocking function. //! //! The [`test`](crate::test) attribute disables monitoring by default. //! See [`main`](crate::main) and [`test`](crate::test) for their other defaults. //! +//! ## Monitoring +//! +//! A monitor is a local connection through which the Seismograph CLI can +//! collect diagnostics. Each enabled runtime owns one connection and removes it +//! after its workers stop. [`MonitorPolicy::name`] sets the displayed runtime name. +//! Use [`MonitorPolicy::disabled`] to run without this connection or task inspection. +//! +//! Collection reads live async tasks and queued or running blocking functions +//! from their native storage. Each request limits the number of tasks per worker. +//! A blocked async worker can time out. The result reports incomplete collection +//! rather than treating missing tasks as an empty worker. +//! +//! Poll timing runs only during a requested measurement interval or event recording. +//! Completed tasks are not kept in a second task registry. +//! History that was not recorded stays unknown. A blocking function that started +//! before observation has no measured start time for that call. +//! +//! Monitor connections are separate, but the recorder and diagnostic sources are +//! process-wide. Recording, clearing event buffers, and snapshot capture through +//! one connection can affect or include other runtimes in the same process. +//! Monitor startup does not change recorder configuration or install an allocator. +//! //! ## Policy types //! //! - [`WorkersPolicy`] selects how many async workers run and which processors they use. @@ -132,6 +155,8 @@ mod control; pub(crate) mod dispatch; mod error; mod handle; +#[cfg(feature = "monitor")] +pub(crate) mod monitoring; mod shutdown; pub(crate) mod worker; mod workers; diff --git a/crates/arty/src/runtime/monitoring/asynchronous.rs b/crates/arty/src/runtime/monitoring/asynchronous.rs new file mode 100644 index 0000000000..02b10de72d --- /dev/null +++ b/crates/arty/src/runtime/monitoring/asynchronous.rs @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::any::Any; +use std::cell::{Cell, RefCell}; + +use arty_executor::{Executor, InspectionError, InspectionStatus, PollObserver}; +use seismograph_runtime::RuntimeHandle; +use seismograph_runtime::snapshot::TaskCollection; +use seismograph_runtime::task::{TaskPoll, TaskRequest}; +use seismograph_runtime::worker::{WorkerHandle, WorkerMetadata, WorkerRegistration, WorkerRole}; + +use crate::runtime::dispatch::DispatcherClient; +use crate::runtime::monitoring::task::{Description, set_current, set_events}; + +#[derive(Debug)] +pub(crate) struct WorkerDiagnostics { + _registration: WorkerRegistration, + handle: WorkerHandle, + poll: RefCell>, + failed: Cell, +} + +impl WorkerDiagnostics { + pub(crate) fn new(runtime: &RuntimeHandle, dispatcher: &DispatcherClient, thread: &thread_aware::Thread) -> Self { + let registration = runtime.register_worker(WorkerMetadata::new(WorkerRole::Core)); + let handle = registration.handle(); + dispatcher.register_task_provider(thread.id(), &handle); + registration.attach_current_thread(); + set_events(Some(handle.task_events())); + Self { + _registration: registration, + handle, + poll: RefCell::new(None), + failed: Cell::new(false), + } + } + + pub(crate) fn is_observed(&self) -> bool { + self.handle.is_observed() + } + + pub(crate) fn collect(&self, executor: &mut Executor, request: TaskRequest) { + if request.is_canceled() { + return; + } + let mut tasks = Vec::new(); + let status = match executor.inspect(request.limit(), |description: &Description| { + if let Description::Snapshot(task) = description { + tasks.push(task.clone()); + } + }) { + Ok(InspectionStatus::Complete) => TaskCollection::Complete, + Ok(InspectionStatus::Truncated) => TaskCollection::Truncated, + Err(InspectionError::Busy) => TaskCollection::Busy, + Err(InspectionError::InvalidLimit | InspectionError::Panicked) => TaskCollection::Failed, + }; + let status = if self.failed.replace(false) { + TaskCollection::Failed + } else { + status + }; + request.respond(tasks, status); + } + + pub(crate) fn parked(&self) { + self.handle.parked(); + } + + pub(crate) fn unparked(&self) { + self.handle.unparked(); + } +} + +impl PollObserver for WorkerDiagnostics { + fn poll_started(&self, description: &dyn Any) { + if let Some(Description::Poll(id)) = description.downcast_ref::() { + set_current(Some(*id)); + *self.poll.borrow_mut() = Some(self.handle.task_poll_started(*id)); + } else { + self.failed.set(true); + } + } + + fn poll_finished(&self, _completed: bool) { + set_current(None); + if let Some(poll) = self.poll.borrow_mut().take() { + self.handle.task_poll_finished(poll); + } + } + + fn inspection_failed(&self, _error: InspectionError) { + self.failed.set(true); + } +} + +impl Drop for WorkerDiagnostics { + fn drop(&mut self) { + set_current(None); + set_events(None); + } +} diff --git a/crates/arty/src/runtime/monitoring/mod.rs b/crates/arty/src/runtime/monitoring/mod.rs new file mode 100644 index 0000000000..48bf4876ca --- /dev/null +++ b/crates/arty/src/runtime/monitoring/mod.rs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Runtime-owned Seismograph connections and native task inspection. + +mod asynchronous; +mod runtime; +mod task; + +pub(crate) use asynchronous::WorkerDiagnostics; +pub(crate) use runtime::RuntimeDiagnostics; +pub(crate) use task::{TaskInfo, add_task}; diff --git a/crates/arty/src/runtime/monitoring/runtime.rs b/crates/arty/src/runtime/monitoring/runtime.rs new file mode 100644 index 0000000000..93b42e35be --- /dev/null +++ b/crates/arty/src/runtime/monitoring/runtime.rs @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use seismograph::monitor::Monitor; +use seismograph_runtime::{RuntimeHandle, RuntimeMetadata, RuntimeRegistration}; + +use crate::runtime::{Error, MonitorPolicy}; + +#[derive(Debug)] +pub(crate) struct RuntimeDiagnostics { + monitor: Monitor, + registration: RuntimeRegistration, +} + +impl RuntimeDiagnostics { + pub(crate) fn start(policy: &MonitorPolicy, workers: usize) -> Result, Error> { + if !policy.is_enabled() { + return Ok(None); + } + let workers = u32::try_from(workers).map_err(Error::new)?; + let mut builder = Monitor::builder().name(policy.runtime_name()); + if let Some(io) = policy.monitor_io() { + builder = builder.io(io); + } + let monitor = builder.start().map_err(Error::new)?; + let registration = seismograph_runtime::register_runtime(RuntimeMetadata::new(policy.runtime_name(), workers)); + Ok(Some(Self { monitor, registration })) + } + + pub(crate) fn handle(&self) -> RuntimeHandle { + self.registration.handle() + } + + pub(crate) fn finish(self) -> Result<(), Error> { + self.registration.stopped(); + let error = self.monitor.last_error(); + drop(self); + error.map_or(Ok(()), |message| Err(Error::new(MonitorFailure::new(message)))) + } +} + +#[ohno::error] +#[display("the runtime monitor failed: {message}")] +struct MonitorFailure { + message: String, +} diff --git a/crates/arty/src/runtime/monitoring/task.rs b/crates/arty/src/runtime/monitoring/task.rs new file mode 100644 index 0000000000..92f521bf5a --- /dev/null +++ b/crates/arty/src/runtime/monitoring/task.rs @@ -0,0 +1,258 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::any::TypeId; +use std::cell::{Cell, OnceCell, RefCell}; +use std::collections::HashMap; +use std::sync::OnceLock; + +use arty_executor::{ShutdownCause, TaskCompletion, TaskOptions, TaskPanicPayload, TaskSet, TaskState}; +use seismograph::recorder::event::EventTimestamp; +use seismograph::recorder::runtime::{TaskId, TypeDescriptorId}; +use seismograph_runtime::snapshot::{Task, TaskActivityState}; +use seismograph_runtime::task::EventContext; + +thread_local! { + static EVENTS: Cell> = const { Cell::new(None) }; + static CURRENT: Cell> = const { Cell::new(None) }; + // This cache contains types, not live tasks. It is touched only during observation. + static TYPES: RefCell> = RefCell::new(HashMap::new()); +} + +pub(crate) fn current_events() -> Option { + EVENTS.with(Cell::get) +} + +pub(super) fn set_events(events: Option) { + EVENTS.with(|slot| slot.set(events)); +} + +pub(super) fn set_current(task: Option) { + CURRENT.with(|slot| slot.set(task)); +} + +#[expect( + clippy::large_enum_variant, + reason = "descriptions are transient stack values; boxing snapshots would allocate during native inspection" +)] +pub(super) enum Description { + Snapshot(Task), + Poll(TaskId), +} + +fn descriptor() -> TypeDescriptorId { + TYPES.with_borrow_mut(|types| { + *types + .entry(TypeId::of::()) + .or_insert_with(seismograph_runtime::allocate_type_descriptor_id) + }) +} + +#[derive(Debug)] +struct TaskMetadata { + describe_type: fn() -> TypeDescriptorId, + body_size: usize, + events: EventContext, + parent: Option, + spawned_at: Option, +} + +impl TaskMetadata { + fn new(events: EventContext) -> Self { + Self { + describe_type: descriptor::, + body_size: size_of::(), + events, + parent: CURRENT.with(Cell::get), + spawned_at: None, + } + } + + fn record_spawn(&mut self) -> Option { + self.events + .spawned_with(self.describe_type, self.parent, self.body_size) + .map(|spawn| { + self.spawned_at = Some(spawn.timestamp); + TaskIdentity { + id: spawn.id, + descriptor: spawn.descriptor, + } + }) + } + + fn identity(&self) -> TaskIdentity { + TaskIdentity { + id: seismograph_runtime::allocate_task_id(), + descriptor: (self.describe_type)(), + } + } + + fn describe(&self, identity: &TaskIdentity, state: TaskActivityState) -> Task { + let mut task = Task::new(identity.id, identity.descriptor, state); + task.parent = self.parent; + task.spawned_at = self.spawned_at; + task.future_size_bytes = Some(u64::try_from(self.body_size).expect("a task body fits the platform address space")); + task + } +} + +#[derive(Debug)] +struct TaskIdentity { + id: TaskId, + descriptor: TypeDescriptorId, +} + +#[derive(Debug)] +pub(crate) struct TaskInfo { + metadata: TaskMetadata, + identity: OnceLock, + execution_events: OnceLock, +} + +impl TaskInfo { + pub(crate) fn new(events: EventContext) -> Self { + let mut metadata = TaskMetadata::new::(events); + let identity = metadata.record_spawn().map_or_else(OnceLock::new, OnceLock::from); + Self { + metadata, + identity, + execution_events: OnceLock::new(), + } + } + + pub(crate) fn id(&self) -> TaskId { + self.identity.get_or_init(|| self.metadata.identity()).id + } + + pub(crate) fn describe(&self, state: TaskActivityState) -> Task { + self.metadata + .describe(self.identity.get_or_init(|| self.metadata.identity()), state) + } + + pub(crate) fn started_on(&self, events: EventContext) { + self.execution_events + .set(events) + .expect("a blocking function starts on exactly one worker"); + } + + pub(crate) fn completed(&self) { + if let Some(identity) = self.identity.get() { + self.execution_events.get().unwrap_or(&self.metadata.events).completed(identity.id); + } + } + + pub(crate) fn panicked(&self) { + if let Some(identity) = self.identity.get() { + self.execution_events.get().unwrap_or(&self.metadata.events).panicked(identity.id); + } + } + + pub(crate) fn canceled(&self) { + if let Some(identity) = self.identity.get() { + self.execution_events.get().unwrap_or(&self.metadata.events).canceled(identity.id); + } + } +} + +// Async metadata stays in the executor's owner-only payload. Blocking metadata +// above remains synchronized because queue inspection and execution can overlap. +#[derive(Debug)] +struct AsyncTaskInfo { + events: EventContext, + parent: Option, + spawned_at: Option, + identity: OnceCell, +} + +impl AsyncTaskInfo { + fn new(events: EventContext) -> Self { + let parent = CURRENT.with(Cell::get); + let spawn = events.spawned_with(descriptor::, parent, size_of::()); + Self { + events, + parent, + spawned_at: spawn.map(|spawn| spawn.timestamp), + identity: spawn.map_or_else(OnceCell::new, |spawn| OnceCell::from(spawn.id)), + } + } + + fn id(&self) -> TaskId { + *self.identity.get_or_init(seismograph_runtime::allocate_task_id) + } + + fn describe(&self, state: TaskActivityState) -> Task { + let mut task = Task::new(self.id(), descriptor::(), state); + task.parent = self.parent; + task.spawned_at = self.spawned_at; + task.future_size_bytes = Some(u64::try_from(size_of::()).expect("a task body fits the platform address space")); + task + } +} + +#[derive(Debug)] +struct Completion { + target: T, + info: AsyncTaskInfo, +} + +impl> TaskCompletion for Completion { + fn completed(self, result: R) { + if let Some(id) = self.info.identity.get() { + self.info.events.completed(*id); + } + self.target.completed(result); + } + + fn panicked(self, panic: TaskPanicPayload, cause: Option) { + if let Some(id) = self.info.identity.get() { + self.info.events.panicked(*id); + } + self.target.panicked(panic, cause); + } + + fn cancelled(self, cause: ShutdownCause) { + if let Some(id) = self.info.identity.get() { + self.info.events.canceled(*id); + } + self.target.cancelled(cause); + } +} + +pub(crate) fn add_task(tasks: &TaskSet, future: F, options: TaskOptions, completion: T) +where + Body: 'static, + F: Future + 'static, + R: 'static, + T: TaskCompletion, +{ + let Some(events) = current_events() else { + tasks.add(future, options, completion); + return; + }; + let completion = Completion { + target: completion, + info: AsyncTaskInfo::new::(events), + }; + tasks.add_with_completion_inspection(future, options, completion, |_, completion, state| { + let info = &completion.info; + Some(seismograph::recorder::without_recording(|| match state { + TaskState::Ready => Description::Snapshot(info.describe::(TaskActivityState::Ready)), + TaskState::Waiting => Description::Snapshot(info.describe::(TaskActivityState::Waiting)), + TaskState::Running => Description::Poll(info.id()), + })) + }); +} + +#[cfg(test)] +mod tests { + use super::{AsyncTaskInfo, TaskInfo}; + + #[test] + fn async_metadata_is_smaller_and_stays_owner_local() { + static_assertions::assert_not_impl_any!(AsyncTaskInfo: Sync); + static_assertions::assert_impl_all!(TaskInfo: Send, Sync); + assert!(size_of::() < size_of::()); + #[cfg(target_pointer_width = "64")] + assert_eq!(size_of::(), 56); + } +} diff --git a/crates/arty/src/runtime/worker/asynchronous/protocol.rs b/crates/arty/src/runtime/worker/asynchronous/protocol.rs index da38bf4636..54120974a0 100644 --- a/crates/arty/src/runtime/worker/asynchronous/protocol.rs +++ b/crates/arty/src/runtime/worker/asynchronous/protocol.rs @@ -63,6 +63,10 @@ pub(crate) enum Command { /// Executes synchronous I/O context access on this worker. AccessIo { access: QueuedIoAccess }, + /// Copies task descriptions from native executor storage between cycles. + #[cfg(feature = "monitor")] + Inspect { request: seismograph_runtime::task::TaskRequest }, + /// Starts shutdown on this worker. /// /// The worker stops accepting new tasks. It discards tasks enqueued after @@ -78,6 +82,8 @@ impl fmt::Debug for Command { match self { Self::EnqueueTask { .. } => write!(f, "EnqueueTask"), Self::AccessIo { .. } => write!(f, "AccessIo"), + #[cfg(feature = "monitor")] + Self::Inspect { .. } => write!(f, "Inspect"), Self::Shutdown => write!(f, "Shutdown"), } } diff --git a/crates/arty/src/runtime/worker/asynchronous/thread.rs b/crates/arty/src/runtime/worker/asynchronous/thread.rs index 8edf17e650..08fdde74e5 100644 --- a/crates/arty/src/runtime/worker/asynchronous/thread.rs +++ b/crates/arty/src/runtime/worker/asynchronous/thread.rs @@ -147,7 +147,12 @@ impl ThreadEntry { arty_executor::PanicPolicy::Ignore => blocking::PanicPolicy::Ignore, arty_executor::PanicPolicy::Shutdown => blocking::PanicPolicy::Shutdown(worker_waker.clone()), }; - let blocking_worker = blocking::Worker::new(blocking_pools.pool_for_worker(), shutdown, blocking_panic_policy); + let blocking_pool = blocking_pools.pool_for_worker(); + #[cfg(feature = "monitor")] + if let Some(runtime) = monitor_options.runtime() { + blocking_pool.enable_monitoring(runtime); + } + let blocking_worker = blocking::Worker::new(blocking_pool, shutdown, blocking_panic_policy); endpoint_tx .send(Endpoint { diff --git a/crates/arty/src/runtime/worker/asynchronous/worker.rs b/crates/arty/src/runtime/worker/asynchronous/worker.rs index 4ad5521fc3..b2fe90ccb7 100644 --- a/crates/arty/src/runtime/worker/asynchronous/worker.rs +++ b/crates/arty/src/runtime/worker/asynchronous/worker.rs @@ -92,6 +92,8 @@ pub(crate) struct Worker { cycle_clock: SimpleClock, signal: Arc, monitor_options: MonitorOptions, + #[cfg(feature = "monitor")] + diagnostics: Option, _single_threaded: PhantomData<*const ()>, } @@ -123,6 +125,10 @@ impl Worker { let tasks = executor.tasks(); let executor_shutdown = executor.shutdown(); + #[cfg(feature = "monitor")] + let diagnostics = monitor_options + .runtime() + .map(|runtime| crate::runtime::monitoring::WorkerDiagnostics::new(runtime, &dispatcher, &thread)); let context = Context::new(dispatcher, clock, io, thread); LocalWorker::register_scheduler(context.clone(), tasks.clone(), executor_shutdown); @@ -136,6 +142,8 @@ impl Worker { cycle_clock, signal, monitor_options, + #[cfg(feature = "monitor")] + diagnostics, _single_threaded: PhantomData, } } @@ -144,9 +152,7 @@ impl Worker { /// The configuration-facing `RuntimeBuilder` delegates construction to bootstrap. /// After this method returns, the thread will end. pub(crate) fn run(&mut self) { - // Keep the worker-owned policy available at the boundary where monitoring - // will be attached in a later stage. - let _monitor_policy = self.monitor_options.policy(); + let _monitoring_enabled = self.monitor_options.policy().is_enabled(); self.execute_phase(); } @@ -163,6 +169,13 @@ impl Worker { CycleAction::Poll | CycleAction::WaitForSignal(_) => Duration::ZERO, }; + #[cfg(feature = "monitor")] + if !matches!(action, CycleAction::Poll) + && let Some(diagnostics) = &self.diagnostics + { + diagnostics.parked(); + } + if RuntimeDriverHost::execute_cycle_current(max_wait).is_err() { return self.finish_shutdown(); } @@ -174,6 +187,13 @@ impl Worker { } } + #[cfg(feature = "monitor")] + if !matches!(action, CycleAction::Poll) + && let Some(diagnostics) = &self.diagnostics + { + diagnostics.unparked(); + } + ControlFlow::Continue(()) } @@ -189,6 +209,12 @@ impl Worker { .as_mut() .expect("executor removal publishes shutdown, which is handled above"); + #[cfg(feature = "monitor")] + let cycle_outcome = match &self.diagnostics { + Some(diagnostics) if diagnostics.is_observed() => executor.execute_cycle_observed(diagnostics), + _ => executor.execute_cycle(), + }; + #[cfg(not(feature = "monitor"))] let cycle_outcome = executor.execute_cycle(); if let CycleOutcome::Shutdown(cause) = cycle_outcome { @@ -256,6 +282,12 @@ impl Worker { Command::AccessIo { access } => { access.into_inner()(context); } + #[cfg(feature = "monitor")] + Command::Inspect { request } => { + if let (Some(diagnostics), Some(executor)) = (&self.diagnostics, &mut self.executor) { + diagnostics.collect(executor, request); + } + } Command::Shutdown => { self.begin_shutdown(); @@ -298,6 +330,8 @@ impl Worker { // Safe executor shutdown synchronously cancels and releases all remaining task storage. executor.shutdown().trigger(cause); drop(executor); + #[cfg(feature = "monitor")] + drop(self.diagnostics.take()); let driver_host = RuntimeDriverHost::uninstall(); drop(driver_host); } diff --git a/crates/arty/src/runtime/worker/blocking/mod.rs b/crates/arty/src/runtime/worker/blocking/mod.rs index 0f2bac39f6..d6acfd0873 100644 --- a/crates/arty/src/runtime/worker/blocking/mod.rs +++ b/crates/arty/src/runtime/worker/blocking/mod.rs @@ -10,6 +10,7 @@ mod local_worker; mod panic_policy; mod pools; +mod queue; mod worker; pub(crate) use local_worker::LocalWorker; diff --git a/crates/arty/src/runtime/worker/blocking/queue.rs b/crates/arty/src/runtime/worker/blocking/queue.rs new file mode 100644 index 0000000000..bb48efa02b --- /dev/null +++ b/crates/arty/src/runtime/worker/blocking/queue.rs @@ -0,0 +1,239 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Native storage for queued blocking functions and active worker slots. +//! Worker handles are retained on their execution thread. The pool shares one +//! immutable task-event context so submissions do not lock this queue for it. + +use std::collections::VecDeque; +#[cfg(feature = "monitor")] +use std::sync::OnceLock; +use std::sync::{Arc, Condvar, Mutex}; + +#[cfg(feature = "monitor")] +use crate::runtime::monitoring::TaskInfo; + +pub(super) type Job = Box; + +pub(super) struct QueuedJob { + pub(super) body: Job, + #[cfg(feature = "monitor")] + pub(super) info: Option>, +} + +#[derive(Default)] +struct WorkerSlot { + #[cfg(feature = "monitor")] + running: bool, + #[cfg(feature = "monitor")] + registration: Option, + #[cfg(feature = "monitor")] + current: Option>, +} + +#[derive(Default)] +struct State { + jobs: VecDeque, + workers: Vec, + stopped: bool, + #[cfg(feature = "monitor")] + runtime: Option, +} + +#[derive(Default)] +pub(super) struct Queue { + state: Mutex, + ready: Condvar, + #[cfg(feature = "monitor")] + pub(super) events: Arc>, +} + +impl Queue { + pub(super) fn push(&self, job: QueuedJob) -> bool { + let mut state = self + .state + .lock() + .expect("a blocking queue operation must not panic while holding its lock"); + if state.stopped { + drop(state); + drop(job); + return false; + } + state.jobs.push_back(job); + drop(state); + self.ready.notify_one(); + true + } + + pub(super) fn add_worker(self: &Arc) -> usize { + let mut state = self + .state + .lock() + .expect("a blocking queue operation must not panic while holding its lock"); + let index = state.workers.len(); + #[cfg(feature = "monitor")] + let registration = state.runtime.as_ref().map(|runtime| self.register_worker(runtime, index)); + let slot = WorkerSlot::default(); + #[cfg(feature = "monitor")] + let slot = WorkerSlot { registration, ..slot }; + state.workers.push(slot); + index + } + + pub(super) fn pop( + &self, + index: usize, + #[cfg(feature = "monitor")] worker: &mut Option, + ) -> Option { + #[cfg(not(feature = "monitor"))] + let _ = index; + let mut state = self + .state + .lock() + .expect("a blocking queue operation must not panic while holding its lock"); + loop { + #[cfg(feature = "monitor")] + if worker.is_none() + && let Some(registration) = &state.workers[index].registration + { + let handle = registration.handle(); + handle.attach_current_thread(); + *worker = Some(handle); + } + if let Some(job) = state.jobs.pop_front() { + #[cfg(feature = "monitor")] + { + state.workers[index].running = true; + state.workers[index].current.clone_from(&job.info); + } + return Some(job); + } + if state.stopped { + return None; + } + #[cfg(feature = "monitor")] + if let Some(worker) = worker.as_ref() { + worker.parked(); + } + state = self + .ready + .wait(state) + .expect("a blocking queue operation must not panic while holding its lock"); + #[cfg(feature = "monitor")] + if let Some(worker) = worker.as_ref() { + worker.unparked(); + } + } + } + + #[cfg(feature = "monitor")] + pub(super) fn finish(&self, index: usize) { + let mut state = self + .state + .lock() + .expect("a blocking queue operation must not panic while holding its lock"); + state.workers[index].current = None; + state.workers[index].running = false; + } + + pub(super) fn stop(&self) { + self.state + .lock() + .expect("a blocking queue operation must not panic while holding its lock") + .stopped = true; + self.ready.notify_all(); + } + + #[cfg(feature = "monitor")] + pub(super) fn retire(&self, index: usize) { + let mut state = self + .state + .lock() + .expect("a blocking queue operation must not panic while holding its lock"); + let registration = state.workers[index].registration.take(); + drop(state); + drop(registration); + } + + #[cfg(feature = "monitor")] + pub(super) fn enable_monitoring(self: &Arc, runtime: &seismograph_runtime::RuntimeHandle) { + let mut state = self + .state + .lock() + .expect("a blocking queue operation must not panic while holding its lock"); + if state.runtime.is_some() { + return; + } + for (index, slot) in state.workers.iter_mut().enumerate() { + slot.registration = Some(self.register_worker(runtime, index)); + } + let events = state + .workers + .first() + .and_then(|slot| slot.registration.as_ref()) + .expect("a blocking pool starts with at least one worker") + .handle() + .task_events(); + self.events.set(events).expect("monitoring is enabled once under the queue lock"); + state.runtime = Some(runtime.clone()); + drop(state); + self.ready.notify_all(); + } + + #[cfg(feature = "monitor")] + fn register_worker( + self: &Arc, + runtime: &seismograph_runtime::RuntimeHandle, + index: usize, + ) -> seismograph_runtime::worker::WorkerRegistration { + use seismograph_runtime::worker::{WorkerMetadata, WorkerRole}; + + let registration = runtime.register_worker(WorkerMetadata::new(WorkerRole::Blocking)); + let queue = Arc::downgrade(self); + registration + .handle() + .register_task_provider(move |request| { + if let Some(queue) = queue.upgrade() { + queue.collect(index, request); + } + }) + .expect("a newly registered blocking worker has not stopped"); + registration + } + + #[cfg(feature = "monitor")] + fn collect(&self, index: usize, request: seismograph_runtime::task::TaskRequest) { + use seismograph_runtime::snapshot::{TaskActivityState, TaskCollection}; + + if request.is_canceled() { + return; + } + let state = self + .state + .lock() + .expect("a blocking queue operation must not panic while holding its lock"); + let current = state.workers[index] + .running + .then_some((state.workers[index].current.as_ref(), TaskActivityState::Running)) + .into_iter(); + // The first worker owns inspection of the shared submission queue. + let queued = state + .jobs + .iter() + .take(if index == 0 { usize::MAX } else { 0 }) + .map(|job| (job.info.as_ref(), TaskActivityState::Ready)); + let mut descriptions = current.chain(queued); + let tasks = descriptions + .by_ref() + .take(request.limit()) + .filter_map(|(info, activity)| info.map(|info| info.describe(activity))) + .collect(); + let status = if descriptions.next().is_some() { + TaskCollection::Truncated + } else { + TaskCollection::Complete + }; + drop(state); + request.respond(tasks, status); + } +} diff --git a/crates/arty/src/runtime/worker/blocking/worker.rs b/crates/arty/src/runtime/worker/blocking/worker.rs index c425767bc9..58628a2544 100644 --- a/crates/arty/src/runtime/worker/blocking/worker.rs +++ b/crates/arty/src/runtime/worker/blocking/worker.rs @@ -9,18 +9,17 @@ use std::{fmt, thread}; use arty_executor::discard_task_panic_payload; use performables::arc::Arc; -use performables::sync::channel; use performables::sync::mutex::Mutex; use crate::runtime::shutdown::{Shutdown, ShutdownState}; +use crate::runtime::worker::blocking::queue::{Queue, QueuedJob}; use crate::runtime::worker::blocking::{LocalWorker, PanicPolicy, PoolIdentity}; use crate::task::handle::{JoinHandle, JoinHandlePool}; +#[cfg(not(feature = "monitor"))] use crate::task::preparation::prepare_blocking; const ERR_POISONED_LOCK: &str = "poisoned lock - cannot continue execution because security and privacy guarantees can no longer be upheld"; -type Job = Box; - #[cfg_attr(test, mutants::skip)] // Missing pool identity disables the same-pool join deadlock guard. fn attach_thread(pool: PoolIdentity) { LocalWorker::attach(pool); @@ -28,19 +27,19 @@ fn attach_thread(pool: PoolIdentity) { /// Grow-only blocking pool that retains worker handles for deterministic shutdown. struct JoinableThreadPool { - jobs: Option>, + jobs: Option>, workers: Vec>, identity: PoolIdentity, - receiver: channel::Receiver, + receiver: std::sync::Arc, queued_count: Arc, active_count: Arc, } impl JoinableThreadPool { fn new(thread_count: usize, identity: PoolIdentity) -> Self { - let (jobs, receiver) = channel::unbounded(); + let receiver = std::sync::Arc::new(Queue::default()); let mut pool = Self { - jobs: Some(jobs), + jobs: Some(std::sync::Arc::clone(&receiver)), workers: Vec::with_capacity(thread_count), identity, receiver, @@ -51,15 +50,24 @@ impl JoinableThreadPool { pool } + #[cfg(test)] fn execute(&self, job: F) -> bool where F: FnOnce() + Send + 'static, { + self.enqueue(QueuedJob { + body: Box::new(job), + #[cfg(feature = "monitor")] + info: None, + }) + } + + fn enqueue(&self, job: QueuedJob) -> bool { let Some(jobs) = &self.jobs else { return false; }; self.queued_count.fetch_add(1, Ordering::SeqCst); - if jobs.send(Box::new(job)).is_err() { + if !jobs.push(job) { self.queued_count.fetch_sub(1, Ordering::SeqCst); return false; } @@ -69,7 +77,8 @@ impl JoinableThreadPool { #[cfg_attr(test, mutants::skip)] // Mutating worker construction creates non-terminating pools. fn set_num_threads(&mut self, thread_count: usize) { while self.workers.len() < thread_count { - let receiver = self.receiver.clone(); + let receiver = std::sync::Arc::clone(&self.receiver); + let index = receiver.add_worker(); let queued_count = Arc::clone(&self.queued_count); let active_count = Arc::clone(&self.active_count); let identity = Arc::clone(&self.identity); @@ -77,15 +86,37 @@ impl JoinableThreadPool { .name("arty-blocking".to_string()) .spawn(move || { attach_thread(identity); - while let Ok(job) = receiver.recv() { + #[cfg(feature = "monitor")] + let mut diagnostics = None; + while let Some(job) = receiver.pop( + index, + #[cfg(feature = "monitor")] + &mut diagnostics, + ) { active_count.fetch_add(1, Ordering::SeqCst); queued_count.fetch_sub(1, Ordering::SeqCst); - let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(job)); + #[cfg(feature = "monitor")] + let observation = diagnostics.as_ref().and_then(|worker| { + job.info.as_ref().map(|info| { + info.started_on(worker.task_events()); + let poll = worker.is_observed().then(|| worker.task_poll_started(info.id())); + (worker, poll) + }) + }); + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(job.body)); + #[cfg(feature = "monitor")] + if let Some((worker, Some(poll))) = observation { + worker.task_poll_finished(poll); + } + #[cfg(feature = "monitor")] + receiver.finish(index); active_count.fetch_sub(1, Ordering::SeqCst); if let Err(panic) = outcome { discard_task_panic_payload(panic); } } + #[cfg(feature = "monitor")] + receiver.retire(index); }) .expect("blocking worker thread creation must succeed"); self.workers.push(worker); @@ -110,7 +141,9 @@ impl JoinableThreadPool { } fn shutdown_and_join(&mut self) { - drop(self.jobs.take()); + if let Some(jobs) = self.jobs.take() { + jobs.stop(); + } for worker in self.workers.drain(..) { if let Err(panic) = worker.join() { discard_task_panic_payload(panic); @@ -162,6 +195,14 @@ impl Worker { R: Send + 'static, { let runtime_shutdown = self.remote_results.shutdown().clone(); + #[cfg(feature = "monitor")] + let info = matches!(runtime_shutdown.state(), ShutdownState::Running) + .then(|| self.pool.task_info::()) + .flatten(); + #[cfg(feature = "monitor")] + let (task, join_handle) = + crate::task::preparation::prepare_blocking_with_info(&self.remote_results, body, self.panic_policy.clone(), info.clone()); + #[cfg(not(feature = "monitor"))] let (task, join_handle) = prepare_blocking(&self.remote_results, body, self.panic_policy.clone()); if matches!(runtime_shutdown.state(), ShutdownState::Started) { drop(task); @@ -176,7 +217,12 @@ impl Worker { } }; - if matches!(self.remote_results.shutdown().state(), ShutdownState::Running) && self.pool.execute(task) { + let job = QueuedJob { + body: Box::new(task), + #[cfg(feature = "monitor")] + info, + }; + if matches!(self.remote_results.shutdown().state(), ShutdownState::Running) && self.pool.enqueue(job) { self.pool.grow_if_overloaded(); } @@ -200,6 +246,8 @@ pub(crate) struct Pool { inner: Arc>>, identity: PoolIdentity, max_thread_count: NonZeroUsize, + #[cfg(feature = "monitor")] + events: std::sync::Arc>, } impl Pool { @@ -224,24 +272,53 @@ impl Pool { // Start with one thread and let the pool grow as needed. let identity = Arc::new(()); let thread_pool = JoinableThreadPool::new(Self::INITIAL_THREAD_COUNT, Arc::clone(&identity)); + #[cfg(feature = "monitor")] + let events = std::sync::Arc::clone(&thread_pool.receiver.events); Self { inner: Arc::new(Mutex::new(Some(thread_pool))), identity, max_thread_count: max_thread_count.unwrap_or(Self::MAX_THREAD_COUNT_DEFAULT), + #[cfg(feature = "monitor")] + events, } } + #[cfg(test)] pub(crate) fn execute(&self, f: F) -> bool where F: FnOnce() + Send + 'static, { + self.enqueue(QueuedJob { + body: Box::new(f), + #[cfg(feature = "monitor")] + info: None, + }) + } + + fn enqueue(&self, job: QueuedJob) -> bool { let pool = self.inner.lock_result().expect(ERR_POISONED_LOCK); let Some(pool) = pool.as_ref() else { // Shutdown closed the shared pool before this racing submission arrived. return false; }; - pool.execute(f) + pool.enqueue(job) + } + + #[cfg(feature = "monitor")] + pub(crate) fn enable_monitoring(&self, runtime: &seismograph_runtime::RuntimeHandle) { + let pool = self.inner.lock_result().expect(ERR_POISONED_LOCK); + if let Some(pool) = pool.as_ref() { + pool.receiver.enable_monitoring(runtime); + } + } + + #[cfg(feature = "monitor")] + fn task_info(&self) -> Option> { + self.events + .get() + .copied() + .map(|events| std::sync::Arc::new(crate::runtime::monitoring::TaskInfo::new::(events))) } #[cfg_attr(test, mutants::skip)] // Impractical to test without overly-expensive timeout logic. @@ -317,7 +394,7 @@ mod tests { use performables::sync::mutex::Mutex; use testing_aids::{TEST_TIMEOUT, execute_or_abandon}; - use super::{Job, JoinableThreadPool, LocalWorker, PanicPolicy, Pool, Worker, attach_thread}; + use super::{JoinableThreadPool, LocalWorker, PanicPolicy, Pool, Worker, attach_thread}; use crate::runtime::shutdown::Shutdown; fn limit(value: usize) -> NonZeroUsize { @@ -489,12 +566,11 @@ mod tests { } #[test] - fn joinable_pool_reports_disconnected_submission() { - let (jobs, disconnected) = performables::sync::channel::unbounded::(); - drop(disconnected); - let (_, receiver) = performables::sync::channel::unbounded(); + fn joinable_pool_reports_closed_submission() { + let receiver = std::sync::Arc::new(super::Queue::default()); + receiver.stop(); let mut pool = JoinableThreadPool { - jobs: Some(jobs), + jobs: Some(std::sync::Arc::clone(&receiver)), workers: Vec::new(), identity: Arc::new(()), receiver, @@ -521,7 +597,7 @@ mod tests { #[test] fn joinable_pool_discards_worker_panic() { let worker = thread::spawn(|| panic!("worker failure")); - let (_, receiver) = performables::sync::channel::unbounded(); + let receiver = std::sync::Arc::new(super::Queue::default()); let mut pool = JoinableThreadPool { jobs: None, workers: vec![worker], diff --git a/crates/arty/src/task/preparation/blocking.rs b/crates/arty/src/task/preparation/blocking.rs index a29fce631e..7d101e7a92 100644 --- a/crates/arty/src/task/preparation/blocking.rs +++ b/crates/arty/src/task/preparation/blocking.rs @@ -10,11 +10,45 @@ use crate::runtime::worker::blocking::PanicPolicy; use crate::task::handle::{JoinHandle, JoinHandlePool, Sender}; use crate::task::preparation::discard_panicking_drop; +#[cfg(any(test, not(feature = "monitor")))] pub(crate) fn prepare_blocking( result_pool: &JoinHandlePool, body: F, panic_policy: PanicPolicy, ) -> (impl FnOnce() + Send + 'static, JoinHandle) +where + F: FnOnce() -> R + Send + 'static, + R: Send + 'static, +{ + prepare_blocking_inner( + result_pool, + body, + panic_policy, + #[cfg(feature = "monitor")] + None, + ) +} + +#[cfg(feature = "monitor")] +pub(crate) fn prepare_blocking_with_info( + result_pool: &JoinHandlePool, + body: F, + panic_policy: PanicPolicy, + info: Option>, +) -> (impl FnOnce() + Send + 'static, JoinHandle) +where + F: FnOnce() -> R + Send + 'static, + R: Send + 'static, +{ + prepare_blocking_inner(result_pool, body, panic_policy, info) +} + +fn prepare_blocking_inner( + result_pool: &JoinHandlePool, + body: F, + panic_policy: PanicPolicy, + #[cfg(feature = "monitor")] info: Option>, +) -> (impl FnOnce() + Send + 'static, JoinHandle) where F: FnOnce() -> R + Send + 'static, R: Send + 'static, @@ -23,6 +57,8 @@ where let task = BlockingTask { pending: Some((body, result_tx)), panic_policy, + #[cfg(feature = "monitor")] + info, }; (move || task.run(), handle) } @@ -30,6 +66,8 @@ where struct BlockingTask { pending: Option<(F, Sender)>, panic_policy: PanicPolicy, + #[cfg(feature = "monitor")] + info: Option>, } impl BlockingTask @@ -41,9 +79,17 @@ where let (body, result_tx) = self.pending.take().expect("a blocking task is consumed exactly once"); match catch_unwind(AssertUnwindSafe(body)) { Ok(result) => { + #[cfg(feature = "monitor")] + if let Some(info) = &self.info { + info.completed(); + } result_tx.completed(result); } Err(panic) => { + #[cfg(feature = "monitor")] + if let Some(info) = &self.info { + info.panicked(); + } if let PanicPolicy::Shutdown(waker) = &self.panic_policy && result_tx.shutdown().begin(ShutdownCause::panic()) == ShutdownStart::Started { @@ -60,6 +106,10 @@ impl Drop for BlockingTask { let Some((body, result_tx)) = self.pending.take() else { return; }; + #[cfg(feature = "monitor")] + if let Some(info) = &self.info { + info.canceled(); + } discard_panicking_drop(body); drop(result_tx); } diff --git a/crates/arty/src/task/preparation/mod.rs b/crates/arty/src/task/preparation/mod.rs index 39b777e505..b814ce9c8e 100644 --- a/crates/arty/src/task/preparation/mod.rs +++ b/crates/arty/src/task/preparation/mod.rs @@ -11,7 +11,10 @@ mod cleanup; mod deferred_task; mod remote; +#[cfg(any(test, not(feature = "monitor")))] pub(crate) use blocking::prepare_blocking; +#[cfg(feature = "monitor")] +pub(crate) use blocking::prepare_blocking_with_info; pub(crate) use cleanup::discard_panicking_drop; pub(crate) use deferred_task::DeferredTask; pub(crate) use remote::{RemoteFutureFactory, prepare_remote}; diff --git a/crates/arty/src/task/preparation/remote.rs b/crates/arty/src/task/preparation/remote.rs index 2e19ad9bf2..82822b30b7 100644 --- a/crates/arty/src/task/preparation/remote.rs +++ b/crates/arty/src/task/preparation/remote.rs @@ -104,6 +104,9 @@ where let future_factory = future_factory.into_inner(); let inner = DeferredTask::new(future_factory, cx); let options = TaskOptions::new(placement); + #[cfg(feature = "monitor")] + crate::runtime::monitoring::add_task::(tasks, inner, options, result_tx); + #[cfg(not(feature = "monitor"))] tasks.add(inner, options, result_tx); }); (future_factory, handle) diff --git a/crates/arty/src/task/runtime_scheduler.rs b/crates/arty/src/task/runtime_scheduler.rs index 9a0f683b90..9dfaa48a53 100644 --- a/crates/arty/src/task/runtime_scheduler.rs +++ b/crates/arty/src/task/runtime_scheduler.rs @@ -191,10 +191,10 @@ impl RuntimeScheduler { F: Future + 'static, R: Send + 'static, { - self.dispatcher.spawn(async move |cx| { + self.dispatcher.spawn(move |cx| { let mut data = data; data.relocate(None, cx.thread()); - future_factory(cx, data).await + future_factory(cx, data) }) } diff --git a/crates/arty/src/task/scheduler.rs b/crates/arty/src/task/scheduler.rs index c63a3aabb1..e985301eca 100644 --- a/crates/arty/src/task/scheduler.rs +++ b/crates/arty/src/task/scheduler.rs @@ -167,6 +167,9 @@ impl Scheduler { let (handle, result_tx) = current.results().handle(); let inner = DeferredTask::new(future_factory, context); let options = TaskOptions::new(TaskPlacement::SameThread); + #[cfg(feature = "monitor")] + crate::runtime::monitoring::add_task::(current.tasks(), inner, options, result_tx); + #[cfg(not(feature = "monitor"))] current.tasks().add(inner, options, result_tx); if !current.tasks().is_executing() { self.dispatcher.wake_worker(self.binding.worker_index); @@ -226,10 +229,10 @@ impl Scheduler { R: ThreadAware + 'static, { let source = self.binding.thread.clone(); - self.dispatcher.spawn_relocating(async move |cx| { + self.dispatcher.spawn_relocating(move |cx| { let mut data = data; data.relocate(Some(&source), cx.thread()); - f(data).await + f(data) }) } diff --git a/crates/arty/tests/monitoring.rs b/crates/arty/tests/monitoring.rs new file mode 100644 index 0000000000..7fb65e09c6 --- /dev/null +++ b/crates/arty/tests/monitoring.rs @@ -0,0 +1,858 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Arty supplies native task state independently of event recording. + +// Collection uses real worker-response deadlines. Native and careful tests cover +// this integration; the executor retains direct storage coverage under Miri. +#![cfg(all(feature = "rt", feature = "monitor", feature = "test-util", not(miri)))] +#![expect(clippy::unwrap_used, reason = "test failures use unwrap for concise backtraces")] + +mod support; + +use std::future::{pending, poll_fn}; +use std::pin::Pin; +use std::sync::{Arc, Condvar, Mutex, MutexGuard, mpsc}; +use std::task::{Context, Poll}; +use std::time::Duration; + +use arty::runtime::{BlockingPoolPolicy, MonitorPolicy, PanicPolicy, Runtime, WorkersPolicy}; +use seismograph::recorder::event::{EventClass, EventKind}; +use seismograph::recorder::{Configuration, Policy}; +use seismograph::snapshot::SnapshotOptions; +use seismograph_protocol::discovery::Directory; +use seismograph_protocol::io::{IoOperation, MonitorIo}; +use seismograph_protocol::message::{RecordingConfiguration, RecordingPolicy, Request, Response}; +use seismograph_protocol::transport::Connection; +use seismograph_protocol::{read_response, write_request}; +use seismograph_runtime::snapshot::{ + CaptureOptions, Runtime as RuntimeSnapshot, RuntimeState, TaskActivityState, TaskCollection, TaskMetricsScope, +}; +use seismograph_runtime::worker::WorkerRole; +use support::JoinHandleExt as _; +use testing_aids::TEST_TIMEOUT; +use thread_aware::Unaware; + +static RECORDING: Mutex<()> = Mutex::new(()); + +struct RecorderGuard { + _lock: MutexGuard<'static, ()>, +} + +impl RecorderGuard { + fn new() -> Self { + let lock = RECORDING.lock().unwrap(); + seismograph::recorder(Configuration::default()); + Self { _lock: lock } + } +} + +impl Drop for RecorderGuard { + fn drop(&mut self) { + seismograph::recorder(Configuration::default()); + } +} + +fn runtime(name: &str, io: &MonitorIo) -> Runtime { + Runtime::builder() + .workers(WorkersPolicy::exactly(1)) + .blocking_pool(BlockingPoolPolicy::shared().max(1)) + .monitor(MonitorPolicy::enabled().name(name).io(io.clone())) + .panic(PanicPolicy::ignore()) + .build() + .unwrap() +} + +fn runtimes() -> Vec { + let bytes = seismograph::snapshot(SnapshotOptions::default()).unwrap(); + let snapshot = seismograph::snapshot::decode(bytes.as_bytes()).unwrap(); + snapshot + .sources + .iter() + .find(|source| source.id == seismograph_runtime::snapshot::source::ID) + .map_or_else(Vec::new, |source| { + seismograph_runtime::snapshot::decode(&source.data).unwrap().runtimes + }) +} + +fn capture(runtime: &Runtime) -> RuntimeSnapshot { + runtime + .diagnostics() + .unwrap() + .capture(CaptureOptions { + timeout: Duration::from_secs(5), + ..CaptureOptions::default() + }) + .unwrap() +} + +fn names(io: &MonitorIo) -> Vec { + let directory = Directory::current(io).unwrap(); + let mut names = directory + .descriptors() + .unwrap() + .map(|descriptor| descriptor.unwrap().name) + .collect::>(); + names.sort(); + names +} + +#[test] +fn disabled_monitoring_registers_no_runtime_or_endpoint() { + let _recording = RecorderGuard::new(); + let (io, _) = MonitorIo::new_mocked(); + let runtime = Runtime::builder() + .monitor(MonitorPolicy::disabled().name("disabled-monitor").io(io.clone())) + .build() + .unwrap(); + + let value = runtime.scheduler().block_on(async |_| 42).unwrap(); + let registered = runtimes().iter().any(|runtime| runtime.name == "disabled-monitor"); + runtime.stop().unwrap(); + + assert_eq!((value, registered, names(&io)), (42, false, Vec::new())); +} + +#[test] +fn local_and_remote_tasks_are_collected_without_recording() { + let _recording = RecorderGuard::new(); + let (io, _) = MonitorIo::new_mocked(); + let runtime = runtime("native-tasks", &io); + let (started, ready) = mpsc::channel(); + let parent = runtime + .scheduler() + .spawn_anywhere(Unaware(started), |cx, Unaware(started)| async move { + cx.scheduler() + .spawn(move |_| { + let mut started = Some(started); + poll_fn(move |_| { + if let Some(started) = started.take() { + started.send(()).unwrap(); + } + Poll::<()>::Pending + }) + }) + .await + .unwrap(); + }); + ready.recv_timeout(TEST_TIMEOUT).unwrap(); + + let snapshot = capture(&runtime); + runtime.stop().unwrap(); + assert!(parent.join().unwrap_err().is_shutdown()); + let observed = snapshot + .tasks + .iter() + .map(|task| { + ( + task.activity.unwrap().state, + task.metrics_scope, + task.spawned_at, + task.last_worker_id.is_some(), + task.future_size_bytes.is_some(), + ) + }) + .collect::>(); + + assert_eq!( + observed, + vec![(TaskActivityState::Waiting, TaskMetricsScope::Uncollected, None, true, true); 2] + ); + assert!( + snapshot + .workers + .iter() + .all(|worker| worker.task_collection == TaskCollection::Complete) + ); +} + +#[test] +fn task_identity_survives_repeated_collection_and_recording_changes() { + let _recording = RecorderGuard::new(); + let (io, _) = MonitorIo::new_mocked(); + let runtime = runtime("stable-identities", &io); + let (started, ready) = mpsc::channel(); + let task = runtime.scheduler().spawn_anywhere(Unaware(started), |_, Unaware(started)| { + let mut started = Some(started); + poll_fn(move |_| { + if let Some(started) = started.take() { + started.send(()).unwrap(); + } + Poll::<()>::Pending + }) + }); + ready.recv_timeout(TEST_TIMEOUT).unwrap(); + let first = capture(&runtime); + + seismograph::recorder(Configuration { + runtime_tasks: Policy::all(false), + ..Configuration::default() + }); + let second = capture(&runtime); + seismograph::recorder(Configuration::default()); + let third = capture(&runtime); + runtime.stop().unwrap(); + assert!(task.join().unwrap_err().is_shutdown()); + + assert_eq!( + (second.tasks[0].id, third.tasks[0].id, second.tasks[0].spawned_at), + (first.tasks[0].id, first.tasks[0].id, None) + ); + assert_eq!( + (second.tasks[0].type_descriptor, third.tasks[0].type_descriptor), + (first.tasks[0].type_descriptor, first.tasks[0].type_descriptor) + ); +} + +#[test] +fn queued_and_running_blocking_tasks_come_from_native_pool_storage() { + let _recording = RecorderGuard::new(); + let (io, _) = MonitorIo::new_mocked(); + let runtime = runtime("blocking-tasks", &io); + let (started, ready) = mpsc::channel(); + let (release, released) = mpsc::channel(); + let first = runtime.scheduler().spawn_blocking(move || { + started.send(()).unwrap(); + released.recv_timeout(TEST_TIMEOUT).unwrap(); + 1 + }); + ready.recv_timeout(TEST_TIMEOUT).unwrap(); + let second = runtime.scheduler().spawn_blocking(|| 2); + + let snapshot = capture(&runtime); + release.send(()).unwrap(); + assert_eq!((first.join().unwrap(), second.join().unwrap()), (1, 2)); + runtime.stop().unwrap(); + let states = snapshot + .tasks + .iter() + .map(|task| { + let worker = snapshot + .workers + .iter() + .find(|worker| Some(worker.id) == task.last_worker_id) + .unwrap(); + (worker.role, task.activity.unwrap().state, task.metrics_scope) + }) + .collect::>(); + + assert_eq!( + states, + [ + (WorkerRole::Blocking, TaskActivityState::Running, TaskMetricsScope::Uncollected), + (WorkerRole::Blocking, TaskActivityState::Ready, TaskMetricsScope::Uncollected), + ] + ); +} + +#[test] +fn draining_queued_blocking_tasks_does_not_report_worker_resumes() { + let _recording = RecorderGuard::new(); + seismograph::recorder(Configuration { + runtime_tasks: Policy::all(false), + ..Configuration::default() + }); + let (io, _) = MonitorIo::new_mocked(); + let runtime = runtime("blocking-queue-resumes", &io); + let (started, ready) = mpsc::channel(); + let (release, released) = mpsc::channel(); + let first = runtime.scheduler().spawn_blocking(move || { + started.send(seismograph::recorder::thread_id()).unwrap(); + released.recv_timeout(TEST_TIMEOUT).unwrap(); + }); + let thread_id = ready.recv_timeout(TEST_TIMEOUT).unwrap(); + let second = runtime.scheduler().spawn_blocking(|| ()); + let third = runtime.scheduler().spawn_blocking(|| ()); + let (started, ready) = mpsc::channel(); + let (finish, finished) = mpsc::channel(); + let last = runtime.scheduler().spawn_blocking(move || { + started.send(()).unwrap(); + finished.recv_timeout(TEST_TIMEOUT).unwrap(); + }); + release.send(()).unwrap(); + ready.recv_timeout(TEST_TIMEOUT).unwrap(); + let runtime_snapshot = capture(&runtime); + let bytes = seismograph::snapshot(SnapshotOptions::default()).unwrap(); + let snapshot = seismograph::snapshot::decode(bytes.as_bytes()).unwrap(); + finish.send(()).unwrap(); + first.join().unwrap(); + second.join().unwrap(); + third.join().unwrap(); + last.join().unwrap(); + runtime.stop().unwrap(); + + let worker = runtime_snapshot + .workers + .iter() + .find(|worker| worker.role == WorkerRole::Blocking) + .unwrap(); + let events = snapshot + .events + .events + .iter() + .filter(|event| { + event + .runtime() + .is_some_and(|payload| payload.runtime_id == runtime_snapshot.id && payload.worker_id == Some(worker.id)) + }) + .skip_while(|event| event.kind != EventKind::TaskPollStarted) + .collect::>(); + let polls = events.iter().filter(|event| event.kind == EventKind::TaskPollStarted).count(); + let transitions = events + .iter() + .filter_map(|event| match event.kind { + EventKind::WorkerParked | EventKind::WorkerUnparked => Some(event.kind), + _ => None, + }) + .collect::>(); + assert_eq!((worker.thread_id, polls, transitions), (Some(thread_id), 4, Vec::new())); +} + +#[test] +fn stopped_runtime_does_not_record_new_blocking_tasks() { + let _recording = RecorderGuard::new(); + let (io, _) = MonitorIo::new_mocked(); + let runtime = runtime("stopped-blocking-submission", &io); + let id = runtime.diagnostics().unwrap().id(); + let scheduler = runtime.scheduler().block_on(async |cx| cx.scheduler().clone()).unwrap(); + runtime.stop().unwrap(); + seismograph::recorder(Configuration { + runtime_tasks: Policy::all(false), + ..Configuration::default() + }); + let outcome = scheduler.spawn_blocking(|| panic!("a stopped pool must not run a function")).join(); + let bytes = seismograph::snapshot(SnapshotOptions::default()).unwrap(); + let snapshot = seismograph::snapshot::decode(bytes.as_bytes()).unwrap(); + let spawned = snapshot + .events + .events + .iter() + .filter(|event| event.kind == EventKind::TaskSpawned && event.runtime().is_some_and(|payload| payload.runtime_id == id)) + .count(); + + assert_eq!((outcome.unwrap_err().is_shutdown(), spawned), (true, 0)); +} + +#[test] +fn recording_reports_real_async_and_blocking_outcomes() { + let _recording = RecorderGuard::new(); + seismograph::recorder(Configuration { + runtime_tasks: Policy::all(false), + ..Configuration::default() + }); + let (io, _) = MonitorIo::new_mocked(); + let runtime = runtime("recorded-outcomes", &io); + let id = runtime.diagnostics().unwrap().id(); + let scheduler = runtime.scheduler().block_on(async |cx| cx.scheduler().clone()).unwrap(); + let completed = scheduler.spawn(async |_| 42); + assert_eq!(completed.join().unwrap(), 42); + assert!( + scheduler + .spawn(async |_| panic!("recorded async panic")) + .join() + .unwrap_err() + .is_panic() + ); + assert_eq!(scheduler.spawn_blocking(|| 7).join().unwrap(), 7); + assert!( + scheduler + .spawn_blocking(|| panic!("recorded blocking panic")) + .join() + .unwrap_err() + .is_panic() + ); + let (started, ready) = mpsc::channel(); + let canceled = scheduler.spawn(move |_| async move { + started.send(()).unwrap(); + pending::<()>().await; + }); + ready.recv_timeout(TEST_TIMEOUT).unwrap(); + + runtime.stop().unwrap(); + assert!(canceled.join().unwrap_err().is_shutdown()); + let bytes = seismograph::snapshot(SnapshotOptions::default()).unwrap(); + let snapshot = seismograph::snapshot::decode(bytes.as_bytes()).unwrap(); + let outcomes = snapshot + .events + .events + .iter() + .filter(|event| event.runtime().is_some_and(|event| event.runtime_id == id)) + .filter_map(|event| match event.kind { + EventKind::TaskCompleted | EventKind::TaskPanicked | EventKind::TaskCanceled => Some(event.kind), + _ => None, + }) + .collect::>(); + + let mut starts = Vec::new(); + let mut finishes = Vec::new(); + for event in &snapshot.events.events { + if let Some(payload) = event.runtime() + && payload.runtime_id == id + { + match event.kind { + EventKind::TaskPollStarted => starts.push(payload.subject_id), + EventKind::TaskPollFinished => finishes.push(payload.subject_id), + _ => {} + } + } + } + starts.sort_unstable(); + finishes.sort_unstable(); + assert_eq!( + ( + outcomes.iter().filter(|kind| **kind == EventKind::TaskCompleted).count(), + outcomes.iter().filter(|kind| **kind == EventKind::TaskPanicked).count(), + outcomes.iter().filter(|kind| **kind == EventKind::TaskCanceled).count(), + starts.len(), + starts == finishes, + ), + (3, 2, 1, 6, true) + ); +} + +#[test] +fn native_tasks_reuse_their_recorded_spawn_timestamp_and_identity() { + let _recording = RecorderGuard::new(); + seismograph::recorder(Configuration { + runtime_tasks: Policy::all(false), + ..Configuration::default() + }); + let (io, _) = MonitorIo::new_mocked(); + let runtime = runtime("coherent-spawn-metadata", &io); + let (started, ready) = mpsc::channel(); + let asynchronous = runtime + .scheduler() + .spawn_anywhere(Unaware(started.clone()), |_, Unaware(started)| async move { + started.send(()).unwrap(); + pending::<()>().await; + }); + let (release, released) = mpsc::channel(); + let blocking = runtime.scheduler().spawn_blocking(move || { + started.send(()).unwrap(); + released.recv().unwrap(); + }); + ready.recv_timeout(TEST_TIMEOUT).unwrap(); + ready.recv_timeout(TEST_TIMEOUT).unwrap(); + + let native = capture(&runtime); + release.send(()).unwrap(); + blocking.join().unwrap(); + runtime.stop().unwrap(); + assert!(asynchronous.join().unwrap_err().is_shutdown()); + let bytes = seismograph::snapshot(SnapshotOptions::default()).unwrap(); + let recorded = seismograph::snapshot::decode(bytes.as_bytes()).unwrap(); + + assert_eq!(native.tasks.len(), 2); + for task in &native.tasks { + let spawn = recorded + .events + .events + .iter() + .find(|event| { + event.kind == EventKind::TaskSpawned + && event + .runtime() + .is_some_and(|payload| payload.runtime_id == native.id && payload.subject_id == task.id.get()) + }) + .unwrap(); + let payload = spawn.runtime().unwrap(); + assert_eq!( + (task.spawned_at, task.type_descriptor.get(), task.future_size_bytes), + (Some(spawn.timestamp), payload.value_0, Some(payload.value_1 - 1)) + ); + } +} + +#[test] +fn collection_reports_the_native_task_limit_instead_of_claiming_completeness() { + let _recording = RecorderGuard::new(); + let (io, _) = MonitorIo::new_mocked(); + let runtime = runtime("bounded-tasks", &io); + let scheduler = runtime.scheduler().block_on(async |cx| cx.scheduler().clone()).unwrap(); + for _ in 0..260 { + drop(scheduler.spawn(async |_| pending::<()>().await)); + } + let (started, ready) = mpsc::channel(); + drop(scheduler.spawn(move |_| async move { + started.send(()).unwrap(); + pending::<()>().await; + })); + ready.recv_timeout(TEST_TIMEOUT).unwrap(); + + let snapshot = capture(&runtime); + runtime.stop().unwrap(); + let core = snapshot.workers.iter().find(|worker| worker.role == WorkerRole::Core).unwrap(); + + assert_eq!((snapshot.tasks.len(), core.task_collection), (256, TaskCollection::Truncated)); +} + +#[test] +fn a_blocked_async_worker_times_out_and_can_answer_a_later_request() { + let _recording = RecorderGuard::new(); + let (io, _) = MonitorIo::new_mocked(); + let runtime = runtime("blocked-worker", &io); + let (started, ready) = mpsc::channel(); + let (release, released) = mpsc::channel(); + let task = runtime + .scheduler() + .spawn_anywhere(Unaware((started, released)), |_, Unaware((started, released))| async move { + started.send(()).unwrap(); + released.recv_timeout(TEST_TIMEOUT).unwrap(); + }); + ready.recv_timeout(TEST_TIMEOUT).unwrap(); + + let blocked = runtime.diagnostics().unwrap().capture(CaptureOptions::default()).unwrap(); + release.send(()).unwrap(); + task.join().unwrap(); + // Completion precedes the next command batch, which retires the canceled request. + runtime.scheduler().block_on(async |_| ()).unwrap(); + let recovered = capture(&runtime); + runtime.stop().unwrap(); + let core_status = |snapshot: &RuntimeSnapshot| { + snapshot + .workers + .iter() + .find(|worker| worker.role == WorkerRole::Core) + .unwrap() + .task_collection + }; + + assert_eq!( + (core_status(&blocked), core_status(&recovered)), + (TaskCollection::TimedOut, TaskCollection::Complete) + ); +} + +#[test] +fn stopping_one_runtime_retires_only_its_endpoint_and_workers() { + let _recording = RecorderGuard::new(); + let (io, _) = MonitorIo::new_mocked(); + let first = runtime("endpoint-one", &io); + let second = runtime("endpoint-two", &io); + assert_eq!(names(&io), ["endpoint-one", "endpoint-two"]); + + first.stop().unwrap(); + let first = runtimes().into_iter().find(|runtime| runtime.name == "endpoint-one").unwrap(); + let second_snapshot = capture(&second); + let remaining = names(&io); + second.stop().unwrap(); + + assert_eq!( + (first.state, first.tasks.is_empty(), second_snapshot.state, remaining, names(&io)), + ( + RuntimeState::Stopped, + true, + RuntimeState::Running, + vec!["endpoint-two".to_owned()], + Vec::new() + ) + ); +} + +#[test] +fn monitor_startup_failure_is_returned_without_publishing_an_endpoint() { + let _recording = RecorderGuard::new(); + let (io, control) = MonitorIo::new_mocked(); + control.fail_next(IoOperation::Bind, std::io::ErrorKind::PermissionDenied.into()); + + let result = Runtime::builder() + .monitor(MonitorPolicy::enabled().name("failed-monitor").io(io.clone())) + .build(); + + drop(result.unwrap_err()); + assert_eq!(names(&io), Vec::::new()); +} + +#[test] +fn stopping_reports_a_monitor_failure_after_removing_the_endpoint() { + let _recording = RecorderGuard::new(); + let (io, control) = MonitorIo::new_mocked(); + control.fail_next( + IoOperation::Accept, + std::io::Error::new(std::io::ErrorKind::PermissionDenied, "injected monitor accept failure"), + ); + let runtime = runtime("failed-listener", &io); + let descriptor = Directory::current(&io).unwrap().descriptors().unwrap().next().unwrap().unwrap(); + let mut connection = Connection::connect(&io, &descriptor, TEST_TIMEOUT).unwrap(); + connection.set_read_timeout(Some(TEST_TIMEOUT)).unwrap(); + write_request(&mut connection, 1, &Request::Hello).unwrap(); + assert!(matches!(read_response(&mut connection).unwrap(), (1, Response::Hello { .. }))); + + let error = runtime.stop().unwrap_err(); + + assert_eq!( + (error.to_string().contains("injected monitor accept failure"), names(&io)), + (true, Vec::new()) + ); +} + +#[test] +fn shutdown_records_cancellation_for_a_queued_blocking_function() { + let _recording = RecorderGuard::new(); + seismograph::recorder(Configuration { + runtime_tasks: Policy::all(false), + ..Configuration::default() + }); + let (io, _) = MonitorIo::new_mocked(); + let runtime = runtime("canceled-blocking", &io); + let control = support::runtime_control(&runtime); + let (started, ready) = mpsc::channel(); + let (release, released) = mpsc::channel(); + let running = runtime.scheduler().spawn_blocking(move || { + started.send(()).unwrap(); + released.recv_timeout(TEST_TIMEOUT).unwrap(); + }); + ready.recv_timeout(TEST_TIMEOUT).unwrap(); + let queued = runtime.scheduler().spawn_blocking(|| panic!("a canceled function must not run")); + let snapshot = capture(&runtime); + let id = snapshot + .tasks + .iter() + .find(|task| task.activity.unwrap().state == TaskActivityState::Ready) + .unwrap() + .id; + + control.request_stop(); + release.send(()).unwrap(); + runtime.stop().unwrap(); + running.join().unwrap(); + let canceled = queued.join().unwrap_err(); + let bytes = seismograph::snapshot(SnapshotOptions::default()).unwrap(); + let events = seismograph::snapshot::decode(bytes.as_bytes()).unwrap(); + let outcomes = events + .events + .events + .iter() + .filter_map(|event| { + event + .runtime() + .filter(|payload| { + payload.runtime_id == snapshot.id + && payload.subject_id == id.get() + && matches!( + event.kind, + EventKind::TaskCompleted | EventKind::TaskPanicked | EventKind::TaskCanceled + ) + }) + .map(|_| event.kind) + }) + .collect::>(); + + assert_eq!((canceled.is_shutdown(), outcomes), (true, vec![EventKind::TaskCanceled])); +} + +#[test] +fn the_monitor_connection_captures_runtime_diagnostics_and_preserves_recording_configuration() { + let _recording = RecorderGuard::new(); + seismograph::recorder(Configuration { + general_events: Policy::all(false), + ..Configuration::default() + }); + let (io, _) = MonitorIo::new_mocked(); + let runtime = runtime("connected-monitor", &io); + let id = runtime.diagnostics().unwrap().id(); + let descriptor = Directory::current(&io).unwrap().descriptors().unwrap().next().unwrap().unwrap(); + let mut connection = Connection::connect(&io, &descriptor, TEST_TIMEOUT).unwrap(); + connection.set_read_timeout(Some(TEST_TIMEOUT)).unwrap(); + + write_request(&mut connection, 1, &Request::Hello).unwrap(); + let hello = read_response(&mut connection).unwrap(); + write_request( + &mut connection, + 2, + &Request::CaptureSnapshot(seismograph_protocol::message::SnapshotOptions::default()), + ) + .unwrap(); + let (request, Response::Snapshot(bytes)) = read_response(&mut connection).unwrap() else { + panic!("the monitor must return the requested snapshot"); + }; + let snapshot = seismograph::snapshot::decode(&bytes).unwrap(); + let source = snapshot + .sources + .iter() + .find(|source| source.id == seismograph_runtime::snapshot::source::ID) + .unwrap(); + let snapshot = seismograph_runtime::snapshot::decode(&source.data).unwrap(); + let captured = snapshot + .runtimes + .iter() + .find(|runtime| runtime.name == "connected-monitor") + .unwrap(); + runtime.stop().unwrap(); + + assert_eq!( + ( + hello, + request, + (captured.id, captured.state), + seismograph::recorder::is_enabled(EventClass::General), + seismograph::recorder::is_enabled(EventClass::RuntimeTask), + names(&io), + ), + ( + ( + 1, + Response::Hello { + instance_id: descriptor.instance_id, + recording: RecordingConfiguration { + general_events: RecordingPolicy { + enabled: true, + ..RecordingPolicy::default() + }, + ..RecordingConfiguration::default() + }, + }, + ), + 2, + (id, RuntimeState::Running), + true, + false, + Vec::new(), + ) + ); +} + +#[test] +fn task_sizes_describe_user_futures_before_runtime_wrapping() { + struct Body { + started: Option>, + _data: [u8; 37], + } + + impl Future for Body { + type Output = (); + + fn poll(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<()> { + if let Some(started) = self.get_mut().started.take() { + started.send(()).unwrap(); + } + Poll::Pending + } + } + + let _recording = RecorderGuard::new(); + let (io, _) = MonitorIo::new_mocked(); + let runtime = runtime("user-bodies", &io); + let (started, ready) = mpsc::channel(); + drop( + runtime + .scheduler() + .spawn_anywhere(Unaware(started.clone()), |_, Unaware(started)| Body { + started: Some(started), + _data: [0; 37], + }), + ); + let scheduler = runtime.scheduler().block_on(async |cx| cx.scheduler().clone()).unwrap(); + drop(scheduler.spawn_anywhere(Unaware(started.clone()), |Unaware(started)| Body { + started: Some(started), + _data: [0; 37], + })); + runtime + .scheduler() + .block_on(async move |cx| { + drop(cx.scheduler().spawn(move |_| Body { + started: Some(started), + _data: [0; 37], + })); + }) + .unwrap(); + for _ in 0..3 { + ready.recv_timeout(TEST_TIMEOUT).unwrap(); + } + + let snapshot = capture(&runtime); + runtime.stop().unwrap(); + let sizes = snapshot.tasks.iter().map(|task| task.future_size_bytes).collect::>(); + + assert_eq!(sizes, vec![Some(size_of::() as u64); 3]); +} + +fn growing_pool(name: &str, policy: BlockingPoolPolicy, expected_blocking_workers: usize) { + let _recording = RecorderGuard::new(); + seismograph::recorder(Configuration { + runtime_tasks: Policy::all(false), + ..Configuration::default() + }); + let (io, _) = MonitorIo::new_mocked(); + let runtime = Runtime::builder() + .workers(WorkersPolicy::exactly(2)) + .blocking_pool(policy.max(2)) + .monitor(MonitorPolicy::enabled().name(name).io(io)) + .build() + .unwrap(); + let scheduler = runtime.scheduler().block_on(async |cx| cx.scheduler().clone()).unwrap(); + let release = Arc::new((Mutex::new(false), Condvar::new())); + let (started, ready) = mpsc::channel(); + let tasks = (0..6) + .map(|_| { + let release = Arc::clone(&release); + let started = started.clone(); + scheduler.spawn_blocking(move || { + started.send(()).unwrap(); + let (released, changed) = &*release; + let mut released = released.lock().unwrap(); + while !*released { + let waited = changed.wait_timeout(released, TEST_TIMEOUT).unwrap(); + assert!(!waited.1.timed_out(), "the test must release its blocking functions"); + released = waited.0; + } + }) + }) + .collect::>(); + for _ in 0..2 { + ready.recv_timeout(TEST_TIMEOUT).unwrap(); + } + + let snapshot = capture(&runtime); + *release.0.lock().unwrap() = true; + release.1.notify_all(); + for task in tasks { + task.join().unwrap(); + } + runtime.stop().unwrap(); + let bytes = seismograph::snapshot(SnapshotOptions::default()).unwrap(); + let events = seismograph::snapshot::decode(bytes.as_bytes()).unwrap(); + let mut matched_workers = Vec::new(); + for task in &snapshot.tasks { + let worker_for = |kind| { + events + .events + .events + .iter() + .find_map(|event| { + event + .runtime() + .filter(|payload| event.kind == kind && payload.runtime_id == snapshot.id && payload.subject_id == task.id.get()) + .and_then(|payload| payload.worker_id) + }) + .unwrap() + }; + matched_workers.push(worker_for(EventKind::TaskPollStarted) == worker_for(EventKind::TaskCompleted)); + } + let running = snapshot + .tasks + .iter() + .filter(|task| task.activity.unwrap().state == TaskActivityState::Running) + .count(); + let queued = snapshot + .tasks + .iter() + .filter(|task| task.activity.unwrap().state == TaskActivityState::Ready) + .count(); + let workers = snapshot.workers.iter().filter(|worker| worker.role == WorkerRole::Blocking).count(); + + assert_eq!( + (workers, running, queued, matched_workers), + (expected_blocking_workers, 2, 4, vec![true; 6]) + ); +} + +#[test] +fn a_shared_pool_registers_new_threads_and_records_outcomes_on_the_execution_worker() { + growing_pool("growing-shared-pool", BlockingPoolPolicy::shared(), 2); +} + +#[test] +fn a_per_worker_pool_registers_new_threads_without_merging_other_pools() { + growing_pool("growing-per-worker-pool", BlockingPoolPolicy::per_worker(), 3); +} diff --git a/crates/arty_executor/src/executor_shared.rs b/crates/arty_executor/src/executor_shared.rs index 38fd1a7880..77c85f3a6c 100644 --- a/crates/arty_executor/src/executor_shared.rs +++ b/crates/arty_executor/src/executor_shared.rs @@ -47,7 +47,7 @@ impl ExecutorShared { F: Future + 'static, R: 'static, T: TaskCompletion, - I: crate::inspection::Inspector, + I: crate::inspection::Inspector, { if self.shutdown().is_triggered() { if let Err(panic) = catch_unwind(AssertUnwindSafe(|| drop(inspector))) { diff --git a/crates/arty_executor/src/inspection.rs b/crates/arty_executor/src/inspection.rs index f042322206..29228eaa84 100644 --- a/crates/arty_executor/src/inspection.rs +++ b/crates/arty_executor/src/inspection.rs @@ -70,14 +70,14 @@ pub trait PollObserver: std::fmt::Debug { } /// Builds a temporary description while the native future is borrowed on its owner. -pub(crate) trait Inspector: 'static { +pub(crate) trait Inspector: 'static { #[cfg(feature = "inspection")] - fn inspect(&self, future: &F, state: TaskState, visitor: &mut dyn FnMut(&dyn Any)); + fn inspect(&self, future: &F, completion: &T, state: TaskState, visitor: &mut dyn FnMut(&dyn Any)); } -impl Inspector for () { +impl Inspector for () { #[cfg(feature = "inspection")] - fn inspect(&self, _future: &F, _state: TaskState, _visitor: &mut dyn FnMut(&dyn Any)) {} + fn inspect(&self, _future: &F, _completion: &T, _state: TaskState, _visitor: &mut dyn FnMut(&dyn Any)) {} } #[cfg(feature = "inspection")] @@ -98,13 +98,13 @@ impl InspectWith { } #[cfg(feature = "inspection")] -impl Inspector for InspectWith +impl Inspector for InspectWith where - I: Fn(&F, TaskState) -> Option + 'static, + I: Fn(&F, &T, TaskState) -> Option + 'static, D: 'static, { - fn inspect(&self, future: &F, state: TaskState, visitor: &mut dyn FnMut(&dyn Any)) { - if let Some(description) = (self.inspect)(future, state) { + fn inspect(&self, future: &F, completion: &T, state: TaskState, visitor: &mut dyn FnMut(&dyn Any)) { + if let Some(description) = (self.inspect)(future, completion, state) { visitor(&description); } } diff --git a/crates/arty_executor/src/task.rs b/crates/arty_executor/src/task.rs index e9c6cc10a8..e457152e63 100644 --- a/crates/arty_executor/src/task.rs +++ b/crates/arty_executor/src/task.rs @@ -40,7 +40,7 @@ where F: Future + 'static, R: 'static, T: TaskCompletion, - I: Inspector, + I: Inspector, { /// `None` after the task completes or the executor abandons it during shutdown. /// @@ -80,7 +80,7 @@ where F: Future + 'static, R: 'static, T: TaskCompletion, - I: Inspector, + I: Inspector, { } @@ -90,7 +90,7 @@ where F: Future + 'static, R: 'static, T: TaskCompletion, - I: Inspector, + I: Inspector, { } @@ -110,7 +110,7 @@ where F: Future + 'static, R: 'static, T: TaskCompletion, - I: Inspector, + I: Inspector, { fn new_inspected(future: F, result_target: T, options: TaskOptions, sink: Sink, inspector: I) -> Self { Self { @@ -146,7 +146,7 @@ where F: Future + 'static, R: 'static, T: TaskCompletion, - I: Inspector, + I: Inspector, { TaskHandle::new(pool.alloc_arc_pin(Task::new_inspected(future, completion, options, sink, inspector))) } @@ -193,7 +193,7 @@ where F: Future + 'static, R: 'static, T: TaskCompletion, - I: Inspector, + I: Inspector, { #[cfg(feature = "inspection")] fn inspect(&self, state: TaskState, visitor: &mut dyn FnMut(&dyn Any)) { @@ -202,8 +202,9 @@ where let payload = unsafe { &*self.payload.get() }; if let Some(payload) = payload && let Some(future) = &payload.future.inner + && let Some(completion) = &payload.result_target { - payload.inspector.inspect(future, state, visitor); + payload.inspector.inspect(future, completion, state, visitor); } } @@ -268,7 +269,7 @@ where F: Future + 'static, R: 'static, T: TaskCompletion, - I: Inspector, + I: Inspector, { fn pinned_wake_signal(self: Pin<&Self>) -> Pin<&WakeSignal> { let signal = self.get_ref().wake_signal.get().expect("task must be initialized before poll()"); @@ -413,7 +414,7 @@ where F: Future + 'static, R: 'static, T: TaskCompletion, - I: Inspector, + I: Inspector, { /// The future that will be polled by the executor to progress the task. future: FutureStorage, diff --git a/crates/arty_executor/src/task_set.rs b/crates/arty_executor/src/task_set.rs index d723790f85..04f30c277f 100644 --- a/crates/arty_executor/src/task_set.rs +++ b/crates/arty_executor/src/task_set.rs @@ -25,6 +25,56 @@ impl TaskSet { T: TaskCompletion, D: 'static, I: Fn(&F, crate::TaskState) -> Option + 'static, + { + self.add_with_completion_inspection(future, options, completion, move |future, _, state| inspect(future, state)); + } + + /// Adds a task whose inspector borrows its future and completion target. + /// + /// Store shared diagnostic metadata in the completion target to avoid a + /// separate allocation. The inspector borrows it only during explicit + /// inspection or observed polls on the executor thread. Ordinary cycles + /// do not call the inspector. Terminal delivery consumes the same target + /// through [`TaskCompletion`]. + /// + /// # Examples + /// + /// ``` + /// # #[cfg(feature = "inspection")] { + /// use arty_executor::{Executor, ShutdownCause, TaskCompletion, TaskOptions, TaskPanicPayload}; + /// + /// #[derive(Debug)] + /// struct Completion { + /// id: u64, + /// } + /// impl TaskCompletion<()> for Completion { + /// fn completed(self, _: ()) {} + /// fn panicked(self, panic: TaskPanicPayload, _: Option) { + /// arty_executor::discard_task_panic_payload(panic); + /// } + /// fn cancelled(self, _: ShutdownCause) {} + /// } + /// + /// let mut executor = Executor::builder().build(); + /// executor.tasks().add_with_completion_inspection( + /// std::future::pending::<()>(), + /// TaskOptions::default(), + /// Completion { id: 7 }, + /// |_, completion, _| Some(completion.id), + /// ); + /// let mut ids = Vec::new(); + /// executor.inspect(1, |id: &u64| ids.push(*id)).unwrap(); + /// assert_eq!(ids, [7]); + /// # } + /// ``` + #[cfg(feature = "inspection")] + pub fn add_with_completion_inspection(&self, future: F, options: TaskOptions, completion: T, inspect: I) + where + F: Future + 'static, + R: 'static, + T: TaskCompletion, + D: 'static, + I: Fn(&F, &T, crate::TaskState) -> Option + 'static, { self.shared .add_inspected_task(future, options, completion, crate::inspection::InspectWith::new(inspect)); diff --git a/crates/arty_executor/tests/inspection.rs b/crates/arty_executor/tests/inspection.rs index ab5ab88168..6a20598009 100644 --- a/crates/arty_executor/tests/inspection.rs +++ b/crates/arty_executor/tests/inspection.rs @@ -25,6 +25,91 @@ impl TaskCompletion for Completion { fn cancelled(self, _cause: ShutdownCause) {} } +#[derive(Debug)] +struct InspectedCompletion { + inspections: Cell, + outcome: Rc>>, +} + +impl TaskCompletion<()> for InspectedCompletion { + fn completed(self, (): ()) { + self.outcome.set(Some(("completed", self.inspections.get()))); + } + + fn panicked(self, panic: TaskPanicPayload, _: Option) { + arty_executor::discard_task_panic_payload(panic); + self.outcome.set(Some(("panicked", self.inspections.get()))); + } + + fn cancelled(self, _: ShutdownCause) { + self.outcome.set(Some(("cancelled", self.inspections.get()))); + } +} + +#[expect(clippy::unwrap_used, reason = "test helpers use unwrap for concise failure backtraces")] +fn inspect_completion(future: impl Future + 'static, poll: bool) -> (Vec, Option<(&'static str, u32)>) { + let mut executor = Executor::builder().build(); + let outcome = Rc::new(Cell::new(None)); + executor.tasks().add_with_completion_inspection( + future, + TaskOptions::default(), + InspectedCompletion { + inspections: Cell::new(0), + outcome: Rc::clone(&outcome), + }, + |_, completion, _| { + completion.inspections.set(completion.inspections.get() + 1); + Some(completion.inspections.get()) + }, + ); + let mut descriptions = Vec::new(); + executor.inspect(1, |value: &u32| descriptions.push(*value)).unwrap(); + if poll { + let _ = executor.execute_cycle(); + } + drop(executor); + (descriptions, outcome.get()) +} + +#[test] +fn inspection_borrows_the_target_that_receives_success() { + assert_eq!(inspect_completion(std::future::ready(()), true), (vec![1], Some(("completed", 1)))); +} + +#[test] +fn inspection_borrows_the_target_that_receives_a_panic() { + assert_eq!( + inspect_completion(async { panic!("inspected task panic") }, true), + (vec![1], Some(("panicked", 1))) + ); +} + +#[test] +fn inspection_borrows_the_target_that_receives_cancellation() { + assert_eq!( + inspect_completion(std::future::pending::<()>(), false), + (vec![1], Some(("cancelled", 1))) + ); +} + +#[test] +fn ordinary_cycles_do_not_inspect_the_completion_target() { + let mut executor = Executor::builder().build(); + let outcome = Rc::new(Cell::new(None)); + executor.tasks().add_with_completion_inspection( + std::future::ready(()), + TaskOptions::default(), + InspectedCompletion { + inspections: Cell::new(0), + outcome: Rc::clone(&outcome), + }, + |_, _, _| -> Option<()> { panic!("ordinary cycles must not inspect completion metadata") }, + ); + let _ = executor.execute_cycle(); + + assert_eq!(outcome.get(), Some(("completed", 0))); +} + #[test] fn ordinary_cycles_never_construct_diagnostic_descriptions() { let mut executor = Executor::builder().build(); @@ -464,6 +549,50 @@ fn observed_polls_pair_callbacks_only_for_described_tasks() { assert_eq!((observer.starts.get(), observer.finishes.get(), observer.errors.get()), (1, 1, 0)); } +#[test] +fn observed_polls_borrow_the_completion_target() { + let mut executor = Executor::builder().build(); + let outcome = Rc::new(Cell::new(None)); + executor.tasks().add_with_completion_inspection( + std::future::ready(()), + TaskOptions::default(), + InspectedCompletion { + inspections: Cell::new(41), + outcome: Rc::clone(&outcome), + }, + |_, completion, state| { + assert_eq!(state, TaskState::Running); + completion.inspections.set(completion.inspections.get() + 1); + Some(completion.inspections.get()) + }, + ); + let observer = Observer::default(); + + let _ = executor.execute_cycle_observed(&observer); + + assert_eq!((observer.starts.get(), observer.finishes.get(), observer.errors.get()), (1, 1, 0)); + assert_eq!(outcome.get(), Some(("completed", 42))); +} + +#[test] +fn rejected_tasks_cancel_the_completion_target_without_inspecting_it() { + let executor = Executor::builder().build(); + executor.shutdown().trigger(ShutdownCause::default()); + let outcome = Rc::new(Cell::new(None)); + + executor.tasks().add_with_completion_inspection( + std::future::pending::<()>(), + TaskOptions::default(), + InspectedCompletion { + inspections: Cell::new(7), + outcome: Rc::clone(&outcome), + }, + |_, _, _| -> Option<()> { panic!("a stopped executor must not inspect rejected tasks") }, + ); + + assert_eq!(outcome.get(), Some(("cancelled", 7))); +} + #[test] fn a_panicking_description_does_not_prevent_its_task_from_running() { let mut executor = Executor::builder().build(); diff --git a/crates/seismograph_runtime/src/task.rs b/crates/seismograph_runtime/src/task.rs index dcb2cb30cf..6288e4fe42 100644 --- a/crates/seismograph_runtime/src/task.rs +++ b/crates/seismograph_runtime/src/task.rs @@ -23,6 +23,20 @@ pub struct EventContext { backtrace: BacktraceCapture, } +/// Identity and timestamp assigned while building a task creation event. +/// +/// Sampling or a concurrent recording change can omit the event after this +/// metadata is assigned. Keep the identity for inspection and later outcomes. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct TaskSpawn { + /// Process-wide task identity. + pub id: TaskId, + /// Concrete task type identity used by the event. + pub descriptor: TypeDescriptorId, + /// Timestamp used by the task creation event. + pub timestamp: EventTimestamp, +} + impl EventContext { pub(crate) const fn new(runtime: RuntimeId, worker: Option, backtrace: BacktraceCapture) -> Self { Self { @@ -44,7 +58,30 @@ impl EventContext { /// Panics if process task identities are exhausted or `body_size` cannot be /// encoded as a value smaller than `u64::MAX`. #[must_use] + #[inline] pub fn spawned(&self, descriptor: TypeDescriptorId, parent: Option, body_size: usize) -> Option { + self.spawned_with(|| descriptor, parent, body_size).map(|spawn| spawn.id) + } + + /// Builds a task creation event and returns its identity and timestamp. + /// + /// Calls `descriptor` only when the recorder runs the event builder. + /// Disabled or suppressed recording does not call it. Sampling and + /// concurrent recording changes can still omit the event after it runs. + /// Reuse the returned timestamp in native task metadata to avoid another + /// clock read. Measure `body_size` before wrapping or type erasure. + /// + /// # Panics + /// + /// Propagates a panic from `descriptor`. Panics if process task identities + /// are exhausted or `body_size` cannot be encoded below `u64::MAX`. + #[must_use] + pub fn spawned_with( + &self, + descriptor: impl FnOnce() -> TypeDescriptorId, + parent: Option, + body_size: usize, + ) -> Option { let mut assigned = None; seismograph::record(EventClass::RuntimeTask, || { let size = u64::try_from(body_size) @@ -52,9 +89,11 @@ impl EventContext { .and_then(|size| size.checked_add(1)) .expect("a concrete task body must be smaller than u64::MAX"); let id = crate::allocate_task_id(); - assigned = Some(id); + let descriptor = descriptor(); + let timestamp = EventTimestamp::now(); + assigned = Some(TaskSpawn { id, descriptor, timestamp }); Some(Record::runtime( - EventTimestamp::now(), + timestamp, EventKind::TaskSpawned, RuntimeEvent { runtime_id: self.runtime, diff --git a/crates/seismograph_runtime/tests/on_demand.rs b/crates/seismograph_runtime/tests/on_demand.rs index 802c411231..7e6e8b9318 100644 --- a/crates/seismograph_runtime/tests/on_demand.rs +++ b/crates/seismograph_runtime/tests/on_demand.rs @@ -377,6 +377,8 @@ fn event_and_capture_values_preserve_copy_and_unwind_contracts() { fn copied() {} value::(); copied::(); + value::(); + copied::(); value::(); value::(); value::(); @@ -676,6 +678,14 @@ fn recording_retains_task_creation_and_real_outcomes_without_a_task_registry() { fn disabled_task_event_context_does_not_initialize_event_buffers() { let _lock = RECORDING.lock().unwrap(); seismograph::recorder(Configuration::default()); + // Disabling recording preserves earlier tests' buffers. Release them before + // checking that this disabled path creates none. + drop( + seismograph::snapshot(SnapshotOptions { + event_buffers: EventBufferDisposition::Stop, + }) + .unwrap(), + ); let runtime = register_runtime(RuntimeMetadata::new("disabled-task-events", 1)); let worker = runtime.register_worker(WorkerMetadata::new(WorkerRole::Core)); let context = worker.handle().task_events(); @@ -691,3 +701,77 @@ fn disabled_task_event_context_does_not_initialize_event_buffers() { assert_eq!((created, snapshot.events.events.len(), snapshot.events.threads.len()), (None, 0, 0)); } + +#[test] +fn lazy_spawn_metadata_matches_the_recorded_event() { + let _lock = RECORDING.lock().unwrap(); + seismograph::recorder(Configuration::default()); + let runtime = register_runtime(RuntimeMetadata::new("lazy-spawn-metadata", 1)); + let worker = runtime.register_worker(WorkerMetadata::new(WorkerRole::Core)); + let context = worker.handle().task_events(); + let descriptor = allocate_type_descriptor_id(); + let parent = allocate_task_id(); + let calls = std::cell::Cell::new(0); + seismograph::recorder(Configuration { + runtime_tasks: Policy::all(false), + ..Configuration::default() + }); + + let spawned = context + .spawned_with( + || { + calls.set(calls.get() + 1); + descriptor + }, + Some(parent), + 64, + ) + .unwrap(); + let encoded = seismograph::snapshot(SnapshotOptions::default()).unwrap(); + let snapshot = seismograph::snapshot::decode(encoded.as_bytes()).unwrap(); + seismograph::recorder(Configuration::default()); + let event = snapshot + .events + .events + .iter() + .find(|event| { + event.kind == EventKind::TaskSpawned + && event + .runtime() + .is_some_and(|payload| payload.runtime_id == runtime.id() && payload.subject_id == spawned.id.get()) + }) + .unwrap(); + let payload = event.runtime().unwrap(); + + assert_eq!( + ( + calls.get(), + spawned.descriptor, + spawned.timestamp, + payload.related_id, + payload.value_0, + payload.value_1 + ), + (1, descriptor, event.timestamp, parent.get(), descriptor.get(), 65) + ); +} + +#[test] +fn disabled_and_suppressed_spawns_do_not_build_type_metadata() { + let _lock = RECORDING.lock().unwrap(); + seismograph::recorder(Configuration::default()); + let runtime = register_runtime(RuntimeMetadata::new("lazy-disabled-spawn", 1)); + let worker = runtime.register_worker(WorkerMetadata::new(WorkerRole::Core)); + let context = worker.handle().task_events(); + let descriptor = || panic!("disabled or suppressed recording must not build task type metadata"); + + let disabled = context.spawned_with(descriptor, None, 0); + seismograph::recorder(Configuration { + runtime_tasks: Policy::all(false), + ..Configuration::default() + }); + let suppressed = without_recording(|| context.spawned_with(descriptor, None, 0)); + seismograph::recorder(Configuration::default()); + + assert_eq!((disabled, suppressed), (None, None)); +} From 4cdc6693661c6da567c69c3e9253362275bd3912 Mon Sep 17 00:00:00 2001 From: Ralf Biedert Date: Mon, 9 Nov 2026 15:31:19 +0100 Subject: [PATCH 02/12] fix(arty): preserve task monitoring metadata across dispatch Capture remote task parents before queueing and restore blocking task context after execution. Report the original scoped future size before erasure without enlarging ordinary async metadata. Add regression coverage and a feature-gated simulated timeout for deterministic capture recovery tests. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- crates/arty/Cargo.toml | 4 +- crates/arty/src/runtime/dispatch/client.rs | 10 + crates/arty/src/runtime/dispatch/core.rs | 30 ++- crates/arty/src/runtime/monitoring/mod.rs | 2 +- crates/arty/src/runtime/monitoring/task.rs | 80 ++++++- .../src/runtime/worker/blocking/worker.rs | 10 +- crates/arty/src/task/preparation/mod.rs | 4 +- crates/arty/src/task/preparation/remote.rs | 22 +- crates/arty/src/task/runtime_scheduler.rs | 6 +- crates/arty/tests/monitoring.rs | 218 +++++++++++++++++- crates/seismograph_runtime/Cargo.toml | 3 + crates/seismograph_runtime/src/collection.rs | 11 + crates/seismograph_runtime/src/lib.rs | 21 +- 13 files changed, 399 insertions(+), 22 deletions(-) diff --git a/crates/arty/Cargo.toml b/crates/arty/Cargo.toml index ea7d2c30c9..251fe12dc2 100644 --- a/crates/arty/Cargo.toml +++ b/crates/arty/Cargo.toml @@ -62,7 +62,7 @@ rt = [ "dep:plurality", "dep:thread_aware", ] -test-util = ["tick/test-util", "many_cpus?/test-util", "seismograph_protocol?/test-util"] +test-util = ["tick/test-util", "many_cpus?/test-util", "seismograph_protocol?/test-util", "seismograph_runtime?/test-util"] [dependencies] arty_executor = { workspace = true, optional = true } @@ -112,7 +112,7 @@ plurality = { path = "../plurality", features = ["std"] } scopeguard = { workspace = true } seismograph = { path = "../seismograph", features = ["monitor"] } seismograph_protocol = { path = "../seismograph_protocol", features = ["test-util"] } -seismograph_runtime = { path = "../seismograph_runtime" } +seismograph_runtime = { path = "../seismograph_runtime", features = ["test-util"] } static_assertions = { workspace = true } testing_aids = { path = "../testing_aids" } thread_aware = { path = "../thread_aware", features = ["std", "derive", "test-utils"] } diff --git a/crates/arty/src/runtime/dispatch/client.rs b/crates/arty/src/runtime/dispatch/client.rs index f4eb39d11d..a8cc551389 100644 --- a/crates/arty/src/runtime/dispatch/client.rs +++ b/crates/arty/src/runtime/dispatch/client.rs @@ -106,6 +106,16 @@ impl DispatcherClient { self.core.spawn(future_factory) } + #[cfg(feature = "monitor")] + pub(crate) fn spawn_with_size(&self, future_factory: FF, body_size: usize) -> JoinHandle + where + FF: FnOnce(Context) -> F + Send + 'static, + F: Future + 'static, + R: Send + 'static, + { + self.core.spawn_with_size(future_factory, body_size) + } + pub(crate) fn spawn_relocating(&self, source: &thread_aware::Thread, future_factory: FF) -> JoinHandle where FF: FnOnce(Context) -> F + Send + 'static, diff --git a/crates/arty/src/runtime/dispatch/core.rs b/crates/arty/src/runtime/dispatch/core.rs index 82f7cc4860..481fb32f45 100644 --- a/crates/arty/src/runtime/dispatch/core.rs +++ b/crates/arty/src/runtime/dispatch/core.rs @@ -20,7 +20,7 @@ use crate::runtime::worker::blocking::Worker; use crate::runtime::{Error, Workers}; use crate::task::Context; use crate::task::handle::{JoinHandle, JoinHandlePool}; -use crate::task::preparation::prepare_remote; +use crate::task::preparation::prepare_remote_with_size; #[derive(Clone, Debug)] pub(crate) struct WorkerEndpoint { @@ -172,6 +172,16 @@ impl DispatcherCore { self.enqueue(self.next_worker_index(), TaskPlacement::Any, future_factory) } + #[cfg(feature = "monitor")] + pub(crate) fn spawn_with_size(&self, future_factory: FF, body_size: usize) -> JoinHandle + where + FF: FnOnce(Context) -> F + Send + 'static, + F: Future + 'static, + R: Send + 'static, + { + self.enqueue_with_size(self.next_worker_index(), TaskPlacement::Any, future_factory, move || body_size) + } + pub(crate) fn spawn_relocating(&self, source: &Thread, future_factory: FF) -> JoinHandle where FF: FnOnce(Context) -> F + Send + 'static, @@ -297,6 +307,22 @@ impl DispatcherCore { FF: FnOnce(Context) -> F + Send + 'static, F: Future + 'static, R: Send + 'static, + { + self.enqueue_with_size(worker_index, placement, future_factory, size_of::) + } + + fn enqueue_with_size( + &self, + worker_index: WorkerIndex, + placement: TaskPlacement, + future_factory: FF, + body_size: S, + ) -> JoinHandle + where + FF: FnOnce(Context) -> F + Send + 'static, + F: Future + 'static, + R: Send + 'static, + S: Fn() -> usize + Send + 'static, { if let Some(join_handle) = JoinHandle::try_from_shutdown(self.shutdown()) { drop(future_factory); @@ -306,7 +332,7 @@ impl DispatcherCore { .worker_endpoints .get(usize::from(worker_index)) .expect("worker index must identify a registered runtime worker"); - let (future_factory, join_handle) = prepare_remote(&self.remote_results, future_factory, placement); + let (future_factory, join_handle) = prepare_remote_with_size(&self.remote_results, future_factory, placement, body_size); // A worker can close its channel before this send if shutdown already // reached that worker. The join handle detects the disconnection. diff --git a/crates/arty/src/runtime/monitoring/mod.rs b/crates/arty/src/runtime/monitoring/mod.rs index 48bf4876ca..0e5c81b2a8 100644 --- a/crates/arty/src/runtime/monitoring/mod.rs +++ b/crates/arty/src/runtime/monitoring/mod.rs @@ -9,4 +9,4 @@ mod task; pub(crate) use asynchronous::WorkerDiagnostics; pub(crate) use runtime::RuntimeDiagnostics; -pub(crate) use task::{TaskInfo, add_task}; +pub(crate) use task::{TaskInfo, add_task, add_task_with_metadata, current_task, enter_current}; diff --git a/crates/arty/src/runtime/monitoring/task.rs b/crates/arty/src/runtime/monitoring/task.rs index ab53b14563..1050ca0955 100644 --- a/crates/arty/src/runtime/monitoring/task.rs +++ b/crates/arty/src/runtime/monitoring/task.rs @@ -31,6 +31,24 @@ pub(super) fn set_current(task: Option) { CURRENT.with(|slot| slot.set(task)); } +pub(crate) fn current_task() -> Option { + CURRENT.with(Cell::get) +} + +#[derive(Debug)] +#[must_use] +pub(crate) struct CurrentTask(Option); + +impl Drop for CurrentTask { + fn drop(&mut self) { + set_current(self.0); + } +} + +pub(crate) fn enter_current(task: TaskId) -> CurrentTask { + CurrentTask(CURRENT.with(|slot| slot.replace(Some(task)))) +} + #[expect( clippy::large_enum_variant, reason = "descriptions are transient stack values; boxing snapshots would allocate during native inspection" @@ -63,7 +81,7 @@ impl TaskMetadata { describe_type: descriptor::, body_size: size_of::(), events, - parent: CURRENT.with(Cell::get), + parent: current_task(), spawned_at: None, } } @@ -165,9 +183,8 @@ struct AsyncTaskInfo { } impl AsyncTaskInfo { - fn new(events: EventContext) -> Self { - let parent = CURRENT.with(Cell::get); - let spawn = events.spawned_with(descriptor::, parent, size_of::()); + fn new(events: EventContext, parent: Option, body_size: usize) -> Self { + let spawn = events.spawned_with(descriptor::, parent, body_size); Self { events, parent, @@ -180,11 +197,11 @@ impl AsyncTaskInfo { *self.identity.get_or_init(seismograph_runtime::allocate_task_id) } - fn describe(&self, state: TaskActivityState) -> Task { + fn describe(&self, state: TaskActivityState, body_size: usize) -> Task { let mut task = Task::new(self.id(), descriptor::(), state); task.parent = self.parent; task.spawned_at = self.spawned_at; - task.future_size_bytes = Some(u64::try_from(size_of::()).expect("a task body fits the platform address space")); + task.future_size_bytes = Some(u64::try_from(body_size).expect("a task body fits the platform address space")); task } } @@ -224,6 +241,24 @@ where F: IntoFuture + 'static, R: 'static, T: TaskCompletion, +{ + // The function item stores no bytes. Only erased scoped futures carry a size. + add_task_with_metadata::(tasks, future, options, completion, current_task(), size_of::); +} + +pub(crate) fn add_task_with_metadata( + tasks: &TaskSet, + future: F, + options: TaskOptions, + completion: T, + parent: Option, + body_size: S, +) where + Body: 'static, + F: IntoFuture + 'static, + R: 'static, + T: TaskCompletion, + S: Fn() -> usize + 'static, { let Some(events) = current_events() else { tasks.add(future, options, completion); @@ -231,13 +266,13 @@ where }; let completion = Completion { target: completion, - info: AsyncTaskInfo::new::(events), + info: AsyncTaskInfo::new::(events, parent, body_size()), }; - tasks.add_with_completion_inspection(future.into_future(), options, completion, |_, completion, state| { + tasks.add_with_completion_inspection(future.into_future(), options, completion, move |_, completion, state| { let info = &completion.info; Some(seismograph::recorder::without_recording(|| match state { - TaskState::Ready => Description::Snapshot(info.describe::(TaskActivityState::Ready)), - TaskState::Waiting => Description::Snapshot(info.describe::(TaskActivityState::Waiting)), + TaskState::Ready => Description::Snapshot(info.describe::(TaskActivityState::Ready, body_size())), + TaskState::Waiting => Description::Snapshot(info.describe::(TaskActivityState::Waiting, body_size())), TaskState::Running => Description::Poll(info.id()), })) }); @@ -245,7 +280,30 @@ where #[cfg(test)] mod tests { - use super::{AsyncTaskInfo, TaskInfo}; + use super::{AsyncTaskInfo, TaskInfo, current_task, enter_current}; + + #[test] + fn current_task_scope_restores_its_parent_after_return_and_panic() { + let parent = seismograph_runtime::allocate_task_id(); + let child = seismograph_runtime::allocate_task_id(); + assert_eq!(current_task(), None); + { + let _parent = enter_current(parent); + { + let _child = enter_current(child); + assert_eq!(current_task(), Some(child)); + } + assert_eq!(current_task(), Some(parent)); + let result = std::panic::catch_unwind(|| { + let _child = enter_current(child); + assert_eq!(current_task(), Some(child)); + panic!("scope restores the parent during unwinding"); + }); + assert!(result.is_err()); + assert_eq!(current_task(), Some(parent)); + } + assert_eq!(current_task(), None); + } #[test] fn async_metadata_is_smaller_and_stays_owner_local() { diff --git a/crates/arty/src/runtime/worker/blocking/worker.rs b/crates/arty/src/runtime/worker/blocking/worker.rs index 58628a2544..90968757a1 100644 --- a/crates/arty/src/runtime/worker/blocking/worker.rs +++ b/crates/arty/src/runtime/worker/blocking/worker.rs @@ -103,7 +103,15 @@ impl JoinableThreadPool { (worker, poll) }) }); - let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(job.body)); + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + #[cfg(feature = "monitor")] + let _current = observation + .as_ref() + .and_then(|(_, poll)| poll.as_ref()) + .and(job.info.as_ref()) + .map(|info| crate::runtime::monitoring::enter_current(info.id())); + (job.body)(); + })); #[cfg(feature = "monitor")] if let Some((worker, Some(poll))) = observation { worker.task_poll_finished(poll); diff --git a/crates/arty/src/task/preparation/mod.rs b/crates/arty/src/task/preparation/mod.rs index ed7889094f..b75f135fba 100644 --- a/crates/arty/src/task/preparation/mod.rs +++ b/crates/arty/src/task/preparation/mod.rs @@ -19,7 +19,9 @@ pub(crate) use cleanup::discard_panicking_drop; #[cfg(test)] use deferred_task::DeferredTask; pub(crate) use deferred_task::DeferredTaskFactory; -pub(crate) use remote::{RemoteFutureFactory, prepare_remote}; +#[cfg(test)] +pub(crate) use remote::prepare_remote; +pub(crate) use remote::{RemoteFutureFactory, prepare_remote_with_size}; #[cfg(test)] #[cfg_attr(coverage_nightly, coverage(off))] // Test scaffolding is not runtime behavior. diff --git a/crates/arty/src/task/preparation/remote.rs b/crates/arty/src/task/preparation/remote.rs index 59be386f7c..325660cf54 100644 --- a/crates/arty/src/task/preparation/remote.rs +++ b/crates/arty/src/task/preparation/remote.rs @@ -88,6 +88,7 @@ impl RemoteFutureFactory { } } +#[cfg(test)] pub(crate) fn prepare_remote( result_pool: &JoinHandlePool, future_factory: FF, @@ -97,15 +98,34 @@ where FF: FnOnce(Context) -> F + Send + 'static, F: Future + 'static, R: Send + 'static, +{ + prepare_remote_with_size(result_pool, future_factory, placement, size_of::) +} + +pub(crate) fn prepare_remote_with_size( + result_pool: &JoinHandlePool, + future_factory: FF, + placement: TaskPlacement, + body_size: S, +) -> (RemoteFutureFactory, JoinHandle) +where + FF: FnOnce(Context) -> F + Send + 'static, + F: Future + 'static, + R: Send + 'static, + S: Fn() -> usize + Send + 'static, { let (handle, result_tx) = result_pool.handle(); + #[cfg(feature = "monitor")] + let parent = crate::runtime::monitoring::current_task(); + #[cfg(not(feature = "monitor"))] + let _ = body_size; let future_factory = TaskFactory::new(future_factory); let future_factory = RemoteFutureFactory::new(move |cx, tasks| { let future_factory = future_factory.into_inner(); let inner = DeferredTaskFactory::new(future_factory, cx); let options = TaskOptions::new(placement); #[cfg(feature = "monitor")] - crate::runtime::monitoring::add_task::(tasks, inner, options, result_tx); + crate::runtime::monitoring::add_task_with_metadata::(tasks, inner, options, result_tx, parent, body_size); #[cfg(not(feature = "monitor"))] tasks.add(inner, options, result_tx); }); diff --git a/crates/arty/src/task/runtime_scheduler.rs b/crates/arty/src/task/runtime_scheduler.rs index c4693305c8..49f664a39b 100644 --- a/crates/arty/src/task/runtime_scheduler.rs +++ b/crates/arty/src/task/runtime_scheduler.rs @@ -135,7 +135,11 @@ impl RuntimeScheduler { // is destroyed. The final sender is dropped after those fields, including on // cancellation or panic. Receiving the result alone is not a destruction guarantee. let factory = unsafe { std::mem::transmute::, BoxedFutureFactory<'static, R>>(factory) }; - futures::executor::block_on(self.dispatcher.spawn(factory)).map_err(Error::new) + #[cfg(feature = "monitor")] + let task = self.dispatcher.spawn_with_size(factory, size_of::()); + #[cfg(not(feature = "monitor"))] + let task = self.dispatcher.spawn(factory); + futures::executor::block_on(task).map_err(Error::new) } /// Lets the runtime choose a worker for an async task and relocates its data. diff --git a/crates/arty/tests/monitoring.rs b/crates/arty/tests/monitoring.rs index 7fb65e09c6..04d7318d57 100644 --- a/crates/arty/tests/monitoring.rs +++ b/crates/arty/tests/monitoring.rs @@ -463,6 +463,218 @@ fn native_tasks_reuse_their_recorded_spawn_timestamp_and_identity() { } } +struct PendingBody { + started: Option>, + release: Option>, + _payload: [u8; N], +} + +impl PendingBody { + fn new(started: mpsc::Sender<()>) -> Self { + Self { + started: Some(started), + release: None, + _payload: [0; N], + } + } +} + +impl Future for PendingBody { + type Output = (); + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { + if let Some(started) = self.started.take() { + started.send(()).unwrap(); + } + self.release + .as_mut() + .map_or(Poll::Pending, |release| Pin::new(release).poll(cx).map(Result::unwrap)) + } +} + +#[test] +fn async_task_sizes_precede_wrapping_and_erasure() { + let _recording = RecorderGuard::new(); + seismograph::recorder(Configuration { + runtime_tasks: Policy::all(false), + ..Configuration::default() + }); + let (io, _) = MonitorIo::new_mocked(); + let runtime = runtime("original-body-sizes", &io); + let (scheduler, destination) = runtime + .scheduler() + .block_on(async |cx| (cx.scheduler().clone(), cx.thread().clone())) + .unwrap(); + let (started, ready) = mpsc::channel(); + let anywhere = runtime + .scheduler() + .spawn_anywhere(Unaware(started.clone()), |_, Unaware(started)| PendingBody::<64>::new(started)); + let at = scheduler.spawn_at(&destination, Unaware(started.clone()), |Unaware(started)| { + PendingBody::<128>::new(started) + }); + let (release, released) = futures::channel::oneshot::channel(); + let native = std::thread::scope(|scope| { + let borrowed = &started; + let runtime = &runtime; + let blocked = scope.spawn(move || { + runtime.scheduler().block_on(|_| { + let mut body = PendingBody::<256>::new(borrowed.clone()); + body.release = Some(released); + body + }) + }); + for _ in 0..3 { + ready.recv_timeout(TEST_TIMEOUT).unwrap(); + } + let native = capture(runtime); + release.send(()).unwrap(); + blocked.join().unwrap().unwrap(); + native + }); + runtime.stop().unwrap(); + assert!(anywhere.join().unwrap_err().is_shutdown()); + assert!(at.join().unwrap_err().is_shutdown()); + let mut sizes = native.tasks.iter().map(|task| task.future_size_bytes.unwrap()).collect::>(); + sizes.sort_unstable(); + assert_eq!( + sizes, + [ + u64::try_from(size_of::>()).unwrap(), + u64::try_from(size_of::>()).unwrap(), + u64::try_from(size_of::>()).unwrap(), + ] + ); + let bytes = seismograph::snapshot(SnapshotOptions::default()).unwrap(); + let recorded = seismograph::snapshot::decode(bytes.as_bytes()).unwrap(); + for task in &native.tasks { + let spawn = recorded + .events + .events + .iter() + .find(|event| { + event.kind == EventKind::TaskSpawned + && event + .runtime() + .is_some_and(|payload| payload.runtime_id == native.id && payload.subject_id == task.id.get()) + }) + .unwrap(); + assert_eq!(task.future_size_bytes, Some(spawn.runtime().unwrap().value_1 - 1)); + } +} + +#[test] +fn queued_remote_children_retain_their_recorded_parent() { + let _recording = RecorderGuard::new(); + seismograph::recorder(Configuration { + runtime_tasks: Policy::all(false), + ..Configuration::default() + }); + let (io, _) = MonitorIo::new_mocked(); + let runtime = Runtime::builder() + .workers(WorkersPolicy::exactly(2)) + .monitor(MonitorPolicy::enabled().name("remote-parent-links").io(io)) + .build() + .unwrap(); + let (started, ready) = mpsc::channel(); + let parent = runtime + .scheduler() + .spawn_anywhere(Unaware(started), |cx, Unaware(started)| async move { + let destination = cx + .control() + .workers() + .into_iter() + .find(|worker| *worker != cx.thread()) + .unwrap() + .clone(); + let at = cx.scheduler().spawn_at(&destination, Unaware(started.clone()), |Unaware(started)| { + PendingBody::<64>::new(started) + }); + let anywhere = cx + .scheduler() + .spawn_anywhere(Unaware(started), |Unaware(started)| PendingBody::<128>::new(started)); + at.await.unwrap(); + anywhere.await.unwrap(); + }); + for _ in 0..2 { + ready.recv_timeout(TEST_TIMEOUT).unwrap(); + } + let native = capture(&runtime); + runtime.stop().unwrap(); + assert!(parent.join().unwrap_err().is_shutdown()); + assert_eq!(native.tasks.len(), 3); + let root = native.tasks.iter().find(|task| task.parent.is_none()).unwrap(); + let children = native.tasks.iter().filter(|task| task.id != root.id).collect::>(); + assert!(children.iter().all(|task| task.parent == Some(root.id))); + let bytes = seismograph::snapshot(SnapshotOptions::default()).unwrap(); + let recorded = seismograph::snapshot::decode(bytes.as_bytes()).unwrap(); + for child in children { + let spawn = recorded + .events + .events + .iter() + .find(|event| { + event.kind == EventKind::TaskSpawned + && event + .runtime() + .is_some_and(|payload| payload.runtime_id == native.id && payload.subject_id == child.id.get()) + }) + .unwrap(); + assert_eq!(spawn.runtime().unwrap().related_id, root.id.get()); + } +} + +#[test] +fn blocking_children_retain_their_recorded_parent() { + let _recording = RecorderGuard::new(); + seismograph::recorder(Configuration { + runtime_tasks: Policy::all(false), + ..Configuration::default() + }); + let (io, _) = MonitorIo::new_mocked(); + let runtime = runtime("blocking-parent-links", &io); + let scheduler = runtime.scheduler().block_on(async |cx| cx.scheduler().clone()).unwrap(); + let (started, ready) = mpsc::channel(); + let (release, released) = mpsc::channel(); + let parent = runtime.scheduler().spawn_blocking(move || { + let child = scheduler.spawn_blocking(|| 42); + started.send(child).unwrap(); + released.recv_timeout(TEST_TIMEOUT).unwrap(); + panic!("recorded parent panic"); + }); + let child = ready.recv_timeout(TEST_TIMEOUT).unwrap(); + let native = capture(&runtime); + release.send(()).unwrap(); + assert!(parent.join().unwrap_err().is_panic()); + assert_eq!(child.join().unwrap(), 42); + runtime.stop().unwrap(); + assert_eq!(native.tasks.len(), 2); + let parent = native + .tasks + .iter() + .find(|task| task.activity.unwrap().state == TaskActivityState::Running) + .unwrap(); + let child = native + .tasks + .iter() + .find(|task| task.activity.unwrap().state == TaskActivityState::Ready) + .unwrap(); + assert_eq!(child.parent, Some(parent.id)); + let bytes = seismograph::snapshot(SnapshotOptions::default()).unwrap(); + let recorded = seismograph::snapshot::decode(bytes.as_bytes()).unwrap(); + let spawn = recorded + .events + .events + .iter() + .find(|event| { + event.kind == EventKind::TaskSpawned + && event + .runtime() + .is_some_and(|payload| payload.runtime_id == native.id && payload.subject_id == child.id.get()) + }) + .unwrap(); + assert_eq!(spawn.runtime().unwrap().related_id, parent.id.get()); +} + #[test] fn collection_reports_the_native_task_limit_instead_of_claiming_completeness() { let _recording = RecorderGuard::new(); @@ -501,7 +713,11 @@ fn a_blocked_async_worker_times_out_and_can_answer_a_later_request() { }); ready.recv_timeout(TEST_TIMEOUT).unwrap(); - let blocked = runtime.diagnostics().unwrap().capture(CaptureOptions::default()).unwrap(); + let blocked = runtime + .diagnostics() + .unwrap() + .capture_with_timeout(CaptureOptions::default()) + .unwrap(); release.send(()).unwrap(); task.join().unwrap(); // Completion precedes the next command batch, which retires the canceled request. diff --git a/crates/seismograph_runtime/Cargo.toml b/crates/seismograph_runtime/Cargo.toml index d892c0399a..b33c291d64 100644 --- a/crates/seismograph_runtime/Cargo.toml +++ b/crates/seismograph_runtime/Cargo.toml @@ -25,6 +25,9 @@ exclude = true [package.metadata.cargo_check_external_types] allowed_external_types = ["seismograph::*"] +[features] +test-util = ["tick/test-util"] + [dependencies] seismograph.workspace = true tick.workspace = true diff --git a/crates/seismograph_runtime/src/collection.rs b/crates/seismograph_runtime/src/collection.rs index 250b7f7094..7ac5577328 100644 --- a/crates/seismograph_runtime/src/collection.rs +++ b/crates/seismograph_runtime/src/collection.rs @@ -115,6 +115,17 @@ pub(crate) fn capture(runtime: &Arc, options: CaptureOptions) -> ) } +#[cfg(feature = "test-util")] +pub(crate) fn capture_with_timeout(runtime: &Arc, options: CaptureOptions) -> Runtime { + capture_with( + runtime, + options, + &SimpleClock::new_frozen(), + |_| {}, + |_, _| Err(mpsc::RecvTimeoutError::Timeout), + ) +} + fn capture_with( runtime: &Arc, options: CaptureOptions, diff --git a/crates/seismograph_runtime/src/lib.rs b/crates/seismograph_runtime/src/lib.rs index 06c1b2799c..697837bb24 100644 --- a/crates/seismograph_runtime/src/lib.rs +++ b/crates/seismograph_runtime/src/lib.rs @@ -213,11 +213,30 @@ impl RuntimeHandle { /// Returns an error for invalid limits or a runtime that already stopped. /// Worker timeouts and provider failures appear in the returned worker rows. pub fn capture(&self, options: snapshot::CaptureOptions) -> Result { + self.validate_capture(&options)?; + Ok(without_recording(|| collection::capture(&self.control, options))) + } + + /// Dispatches a capture and simulates a timeout while waiting for responses. + /// + /// Uses a frozen clock and does not wait for measurement or responses. + /// Queued requests are canceled, just as after a real timeout. + /// + /// # Errors + /// + /// Returns an error for invalid limits or a runtime that already stopped. + #[cfg(feature = "test-util")] + pub fn capture_with_timeout(&self, options: snapshot::CaptureOptions) -> Result { + self.validate_capture(&options)?; + Ok(without_recording(|| collection::capture_with_timeout(&self.control, options))) + } + + fn validate_capture(&self, options: &snapshot::CaptureOptions) -> Result<(), seismograph::Error> { options.validate()?; if self.control.state.load(Ordering::Acquire) == RuntimeState::Stopped.wire_value() { return Err(seismograph::Error::new("cannot inspect a stopped runtime")); } - Ok(without_recording(|| collection::capture(&self.control, options))) + Ok(()) } /// Marks the runtime as stopping and records the transition once. From b2929a40fa156ae417bfa6821bc3e265fd60dd65 Mon Sep 17 00:00:00 2001 From: Ralf Biedert Date: Mon, 9 Nov 2026 16:13:03 +0100 Subject: [PATCH 03/12] fix(arty): handle recording and blocking queue transitions Record real terminal outcomes when recording starts during execution. Keep one task row across shared-queue transitions and skip poll events for shutdown cancellations. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- crates/arty/src/runtime/monitoring/task.rs | 40 +++-- .../arty/src/runtime/worker/blocking/queue.rs | 1 + .../src/runtime/worker/blocking/worker.rs | 23 +-- crates/arty/tests/monitoring.rs | 143 +++++++++++++++++- crates/seismograph_runtime/src/collection.rs | 11 +- crates/seismograph_runtime/src/snapshot.rs | 3 + crates/seismograph_runtime/tests/on_demand.rs | 57 +++++++ 7 files changed, 253 insertions(+), 25 deletions(-) diff --git a/crates/arty/src/runtime/monitoring/task.rs b/crates/arty/src/runtime/monitoring/task.rs index 1050ca0955..b36e23e74d 100644 --- a/crates/arty/src/runtime/monitoring/task.rs +++ b/crates/arty/src/runtime/monitoring/task.rs @@ -7,7 +7,7 @@ use std::collections::HashMap; use std::sync::OnceLock; use arty_executor::{ShutdownCause, TaskCompletion, TaskOptions, TaskPanicPayload, TaskSet, TaskState}; -use seismograph::recorder::event::EventTimestamp; +use seismograph::recorder::event::{EventClass, EventTimestamp}; use seismograph::recorder::runtime::{TaskId, TypeDescriptorId}; use seismograph_runtime::snapshot::{Task, TaskActivityState}; use seismograph_runtime::task::EventContext; @@ -154,22 +154,29 @@ impl TaskInfo { } pub(crate) fn completed(&self) { - if let Some(identity) = self.identity.get() { - self.execution_events.get().unwrap_or(&self.metadata.events).completed(identity.id); + if let Some(id) = self.terminal_id() { + self.execution_events.get().unwrap_or(&self.metadata.events).completed(id); } } pub(crate) fn panicked(&self) { - if let Some(identity) = self.identity.get() { - self.execution_events.get().unwrap_or(&self.metadata.events).panicked(identity.id); + if let Some(id) = self.terminal_id() { + self.execution_events.get().unwrap_or(&self.metadata.events).panicked(id); } } pub(crate) fn canceled(&self) { - if let Some(identity) = self.identity.get() { - self.execution_events.get().unwrap_or(&self.metadata.events).canceled(identity.id); + if let Some(id) = self.terminal_id() { + self.execution_events.get().unwrap_or(&self.metadata.events).canceled(id); } } + + fn terminal_id(&self) -> Option { + self.identity + .get() + .map(|identity| identity.id) + .or_else(|| seismograph::recorder::is_enabled(EventClass::RuntimeTask).then(|| self.id())) + } } // Async metadata stays in the executor's owner-only payload. Blocking metadata @@ -204,6 +211,13 @@ impl AsyncTaskInfo { task.future_size_bytes = Some(u64::try_from(body_size).expect("a task body fits the platform address space")); task } + + fn terminal_id(&self) -> Option { + self.identity + .get() + .copied() + .or_else(|| seismograph::recorder::is_enabled(EventClass::RuntimeTask).then(|| self.id())) + } } #[derive(Debug)] @@ -214,22 +228,22 @@ struct Completion { impl> TaskCompletion for Completion { fn completed(self, result: R) { - if let Some(id) = self.info.identity.get() { - self.info.events.completed(*id); + if let Some(id) = self.info.terminal_id() { + self.info.events.completed(id); } self.target.completed(result); } fn panicked(self, panic: TaskPanicPayload, cause: Option) { - if let Some(id) = self.info.identity.get() { - self.info.events.panicked(*id); + if let Some(id) = self.info.terminal_id() { + self.info.events.panicked(id); } self.target.panicked(panic, cause); } fn cancelled(self, cause: ShutdownCause) { - if let Some(id) = self.info.identity.get() { - self.info.events.canceled(*id); + if let Some(id) = self.info.terminal_id() { + self.info.events.canceled(id); } self.target.cancelled(cause); } diff --git a/crates/arty/src/runtime/worker/blocking/queue.rs b/crates/arty/src/runtime/worker/blocking/queue.rs index bb48efa02b..bf6b5f9a1e 100644 --- a/crates/arty/src/runtime/worker/blocking/queue.rs +++ b/crates/arty/src/runtime/worker/blocking/queue.rs @@ -17,6 +17,7 @@ pub(super) type Job = Box; pub(super) struct QueuedJob { pub(super) body: Job, + pub(super) shutdown: Option, #[cfg(feature = "monitor")] pub(super) info: Option>, } diff --git a/crates/arty/src/runtime/worker/blocking/worker.rs b/crates/arty/src/runtime/worker/blocking/worker.rs index 90968757a1..94d0cdbaf0 100644 --- a/crates/arty/src/runtime/worker/blocking/worker.rs +++ b/crates/arty/src/runtime/worker/blocking/worker.rs @@ -57,6 +57,7 @@ impl JoinableThreadPool { { self.enqueue(QueuedJob { body: Box::new(job), + shutdown: None, #[cfg(feature = "monitor")] info: None, }) @@ -95,8 +96,12 @@ impl JoinableThreadPool { ) { active_count.fetch_add(1, Ordering::SeqCst); queued_count.fetch_sub(1, Ordering::SeqCst); + let canceled = job + .shutdown + .as_ref() + .is_some_and(|shutdown| matches!(shutdown.state(), ShutdownState::Started)); #[cfg(feature = "monitor")] - let observation = diagnostics.as_ref().and_then(|worker| { + let observation = diagnostics.as_ref().filter(|_| !canceled).and_then(|worker| { job.info.as_ref().map(|info| { info.started_on(worker.task_events()); let poll = worker.is_observed().then(|| worker.task_poll_started(info.id())); @@ -110,7 +115,11 @@ impl JoinableThreadPool { .and_then(|(_, poll)| poll.as_ref()) .and(job.info.as_ref()) .map(|info| crate::runtime::monitoring::enter_current(info.id())); - (job.body)(); + if canceled { + drop(job.body); + } else { + (job.body)(); + } })); #[cfg(feature = "monitor")] if let Some((worker, Some(poll))) = observation { @@ -217,16 +226,9 @@ impl Worker { return join_handle; } let join_handle = join_handle.blocking_pool(Arc::clone(&self.pool.identity)); - let task = move || { - if matches!(runtime_shutdown.state(), ShutdownState::Started) { - drop(task); - } else { - task(); - } - }; - let job = QueuedJob { body: Box::new(task), + shutdown: Some(runtime_shutdown), #[cfg(feature = "monitor")] info, }; @@ -299,6 +301,7 @@ impl Pool { { self.enqueue(QueuedJob { body: Box::new(f), + shutdown: None, #[cfg(feature = "monitor")] info: None, }) diff --git a/crates/arty/tests/monitoring.rs b/crates/arty/tests/monitoring.rs index 04d7318d57..568119445e 100644 --- a/crates/arty/tests/monitoring.rs +++ b/crates/arty/tests/monitoring.rs @@ -333,6 +333,143 @@ fn stopped_runtime_does_not_record_new_blocking_tasks() { assert_eq!((outcome.unwrap_err().is_shutdown(), spawned), (true, 0)); } +#[test] +fn recording_enabled_during_blocking_execution_reports_real_outcomes() { + let _recording = RecorderGuard::new(); + for (panics, stops) in [(false, false), (true, false), (false, true)] { + seismograph::recorder(Configuration::default()); + let (io, _) = MonitorIo::new_mocked(); + let runtime = runtime("late-blocking-recording", &io); + let id = runtime.diagnostics().unwrap().id(); + let control = support::runtime_control(&runtime); + let (started, ready) = mpsc::channel(); + let (release, released) = mpsc::channel(); + let running = runtime.scheduler().spawn_blocking(move || { + started.send(()).unwrap(); + released.recv_timeout(TEST_TIMEOUT).unwrap(); + assert!(!panics, "recorded blocking panic"); + }); + ready.recv_timeout(TEST_TIMEOUT).unwrap(); + let queued = stops.then(|| runtime.scheduler().spawn_blocking(|| panic!("canceled function must not run"))); + seismograph::recorder(Configuration { + runtime_tasks: Policy::all(false), + ..Configuration::default() + }); + if stops { + control.request_stop(); + } + release.send(()).unwrap(); + let outcome = running.join(); + if panics { + assert!(outcome.unwrap_err().is_panic()); + } else { + outcome.unwrap(); + } + if let Some(queued) = queued { + assert!(queued.join().unwrap_err().is_shutdown()); + } + runtime.stop().unwrap(); + let bytes = seismograph::snapshot(SnapshotOptions::default()).unwrap(); + let snapshot = seismograph::snapshot::decode(bytes.as_bytes()).unwrap(); + let tasks = snapshot + .events + .events + .iter() + .filter(|event| { + matches!( + event.kind, + EventKind::TaskSpawned + | EventKind::TaskPollStarted + | EventKind::TaskPollFinished + | EventKind::TaskCompleted + | EventKind::TaskPanicked + | EventKind::TaskCanceled + ) && event.runtime().is_some_and(|payload| payload.runtime_id == id) + }) + .collect::>(); + let outcomes = tasks.iter().map(|event| event.kind).collect::>(); + let expected = if panics { + vec![EventKind::TaskPanicked] + } else if stops { + vec![EventKind::TaskCompleted, EventKind::TaskCanceled] + } else { + vec![EventKind::TaskCompleted] + }; + assert_eq!(outcomes, expected); + let identities = tasks.iter().map(|event| event.runtime().unwrap().subject_id).collect::>(); + assert!(identities.iter().all(|id| *id != 0)); + if stops { + assert_ne!(identities[0], identities[1]); + } + } +} + +#[test] +fn recording_enabled_during_an_async_poll_reports_real_outcomes() { + let _recording = RecorderGuard::new(); + for (panics, stops) in [(false, false), (true, false), (false, true)] { + seismograph::recorder(Configuration::default()); + let (io, _) = MonitorIo::new_mocked(); + let runtime = runtime("late-async-recording", &io); + let id = runtime.diagnostics().unwrap().id(); + let control = support::runtime_control(&runtime); + let (started, ready) = mpsc::channel(); + let (release, released) = mpsc::channel(); + let running = runtime.scheduler().spawn_anywhere( + Unaware((started, released, panics, stops)), + |_, Unaware((started, released, panics, stops))| async move { + started.send(()).unwrap(); + released.recv_timeout(TEST_TIMEOUT).unwrap(); + if stops { + pending::<()>().await; + } + assert!(!panics, "recorded async panic"); + }, + ); + ready.recv_timeout(TEST_TIMEOUT).unwrap(); + seismograph::recorder(Configuration { + runtime_tasks: Policy::all(false), + ..Configuration::default() + }); + if stops { + control.request_stop(); + } + release.send(()).unwrap(); + let outcome = running.join(); + let expected = if panics { + assert!(outcome.unwrap_err().is_panic()); + EventKind::TaskPanicked + } else if stops { + assert!(outcome.unwrap_err().is_shutdown()); + EventKind::TaskCanceled + } else { + outcome.unwrap(); + EventKind::TaskCompleted + }; + runtime.stop().unwrap(); + let bytes = seismograph::snapshot(SnapshotOptions::default()).unwrap(); + let snapshot = seismograph::snapshot::decode(bytes.as_bytes()).unwrap(); + let tasks = snapshot + .events + .events + .iter() + .filter(|event| { + matches!( + event.kind, + EventKind::TaskSpawned + | EventKind::TaskPollStarted + | EventKind::TaskPollFinished + | EventKind::TaskCompleted + | EventKind::TaskPanicked + | EventKind::TaskCanceled + ) && event.runtime().is_some_and(|payload| payload.runtime_id == id) + }) + .collect::>(); + assert_eq!(tasks.iter().map(|event| event.kind).collect::>(), [expected]); + assert_ne!(tasks[0].runtime().unwrap().subject_id, 0); + } +} + #[test] fn recording_reports_real_async_and_blocking_outcomes() { let _recording = RecorderGuard::new(); @@ -847,7 +984,11 @@ fn shutdown_records_cancellation_for_a_queued_blocking_function() { && payload.subject_id == id.get() && matches!( event.kind, - EventKind::TaskCompleted | EventKind::TaskPanicked | EventKind::TaskCanceled + EventKind::TaskCompleted + | EventKind::TaskPanicked + | EventKind::TaskCanceled + | EventKind::TaskPollStarted + | EventKind::TaskPollFinished ) }) .map(|_| event.kind) diff --git a/crates/seismograph_runtime/src/collection.rs b/crates/seismograph_runtime/src/collection.rs index 7ac5577328..13d36d469e 100644 --- a/crates/seismograph_runtime/src/collection.rs +++ b/crates/seismograph_runtime/src/collection.rs @@ -188,7 +188,16 @@ fn capture_with( worker.snapshot(batch.collection) }) .collect(); - tasks.sort_unstable_by_key(|task| task.id); + // A shared-queue task can start between two workers' responses. Keep its + // executing observation rather than also reporting its earlier queue entry. + tasks.sort_unstable_by_key(|task| { + ( + task.id, + task.activity + .is_none_or(|activity| activity.state != crate::snapshot::TaskActivityState::Running), + ) + }); + tasks.dedup_by_key(|task| task.id); let state = RuntimeState::from_wire_value(runtime.state.load(Ordering::Acquire)).expect("runtime state stores enum values"); Runtime { id: runtime.id, diff --git a/crates/seismograph_runtime/src/snapshot.rs b/crates/seismograph_runtime/src/snapshot.rs index 7fc0ea7096..6afa6fce23 100644 --- a/crates/seismograph_runtime/src/snapshot.rs +++ b/crates/seismograph_runtime/src/snapshot.rs @@ -395,6 +395,9 @@ pub struct Runtime { /// Active and retired worker metadata. pub workers: Vec, /// Spawn metadata for tasks that have not reached a terminal state. + /// + /// Captures contain at most one row per task identity, sorted by identity. + /// If worker responses contain both queued and running observations, the capture keeps the running observation. pub tasks: Vec, } diff --git a/crates/seismograph_runtime/tests/on_demand.rs b/crates/seismograph_runtime/tests/on_demand.rs index 7e6e8b9318..03cf6a2736 100644 --- a/crates/seismograph_runtime/tests/on_demand.rs +++ b/crates/seismograph_runtime/tests/on_demand.rs @@ -73,6 +73,63 @@ fn all_native_constructor_states_survive_process_snapshot_encoding() { ); } +#[test] +fn captures_keep_one_running_row_when_a_task_also_appears_in_a_queue() { + let _lock = RECORDING.lock().unwrap(); + for running_first in [false, true] { + let runtime = register_runtime(RuntimeMetadata::new("shared-queue-transition", 2)); + let first = runtime.register_worker(WorkerMetadata::new(WorkerRole::Blocking)); + let second = runtime.register_worker(WorkerMetadata::new(WorkerRole::Blocking)); + let task = allocate_task_id(); + let other_task = allocate_task_id(); + let descriptor = allocate_type_descriptor_id(); + let (running, ready) = if running_first { (&first, &second) } else { (&second, &first) }; + running + .handle() + .register_task_provider(move |request| { + request.respond( + vec![Task::new(task, descriptor, TaskActivityState::Running)], + TaskCollection::Complete, + ); + }) + .unwrap(); + ready + .handle() + .register_task_provider(move |request| { + request.respond( + vec![ + Task::new(task, descriptor, TaskActivityState::Ready), + Task::new(other_task, descriptor, TaskActivityState::Ready), + ], + TaskCollection::Truncated, + ); + }) + .unwrap(); + + let captured = runtime.handle().capture(CaptureOptions::default()).unwrap(); + + assert_eq!(captured.tasks.len(), 2); + assert_eq!( + ( + captured.tasks[0].id, + captured.tasks[0].last_worker_id, + captured.tasks[0].activity.unwrap().state + ), + (task, Some(running.id()), TaskActivityState::Running) + ); + assert_eq!(captured.tasks[1].id, other_task); + assert_eq!( + captured + .workers + .iter() + .find(|worker| worker.id == ready.id()) + .unwrap() + .task_collection, + TaskCollection::Truncated + ); + } +} + #[test] fn worker_metadata_transitions_and_runtime_retirement_remain_visible() { use seismograph_runtime::snapshot::{RuntimeState, WorkerState}; From 69134976253318e747c32026ec3566579c4421d9 Mon Sep 17 00:00:00 2001 From: Ralf Biedert Date: Mon, 9 Nov 2026 16:42:55 +0100 Subject: [PATCH 04/12] test(arty): isolate monitoring dependencies and capture sequencing Wait for the collection request slot to retire before another capture. Assert complete snapshots before comparing task identities. Disable native monitoring in scheduling tests and examples. Inject owned monitor I/O in enabled-policy construction tests and compile native-monitor examples without running them. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- crates/arty/README.md | 4 +- crates/arty/src/io/handle.rs | 5 +- crates/arty/src/io/mod.rs | 2 +- crates/arty/src/lib.rs | 14 +-- crates/arty/src/runtime/bootstrap/startup.rs | 2 +- crates/arty/src/runtime/builder.rs | 16 +++- .../arty/src/runtime/config/monitor_policy.rs | 2 +- .../arty/src/runtime/config/panic_policy.rs | 5 +- crates/arty/src/runtime/control.rs | 6 +- crates/arty/src/runtime/handle.rs | 17 +++- crates/arty/src/runtime/mod.rs | 1 + .../runtime/worker/asynchronous/protocol.rs | 1 + .../src/runtime/worker/blocking/worker.rs | 1 + crates/arty/src/task/context.rs | 8 +- crates/arty/src/task/handle/error.rs | 10 +- crates/arty/src/task/handle/join_handle.rs | 2 +- crates/arty/src/task/mod.rs | 2 +- crates/arty/src/task/preparation/tests.rs | 6 +- crates/arty/src/task/runtime_scheduler.rs | 17 +++- crates/arty/src/task/scheduler.rs | 12 ++- crates/arty/tests/behavior_contracts.rs | 14 ++- crates/arty/tests/block_on_spawn.rs | 16 +++- crates/arty/tests/construction.rs | 26 ++++- crates/arty/tests/io/cycles.rs | 1 + crates/arty/tests/io/main.rs | 6 +- crates/arty/tests/join_failures.rs | 2 + crates/arty/tests/monitoring.rs | 11 +++ crates/arty/tests/panic/policy.rs | 2 +- crates/arty/tests/panic/self_wake.rs | 1 + crates/arty/tests/panic_support/mod.rs | 1 + crates/arty/tests/relocation.rs | 96 ++++++++++++++++--- crates/arty/tests/runtime_contracts.rs | 9 +- crates/arty/tests/spawning.rs | 24 ++++- crates/arty/tests/stop.rs | 19 +++- crates/arty/tests/test_macro.rs | 2 +- crates/arty/tests/time.rs | 1 + 36 files changed, 287 insertions(+), 77 deletions(-) diff --git a/crates/arty/README.md b/crates/arty/README.md index 2eb773982e..df3afe79ba 100644 --- a/crates/arty/README.md +++ b/crates/arty/README.md @@ -34,7 +34,7 @@ cargo add arty ```rust use arty::task::{Context, JoinError}; -#[arty::main] +#[arty::main(monitor = disabled)] async fn main(cx: Context) -> Result<(), JoinError> { let answer = cx.scheduler().spawn(async |_| 6 * 7).await?; println!("{answer}"); @@ -80,7 +80,7 @@ The default feature set enables `rt`, `macros`, and `monitor`. This crate was developed as part of The Oxidizer Project. Browse this crate's source code. - [__cargo_doc2readme_dependencies_info]: ggGmYW0CYXZlMC43LjNhdIQborR2_k_xJd4bTcf2krrNPIcbP72Pw1UdRjkbim_eMDe2BBthYvRhcoQbpjTzkOovhJgb4j8Xd8b5dycb1Kf_wRXWtfYbr0QtKZVggRRhZIKCZGFydHllMC42LjGCa2FydHlfbWFjcm9zZTAuNi4x + [__cargo_doc2readme_dependencies_info]: ggGmYW0CYXZlMC43LjNhdIQborR2_k_xJd4bTcf2krrNPIcbP72Pw1UdRjkbim_eMDe2BBthYvRhcoQb94geMR9_5x8bD0NI7vWvQU8bjBeZnbfaMTgbnBA2qp7TcJRhZIKCZGFydHllMC42LjGCa2FydHlfbWFjcm9zZTAuNi4x [__link0]: https://doc.rust-lang.org/stable/std/marker/trait.Send.html [__link1]: https://docs.rs/arty/0.6.1/arty/?search=core::ThreadAware [__link10]: https://docs.rs/arty/0.6.1/arty/core/index.html diff --git a/crates/arty/src/io/handle.rs b/crates/arty/src/io/handle.rs index b8d7aa9691..98cfba6e5b 100644 --- a/crates/arty/src/io/handle.rs +++ b/crates/arty/src/io/handle.rs @@ -236,7 +236,10 @@ mod tests { #[test] fn blocking_marker_io_lookup_survives_scheduler_slot_teardown() { - let runtime = crate::runtime::Runtime::new().unwrap(); + let runtime = crate::runtime::Runtime::builder() + .monitor(crate::runtime::MonitorPolicy::disabled()) + .build() + .unwrap(); let context = runtime.scheduler().block_on(async |cx| cx).unwrap(); let scheduler = context.scheduler().clone(); let io = context.io().clone(); diff --git a/crates/arty/src/io/mod.rs b/crates/arty/src/io/mod.rs index 8375bdb977..7c195d276e 100644 --- a/crates/arty/src/io/mod.rs +++ b/crates/arty/src/io/mod.rs @@ -60,7 +60,7 @@ //! # } //! # } //! # #[cfg(all(feature = "macros", feature = "rt"))] -//! #[arty::main] +//! #[arty::main(monitor = disabled)] //! async fn main(cx: Context) { //! let io = cx.io().context::(&()); //! io.submit("request"); diff --git a/crates/arty/src/lib.rs b/crates/arty/src/lib.rs index eda623d1f0..bd410c161e 100644 --- a/crates/arty/src/lib.rs +++ b/crates/arty/src/lib.rs @@ -37,7 +37,7 @@ //! use arty::task::{Context, JoinError}; //! //! # #[cfg(all(feature = "macros", feature = "rt"))] -//! #[arty::main] +//! #[arty::main(monitor = disabled)] //! async fn main(cx: Context) -> Result<(), JoinError> { //! let answer = cx.scheduler().spawn(async |_| 6 * 7).await?; //! # assert_eq!(answer, 42); @@ -142,7 +142,7 @@ use arty_io_core as _; /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] -/// # #[arty::main] +/// # #[arty::main(monitor = disabled)] /// # async fn main(cx: arty::task::Context) { /// cx.clock().delay(std::time::Duration::from_millis(1)).await; /// println!("Hello from Arty!"); @@ -163,7 +163,7 @@ use arty_io_core as _; /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] -/// #[arty::main(panic = ignore)] +/// #[arty::main(panic = ignore, monitor = disabled)] /// async fn main(cx: arty::task::Context) -> Result<(), arty::task::JoinError> { /// assert!( /// cx.scheduler() @@ -182,7 +182,7 @@ use arty_io_core as _; /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] -/// #[arty::main(workers = 4)] +/// #[arty::main(workers = 4, monitor = disabled)] /// async fn main(cx: arty::task::Context) -> Result<(), arty::task::JoinError> { /// let answer = cx.scheduler().spawn(async |_| 42).await?; /// assert_eq!(answer, 42); @@ -262,7 +262,7 @@ pub use arty_macros::main; /// /// Enable monitoring for a test that exercises monitoring behavior: /// -/// ```test_harness +/// ```no_run,test_harness /// # #[cfg(all(feature = "macros", feature = "rt"))] /// #[arty::test(monitor = enabled)] /// async fn monitored(_cx: arty::task::Context) {} @@ -326,7 +326,7 @@ pub mod task; /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] -/// # #[arty::main] +/// # #[arty::main(monitor = disabled)] /// # async fn main(cx: arty::task::Context) { /// let worker: &arty::core::Thread = cx.thread(); /// assert_eq!(worker.id(), std::thread::current().id()); @@ -358,7 +358,7 @@ pub mod core { /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] -/// # #[arty::main] +/// # #[arty::main(monitor = disabled)] /// # async fn main(cx: arty::task::Context) { /// use std::future::pending; /// use std::time::Duration; diff --git a/crates/arty/src/runtime/bootstrap/startup.rs b/crates/arty/src/runtime/bootstrap/startup.rs index a87a5a78a6..7178214d8a 100644 --- a/crates/arty/src/runtime/bootstrap/startup.rs +++ b/crates/arty/src/runtime/bootstrap/startup.rs @@ -199,7 +199,7 @@ mod tests { let hardware = fake_hardware(PROCESSORS); let runtime = runtime_with_processors( WorkersPolicy::exactly(WORKERS), - MonitorPolicy::enabled().into(), + MonitorPolicy::disabled().into(), &hardware.processors(), ); let cx = runtime.scheduler().block_on(async |cx| cx).unwrap(); diff --git a/crates/arty/src/runtime/builder.rs b/crates/arty/src/runtime/builder.rs index e173a67024..41dd3c2af3 100644 --- a/crates/arty/src/runtime/builder.rs +++ b/crates/arty/src/runtime/builder.rs @@ -27,6 +27,7 @@ use crate::runtime::handle::Runtime; /// use arty::runtime::{BlockingPoolPolicy, Runtime, WorkersPolicy}; /// /// let runtime = Runtime::builder() +/// .monitor(arty::runtime::MonitorPolicy::disabled()) /// .workers(WorkersPolicy::at_most(4)) /// .blocking_pool(BlockingPoolPolicy::shared().max(4)) /// .build()?; @@ -164,7 +165,10 @@ impl RuntimeBuilder { /// # #[cfg(all(feature = "rt", feature = "test-util"))] /// # fn sequential_delay() -> Result<(), Box> { /// let control = arty::time::ClockControl::new().auto_advance_timers(true); - /// let runtime = arty::runtime::Runtime::builder().clock(control).build()?; + /// let runtime = arty::runtime::Runtime::builder() + /// .monitor(arty::runtime::MonitorPolicy::disabled()) + /// .clock(control) + /// .build()?; /// runtime.scheduler().block_on(async |cx| { /// let duration = std::time::Duration::from_secs(30); /// let watch = cx.clock().stopwatch(); @@ -205,7 +209,9 @@ impl RuntimeBuilder { /// ``` /// use arty::runtime::Runtime; /// - /// let runtime = Runtime::builder().build()?; + /// let runtime = Runtime::builder() + /// .monitor(arty::runtime::MonitorPolicy::disabled()) + /// .build()?; /// assert_eq!(runtime.scheduler().block_on(async |_| 42)?, 42); /// # Ok::<(), Box>(()) /// ``` @@ -328,7 +334,11 @@ mod tests { #[test] fn build_and_stop_inside_futures_executor() { futures::executor::block_on(async { - let runtime = Runtime::builder().workers(WorkersPolicy::exactly(1)).build().unwrap(); + let runtime = Runtime::builder() + .monitor(MonitorPolicy::disabled()) + .workers(WorkersPolicy::exactly(1)) + .build() + .unwrap(); let (value, scheduler) = runtime .scheduler() diff --git a/crates/arty/src/runtime/config/monitor_policy.rs b/crates/arty/src/runtime/config/monitor_policy.rs index bcd1aa77ed..8f64490b00 100644 --- a/crates/arty/src/runtime/config/monitor_policy.rs +++ b/crates/arty/src/runtime/config/monitor_policy.rs @@ -14,7 +14,7 @@ /// /// # Examples /// -/// ``` +/// ```no_run /// use arty::runtime::{MonitorPolicy, Runtime}; /// /// let runtime = Runtime::builder() diff --git a/crates/arty/src/runtime/config/panic_policy.rs b/crates/arty/src/runtime/config/panic_policy.rs index 7cee4502e1..7e4d20aa27 100644 --- a/crates/arty/src/runtime/config/panic_policy.rs +++ b/crates/arty/src/runtime/config/panic_policy.rs @@ -19,7 +19,10 @@ /// ``` /// use arty::runtime::{PanicPolicy, Runtime}; /// -/// let runtime = Runtime::builder().panic(PanicPolicy::ignore()).build()?; +/// let runtime = Runtime::builder() +/// .monitor(arty::runtime::MonitorPolicy::disabled()) +/// .panic(PanicPolicy::ignore()) +/// .build()?; /// runtime.stop()?; /// # Ok::<(), Box>(()) /// ``` diff --git a/crates/arty/src/runtime/control.rs b/crates/arty/src/runtime/control.rs index 9652a3cf6d..74181a81b8 100644 --- a/crates/arty/src/runtime/control.rs +++ b/crates/arty/src/runtime/control.rs @@ -32,7 +32,7 @@ impl RuntimeControl { /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] - /// # #[arty::main] + /// # #[arty::main(monitor = disabled)] /// # async fn main(cx: arty::task::Context) -> Result<(), arty::task::JoinError> { /// let tasks: Vec<_> = cx /// .control() @@ -69,7 +69,9 @@ impl RuntimeControl { /// ``` /// use arty::runtime::Runtime; /// - /// let runtime = Runtime::new()?; + /// let runtime = Runtime::builder() + /// .monitor(arty::runtime::MonitorPolicy::disabled()) + /// .build()?; /// let control = runtime /// .scheduler() /// .block_on(async |cx| cx.control().clone())?; diff --git a/crates/arty/src/runtime/handle.rs b/crates/arty/src/runtime/handle.rs index 0f0113d3f2..1bac0775c6 100644 --- a/crates/arty/src/runtime/handle.rs +++ b/crates/arty/src/runtime/handle.rs @@ -45,7 +45,9 @@ use crate::task::RuntimeScheduler; /// ``` /// use arty::runtime::Runtime; /// -/// let runtime = Runtime::new()?; +/// let runtime = Runtime::builder() +/// .monitor(arty::runtime::MonitorPolicy::disabled()) +/// .build()?; /// let task = runtime.scheduler().spawn_anywhere((), |_, ()| async { 42 }); /// assert_eq!(futures::executor::block_on(task)?, 42); /// runtime.stop()?; @@ -78,7 +80,7 @@ impl Runtime { /// /// # Examples /// - /// ``` + /// ```no_run /// use arty::runtime::Runtime; /// /// let runtime = Runtime::new()?; @@ -120,7 +122,9 @@ impl Runtime { /// ``` /// use arty::runtime::Runtime; /// - /// let runtime = Runtime::new()?; + /// let runtime = Runtime::builder() + /// .monitor(arty::runtime::MonitorPolicy::disabled()) + /// .build()?; /// let scheduler = runtime.scheduler(); /// let first = scheduler.spawn_anywhere((), |_, ()| async { 20 }); /// let second = scheduler.spawn_anywhere((), |_, ()| async { 22 }); @@ -179,7 +183,9 @@ impl Runtime { /// ``` /// use arty::runtime::Runtime; /// - /// let runtime = Runtime::new()?; + /// let runtime = Runtime::builder() + /// .monitor(arty::runtime::MonitorPolicy::disabled()) + /// .build()?; /// let error = runtime.scheduler().block_on(async |cx| { /// cx.control().request_stop(); /// cx.scheduler() @@ -346,6 +352,7 @@ mod tests { #[test] fn ignored_task_worker_stop_skips_a_later_active_task() { let runtime = Runtime::builder() + .monitor(crate::runtime::MonitorPolicy::disabled()) .workers(crate::runtime::WorkersPolicy::exactly(1)) .panic(crate::runtime::PanicPolicy::ignore()) .build() @@ -399,6 +406,7 @@ mod tests { #[test] fn ignored_task_worker_drop_skips_a_later_active_task() { let runtime = Runtime::builder() + .monitor(crate::runtime::MonitorPolicy::disabled()) .workers(crate::runtime::WorkersPolicy::exactly(1)) .panic(crate::runtime::PanicPolicy::ignore()) .build() @@ -525,6 +533,7 @@ mod tests { fn single_worker_runtime() -> Runtime { Runtime::builder() + .monitor(crate::runtime::MonitorPolicy::disabled()) .workers(crate::runtime::WorkersPolicy::exactly(1)) .blocking_pool(crate::runtime::BlockingPoolPolicy::shared().max(1)) .build() diff --git a/crates/arty/src/runtime/mod.rs b/crates/arty/src/runtime/mod.rs index 4ca6ac31c3..64198b1688 100644 --- a/crates/arty/src/runtime/mod.rs +++ b/crates/arty/src/runtime/mod.rs @@ -92,6 +92,7 @@ //! //! fn main() -> Result<(), Box> { //! let runtime = Runtime::builder() +//! .monitor(arty::runtime::MonitorPolicy::disabled()) //! .workers(WorkersPolicy::at_most(2)) //! .blocking_pool(BlockingPoolPolicy::shared().max(4)) //! .build()?; diff --git a/crates/arty/src/runtime/worker/asynchronous/protocol.rs b/crates/arty/src/runtime/worker/asynchronous/protocol.rs index 54120974a0..6d4d4371d6 100644 --- a/crates/arty/src/runtime/worker/asynchronous/protocol.rs +++ b/crates/arty/src/runtime/worker/asynchronous/protocol.rs @@ -123,6 +123,7 @@ mod tests { #[test] fn queued_factory_into_inner_returns_the_factory() { let runtime = crate::runtime::Runtime::builder() + .monitor(crate::runtime::MonitorPolicy::disabled()) .workers(crate::runtime::WorkersPolicy::exactly(1)) .build() .unwrap(); diff --git a/crates/arty/src/runtime/worker/blocking/worker.rs b/crates/arty/src/runtime/worker/blocking/worker.rs index 94d0cdbaf0..d34d09cb39 100644 --- a/crates/arty/src/runtime/worker/blocking/worker.rs +++ b/crates/arty/src/runtime/worker/blocking/worker.rs @@ -489,6 +489,7 @@ mod tests { fn runtime_releases_pool_even_when_a_scheduler_is_retained() { execute_or_abandon(|| { let runtime = crate::runtime::Runtime::builder() + .monitor(crate::runtime::MonitorPolicy::disabled()) .workers(crate::runtime::WorkersPolicy::exactly(1)) .build() .unwrap(); diff --git a/crates/arty/src/task/context.rs b/crates/arty/src/task/context.rs index 4767056d37..69894bbf40 100644 --- a/crates/arty/src/task/context.rs +++ b/crates/arty/src/task/context.rs @@ -33,7 +33,7 @@ use crate::task::scheduler::Scheduler; /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] -/// # #[arty::main] +/// # #[arty::main(monitor = disabled)] /// # async fn main(cx: arty::task::Context) -> Result<(), arty::task::JoinError> { /// let answer = cx /// .scheduler() @@ -75,7 +75,7 @@ impl Context { /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] - /// # #[arty::main] + /// # #[arty::main(monitor = disabled)] /// # async fn main(cx: arty::task::Context) -> Result<(), arty::task::JoinError> { /// let parent_thread = cx.thread().id(); /// let child_thread = cx @@ -114,7 +114,7 @@ impl Context { /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] - /// # #[arty::main] + /// # #[arty::main(monitor = disabled)] /// # async fn main(cx: arty::task::Context) { /// assert_eq!(cx.thread().id(), std::thread::current().id()); /// # } @@ -137,7 +137,7 @@ impl Context { /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] - /// # #[arty::main] + /// # #[arty::main(monitor = disabled)] /// # async fn main(cx: arty::task::Context) { /// let duration = std::time::Duration::from_millis(1); /// let watch = cx.clock().stopwatch(); diff --git a/crates/arty/src/task/handle/error.rs b/crates/arty/src/task/handle/error.rs index 4b5a8e7c01..cecd87a5cf 100644 --- a/crates/arty/src/task/handle/error.rs +++ b/crates/arty/src/task/handle/error.rs @@ -36,7 +36,9 @@ use performables::sync::mutex::Mutex; /// ``` /// use arty::runtime::Runtime; /// -/// let runtime = Runtime::new()?; +/// let runtime = Runtime::builder() +/// .monitor(arty::runtime::MonitorPolicy::disabled()) +/// .build()?; /// let error = runtime.scheduler().block_on(async |cx| { /// cx.control().request_stop(); /// cx.scheduler() @@ -71,7 +73,7 @@ impl JoinError { /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] - /// # #[arty::main(panic = ignore)] + /// # #[arty::main(panic = ignore, monitor = disabled)] /// # async fn main(cx: arty::task::Context) -> Result<(), arty::task::JoinError> { /// let result: Result<(), arty::task::JoinError> = /// cx.scheduler().spawn(async |_| panic!("task failed")).await; @@ -93,7 +95,9 @@ impl JoinError { /// ``` /// use arty::runtime::Runtime; /// - /// let runtime = Runtime::new()?; + /// let runtime = Runtime::builder() + /// .monitor(arty::runtime::MonitorPolicy::disabled()) + /// .build()?; /// let error = runtime.scheduler().block_on(async |cx| { /// cx.control().request_stop(); /// cx.scheduler() diff --git a/crates/arty/src/task/handle/join_handle.rs b/crates/arty/src/task/handle/join_handle.rs index a051fc17bf..2fbcdd6f04 100644 --- a/crates/arty/src/task/handle/join_handle.rs +++ b/crates/arty/src/task/handle/join_handle.rs @@ -46,7 +46,7 @@ use crate::runtime::worker::blocking::{LocalWorker as BlockingLocalWorker, PoolI /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] -/// # #[arty::main] +/// # #[arty::main(monitor = disabled)] /// # async fn main(cx: arty::task::Context) -> Result<(), arty::task::JoinError> { /// let task = cx.scheduler().spawn(async |_| 42); /// assert_eq!(task.await?, 42); diff --git a/crates/arty/src/task/mod.rs b/crates/arty/src/task/mod.rs index 2e9eec0886..ccbaccf721 100644 --- a/crates/arty/src/task/mod.rs +++ b/crates/arty/src/task/mod.rs @@ -70,7 +70,7 @@ //! use std::rc::Rc; //! //! # #[cfg(all(feature = "macros", feature = "rt"))] -//! # #[arty::main] +//! # #[arty::main(monitor = disabled)] //! # async fn main( //! # cx: arty::task::Context, //! # ) -> Result<(), Box> { diff --git a/crates/arty/src/task/preparation/tests.rs b/crates/arty/src/task/preparation/tests.rs index 6d00641695..b346a1d4e4 100644 --- a/crates/arty/src/task/preparation/tests.rs +++ b/crates/arty/src/task/preparation/tests.rs @@ -19,7 +19,11 @@ use crate::task::Context; use crate::task::handle::JoinHandlePool; fn test_context() -> (Runtime, Context) { - let runtime = Runtime::builder().workers(WorkersPolicy::exactly(1)).build().unwrap(); + let runtime = Runtime::builder() + .monitor(crate::runtime::MonitorPolicy::disabled()) + .workers(WorkersPolicy::exactly(1)) + .build() + .unwrap(); let cx = runtime.scheduler().block_on(async |cx| cx).unwrap(); (runtime, cx) } diff --git a/crates/arty/src/task/runtime_scheduler.rs b/crates/arty/src/task/runtime_scheduler.rs index 49f664a39b..ad039f49bf 100644 --- a/crates/arty/src/task/runtime_scheduler.rs +++ b/crates/arty/src/task/runtime_scheduler.rs @@ -105,7 +105,9 @@ impl RuntimeScheduler { /// ``` /// use arty::runtime::Runtime; /// - /// let runtime = Runtime::new()?; + /// let runtime = Runtime::builder() + /// .monitor(arty::runtime::MonitorPolicy::disabled()) + /// .build()?; /// let mut message = String::from("Hello"); /// runtime /// .scheduler() @@ -164,7 +166,9 @@ impl RuntimeScheduler { /// ``` /// use arty::runtime::Runtime; /// - /// let runtime = Runtime::new()?; + /// let runtime = Runtime::builder() + /// .monitor(arty::runtime::MonitorPolicy::disabled()) + /// .build()?; /// let task = runtime /// .scheduler() /// .spawn_anywhere(String::from("Arty"), |_, name| async move { name.len() }); @@ -179,7 +183,9 @@ impl RuntimeScheduler { /// use arty::runtime::Runtime; /// /// struct SendOnly(u32); - /// let runtime = Runtime::new()?; + /// let runtime = Runtime::builder() + /// .monitor(arty::runtime::MonitorPolicy::disabled()) + /// .build()?; /// let task = runtime /// .scheduler() /// .spawn_anywhere((), |_, ()| async { SendOnly(42) }); @@ -221,7 +227,9 @@ impl RuntimeScheduler { /// ``` /// use arty::runtime::Runtime; /// - /// let runtime = Runtime::new()?; + /// let runtime = Runtime::builder() + /// .monitor(arty::runtime::MonitorPolicy::disabled()) + /// .build()?; /// let task = runtime.scheduler().spawn_blocking(|| 6 * 7); /// assert_eq!(futures::executor::block_on(task)?, 42); /// # Ok::<(), Box>(()) @@ -416,6 +424,7 @@ mod tests { #[cfg(not(miri))] fn check_borrowed_future_completion(panics: bool) { let runtime = Runtime::builder() + .monitor(crate::runtime::MonitorPolicy::disabled()) .workers(crate::runtime::WorkersPolicy::exactly(1)) .panic(crate::runtime::PanicPolicy::ignore()) .build() diff --git a/crates/arty/src/task/scheduler.rs b/crates/arty/src/task/scheduler.rs index c5207b6098..81857535e7 100644 --- a/crates/arty/src/task/scheduler.rs +++ b/crates/arty/src/task/scheduler.rs @@ -38,7 +38,7 @@ use crate::task::handle::JoinHandle; /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] -/// # #[arty::main] +/// # #[arty::main(monitor = disabled)] /// # async fn main(cx: arty::task::Context) -> Result<(), arty::task::JoinError> { /// let scheduler = cx.scheduler().clone(); /// let task = scheduler.spawn(async |_| 42); @@ -112,7 +112,7 @@ impl Scheduler { /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] - /// # #[arty::main] + /// # #[arty::main(monitor = disabled)] /// # async fn main(cx: arty::task::Context) -> Result<(), arty::task::JoinError> { /// use std::rc::Rc; /// use std::time::Duration; @@ -186,7 +186,7 @@ impl Scheduler { /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] - /// # #[arty::main] + /// # #[arty::main(monitor = disabled)] /// # async fn main(cx: arty::task::Context) -> Result<(), arty::task::JoinError> { /// let answer = cx /// .scheduler() @@ -243,7 +243,7 @@ impl Scheduler { /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] - /// # #[arty::main] + /// # #[arty::main(monitor = disabled)] /// # async fn main( /// # cx: arty::task::Context, /// # ) -> Result<(), Box> { @@ -411,6 +411,7 @@ mod tests { #[test] fn worker_execution_registers_the_current_scheduler() { let runtime = crate::runtime::Runtime::builder() + .monitor(crate::runtime::MonitorPolicy::disabled()) .workers(crate::runtime::WorkersPolicy::exactly(1)) .build() .unwrap(); @@ -429,6 +430,7 @@ mod tests { #[test] fn dispatcher_shutdown_rejects_direct_submissions_immediately() { let runtime = crate::runtime::Runtime::builder() + .monitor(crate::runtime::MonitorPolicy::disabled()) .workers(crate::runtime::WorkersPolicy::exactly(1)) .build() .unwrap(); @@ -465,6 +467,7 @@ mod tests { } let runtime = crate::runtime::Runtime::builder() + .monitor(crate::runtime::MonitorPolicy::disabled()) .workers(crate::runtime::WorkersPolicy::exactly(1)) .build() .unwrap(); @@ -491,6 +494,7 @@ mod tests { #[test] fn external_spawn_survives_scheduler_slot_teardown() { let runtime = crate::runtime::Runtime::builder() + .monitor(crate::runtime::MonitorPolicy::disabled()) .workers(crate::runtime::WorkersPolicy::exactly(1)) .build() .unwrap(); diff --git a/crates/arty/tests/behavior_contracts.rs b/crates/arty/tests/behavior_contracts.rs index 4641c316d9..dd998e634d 100644 --- a/crates/arty/tests/behavior_contracts.rs +++ b/crates/arty/tests/behavior_contracts.rs @@ -26,7 +26,11 @@ use testing_aids::TEST_TIMEOUT; use thread_aware::Unaware; fn runtime(workers: usize) -> Runtime { - Runtime::builder().workers(WorkersPolicy::exactly(workers)).build().unwrap() + Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .workers(WorkersPolicy::exactly(workers)) + .build() + .unwrap() } fn worker_scheduler(runtime: &Runtime) -> Scheduler { @@ -96,6 +100,7 @@ fn assert_blocking_pool_limit(max_threads: usize) { const QUEUED_TASK_COUNT: usize = 6; let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) .workers(WorkersPolicy::exactly(1)) .blocking_pool(BlockingPoolPolicy::shared().max(max_threads)) .build() @@ -172,6 +177,7 @@ fn the_last_processor_setting_replaces_invalid_and_previous_counts() { WorkersPolicy::default(), ] { let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) .workers(previous) .workers(WorkersPolicy::at_most(1)) .build() @@ -190,6 +196,7 @@ fn zero_stack_size_is_rejected_by_the_setter() { #[test] fn resource_setters_preserve_worker_count_and_pool_policy() { let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) .blocking_pool(BlockingPoolPolicy::per_worker()) .workers(WorkersPolicy::at_most(1)) .blocking_pool(BlockingPoolPolicy::shared().max(1)) @@ -272,6 +279,7 @@ fn nested_block_on_rejects_the_factory_before_submission() { fn worker_context_belongs_to_one_runtime_and_rejects_nested_block_on() { completes_without_deadlock(|| { let first = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) .workers(WorkersPolicy::exactly(1)) .blocking_pool(BlockingPoolPolicy::shared().max(1)) .build() @@ -374,6 +382,7 @@ fn configured_clock_control_remains_the_worker_time_source() { let control = ClockControl::new(); let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) .workers(WorkersPolicy::exactly(1)) .clock(control.clone()) .build() @@ -423,6 +432,7 @@ fn stopping_one_runtime_does_not_stop_another() { fn dropping_another_runtime_from_a_worker_panics() { let target = runtime(1); let caller = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) .workers(WorkersPolicy::exactly(1)) .panic(PanicPolicy::ignore()) .build() @@ -504,7 +514,7 @@ fn rejected_remote_joins_report_shutdown_then_reject_repolling() { mod entry_points { use arty::task::Context; - #[arty::main(workers = 1)] + #[arty::main(workers = 1, monitor = disabled)] async fn application_error(_cx: Context) -> Result<(), &'static str> { Err("application") } diff --git a/crates/arty/tests/block_on_spawn.rs b/crates/arty/tests/block_on_spawn.rs index c0a632311c..09801ec2dc 100644 --- a/crates/arty/tests/block_on_spawn.rs +++ b/crates/arty/tests/block_on_spawn.rs @@ -11,7 +11,9 @@ use arty::runtime::Runtime; fn capture_by_value() { #[expect(clippy::useless_vec, reason = "using a vec to test with a type using pointers on the inside")] let vec = vec![10, 11]; - let result = Runtime::new() + let result = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .build() .expect("Failed to create runtime") .scheduler() .block_on(async move |_ctx| vec.len() + 10) @@ -23,7 +25,9 @@ fn capture_by_value() { fn move_and_spawn() { #[expect(clippy::useless_vec, reason = "using a vec to test with a type using pointers on the inside")] let vec = vec![10, 11]; - let result = Runtime::new() + let result = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .build() .expect("Failed to create runtime") .scheduler() .block_on(async move |ctx| { @@ -37,7 +41,9 @@ fn move_and_spawn() { #[test] fn capture_reference() { let vec = vec![10, 11]; - let result = Runtime::new() + let result = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .build() .expect("Failed to create runtime") .scheduler() .block_on(async |_ctx| { @@ -51,7 +57,9 @@ fn capture_reference() { #[test] fn capture_mut_reference() { let mut vec = vec![10, 11]; - let result = Runtime::new() + let result = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .build() .expect("Failed to create runtime") .scheduler() .block_on(async |_ctx| { diff --git a/crates/arty/tests/construction.rs b/crates/arty/tests/construction.rs index 1284d967ed..789a72adc0 100644 --- a/crates/arty/tests/construction.rs +++ b/crates/arty/tests/construction.rs @@ -11,10 +11,14 @@ use std::error::Error as StdError; use arty::runtime::{BlockingPoolPolicy, Error, MonitorPolicy, PanicPolicy, Runtime, WorkersPolicy}; #[test] +#[cfg(any(not(feature = "monitor"), feature = "test-util"))] fn monitor_policy_defaults_to_enabled() { + let policy = MonitorPolicy::default(); + #[cfg(feature = "monitor")] + let policy = policy.io(seismograph_protocol::io::MonitorIo::new_mocked().0); let runtime = Runtime::builder() .workers(WorkersPolicy::exactly(1)) - .monitor(MonitorPolicy::default()) + .monitor(policy) .build() .unwrap(); @@ -23,8 +27,11 @@ fn monitor_policy_defaults_to_enabled() { } #[test] +#[cfg(any(not(feature = "monitor"), feature = "test-util"))] fn explicit_monitor_policies_preserve_runtime_behavior() { for policy in [MonitorPolicy::disabled(), MonitorPolicy::enabled()] { + #[cfg(feature = "monitor")] + let policy = policy.io(seismograph_protocol::io::MonitorIo::new_mocked().0); let runtime = Runtime::builder() .workers(WorkersPolicy::exactly(1)) .monitor(policy) @@ -39,9 +46,14 @@ fn explicit_monitor_policies_preserve_runtime_behavior() { #[test] fn default_runtime_construction_uses_one_worker() { for runtime in [ + #[cfg(not(feature = "monitor"))] Runtime::new().unwrap(), - Runtime::builder().build().unwrap(), - Runtime::builder().workers(WorkersPolicy::default()).build().unwrap(), + Runtime::builder().monitor(MonitorPolicy::disabled()).build().unwrap(), + Runtime::builder() + .monitor(MonitorPolicy::disabled()) + .workers(WorkersPolicy::default()) + .build() + .unwrap(), ] { runtime .scheduler() @@ -57,7 +69,12 @@ fn default_runtime_construction_uses_one_worker() { #[test] fn explicit_panic_policies_preserve_runtime_construction() { for policy in [PanicPolicy::shutdown(), PanicPolicy::ignore()] { - let runtime = Runtime::builder().workers(WorkersPolicy::exactly(1)).panic(policy).build().unwrap(); + let runtime = Runtime::builder() + .monitor(MonitorPolicy::disabled()) + .workers(WorkersPolicy::exactly(1)) + .panic(policy) + .build() + .unwrap(); assert_eq!(runtime.scheduler().block_on(async |_| 42).unwrap(), 42); runtime.stop().unwrap(); @@ -95,6 +112,7 @@ fn zero_shared_blocking_limits_are_rejected_when_the_runtime_is_built() { #[test] fn zero_shared_blocking_limit_can_be_replaced_before_building() { let runtime = Runtime::builder() + .monitor(MonitorPolicy::disabled()) .blocking_pool(BlockingPoolPolicy::shared().max(0)) .blocking_pool(BlockingPoolPolicy::shared().max(1)) .build() diff --git a/crates/arty/tests/io/cycles.rs b/crates/arty/tests/io/cycles.rs index b1370cedf4..25e69c4fe1 100644 --- a/crates/arty/tests/io/cycles.rs +++ b/crates/arty/tests/io/cycles.rs @@ -214,6 +214,7 @@ fn notification_during_first_driver_construction_prevents_primary_wait() { fn controlled_timer_fires_while_a_primary_driver_is_initialized() { let control = tick::ClockControl::new(); let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) .workers(WorkersPolicy::exactly(1)) .clock(control.clone()) .build() diff --git a/crates/arty/tests/io/main.rs b/crates/arty/tests/io/main.rs index d6b289e0ba..84f3612319 100644 --- a/crates/arty/tests/io/main.rs +++ b/crates/arty/tests/io/main.rs @@ -558,7 +558,11 @@ impl Driver for TestDriver { } fn runtime(workers: usize) -> Runtime { - Runtime::builder().workers(WorkersPolicy::exactly(workers)).build().unwrap() + Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .workers(WorkersPolicy::exactly(workers)) + .build() + .unwrap() } fn worker_schedulers(runtime: &Runtime, workers: usize) -> Vec { diff --git a/crates/arty/tests/join_failures.rs b/crates/arty/tests/join_failures.rs index 4d8bbfd77a..a8348a9824 100644 --- a/crates/arty/tests/join_failures.rs +++ b/crates/arty/tests/join_failures.rs @@ -26,6 +26,7 @@ use thread_aware::Unaware; #[cfg(test)] fn runtime_builder() -> RuntimeBuilder { Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) .workers(WorkersPolicy::exactly(1)) .blocking_pool(BlockingPoolPolicy::shared().max(1)) } @@ -55,6 +56,7 @@ async fn async_and_blocking_panics_are_join_errors(cx: Context) { #[test] fn blocking_callback_cannot_wait_for_blocking_work() { let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) .workers(WorkersPolicy::exactly(1)) .blocking_pool(BlockingPoolPolicy::shared().max(2)) .build() diff --git a/crates/arty/tests/monitoring.rs b/crates/arty/tests/monitoring.rs index 568119445e..436e396190 100644 --- a/crates/arty/tests/monitoring.rs +++ b/crates/arty/tests/monitoring.rs @@ -183,17 +183,28 @@ fn task_identity_survives_repeated_collection_and_recording_changes() { }); ready.recv_timeout(TEST_TIMEOUT).unwrap(); let first = capture(&runtime); + runtime.scheduler().block_on(async |_| ()).unwrap(); seismograph::recorder(Configuration { runtime_tasks: Policy::all(false), ..Configuration::default() }); let second = capture(&runtime); + runtime.scheduler().block_on(async |_| ()).unwrap(); seismograph::recorder(Configuration::default()); let third = capture(&runtime); runtime.stop().unwrap(); assert!(task.join().unwrap_err().is_shutdown()); + for snapshot in [&first, &second, &third] { + assert_eq!(snapshot.tasks.len(), 1); + assert!( + snapshot + .workers + .iter() + .all(|worker| worker.task_collection == TaskCollection::Complete) + ); + } assert_eq!( (second.tasks[0].id, third.tasks[0].id, second.tasks[0].spawned_at), (first.tasks[0].id, first.tasks[0].id, None) diff --git a/crates/arty/tests/panic/policy.rs b/crates/arty/tests/panic/policy.rs index 408813c0ca..e263a562b6 100644 --- a/crates/arty/tests/panic/policy.rs +++ b/crates/arty/tests/panic/policy.rs @@ -20,7 +20,7 @@ use crate::support; use crate::support::JoinHandleExt as _; fn runtime_builder(workers: usize, policy: PanicPolicy) -> RuntimeBuilder { - Runtime::builder() + Runtime::builder().monitor(arty::runtime::MonitorPolicy::disabled()) .workers(WorkersPolicy::exactly(workers)) // Two threads keep blocking-panic scenarios bounded without serializing // the callback whose completion or cancellation the test observes. diff --git a/crates/arty/tests/panic/self_wake.rs b/crates/arty/tests/panic/self_wake.rs index 41cad19c40..776f90052a 100644 --- a/crates/arty/tests/panic/self_wake.rs +++ b/crates/arty/tests/panic/self_wake.rs @@ -44,6 +44,7 @@ impl Drop for SelfWakePanic { fn every_worker_retires_self_waking_panics_before_reusing_task_storage() { let repetitions = if cfg!(miri) { 2 } else { 32 }; let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) .workers(WorkersPolicy::exactly(2)) .blocking_pool(BlockingPoolPolicy::shared().max(1)) .panic(PanicPolicy::ignore()) diff --git a/crates/arty/tests/panic_support/mod.rs b/crates/arty/tests/panic_support/mod.rs index c306005f86..7b7760aaff 100644 --- a/crates/arty/tests/panic_support/mod.rs +++ b/crates/arty/tests/panic_support/mod.rs @@ -7,6 +7,7 @@ use arty::runtime::{BlockingPoolPolicy, PanicPolicy, Runtime, WorkersPolicy}; pub(crate) fn runtime() -> Runtime { Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) .workers(WorkersPolicy::exactly(1)) .blocking_pool(BlockingPoolPolicy::shared().max(1)) .panic(PanicPolicy::ignore()) diff --git a/crates/arty/tests/relocation.rs b/crates/arty/tests/relocation.rs index b41b0ae963..8550ba4b79 100644 --- a/crates/arty/tests/relocation.rs +++ b/crates/arty/tests/relocation.rs @@ -65,7 +65,11 @@ impl Driver for RelocationIoDriver { #[test] fn relocating_context_leaves_shared_source_clones_on_the_source_worker() { - let runtime = Runtime::builder().workers(WorkersPolicy::exactly(2)).build().unwrap(); + let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .workers(WorkersPolicy::exactly(2)) + .build() + .unwrap(); let (source, original, scheduler) = runtime .scheduler() .spawn_anywhere( @@ -119,7 +123,11 @@ impl ThreadAware for ReturnedProbe { #[test] fn spawn_anywhere_keeps_round_robin_selection_and_local_children() { - let runtime = Runtime::builder().workers(WorkersPolicy::exactly(2)).build().unwrap(); + let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .workers(WorkersPolicy::exactly(2)) + .build() + .unwrap(); runtime .scheduler() .block_on(async |cx| { @@ -155,8 +163,21 @@ fn spawn_anywhere_keeps_round_robin_selection_and_local_children() { #[test] fn spawn_anywhere_relocates_results_on_the_source_worker() { for (runtime, workers) in [ - (Runtime::new().unwrap(), 1), - (Runtime::builder().workers(WorkersPolicy::exactly(2)).build().unwrap(), 2), + ( + Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .build() + .unwrap(), + 1, + ), + ( + Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .workers(WorkersPolicy::exactly(2)) + .build() + .unwrap(), + 2, + ), ] { runtime .scheduler() @@ -197,7 +218,11 @@ fn spawn_anywhere_relocates_results_on_the_source_worker() { #[test] fn spawn_at_relocates_results_to_the_source_worker() { - let runtime = Runtime::builder().workers(WorkersPolicy::exactly(2)).build().unwrap(); + let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .workers(WorkersPolicy::exactly(2)) + .build() + .unwrap(); let source = runtime.scheduler().block_on(async |cx| cx).unwrap(); let destination = source .control() @@ -239,7 +264,11 @@ fn spawn_at_relocates_results_to_the_source_worker() { // to the worker that receives the result. #[test] fn spawn_anywhere_returns_context_bound_to_the_source_worker() { - let runtime = Runtime::builder().workers(WorkersPolicy::exactly(2)).build().unwrap(); + let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .workers(WorkersPolicy::exactly(2)) + .build() + .unwrap(); runtime .scheduler() .block_on(async |cx| { @@ -261,7 +290,11 @@ fn spawn_anywhere_returns_context_bound_to_the_source_worker() { // selects the result destination, not the submitting worker. #[test] fn spawn_anywhere_relocates_results_to_the_worker_that_receives_the_join() { - let runtime = Runtime::builder().workers(WorkersPolicy::exactly(2)).build().unwrap(); + let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .workers(WorkersPolicy::exactly(2)) + .build() + .unwrap(); let source = runtime.scheduler().block_on(async |cx| cx).unwrap(); let destination = runtime.scheduler().block_on(async |cx| cx).unwrap(); assert_ne!(source.thread(), destination.thread()); @@ -291,7 +324,11 @@ fn spawn_anywhere_relocates_results_to_the_worker_that_receives_the_join() { // receiving Arty worker. #[test] fn spawn_anywhere_results_joined_outside_arty_are_not_relocated() { - let runtime = Runtime::builder().workers(WorkersPolicy::exactly(1)).build().unwrap(); + let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .workers(WorkersPolicy::exactly(1)) + .build() + .unwrap(); let source = runtime.scheduler().block_on(async |cx| cx.thread().clone()).unwrap(); let scheduler = runtime.scheduler().block_on(async |cx| cx.scheduler().clone()).unwrap(); let result = scheduler @@ -321,7 +358,11 @@ fn a_foreign_owner_cannot_rebind_a_registered_thread_id() { } } - let runtime = Runtime::builder().workers(WorkersPolicy::exactly(1)).build().unwrap(); + let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .workers(WorkersPolicy::exactly(1)) + .build() + .unwrap(); let (source, mut scheduler) = runtime .scheduler() .spawn_anywhere((), |cx, ()| async move { (cx.thread().clone(), cx.scheduler().clone()) }) @@ -372,6 +413,7 @@ struct RelocationObservation { #[cfg(test)] fn relocate_and_observe(policy: BlockingPoolPolicy, target: RelocationTarget) -> RelocationObservation { let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) .workers(WorkersPolicy::exactly(2)) .blocking_pool(policy) .build() @@ -508,7 +550,11 @@ fn relocating_context_uses_same_pool_when_shared() { #[test] fn relocated_context_initializes_io_on_the_destination_worker() { - let runtime = Runtime::builder().workers(WorkersPolicy::exactly(2)).build().unwrap(); + let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .workers(WorkersPolicy::exactly(2)) + .build() + .unwrap(); let initialized = Arc::new(Mutex::new(Vec::new())); let destination = runtime .scheduler() @@ -550,8 +596,16 @@ fn relocated_context_initializes_io_on_the_destination_worker() { #[test] fn foreign_owner_relocation_preserves_runtime_binding() { - let source_runtime = Runtime::builder().workers(WorkersPolicy::exactly(1)).build().unwrap(); - let destination_runtime = Runtime::builder().workers(WorkersPolicy::exactly(1)).build().unwrap(); + let source_runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .workers(WorkersPolicy::exactly(1)) + .build() + .unwrap(); + let destination_runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .workers(WorkersPolicy::exactly(1)) + .build() + .unwrap(); let mut cx = source_runtime .scheduler() .spawn_anywhere((), |cx, ()| async move { cx }) @@ -582,7 +636,11 @@ fn repeated_spawn_after_relocation_uses_destination() { // Exercise the cached route repeatedly without turning this regression test into a load test. const SPAWN_COUNT: usize = 100; - let runtime = Runtime::builder().workers(WorkersPolicy::exactly(2)).build().unwrap(); + let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .workers(WorkersPolicy::exactly(2)) + .build() + .unwrap(); let workers: Vec<_> = (0..2) .map(|_| runtime.scheduler().spawn_anywhere((), |cx, ()| async move { cx })) .map(|handle| handle.join().unwrap()) @@ -602,8 +660,16 @@ fn repeated_spawn_after_relocation_uses_destination() { #[test] fn foreign_owner_relocation_preserves_bare_scheduler_binding() { - let source_runtime = Runtime::builder().workers(WorkersPolicy::exactly(1)).build().unwrap(); - let destination_runtime = Runtime::builder().workers(WorkersPolicy::exactly(1)).build().unwrap(); + let source_runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .workers(WorkersPolicy::exactly(1)) + .build() + .unwrap(); + let destination_runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .workers(WorkersPolicy::exactly(1)) + .build() + .unwrap(); let cx = source_runtime .scheduler() .spawn_anywhere((), |cx, ()| async move { cx }) diff --git a/crates/arty/tests/runtime_contracts.rs b/crates/arty/tests/runtime_contracts.rs index 3a65e70f3a..e3b4069e2c 100644 --- a/crates/arty/tests/runtime_contracts.rs +++ b/crates/arty/tests/runtime_contracts.rs @@ -33,6 +33,7 @@ use tick::FutureExt; #[cfg(test)] fn runtime(workers: usize) -> Runtime { Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) .workers(WorkersPolicy::exactly(workers)) .blocking_pool(BlockingPoolPolicy::shared().max(1)) .build() @@ -320,6 +321,7 @@ fn factory_and_poll_panics_return_errors_and_preserve_runtime_usability() { panic_any(Payload(1)) } let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) .workers(WorkersPolicy::exactly(1)) .blocking_pool(BlockingPoolPolicy::shared().max(1)) .panic(PanicPolicy::ignore()) @@ -369,6 +371,7 @@ fn scoped_borrowed_storage_is_destroyed_before_success_or_error() { fn dropping_a_runtime_on_a_blocking_worker_panics() { for policy in [BlockingPoolPolicy::per_worker(), BlockingPoolPolicy::shared().max(1)] { let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) .workers(WorkersPolicy::exactly(1)) .blocking_pool(policy) .build() @@ -385,7 +388,11 @@ fn dropping_a_runtime_on_a_blocking_worker_panics() { #[test] fn repeated_stop_requests_do_not_break_shutdown() { - let runtime = Runtime::builder().workers(WorkersPolicy::exactly(1)).build().unwrap(); + let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .workers(WorkersPolicy::exactly(1)) + .build() + .unwrap(); let control = support::runtime_control(&runtime); control.request_stop(); control.request_stop(); diff --git a/crates/arty/tests/spawning.rs b/crates/arty/tests/spawning.rs index eabd55d598..f820b5cb84 100644 --- a/crates/arty/tests/spawning.rs +++ b/crates/arty/tests/spawning.rs @@ -37,6 +37,7 @@ fn assert_send_sync() { #[test] fn spawned_future_preserves_mutable_self_borrows_across_polls() { let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) .workers(arty::runtime::WorkersPolicy::exactly(1)) .build() .unwrap(); @@ -108,6 +109,7 @@ async fn spawn_some_tasks(cx: Context) { #[test] fn local_factory_runs_once_and_preserves_future_borrows() { let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) .workers(arty::runtime::WorkersPolicy::exactly(1)) .build() .unwrap(); @@ -183,6 +185,7 @@ fn relocating_factories_remain_deferred_and_preserve_future_borrows() { }; let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) .workers(arty::runtime::WorkersPolicy::exactly(1)) .build() .unwrap(); @@ -221,7 +224,11 @@ fn test_worker_affinity() { eprintln!("requires six processors; two-worker affinity is covered by runtime_contracts"); return; } - let runtime = Runtime::builder().workers(WorkersPolicy::exactly(6)).build().unwrap(); + let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .workers(WorkersPolicy::exactly(6)) + .build() + .unwrap(); let (thread1, scheduler1) = runtime .scheduler() .spawn_anywhere((), |cx, ()| async move { (thread::current().id(), cx.scheduler().clone()) }) @@ -242,7 +249,10 @@ fn test_worker_affinity() { #[test] fn remote_factories_create_non_send_futures_on_the_worker() { - let runtime = Runtime::new().unwrap(); + let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .build() + .unwrap(); let (created, polled, associated) = runtime .scheduler() .spawn_anywhere((), |cx: arty::task::Context, ()| { @@ -274,6 +284,7 @@ fn queued_factory_outlives_its_submitting_thread_and_accepts_non_unpin_captures( static_assertions::assert_impl_all!(Capture: Send); static_assertions::assert_not_impl_any!(Capture: Unpin); let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) .workers(arty::runtime::WorkersPolicy::at_most(1)) .build() .unwrap(); @@ -329,7 +340,10 @@ fn runtime_spawn_anywhere_relocates_payload_with_unknown_source() { } } - let runtime = Runtime::new().unwrap(); + let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .build() + .unwrap(); let (source, destination, worker, executed_on) = runtime .scheduler() .spawn_anywhere( @@ -363,6 +377,7 @@ fn runtime_spawn_anywhere_accepts_send_only_non_sync_results() { static_assertions::assert_impl_all!(ResultValue: Send); static_assertions::assert_not_impl_any!(ResultValue: Sync, ThreadAware); let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) .workers(arty::runtime::WorkersPolicy::at_most(1)) .build() .unwrap(); @@ -440,6 +455,7 @@ async fn spawn_at_relocates_payload_to_each_selected_worker_despite_interleaving #[test] fn spawn_at_returns_shutdown_joins_without_invoking_factories() { let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) .workers(arty::runtime::WorkersPolicy::at_most(2)) .build() .unwrap(); @@ -475,6 +491,7 @@ fn spawn_at_rejects_a_worker_from_another_runtime() { } let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) .workers(arty::runtime::WorkersPolicy::exactly(1)) .build() .unwrap(); @@ -483,6 +500,7 @@ fn spawn_at_rejects_a_worker_from_another_runtime() { .block_on(async |cx| (cx.scheduler().clone(), cx.control().clone())) .unwrap(); let other = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) .workers(arty::runtime::WorkersPolicy::exactly(1)) .build() .unwrap(); diff --git a/crates/arty/tests/stop.rs b/crates/arty/tests/stop.rs index a8fd12092e..04f9d6cb8d 100644 --- a/crates/arty/tests/stop.rs +++ b/crates/arty/tests/stop.rs @@ -28,7 +28,10 @@ fn canary() -> (impl Future, BoxedReceiver<()>, std::sync::Weak<()> #[test] fn stop_via_runtime() { execute_or_abandon(|| { - let runtime = Runtime::new().unwrap(); + let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .build() + .unwrap(); let (canary, started, observer) = canary(); @@ -46,7 +49,10 @@ fn stop_via_runtime() { #[test] fn stop_via_async_task() { execute_or_abandon(|| { - let runtime = Runtime::new().unwrap(); + let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .build() + .unwrap(); let (canary, started, observer) = canary(); @@ -74,7 +80,10 @@ fn stop_via_async_task() { #[test] fn stop_scoped() { execute_or_abandon(move || { - _ = Runtime::new().unwrap(); + _ = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .build() + .unwrap(); }) .unwrap(); } @@ -87,7 +96,9 @@ fn stop_in_run() { let task_was_executed = Arc::clone(&task_was_executed); move || { - Runtime::new() + Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) + .build() .unwrap() .scheduler() .block_on(async move |_| { diff --git a/crates/arty/tests/test_macro.rs b/crates/arty/tests/test_macro.rs index b11f7d6da7..0d89b0359d 100644 --- a/crates/arty/tests/test_macro.rs +++ b/crates/arty/tests/test_macro.rs @@ -130,7 +130,7 @@ async fn worker_count_is_exact_even_when_processors_are_shared(cx: Context) { assert_eq!(workers.len(), 3); } -#[arty::main] +#[arty::main(monitor = disabled)] #[::core::prelude::v1::test] async fn default_main_runtime_has_exactly_one_worker(cx: Context) { let tasks: Vec<_> = cx diff --git a/crates/arty/tests/time.rs b/crates/arty/tests/time.rs index e00785e2b9..42684ff2ca 100644 --- a/crates/arty/tests/time.rs +++ b/crates/arty/tests/time.rs @@ -109,6 +109,7 @@ fn controlled_timer_callback_submission_wakes_bound_worker() { let control = ClockControl::new(); let runtime = Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) .workers(WorkersPolicy::exactly(1)) .clock(control.clone()) .build() From 159a06d32804002e95e40489ad0312760d14bbc0 Mon Sep 17 00:00:00 2001 From: Ralf Biedert Date: Mon, 9 Nov 2026 19:04:28 +0100 Subject: [PATCH 05/12] refactor(arty): remove public monitoring test hooks Keep task collection tests private and remove duplicated monitor protocol tests. Preserve production monitoring and add targeted CI regression coverage. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- crates/arty/Cargo.toml | 6 +- .../arty/src/runtime/config/monitor_policy.rs | 30 +-- crates/arty/src/runtime/dispatch/core.rs | 21 ++ crates/arty/src/runtime/handle.rs | 4 +- .../src/runtime/monitoring/asynchronous.rs | 125 ++++++++- crates/arty/src/runtime/monitoring/mod.rs | 3 + crates/arty/src/runtime/monitoring/runtime.rs | 7 +- crates/arty/src/runtime/monitoring/task.rs | 23 ++ .../runtime/monitoring/tests.rs} | 251 +++++------------- .../src/runtime/worker/asynchronous/worker.rs | 92 +++++++ .../arty/src/runtime/worker/blocking/queue.rs | 79 ++++++ crates/arty/tests/construction.rs | 33 --- crates/seismograph_runtime/src/collection.rs | 25 ++ 13 files changed, 431 insertions(+), 268 deletions(-) rename crates/arty/{tests/monitoring.rs => src/runtime/monitoring/tests.rs} (83%) diff --git a/crates/arty/Cargo.toml b/crates/arty/Cargo.toml index 251fe12dc2..bd0f726f60 100644 --- a/crates/arty/Cargo.toml +++ b/crates/arty/Cargo.toml @@ -21,8 +21,6 @@ allowed_external_types = [ "arty_io_core::io_context::IoContext", "arty_macros::main", "arty_macros::test", - "seismograph_protocol::io::MonitorIo", - "seismograph_runtime::RuntimeHandle", "thread_aware_core::thread::NumaNode", "thread_aware_core::thread::Owner", "thread_aware_core::thread::Thread", @@ -40,7 +38,6 @@ default = ["rt", "macros", "monitor"] macros = ["rt", "dep:arty_macros"] monitor = [ "dep:seismograph", - "dep:seismograph_protocol", "dep:seismograph_runtime", "arty_executor?/inspection", "seismograph/monitor", @@ -62,7 +59,7 @@ rt = [ "dep:plurality", "dep:thread_aware", ] -test-util = ["tick/test-util", "many_cpus?/test-util", "seismograph_protocol?/test-util", "seismograph_runtime?/test-util"] +test-util = ["tick/test-util", "many_cpus?/test-util"] [dependencies] arty_executor = { workspace = true, optional = true } @@ -81,7 +78,6 @@ performables = { workspace = true, optional = true } pin-project = { workspace = true, optional = true } plurality = { workspace = true, features = ["std"], optional = true } seismograph = { workspace = true, optional = true } -seismograph_protocol = { workspace = true, optional = true } seismograph_runtime = { workspace = true, optional = true } thread_aware = { workspace = true, features = ["std", "derive"], optional = true } thread_aware_core = { workspace = true, features = ["std"] } diff --git a/crates/arty/src/runtime/config/monitor_policy.rs b/crates/arty/src/runtime/config/monitor_policy.rs index 8f64490b00..3156b0ec27 100644 --- a/crates/arty/src/runtime/config/monitor_policy.rs +++ b/crates/arty/src/runtime/config/monitor_policy.rs @@ -27,20 +27,13 @@ pub struct MonitorPolicy { enabled: bool, name: Option, - #[cfg(feature = "monitor")] - io: Option, } impl MonitorPolicy { /// Creates a policy with monitoring enabled in the runtime configuration. #[must_use] pub const fn enabled() -> Self { - Self { - enabled: true, - name: None, - #[cfg(feature = "monitor")] - io: None, - } + Self { enabled: true, name: None } } /// Creates a policy with monitoring disabled in the runtime configuration. @@ -49,8 +42,6 @@ impl MonitorPolicy { Self { enabled: false, name: None, - #[cfg(feature = "monitor")] - io: None, } } @@ -63,16 +54,6 @@ impl MonitorPolicy { self } - /// Selects isolated monitor I/O for tests. - /// - /// This does not change the process-wide recorder or runtime source. - #[cfg(all(feature = "monitor", any(test, feature = "test-util")))] - #[must_use] - pub fn io(mut self, io: seismograph_protocol::io::MonitorIo) -> Self { - self.io = Some(io); - self - } - pub(crate) const fn is_enabled(&self) -> bool { self.enabled } @@ -81,15 +62,6 @@ impl MonitorPolicy { pub(crate) fn runtime_name(&self) -> &str { self.name.as_deref().unwrap_or("arty") } - - #[cfg(feature = "monitor")] - pub(crate) fn monitor_io(&self) -> Option { - #[cfg(all(miri, any(test, feature = "test-util")))] - if self.io.is_none() { - return Some(seismograph_protocol::io::MonitorIo::new_mocked().0); - } - self.io.clone() - } } impl Default for MonitorPolicy { diff --git a/crates/arty/src/runtime/dispatch/core.rs b/crates/arty/src/runtime/dispatch/core.rs index 481fb32f45..7d32f34687 100644 --- a/crates/arty/src/runtime/dispatch/core.rs +++ b/crates/arty/src/runtime/dispatch/core.rs @@ -555,6 +555,27 @@ mod tests { assert_eq!(drain_rx(&worker2_rx).len(), 0); } + #[cfg(feature = "monitor")] + #[test] + fn dispatcher_stop_updates_its_registered_runtime() { + use seismograph_runtime::snapshot::{CaptureOptions, RuntimeState}; + + let runtime = seismograph_runtime::register_runtime(seismograph_runtime::RuntimeMetadata::new("dispatcher-stop", 1)); + let (worker_tx, _worker_rx) = channel::unbounded(); + let threads = test_threads(1); + let core = DispatcherCore::new(NonEmpty::new(endpoint(worker_tx, &threads[0])), Shutdown::default()); + let client = DispatcherClient::new(core); + client.set_diagnostics(None); + client.set_diagnostics(Some(runtime.handle())); + + client.stop(ShutdownCause::default()); + + assert_eq!( + runtime.handle().capture(CaptureOptions::default()).unwrap().state, + RuntimeState::Stopping + ); + } + #[test] fn spawn_after_command_channel_closed_returns_disconnected_join_handle() { // After a worker closes its command channel during shutdown, later diff --git a/crates/arty/src/runtime/handle.rs b/crates/arty/src/runtime/handle.rs index 1bac0775c6..5b5ce0227e 100644 --- a/crates/arty/src/runtime/handle.rs +++ b/crates/arty/src/runtime/handle.rs @@ -145,9 +145,9 @@ impl Runtime { /// Returns `None` when monitoring is disabled. Capture from a diagnostic /// thread, not from an Arty worker that must answer the collection request. /// This does not change the process-wide recorder. - #[cfg(all(feature = "monitor", any(test, feature = "test-util")))] + #[cfg(all(feature = "monitor", test, not(miri)))] #[must_use] - pub fn diagnostics(&self) -> Option { + pub(crate) fn diagnostics(&self) -> Option { self.diagnostics .as_ref() .map(crate::runtime::monitoring::RuntimeDiagnostics::handle) diff --git a/crates/arty/src/runtime/monitoring/asynchronous.rs b/crates/arty/src/runtime/monitoring/asynchronous.rs index 02b10de72d..299bb4aeeb 100644 --- a/crates/arty/src/runtime/monitoring/asynchronous.rs +++ b/crates/arty/src/runtime/monitoring/asynchronous.rs @@ -45,16 +45,11 @@ impl WorkerDiagnostics { return; } let mut tasks = Vec::new(); - let status = match executor.inspect(request.limit(), |description: &Description| { + let status = collection_status(executor.inspect(request.limit(), |description: &Description| { if let Description::Snapshot(task) = description { tasks.push(task.clone()); } - }) { - Ok(InspectionStatus::Complete) => TaskCollection::Complete, - Ok(InspectionStatus::Truncated) => TaskCollection::Truncated, - Err(InspectionError::Busy) => TaskCollection::Busy, - Err(InspectionError::InvalidLimit | InspectionError::Panicked) => TaskCollection::Failed, - }; + })); let status = if self.failed.replace(false) { TaskCollection::Failed } else { @@ -100,3 +95,119 @@ impl Drop for WorkerDiagnostics { set_events(None); } } + +fn collection_status(result: Result) -> TaskCollection { + match result { + Ok(InspectionStatus::Complete) => TaskCollection::Complete, + Ok(InspectionStatus::Truncated) => TaskCollection::Truncated, + Err(InspectionError::Busy) => TaskCollection::Busy, + Err(InspectionError::InvalidLimit | InspectionError::Panicked) => TaskCollection::Failed, + } +} + +#[cfg(test)] +mod tests { + #[cfg(all(feature = "test-util", not(miri)))] + use std::sync::mpsc; + + use super::*; + + #[test] + fn inspection_outcomes_preserve_completeness_and_failure() { + assert_eq!( + [ + Ok(InspectionStatus::Complete), + Ok(InspectionStatus::Truncated), + Err(InspectionError::Busy), + Err(InspectionError::InvalidLimit), + Err(InspectionError::Panicked), + ] + .map(collection_status), + [ + TaskCollection::Complete, + TaskCollection::Truncated, + TaskCollection::Busy, + TaskCollection::Failed, + TaskCollection::Failed, + ] + ); + } + + #[cfg(all(feature = "test-util", not(miri)))] + #[test] + fn worker_bridge_preserves_demand_failures_and_thread_local_lifecycle() { + use seismograph_runtime::RuntimeMetadata; + use seismograph_runtime::snapshot::{CaptureOptions, WorkerState}; + use testing_aids::TEST_TIMEOUT; + + use crate::runtime::monitoring::task::{current_events, current_task}; + + let runtime = seismograph_runtime::register_runtime(RuntimeMetadata::new("worker-bridge", 1)); + let thread = thread_aware::ThreadBuilder::default().build(std::thread::current().id()); + let dispatcher = DispatcherClient::for_test(&thread); + let diagnostics = WorkerDiagnostics::new(&runtime.handle(), &dispatcher, &thread); + let (sender, receiver) = mpsc::channel(); + diagnostics + .handle + .register_task_provider(move |request| sender.send(request).unwrap()) + .unwrap(); + let mut executor = Executor::builder().build(); + let capture = |executor: &mut Executor| { + std::thread::scope(|scope| { + let handle = runtime.handle(); + let captured = scope.spawn(move || { + handle + .capture(CaptureOptions { + timeout: TEST_TIMEOUT.min(std::time::Duration::from_secs(5)), + ..CaptureOptions::default() + }) + .unwrap() + }); + let request = receiver.recv_timeout(TEST_TIMEOUT).unwrap(); + assert!(diagnostics.is_observed()); + diagnostics.collect(executor, request); + captured.join().unwrap() + }) + }; + + assert!(current_events().is_some()); + assert!(!seismograph::recorder::without_recording(|| diagnostics.is_observed())); + let completion = crate::task::handle::JoinHandlePool::new(crate::runtime::shutdown::Shutdown::default()); + let (_handle, sender) = completion.handle(); + crate::runtime::monitoring::add_task::<(), _, _, _>( + &executor.tasks(), + std::future::pending::<()>(), + arty_executor::TaskOptions::default(), + sender, + ); + assert_eq!( + capture(&mut executor).tasks[0].activity.unwrap().state, + seismograph_runtime::snapshot::TaskActivityState::Ready + ); + diagnostics.parked(); + assert_eq!(capture(&mut executor).workers[0].state, WorkerState::Parked); + diagnostics.unparked(); + assert_eq!(capture(&mut executor).workers[0].state, WorkerState::Running); + + diagnostics.inspection_failed(InspectionError::Panicked); + assert_eq!(capture(&mut executor).workers[0].task_collection, TaskCollection::Failed); + assert_eq!(capture(&mut executor).workers[0].task_collection, TaskCollection::Complete); + diagnostics.poll_started(&()); + diagnostics.poll_finished(false); + assert_eq!(capture(&mut executor).workers[0].task_collection, TaskCollection::Failed); + + let task = seismograph_runtime::allocate_task_id(); + diagnostics.poll_started(&Description::Poll(task)); + assert_eq!(current_task(), Some(task)); + diagnostics.poll_finished(false); + assert_eq!(current_task(), None); + diagnostics.poll_started(&Description::Poll(task)); + drop(diagnostics); + assert_eq!(current_task(), None); + assert!(current_events().is_none()); + assert_eq!( + runtime.handle().capture(CaptureOptions::default()).unwrap().workers[0].state, + WorkerState::Stopped + ); + } +} diff --git a/crates/arty/src/runtime/monitoring/mod.rs b/crates/arty/src/runtime/monitoring/mod.rs index 0e5c81b2a8..c28e9141b3 100644 --- a/crates/arty/src/runtime/monitoring/mod.rs +++ b/crates/arty/src/runtime/monitoring/mod.rs @@ -7,6 +7,9 @@ mod asynchronous; mod runtime; mod task; +#[cfg(all(test, not(miri)))] +mod tests; + pub(crate) use asynchronous::WorkerDiagnostics; pub(crate) use runtime::RuntimeDiagnostics; pub(crate) use task::{TaskInfo, add_task, add_task_with_metadata, current_task, enter_current}; diff --git a/crates/arty/src/runtime/monitoring/runtime.rs b/crates/arty/src/runtime/monitoring/runtime.rs index 93b42e35be..70838cd94b 100644 --- a/crates/arty/src/runtime/monitoring/runtime.rs +++ b/crates/arty/src/runtime/monitoring/runtime.rs @@ -18,10 +18,9 @@ impl RuntimeDiagnostics { return Ok(None); } let workers = u32::try_from(workers).map_err(Error::new)?; - let mut builder = Monitor::builder().name(policy.runtime_name()); - if let Some(io) = policy.monitor_io() { - builder = builder.io(io); - } + let builder = Monitor::builder().name(policy.runtime_name()); + #[cfg(test)] + let builder = builder.io(seismograph_protocol::io::MonitorIo::new_mocked().0); let monitor = builder.start().map_err(Error::new)?; let registration = seismograph_runtime::register_runtime(RuntimeMetadata::new(policy.runtime_name(), workers)); Ok(Some(Self { monitor, registration })) diff --git a/crates/arty/src/runtime/monitoring/task.rs b/crates/arty/src/runtime/monitoring/task.rs index b36e23e74d..5715b96c13 100644 --- a/crates/arty/src/runtime/monitoring/task.rs +++ b/crates/arty/src/runtime/monitoring/task.rs @@ -327,4 +327,27 @@ mod tests { #[cfg(target_pointer_width = "64")] assert_eq!(size_of::(), 56); } + + #[test] + fn absent_worker_events_preserve_plain_task_execution() { + super::set_events(None); + let mut executor = arty_executor::Executor::builder().build(); + let ran = std::rc::Rc::new(std::cell::Cell::new(false)); + let completed = std::rc::Rc::clone(&ran); + let future = async move { completed.set(true) }; + let completion = crate::task::handle::JoinHandlePool::new(crate::runtime::shutdown::Shutdown::default()); + let (_handle, sender) = completion.handle(); + + super::add_task_with_metadata::<(), _, _, _, _>( + &executor.tasks(), + future, + arty_executor::TaskOptions::default(), + sender, + None, + || 0, + ); + let _ = executor.execute_cycle(); + + assert!(ran.get()); + } } diff --git a/crates/arty/tests/monitoring.rs b/crates/arty/src/runtime/monitoring/tests.rs similarity index 83% rename from crates/arty/tests/monitoring.rs rename to crates/arty/src/runtime/monitoring/tests.rs index 436e396190..fb1eadb184 100644 --- a/crates/arty/tests/monitoring.rs +++ b/crates/arty/src/runtime/monitoring/tests.rs @@ -3,36 +3,26 @@ //! Arty supplies native task state independently of event recording. -// Collection uses real worker-response deadlines. Native and careful tests cover -// this integration; the executor retains direct storage coverage under Miri. -#![cfg(all(feature = "rt", feature = "monitor", feature = "test-util", not(miri)))] #![expect(clippy::unwrap_used, reason = "test failures use unwrap for concise backtraces")] -mod support; - use std::future::{pending, poll_fn}; use std::pin::Pin; use std::sync::{Arc, Condvar, Mutex, MutexGuard, mpsc}; use std::task::{Context, Poll}; use std::time::Duration; -use arty::runtime::{BlockingPoolPolicy, MonitorPolicy, PanicPolicy, Runtime, WorkersPolicy}; -use seismograph::recorder::event::{EventClass, EventKind}; +use seismograph::recorder::event::EventKind; use seismograph::recorder::{Configuration, Policy}; use seismograph::snapshot::SnapshotOptions; -use seismograph_protocol::discovery::Directory; -use seismograph_protocol::io::{IoOperation, MonitorIo}; -use seismograph_protocol::message::{RecordingConfiguration, RecordingPolicy, Request, Response}; -use seismograph_protocol::transport::Connection; -use seismograph_protocol::{read_response, write_request}; use seismograph_runtime::snapshot::{ CaptureOptions, Runtime as RuntimeSnapshot, RuntimeState, TaskActivityState, TaskCollection, TaskMetricsScope, }; use seismograph_runtime::worker::WorkerRole; -use support::JoinHandleExt as _; use testing_aids::TEST_TIMEOUT; use thread_aware::Unaware; +use crate::runtime::{BlockingPoolPolicy, MonitorPolicy, PanicPolicy, Runtime, WorkersPolicy}; + static RECORDING: Mutex<()> = Mutex::new(()); struct RecorderGuard { @@ -53,11 +43,11 @@ impl Drop for RecorderGuard { } } -fn runtime(name: &str, io: &MonitorIo) -> Runtime { +fn runtime(name: &str) -> Runtime { Runtime::builder() .workers(WorkersPolicy::exactly(1)) .blocking_pool(BlockingPoolPolicy::shared().max(1)) - .monitor(MonitorPolicy::enabled().name(name).io(io.clone())) + .monitor(MonitorPolicy::enabled().name(name)) .panic(PanicPolicy::ignore()) .build() .unwrap() @@ -86,23 +76,42 @@ fn capture(runtime: &Runtime) -> RuntimeSnapshot { .unwrap() } -fn names(io: &MonitorIo) -> Vec { - let directory = Directory::current(io).unwrap(); - let mut names = directory - .descriptors() - .unwrap() - .map(|descriptor| descriptor.unwrap().name) - .collect::>(); - names.sort(); - names +#[test] +fn monitor_policy_defaults_to_enabled() { + let _recording = RecorderGuard::new(); + let runtime = Runtime::new().unwrap(); + let diagnostics = runtime.diagnostics().unwrap(); + let workers = runtime + .scheduler() + .block_on(async |cx| cx.control().workers().into_iter().count()) + .unwrap(); + let active = diagnostics.capture(CaptureOptions::default()).unwrap().state; + runtime.stop().unwrap(); + let stopped = runtimes().into_iter().find(|runtime| runtime.id == diagnostics.id()).unwrap().state; + + assert_eq!((workers, active, stopped), (1, RuntimeState::Running, RuntimeState::Stopped)); } #[test] -fn disabled_monitoring_registers_no_runtime_or_endpoint() { +fn explicit_monitor_policies_preserve_runtime_behavior() { + let _recording = RecorderGuard::new(); + for policy in [MonitorPolicy::disabled(), MonitorPolicy::enabled()] { + let runtime = Runtime::builder() + .workers(WorkersPolicy::exactly(1)) + .monitor(policy) + .build() + .unwrap(); + + assert_eq!(runtime.scheduler().block_on(async |_| 42).unwrap(), 42); + runtime.stop().unwrap(); + } +} + +#[test] +fn disabled_monitoring_registers_no_runtime() { let _recording = RecorderGuard::new(); - let (io, _) = MonitorIo::new_mocked(); let runtime = Runtime::builder() - .monitor(MonitorPolicy::disabled().name("disabled-monitor").io(io.clone())) + .monitor(MonitorPolicy::disabled().name("disabled-monitor")) .build() .unwrap(); @@ -110,14 +119,13 @@ fn disabled_monitoring_registers_no_runtime_or_endpoint() { let registered = runtimes().iter().any(|runtime| runtime.name == "disabled-monitor"); runtime.stop().unwrap(); - assert_eq!((value, registered, names(&io)), (42, false, Vec::new())); + assert_eq!((value, registered), (42, false)); } #[test] fn local_and_remote_tasks_are_collected_without_recording() { let _recording = RecorderGuard::new(); - let (io, _) = MonitorIo::new_mocked(); - let runtime = runtime("native-tasks", &io); + let runtime = runtime("native-tasks"); let (started, ready) = mpsc::channel(); let parent = runtime .scheduler() @@ -169,8 +177,7 @@ fn local_and_remote_tasks_are_collected_without_recording() { #[test] fn task_identity_survives_repeated_collection_and_recording_changes() { let _recording = RecorderGuard::new(); - let (io, _) = MonitorIo::new_mocked(); - let runtime = runtime("stable-identities", &io); + let runtime = runtime("stable-identities"); let (started, ready) = mpsc::channel(); let task = runtime.scheduler().spawn_anywhere(Unaware(started), |_, Unaware(started)| { let mut started = Some(started); @@ -218,8 +225,7 @@ fn task_identity_survives_repeated_collection_and_recording_changes() { #[test] fn queued_and_running_blocking_tasks_come_from_native_pool_storage() { let _recording = RecorderGuard::new(); - let (io, _) = MonitorIo::new_mocked(); - let runtime = runtime("blocking-tasks", &io); + let runtime = runtime("blocking-tasks"); let (started, ready) = mpsc::channel(); let (release, released) = mpsc::channel(); let first = runtime.scheduler().spawn_blocking(move || { @@ -263,8 +269,7 @@ fn draining_queued_blocking_tasks_does_not_report_worker_resumes() { runtime_tasks: Policy::all(false), ..Configuration::default() }); - let (io, _) = MonitorIo::new_mocked(); - let runtime = runtime("blocking-queue-resumes", &io); + let runtime = runtime("blocking-queue-resumes"); let (started, ready) = mpsc::channel(); let (release, released) = mpsc::channel(); let first = runtime.scheduler().spawn_blocking(move || { @@ -322,8 +327,7 @@ fn draining_queued_blocking_tasks_does_not_report_worker_resumes() { #[test] fn stopped_runtime_does_not_record_new_blocking_tasks() { let _recording = RecorderGuard::new(); - let (io, _) = MonitorIo::new_mocked(); - let runtime = runtime("stopped-blocking-submission", &io); + let runtime = runtime("stopped-blocking-submission"); let id = runtime.diagnostics().unwrap().id(); let scheduler = runtime.scheduler().block_on(async |cx| cx.scheduler().clone()).unwrap(); runtime.stop().unwrap(); @@ -349,10 +353,9 @@ fn recording_enabled_during_blocking_execution_reports_real_outcomes() { let _recording = RecorderGuard::new(); for (panics, stops) in [(false, false), (true, false), (false, true)] { seismograph::recorder(Configuration::default()); - let (io, _) = MonitorIo::new_mocked(); - let runtime = runtime("late-blocking-recording", &io); + let runtime = runtime("late-blocking-recording"); let id = runtime.diagnostics().unwrap().id(); - let control = support::runtime_control(&runtime); + let control = runtime.scheduler().block_on(async |cx| cx.control().clone()).unwrap(); let (started, ready) = mpsc::channel(); let (release, released) = mpsc::channel(); let running = runtime.scheduler().spawn_blocking(move || { @@ -420,10 +423,9 @@ fn recording_enabled_during_an_async_poll_reports_real_outcomes() { let _recording = RecorderGuard::new(); for (panics, stops) in [(false, false), (true, false), (false, true)] { seismograph::recorder(Configuration::default()); - let (io, _) = MonitorIo::new_mocked(); - let runtime = runtime("late-async-recording", &io); + let runtime = runtime("late-async-recording"); let id = runtime.diagnostics().unwrap().id(); - let control = support::runtime_control(&runtime); + let control = runtime.scheduler().block_on(async |cx| cx.control().clone()).unwrap(); let (started, ready) = mpsc::channel(); let (release, released) = mpsc::channel(); let running = runtime.scheduler().spawn_anywhere( @@ -488,8 +490,7 @@ fn recording_reports_real_async_and_blocking_outcomes() { runtime_tasks: Policy::all(false), ..Configuration::default() }); - let (io, _) = MonitorIo::new_mocked(); - let runtime = runtime("recorded-outcomes", &io); + let runtime = runtime("recorded-outcomes"); let id = runtime.diagnostics().unwrap().id(); let scheduler = runtime.scheduler().block_on(async |cx| cx.scheduler().clone()).unwrap(); let completed = scheduler.spawn(async |_| 42); @@ -565,8 +566,7 @@ fn native_tasks_reuse_their_recorded_spawn_timestamp_and_identity() { runtime_tasks: Policy::all(false), ..Configuration::default() }); - let (io, _) = MonitorIo::new_mocked(); - let runtime = runtime("coherent-spawn-metadata", &io); + let runtime = runtime("coherent-spawn-metadata"); let (started, ready) = mpsc::channel(); let asynchronous = runtime .scheduler() @@ -647,8 +647,7 @@ fn async_task_sizes_precede_wrapping_and_erasure() { runtime_tasks: Policy::all(false), ..Configuration::default() }); - let (io, _) = MonitorIo::new_mocked(); - let runtime = runtime("original-body-sizes", &io); + let runtime = runtime("original-body-sizes"); let (scheduler, destination) = runtime .scheduler() .block_on(async |cx| (cx.scheduler().clone(), cx.thread().clone())) @@ -717,10 +716,9 @@ fn queued_remote_children_retain_their_recorded_parent() { runtime_tasks: Policy::all(false), ..Configuration::default() }); - let (io, _) = MonitorIo::new_mocked(); let runtime = Runtime::builder() .workers(WorkersPolicy::exactly(2)) - .monitor(MonitorPolicy::enabled().name("remote-parent-links").io(io)) + .monitor(MonitorPolicy::enabled().name("remote-parent-links")) .build() .unwrap(); let (started, ready) = mpsc::channel(); @@ -778,8 +776,7 @@ fn blocking_children_retain_their_recorded_parent() { runtime_tasks: Policy::all(false), ..Configuration::default() }); - let (io, _) = MonitorIo::new_mocked(); - let runtime = runtime("blocking-parent-links", &io); + let runtime = runtime("blocking-parent-links"); let scheduler = runtime.scheduler().block_on(async |cx| cx.scheduler().clone()).unwrap(); let (started, ready) = mpsc::channel(); let (release, released) = mpsc::channel(); @@ -826,8 +823,7 @@ fn blocking_children_retain_their_recorded_parent() { #[test] fn collection_reports_the_native_task_limit_instead_of_claiming_completeness() { let _recording = RecorderGuard::new(); - let (io, _) = MonitorIo::new_mocked(); - let runtime = runtime("bounded-tasks", &io); + let runtime = runtime("bounded-tasks"); let scheduler = runtime.scheduler().block_on(async |cx| cx.scheduler().clone()).unwrap(); for _ in 0..260 { drop(scheduler.spawn(async |_| pending::<()>().await)); @@ -849,8 +845,7 @@ fn collection_reports_the_native_task_limit_instead_of_claiming_completeness() { #[test] fn a_blocked_async_worker_times_out_and_can_answer_a_later_request() { let _recording = RecorderGuard::new(); - let (io, _) = MonitorIo::new_mocked(); - let runtime = runtime("blocked-worker", &io); + let runtime = runtime("blocked-worker"); let (started, ready) = mpsc::channel(); let (release, released) = mpsc::channel(); let task = runtime @@ -888,65 +883,19 @@ fn a_blocked_async_worker_times_out_and_can_answer_a_later_request() { } #[test] -fn stopping_one_runtime_retires_only_its_endpoint_and_workers() { +fn stopping_one_runtime_retires_only_its_workers() { let _recording = RecorderGuard::new(); - let (io, _) = MonitorIo::new_mocked(); - let first = runtime("endpoint-one", &io); - let second = runtime("endpoint-two", &io); - assert_eq!(names(&io), ["endpoint-one", "endpoint-two"]); + let first = runtime("runtime-one"); + let second = runtime("runtime-two"); first.stop().unwrap(); - let first = runtimes().into_iter().find(|runtime| runtime.name == "endpoint-one").unwrap(); + let first = runtimes().into_iter().find(|runtime| runtime.name == "runtime-one").unwrap(); let second_snapshot = capture(&second); - let remaining = names(&io); second.stop().unwrap(); assert_eq!( - (first.state, first.tasks.is_empty(), second_snapshot.state, remaining, names(&io)), - ( - RuntimeState::Stopped, - true, - RuntimeState::Running, - vec!["endpoint-two".to_owned()], - Vec::new() - ) - ); -} - -#[test] -fn monitor_startup_failure_is_returned_without_publishing_an_endpoint() { - let _recording = RecorderGuard::new(); - let (io, control) = MonitorIo::new_mocked(); - control.fail_next(IoOperation::Bind, std::io::ErrorKind::PermissionDenied.into()); - - let result = Runtime::builder() - .monitor(MonitorPolicy::enabled().name("failed-monitor").io(io.clone())) - .build(); - - drop(result.unwrap_err()); - assert_eq!(names(&io), Vec::::new()); -} - -#[test] -fn stopping_reports_a_monitor_failure_after_removing_the_endpoint() { - let _recording = RecorderGuard::new(); - let (io, control) = MonitorIo::new_mocked(); - control.fail_next( - IoOperation::Accept, - std::io::Error::new(std::io::ErrorKind::PermissionDenied, "injected monitor accept failure"), - ); - let runtime = runtime("failed-listener", &io); - let descriptor = Directory::current(&io).unwrap().descriptors().unwrap().next().unwrap().unwrap(); - let mut connection = Connection::connect(&io, &descriptor, TEST_TIMEOUT).unwrap(); - connection.set_read_timeout(Some(TEST_TIMEOUT)).unwrap(); - write_request(&mut connection, 1, &Request::Hello).unwrap(); - assert!(matches!(read_response(&mut connection).unwrap(), (1, Response::Hello { .. }))); - - let error = runtime.stop().unwrap_err(); - - assert_eq!( - (error.to_string().contains("injected monitor accept failure"), names(&io)), - (true, Vec::new()) + (first.state, first.tasks.is_empty(), second_snapshot.state), + (RuntimeState::Stopped, true, RuntimeState::Running) ); } @@ -957,9 +906,8 @@ fn shutdown_records_cancellation_for_a_queued_blocking_function() { runtime_tasks: Policy::all(false), ..Configuration::default() }); - let (io, _) = MonitorIo::new_mocked(); - let runtime = runtime("canceled-blocking", &io); - let control = support::runtime_control(&runtime); + let runtime = runtime("canceled-blocking"); + let control = runtime.scheduler().block_on(async |cx| cx.control().clone()).unwrap(); let (started, ready) = mpsc::channel(); let (release, released) = mpsc::channel(); let running = runtime.scheduler().spawn_blocking(move || { @@ -1009,77 +957,6 @@ fn shutdown_records_cancellation_for_a_queued_blocking_function() { assert_eq!((canceled.is_shutdown(), outcomes), (true, vec![EventKind::TaskCanceled])); } -#[test] -fn the_monitor_connection_captures_runtime_diagnostics_and_preserves_recording_configuration() { - let _recording = RecorderGuard::new(); - seismograph::recorder(Configuration { - general_events: Policy::all(false), - ..Configuration::default() - }); - let (io, _) = MonitorIo::new_mocked(); - let runtime = runtime("connected-monitor", &io); - let id = runtime.diagnostics().unwrap().id(); - let descriptor = Directory::current(&io).unwrap().descriptors().unwrap().next().unwrap().unwrap(); - let mut connection = Connection::connect(&io, &descriptor, TEST_TIMEOUT).unwrap(); - connection.set_read_timeout(Some(TEST_TIMEOUT)).unwrap(); - - write_request(&mut connection, 1, &Request::Hello).unwrap(); - let hello = read_response(&mut connection).unwrap(); - write_request( - &mut connection, - 2, - &Request::CaptureSnapshot(seismograph_protocol::message::SnapshotOptions::default()), - ) - .unwrap(); - let (request, Response::Snapshot(bytes)) = read_response(&mut connection).unwrap() else { - panic!("the monitor must return the requested snapshot"); - }; - let snapshot = seismograph::snapshot::decode(&bytes).unwrap(); - let source = snapshot - .sources - .iter() - .find(|source| source.id == seismograph_runtime::snapshot::source::ID) - .unwrap(); - let snapshot = seismograph_runtime::snapshot::decode(&source.data).unwrap(); - let captured = snapshot - .runtimes - .iter() - .find(|runtime| runtime.name == "connected-monitor") - .unwrap(); - runtime.stop().unwrap(); - - assert_eq!( - ( - hello, - request, - (captured.id, captured.state), - seismograph::recorder::is_enabled(EventClass::General), - seismograph::recorder::is_enabled(EventClass::RuntimeTask), - names(&io), - ), - ( - ( - 1, - Response::Hello { - instance_id: descriptor.instance_id, - recording: RecordingConfiguration { - general_events: RecordingPolicy { - enabled: true, - ..RecordingPolicy::default() - }, - ..RecordingConfiguration::default() - }, - }, - ), - 2, - (id, RuntimeState::Running), - true, - false, - Vec::new(), - ) - ); -} - #[test] fn task_sizes_describe_user_futures_before_runtime_wrapping() { struct Body { @@ -1099,8 +976,7 @@ fn task_sizes_describe_user_futures_before_runtime_wrapping() { } let _recording = RecorderGuard::new(); - let (io, _) = MonitorIo::new_mocked(); - let runtime = runtime("user-bodies", &io); + let runtime = runtime("user-bodies"); let (started, ready) = mpsc::channel(); drop( runtime @@ -1141,11 +1017,10 @@ fn growing_pool(name: &str, policy: BlockingPoolPolicy, expected_blocking_worker runtime_tasks: Policy::all(false), ..Configuration::default() }); - let (io, _) = MonitorIo::new_mocked(); let runtime = Runtime::builder() .workers(WorkersPolicy::exactly(2)) .blocking_pool(policy.max(2)) - .monitor(MonitorPolicy::enabled().name(name).io(io)) + .monitor(MonitorPolicy::enabled().name(name)) .build() .unwrap(); let scheduler = runtime.scheduler().block_on(async |cx| cx.scheduler().clone()).unwrap(); diff --git a/crates/arty/src/runtime/worker/asynchronous/worker.rs b/crates/arty/src/runtime/worker/asynchronous/worker.rs index b2fe90ccb7..a833d4bb04 100644 --- a/crates/arty/src/runtime/worker/asynchronous/worker.rs +++ b/crates/arty/src/runtime/worker/asynchronous/worker.rs @@ -761,6 +761,98 @@ mod tests { assert!(!worker.monitor_options.policy().is_enabled()); } + #[cfg(all(feature = "monitor", feature = "test-util", not(miri)))] + #[test] + fn monitoring_cycles_only_park_when_waiting_and_leave_idle_polls_unobserved() { + use seismograph_runtime::RuntimeHandle; + use seismograph_runtime::snapshot::{CaptureOptions, WorkerState}; + + #[derive(Clone)] + struct ProbeContext(RuntimeHandle, Arc>>); + + impl ThreadAware for ProbeContext { + fn relocate(&mut self, _source: Option<&Thread>, _destination: &Thread) {} + } + + impl IoContext for ProbeContext { + type ContextOptions = Self; + type Driver = Self; + + fn context(options: Self) -> Self { + options + } + + fn driver(&mut self, _options: DriverOptions) -> Result { + Ok(self.clone()) + } + } + + impl Driver for ProbeContext { + fn waker(&self) -> Waker { + Waker::noop().clone() + } + + fn execute_cycle(&mut self, _cycle: &mut Cycle) -> Result<(), DriverError> { + let snapshot = self.0.capture_with_timeout(CaptureOptions::default()).unwrap(); + self.1.lock().unwrap().push(snapshot.workers[0].state); + Ok(()) + } + + fn shutdown(self) -> Result<(), DriverError> { + Ok(()) + } + } + + let runtime = seismograph_runtime::register_runtime(seismograph_runtime::RuntimeMetadata::new("cycle-observation", 1)); + let mut options = MonitorOptions::default(); + options.set_runtime(Some(runtime.handle())); + let (command_tx, command_rx) = channel::unbounded(); + let mut worker = test_worker_with_monitor_options( + command_tx.clone(), + command_rx, + test_blocking_worker(), + InactiveClock::new_isolated(), + Arc::new(Signal::default()), + options, + ); + let polled = std::rc::Rc::new(std::cell::Cell::new(false)); + let completed = std::rc::Rc::clone(&polled); + crate::runtime::monitoring::add_task::<(), _, _, _>( + &worker.tasks, + async move { + assert_eq!(crate::runtime::monitoring::current_task(), None); + completed.set(true); + }, + TaskOptions::default(), + DiscardCompletion, + ); + seismograph::recorder::without_recording(|| { + worker.prepare_cycle(); + }); + assert!(polled.get()); + + let states = Arc::new(std::sync::Mutex::new(Vec::new())); + let _io = worker + .context + .as_ref() + .unwrap() + .io() + .context::(&ProbeContext(runtime.handle(), Arc::clone(&states))); + assert!(worker.execute_step().is_continue()); + assert_eq!( + runtime.handle().capture_with_timeout(CaptureOptions::default()).unwrap().workers[0].state, + WorkerState::Running + ); + for _ in 0..COMMANDS_PER_CYCLE { + command_tx.send(enqueue(RemoteFutureFactory::new(|_, _| {}))).unwrap(); + } + assert!(worker.execute_step().is_continue()); + + assert_eq!(*states.lock().unwrap(), vec![WorkerState::Parked, WorkerState::Running]); + command_tx.send(Command::Shutdown).unwrap(); + worker.run(); + } + #[cfg(not(miri))] #[test] fn worker_without_primary_waits_for_timer_or_notification() { diff --git a/crates/arty/src/runtime/worker/blocking/queue.rs b/crates/arty/src/runtime/worker/blocking/queue.rs index bf6b5f9a1e..b47470fce4 100644 --- a/crates/arty/src/runtime/worker/blocking/queue.rs +++ b/crates/arty/src/runtime/worker/blocking/queue.rs @@ -22,6 +22,85 @@ pub(super) struct QueuedJob { pub(super) info: Option>, } +#[cfg(all(test, feature = "monitor", feature = "test-util"))] +mod tests { + use seismograph_runtime::RuntimeMetadata; + use seismograph_runtime::snapshot::{CaptureOptions, TaskActivityState, TaskCollection, WorkerState}; + use seismograph_runtime::worker::{WorkerMetadata, WorkerRole}; + + use super::*; + + #[test] + fn blocking_slots_finish_and_retire_without_retaining_a_running_task() { + let runtime = seismograph_runtime::register_runtime(RuntimeMetadata::new("blocking-slots", 1)); + let queue = Arc::new(Queue::default()); + let index = queue.add_worker(); + queue.enable_monitoring(&runtime.handle()); + queue.enable_monitoring(&runtime.handle()); + let info = Arc::new(TaskInfo::new::<()>(*queue.events.get().unwrap())); + assert!(queue.push(QueuedJob { + body: Box::new(|| {}), + shutdown: None, + info: Some(Arc::clone(&info)), + })); + let mut worker = None; + let job = queue.pop(index, &mut worker).unwrap(); + let active = runtime.handle().capture(CaptureOptions::default()).unwrap(); + + queue.finish(index); + let finished = runtime.handle().capture(CaptureOptions::default()).unwrap(); + queue.retire(index); + let retired = runtime.handle().capture(CaptureOptions::default()).unwrap(); + drop(job); + + assert_eq!(active.tasks.len(), 1); + assert_eq!(active.tasks[0].id, info.id()); + assert_eq!(active.tasks[0].activity.unwrap().state, TaskActivityState::Running); + assert!(finished.tasks.is_empty()); + assert!(retired.tasks.is_empty()); + assert_eq!(retired.workers[0].state, WorkerState::Stopped); + } + + #[test] + fn blocking_collection_handles_missing_metadata_limits_and_canceled_requests() { + let runtime = seismograph_runtime::register_runtime(RuntimeMetadata::new("blocking-limits", 1)); + let queue = Arc::new(Queue::default()); + queue.add_worker(); + queue.enable_monitoring(&runtime.handle()); + for info in [None, Some(Arc::new(TaskInfo::new::<()>(*queue.events.get().unwrap())))] { + assert!(queue.push(QueuedJob { + body: Box::new(|| {}), + shutdown: None, + info, + })); + } + let truncated = runtime + .handle() + .capture(CaptureOptions { + limit_per_worker: 1, + ..CaptureOptions::default() + }) + .unwrap(); + let complete = runtime.handle().capture(CaptureOptions::default()).unwrap(); + + let source = seismograph_runtime::register_runtime(RuntimeMetadata::new("canceled-blocking-request", 1)); + let worker = source.register_worker(WorkerMetadata::new(WorkerRole::Core)); + let (sender, receiver) = std::sync::mpsc::channel(); + worker + .handle() + .register_task_provider(move |request| sender.send(request).unwrap()) + .unwrap(); + source.handle().capture_with_timeout(CaptureOptions::default()).unwrap(); + queue.collect(0, receiver.try_recv().unwrap()); + + assert!(truncated.tasks.is_empty()); + assert_eq!(truncated.workers[0].task_collection, TaskCollection::Truncated); + assert_eq!(complete.tasks.len(), 1); + assert_eq!(complete.tasks[0].activity.unwrap().state, TaskActivityState::Ready); + assert_eq!(complete.workers[0].task_collection, TaskCollection::Complete); + } +} + #[derive(Default)] struct WorkerSlot { #[cfg(feature = "monitor")] diff --git a/crates/arty/tests/construction.rs b/crates/arty/tests/construction.rs index 789a72adc0..8b142a6a29 100644 --- a/crates/arty/tests/construction.rs +++ b/crates/arty/tests/construction.rs @@ -10,39 +10,6 @@ use std::error::Error as StdError; use arty::runtime::{BlockingPoolPolicy, Error, MonitorPolicy, PanicPolicy, Runtime, WorkersPolicy}; -#[test] -#[cfg(any(not(feature = "monitor"), feature = "test-util"))] -fn monitor_policy_defaults_to_enabled() { - let policy = MonitorPolicy::default(); - #[cfg(feature = "monitor")] - let policy = policy.io(seismograph_protocol::io::MonitorIo::new_mocked().0); - let runtime = Runtime::builder() - .workers(WorkersPolicy::exactly(1)) - .monitor(policy) - .build() - .unwrap(); - - assert_eq!(runtime.scheduler().block_on(async |_| 42).unwrap(), 42); - runtime.stop().unwrap(); -} - -#[test] -#[cfg(any(not(feature = "monitor"), feature = "test-util"))] -fn explicit_monitor_policies_preserve_runtime_behavior() { - for policy in [MonitorPolicy::disabled(), MonitorPolicy::enabled()] { - #[cfg(feature = "monitor")] - let policy = policy.io(seismograph_protocol::io::MonitorIo::new_mocked().0); - let runtime = Runtime::builder() - .workers(WorkersPolicy::exactly(1)) - .monitor(policy) - .build() - .unwrap(); - - assert_eq!(runtime.scheduler().block_on(async |_| 42).unwrap(), 42); - runtime.stop().unwrap(); - } -} - #[test] fn default_runtime_construction_uses_one_worker() { for runtime in [ diff --git a/crates/seismograph_runtime/src/collection.rs b/crates/seismograph_runtime/src/collection.rs index 13d36d469e..315fcc4b88 100644 --- a/crates/seismograph_runtime/src/collection.rs +++ b/crates/seismograph_runtime/src/collection.rs @@ -326,6 +326,31 @@ mod tests { ) } + #[cfg(feature = "test-util")] + #[test] + fn forced_timeout_marks_an_unanswered_worker_without_waiting() { + let runtime = register_runtime(RuntimeMetadata::new("forced-timeout", 1)); + let worker = runtime.register_worker(WorkerMetadata::new(WorkerRole::Core)); + let (sender, receiver) = mpsc::channel(); + worker + .handle() + .register_task_provider(move |request| sender.send(request).unwrap()) + .unwrap(); + + let captured = runtime + .handle() + .capture_with_timeout(CaptureOptions { + measure_for: Duration::from_secs(1), + ..CaptureOptions::default() + }) + .unwrap(); + let request = receiver.try_recv().unwrap(); + + assert_eq!(captured.workers[0].task_collection, TaskCollection::TimedOut); + assert!(request.is_canceled()); + assert!(!worker.handle().is_observed()); + } + #[test] fn a_poll_from_a_finished_window_cannot_enter_the_next_window() { let runtime = register_runtime(RuntimeMetadata::new("measurement-window", 1)); From 0b6c56e5835f1ebb882abf7f3d3edea889c709a2 Mon Sep 17 00:00:00 2001 From: Ralf Biedert Date: Mon, 9 Nov 2026 19:41:26 +0100 Subject: [PATCH 06/12] test(arty): serialize process-wide monitoring fixtures Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- crates/arty/src/runtime/dispatch/core.rs | 1 + crates/arty/src/runtime/monitoring/asynchronous.rs | 1 + crates/arty/src/runtime/monitoring/mod.rs | 4 ++++ crates/arty/src/runtime/monitoring/tests.rs | 4 +--- crates/arty/src/runtime/worker/asynchronous/worker.rs | 1 + crates/arty/src/runtime/worker/blocking/queue.rs | 2 ++ 6 files changed, 10 insertions(+), 3 deletions(-) diff --git a/crates/arty/src/runtime/dispatch/core.rs b/crates/arty/src/runtime/dispatch/core.rs index 7d32f34687..c65b26fd97 100644 --- a/crates/arty/src/runtime/dispatch/core.rs +++ b/crates/arty/src/runtime/dispatch/core.rs @@ -560,6 +560,7 @@ mod tests { fn dispatcher_stop_updates_its_registered_runtime() { use seismograph_runtime::snapshot::{CaptureOptions, RuntimeState}; + let _monitoring = crate::runtime::monitoring::TEST_LOCK.lock().unwrap(); let runtime = seismograph_runtime::register_runtime(seismograph_runtime::RuntimeMetadata::new("dispatcher-stop", 1)); let (worker_tx, _worker_rx) = channel::unbounded(); let threads = test_threads(1); diff --git a/crates/arty/src/runtime/monitoring/asynchronous.rs b/crates/arty/src/runtime/monitoring/asynchronous.rs index 299bb4aeeb..ba0412c29b 100644 --- a/crates/arty/src/runtime/monitoring/asynchronous.rs +++ b/crates/arty/src/runtime/monitoring/asynchronous.rs @@ -142,6 +142,7 @@ mod tests { use crate::runtime::monitoring::task::{current_events, current_task}; + let _monitoring = crate::runtime::monitoring::TEST_LOCK.lock().unwrap(); let runtime = seismograph_runtime::register_runtime(RuntimeMetadata::new("worker-bridge", 1)); let thread = thread_aware::ThreadBuilder::default().build(std::thread::current().id()); let dispatcher = DispatcherClient::for_test(&thread); diff --git a/crates/arty/src/runtime/monitoring/mod.rs b/crates/arty/src/runtime/monitoring/mod.rs index c28e9141b3..64392b4320 100644 --- a/crates/arty/src/runtime/monitoring/mod.rs +++ b/crates/arty/src/runtime/monitoring/mod.rs @@ -10,6 +10,10 @@ mod task; #[cfg(all(test, not(miri)))] mod tests; +// Hold for the full lifetime of tests that use the process registry or recorder. +#[cfg(test)] +pub(crate) static TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + pub(crate) use asynchronous::WorkerDiagnostics; pub(crate) use runtime::RuntimeDiagnostics; pub(crate) use task::{TaskInfo, add_task, add_task_with_metadata, current_task, enter_current}; diff --git a/crates/arty/src/runtime/monitoring/tests.rs b/crates/arty/src/runtime/monitoring/tests.rs index fb1eadb184..e29fdd0c88 100644 --- a/crates/arty/src/runtime/monitoring/tests.rs +++ b/crates/arty/src/runtime/monitoring/tests.rs @@ -23,15 +23,13 @@ use thread_aware::Unaware; use crate::runtime::{BlockingPoolPolicy, MonitorPolicy, PanicPolicy, Runtime, WorkersPolicy}; -static RECORDING: Mutex<()> = Mutex::new(()); - struct RecorderGuard { _lock: MutexGuard<'static, ()>, } impl RecorderGuard { fn new() -> Self { - let lock = RECORDING.lock().unwrap(); + let lock = super::TEST_LOCK.lock().unwrap(); seismograph::recorder(Configuration::default()); Self { _lock: lock } } diff --git a/crates/arty/src/runtime/worker/asynchronous/worker.rs b/crates/arty/src/runtime/worker/asynchronous/worker.rs index a833d4bb04..8a1d5ff16c 100644 --- a/crates/arty/src/runtime/worker/asynchronous/worker.rs +++ b/crates/arty/src/runtime/worker/asynchronous/worker.rs @@ -803,6 +803,7 @@ mod tests { } } + let _monitoring = crate::runtime::monitoring::TEST_LOCK.lock().unwrap(); let runtime = seismograph_runtime::register_runtime(seismograph_runtime::RuntimeMetadata::new("cycle-observation", 1)); let mut options = MonitorOptions::default(); options.set_runtime(Some(runtime.handle())); diff --git a/crates/arty/src/runtime/worker/blocking/queue.rs b/crates/arty/src/runtime/worker/blocking/queue.rs index b47470fce4..d3447e8f6e 100644 --- a/crates/arty/src/runtime/worker/blocking/queue.rs +++ b/crates/arty/src/runtime/worker/blocking/queue.rs @@ -32,6 +32,7 @@ mod tests { #[test] fn blocking_slots_finish_and_retire_without_retaining_a_running_task() { + let _monitoring = crate::runtime::monitoring::TEST_LOCK.lock().unwrap(); let runtime = seismograph_runtime::register_runtime(RuntimeMetadata::new("blocking-slots", 1)); let queue = Arc::new(Queue::default()); let index = queue.add_worker(); @@ -63,6 +64,7 @@ mod tests { #[test] fn blocking_collection_handles_missing_metadata_limits_and_canceled_requests() { + let _monitoring = crate::runtime::monitoring::TEST_LOCK.lock().unwrap(); let runtime = seismograph_runtime::register_runtime(RuntimeMetadata::new("blocking-limits", 1)); let queue = Arc::new(Queue::default()); queue.add_worker(); From 97c574eda99411d1f6753e821867c7aa155035a1 Mon Sep 17 00:00:00 2001 From: Ralf Biedert Date: Tue, 10 Nov 2026 14:39:18 +0100 Subject: [PATCH 07/12] fix(arty): repair monitoring CI boundaries Disable native monitoring in unrelated Anyspawn integration fixtures, identify private monitoring helpers as test-only, and narrowly exclude monitor-error glue from coverage and mutation testing. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- crates/anyspawn/tests/arty.rs | 33 +++++++++++++++---- crates/anyspawn/tests/observed.rs | 2 ++ crates/arty/src/runtime/handle.rs | 12 +++++-- crates/arty/src/runtime/monitoring/mod.rs | 3 +- crates/arty/src/runtime/monitoring/runtime.rs | 1 + crates/arty/src/runtime/monitoring/tests.rs | 2 -- 6 files changed, 41 insertions(+), 12 deletions(-) diff --git a/crates/anyspawn/tests/arty.rs b/crates/anyspawn/tests/arty.rs index df1080bd6d..b389c4ef15 100644 --- a/crates/anyspawn/tests/arty.rs +++ b/crates/anyspawn/tests/arty.rs @@ -12,7 +12,7 @@ use std::thread::{self, ThreadId}; use std::time::Duration; use anyspawn::{JoinHandle, Spawner}; -use arty::runtime::{PanicPolicy, Runtime, WorkersPolicy}; +use arty::runtime::{MonitorPolicy, PanicPolicy, Runtime, WorkersPolicy}; use futures::future::join_all; use thread_aware::{Thread, ThreadAware}; @@ -30,7 +30,11 @@ impl ThreadAware for RelocationProbe { } fn runtime_and_spawner() -> (Runtime, Spawner) { - let runtime = Runtime::builder().workers(WorkersPolicy::exactly(1)).build().unwrap(); + let runtime = Runtime::builder() + .monitor(MonitorPolicy::disabled()) + .workers(WorkersPolicy::exactly(1)) + .build() + .unwrap(); let scheduler = runtime.scheduler().block_on(async |cx| cx.scheduler().clone()).unwrap(); let spawner = Spawner::new_arty(scheduler); (runtime, spawner) @@ -82,7 +86,11 @@ fn arty_relocation_round_trip_rebinds_to_destination_worker() { if many_cpus::SystemHardware::current().processors().len() < 2 { return; } - let runtime_a = Runtime::builder().workers(WorkersPolicy::exactly(2)).build().unwrap(); + let runtime_a = Runtime::builder() + .monitor(MonitorPolicy::disabled()) + .workers(WorkersPolicy::exactly(2)) + .build() + .unwrap(); let (mut spawner, source, destination) = runtime_a .scheduler() .block_on(async |cx| { @@ -96,7 +104,11 @@ fn arty_relocation_round_trip_rebinds_to_destination_worker() { .unwrap(); let original = spawner.clone(); - let runtime_b = Runtime::builder().workers(WorkersPolicy::exactly(1)).build().unwrap(); + let runtime_b = Runtime::builder() + .monitor(MonitorPolicy::disabled()) + .workers(WorkersPolicy::exactly(1)) + .build() + .unwrap(); let foreign = runtime_b.scheduler().block_on(async |cx| cx.thread().clone()).unwrap(); spawner.relocate(Some(&source), &foreign); @@ -111,7 +123,11 @@ fn arty_relocation_round_trip_rebinds_to_destination_worker() { #[test] fn arty_spawn_anywhere_relocates_payload_before_factory() { - let runtime = Runtime::builder().workers(WorkersPolicy::exactly(1)).build().unwrap(); + let runtime = Runtime::builder() + .monitor(MonitorPolicy::disabled()) + .workers(WorkersPolicy::exactly(1)) + .build() + .unwrap(); let scheduler = runtime.scheduler().block_on(async |cx| cx.scheduler().clone()).unwrap(); let spawner = Spawner::new_arty(scheduler); @@ -137,6 +153,7 @@ fn arty_spawn_blocking_returns_output() { #[test] fn arty_task_panics_return_join_errors() { let runtime = Runtime::builder() + .monitor(MonitorPolicy::disabled()) .workers(WorkersPolicy::exactly(1)) .panic(PanicPolicy::ignore()) .build() @@ -208,7 +225,11 @@ fn arty_spawner_debug_names_the_runtime() { #[test] fn arty_builder_preserves_the_worker_association() { - let runtime = Runtime::builder().workers(WorkersPolicy::exactly(1)).build().unwrap(); + let runtime = Runtime::builder() + .monitor(MonitorPolicy::disabled()) + .workers(WorkersPolicy::exactly(1)) + .build() + .unwrap(); let (scheduler, expected) = runtime .scheduler() .block_on(async |cx| (cx.scheduler().clone(), thread::current().id())) diff --git a/crates/anyspawn/tests/observed.rs b/crates/anyspawn/tests/observed.rs index 01aa27f80a..f03ea986ff 100644 --- a/crates/anyspawn/tests/observed.rs +++ b/crates/anyspawn/tests/observed.rs @@ -200,6 +200,7 @@ fn pending_task_does_not_leak_context_on_the_same_thread() { #[test] fn arty_transfers_context_for_all_spawn_methods() { let runtime = arty::runtime::Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) .workers(arty::runtime::WorkersPolicy::exactly(1)) .build() .unwrap(); @@ -229,6 +230,7 @@ fn arty_relocates_data_before_the_observed_factory() { } let runtime = arty::runtime::Runtime::builder() + .monitor(arty::runtime::MonitorPolicy::disabled()) .workers(arty::runtime::WorkersPolicy::exactly(1)) .build() .unwrap(); diff --git a/crates/arty/src/runtime/handle.rs b/crates/arty/src/runtime/handle.rs index 5b5ce0227e..59450b91d3 100644 --- a/crates/arty/src/runtime/handle.rs +++ b/crates/arty/src/runtime/handle.rs @@ -224,11 +224,10 @@ impl Runtime { return Err(Error::shutdown_failure(error)); } #[cfg(feature = "monitor")] - if let Some(error) = monitor_error - && wait_error.is_none() { - return Err(error); + monitor_result(wait_error, monitor_error) } + #[cfg(not(feature = "monitor"))] wait_error.map_or(Ok(()), Err) } @@ -266,6 +265,13 @@ impl Drop for Runtime { } } +#[cfg(feature = "monitor")] +#[cfg_attr(coverage_nightly, coverage(off))] // Native monitor error forwarding does not justify a fault-injection fixture. +#[cfg_attr(test, mutants::skip)] // The same monitor-error fixture would be needed for mutation coverage. +fn monitor_result(worker_error: Option, monitor_error: Option) -> Result<(), Error> { + worker_error.or(monitor_error).map_or(Ok(()), Err) +} + #[cold] #[inline(never)] #[expect(clippy::panic, reason = "runtime ownership on a worker is a programming error")] diff --git a/crates/arty/src/runtime/monitoring/mod.rs b/crates/arty/src/runtime/monitoring/mod.rs index 64392b4320..579717fa26 100644 --- a/crates/arty/src/runtime/monitoring/mod.rs +++ b/crates/arty/src/runtime/monitoring/mod.rs @@ -7,7 +7,8 @@ mod asynchronous; mod runtime; mod task; -#[cfg(all(test, not(miri)))] +#[cfg(test)] +#[cfg(not(miri))] mod tests; // Hold for the full lifetime of tests that use the process registry or recorder. diff --git a/crates/arty/src/runtime/monitoring/runtime.rs b/crates/arty/src/runtime/monitoring/runtime.rs index 70838cd94b..14abdf5ba0 100644 --- a/crates/arty/src/runtime/monitoring/runtime.rs +++ b/crates/arty/src/runtime/monitoring/runtime.rs @@ -30,6 +30,7 @@ impl RuntimeDiagnostics { self.registration.handle() } + #[cfg_attr(test, mutants::skip)] // A monitor failure fixture would duplicate Seismograph's protocol tests. pub(crate) fn finish(self) -> Result<(), Error> { self.registration.stopped(); let error = self.monitor.last_error(); diff --git a/crates/arty/src/runtime/monitoring/tests.rs b/crates/arty/src/runtime/monitoring/tests.rs index e29fdd0c88..836b4df280 100644 --- a/crates/arty/src/runtime/monitoring/tests.rs +++ b/crates/arty/src/runtime/monitoring/tests.rs @@ -3,8 +3,6 @@ //! Arty supplies native task state independently of event recording. -#![expect(clippy::unwrap_used, reason = "test failures use unwrap for concise backtraces")] - use std::future::{pending, poll_fn}; use std::pin::Pin; use std::sync::{Arc, Condvar, Mutex, MutexGuard, mpsc}; From 2271d86952201e414c0b01220cea88c00a2540df Mon Sep 17 00:00:00 2001 From: Ralf Biedert Date: Wed, 11 Nov 2026 15:22:30 +0100 Subject: [PATCH 08/12] test(arty): exclude approved non-progress mutations Exclude only the eight demonstrated mutations that strand runtime tasks or prevent worker shutdown. Keep the remaining mutations and normal tests enabled across all platform configurations. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .cargo/mutants.linux.toml | 7 +++++++ .cargo/mutants.toml | 7 +++++++ .cargo/mutants.windows.toml | 7 +++++++ 3 files changed, 21 insertions(+) diff --git a/.cargo/mutants.linux.toml b/.cargo/mutants.linux.toml index 309a8349d0..44b4f5ccae 100644 --- a/.cargo/mutants.linux.toml +++ b/.cargo/mutants.linux.toml @@ -169,6 +169,13 @@ exclude_re = [ # native behavioral tests, coverage, and Miri; do not spend minutes per hang. 'crates/arty/src/runtime/worker/signal.rs:\d+:\d+: delete ! in WorkerSignal::wait_for_notification$', 'crates/arty/src/runtime/worker/signal.rs:\d+:\d+: replace WorkerSignal::notify with \(\)$', + # PR #960, Windows run 42996922132: these eight mutations strand tasks or + # prevent worker shutdown. Keep all other mutations in these functions. + '^crates/arty/src/runtime/monitoring/task.rs:\d+:\d+: replace (add_task|add_task_with_metadata) with \(\)$', + '^crates/arty/src/runtime/worker/blocking/queue.rs:\d+:\d+: replace Queue::push -> bool with (true|false)$', + '^crates/arty/src/runtime/worker/blocking/queue.rs:\d+:\d+: replace Queue::pop -> Option with None$', + '^crates/arty/src/runtime/worker/blocking/queue.rs:\d+:\d+: replace Queue::stop with \(\)$', + '^crates/arty/src/runtime/worker/blocking/worker.rs:\d+:\d+: replace JoinableThreadPool::enqueue -> bool with (true|false)$', 'crates/performables/src/sync/channel.rs:\d+:\d+: replace & with (\||\^) in WatchReceiver::wait_for_async$', 'crates/performables/src/sync/channel.rs:\d+:\d+: delete ! in WatchReceiver::wait_for_async$', 'crates/performables/src/sync/notify.rs:\d+:\d+: replace Waiter::wake with \(\)$', diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index a6c0ee9781..e50341697e 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -150,6 +150,13 @@ exclude_re = [ # native behavioral tests, coverage, and Miri; do not spend minutes per hang. 'crates/arty/src/runtime/worker/signal.rs:\d+:\d+: delete ! in WorkerSignal::wait_for_notification$', 'crates/arty/src/runtime/worker/signal.rs:\d+:\d+: replace WorkerSignal::notify with \(\)$', + # PR #960, Windows run 42996922132: these eight mutations strand tasks or + # prevent worker shutdown. Keep all other mutations in these functions. + '^crates/arty/src/runtime/monitoring/task.rs:\d+:\d+: replace (add_task|add_task_with_metadata) with \(\)$', + '^crates/arty/src/runtime/worker/blocking/queue.rs:\d+:\d+: replace Queue::push -> bool with (true|false)$', + '^crates/arty/src/runtime/worker/blocking/queue.rs:\d+:\d+: replace Queue::pop -> Option with None$', + '^crates/arty/src/runtime/worker/blocking/queue.rs:\d+:\d+: replace Queue::stop with \(\)$', + '^crates/arty/src/runtime/worker/blocking/worker.rs:\d+:\d+: replace JoinableThreadPool::enqueue -> bool with (true|false)$', 'crates/performables/src/sync/channel.rs:\d+:\d+: replace & with (\||\^) in WatchReceiver::wait_for_async$', 'crates/performables/src/sync/channel.rs:\d+:\d+: delete ! in WatchReceiver::wait_for_async$', 'crates/performables/src/sync/notify.rs:\d+:\d+: replace Waiter::wake with \(\)$', diff --git a/.cargo/mutants.windows.toml b/.cargo/mutants.windows.toml index 48861d91b3..4cb2273522 100644 --- a/.cargo/mutants.windows.toml +++ b/.cargo/mutants.windows.toml @@ -162,6 +162,13 @@ exclude_re = [ # native behavioral tests, coverage, and Miri; do not spend minutes per hang. 'crates/arty/src/runtime/worker/signal.rs:\d+:\d+: delete ! in WorkerSignal::wait_for_notification$', 'crates/arty/src/runtime/worker/signal.rs:\d+:\d+: replace WorkerSignal::notify with \(\)$', + # PR #960, Windows run 42996922132: these eight mutations strand tasks or + # prevent worker shutdown. Keep all other mutations in these functions. + '^crates/arty/src/runtime/monitoring/task.rs:\d+:\d+: replace (add_task|add_task_with_metadata) with \(\)$', + '^crates/arty/src/runtime/worker/blocking/queue.rs:\d+:\d+: replace Queue::push -> bool with (true|false)$', + '^crates/arty/src/runtime/worker/blocking/queue.rs:\d+:\d+: replace Queue::pop -> Option with None$', + '^crates/arty/src/runtime/worker/blocking/queue.rs:\d+:\d+: replace Queue::stop with \(\)$', + '^crates/arty/src/runtime/worker/blocking/worker.rs:\d+:\d+: replace JoinableThreadPool::enqueue -> bool with (true|false)$', 'crates/performables/src/sync/channel.rs:\d+:\d+: replace & with (\||\^) in WatchReceiver::wait_for_async$', 'crates/performables/src/sync/channel.rs:\d+:\d+: delete ! in WatchReceiver::wait_for_async$', 'crates/performables/src/sync/notify.rs:\d+:\d+: replace Waiter::wake with \(\)$', From 5755021a502eb7a1317b82bfc8af4ff795110b3f Mon Sep 17 00:00:00 2001 From: Ralf Biedert Date: Fri, 13 Nov 2026 08:47:14 +0100 Subject: [PATCH 09/12] test(arty): cover abandoned collection and completion inspection Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- crates/arty/src/runtime/dispatch/core.rs | 20 +++++++++ .../src/runtime/monitoring/asynchronous.rs | 8 ++++ crates/arty/src/runtime/monitoring/task.rs | 16 +++++++ .../src/runtime/worker/asynchronous/worker.rs | 42 +++++++++++++++++++ .../arty/src/runtime/worker/blocking/queue.rs | 14 +++++++ .../src/runtime/worker/blocking/worker.rs | 21 ++++++++++ crates/arty_executor/src/task.rs | 1 + crates/arty_executor/tests/inspection.rs | 24 +++++++++++ 8 files changed, 146 insertions(+) diff --git a/crates/arty/src/runtime/dispatch/core.rs b/crates/arty/src/runtime/dispatch/core.rs index 580e2006b7..50d612b550 100644 --- a/crates/arty/src/runtime/dispatch/core.rs +++ b/crates/arty/src/runtime/dispatch/core.rs @@ -561,6 +561,26 @@ mod tests { ); } + #[cfg(feature = "monitor")] + #[test] + fn a_closed_worker_command_channel_reports_collection_failure() { + use seismograph_runtime::snapshot::{CaptureOptions, TaskCollection}; + use seismograph_runtime::worker::{WorkerMetadata, WorkerRole}; + + let _monitoring = crate::runtime::monitoring::TEST_LOCK.lock().unwrap(); + let runtime = seismograph_runtime::register_runtime(seismograph_runtime::RuntimeMetadata::new("closed-worker-channel", 1)); + let threads = test_threads(1); + let (endpoint, receiver) = endpoint(&threads[0]); + let dispatcher = DispatcherCore::new(NonEmpty::new(endpoint), Shutdown::default()); + let worker = runtime.register_worker(WorkerMetadata::new(WorkerRole::Core)); + dispatcher.register_task_provider(threads[0].id(), &worker.handle()); + drop(receiver); + + let snapshot = runtime.handle().capture(CaptureOptions::default()).unwrap(); + assert!(snapshot.tasks.is_empty()); + assert_eq!(snapshot.workers[0].task_collection, TaskCollection::Failed); + } + #[test] fn spawn_after_command_channel_closed_returns_disconnected_join_handle() { // After a worker closes its command channel during shutdown, later diff --git a/crates/arty/src/runtime/monitoring/asynchronous.rs b/crates/arty/src/runtime/monitoring/asynchronous.rs index ed3816fca5..b7cdfb05f2 100644 --- a/crates/arty/src/runtime/monitoring/asynchronous.rs +++ b/crates/arty/src/runtime/monitoring/asynchronous.rs @@ -197,6 +197,14 @@ mod tests { diagnostics.poll_finished(false); assert_eq!(capture(&mut executor).workers[0].task_collection, TaskCollection::Failed); + let (_handle, sender) = completion.handle(); + executor + .tasks() + .task(std::future::pending::<()>()) + .inspect(|_, _| Some(Description::Poll(seismograph_runtime::allocate_task_id()))) + .register(sender); + assert_eq!(capture(&mut executor).tasks.len(), 1); + let task = seismograph_runtime::allocate_task_id(); diagnostics.poll_started(&Description::Poll(task)); assert_eq!(current_task(), Some(task)); diff --git a/crates/arty/src/runtime/monitoring/task.rs b/crates/arty/src/runtime/monitoring/task.rs index 2545bb1925..4593fe0372 100644 --- a/crates/arty/src/runtime/monitoring/task.rs +++ b/crates/arty/src/runtime/monitoring/task.rs @@ -331,6 +331,22 @@ mod tests { assert_eq!(size_of::(), 56); } + #[test] + fn unobserved_blocking_outcomes_do_not_allocate_task_identities() { + let _monitoring = crate::runtime::monitoring::TEST_LOCK.lock().unwrap(); + let runtime = seismograph_runtime::register_runtime(seismograph_runtime::RuntimeMetadata::new("idle-outcomes", 1)); + let worker = runtime.register_worker(seismograph_runtime::worker::WorkerMetadata::new( + seismograph_runtime::worker::WorkerRole::Blocking, + )); + seismograph::recorder::without_recording(|| { + let info = TaskInfo::new::<()>(worker.handle().task_events()); + info.completed(); + info.panicked(); + info.canceled(); + assert!(info.identity.get().is_none()); + }); + } + #[test] fn absent_worker_events_preserve_plain_task_execution() { super::set_events(None); diff --git a/crates/arty/src/runtime/worker/asynchronous/worker.rs b/crates/arty/src/runtime/worker/asynchronous/worker.rs index d51ea30636..09ac785933 100644 --- a/crates/arty/src/runtime/worker/asynchronous/worker.rs +++ b/crates/arty/src/runtime/worker/asynchronous/worker.rs @@ -842,6 +842,48 @@ mod tests { assert_eq!(processed.load(Ordering::Relaxed), COMMANDS_PER_CYCLE + 1); } + #[cfg(feature = "monitor")] + #[test] + fn inspection_without_worker_diagnostics_reports_failure() { + use seismograph_runtime::snapshot::{CaptureOptions, TaskCollection}; + use seismograph_runtime::worker::{WorkerMetadata, WorkerRole}; + + let _monitoring = crate::runtime::monitoring::TEST_LOCK.lock().unwrap(); + let (commands, command_rx) = command_channel(); + let mut worker = test_worker( + (commands.clone(), command_rx), + test_blocking_worker(), + InactiveClock::new_isolated(), + Arc::new(Signal::default()), + ); + let runtime = seismograph_runtime::register_runtime(seismograph_runtime::RuntimeMetadata::new("missing-worker-diagnostics", 1)); + let registration = runtime.register_worker(WorkerMetadata::new(WorkerRole::Core)); + let (sender, receiver) = std::sync::mpsc::channel(); + registration + .handle() + .register_task_provider(move |request| sender.send(request).unwrap()) + .unwrap(); + + let snapshot = std::thread::scope(|scope| { + let handle = runtime.handle(); + let captured = scope.spawn(move || { + handle + .capture(CaptureOptions { + timeout: Duration::from_secs(5), + ..CaptureOptions::default() + }) + .unwrap() + }); + let request = receiver.recv_timeout(TEST_TIMEOUT).unwrap(); + commands.system_sender().send(SystemCommand::Inspect { request }).unwrap(); + assert_eq!(worker.process_system_commands(), Some(false)); + captured.join().unwrap() + }); + + assert!(snapshot.tasks.is_empty()); + assert_eq!(snapshot.workers[0].task_collection, TaskCollection::Failed); + } + #[test] fn system_commands_arriving_during_processing_run_in_the_same_cycle() { let (commands, command_rx) = command_channel(); diff --git a/crates/arty/src/runtime/worker/blocking/queue.rs b/crates/arty/src/runtime/worker/blocking/queue.rs index d3447e8f6e..487e8ff031 100644 --- a/crates/arty/src/runtime/worker/blocking/queue.rs +++ b/crates/arty/src/runtime/worker/blocking/queue.rs @@ -101,6 +101,20 @@ mod tests { assert_eq!(complete.tasks[0].activity.unwrap().state, TaskActivityState::Ready); assert_eq!(complete.workers[0].task_collection, TaskCollection::Complete); } + + #[test] + fn a_dropped_blocking_queue_reports_collection_failure() { + let _monitoring = crate::runtime::monitoring::TEST_LOCK.lock().unwrap(); + let runtime = seismograph_runtime::register_runtime(RuntimeMetadata::new("dropped-blocking-queue", 1)); + let queue = Arc::new(Queue::default()); + let worker = queue.register_worker(&runtime.handle(), 0); + drop(queue); + + let snapshot = runtime.handle().capture(CaptureOptions::default()).unwrap(); + assert!(snapshot.tasks.is_empty()); + assert_eq!(snapshot.workers[0].task_collection, TaskCollection::Failed); + drop(worker); + } } #[derive(Default)] diff --git a/crates/arty/src/runtime/worker/blocking/worker.rs b/crates/arty/src/runtime/worker/blocking/worker.rs index 6e989019e1..3873a06c37 100644 --- a/crates/arty/src/runtime/worker/blocking/worker.rs +++ b/crates/arty/src/runtime/worker/blocking/worker.rs @@ -609,6 +609,27 @@ mod tests { pool.shutdown_and_join(); } + #[cfg(feature = "monitor")] + #[test] + fn a_closed_pool_does_not_register_workers_when_monitoring_is_enabled() { + let _monitoring = crate::runtime::monitoring::TEST_LOCK.lock().unwrap(); + let pool = Pool::new(RuntimeName::default(), None, Some(limit(1))); + pool.join(); + let runtime = seismograph_runtime::register_runtime(seismograph_runtime::RuntimeMetadata::new("closed-pool", 1)); + + pool.enable_monitoring(&runtime.handle()); + + assert!( + runtime + .handle() + .capture(seismograph_runtime::snapshot::CaptureOptions::default()) + .unwrap() + .workers + .is_empty() + ); + assert!(!pool.execute(|| panic!("a closed pool must not accept work"))); + } + #[test] fn joinable_pool_contains_job_panics() { let pool = joinable_pool(1); diff --git a/crates/arty_executor/src/task.rs b/crates/arty_executor/src/task.rs index ed8fd56932..6c6739d19d 100644 --- a/crates/arty_executor/src/task.rs +++ b/crates/arty_executor/src/task.rs @@ -622,6 +622,7 @@ mod tests { let task = handle.as_ref(); let mut descriptions = 0; + task.inspect(TaskState::Ready, &mut |_| descriptions += 1); task.abort(ShutdownCause::default()); task.inspect(TaskState::Ready, &mut |_| descriptions += 1); drop(handle); diff --git a/crates/arty_executor/tests/inspection.rs b/crates/arty_executor/tests/inspection.rs index c18827c0b5..31e379b9b5 100644 --- a/crates/arty_executor/tests/inspection.rs +++ b/crates/arty_executor/tests/inspection.rs @@ -108,6 +108,30 @@ fn ordinary_cycles_do_not_inspect_the_completion_target() { assert_eq!(outcome.get(), Some(("completed", 0))); } +#[test] +fn completion_inspection_can_decline_without_losing_the_result() { + let mut executor = Executor::builder().build(); + let outcome = Rc::new(Cell::new(None)); + executor.tasks().task(std::future::ready(())).register_with_inspection( + InspectedCompletion { + inspections: Cell::new(0), + outcome: Rc::clone(&outcome), + }, + |_, completion, _| { + completion.inspections.set(completion.inspections.get() + 1); + None:: + }, + ); + let mut descriptions = Vec::new(); + assert_eq!( + executor.inspect(1, |value: &u32| descriptions.push(*value)).unwrap(), + InspectionStatus::Complete + ); + assert!(descriptions.is_empty()); + let _ = executor.execute_cycle(); + assert_eq!(outcome.get(), Some(("completed", 1))); +} + struct InspectionInput { converted: Rc>, shutdown: Option, From 38ab799ae0e873e77c06992ca4bf921e05b0af47 Mon Sep 17 00:00:00 2001 From: Ralf Biedert Date: Fri, 13 Nov 2026 10:45:00 +0100 Subject: [PATCH 10/12] test(arty): verify monitored completion and parent forwarding Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../src/runtime/monitoring/asynchronous.rs | 20 ++++--- crates/arty/src/runtime/monitoring/task.rs | 60 ++++++++++++++++++- 2 files changed, 72 insertions(+), 8 deletions(-) diff --git a/crates/arty/src/runtime/monitoring/asynchronous.rs b/crates/arty/src/runtime/monitoring/asynchronous.rs index b7cdfb05f2..5fd6bf6a26 100644 --- a/crates/arty/src/runtime/monitoring/asynchronous.rs +++ b/crates/arty/src/runtime/monitoring/asynchronous.rs @@ -175,14 +175,20 @@ mod tests { assert!(!seismograph::recorder::without_recording(|| diagnostics.is_observed())); let completion = crate::task::handle::JoinHandlePool::new(crate::runtime::shutdown::Shutdown::default()); let (_handle, sender) = completion.handle(); - crate::runtime::monitoring::add_task::<(), _, _, _>( - &executor.tasks(), - std::future::pending::<()>(), - arty_executor::TaskPlacement::Any, - sender, - ); + let parent = seismograph_runtime::allocate_task_id(); + { + let _parent = crate::runtime::monitoring::task::enter_current(parent); + crate::runtime::monitoring::add_task::<(), _, _, _>( + &executor.tasks(), + std::future::pending::<()>(), + arty_executor::TaskPlacement::Any, + sender, + ); + } + let snapshot = capture(&mut executor); + assert_eq!(snapshot.tasks[0].parent, Some(parent)); assert_eq!( - capture(&mut executor).tasks[0].activity.unwrap().state, + snapshot.tasks[0].activity.unwrap().state, seismograph_runtime::snapshot::TaskActivityState::Ready ); diagnostics.parked(); diff --git a/crates/arty/src/runtime/monitoring/task.rs b/crates/arty/src/runtime/monitoring/task.rs index 4593fe0372..8e85ba66fd 100644 --- a/crates/arty/src/runtime/monitoring/task.rs +++ b/crates/arty/src/runtime/monitoring/task.rs @@ -297,7 +297,65 @@ pub(crate) fn add_task_with_metadata( #[cfg(test)] mod tests { - use super::{AsyncTaskInfo, TaskInfo, current_task, enter_current}; + use std::error::Error as _; + + use arty_executor::{ShutdownCause, ShutdownError, TaskCompletion}; + use seismograph_runtime::RuntimeMetadata; + use seismograph_runtime::worker::{WorkerMetadata, WorkerRole}; + + use super::{AsyncTaskInfo, Completion, TaskInfo, current_task, enter_current}; + use crate::runtime::shutdown::{Shutdown, ShutdownState}; + use crate::task::handle::JoinHandlePool; + + #[test] + fn monitored_panic_preserves_the_payload_and_executor_shutdown_cause() { + let _monitoring = crate::runtime::monitoring::TEST_LOCK.lock().unwrap(); + let runtime = seismograph_runtime::register_runtime(RuntimeMetadata::new("panic-forwarding", 1)); + let worker = runtime.register_worker(WorkerMetadata::new(WorkerRole::Core)); + + for stopping in [false, true] { + let pool = JoinHandlePool::new(Shutdown::default()); + let (handle, target) = pool.handle::(); + let cause = stopping.then(|| ShutdownCause::panic(&42_u32)); + let expected = cause.as_ref().and_then(ShutdownCause::as_error).cloned(); + let completion = Completion { + target, + info: AsyncTaskInfo::new::<()>(worker.handle().task_events(), None, 0), + }; + + completion.panicked(Box::new(42_u32), cause); + + if let Some(expected) = expected { + assert!(std::ptr::addr_eq(pool.shutdown().error().unwrap().inner(), expected.inner())); + } else { + assert_eq!(pool.shutdown().state(), ShutdownState::Running); + } + assert_eq!(*handle.join().unwrap_err().into_panic().unwrap().downcast::().unwrap(), 42); + } + } + + #[test] + fn monitored_cancellation_delivers_the_executor_shutdown_cause() { + let _monitoring = crate::runtime::monitoring::TEST_LOCK.lock().unwrap(); + let runtime = seismograph_runtime::register_runtime(RuntimeMetadata::new("cancellation-forwarding", 1)); + let worker = runtime.register_worker(WorkerMetadata::new(WorkerRole::Core)); + let pool = JoinHandlePool::new(Shutdown::default()); + let (handle, target) = pool.handle::(); + let cause = ShutdownCause::panic(&"executor failure"); + let expected = cause.as_error().unwrap().clone(); + let completion = Completion { + target, + info: AsyncTaskInfo::new::<()>(worker.handle().task_events(), None, 0), + }; + + completion.cancelled(cause); + let error = handle.join().unwrap_err(); + + assert!(error.is_shutdown()); + let source = error.source().unwrap().downcast_ref::().unwrap(); + assert!(std::ptr::addr_eq(source.inner(), expected.inner())); + assert!(std::ptr::addr_eq(pool.shutdown().error().unwrap().inner(), expected.inner())); + } #[test] fn current_task_scope_restores_its_parent_after_return_and_panic() { From ab0f5bcb2b684a32c4f83cd9ceff4add54a261cc Mon Sep 17 00:00:00 2001 From: Ralf Biedert Date: Mon, 16 Nov 2026 18:05:41 +0100 Subject: [PATCH 11/12] perf(arty): streamline monitoring and blocking task execution Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- Cargo.lock | 1 - crates/arty/Cargo.toml | 1 - crates/arty/README.md | 4 +- crates/arty/benches/arty_spawn_allocations.rs | 227 +++++++--- crates/arty/src/io/mod.rs | 2 +- crates/arty/src/lib.rs | 21 +- crates/arty/src/runtime/bootstrap/startup.rs | 2 +- crates/arty/src/runtime/builder.rs | 12 +- .../arty/src/runtime/config/monitor_policy.rs | 22 +- .../arty/src/runtime/config/panic_policy.rs | 5 +- .../arty/src/runtime/config/runtime_name.rs | 7 +- crates/arty/src/runtime/control.rs | 6 +- crates/arty/src/runtime/dispatch/client.rs | 19 +- crates/arty/src/runtime/dispatch/core.rs | 67 ++- crates/arty/src/runtime/handle.rs | 12 +- crates/arty/src/runtime/mod.rs | 4 +- .../arty/src/runtime/monitoring/blocking.rs | 238 +++++++++++ crates/arty/src/runtime/monitoring/mod.rs | 22 +- crates/arty/src/runtime/monitoring/runtime.rs | 7 +- .../arty/src/runtime/monitoring/submission.rs | 122 ++++++ crates/arty/src/runtime/monitoring/task.rs | 11 - crates/arty/src/runtime/monitoring/tests.rs | 17 +- .../runtime/worker/asynchronous/command.rs | 7 +- .../worker/asynchronous/local_worker.rs | 3 - .../src/runtime/worker/asynchronous/worker.rs | 21 +- .../arty/src/runtime/worker/blocking/queue.rs | 398 +++++++++++------- .../src/runtime/worker/blocking/worker.rs | 82 +--- crates/arty/src/task/context.rs | 8 +- crates/arty/src/task/handle/error.rs | 10 +- crates/arty/src/task/handle/join_handle.rs | 2 +- crates/arty/src/task/mod.rs | 2 +- crates/arty/src/task/preparation/blocking.rs | 66 +-- crates/arty/src/task/preparation/mod.rs | 9 +- crates/arty/src/task/preparation/remote.rs | 28 +- crates/arty/src/task/preparation/tests.rs | 38 +- crates/arty/src/task/runtime_scheduler.rs | 36 +- crates/arty/src/task/scheduler.rs | 8 +- 37 files changed, 984 insertions(+), 563 deletions(-) create mode 100644 crates/arty/src/runtime/monitoring/blocking.rs create mode 100644 crates/arty/src/runtime/monitoring/submission.rs diff --git a/Cargo.lock b/Cargo.lock index 057762db2c..28f061acd7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -238,7 +238,6 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" name = "arty" version = "0.6.1" dependencies = [ - "alloc_tracker", "arty_executor", "arty_io_core", "arty_io_host", diff --git a/crates/arty/Cargo.toml b/crates/arty/Cargo.toml index 63581fc67c..f1e0e91225 100644 --- a/crates/arty/Cargo.toml +++ b/crates/arty/Cargo.toml @@ -84,7 +84,6 @@ thread_aware_core = { workspace = true, features = ["std"] } tick = { workspace = true } [dev-dependencies] -alloc_tracker = { workspace = true } arty_executor = { path = "../arty_executor", features = ["inspection"] } arty_io_host = { path = "../arty_io_host" } arty_macros = { path = "../arty_macros" } diff --git a/crates/arty/README.md b/crates/arty/README.md index df3afe79ba..2eb773982e 100644 --- a/crates/arty/README.md +++ b/crates/arty/README.md @@ -34,7 +34,7 @@ cargo add arty ```rust use arty::task::{Context, JoinError}; -#[arty::main(monitor = disabled)] +#[arty::main] async fn main(cx: Context) -> Result<(), JoinError> { let answer = cx.scheduler().spawn(async |_| 6 * 7).await?; println!("{answer}"); @@ -80,7 +80,7 @@ The default feature set enables `rt`, `macros`, and `monitor`. This crate was developed as part of The Oxidizer Project. Browse this crate's source code. - [__cargo_doc2readme_dependencies_info]: ggGmYW0CYXZlMC43LjNhdIQborR2_k_xJd4bTcf2krrNPIcbP72Pw1UdRjkbim_eMDe2BBthYvRhcoQb94geMR9_5x8bD0NI7vWvQU8bjBeZnbfaMTgbnBA2qp7TcJRhZIKCZGFydHllMC42LjGCa2FydHlfbWFjcm9zZTAuNi4x + [__cargo_doc2readme_dependencies_info]: ggGmYW0CYXZlMC43LjNhdIQborR2_k_xJd4bTcf2krrNPIcbP72Pw1UdRjkbim_eMDe2BBthYvRhcoQbpjTzkOovhJgb4j8Xd8b5dycb1Kf_wRXWtfYbr0QtKZVggRRhZIKCZGFydHllMC42LjGCa2FydHlfbWFjcm9zZTAuNi4x [__link0]: https://doc.rust-lang.org/stable/std/marker/trait.Send.html [__link1]: https://docs.rs/arty/0.6.1/arty/?search=core::ThreadAware [__link10]: https://docs.rs/arty/0.6.1/arty/core/index.html diff --git a/crates/arty/benches/arty_spawn_allocations.rs b/crates/arty/benches/arty_spawn_allocations.rs index 25caf2a360..ef38b0eb85 100644 --- a/crates/arty/benches/arty_spawn_allocations.rs +++ b/crates/arty/benches/arty_spawn_allocations.rs @@ -1,85 +1,182 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -//! Counts process-wide allocations for warmed async spawning. +//! Measures warmed task spawning with Metabench. //! -//! Run each case in a separate process. Select `local` or `remote`, followed by -//! `idle` or `disabled`. To exclude monitoring dependencies, build this source -//! in a separate package with Arty as a normal, runtime-only dependency. +//! Run with `--criterion --allocations`. Metabench runs allocation cases in +//! separate processes. To exclude monitoring dependencies, use a separate +//! package with Arty as a normal, runtime-only dependency. //! Arty's own dev-dependencies can enable monitoring through feature unification. //! Runtime setup and warm-up are excluded. //! Unrelated process threads can contribute allocations. The local case measures //! 4096 sequential children in one parent. The remote case measures 64 batches -//! of 64 tasks on four workers, including dispatch and joins. +//! of 64 tasks on four workers, including dispatch and joins. The blocking case +//! uses the same batches and one blocking worker. Reports show totals per +//! 4096-task workload, not per task. +//! The blocking backlog case queues 4096 tasks behind a gate. Its timer covers +//! releasing the gate and joining all tasks, but excludes submission. +//! Its allocation counts include the gate and its task. -use std::alloc::System; +use std::borrow::BorrowMut; use std::hint::black_box; +use std::sync::{Arc, Barrier}; +use std::time::{Duration, Instant}; -use alloc_tracker::{Allocator, Session}; -use arty::runtime::{MonitorPolicy, Runtime, WorkersPolicy}; +use arty::runtime::{BlockingPoolPolicy, MonitorPolicy, Runtime, WorkersPolicy}; +use arty::task::JoinHandle; +use criterion::{BenchmarkId, Criterion, Throughput}; +use metabench::benchmark; -#[global_allocator] -static ALLOCATOR: Allocator = Allocator::system(); +#[derive(Clone, Copy, Debug)] +enum Workload { + Local, + Remote, + Blocking, + BlockingBacklog, +} -fn main() -> Result<(), &'static str> { - let arguments = std::env::args().skip(1).collect::>(); - let [workload, mode] = arguments.as_slice() else { - return Err("pass a workload (local or remote) and a monitoring mode (idle or disabled)"); - }; - let local = match workload.as_str() { - "local" => true, - "remote" => false, - _ => return Err("workload must be local or remote"), - }; - let monitor = match mode.as_str() { - "idle" => MonitorPolicy::enabled(), - "disabled" => MonitorPolicy::disabled(), - _ => return Err("monitoring mode must be idle or disabled"), - }; - let runtime = Runtime::builder() - .workers(WorkersPolicy::exactly(if local { 1 } else { 4 })) - .monitor(monitor) - .build() - .expect("benchmark requires the configured worker count"); - let session = Session::new().no_stdout().no_file(); - let operation = session.operation("spawn"); - let mut handles = Vec::with_capacity(64); - for measured in [false, true] { - let measurement = measured.then(|| operation.measure_process().iterations(4096)); - if local { - let parent = runtime.scheduler().spawn_anywhere((), |cx, ()| async move { - for _ in 0..4096 { - cx.scheduler() - .spawn(async |_| black_box(())) - .await - .expect("benchmark child succeeds"); - } +impl Workload { + const fn name(self) -> &'static str { + match self { + Self::Local => "local", + Self::Remote => "remote", + Self::Blocking => "blocking", + Self::BlockingBacklog => "blocking_backlog", + } + } +} + +#[derive(Debug)] +struct Case { + runtime: Runtime, + handles: Vec>, + workload: Workload, +} + +impl Case { + fn new(workload: Workload, enabled: bool) -> Self { + let runtime = Runtime::builder() + .workers(WorkersPolicy::exactly(if matches!(workload, Workload::Local) { 1 } else { 4 })) + .blocking_pool(BlockingPoolPolicy::shared().max(1)) + .monitor(if enabled { + MonitorPolicy::enabled() + } else { + MonitorPolicy::disabled() + }) + .build() + .expect("benchmark requires the configured worker count"); + let mut case = Self { + runtime, + handles: Vec::with_capacity(if matches!(workload, Workload::BlockingBacklog) { 4096 } else { 64 }), + workload, + }; + black_box(case.run()); + case + } + + fn run(&mut self) -> Duration { + if matches!(self.workload, Workload::BlockingBacklog) { + self.handles.clear(); + let gate = Arc::new(Barrier::new(2)); + let worker_gate = Arc::clone(&gate); + let first = self.runtime.scheduler().spawn_blocking(move || { + worker_gate.wait(); }); - futures::executor::block_on(parent).expect("benchmark parent succeeds"); - } else { - for _ in 0..64 { - handles.clear(); + for _ in 0..4096 { + self.handles.push(self.runtime.scheduler().spawn_blocking(|| black_box(()))); + } + let start = Instant::now(); + gate.wait(); + futures::executor::block_on(first).expect("benchmark gate succeeds"); + for task in &mut self.handles { + futures::executor::block_on(task).expect("benchmark task succeeds"); + } + return start.elapsed(); + } + let start = Instant::now(); + match self.workload { + Workload::Local => { + let parent = self.runtime.scheduler().spawn_anywhere((), |cx, ()| async move { + for _ in 0..4096 { + cx.scheduler() + .spawn(async |_| black_box(())) + .await + .expect("benchmark child succeeds"); + } + }); + futures::executor::block_on(parent).expect("benchmark parent succeeds"); + } + Workload::Remote | Workload::Blocking | Workload::BlockingBacklog => { for _ in 0..64 { - handles.push(runtime.scheduler().spawn_anywhere((), |_, ()| async { black_box(()) })); - } - for task in &mut handles { - futures::executor::block_on(task).expect("benchmark remote task succeeds"); + self.handles.clear(); + for _ in 0..64 { + let task = match self.workload { + Workload::Blocking => self.runtime.scheduler().spawn_blocking(|| black_box(())), + Workload::Remote => self.runtime.scheduler().spawn_anywhere((), |_, ()| async { black_box(()) }), + Workload::Local | Workload::BlockingBacklog => unreachable!("local and backlog spawning use separate paths"), + }; + self.handles.push(task); + } + for task in &mut self.handles { + futures::executor::block_on(task).expect("benchmark task succeeds"); + } } } } - drop(measurement); + start.elapsed() + } +} + +fn configured_criterion() -> Criterion { + Criterion::default() + .sample_size(50) + .warm_up_time(Duration::from_millis(500)) + .measurement_time(Duration::from_secs(2)) +} + +fn criterion_benchmarks(criterion: &mut Criterion) { + let mut group = criterion.benchmark_group(SPAWN.group_name()); + group.throughput(Throughput::Elements(4096)); + for workload in [Workload::Local, Workload::Remote, Workload::Blocking, Workload::BlockingBacklog] { + for enabled in [true, false] { + let mode = if enabled { "idle" } else { "disabled" }; + group.bench_function( + BenchmarkId::new(SPAWN.benchmark_name(), format!("{}_{mode}", workload.name())), + |bencher| { + let mut case = Case::new(workload, enabled); + bencher.iter_custom(|iterations| { + let mut elapsed = Duration::ZERO; + for _ in 0..iterations { + elapsed += spawn_workload(&mut case).0; + } + elapsed + }); + }, + ); + } } - let report = session.to_report(); - let (_, operation) = report - .operations() - .find(|(name, _)| *name == "spawn") - .expect("the measured operation exists"); - println!( - "{workload},{mode},{},{},{}", - operation.total_iterations(), - operation.total_allocations_count(), - operation.total_bytes_allocated() - ); - runtime.stop().expect("benchmark runtime stops"); - Ok(()) + group.finish(); } + +#[benchmark(SPAWN, "arty_spawn_allocations/tasks", "spawn")] +#[bench::local_idle(Box::new(Case::new(Workload::Local, true)))] +#[bench::local_disabled(Box::new(Case::new(Workload::Local, false)))] +#[bench::remote_idle(Box::new(Case::new(Workload::Remote, true)))] +#[bench::remote_disabled(Box::new(Case::new(Workload::Remote, false)))] +#[bench::blocking_idle(Box::new(Case::new(Workload::Blocking, true)))] +#[bench::blocking_disabled(Box::new(Case::new(Workload::Blocking, false)))] +#[bench::blocking_backlog_idle(Box::new(Case::new(Workload::BlockingBacklog, true)))] +#[bench::blocking_backlog_disabled(Box::new(Case::new(Workload::BlockingBacklog, false)))] +fn spawn_workload>(mut input: S) -> (Duration, S) { + let elapsed = input.borrow_mut().run(); + (elapsed, input) +} + +metabench::main!( + criterion = { + factory = configured_criterion, + benchmarks = criterion_benchmarks, + unit = "ns", + }, + benchmarks = [SPAWN], +); diff --git a/crates/arty/src/io/mod.rs b/crates/arty/src/io/mod.rs index 4a027f5a91..e14364354e 100644 --- a/crates/arty/src/io/mod.rs +++ b/crates/arty/src/io/mod.rs @@ -60,7 +60,7 @@ //! # } //! # } //! # #[cfg(all(feature = "macros", feature = "rt"))] -//! #[arty::main(monitor = disabled)] +//! #[arty::main] //! async fn main(cx: Context) { //! let io = cx.io().context::(&()); //! io.submit("request"); diff --git a/crates/arty/src/lib.rs b/crates/arty/src/lib.rs index bd410c161e..d4ff8205f1 100644 --- a/crates/arty/src/lib.rs +++ b/crates/arty/src/lib.rs @@ -37,7 +37,7 @@ //! use arty::task::{Context, JoinError}; //! //! # #[cfg(all(feature = "macros", feature = "rt"))] -//! #[arty::main(monitor = disabled)] +//! #[arty::main] //! async fn main(cx: Context) -> Result<(), JoinError> { //! let answer = cx.scheduler().spawn(async |_| 6 * 7).await?; //! # assert_eq!(answer, 42); @@ -142,7 +142,7 @@ use arty_io_core as _; /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] -/// # #[arty::main(monitor = disabled)] +/// # #[arty::main] /// # async fn main(cx: arty::task::Context) { /// cx.clock().delay(std::time::Duration::from_millis(1)).await; /// println!("Hello from Arty!"); @@ -150,20 +150,11 @@ use arty_io_core as _; /// # #[cfg(not(all(feature = "macros", feature = "rt")))] fn main() {} /// ``` /// -/// Disable monitoring for the runtime: -/// -/// ``` -/// # #[cfg(all(feature = "macros", feature = "rt"))] -/// #[arty::main(monitor = disabled)] -/// async fn main(_cx: arty::task::Context) {} -/// # #[cfg(not(all(feature = "macros", feature = "rt")))] fn main() {} -/// ``` -/// /// Keep the runtime running after a task panic that Arty catches: /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] -/// #[arty::main(panic = ignore, monitor = disabled)] +/// #[arty::main(panic = ignore)] /// async fn main(cx: arty::task::Context) -> Result<(), arty::task::JoinError> { /// assert!( /// cx.scheduler() @@ -182,7 +173,7 @@ use arty_io_core as _; /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] -/// #[arty::main(workers = 4, monitor = disabled)] +/// #[arty::main(workers = 4)] /// async fn main(cx: arty::task::Context) -> Result<(), arty::task::JoinError> { /// let answer = cx.scheduler().spawn(async |_| 42).await?; /// assert_eq!(answer, 42); @@ -326,7 +317,7 @@ pub mod task; /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] -/// # #[arty::main(monitor = disabled)] +/// # #[arty::main] /// # async fn main(cx: arty::task::Context) { /// let worker: &arty::core::Thread = cx.thread(); /// assert_eq!(worker.id(), std::thread::current().id()); @@ -358,7 +349,7 @@ pub mod core { /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] -/// # #[arty::main(monitor = disabled)] +/// # #[arty::main] /// # async fn main(cx: arty::task::Context) { /// use std::future::pending; /// use std::time::Duration; diff --git a/crates/arty/src/runtime/bootstrap/startup.rs b/crates/arty/src/runtime/bootstrap/startup.rs index 094b949f21..64ce154e7b 100644 --- a/crates/arty/src/runtime/bootstrap/startup.rs +++ b/crates/arty/src/runtime/bootstrap/startup.rs @@ -45,7 +45,7 @@ fn build_with_processors( let blocking_pools = processor_config.blocking_pool_policy.into_pools(name.clone())?; #[cfg(feature = "monitor")] let (diagnostics, monitor_options) = { - let diagnostics = crate::runtime::monitoring::RuntimeDiagnostics::start(monitor_options.policy(), worker_count)?; + let diagnostics = crate::runtime::monitoring::RuntimeDiagnostics::start(monitor_options.policy(), &name, worker_count)?; let mut monitor_options = monitor_options; monitor_options.set_runtime(diagnostics.as_ref().map(crate::runtime::monitoring::RuntimeDiagnostics::handle)); (diagnostics, monitor_options) diff --git a/crates/arty/src/runtime/builder.rs b/crates/arty/src/runtime/builder.rs index ac11dfa570..320f83de0d 100644 --- a/crates/arty/src/runtime/builder.rs +++ b/crates/arty/src/runtime/builder.rs @@ -27,7 +27,6 @@ use crate::runtime::handle::Runtime; /// use arty::runtime::{BlockingPoolPolicy, Runtime, WorkersPolicy}; /// /// let runtime = Runtime::builder() -/// .monitor(arty::runtime::MonitorPolicy::disabled()) /// .workers(WorkersPolicy::at_most(4)) /// .blocking_pool(BlockingPoolPolicy::shared().max(4)) /// .build()?; @@ -41,7 +40,7 @@ pub struct RuntimeBuilder { } impl RuntimeBuilder { - /// Sets the runtime name used in worker thread names. + /// Sets the runtime name used in worker thread names and monitoring. /// /// Async threads use `-async-`, with indices starting at zero. /// Shared blocking pools use `-blocking-`. @@ -194,10 +193,7 @@ impl RuntimeBuilder { /// # #[cfg(all(feature = "rt", feature = "test-util"))] /// # fn sequential_delay() -> Result<(), Box> { /// let control = arty::time::ClockControl::new().auto_advance_timers(true); - /// let runtime = arty::runtime::Runtime::builder() - /// .monitor(arty::runtime::MonitorPolicy::disabled()) - /// .clock(control) - /// .build()?; + /// let runtime = arty::runtime::Runtime::builder().clock(control).build()?; /// runtime.scheduler().block_on(async |cx| { /// let duration = std::time::Duration::from_secs(30); /// let watch = cx.clock().stopwatch(); @@ -239,9 +235,7 @@ impl RuntimeBuilder { /// ``` /// use arty::runtime::Runtime; /// - /// let runtime = Runtime::builder() - /// .monitor(arty::runtime::MonitorPolicy::disabled()) - /// .build()?; + /// let runtime = Runtime::builder().build()?; /// assert_eq!(runtime.scheduler().block_on(async |_| 42)?, 42); /// # Ok::<(), Box>(()) /// ``` diff --git a/crates/arty/src/runtime/config/monitor_policy.rs b/crates/arty/src/runtime/config/monitor_policy.rs index 3156b0ec27..e4db0215c2 100644 --- a/crates/arty/src/runtime/config/monitor_policy.rs +++ b/crates/arty/src/runtime/config/monitor_policy.rs @@ -26,42 +26,24 @@ #[derive(Debug, Clone)] pub struct MonitorPolicy { enabled: bool, - name: Option, } impl MonitorPolicy { /// Creates a policy with monitoring enabled in the runtime configuration. #[must_use] pub const fn enabled() -> Self { - Self { enabled: true, name: None } + Self { enabled: true } } /// Creates a policy with monitoring disabled in the runtime configuration. #[must_use] pub const fn disabled() -> Self { - Self { - enabled: false, - name: None, - } - } - - /// Sets the runtime name shown by the Seismograph CLI. - /// - /// The default name is `arty`. Each runtime has a separate local endpoint. - #[must_use] - pub fn name(mut self, name: impl Into) -> Self { - self.name = Some(name.into()); - self + Self { enabled: false } } pub(crate) const fn is_enabled(&self) -> bool { self.enabled } - - #[cfg(feature = "monitor")] - pub(crate) fn runtime_name(&self) -> &str { - self.name.as_deref().unwrap_or("arty") - } } impl Default for MonitorPolicy { diff --git a/crates/arty/src/runtime/config/panic_policy.rs b/crates/arty/src/runtime/config/panic_policy.rs index 7e4d20aa27..7cee4502e1 100644 --- a/crates/arty/src/runtime/config/panic_policy.rs +++ b/crates/arty/src/runtime/config/panic_policy.rs @@ -19,10 +19,7 @@ /// ``` /// use arty::runtime::{PanicPolicy, Runtime}; /// -/// let runtime = Runtime::builder() -/// .monitor(arty::runtime::MonitorPolicy::disabled()) -/// .panic(PanicPolicy::ignore()) -/// .build()?; +/// let runtime = Runtime::builder().panic(PanicPolicy::ignore()).build()?; /// runtime.stop()?; /// # Ok::<(), Box>(()) /// ``` diff --git a/crates/arty/src/runtime/config/runtime_name.rs b/crates/arty/src/runtime/config/runtime_name.rs index 0a2af59ceb..af57954e2b 100644 --- a/crates/arty/src/runtime/config/runtime_name.rs +++ b/crates/arty/src/runtime/config/runtime_name.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use crate::runtime::error::Error; -/// Creates thread names for one runtime. +/// Names one runtime and its threads. #[derive(Clone, Debug)] pub(crate) struct RuntimeName(Arc); @@ -14,6 +14,11 @@ impl RuntimeName { Self(Arc::from(name.as_ref())) } + #[cfg(feature = "monitor")] + pub(crate) fn as_str(&self) -> &str { + &self.0 + } + pub(crate) fn async_worker(&self, index: usize) -> String { format!("{}-async-{index}", self.0) } diff --git a/crates/arty/src/runtime/control.rs b/crates/arty/src/runtime/control.rs index ee98aacd4f..bfe4cd57f3 100644 --- a/crates/arty/src/runtime/control.rs +++ b/crates/arty/src/runtime/control.rs @@ -32,7 +32,7 @@ impl RuntimeControl { /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] - /// # #[arty::main(monitor = disabled)] + /// # #[arty::main] /// # async fn main(cx: arty::task::Context) -> Result<(), arty::task::JoinError> { /// let tasks: Vec<_> = cx /// .control() @@ -69,9 +69,7 @@ impl RuntimeControl { /// ``` /// use arty::runtime::Runtime; /// - /// let runtime = Runtime::builder() - /// .monitor(arty::runtime::MonitorPolicy::disabled()) - /// .build()?; + /// let runtime = Runtime::builder().build()?; /// let control = runtime /// .scheduler() /// .block_on(async |cx| cx.control().clone())?; diff --git a/crates/arty/src/runtime/dispatch/client.rs b/crates/arty/src/runtime/dispatch/client.rs index 877184838a..6c24d8f391 100644 --- a/crates/arty/src/runtime/dispatch/client.rs +++ b/crates/arty/src/runtime/dispatch/client.rs @@ -77,23 +77,18 @@ impl DispatcherClient { self.core.spawn_on_worker(worker_index, future_factory) } - pub(crate) fn spawn(&self, future_factory: FF) -> JoinHandle + pub(crate) fn spawn( + &self, + future_factory: FF, + monitoring: crate::runtime::monitoring::TaskMonitoring, + ) -> JoinHandle where FF: FnOnce(Context) -> F + Send + 'static, F: Future + 'static, R: Send + 'static, + S: Fn() -> usize + Send + 'static, { - self.core.spawn(future_factory) - } - - #[cfg(feature = "monitor")] - pub(crate) fn spawn_with_size(&self, future_factory: FF, body_size: usize) -> JoinHandle - where - FF: FnOnce(Context) -> F + Send + 'static, - F: Future + 'static, - R: Send + 'static, - { - self.core.spawn_with_size(future_factory, body_size) + self.core.spawn(future_factory, monitoring) } pub(crate) fn spawn_relocating(&self, source: &thread_aware::Thread, future_factory: FF) -> JoinHandle diff --git a/crates/arty/src/runtime/dispatch/core.rs b/crates/arty/src/runtime/dispatch/core.rs index 50d612b550..54bed17d36 100644 --- a/crates/arty/src/runtime/dispatch/core.rs +++ b/crates/arty/src/runtime/dispatch/core.rs @@ -13,13 +13,14 @@ use nonempty::NonEmpty; use performables::arc::Arc; use thread_aware::{Thread, ThreadAware}; +use crate::runtime::monitoring::TaskMonitoring; use crate::runtime::shutdown::{Shutdown, ShutdownState}; use crate::runtime::worker::asynchronous::{CommandSender, LocalWorker, QueuedFactory, QueuedIoAccess, SpawnCommand, SystemCommand}; use crate::runtime::worker::blocking::Worker; use crate::runtime::{Error, Workers}; use crate::task::Context; use crate::task::handle::{JoinHandle, JoinHandlePool}; -use crate::task::preparation::prepare_remote_with_size; +use crate::task::preparation::prepare_remote; #[derive(Clone, Debug)] pub(crate) struct WorkerEndpoint { @@ -161,23 +162,14 @@ impl DispatcherCore { WorkerIndex(self.next_async_worker_index.fetch_add(1, Ordering::Relaxed) % self.worker_endpoints.len()) } - pub(crate) fn spawn(&self, future_factory: FF) -> JoinHandle - where - FF: FnOnce(Context) -> F + Send + 'static, - F: Future + 'static, - R: Send + 'static, - { - self.enqueue(self.next_worker_index(), TaskPlacement::Any, future_factory) - } - - #[cfg(feature = "monitor")] - pub(crate) fn spawn_with_size(&self, future_factory: FF, body_size: usize) -> JoinHandle + pub(crate) fn spawn(&self, future_factory: FF, monitoring: TaskMonitoring) -> JoinHandle where FF: FnOnce(Context) -> F + Send + 'static, F: Future + 'static, R: Send + 'static, + S: Fn() -> usize + Send + 'static, { - self.enqueue_with_size(self.next_worker_index(), TaskPlacement::Any, future_factory, move || body_size) + self.enqueue(self.next_worker_index(), TaskPlacement::Any, future_factory, monitoring) } pub(crate) fn spawn_relocating(&self, source: &Thread, future_factory: FF) -> JoinHandle @@ -285,7 +277,7 @@ impl DispatcherCore { F: Future + 'static, R: Send + 'static, { - self.enqueue(worker_index, TaskPlacement::SameThread, future_factory) + self.enqueue(worker_index, TaskPlacement::SameThread, future_factory, TaskMonitoring::capture()) } /// Records the execution worker so a successful result can relocate when its join is polled. @@ -296,24 +288,16 @@ impl DispatcherCore { R: ThreadAware + 'static, { let source = self.worker_endpoints[usize::from(worker_index)].thread.clone(); - self.enqueue(worker_index, placement, future_factory).relocate_result_from(source) - } - - fn enqueue(&self, worker_index: WorkerIndex, placement: TaskPlacement, future_factory: FF) -> JoinHandle - where - FF: FnOnce(Context) -> F + Send + 'static, - F: Future + 'static, - R: Send + 'static, - { - self.enqueue_with_size(worker_index, placement, future_factory, size_of::) + self.enqueue(worker_index, placement, future_factory, TaskMonitoring::capture()) + .relocate_result_from(source) } - fn enqueue_with_size( + fn enqueue( &self, worker_index: WorkerIndex, placement: TaskPlacement, future_factory: FF, - body_size: S, + monitoring: TaskMonitoring, ) -> JoinHandle where FF: FnOnce(Context) -> F + Send + 'static, @@ -329,7 +313,7 @@ impl DispatcherCore { .worker_endpoints .get(usize::from(worker_index)) .expect("worker index must identify a registered runtime worker"); - let (future_factory, join_handle) = prepare_remote_with_size(&self.remote_results, future_factory, placement, body_size); + let (future_factory, join_handle) = prepare_remote(&self.remote_results, future_factory, placement, monitoring); // A worker can close its channel before this send if shutdown already // reached that worker. The join handle detects the disconnection. @@ -396,9 +380,15 @@ mod tests { let dispatcher = DispatcherCore::new(NonEmpty::new(worker), Shutdown::default()); - dispatcher.spawn(|_| async move { unreachable!("we do not expect the task to be executed") }); + dispatcher.spawn( + |_| async move { unreachable!("we do not expect the task to be executed") }, + TaskMonitoring::capture(), + ); - dispatcher.spawn(|_| async move { unreachable!("we do not expect the task to be executed") }); + dispatcher.spawn( + |_| async move { unreachable!("we do not expect the task to be executed") }, + TaskMonitoring::capture(), + ); assert_eq!(drain_spawn(&worker_rx).len(), 2); } @@ -415,9 +405,15 @@ mod tests { let dispatcher = DispatcherCore::new(NonEmpty::from_vec(vec![worker1, worker2]).unwrap(), Shutdown::default()); - dispatcher.spawn(|_| async move { unreachable!("we do not expect the task to be executed") }); + dispatcher.spawn( + |_| async move { unreachable!("we do not expect the task to be executed") }, + TaskMonitoring::capture(), + ); - dispatcher.spawn(|_| async move { unreachable!("we do not expect the task to be executed") }); + dispatcher.spawn( + |_| async move { unreachable!("we do not expect the task to be executed") }, + TaskMonitoring::capture(), + ); assert_eq!(drain_spawn(&worker1_rx).len(), 1); assert_eq!(drain_spawn(&worker2_rx).len(), 1); @@ -432,7 +428,10 @@ mod tests { let dispatcher = DispatcherCore::new(NonEmpty::from_vec(vec![worker1, worker2]).unwrap(), Shutdown::default()); for _ in 0..4 { - dispatcher.spawn(|_| async move { unreachable!("we do not expect the task to be executed") }); + dispatcher.spawn( + |_| async move { unreachable!("we do not expect the task to be executed") }, + TaskMonitoring::capture(), + ); } assert_eq!(drain_spawn(&worker1_rx).len(), 2); @@ -503,7 +502,7 @@ mod tests { let dispatcher = DispatcherCore::new(NonEmpty::new(endpoint(&threads[0]).0), Shutdown::default()); dispatcher.stop_inner(ShutdownError::new(Error::worker_panicked()).into()); - let mut join = pin!(dispatcher.spawn(|_| async {})); + let mut join = pin!(dispatcher.spawn(|_| async {}, TaskMonitoring::capture())); let Poll::Ready(Err(error)) = join.as_mut().poll(&mut task::Context::from_waker(Waker::noop())) else { panic!("shutdown rejection must be immediately ready"); }; @@ -599,7 +598,7 @@ mod tests { // Close the command channel. drop(command_rx); - let join_handle = dispatcher.spawn(|_| async move {}); + let join_handle = dispatcher.spawn(|_| async move {}, TaskMonitoring::capture()); let join_handle = pin!(join_handle); let mut cx = task::Context::from_waker(Waker::noop()); diff --git a/crates/arty/src/runtime/handle.rs b/crates/arty/src/runtime/handle.rs index 08f831204f..3d82623339 100644 --- a/crates/arty/src/runtime/handle.rs +++ b/crates/arty/src/runtime/handle.rs @@ -45,9 +45,7 @@ use crate::task::RuntimeScheduler; /// ``` /// use arty::runtime::Runtime; /// -/// let runtime = Runtime::builder() -/// .monitor(arty::runtime::MonitorPolicy::disabled()) -/// .build()?; +/// let runtime = Runtime::builder().build()?; /// let task = runtime.scheduler().spawn_anywhere((), |_, ()| async { 42 }); /// assert_eq!(futures::executor::block_on(task)?, 42); /// runtime.stop()?; @@ -122,9 +120,7 @@ impl Runtime { /// ``` /// use arty::runtime::Runtime; /// - /// let runtime = Runtime::builder() - /// .monitor(arty::runtime::MonitorPolicy::disabled()) - /// .build()?; + /// let runtime = Runtime::builder().build()?; /// let scheduler = runtime.scheduler(); /// let first = scheduler.spawn_anywhere((), |_, ()| async { 20 }); /// let second = scheduler.spawn_anywhere((), |_, ()| async { 22 }); @@ -183,9 +179,7 @@ impl Runtime { /// ``` /// use arty::runtime::Runtime; /// - /// let runtime = Runtime::builder() - /// .monitor(arty::runtime::MonitorPolicy::disabled()) - /// .build()?; + /// let runtime = Runtime::builder().build()?; /// let error = runtime.scheduler().block_on(async |cx| { /// cx.control().request_stop(); /// cx.scheduler() diff --git a/crates/arty/src/runtime/mod.rs b/crates/arty/src/runtime/mod.rs index 807e867052..291f581c66 100644 --- a/crates/arty/src/runtime/mod.rs +++ b/crates/arty/src/runtime/mod.rs @@ -57,7 +57,7 @@ //! //! A monitor is a local connection through which the Seismograph CLI can //! collect diagnostics. Each enabled runtime owns one connection and removes it -//! after its workers stop. [`MonitorPolicy::name`] sets the displayed runtime name. +//! after its workers stop. [`RuntimeBuilder::name`] sets the displayed runtime name. //! Use [`MonitorPolicy::disabled`] to run without this connection or task inspection. //! //! Collection reads live async tasks and queued or running blocking functions @@ -96,7 +96,6 @@ //! //! fn main() -> Result<(), Box> { //! let runtime = Runtime::builder() -//! .monitor(arty::runtime::MonitorPolicy::disabled()) //! .workers(WorkersPolicy::at_most(2)) //! .blocking_pool(BlockingPoolPolicy::shared().max(4)) //! .build()?; @@ -158,7 +157,6 @@ mod control; pub(crate) mod dispatch; mod error; mod handle; -#[cfg(feature = "monitor")] pub(crate) mod monitoring; mod shutdown; mod shutdown_cause; diff --git a/crates/arty/src/runtime/monitoring/blocking.rs b/crates/arty/src/runtime/monitoring/blocking.rs new file mode 100644 index 0000000000..e8f2c37d0f --- /dev/null +++ b/crates/arty/src/runtime/monitoring/blocking.rs @@ -0,0 +1,238 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::marker::PhantomData; + +#[cfg(feature = "monitor")] +use super::TaskInfo; +use crate::runtime::shutdown::Shutdown; +#[cfg(feature = "monitor")] +use crate::runtime::shutdown::ShutdownState; +#[cfg(feature = "monitor")] +use crate::runtime::worker::blocking::PanicPolicy; +use crate::task::preparation::BlockingTask; + +pub(crate) enum BlockingJob { + Plain(Box), + #[cfg(feature = "monitor")] + Monitored(std::sync::Arc), +} + +impl BlockingJob { + pub(crate) fn run(self) { + match self { + Self::Plain(body) => body(), + #[cfg(feature = "monitor")] + Self::Monitored(body) => body.run(), + } + } + + #[cfg(feature = "monitor")] + pub(crate) fn monitoring(&self) -> Option<&std::sync::Arc> { + match self { + Self::Plain(_) => None, + Self::Monitored(body) => Some(body), + } + } +} + +#[cfg(feature = "monitor")] +pub(crate) trait MonitoredBlockingJob: Send + Sync { + fn info(&self) -> &TaskInfo; + fn run(&self); +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct BlockingPoolMonitoring { + #[cfg(feature = "monitor")] + events: std::sync::Arc>, +} + +impl BlockingPoolMonitoring { + #[cfg_attr( + not(feature = "monitor"), + expect( + clippy::unused_self, + clippy::extra_unused_type_parameters, + reason = "The monitoring interface is unchanged when monitoring is compiled out." + ) + )] + pub(crate) fn task(&self, shutdown: &Shutdown) -> BlockingTaskMonitoring { + #[cfg(not(feature = "monitor"))] + let _ = shutdown; + BlockingTaskMonitoring { + #[cfg(feature = "monitor")] + info: matches!(shutdown.state(), ShutdownState::Running) + .then(|| self.events().map(TaskInfo::new::)) + .flatten(), + } + } + + #[cfg(feature = "monitor")] + pub(crate) fn enable(&self, events: seismograph_runtime::task::EventContext) { + self.events.set(events).expect("monitoring is enabled once under the queue lock"); + } + + #[cfg(feature = "monitor")] + pub(crate) fn events(&self) -> Option { + self.events.get().copied() + } +} + +#[derive(Default)] +pub(crate) struct BlockingTaskMonitoring { + #[cfg(feature = "monitor")] + info: Option, +} + +impl BlockingTaskMonitoring { + #[cfg_attr( + not(feature = "monitor"), + expect(clippy::unused_self, reason = "Task monitoring is empty when monitoring is compiled out.") + )] + pub(crate) fn prepare(self, task: BlockingTask) -> BlockingJob + where + F: FnOnce() -> R + Send + 'static, + R: Send + 'static, + { + #[cfg(feature = "monitor")] + if let Some(info) = self.info { + return BlockingJob::Monitored(std::sync::Arc::new(MonitoredTask { + task: std::sync::Mutex::new(task), + info, + })); + } + BlockingJob::Plain(Box::new(move || task.run(BlockingTaskEvents::default()))) + } +} + +#[derive(Clone, Copy, Default)] +pub(crate) struct BlockingTaskEvents<'a> { + #[cfg(feature = "monitor")] + info: Option<&'a TaskInfo>, + marker: PhantomData<&'a ()>, +} + +#[cfg_attr( + not(feature = "monitor"), + expect(clippy::unused_self, reason = "Task events do nothing when monitoring is compiled out.") +)] +impl BlockingTaskEvents<'_> { + pub(crate) fn completed(self) { + #[cfg(feature = "monitor")] + if let Some(info) = self.info { + info.completed(); + } + } + + pub(crate) fn panicked(self) { + #[cfg(feature = "monitor")] + if let Some(info) = self.info { + info.panicked(); + } + } +} + +// Inspection shares this allocation with execution. Only taking the callback +// needs the mutex; neither the callback nor its destructor runs under the lock. +#[cfg(feature = "monitor")] +struct MonitoredTask { + task: std::sync::Mutex>, + info: TaskInfo, +} + +#[cfg(feature = "monitor")] +impl MonitoredBlockingJob for MonitoredTask +where + F: FnOnce() -> R + Send + 'static, + R: Send + 'static, +{ + fn info(&self) -> &TaskInfo { + &self.info + } + + fn run(&self) { + let task = std::mem::replace( + &mut *self.task.lock().expect("taking a blocking callback does not execute user code"), + BlockingTask { + pending: None, + panic_policy: PanicPolicy::Ignore, + }, + ); + task.run(BlockingTaskEvents { + info: Some(&self.info), + marker: PhantomData, + }); + } +} + +#[cfg(feature = "monitor")] +impl Drop for MonitoredTask { + fn drop(&mut self) { + if self + .task + .get_mut() + .expect("taking a blocking callback does not execute user code") + .pending + .is_some() + { + self.info.canceled(); + } + } +} + +#[derive(Default)] +pub(crate) struct BlockingWorkerMonitoring { + #[cfg(feature = "monitor")] + pub(crate) handle: Option, +} + +#[cfg_attr( + not(feature = "monitor"), + expect(clippy::unused_self, reason = "Worker monitoring is empty when monitoring is compiled out.") +)] +impl BlockingWorkerMonitoring { + pub(crate) fn parked(&self) { + #[cfg(feature = "monitor")] + if let Some(worker) = &self.handle { + worker.parked(); + } + } + + pub(crate) fn unparked(&self) { + #[cfg(feature = "monitor")] + if let Some(worker) = &self.handle { + worker.unparked(); + } + } + + pub(crate) fn run(&self, body: BlockingJob, canceled: bool) -> std::thread::Result<()> { + #[cfg(feature = "monitor")] + let observation = self.handle.as_ref().filter(|_| !canceled).and_then(|worker| { + body.monitoring().map(|body| { + let info = body.info(); + info.started_on(worker.task_events()); + let poll = worker.is_observed().then(|| worker.task_poll_started(info.id())); + (worker, poll) + }) + }); + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + #[cfg(feature = "monitor")] + let _current = observation + .as_ref() + .and_then(|(_, poll)| poll.as_ref()) + .and_then(|_| body.monitoring()) + .map(|body| super::task::enter_current(body.info().id())); + if canceled { + drop(body); + } else { + body.run(); + } + })); + #[cfg(feature = "monitor")] + if let Some((worker, Some(poll))) = observation { + worker.task_poll_finished(poll); + } + outcome + } +} diff --git a/crates/arty/src/runtime/monitoring/mod.rs b/crates/arty/src/runtime/monitoring/mod.rs index 579717fa26..208bc7e872 100644 --- a/crates/arty/src/runtime/monitoring/mod.rs +++ b/crates/arty/src/runtime/monitoring/mod.rs @@ -1,20 +1,34 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -//! Runtime-owned Seismograph connections and native task inspection. +//! Private task monitoring and native task inspection. +#[cfg(feature = "monitor")] mod asynchronous; +mod blocking; +#[cfg(feature = "monitor")] mod runtime; +mod submission; +#[cfg(feature = "monitor")] mod task; #[cfg(test)] -#[cfg(not(miri))] +#[cfg(all(feature = "monitor", not(miri)))] mod tests; // Hold for the full lifetime of tests that use the process registry or recorder. -#[cfg(test)] +#[cfg(all(test, feature = "monitor"))] pub(crate) static TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); +#[cfg(feature = "monitor")] pub(crate) use asynchronous::WorkerDiagnostics; +#[cfg(feature = "monitor")] +pub(crate) use blocking::MonitoredBlockingJob; +pub(crate) use blocking::{BlockingJob, BlockingPoolMonitoring, BlockingTaskEvents, BlockingTaskMonitoring, BlockingWorkerMonitoring}; +#[cfg(feature = "monitor")] pub(crate) use runtime::RuntimeDiagnostics; -pub(crate) use task::{TaskInfo, add_task, add_task_with_metadata, current_task, enter_current}; +pub(crate) use submission::{TaskMonitoring, add_task}; +#[cfg(feature = "monitor")] +pub(crate) use task::TaskInfo; +#[cfg(all(test, feature = "monitor"))] +pub(crate) use task::current_task; diff --git a/crates/arty/src/runtime/monitoring/runtime.rs b/crates/arty/src/runtime/monitoring/runtime.rs index 70838cd94b..e285d171ac 100644 --- a/crates/arty/src/runtime/monitoring/runtime.rs +++ b/crates/arty/src/runtime/monitoring/runtime.rs @@ -4,6 +4,7 @@ use seismograph::monitor::Monitor; use seismograph_runtime::{RuntimeHandle, RuntimeMetadata, RuntimeRegistration}; +use crate::runtime::config::RuntimeName; use crate::runtime::{Error, MonitorPolicy}; #[derive(Debug)] @@ -13,16 +14,16 @@ pub(crate) struct RuntimeDiagnostics { } impl RuntimeDiagnostics { - pub(crate) fn start(policy: &MonitorPolicy, workers: usize) -> Result, Error> { + pub(crate) fn start(policy: &MonitorPolicy, name: &RuntimeName, workers: usize) -> Result, Error> { if !policy.is_enabled() { return Ok(None); } let workers = u32::try_from(workers).map_err(Error::new)?; - let builder = Monitor::builder().name(policy.runtime_name()); + let builder = Monitor::builder().name(name.as_str()); #[cfg(test)] let builder = builder.io(seismograph_protocol::io::MonitorIo::new_mocked().0); let monitor = builder.start().map_err(Error::new)?; - let registration = seismograph_runtime::register_runtime(RuntimeMetadata::new(policy.runtime_name(), workers)); + let registration = seismograph_runtime::register_runtime(RuntimeMetadata::new(name.as_str(), workers)); Ok(Some(Self { monitor, registration })) } diff --git a/crates/arty/src/runtime/monitoring/submission.rs b/crates/arty/src/runtime/monitoring/submission.rs new file mode 100644 index 0000000000..f73a579453 --- /dev/null +++ b/crates/arty/src/runtime/monitoring/submission.rs @@ -0,0 +1,122 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::marker::PhantomData; + +use arty_executor::{TaskCompletion, TaskPlacement, TaskSet}; + +/// Captures submission metadata before a task crosses threads. +pub(crate) struct TaskMonitoring { + #[cfg(feature = "monitor")] + parent: Option, + #[cfg(feature = "monitor")] + body_size: S, + marker: PhantomData (Body, S)>, +} + +impl TaskMonitoring { + pub(crate) fn capture() -> TaskMonitoring usize + Send> { + // A function item stores no bytes; only erased scoped futures carry a size. + TaskMonitoring::new(size_of::) + } + + pub(crate) fn with_body_size(body_size: usize) -> TaskMonitoring usize + Send> { + TaskMonitoring::new(move || body_size) + } +} + +impl TaskMonitoring { + fn new(body_size: S) -> Self { + #[cfg(not(feature = "monitor"))] + drop(body_size); + Self { + #[cfg(feature = "monitor")] + parent: super::task::current_task(), + #[cfg(feature = "monitor")] + body_size, + marker: PhantomData, + } + } +} + +impl TaskMonitoring +where + Body: 'static, + S: Fn() -> usize + 'static, +{ + #[cfg_attr( + not(feature = "monitor"), + expect(clippy::unused_self, reason = "Submission metadata is empty when monitoring is compiled out.") + )] + pub(crate) fn register(self, tasks: &TaskSet, future: F, placement: TaskPlacement, completion: T) + where + F: IntoFuture + 'static, + R: 'static, + T: TaskCompletion, + { + #[cfg(feature = "monitor")] + super::task::add_task_with_metadata::(tasks, future, placement, completion, self.parent, self.body_size); + #[cfg(not(feature = "monitor"))] + tasks.task(future).placement(placement).register(completion); + } +} + +pub(crate) fn add_task(tasks: &TaskSet, future: F, placement: TaskPlacement, completion: T) +where + Body: 'static, + F: IntoFuture + 'static, + R: 'static, + T: TaskCompletion, +{ + TaskMonitoring::::capture().register(tasks, future, placement, completion); +} + +#[cfg(all(test, feature = "monitor"))] +mod tests { + use std::pin::Pin; + + use arty_executor::{Executor, InspectionStatus}; + use seismograph_runtime::RuntimeMetadata; + use seismograph_runtime::worker::{WorkerMetadata, WorkerRole}; + + use super::*; + use crate::runtime::monitoring::task::{Description, enter_current, set_events}; + use crate::runtime::shutdown::Shutdown; + use crate::task::handle::JoinHandlePool; + + #[test] + fn erased_submission_preserves_its_original_size_and_submission_parent() { + type ErasedFuture = Pin>>; + + let _monitoring = crate::runtime::monitoring::TEST_LOCK.lock().unwrap(); + let runtime = seismograph_runtime::register_runtime(RuntimeMetadata::new("submission-metadata", 1)); + let worker = runtime.register_worker(WorkerMetadata::new(WorkerRole::Core)); + let parent = seismograph_runtime::allocate_task_id(); + let monitoring = { + let _parent = enter_current(parent); + TaskMonitoring::::with_body_size(37) + }; + let _other_parent = enter_current(seismograph_runtime::allocate_task_id()); + set_events(Some(worker.handle().task_events())); + let _events = scopeguard::guard((), |()| set_events(None)); + let mut executor = Executor::builder().build(); + let results = JoinHandlePool::new(Shutdown::default()); + let (_handle, sender) = results.handle(); + let future: ErasedFuture = Box::pin(std::future::pending()); + monitoring.register(&executor.tasks(), future, TaskPlacement::Any, sender); + let mut snapshots = Vec::new(); + + let status = executor + .inspect(1, |description: &Description| { + if let Description::Snapshot(task) = description { + snapshots.push(task.clone()); + } + }) + .unwrap(); + + assert_eq!(status, InspectionStatus::Complete); + assert_eq!(snapshots.len(), 1); + assert_eq!(snapshots[0].parent, Some(parent)); + assert_eq!(snapshots[0].future_size_bytes, Some(37)); + } +} diff --git a/crates/arty/src/runtime/monitoring/task.rs b/crates/arty/src/runtime/monitoring/task.rs index 8e85ba66fd..b9a9d0c700 100644 --- a/crates/arty/src/runtime/monitoring/task.rs +++ b/crates/arty/src/runtime/monitoring/task.rs @@ -249,17 +249,6 @@ impl> TaskCompletion for Completion { } } -pub(crate) fn add_task(tasks: &TaskSet, future: F, placement: TaskPlacement, completion: T) -where - Body: 'static, - F: IntoFuture + 'static, - R: 'static, - T: TaskCompletion, -{ - // The function item stores no bytes. Only erased scoped futures carry a size. - add_task_with_metadata::(tasks, future, placement, completion, current_task(), size_of::); -} - pub(crate) fn add_task_with_metadata( tasks: &TaskSet, future: F, diff --git a/crates/arty/src/runtime/monitoring/tests.rs b/crates/arty/src/runtime/monitoring/tests.rs index 836b4df280..52f764ccae 100644 --- a/crates/arty/src/runtime/monitoring/tests.rs +++ b/crates/arty/src/runtime/monitoring/tests.rs @@ -43,7 +43,7 @@ fn runtime(name: &str) -> Runtime { Runtime::builder() .workers(WorkersPolicy::exactly(1)) .blocking_pool(BlockingPoolPolicy::shared().max(1)) - .monitor(MonitorPolicy::enabled().name(name)) + .name(name) .panic(PanicPolicy::ignore()) .build() .unwrap() @@ -81,11 +81,14 @@ fn monitor_policy_defaults_to_enabled() { .scheduler() .block_on(async |cx| cx.control().workers().into_iter().count()) .unwrap(); - let active = diagnostics.capture(CaptureOptions::default()).unwrap().state; + let active = diagnostics.capture(CaptureOptions::default()).unwrap(); runtime.stop().unwrap(); let stopped = runtimes().into_iter().find(|runtime| runtime.id == diagnostics.id()).unwrap().state; - assert_eq!((workers, active, stopped), (1, RuntimeState::Running, RuntimeState::Stopped)); + assert_eq!( + (workers, active.name.as_str(), active.state, stopped), + (1, "arty", RuntimeState::Running, RuntimeState::Stopped) + ); } #[test] @@ -107,7 +110,8 @@ fn explicit_monitor_policies_preserve_runtime_behavior() { fn disabled_monitoring_registers_no_runtime() { let _recording = RecorderGuard::new(); let runtime = Runtime::builder() - .monitor(MonitorPolicy::disabled().name("disabled-monitor")) + .name("disabled-monitor") + .monitor(MonitorPolicy::disabled()) .build() .unwrap(); @@ -143,6 +147,7 @@ fn local_and_remote_tasks_are_collected_without_recording() { let snapshot = capture(&runtime); runtime.stop().unwrap(); + assert_eq!(snapshot.name, "native-tasks"); assert!(parent.join().unwrap_err().is_shutdown()); let observed = snapshot .tasks @@ -714,7 +719,7 @@ fn queued_remote_children_retain_their_recorded_parent() { }); let runtime = Runtime::builder() .workers(WorkersPolicy::exactly(2)) - .monitor(MonitorPolicy::enabled().name("remote-parent-links")) + .name("remote-parent-links") .build() .unwrap(); let (started, ready) = mpsc::channel(); @@ -1016,7 +1021,7 @@ fn growing_pool(name: &str, policy: BlockingPoolPolicy, expected_blocking_worker let runtime = Runtime::builder() .workers(WorkersPolicy::exactly(2)) .blocking_pool(policy.max(2)) - .monitor(MonitorPolicy::enabled().name(name)) + .name(name) .build() .unwrap(); let scheduler = runtime.scheduler().block_on(async |cx| cx.scheduler().clone()).unwrap(); diff --git a/crates/arty/src/runtime/worker/asynchronous/command.rs b/crates/arty/src/runtime/worker/asynchronous/command.rs index 0bb2009063..79548ce9e6 100644 --- a/crates/arty/src/runtime/worker/asynchronous/command.rs +++ b/crates/arty/src/runtime/worker/asynchronous/command.rs @@ -184,7 +184,12 @@ mod tests { #[test] fn queued_factory_disposal_contains_join_notification_panic() { let result_pool = crate::task::handle::JoinHandlePool::new(crate::runtime::shutdown::Shutdown::default()); - let (factory, join) = prepare_remote(&result_pool, |_| async { 42 }, TaskPlacement::Any); + let (factory, join) = prepare_remote( + &result_pool, + |_| async { 42 }, + TaskPlacement::Any, + crate::runtime::monitoring::TaskMonitoring::capture(), + ); let mut join = pin!(join); let wake = Arc::new(PanicWake(AtomicBool::new(false))); let waker = Waker::from(Arc::clone(&wake)); diff --git a/crates/arty/src/runtime/worker/asynchronous/local_worker.rs b/crates/arty/src/runtime/worker/asynchronous/local_worker.rs index c22cdf78dc..b009898c44 100644 --- a/crates/arty/src/runtime/worker/asynchronous/local_worker.rs +++ b/crates/arty/src/runtime/worker/asynchronous/local_worker.rs @@ -165,10 +165,7 @@ impl SchedulerContext { { let (handle, result_tx) = self.results.handle(); let inner = DeferredTaskFactory::new(future_factory, self.context.clone()); - #[cfg(feature = "monitor")] crate::runtime::monitoring::add_task::(&self.tasks, inner, placement, result_tx); - #[cfg(not(feature = "monitor"))] - self.tasks.task(inner).placement(placement).register(result_tx); handle } } diff --git a/crates/arty/src/runtime/worker/asynchronous/worker.rs b/crates/arty/src/runtime/worker/asynchronous/worker.rs index 09ac785933..2640ef6998 100644 --- a/crates/arty/src/runtime/worker/asynchronous/worker.rs +++ b/crates/arty/src/runtime/worker/asynchronous/worker.rs @@ -946,7 +946,9 @@ mod tests { }))) .unwrap(); } - let join = worker_context(&worker).dispatcher().spawn(|_| async {}); + let join = worker_context(&worker) + .dispatcher() + .spawn(|_| async {}, crate::runtime::monitoring::TaskMonitoring::capture()); commands.system_sender().send(SystemCommand::Shutdown).unwrap(); assert!(worker.execute_step().is_break()); @@ -1688,13 +1690,16 @@ mod tests { }))) .unwrap(); let invoked = Arc::new(AtomicBool::new(false)); - let join = worker_context(&worker).dispatcher().spawn({ - let invoked = Arc::clone(&invoked); - move |_| { - invoked.store(true, Ordering::Relaxed); - std::future::ready(()) - } - }); + let join = worker_context(&worker).dispatcher().spawn( + { + let invoked = Arc::clone(&invoked); + move |_| { + invoked.store(true, Ordering::Relaxed); + std::future::ready(()) + } + }, + crate::runtime::monitoring::TaskMonitoring::capture(), + ); assert!(worker.execute_step().is_break()); diff --git a/crates/arty/src/runtime/worker/blocking/queue.rs b/crates/arty/src/runtime/worker/blocking/queue.rs index 487e8ff031..3bf77b111c 100644 --- a/crates/arty/src/runtime/worker/blocking/queue.rs +++ b/crates/arty/src/runtime/worker/blocking/queue.rs @@ -6,115 +6,15 @@ //! immutable task-event context so submissions do not lock this queue for it. use std::collections::VecDeque; -#[cfg(feature = "monitor")] -use std::sync::OnceLock; use std::sync::{Arc, Condvar, Mutex}; #[cfg(feature = "monitor")] -use crate::runtime::monitoring::TaskInfo; - -pub(super) type Job = Box; +use crate::runtime::monitoring::MonitoredBlockingJob; +use crate::runtime::monitoring::{BlockingJob, BlockingPoolMonitoring, BlockingWorkerMonitoring}; pub(super) struct QueuedJob { - pub(super) body: Job, + pub(super) body: BlockingJob, pub(super) shutdown: Option, - #[cfg(feature = "monitor")] - pub(super) info: Option>, -} - -#[cfg(all(test, feature = "monitor", feature = "test-util"))] -mod tests { - use seismograph_runtime::RuntimeMetadata; - use seismograph_runtime::snapshot::{CaptureOptions, TaskActivityState, TaskCollection, WorkerState}; - use seismograph_runtime::worker::{WorkerMetadata, WorkerRole}; - - use super::*; - - #[test] - fn blocking_slots_finish_and_retire_without_retaining_a_running_task() { - let _monitoring = crate::runtime::monitoring::TEST_LOCK.lock().unwrap(); - let runtime = seismograph_runtime::register_runtime(RuntimeMetadata::new("blocking-slots", 1)); - let queue = Arc::new(Queue::default()); - let index = queue.add_worker(); - queue.enable_monitoring(&runtime.handle()); - queue.enable_monitoring(&runtime.handle()); - let info = Arc::new(TaskInfo::new::<()>(*queue.events.get().unwrap())); - assert!(queue.push(QueuedJob { - body: Box::new(|| {}), - shutdown: None, - info: Some(Arc::clone(&info)), - })); - let mut worker = None; - let job = queue.pop(index, &mut worker).unwrap(); - let active = runtime.handle().capture(CaptureOptions::default()).unwrap(); - - queue.finish(index); - let finished = runtime.handle().capture(CaptureOptions::default()).unwrap(); - queue.retire(index); - let retired = runtime.handle().capture(CaptureOptions::default()).unwrap(); - drop(job); - - assert_eq!(active.tasks.len(), 1); - assert_eq!(active.tasks[0].id, info.id()); - assert_eq!(active.tasks[0].activity.unwrap().state, TaskActivityState::Running); - assert!(finished.tasks.is_empty()); - assert!(retired.tasks.is_empty()); - assert_eq!(retired.workers[0].state, WorkerState::Stopped); - } - - #[test] - fn blocking_collection_handles_missing_metadata_limits_and_canceled_requests() { - let _monitoring = crate::runtime::monitoring::TEST_LOCK.lock().unwrap(); - let runtime = seismograph_runtime::register_runtime(RuntimeMetadata::new("blocking-limits", 1)); - let queue = Arc::new(Queue::default()); - queue.add_worker(); - queue.enable_monitoring(&runtime.handle()); - for info in [None, Some(Arc::new(TaskInfo::new::<()>(*queue.events.get().unwrap())))] { - assert!(queue.push(QueuedJob { - body: Box::new(|| {}), - shutdown: None, - info, - })); - } - let truncated = runtime - .handle() - .capture(CaptureOptions { - limit_per_worker: 1, - ..CaptureOptions::default() - }) - .unwrap(); - let complete = runtime.handle().capture(CaptureOptions::default()).unwrap(); - - let source = seismograph_runtime::register_runtime(RuntimeMetadata::new("canceled-blocking-request", 1)); - let worker = source.register_worker(WorkerMetadata::new(WorkerRole::Core)); - let (sender, receiver) = std::sync::mpsc::channel(); - worker - .handle() - .register_task_provider(move |request| sender.send(request).unwrap()) - .unwrap(); - source.handle().capture_with_timeout(CaptureOptions::default()).unwrap(); - queue.collect(0, receiver.try_recv().unwrap()); - - assert!(truncated.tasks.is_empty()); - assert_eq!(truncated.workers[0].task_collection, TaskCollection::Truncated); - assert_eq!(complete.tasks.len(), 1); - assert_eq!(complete.tasks[0].activity.unwrap().state, TaskActivityState::Ready); - assert_eq!(complete.workers[0].task_collection, TaskCollection::Complete); - } - - #[test] - fn a_dropped_blocking_queue_reports_collection_failure() { - let _monitoring = crate::runtime::monitoring::TEST_LOCK.lock().unwrap(); - let runtime = seismograph_runtime::register_runtime(RuntimeMetadata::new("dropped-blocking-queue", 1)); - let queue = Arc::new(Queue::default()); - let worker = queue.register_worker(&runtime.handle(), 0); - drop(queue); - - let snapshot = runtime.handle().capture(CaptureOptions::default()).unwrap(); - assert!(snapshot.tasks.is_empty()); - assert_eq!(snapshot.workers[0].task_collection, TaskCollection::Failed); - drop(worker); - } } #[derive(Default)] @@ -124,7 +24,7 @@ struct WorkerSlot { #[cfg(feature = "monitor")] registration: Option, #[cfg(feature = "monitor")] - current: Option>, + current: Option>, } #[derive(Default)] @@ -140,8 +40,7 @@ struct State { pub(super) struct Queue { state: Mutex, ready: Condvar, - #[cfg(feature = "monitor")] - pub(super) events: Arc>, + pub(super) monitoring: BlockingPoolMonitoring, } impl Queue { @@ -176,31 +75,43 @@ impl Queue { index } - pub(super) fn pop( - &self, - index: usize, - #[cfg(feature = "monitor")] worker: &mut Option, - ) -> Option { + #[cfg_attr( + not(feature = "monitor"), + expect( + clippy::needless_pass_by_ref_mut, + reason = "The worker handle is assigned only when monitoring is compiled in." + ) + )] + pub(super) fn pop(&self, index: usize, worker: &mut BlockingWorkerMonitoring) -> Option { #[cfg(not(feature = "monitor"))] let _ = index; + // Drop the previous job after the queue guard, including on early return. + #[cfg(feature = "monitor")] + let mut previous; let mut state = self .state .lock() .expect("a blocking queue operation must not panic while holding its lock"); + #[cfg(feature = "monitor")] + { + let slot = &mut state.workers[index]; + previous = slot.current.take(); + slot.running = false; + } loop { #[cfg(feature = "monitor")] - if worker.is_none() + if worker.handle.is_none() && let Some(registration) = &state.workers[index].registration { let handle = registration.handle(); handle.attach_current_thread(); - *worker = Some(handle); + worker.handle = Some(handle); } if let Some(job) = state.jobs.pop_front() { #[cfg(feature = "monitor")] - { + if worker.handle.is_some() { state.workers[index].running = true; - state.workers[index].current.clone_from(&job.info); + state.workers[index].current = job.body.monitoring().cloned(); } return Some(job); } @@ -208,30 +119,24 @@ impl Queue { return None; } #[cfg(feature = "monitor")] - if let Some(worker) = worker.as_ref() { - worker.parked(); + if previous.is_some() { + drop(state); + drop(previous.take()); + state = self + .state + .lock() + .expect("a blocking queue operation must not panic while holding its lock"); + continue; } + worker.parked(); state = self .ready .wait(state) .expect("a blocking queue operation must not panic while holding its lock"); - #[cfg(feature = "monitor")] - if let Some(worker) = worker.as_ref() { - worker.unparked(); - } + worker.unparked(); } } - #[cfg(feature = "monitor")] - pub(super) fn finish(&self, index: usize) { - let mut state = self - .state - .lock() - .expect("a blocking queue operation must not panic while holding its lock"); - state.workers[index].current = None; - state.workers[index].running = false; - } - pub(super) fn stop(&self) { self.state .lock() @@ -240,15 +145,26 @@ impl Queue { self.ready.notify_all(); } - #[cfg(feature = "monitor")] + #[cfg_attr( + not(feature = "monitor"), + expect( + clippy::unused_self, + reason = "There is no worker registration to retire when monitoring is compiled out." + ) + )] pub(super) fn retire(&self, index: usize) { - let mut state = self - .state - .lock() - .expect("a blocking queue operation must not panic while holding its lock"); - let registration = state.workers[index].registration.take(); - drop(state); - drop(registration); + #[cfg(not(feature = "monitor"))] + let _ = index; + #[cfg(feature = "monitor")] + { + let mut state = self + .state + .lock() + .expect("a blocking queue operation must not panic while holding its lock"); + let registration = state.workers[index].registration.take(); + drop(state); + drop(registration); + } } #[cfg(feature = "monitor")] @@ -270,7 +186,7 @@ impl Queue { .expect("a blocking pool starts with at least one worker") .handle() .task_events(); - self.events.set(events).expect("monitoring is enabled once under the queue lock"); + self.monitoring.enable(events); state.runtime = Some(runtime.clone()); drop(state); self.ready.notify_all(); @@ -310,14 +226,17 @@ impl Queue { .expect("a blocking queue operation must not panic while holding its lock"); let current = state.workers[index] .running - .then_some((state.workers[index].current.as_ref(), TaskActivityState::Running)) + .then_some(( + state.workers[index].current.as_ref().map(|body| body.info()), + TaskActivityState::Running, + )) .into_iter(); // The first worker owns inspection of the shared submission queue. let queued = state .jobs .iter() .take(if index == 0 { usize::MAX } else { 0 }) - .map(|job| (job.info.as_ref(), TaskActivityState::Ready)); + .map(|job| (job.body.monitoring().map(|body| body.info()), TaskActivityState::Ready)); let mut descriptions = current.chain(queued); let tasks = descriptions .by_ref() @@ -333,3 +252,198 @@ impl Queue { request.respond(tasks, status); } } + +#[cfg(all(test, feature = "monitor", feature = "test-util"))] +mod tests { + use seismograph_runtime::RuntimeMetadata; + use seismograph_runtime::snapshot::{CaptureOptions, TaskActivityState, TaskCollection, WorkerState}; + use seismograph_runtime::worker::{WorkerMetadata, WorkerRole}; + + use super::*; + use crate::runtime::monitoring::TaskInfo; + + struct TestJob(TaskInfo); + + impl MonitoredBlockingJob for TestJob { + fn info(&self) -> &TaskInfo { + &self.0 + } + + fn run(&self) {} + } + + #[test] + fn blocking_slots_advance_and_retire_without_retaining_a_running_task() { + let _monitoring = crate::runtime::monitoring::TEST_LOCK.lock().unwrap(); + let runtime = seismograph_runtime::register_runtime(RuntimeMetadata::new("blocking-slots", 1)); + let queue = Arc::new(Queue::default()); + let index = queue.add_worker(); + queue.enable_monitoring(&runtime.handle()); + queue.enable_monitoring(&runtime.handle()); + let body: Arc = Arc::new(TestJob(TaskInfo::new::<()>(queue.monitoring.events().unwrap()))); + assert!(queue.push(QueuedJob { + body: BlockingJob::Monitored(Arc::clone(&body)), + shutdown: None, + })); + let mut worker = BlockingWorkerMonitoring::default(); + let job = queue.pop(index, &mut worker).unwrap(); + let active = runtime.handle().capture(CaptureOptions::default()).unwrap(); + + assert!(queue.push(QueuedJob { + body: BlockingJob::Plain(Box::new(|| {})), + shutdown: None, + })); + drop(job); + let next = queue.pop(index, &mut worker).unwrap(); + let finished = runtime.handle().capture(CaptureOptions::default()).unwrap(); + drop(next); + queue.stop(); + assert!(queue.pop(index, &mut worker).is_none()); + let stopped = runtime.handle().capture(CaptureOptions::default()).unwrap(); + queue.retire(index); + let retired = runtime.handle().capture(CaptureOptions::default()).unwrap(); + + assert_eq!(active.tasks.len(), 1); + assert_eq!(active.tasks[0].id, body.info().id()); + assert_eq!(active.tasks[0].activity.unwrap().state, TaskActivityState::Running); + assert!(finished.tasks.is_empty()); + assert!(stopped.tasks.is_empty()); + assert!(retired.tasks.is_empty()); + assert_eq!(retired.workers[0].state, WorkerState::Stopped); + } + + #[test] + fn dequeuing_or_stopping_releases_a_canceled_capture_outside_the_queue_lock() { + struct Capture(Arc, std::sync::mpsc::Sender); + + impl Drop for Capture { + fn drop(&mut self) { + self.1.send(self.0.state.try_lock().is_ok()).unwrap(); + } + } + + for stopped in [false, true] { + let _monitoring = crate::runtime::monitoring::TEST_LOCK.lock().unwrap(); + let runtime = seismograph_runtime::register_runtime(RuntimeMetadata::new("blocking-capture-drop", 1)); + let queue = Arc::new(Queue::default()); + let index = queue.add_worker(); + queue.enable_monitoring(&runtime.handle()); + let (dropped, received) = std::sync::mpsc::channel(); + let capture = Capture(Arc::clone(&queue), dropped); + let pool = crate::task::handle::JoinHandlePool::new(crate::runtime::shutdown::Shutdown::default()); + let (body, handle) = crate::task::preparation::prepare_blocking( + &pool, + move || drop(capture), + crate::runtime::worker::blocking::PanicPolicy::Ignore, + queue.monitoring.task::<()>(&crate::runtime::shutdown::Shutdown::default()), + ); + assert!(queue.push(QueuedJob { body, shutdown: None })); + let mut worker = BlockingWorkerMonitoring::default(); + drop(queue.pop(index, &mut worker).unwrap()); + + if stopped { + queue.stop(); + } else { + assert!(queue.push(QueuedJob { + body: BlockingJob::Plain(Box::new(|| {})), + shutdown: None, + })); + } + let next = queue.pop(index, &mut worker); + + assert_eq!(next.is_none(), stopped); + assert!(received.try_recv().unwrap()); + assert!(handle.join().unwrap_err().is_shutdown()); + } + } + + #[test] + fn an_idle_blocking_worker_releases_its_canceled_capture_before_waiting() { + struct Capture(Arc, std::sync::mpsc::Sender); + + impl Drop for Capture { + fn drop(&mut self) { + let mut state = self.0.state.try_lock().unwrap(); + let released = state.workers[0].current.is_none() && !state.workers[0].running; + state.stopped = true; + self.1.send(released).unwrap(); + } + } + + let _monitoring = crate::runtime::monitoring::TEST_LOCK.lock().unwrap(); + let runtime = seismograph_runtime::register_runtime(RuntimeMetadata::new("idle-blocking-drop", 1)); + let queue = Arc::new(Queue::default()); + let index = queue.add_worker(); + queue.enable_monitoring(&runtime.handle()); + let (dropped, received) = std::sync::mpsc::channel(); + let capture = Capture(Arc::clone(&queue), dropped); + let pool = crate::task::handle::JoinHandlePool::new(crate::runtime::shutdown::Shutdown::default()); + let (body, handle) = crate::task::preparation::prepare_blocking( + &pool, + move || drop(capture), + crate::runtime::worker::blocking::PanicPolicy::Ignore, + queue.monitoring.task::<()>(&crate::runtime::shutdown::Shutdown::default()), + ); + assert!(queue.push(QueuedJob { body, shutdown: None })); + let mut worker = BlockingWorkerMonitoring::default(); + drop(queue.pop(index, &mut worker).unwrap()); + + assert!(queue.pop(index, &mut worker).is_none()); + + assert!(received.try_recv().unwrap()); + assert!(handle.join().unwrap_err().is_shutdown()); + } + + #[test] + fn blocking_collection_handles_missing_metadata_limits_and_canceled_requests() { + let _monitoring = crate::runtime::monitoring::TEST_LOCK.lock().unwrap(); + let runtime = seismograph_runtime::register_runtime(RuntimeMetadata::new("blocking-limits", 1)); + let queue = Arc::new(Queue::default()); + queue.add_worker(); + queue.enable_monitoring(&runtime.handle()); + for body in [ + BlockingJob::Plain(Box::new(|| {})), + BlockingJob::Monitored(Arc::new(TestJob(TaskInfo::new::<()>(queue.monitoring.events().unwrap())))), + ] { + assert!(queue.push(QueuedJob { body, shutdown: None })); + } + let truncated = runtime + .handle() + .capture(CaptureOptions { + limit_per_worker: 1, + ..CaptureOptions::default() + }) + .unwrap(); + let complete = runtime.handle().capture(CaptureOptions::default()).unwrap(); + + let source = seismograph_runtime::register_runtime(RuntimeMetadata::new("canceled-blocking-request", 1)); + let worker = source.register_worker(WorkerMetadata::new(WorkerRole::Core)); + let (sender, receiver) = std::sync::mpsc::channel(); + worker + .handle() + .register_task_provider(move |request| sender.send(request).unwrap()) + .unwrap(); + source.handle().capture_with_timeout(CaptureOptions::default()).unwrap(); + queue.collect(0, receiver.try_recv().unwrap()); + + assert!(truncated.tasks.is_empty()); + assert_eq!(truncated.workers[0].task_collection, TaskCollection::Truncated); + assert_eq!(complete.tasks.len(), 1); + assert_eq!(complete.tasks[0].activity.unwrap().state, TaskActivityState::Ready); + assert_eq!(complete.workers[0].task_collection, TaskCollection::Complete); + } + + #[test] + fn a_dropped_blocking_queue_reports_collection_failure() { + let _monitoring = crate::runtime::monitoring::TEST_LOCK.lock().unwrap(); + let runtime = seismograph_runtime::register_runtime(RuntimeMetadata::new("dropped-blocking-queue", 1)); + let queue = Arc::new(Queue::default()); + let worker = queue.register_worker(&runtime.handle(), 0); + drop(queue); + + let snapshot = runtime.handle().capture(CaptureOptions::default()).unwrap(); + assert!(snapshot.tasks.is_empty()); + assert_eq!(snapshot.workers[0].task_collection, TaskCollection::Failed); + drop(worker); + } +} diff --git a/crates/arty/src/runtime/worker/blocking/worker.rs b/crates/arty/src/runtime/worker/blocking/worker.rs index 3873a06c37..7910584097 100644 --- a/crates/arty/src/runtime/worker/blocking/worker.rs +++ b/crates/arty/src/runtime/worker/blocking/worker.rs @@ -12,11 +12,13 @@ use performables::arc::Arc; use performables::sync::mutex::Mutex; use crate::runtime::config::RuntimeName; +#[cfg(test)] +use crate::runtime::monitoring::BlockingJob; +use crate::runtime::monitoring::{BlockingPoolMonitoring, BlockingWorkerMonitoring}; use crate::runtime::shutdown::{Shutdown, ShutdownState}; use crate::runtime::worker::blocking::queue::{Queue, QueuedJob}; use crate::runtime::worker::blocking::{LocalWorker, PanicPolicy, PoolIdentity}; use crate::task::handle::{JoinHandle, JoinHandlePool}; -#[cfg(not(feature = "monitor"))] use crate::task::preparation::prepare_blocking; const ERR_POISONED_LOCK: &str = "poisoned lock - cannot continue execution because security and privacy guarantees can no longer be upheld"; @@ -68,10 +70,8 @@ impl JoinableThreadPool { F: FnOnce() + Send + 'static, { self.enqueue(QueuedJob { - body: Box::new(job), + body: BlockingJob::Plain(Box::new(job)), shutdown: None, - #[cfg(feature = "monitor")] - info: None, }) } @@ -99,52 +99,20 @@ impl JoinableThreadPool { .name(name) .spawn(move || { attach_thread(identity); - #[cfg(feature = "monitor")] - let mut diagnostics = None; - while let Some(job) = receiver.pop( - index, - #[cfg(feature = "monitor")] - &mut diagnostics, - ) { + let mut diagnostics = BlockingWorkerMonitoring::default(); + while let Some(job) = receiver.pop(index, &mut diagnostics) { active_count.fetch_add(1, Ordering::SeqCst); queued_count.fetch_sub(1, Ordering::SeqCst); let canceled = job .shutdown .as_ref() .is_some_and(|shutdown| matches!(shutdown.state(), ShutdownState::Started)); - #[cfg(feature = "monitor")] - let observation = diagnostics.as_ref().filter(|_| !canceled).and_then(|worker| { - job.info.as_ref().map(|info| { - info.started_on(worker.task_events()); - let poll = worker.is_observed().then(|| worker.task_poll_started(info.id())); - (worker, poll) - }) - }); - let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - #[cfg(feature = "monitor")] - let _current = observation - .as_ref() - .and_then(|(_, poll)| poll.as_ref()) - .and(job.info.as_ref()) - .map(|info| crate::runtime::monitoring::enter_current(info.id())); - if canceled { - drop(job.body); - } else { - (job.body)(); - } - })); - #[cfg(feature = "monitor")] - if let Some((worker, Some(poll))) = observation { - worker.task_poll_finished(poll); - } - #[cfg(feature = "monitor")] - receiver.finish(index); + let outcome = diagnostics.run(job.body, canceled); active_count.fetch_sub(1, Ordering::SeqCst); if let Err(panic) = outcome { discard_task_panic_payload(panic); } } - #[cfg(feature = "monitor")] receiver.retire(index); }) .expect("blocking worker thread creation must succeed"); @@ -213,25 +181,16 @@ impl Worker { R: Send + 'static, { let runtime_shutdown = self.remote_results.shutdown().clone(); - #[cfg(feature = "monitor")] - let info = matches!(runtime_shutdown.state(), ShutdownState::Running) - .then(|| self.pool.task_info::()) - .flatten(); - #[cfg(feature = "monitor")] - let (task, join_handle) = - crate::task::preparation::prepare_blocking_with_info(&self.remote_results, body, self.panic_policy.clone(), info.clone()); - #[cfg(not(feature = "monitor"))] - let (task, join_handle) = prepare_blocking(&self.remote_results, body, self.panic_policy.clone()); + let monitoring = self.pool.monitoring.task::(&runtime_shutdown); + let (task, join_handle) = prepare_blocking(&self.remote_results, body, self.panic_policy.clone(), monitoring); if matches!(runtime_shutdown.state(), ShutdownState::Started) { drop(task); return join_handle; } let join_handle = join_handle.blocking_pool(Arc::clone(&self.pool.identity)); let job = QueuedJob { - body: Box::new(task), + body: task, shutdown: Some(runtime_shutdown), - #[cfg(feature = "monitor")] - info, }; if matches!(self.remote_results.shutdown().state(), ShutdownState::Running) && self.pool.enqueue(job) { self.pool.grow_if_overloaded(); @@ -256,8 +215,7 @@ pub(crate) struct Pool { inner: Arc>>, identity: PoolIdentity, max_thread_count: NonZeroUsize, - #[cfg(feature = "monitor")] - events: std::sync::Arc>, + monitoring: BlockingPoolMonitoring, } impl Pool { @@ -281,15 +239,13 @@ impl Pool { // Start with one thread and let the pool grow as needed. let identity = Arc::new(()); let thread_pool = JoinableThreadPool::new(name, async_worker_index, Self::INITIAL_THREAD_COUNT, Arc::clone(&identity)); - #[cfg(feature = "monitor")] - let events = std::sync::Arc::clone(&thread_pool.receiver.events); + let monitoring = thread_pool.receiver.monitoring.clone(); Self { inner: Arc::new(Mutex::new(Some(thread_pool))), identity, max_thread_count: max_thread_count.unwrap_or(Self::MAX_THREAD_COUNT_DEFAULT), - #[cfg(feature = "monitor")] - events, + monitoring, } } @@ -299,10 +255,8 @@ impl Pool { F: FnOnce() + Send + 'static, { self.enqueue(QueuedJob { - body: Box::new(f), + body: BlockingJob::Plain(Box::new(f)), shutdown: None, - #[cfg(feature = "monitor")] - info: None, }) } @@ -323,14 +277,6 @@ impl Pool { } } - #[cfg(feature = "monitor")] - fn task_info(&self) -> Option> { - self.events - .get() - .copied() - .map(|events| std::sync::Arc::new(crate::runtime::monitoring::TaskInfo::new::(events))) - } - fn join(&self) { // Do not keep the pool alive through retained scheduler handles, and do not // hold its submission lock while waiting for blocking functions to finish. diff --git a/crates/arty/src/task/context.rs b/crates/arty/src/task/context.rs index 69894bbf40..4767056d37 100644 --- a/crates/arty/src/task/context.rs +++ b/crates/arty/src/task/context.rs @@ -33,7 +33,7 @@ use crate::task::scheduler::Scheduler; /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] -/// # #[arty::main(monitor = disabled)] +/// # #[arty::main] /// # async fn main(cx: arty::task::Context) -> Result<(), arty::task::JoinError> { /// let answer = cx /// .scheduler() @@ -75,7 +75,7 @@ impl Context { /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] - /// # #[arty::main(monitor = disabled)] + /// # #[arty::main] /// # async fn main(cx: arty::task::Context) -> Result<(), arty::task::JoinError> { /// let parent_thread = cx.thread().id(); /// let child_thread = cx @@ -114,7 +114,7 @@ impl Context { /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] - /// # #[arty::main(monitor = disabled)] + /// # #[arty::main] /// # async fn main(cx: arty::task::Context) { /// assert_eq!(cx.thread().id(), std::thread::current().id()); /// # } @@ -137,7 +137,7 @@ impl Context { /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] - /// # #[arty::main(monitor = disabled)] + /// # #[arty::main] /// # async fn main(cx: arty::task::Context) { /// let duration = std::time::Duration::from_millis(1); /// let watch = cx.clock().stopwatch(); diff --git a/crates/arty/src/task/handle/error.rs b/crates/arty/src/task/handle/error.rs index 881861c028..38b03730f6 100644 --- a/crates/arty/src/task/handle/error.rs +++ b/crates/arty/src/task/handle/error.rs @@ -43,9 +43,7 @@ use crate::runtime::ShutdownCause; /// ``` /// use arty::runtime::Runtime; /// -/// let runtime = Runtime::builder() -/// .monitor(arty::runtime::MonitorPolicy::disabled()) -/// .build()?; +/// let runtime = Runtime::builder().build()?; /// let error = runtime.scheduler().block_on(async |cx| { /// cx.control().request_stop(); /// cx.scheduler() @@ -80,7 +78,7 @@ impl JoinError { /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] - /// # #[arty::main(panic = ignore, monitor = disabled)] + /// # #[arty::main(panic = ignore)] /// # async fn main(cx: arty::task::Context) -> Result<(), arty::task::JoinError> { /// let result: Result<(), arty::task::JoinError> = /// cx.scheduler().spawn(async |_| panic!("task failed")).await; @@ -102,9 +100,7 @@ impl JoinError { /// ``` /// use arty::runtime::Runtime; /// - /// let runtime = Runtime::builder() - /// .monitor(arty::runtime::MonitorPolicy::disabled()) - /// .build()?; + /// let runtime = Runtime::builder().build()?; /// let error = runtime.scheduler().block_on(async |cx| { /// cx.control().request_stop(); /// cx.scheduler() diff --git a/crates/arty/src/task/handle/join_handle.rs b/crates/arty/src/task/handle/join_handle.rs index 9dcea0fc51..d86ea483f3 100644 --- a/crates/arty/src/task/handle/join_handle.rs +++ b/crates/arty/src/task/handle/join_handle.rs @@ -46,7 +46,7 @@ use crate::runtime::worker::blocking::{LocalWorker as BlockingLocalWorker, PoolI /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] -/// # #[arty::main(monitor = disabled)] +/// # #[arty::main] /// # async fn main(cx: arty::task::Context) -> Result<(), arty::task::JoinError> { /// let task = cx.scheduler().spawn(async |_| 42); /// assert_eq!(task.await?, 42); diff --git a/crates/arty/src/task/mod.rs b/crates/arty/src/task/mod.rs index ccbaccf721..2e9eec0886 100644 --- a/crates/arty/src/task/mod.rs +++ b/crates/arty/src/task/mod.rs @@ -70,7 +70,7 @@ //! use std::rc::Rc; //! //! # #[cfg(all(feature = "macros", feature = "rt"))] -//! # #[arty::main(monitor = disabled)] +//! # #[arty::main] //! # async fn main( //! # cx: arty::task::Context, //! # ) -> Result<(), Box> { diff --git a/crates/arty/src/task/preparation/blocking.rs b/crates/arty/src/task/preparation/blocking.rs index 1838e4db19..7b70928dc3 100644 --- a/crates/arty/src/task/preparation/blocking.rs +++ b/crates/arty/src/task/preparation/blocking.rs @@ -6,49 +6,17 @@ use std::panic::{AssertUnwindSafe, catch_unwind}; use arty_executor::{ShutdownCause, TaskCompletion}; use crate::runtime::ShutdownStart; +use crate::runtime::monitoring::{BlockingJob, BlockingTaskEvents, BlockingTaskMonitoring}; use crate::runtime::worker::blocking::PanicPolicy; use crate::task::handle::{JoinHandle, JoinHandlePool, Sender}; use crate::task::preparation::discard_panicking_drop; -#[cfg(any(test, not(feature = "monitor")))] pub(crate) fn prepare_blocking( result_pool: &JoinHandlePool, body: F, panic_policy: PanicPolicy, -) -> (impl FnOnce() + Send + 'static, JoinHandle) -where - F: FnOnce() -> R + Send + 'static, - R: Send + 'static, -{ - prepare_blocking_inner( - result_pool, - body, - panic_policy, - #[cfg(feature = "monitor")] - None, - ) -} - -#[cfg(feature = "monitor")] -pub(crate) fn prepare_blocking_with_info( - result_pool: &JoinHandlePool, - body: F, - panic_policy: PanicPolicy, - info: Option>, -) -> (impl FnOnce() + Send + 'static, JoinHandle) -where - F: FnOnce() -> R + Send + 'static, - R: Send + 'static, -{ - prepare_blocking_inner(result_pool, body, panic_policy, info) -} - -fn prepare_blocking_inner( - result_pool: &JoinHandlePool, - body: F, - panic_policy: PanicPolicy, - #[cfg(feature = "monitor")] info: Option>, -) -> (impl FnOnce() + Send + 'static, JoinHandle) + monitoring: BlockingTaskMonitoring, +) -> (BlockingJob, JoinHandle) where F: FnOnce() -> R + Send + 'static, R: Send + 'static, @@ -57,17 +25,13 @@ where let task = BlockingTask { pending: Some((body, result_tx)), panic_policy, - #[cfg(feature = "monitor")] - info, }; - (move || task.run(), handle) + (monitoring.prepare(task), handle) } -struct BlockingTask { - pending: Option<(F, Sender)>, - panic_policy: PanicPolicy, - #[cfg(feature = "monitor")] - info: Option>, +pub(crate) struct BlockingTask { + pub(crate) pending: Option<(F, Sender)>, + pub(crate) panic_policy: PanicPolicy, } impl BlockingTask @@ -75,21 +39,15 @@ where F: FnOnce() -> R + Send + 'static, R: Send + 'static, { - fn run(mut self) { + pub(crate) fn run(mut self, monitoring: BlockingTaskEvents<'_>) { let (body, result_tx) = self.pending.take().expect("a blocking task is consumed exactly once"); match catch_unwind(AssertUnwindSafe(body)) { Ok(result) => { - #[cfg(feature = "monitor")] - if let Some(info) = &self.info { - info.completed(); - } + monitoring.completed(); result_tx.completed(result); } Err(panic) => { - #[cfg(feature = "monitor")] - if let Some(info) = &self.info { - info.panicked(); - } + monitoring.panicked(); if let PanicPolicy::Shutdown(waker) = &self.panic_policy && result_tx.shutdown().begin(ShutdownCause::panic(panic.as_ref())) == ShutdownStart::Started { @@ -106,10 +64,6 @@ impl Drop for BlockingTask { let Some((body, result_tx)) = self.pending.take() else { return; }; - #[cfg(feature = "monitor")] - if let Some(info) = &self.info { - info.canceled(); - } discard_panicking_drop(body); drop(result_tx); } diff --git a/crates/arty/src/task/preparation/mod.rs b/crates/arty/src/task/preparation/mod.rs index b75f135fba..7b17817176 100644 --- a/crates/arty/src/task/preparation/mod.rs +++ b/crates/arty/src/task/preparation/mod.rs @@ -11,17 +11,12 @@ mod cleanup; mod deferred_task; mod remote; -#[cfg(any(test, not(feature = "monitor")))] -pub(crate) use blocking::prepare_blocking; -#[cfg(feature = "monitor")] -pub(crate) use blocking::prepare_blocking_with_info; +pub(crate) use blocking::{BlockingTask, prepare_blocking}; pub(crate) use cleanup::discard_panicking_drop; #[cfg(test)] use deferred_task::DeferredTask; pub(crate) use deferred_task::DeferredTaskFactory; -#[cfg(test)] -pub(crate) use remote::prepare_remote; -pub(crate) use remote::{RemoteFutureFactory, prepare_remote_with_size}; +pub(crate) use remote::{RemoteFutureFactory, prepare_remote}; #[cfg(test)] #[cfg_attr(coverage_nightly, coverage(off))] // Test scaffolding is not runtime behavior. diff --git a/crates/arty/src/task/preparation/remote.rs b/crates/arty/src/task/preparation/remote.rs index 9709a16c20..02d443ff73 100644 --- a/crates/arty/src/task/preparation/remote.rs +++ b/crates/arty/src/task/preparation/remote.rs @@ -6,6 +6,7 @@ use std::cell::Cell; use arty_executor::{TaskPlacement, TaskSet}; use plurality::{Box as PoolBox, MultiPool, coerce}; +use crate::runtime::monitoring::TaskMonitoring; use crate::task::Context; use crate::task::handle::{JoinHandle, JoinHandlePool}; use crate::task::preparation::{DeferredTaskFactory, discard_panicking_drop}; @@ -88,25 +89,11 @@ impl RemoteFutureFactory { } } -#[cfg(test)] -pub(crate) fn prepare_remote( - result_pool: &JoinHandlePool, - future_factory: FF, - placement: TaskPlacement, -) -> (RemoteFutureFactory, JoinHandle) -where - FF: FnOnce(Context) -> F + Send + 'static, - F: Future + 'static, - R: Send + 'static, -{ - prepare_remote_with_size(result_pool, future_factory, placement, size_of::) -} - -pub(crate) fn prepare_remote_with_size( +pub(crate) fn prepare_remote( result_pool: &JoinHandlePool, future_factory: FF, placement: TaskPlacement, - body_size: S, + monitoring: TaskMonitoring, ) -> (RemoteFutureFactory, JoinHandle) where FF: FnOnce(Context) -> F + Send + 'static, @@ -115,18 +102,11 @@ where S: Fn() -> usize + Send + 'static, { let (handle, result_tx) = result_pool.handle(); - #[cfg(feature = "monitor")] - let parent = crate::runtime::monitoring::current_task(); - #[cfg(not(feature = "monitor"))] - let _ = body_size; let future_factory = TaskFactory::new(future_factory); let future_factory = RemoteFutureFactory::new(move |cx, tasks| { let future_factory = future_factory.into_inner(); let inner = DeferredTaskFactory::new(future_factory, cx); - #[cfg(feature = "monitor")] - crate::runtime::monitoring::add_task_with_metadata::(tasks, inner, placement, result_tx, parent, body_size); - #[cfg(not(feature = "monitor"))] - tasks.task(inner).placement(placement).register(result_tx); + monitoring.register(tasks, inner, placement, result_tx); }); (future_factory, handle) } diff --git a/crates/arty/src/task/preparation/tests.rs b/crates/arty/src/task/preparation/tests.rs index 76494fe247..e8c0969323 100644 --- a/crates/arty/src/task/preparation/tests.rs +++ b/crates/arty/src/task/preparation/tests.rs @@ -13,6 +13,7 @@ use arty_executor::{CycleOutcome, Executor, ShutdownCause, ShutdownError, TaskPl use futures::executor::block_on; use super::*; +use crate::runtime::monitoring::{BlockingTaskMonitoring, TaskMonitoring}; use crate::runtime::worker::blocking::PanicPolicy; use crate::runtime::{Runtime, Shutdown, WorkersPolicy}; use crate::task::Context; @@ -46,6 +47,7 @@ fn remote_factory_remains_deferred_until_poll() { } }, TaskPlacement::Any, + TaskMonitoring::capture(), ); let mut executor = Executor::builder().owner_waker(Waker::noop().clone()).build(); @@ -115,8 +117,8 @@ fn blocking_task_delivers_its_result() { order: Arc::clone(&order), expected: 0, })); - let (task, handle) = prepare_blocking(&result_pool, || 42, PanicPolicy::Shutdown(waker)); - task(); + let (task, handle) = prepare_blocking(&result_pool, || 42, PanicPolicy::Shutdown(waker), BlockingTaskMonitoring::default()); + task.run(); assert_eq!(block_on(handle).unwrap(), 42); assert!(result_pool.shutdown().cause().is_none()); @@ -126,8 +128,13 @@ fn blocking_task_delivers_its_result() { #[test] fn blocking_panic_is_delivered_to_joiner() { let result_pool = JoinHandlePool::new(Shutdown::default()); - let (task, handle) = prepare_blocking(&result_pool, || panic!("blocking task"), PanicPolicy::Ignore); - task(); + let (task, handle) = prepare_blocking( + &result_pool, + || panic!("blocking task"), + PanicPolicy::Ignore, + BlockingTaskMonitoring::default(), + ); + task.run(); assert!(block_on(handle).unwrap_err().is_panic()); assert!(result_pool.shutdown().cause().is_none()); @@ -151,7 +158,12 @@ fn blocking_panic_publishes_its_cause_before_worker_and_join_notifications() { order: Arc::clone(&order), expected: 0, })); - let (task, handle) = prepare_blocking(&result_pool, || panic_any(42_u32), PanicPolicy::Shutdown(worker_waker)); + let (task, handle) = prepare_blocking( + &result_pool, + || panic_any(42_u32), + PanicPolicy::Shutdown(worker_waker), + BlockingTaskMonitoring::default(), + ); let mut handle = pin!(handle); let worker_notifications = usize::from(existing.is_none()); let join_waker = Waker::from(Arc::new(ObserveShutdownWake { @@ -161,7 +173,7 @@ fn blocking_panic_publishes_its_cause_before_worker_and_join_notifications() { })); assert!(handle.as_mut().poll(&mut PollContext::from_waker(&join_waker)).is_pending()); - task(); + task.run(); assert_eq!(order.load(Ordering::Relaxed), worker_notifications + 1); let panic = block_on(handle).unwrap_err().into_panic().unwrap(); @@ -189,6 +201,7 @@ fn remote_factory_panic_is_delivered_to_joiner() { &result_pool, |_| -> std::future::Ready<()> { panic_any(Payload(42)) }, TaskPlacement::Any, + TaskMonitoring::capture(), ); let mut executor = Executor::builder().owner_waker(Waker::noop().clone()).build(); let (_runtime, cx) = test_context(); @@ -202,7 +215,7 @@ fn discarded_preparation_reports_the_shutdown_cause() { let shutdown = Shutdown::default(); shutdown.begin(arty_executor::ShutdownError::new(std::io::Error::other("blocking shutdown")).into()); let result_pool = JoinHandlePool::new(shutdown); - let (task, handle) = prepare_blocking(&result_pool, || 42, PanicPolicy::Ignore); + let (task, handle) = prepare_blocking(&result_pool, || 42, PanicPolicy::Ignore, BlockingTaskMonitoring::default()); drop(task); let mut handle = pin!(handle); let mut context = PollContext::from_waker(Waker::noop()); @@ -219,7 +232,7 @@ fn discarded_remote_factory_reports_the_shutdown_cause() { let shutdown = Shutdown::default(); shutdown.begin(arty_executor::ShutdownError::new(std::io::Error::other("queued shutdown")).into()); let result_pool = JoinHandlePool::new(shutdown); - let (factory, handle) = prepare_remote(&result_pool, |_| async { 42 }, TaskPlacement::Any); + let (factory, handle) = prepare_remote(&result_pool, |_| async { 42 }, TaskPlacement::Any, TaskMonitoring::capture()); drop(factory); @@ -240,7 +253,12 @@ fn discarded_blocking_task_contains_body_destructor_panic() { let capture = PanicOnDrop; let result_pool = JoinHandlePool::new(Shutdown::default()); - let (task, handle) = prepare_blocking(&result_pool, move || drop(capture), PanicPolicy::Ignore); + let (task, handle) = prepare_blocking( + &result_pool, + move || drop(capture), + PanicPolicy::Ignore, + BlockingTaskMonitoring::default(), + ); catch_unwind(AssertUnwindSafe(|| drop(task))).unwrap(); assert!(block_on(handle).unwrap_err().is_shutdown()); } @@ -248,7 +266,7 @@ fn discarded_blocking_task_contains_body_destructor_panic() { #[test] fn discarded_blocking_task_contains_join_waker_panic() { let result_pool = JoinHandlePool::new(Shutdown::default()); - let (task, handle) = prepare_blocking(&result_pool, || 42, PanicPolicy::Ignore); + let (task, handle) = prepare_blocking(&result_pool, || 42, PanicPolicy::Ignore, BlockingTaskMonitoring::default()); let mut handle = pin!(handle); let waker = Waker::from(Arc::new(PanicWake)); assert!(handle.as_mut().poll(&mut PollContext::from_waker(&waker)).is_pending()); diff --git a/crates/arty/src/task/runtime_scheduler.rs b/crates/arty/src/task/runtime_scheduler.rs index ad039f49bf..a81c5b08a7 100644 --- a/crates/arty/src/task/runtime_scheduler.rs +++ b/crates/arty/src/task/runtime_scheduler.rs @@ -105,9 +105,7 @@ impl RuntimeScheduler { /// ``` /// use arty::runtime::Runtime; /// - /// let runtime = Runtime::builder() - /// .monitor(arty::runtime::MonitorPolicy::disabled()) - /// .build()?; + /// let runtime = Runtime::builder().build()?; /// let mut message = String::from("Hello"); /// runtime /// .scheduler() @@ -137,10 +135,9 @@ impl RuntimeScheduler { // is destroyed. The final sender is dropped after those fields, including on // cancellation or panic. Receiving the result alone is not a destruction guarantee. let factory = unsafe { std::mem::transmute::, BoxedFutureFactory<'static, R>>(factory) }; - #[cfg(feature = "monitor")] - let task = self.dispatcher.spawn_with_size(factory, size_of::()); - #[cfg(not(feature = "monitor"))] - let task = self.dispatcher.spawn(factory); + let task = self + .dispatcher + .spawn(factory, crate::runtime::monitoring::TaskMonitoring::with_body_size(size_of::())); futures::executor::block_on(task).map_err(Error::new) } @@ -166,9 +163,7 @@ impl RuntimeScheduler { /// ``` /// use arty::runtime::Runtime; /// - /// let runtime = Runtime::builder() - /// .monitor(arty::runtime::MonitorPolicy::disabled()) - /// .build()?; + /// let runtime = Runtime::builder().build()?; /// let task = runtime /// .scheduler() /// .spawn_anywhere(String::from("Arty"), |_, name| async move { name.len() }); @@ -183,9 +178,7 @@ impl RuntimeScheduler { /// use arty::runtime::Runtime; /// /// struct SendOnly(u32); - /// let runtime = Runtime::builder() - /// .monitor(arty::runtime::MonitorPolicy::disabled()) - /// .build()?; + /// let runtime = Runtime::builder().build()?; /// let task = runtime /// .scheduler() /// .spawn_anywhere((), |_, ()| async { SendOnly(42) }); @@ -200,11 +193,14 @@ impl RuntimeScheduler { F: Future + 'static, R: Send + 'static, { - self.dispatcher.spawn(move |cx| { - let mut data = data; - data.relocate(None, cx.thread()); - future_factory(cx, data) - }) + self.dispatcher.spawn( + move |cx| { + let mut data = data; + data.relocate(None, cx.thread()); + future_factory(cx, data) + }, + crate::runtime::monitoring::TaskMonitoring::capture(), + ) } /// Runs a function on a thread from a blocking pool. @@ -227,9 +223,7 @@ impl RuntimeScheduler { /// ``` /// use arty::runtime::Runtime; /// - /// let runtime = Runtime::builder() - /// .monitor(arty::runtime::MonitorPolicy::disabled()) - /// .build()?; + /// let runtime = Runtime::builder().build()?; /// let task = runtime.scheduler().spawn_blocking(|| 6 * 7); /// assert_eq!(futures::executor::block_on(task)?, 42); /// # Ok::<(), Box>(()) diff --git a/crates/arty/src/task/scheduler.rs b/crates/arty/src/task/scheduler.rs index 81857535e7..11eef4a5a0 100644 --- a/crates/arty/src/task/scheduler.rs +++ b/crates/arty/src/task/scheduler.rs @@ -38,7 +38,7 @@ use crate::task::handle::JoinHandle; /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] -/// # #[arty::main(monitor = disabled)] +/// # #[arty::main] /// # async fn main(cx: arty::task::Context) -> Result<(), arty::task::JoinError> { /// let scheduler = cx.scheduler().clone(); /// let task = scheduler.spawn(async |_| 42); @@ -112,7 +112,7 @@ impl Scheduler { /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] - /// # #[arty::main(monitor = disabled)] + /// # #[arty::main] /// # async fn main(cx: arty::task::Context) -> Result<(), arty::task::JoinError> { /// use std::rc::Rc; /// use std::time::Duration; @@ -186,7 +186,7 @@ impl Scheduler { /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] - /// # #[arty::main(monitor = disabled)] + /// # #[arty::main] /// # async fn main(cx: arty::task::Context) -> Result<(), arty::task::JoinError> { /// let answer = cx /// .scheduler() @@ -243,7 +243,7 @@ impl Scheduler { /// /// ``` /// # #[cfg(all(feature = "macros", feature = "rt"))] - /// # #[arty::main(monitor = disabled)] + /// # #[arty::main] /// # async fn main( /// # cx: arty::task::Context, /// # ) -> Result<(), Box> { From 83a65e2a7ef4683a68498263dbaeb6345cb89838 Mon Sep 17 00:00:00 2001 From: Ralf Biedert Date: Mon, 16 Nov 2026 18:42:05 +0100 Subject: [PATCH 12/12] test(arty): isolate enabled monitor macro execution Run the enabled test macro with private mocked monitor I/O. Preserve integration compile coverage without opening native monitor endpoints. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- crates/arty/src/lib.rs | 3 +++ crates/arty/src/runtime/monitoring/tests.rs | 8 ++++++++ crates/arty/tests/test_macro.rs | 9 --------- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/crates/arty/src/lib.rs b/crates/arty/src/lib.rs index d4ff8205f1..7b9df475ce 100644 --- a/crates/arty/src/lib.rs +++ b/crates/arty/src/lib.rs @@ -79,6 +79,9 @@ //! [`ClockControl`](crate::time::ClockControl). //! Enable it in dev-dependencies, not production dependencies. +#[cfg(all(test, feature = "macros", feature = "monitor", not(miri)))] +extern crate self as arty; + use arty_io_core as _; /// Runs an async entry point on an Arty runtime. /// diff --git a/crates/arty/src/runtime/monitoring/tests.rs b/crates/arty/src/runtime/monitoring/tests.rs index e73d6a57aa..0d602387e1 100644 --- a/crates/arty/src/runtime/monitoring/tests.rs +++ b/crates/arty/src/runtime/monitoring/tests.rs @@ -107,6 +107,14 @@ fn explicit_monitor_policies_preserve_runtime_behavior() { } } +#[cfg(feature = "macros")] +#[arty_macros::test(monitor = enabled)] +async fn test_macro_can_enable_monitoring(cx: crate::task::Context) { + let _recording = RecorderGuard::new(); + assert!(super::task::current_events().is_some()); + assert_eq!(cx.scheduler().spawn(async |_| 42).await.unwrap(), 42); +} + #[test] fn disabled_monitoring_registers_no_runtime() { let _recording = RecorderGuard::new(); diff --git a/crates/arty/tests/test_macro.rs b/crates/arty/tests/test_macro.rs index ff4a303254..e17923a4b9 100644 --- a/crates/arty/tests/test_macro.rs +++ b/crates/arty/tests/test_macro.rs @@ -107,15 +107,6 @@ async fn default_test_runtime_has_exactly_one_worker(cx: Context) { } } -#[cfg_attr( - all(miri, feature = "monitor"), - ignore = "native monitor uses OS I/O; native and cargo careful tests cover macro policy wiring" -)] -#[test(monitor = enabled)] -async fn test_can_enable_monitoring(cx: Context) { - assert_eq!(cx.scheduler().spawn(async |_| 42).await.unwrap(), 42); -} - #[test(monitor = disabled, workers = 1)] async fn test_can_disable_monitoring_explicitly(cx: Context) { assert_eq!(cx.scheduler().spawn(async |_| 42).await.unwrap(), 42);