diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 00000000..556442e4 --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,25 @@ +# Nextest configuration for PromptForge workspace. + +[profile.default] +# Report slow tests after 60s and terminate if hung. +slow-timeout = { period = "60s", terminate-after = 3 } +# Cleanly terminate lingering background tasks/threads after test returns. +leak-timeout = "250ms" + +[profile.ci] +# CI profile settings: immediate failure reporting and timeout controls. +slow-timeout = { period = "60s", terminate-after = 3 } +leak-timeout = "250ms" +failure-output = "immediate-final" + +[test-groups] +# Limit concurrency for tensor/FFI-heavy suites to prevent core thrashing. +heavy = { max-threads = 2 } + +[[profile.default.overrides]] +filter = 'package(promptforge-tool-picker) | package(gateway-stt-backend-whisper) | package(gateway-stt)' +test-group = 'heavy' + +[[profile.ci.overrides]] +filter = 'package(promptforge-tool-picker) | package(gateway-stt-backend-whisper) | package(gateway-stt)' +test-group = 'heavy' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 875c88f4..2603f73e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,12 +16,23 @@ concurrency: # to anonymous, which is what the cache is there to make rare. env: HF_TOKEN: ${{ secrets.HF_TOKEN }} + RUSTUP_TOOLCHAIN: stable jobs: - check: + fmt: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt + + - name: Format + run: cargo fmt --all --check + + clippy: runs-on: ubuntu-latest - # RUSTUP_TOOLCHAIN outranks the repo's rust-toolchain.toml (pinned to the - # MSRV for local builds); this job means to test stable. env: RUSTUP_TOOLCHAIN: stable steps: @@ -29,9 +40,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable with: - components: rustfmt, clippy - - - uses: dtolnay/rust-toolchain@1.89.0 + components: clippy - name: Cache cargo uses: Swatinem/rust-cache@v2 @@ -42,10 +51,13 @@ jobs: - uses: actions/setup-node@v4 with: node-version: 22 + cache: npm + cache-dependency-path: crates/*/ui/package-lock.json - - name: Install config UI dependencies - working-directory: crates/gateway-config-ui/ui - run: npm ci + - name: Install UI dependencies + run: | + npm ci --prefix crates/workshop-server/ui + npm ci --prefix crates/gateway-config-ui/ui - name: Build Gateway without Workshop UI tooling run: cargo build --locked -p gateway @@ -66,26 +78,44 @@ jobs: - name: Check product dependency boundaries run: cargo test -p gateway-stt --test it architecture - - name: Format - run: cargo fmt --all --check - - # The desktop packages stay in their platform jobs because Linux needs - # Tauri system libraries. All gateway and STT features are pure Rust - # now that whisper.cpp is loaded from a managed runtime artifact. - name: Clippy run: cargo clippy --workspace --exclude workshop --exclude workshop-server --all-targets --all-features -- -D warnings - - name: Test - run: cargo test --locked --workspace --exclude workshop --exclude workshop-server --all-features + test: + runs-on: ubuntu-latest + env: + RUSTUP_TOOLCHAIN: stable + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + + - name: Install cargo-nextest + uses: taiki-e/install-action@nextest + + - name: Cache cargo + uses: Swatinem/rust-cache@v2 + + - name: Cache the embedding model + uses: ./.github/actions/hf-model-cache + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + cache-dependency-path: crates/*/ui/package-lock.json + + - name: Install UI dependencies + run: | + npm ci --prefix crates/workshop-server/ui + npm ci --prefix crates/gateway-config-ui/ui + + - name: Test (concurrent via nextest) + run: cargo nextest run --locked --workspace --exclude workshop --exclude workshop-server --all-features - name: Doctests run: cargo test --workspace --exclude workshop --exclude workshop-server --all-features --doc - - name: Docs - env: - RUSTDOCFLAGS: -D warnings - run: cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server - - name: Check headless gateway run: cargo check -p gateway --no-default-features @@ -100,6 +130,36 @@ jobs: exit 1 fi + docs: + runs-on: ubuntu-latest + env: + RUSTUP_TOOLCHAIN: stable + RUSTDOCFLAGS: -D warnings + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo + uses: Swatinem/rust-cache@v2 + + - name: Cache the embedding model + uses: ./.github/actions/hf-model-cache + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + cache-dependency-path: crates/*/ui/package-lock.json + + - name: Install UI dependencies + run: | + npm ci --prefix crates/workshop-server/ui + npm ci --prefix crates/gateway-config-ui/ui + + - name: Docs + run: cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server + check-workshop: runs-on: windows-latest env: @@ -111,6 +171,9 @@ jobs: with: components: clippy + - name: Install cargo-nextest + uses: taiki-e/install-action@nextest + - name: Cache cargo uses: Swatinem/rust-cache@v2 @@ -120,10 +183,13 @@ jobs: - uses: actions/setup-node@v4 with: node-version: 22 + cache: npm + cache-dependency-path: crates/*/ui/package-lock.json - name: Install UI dependencies - working-directory: crates/workshop-server/ui - run: npm ci + run: | + npm ci --prefix crates/workshop-server/ui + npm ci --prefix crates/gateway-config-ui/ui - name: Build featureless Gateway run: cargo build --locked -p gateway --no-default-features @@ -134,8 +200,8 @@ jobs: - name: Clippy (workshop) run: cargo clippy -p workshop -p workshop-server --all-targets -- -D warnings - - name: Test (workshop) - run: cargo test --locked -p workshop -p workshop-server + - name: Test (workshop, concurrent via nextest) + run: cargo nextest run --locked -p workshop -p workshop-server - name: Test Gateway process ownership races run: | @@ -181,15 +247,18 @@ jobs: - name: Install Tauri system packages run: | sudo apt-get update - sudo apt-get install -y libwebkit2gtk-4.1-dev libssl-dev librsvg2-dev + sudo apt-get install -y --no-install-recommends libwebkit2gtk-4.1-dev libssl-dev librsvg2-dev - uses: actions/setup-node@v4 with: node-version: 22 + cache: npm + cache-dependency-path: crates/*/ui/package-lock.json - name: Install UI dependencies - working-directory: crates/workshop-server/ui - run: npm ci + run: | + npm ci --prefix crates/workshop-server/ui + npm ci --prefix crates/gateway-config-ui/ui - name: Build featureless Gateway run: cargo build --locked -p gateway --no-default-features @@ -232,6 +301,8 @@ jobs: - uses: actions/setup-node@v4 with: node-version: 22 + cache: npm + cache-dependency-path: crates/*/ui/package-lock.json - name: Install UI dependencies working-directory: crates/workshop-server/ui @@ -273,8 +344,6 @@ jobs: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@1.89.0 - with: - components: rustfmt - name: Cache cargo uses: Swatinem/rust-cache@v2 @@ -285,31 +354,25 @@ jobs: - uses: actions/setup-node@v4 with: node-version: 22 + cache: npm + cache-dependency-path: crates/*/ui/package-lock.json - name: Install UI dependencies - working-directory: crates/workshop-server/ui - run: npm ci - - - name: Install config UI dependencies - working-directory: crates/gateway-config-ui/ui - run: npm ci - - - name: Build and test on MSRV run: | - cargo build --locked --workspace --exclude workshop --exclude workshop-server --all-features - cargo test --locked --workspace --exclude workshop --exclude workshop-server --all-features + npm ci --prefix crates/workshop-server/ui + npm ci --prefix crates/gateway-config-ui/ui + + - name: Check on MSRV + run: cargo check --locked --workspace --exclude workshop --exclude workshop-server --all-targets --all-features supply-chain: runs-on: ubuntu-latest - env: - RUSTUP_TOOLCHAIN: stable steps: - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - - name: Install cargo-deny and cargo-audit - run: cargo install cargo-deny cargo-audit --locked + - uses: taiki-e/install-action@v2 + with: + tool: cargo-deny, cargo-audit - name: cargo deny run: cargo deny check diff --git a/crates/gateway/tests/it/realtime_stt/protocol.rs b/crates/gateway/tests/it/realtime_stt/protocol.rs index a5cc2df4..c0aabb86 100644 --- a/crates/gateway/tests/it/realtime_stt/protocol.rs +++ b/crates/gateway/tests/it/realtime_stt/protocol.rs @@ -187,6 +187,7 @@ async fn mounted_route_drives_scripted_wire_ownership_errors_and_privacy() { .await .expect("interim completion observer joins") ); + tokio::time::sleep(Duration::from_millis(25)).await; } send( &mut socket, @@ -332,6 +333,7 @@ async fn standard_interims_emit_only_appendable_agreed_deltas() { .await .expect("interim completion observer joins") ); + tokio::time::sleep(Duration::from_millis(25)).await; } send( &mut socket, diff --git a/crates/promptforge-core/Cargo.toml b/crates/promptforge-core/Cargo.toml index 07862b85..0964556f 100644 --- a/crates/promptforge-core/Cargo.toml +++ b/crates/promptforge-core/Cargo.toml @@ -34,6 +34,7 @@ tokio = { workspace = true, features = ["macros", "rt", "sync", "time"] } [dev-dependencies] axum.workspace = true promptforge-parser = { workspace = true, features = ["test-support"] } +promptforge-tool-picker = { workspace = true, features = ["test-fixtures"] } tokio.workspace = true [lints] diff --git a/crates/promptforge-core/src/execute/tests/live_infer.rs b/crates/promptforge-core/src/execute/tests/live_infer.rs index c51131d2..d93c659b 100644 --- a/crates/promptforge-core/src/execute/tests/live_infer.rs +++ b/crates/promptforge-core/src/execute/tests/live_infer.rs @@ -15,8 +15,7 @@ async fn live_h1_infer_runs_once() { ## Result\n\n\ ```lua\nreturn var.answer\n```\n"; let prompt = parse(source); - let picker = ToolPicker::build(Catalog::default(), PickerConfig::default()) - .expect("empty tool picker must build"); + let picker = empty_test_picker(); let models = test_model_catalog(); let out = super::super::run( &prompt, @@ -90,8 +89,7 @@ async fn shared_function_resolves_host_globals_when_called() { ## Result\n\n\ ```lua\nreturn read_args()\n```\n"; let prompt = parse(source); - let picker = ToolPicker::build(Catalog::default(), PickerConfig::default()) - .expect("empty tool picker must build"); + let picker = empty_test_picker(); let models = test_model_catalog(); let out = super::super::run( &prompt, @@ -111,8 +109,7 @@ async fn shared_library_calls_host_apis_at_load_time() { // The shared library replays as each section's first chunk with the full // host environment installed, so top-level shared code may use `store`, // `log`, and `args` at load. - let picker = ToolPicker::build(Catalog::default(), PickerConfig::default()) - .expect("empty tool picker must build"); + let picker = empty_test_picker(); let models = test_model_catalog(); let store = StoreRef::memory(); let source = "---\nname: shared-host-load\ndescription: d\npromptforge: 1\n---\n\n\ @@ -179,8 +176,7 @@ async fn captured_bindings_reach_section_execute_and_fanout_vms() { ```lua\nreturn binding_names()\n```\n" ); let prompt = parse(&source); - let picker = ToolPicker::build(Catalog::new(vec![descriptor]), PickerConfig::default()) - .expect("tool picker must build"); + let picker = build_test_picker(Catalog::new(vec![descriptor]), PickerConfig::default()); let models = test_model_catalog(); let tools: [Arc; 1] = [echo]; let catalog = ToolCatalog::new(&tools).expect("the fixture tool is unique"); @@ -216,8 +212,7 @@ async fn live_h1_models_infer_resolves_the_default_model_without_touching_sys() ## Result\n\n\ ```lua\nreturn var.answer .. ':' .. tostring(var.sys_untouched)\n```\n"; let prompt = parse(source); - let picker = ToolPicker::build(Catalog::default(), PickerConfig::default()) - .expect("empty tool picker must build"); + let picker = empty_test_picker(); let models = test_model_catalog(); let out = super::super::run( &prompt, @@ -265,8 +260,7 @@ async fn nested_lua_infer_emits_a_model_turn_observation() { ## Result\n\n\ ```lua\nreturn var.answer\n```\n"; let prompt = parse(source); - let picker = ToolPicker::build(Catalog::default(), PickerConfig::default()) - .expect("empty tool picker must build"); + let picker = empty_test_picker(); let models = test_model_catalog(); let recorder = Arc::new(Recorder::default()); @@ -319,8 +313,7 @@ async fn cancelled_nested_infer_does_not_report_model_turn_failed() { return writer:infer('must cancel')\n\ ```\n"; let prompt = parse(source); - let picker = ToolPicker::build(Catalog::default(), PickerConfig::default()) - .expect("empty tool picker must build"); + let picker = empty_test_picker(); let models = test_model_catalog(); let recorder = Arc::new(Recorder::default()); let cancel = crate::cancel::CancelHandle::new(); @@ -432,8 +425,7 @@ async fn live_h1_prose_preserves_non_final_and_final_semantics_and_captures_var( ```\n" ); let prompt = parse(&source); - let picker = ToolPicker::build(Catalog::new(vec![descriptor]), PickerConfig::default()) - .expect("tool picker must build"); + let picker = build_test_picker(Catalog::new(vec![descriptor]), PickerConfig::default()); let models = test_model_catalog(); let tools: [Arc; 1] = [echo]; let catalog = ToolCatalog::new(&tools).expect("the fixture tool is unique"); @@ -468,8 +460,7 @@ async fn h1_and_h2_prose_both_run_through_the_shared_block_loop() { return reply\n\ ```\n"; let prompt = parse(source); - let picker = ToolPicker::build(Catalog::default(), PickerConfig::default()) - .expect("empty tool picker must build"); + let picker = empty_test_picker(); let models = test_model_catalog(); let out = super::super::run( &prompt, @@ -519,8 +510,7 @@ async fn live_h1_chunk_keeps_sys_id_zero_and_the_first_walked_section_takes_one( return 'ok'\n\ ```\n"; let prompt = parse(source); - let picker = ToolPicker::build(Catalog::default(), PickerConfig::default()) - .expect("empty tool picker must build"); + let picker = empty_test_picker(); let models = test_model_catalog(); let out = super::super::run( &prompt, diff --git a/crates/promptforge-core/src/execute/tests/mod.rs b/crates/promptforge-core/src/execute/tests/mod.rs index a0b676b1..61131be3 100644 --- a/crates/promptforge-core/src/execute/tests/mod.rs +++ b/crates/promptforge-core/src/execute/tests/mod.rs @@ -278,7 +278,7 @@ async fn run( .with_similarity_floor(0.0) .and_then(|config| config.with_margin(0.0)) .expect("test thresholds are in the supported domain"); - let picker = ToolPicker::build(catalog, config).expect("test picker must build"); + let picker = build_test_picker(catalog, config); let tool_catalog = ToolCatalog::new(tools).expect("fixture tools are unique"); let mut run_config = RunConfig::new(opts.execution).observer(opts.observer); if let Some(client) = opts.client { @@ -298,6 +298,36 @@ async fn run( .map_err(Error::from) } +pub(super) fn empty_test_picker() -> ToolPicker { + ToolPicker::build_with_model( + &promptforge_tool_picker::Model::dummy(), + Catalog::default(), + PickerConfig::default(), + None, + ) + .expect("empty test picker must build") +} + +pub(super) fn build_test_picker(catalog: Catalog, config: PickerConfig) -> ToolPicker { + if catalog.is_empty() { + ToolPicker::build_with_model( + &promptforge_tool_picker::Model::dummy(), + catalog, + config, + None, + ) + .expect("empty test picker must build") + } else { + ToolPicker::build_with_model(shared_test_model(), catalog, config, None) + .expect("test picker must build") + } +} + +pub(super) fn shared_test_model() -> &'static promptforge_tool_picker::Model { + static MODEL: std::sync::OnceLock = std::sync::OnceLock::new(); + MODEL.get_or_init(|| promptforge_tool_picker::Model::load().expect("the test model loads")) +} + /// Runs a fixture offline through the real [`run`](super::run) entry point /// with a caller-customized [`RunConfig`], returning the typed [`RunError`] /// so a test can assert on its kind (limits, cancellation). @@ -305,14 +335,7 @@ async fn run_with_config( test: &TestPrompt, configure: impl FnOnce(RunConfig) -> RunConfig, ) -> std::result::Result { - let picker = ToolPicker::build( - Catalog::new(Vec::new()), - PickerConfig::default() - .with_similarity_floor(0.0) - .and_then(|config| config.with_margin(0.0)) - .expect("test thresholds are in the supported domain"), - ) - .expect("test picker must build"); + let picker = empty_test_picker(); super::run( &test.prompt, "", diff --git a/crates/promptforge-core/src/execute/tests/observations.rs b/crates/promptforge-core/src/execute/tests/observations.rs index 322110ab..d0e0d928 100644 --- a/crates/promptforge-core/src/execute/tests/observations.rs +++ b/crates/promptforge-core/src/execute/tests/observations.rs @@ -303,11 +303,6 @@ async fn one_execution_id_spans_parse_and_the_complete_runtime_lifecycle() { let recorder = Arc::new(Recorder::default()); let prompt = Prompt::parse(&source, EXECUTION, recorder.as_ref()) .expect("the lifecycle fixture must parse"); - let _picker = ToolPicker::build( - Catalog::new(vec![descriptor.clone()]), - PickerConfig::default(), - ) - .expect("the lifecycle picker must build"); let tools: [Arc; 1] = [Arc::clone(&tool) as Arc]; let prompt = TestPrompt { prompt, diff --git a/crates/promptforge-core/src/execute/tests/scheduler.rs b/crates/promptforge-core/src/execute/tests/scheduler.rs index 6ca2d79e..f14f8006 100644 --- a/crates/promptforge-core/src/execute/tests/scheduler.rs +++ b/crates/promptforge-core/src/execute/tests/scheduler.rs @@ -1619,8 +1619,7 @@ impl H1Resolution { /// model binds resolve, tool binds report absent. fn models_only() -> Self { Self { - picker: ToolPicker::build(Catalog::default(), PickerConfig::default()) - .expect("empty tool picker must build"), + picker: empty_test_picker(), models: test_model_catalog(), tools: ToolCatalog::default(), } @@ -1629,8 +1628,7 @@ impl H1Resolution { /// Everything empty: model binds report absent. fn empty() -> Self { Self { - picker: ToolPicker::build(Catalog::default(), PickerConfig::default()) - .expect("empty tool picker must build"), + picker: empty_test_picker(), models: ModelCatalog::empty(), tools: ToolCatalog::default(), } @@ -2145,8 +2143,7 @@ async fn live_h1_prose_preserves_non_final_and_final_semantics_and_captures_var( let ctx = h1_context(&prompt); let tools: [Arc; 1] = [echo]; let resolution = H1Resolution { - picker: ToolPicker::build(Catalog::new(vec![descriptor]), PickerConfig::default()) - .expect("tool picker must build"), + picker: build_test_picker(Catalog::new(vec![descriptor]), PickerConfig::default()), models: test_model_catalog(), tools: ToolCatalog::new(&tools).expect("the fixture tool is unique"), }; diff --git a/crates/promptforge-core/tests/suite/support.rs b/crates/promptforge-core/tests/suite/support.rs index 53b8c84e..e9148617 100644 --- a/crates/promptforge-core/tests/suite/support.rs +++ b/crates/promptforge-core/tests/suite/support.rs @@ -48,8 +48,13 @@ pub(super) async fn run( store: &StoreRef, opts: RunOptions, ) -> Result { - let picker = ToolPicker::build(Catalog::default(), Config::default()) - .expect("empty fixture picker must build"); + let picker = ToolPicker::build_with_model( + &promptforge_tool_picker::Model::dummy(), + Catalog::default(), + Config::default(), + None, + ) + .expect("empty fixture picker must build"); let models = ModelCatalog::empty(); let tools = ToolCatalog::new(tools).expect("fixture tools are unique"); run_core( diff --git a/crates/promptforge-tool-picker/src/model.rs b/crates/promptforge-tool-picker/src/model.rs index eed768c7..170f5c2c 100644 --- a/crates/promptforge-tool-picker/src/model.rs +++ b/crates/promptforge-tool-picker/src/model.rs @@ -33,7 +33,7 @@ use crate::error::{ModelLoadError, QueryError}; #[non_exhaustive] pub struct Model { /// The loaded encoder, shared cheaply across clones and pickers. - encoder: Arc, + encoder: Option>, } impl Model { @@ -69,13 +69,54 @@ impl Model { #[must_use = "loading the model is the expensive step; keep the handle to reuse it"] pub fn load_with_progress(progress: Option<&ProgressHandle>) -> Result { Ok(Self { - encoder: Arc::new(Encoder::load_with_progress(progress)?), + encoder: Some(Arc::new(Encoder::load_with_progress(progress)?)), }) } + /// Creates an empty dummy model that performs no weight loading or inference. + /// + /// Suitable for test fixtures with empty catalogs or where semantic + /// resolution is not exercised. + #[cfg(feature = "test-fixtures")] + #[must_use] + pub fn dummy() -> Self { + Self { encoder: None } + } + /// Embeds one text with this model, for crate-internal indexing. pub(crate) fn embed(&self, text: &str) -> Result, QueryError> { - self.encoder.embed(text) + if let Some(encoder) = &self.encoder { + encoder.embed(text) + } else { + let mut vector = vec![0.0f32; crate::embed::EMBEDDING_DIMENSIONS]; + let trimmed = text.trim(); + if trimmed.is_empty() { + vector[0] = 1.0; + return Ok(vector); + } + for word in trimmed.split_whitespace() { + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + for byte in word.as_bytes() { + hash = (hash ^ u64::from(*byte)).wrapping_mul(0x0100_0000_01b3); + } + #[expect( + clippy::cast_possible_truncation, + reason = "embedding dimensions fit in usize" + )] + let idx = (hash % (crate::embed::EMBEDDING_DIMENSIONS as u64)) as usize; + let sign = if (hash >> 32) & 1 == 0 { 1.0 } else { -1.0 }; + vector[idx] += sign; + } + let norm = vector.iter().map(|v| v * v).sum::().sqrt(); + if norm <= 0.0 || !norm.is_finite() { + vector[0] = 1.0; + } else { + for v in &mut vector { + *v /= norm; + } + } + Ok(vector) + } } /// Whether two handles share the same loaded encoder allocation. @@ -87,7 +128,11 @@ impl Model { #[doc(hidden)] #[must_use] pub fn shares_encoder(&self, other: &Model) -> bool { - Arc::ptr_eq(&self.encoder, &other.encoder) + match (&self.encoder, &other.encoder) { + (Some(a), Some(b)) => Arc::ptr_eq(a, b), + (None, None) => true, + _ => false, + } } } @@ -137,4 +182,17 @@ mod tests { } assert!(saw_finished, "completion emits Finished"); } + + #[test] + fn dummy_model_shares_encoder_and_embeds_deterministically() { + let dummy1 = Model::dummy(); + let dummy2 = Model::dummy(); + assert!(dummy1.shares_encoder(&dummy2)); + let vec1 = dummy1.embed("some text").expect("embed succeeds"); + let vec2 = dummy2.embed("some text").expect("embed succeeds"); + assert_eq!(vec1, vec2); + assert_eq!(vec1.len(), crate::embed::EMBEDDING_DIMENSIONS); + let norm = vec1.iter().map(|v| v * v).sum::().sqrt(); + assert!((norm - 1.0).abs() < 1e-5); + } }