Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 93 additions & 4 deletions crates/app/peritus-daemon/src/product_run/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,10 @@ use support::{
CORRECT, ScriptedProvider, clean_review, complete_writer, repository, scripted, stalled,
};

// This integration test waits for real Cargo gates on its generated fixture; this is an
// observation watchdog for test completion, not a product-run deadline.
const PRODUCT_RUN_TERMINAL_WAIT: Duration = Duration::from_mins(2);

#[test]
fn candidate_retry_resumes_review_without_repeating_design_or_writing() {
tokio::runtime::Builder::new_current_thread()
Expand Down Expand Up @@ -139,6 +143,62 @@ async fn candidate_cancellation_scenario() {
service.shutdown(Duration::from_secs(5)).await;
}

#[test]
fn terminal_wait_timeout_preserves_the_latest_nonterminal_snapshot() {
interaction::block_on(async {
let repository = repository();
let state = tempfile::tempdir().expect("state");
let writer = stalled(0x86, "writer");
let reviewer = scripted(0x87, "reviewer", Vec::new());
let fixer = scripted(0x88, "fixer", Vec::new());
let workspace_id = WorkspaceId::new([0x89; 16]).expect("workspace");
let run_id = RunId::new([0x8a; 16]).expect("run");
let service =
service(state.path(), repository.path(), workspace_id, [&writer, &reviewer, &fixer]);
service
.start(
ProductRunRequest::new(
run_id,
workspace_id,
ProductProviderSelection::new(
writer.profile.profile_id(),
reviewer.profile.profile_id(),
fixer.profile.profile_id(),
),
"Add a tested answer function that returns 42.".to_owned(),
)
.expect("request"),
)
.await
.expect("start run");
tokio::time::timeout(Duration::from_secs(5), writer.wait_for_stalled_response())
.await
.expect("writer did not start its deliberately stalled response");

let snapshot = tokio::time::timeout(
Duration::from_secs(5),
wait_for_terminal_with_timeout(&service, run_id, Duration::from_millis(25)),
)
.await
.expect("short terminal observation budget should return promptly")
.expect_err("stalled provider must leave the run nonterminal");
assert_eq!(snapshot.phase(), ProductRunPhase::Designing);
let diagnostic = terminal_wait_timeout_diagnostic(&snapshot, Duration::from_millis(25));
assert!(diagnostic.contains(&format!("phase={:?}", snapshot.phase())));
assert!(diagnostic.contains(&format!("status={:?}", snapshot.status())));
assert!(diagnostic.contains(&format!("gates={:?}", snapshot.gates())));
assert!(diagnostic.contains(&format!("summary={:?}", snapshot.summary())));

service.cancel(run_id).expect("cancel stalled run");
let cancelled =
tokio::time::timeout(Duration::from_secs(5), wait_for_terminal(&service, run_id))
.await
.expect("cancellation must remain prompt with the longer observation budget");
assert_eq!(cancelled.phase(), ProductRunPhase::Cancelled);
service.shutdown(Duration::from_secs(5)).await;
});
}

fn service(
state: &std::path::Path,
workspace: &std::path::Path,
Expand Down Expand Up @@ -190,19 +250,48 @@ async fn wait_for_terminal(
service: &ProductRunService,
run_id: RunId,
) -> peritus_app_protocol::ProductRunSnapshot {
for _ in 0..400 {
wait_for_terminal_with_timeout(service, run_id, PRODUCT_RUN_TERMINAL_WAIT).await.unwrap_or_else(
|snapshot| {
panic!("{}", terminal_wait_timeout_diagnostic(&snapshot, PRODUCT_RUN_TERMINAL_WAIT))
},
)
}

async fn wait_for_terminal_with_timeout(
service: &ProductRunService,
run_id: RunId,
timeout: Duration,
) -> Result<peritus_app_protocol::ProductRunSnapshot, peritus_app_protocol::ProductRunSnapshot> {
let deadline = tokio::time::Instant::now() + timeout;
loop {
let snapshot = service
.query(ProductRunQuery::exact(run_id))
.expect("query run")
.into_iter()
.next()
.expect("run snapshot");
if snapshot.phase().terminal() {
return snapshot;
return Ok(snapshot);
}
tokio::time::sleep(Duration::from_millis(25)).await;
let now = tokio::time::Instant::now();
if now >= deadline {
return Err(snapshot);
}
tokio::time::sleep_until((now + Duration::from_millis(25)).min(deadline)).await;
}
panic!("product run did not settle within ten seconds")
}

fn terminal_wait_timeout_diagnostic(
snapshot: &peritus_app_protocol::ProductRunSnapshot,
timeout: Duration,
) -> String {
format!(
"product run did not settle within {timeout:?}; phase={:?}, status={:?}, gates={:?}, summary={:?}",
snapshot.phase(),
snapshot.status(),
snapshot.gates(),
snapshot.summary(),
)
}

async fn wait_for_review_stall(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,40 +15,46 @@ use super::{contract, identity};
/// `SQLite` serializes reservations across independently opened runtimes. Legacy
/// authority and compactor paths remain occupied even without an allocator row.
pub(super) fn reserve(root: &Path, run_id: RunId, after: u64) -> Result<u64, String> {
let mut connection =
Connection::open(root.join("command-ordinals.sqlite3")).map_err(|error| detail(&error))?;
let mut connection = Connection::open(root.join("command-ordinals.sqlite3"))
.map_err(|error| detail("open store", &error))?;
// Independent runtimes share this writer lock. Allow durable commits and
// scheduling delays under contention without failing after only 250 ms.
connection.busy_timeout(Duration::from_secs(5)).map_err(|error| detail(&error))?;
connection
.busy_timeout(Duration::from_secs(5))
.map_err(|error| detail("configure busy timeout", &error))?;
// EXTRA also syncs the rollback-journal directory after commit. FULL alone
// can lose the last acknowledged reservation on power loss in DELETE mode.
connection.pragma_update(None, "synchronous", "EXTRA").map_err(|error| detail(&error))?;
connection
.pragma_update(None, "synchronous", "EXTRA")
.map_err(|error| detail("configure durable synchronization", &error))?;
// Acquire the writer before installing the schema. First-open contenders
// must not commit a separate schema transaction before reserving an ordinal.
let transaction = connection
.transaction_with_behavior(TransactionBehavior::Immediate)
.map_err(|error| detail("begin reservation", &error))?;
transaction
.execute_batch(
"CREATE TABLE IF NOT EXISTS command_ordinals (
run_id BLOB PRIMARY KEY NOT NULL CHECK(length(run_id) = 16),
last_ordinal INTEGER NOT NULL
CHECK(typeof(last_ordinal) = 'integer' AND last_ordinal >= 0)
);",
)
.map_err(|error| detail(&error))?;
let transaction = connection
.transaction_with_behavior(TransactionBehavior::Immediate)
.map_err(|error| detail(&error))?;
.map_err(|error| detail("install schema", &error))?;
transaction
.execute(
"INSERT INTO command_ordinals(run_id, last_ordinal) VALUES (?1, 0)
ON CONFLICT(run_id) DO NOTHING",
params![run_id.as_bytes().as_slice()],
)
.map_err(|error| detail(&error))?;
.map_err(|error| detail("initialize run counter", &error))?;
let stored: i64 = transaction
.query_row(
"SELECT last_ordinal FROM command_ordinals WHERE run_id = ?1",
params![run_id.as_bytes().as_slice()],
|row| row.get(0),
)
.map_err(|error| detail(&error))?;
.map_err(|error| detail("read run counter", &error))?;
let mut ordinal = u64::try_from(stored)
.map_err(|_| "command ordinal store contains a negative number".to_owned())?
.max(after);
Expand All @@ -73,9 +79,9 @@ pub(super) fn reserve(root: &Path, run_id: RunId, after: u64) -> Result<u64, Str
"UPDATE command_ordinals SET last_ordinal = ?1 WHERE run_id = ?2",
params![stored, run_id.as_bytes().as_slice()],
)
.map_err(|error| detail(&error))?;
.map_err(|error| detail("update run counter", &error))?;
// Never expose an allocation whose durable commit was not acknowledged.
transaction.commit().map_err(|error| detail(&error))?;
transaction.commit().map_err(|error| detail("commit reservation", &error))?;
Ok(ordinal)
}

Expand All @@ -87,8 +93,8 @@ fn occupied(path: &Path) -> Result<bool, String> {
}
}

fn detail(error: &rusqlite::Error) -> String {
format!("reserve durable command ordinal: {error}")
fn detail(operation: &str, error: &rusqlite::Error) -> String {
format!("reserve durable command ordinal ({operation}): {error}")
}

#[cfg(test)]
Expand Down Expand Up @@ -207,6 +213,26 @@ mod tests {
assert_eq!(reserve(root.path(), second, 0).expect("second run reopened"), 2);
}

#[test]
fn failed_first_reservation_does_not_publish_schema_separately() {
let root = tempfile::tempdir().expect("state directory");
let run = RunId::new([9; 16]).expect("run");
assert!(reserve(root.path(), run, u64::MAX).is_err());
let connection =
Connection::open(root.path().join("command-ordinals.sqlite3")).expect("reopen store");
let tables: i64 = connection
.query_row(
"SELECT count(*) FROM sqlite_schema WHERE name = 'command_ordinals'",
[],
|row| row.get(0),
)
.expect("inspect schema after rollback");
assert_eq!(tables, 0, "schema installation must share the reservation transaction");
drop(connection);
assert_eq!(reserve(root.path(), run, 0).expect("first committed reservation"), 1);
assert_eq!(reserve(root.path(), run, 0).expect("next reservation"), 2);
}

#[test]
fn invalid_and_exhausted_stored_counters_are_never_reset() {
for stored in [
Expand Down
16 changes: 16 additions & 0 deletions verification/actor-provenance.json
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,22 @@
"record_locators": [
"codex-collaboration:Corvidae-Coding-Projects/Project-Peritus/session/102/task/root/batch3_formal_review_sol"
]
},
{
"actor_id": "ACTOR-0011",
"kind": "codex-subagent",
"principal": "Corvidae-Coding-Projects/Project-Peritus/session/102/task/root/ordinal_formal_review_sol",
"repository": "Corvidae-Coding-Projects/Project-Peritus",
"issue": 103,
"issue_created_at": "2026-10-05T14:30:16.158065741Z",
"session": 102,
"task": "/root/ordinal_formal_review_sol",
"mode": "read-only-review",
"model": "gpt-5.6-sol",
"reasoning_effort": "xhigh",
"record_locators": [
"codex-collaboration:Corvidae-Coding-Projects/Project-Peritus/session/102/task/root/ordinal_formal_review_sol"
]
}
]
}
28 changes: 18 additions & 10 deletions verification/actors.toml
Original file line number Diff line number Diff line change
Expand Up @@ -10,76 +10,84 @@ kind = "crosslink-agent"
principal = "SHA256:eV8eZPaZxut5mrkihmvsOTrGWClwD+B/HR//do+oIeI"
display_name = "Project Peritus A1 implementation agent"
roles = ["owner"]
provenance = { record_path = "verification/actor-provenance.json", record_sha256 = "d49cf4e92e91adaf48a6487796d5dd9718bf532912923ab168321f1fd13893a8" }
provenance = { record_path = "verification/actor-provenance.json", record_sha256 = "cd7dd74a43a8f66bbb00cf20b401b4112110318750b1d079e82e0c06025b23f0" }

[[entries]]
id = "ACTOR-0002"
kind = "codex-subagent"
principal = "Corvidae-Coding-Projects/Project-Peritus/session/2/task/root/a1_final_gate_review"
display_name = "Banach independent final Gate A reviewer"
roles = ["reviewer"]
provenance = { record_path = "verification/actor-provenance.json", record_sha256 = "d49cf4e92e91adaf48a6487796d5dd9718bf532912923ab168321f1fd13893a8" }
provenance = { record_path = "verification/actor-provenance.json", record_sha256 = "cd7dd74a43a8f66bbb00cf20b401b4112110318750b1d079e82e0c06025b23f0" }

[[entries]]
id = "ACTOR-0003"
kind = "crosslink-agent"
principal = "SHA256:IeCr62P7gzB0M5HnHg0Ncs6fEmNgIObFyetKV6DXuCc"
display_name = "Workbench implementation and formal repair owner"
roles = ["owner"]
provenance = { record_path = "verification/actor-provenance.json", record_sha256 = "d49cf4e92e91adaf48a6487796d5dd9718bf532912923ab168321f1fd13893a8" }
provenance = { record_path = "verification/actor-provenance.json", record_sha256 = "cd7dd74a43a8f66bbb00cf20b401b4112110318750b1d079e82e0c06025b23f0" }

[[entries]]
id = "ACTOR-0004"
kind = "codex-subagent"
principal = "Corvidae-Coding-Projects/Project-Peritus/session/23/task/root/workbench_formal_review"
display_name = "Independent workbench and formal authorization reviewer"
roles = ["reviewer"]
provenance = { record_path = "verification/actor-provenance.json", record_sha256 = "d49cf4e92e91adaf48a6487796d5dd9718bf532912923ab168321f1fd13893a8" }
provenance = { record_path = "verification/actor-provenance.json", record_sha256 = "cd7dd74a43a8f66bbb00cf20b401b4112110318750b1d079e82e0c06025b23f0" }

[[entries]]
id = "ACTOR-0005"
kind = "crosslink-agent"
principal = "SHA256:F4pa6Img4SKcIhtZ3zyHSwUehhPFSMjlvpf7eTU6qsE"
display_name = "GAP-02 implementation and reconciliation owner"
roles = ["owner"]
provenance = { record_path = "verification/actor-provenance.json", record_sha256 = "d49cf4e92e91adaf48a6487796d5dd9718bf532912923ab168321f1fd13893a8" }
provenance = { record_path = "verification/actor-provenance.json", record_sha256 = "cd7dd74a43a8f66bbb00cf20b401b4112110318750b1d079e82e0c06025b23f0" }

[[entries]]
id = "ACTOR-0006"
kind = "codex-subagent"
principal = "Corvidae-Coding-Projects/Project-Peritus/session/4/task/root/gap2_final_review"
display_name = "Independent final GAP-02 source and proof-impact reviewer"
roles = ["reviewer"]
provenance = { record_path = "verification/actor-provenance.json", record_sha256 = "d49cf4e92e91adaf48a6487796d5dd9718bf532912923ab168321f1fd13893a8" }
provenance = { record_path = "verification/actor-provenance.json", record_sha256 = "cd7dd74a43a8f66bbb00cf20b401b4112110318750b1d079e82e0c06025b23f0" }

[[entries]]
id = "ACTOR-0007"
kind = "codex-subagent"
principal = "Corvidae-Coding-Projects/Project-Peritus/session/4/task/root/gap4_final_review"
display_name = "Independent final GAP-04 context correspondence reviewer"
roles = ["reviewer"]
provenance = { record_path = "verification/actor-provenance.json", record_sha256 = "d49cf4e92e91adaf48a6487796d5dd9718bf532912923ab168321f1fd13893a8" }
provenance = { record_path = "verification/actor-provenance.json", record_sha256 = "cd7dd74a43a8f66bbb00cf20b401b4112110318750b1d079e82e0c06025b23f0" }

[[entries]]
id = "ACTOR-0008"
kind = "codex-subagent"
principal = "Corvidae-Coding-Projects/Project-Peritus/session/61/task/root/pcr7_final_review_sol"
display_name = "Independent PCR-0007 product reliability and proof-impact reviewer"
roles = ["reviewer"]
provenance = { record_path = "verification/actor-provenance.json", record_sha256 = "d49cf4e92e91adaf48a6487796d5dd9718bf532912923ab168321f1fd13893a8" }
provenance = { record_path = "verification/actor-provenance.json", record_sha256 = "cd7dd74a43a8f66bbb00cf20b401b4112110318750b1d079e82e0c06025b23f0" }

[[entries]]
id = "ACTOR-0009"
kind = "codex-subagent"
principal = "Corvidae-Coding-Projects/Project-Peritus/session/61/task/root/pcr8_final_review_sol"
display_name = "Independent PCR-0008 Windows diagnostic portability reviewer"
roles = ["reviewer"]
provenance = { record_path = "verification/actor-provenance.json", record_sha256 = "d49cf4e92e91adaf48a6487796d5dd9718bf532912923ab168321f1fd13893a8" }
provenance = { record_path = "verification/actor-provenance.json", record_sha256 = "cd7dd74a43a8f66bbb00cf20b401b4112110318750b1d079e82e0c06025b23f0" }

[[entries]]
id = "ACTOR-0010"
kind = "codex-subagent"
principal = "Corvidae-Coding-Projects/Project-Peritus/session/102/task/root/batch3_formal_review_sol"
display_name = "Independent minimum-repair batch and proof-impact reviewer"
roles = ["reviewer"]
provenance = { record_path = "verification/actor-provenance.json", record_sha256 = "d49cf4e92e91adaf48a6487796d5dd9718bf532912923ab168321f1fd13893a8" }
provenance = { record_path = "verification/actor-provenance.json", record_sha256 = "cd7dd74a43a8f66bbb00cf20b401b4112110318750b1d079e82e0c06025b23f0" }

[[entries]]
id = "ACTOR-0011"
kind = "codex-subagent"
principal = "Corvidae-Coding-Projects/Project-Peritus/session/102/task/root/ordinal_formal_review_sol"
display_name = "Independent command ordinal contention and proof-impact reviewer"
roles = ["reviewer"]
provenance = { record_path = "verification/actor-provenance.json", record_sha256 = "cd7dd74a43a8f66bbb00cf20b401b4112110318750b1d079e82e0c06025b23f0" }
Loading
Loading