From 7379928515928d8a925682b5f7a5999652d27851 Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Thu, 8 Oct 2026 11:17:15 -0400 Subject: [PATCH 1/2] Fail refused Security sessions and log their answering Models (#547) --- src/harness/interpretation.rs | 37 ++++- src/harness/interpretation/security.rs | 99 +++++++++++++ src/harness/interpretation_tests.rs | 61 ++++++++ src/notification.rs | 12 +- src/run_ending.rs | 23 ++- src/session.rs | 4 + tests/security_run.rs | 186 +++++++++++++++++++++++++ 7 files changed, 410 insertions(+), 12 deletions(-) create mode 100644 src/harness/interpretation/security.rs diff --git a/src/harness/interpretation.rs b/src/harness/interpretation.rs index 03bef9a1..490bf067 100644 --- a/src/harness/interpretation.rs +++ b/src/harness/interpretation.rs @@ -10,6 +10,9 @@ use crate::interrupt; mod stream; pub(super) use stream::Stream; +mod security; +pub use security::SafeguardRefusal; +use security::Security; /// Live interpretation has only line condensation. Consume it after the /// process owner has stopped or waited for the child and joined its workers. @@ -17,6 +20,7 @@ pub struct Interpretation { cli: &'static str, decoder: Box, retained: Retained, + security: Option, } /// Reporting survives ordinary failures; only success supplies Resume facts. @@ -118,6 +122,7 @@ enum Failure { Rejected { status: ExitStatus, diagnostic: Option, + refusal: Option, }, } @@ -125,7 +130,11 @@ impl Failure { fn session_error(self, cli: &str) -> anyhow::Error { match self { Self::Execution(error) => error, - Self::Rejected { status, diagnostic } => { + Self::Rejected { + status, + diagnostic, + refusal, + } => { let ended = if status.success() { "'s turn failed".to_string() } else { @@ -136,9 +145,13 @@ impl Failure { .map_or("by signal".to_string(), |code| code.to_string()) ) }; - match diagnostic { + let error = match diagnostic { Some(error) => anyhow!("{cli}{ended}: {error}"), None => anyhow!("{cli}{ended}"), + }; + match refusal { + Some(refusal) => error.context(refusal), + None => error, } } } @@ -151,12 +164,25 @@ impl Interpretation { cli, decoder, retained, + security: None, } } + /// Apply refusal and Model reporting rules independently of the session's log label. + pub fn for_security(mut self, requested_model: Option<&str>) -> Self { + self.security = Some(Security::new(requested_model)); + self + } + /// Unknown or malformed lines produce no progress, never an error. pub fn condense(&mut self, raw: &str) -> Vec { - self.decoder.condense(raw) + let mut lines = self.decoder.condense(raw); + if let Some(security) = &mut self.security + && let Some(line) = security.observe(self.cli, raw) + { + lines.push(line); + } + lines } pub fn finish(self, execution: Result) -> Completion { @@ -188,6 +214,10 @@ impl Interpretation { return (None, Err(Failure::Execution(error))); } let mut facts = self.decoder.complete(); + let refusal = self.security.and_then(|security| security.refusal); + if refusal.is_some() { + facts.outcome = TurnOutcome::Failed; + } let recovered = self.retained.reconcile(&mut facts); if let Err(error) = interrupt::check() { return (None, Err(Failure::Execution(error))); @@ -200,6 +230,7 @@ impl Interpretation { Err(Failure::Rejected { status, diagnostic: facts.diagnostic, + refusal, }) } else { Ok(facts.ended) diff --git a/src/harness/interpretation/security.rs b/src/harness/interpretation/security.rs new file mode 100644 index 00000000..d5cd3b73 --- /dev/null +++ b/src/harness/interpretation/security.rs @@ -0,0 +1,99 @@ +//! Security session evidence: safeguard refusals are failures even when a +//! Harness reports a successful turn. Raw diagnostics stay in local logs. + +use std::fmt; + +use serde_json::Value; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SafeguardRefusal { + ClaudeCyber, + CodexCyber, +} + +impl SafeguardRefusal { + pub fn description(self) -> &'static str { + match self { + Self::ClaudeCyber => "Claude Code's [cyber] safeguard refusal", + Self::CodexCyber => "Codex's cybersecurity safeguard refusal", + } + } +} + +impl fmt::Display for SafeguardRefusal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.description()) + } +} + +impl std::error::Error for SafeguardRefusal {} + +#[derive(Default)] +pub(super) struct Security { + pub refusal: Option, + last_model: Option, + requested_model: Option, +} + +impl Security { + pub fn new(requested_model: Option<&str>) -> Self { + Self { + requested_model: requested_model.map(String::from), + ..Self::default() + } + } + + pub fn observe(&mut self, cli: &str, raw: &str) -> Option { + let Ok(event) = serde_json::from_str::(raw) else { + return None; + }; + match (cli, event["type"].as_str()) { + ("claude", Some("system")) + if event["subtype"] == "model_refusal_no_fallback" + && event["api_refusal_category"] == "cyber" => + { + self.refusal = Some(SafeguardRefusal::ClaudeCyber); + } + ("claude", Some("result")) + if event["result"].as_str().is_some_and(|text| { + let text = text.trim_start(); + text.starts_with("[cyber]") + || (text.starts_with("API Error:") && text.contains("[cyber]")) + }) => + { + self.refusal = Some(SafeguardRefusal::ClaudeCyber); + } + ("codex", Some("turn.failed")) + if event["error"]["message"] + .as_str() + .is_some_and(|text| text.to_ascii_lowercase().contains("cybersecurity")) => + { + self.refusal = Some(SafeguardRefusal::CodexCyber); + } + _ => {} + } + // Init names the requested Model; result.modelUsage includes sub-agents. + // Only main-loop assistant messages identify the Model that answered. + if cli == "claude" + && event["type"] == "assistant" + && event["parent_tool_use_id"].is_null() + && let Some(model) = event["message"]["model"].as_str() + && !model.is_empty() + && self.last_model.as_deref() != Some(model) + { + self.last_model = Some(model.to_string()); + return Some(format!("Model: {model}")); + } + if cli == "codex" && event["type"] == "thread.started" && self.last_model.is_none() { + let model = self.requested_model.as_deref().unwrap_or("Harness default"); + self.last_model = Some(model.to_string()); + let basis = if self.requested_model.is_some() { + "requested" + } else { + "no Model requested" + }; + return Some(format!("Model: {model} ({basis})")); + } + None + } +} diff --git a/src/harness/interpretation_tests.rs b/src/harness/interpretation_tests.rs index 5ce9e157..05480499 100644 --- a/src/harness/interpretation_tests.rs +++ b/src/harness/interpretation_tests.rs @@ -1,6 +1,67 @@ use super::*; use std::os::unix::process::ExitStatusExt; +#[test] +fn a_security_session_can_discuss_the_cyber_marker_without_being_refused() { + let mut interpretation = stream(Harness::Claude, "").for_security(None); + interpretation.condense( + r#"{"type":"result","subtype":"success","result":"The [cyber] refusal test passed.\nSecurity audit: complete"}"#, + ); + let ended = finish(interpretation).outcome.unwrap(); + assert_eq!( + ended.final_message.as_deref(), + Some("The [cyber] refusal test passed.\nSecurity audit: complete") + ); +} + +#[test] +fn a_terminal_claude_cyber_refusal_overrides_a_successful_security_result() { + let event = r#"{"type":"system","subtype":"model_refusal_no_fallback","api_refusal_category":"cyber","content":"Private safeguard explanation."}"#; + for security in [false, true] { + let mut interpretation = stream(Harness::Claude, ""); + if security { + interpretation = interpretation.for_security(None); + } + interpretation.condense(event); + interpretation.condense( + r#"{"type":"result","subtype":"success","result":"Security audit: complete"}"#, + ); + let completion = finish(interpretation); + if security { + let error = completion.outcome.unwrap_err(); + assert_eq!( + error.downcast_ref::(), + Some(&interpretation::SafeguardRefusal::ClaudeCyber) + ); + } else { + assert!(completion.outcome.is_ok()); + } + } +} + +#[test] +fn security_model_progress_names_only_new_main_loop_answers() { + let mut interpretation = stream(Harness::Claude, "").for_security(Some("opus")); + for ignored in [ + r#"{"type":"system","subtype":"init","model":"requested-model"}"#, + r#"{"type":"assistant","parent_tool_use_id":"child","message":{"model":"child-model","content":[]}}"#, + r#"{"type":"result","subtype":"success","modelUsage":{"child-model":{"inputTokens":100}}}"#, + ] { + assert!( + interpretation + .condense(ignored) + .iter() + .all(|line| !line.starts_with("Model:")) + ); + } + let answer = r#"{"type":"assistant","message":{"model":"claude-opus-4-8","content":[]}}"#; + assert_eq!( + interpretation.condense(answer), + vec!["Model: claude-opus-4-8"] + ); + assert!(interpretation.condense(answer).is_empty()); +} + #[test] fn claude_completion_keeps_the_last_result_and_the_last_complete_usage() { let mut interpretation = Harness::Claude diff --git a/src/notification.rs b/src/notification.rs index bd51ae28..4e908bf1 100644 --- a/src/notification.rs +++ b/src/notification.rs @@ -239,12 +239,18 @@ fn body( text += &format!("Pull request: {pr_url}\n"); } // Security failure causes can quote private verifier work or Harness - // diagnostics. Their details belong in the local logs. + // diagnostics. Only the fixed safeguard category is safe to send. if let Err(cause) = &account.ended && !account.interrupted - && account.security_findings.is_none() { - text += &format!("Cause: {}\n", cause.full()); + let cause = if account.security_findings.is_some() { + cause.safeguard_refusal() + } else { + Some(cause.full()) + }; + if let Some(cause) = cause { + text += &format!("Cause: {cause}\n"); + } } for line in account.advice { text += &format!("{:<14}{}\n", format!("{}:", line.label), line.value); diff --git a/src/run_ending.rs b/src/run_ending.rs index 722bc5ab..8eb50b97 100644 --- a/src/run_ending.rs +++ b/src/run_ending.rs @@ -117,22 +117,33 @@ pub struct Review<'a> { /// A failure's cause, as stderr gives it, down to what a session left /// running. #[derive(Debug, PartialEq, Eq)] -pub struct Cause(String); +pub struct Cause { + full: String, + safeguard_refusal: Option, +} impl Cause { /// The cause of a failure that failed with `error`. pub fn of(error: &anyhow::Error) -> Self { - Cause(format!("{error:#}")) + Self { + full: format!("{error:#}"), + safeguard_refusal: error.downcast_ref().copied(), + } } /// The whole cause, of as many lines as it has. pub fn full(&self) -> &str { - &self.0 + &self.full } /// Its first line, all of it that a short account gives. pub fn first_line(&self) -> &str { - self.0.lines().next().unwrap_or_default() + self.full.lines().next().unwrap_or_default() + } + + /// A fixed refusal category safe to send without the Harness's private diagnostic. + pub fn safeguard_refusal(&self) -> Option<&'static str> { + self.safeguard_refusal.map(|refusal| refusal.description()) } } @@ -427,7 +438,7 @@ mod tests { } fn cause(cause: &str) -> Cause { - Cause(cause.to_string()) + Cause::of(&anyhow!("{cause}")) } fn strings(lines: &[&str]) -> Vec { @@ -470,7 +481,7 @@ mod tests { fn failed_account(cause: &str) -> Account<'static> { Account { outcome: "failed", - ended: Err(Cause(cause.to_string())), + ended: Err(Cause::of(&anyhow!("{cause}"))), interrupted: false, pr_url: None, advice: &[], diff --git a/src/session.rs b/src/session.rs index bffa7665..62b0e7d0 100644 --- a/src/session.rs +++ b/src/session.rs @@ -298,6 +298,10 @@ impl Outside for OnMachine { Purpose::Ordinary => self.adapter.session(&self.harness, resume, prompt), }; let interpretation = self.adapter.interpretation(&self.worktree, prompt); + let interpretation = match purpose { + Purpose::Security => interpretation.for_security(self.harness.model.as_deref()), + Purpose::Ordinary => interpretation, + }; run( kind, self.adapter, diff --git a/tests/security_run.rs b/tests/security_run.rs index 2041aa56..056cc255 100644 --- a/tests/security_run.rs +++ b/tests/security_run.rs @@ -35,6 +35,192 @@ fn finding(fingerprint: &str) -> Value { }) } +#[test] +fn a_claude_cyber_refusal_fails_the_security_audit_with_its_cause() { + let scenario = Scenario::new(); + scenario.agent_does( + r#" +printf '%s\n' 'API Error: [cyber] This request was refused by the safeguard.' > "$FAKE_CLAUDE_FINAL_MESSAGE" +"#, + ); + let result = scenario.run(&["secure"]); + assert_eq!(result.code, Some(1), "{}", result.stderr); + assert!( + result + .stderr + .contains("Claude Code's [cyber] safeguard refusal"), + "{}", + result.stderr + ); + assert!(scenario.gh_state()["advisories"].is_null()); + assert_eq!(scenario.claude_calls().len(), 1); + assert_eq!(scenario.entries("work"), vec![REPO]); +} + +#[test] +fn safeguard_refusals_notify_the_cause_without_private_diagnostics() { + for (harness, script, cause) in [ + ( + "claude", + r#"printf '%s\n' 'API Error: [cyber] Private refusal evidence.' > "$FAKE_CLAUDE_FINAL_MESSAGE""#, + "Claude Code's [cyber] safeguard refusal", + ), + ( + "codex", + r#"printf '%s\n' 'Cybersecurity safeguard refused: Private refusal evidence.' > "$FAKE_CODEX_ERROR" +exit 1"#, + "Codex's cybersecurity safeguard refusal", + ), + ] { + let scenario = Scenario::new(); + scenario.agent_does(script); + let resend = ResendStandIn::replying(200, r#"{"id":"1"}"#); + let result = scenario.run_with_env( + &["secure", "harness", harness, "email", "me@example.com"], + &resend_env(&resend), + ); + assert_eq!(result.code, Some(1), "{}", result.stderr); + assert!(result.stderr.contains(cause), "{}", result.stderr); + let (subject, text) = the_one_notification(&resend); + assert_eq!( + subject, + "[thirdshift] acme/widgets Security run: audit failed" + ); + assert!(text.contains(&format!("Cause: {cause}")), "{text}"); + assert!(!text.contains("Private refusal evidence."), "{text}"); + } +} + +#[test] +fn security_sessions_log_claudes_answering_models_in_progress_and_the_command_log() { + let scenario = Scenario::new(); + scenario.agent_does_in_session(1, "true\n"); // Requested Model check. + scenario.agent_does_in_session( + 2, + &format!( + r#" +printf '%s\n' '{{"type":"assistant","message":{{"model":"claude-opus-5-5","content":[]}}}}' +printf '%s\n' '{{"type":"assistant","parent_tool_use_id":"child","message":{{"model":"claude-sonnet-5","content":[]}}}}' +printf '%s\n' '{{"type":"system","subtype":"model_refusal_fallback","original_model":"claude-opus-5-5","fallback_model":"claude-opus-4-8","api_refusal_category":"cyber"}}' +printf '%s\n' '{{"type":"assistant","message":{{"model":"claude-opus-4-8","content":[]}}}}' +{} +"#, + audit_script(&json!([finding("model-switch")]).to_string()) + ), + ); + scenario.agent_does_in_session( + 3, + &format!( + "printf '%s\\n' '{{\"type\":\"assistant\",\"message\":{{\"model\":\"claude-opus-4-8\",\"content\":[]}}}}'\n{}", + reproduction_script("not reproduced") + ), + ); + let result = scenario.run(&["secure", "model", "opus"]); + assert_eq!(result.code, Some(0), "{}", result.stderr); + let logs = scenario.entries("home/.thirdshift/logs/acme/widgets/commands/secure"); + let command_log = fs::read_to_string(scenario.path(&format!( + "home/.thirdshift/logs/acme/widgets/commands/secure/{}", + logs[0] + ))) + .unwrap(); + for line in [ + "security-audit: Model: claude-opus-5-5", + "security-audit: Model: claude-opus-4-8", + "security-reproduction-1: Model: claude-opus-4-8", + ] { + assert!(result.stderr.contains(line), "{}", result.stderr); + assert!(command_log.contains(line), "{command_log}"); + } + assert!( + !command_log.contains("Model: claude-sonnet-5"), + "{command_log}" + ); +} + +#[test] +fn codex_logs_the_requested_model_for_each_security_session_and_resume() { + let scenario = Scenario::new(); + scenario.agent_does_in_session( + 1, + &format!( + "{}\nprintf '%s\\n' '{{\"type\":\"item.started\",\"item\":{{\"id\":\"child\",\"type\":\"collab_tool_call\",\"tool\":\"spawn_agent\",\"status\":\"in_progress\"}}}}'\n", + audit_script(&json!([finding("codex-model")]).to_string()) + ), + ); + scenario.agent_does_in_session( + 2, + r#"printf '%s\n' 'Security audit: complete' > "$FAKE_CLAUDE_FINAL_MESSAGE""#, + ); + scenario.agent_does_in_session(3, &reproduction_script("not reproduced")); + let result = scenario.run(&["secure", "harness", "codex", "model", "GPT-6.1-Sol"]); + assert_eq!(result.code, Some(0), "{}", result.stderr); + let logs = scenario.entries("home/.thirdshift/logs/acme/widgets/commands/secure"); + let command_log = fs::read_to_string(scenario.path(&format!( + "home/.thirdshift/logs/acme/widgets/commands/secure/{}", + logs[0] + ))) + .unwrap(); + for kind in [ + "security-audit", + "security-audit-resume", + "security-reproduction-1", + ] { + let line = format!("{kind}: Model: gpt-6.1-sol (requested)"); + assert!(result.stderr.contains(&line), "{}", result.stderr); + assert!(command_log.contains(&line), "{command_log}"); + } +} + +#[test] +fn refused_security_reproductions_keep_the_record_and_stop_before_the_next_finding() { + for (harness, script, cause) in [ + ( + "claude", + r#"printf '%s\n' 'API Error: [cyber] Private refusal evidence.' > "$FAKE_CLAUDE_FINAL_MESSAGE""#, + "Claude Code's [cyber] safeguard refusal", + ), + ( + "codex", + r#"printf '%s\n' 'Cybersecurity safeguard refused: Private refusal evidence.' > "$FAKE_CODEX_ERROR" +exit 1"#, + "Codex's cybersecurity safeguard refusal", + ), + ] { + let scenario = Scenario::new(); + scenario.agent_does_in_session( + 1, + &audit_script(&json!([finding("first"), finding("second")]).to_string()), + ); + scenario.agent_does_in_session(2, script); + scenario.agent_does_in_session(3, "exit 99\n"); + let resend = ResendStandIn::replying(200, r#"{"id":"1"}"#); + let result = scenario.run_with_env( + &["secure", "harness", harness, "email", "me@example.com"], + &resend_env(&resend), + ); + assert_eq!(result.code, Some(1), "{}", result.stderr); + assert!(result.stderr.contains(cause), "{}", result.stderr); + let (_, text) = the_one_notification(&resend); + assert!(text.contains(cause), "{text}"); + assert!(!text.contains("Private refusal evidence."), "{text}"); + let state = scenario.gh_state(); + for record in state["advisories"].as_array().unwrap() { + assert!(record["severity"].is_null()); + assert!( + !record["description"] + .as_str() + .unwrap() + .contains("## Reproduction") + ); + } + assert_eq!( + scenario.claude_calls().len() + scenario.codex_calls().len(), + 2 + ); + assert_eq!(scenario.entries("work"), vec![REPO]); + } +} + #[test] fn a_reproduction_scores_the_finding_and_keeps_its_test_in_the_private_record() { let scenario = Scenario::new(); From 3718f525379ff7223b42602618a62d05af124aec Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Thu, 8 Oct 2026 11:37:46 -0400 Subject: [PATCH 2/2] Address #547 review: ignore synthetic Models and report all Harnesses --- src/harness/agy/stream.rs | 1 + src/harness/codex/stream.rs | 1 + src/harness/interpretation.rs | 13 +- src/harness/interpretation/security.rs | 117 +++++++++------- src/harness/interpretation/stream.rs | 1 + src/harness/interpretation_tests.rs | 12 ++ src/harness/muse/log.rs | 6 + src/harness/muse/stream.rs | 1 + src/harness/opencode/export.rs | 11 ++ src/harness/opencode/stream.rs | 1 + src/session.rs | 19 ++- tests/security_run.rs | 181 +++++++++++++++++++------ 12 files changed, 267 insertions(+), 97 deletions(-) diff --git a/src/harness/agy/stream.rs b/src/harness/agy/stream.rs index eee6337f..0206808b 100644 --- a/src/harness/agy/stream.rs +++ b/src/harness/agy/stream.rs @@ -75,6 +75,7 @@ impl Decoder for AgyProgress { report: Report { warnings: Vec::new(), summary, + ..Report::default() }, ended: Ended { session_id: self.session_id.clone(), diff --git a/src/harness/codex/stream.rs b/src/harness/codex/stream.rs index 9d64f017..4ac91fcf 100644 --- a/src/harness/codex/stream.rs +++ b/src/harness/codex/stream.rs @@ -187,6 +187,7 @@ impl Decoder for CodexProgress { report: Report { warnings: Vec::new(), summary, + ..Report::default() }, ended: Ended { session_id: self.session_id, diff --git a/src/harness/interpretation.rs b/src/harness/interpretation.rs index 490bf067..1e25933a 100644 --- a/src/harness/interpretation.rs +++ b/src/harness/interpretation.rs @@ -29,9 +29,12 @@ pub struct Completion { pub outcome: Result, } +#[derive(Default)] pub struct Report { pub warnings: Vec, pub summary: Option, + /// Answering Models recovered from retained records, in response order. + pub models: Vec, } #[derive(Debug)] @@ -97,6 +100,7 @@ impl Retained { if log.message.is_some() { facts.ended.final_message = log.message; } + facts.report.models = log.models; } } Self::OpenCode(worktree) => { @@ -106,6 +110,7 @@ impl Retained { // A readable export is authoritative, even without text. facts.ended.final_message = export.message; facts.report.summary = export.summary; + facts.report.models = export.models; if export.failure.is_some() { facts.outcome = TurnOutcome::Failed; facts.diagnostic = facts.diagnostic.take().or(export.failure); @@ -170,7 +175,7 @@ impl Interpretation { /// Apply refusal and Model reporting rules independently of the session's log label. pub fn for_security(mut self, requested_model: Option<&str>) -> Self { - self.security = Some(Security::new(requested_model)); + self.security = Some(Security::new(self.cli, requested_model)); self } @@ -178,7 +183,7 @@ impl Interpretation { pub fn condense(&mut self, raw: &str) -> Vec { let mut lines = self.decoder.condense(raw); if let Some(security) = &mut self.security - && let Some(line) = security.observe(self.cli, raw) + && let Some(line) = security.observe(raw) { lines.push(line); } @@ -213,12 +218,16 @@ impl Interpretation { if let Err(error) = interrupt::check() { return (None, Err(Failure::Execution(error))); } + let security_session = self.security.is_some(); let mut facts = self.decoder.complete(); let refusal = self.security.and_then(|security| security.refusal); if refusal.is_some() { facts.outcome = TurnOutcome::Failed; } let recovered = self.retained.reconcile(&mut facts); + if !security_session { + facts.report.models.clear(); + } if let Err(error) = interrupt::check() { return (None, Err(Failure::Execution(error))); } diff --git a/src/harness/interpretation/security.rs b/src/harness/interpretation/security.rs index d5cd3b73..2ed61ea9 100644 --- a/src/harness/interpretation/security.rs +++ b/src/harness/interpretation/security.rs @@ -5,6 +5,8 @@ use std::fmt; use serde_json::Value; +use crate::harness::Harness; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SafeguardRefusal { ClaudeCyber, @@ -28,72 +30,87 @@ impl fmt::Display for SafeguardRefusal { impl std::error::Error for SafeguardRefusal {} -#[derive(Default)] pub(super) struct Security { + harness: Option, pub refusal: Option, last_model: Option, requested_model: Option, } impl Security { - pub fn new(requested_model: Option<&str>) -> Self { + pub fn new(cli: &str, requested_model: Option<&str>) -> Self { Self { + harness: Harness::named(cli), + refusal: None, + last_model: None, requested_model: requested_model.map(String::from), - ..Self::default() } } - pub fn observe(&mut self, cli: &str, raw: &str) -> Option { - let Ok(event) = serde_json::from_str::(raw) else { - return None; - }; - match (cli, event["type"].as_str()) { - ("claude", Some("system")) - if event["subtype"] == "model_refusal_no_fallback" - && event["api_refusal_category"] == "cyber" => - { - self.refusal = Some(SafeguardRefusal::ClaudeCyber); + pub fn observe(&mut self, raw: &str) -> Option { + let event = serde_json::from_str::(raw).ok()?; + let model = match self.harness { + Some(Harness::Claude) => { + let refused = match event["type"].as_str() { + Some("system") => { + event["subtype"] == "model_refusal_no_fallback" + && event["api_refusal_category"] == "cyber" + } + Some("result") => event["result"].as_str().is_some_and(|text| { + let text = text.trim_start(); + text.starts_with("[cyber]") + || (text.starts_with("API Error:") && text.contains("[cyber]")) + }), + _ => false, + }; + if refused { + self.refusal = Some(SafeguardRefusal::ClaudeCyber); + } + Self::answer_model(&event) } - ("claude", Some("result")) - if event["result"].as_str().is_some_and(|text| { - let text = text.trim_start(); - text.starts_with("[cyber]") - || (text.starts_with("API Error:") && text.contains("[cyber]")) - }) => - { - self.refusal = Some(SafeguardRefusal::ClaudeCyber); + Some(Harness::Grok) => Self::answer_model(&event), + Some(Harness::Codex) => { + if event["type"] == "turn.failed" + && event["error"]["message"] + .as_str() + .is_some_and(|text| text.to_ascii_lowercase().contains("cybersecurity")) + { + self.refusal = Some(SafeguardRefusal::CodexCyber); + } + if event["type"] == "thread.started" && self.last_model.is_none() { + let model = self.requested_model.as_deref().unwrap_or("Harness default"); + let basis = if self.requested_model.is_some() { + "requested" + } else { + "no Model requested" + }; + let line = format!("Model: {model} ({basis})"); + self.last_model = Some(model.to_string()); + return Some(line); + } + None } - ("codex", Some("turn.failed")) - if event["error"]["message"] - .as_str() - .is_some_and(|text| text.to_ascii_lowercase().contains("cybersecurity")) => - { - self.refusal = Some(SafeguardRefusal::CodexCyber); - } - _ => {} + Some(Harness::Agy) if event["event"] == "init" => event["init"]["model"].as_str(), + _ => None, + }?; + if model.is_empty() || self.last_model.as_deref() == Some(model) { + return None; } - // Init names the requested Model; result.modelUsage includes sub-agents. - // Only main-loop assistant messages identify the Model that answered. - if cli == "claude" - && event["type"] == "assistant" - && event["parent_tool_use_id"].is_null() - && let Some(model) = event["message"]["model"].as_str() - && !model.is_empty() - && self.last_model.as_deref() != Some(model) + self.last_model = Some(model.to_string()); + Some(format!("Model: {model}")) + } + + // Init names the requested Model; result.modelUsage includes sub-agents. + // Only main-loop assistant messages identify the Model that answered. + fn answer_model(event: &Value) -> Option<&str> { + if event["type"] != "assistant" + || !event["parent_tool_use_id"].is_null() + || event["is_api_error_message"] == true { - self.last_model = Some(model.to_string()); - return Some(format!("Model: {model}")); - } - if cli == "codex" && event["type"] == "thread.started" && self.last_model.is_none() { - let model = self.requested_model.as_deref().unwrap_or("Harness default"); - self.last_model = Some(model.to_string()); - let basis = if self.requested_model.is_some() { - "requested" - } else { - "no Model requested" - }; - return Some(format!("Model: {model} ({basis})")); + return None; } - None + event["message"]["model"] + .as_str() + .filter(|model| *model != "") } } diff --git a/src/harness/interpretation/stream.rs b/src/harness/interpretation/stream.rs index a8182b35..a51e4901 100644 --- a/src/harness/interpretation/stream.rs +++ b/src/harness/interpretation/stream.rs @@ -209,6 +209,7 @@ impl Decoder for Stream { report: Report { warnings: Vec::new(), summary, + ..Report::default() }, ended, outcome: TurnOutcome::from_failed(failed), diff --git a/src/harness/interpretation_tests.rs b/src/harness/interpretation_tests.rs index 05480499..e467e2df 100644 --- a/src/harness/interpretation_tests.rs +++ b/src/harness/interpretation_tests.rs @@ -1,6 +1,18 @@ use super::*; use std::os::unix::process::ExitStatusExt; +#[test] +fn agy_security_progress_names_the_resolved_model() { + let mut interpretation = stream(Harness::Agy, "").for_security(Some("gemini-3.8-flash")); + let lines = interpretation.condense( + r#"{"event":"init","conversation_id":"s1","init":{"model":"gemini-3.8-flash-high"}}"#, + ); + assert!( + lines.contains(&"Model: gemini-3.8-flash-high".to_string()), + "{lines:?}" + ); +} + #[test] fn a_security_session_can_discuss_the_cyber_marker_without_being_refused() { let mut interpretation = stream(Harness::Claude, "").for_security(None); diff --git a/src/harness/muse/log.rs b/src/harness/muse/log.rs index 46c00322..7d997fcb 100644 --- a/src/harness/muse/log.rs +++ b/src/harness/muse/log.rs @@ -8,6 +8,7 @@ use std::path::{Path, PathBuf}; #[derive(Default)] pub(in crate::harness) struct SessionLog { pub(in crate::harness) message: Option, + pub(in crate::harness) models: Vec, tokens: Option, } @@ -46,6 +47,11 @@ impl SessionLog { self.message = event["text"].as_str().map(String::from); } Some("model_completed") => { + if let Some(model) = event["model"].as_str().filter(|model| !model.is_empty()) + && self.models.last().map(String::as_str) != Some(model) + { + self.models.push(model.to_string()); + } if let Some(usage) = event["usage"].as_object() { let tokens = self.tokens.get_or_insert_default(); tokens.input += usage diff --git a/src/harness/muse/stream.rs b/src/harness/muse/stream.rs index 4ed46f95..09faba82 100644 --- a/src/harness/muse/stream.rs +++ b/src/harness/muse/stream.rs @@ -135,6 +135,7 @@ impl Decoder for MuseProgress { report: Report { warnings: self.skill_load.warnings(), summary: None, + ..Report::default() }, ended: Ended { session_id: self.session_id, diff --git a/src/harness/opencode/export.rs b/src/harness/opencode/export.rs index f3628df1..50d76ad6 100644 --- a/src/harness/opencode/export.rs +++ b/src/harness/opencode/export.rs @@ -11,6 +11,7 @@ pub(in crate::harness) struct SessionExport { pub(in crate::harness) message: Option, pub(in crate::harness) summary: Option, pub(in crate::harness) failure: Option, + pub(in crate::harness) models: Vec, } /// OpenCode records uncached input and visible output separately from cache @@ -64,12 +65,22 @@ impl SessionExport { message: None, summary: None, failure: None, + models: Vec::new(), }; let mut usage: Option = None; for message in messages .iter() .filter(|message| message["type"] == "assistant") { + if let Some(id) = message["model"]["id"].as_str().filter(|id| !id.is_empty()) { + let model = message["model"]["providerID"] + .as_str() + .filter(|provider| !provider.is_empty()) + .map_or_else(|| id.to_string(), |provider| format!("{provider}/{id}")); + if result.models.last() != Some(&model) { + result.models.push(model); + } + } result.message = message["content"] .as_array() .and_then(|parts| parts.iter().rev().find(|part| part["type"] == "text")) diff --git a/src/harness/opencode/stream.rs b/src/harness/opencode/stream.rs index 9c9f0e4f..0d6fd4e1 100644 --- a/src/harness/opencode/stream.rs +++ b/src/harness/opencode/stream.rs @@ -85,6 +85,7 @@ impl Decoder for OpenCodeProgress { report: Report { warnings: self.skill_load.warnings(), summary: None, + ..Report::default() }, ended: Ended { session_id: self.session_id, diff --git a/src/session.rs b/src/session.rs index 62b0e7d0..9b763a91 100644 --- a/src/session.rs +++ b/src/session.rs @@ -293,14 +293,16 @@ impl Outside for OnMachine { prompt: &str, log: &Path, ) -> Result { - let invocation = match purpose { - Purpose::Security => self.adapter.security_session(&self.harness, resume, prompt), - Purpose::Ordinary => self.adapter.session(&self.harness, resume, prompt), - }; let interpretation = self.adapter.interpretation(&self.worktree, prompt); - let interpretation = match purpose { - Purpose::Security => interpretation.for_security(self.harness.model.as_deref()), - Purpose::Ordinary => interpretation, + let (invocation, interpretation) = match purpose { + Purpose::Security => ( + self.adapter.security_session(&self.harness, resume, prompt), + interpretation.for_security(self.harness.model.as_deref()), + ), + Purpose::Ordinary => ( + self.adapter.session(&self.harness, resume, prompt), + interpretation, + ), }; run( kind, @@ -364,6 +366,9 @@ fn run( let completion = executed.state.finish(executed.execution); if let Some(report) = completion.report { + for model in report.models { + progress::step(format_args!("{kind}: Model: {model}")); + } for warning in report.warnings { progress::step(format_args!("{kind}: {warning}")); } diff --git a/tests/security_run.rs b/tests/security_run.rs index 056cc255..1172b5b4 100644 --- a/tests/security_run.rs +++ b/tests/security_run.rs @@ -35,6 +35,31 @@ fn finding(fingerprint: &str) -> Value { }) } +fn safeguard_refusals() -> [(&'static str, &'static str, &'static str); 2] { + [ + ( + "claude", + r#"printf '%s\n' 'API Error: [cyber] Private refusal evidence.' > "$FAKE_CLAUDE_FINAL_MESSAGE""#, + "Claude Code's [cyber] safeguard refusal", + ), + ( + "codex", + r#"printf '%s\n' 'Cybersecurity safeguard refused: Private refusal evidence.' > "$FAKE_CODEX_ERROR" +exit 1"#, + "Codex's cybersecurity safeguard refusal", + ), + ] +} + +fn security_command_log(scenario: &Scenario) -> String { + let logs = scenario.entries("home/.thirdshift/logs/acme/widgets/commands/secure"); + fs::read_to_string(scenario.path(&format!( + "home/.thirdshift/logs/acme/widgets/commands/secure/{}", + logs[0] + ))) + .unwrap() +} + #[test] fn a_claude_cyber_refusal_fails_the_security_audit_with_its_cause() { let scenario = Scenario::new(); @@ -59,19 +84,7 @@ printf '%s\n' 'API Error: [cyber] This request was refused by the safeguard.' > #[test] fn safeguard_refusals_notify_the_cause_without_private_diagnostics() { - for (harness, script, cause) in [ - ( - "claude", - r#"printf '%s\n' 'API Error: [cyber] Private refusal evidence.' > "$FAKE_CLAUDE_FINAL_MESSAGE""#, - "Claude Code's [cyber] safeguard refusal", - ), - ( - "codex", - r#"printf '%s\n' 'Cybersecurity safeguard refused: Private refusal evidence.' > "$FAKE_CODEX_ERROR" -exit 1"#, - "Codex's cybersecurity safeguard refusal", - ), - ] { + for (harness, script, cause) in safeguard_refusals() { let scenario = Scenario::new(); scenario.agent_does(script); let resend = ResendStandIn::replying(200, r#"{"id":"1"}"#); @@ -117,12 +130,7 @@ printf '%s\n' '{{"type":"assistant","message":{{"model":"claude-opus-4-8","conte ); let result = scenario.run(&["secure", "model", "opus"]); assert_eq!(result.code, Some(0), "{}", result.stderr); - let logs = scenario.entries("home/.thirdshift/logs/acme/widgets/commands/secure"); - let command_log = fs::read_to_string(scenario.path(&format!( - "home/.thirdshift/logs/acme/widgets/commands/secure/{}", - logs[0] - ))) - .unwrap(); + let command_log = security_command_log(&scenario); for line in [ "security-audit: Model: claude-opus-5-5", "security-audit: Model: claude-opus-4-8", @@ -154,12 +162,7 @@ fn codex_logs_the_requested_model_for_each_security_session_and_resume() { scenario.agent_does_in_session(3, &reproduction_script("not reproduced")); let result = scenario.run(&["secure", "harness", "codex", "model", "GPT-6.1-Sol"]); assert_eq!(result.code, Some(0), "{}", result.stderr); - let logs = scenario.entries("home/.thirdshift/logs/acme/widgets/commands/secure"); - let command_log = fs::read_to_string(scenario.path(&format!( - "home/.thirdshift/logs/acme/widgets/commands/secure/{}", - logs[0] - ))) - .unwrap(); + let command_log = security_command_log(&scenario); for kind in [ "security-audit", "security-audit-resume", @@ -173,19 +176,7 @@ fn codex_logs_the_requested_model_for_each_security_session_and_resume() { #[test] fn refused_security_reproductions_keep_the_record_and_stop_before_the_next_finding() { - for (harness, script, cause) in [ - ( - "claude", - r#"printf '%s\n' 'API Error: [cyber] Private refusal evidence.' > "$FAKE_CLAUDE_FINAL_MESSAGE""#, - "Claude Code's [cyber] safeguard refusal", - ), - ( - "codex", - r#"printf '%s\n' 'Cybersecurity safeguard refused: Private refusal evidence.' > "$FAKE_CODEX_ERROR" -exit 1"#, - "Codex's cybersecurity safeguard refusal", - ), - ] { + for (harness, script, cause) in safeguard_refusals() { let scenario = Scenario::new(); scenario.agent_does_in_session( 1, @@ -2006,3 +1997,117 @@ fn a_reproduction_failure_notifies_recorded_metadata_without_private_evidence() assert_eq!(scenario.claude_calls().len(), 3); } } + +#[test] +fn claude_security_progress_ignores_synthetic_api_errors() { + let scenario = Scenario::new(); + scenario.agent_does(&format!( + r#" +printf '%s\n' '{{"type":"assistant","message":{{"model":"claude-opus-4-8","content":[]}}}}' +printf '%s\n' '{{"type":"assistant","parent_tool_use_id":null,"is_api_error_message":true,"error":"rate_limit","message":{{"model":"","role":"assistant","content":[{{"type":"text","text":"API Error: Rate limit reached"}}]}}}}' +printf '%s\n' '{{"type":"assistant","message":{{"model":"claude-opus-4-8","content":[]}}}}' +{} +"#, + audit_script("[]") + )); + let result = scenario.run(&["secure"]); + assert_eq!(result.code, Some(0), "{}", result.stderr); + let command_log = security_command_log(&scenario); + for text in [&result.stderr, &command_log] { + assert!(!text.contains("Model: "), "{text}"); + assert_eq!( + text.matches("security-audit: Model: claude-opus-4-8") + .count(), + 1, + "{text}" + ); + } +} + +#[test] +fn grok_security_audits_log_the_model_that_answered() { + let scenario = Scenario::new(); + scenario.agent_does( + r#" +node -e 'const fs = require("fs"); const calls = JSON.parse(fs.readFileSync(process.env.FAKE_GROK_RECORD, "utf8")); process.stdout.write(calls[calls.length - 1].prompt);' > prompt.txt +output=$(sed -n 's/^Output directory: `\(.*\)`\.$/\1/p' prompt.txt) +test -n "$output" +printf '%s\n' '[]' > "$output/findings.json" +printf '%s\n' '[]' > "$output/coverage-ledger.json" +printf '%s\n' '{"run_status":"complete"}' > "$output/run-metadata.json" +printf '%s\n' '{"type":"assistant","message":{"model":"grok-4.7","content":[]}}' +printf '%s\n' 'Security audit: complete' > "$FAKE_CLAUDE_FINAL_MESSAGE" +"#, + ); + let result = scenario.run(&["secure", "harness", "grok", "model", "grok-4.7"]); + assert_eq!(result.code, Some(0), "{}", result.stderr); + let command_log = security_command_log(&scenario); + for text in [&result.stderr, &command_log] { + assert!(text.contains("security-audit: Model: grok-4.7"), "{text}"); + } +} + +#[test] +fn muse_security_audits_log_the_answering_model_from_the_session_record() { + let scenario = Scenario::new(); + scenario.agent_does(&format!( + r#" +export FAKE_CLAUDE_PROMPT="$(node -e 'const fs = require("fs"); const calls = JSON.parse(fs.readFileSync(process.env.FAKE_MUSE_RECORD, "utf8")); process.stdout.write(calls[calls.length - 1].prompt);')" +{} +root="$HOME/.local/share/muse/sessions/2026/10/06/fake-muse-1" +mkdir -p "$root" +printf '%s\n' '{{"stream":{{"id":"child"}},"payload":{{"kind":"run","event":{{"kind":"model_completed","model":"child-model"}}}}}}' '{{"stream":{{"id":"fake-muse-1"}},"payload":{{"kind":"run","event":{{"kind":"model_completed","model":"muse-spark-1.3"}}}}}}' > "$root/session.jsonl" +"#, + audit_script("[]") + )); + let result = + scenario.run_with_env(&["secure", "harness", "muse"], &[("FAKE_MUSE_NO_LOG", "1")]); + assert_eq!(result.code, Some(0), "{}", result.stderr); + assert!( + result + .stderr + .contains("security-audit: Model: muse-spark-1.3"), + "{}", + result.stderr + ); + assert!( + !result.stderr.contains("Model: child-model"), + "{}", + result.stderr + ); +} + +#[test] +fn opencode_security_audits_log_the_answering_model_from_the_export() { + let scenario = Scenario::new(); + scenario.agent_does(&format!( + r#" +export FAKE_CLAUDE_PROMPT="$(node -e 'const fs = require("fs"); const calls = JSON.parse(fs.readFileSync(process.env.FAKE_OPENCODE_RECORD, "utf8")); process.stdout.write(calls[calls.length - 1].prompt);')" +{} +"#, + audit_script("[]") + )); + let script = scenario.path("export-model.sh"); + fs::write(&script, format!( + r#"node -e 'const fs = require("fs"); const p = process.argv[1]; const record = JSON.parse(fs.readFileSync(p, "utf8")); record.info.model = {{id:"title-model",providerID:"title"}}; for (const message of record.messages) {{ if (message.type === "assistant") message.model = {{id:"MiMo-V2.6-Pro",providerID:"primalabs"}}; }} fs.writeFileSync(p, JSON.stringify(record));' "{}" +"#, + scenario.path("opencode-calls.fake-opencode-1.export.json").display() + )).unwrap(); + let result = scenario.run_with_env( + &["secure", "harness", "opencode"], + &[("FAKE_OPENCODE_EXPORT_SCRIPT", script.to_str().unwrap())], + ); + assert_eq!(result.code, Some(0), "{}", result.stderr); + assert!( + result + .stderr + .contains("security-audit: Model: primalabs/MiMo-V2.6-Pro"), + "{}", + result.stderr + ); + assert!( + !result.stderr.contains("Model: title/"), + "{}", + result.stderr + ); +}