diff --git a/SCIENT.md b/SCIENT.md index 6f11629ef8..2116b3317d 100644 --- a/SCIENT.md +++ b/SCIENT.md @@ -26,7 +26,7 @@ Everything else is upstream's code and behavior. | Usage-limit reset | A `prompt_result` for a turn stopped by a usage limit carries `error.resetAt`, the time the provider said the limit resets (epoch ms): from its retry hint, the account's usage report, or an earlier stated block, whichever is latest. It is left out when the provider gave no time; the 30-minute guess for a hintless error is never reported. A host can resume the turn at that time. | | Failure kinds | A `prompt_result` for a failed turn carries `error.kinds`, the error classifier's labels for that failure (`auth-failed`, `context-overflow`, `transient`, `usage-limit`, ...), so a host can report why turns fail. It is left out when the classifier recognized no kind; the labels carry no provider text. | | macOS Accessibility prompt | The agent sends no Accessibility message while macOS doesn't trust it, so a computer call that refuses for lack of a permission never makes macOS ask. Only the host asks for permissions. | -| Live voice delegation | `live_connect` with `hostDelegation` sends each request the voice model delegates to the host as a `live_delegation` frame instead of starting a turn, so a host that owns its turns (Scient Desktop) sends it as the user's message. The live session still reads that turn's answer. Without the flag nothing changes. | +| Live voice delegation | `live_connect` with `hostDelegation` sends each request the voice model delegates to the host as a `live_delegation` frame instead of starting a turn, so a host that owns its turns (Scient Desktop) sends it as the user's message. The live session still reads that turn's answer. A host that runs the turn elsewhere, such as in the conversation's own agent while the call runs in a separate one, reports its progress and answer with `live_report`. Without the flag nothing changes. | | Dictation | `scient-agent dictate ` transcribes a recording with ChatGPT's dictation service, the one the ChatGPT and Codex apps use, through the stored ChatGPT sign-in. With `--json` it prints `{ "text" }`, or `{ "error": { "kind", "message" } }` and exits 1, so a host (Scient Desktop's microphone) can show its own message. ChatGPT keeps the uploaded recording as a file for 30 days, as it does for its own apps' dictation. | | HTML exports | Oh My Pi's binaries load the export viewer from its CDN. Scient's viewer differs by its wording, so a compiled Scient Agent carries its viewer inside the exported page. The page still loads two pinned libraries from `stencil.so`. | | Stats dashboard | The dashboard's tab icon is Scient Agent's symbol. With `SCIENT_AGENT_HOST_PAGES=1` the agent reports the dashboard's address and opens no system browser, so a host can show it. | diff --git a/packages/coding-agent/src/live/controller.ts b/packages/coding-agent/src/live/controller.ts index 90c6e093e2..c11ffe57f9 100644 --- a/packages/coding-agent/src/live/controller.ts +++ b/packages/coding-agent/src/live/controller.ts @@ -246,6 +246,25 @@ export class LiveSessionController { throw error; } } + // scient(SA-LIVE-HOST-DELEGATION): begin - A host that runs the delegated turn reports its progress and answer. + + /** + * Gives the voice model the host's account of the delegated request's turn: its progress, or + * with `final` its answer, which ends the delegation. Without an active delegation it is dropped. + */ + reportHostWork(text: string, final: boolean): void { + const delegationId = this.#activeDelegationId; + const report = text.trim(); + if (!delegationId || !report) return; + const content = final ? prompt.render(agentFinalMessageTemplate, { message: report }) : report; + for (const chunk of chunkLiveContext(content)) { + this.#queueSend(buildDelegationContextAppend(delegationId, chunk, final ? undefined : "commentary")); + } + if (!final) return; + this.#activeDelegationId = undefined; + this.#refreshAudioPhase(); + } + // scient(SA-LIVE-HOST-DELEGATION): end /** Toggles microphone capture while leaving output and the session connected. */ toggleMute(): void { diff --git a/packages/coding-agent/src/modes/rpc/rpc-live.ts b/packages/coding-agent/src/modes/rpc/rpc-live.ts index efb142911c..34bb03bbc5 100644 --- a/packages/coding-agent/src/modes/rpc/rpc-live.ts +++ b/packages/coding-agent/src/modes/rpc/rpc-live.ts @@ -17,7 +17,10 @@ import { hostLiveDelegate } from "../../scient/live-host-delegation"; export const RPC_LIVE_LEVELS_INTERVAL_MS = 100; /** The controller surface the RPC bridge drives. */ -export type RpcLiveSession = Pick; +// scient(SA-LIVE-HOST-DELEGATION): begin - A host that runs the delegated turn reports its progress and answer. +export type RpcLiveSession = Pick & + Partial>; +// scient(SA-LIVE-HOST-DELEGATION): end /** Builds the controller for one live session; tests substitute a fake. */ export type RpcLiveSessionFactory = (options: LiveSessionControllerOptions) => RpcLiveSession; @@ -148,6 +151,15 @@ export class RpcLiveBridge { if (muted === undefined || muted !== controller.muted) controller.toggleMute(); return { muted: controller.muted }; } + // scient(SA-LIVE-HOST-DELEGATION): begin - A host that runs the delegated turn reports its progress and answer. + + /** Passes the host's report of the delegated turn to the active session. */ + report(text: string, final: boolean): void { + const controller = this.#controller; + if (!controller || this.#closing) throw new Error("No live session is active"); + controller.reportHostWork?.(text, final); + } + // scient(SA-LIVE-HOST-DELEGATION): end /** Stops `controller` and releases the slot once it has fully stopped. Idempotent. */ #retire(controller: RpcLiveSession): Promise { diff --git a/packages/coding-agent/src/modes/rpc/rpc-mode.ts b/packages/coding-agent/src/modes/rpc/rpc-mode.ts index 4cb6a4ff94..6d76396f75 100644 --- a/packages/coding-agent/src/modes/rpc/rpc-mode.ts +++ b/packages/coding-agent/src/modes/rpc/rpc-mode.ts @@ -2450,6 +2450,18 @@ export async function runRpcMode(session: AgentSession, options: RpcModeOptions return error(id, command.type, err instanceof Error ? err.message : String(err)); } } + // scient(SA-LIVE-HOST-DELEGATION): begin - A host that runs the delegated turn reports its progress and answer. + + case "live_report": { + if (typeof command.text !== "string") return error(id, "live_report", "live_report requires text"); + try { + liveBridge.report(command.text, command.final === true); + return success(id, "live_report"); + } catch (err) { + return error(id, "live_report", err instanceof Error ? err.message : String(err)); + } + } + // scient(SA-LIVE-HOST-DELEGATION): end case "live_stop": { await liveBridge.stop(); diff --git a/packages/coding-agent/src/modes/rpc/rpc-types.ts b/packages/coding-agent/src/modes/rpc/rpc-types.ts index 3df9f87b9e..d09bf3e738 100644 --- a/packages/coding-agent/src/modes/rpc/rpc-types.ts +++ b/packages/coding-agent/src/modes/rpc/rpc-types.ts @@ -106,6 +106,7 @@ export type RpcCommand = instructions?: string; hostDelegation?: boolean; } + | { id?: string; type: "live_report"; text: string; final?: boolean } // scient(SA-LIVE-HOST-DELEGATION): end | { id?: string; type: "live_stop" } | { id?: string; type: "live_mute"; muted?: boolean } @@ -670,6 +671,9 @@ export type RpcResponse = success: true; data: { voice: string; sdpAnswer: string }; } + // scient(SA-LIVE-HOST-DELEGATION): begin - A host that runs the delegated turn reports its progress and answer. + | { id?: string; type: "response"; command: "live_report"; success: true } + // scient(SA-LIVE-HOST-DELEGATION): end | { id?: string; type: "response"; command: "live_stop"; success: true } | { id?: string; type: "response"; command: "live_mute"; success: true; data: { muted: boolean } } // Draft-only speech-to-text diff --git a/packages/coding-agent/src/modes/rpc/wire/commands.ts b/packages/coding-agent/src/modes/rpc/wire/commands.ts index 6494ed0658..e67e811d7f 100644 --- a/packages/coding-agent/src/modes/rpc/wire/commands.ts +++ b/packages/coding-agent/src/modes/rpc/wire/commands.ts @@ -213,6 +213,13 @@ export const rpcCommands: readonly RpcCommandSpec[] = [ // scient(SA-LIVE-HOST-DELEGATION): end result: { voice: "string", sdpAnswer: "string" }, }, + // scient(SA-LIVE-HOST-DELEGATION): begin - A host that runs the delegated turn reports its progress and answer. + { + name: "live_report", + doc: "Report the delegated request's progress, or with `final` its answer, to the live voice session.", + params: { text: "string", "final?": "boolean" }, + }, + // scient(SA-LIVE-HOST-DELEGATION): end { name: "live_stop", doc: "Stop the live voice session, if any." }, { name: "live_mute", diff --git a/packages/coding-agent/src/modes/rpc/wire/rpc-wire.generated.ts b/packages/coding-agent/src/modes/rpc/wire/rpc-wire.generated.ts index 255e0c0b71..efaf398693 100644 --- a/packages/coding-agent/src/modes/rpc/wire/rpc-wire.generated.ts +++ b/packages/coding-agent/src/modes/rpc/wire/rpc-wire.generated.ts @@ -1754,6 +1754,11 @@ export interface LiveConnectResult { sdpAnswer: string; } +export interface LiveReportParams { + text: string; + final?: boolean; +} + export interface LiveMuteParams { muted?: boolean; } @@ -1997,6 +2002,7 @@ export interface RpcWireCommands { steer_subagent: { params: SteerSubagentParams; result: undefined }; live_start: { params: LiveStartParams; result: LiveStartResult }; live_connect: { params: LiveConnectParams; result: LiveConnectResult }; + live_report: { params: LiveReportParams; result: undefined }; live_stop: { params: undefined; result: undefined }; live_mute: { params: LiveMuteParams; result: LiveMuteResult }; dictation_start: { params: DictationStartParams; result: undefined }; diff --git a/packages/coding-agent/src/modes/rpc/wire/rpc-wire.schema.json b/packages/coding-agent/src/modes/rpc/wire/rpc-wire.schema.json index cba439450c..ec6c64dcf9 100644 --- a/packages/coding-agent/src/modes/rpc/wire/rpc-wire.schema.json +++ b/packages/coding-agent/src/modes/rpc/wire/rpc-wire.schema.json @@ -352,6 +352,17 @@ "completion": null, "clientOmit": [] }, + { + "name": "live_report", + "doc": "Report the delegated request's progress, or with `final` its answer, to the live voice session.", + "params": "LiveReportParams", + "result": null, + "nullable": false, + "unwrap": null, + "timeoutMs": null, + "completion": null, + "clientOmit": [] + }, { "name": "live_stop", "doc": "Stop the live voice session, if any.", @@ -6804,6 +6815,20 @@ "sdpAnswer" ] }, + "LiveReportParams": { + "type": "object", + "properties": { + "text": { + "type": "string" + }, + "final": { + "type": "boolean" + } + }, + "required": [ + "text" + ] + }, "LiveMuteParams": { "type": "object", "properties": { diff --git a/packages/coding-agent/test/scient/live-host-delegation.test.ts b/packages/coding-agent/test/scient/live-host-delegation.test.ts index 1c8e6eadc1..7808d1e9d3 100644 --- a/packages/coding-agent/test/scient/live-host-delegation.test.ts +++ b/packages/coding-agent/test/scient/live-host-delegation.test.ts @@ -102,6 +102,24 @@ describe("live host delegation", () => { await controller.stop(); }); + it("speaks the host's report of a turn it ran elsewhere, and only during a delegation", async () => { + const { controller, emit, sentToModel } = await startController({ delegate: () => {} }); + controller.reportHostWork("nothing is delegated yet", true); + expect(sentToModel).toEqual([]); + emit(delegationCreated("d1", "run the analysis")); + controller.reportHostWork("Loading the cohort", false); + controller.reportHostWork("The hazard ratio is 1.4", true); + controller.reportHostWork("a late report", false); + await Bun.sleep(0); + const spoken = JSON.stringify(sentToModel); + expect(spoken).toContain('"channel":"commentary"'); + expect(spoken).toContain("Loading the cohort"); + expect(spoken).toContain("The hazard ratio is 1.4"); + expect(spoken).not.toContain("a late report"); + expect(controller.phase).not.toBe("working"); + await controller.stop(); + }); + it("starts the turn itself when no delegate is set", async () => { const { controller, emit, sentToSession } = await startController({}); emit(delegationCreated("d1", "list the files")); diff --git a/scripts/scient/divergences.json b/scripts/scient/divergences.json index 5a99606ab0..ac465cceb8 100644 --- a/scripts/scient/divergences.json +++ b/scripts/scient/divergences.json @@ -333,7 +333,7 @@ }, "SA-LIVE-HOST-DELEGATION": { "status": "active", - "contract": "live_connect accepts hostDelegation. When it is true, each request the realtime model delegates goes to the host as a live_delegation frame ({ text }) instead of starting a turn in the session; the host sends it as the user's message, and the live session reads that turn's progress and final answer as before. Without the flag, delegation is unchanged.", + "contract": "live_connect accepts hostDelegation. When it is true, each request the realtime model delegates goes to the host as a live_delegation frame ({ text }) instead of starting a turn in the session; the host sends it as the user's message, and the live session reads that turn's progress and final answer as before. A host that runs the turn somewhere else, such as another session, reports it with live_report ({ text, final? }): progress as commentary, and with final the answer, which ends the delegation; a report without an active delegation is dropped. Without the flag, delegation is unchanged.", "owned": [ "packages/coding-agent/src/scient/live-host-delegation.ts", "packages/coding-agent/test/scient/live-host-delegation.test.ts" diff --git a/sdk/go/omp-rpc/wire.go b/sdk/go/omp-rpc/wire.go index 2cbdcce589..199fe673e7 100644 --- a/sdk/go/omp-rpc/wire.go +++ b/sdk/go/omp-rpc/wire.go @@ -8317,6 +8317,17 @@ func (c Commands) LiveConnect(ctx context.Context, p LiveConnectCommand) (LiveCo return out, err } +// LiveReportCommand holds the parameters of "live_report". +type LiveReportCommand struct { + Text string `json:"text"` + Final *bool `json:"final,omitempty"` +} + +// LiveReport sends "live_report": Report the delegated request's progress, or with `final` its answer, to the live voice session. +func (c Commands) LiveReport(ctx context.Context, p LiveReportCommand) error { + return c.call(ctx, "live_report", p, 0, nil) +} + // LiveStop sends "live_stop": Stop the live voice session, if any. func (c Commands) LiveStop(ctx context.Context) error { return c.call(ctx, "live_stop", nil, 0, nil) diff --git a/sdk/python/omp-rpc/src/omp_rpc/_wire.py b/sdk/python/omp-rpc/src/omp_rpc/_wire.py index 3a4ba5ddac..c3076046e8 100644 --- a/sdk/python/omp-rpc/src/omp_rpc/_wire.py +++ b/sdk/python/omp-rpc/src/omp_rpc/_wire.py @@ -3565,6 +3565,14 @@ def live_connect(self, sdp_offer: str, *, voice: str | None = None, instructions params["hostDelegation"] = host_delegation return parse_live_connect_result(self._command("live_connect", params), "live_connect") + def live_report(self, text: str, *, final: bool | None = None) -> None: + """Report the delegated request's progress, or with `final` its answer, to the live voice session.""" + params: dict[str, object] = {} + params["text"] = text + if final is not None: + params["final"] = final + self._command("live_report", params) + def live_stop(self) -> None: """Stop the live voice session, if any.""" params: dict[str, object] = {} diff --git a/sdk/rust/omp-rpc/src/wire.rs b/sdk/rust/omp-rpc/src/wire.rs index 9922f21686..233d10d16f 100644 --- a/sdk/rust/omp-rpc/src/wire.rs +++ b/sdk/rust/omp-rpc/src/wire.rs @@ -5512,6 +5512,13 @@ pub struct LiveConnectResult { pub sdp_answer: String, } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct LiveReportParams { + pub text: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub r#final: Option, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct LiveMuteParams { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -6797,6 +6804,25 @@ impl Command for LiveConnectCommand { } } +/// Report the delegated request's progress, or with `final` its answer, to the live voice session. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct LiveReportCommand { + pub text: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub r#final: Option, +} + +impl Command for LiveReportCommand { + const NAME: &'static str = "live_report"; + const TIMEOUT_MS: Option = None; + type Output = (); + + fn decode(data: Option) -> Result { + let _ = data; + Ok(()) + } +} + /// Stop the live voice session, if any. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)] pub struct LiveStopCommand {}