From a4cb7caef6da5bce3f58dfc43c7230a6ace0bdb3 Mon Sep 17 00:00:00 2001 From: Yaacov Date: Sat, 14 Nov 2026 14:33:08 +0200 Subject: [PATCH 1/2] feat(rpc): let a host report a delegated turn it ran elsewhere With hostDelegation the host sends each spoken request as the user's message, and the live session read that turn's progress and answer from its own session. That ties a call to the conversation's agent: no call before the conversation has one, or after it stopped. live_report ({ text, final? }) lets the host run the turn anywhere and report it instead: progress as commentary, and with final the answer, which ends the delegation. A report without an active delegation is dropped. Scient-Divergence: SA-LIVE-HOST-DELEGATION Co-Authored-By: Claude Opus 5.5 --- SCIENT.md | 2 +- packages/coding-agent/src/live/controller.ts | 19 +++++++++++++++++++ .../coding-agent/src/modes/rpc/rpc-live.ts | 14 +++++++++++++- .../coding-agent/src/modes/rpc/rpc-mode.ts | 12 ++++++++++++ .../coding-agent/src/modes/rpc/rpc-types.ts | 4 ++++ .../src/modes/rpc/wire/commands.ts | 7 +++++++ .../test/scient/live-host-delegation.test.ts | 18 ++++++++++++++++++ scripts/scient/divergences.json | 2 +- 8 files changed, 75 insertions(+), 3 deletions(-) diff --git a/SCIENT.md b/SCIENT.md index 6f11629ef8..2116b3317d 100644 --- a/SCIENT.md +++ b/SCIENT.md @@ -26,7 +26,7 @@ Everything else is upstream's code and behavior. | Usage-limit reset | A `prompt_result` for a turn stopped by a usage limit carries `error.resetAt`, the time the provider said the limit resets (epoch ms): from its retry hint, the account's usage report, or an earlier stated block, whichever is latest. It is left out when the provider gave no time; the 30-minute guess for a hintless error is never reported. A host can resume the turn at that time. | | Failure kinds | A `prompt_result` for a failed turn carries `error.kinds`, the error classifier's labels for that failure (`auth-failed`, `context-overflow`, `transient`, `usage-limit`, ...), so a host can report why turns fail. It is left out when the classifier recognized no kind; the labels carry no provider text. | | macOS Accessibility prompt | The agent sends no Accessibility message while macOS doesn't trust it, so a computer call that refuses for lack of a permission never makes macOS ask. Only the host asks for permissions. | -| Live voice delegation | `live_connect` with `hostDelegation` sends each request the voice model delegates to the host as a `live_delegation` frame instead of starting a turn, so a host that owns its turns (Scient Desktop) sends it as the user's message. The live session still reads that turn's answer. Without the flag nothing changes. | +| Live voice delegation | `live_connect` with `hostDelegation` sends each request the voice model delegates to the host as a `live_delegation` frame instead of starting a turn, so a host that owns its turns (Scient Desktop) sends it as the user's message. The live session still reads that turn's answer. A host that runs the turn elsewhere, such as in the conversation's own agent while the call runs in a separate one, reports its progress and answer with `live_report`. Without the flag nothing changes. | | Dictation | `scient-agent dictate ` transcribes a recording with ChatGPT's dictation service, the one the ChatGPT and Codex apps use, through the stored ChatGPT sign-in. With `--json` it prints `{ "text" }`, or `{ "error": { "kind", "message" } }` and exits 1, so a host (Scient Desktop's microphone) can show its own message. ChatGPT keeps the uploaded recording as a file for 30 days, as it does for its own apps' dictation. | | HTML exports | Oh My Pi's binaries load the export viewer from its CDN. Scient's viewer differs by its wording, so a compiled Scient Agent carries its viewer inside the exported page. The page still loads two pinned libraries from `stencil.so`. | | Stats dashboard | The dashboard's tab icon is Scient Agent's symbol. With `SCIENT_AGENT_HOST_PAGES=1` the agent reports the dashboard's address and opens no system browser, so a host can show it. | diff --git a/packages/coding-agent/src/live/controller.ts b/packages/coding-agent/src/live/controller.ts index 90c6e093e2..c11ffe57f9 100644 --- a/packages/coding-agent/src/live/controller.ts +++ b/packages/coding-agent/src/live/controller.ts @@ -246,6 +246,25 @@ export class LiveSessionController { throw error; } } + // scient(SA-LIVE-HOST-DELEGATION): begin - A host that runs the delegated turn reports its progress and answer. + + /** + * Gives the voice model the host's account of the delegated request's turn: its progress, or + * with `final` its answer, which ends the delegation. Without an active delegation it is dropped. + */ + reportHostWork(text: string, final: boolean): void { + const delegationId = this.#activeDelegationId; + const report = text.trim(); + if (!delegationId || !report) return; + const content = final ? prompt.render(agentFinalMessageTemplate, { message: report }) : report; + for (const chunk of chunkLiveContext(content)) { + this.#queueSend(buildDelegationContextAppend(delegationId, chunk, final ? undefined : "commentary")); + } + if (!final) return; + this.#activeDelegationId = undefined; + this.#refreshAudioPhase(); + } + // scient(SA-LIVE-HOST-DELEGATION): end /** Toggles microphone capture while leaving output and the session connected. */ toggleMute(): void { diff --git a/packages/coding-agent/src/modes/rpc/rpc-live.ts b/packages/coding-agent/src/modes/rpc/rpc-live.ts index efb142911c..34bb03bbc5 100644 --- a/packages/coding-agent/src/modes/rpc/rpc-live.ts +++ b/packages/coding-agent/src/modes/rpc/rpc-live.ts @@ -17,7 +17,10 @@ import { hostLiveDelegate } from "../../scient/live-host-delegation"; export const RPC_LIVE_LEVELS_INTERVAL_MS = 100; /** The controller surface the RPC bridge drives. */ -export type RpcLiveSession = Pick; +// scient(SA-LIVE-HOST-DELEGATION): begin - A host that runs the delegated turn reports its progress and answer. +export type RpcLiveSession = Pick & + Partial>; +// scient(SA-LIVE-HOST-DELEGATION): end /** Builds the controller for one live session; tests substitute a fake. */ export type RpcLiveSessionFactory = (options: LiveSessionControllerOptions) => RpcLiveSession; @@ -148,6 +151,15 @@ export class RpcLiveBridge { if (muted === undefined || muted !== controller.muted) controller.toggleMute(); return { muted: controller.muted }; } + // scient(SA-LIVE-HOST-DELEGATION): begin - A host that runs the delegated turn reports its progress and answer. + + /** Passes the host's report of the delegated turn to the active session. */ + report(text: string, final: boolean): void { + const controller = this.#controller; + if (!controller || this.#closing) throw new Error("No live session is active"); + controller.reportHostWork?.(text, final); + } + // scient(SA-LIVE-HOST-DELEGATION): end /** Stops `controller` and releases the slot once it has fully stopped. Idempotent. */ #retire(controller: RpcLiveSession): Promise { diff --git a/packages/coding-agent/src/modes/rpc/rpc-mode.ts b/packages/coding-agent/src/modes/rpc/rpc-mode.ts index 4cb6a4ff94..6d76396f75 100644 --- a/packages/coding-agent/src/modes/rpc/rpc-mode.ts +++ b/packages/coding-agent/src/modes/rpc/rpc-mode.ts @@ -2450,6 +2450,18 @@ export async function runRpcMode(session: AgentSession, options: RpcModeOptions return error(id, command.type, err instanceof Error ? err.message : String(err)); } } + // scient(SA-LIVE-HOST-DELEGATION): begin - A host that runs the delegated turn reports its progress and answer. + + case "live_report": { + if (typeof command.text !== "string") return error(id, "live_report", "live_report requires text"); + try { + liveBridge.report(command.text, command.final === true); + return success(id, "live_report"); + } catch (err) { + return error(id, "live_report", err instanceof Error ? err.message : String(err)); + } + } + // scient(SA-LIVE-HOST-DELEGATION): end case "live_stop": { await liveBridge.stop(); diff --git a/packages/coding-agent/src/modes/rpc/rpc-types.ts b/packages/coding-agent/src/modes/rpc/rpc-types.ts index 3df9f87b9e..d09bf3e738 100644 --- a/packages/coding-agent/src/modes/rpc/rpc-types.ts +++ b/packages/coding-agent/src/modes/rpc/rpc-types.ts @@ -106,6 +106,7 @@ export type RpcCommand = instructions?: string; hostDelegation?: boolean; } + | { id?: string; type: "live_report"; text: string; final?: boolean } // scient(SA-LIVE-HOST-DELEGATION): end | { id?: string; type: "live_stop" } | { id?: string; type: "live_mute"; muted?: boolean } @@ -670,6 +671,9 @@ export type RpcResponse = success: true; data: { voice: string; sdpAnswer: string }; } + // scient(SA-LIVE-HOST-DELEGATION): begin - A host that runs the delegated turn reports its progress and answer. + | { id?: string; type: "response"; command: "live_report"; success: true } + // scient(SA-LIVE-HOST-DELEGATION): end | { id?: string; type: "response"; command: "live_stop"; success: true } | { id?: string; type: "response"; command: "live_mute"; success: true; data: { muted: boolean } } // Draft-only speech-to-text diff --git a/packages/coding-agent/src/modes/rpc/wire/commands.ts b/packages/coding-agent/src/modes/rpc/wire/commands.ts index 6494ed0658..e67e811d7f 100644 --- a/packages/coding-agent/src/modes/rpc/wire/commands.ts +++ b/packages/coding-agent/src/modes/rpc/wire/commands.ts @@ -213,6 +213,13 @@ export const rpcCommands: readonly RpcCommandSpec[] = [ // scient(SA-LIVE-HOST-DELEGATION): end result: { voice: "string", sdpAnswer: "string" }, }, + // scient(SA-LIVE-HOST-DELEGATION): begin - A host that runs the delegated turn reports its progress and answer. + { + name: "live_report", + doc: "Report the delegated request's progress, or with `final` its answer, to the live voice session.", + params: { text: "string", "final?": "boolean" }, + }, + // scient(SA-LIVE-HOST-DELEGATION): end { name: "live_stop", doc: "Stop the live voice session, if any." }, { name: "live_mute", diff --git a/packages/coding-agent/test/scient/live-host-delegation.test.ts b/packages/coding-agent/test/scient/live-host-delegation.test.ts index 1c8e6eadc1..7808d1e9d3 100644 --- a/packages/coding-agent/test/scient/live-host-delegation.test.ts +++ b/packages/coding-agent/test/scient/live-host-delegation.test.ts @@ -102,6 +102,24 @@ describe("live host delegation", () => { await controller.stop(); }); + it("speaks the host's report of a turn it ran elsewhere, and only during a delegation", async () => { + const { controller, emit, sentToModel } = await startController({ delegate: () => {} }); + controller.reportHostWork("nothing is delegated yet", true); + expect(sentToModel).toEqual([]); + emit(delegationCreated("d1", "run the analysis")); + controller.reportHostWork("Loading the cohort", false); + controller.reportHostWork("The hazard ratio is 1.4", true); + controller.reportHostWork("a late report", false); + await Bun.sleep(0); + const spoken = JSON.stringify(sentToModel); + expect(spoken).toContain('"channel":"commentary"'); + expect(spoken).toContain("Loading the cohort"); + expect(spoken).toContain("The hazard ratio is 1.4"); + expect(spoken).not.toContain("a late report"); + expect(controller.phase).not.toBe("working"); + await controller.stop(); + }); + it("starts the turn itself when no delegate is set", async () => { const { controller, emit, sentToSession } = await startController({}); emit(delegationCreated("d1", "list the files")); diff --git a/scripts/scient/divergences.json b/scripts/scient/divergences.json index 5a99606ab0..ac465cceb8 100644 --- a/scripts/scient/divergences.json +++ b/scripts/scient/divergences.json @@ -333,7 +333,7 @@ }, "SA-LIVE-HOST-DELEGATION": { "status": "active", - "contract": "live_connect accepts hostDelegation. When it is true, each request the realtime model delegates goes to the host as a live_delegation frame ({ text }) instead of starting a turn in the session; the host sends it as the user's message, and the live session reads that turn's progress and final answer as before. Without the flag, delegation is unchanged.", + "contract": "live_connect accepts hostDelegation. When it is true, each request the realtime model delegates goes to the host as a live_delegation frame ({ text }) instead of starting a turn in the session; the host sends it as the user's message, and the live session reads that turn's progress and final answer as before. A host that runs the turn somewhere else, such as another session, reports it with live_report ({ text, final? }): progress as commentary, and with final the answer, which ends the delegation; a report without an active delegation is dropped. Without the flag, delegation is unchanged.", "owned": [ "packages/coding-agent/src/scient/live-host-delegation.ts", "packages/coding-agent/test/scient/live-host-delegation.test.ts" From 49b8e9e9728b961f6dc7fcd5e98d5669ecb65957 Mon Sep 17 00:00:00 2001 From: Yaacov Date: Sat, 14 Nov 2026 14:33:08 +0200 Subject: [PATCH 2/2] chore(rpc): regenerate clients for live_report (generated) Scient-Generator: rpc Co-Authored-By: Claude Opus 5.5 --- .../src/modes/rpc/wire/rpc-wire.generated.ts | 6 +++++ .../src/modes/rpc/wire/rpc-wire.schema.json | 25 ++++++++++++++++++ sdk/go/omp-rpc/wire.go | 11 ++++++++ sdk/python/omp-rpc/src/omp_rpc/_wire.py | 8 ++++++ sdk/rust/omp-rpc/src/wire.rs | 26 +++++++++++++++++++ 5 files changed, 76 insertions(+) diff --git a/packages/coding-agent/src/modes/rpc/wire/rpc-wire.generated.ts b/packages/coding-agent/src/modes/rpc/wire/rpc-wire.generated.ts index 255e0c0b71..efaf398693 100644 --- a/packages/coding-agent/src/modes/rpc/wire/rpc-wire.generated.ts +++ b/packages/coding-agent/src/modes/rpc/wire/rpc-wire.generated.ts @@ -1754,6 +1754,11 @@ export interface LiveConnectResult { sdpAnswer: string; } +export interface LiveReportParams { + text: string; + final?: boolean; +} + export interface LiveMuteParams { muted?: boolean; } @@ -1997,6 +2002,7 @@ export interface RpcWireCommands { steer_subagent: { params: SteerSubagentParams; result: undefined }; live_start: { params: LiveStartParams; result: LiveStartResult }; live_connect: { params: LiveConnectParams; result: LiveConnectResult }; + live_report: { params: LiveReportParams; result: undefined }; live_stop: { params: undefined; result: undefined }; live_mute: { params: LiveMuteParams; result: LiveMuteResult }; dictation_start: { params: DictationStartParams; result: undefined }; diff --git a/packages/coding-agent/src/modes/rpc/wire/rpc-wire.schema.json b/packages/coding-agent/src/modes/rpc/wire/rpc-wire.schema.json index cba439450c..ec6c64dcf9 100644 --- a/packages/coding-agent/src/modes/rpc/wire/rpc-wire.schema.json +++ b/packages/coding-agent/src/modes/rpc/wire/rpc-wire.schema.json @@ -352,6 +352,17 @@ "completion": null, "clientOmit": [] }, + { + "name": "live_report", + "doc": "Report the delegated request's progress, or with `final` its answer, to the live voice session.", + "params": "LiveReportParams", + "result": null, + "nullable": false, + "unwrap": null, + "timeoutMs": null, + "completion": null, + "clientOmit": [] + }, { "name": "live_stop", "doc": "Stop the live voice session, if any.", @@ -6804,6 +6815,20 @@ "sdpAnswer" ] }, + "LiveReportParams": { + "type": "object", + "properties": { + "text": { + "type": "string" + }, + "final": { + "type": "boolean" + } + }, + "required": [ + "text" + ] + }, "LiveMuteParams": { "type": "object", "properties": { diff --git a/sdk/go/omp-rpc/wire.go b/sdk/go/omp-rpc/wire.go index 2cbdcce589..199fe673e7 100644 --- a/sdk/go/omp-rpc/wire.go +++ b/sdk/go/omp-rpc/wire.go @@ -8317,6 +8317,17 @@ func (c Commands) LiveConnect(ctx context.Context, p LiveConnectCommand) (LiveCo return out, err } +// LiveReportCommand holds the parameters of "live_report". +type LiveReportCommand struct { + Text string `json:"text"` + Final *bool `json:"final,omitempty"` +} + +// LiveReport sends "live_report": Report the delegated request's progress, or with `final` its answer, to the live voice session. +func (c Commands) LiveReport(ctx context.Context, p LiveReportCommand) error { + return c.call(ctx, "live_report", p, 0, nil) +} + // LiveStop sends "live_stop": Stop the live voice session, if any. func (c Commands) LiveStop(ctx context.Context) error { return c.call(ctx, "live_stop", nil, 0, nil) diff --git a/sdk/python/omp-rpc/src/omp_rpc/_wire.py b/sdk/python/omp-rpc/src/omp_rpc/_wire.py index 3a4ba5ddac..c3076046e8 100644 --- a/sdk/python/omp-rpc/src/omp_rpc/_wire.py +++ b/sdk/python/omp-rpc/src/omp_rpc/_wire.py @@ -3565,6 +3565,14 @@ def live_connect(self, sdp_offer: str, *, voice: str | None = None, instructions params["hostDelegation"] = host_delegation return parse_live_connect_result(self._command("live_connect", params), "live_connect") + def live_report(self, text: str, *, final: bool | None = None) -> None: + """Report the delegated request's progress, or with `final` its answer, to the live voice session.""" + params: dict[str, object] = {} + params["text"] = text + if final is not None: + params["final"] = final + self._command("live_report", params) + def live_stop(self) -> None: """Stop the live voice session, if any.""" params: dict[str, object] = {} diff --git a/sdk/rust/omp-rpc/src/wire.rs b/sdk/rust/omp-rpc/src/wire.rs index 9922f21686..233d10d16f 100644 --- a/sdk/rust/omp-rpc/src/wire.rs +++ b/sdk/rust/omp-rpc/src/wire.rs @@ -5512,6 +5512,13 @@ pub struct LiveConnectResult { pub sdp_answer: String, } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct LiveReportParams { + pub text: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub r#final: Option, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct LiveMuteParams { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -6797,6 +6804,25 @@ impl Command for LiveConnectCommand { } } +/// Report the delegated request's progress, or with `final` its answer, to the live voice session. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct LiveReportCommand { + pub text: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub r#final: Option, +} + +impl Command for LiveReportCommand { + const NAME: &'static str = "live_report"; + const TIMEOUT_MS: Option = None; + type Output = (); + + fn decode(data: Option) -> Result { + let _ = data; + Ok(()) + } +} + /// Stop the live voice session, if any. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)] pub struct LiveStopCommand {}