diff --git a/SCIENT.md b/SCIENT.md index 2116b3317d..acce4247f3 100644 --- a/SCIENT.md +++ b/SCIENT.md @@ -28,6 +28,7 @@ Everything else is upstream's code and behavior. | macOS Accessibility prompt | The agent sends no Accessibility message while macOS doesn't trust it, so a computer call that refuses for lack of a permission never makes macOS ask. Only the host asks for permissions. | | Live voice delegation | `live_connect` with `hostDelegation` sends each request the voice model delegates to the host as a `live_delegation` frame instead of starting a turn, so a host that owns its turns (Scient Desktop) sends it as the user's message. The live session still reads that turn's answer. A host that runs the turn elsewhere, such as in the conversation's own agent while the call runs in a separate one, reports its progress and answer with `live_report`. Without the flag nothing changes. | | Dictation | `scient-agent dictate ` transcribes a recording with ChatGPT's dictation service, the one the ChatGPT and Codex apps use, through the stored ChatGPT sign-in. With `--json` it prints `{ "text" }`, or `{ "error": { "kind", "message" } }` and exits 1, so a host (Scient Desktop's microphone) can show its own message. ChatGPT keeps the uploaded recording as a file for 30 days, as it does for its own apps' dictation. | +| ChatGPT apps | `scient-agent apps [--json]` lists the apps the person connected in ChatGPT (Gmail, Google Drive and the others), through the stored ChatGPT sign-in and the apps server the Codex apps use. A host that sets `SCIENT_AGENT_CHATGPT_APPS` (`all`, app ids, `-` to leave one out) gets those apps' tools in its sessions, named `__` and loaded on demand. ChatGPT's own built-in features on that server are left out. | | HTML exports | Oh My Pi's binaries load the export viewer from its CDN. Scient's viewer differs by its wording, so a compiled Scient Agent carries its viewer inside the exported page. The page still loads two pinned libraries from `stencil.so`. | | Stats dashboard | The dashboard's tab icon is Scient Agent's symbol. With `SCIENT_AGENT_HOST_PAGES=1` the agent reports the dashboard's address and opens no system browser, so a host can show it. | | RPC output spool | While a host is not reading, the agent spools the frames it cannot deliver to a temporary folder. Past 2 GiB (`PI_RPC_SPOOL_MAX_BYTES`) it stops with exit code 1 instead of filling the disk. The folder is named `scient-agent-rpc-output--…`, and an agent removes the folders of agents that were killed when it starts; other products' folders stay. Proposed to Oh My Pi as [#16139](https://github.com/can1357/oh-my-pi/issues/16139). | diff --git a/packages/coding-agent/src/cli-commands.ts b/packages/coding-agent/src/cli-commands.ts index f421eb90b7..8b44838e51 100644 --- a/packages/coding-agent/src/cli-commands.ts +++ b/packages/coding-agent/src/cli-commands.ts @@ -21,6 +21,9 @@ import type * as LaunchHelp from "./commands/launch-help"; // scient(SA-DICTATION): begin - Scient's dictation command. import { dictateHelp } from "./scient/command-help"; // scient(SA-DICTATION): end +// scient(SA-CHATGPT-APPS): begin - Scient's ChatGPT apps command. +import { appsHelp } from "./scient/command-help"; +// scient(SA-CHATGPT-APPS): end function loadLaunchHelp(): typeof LaunchHelp.launchHelp { const module: typeof LaunchHelp = require("./commands/launch-help"); @@ -289,6 +292,13 @@ export const commands: CommandEntry[] = [ help: dictateHelp, }, // scient(SA-DICTATION): end + // scient(SA-CHATGPT-APPS): begin - Scient's ChatGPT apps command. + { + name: "apps", + load: () => import("./scient/apps-command").then(m => m.default), + help: appsHelp, + }, + // scient(SA-CHATGPT-APPS): end { name: "usage", load: () => import("./commands/usage").then(m => m.default), diff --git a/packages/coding-agent/src/scient/apps-command.ts b/packages/coding-agent/src/scient/apps-command.ts new file mode 100644 index 0000000000..2540297ef8 --- /dev/null +++ b/packages/coding-agent/src/scient/apps-command.ts @@ -0,0 +1,43 @@ +/** + * `scient-agent apps`: the ChatGPT apps the stored ChatGPT sign-in has connected. With --json it + * prints `{ "apps": [{ "id", "name", "description"?, "tools", "readOnlyTools" }] }`, or + * `{ "error": { "kind", "message" } }` and exits 1, for a host that shows the apps itself. + */ +import { Command, Flags } from "@oh-my-pi/pi-utils/cli"; +import { Settings } from "../config/settings"; +import { discoverAuthStorage } from "../sdk"; +import { ChatGptAppsClient, ChatGptAppsError } from "./chatgpt-apps"; +import { appsHelp } from "./command-help"; + +export default class Apps extends Command { + static description = appsHelp.description; + static flags = { + json: Flags.boolean({ description: "Print one JSON document" }), + }; + static examples = ["scient-agent apps", "scient-agent apps --json"]; + + async run(): Promise { + const { flags } = await this.parse(Apps); + const settings = await Settings.loadReadOnly(); + const authStorage = await discoverAuthStorage(undefined, { settings }); + try { + const apps = (await new ChatGptAppsClient(authStorage).listApps()).map(app => ({ + id: app.id, + name: app.name, + ...(app.description ? { description: app.description } : {}), + tools: app.tools.length, + readOnlyTools: app.tools.filter(tool => tool.readOnly).length, + })); + if (flags.json) process.stdout.write(`${JSON.stringify({ apps })}\n`); + else for (const app of apps) process.stdout.write(`${app.name} (${app.id}): ${app.tools} tools\n`); + } catch (error) { + const kind = error instanceof ChatGptAppsError ? error.kind : "unavailable"; + const message = error instanceof ChatGptAppsError ? error.message : "Listing ChatGPT apps failed."; + if (flags.json) process.stdout.write(`${JSON.stringify({ error: { kind, message } })}\n`); + else process.stderr.write(`error: ${message}\n`); + process.exitCode = 1; + } finally { + authStorage.close(); + } + } +} diff --git a/packages/coding-agent/src/scient/chatgpt-apps-extension.ts b/packages/coding-agent/src/scient/chatgpt-apps-extension.ts new file mode 100644 index 0000000000..b01ec0bc43 --- /dev/null +++ b/packages/coding-agent/src/scient/chatgpt-apps-extension.ts @@ -0,0 +1,169 @@ +/** + * A host that wants Scient Agent to use the person's ChatGPT apps sets SCIENT_AGENT_CHATGPT_APPS + * to a comma-separated list: `all` for every app the ChatGPT sign-in has connected, an app id to + * name one, and `-` to leave one out (an app the host already reaches another way). At + * session start the agent lists the connected apps and registers the chosen ones' tools as + * discoverable tools; an app the account has not connected is skipped. Without the variable, + * nothing changes. + */ +import { createHash } from "node:crypto"; +import type { AuthStorage } from "@oh-my-pi/pi-ai"; +import { logger } from "@oh-my-pi/pi-utils"; +import type { ExtensionFactory } from "../extensibility/extensions"; +import { type ChatGptApp, ChatGptAppsClient, type ChatGptToolResult, chatGptAppConnectUrl } from "./chatgpt-apps"; + +export interface ChatGptAppsPolicy { + readonly all: boolean; + readonly include: readonly string[]; + readonly exclude: readonly string[]; +} + +const APP_ID = /^[A-Za-z0-9_-]{1,128}$/; + +/** The apps the host asked for, or undefined when it asked for none. */ +export function requestedChatGptApps(env: NodeJS.ProcessEnv = process.env): ChatGptAppsPolicy | undefined { + const include = new Set(); + const exclude = new Set(); + let all = false; + for (const raw of (env.SCIENT_AGENT_CHATGPT_APPS ?? "").split(",")) { + const token = raw.trim(); + if (token === "all") all = true; + else if (token.startsWith("-") && APP_ID.test(token.slice(1))) exclude.add(token.slice(1)); + else if (APP_ID.test(token)) include.add(token); + } + if (!all && include.size === 0) return undefined; + return { all, include: [...include].sort(), exclude: [...exclude].sort() }; +} + +export function chosenByPolicy(policy: ChatGptAppsPolicy, id: string): boolean { + return !policy.exclude.includes(id) && (policy.all || policy.include.includes(id)); +} + +/** The tool's model-facing name: `gmail.search_emails` becomes `gmail__search_emails`. */ +export function chatGptToolName(app: ChatGptApp, remoteName: string): string { + const dot = remoteName.indexOf("."); + const prefix = dot > 0 ? remoteName.slice(0, dot) : app.name; + const rest = dot > 0 ? remoteName.slice(dot + 1) : remoteName; + const clean = (value: string) => value.replace(/[^A-Za-z0-9_-]/g, "_"); + let name = `${clean(prefix).toLowerCase()}__${clean(rest)}`; + if (!/^[A-Za-z_]/.test(name)) name = `_${name}`; + if (name.length <= 64) return name; + const digest = createHash("sha256").update(`${app.id}\0${remoteName}`).digest("hex").slice(0, 8); + return `${name.slice(0, 55)}_${digest}`; +} + +/** Codex marks a call an app could not make because its link to the service needs signing in again. */ +function needsReconnect(result: ChatGptToolResult): boolean { + const meta = result._meta as Record | undefined; + const failure = meta?.connector_auth_failure as Record | undefined; + return failure?.is_auth_failure === true; +} + +function modelContent(app: ChatGptApp, result: ChatGptToolResult) { + if (needsReconnect(result)) + throw new Error( + `${app.name} needs to be connected again in ChatGPT. The person can reconnect it at ${chatGptAppConnectUrl(app)}.`, + ); + const content: Array<{ type: "text"; text: string } | { type: "image"; data: string; mimeType: string }> = []; + for (const part of result.content) { + if (part.type === "text" && typeof part.text === "string") content.push({ type: "text", text: part.text }); + else if (part.type === "image" && typeof part.data === "string" && typeof part.mimeType === "string") + content.push({ type: "image", data: part.data, mimeType: part.mimeType }); + else content.push({ type: "text", text: JSON.stringify(part) }); + } + // ChatGPT's apps put their data in structured content and only a summary in the text, so the + // model gets both, unless a text part already repeats the structured payload. + const structured = result.structuredContent; + if (structured !== undefined && structured !== null && !repeatsInText(structured, content)) + content.push({ type: "text", text: JSON.stringify(structured) }); + return content; +} + +function repeatsInText(structured: unknown, content: ReadonlyArray<{ type: string; text?: string }>): boolean { + return content.some(part => { + if (part.type !== "text" || !part.text?.trim()) return false; + try { + return Bun.deepEquals(JSON.parse(part.text), structured); + } catch { + return false; + } + }); +} + +function toolOutput(app: ChatGptApp, result: ChatGptToolResult) { + const content = modelContent(app, result); + if (result.isError === true) { + const message = content.flatMap(part => (part.type === "text" ? [part.text] : [])).join("\n"); + throw new Error(`${app.name} reported an error${message ? `: ${message}` : "."}`); + } + // Code that calls the tool (eval) reads the structured data without parsing the text. + return { + content, + details: result.structuredContent === undefined ? undefined : { structuredContent: result.structuredContent }, + }; +} + +/** + * One listing per credential store and app set, shared by a session and its subagents. Another + * store (another host's sessions in the same process) gets its own client and account. + */ +const listingsByStore = new WeakMap< + AuthStorage, + Map> +>(); + +function listing(storage: AuthStorage, policy: ChatGptAppsPolicy) { + const listings = listingsByStore.get(storage) ?? new Map(); + listingsByStore.set(storage, listings); + const key = JSON.stringify(policy); + let pending = listings.get(key); + if (!pending) { + const client = new ChatGptAppsClient(storage); + pending = client.listApps(AbortSignal.timeout(30_000)).then( + all => ({ client, apps: all.filter(app => chosenByPolicy(policy, app.id)) }), + error => { + // The next session may try again; this one goes on without the apps. + listings.delete(key); + throw error; + }, + ); + listings.set(key, pending); + } + return pending; +} + +export function createChatGptAppsExtension(storage: AuthStorage, policy: ChatGptAppsPolicy): ExtensionFactory { + return async pi => { + let found: { client: ChatGptAppsClient; apps: ChatGptApp[] }; + try { + found = await listing(storage, policy); + } catch (error) { + logger.warn("ChatGPT apps unavailable", { error: error instanceof Error ? error.message : String(error) }); + return; + } + const { client, apps } = found; + const taken = new Set(); + for (const app of apps) { + for (const tool of app.tools) { + const name = chatGptToolName(app, tool.name); + if (taken.has(name)) continue; + taken.add(name); + pi.registerTool({ + name, + label: name, + description: `${app.name}, through the person's ChatGPT account: ${tool.description}`, + parameters: tool.inputSchema as never, + // Like an MCP tool: changes stay conservative, the read-only hint only informs. + approval: "write", + mcpServerName: app.name, + mcpToolName: tool.name, + mcpReadOnlyHint: tool.readOnly, + strict: false, + async execute(_id, args, signal) { + return toolOutput(app, await client.callTool(tool.name, args, signal)); + }, + }); + } + } + }; +} diff --git a/packages/coding-agent/src/scient/chatgpt-apps.ts b/packages/coding-agent/src/scient/chatgpt-apps.ts new file mode 100644 index 0000000000..d20e5ebd6f --- /dev/null +++ b/packages/coding-agent/src/scient/chatgpt-apps.ts @@ -0,0 +1,324 @@ +/** + * ChatGPT apps through the ChatGPT sign-in. The apps a person connected in ChatGPT (Gmail, Google + * Drive and the rest) sit behind one MCP server of ChatGPT's, the one the Codex apps use. Each + * request carries the stored openai-codex sign-in, which the shared OAuth retry refreshes or + * rotates as for a model request. + */ +import { type AuthStorage, withOAuthAccess } from "@oh-my-pi/pi-ai"; +import { MissingApiKeyError, ProviderHttpError } from "@oh-my-pi/pi-ai/error"; +import { wrapFetchForProxy } from "@oh-my-pi/pi-ai/utils/proxy"; +import { + CODEX_BASE_URL, + CODEX_CLIENT_VERSION, + getCodexAccountId, + OPENAI_HEADERS, +} from "@oh-my-pi/pi-catalog/wire/codex"; + +const PROVIDER = "openai-codex"; +export const CHATGPT_APPS_URL = `${CODEX_BASE_URL}/ps/mcp`; +const ORIGINATOR = "Codex Desktop"; +const PROTOCOL_VERSION = "2025-06-18"; +/** Bounds one response: a catalog of every connected app is a few megabytes at most. */ +const MAX_RESPONSE_BYTES = 32 * 1024 * 1024; +const LIST_TIMEOUT_MS = 60_000; +/** Far above any app call; the caller's signal cancels sooner. */ +const CALL_TIMEOUT_MS = 10 * 60_000; + +export type ChatGptAppsFailureKind = + /** No ChatGPT sign-in, or ChatGPT refused it after a refresh. */ + "signed-out" | "unavailable"; + +export class ChatGptAppsError extends Error { + override readonly name = "ChatGptAppsError"; + constructor( + readonly kind: ChatGptAppsFailureKind, + message: string, + ) { + super(message); + } +} + +export interface ChatGptAppTool { + /** The name ChatGPT knows the tool by, such as `gmail.search_emails`. */ + readonly name: string; + readonly description: string; + readonly inputSchema: Record; + readonly readOnly: boolean; +} + +export interface ChatGptApp { + /** ChatGPT's app id, such as `connector_2128aebfecb84f64a069897515042a44`. */ + readonly id: string; + readonly name: string; + readonly description?: string; + readonly tools: readonly ChatGptAppTool[]; +} + +export interface ChatGptToolContent { + readonly type: string; + readonly text?: string; + readonly data?: string; + readonly mimeType?: string; +} + +export interface ChatGptToolResult { + readonly content: readonly ChatGptToolContent[]; + readonly structuredContent?: unknown; + readonly isError?: boolean; + readonly _meta?: unknown; +} + +type JsonRecord = Record; +const record = (value: unknown): JsonRecord | undefined => + typeof value === "object" && value !== null && !Array.isArray(value) ? (value as JsonRecord) : undefined; +const text = (value: unknown): string | undefined => (typeof value === "string" && value ? value : undefined); + +/** The JSON-RPC response for `id` in a JSON or event-stream body. */ +export function rpcResponse(body: string, eventStream: boolean, id: number): JsonRecord | undefined { + const messages: unknown[] = []; + if (!eventStream) messages.push(JSON.parse(body)); + else { + let data: string[] = []; + for (const line of `${body}\n\n`.split(/\r\n|\r|\n/)) { + if (line === "") { + if (data.length > 0) messages.push(JSON.parse(data.join("\n"))); + data = []; + continue; + } + const colon = line.indexOf(":"); + if ((colon === -1 ? line : line.slice(0, colon)) !== "data") continue; + const value = colon === -1 ? "" : line.slice(colon + 1); + data.push(value.startsWith(" ") ? value.slice(1) : value); + } + } + for (const message of messages.flat()) { + const envelope = record(message); + // A server request may reuse the id; only a response (no method) answers it. + if (envelope?.id === id && !("method" in envelope)) return envelope; + } + return undefined; +} + +function toolOf( + value: unknown, +): { app: { id: string; name: string; description?: string }; tool: ChatGptAppTool } | undefined { + const tool = record(value); + const meta = record(tool?._meta); + const name = text(tool?.name); + const appId = text(meta?.connector_id); + const inputSchema = record(tool?.inputSchema); + if (!name || !appId || !inputSchema) return undefined; + return { + app: { + id: appId, + name: text(meta?.connector_name) ?? appId, + ...(text(meta?.connector_description) ? { description: text(meta?.connector_description) } : {}), + }, + tool: { + name, + description: text(tool?.description) ?? name, + inputSchema, + readOnly: record(tool?.annotations)?.readOnlyHint === true, + }, + }; +} + +/** + * ChatGPT's own features (pets, safety settings, spaces, search and the like) ride on the same + * server under `connector_openai_*` ids. They are ChatGPT's, not apps the person connected. + */ +export function isChatGptOwnApp(id: string): boolean { + return id.startsWith("connector_openai_"); +} + +/** Groups a tool catalog by the app each tool belongs to, in the catalog's order. */ +export function appsFromTools(tools: readonly unknown[]): ChatGptApp[] { + const apps = new Map(); + for (const value of tools) { + const parsed = toolOf(value); + if (!parsed) continue; + const app = apps.get(parsed.app.id) ?? { ...parsed.app, tools: [] }; + app.tools.push(parsed.tool); + apps.set(parsed.app.id, app); + } + return [...apps.values()]; +} + +/** The ChatGPT page that connects an app to the person's ChatGPT account. */ +export function chatGptAppConnectUrl(app: { readonly id: string; readonly name: string }): string { + const slug = app.name + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, ""); + return `${new URL(CODEX_BASE_URL).origin}/apps/${slug}/${app.id}`; +} + +function failure(error: unknown): ChatGptAppsError { + if (error instanceof ChatGptAppsError) return error; + if (error instanceof MissingApiKeyError) + return new ChatGptAppsError("signed-out", "Sign in to ChatGPT to use its apps."); + if (error instanceof ProviderHttpError && (error.status === 401 || error.status === 403)) + return new ChatGptAppsError("signed-out", "ChatGPT refused the sign-in. Sign in to ChatGPT again."); + if (error instanceof ProviderHttpError) + return new ChatGptAppsError("unavailable", `ChatGPT apps failed (${error.status}): ${error.message}`); + return new ChatGptAppsError("unavailable", "Scient Agent could not reach ChatGPT's apps."); +} + +/** + * One MCP session with ChatGPT's apps server. The client keeps to the ChatGPT account its first + * request used: the apps it listed belong to that account, so no later call may run through + * another stored sign-in. + */ +export class ChatGptAppsClient { + #session: { account: string; id: string | undefined } | undefined; + #nextId = 0; + /** Keeps credential resolution on one stored sign-in. */ + readonly #stickiness = crypto.randomUUID(); + #account: string | undefined; + + constructor( + private readonly storage: AuthStorage, + /** Defaults to the global fetch at each request. */ + private readonly fetchImpl?: typeof fetch, + ) {} + + async #post( + access: { accessToken: string; accountId: string | undefined }, + method: string, + params: unknown, + signal: AbortSignal, + notification = false, + ): Promise { + const id = ++this.#nextId; + const headers: Record = { + Authorization: `Bearer ${access.accessToken}`, + "Content-Type": "application/json", + Accept: "application/json, text/event-stream", + "User-Agent": `${ORIGINATOR}/${CODEX_CLIENT_VERSION}`, + "MCP-Protocol-Version": PROTOCOL_VERSION, + "X-OpenAI-Product-Sku": "codex", + [OPENAI_HEADERS.ORIGINATOR]: ORIGINATOR, + [OPENAI_HEADERS.VERSION]: CODEX_CLIENT_VERSION, + }; + if (access.accountId) headers[OPENAI_HEADERS.ACCOUNT_ID] = access.accountId; + const sessionId = this.#session?.id; + if (sessionId) headers["Mcp-Session-Id"] = sessionId; + const response = await wrapFetchForProxy(this.fetchImpl ?? fetch, PROVIDER)(CHATGPT_APPS_URL, { + method: "POST", + redirect: "error", + headers, + body: JSON.stringify({ jsonrpc: "2.0", ...(notification ? {} : { id }), method, params }), + signal, + }); + const received = response.headers.get("mcp-session-id"); + if (received && this.#session) this.#session.id = received; + if (!response.ok) { + const body = (await response.text().catch(() => "")).trim().replaceAll(/\s+/g, " ").slice(0, 300); + // A lost session (404) starts a new one on the next attempt. + if (response.status === 404) this.#session = undefined; + // The shared retry reads the status: a 401 refreshes the credential once before failing. + throw new ProviderHttpError(body || response.statusText, response.status); + } + if (notification) { + await response.body?.cancel(); + return undefined; + } + const length = Number(response.headers.get("content-length") ?? 0); + if (length > MAX_RESPONSE_BYTES) + throw new ChatGptAppsError("unavailable", "ChatGPT's apps sent too large a response."); + const body = await response.text(); + if (body.length > MAX_RESPONSE_BYTES) + throw new ChatGptAppsError("unavailable", "ChatGPT's apps sent too large a response."); + const eventStream = response.headers.get("content-type")?.includes("text/event-stream") === true; + const envelope = rpcResponse(body, eventStream, id); + if (!envelope) throw new ChatGptAppsError("unavailable", "ChatGPT's apps sent no answer to the request."); + const error = record(envelope.error); + if (error) + throw new ChatGptAppsError( + "unavailable", + `ChatGPT's apps refused ${method}: ${text(error.message) ?? "error"}`, + ); + return record(envelope.result) ?? {}; + } + + /** Runs `method` in an initialized session, through the shared OAuth retry. */ + async #request(method: string, params: unknown, timeoutMs: number, signal?: AbortSignal): Promise { + const bounded = signal + ? AbortSignal.any([signal, AbortSignal.timeout(timeoutMs)]) + : AbortSignal.timeout(timeoutMs); + try { + return await withOAuthAccess( + this.storage, + PROVIDER, + async oauth => { + const access = { + accessToken: oauth.accessToken, + accountId: oauth.accountId ?? getCodexAccountId(oauth.accessToken), + }; + const account = access.accountId ?? ""; + if (this.#account !== undefined && this.#account !== account) + throw new ChatGptAppsError( + "signed-out", + "The ChatGPT sign-in changed. Start a new conversation to use its apps.", + ); + this.#account = account; + const open = async () => { + // A session belongs to one ChatGPT account; a rotated sign-in starts its own. + this.#session = { account, id: undefined }; + await this.#post( + access, + "initialize", + { + protocolVersion: PROTOCOL_VERSION, + capabilities: {}, + clientInfo: { name: "scient-agent", version: "1" }, + }, + bounded, + ); + await this.#post(access, "notifications/initialized", {}, bounded, true); + }; + const reused = this.#session?.account === account; + if (!reused) await open(); + try { + return (await this.#post(access, method, params, bounded)) ?? {}; + } catch (error) { + // ChatGPT forgot a session this process kept: open a new one and ask once more. + if (!reused || !(error instanceof ProviderHttpError) || error.status !== 404) throw error; + await open(); + return (await this.#post(access, method, params, bounded)) ?? {}; + } + }, + { + signal: bounded, + sessionId: this.#stickiness, + missingAccessMessage: "Sign in to ChatGPT to use its apps.", + }, + ); + } catch (error) { + if (signal?.aborted) throw error; + throw failure(error); + } + } + + /** Every app the person connected to the ChatGPT account, with its tools. */ + async listApps(signal?: AbortSignal): Promise { + const tools: unknown[] = []; + const seen = new Set(); + let cursor: string | undefined; + do { + const page = await this.#request("tools/list", cursor ? { cursor } : {}, LIST_TIMEOUT_MS, signal); + if (Array.isArray(page.tools)) tools.push(...page.tools); + cursor = text(page.nextCursor); + if (cursor && seen.has(cursor)) + throw new ChatGptAppsError("unavailable", "ChatGPT's app list did not finish."); + if (cursor) seen.add(cursor); + } while (cursor); + return appsFromTools(tools).filter(app => !isChatGptOwnApp(app.id)); + } + + async callTool(name: string, args: unknown, signal?: AbortSignal): Promise { + const result = await this.#request("tools/call", { name, arguments: args ?? {} }, CALL_TIMEOUT_MS, signal); + const content = Array.isArray(result.content) ? (result.content as ChatGptToolContent[]) : []; + return { ...result, content }; + } +} diff --git a/packages/coding-agent/src/scient/command-help.ts b/packages/coding-agent/src/scient/command-help.ts index 578fcab5f6..acc177c471 100644 --- a/packages/coding-agent/src/scient/command-help.ts +++ b/packages/coding-agent/src/scient/command-help.ts @@ -4,3 +4,7 @@ import type { CommandMetadata } from "@oh-my-pi/pi-utils/cli"; export const dictateHelp = { description: "Transcribe a recording with ChatGPT's dictation, using the ChatGPT sign-in", } satisfies CommandMetadata; + +export const appsHelp = { + description: "List the ChatGPT apps (Gmail, Google Drive and others) the ChatGPT sign-in has connected", +} satisfies CommandMetadata; diff --git a/packages/coding-agent/src/scient/runtime-info.ts b/packages/coding-agent/src/scient/runtime-info.ts index d5c7763b4c..aa50b84928 100644 --- a/packages/coding-agent/src/scient/runtime-info.ts +++ b/packages/coding-agent/src/scient/runtime-info.ts @@ -19,6 +19,11 @@ export interface RuntimeInfo { rpcProtocolVersions: RpcReadyFrame["supportedProtocolVersions"]; /** Source revision of this build, or "source" when it was not built by Scient's workflow. */ buildId: string; + /** + * Scient's own commands a host may run. An executable without one reads the word as a + * prompt, so a host checks this list instead of guessing from the version. + */ + hostCommands: string[]; } export function formatRuntimeInfo(): string { @@ -28,6 +33,7 @@ export function formatRuntimeInfo(): string { upstream: UPSTREAM, rpcProtocolVersions: [1, 2], buildId: BUILD_ID, + hostCommands: ["apps", "dictate"], }; return `${JSON.stringify(info)}\n`; } diff --git a/packages/coding-agent/src/sdk.ts b/packages/coding-agent/src/sdk.ts index 8bf24940f3..937b02fa63 100644 --- a/packages/coding-agent/src/sdk.ts +++ b/packages/coding-agent/src/sdk.ts @@ -52,6 +52,9 @@ import { import { AsyncJobManager, type AsyncJobManagerApi } from "./async"; import { AutoLearnController, buildAutoLearnInstructions } from "./autolearn/controller"; import { createAutoresearchExtension } from "./autoresearch"; +// scient(SA-CHATGPT-APPS): begin - the person's ChatGPT apps, when the host asks for them. +import { createChatGptAppsExtension, requestedChatGptApps } from "./scient/chatgpt-apps-extension"; +// scient(SA-CHATGPT-APPS): end import { loadCapability, reset as resetCapabilities } from "./capability"; import { MAIN_AGENT_RULE_NAME, @@ -2981,6 +2984,10 @@ async function createAgentSessionScoped(options: CreateAgentSessionOptions): Pro inlineExtensions.push(...(options.extensions ?? [])); inlineExtensions.push(createAutoresearchExtension); + // scient(SA-CHATGPT-APPS): begin - the person's ChatGPT apps, when the host asks for them. + const chatGptApps = requestedChatGptApps(); + if (chatGptApps) inlineExtensions.push(createChatGptAppsExtension(authStorage, chatGptApps)); + // scient(SA-CHATGPT-APPS): end if (customTools.length > 0) { inlineExtensions.push(createCustomToolsExtension(customTools, customToolSourcePaths)); } diff --git a/packages/coding-agent/test/scient/chatgpt-apps.test.ts b/packages/coding-agent/test/scient/chatgpt-apps.test.ts new file mode 100644 index 0000000000..472d39ee1e --- /dev/null +++ b/packages/coding-agent/test/scient/chatgpt-apps.test.ts @@ -0,0 +1,245 @@ +import { afterEach, beforeEach, describe, expect, it, spyOn } from "bun:test"; +import { ChatGptAppsClient, ChatGptAppsError, chatGptAppConnectUrl, rpcResponse } from "../../src/scient/chatgpt-apps"; +import { + chatGptToolName, + chosenByPolicy, + createChatGptAppsExtension, + requestedChatGptApps, +} from "../../src/scient/chatgpt-apps-extension"; +import { AuthStorage } from "../../src/session/auth-storage"; + +const GMAIL = "connector_2128aebfecb84f64a069897515042a44"; +const DRIVE = "connector_5f3c8c41a1e54ad7a76272c89e2554fa"; + +const tool = (name: string, appId: string, appName: string, readOnly = false) => ({ + name, + description: `${name} description`, + inputSchema: { type: "object", properties: {} }, + annotations: { readOnlyHint: readOnly }, + _meta: { connector_id: appId, connector_name: appName, connector_description: `${appName} tools` }, +}); + +const CATALOG = [ + tool("gmail.search_emails", GMAIL, "Gmail", true), + tool("gmail.send_email", GMAIL, "Gmail"), + tool("google_drive.fetch", DRIVE, "Google Drive", true), + tool("pets.adopt", "connector_openai_chatgpt_pets", "Pets"), +]; + +/** A ChatGPT apps server: event-stream answers, one session id, and the calls it received. */ +function fakeServer(options: { expireSessionOnce?: boolean; call?: unknown } = {}) { + const requests: Array<{ method: string; session: string | null; headers: Headers; params: unknown }> = []; + let sessions = 0; + let expire = options.expireSessionOnce === true; + const spy = spyOn(globalThis, "fetch").mockImplementation((async (_url: unknown, init?: RequestInit) => { + const headers = new Headers(init?.headers); + const body = JSON.parse(String(init?.body)); + requests.push({ method: body.method, session: headers.get("mcp-session-id"), headers, params: body.params }); + if (body.method === "initialize") { + sessions++; + return new Response( + `event: message\ndata: ${JSON.stringify({ jsonrpc: "2.0", id: body.id, result: {} })}\n\n`, + { + headers: { "content-type": "text/event-stream", "mcp-session-id": `session-${sessions}` }, + }, + ); + } + if (body.id === undefined) return new Response(null, { status: 202 }); + if (expire && body.method !== "initialize") { + expire = false; + return new Response("Session not found", { status: 404 }); + } + const result = body.method === "tools/list" ? { tools: CATALOG } : (options.call ?? { content: [] }); + return Response.json({ jsonrpc: "2.0", id: body.id, result }); + }) as typeof fetch); + return { spy, requests }; +} + +describe("ChatGPT apps through the ChatGPT sign-in", () => { + let authStorage: AuthStorage; + + beforeEach(async () => { + authStorage = await AuthStorage.create(":memory:"); + }); + afterEach(() => { + authStorage.close(); + }); + + const signIn = () => + authStorage.credentials.set("openai-codex", [ + { + type: "oauth", + access: "access-1", + refresh: "refresh-1", + expires: Date.now() + 600_000, + accountId: "acct-1", + }, + ]); + + it("lists the connected apps with the sign-in, leaving out ChatGPT's own features", async () => { + await signIn(); + const { spy, requests } = fakeServer(); + try { + const apps = await new ChatGptAppsClient(authStorage).listApps(); + expect(apps.map(app => [app.name, app.tools.map(t => t.name)])).toEqual([ + ["Gmail", ["gmail.search_emails", "gmail.send_email"]], + ["Google Drive", ["google_drive.fetch"]], + ]); + expect(apps[0]?.tools[0]?.readOnly).toBe(true); + expect(String(spy.mock.calls[0]![0])).toBe("https://chatgpt.com/backend-api/ps/mcp"); + expect(requests.map(r => r.method)).toEqual(["initialize", "notifications/initialized", "tools/list"]); + const list = requests[2]!; + expect(list.session).toBe("session-1"); + expect(list.headers.get("authorization")).toBe("Bearer access-1"); + expect(list.headers.get("chatgpt-account-id")).toBe("acct-1"); + expect(list.headers.get("x-openai-product-sku")).toBe("codex"); + } finally { + spy.mockRestore(); + } + }); + + it("opens a new session once when ChatGPT has forgotten the old one", async () => { + await signIn(); + const client = new ChatGptAppsClient(authStorage); + const first = fakeServer(); + await client.listApps(); + first.spy.mockRestore(); + const { spy, requests } = fakeServer({ + expireSessionOnce: true, + call: { content: [{ type: "text", text: "ok" }] }, + }); + try { + const result = await client.callTool("gmail.search_emails", { query: "" }); + expect(result.content).toEqual([{ type: "text", text: "ok" }]); + expect(requests.map(r => r.method)).toEqual([ + "tools/call", + "initialize", + "notifications/initialized", + "tools/call", + ]); + expect(requests.filter(r => r.method === "tools/call").map(r => r.params)).toEqual([ + { name: "gmail.search_emails", arguments: { query: "" } }, + { name: "gmail.search_emails", arguments: { query: "" } }, + ]); + } finally { + spy.mockRestore(); + } + }); + + it("asks for a sign-in when there is none, without calling ChatGPT", async () => { + const spy = spyOn(globalThis, "fetch"); + try { + const error = await new ChatGptAppsClient(authStorage).listApps().catch(e => e); + expect(error).toBeInstanceOf(ChatGptAppsError); + expect((error as ChatGptAppsError).kind).toBe("signed-out"); + expect(spy).not.toHaveBeenCalled(); + } finally { + spy.mockRestore(); + } + }); + + it("registers the chosen apps' tools and runs them through ChatGPT", async () => { + await signIn(); + const { spy, requests } = fakeServer({ + call: { content: [], structuredContent: { emails: [{ subject: "Lab meeting" }] } }, + }); + const registered: Array<{ name: string; description: string; execute: Function; mcpReadOnlyHint?: boolean }> = []; + try { + const policy = requestedChatGptApps({ SCIENT_AGENT_CHATGPT_APPS: `all,-${DRIVE}` })!; + await createChatGptAppsExtension( + authStorage, + policy, + )({ registerTool: (t: never) => registered.push(t) } as never); + expect(registered.map(t => t.name)).toEqual(["gmail__search_emails", "gmail__send_email"]); + expect(registered[0]?.description).toStartWith("Gmail, through the person's ChatGPT account:"); + expect(registered[0]?.mcpReadOnlyHint).toBe(true); + const output = await registered[0]!.execute("call-1", { query: "lab" }, undefined); + expect(output.content).toEqual([{ type: "text", text: '{"emails":[{"subject":"Lab meeting"}]}' }]); + expect(requests.at(-1)?.params).toEqual({ name: "gmail.search_emails", arguments: { query: "lab" } }); + } finally { + spy.mockRestore(); + } + }); + + it("turns an app's error and a lost link into errors the model can explain", async () => { + await signIn(); + const { spy } = fakeServer({ + call: { content: [], isError: true, _meta: { connector_auth_failure: { is_auth_failure: true } } }, + }); + const registered: Array<{ execute: Function }> = []; + try { + const policy = requestedChatGptApps({ SCIENT_AGENT_CHATGPT_APPS: GMAIL })!; + await createChatGptAppsExtension( + authStorage, + policy, + )({ registerTool: (t: never) => registered.push(t) } as never); + const error = await registered[0]!.execute("call-1", {}, undefined).catch((e: Error) => e); + expect(String(error.message)).toContain(`connected again in ChatGPT`); + expect(String(error.message)).toContain(`https://chatgpt.com/apps/gmail/${GMAIL}`); + } finally { + spy.mockRestore(); + } + }); +}); + +describe("the host's choice of apps", () => { + it("reads all, ids and exclusions, and nothing without the variable", () => { + expect(requestedChatGptApps({})).toBeUndefined(); + expect(requestedChatGptApps({ SCIENT_AGENT_CHATGPT_APPS: `-${GMAIL}` })).toBeUndefined(); + const policy = requestedChatGptApps({ SCIENT_AGENT_CHATGPT_APPS: ` all , -${GMAIL}, bad id ` })!; + expect(policy).toEqual({ all: true, include: [], exclude: [GMAIL] }); + expect(chosenByPolicy(policy, GMAIL)).toBe(false); + expect(chosenByPolicy(policy, DRIVE)).toBe(true); + const named = requestedChatGptApps({ SCIENT_AGENT_CHATGPT_APPS: DRIVE })!; + expect(chosenByPolicy(named, DRIVE)).toBe(true); + expect(chosenByPolicy(named, GMAIL)).toBe(false); + }); + + it("names tools by app and keeps long names within 64 characters", () => { + const app = { id: GMAIL, name: "Gmail", tools: [] }; + expect(chatGptToolName(app, "gmail.search_emails")).toBe("gmail__search_emails"); + expect(chatGptToolName({ ...app, name: "Google Drive" }, "fetch")).toBe("google_drive__fetch"); + const long = chatGptToolName(app, `gmail.${"x".repeat(80)}`); + expect(long.length).toBe(64); + expect(long).toMatch(/^gmail__x+_[0-9a-f]{8}$/); + }); + + it("builds ChatGPT's connect page and reads event-stream answers", () => { + expect(chatGptAppConnectUrl({ id: "connector_x", name: "Google Calendar" })).toBe( + "https://chatgpt.com/apps/google-calendar/connector_x", + ); + const body = `event: message\r\ndata: {"jsonrpc":"2.0","method":"note","id":3}\r\n\r\ndata: {"jsonrpc":"2.0",\r\ndata: "id":3,"result":{"ok":true}}\r\n\r\n`; + expect(rpcResponse(body, true, 3)).toEqual({ jsonrpc: "2.0", id: 3, result: { ok: true } }); + }); +}); + +describe("one ChatGPT account per client", () => { + it("refuses a call once the sign-in resolves to another account", async () => { + const authStorage = await AuthStorage.create(":memory:"); + try { + await authStorage.credentials.set("openai-codex", [ + { type: "oauth", access: "access-a", refresh: "r", expires: Date.now() + 600_000, accountId: "acct-a" }, + ]); + const client = new ChatGptAppsClient(authStorage); + const spy = spyOn(globalThis, "fetch").mockImplementation((async (_url: unknown, init?: RequestInit) => { + const body = JSON.parse(String(init?.body)); + if (body.id === undefined) return new Response(null, { status: 202 }); + return Response.json({ jsonrpc: "2.0", id: body.id, result: { tools: [], content: [] } }); + }) as typeof fetch); + try { + await client.listApps(); + await authStorage.credentials.set("openai-codex", [ + { type: "oauth", access: "access-b", refresh: "r", expires: Date.now() + 600_000, accountId: "acct-b" }, + ]); + const calls = spy.mock.calls.length; + const error = await client.callTool("gmail.send_email", {}).catch(e => e); + expect((error as ChatGptAppsError).kind).toBe("signed-out"); + expect(spy.mock.calls.length).toBe(calls); + } finally { + spy.mockRestore(); + } + } finally { + authStorage.close(); + } + }); +}); diff --git a/packages/coding-agent/test/scient/runtime-info.test.ts b/packages/coding-agent/test/scient/runtime-info.test.ts index 4977b6a183..8a1c6eed9f 100644 --- a/packages/coding-agent/test/scient/runtime-info.test.ts +++ b/packages/coding-agent/test/scient/runtime-info.test.ts @@ -15,6 +15,7 @@ describe("--runtime-info", () => { upstream: UPSTREAM, rpcProtocolVersions: [1, 2], buildId: BUILD_ID, + hostCommands: ["apps", "dictate"], }); }); }); diff --git a/scripts/scient/divergences.json b/scripts/scient/divergences.json index ac465cceb8..983464fba4 100644 --- a/scripts/scient/divergences.json +++ b/scripts/scient/divergences.json @@ -58,7 +58,7 @@ }, "SA-PRODUCT": { "status": "active", - "contract": "Scient Agent reports its own release: scient.json names the product, its version and the exact Oh My Pi release; --version, the startup screen, completions and the User-Agent report the version, and --runtime-info reports all of it with the build id Scient's workflow stamps.", + "contract": "Scient Agent reports its own release: scient.json names the product, its version and the exact Oh My Pi release; --version, the startup screen, completions and the User-Agent report the version, and --runtime-info reports all of it with the build id Scient's workflow stamps and the Scient commands a host may run (hostCommands).", "owned": [ "scient.json", "scripts/scient/stamp-build-id.ts", @@ -349,6 +349,19 @@ ], "verification": ["bun --cwd=packages/coding-agent test test/scient/live-host-delegation.test.ts"] }, + "SA-CHATGPT-APPS": { + "status": "active", + "contract": "The person's ChatGPT apps (Gmail, Google Drive and the others they connected in ChatGPT) through the stored openai-codex sign-in, from the MCP server the Codex apps use (chatgpt.com/backend-api/ps/mcp), with the shared OAuth retry refreshing or rotating the credential. `scient-agent apps [--json]` lists the connected apps as {\"apps\":[{\"id\",\"name\",\"description\"?,\"tools\",\"readOnlyTools\"}]}, or prints {\"error\":{\"kind\",\"message\"}} and exits 1, the kind being signed-out or unavailable. A host that sets SCIENT_AGENT_CHATGPT_APPS (`all`, app ids, `-` to leave one out) gets the chosen apps' tools registered at session start as discoverable tools named __; ChatGPT's own connector_openai_* features are never listed or registered. Without the variable nothing changes.", + "owned": [ + "packages/coding-agent/src/scient/apps-command.ts", + "packages/coding-agent/src/scient/chatgpt-apps.ts", + "packages/coding-agent/src/scient/chatgpt-apps-extension.ts", + "packages/coding-agent/src/scient/command-help.ts", + "packages/coding-agent/test/scient/chatgpt-apps.test.ts" + ], + "upstream": ["packages/coding-agent/src/cli-commands.ts", "packages/coding-agent/src/sdk.ts"], + "verification": ["bun --cwd=packages/coding-agent test test/scient/chatgpt-apps.test.ts"] + }, "SA-DICTATION": { "status": "active", "contract": "`scient-agent dictate [--language xx] [--json]` transcribes a recording with ChatGPT's dictation service, the one the ChatGPT and Codex apps use, through the stored openai-codex sign-in, which the shared OAuth retry refreshes or rotates. With --json it prints {\"text\"} on success, or {\"error\":{\"kind\",\"message\"}} and exits 1, the kind being signed-out, invalid-audio, rate-limited or unavailable. Nothing else changes.",