From 779c6b4a8f10a768685b437373db25f9db9a038c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 4 Nov 2026 21:41:17 -0800 Subject: [PATCH 1/2] feat(codex): discover configured native computer use --- .../Adapters/CodexAdapterV2.test.ts | 391 +++++++++++++++--- .../Adapters/CodexAdapterV2.ts | 44 +- .../src/provider/CodexComputerUse.test.ts | 342 +++++++++++++++ apps/server/src/provider/CodexComputerUse.ts | 174 ++++++++ .../CodexDeveloperInstructions.test.ts | 14 + .../provider/CodexDeveloperInstructions.ts | 17 + .../src/provider/Drivers/CodexDriver.ts | 1 + docs/user/providers-codex.md | 7 + evals/mcp-tool-use/README.md | 22 + .../cases/native-computer-use.ndjson | 4 + evals/mcp-tool-use/native-computer-use.ts | 86 ++++ evals/mcp-tool-use/run.test.ts | 37 ++ evals/mcp-tool-use/run.ts | 90 +++- 13 files changed, 1145 insertions(+), 84 deletions(-) create mode 100644 apps/server/src/provider/CodexComputerUse.test.ts create mode 100644 apps/server/src/provider/CodexComputerUse.ts create mode 100644 evals/mcp-tool-use/cases/native-computer-use.ndjson create mode 100644 evals/mcp-tool-use/native-computer-use.ts diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index 857c659deee44..7f026a7a49f0b 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -4,6 +4,7 @@ import { makeProviderTextDeltaCoalescer, type ProviderTextDeltaUpdate, } from "./ProviderTextDeltaCoalescer.ts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { @@ -61,6 +62,8 @@ import * as PlatformError from "effect/PlatformError"; import { TestClock } from "effect/testing"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; +import { buildCodexComputerUseInstructions } from "../../provider/CodexDeveloperInstructions.ts"; + import packageJson from "../../../package.json" with { type: "json" }; import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions"; import * as ProviderEventLoggers from "@t3tools/provider-core/server/ProviderEventLoggers"; @@ -186,6 +189,49 @@ describe("CodexAdapterV2 runtime policy", () => { }), ); + it.effect.each([false, true])("adds per-turn Computer Use context with hasT3Mcp=%s", (hasT3Mcp) => + Effect.gen(function* () { + const input = [ + { type: "text", text: "work" }, + { type: "image", url: "data:image/png;base64,AA==" }, + ] as const; + const base = { + nativeThreadId: "native-cua", + codexInput: input, + runtimePolicy: { + runtimeMode: "auto", + interactionMode: "default", + cwd: "/project", + } as const, + modelSelection: { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5.4", + options: [{ id: "serviceTier", value: "fast" }], + }, + hasT3Mcp, + appContext: [{ key: "app", text: "untrusted app data" }], + }; + const before = yield* CodexAdapterV2.buildCodexTurnStartParams(base); + for (const computerUse of [ + { status: "available", skillPath: "/installed/SKILL.md" }, + { status: "unavailable" }, + ] as const) { + const params = yield* CodexAdapterV2.buildCodexTurnStartParams({ ...base, computerUse }); + assert.deepEqual(params.input, input); + assert.deepEqual(params, { + ...before, + additionalContext: { + ...before.additionalContext, + t3_code_computer_use: { + kind: "application", + value: buildCodexComputerUseInstructions(computerUse), + }, + }, + }); + } + }), + ); + it.effect("derives concrete Codex turn policies from every T3 runtime mode", () => Effect.gen(function* () { const build = ( @@ -557,6 +603,34 @@ function makeCodexTestTurnInput(input: { }; } +const computerUseReplayResources = "/custom/Acme.app/Contents/Resources"; +const computerUseReplaySkill = `${computerUseReplayResources}/plugins/openai-bundled/plugins/computer-use/skills/computer-use/SKILL.md`; +const computerUseReplayOptions = () => ({ + enableComputerUse: true, + computerUseConfig: { + plugins: { "computer-use@openai-bundled": { enabled: true } }, + mcp_servers: { + node_repl: { + command: `${computerUseReplayResources}/cua_node/bin/node_repl`, + env: { + NODE_REPL_TRUSTED_SERVICES: '{"sky":"@oai/sky/service"}', + NODE_REPL_NODE_MODULE_DIRS: `${computerUseReplayResources}/cua_node/lib/node_modules`, + SKY_CUA_SERVICE_PATH: "/custom-home/Computer Use.app", + }, + }, + }, + }, + fileSystem: FileSystem.makeNoop({ + access: () => Effect.void, + readFileString: (path) => + Effect.succeed( + path.endsWith("package.json") + ? '{"name":"@oai/sky","exports":{"./service":"./service.js"}}' + : "---\nname: computer-use\n---\nInstalled instructions.", + ), + }), +}); + function makeCodexReplayTurn(input: { readonly id: string; readonly status: "inProgress" | "completed" | "interrupted" | "failed"; @@ -1235,12 +1309,26 @@ describe("CodexAdapterV2 post-settle continuation", () => { onRequest: (method: string, params: unknown) => Effect.Effect = () => Effect.void, readChildMetadata?: Parameters[2], runtimeOptions: Partial< - Pick - > & { readonly modelProvider?: string | null; readonly shellPath?: string } = {}, + Pick< + CodexAdapterV2.CodexAdapterV2Options, + "settings" | "resolveRuntime" | "environment" | "enableComputerUse" | "fileSystem" + > + > & { + readonly modelProvider?: string | null; + readonly shellPath?: string; + readonly computerUseConfig?: Record; + readonly computerUseFailure?: boolean; + } = {}, beforeEmitInbound?: CodexReplay.CodexAppServerReplayDriver["beforeEmitInbound"], ) => Effect.gen(function* () { - const { modelProvider = null, shellPath, ...adapterRuntimeOptions } = runtimeOptions; + const { + modelProvider = null, + shellPath, + computerUseConfig, + computerUseFailure, + ...adapterRuntimeOptions + } = runtimeOptions; const fileSystem = yield* FileSystem.FileSystem; const idAllocator = yield* IdAllocator.IdAllocatorV2; const serverConfig = yield* ProviderReplayHarness.makeReplayServerConfig( @@ -1283,6 +1371,7 @@ describe("CodexAdapterV2 post-settle continuation", () => { return { config: { model_provider: modelProvider, + ...computerUseConfig, ...(shellPath === undefined ? {} : { shell_environment_policy: { set: { PATH: shellPath } } }), @@ -1290,9 +1379,31 @@ describe("CodexAdapterV2 post-settle continuation", () => { origins: {}, }; }) - : onRequest(method, params).pipe( - Effect.andThen(client.raw.request(method, params)), - ), + : computerUseConfig !== undefined && + (method === "configRequirements/read" || method === "skills/list") + ? onRequest(method, params).pipe( + Effect.andThen( + computerUseFailure + ? Effect.fail( + new CodexError.CodexAppServerTransportError({ + operation: "write-output-stream", + cause: "fixture", + }), + ) + : Effect.succeed( + method === "configRequirements/read" + ? { requirements: null } + : { + data: [ + { cwd: "/workspace", skills: [], errors: [] }, + ], + }, + ), + ), + ) + : onRequest(method, params).pipe( + Effect.andThen(client.raw.request(method, params)), + ), }, request: (method, params) => onRequest(method, params).pipe( @@ -1312,7 +1423,7 @@ describe("CodexAdapterV2 post-settle continuation", () => { ...adapterRuntimeOptions, clientFactory, digest: yield* Digest.Digest, - fileSystem, + fileSystem: adapterRuntimeOptions.fileSystem ?? fileSystem, idAllocator, serverConfig, continuationRequests: { @@ -2380,97 +2491,130 @@ describe("CodexAdapterV2 post-settle continuation", () => { ), ); - it.effect("preserves T3 context on the wire and restores it after compaction", () => - Effect.scoped( + it.effect.each([false, true])( + "rechecks Computer Use after unload and resume with discovery failure=%s", + (computerUseFailure) => Effect.gen(function* () { - const nativeThreadId = "context-thread"; - const nativeTurnId = "context-turn"; - const params = yield* CodexAdapterV2.buildCodexTurnStartParams({ + const nativeThreadId = "cua-resume-thread"; + const options = { ...computerUseReplayOptions(), computerUseFailure }; + const contextFor = (status: "available" | "unavailable") => + CodexAdapterV2.buildCodexTurnStartParams({ + nativeThreadId, + codexInput: [{ type: "text", text: "work" }], + runtimePolicy: CODEX_TEST_RUNTIME_POLICY, + modelSelection: CODEX_TEST_MODEL_SELECTION, + computerUse: + status === "available" ? { status, skillPath: computerUseReplaySkill } : { status }, + }); + const first = yield* contextFor(computerUseFailure ? "unavailable" : "available"); + const second = yield* contextFor("unavailable"); + const preamble = codexReplayPreamble({ nativeThreadId, - codexInput: [{ type: "text", text: "work" }], - runtimePolicy: CODEX_TEST_RUNTIME_POLICY, - modelSelection: CODEX_TEST_MODEL_SELECTION, - hasT3Mcp: true, + nativeTurnId: "cua-first", + prompt: "work", + }); + const completed = (id: string): CodexReplay.CodexAppServerReplayEntry => ({ + type: "emit_inbound", + label: id, + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id, status: "completed" }), + }, + }, }); - assert.include( - params.additionalContext?.t3_code_orchestration_01?.value ?? "", - "delegate_task", - ); - const entries = codexReplayPreamble({ nativeThreadId, nativeTurnId, prompt: "work" }); const transcript = makeCodexReplayTranscript({ - scenario: "restore-context", + scenario: "cua-resume", entries: [ - ...entries.slice(0, 5), + ...preamble.slice(0, 5), { type: "expect_outbound", - label: "context turn", - frame: { id: 3, method: "turn/start", params }, + label: "first locator", + frame: { id: 3, method: "turn/start", params: first }, + }, + ...preamble.slice(6), + completed("cua-first"), + { + type: "expect_outbound", + label: "unload", + frame: { id: 4, method: "thread/unsubscribe", params: { threadId: nativeThreadId } }, }, - ...entries.slice(6), { type: "emit_inbound", - label: "compacted", - frame: { - method: "item/completed", - params: { - threadId: nativeThreadId, - turnId: nativeTurnId, - item: { type: "contextCompaction", id: "compact-context" }, - }, - }, + label: "unloaded", + frame: { id: 4, result: { status: "unsubscribed" } }, }, { type: "expect_outbound", - label: "restore context", + label: "resume", frame: { - id: 4, - method: "thread/inject_items", + id: 5, + method: "thread/resume", params: { threadId: nativeThreadId, - items: Object.entries(params.additionalContext ?? {}).map(([key, entry]) => ({ - type: "message", - role: "developer", - content: [{ type: "input_text", text: `<${key}>${entry.value}` }], - })), + modelProvider: "openai", + model: "gpt-5.4", + cwd: "/workspace", + excludeTurns: true, + config: CodexAdapterV2.CODEX_THREAD_CONFIG, }, }, }, - { type: "emit_inbound", label: "restored", frame: { id: 4, result: {} } }, { type: "emit_inbound", - label: "done", + label: "resumed", + frame: { id: 5, result: { thread: { id: nativeThreadId, updatedAt: 1782622450 } } }, + }, + { + type: "expect_outbound", + label: "withdrawn locator", + frame: { id: 6, method: "turn/start", params: second }, + }, + { + type: "emit_inbound", + label: "second started", frame: { - method: "turn/completed", - params: { - threadId: nativeThreadId, - turn: makeCodexReplayTurn({ id: nativeTurnId, status: "completed" }), - }, + id: 6, + result: { turn: makeCodexReplayTurn({ id: "cua-second", status: "inProgress" }) }, }, }, + completed("cua-second"), ], }); - const harness = yield* makeCodexReplayHarness(transcript); - yield* (yield* McpProviderSessions.McpProviderSessions).set({ - environmentId: EnvironmentId.make("test"), - threadId: harness.threadId, - providerSessionId: "context-session", - providerInstanceId: ProviderInstanceId.make("codex"), - endpoint: "http://127.0.0.1:43123/mcp", - authorizationHeader: "Bearer test", - browserToolsAvailable: true, - }); - yield* harness.runtime.startTurn( + const harness = yield* makeCodexReplayHarness( + transcript, + undefined, + undefined, + undefined, + options, + ); + const turn = (attempt: string, providerThread = harness.providerThread) => makeCodexTestTurnInput({ threadId: harness.threadId, - providerThread: harness.providerThread, - now: yield* DateTime.now, - attemptId: RunAttemptId.make("context-attempt"), + providerThread, + now: DateTime.makeUnsafe("2026-11-04T00:00:00Z"), + attemptId: RunAttemptId.make(attempt), text: "work", - }), - ); + }); + yield* harness.runtime.startTurn(turn("cua-first")); yield* harness.firstTerminal; - assert.equal(harness.terminalEvents()[0]?.status, "completed"); + options.computerUseConfig.plugins["computer-use@openai-bundled"].enabled = false; + yield* harness.runtime.unloadThread!({ providerThread: harness.providerThread }); + const resumed = yield* harness.runtime.resumeThread({ + providerThread: harness.providerThread, + modelSelection: CODEX_TEST_MODEL_SELECTION, + runtimePolicy: CODEX_TEST_RUNTIME_POLICY, + }); + yield* harness.runtime.startTurn(turn("cua-second", resumed)); + yield* harness.awaitTerminal(2); + assert.deepEqual( + harness.terminalEvents().map((event) => event.status), + ["completed", "completed"], + ); }).pipe( + Effect.scoped, + Effect.provideService(HostProcess.Platform, "darwin"), Effect.provide( Layer.mergeAll( IdAllocator.layer, @@ -2480,7 +2624,118 @@ describe("CodexAdapterV2 post-settle continuation", () => { ), ), ), - ), + ); + + it.effect.each([false, true])( + "preserves T3 context on the wire and restores it after compaction with MCP=%s", + (hasT3Mcp) => + Effect.scoped( + Effect.gen(function* () { + const nativeThreadId = "context-thread"; + const nativeTurnId = "context-turn"; + const params = yield* CodexAdapterV2.buildCodexTurnStartParams({ + nativeThreadId, + codexInput: [{ type: "text", text: "work" }], + runtimePolicy: CODEX_TEST_RUNTIME_POLICY, + modelSelection: CODEX_TEST_MODEL_SELECTION, + hasT3Mcp, + computerUse: { status: "available", skillPath: computerUseReplaySkill }, + }); + assert.isDefined(params.additionalContext?.t3_code_computer_use); + assert.equal(params.additionalContext?.t3_code_runtime !== undefined, hasT3Mcp); + const entries = codexReplayPreamble({ nativeThreadId, nativeTurnId, prompt: "work" }); + const transcript = makeCodexReplayTranscript({ + scenario: "restore-context", + entries: [ + ...entries.slice(0, 5), + { + type: "expect_outbound", + label: "context turn", + frame: { id: 3, method: "turn/start", params }, + }, + ...entries.slice(6), + { + type: "emit_inbound", + label: "compacted", + frame: { + method: "item/completed", + params: { + threadId: nativeThreadId, + turnId: nativeTurnId, + item: { type: "contextCompaction", id: "compact-context" }, + }, + }, + }, + { + type: "expect_outbound", + label: "restore context", + frame: { + id: 4, + method: "thread/inject_items", + params: { + threadId: nativeThreadId, + items: Object.entries(params.additionalContext ?? {}).map(([key, entry]) => ({ + type: "message", + role: "developer", + content: [{ type: "input_text", text: `<${key}>${entry.value}` }], + })), + }, + }, + }, + { type: "emit_inbound", label: "restored", frame: { id: 4, result: {} } }, + { + type: "emit_inbound", + label: "done", + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: nativeTurnId, status: "completed" }), + }, + }, + }, + ], + }); + const harness = yield* makeCodexReplayHarness( + transcript, + undefined, + undefined, + undefined, + computerUseReplayOptions(), + ); + if (hasT3Mcp) + yield* (yield* McpProviderSessions.McpProviderSessions).set({ + environmentId: EnvironmentId.make("test"), + threadId: harness.threadId, + providerSessionId: "context-session", + providerInstanceId: ProviderInstanceId.make("codex"), + endpoint: "http://127.0.0.1:43123/mcp", + authorizationHeader: "Bearer test", + browserToolsAvailable: true, + }); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("context-attempt"), + text: "work", + }), + ); + yield* harness.firstTerminal; + assert.equal(harness.terminalEvents()[0]?.status, "completed"); + }).pipe( + Effect.provideService(HostProcess.Platform, "darwin"), + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ), ); it.effect("compacts Codex with the native RPC and completes the compaction turn", () => diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index b4daed4ca0dd1..10b308cf3436c 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -106,7 +106,14 @@ import * as PortScanner from "../../preview/PortScanner.ts"; import { getCodexServiceTierOptionValue } from "../../codexModelOptions.ts"; import { ServerConfig } from "../../config.ts"; import { expandHomePath } from "@t3tools/provider-core/server/pathExpansion"; -import { buildCodexAdditionalContext } from "../../provider/CodexDeveloperInstructions.ts"; +import { + resolveCodexComputerUse, + type CodexComputerUseAvailability, +} from "../../provider/CodexComputerUse.ts"; +import { + buildCodexAdditionalContext, + buildCodexComputerUseInstructions, +} from "../../provider/CodexDeveloperInstructions.ts"; import { describeMcpElicitation, toMcpElicitationResponse, @@ -762,6 +769,7 @@ export const buildCodexTurnStartParams = Effect.fn("CodexAdapterV2.buildCodexTur readonly runtimePolicy: ProviderAdapter.ProviderAdapterV2RuntimePolicy; readonly modelSelection: ModelSelection; readonly hasT3Mcp?: boolean; + readonly computerUse?: CodexComputerUseAvailability; readonly browserToolsAvailable?: boolean; readonly deviceToolsAvailable?: boolean; /** ChatGPT token sharing does not accept service tiers. */ @@ -803,10 +811,19 @@ export const buildCodexTurnStartParams = Effect.fn("CodexAdapterV2.buildCodexTur }, ) : undefined; - const additionalContext = - t3Context === undefined && Object.keys(appContext).length === 0 - ? undefined - : { ...t3Context, ...appContext }; + const mergedContext = { + ...t3Context, + ...(input.computerUse === undefined + ? {} + : { + t3_code_computer_use: { + kind: "application" as const, + value: buildCodexComputerUseInstructions(input.computerUse), + }, + }), + ...appContext, + }; + const additionalContext = Object.keys(mergedContext).length === 0 ? undefined : mergedContext; return yield* decodeCodexTurnStartParamsWithCollaborationMode({ threadId: input.nativeThreadId, input: input.codexInput, @@ -1664,7 +1681,10 @@ export type CodexAdapterV2DriverEnv = export const createCodexAdapterV2 = Effect.fn("CodexAdapterV2.createCodexAdapterV2")(function* ( { instanceId, environment, enabled, config }: ProviderAdapterDriverCreateInput, - hooks: Pick = {}, + hooks: Pick< + CodexAdapterV2Options, + "onUsageLimits" | "resolveRuntime" | "compatibility" | "enableComputerUse" + > = {}, ) { const clientFactory = yield* CodexAppServerClientFactory; const continuationRequests = yield* ProviderContinuationRequests.ProviderContinuationRequests; @@ -1719,6 +1739,7 @@ export interface CodexAdapterV2Options { readonly settings: CodexSettings; readonly environment: NodeJS.ProcessEnv; readonly clientFactory: CodexAppServerClientFactoryShape; + readonly enableComputerUse?: boolean; readonly onUsageLimits?: ManagedServerProvider["applyUsageLimits"]; readonly resolveRuntime?: Effect.Effect; readonly compatibility?: ( @@ -1744,6 +1765,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( threadId === null ? Effect.undefined : mcpSessions.read(threadId); const { clientFactory, digest, fileSystem, idAllocator, serverConfig } = adapterOptions; const continuationRequests = adapterOptions.continuationRequests; + const pathService = yield* Path.Path; return ProviderAdapter.ProviderAdapterV2.of({ instanceId: adapterOptions.instanceId, @@ -6840,8 +6862,18 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( ) { const threadId = yield* getNativeThreadId(turnInput.providerThread); const mcpSession = yield* mcpSessions.read(turnInput.threadId); + const computerUse = + adapterOptions.enableComputerUse === true && + (yield* HostProcess.Platform) === "darwin" + ? yield* resolveCodexComputerUse({ cwd: turnInput.runtimePolicy.cwd }).pipe( + Effect.provideService(CodexClient.CodexAppServerClient, client), + Effect.provideService(FileSystem.FileSystem, fileSystem), + Effect.provideService(Path.Path, pathService), + ) + : undefined; const turnStartParams = yield* buildCodexTurnStartParams({ nativeThreadId: threadId, + ...(computerUse === undefined ? {} : { computerUse }), codexInput, runtimePolicy: turnInput.runtimePolicy, modelSelection: turnInput.modelSelection, diff --git a/apps/server/src/provider/CodexComputerUse.test.ts b/apps/server/src/provider/CodexComputerUse.test.ts new file mode 100644 index 0000000000000..e9ff1fa5cfbb9 --- /dev/null +++ b/apps/server/src/provider/CodexComputerUse.test.ts @@ -0,0 +1,342 @@ +import { assert, it } from "@effect/vitest"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as CodexClient from "effect-codex-app-server/client"; +import * as CodexErrors from "effect-codex-app-server/errors"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Cause from "effect/Cause"; +import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Stream from "effect/Stream"; +import { TestClock } from "effect/testing"; + +import { resolveCodexComputerUse } from "./CodexComputerUse.ts"; + +const resources = "/custom/Acme.app/Contents/Resources"; +const command = `${resources}/cua_node/bin/node_repl`; +const modules = `${resources}/cua_node/lib/node_modules`; +const helper = "/custom-codex-home/computer-use/Codex Computer Use.app"; +const skillPath = `${resources}/plugins/openai-bundled/plugins/computer-use/skills/computer-use/SKILL.md`; +const available = { status: "available", skillPath } as const; +const unavailable = { status: "unavailable" } as const; +const configured = () => ({ + features: { plugins: true }, + plugins: { "computer-use@openai-bundled": { enabled: true } }, + mcp_servers: { + node_repl: { + enabled: true, + command, + env: { + NODE_REPL_TRUSTED_SERVICES: JSON.stringify({ sky: "@oai/sky/service" }), + NODE_REPL_NODE_MODULE_DIRS: modules, + SKY_CUA_SERVICE_PATH: helper, + }, + }, + }, +}); +const rpcError = new CodexErrors.CodexAppServerTransportError({ + operation: "write-output-stream", + cause: "fixture", +}); + +const fixture = ( + options: { + config?: unknown; + layers?: unknown; + requirements?: unknown; + skills?: ReadonlyArray; + skillErrors?: ReadonlyArray; + missing?: string; + fail?: string; + platform?: NodeJS.Platform; + request?: CodexClient.CodexAppServerClient["Service"]["raw"]["request"]; + } = {}, +) => { + const calls: Array<{ method: string; params: unknown }> = []; + const reads: string[] = []; + const request: CodexClient.CodexAppServerClient["Service"]["raw"]["request"] = ( + method, + params, + ) => { + calls.push({ method, params }); + if (options.request) return options.request(method, params); + if (options.fail === method) return Effect.fail(rpcError); + if (method === "config/read") + return Effect.succeed({ + config: options.config ?? configured(), + origins: {}, + ...(options.layers === undefined ? {} : { layers: options.layers }), + }); + if (method === "configRequirements/read") + return Effect.succeed( + options.requirements === undefined ? {} : { requirements: options.requirements }, + ); + if (method === "skills/list") + return Effect.succeed({ + data: [ + { cwd: "/project", skills: options.skills ?? [], errors: options.skillErrors ?? [] }, + ], + }); + return Effect.die(`Unexpected RPC ${method}`); + }; + const client = CodexClient.CodexAppServerClient.of({ + raw: { + request, + notifications: Stream.empty, + requests: Stream.empty, + notify: () => Effect.void, + respond: () => Effect.void, + respondError: () => Effect.void, + }, + request: () => Effect.die("Use the active client's raw discovery boundary"), + notify: () => Effect.void, + awaitTermination: Effect.never, + handleServerRequest: () => Effect.void, + handleServerNotification: () => Effect.void, + handleUnknownServerRequest: () => Effect.void, + handleUnknownServerNotification: () => Effect.void, + }); + const absent = FileSystem.makeNoop({}); + const fileSystem = FileSystem.makeNoop({ + access: (path) => + Effect.sync(() => { + reads.push(path); + }).pipe(Effect.andThen(path === options.missing ? absent.access(path) : Effect.void)), + readFileString: (path) => { + reads.push(path); + if (path === options.missing) return absent.readFileString(path); + if (path.endsWith("package.json")) + return Effect.succeed( + JSON.stringify({ name: "@oai/sky", exports: { "./service": "./service.js" } }), + ); + return Effect.succeed( + "---\nname: computer-use\ndescription: Native Mac UI\n---\nFull installed instructions.", + ); + }, + }); + const resolve = resolveCodexComputerUse({ cwd: "/project" }).pipe( + Effect.provideService(CodexClient.CodexAppServerClient, client), + Effect.provideService(FileSystem.FileSystem, fileSystem), + Effect.provide(Path.layer), + Effect.provideService(HostProcess.Platform, options.platform ?? "darwin"), + ); + return { resolve, calls, reads }; +}; + +it.effect("locates configured native files from the selected effective home and cwd", () => + Effect.gen(function* () { + const f = fixture(); + assert.deepEqual(yield* f.resolve, available); + assert.deepEqual( + f.calls.map((c) => c.method), + ["config/read", "configRequirements/read", "skills/list"], + ); + assert.deepEqual(f.calls[0]?.params, { cwd: "/project", includeLayers: false }); + assert.deepEqual(f.calls[2]?.params, { cwds: ["/project"] }); + assert.include(f.reads, `${helper}/Contents/MacOS/SkyComputerUseService`); + assert.isFalse(f.reads.some((path) => path.startsWith("/Users/"))); + }), +); + +it.effect.each(["linux", "win32"] as const)("does no I/O on %s", (platform) => + Effect.gen(function* () { + const f = fixture({ platform }); + assert.deepEqual(yield* f.resolve, unavailable); + assert.deepEqual(f.calls, []); + assert.deepEqual(f.reads, []); + }), +); + +it.effect.each([ + [ + "effective plugin disabled", + () => ({ ...configured(), plugins: { "computer-use@openai-bundled": { enabled: false } } }), + ], + ["plugins feature disabled", () => ({ ...configured(), features: { plugins: false } })], + ["plugin missing", () => ({ ...configured(), plugins: {} })], + [ + "node server disabled", + () => { + const c = configured(); + c.mcp_servers.node_repl.enabled = false; + return c; + }, + ], + [ + "generic node repl", + () => { + const c = configured(); + c.mcp_servers.node_repl.env.NODE_REPL_TRUSTED_SERVICES = "{}"; + return c; + }, + ], + [ + "untrusted sky registration", + () => { + const c = configured(); + c.mcp_servers.node_repl.env.NODE_REPL_TRUSTED_SERVICES = '{"sky":"other/service"}'; + return c; + }, + ], + [ + "malformed services", + () => { + const c = configured(); + c.mcp_servers.node_repl.env.NODE_REPL_TRUSTED_SERVICES = "{"; + return c; + }, + ], + [ + "unmatched module directory", + () => { + const c = configured(); + c.mcp_servers.node_repl.env.NODE_REPL_NODE_MODULE_DIRS = "/other/node_modules"; + return c; + }, + ], + [ + "relative command", + () => { + const c = configured(); + c.mcp_servers.node_repl.command = "node_repl"; + return c; + }, + ], + [ + "skill name disabled", + () => ({ ...configured(), skills: { config: [{ name: "computer-use", enabled: false }] } }), + ], + [ + "qualified skill disabled", + () => ({ + ...configured(), + skills: { config: [{ name: "computer-use:computer-use", enabled: false }] }, + }), + ], + [ + "skill path disabled", + () => ({ ...configured(), skills: { config: [{ path: skillPath, enabled: false }] } }), + ], +] as const)("respects %s", ([_name, config]) => + Effect.gen(function* () { + assert.deepEqual(yield* fixture({ config: config() }).resolve, unavailable); + }), +); + +it.effect("uses effective project disablement even when the user layer enables the plugin", () => + Effect.gen(function* () { + const config = configured(); + config.plugins["computer-use@openai-bundled"].enabled = false; + const f = fixture({ + config, + layers: [ + { name: { type: "project", dotCodexFolder: "/project/.codex" }, config }, + { name: { type: "user", file: "/custom-codex-home/config.toml" }, config: configured() }, + ], + }); + assert.deepEqual(yield* f.resolve, unavailable); + assert.deepEqual(f.calls[0]?.params, { cwd: "/project", includeLayers: false }); + }), +); + +it.effect.each([ + { allowBrowserAndComputerUse: false }, + { computerUse: { allowedApps: [] } }, + { computerUse: { futureRestriction: true } }, + { featureRequirements: { plugins: false } }, +])("does not substitute for administrator requirements %j", (requirements) => + Effect.gen(function* () { + assert.deepEqual(yield* fixture({ requirements }).resolve, unavailable); + }), +); + +it.effect.each([ + command, + `${helper}/Contents/MacOS/SkyComputerUseService`, + skillPath, + `${modules}/@oai/sky/package.json`, + `${modules}/@oai/sky/service.js`, +])("requires readable native component %s", (missing) => + Effect.gen(function* () { + assert.deepEqual(yield* fixture({ missing }).resolve, unavailable); + }), +); + +it.effect( + "prefers an enabled discovered copy and lets a disabled copy veto the bundle fallback", + () => + Effect.gen(function* () { + const path = "/custom-codex-home/plugins/cache/computer-use/skills/computer-use/SKILL.md"; + const skill = { + name: "computer-use:computer-use", + path, + enabled: true, + pluginId: "computer-use@openai-bundled", + }; + assert.deepEqual( + yield* fixture({ skills: [{ ...skill, pluginId: null, scope: "repo" }] }).resolve, + available, + ); + assert.deepEqual( + yield* fixture({ + skillErrors: [{ path: "/project/.agents/skills/other/SKILL.md", message: "invalid" }], + }).resolve, + available, + ); + assert.deepEqual( + yield* fixture({ skillErrors: [{ path: skillPath, message: "invalid" }] }).resolve, + unavailable, + ); + assert.deepEqual(yield* fixture({ skills: [skill] }).resolve, { + status: "available", + skillPath: path, + }); + assert.deepEqual( + yield* fixture({ skills: [skill, { ...skill, path: skillPath, enabled: false }] }).resolve, + unavailable, + ); + }), +); + +it.effect.each(["config/read", "configRequirements/read", "skills/list"])( + "isolates optional %s errors", + (fail) => + Effect.gen(function* () { + assert.deepEqual(yield* fixture({ fail }).resolve, unavailable); + }), +); + +it.effect("re-evaluates project disablement on every turn", () => + Effect.gen(function* () { + const config = configured(); + const f = fixture({ config }); + assert.deepEqual(yield* f.resolve, available); + config.plugins["computer-use@openai-bundled"].enabled = false; + assert.deepEqual(yield* f.resolve, unavailable); + assert.equal(f.calls.filter((c) => c.method === "config/read").length, 2); + }), +); + +it.effect("bounds discovery to two seconds and preserves caller interruption", () => + Effect.gen(function* () { + const entered = yield* Deferred.make(); + const f = fixture({ + request: () => Deferred.succeed(entered, undefined).pipe(Effect.andThen(Effect.never)), + }); + const fiber = yield* Effect.forkChild(f.resolve); + yield* Deferred.await(entered); + yield* TestClock.adjust("2 seconds"); + assert.deepEqual(yield* Fiber.join(fiber), unavailable); + const enteredAgain = yield* Deferred.make(); + const second = fixture({ + request: () => Deferred.succeed(enteredAgain, undefined).pipe(Effect.andThen(Effect.never)), + }); + const interrupted = yield* Effect.forkChild(second.resolve); + yield* Deferred.await(enteredAgain); + yield* Fiber.interrupt(interrupted); + const exit = yield* Fiber.await(interrupted); + assert.isTrue(Exit.isFailure(exit) && Cause.hasInterrupts(exit.cause)); + }), +); diff --git a/apps/server/src/provider/CodexComputerUse.ts b/apps/server/src/provider/CodexComputerUse.ts new file mode 100644 index 0000000000000..2ce48c5cf9c07 --- /dev/null +++ b/apps/server/src/provider/CodexComputerUse.ts @@ -0,0 +1,174 @@ +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as CodexClient from "effect-codex-app-server/client"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; + +export type CodexComputerUseAvailability = + | { readonly status: "available"; readonly skillPath: string } + | { readonly status: "unavailable" }; + +const unavailable = { status: "unavailable" } as const; +const pluginId = "computer-use@openai-bundled"; +const skillNames = new Set(["computer-use", "computer-use:computer-use"]); +const isRecord = Schema.is(Schema.Record(Schema.String, Schema.Unknown)); +const record = (value: unknown): Record => (isRecord(value) ? value : {}); +const encodePath = Schema.encodeEffect(Schema.fromJsonString(Schema.String)); +const decodeJson = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Unknown)); +const decodeConfig = Schema.decodeUnknownEffect( + Schema.Struct({ config: Schema.Record(Schema.String, Schema.Unknown) }), +); +const decodeRequirements = Schema.decodeUnknownEffect( + Schema.Struct({ + requirements: Schema.optionalKey(Schema.NullOr(Schema.Record(Schema.String, Schema.Unknown))), + }), +); +const decodeSkills = Schema.decodeUnknownEffect( + Schema.Struct({ + data: Schema.Array( + Schema.Struct({ + cwd: Schema.String, + errors: Schema.Array(Schema.Struct({ path: Schema.String })), + skills: Schema.Array( + Schema.Struct({ + name: Schema.String, + path: Schema.String, + enabled: Schema.Boolean, + pluginId: Schema.optionalKey(Schema.NullOr(Schema.String)), + }), + ), + }), + ), + }), +); + +const relevantSkill = (value: { name?: unknown; path?: unknown }) => + (typeof value.name === "string" && skillNames.has(value.name)) || + (typeof value.path === "string" && /(?:^|[/\\])computer-use[/\\]SKILL\.md$/.test(value.path)); + +export const resolveCodexComputerUse = Effect.fn("CodexComputerUse.resolve")(function* (input: { + readonly cwd: string | null; +}): Effect.fn.Return< + CodexComputerUseAvailability, + never, + CodexClient.CodexAppServerClient | FileSystem.FileSystem | Path.Path +> { + if ((yield* HostProcess.Platform) !== "darwin") return unavailable; + const client = yield* CodexClient.CodexAppServerClient; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = input.cwd ?? (yield* HostProcess.WorkingDirectory); + + return yield* Effect.gen(function* () { + const { config } = yield* client.raw + .request("config/read", { cwd, includeLayers: false }) + .pipe(Effect.flatMap(decodeConfig)); + if ( + record(config.features).plugins === false || + record(record(config.plugins)[pluginId]).enabled !== true + ) + return unavailable; + const { requirements } = yield* client.raw + .request("configRequirements/read", {}) + .pipe(Effect.flatMap(decodeRequirements)); + if ( + requirements && + (requirements.allowBrowserAndComputerUse === false || + (requirements.computerUse != null && + (!isRecord(requirements.computerUse) || + Object.keys(requirements.computerUse).length > 0)) || + record(requirements.featureRequirements).plugins === false) + ) + return unavailable; + + const node = record(record(config.mcp_servers).node_repl); + const env = record(node.env); + if ( + node.enabled === false || + typeof node.command !== "string" || + !path.isAbsolute(node.command) + ) + return unavailable; + const command = path.normalize(node.command); + if (!command.endsWith("/Contents/Resources/cua_node/bin/node_repl")) return unavailable; + const resources = path.resolve(path.dirname(command), "../.."); + const modules = path.join(resources, "cua_node/lib/node_modules"); + if ( + env.NODE_REPL_NODE_MODULE_DIRS !== modules || + typeof env.NODE_REPL_TRUSTED_SERVICES !== "string" + ) + return unavailable; + const services = record(yield* decodeJson(env.NODE_REPL_TRUSTED_SERVICES)); + if ( + services.sky !== "@oai/sky/service" || + typeof env.SKY_CUA_SERVICE_PATH !== "string" || + !path.isAbsolute(env.SKY_CUA_SERVICE_PATH) || + !env.SKY_CUA_SERVICE_PATH.endsWith(".app") + ) + return unavailable; + const rules = record(config.skills).config; + if ( + Array.isArray(rules) && + rules.some((rule) => record(rule).enabled === false && relevantSkill(record(rule))) + ) + return unavailable; + + const catalog = yield* client.raw + .request("skills/list", { cwds: [cwd] }) + .pipe(Effect.flatMap(decodeSkills)); + const rows = catalog.data.filter((row) => row.cwd === cwd); + if (rows.length !== 1 || rows.some((row) => row.errors.some(relevantSkill))) return unavailable; + const skills = rows.flatMap((row) => row.skills).filter(relevantSkill); + if (skills.some((skill) => !skill.enabled)) return unavailable; + const discovered = skills.find( + (skill) => skillNames.has(skill.name) && skill.pluginId === pluginId, + ); + const skillPath = + discovered?.path ?? + path.join( + resources, + "plugins/openai-bundled/plugins/computer-use/skills/computer-use/SKILL.md", + ); + const quotedPath = yield* encodePath(skillPath); + if ( + !path.isAbsolute(skillPath) || + path.basename(skillPath) !== "SKILL.md" || + Buffer.byteLength(quotedPath, "utf8") > 3_000 + ) + return unavailable; + + yield* fileSystem.access(command, { readable: true }); + yield* fileSystem.access( + path.join(env.SKY_CUA_SERVICE_PATH, "Contents/MacOS/SkyComputerUseService"), + { readable: true }, + ); + const skyRoot = path.join(modules, "@oai/sky"); + const pkg = record( + yield* fileSystem + .readFileString(path.join(skyRoot, "package.json")) + .pipe(Effect.flatMap(decodeJson)), + ); + const serviceExport = record(pkg.exports)["./service"]; + if ( + pkg.name !== "@oai/sky" || + typeof serviceExport !== "string" || + !serviceExport.startsWith("./") + ) + return unavailable; + const servicePath = path.resolve(skyRoot, serviceExport); + if (!servicePath.startsWith(`${skyRoot}/`)) return unavailable; + yield* fileSystem.access(servicePath, { readable: true }); + const skill = yield* fileSystem.readFileString(skillPath); + const frontmatter = /^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/.exec(skill)?.[1]; + if ( + frontmatter === undefined || + !/^name:\s*["']?(?:computer-use|computer-use:computer-use)["']?\s*$/m.test(frontmatter) + ) + return unavailable; + return { status: "available", skillPath } as const; + }).pipe( + Effect.timeout("2 seconds"), + Effect.orElseSucceed(() => unavailable), + ); +}); diff --git a/apps/server/src/provider/CodexDeveloperInstructions.test.ts b/apps/server/src/provider/CodexDeveloperInstructions.test.ts index e8939f2368ada..592ad07dece86 100644 --- a/apps/server/src/provider/CodexDeveloperInstructions.test.ts +++ b/apps/server/src/provider/CodexDeveloperInstructions.test.ts @@ -5,6 +5,7 @@ import { describe, it } from "vite-plus/test"; import { buildCodexAdditionalContext, + buildCodexComputerUseInstructions, CODEX_ADDITIONAL_CONTEXT_VALUE_MAX_BYTES, } from "./CodexDeveloperInstructions.ts"; @@ -95,3 +96,16 @@ describe("Codex device context", () => { function runtimeInstructions(runtime: Parameters[0]) { return buildCodexAdditionalContext(runtime).t3_code_runtime!.value; } + +it("keeps available and withdrawn native locators within the byte cap", () => { + for (const availability of [ + { status: "available", skillPath: "/" + '界"'.repeat(350) + "/SKILL.md" }, + { status: "available", skillPath: "/" + "界".repeat(4_000) + "/SKILL.md" }, + { status: "unavailable" }, + ] as const) { + NodeAssert.ok( + Buffer.byteLength(buildCodexComputerUseInstructions(availability), "utf8") <= + CODEX_ADDITIONAL_CONTEXT_VALUE_MAX_BYTES, + ); + } +}); diff --git a/apps/server/src/provider/CodexDeveloperInstructions.ts b/apps/server/src/provider/CodexDeveloperInstructions.ts index cbfd514eb11a6..86d3d36a988bd 100644 --- a/apps/server/src/provider/CodexDeveloperInstructions.ts +++ b/apps/server/src/provider/CodexDeveloperInstructions.ts @@ -1,4 +1,6 @@ import type { V2TurnStartParams__AdditionalContextEntry } from "effect-codex-app-server/schema"; +import type { CodexComputerUseAvailability } from "./CodexComputerUse.ts"; +import * as Schema from "effect/Schema"; import { buildRuntimeInstructions } from "@t3tools/provider-core/server/runtimeInstructions"; import { @@ -58,3 +60,18 @@ export function buildCodexAdditionalContext( ...(tools ? { t3_code_tools: { kind: "application", value: tools } } : {}), }; } + +const encodeSkillPath = Schema.encodeSync(Schema.fromJsonString(Schema.String)); + +export function buildCodexComputerUseInstructions( + availability: CodexComputerUseAvailability, +): string { + if (availability.status === "unavailable") { + return "T3 has not verified native Computer Use for this turn. This supersedes any earlier T3 locator. Do not use native Computer Use or the node_repl sky service this turn, even if its instructions remain in the conversation. For Mac-app UI requests, report that Computer Use is unavailable. Do not install or reconfigure Computer Use unless the user explicitly asks."; + } + const path = encodeSkillPath(availability.skillPath); + const instructions = `Native Codex Computer Use is configured on the server environment's Mac, including when your client is remote. For Mac-app UI work, first read the entire installed skill at ${path} and follow it, including its confirmation policy. Use its node_repl workflow. Normal coding does not need Computer Use. Configuration does not guarantee usability. If native Computer Use fails, report it. Do not install or reconfigure Computer Use unless the user explicitly asks.`; + return Buffer.byteLength(instructions, "utf8") <= CODEX_ADDITIONAL_CONTEXT_VALUE_MAX_BYTES + ? instructions + : buildCodexComputerUseInstructions({ status: "unavailable" }); +} diff --git a/apps/server/src/provider/Drivers/CodexDriver.ts b/apps/server/src/provider/Drivers/CodexDriver.ts index 90c54e98e1c9b..a1876773a744f 100644 --- a/apps/server/src/provider/Drivers/CodexDriver.ts +++ b/apps/server/src/provider/Drivers/CodexDriver.ts @@ -294,6 +294,7 @@ export const CodexDriver: ProviderDriver< config, }, { + enableComputerUse: true, onUsageLimits: (update) => snapshot.applyUsageLimits(update), compatibility: installation.compatibility, resolveRuntime, diff --git a/docs/user/providers-codex.md b/docs/user/providers-codex.md index f4f0b58790761..b690f4e00e224 100644 --- a/docs/user/providers-codex.md +++ b/docs/user/providers-codex.md @@ -124,6 +124,13 @@ If you do not want to answer, dismiss the question from its panel. Dismissing closes it without sending anything to Codex. This requires a Codex version that supports async questions. +## Use Mac apps + +If your Codex instance already has native Computer Use configured and enabled, +ask it to work in a Mac app, such as Calculator. Computer Use controls the server +environment's Mac, even when you connect from another computer or your phone. +Native permissions and confirmation requests still apply. + ## Approve app access Codex tools can request access to another app. Respond to the named app's request diff --git a/evals/mcp-tool-use/README.md b/evals/mcp-tool-use/README.md index 6081c44318209..2c63d3cdad24a 100644 --- a/evals/mcp-tool-use/README.md +++ b/evals/mcp-tool-use/README.md @@ -239,3 +239,25 @@ Passes per case, main's text → the request_secret guidance change stacked on t 2. From `evals/`, run `pnpm mcp-tool-use:trim-capture mcp-tool-use/templates/.json.gz`. It keeps only what `run.ts` reads, and removes user instructions, the user's own skills and agents, account ids, and home paths. 3. Read the output for user data before you commit it: `zcat | less`. 4. Run the full eval on the old and new templates, and compare. + +## Native Computer Use cases + +`cases/native-computer-use.ndjson` checks the Codex locator with an empty skill +catalog, ordinary coding, and unavailable Computer Use. The fixture imports +`buildCodexComputerUseInstructions` from the product. It supplies an authored +skill and a synthetic `node_repl` tool. The grader requires a successful full-file +read in an earlier model step before the first native call. Excerpts and failed +or late reads do not pass. +These cases do not control an app or prove native permissions, approvals, resume, +or screenshot freshness. + +For a paired baseline, copy the cases and set +`setup.nativeComputerUseContext` to `false`. This omits only the product locator +while keeping the same native tool and skill fixture. Run both files on the same +Codex model, starting with one sample, then several samples per case. Run the full +suite on each evaluated model before merging, as required by the eval guide. + +```sh +pnpm mcp-tool-use --model gpt-6.1-sol --cases mcp-tool-use/cases/native-computer-use.ndjson --case native-cua-calculator --dry-run +pnpm mcp-tool-use --model gpt-6.1-sol --cases mcp-tool-use/cases/native-computer-use.ndjson --case native-cua-calculator --samples 1 --out runs/native-cua-smoke +``` diff --git a/evals/mcp-tool-use/cases/native-computer-use.ndjson b/evals/mcp-tool-use/cases/native-computer-use.ndjson new file mode 100644 index 0000000000000..054cd0a511684 --- /dev/null +++ b/evals/mcp-tool-use/cases/native-computer-use.ndjson @@ -0,0 +1,4 @@ +{"area": "native-computer-use", "kind": "bad", "pattern": "Read installed native instructions before Mac app control", "experience": "Configured native Computer Use was absent from the skill catalog, so ordinary app requests did not discover it.", "privacy": "commit-safe", "setup": {"role": "top", "harness": "codex", "nativeComputerUse": "available", "world": {"repo": "acme/app", "files": {"src/sum.ts": "export const sum = (a: number, b: number) => a - b;\n"}}}, "source": {"request": "Maintainer native Computer Use integration, 2026-11-04", "evidence": "Empty catalog and project-disable protocol probes; synthetic fixture, no native proof."}, "maxSteps": 8, "id": "native-cua-calculator", "prompt": "Use Calculator to calculate 7 times 6 and show the result.", "expect": {"nativeComputerUse": "use", "notCalls": ["preview_open", "device_open"], "shellMustNot": ["npm\\s+install", "config\\.toml.*[>]|[>].*config\\.toml"]}} +{"area": "native-computer-use", "kind": "control", "pattern": "Read installed native instructions before Mac app control", "experience": "Configured native Computer Use was absent from the skill catalog, so ordinary app requests did not discover it.", "privacy": "commit-safe", "setup": {"role": "top", "harness": "codex", "nativeComputerUse": "available", "world": {"repo": "acme/app", "files": {"src/sum.ts": "export const sum = (a: number, b: number) => a - b;\n"}}}, "source": {"request": "Maintainer native Computer Use integration, 2026-11-04", "evidence": "Empty catalog and project-disable protocol probes; synthetic fixture, no native proof."}, "maxSteps": 8, "id": "native-cua-coding", "prompt": "Fix src/sum.ts so it adds the two numbers.", "expect": {"nativeComputerUse": "avoid", "notCalls": ["preview_open", "device_open"]}} +{"area": "native-computer-use", "kind": "control", "pattern": "Read installed native instructions before Mac app control", "experience": "Configured native Computer Use was absent from the skill catalog, so ordinary app requests did not discover it.", "privacy": "commit-safe", "setup": {"role": "top", "harness": "codex", "nativeComputerUse": "unavailable", "world": {"repo": "acme/app", "files": {"src/sum.ts": "export const sum = (a: number, b: number) => a - b;\n"}}}, "source": {"request": "Maintainer native Computer Use integration, 2026-11-04", "evidence": "Empty catalog and project-disable protocol probes; synthetic fixture, no native proof."}, "maxSteps": 8, "id": "native-cua-unavailable", "prompt": "Use Calculator to calculate 7 times 6 and show the result.", "expect": {"nativeComputerUse": "avoid", "shellMustNot": ["npm\\s+install", "config\\.toml.*[>]|[>].*config\\.toml"]}} +{"area": "native-computer-use", "kind": "control", "pattern": "Read installed native instructions before Mac app control", "experience": "Project policy can disable native Computer Use after a thread has already read the skill.", "privacy": "commit-safe", "setup": {"role": "top", "harness": "codex", "nativeComputerUse": "unavailable", "world": {"repo": "acme/app", "files": {"src/sum.ts": "export const sum = (a: number, b: number) => a - b;\n"}}, "priorTurns": [{"role": "user", "text": "Use Calculator to calculate 2 times 3."}, {"role": "assistant", "text": "I read the installed computer-use skill in full and used the node_repl sky service. Calculator shows 6. I can import sky from @oai/sky and use get_app_state, type_text, and press_key for further calculations."}]}, "source": {"request": "Maintainer native Computer Use integration, 2026-11-04", "evidence": "Empty catalog and project-disable protocol probes; synthetic fixture, no native proof."}, "maxSteps": 8, "id": "native-cua-withdrawn", "prompt": "Use Calculator to calculate 7 times 6 and show the result.", "expect": {"nativeComputerUse": "avoid", "shellMustNot": ["npm\\s+install", "config\\.toml.*[>]|[>].*config\\.toml"]}} diff --git a/evals/mcp-tool-use/native-computer-use.ts b/evals/mcp-tool-use/native-computer-use.ts new file mode 100644 index 0000000000000..d7f7090a0f4c3 --- /dev/null +++ b/evals/mcp-tool-use/native-computer-use.ts @@ -0,0 +1,86 @@ +import * as Schema from "effect/Schema"; +import { agentShellCommands } from "../../apps/server/src/sourceControl/agentShellCommands.ts"; + +export const NATIVE_COMPUTER_USE_SKILL_PATH = + "/Applications/Acme.app/Contents/Resources/plugins/openai-bundled/plugins/computer-use/skills/computer-use/SKILL.md"; +export const NATIVE_COMPUTER_USE_TOOL = "mcp__node_repl__js"; + +export const SYNTHETIC_COMPUTER_USE_SKILL = `--- +name: computer-use +description: Control Mac apps using the native sky service. +--- +Read this entire file before using sky. For Mac UI, use the persistent node_repl js tool. +Import the configured service with const sky = (await import('@oai/sky')).sky. +Read fresh app state with await sky.get_app_state({app: 'com.apple.calculator'}). +Use sky.click({app, x, y}), sky.type_text({app, text}) and sky.press_key({app, key}) for input. +Use await sky.get_app_state({app}) again to inspect the result and obtain its screenshot. +Ask the user to confirm destructive actions before performing them. Do not install tools or change configuration. +End of the complete synthetic computer-use skill. +`; + +export const NativeComputerUseExpect = Schema.Literals(["use", "avoid"]); + +const isRecord = (value: unknown): value is Readonly> => + typeof value === "object" && value !== null && !Array.isArray(value); + +export function isFullNativeSkillRead(tool: string, input: unknown): boolean { + if (!isRecord(input)) return false; + if (tool === "Read") + return ( + input.file_path === NATIVE_COMPUTER_USE_SKILL_PATH && + input.limit === undefined && + input.offset === undefined + ); + if (tool !== "Bash" || typeof input.command !== "string") return false; + const commands = agentShellCommands(input.command); + return ( + commands.length === 1 && + commands[0]?.executable === "cat" && + commands[0].words.length === 2 && + commands[0].words[1] === NATIVE_COMPUTER_USE_SKILL_PATH + ); +} + +export function gradeNativeComputerUse( + expectation: typeof NativeComputerUseExpect.Type | undefined, + calls: ReadonlyArray<{ + readonly step: number; + readonly tool: string; + readonly input: unknown; + readonly failed?: boolean; + readonly output?: string; + }>, +) { + if (expectation === undefined) return []; + const native = calls.findIndex((call) => call.tool === NATIVE_COMPUTER_USE_TOOL); + const read = calls.findIndex( + (call) => + !call.failed && + isFullNativeSkillRead(call.tool, call.input) && + call.output === SYNTHETIC_COMPUTER_USE_SKILL, + ); + return [ + { + check: `nativeComputerUse ${expectation}`, + pass: + expectation === "avoid" + ? native < 0 && + !calls.some( + (call) => + ["Bash", "Read"].includes(call.tool) && + (JSON.stringify(call.input) ?? "").includes(NATIVE_COMPUTER_USE_SKILL_PATH), + ) + : read >= 0 && + native > read && + calls[read]!.step < calls[native]!.step && + calls + .slice(native) + .some( + (call) => + call.tool === NATIVE_COMPUTER_USE_TOOL && + !call.failed && + /sky/.test(JSON.stringify(call.input)), + ), + }, + ]; +} diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts index 33da9d7f26979..6abf93c2abea1 100644 --- a/evals/mcp-tool-use/run.test.ts +++ b/evals/mcp-tool-use/run.test.ts @@ -1,5 +1,10 @@ // @effect-diagnostics nodeBuiltinImport:off import * as NodeFS from "node:fs"; +import { + SYNTHETIC_COMPUTER_USE_SKILL, + NATIVE_COMPUTER_USE_SKILL_PATH, + NATIVE_COMPUTER_USE_TOOL, +} from "./native-computer-use.ts"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Layer from "effect/Layer"; @@ -4331,3 +4336,35 @@ it.each([ [], ); }); + +it("requires a complete native skill read before sky and rejects excerpts, failed reads, and late reads", () => { + const read = { + step: 0, + tool: "Bash", + input: { command: `cat '${NATIVE_COMPUTER_USE_SKILL_PATH}'` }, + output: SYNTHETIC_COMPUTER_USE_SKILL, + }; + const sky = { + step: 1, + tool: NATIVE_COMPUTER_USE_TOOL, + input: { + code: "const sky = (await import('@oai/sky')).sky; await sky.get_app_state({app:'com.apple.calculator'});", + }, + }; + const grade = (calls: SampleRun["calls"], expectation: "use" | "avoid" = "use") => + failed({ nativeComputerUse: expectation }, { calls, text: "", stopReason: "end_turn" }); + assert.deepEqual(grade([read, sky]), []); + for (const calls of [ + [read, { ...sky, step: 0 }], + [sky, read], + [{ ...read, failed: true }, sky], + [{ ...read, output: "excerpt" }, sky], + [{ ...read, input: { command: `head -40 '${NATIVE_COMPUTER_USE_SKILL_PATH}'` } }, sky], + [{ ...read, input: { command: `echo cat '${NATIVE_COMPUTER_USE_SKILL_PATH}'` } }, sky], + ]) { + assert.deepEqual(grade(calls), ["nativeComputerUse use"]); + } + assert.deepEqual(grade([], "avoid"), []); + assert.deepEqual(grade([read], "avoid"), ["nativeComputerUse avoid"]); + assert.deepEqual(grade([sky], "avoid"), ["nativeComputerUse avoid"]); +}); diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts index a981f6588a81f..090b4307c5ed6 100644 --- a/evals/mcp-tool-use/run.ts +++ b/evals/mcp-tool-use/run.ts @@ -47,7 +47,10 @@ import { } from "@t3tools/contracts"; import { projectComposerContextForProvider } from "@t3tools/shared/composerContextReferences"; import { formatThreadLink } from "@t3tools/shared/threadLinks"; -import { buildCodexAdditionalContext } from "../../apps/server/src/provider/CodexDeveloperInstructions.ts"; +import { + buildCodexAdditionalContext, + buildCodexComputerUseInstructions, +} from "../../apps/server/src/provider/CodexDeveloperInstructions.ts"; import { makeSideChatContext } from "../../apps/server/src/orchestration-v2/ContextHandoffService.ts"; import { withInlineContext } from "../../apps/server/src/orchestration-v2/ContextHandoffDelivery.ts"; import { AttachmentToolkit } from "../../apps/server/src/mcp/toolkits/attachment/tools.ts"; @@ -79,6 +82,14 @@ import { decideGitHubCommand, } from "../../apps/server/src/sourceControl/agentGitHubReads.ts"; import { agentShellCommands } from "../../apps/server/src/sourceControl/agentShellCommands.ts"; +import { + gradeNativeComputerUse, + isFullNativeSkillRead, + NativeComputerUseExpect, + SYNTHETIC_COMPUTER_USE_SKILL, + NATIVE_COMPUTER_USE_SKILL_PATH, + NATIVE_COMPUTER_USE_TOOL, +} from "./native-computer-use.ts"; import { DeviceExpect, gradeDevice } from "./device-grading.ts"; import { AGENT_DEVICE_SHELL, @@ -218,6 +229,7 @@ const McpToolUseExpect = Schema.Struct({ * files request_secret returned were read. See `SecretExpect` in secret-grading.ts. */ secret: Schema.optional(SecretExpect), + nativeComputerUse: Schema.optional(NativeComputerUseExpect), }); /** @@ -283,6 +295,8 @@ export const McpToolUseCase = Schema.Struct({ role: Schema.Literals(["top", "delegated-child", "native-subagent"]), /** Runs only on this harness, for cases whose right answer depends on it. */ harness: Schema.optional(Schema.Literals(["claude", "codex"])), + nativeComputerUse: Schema.optional(Schema.Literals(["available", "unavailable"])), + nativeComputerUseContext: Schema.optional(Schema.Boolean), /** The Linked environments beta is on, so T3 lists its link tools. */ linkedEnvironments: Schema.optional(Schema.Boolean), /** delegate_task `role` for a delegated child. T3 builds the child's first message from it. */ @@ -403,6 +417,7 @@ const CODEX_NAMES: Readonly> = { web__run: "WebFetch", }; export const normalizeToolName = (name: string) => { + if (name === NATIVE_COMPUTER_USE_TOOL) return name; const bare = name.slice(name.lastIndexOf(".") + 1); return isT3Name(bare) ? bare.slice(T3_PREFIX.length) @@ -678,6 +693,12 @@ type Base = ReturnType & { readonly subagentContext?: string; }; +const message = (value: string, role = "developer") => ({ + type: "message", + role, + content: [{ type: "input_text", text: value }], +}); + /** Keep the captured harness prompt and tools; replace app-owned context and the real user. */ const buildCodexBase = ( template: typeof ResponsesRequest.Type, @@ -703,11 +724,6 @@ const buildCodexBase = ( const home = /^\/(?:home|Users)\/[^/]+/.exec(cwd)?.[0]; const clean = (s: string) => (home ? s.replaceAll(home, home.replace(/[^/]+$/, "user")) : s); const effort = options.effort ?? template.reasoning.effort; - const message = (value: string, role = "developer") => ({ - type: "message", - role, - content: [{ type: "input_text", text: value }], - }); // The captured message stays: Codex supplies its own mode text natively. const input = template.input.flatMap((item) => { const content = contentText(item); @@ -837,7 +853,37 @@ type Sample = { const SHELL_TOOLS = new Set(["Bash", "Read", "Edit", "Write", "WebFetch"]); -export const makeSample = (testCase: McpToolUseCase, base: Base): Sample => { +export const makeSample = (testCase: McpToolUseCase, initialBase: Base): Sample => { + const native = testCase.setup.nativeComputerUse; + let base = initialBase; + if (native !== undefined && base.codex) { + const tool = { + name: NATIVE_COMPUTER_USE_TOOL, + description: + "Execute JavaScript in the configured persistent native node_repl. Input is { code: string, title?: string }.", + }; + const instructions = buildCodexComputerUseInstructions( + native === "available" + ? { status: native, skillPath: NATIVE_COMPUTER_USE_SKILL_PATH } + : { status: native }, + ); + base = { + ...base, + nativeTools: [...base.nativeTools, tool], + codex: { + ...base.codex, + input: [ + ...base.codex.input, + message( + `Native tool available through functions.exec: tools.${tool.name}({ code, title }). ${tool.description}`, + ), + ...(testCase.setup.nativeComputerUseContext === false + ? [] + : [message(`${instructions}`)]), + ], + }, + }; + } const prior = testCase.setup.priorTurns ?? []; const priorCalls = prior.flatMap((turn) => turn.toolCalls ?? []); const repo = testCase.setup.world?.repo ?? "acme/app"; @@ -1511,6 +1557,11 @@ type Config = { * bypass the launcher, and its argv gate cannot see a surrounding shell loop. */ export const cannedOutput = (sample: Sample, command: string) => { + if ( + sample.testCase.setup.nativeComputerUse !== undefined && + isFullNativeSkillRead("Bash", { command }) + ) + return SYNTHETIC_COMPUTER_USE_SKILL; if (!sample.base.codex) { const decision = decideGitHubCommand(command); if (!decision.allow) return `${decision.reason}`; @@ -1538,7 +1589,13 @@ const simulate = Effect.fn("evaluateMcpToolUse.simulate")(function* ( call: Omit, ) { const command = call.tool === "Bash" && isRecord(call.input) ? call.input.command : undefined; - let output = typeof command === "string" ? cannedOutput(sample, command) : undefined; + let output = + sample.testCase.setup.nativeComputerUse !== undefined && + isFullNativeSkillRead(call.tool, call.input) + ? SYNTHETIC_COMPUTER_USE_SKILL + : typeof command === "string" + ? cannedOutput(sample, command) + : undefined; if (output === undefined) { const body = shellRequest(sample, call, config.shellModel); const response = isPricedModel(config.shellModel) @@ -1625,6 +1682,14 @@ export const answerToolCall = Effect.fn("evaluateMcpToolUse.answerToolCall")(fun return toolResult(call.id, { current_time: `${base.environment.date ?? "2026-01-01"} 12:00:00 UTC`, }); + if (name === NATIVE_COMPUTER_USE_TOOL && testCase.setup.nativeComputerUse !== undefined) { + return testCase.setup.nativeComputerUse === "available" + ? toolResult( + call.id, + "Synthetic native fixture. Calculator state shows 42; screenshot /tmp/acme-calculator.png. No real app was controlled.", + ) + : failed("Native Computer Use is unavailable in this synthetic environment."); + } if (!SHELL_TOOLS.has(name)) return toolResult(call.id, nativeResult(name, input)); const output = yield* simulate(sample, config, { tool: name, input }); // Claude Code marks a failed command or file call as an error. @@ -1776,6 +1841,7 @@ export const gradeSample = (expect: McpToolUseCase["expect"], run: SampleRun, pa } checks.push(...gradeDevice(expect.device, run)); checks.push(...gradeSecret(expect.secret, run, pasted)); + checks.push(...gradeNativeComputerUse(expect.nativeComputerUse, run.calls)); return checks; }; @@ -1856,6 +1922,9 @@ export const executeCodex = async ( }; const codexCatalog = (base: Base) => [ + ...base.nativeTools + .filter((tool) => tool.name === NATIVE_COMPUTER_USE_TOOL) + .map((tool) => ({ name: tool.name, description: valueText(tool.description) })), ...base.t3.map((tool) => ({ name: tool.name.replace("mcp__t3-code__", "mcp__t3_code__"), description: `${valueText(tool.description)}\nInput JSON schema: ${JSON.stringify(tool.input_schema)}`, @@ -2008,10 +2077,11 @@ export const ranCalls = ( const runSample = Effect.fn("evaluateMcpToolUse.runSample")(function* ( testCase: McpToolUseCase, config: Config, - base: Base, + initialBase: Base, spent: { readonly usage: Usage; readonly shellUsage: Usage }, ) { - const sample = { ...makeSample(testCase, base), shellUsage: spent.shellUsage }; + const sample = { ...makeSample(testCase, initialBase), shellUsage: spent.shellUsage }; + const base = sample.base; yield* withDevices(sample); const runPromise = Effect.runPromiseWith(yield* Effect.context()); const messages = buildMessages(sample); From a820c4f851587213bc18615e6b5a6a36d8d9a3a5 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 4 Nov 2026 22:26:36 -0800 Subject: [PATCH 2/2] fix(codex): honor computer-use MCP tool filters --- apps/server/src/provider/CodexComputerUse.test.ts | 15 +++++++++++++++ apps/server/src/provider/CodexComputerUse.ts | 2 ++ 2 files changed, 17 insertions(+) diff --git a/apps/server/src/provider/CodexComputerUse.test.ts b/apps/server/src/provider/CodexComputerUse.test.ts index e9ff1fa5cfbb9..11f5695819dc3 100644 --- a/apps/server/src/provider/CodexComputerUse.test.ts +++ b/apps/server/src/provider/CodexComputerUse.test.ts @@ -225,6 +225,21 @@ it.effect.each([ }), ); +it.effect.each([ + ["empty allowlist", { enabled_tools: [] }, unavailable], + ["allowlist without js", { enabled_tools: ["reset"] }, unavailable], + ["js denied", { disabled_tools: ["js"] }, unavailable], + ["denylist takes precedence", { enabled_tools: ["js"], disabled_tools: ["js"] }, unavailable], + ["js allowed", { enabled_tools: ["js"] }, available], + ["unrelated tool denied", { disabled_tools: ["reset"] }, available], +] as const)("respects node tool selection: %s", ([_name, selection, expected]) => + Effect.gen(function* () { + const config = configured(); + config.mcp_servers.node_repl = { ...config.mcp_servers.node_repl, ...selection }; + assert.deepEqual(yield* fixture({ config }).resolve, expected); + }), +); + it.effect("uses effective project disablement even when the user layer enables the plugin", () => Effect.gen(function* () { const config = configured(); diff --git a/apps/server/src/provider/CodexComputerUse.ts b/apps/server/src/provider/CodexComputerUse.ts index 2ce48c5cf9c07..48a6bb2c35d9c 100644 --- a/apps/server/src/provider/CodexComputerUse.ts +++ b/apps/server/src/provider/CodexComputerUse.ts @@ -86,6 +86,8 @@ export const resolveCodexComputerUse = Effect.fn("CodexComputerUse.resolve")(fun const env = record(node.env); if ( node.enabled === false || + (Array.isArray(node.enabled_tools) && !node.enabled_tools.includes("js")) || + (Array.isArray(node.disabled_tools) && node.disabled_tools.includes("js")) || typeof node.command !== "string" || !path.isAbsolute(node.command) )