From fa6f917c8720a653b6a0313a8258217753947764 Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Wed, 21 Oct 2026 18:43:30 -0500 Subject: [PATCH 01/12] test(mcp): serve every parameter description agents were given Co-Authored-By: Claude Opus 5.5 --- apps/server/src/mcp/toolkits/tools.test.ts | 83 ++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 apps/server/src/mcp/toolkits/tools.test.ts diff --git a/apps/server/src/mcp/toolkits/tools.test.ts b/apps/server/src/mcp/toolkits/tools.test.ts new file mode 100644 index 0000000000000..4af3f354f3e57 --- /dev/null +++ b/apps/server/src/mcp/toolkits/tools.test.ts @@ -0,0 +1,83 @@ +import { expect, it } from "@effect/vitest"; +import * as Schema from "effect/Schema"; +import { Tool } from "effect/ai"; + +import { AttachmentToolkit } from "./attachment/tools.ts"; +import { DeviceToolkit } from "./device/tools.ts"; +import { EnvironmentToolkit } from "./environment/tools.ts"; +import { HtmlToolkit } from "./html/tools.ts"; +import { OrchestratorToolkit } from "./orchestrator/tools.ts"; +import { PreviewToolkit } from "./preview/tools.ts"; +import { PreviewControlsToolkit } from "./previewControls/tools.ts"; +import { ProjectToolkit } from "./project/tools.ts"; +import { PullRequestsToolkit } from "./pullRequests/tools.ts"; +import { ThreadToolkit } from "./thread/tools.ts"; +import { WorktreeToolkit } from "./worktree/tools.ts"; + +interface JsonSchemaNode { + readonly description?: string; + readonly properties?: Readonly>; + readonly anyOf?: ReadonlyArray; +} + +const descriptionOf = (schema: JsonSchemaNode | undefined): string | undefined => + schema?.description ?? schema?.anyOf?.map(descriptionOf).find(Boolean); + +const objectOf = (schema: JsonSchemaNode | undefined): JsonSchemaNode | undefined => + schema?.properties ? schema : schema?.anyOf?.map(objectOf).find(Boolean); + +const missingDescriptions = ( + decoded: JsonSchemaNode | undefined, + served: JsonSchemaNode | undefined, + path: string, +): ReadonlyArray => + Object.entries(objectOf(decoded)?.properties ?? {}).flatMap(([field, fieldSchema]) => { + const servedField = objectOf(served)?.properties?.[field]; + const description = descriptionOf(fieldSchema); + const missing = + description !== undefined && descriptionOf(servedField) !== description + ? [`${path}.${field}`] + : []; + return [...missing, ...missingDescriptions(fieldSchema, servedField, `${path}.${field}`)]; + }); + +const tools = [ + AttachmentToolkit, + DeviceToolkit, + EnvironmentToolkit, + HtmlToolkit, + OrchestratorToolkit, + PreviewToolkit, + PreviewControlsToolkit, + ProjectToolkit, + PullRequestsToolkit, + ThreadToolkit, + WorktreeToolkit, +].flatMap((toolkit) => Object.values(toolkit.tools) as Tool.Any[]); + +// Agents receive the encoded-side JSON schema, so a description written with +// `.annotate` on a transformed field, such as a trimmed string, never reaches them. +it("serves every parameter description written on the decoded side", () => { + const missing = tools.flatMap((tool) => + missingDescriptions( + Schema.toJsonSchemaDocument(Schema.toType(tool.parametersSchema)).schema as JsonSchemaNode, + Tool.getJsonSchema(tool) as JsonSchemaNode, + tool.name, + ), + ); + expect(missing).toEqual([]); +}); + +it.each([ + ["delegate_task", "task"], + ["delegate_task", "clientRequestId"], + ["request_secret", "label"], + ["schedule_task", "projectId"], + ["link_pull_request", "url"], + ["t3_worktree_handoff", "branch"], + ["device_open", "deviceId"], +])("describes %s.%s to agents", (name, field) => { + const tool = tools.find((candidate) => candidate.name === name); + const served = tool && (Tool.getJsonSchema(tool) as JsonSchemaNode); + expect(descriptionOf(served?.properties?.[field])).toBeTruthy(); +}); From 5afab7448bdf7b24757cd84c8102915ad5081652 Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Wed, 21 Oct 2026 18:52:11 -0500 Subject: [PATCH 02/12] fix(mcp): keep parameter descriptions on trimmed-string tool fields Agents read each tool's encoded-side JSON schema. A description set with .annotate on a trimmed string lands on the decoded side and never reaches them. Annotate the optional wrapper, as this code already does elsewhere, or use Schema.annotateEncoded where the field is required. Co-Authored-By: Claude Opus 5.5 --- .../src/mcp/toolkits/pullRequests/tools.ts | 44 ++++---- packages/contracts/src/device.ts | 34 +++--- packages/contracts/src/orchestratorMcp.ts | 102 +++++++++--------- packages/contracts/src/threadMetadataMcp.ts | 16 +-- packages/contracts/src/worktreeMcp.ts | 24 ++--- 5 files changed, 107 insertions(+), 113 deletions(-) diff --git a/apps/server/src/mcp/toolkits/pullRequests/tools.ts b/apps/server/src/mcp/toolkits/pullRequests/tools.ts index a3a75eabaa385..26433919da459 100644 --- a/apps/server/src/mcp/toolkits/pullRequests/tools.ts +++ b/apps/server/src/mcp/toolkits/pullRequests/tools.ts @@ -35,34 +35,26 @@ const REGISTER_EVERY_PR = "Register every pull request you open for this thread, including each layer of a stack, right after creating it."; export const PullRequestTargetInput = Schema.Struct({ - threadId: Schema.optional( - ThreadId.annotate({ - description: "Thread to act on. Omit for this thread.", - }), - ), - url: Schema.optional( - TrimmedNonEmptyString.annotate({ - description: - "The pull request's web URL, for example https://github.com/owner/repo/pull/123. Preferred when you have it; host, repository and number are read from it.", - }), - ), - repository: Schema.optional( - TrimmedNonEmptyString.annotate({ - description: - "Repository path below the host, for example owner/repo. Required with number when url is omitted.", - }), - ), + threadId: Schema.optional(ThreadId).annotate({ + description: "Thread to act on. Omit for this thread.", + }), + url: Schema.optional(TrimmedNonEmptyString).annotate({ + description: + "The pull request's web URL, for example https://github.com/owner/repo/pull/123. Preferred when you have it; host, repository and number are read from it.", + }), + repository: Schema.optional(TrimmedNonEmptyString).annotate({ + description: + "Repository path below the host, for example owner/repo. Required with number when url is omitted.", + }), number: Schema.optional( PositiveInt.annotate({ description: "Pull request number. Required with repository when url is omitted.", }), ), - host: Schema.optional( - TrimmedNonEmptyString.annotate({ - description: - "Host the repository lives on, for example github.com. Defaults to the host of this thread's project.", - }), - ), + host: Schema.optional(TrimmedNonEmptyString).annotate({ + description: + "Host the repository lives on, for example github.com. Defaults to the host of this thread's project.", + }), }); export type PullRequestTargetInput = typeof PullRequestTargetInput.Type; @@ -300,9 +292,9 @@ const UnlinkPullRequestTool = Tool.make("unlink_pull_request", { const ListThreadPullRequestsTool = Tool.make("list_thread_pull_requests", { description: `List the pull requests linked to a thread (omit threadId for this thread) with their last known host state, and how they chain into stacks (bottom to top). ${REGISTER_EVERY_PR}`, parameters: Schema.Struct({ - threadId: Schema.optional( - ThreadId.annotate({ description: "Thread to list. Omit for this thread." }), - ), + threadId: Schema.optional(ThreadId).annotate({ + description: "Thread to list. Omit for this thread.", + }), }), success: ListThreadPullRequestsResult, failure: PullRequestToolError, diff --git a/packages/contracts/src/device.ts b/packages/contracts/src/device.ts index 0fdfb3049443e..4a33664e41ed7 100644 --- a/packages/contracts/src/device.ts +++ b/packages/contracts/src/device.ts @@ -563,20 +563,18 @@ export const DeviceToolListResult = Schema.Struct({ export type DeviceToolListResult = typeof DeviceToolListResult.Type; export const DeviceToolOpenInput = Schema.Struct({ - deviceId: Schema.optional( - DeviceId.annotate({ - description: - "Simulator udid or emulator serial from device_list. Omit to use the booted device for the platform, or the most recently used one.", - }), - ), + deviceId: Schema.optional(DeviceId).annotate({ + description: + "Simulator udid or emulator serial from device_list. Omit to use the booted device for the platform, or the most recently used one.", + }), platform: Schema.optional( DevicePlatform.annotate({ description: "Required when deviceId is omitted and both platforms are available.", }), ), - hostId: Schema.optional( - DeviceHostId.annotate({ description: "Device host from device_list. Defaults to local." }), - ), + hostId: Schema.optional(DeviceHostId).annotate({ + description: "Device host from device_list. Defaults to local.", + }), }).annotate({ description: "Boots the device if needed, starts its live stream, and opens the Device panel so the user can watch. Returns how to drive it with the agent-device CLI.", @@ -594,12 +592,10 @@ export const DeviceToolOpenResult = Schema.Struct({ export type DeviceToolOpenResult = typeof DeviceToolOpenResult.Type; export const DeviceToolTargetInput = Schema.Struct({ - deviceId: Schema.optional( - DeviceId.annotate({ - description: - "Device from device_list. Omit to use the device most recently opened in this thread.", - }), - ), + deviceId: Schema.optional(DeviceId).annotate({ + description: + "Device from device_list. Omit to use the device most recently opened in this thread.", + }), hostId: Schema.optional(DeviceHostId), }); export type DeviceToolTargetInput = typeof DeviceToolTargetInput.Type; @@ -616,11 +612,9 @@ export const DeviceToolScreenshotResult = Schema.Struct({ export type DeviceToolScreenshotResult = typeof DeviceToolScreenshotResult.Type; export const DeviceToolCloseInput = Schema.Struct({ - deviceId: Schema.optional( - DeviceId.annotate({ - description: "Device to close. Omit to close every device in this thread.", - }), - ), + deviceId: Schema.optional(DeviceId).annotate({ + description: "Device to close. Omit to close every device in this thread.", + }), hostId: Schema.optional(DeviceHostId), shutdown: Schema.optional( Schema.Boolean.annotate({ diff --git a/packages/contracts/src/orchestratorMcp.ts b/packages/contracts/src/orchestratorMcp.ts index 7d643be910789..e93f737c1a6af 100644 --- a/packages/contracts/src/orchestratorMcp.ts +++ b/packages/contracts/src/orchestratorMcp.ts @@ -44,15 +44,21 @@ import { } from "./model.ts"; import { ProviderDriverKind, ProviderInstanceId } from "./providerInstance.ts"; -const OrchestratorMcpPrompt = TrimmedNonEmptyString.check(Schema.isMaxLength(120_000)).annotate({ - description: "Complete task or message text for the target agent.", -}); -const OrchestratorMcpTitle = TrimmedNonEmptyString.check(Schema.isMaxLength(512)).annotate({ - description: "Optional concise display title.", -}); -const OrchestratorMcpClientRequestId = TrimmedNonEmptyString.check( - Schema.isMaxLength(256), -).annotate({ description: "Stable idempotency key to reuse when retrying this mutation." }); +const OrchestratorMcpPrompt = TrimmedNonEmptyString.check(Schema.isMaxLength(120_000)).pipe( + Schema.annotateEncoded({ + description: "Complete task or message text for the target agent.", + }), +); +const OrchestratorMcpTitle = TrimmedNonEmptyString.check(Schema.isMaxLength(512)).pipe( + Schema.annotateEncoded({ + description: "Optional concise display title.", + }), +); +const OrchestratorMcpClientRequestId = TrimmedNonEmptyString.check(Schema.isMaxLength(256)).pipe( + Schema.annotateEncoded({ + description: "Stable idempotency key to reuse when retrying this mutation.", + }), +); /** * OpenCode 1.15 has been observed serializing nested MCP union objects as JSON @@ -98,21 +104,15 @@ export const OrchestratorMcpTargetOptions = Schema.Union([ export type OrchestratorMcpTargetOptions = typeof OrchestratorMcpTargetOptions.Type; export const OrchestratorMcpTarget = Schema.Struct({ - providerInstanceId: Schema.optional( - ProviderInstanceId.annotate({ - description: "Configured provider instance id from orchestrator_capabilities.", - }), - ), - driverKind: Schema.optional( - ProviderDriverKind.annotate({ - description: "Provider driver kind; prefer providerInstanceId when available.", - }), - ), - model: Schema.optional( - TrimmedNonEmptyString.annotate({ - description: "Model id advertised for the selected provider instance.", - }), - ), + providerInstanceId: Schema.optional(ProviderInstanceId).annotate({ + description: "Configured provider instance id from orchestrator_capabilities.", + }), + driverKind: Schema.optional(ProviderDriverKind).annotate({ + description: "Provider driver kind; prefer providerInstanceId when available.", + }), + model: Schema.optional(TrimmedNonEmptyString).annotate({ + description: "Model id advertised for the selected provider instance.", + }), options: Schema.optional( OrchestratorMcpTargetOptions.annotate({ description: @@ -162,9 +162,11 @@ export type OrchestratorMcpTerminalDelegatedTaskStatus = typeof OrchestratorMcpTerminalDelegatedTaskStatus.Type; export const OrchestratorMcpDelegateTaskInput = Schema.Struct({ - task: OrchestratorMcpPrompt.annotate({ - description: "Self-contained task for one delegated child agent/subagent.", - }), + task: OrchestratorMcpPrompt.pipe( + Schema.annotateEncoded({ + description: "Self-contained task for one delegated child agent/subagent.", + }), + ), target: Schema.optional(OrchestratorMcpTarget), title: Schema.optional(OrchestratorMcpTitle), role: Schema.optional(OrchestratorMcpTaskRole), @@ -302,12 +304,10 @@ export const OrchestratorMcpThreadStatus = Schema.Union([ ]); export type OrchestratorMcpThreadStatus = typeof OrchestratorMcpThreadStatus.Type; -const OrchestratorMcpProjectTarget = Schema.optional( - ProjectId.annotate({ - description: - "Project to act on. Omit for the calling thread's project; required when the caller is not a T3 thread.", - }), -); +const OrchestratorMcpProjectTarget = Schema.optional(ProjectId).annotate({ + description: + "Project to act on. Omit for the calling thread's project; required when the caller is not a T3 thread.", +}); export const OrchestratorMcpThreadListInput = Schema.Struct({ projectId: OrchestratorMcpProjectTarget, @@ -548,9 +548,11 @@ export type OrchestratorMcpCapabilitiesResult = typeof OrchestratorMcpCapabiliti export const OrchestratorMcpScheduleTaskInput = Schema.Struct({ projectId: OrchestratorMcpProjectTarget, - prompt: OrchestratorMcpPrompt.annotate({ - description: "Prompt executed on every scheduled run.", - }), + prompt: OrchestratorMcpPrompt.pipe( + Schema.annotateEncoded({ + description: "Prompt executed on every scheduled run.", + }), + ), schedule: OrchestratorMcpSchedule, title: Schema.optional(OrchestratorMcpTitle), enabled: Schema.optional( @@ -590,12 +592,10 @@ export const OrchestratorMcpScheduleTaskResult = OrchestratorMcpScheduledTask; export type OrchestratorMcpScheduleTaskResult = typeof OrchestratorMcpScheduleTaskResult.Type; export const OrchestratorMcpListScheduledTasksInput = Schema.Struct({ - projectId: Schema.optional( - ProjectId.annotate({ - description: - "Only list this project's tasks. Omit for the calling thread's project, or for every project when the caller is not a T3 thread.", - }), - ), + projectId: Schema.optional(ProjectId).annotate({ + description: + "Only list this project's tasks. Omit for the calling thread's project, or for every project when the caller is not a T3 thread.", + }), }); export type OrchestratorMcpListScheduledTasksInput = typeof OrchestratorMcpListScheduledTasksInput.Type; @@ -618,13 +618,17 @@ export type OrchestratorMcpUpdateScheduledTaskInput = typeof OrchestratorMcpUpdateScheduledTaskInput.Type; export const OrchestratorMcpRequestSecretInput = Schema.Struct({ - label: TrimmedNonEmptyString.annotate({ - description: "What you need, shown as the card's title, e.g. 'GitHub webhook secret'.", - }), - reason: TrimmedNonEmptyString.annotate({ - description: - "One or two sentences on what it is for and where the user gets or also enters it.", - }), + label: TrimmedNonEmptyString.pipe( + Schema.annotateEncoded({ + description: "What you need, shown as the card's title, e.g. 'GitHub webhook secret'.", + }), + ), + reason: TrimmedNonEmptyString.pipe( + Schema.annotateEncoded({ + description: + "One or two sentences on what it is for and where the user gets or also enters it.", + }), + ), placeholder: Schema.optional(TrimmedNonEmptyString).annotate({ description: "Hint inside the input, e.g. 'Paste your GitHub token'.", }), diff --git a/packages/contracts/src/threadMetadataMcp.ts b/packages/contracts/src/threadMetadataMcp.ts index 7c132e4d45f4b..1bb2f0bddfa73 100644 --- a/packages/contracts/src/threadMetadataMcp.ts +++ b/packages/contracts/src/threadMetadataMcp.ts @@ -36,9 +36,11 @@ function isHttpUrl(value: string): boolean { } } -const ThreadMetadataTitle = TrimmedNonEmptyString.check(Schema.isMaxLength(512)).annotate({ - description: "New concise display title. Required only when action is rename.", -}); +const ThreadMetadataTitle = TrimmedNonEmptyString.check(Schema.isMaxLength(512)).pipe( + Schema.annotateEncoded({ + description: "New concise display title. Required only when action is rename.", + }), +); const ThreadMetadataClientRequestId = TrimmedNonEmptyString.check(Schema.isMaxLength(256)) .check( @@ -70,9 +72,11 @@ export const ThreadMetadataMcpAction = Schema.Literals([ export type ThreadMetadataMcpAction = typeof ThreadMetadataMcpAction.Type; export const ThreadMetadataMcpPullRequest = Schema.Struct({ - repository: TrimmedNonEmptyString.annotate({ - description: "Repository name as owner/name.", - }), + repository: TrimmedNonEmptyString.pipe( + Schema.annotateEncoded({ + description: "Repository name as owner/name.", + }), + ), number: PositiveInt.annotate({ description: "Pull request number." }), url: ThreadMetadataMcpPullRequestUrl, }); diff --git a/packages/contracts/src/worktreeMcp.ts b/packages/contracts/src/worktreeMcp.ts index ddc4fd269608d..280b40edb0c9f 100644 --- a/packages/contracts/src/worktreeMcp.ts +++ b/packages/contracts/src/worktreeMcp.ts @@ -9,15 +9,15 @@ import { TrimmedNonEmptyString } from "./baseSchemas.ts"; * continues inside the worktree on the thread's next run. */ export const WorktreeMcpHandoffInput = Schema.Struct({ - branch: TrimmedNonEmptyString.annotate({ - description: "Branch name to create for the worktree (e.g. 'feature/my-change').", - }), - baseRef: Schema.optional( - TrimmedNonEmptyString.annotate({ - description: - "Branch or ref the worktree branch starts from. Defaults to the branch currently checked out in the project workspace.", + branch: TrimmedNonEmptyString.pipe( + Schema.annotateEncoded({ + description: "Branch name to create for the worktree (e.g. 'feature/my-change').", }), ), + baseRef: Schema.optional(TrimmedNonEmptyString).annotate({ + description: + "Branch or ref the worktree branch starts from. Defaults to the branch currently checked out in the project workspace.", + }), startFromOrigin: Schema.optional( Schema.Boolean.annotate({ description: @@ -31,11 +31,11 @@ export const WorktreeMcpHandoffInput = Schema.Struct({ }), ), continuationPrompt: Schema.optional( - TrimmedNonEmptyString.check(Schema.isMaxLength(120_000)).annotate({ - description: - "Message queued as the thread's next turn after the handoff. The handoff detaches the current provider session, so pass the remaining work here to automatically resume inside the worktree; omit it to stop after the handoff and wait for the next message.", - }), - ), + TrimmedNonEmptyString.check(Schema.isMaxLength(120_000)), + ).annotate({ + description: + "Message queued as the thread's next turn after the handoff. The handoff detaches the current provider session, so pass the remaining work here to automatically resume inside the worktree; omit it to stop after the handoff and wait for the next message.", + }), }); export type WorktreeMcpHandoffInput = typeof WorktreeMcpHandoffInput.Type; From 53fc06dc300af12fb659b834f99c5a055b66cd37 Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Wed, 21 Oct 2026 19:00:33 -0500 Subject: [PATCH 03/12] fix(mcp): keep the orchestrator_capabilities search description Co-Authored-By: Claude Opus 5.5 --- packages/contracts/src/orchestratorMcp.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/contracts/src/orchestratorMcp.ts b/packages/contracts/src/orchestratorMcp.ts index b36be2663fae4..3a531bd895da8 100644 --- a/packages/contracts/src/orchestratorMcp.ts +++ b/packages/contracts/src/orchestratorMcp.ts @@ -524,12 +524,10 @@ export const OrchestratorMcpThreadInterruptResult = Schema.Struct({ export type OrchestratorMcpThreadInterruptResult = typeof OrchestratorMcpThreadInterruptResult.Type; export const OrchestratorMcpCapabilitiesInput = Schema.Struct({ - search: Schema.optional( - TrimmedNonEmptyString.check(Schema.isMaxLength(200)).annotate({ - description: - "Words to match against model ids and labels. Lists matching models instead of the default short list.", - }), - ), + search: Schema.optional(TrimmedNonEmptyString.check(Schema.isMaxLength(200))).annotate({ + description: + "Words to match against model ids and labels. Lists matching models instead of the default short list.", + }), }); export type OrchestratorMcpCapabilitiesInput = typeof OrchestratorMcpCapabilitiesInput.Type; From 0a352260f6049fa111da67457178ba48ea3db2c9 Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Wed, 21 Oct 2026 20:13:07 -0500 Subject: [PATCH 04/12] fix(mcp): describe t3_thread_update clientRequestId and pull request url Both fields add their description with `.annotate` after a `Schema.makeFilter` check, which loses it from the encoded and the decoded projection alike, so agents got bare strings and the walk in tools.test.ts could not see the gap. Move both to `Schema.annotateEncoded`, pin them by name, and say in the walk's comment what it cannot catch. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/mcp/toolkits/tools.test.ts | 19 ++++++++++++++----- packages/contracts/src/threadMetadataMcp.ts | 14 ++++++++++---- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/apps/server/src/mcp/toolkits/tools.test.ts b/apps/server/src/mcp/toolkits/tools.test.ts index 4af3f354f3e57..fd556eb5450f5 100644 --- a/apps/server/src/mcp/toolkits/tools.test.ts +++ b/apps/server/src/mcp/toolkits/tools.test.ts @@ -55,8 +55,10 @@ const tools = [ WorktreeToolkit, ].flatMap((toolkit) => Object.values(toolkit.tools) as Tool.Any[]); -// Agents receive the encoded-side JSON schema, so a description written with -// `.annotate` on a transformed field, such as a trimmed string, never reaches them. +// Agents receive the encoded-side JSON schema, so this compares the two +// projections: a description on the decoded side but not on the encoded side is +// one no agent sees. A description lost from both projections, such as +// `.annotate` after `Schema.makeFilter`, stays invisible here and needs a pin. it("serves every parameter description written on the decoded side", () => { const missing = tools.flatMap((tool) => missingDescriptions( @@ -76,8 +78,15 @@ it.each([ ["link_pull_request", "url"], ["t3_worktree_handoff", "branch"], ["device_open", "deviceId"], -])("describes %s.%s to agents", (name, field) => { + ["t3_thread_update", "clientRequestId"], + ["t3_thread_update", "pullRequest.url"], +])("describes %s.%s to agents", (name, path) => { const tool = tools.find((candidate) => candidate.name === name); - const served = tool && (Tool.getJsonSchema(tool) as JsonSchemaNode); - expect(descriptionOf(served?.properties?.[field])).toBeTruthy(); + const served = path + .split(".") + .reduce( + (schema, field) => objectOf(schema)?.properties?.[field], + tool && (Tool.getJsonSchema(tool) as JsonSchemaNode), + ); + expect(descriptionOf(served)).toBeTruthy(); }); diff --git a/packages/contracts/src/threadMetadataMcp.ts b/packages/contracts/src/threadMetadataMcp.ts index 1bb2f0bddfa73..3141d31675478 100644 --- a/packages/contracts/src/threadMetadataMcp.ts +++ b/packages/contracts/src/threadMetadataMcp.ts @@ -50,15 +50,21 @@ const ThreadMetadataClientRequestId = TrimmedNonEmptyString.check(Schema.isMaxLe : "clientRequestId must contain valid paired Unicode surrogates.", ), ) - .annotate({ description: "Stable idempotency key to reuse when retrying this mutation." }); + .pipe( + Schema.annotateEncoded({ + description: "Stable idempotency key to reuse when retrying this mutation.", + }), + ); const ThreadMetadataMcpPullRequestUrl = TrimmedNonEmptyString.check( Schema.makeFilter((value) => isHttpUrl(value) ? true : "Pull request URL must be a well-formed HTTP(S) URL.", ), -).annotate({ - description: "Canonical HTTP(S) pull request URL, including self-hosted repository URLs.", -}); +).pipe( + Schema.annotateEncoded({ + description: "Canonical HTTP(S) pull request URL, including self-hosted repository URLs.", + }), +); export const ThreadMetadataMcpAction = Schema.Literals([ "rename", From aaad4ec2a1140f128b840f5c91bdb5019a7a9ea0 Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Wed, 21 Oct 2026 21:12:32 -0500 Subject: [PATCH 05/12] fix(mcp): describe scheduled task fields inside unions and arrays The guard walk now follows every anyOf member and array items, which exposes timeOfDay, signature.header and filters[].path on schedule_task and update_scheduled_task. They move to Schema.annotateEncoded like the other trimmed-string fields. Three more pins cover array items and union members. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/mcp/toolkits/tools.test.ts | 59 ++++++++++++++-------- packages/contracts/src/scheduledTask.ts | 24 ++++++--- 2 files changed, 55 insertions(+), 28 deletions(-) diff --git a/apps/server/src/mcp/toolkits/tools.test.ts b/apps/server/src/mcp/toolkits/tools.test.ts index fd556eb5450f5..465475eae06e7 100644 --- a/apps/server/src/mcp/toolkits/tools.test.ts +++ b/apps/server/src/mcp/toolkits/tools.test.ts @@ -18,28 +18,42 @@ interface JsonSchemaNode { readonly description?: string; readonly properties?: Readonly>; readonly anyOf?: ReadonlyArray; + readonly items?: JsonSchemaNode; } const descriptionOf = (schema: JsonSchemaNode | undefined): string | undefined => schema?.description ?? schema?.anyOf?.map(descriptionOf).find(Boolean); -const objectOf = (schema: JsonSchemaNode | undefined): JsonSchemaNode | undefined => - schema?.properties ? schema : schema?.anyOf?.map(objectOf).find(Boolean); +// Every object a field can hold: each union member and each array's items. +const objectsOf = (schema: JsonSchemaNode | undefined): ReadonlyArray => + schema?.properties + ? [schema] + : [...(schema?.anyOf ?? []), ...(schema?.items ? [schema.items] : [])].flatMap(objectsOf); +const fieldsOf = ( + schemas: ReadonlyArray, + field: string, +): ReadonlyArray => + schemas.flatMap(objectsOf).flatMap((object) => object.properties?.[field] ?? []); + +// Union members are matched by field name, so every variant of a union is walked. const missingDescriptions = ( decoded: JsonSchemaNode | undefined, - served: JsonSchemaNode | undefined, + served: ReadonlyArray, path: string, ): ReadonlyArray => - Object.entries(objectOf(decoded)?.properties ?? {}).flatMap(([field, fieldSchema]) => { - const servedField = objectOf(served)?.properties?.[field]; - const description = descriptionOf(fieldSchema); - const missing = - description !== undefined && descriptionOf(servedField) !== description - ? [`${path}.${field}`] - : []; - return [...missing, ...missingDescriptions(fieldSchema, servedField, `${path}.${field}`)]; - }); + objectsOf(decoded).flatMap((object) => + Object.entries(object.properties ?? {}).flatMap(([field, fieldSchema]) => { + const servedFields = fieldsOf(served, field); + const description = descriptionOf(fieldSchema); + const missing = + description !== undefined && + !servedFields.some((servedField) => descriptionOf(servedField) === description) + ? [`${path}.${field}`] + : []; + return [...missing, ...missingDescriptions(fieldSchema, servedFields, `${path}.${field}`)]; + }), + ); const tools = [ AttachmentToolkit, @@ -57,17 +71,18 @@ const tools = [ // Agents receive the encoded-side JSON schema, so this compares the two // projections: a description on the decoded side but not on the encoded side is -// one no agent sees. A description lost from both projections, such as -// `.annotate` after `Schema.makeFilter`, stays invisible here and needs a pin. +// one no agent sees. The walk follows object properties, every anyOf member and +// array items. A description lost from both projections, such as `.annotate` +// after `Schema.makeFilter`, is not seen here and needs a pin. it("serves every parameter description written on the decoded side", () => { const missing = tools.flatMap((tool) => missingDescriptions( Schema.toJsonSchemaDocument(Schema.toType(tool.parametersSchema)).schema as JsonSchemaNode, - Tool.getJsonSchema(tool) as JsonSchemaNode, + [Tool.getJsonSchema(tool) as JsonSchemaNode], tool.name, ), ); - expect(missing).toEqual([]); + expect([...new Set(missing)]).toEqual([]); }); it.each([ @@ -80,13 +95,17 @@ it.each([ ["device_open", "deviceId"], ["t3_thread_update", "clientRequestId"], ["t3_thread_update", "pullRequest.url"], + ["create_threads", "threads.prompt"], + ["schedule_task", "schedule.timeOfDay"], + ["update_scheduled_task", "schedule.filters.path"], ])("describes %s.%s to agents", (name, path) => { const tool = tools.find((candidate) => candidate.name === name); const served = path .split(".") - .reduce( - (schema, field) => objectOf(schema)?.properties?.[field], - tool && (Tool.getJsonSchema(tool) as JsonSchemaNode), + .reduce>( + fieldsOf, + tool ? [Tool.getJsonSchema(tool) as JsonSchemaNode] : [], ); - expect(descriptionOf(served)).toBeTruthy(); + expect(served).not.toEqual([]); + expect(served.filter((field) => !descriptionOf(field))).toEqual([]); }); diff --git a/packages/contracts/src/scheduledTask.ts b/packages/contracts/src/scheduledTask.ts index 67e2314a8b010..955ce0a62db87 100644 --- a/packages/contracts/src/scheduledTask.ts +++ b/packages/contracts/src/scheduledTask.ts @@ -21,7 +21,11 @@ import { ProviderInteractionMode, RuntimeMode } from "./providerPolicy.ts"; /** 24-hour "HH:MM" wall-clock time. Mirrors `parseTimeOfDay` on the server. */ const TimeOfDay = TrimmedNonEmptyString.check( Schema.isPattern(/^([01]?\d|2[0-3]):([0-5]\d)$/), -).annotate({ description: "Local wall-clock time in 24-hour HH:MM form, such as 09:30." }); +).pipe( + Schema.annotateEncoded({ + description: "Local wall-clock time in 24-hour HH:MM form, such as 09:30.", + }), +); export const MIN_SCHEDULED_TASK_INTERVAL_MS = 60_000; @@ -57,9 +61,11 @@ const ScheduledTaskFixedTimeSchedule = Schema.Struct({ }); const ScheduledTaskWebhookSignatureFields = { - header: TrimmedNonEmptyString.annotate({ - description: "Request header carrying the signature, such as x-hub-signature-256.", - }), + header: TrimmedNonEmptyString.pipe( + Schema.annotateEncoded({ + description: "Request header carrying the signature, such as x-hub-signature-256.", + }), + ), encoding: Schema.Literals(["hex", "base64"]).annotate({ description: "How the HMAC-SHA256 digest is encoded in the header.", }), @@ -107,10 +113,12 @@ export const MAX_WEBHOOK_FILTERS = 16; const WebhookFilterPath = TrimmedNonEmptyString.check( Schema.isPattern(/^(body|headers|query)(\.|$)/), -).annotate({ - description: - "Request value to test, in placeholder syntax without braces: headers.x-github-event, body.action, query.source.", -}); +).pipe( + Schema.annotateEncoded({ + description: + "Request value to test, in placeholder syntax without braces: headers.x-github-event, body.action, query.source.", + }), +); /** * Forms edit an `in` list as comma-separated text, so a value that has no From a20bbc3baf87bca3ca0978cb75ad415128bf2ece Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Mon, 26 Oct 2026 12:32:41 +0000 Subject: [PATCH 06/12] test(mcp): verify descriptions through authenticated tool discovery --- apps/server/src/mcp/McpHttpServer.test.ts | 11 +++++++++++ apps/server/src/mcp/toolkits/core.test.ts | 5 ++++- packages/contracts/src/orchestratorMcp.ts | 8 +++----- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts index 8b73483380e2c..dd7b846f33d0b 100644 --- a/apps/server/src/mcp/McpHttpServer.test.ts +++ b/apps/server/src/mcp/McpHttpServer.test.ts @@ -889,6 +889,17 @@ it.effect("lists pull request watch tools only to threads that are not delegated const body = (yield* listed.json) as { readonly result: { readonly tools: ReadonlyArray<{ readonly name: string }> }; }; + expect(body.result.tools.find((tool) => tool.name === "link_pull_request")).toMatchObject( + { + inputSchema: { + properties: { + threadId: { description: "Thread. Default: this thread." }, + repository: { description: "With `number`, when no `url`." }, + host: { description: "Default: this project's host." }, + }, + }, + }, + ); return body.result.tools.map((tool) => tool.name); }); diff --git a/apps/server/src/mcp/toolkits/core.test.ts b/apps/server/src/mcp/toolkits/core.test.ts index c6293ee0acfa6..1ebfd8ab689a7 100644 --- a/apps/server/src/mcp/toolkits/core.test.ts +++ b/apps/server/src/mcp/toolkits/core.test.ts @@ -901,6 +901,9 @@ interface ParameterSchema { readonly items?: ParameterSchema; } +const parameterDescription = (schema: ParameterSchema): string | undefined => + schema.description ?? schema.anyOf?.map(parameterDescription).find(Boolean); + const parameterFields = (schema: ParameterSchema, field: string): ParameterSchema[] => [ ...(schema.properties?.[field] ? [schema.properties[field]] : []), ...(schema.anyOf ?? []).flatMap((member) => parameterFields(member, field)), @@ -949,7 +952,7 @@ it("advertises existing string parameter guidance to MCP clients", () => { ); expect(fields.length, name + "." + path).toBeGreaterThan(0); for (const field of fields) { - expect(field.description, name + "." + path).toEqual(expect.any(String)); + expect(parameterDescription(field), name + "." + path).toEqual(expect.any(String)); } } }); diff --git a/packages/contracts/src/orchestratorMcp.ts b/packages/contracts/src/orchestratorMcp.ts index 7151f67686493..e595912e73598 100644 --- a/packages/contracts/src/orchestratorMcp.ts +++ b/packages/contracts/src/orchestratorMcp.ts @@ -90,11 +90,9 @@ export const OrchestratorMcpTarget = Schema.Struct({ providerInstanceId: Schema.optional(ProviderInstanceId).annotate({ description: "Provider instance id from `orchestrator_capabilities`.", }), - driverKind: Schema.optional( - ProviderDriverKind.annotate({ - description: "Prefer `providerInstanceId`.", - }), - ), + driverKind: Schema.optional(ProviderDriverKind).annotate({ + description: "Prefer `providerInstanceId`.", + }), model: Schema.optional(TrimmedNonEmptyString), options: Schema.optional( OrchestratorMcpTargetOptions.annotate({ From 833c61be35966feba86c36e670a7416ff12ec605 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Wed, 28 Oct 2026 09:44:21 +0000 Subject: [PATCH 07/12] fix(mcp): expose optional parameter guidance to Codex --- apps/server/src/mcp/toolkits/core.test.ts | 5 +--- packages/contracts/src/orchestratorMcp.ts | 24 +++++++++--------- packages/contracts/src/threadMetadataMcp.ts | 28 ++++++++++----------- 3 files changed, 26 insertions(+), 31 deletions(-) diff --git a/apps/server/src/mcp/toolkits/core.test.ts b/apps/server/src/mcp/toolkits/core.test.ts index 8f82513df9b0d..03eb3735ac5c7 100644 --- a/apps/server/src/mcp/toolkits/core.test.ts +++ b/apps/server/src/mcp/toolkits/core.test.ts @@ -903,9 +903,6 @@ interface ParameterSchema { readonly items?: ParameterSchema; } -const parameterDescription = (schema: ParameterSchema): string | undefined => - schema.description ?? schema.anyOf?.map(parameterDescription).find(Boolean); - const parameterFields = (schema: ParameterSchema, field: string): ParameterSchema[] => [ ...(schema.properties?.[field] ? [schema.properties[field]] : []), ...(schema.anyOf ?? []).flatMap((member) => parameterFields(member, field)), @@ -952,7 +949,7 @@ it("advertises existing string parameter guidance to MCP clients", () => { ); expect(fields.length, name + "." + path).toBeGreaterThan(0); for (const field of fields) { - expect(parameterDescription(field), name + "." + path).toEqual(expect.any(String)); + expect(field.description, name + "." + path).toEqual(expect.any(String)); } } }); diff --git a/packages/contracts/src/orchestratorMcp.ts b/packages/contracts/src/orchestratorMcp.ts index f2936cb0a0ae3..90d40e9cb5ecf 100644 --- a/packages/contracts/src/orchestratorMcp.ts +++ b/packages/contracts/src/orchestratorMcp.ts @@ -42,11 +42,11 @@ import { ProviderDriverKind, ProviderInstanceId } from "./providerInstance.ts"; const OrchestratorMcpPrompt = TrimmedNonEmptyString.check(Schema.isMaxLength(120_000)); const OrchestratorMcpTitle = TrimmedNonEmptyString.check(Schema.isMaxLength(512)); -const OrchestratorMcpClientRequestId = TrimmedNonEmptyString.check(Schema.isMaxLength(256)).pipe( - Schema.annotateEncoded({ - description: "Reuse it to retry the same call. A new call gets a new key.", - }), -); +const OrchestratorMcpClientRequestId = Schema.optional( + TrimmedNonEmptyString.check(Schema.isMaxLength(256)), +).annotate({ + description: "Reuse it to retry the same call. A new call gets a new key.", +}); // OpenCode 1.15 serializes nested MCP union objects as JSON strings; decode that form at the boundary. const OrchestratorMcpScheduleFromJsonString = Schema.fromJsonString( @@ -162,7 +162,7 @@ export const OrchestratorMcpDelegateTaskInput = Schema.Struct({ timeoutMs: Schema.optional(Schema.Finite).annotate({ description: "Mode 'wait' only. Default 10 min. A timeout does not cancel the child.", }), - clientRequestId: Schema.optional(OrchestratorMcpClientRequestId), + clientRequestId: OrchestratorMcpClientRequestId, runtimeMode: Schema.optional( OrchestratorMcpRuntimeMode.annotate({ description: @@ -220,7 +220,7 @@ export type OrchestratorMcpTaskStatusInput = typeof OrchestratorMcpTaskStatusInp export const OrchestratorMcpTaskCancelInput = Schema.Struct({ taskId: NodeId, reason: Schema.optional(Schema.String.check(Schema.isMaxLength(2_000))), - clientRequestId: Schema.optional(OrchestratorMcpClientRequestId), + clientRequestId: OrchestratorMcpClientRequestId, }); export type OrchestratorMcpTaskCancelInput = typeof OrchestratorMcpTaskCancelInput.Type; @@ -244,7 +244,7 @@ export const OrchestratorMcpCreateThreadsInput = Schema.Struct({ Schema.isMinLength(1), Schema.isMaxLength(20), ), - clientRequestId: Schema.optional(OrchestratorMcpClientRequestId), + clientRequestId: OrchestratorMcpClientRequestId, }); export type OrchestratorMcpCreateThreadsInput = typeof OrchestratorMcpCreateThreadsInput.Type; @@ -455,7 +455,7 @@ export const OrchestratorMcpThreadSendInput = Schema.Struct({ threadId: ThreadId, message: OrchestratorMcpPrompt, mode: Schema.optional(Schema.Literals(["auto", "queue", "steer", "restart"])), - clientRequestId: Schema.optional(OrchestratorMcpClientRequestId), + clientRequestId: OrchestratorMcpClientRequestId, }); export type OrchestratorMcpThreadSendInput = typeof OrchestratorMcpThreadSendInput.Type; @@ -490,7 +490,7 @@ export const OrchestratorMcpThreadInterruptInput = Schema.Struct({ threadId: ThreadId, runId: Schema.optional(RunId), reason: Schema.optional(Schema.String.check(Schema.isMaxLength(2_000))), - clientRequestId: Schema.optional(OrchestratorMcpClientRequestId), + clientRequestId: OrchestratorMcpClientRequestId, }); export type OrchestratorMcpThreadInterruptInput = typeof OrchestratorMcpThreadInterruptInput.Type; @@ -572,7 +572,7 @@ export const OrchestratorMcpScheduleTaskInput = Schema.Struct({ description: "Default true: runs post in this thread. False: new thread per run.", }), ), - clientRequestId: Schema.optional(OrchestratorMcpClientRequestId), + clientRequestId: OrchestratorMcpClientRequestId, }); export type OrchestratorMcpScheduleTaskInput = typeof OrchestratorMcpScheduleTaskInput.Type; @@ -651,7 +651,7 @@ export const OrchestratorMcpRequestSecretInput = Schema.Struct({ timeoutMs: Schema.optional( Schema.Int.check(Schema.isBetween({ minimum: 1_000, maximum: 60 * 60 * 1_000 })), ).annotate({ description: "Default 10 min." }), - clientRequestId: Schema.optional(OrchestratorMcpClientRequestId).annotate({ + clientRequestId: OrchestratorMcpClientRequestId.annotate({ description: "Retry key for this ask. New key after a timeout or cancel.", }), }); diff --git a/packages/contracts/src/threadMetadataMcp.ts b/packages/contracts/src/threadMetadataMcp.ts index 02c15c6ebf099..8e22c2f71b2e8 100644 --- a/packages/contracts/src/threadMetadataMcp.ts +++ b/packages/contracts/src/threadMetadataMcp.ts @@ -26,25 +26,23 @@ function hasOnlyPairedSurrogates(value: string): boolean { return true; } -const ThreadMetadataTitle = TrimmedNonEmptyString.check(Schema.isMaxLength(512)).pipe( - Schema.annotateEncoded({ - description: "New concise display title. Required only when action is rename.", - }), -); +const ThreadMetadataTitle = Schema.optional( + TrimmedNonEmptyString.check(Schema.isMaxLength(512)), +).annotate({ + description: "New concise display title. Required only when action is rename.", +}); -const ThreadMetadataClientRequestId = TrimmedNonEmptyString.check(Schema.isMaxLength(256)) - .check( +const ThreadMetadataClientRequestId = Schema.optional( + TrimmedNonEmptyString.check(Schema.isMaxLength(256)).check( Schema.makeFilter((value) => hasOnlyPairedSurrogates(value) ? true : "clientRequestId must contain valid paired Unicode surrogates.", ), - ) - .pipe( - Schema.annotateEncoded({ - description: "Stable idempotency key to reuse when retrying this mutation.", - }), - ); + ), +).annotate({ + description: "Stable idempotency key to reuse when retrying this mutation.", +}); export const ThreadMetadataMcpAction = Schema.Literals(["rename", "regenerate_title"]); export type ThreadMetadataMcpAction = typeof ThreadMetadataMcpAction.Type; @@ -52,8 +50,8 @@ export type ThreadMetadataMcpAction = typeof ThreadMetadataMcpAction.Type; export const ThreadMetadataMcpUpdateInput = Schema.Struct({ threadId: Schema.optional(ThreadId), action: ThreadMetadataMcpAction, - title: Schema.optional(ThreadMetadataTitle), - clientRequestId: Schema.optional(ThreadMetadataClientRequestId), + title: ThreadMetadataTitle, + clientRequestId: ThreadMetadataClientRequestId, }).check( Schema.makeFilter((input) => (input.action === "rename") === (input.title !== undefined) From 7f0a33885a15b193cc94583ebf3fe6588412ada5 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Fri, 30 Oct 2026 00:51:22 +0000 Subject: [PATCH 08/12] fix(mcp): record restored parameter descriptions in context budget --- apps/server/src/mcp/toolkits/pullRequests/tools.ts | 1 - apps/server/src/provider/agentContextBudget.json | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/server/src/mcp/toolkits/pullRequests/tools.ts b/apps/server/src/mcp/toolkits/pullRequests/tools.ts index a344f28bcb0ef..b8a8ef438735f 100644 --- a/apps/server/src/mcp/toolkits/pullRequests/tools.ts +++ b/apps/server/src/mcp/toolkits/pullRequests/tools.ts @@ -38,7 +38,6 @@ export const PullRequestTargetInput = Schema.Struct({ threadId: Schema.optional(ThreadId).annotate({ description: "Thread. Default: this thread.", }), - url: Schema.optional(TrimmedNonEmptyString), repository: Schema.optional(TrimmedNonEmptyString).annotate({ description: "With `number`, when no `url`.", diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index 1711e148d4ca4..a149da8d9eeaf 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { "claudeSystemAppendBytes": 5135, "codexAdditionalContextBytes": 6865, - "mcpToolListJsonBytes": 46007 + "mcpToolListJsonBytes": 48730 } From 7e6971d28534f24bffd5f2786d9686dc3cd70fcf Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Sun, 8 Nov 2026 10:06:20 +0000 Subject: [PATCH 09/12] fix(mcp): schedule_task lists the webhook prompt variables The webhook branch described the {{body.path}}-style prompt variables, but advertisedJsonSchema merges schedule's tagged union into one object and drops branch descriptions, so agents never saw them. The text now sits on the outer schedule description, which reaches every client. Adds 96 bytes to the MCP tool list. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/provider/agentContextBudget.json | 2 +- packages/contracts/src/orchestratorMcp.ts | 2 +- packages/contracts/src/scheduledTask.ts | 3 --- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index 99d460d822f41..3bfab36034d5e 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { "claudeSystemAppendBytes": 5350, "codexAdditionalContextBytes": 7080, - "mcpToolListJsonBytes": 50839 + "mcpToolListJsonBytes": 50935 } diff --git a/packages/contracts/src/orchestratorMcp.ts b/packages/contracts/src/orchestratorMcp.ts index dc4d558840a61..3c66bba6425e6 100644 --- a/packages/contracts/src/orchestratorMcp.ts +++ b/packages/contracts/src/orchestratorMcp.ts @@ -77,7 +77,7 @@ const OrchestratorMcpSchedule = Schema.Union([ OrchestratorMcpScheduleFromJsonString, ]).annotate({ description: - "Trigger: {type:'interval',everyMs}, {type:'fixed_time',timeOfDay,weekdays?}, {type:'once',runAt}, or {type:'webhook'}.", + "Trigger: {type:'interval',everyMs}, {type:'fixed_time',timeOfDay,weekdays?}, {type:'once',runAt}, or {type:'webhook'}. Webhook prompts can use {{body.path}}, {{headers.name}}, {{query.name}}, {{body}}, {{request}}.", }); const OrchestratorMcpTargetOptionsFromRecord = Schema.Record( diff --git a/packages/contracts/src/scheduledTask.ts b/packages/contracts/src/scheduledTask.ts index a092d35fed290..667c2a796b55b 100644 --- a/packages/contracts/src/scheduledTask.ts +++ b/packages/contracts/src/scheduledTask.ts @@ -187,9 +187,6 @@ const ScheduledTaskUpsertWebhookSchedule = Schema.Struct({ maxDeliveryAgeMinutes: Schema.optional(Schema.NullOr(WebhookMaxDeliveryAgeMinutes)), dedupe: Schema.optional(ScheduledTaskWebhookDedupe), filters: Schema.optional(ScheduledTaskWebhookFilters), -}).annotate({ - description: - "Run per webhook request. Prompt can use {{body.path}}, {{headers.name}}, {{query.name}}, {{body}}, {{request}}.", }); /** Accept legacy sub-minute schedules so users can still manage them. */ From f5840bbf05e6195c4283aba787716ed1f194a073 Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Sun, 8 Nov 2026 14:02:36 +0000 Subject: [PATCH 10/12] fix(mcp): link_pull_requests and unlink_pull_requests show their threadId description Co-Authored-By: Claude Opus 5.5 --- apps/server/src/mcp/toolkits/pullRequests/tools.ts | 4 ++-- apps/server/src/provider/agentContextBudget.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/server/src/mcp/toolkits/pullRequests/tools.ts b/apps/server/src/mcp/toolkits/pullRequests/tools.ts index 0959f353a013d..3edff7bcc6bc0 100644 --- a/apps/server/src/mcp/toolkits/pullRequests/tools.ts +++ b/apps/server/src/mcp/toolkits/pullRequests/tools.ts @@ -304,7 +304,7 @@ const LinkPullRequestsTool = Tool.make("link_pull_requests", { "Link several PRs to a thread in one call. Use it instead of one `link_pull_request` per PR. URLs that are not PRs come back in `invalidUrls`.", parameters: Schema.Struct({ environmentId: OrchestratorMcpEnvironmentTarget, - threadId: Schema.optional(ThreadId.annotate({ description: "Thread. Default: this thread." })), + threadId: Schema.optional(ThreadId).annotate({ description: "Thread. Default: this thread." }), urls: Schema.Array(TrimmedNonEmptyString).check(Schema.isMinLength(1)), }), success: LinkPullRequestsResult, @@ -336,7 +336,7 @@ const UnlinkPullRequestsTool = Tool.make("unlink_pull_requests", { "Unlink every PR on a thread whose last known state (as `list_thread_pull_requests` shows) is in `states`, in one call. Use it instead of one `unlink_pull_request` per PR. PRs with no known state stay linked.", parameters: Schema.Struct({ environmentId: OrchestratorMcpEnvironmentTarget, - threadId: Schema.optional(ThreadId.annotate({ description: "Thread. Default: this thread." })), + threadId: Schema.optional(ThreadId).annotate({ description: "Thread. Default: this thread." }), states: Schema.Array(PullRequestState).check(Schema.isMinLength(1)), }), success: UnlinkPullRequestsResult, diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index 3bfab36034d5e..ebeba364d7e03 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { "claudeSystemAppendBytes": 5350, "codexAdditionalContextBytes": 7080, - "mcpToolListJsonBytes": 50935 + "mcpToolListJsonBytes": 51027 } From c8cfbdf66c8a78188928d1049f67d094a6e221dc Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Sun, 8 Nov 2026 10:06:42 +0000 Subject: [PATCH 11/12] test(mcp): check every parameter description reaches agents Replaces the fixed list of 22 fields with a walk over every registered tool's parameter schema. Each description written in the source must appear at the same path in advertisedJsonSchema, so a new field that loses its description fails the test. The environmentId and target.projectId descriptions are skipped until maintainers decide whether to restore or remove them. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/mcp/advertisedJsonSchema.ts | 2 +- apps/server/src/mcp/toolkits/core.test.ts | 135 ++++++++++++-------- 2 files changed, 86 insertions(+), 51 deletions(-) diff --git a/apps/server/src/mcp/advertisedJsonSchema.ts b/apps/server/src/mcp/advertisedJsonSchema.ts index 10001307f2d6c..a2aa7e08a72b8 100644 --- a/apps/server/src/mcp/advertisedJsonSchema.ts +++ b/apps/server/src/mcp/advertisedJsonSchema.ts @@ -11,7 +11,7 @@ const isBareAnyOf = (value: Json): value is { anyOf: Array } => Predicate.isObject(value) && Array.isArray(value.anyOf) && Object.keys(value).length === 1; // A `Schema.fromJsonString` fallback. Advertised beside object shapes, it invites stringified input. -const isJsonStringSchema = (value: Json) => +export const isJsonStringSchema = (value: Json) => Predicate.isObject(value) && value.type === "string" && value.contentMediaType === "application/json"; diff --git a/apps/server/src/mcp/toolkits/core.test.ts b/apps/server/src/mcp/toolkits/core.test.ts index 65b207768a6c4..cc0551d80b4b6 100644 --- a/apps/server/src/mcp/toolkits/core.test.ts +++ b/apps/server/src/mcp/toolkits/core.test.ts @@ -26,7 +26,9 @@ import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as TestClock from "effect/testing/TestClock"; import * as Layer from "effect/Layer"; +import * as Predicate from "effect/Predicate"; import * as Schema from "effect/Schema"; +import * as SchemaAST from "effect/SchemaAST"; import { McpAttachmentInput } from "./attachment/input.ts"; import { McpSchema, McpServer, Tool } from "effect/ai"; import { FetchHttpClient } from "effect/http"; @@ -1050,67 +1052,100 @@ it.effect("only the caller that prepared a pending upload can discard it", () => ), ); +import { advertisedJsonSchema, isJsonStringSchema } from "../advertisedJsonSchema.ts"; +import { AttachmentToolkit } from "./attachment/tools.ts"; +import { DeviceToolkit } from "./device/tools.ts"; +import { EnvironmentToolkit } from "./environment/tools.ts"; +import { HtmlToolkit } from "./html/tools.ts"; import { OrchestratorToolkit } from "./orchestrator/tools.ts"; +import { PreviewToolkit } from "./preview/tools.ts"; +import { PreviewControlsToolkit } from "./previewControls/tools.ts"; +import { ProjectToolkit } from "./project/tools.ts"; import { PullRequestsToolkit } from "./pullRequests/tools.ts"; -import { DeviceToolkit } from "./device/tools.ts"; +import { ThreadToolkit } from "./thread/tools.ts"; import { WorktreeToolkit } from "./worktree/tools.ts"; -interface ParameterSchema { - readonly description?: string; - readonly properties?: Readonly>; - readonly anyOf?: ReadonlyArray; - readonly items?: ParameterSchema; -} +// advertisedJsonSchema drops `Schema.fromJsonString` union branches, and their text with them. +const isJsonStringFallback = (ast: SchemaAST.AST) => + isJsonStringSchema(Schema.toJsonSchemaDocument(Schema.make(ast)).schema); + +// An outer description replaces the one on its encoded side, and on a union's only kept +// branch, so only the outermost counts. +const sourceDescriptions = ( + ast: SchemaAST.AST, + path: string, + out: Array, + described = false, +): void => { + const description = SchemaAST.resolveDescription(ast); + if (!described && description !== undefined) out.push([path, description]); + const covered = described || description !== undefined; + const end = ast.encoding?.at(-1)?.to; + if (end) return sourceDescriptions(end, path, out, covered); + if (SchemaAST.isObjects(ast)) { + for (const property of ast.propertySignatures) + sourceDescriptions(property.type, `${path}.${String(property.name)}`, out); + for (const index of ast.indexSignatures) sourceDescriptions(index.type, `${path}.*`, out); + } else if (SchemaAST.isUnion(ast)) { + const kept = ast.types.filter((member) => !isJsonStringFallback(member)); + const branches = kept.filter( + (member) => !SchemaAST.isUndefined(member) && !SchemaAST.isNull(member), + ); + for (const member of kept) + sourceDescriptions(member, path, out, branches.length === 1 && covered); + } else if (SchemaAST.isArrays(ast)) { + for (const item of ast.rest) sourceDescriptions(item, `${path}[]`, out); + } +}; -const parameterFields = (schema: ParameterSchema, field: string): ParameterSchema[] => [ - ...(schema.properties?.[field] ? [schema.properties[field]] : []), - ...(schema.anyOf ?? []).flatMap((member) => parameterFields(member, field)), - ...(schema.items ? parameterFields(schema.items, field) : []), -]; +const advertisedDescriptions = (schema: unknown, path: string, out: Set): void => { + if (Array.isArray(schema)) { + for (const member of schema) advertisedDescriptions(member, path, out); + return; + } + if (!Predicate.isObject(schema)) return; + if (typeof schema.description === "string") out.add(`${path}: ${schema.description}`); + if (Predicate.isObject(schema.properties)) + for (const [name, property] of Object.entries(schema.properties)) + advertisedDescriptions(property, `${path}.${name}`, out); + advertisedDescriptions(schema.anyOf, path, out); + advertisedDescriptions(schema.items, `${path}[]`, out); + advertisedDescriptions(schema.additionalProperties, `${path}.*`, out); +}; -it("advertises existing string parameter guidance to MCP clients", () => { - const tools = { +// Restoring these is a context-budget call: about 3.6 KB on every thread's tool list. +const awaitingBudgetCall = new Set([ + "Linked environment to act in. Omit for this one. Ids are then that environment's.", + "Linked environment to run in. Provider and model default to yours; others come from `orchestrator_capabilities` with this `environmentId`.", + "With `environmentId`: its project to run in. Default: the one with this project's repository.", + "The linked environment to forget, from t3_environment_links.", +]); + +it("advertises every parameter description written in the tool schemas", () => { + const tools: Record = { + ...AttachmentToolkit.tools, + ...DeviceToolkit.tools, + ...EnvironmentToolkit.tools, + ...HtmlToolkit.tools, ...OrchestratorToolkit.tools, + ...PreviewToolkit.tools, + ...PreviewControlsToolkit.tools, + ...ProjectToolkit.tools, ...PullRequestsToolkit.tools, - ...DeviceToolkit.tools, + ...ThreadToolkit.tools, ...WorktreeToolkit.tools, }; - for (const [name, path] of [ - ["delegate_task", "clientRequestId"], - ["delegate_task", "target.providerInstanceId"], - ["delegate_task", "target.driverKind"], - ["request_secret", "label"], - ["request_secret", "reason"], - ["schedule_task", "projectId"], - ["list_scheduled_tasks", "projectId"], - ["orchestrator_capabilities", "search"], - ["link_pull_request", "threadId"], - ["link_pull_request", "repository"], - ["link_pull_request", "host"], - ["list_thread_pull_requests", "threadId"], - ["t3_worktree_handoff", "baseRef"], - ["t3_worktree_handoff", "continuationPrompt"], - ["device_open", "deviceId"], - ["device_open", "hostId"], - ["device_screenshot", "deviceId"], - ["device_close", "deviceId"], - ["t3_thread_update", "title"], - ["t3_thread_update", "clientRequestId"], - ["schedule_task", "schedule.timeOfDay"], - ["schedule_task", "schedule.signature.header"], - ] as const) { - const schema = Tool.getJsonSchema(tools[name]) as ParameterSchema; - const fields = path - .split(".") - .reduce( - (nodes, field) => nodes.flatMap((node) => parameterFields(node, field)), - [schema], - ); - expect(fields.length, name + "." + path).toBeGreaterThan(0); - for (const field of fields) { - expect(field.description, name + "." + path).toEqual(expect.any(String)); - } + const missing: string[] = []; + for (const [name, tool] of Object.entries(tools)) { + const source: Array = []; + sourceDescriptions(tool.parametersSchema.ast, name, source); + const advertised = new Set(); + advertisedDescriptions(advertisedJsonSchema(tool), name, advertised); + for (const [path, text] of source) + if (!awaitingBudgetCall.has(text) && !advertised.has(`${path}: ${text}`)) + missing.push(`${path}: ${text}`); } + expect(missing).toEqual([]); }); it.effect( From 93da9d92a6bbac230025e73645c8247d12906875 Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Wed, 11 Nov 2026 11:22:08 +0000 Subject: [PATCH 12/12] test(evals): webhook task prompts use the payload placeholder Adds an mcp-tool-use case where the agent sets up a webhook task that fills the pushed branch into its prompt, and a fixed-time control whose prompt needs no placeholder. Both are graded on the saved call and one fixed-shape last line. A call argument written as /pattern/ now matches as a case-sensitive regular expression, since webhook placeholders are case-sensitive and ~text is not. Co-Authored-By: Claude Opus 5.5 --- evals/mcp-tool-use/README.md | 2 +- evals/mcp-tool-use/cases/pull-requests.ndjson | 2 ++ evals/mcp-tool-use/run.test.ts | 9 +++++++++ evals/mcp-tool-use/run.ts | 3 +++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/evals/mcp-tool-use/README.md b/evals/mcp-tool-use/README.md index c5802a35af63b..f222792d1b1c4 100644 --- a/evals/mcp-tool-use/README.md +++ b/evals/mcp-tool-use/README.md @@ -28,7 +28,7 @@ Cases are JSON lines in `cases/`, one file per area. The case schema in `run.ts` - `experience`: the real mistake, in a sentence. `source`: where it happened. - `setup`: the fake thread, its `world` (repo, branch, files, PRs, pages), `priorTurns`, and `mocks` for tool results. - `setup.priorTurns[].toolCalls[].resultFile`: load a repository-relative file as the tool result instead of copying its contents into the case. An empty `prompt` resumes a top-level thread after its last prior tool result without adding a user message. Side-chat and subagent setups require a nonempty prompt. -- `expect`: the checks, such as `calls`, `callsMustNot`, `maxCalls`, `order`, `shellMust`, `textMust`, `finalTextMust` (the last text, once the turn ended), and `linked`. A call's `args` match partially: `~text` is a case-insensitive substring, `"<=N"` / `">=N"` bound a number, `null` matches a missing key, and keys may be dotted paths. Device cases also use `device` to check every child brief, forbid raw UI commands, bound failed device actions per user turn, and require artifact paths in completed final answers. Secrets cases use `secret` (see [Secrets cases](#secrets-cases)). +- `expect`: the checks, such as `calls`, `callsMustNot`, `maxCalls`, `order`, `shellMust`, `textMust`, `finalTextMust` (the last text, once the turn ended), and `linked`. A call's `args` match partially: `~text` is a case-insensitive substring, `/pattern/` is a case-sensitive regular expression, `"<=N"` / `">=N"` bound a number, `null` matches a missing key, and keys may be dotted paths. Device cases also use `device` to check every child brief, forbid raw UI commands, bound failed device actions per user turn, and require artifact paths in completed final answers. Secrets cases use `secret` (see [Secrets cases](#secrets-cases)). Check that the case builds with `--dry-run --case `. Then run it on the text before your fix, where it must fail, and on the text after, where it must pass. diff --git a/evals/mcp-tool-use/cases/pull-requests.ndjson b/evals/mcp-tool-use/cases/pull-requests.ndjson index 2e14e45f7f250..f01b612acb3a4 100644 --- a/evals/mcp-tool-use/cases/pull-requests.ndjson +++ b/evals/mcp-tool-use/cases/pull-requests.ndjson @@ -37,3 +37,5 @@ {"id": "link-one-pr", "area": "pull-requests", "privacy": "commit-safe", "kind": "control", "pattern": "single link stays single", "experience": "Linking one named PR is ordinary work.", "setup": {"role": "top", "context": "No pull requests are linked to this thread.", "world": {"repo": "acme/app", "branch": "main"}}, "prompt": "link https://github.com/acme/app/pull/130 here", "expect": {"calls": [{"tool": "link_pull_request", "args": {"url": "~/pull/130"}}], "notCalls": ["link_pull_requests"]}, "maxSteps": 6, "source": {"thread": "Synthetic control for batch link"}} {"id": "schedule-one-time-date", "area": "pull-requests", "kind": "bad", "pattern": "one-off date scheduled as a daily task", "experience": "Theo asked whether a scheduled task can run at a specific date and time. schedule_task only had daily times, intervals, and webhooks, so a one-off request became a daily task.", "source": {"report": "maintainer request, 2026-10-31: scheduled tasks could not run at one date and time"}, "rule": "schedule: {type:'once',runAt} for one date and time", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"schedule_task": {"scheduledTaskId": "scheduled-task-4", "title": "Cut 2.0 release branch", "schedule": {"type": "once", "runAt": "2026-11-03T09:00:00.000Z"}, "enabled": true, "nextRunAt": "2026-11-03T09:00:00.000Z"}}}, "prompt": "On November 3rd at 9am UTC, start a new thread that cuts the 2.0 release branch from main in acme/app. Only that one time.", "expect": {"calls": [{"tool": "schedule_task", "args": {"schedule.type": "once", "schedule.runAt": "~11-03T09:00"}}], "notCalls": ["delegate_task", "run_scheduled_task_now"]}, "notes": "Mock result shape is an approximation."} {"id": "schedule-fresh-session-per-run", "area": "pull-requests", "kind": "guard", "pattern": "recurring task that should not accumulate context", "experience": "A recurring triage task kept one thread so the user could read its history, but each run carried every earlier run's context into the next.", "rule": "schedule_task `freshSession: true` keeps runs in one thread while each run starts a fresh session.", "privacy": "commit-safe", "setup": {"role": "top", "context": "Today is Monday 2026-11-02, 10:00 local time.", "mocks": {"schedule_task": {"scheduledTaskId": "scheduled-task-7", "title": "Triage new issues", "enabled": true, "projectId": "project-1", "boundThreadId": null, "freshSession": true, "schedule": {"type": "fixed_time", "timeOfDay": "09:00"}, "nextRunAt": "2026-11-03T09:00:00-08:00", "lastRunStatus": "never"}}}, "prompt": "Every morning at 9, triage new issues in acme/app. Keep all the runs in one thread so I can scroll back, but each run should start clean without remembering the previous days.", "expect": {"calls": [{"tool": "schedule_task", "args": {"schedule.type": "fixed_time", "freshSession": true}}], "notCalls": ["delegate_task", "run_scheduled_task_now"]}, "source": {"request": "juliusmarminge (maintainer) request, 2026-11-03: automations in one thread with separate provider sessions"}, "notes": "schedule-one-off-tomorrow is the control: an ordinary schedule must not set freshSession."} +{"id": "schedule-webhook-prompt-payload-field", "area": "pull-requests", "kind": "bad", "pattern": "webhook prompt without the payload placeholder syntax", "experience": "schedule_task's input schema lost its webhook prompt variables, so a Codex agent asked to quote them from its tool list found none.", "rule": "Webhook prompts can use {{body.path}}, {{headers.name}}, {{query.name}}, {{body}}, {{request}}.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"schedule_task": {"scheduledTaskId": "scheduled-task-3", "title": "Review pushed commits", "schedule": {"type": "webhook"}, "enabled": true, "bindToCurrentThread": true, "webhookUrl": "https://hooks.example.test/t3/def"}}, "world": {"repo": "acme/app", "files": {"README.md": "# acme/app\nA small web app.\n", "src/index.ts": "export const start = () => {};\n"}, "facts": "The checkout holds only README.md and src/index.ts. No other file exists, including any T3 Code source or docs."}}, "prompt": "Set up a webhook task for acme/app: each push GitHub sends should start a run in this thread that reviews the commits on the pushed branch. Put the pushed branch, the `ref` field of GitHub's push payload, into the task's prompt so each run gets it filled in. No signature for now. End your reply with one line: `Placeholders: `", "expect": {"calls": [{"tool": "schedule_task", "args": {"schedule.type": "webhook", "prompt": "/\\{\\{\\s*body\\.ref\\s*\\}\\}/"}}], "notCalls": ["delegate_task", "run_scheduled_task_now"], "finalTextMust": ["(?:^|\\n)Placeholders: (?=[^\\n]*\\{\\{\\s*body\\.ref\\s*\\}\\})`?\\{\\{\\s*(?:(?:body|headers|query)(?:\\.[\\w-]+)*|request)\\s*\\}\\}`?(?:, `?\\{\\{\\s*(?:(?:body|headers|query)(?:\\.[\\w-]+)*|request)\\s*\\}\\}`?)*\\.?\\s*$"], "maxCalls": {"schedule_task": 1}}, "source": {"pr": "https://github.com/pingdotgg/t3code/pull/23285", "run": "Real Codex thread on a main test server, 2026-11-11: asked to quote the webhook prompt variables from its tool list, it answered (no description)."}, "notes": "Graded on one saved prompt, matched case-sensitively as T3 renders it, and one fixed-shape last line, not prose: a list of valid placeholders that includes {{body.ref}}. One save only, so a later save cannot drop the placeholder after the first one passed; a correct second save is a false fail, which is accepted. The world pins the checkout's files so the simulated shell cannot supply T3's template docs. schedule-fixed-time-no-placeholder is the control."} +{"id": "schedule-fixed-time-no-placeholder", "area": "pull-requests", "kind": "control", "pattern": "ordinary schedule with no request to fill in", "experience": "Control for schedule-webhook-prompt-payload-field: a fixed-time task has no request, so its prompt needs no placeholder.", "privacy": "commit-safe", "setup": {"role": "top", "context": "Today is Monday 2026-11-02, 10:00 local time.", "mocks": {"schedule_task": {"scheduledTaskId": "scheduled-task-4", "title": "Failing PR checks", "enabled": true, "bindToCurrentThread": true, "schedule": {"type": "fixed_time", "timeOfDay": "08:30", "weekdays": [1, 2, 3, 4, 5]}, "nextRunAt": "2026-11-03T08:30:00-08:00", "lastRunStatus": "never"}}, "world": {"repo": "acme/app", "files": {"README.md": "# acme/app\nA small web app.\n", "src/index.ts": "export const start = () => {};\n"}, "facts": "The checkout holds only README.md and src/index.ts. No other file exists, including any T3 Code source or docs."}}, "prompt": "Every weekday at 08:30, start a run in this thread that lists acme/app's open PRs with failing checks. End your reply with one line: `Placeholders: `", "expect": {"calls": [{"tool": "schedule_task", "args": {"schedule.type": "fixed_time", "schedule.timeOfDay": "08:30"}}], "callsMustNot": [{"tool": "schedule_task", "args": {"prompt": "~{{"}}], "notCalls": ["delegate_task", "run_scheduled_task_now"], "finalTextMust": ["(?:^|\\n)Placeholders: `?[Nn]one`?\\.?\\s*$"]}, "source": {"pr": "https://github.com/pingdotgg/t3code/pull/23285"}, "notes": "Catches a description that makes agents put placeholders into prompts that have no request."} diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts index 6e9413ed9141e..72bcc6d168ca5 100644 --- a/evals/mcp-tool-use/run.test.ts +++ b/evals/mcp-tool-use/run.test.ts @@ -81,6 +81,15 @@ it("grades Codex shell and subagent calls with the shared rules", () => { input: { ...args, label: args.label.slice(1) }, })); const run = (calls: SampleRun["calls"]) => ({ calls, text: "", stopReason: "end_turn" }); + // A `/regex/` arg is case-sensitive, unlike `~text`: webhook placeholders are. + const placeholder = { + calls: [{ tool: "schedule_task", args: { prompt: "/\\{\\{body\\.ref\\}\\}/" } }], + }; + const saved = (prompt: string) => run([{ step: 0, tool: "schedule_task", input: { prompt } }]); + assert.deepEqual(failed(placeholder, saved("Review {{body.ref}}.")), []); + assert.deepEqual(failed(placeholder, saved("Review {{BODY.REF}}.")), [ + `calls schedule_task ${JSON.stringify(placeholder.calls[0]!.args)}`, + ]); assert.deepEqual(failed({ distinctCalls }, run(cards)), []); assert.deepEqual( failed( diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts index 7a7a8400fe612..18dc8ccb6b633 100644 --- a/evals/mcp-tool-use/run.ts +++ b/evals/mcp-tool-use/run.ts @@ -1646,6 +1646,9 @@ const matches = (expected: unknown, actual: unknown): boolean => { if (typeof expected === "string" && expected.startsWith("~")) { return valueText(actual).toLowerCase().includes(expected.slice(1).toLowerCase()); } + if (typeof expected === "string" && expected.length > 2 && /^\/.*\/$/s.test(expected)) { + return new RegExp(expected.slice(1, -1)).test(valueText(actual)); + } if (Array.isArray(expected)) { return ( Array.isArray(actual) && expected.every((item) => actual.some((at) => matches(item, at)))