From 56dba87bc30f7497ecdf1fb9d27183c349bf35bf Mon Sep 17 00:00:00 2001 From: Bazyli Brzoska Date: Sat, 31 Oct 2026 23:21:51 -0700 Subject: [PATCH 1/2] fix(server): list peer links without waiting for reachability probes --- apps/server/src/peer/PeerLinks.test.ts | 52 +++++++++++++++++++ apps/server/src/peer/PeerLinks.ts | 25 +++------ .../client-runtime/src/state/peerLinks.ts | 7 +-- packages/contracts/src/peerLink.ts | 2 +- 4 files changed, 61 insertions(+), 25 deletions(-) diff --git a/apps/server/src/peer/PeerLinks.test.ts b/apps/server/src/peer/PeerLinks.test.ts index f16a91f26c0ad..221b9ddbdd86a 100644 --- a/apps/server/src/peer/PeerLinks.test.ts +++ b/apps/server/src/peer/PeerLinks.test.ts @@ -2,6 +2,7 @@ import { NodeHttpServer } from "@effect/platform-node"; import { type ExecutionEnvironmentDescriptor, OrchestratorMcpFailure } from "@t3tools/contracts"; import { expect, it } from "@effect/vitest"; import * as Context from "effect/Context"; +import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; @@ -279,6 +280,53 @@ const serveRaw = ( ), ); +it.effect("lists stored outcomes without waiting for an unreachable peer", () => + Effect.scoped( + Effect.gen(function* () { + const b = yield* serveB(box); + const a = yield* makeA(laptop); + const pairing = yield* b.auth.issuePairingCredential(); + const linked = yield* a.links.link({ + url: b.url, + pairingCode: pairing.credential, + access: "auto", + }); + const probed = yield* Deferred.make(); + const release = yield* Deferred.make(); + const asleep = yield* serveRaw(() => + Deferred.succeed(probed, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.as(HttpServerResponse.empty({ status: 503 })), + ), + ); + yield* a.sql`UPDATE peer_environment_links SET urls_json = ${JSON.stringify([asleep])}`; + + const listed = yield* Effect.raceFirst( + a.links.list, + Deferred.await(probed).pipe(Effect.as("blocked on peer")), + ); + yield* Deferred.succeed(release, undefined); + expect(listed).toEqual([{ ...linked, urls: [asleep], status: "reachable" }]); + + yield* a.links.recordOutcome(box.environmentId, { error: "Peer did not answer" }); + expect(yield* a.links.list).toMatchObject([ + { + status: "unreachable", + lastError: "Peer did not answer", + lastReachedAt: linked.lastReachedAt, + }, + ]); + yield* a.links.recordOutcome(box.environmentId, { error: null }); + expect(yield* a.links.list).toMatchObject([{ status: "reachable", lastError: null }]); + yield* a.sql`UPDATE peer_environment_links SET last_reached_at = NULL`; + expect(yield* a.links.list).toMatchObject([{ status: "unreachable" }]); + yield* TestClock.adjust(Duration.days(31)); + expect(yield* a.links.list).toMatchObject([{ status: "expired" }]); + expect(yield* Deferred.isDone(probed)).toBe(false); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + it.effect("never follows a peer's redirect with the pairing code", () => Effect.scoped( Effect.gen(function* () { @@ -335,6 +383,10 @@ it.effect("never sends a link's token to an address that stops answering as that expect(failed.message).toContain("Box did not answer"); expect(yield* Ref.get(b.bearers)).toEqual([]); expect((yield* a.links.list).map((link) => link.status)).toEqual(["unreachable"]); + + yield* Ref.set(b.descriptor, box); + yield* probe(a); + expect((yield* a.links.list).map((link) => link.status)).toEqual(["reachable"]); }), ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); diff --git a/apps/server/src/peer/PeerLinks.ts b/apps/server/src/peer/PeerLinks.ts index 11d058744084b..bb00aad510fd7 100644 --- a/apps/server/src/peer/PeerLinks.ts +++ b/apps/server/src/peer/PeerLinks.ts @@ -75,6 +75,7 @@ export class PeerLinks extends Context.Service< ) => Effect.Effect< Option.Option<{ readonly environmentId: EnvironmentId; readonly label: string }> >; + /** Stored outcomes and current expiry; does not probe peers. */ readonly list: Effect.Effect, PeerLinkError>; /** The stored link, without asking the peer whether it answers. */ readonly get: ( @@ -486,29 +487,17 @@ const make = Effect.gen(function* () { Effect.orElseSucceed(() => Option.none()), ); - const statusOf = (link: PeerLink, now: DateTime.Utc) => + const statusOf = (link: PeerLink, now: DateTime.Utc): PeerLinkStatus => DateTime.isLessThanOrEqualTo(link.expiresAt, now) - ? Effect.succeed("expired") - : Effect.findFirst(link.urls, (url) => - describe(url).pipe( - Effect.map( - (found) => Option.isSome(found) && found.value.environmentId === link.environmentId, - ), - ), - ).pipe( - Effect.map((answered): PeerLinkStatus => - Option.isSome(answered) ? "reachable" : "unreachable", - ), - ); + ? "expired" + : link.lastReachedAt !== null && link.lastError === null + ? "reachable" + : "unreachable"; const list: PeerLinks["Service"]["list"] = Effect.gen(function* () { const links = yield* Effect.forEach(yield* readRows, toLink); const now = yield* DateTime.now; - return yield* Effect.forEach( - links, - (link) => statusOf(link, now).pipe(Effect.map((status) => ({ ...link, status }))), - { concurrency: "unbounded" }, - ); + return links.map((link) => ({ ...link, status: statusOf(link, now) })); }); const get: PeerLinks["Service"]["get"] = (environmentId) => diff --git a/packages/client-runtime/src/state/peerLinks.ts b/packages/client-runtime/src/state/peerLinks.ts index 9295bf1b40156..045cd976b6e4b 100644 --- a/packages/client-runtime/src/state/peerLinks.ts +++ b/packages/client-runtime/src/state/peerLinks.ts @@ -13,7 +13,6 @@ export function createPeerLinkEnvironmentAtoms( const list = createEnvironmentRpcQueryAtomFamily(runtime, { label: "environment-data:peer-links:list", tag: WS_METHODS.peerLinksList, - // Listing probes every linked environment, so a list stays fresh briefly. staleTimeMs: 10_000, }); return { @@ -53,11 +52,7 @@ export type PeerLinkHealth = | { readonly kind: "failing"; readonly detail: string } | { readonly kind: "expired" }; -/** - * Listing only checks that the peer answers, so a link whose last call failed - * (the peer revoked it, say) is `failing` with the recorded reason; the next - * successful call clears it. - */ +/** Stored outcomes describe the last call; expiry is checked against the client's clock too. */ export function peerLinkHealth(link: PeerLinkSummary, now: DateTime.Utc): PeerLinkHealth { if (link.status === "expired" || DateTime.isLessThanOrEqualTo(link.expiresAt, now)) { return { kind: "expired" }; diff --git a/packages/contracts/src/peerLink.ts b/packages/contracts/src/peerLink.ts index f8cf83c1fdbdd..c50fa5e771f34 100644 --- a/packages/contracts/src/peerLink.ts +++ b/packages/contracts/src/peerLink.ts @@ -22,7 +22,7 @@ export const PeerLink = Schema.Struct({ }); export type PeerLink = typeof PeerLink.Type; -/** Whether a link can be used now, as worked out when it is listed. */ +/** Last recorded call outcome, with current expiry taking precedence; not a live probe. */ export const PeerLinkStatus = Schema.Literals(["reachable", "unreachable", "expired"]); export type PeerLinkStatus = typeof PeerLinkStatus.Type; From d06c15ab8a9cc8caad8de4e267530c4ab848b0bd Mon Sep 17 00:00:00 2001 From: Bazyli Brzoska Date: Sun, 1 Nov 2026 11:06:36 -0800 Subject: [PATCH 2/2] fix(server): describe stored peer status and on-demand retries --- apps/server/src/cli/environment.ts | 4 +++- apps/server/src/mcp/toolkits/orchestrator/tools.ts | 2 +- apps/server/src/provider/agentContextBudget.json | 2 +- evals/mcp-tool-use/cases/orchestration.ndjson | 1 + 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/server/src/cli/environment.ts b/apps/server/src/cli/environment.ts index 2b023f6562eb3..4fedde1cae79f 100644 --- a/apps/server/src/cli/environment.ts +++ b/apps/server/src/cli/environment.ts @@ -117,7 +117,9 @@ const environmentLinkCommand = Command.make("link", { ); const environmentListCommand = Command.make("list", { ...authLocationFlags, json: jsonFlag }).pipe( - Command.withDescription("List linked environments and whether each answers now."), + Command.withDescription( + "List linked environments with their last recorded call outcomes and current expiry, without probing peers. Calling a non-expired peer retries it and updates its stored outcome.", + ), Command.withHandler((flags) => runWithPeerLinks( flags, diff --git a/apps/server/src/mcp/toolkits/orchestrator/tools.ts b/apps/server/src/mcp/toolkits/orchestrator/tools.ts index 7c05f5c8d8e25..1e9ed737a9345 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/tools.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/tools.ts @@ -307,7 +307,7 @@ const ThreadInterruptTool = Tool.make("t3_thread_interrupt", { const EnvironmentLinksTool = Tool.make("t3_environment_links", { description: - "List linked environments (the user's other machines) and whether each answers now. Pass one's `environmentId` to a tool to act there; `delegate_task` takes `target.environmentId`. Expired: link it again with `t3_environment_link`.", + "List linked peers' stored call outcomes; no live probes. Retry non-expired unreachable peers with a tool call to update status. Tools take `environmentId`; `delegate_task`: `target.environmentId`. Expired: `t3_environment_link`.", success: OrchestratorMcpEnvironmentLinksResult, failure: OrchestratorMcpFailure, failureMode: "return", diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index 3e1e5dc80f702..537fcd7520b1c 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { "claudeSystemAppendBytes": 5153, "codexAdditionalContextBytes": 6883, - "mcpToolListJsonBytes": 47074 + "mcpToolListJsonBytes": 47072 } diff --git a/evals/mcp-tool-use/cases/orchestration.ndjson b/evals/mcp-tool-use/cases/orchestration.ndjson index dc79eb774df7e..c2ebd0e91d818 100644 --- a/evals/mcp-tool-use/cases/orchestration.ndjson +++ b/evals/mcp-tool-use/cases/orchestration.ndjson @@ -34,3 +34,4 @@ {"id": "two-children-same-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for two Sol xhigh subagents, an Astra parent started one and refused the second, citing 'One child per named model' (#24967).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}}, "prompt": "Launch two Sol xhigh subagents. Give each a trivial task that only replies OK.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "gpt-6.1-sol", "target.options.reasoningEffort": "xhigh"}}], "minCalls": {"delegate_task": 2}, "callsMustNot": [{"tool": "delegate_task", "args": {"target.model": "~muse"}}, {"tool": "delegate_task", "args": {"target.model": "~claude"}}]}, "notes": "From #24967. One child plus the parent doing the work fails, and so does a second model in place of Sol."} {"id": "worker-counts-per-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for 20 Sol workers and 20 Luna workers, the agent started one worker per model (#25359).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}, "world": {"files": {"tests/flaky/retry.test.ts": "it('retry is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/upload.test.ts": "it('upload is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/session.test.ts": "it('session is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/search.test.ts": "it('search is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/billing.test.ts": "it('billing is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/webhook.test.ts": "it('webhook is flaky', async () => { /* fails 1 in 5 runs */ });\n"}}}, "prompt": "Six tests in tests/flaky/ fail now and then. Start 3 Muse Wonder Full workers and 3 Grok workers, one test file each, to find and fix the flake.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "muse-wonder-full"}}, {"tool": "delegate_task", "args": {"target.model": "grok-4.7"}}], "minCalls": {"delegate_task": 6}}, "notes": "Scaled down from the real 20 Sol + 20 Luna ask. Muse and Grok need delegate_task from every parent, so a native Sol helper cannot pass or fail it."} {"id": "linked-thread-chip-reads-there", "area": "orchestration", "kind": "guard", "pattern": "thread chip from a linked machine read on this one", "experience": "A thread chip from a linked machine names its environmentId; an agent that reads it without passing that id reads this machine, where the thread does not exist.", "rule": "The thread chip names its environmentId; t3_thread_read takes it.", "privacy": "commit-safe", "setup": {"role": "top", "linkedEnvironments": true, "mocks": {"t3_environment_links": {"environmentId": "environment-laptop", "links": [{"environmentId": "environment-vps", "label": "Hetzner VPS", "status": "reachable", "access": "auto-accept-edits", "expiresAt": "2026-12-01T00:00:00.000Z", "lastError": null}], "notListed": "Not listed: not linked yet. Call t3_environment_link with a hint from the user's words."}, "t3_thread_read": {"thread": {"threadId": "thread-deploy-api", "title": "Deploy the API", "status": "idle"}, "items": [{"position": 1, "createdBy": "user", "text": "Deploy the API to the VPS and check the health endpoint."}, {"position": 2, "createdBy": "assistant", "text": "Deployed. /health returns 200, but /health/db times out after 5s: the API can't reach Postgres on 10.0.0.5:5432."}], "nextPosition": 3, "hasMore": false}}}, "prompt": "Pick up where [Deploy the API](t3-context://v1/thread/thread_deploy-api) left off. What is still failing?", "promptContext": {"records": [{"version": 1, "kind": "thread", "contextId": "thread_deploy-api", "label": "Deploy the API", "environmentId": "environment-vps", "threadId": "thread-deploy-api", "title": "Deploy the API"}]}, "expect": {"calls": [{"tool": "t3_thread_read", "args": {"threadId": "thread-deploy-api", "environmentId": "environment-vps"}}], "notCalls": ["t3_environment_link"]}, "source": {"request": "akj on #17130: attach a linked machine's thread without dragging"}} +{"id":"peer-inventory-explains-stored-status","area":"orchestration","kind":"bad","pattern":"cached failure presented as live reachability","experience":"PR #28533 left the inventory tool advertising live reachability after listing switched to stored outcomes.","rule":"Inventory reports the last recorded outcome; a call to a non-expired peer retries it and updates status.","privacy":"commit-safe","setup":{"role":"top","linkedEnvironments":true,"mocks":{"t3_environment_links":{"environmentId":"environment-local","links":[{"environmentId":"environment-build","label":"build-box","status":"unreachable","access":"auto","expiresAt":"2026-12-01T00:00:00.000Z","lastError":"build-box did not answer."}],"notListed":"Not listed: not linked yet."}}},"prompt":"Check linked environments. Does build-box being unreachable in this inventory prove it is offline right now? Explain what the status means and how to refresh it. Do not contact build-box or renew its link.","expect":{"calls":[{"tool":"t3_environment_links"}],"notCalls":["t3_environment_link","t3_project_list","delegate_task"],"textMust":["([Ss]tored|[Hh]istorical|[Ll]ast (recorded|call|known)|[Cc]ached)","([Cc]all|[Rr]etry|[Aa]ttempt)"]},"source":{"request":"https://github.com/pingdotgg/t3code/pull/28533#discussion_r4434640215"}}