Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion apps/server/src/cli/environment.ts
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,9 @@ const environmentLinkCommand = Command.make("link", {
);

const environmentListCommand = Command.make("list", { ...authLocationFlags, json: jsonFlag }).pipe(
Command.withDescription("List linked environments and whether each answers now."),
Command.withDescription(
"List linked environments with their last recorded call outcomes and current expiry, without probing peers. Calling a non-expired peer retries it and updates its stored outcome.",
),
Command.withHandler((flags) =>
runWithPeerLinks(
flags,
Expand Down
2 changes: 1 addition & 1 deletion apps/server/src/mcp/toolkits/orchestrator/tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,7 @@ const ThreadInterruptTool = Tool.make("t3_thread_interrupt", {

const EnvironmentLinksTool = Tool.make("t3_environment_links", {
description:
"List linked environments (the user's other machines) and whether each answers now. Pass one's `environmentId` to a tool to act there; `delegate_task` takes `target.environmentId`. Expired: link it again with `t3_environment_link`.",
"List linked peers' stored call outcomes; no live probes. Retry non-expired unreachable peers with a tool call to update status. Tools take `environmentId`; `delegate_task`: `target.environmentId`. Expired: `t3_environment_link`.",
success: OrchestratorMcpEnvironmentLinksResult,
failure: OrchestratorMcpFailure,
failureMode: "return",
Expand Down
52 changes: 52 additions & 0 deletions apps/server/src/peer/PeerLinks.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { NodeHttpServer } from "@effect/platform-node";
import { type ExecutionEnvironmentDescriptor, OrchestratorMcpFailure } from "@t3tools/contracts";
import { expect, it } from "@effect/vitest";
import * as Context from "effect/Context";
import * as Deferred from "effect/Deferred";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
Expand Down Expand Up @@ -279,6 +280,53 @@ const serveRaw = <E, R>(
),
);

it.effect("lists stored outcomes without waiting for an unreachable peer", () =>
Effect.scoped(
Effect.gen(function* () {
const b = yield* serveB(box);
const a = yield* makeA(laptop);
const pairing = yield* b.auth.issuePairingCredential();
const linked = yield* a.links.link({
url: b.url,
pairingCode: pairing.credential,
access: "auto",
});
const probed = yield* Deferred.make<void>();
const release = yield* Deferred.make<void>();
const asleep = yield* serveRaw(() =>
Deferred.succeed(probed, undefined).pipe(
Effect.andThen(Deferred.await(release)),
Effect.as(HttpServerResponse.empty({ status: 503 })),
),
);
yield* a.sql`UPDATE peer_environment_links SET urls_json = ${JSON.stringify([asleep])}`;

const listed = yield* Effect.raceFirst(
a.links.list,
Deferred.await(probed).pipe(Effect.as("blocked on peer")),
);
yield* Deferred.succeed(release, undefined);
expect(listed).toEqual([{ ...linked, urls: [asleep], status: "reachable" }]);

yield* a.links.recordOutcome(box.environmentId, { error: "Peer did not answer" });
expect(yield* a.links.list).toMatchObject([
{
status: "unreachable",
lastError: "Peer did not answer",
lastReachedAt: linked.lastReachedAt,
},
]);
yield* a.links.recordOutcome(box.environmentId, { error: null });
expect(yield* a.links.list).toMatchObject([{ status: "reachable", lastError: null }]);
yield* a.sql`UPDATE peer_environment_links SET last_reached_at = NULL`;
expect(yield* a.links.list).toMatchObject([{ status: "unreachable" }]);
yield* TestClock.adjust(Duration.days(31));
expect(yield* a.links.list).toMatchObject([{ status: "expired" }]);
expect(yield* Deferred.isDone(probed)).toBe(false);
}),
).pipe(Effect.provide(NodeHttpServer.layerTest)),
);

it.effect("never follows a peer's redirect with the pairing code", () =>
Effect.scoped(
Effect.gen(function* () {
Expand Down Expand Up @@ -335,6 +383,10 @@ it.effect("never sends a link's token to an address that stops answering as that
expect(failed.message).toContain("Box did not answer");
expect(yield* Ref.get(b.bearers)).toEqual([]);
expect((yield* a.links.list).map((link) => link.status)).toEqual(["unreachable"]);

yield* Ref.set(b.descriptor, box);
yield* probe(a);
expect((yield* a.links.list).map((link) => link.status)).toEqual(["reachable"]);
}),
).pipe(Effect.provide(NodeHttpServer.layerTest)),
);
Expand Down
25 changes: 7 additions & 18 deletions apps/server/src/peer/PeerLinks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ export class PeerLinks extends Context.Service<
) => Effect.Effect<
Option.Option<{ readonly environmentId: EnvironmentId; readonly label: string }>
>;
/** Stored outcomes and current expiry; does not probe peers. */
readonly list: Effect.Effect<ReadonlyArray<PeerLinkSummary>, PeerLinkError>;
/** The stored link, without asking the peer whether it answers. */
readonly get: (
Expand Down Expand Up @@ -486,29 +487,17 @@ const make = Effect.gen(function* () {
Effect.orElseSucceed(() => Option.none()),
);

const statusOf = (link: PeerLink, now: DateTime.Utc) =>
const statusOf = (link: PeerLink, now: DateTime.Utc): PeerLinkStatus =>
DateTime.isLessThanOrEqualTo(link.expiresAt, now)
? Effect.succeed<PeerLinkStatus>("expired")
: Effect.findFirst(link.urls, (url) =>
describe(url).pipe(
Effect.map(
(found) => Option.isSome(found) && found.value.environmentId === link.environmentId,
),
),
).pipe(
Effect.map((answered): PeerLinkStatus =>
Option.isSome(answered) ? "reachable" : "unreachable",
),
);
? "expired"
: link.lastReachedAt !== null && link.lastError === null
? "reachable"
: "unreachable";
Comment thread
niieani marked this conversation as resolved.
Resolved

const list: PeerLinks["Service"]["list"] = Effect.gen(function* () {
const links = yield* Effect.forEach(yield* readRows, toLink);
const now = yield* DateTime.now;
return yield* Effect.forEach(
links,
(link) => statusOf(link, now).pipe(Effect.map((status) => ({ ...link, status }))),
{ concurrency: "unbounded" },
);
return links.map((link) => ({ ...link, status: statusOf(link, now) }));
});

const get: PeerLinks["Service"]["get"] = (environmentId) =>
Expand Down
2 changes: 1 addition & 1 deletion apps/server/src/provider/agentContextBudget.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"claudeSystemAppendBytes": 5153,
"codexAdditionalContextBytes": 6883,
"mcpToolListJsonBytes": 47074
"mcpToolListJsonBytes": 47072
}
1 change: 1 addition & 0 deletions evals/mcp-tool-use/cases/orchestration.ndjson
Original file line number Diff line number Diff line change
Expand Up @@ -34,3 +34,4 @@
{"id": "two-children-same-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for two Sol xhigh subagents, an Astra parent started one and refused the second, citing 'One child per named model' (#24967).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}}, "prompt": "Launch two Sol xhigh subagents. Give each a trivial task that only replies OK.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "gpt-6.1-sol", "target.options.reasoningEffort": "xhigh"}}], "minCalls": {"delegate_task": 2}, "callsMustNot": [{"tool": "delegate_task", "args": {"target.model": "~muse"}}, {"tool": "delegate_task", "args": {"target.model": "~claude"}}]}, "notes": "From #24967. One child plus the parent doing the work fails, and so does a second model in place of Sol."}
{"id": "worker-counts-per-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for 20 Sol workers and 20 Luna workers, the agent started one worker per model (#25359).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}, "world": {"files": {"tests/flaky/retry.test.ts": "it('retry is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/upload.test.ts": "it('upload is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/session.test.ts": "it('session is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/search.test.ts": "it('search is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/billing.test.ts": "it('billing is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/webhook.test.ts": "it('webhook is flaky', async () => { /* fails 1 in 5 runs */ });\n"}}}, "prompt": "Six tests in tests/flaky/ fail now and then. Start 3 Muse Wonder Full workers and 3 Grok workers, one test file each, to find and fix the flake.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "muse-wonder-full"}}, {"tool": "delegate_task", "args": {"target.model": "grok-4.7"}}], "minCalls": {"delegate_task": 6}}, "notes": "Scaled down from the real 20 Sol + 20 Luna ask. Muse and Grok need delegate_task from every parent, so a native Sol helper cannot pass or fail it."}
{"id": "linked-thread-chip-reads-there", "area": "orchestration", "kind": "guard", "pattern": "thread chip from a linked machine read on this one", "experience": "A thread chip from a linked machine names its environmentId; an agent that reads it without passing that id reads this machine, where the thread does not exist.", "rule": "The thread chip names its environmentId; t3_thread_read takes it.", "privacy": "commit-safe", "setup": {"role": "top", "linkedEnvironments": true, "mocks": {"t3_environment_links": {"environmentId": "environment-laptop", "links": [{"environmentId": "environment-vps", "label": "Hetzner VPS", "status": "reachable", "access": "auto-accept-edits", "expiresAt": "2026-12-01T00:00:00.000Z", "lastError": null}], "notListed": "Not listed: not linked yet. Call t3_environment_link with a hint from the user's words."}, "t3_thread_read": {"thread": {"threadId": "thread-deploy-api", "title": "Deploy the API", "status": "idle"}, "items": [{"position": 1, "createdBy": "user", "text": "Deploy the API to the VPS and check the health endpoint."}, {"position": 2, "createdBy": "assistant", "text": "Deployed. /health returns 200, but /health/db times out after 5s: the API can't reach Postgres on 10.0.0.5:5432."}], "nextPosition": 3, "hasMore": false}}}, "prompt": "Pick up where [Deploy the API](t3-context://v1/thread/thread_deploy-api) left off. What is still failing?", "promptContext": {"records": [{"version": 1, "kind": "thread", "contextId": "thread_deploy-api", "label": "Deploy the API", "environmentId": "environment-vps", "threadId": "thread-deploy-api", "title": "Deploy the API"}]}, "expect": {"calls": [{"tool": "t3_thread_read", "args": {"threadId": "thread-deploy-api", "environmentId": "environment-vps"}}], "notCalls": ["t3_environment_link"]}, "source": {"request": "akj on #17130: attach a linked machine's thread without dragging"}}
{"id":"peer-inventory-explains-stored-status","area":"orchestration","kind":"bad","pattern":"cached failure presented as live reachability","experience":"PR #28533 left the inventory tool advertising live reachability after listing switched to stored outcomes.","rule":"Inventory reports the last recorded outcome; a call to a non-expired peer retries it and updates status.","privacy":"commit-safe","setup":{"role":"top","linkedEnvironments":true,"mocks":{"t3_environment_links":{"environmentId":"environment-local","links":[{"environmentId":"environment-build","label":"build-box","status":"unreachable","access":"auto","expiresAt":"2026-12-01T00:00:00.000Z","lastError":"build-box did not answer."}],"notListed":"Not listed: not linked yet."}}},"prompt":"Check linked environments. Does build-box being unreachable in this inventory prove it is offline right now? Explain what the status means and how to refresh it. Do not contact build-box or renew its link.","expect":{"calls":[{"tool":"t3_environment_links"}],"notCalls":["t3_environment_link","t3_project_list","delegate_task"],"textMust":["([Ss]tored|[Hh]istorical|[Ll]ast (recorded|call|known)|[Cc]ached)","([Cc]all|[Rr]etry|[Aa]ttempt)"]},"source":{"request":"https://github.com/pingdotgg/t3code/pull/28533#discussion_r4434640215"}}
7 changes: 1 addition & 6 deletions packages/client-runtime/src/state/peerLinks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ export function createPeerLinkEnvironmentAtoms<R, E>(
const list = createEnvironmentRpcQueryAtomFamily(runtime, {
label: "environment-data:peer-links:list",
tag: WS_METHODS.peerLinksList,
// Listing probes every linked environment, so a list stays fresh briefly.
staleTimeMs: 10_000,
});
return {
Expand Down Expand Up @@ -53,11 +52,7 @@ export type PeerLinkHealth =
| { readonly kind: "failing"; readonly detail: string }
| { readonly kind: "expired" };

/**
* Listing only checks that the peer answers, so a link whose last call failed
* (the peer revoked it, say) is `failing` with the recorded reason; the next
* successful call clears it.
*/
/** Stored outcomes describe the last call; expiry is checked against the client's clock too. */
export function peerLinkHealth(link: PeerLinkSummary, now: DateTime.Utc): PeerLinkHealth {
if (link.status === "expired" || DateTime.isLessThanOrEqualTo(link.expiresAt, now)) {
return { kind: "expired" };
Expand Down
2 changes: 1 addition & 1 deletion packages/contracts/src/peerLink.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ export const PeerLink = Schema.Struct({
});
export type PeerLink = typeof PeerLink.Type;

/** Whether a link can be used now, as worked out when it is listed. */
/** Last recorded call outcome, with current expiry taking precedence; not a live probe. */
export const PeerLinkStatus = Schema.Literals(["reachable", "unreachable", "expired"]);
export type PeerLinkStatus = typeof PeerLinkStatus.Type;

Expand Down
Loading