diff --git a/apps/server/package.json b/apps/server/package.json index cb8352940f95..ed3045828257 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -1,6 +1,6 @@ { "name": "t3", - "version": "0.1.7", + "version": "0.1.9", "license": "MIT", "type": "module", "scripts": { @@ -29,7 +29,7 @@ "yauzl": "^3.4.0" }, "devDependencies": { - "@anthropic-ai/claude-agent-sdk": "0.3.316", + "@anthropic-ai/claude-agent-sdk": "0.3.321", "@anthropic-ai/sdk": "0.93.0", "@effect/vitest": "catalog:", "@t3tools/contracts": "workspace:*", diff --git a/apps/server/scripts/acp-mock-agent.ts b/apps/server/scripts/acp-mock-agent.ts index a646b45c570d..660853e7d77e 100644 --- a/apps/server/scripts/acp-mock-agent.ts +++ b/apps/server/scripts/acp-mock-agent.ts @@ -28,6 +28,7 @@ const emitInterleavedAssistantToolCalls = process.env.T3_ACP_EMIT_INTERLEAVED_ASSISTANT_TOOL_CALLS === "1"; const emitV2Fidelity = process.env.T3_ACP_EMIT_V2_FIDELITY === "1"; const emitNotice = process.env.T3_ACP_EMIT_NOTICE === "1"; +const hangCloseSession = process.env.T3_ACP_HANG_CLOSE_SESSION === "1"; const vibeRetryOutcome = process.env.T3_ACP_VIBE_RETRY_OUTCOME; const emitGenericToolPlaceholders = process.env.T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS === "1"; const emitInTurnTaskOutputThenLateDuplicate = @@ -85,6 +86,7 @@ const supportsSessionLifecycle = process.env.T3_ACP_SESSION_LIFECYCLE === "1"; const supportsAcpMcp = process.env.T3_ACP_MCP_ACP === "1"; const supportsHttpMcp = process.env.T3_ACP_MCP_HTTP === "1"; const supportsV2Management = process.env.T3_ACP_V2_MANAGEMENT === "1"; +const failDeleteSession = process.env.T3_ACP_FAIL_DELETE_SESSION === "1"; const advertisedAuthMethodId = process.env.T3_ACP_AUTH_METHOD_ID?.trim(); const initializeAuthMethodId = advertisedAuthMethodId ?? (supportsSessionLifecycle ? "test" : undefined); @@ -760,6 +762,9 @@ const program = Effect.gen(function* () { yield* agent.handleCloseSession(() => Effect.gen(function* () { yield* requireAuthentication(); + if (hangCloseSession) { + return yield* Effect.never; + } return {}; }), ); @@ -767,6 +772,9 @@ const program = Effect.gen(function* () { yield* agent.handleDeleteSession(() => Effect.gen(function* () { yield* requireAuthentication(); + if (failDeleteSession) { + return yield* AcpError.AcpRequestError.internalError("delete failed"); + } return {}; }), ); diff --git a/apps/server/src/checkpointing/CheckpointStore.test.ts b/apps/server/src/checkpointing/CheckpointStore.test.ts index 36e58024adba..62c764b537a8 100644 --- a/apps/server/src/checkpointing/CheckpointStore.test.ts +++ b/apps/server/src/checkpointing/CheckpointStore.test.ts @@ -120,6 +120,19 @@ it.layer(layerTest)("CheckpointStore.layer", (it) => { expect(yield* checkpointStore.isGitRepository(nested)).toBe(true); }), ); + it.effect("fails the repository probe that other callers read as no repository", () => + Effect.gen(function* () { + const tmp = yield* makeTmpDir(); + yield* git(tmp, ["init"]); + yield* writeTextFile(NodePath.join(tmp, ".git", "config"), "invalid config\n"); + const registry = yield* VcsDriverRegistry.VcsDriverRegistry; + const checkpointStore = yield* CheckpointStore.CheckpointStore; + + expect(yield* registry.detect({ cwd: tmp })).toBeNull(); + const probe = yield* Effect.flip(checkpointStore.isGitRepository(tmp)); + expect(probe._tag).toBe("VcsProcessExitError"); + }), + ); describe("diffCheckpoints", () => { it.effect("returns full oversized checkpoint diffs without truncation", () => Effect.gen(function* () { diff --git a/apps/server/src/checkpointing/CheckpointStore.ts b/apps/server/src/checkpointing/CheckpointStore.ts index 3d180e9c66c4..e5d3006568a6 100644 --- a/apps/server/src/checkpointing/CheckpointStore.ts +++ b/apps/server/src/checkpointing/CheckpointStore.ts @@ -85,10 +85,14 @@ export const make = Effect.gen(function* () { return handle.driver.checkpoints satisfies VcsCheckpointOps; }); - const isGitRepository: CheckpointStore["Service"]["isGitRepository"] = (cwd) => - vcsRegistry - .detect({ cwd, requestedKind: "git" }) - .pipe(Effect.map((repository) => repository !== null)); + // The registry reads an unreadable repository as none; ask Git again so that fails here. + const isGitRepository: CheckpointStore["Service"]["isGitRepository"] = Effect.fn( + "CheckpointStore.isGitRepository", + )(function* (cwd) { + if ((yield* vcsRegistry.detect({ cwd, requestedKind: "git" })) !== null) return true; + const git = yield* vcsRegistry.get("git"); + return (yield* git.detectRepository(cwd)) !== null; + }); const captureCheckpoint: CheckpointStore["Service"]["captureCheckpoint"] = Effect.fn( "captureCheckpoint", diff --git a/apps/server/src/coderEnvironment.ts b/apps/server/src/coderEnvironment.ts index b51c7ab71461..b713297bfb2f 100644 --- a/apps/server/src/coderEnvironment.ts +++ b/apps/server/src/coderEnvironment.ts @@ -126,6 +126,8 @@ export const CODER_OMITTED_CAPABILITIES = { "No Add to another machine: ignored files such as .env would pass through the laptop browser.", mcpModeLimitHeader: "No /mcp HTTP server; the gateway hub carries a linked call's mode limits with the call.", + modelCatalogRefresh: + "The model catalog is bundled (`ModelManifest.layerBundled`), so an opened picker has nothing new to fetch.", triage: "No in-app triage: it clones upstream T3 Code to help file upstream issues.", } satisfies Partial>; diff --git a/apps/server/src/coderServer.ts b/apps/server/src/coderServer.ts index 36a585f0d693..dbfd4e97f3de 100644 --- a/apps/server/src/coderServer.ts +++ b/apps/server/src/coderServer.ts @@ -28,6 +28,8 @@ import * as ProjectFaviconResolver from "./project/ProjectFaviconResolver.ts"; import * as T3ProjectFileLoader from "./project/T3ProjectFileLoader.ts"; import * as AgentScopeLive from "./process/agentScope.ts"; import * as ProviderInstanceRegistryHydration from "./provider/ProviderInstanceRegistryHydration.ts"; +import * as ProviderInstallationGate from "./provider/ProviderInstallationGate.ts"; +import * as AcpRegistrySessions from "./provider/AcpRegistrySessions.ts"; import * as AcpRegistryRuntimeCoordinator from "@t3tools/provider-acp-registry/server/AcpRegistryRuntimeCoordinator"; import * as ProviderRegistryLayer from "./provider/ProviderRegistry.ts"; import * as ProviderEventLoggers from "@t3tools/provider-core/server/ProviderEventLoggers"; @@ -59,7 +61,6 @@ import * as PeerForwarding from "./peer/PeerForwarding.ts"; import * as PeerLinks from "./peer/PeerLinks.ts"; import * as PeerMcpClient from "./peer/PeerMcpClient.ts"; import * as RemoteDelegation from "./peer/RemoteDelegation.ts"; -import * as AgentSessionScanner from "./project/AgentSessionScanner.ts"; import * as ProviderRuntimeRecovery from "./orchestration-v2/ProviderRuntimeRecoveryService.ts"; import * as ProviderSessionManager from "./orchestration-v2/ProviderSessionManager.ts"; import * as RunFinalizationService from "./orchestration-v2/RunFinalizationService.ts"; @@ -241,6 +242,8 @@ const CoderRuntimeDependenciesLive = CoderRuntimeCoreLive.pipe( ), Layer.provideMerge(CoderProviderRegistryLive), Layer.provideMerge(CoderProviderInstancesLive), + // As upstream: one gate for the server, below hydration so the Claude adapter sees it. + Layer.provideMerge(ProviderInstallationGate.layer), // Provider hydration builds the Claude runner, which reports agents to the same scanner the // terminal layer and the RPC handlers share. Layer.provideMerge(layerPortScanner), @@ -377,9 +380,9 @@ export const makeCoderRuntimeLayer = () => { // toolkit read office documents' extracted text; Node's crypto backs random ids and digests. Layer.provideMerge(DocumentText.layer.pipe(Layer.provideMerge(NodeCrypto.layer))), ); - // Upstream provides the agent-session scanner beside its WS layer; the helper RPC layer - // takes it here. - const rpcServices = AgentSessionScanner.layer.pipe(Layer.provideMerge(services)); + // Upstream provides the ACP session service beside its WS layer; the helper RPC layer takes it + // here. + const rpcServices = AcpRegistrySessions.layer.pipe(Layer.provideMerge(services)); // Coder: the gateway hub methods (helper protocol v3) share the runtime's CoderHub. return Layer.mergeAll(CoderWs.layer, CoderHub.handlersLayer).pipe(Layer.provide(rpcServices)); diff --git a/apps/server/src/git/GitManager.test.ts b/apps/server/src/git/GitManager.test.ts index ed3dd48845e7..dbcb47775141 100644 --- a/apps/server/src/git/GitManager.test.ts +++ b/apps/server/src/git/GitManager.test.ts @@ -2615,6 +2615,46 @@ it.layer(layerGitManagerTest)("GitManager", (it) => { }), ); + it.effect( + "status does not inherit a release PR from a worktree's non-default base upstream", + () => + Effect.gen(function* () { + const repoDir = yield* makeTempDir("t3code-git-manager-"); + yield* initRepo(repoDir); + const remoteDir = yield* createBareRemote(); + yield* runGit(repoDir, ["remote", "add", "origin", remoteDir]); + yield* runGit(repoDir, ["push", "-u", "origin", "main"]); + yield* runGit(repoDir, ["push", "origin", "main:dev"]); + yield* runGit(repoDir, ["fetch", "origin"]); + yield* runGit(repoDir, ["remote", "set-head", "origin", "main"]); + yield* runGit(repoDir, ["checkout", "-b", "feature/from-dev", "origin/dev"]); + yield* runGit(repoDir, ["config", "branch.feature/from-dev.t3-merge-base", "dev"]); + + const { manager, providerCalls } = yield* makeManager({ + providerScenario: { + prListByHeadSelector: { + dev: [ + { + number: 49, + title: "Release dev to main", + url: "https://github.com/pingdotgg/codething-mvp/pull/49", + baseRefName: "main", + headRefName: "dev", + state: "merged", + updatedAt: "2026-08-20T10:00:00Z", + }, + ], + }, + }, + }); + + const status = yield* manager.status({ cwd: repoDir }); + expect(status.refName).toBe("feature/from-dev"); + expect(status.pr).toBeNull(); + expect(listedSelectors(providerCalls).includes("dev")).toBe(false); + }), + ); + it.effect("status finds a PR pushed under the branch's own name despite a default upstream", () => Effect.gen(function* () { const repoDir = yield* makeTempDir("t3code-git-manager-"); @@ -3489,6 +3529,125 @@ it.layer(layerGitManagerTest)("GitManager", (it) => { }), ); + it.effect.each([ + { localBranch: "develop" }, + // Pushes to origin's develop under a name that differs from the base. + { localBranch: "origin/develop" }, + ] as const)( + "commit_push_pr refuses to commit or push on the default branch when origin/HEAD is missing ($localBranch)", + ({ localBranch }) => + Effect.gen(function* () { + const repoDir = yield* makeTempDir("t3code-git-manager-"); + yield* initRepo(repoDir); + yield* runGit(repoDir, ["branch", "-m", "main", "develop"]); + const remoteDir = yield* createBareRemote(); + yield* runGit(repoDir, ["remote", "add", "origin", remoteDir]); + yield* runGit(repoDir, ["push", "-u", "origin", "develop"]); + if (localBranch !== "develop") yield* runGit(repoDir, ["checkout", "-b", localBranch]); + const readRemoteDevelop = runGit(repoDir, [ + "ls-remote", + "origin", + "refs/heads/develop", + ]).pipe(Effect.map((output) => output.stdout)); + const remoteBefore = yield* readRemoteDevelop; + const localBefore = (yield* runGit(repoDir, ["rev-parse", "HEAD"])).stdout; + NodeFS.writeFileSync(NodePath.join(repoDir, "change.txt"), "change\n"); + + const { manager } = yield* makeManager({ + providerScenario: { defaultBranch: "develop", prListSequence: [[], []] }, + }); + + const error = yield* runStackedAction(manager, { + cwd: repoDir, + action: "commit_push_pr", + }).pipe(Effect.flip); + + expect(yield* readRemoteDevelop).toBe(remoteBefore); + expect((yield* runGit(repoDir, ["rev-parse", "HEAD"])).stdout).toBe(localBefore); + expect(error.message).toContain("develop is the base branch"); + }), + ); + + it.effect("commit_push_pr on the default branch returns its open PR into another base", () => + Effect.gen(function* () { + const repoDir = yield* makeTempDir("t3code-git-manager-"); + yield* initRepo(repoDir); + const remoteDir = yield* createBareRemote(); + yield* runGit(repoDir, ["remote", "add", "origin", remoteDir]); + yield* runGit(repoDir, ["push", "-u", "origin", "main"]); + const releasePr = { + number: 43, + title: "Release", + url: "https://github.com/pingdotgg/codething-mvp/pull/43", + baseRefName: "release", + headRefName: "main", + }; + + const { manager } = yield* makeManager({ + providerScenario: { defaultBranch: "main", prListSequence: [[releasePr], [releasePr]] }, + }); + const result = yield* runStackedAction(manager, { + cwd: repoDir, + action: "commit_push_pr", + }); + + expect(result.pr.status).toBe("opened_existing"); + expect(result.pr.number).toBe(43); + }), + ); + + it.effect( + "commit_push_pr publishes a branch that tracks the default branch under its own name", + () => + Effect.gen(function* () { + const repoDir = yield* makeTempDir("t3code-git-manager-"); + yield* initRepo(repoDir); + yield* runGit(repoDir, ["branch", "-m", "main", "develop"]); + const remoteDir = yield* createBareRemote(); + yield* runGit(repoDir, ["remote", "add", "origin", remoteDir]); + yield* runGit(repoDir, ["push", "-u", "origin", "develop"]); + yield* runGit(repoDir, ["checkout", "-b", "t3code/feature", "--track", "origin/develop"]); + const readRemoteDevelop = runGit(repoDir, [ + "ls-remote", + "origin", + "refs/heads/develop", + ]).pipe(Effect.map((output) => output.stdout)); + const remoteBefore = yield* readRemoteDevelop; + NodeFS.writeFileSync(NodePath.join(repoDir, "change.txt"), "change\n"); + + const { manager, providerCalls } = yield* makeManager({ + providerScenario: { + defaultBranch: "develop", + prListSequence: [ + [], + [ + { + number: 506, + title: "Feature", + url: "https://github.com/pingdotgg/codething-mvp/pull/506", + baseRefName: "develop", + headRefName: "t3code/feature", + }, + ], + ], + }, + }); + + const result = yield* runStackedAction(manager, { + cwd: repoDir, + action: "commit_push_pr", + }); + + expect(result.pr.status).toBe("created"); + expect(yield* readRemoteDevelop).toBe(remoteBefore); + expect( + created(providerCalls).some( + (call) => call.baseRefName === "develop" && call.headSelector === "t3code/feature", + ), + ).toBe(true); + }), + ); + it.effect("create_pr reports a created PR when the lookup after creation fails", () => Effect.gen(function* () { const repoDir = yield* makeTempDir("t3code-git-manager-"); @@ -4345,6 +4504,74 @@ it.layer(layerGitManagerTest)("GitManager", (it) => { }), ); + it.effect("commit_push_pr publishes the default branch to the push remote's fork", () => + Effect.gen(function* () { + const repoDir = yield* makeTempDir("t3code-git-manager-"); + yield* initRepo(repoDir); + const originDir = yield* createBareRemote(); + const forkDir = yield* createBareRemote(); + yield* runGit(repoDir, ["remote", "add", "origin", originDir]); + yield* runGit(repoDir, ["push", "origin", "main"]); + yield* runGit(repoDir, ["remote", "add", "fork", forkDir]); + yield* runGit(repoDir, ["push", "fork", "main"]); + yield* runGit(repoDir, ["config", "remote.pushDefault", "fork"]); + yield* configureVisibleRemoteUrlWithLocalRewrite( + repoDir, + "origin", + "git@github.com:pingdotgg/codething-mvp.git", + originDir, + ); + yield* configureVisibleRemoteUrlWithLocalRewrite( + repoDir, + "fork", + "git@github.com:octocat/codething-mvp.git", + forkDir, + ); + const readOriginMain = runGit(repoDir, ["ls-remote", "origin", "refs/heads/main"]).pipe( + Effect.map((output) => output.stdout), + ); + const originBefore = yield* readOriginMain; + NodeFS.writeFileSync(NodePath.join(repoDir, "change.txt"), "change\n"); + + const { manager, providerCalls } = yield* makeManager({ + providerScenario: { + defaultBranch: "main", + prListSequenceByHeadSelector: { + "octocat:main": [ + [], + [ + { + number: 189, + title: "Contribute from main", + url: "https://github.com/pingdotgg/codething-mvp/pull/189", + baseRefName: "main", + headRefName: "main", + state: "open", + isCrossRepository: true, + headRepositoryNameWithOwner: "octocat/codething-mvp", + headRepositoryOwnerLogin: "octocat", + }, + ], + ], + }, + }, + }); + + const result = yield* runStackedAction(manager, { + cwd: repoDir, + action: "commit_push_pr", + }); + + expect(result.pr.status).toBe("created"); + expect(yield* readOriginMain).toBe(originBefore); + expect( + created(providerCalls).some( + (call) => call.baseRefName === "main" && call.headSelector === "octocat:main", + ), + ).toBe(true); + }), + ); + it.effect("rejects push/pr actions from detached HEAD", () => Effect.gen(function* () { const repoDir = yield* makeTempDir("t3code-git-manager-"); @@ -4765,6 +4992,9 @@ it.layer(layerGitManagerTest)("GitManager", (it) => { expect(NodeFS.readFileSync(NodePath.join(repoDir, "untracked.txt"), "utf8")).toBe( "keep me\n", ); + expect( + (yield* runGit(repoDir, ["config", "branch.feature/pr-safe.t3-pull-request"])).stdout, + ).toBe("https://github.com/pingdotgg/codething-mvp/pull/91\n"); }), ); diff --git a/apps/server/src/git/GitManager.ts b/apps/server/src/git/GitManager.ts index fa04333d55c7..84fe8438c7e8 100644 --- a/apps/server/src/git/GitManager.ts +++ b/apps/server/src/git/GitManager.ts @@ -86,6 +86,7 @@ import { detachStackFrame } from "./detachStackFrame.ts"; import * as ServerSettings from "../serverSettings.ts"; import type { GitManagerServiceError } from "@t3tools/contracts"; import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; +import { isNonRepositoryGitStderr } from "../vcs/GitVcsDriverCore.ts"; import * as SourceControlProvider from "@t3tools/source-control-core/server/SourceControlProvider"; import * as SourceControlRateLimit from "@t3tools/source-control-core/server/SourceControlRateLimit"; import * as SourceControlProviderRegistry from "../sourceControl/SourceControlProviderRegistry.ts"; @@ -114,6 +115,7 @@ export type GitBranchPullRequest = NonNullable & { interface SourceControlTextGenerationSettings { readonly modelSelection: ModelSelection; + readonly projectId: ProjectId | null; readonly style: SourceControlWritingStyleSettings; } @@ -139,6 +141,12 @@ export class GitManager extends Context.Service< input: { readonly cwd: string; readonly branch: string }, options?: { readonly refresh?: boolean; readonly freshGitState?: boolean }, ) => Effect.Effect; + /** Whether T3 checked this pull request out onto the branch, which ties it to later threads. */ + readonly checkedOutPullRequest: (input: { + readonly cwd: string; + readonly branch: string; + readonly url: string; + }) => Effect.Effect; readonly invalidateLocalStatus: (cwd: string) => Effect.Effect; readonly invalidateRemoteStatus: (cwd: string) => Effect.Effect; readonly invalidateStatus: (cwd: string) => Effect.Effect; @@ -191,7 +199,7 @@ type GitActionProgressPayload = StripProgressContext; type GitActionProgressEmitter = (event: GitActionProgressPayload) => Effect.Effect; function isNotGitRepositoryError(error: GitCommandError): boolean { - return error.message.toLowerCase().includes("not a git repository"); + return isNonRepositoryGitStderr(error.message); } interface OpenPrInfo { @@ -1777,7 +1785,7 @@ export const make = Effect.gen(function* () { return remoteUrl === null ? null : remoteName; }); - // worktree add can set origin/main as upstream; this base is not the branch's published PR head. + // worktree add can set the branch's base as upstream; that is not its published PR head. const resolveLookupHeadContext = Effect.fn("resolveLookupHeadContext")(function* ( cwd: string, details: { @@ -1799,15 +1807,16 @@ export const make = Effect.gen(function* () { }; } } - const upstreamHeadIsDefault = + if (headContext.headBranch === details.branch || headContext.isCrossRepository) { + return { headContext, lookup: true }; + } + const upstreamHeadIsBase = headContext.headBranch === details.defaultBranch || (details.defaultBranch === null && - (headContext.headBranch === "main" || headContext.headBranch === "master")); - if ( - headContext.headBranch === details.branch || - !upstreamHeadIsDefault || - headContext.isCrossRepository - ) { + (headContext.headBranch === "main" || headContext.headBranch === "master")) || + headContext.headBranch === + (yield* gitCore.readBranchBase(cwd, details.branch).pipe(Effect.orElseSucceed(() => null))); + if (!upstreamHeadIsBase) { return { headContext, lookup: true }; } const remoteName = yield* findRemoteTrackingRemote( @@ -2147,6 +2156,7 @@ export const make = Effect.gen(function* () { ...(input.includeBranch ? { includeBranch: true } : {}), ...(policy ? { policy } : {}), modelSelection: input.settings.modelSelection, + projectId: input.settings.projectId, }) .pipe(Effect.map((result) => sanitizeCommitMessage(result))); @@ -2273,6 +2283,34 @@ export const make = Effect.gen(function* () { }; }); + const ensurePrHeadIsNotBase = Effect.fn("ensurePrHeadIsNotBase")(function* ( + cwd: string, + status: { branch: string; upstreamRef: string | null; hasUpstream: boolean }, + changeRequestLabel: string, + ) { + const target = yield* gitCore.resolvePushTarget(cwd, status); + if (!target) return; + const headContext = yield* resolveBranchHeadContext(cwd, { + branch: status.branch, + upstreamRef: `${target.remoteName}/${target.branchName}`, + remoteName: target.remoteName, + }); + if (headContext.isCrossRepository) return; + const { baseBranch } = yield* resolvePrBase( + cwd, + status.branch, + status.upstreamRef, + headContext, + ); + if (baseBranch !== target.branchName) return; + if (yield* findOpenPr(cwd, headContext)) return; + return yield* new GitManagerError({ + operation: "runStackedAction", + cwd, + detail: `${baseBranch} is the base branch. Check out a feature branch to create ${changeRequestLabel}.`, + }); + }); + const runPrStep = Effect.fn("runPrStep")(function* ( settings: SourceControlTextGenerationSettings, cwd: string, @@ -2349,6 +2387,7 @@ export const make = Effect.gen(function* () { ...(threadContext ? { threadContext } : {}), ...(policy ? { policy } : {}), modelSelection: settings.modelSelection, + projectId: settings.projectId, }); const bodyFile = path.join( @@ -2714,6 +2753,9 @@ export const make = Effect.gen(function* () { repositoryKey: pullRequestRepositoryKey(latest.url), }; }); + + const checkedOutPullRequestKey = (branch: string) => `branch.${branch}.t3-pull-request`; + const branchPullRequest: GitManager["Service"]["branchPullRequest"] = Effect.fn( "branchPullRequest", )(function* ({ cwd, branch }, options) { @@ -2758,6 +2800,16 @@ export const make = Effect.gen(function* () { } return confirmed; }); + + const checkedOutPullRequest: GitManager["Service"]["checkedOutPullRequest"] = Effect.fn( + "checkedOutPullRequest", + )(function* ({ cwd, branch, url }) { + const checkedOutUrl = yield* gitCore + .readConfigValue(cwd, checkedOutPullRequestKey(branch)) + .pipe(Effect.orElseSucceed(() => null)); + return checkedOutUrl === url; + }); + const invalidateLocalStatus: GitManager["Service"]["invalidateLocalStatus"] = Effect.fn( "invalidateLocalStatus", )(function* (cwd) { @@ -3093,7 +3145,25 @@ export const make = Effect.gen(function* () { worktreePath: worktree.worktree.path, isOnPullRequestHead: true, }; - }).pipe(Effect.ensuring(invalidateStatus(input.cwd))); + }).pipe( + Effect.tap((result) => + gitCore + .writeConfigValue({ + cwd: input.cwd, + key: checkedOutPullRequestKey(result.branch), + value: result.pullRequest.url, + }) + .pipe( + Effect.catch((cause) => + Effect.logWarning("GitManager.preparePullRequestThread could not mark the branch", { + branch: result.branch, + cause, + }), + ), + ), + ), + Effect.ensuring(invalidateStatus(input.cwd)), + ); }, asUserAction); const runFeatureBranchStep = Effect.fn("runFeatureBranchStep")(function* ( @@ -3195,6 +3265,7 @@ export const make = Effect.gen(function* () { settings.sourceControlWriterModelSelection === null ? Effect.succeed({ modelSelection: settings.textGenerationModelSelection, + projectId, style: settings.sourceControlWritingStyle, }) : providerRegistry.getProviders.pipe( @@ -3204,6 +3275,7 @@ export const make = Effect.gen(function* () { providers, projectId, ), + projectId, style: settings.sourceControlWritingStyle, })), ), @@ -3243,6 +3315,15 @@ export const make = Effect.gen(function* () { const currentBranch = branchStep.name ?? initialStatus.branch; const commitAction = isCommitAction(input.action) ? input.action : null; const changeRequestTerms = wantsPr ? yield* changeRequestTermsFor(input.cwd) : null; + // A stale or missing origin/HEAD hides the default branch from status, so check here + // before anything is committed or pushed to it. + if (changeRequestTerms && !input.featureBranch && initialStatus.branch) { + yield* ensurePrHeadIsNotBase( + input.cwd, + { ...initialStatus, branch: initialStatus.branch }, + changeRequestTerms.shortLabelWithArticle, + ); + } const commit = commitAction ? yield* Ref.set(currentPhase, Option.some("commit")).pipe( @@ -3341,6 +3422,7 @@ export const make = Effect.gen(function* () { remoteStatus, status, branchPullRequest, + checkedOutPullRequest, invalidateLocalStatus, invalidateRemoteStatus, invalidateStatus, diff --git a/apps/server/src/git/GitWorkflowService.ts b/apps/server/src/git/GitWorkflowService.ts index 50731b82cce8..6222cab58bff 100644 --- a/apps/server/src/git/GitWorkflowService.ts +++ b/apps/server/src/git/GitWorkflowService.ts @@ -38,7 +38,7 @@ import { withWorkspaceLease } from "../workspace/workspaceLease.ts"; export class GitWorkflowService extends Context.Service< GitWorkflowService, { - readonly isRepository: (cwd: string) => Effect.Effect; + readonly repositoryRoot: (cwd: string) => Effect.Effect; readonly hasCommit: (input: { readonly cwd: string; readonly refName: string; @@ -68,6 +68,8 @@ export class GitWorkflowService extends Context.Service< readonly localRefStatus: ( input: VcsStatusInput, ) => Effect.Effect; + /** The checkout's own HEAD file, inside a linked worktree's admin directory when there is one. */ + readonly headPath: (cwd: string) => Effect.Effect; readonly pullCurrentBranch: (cwd: string) => Effect.Effect; readonly runStackedAction: ( input: GitRunStackedActionInput, @@ -279,13 +281,13 @@ export const make = Effect.gen(function* () { ensureGit(operation, input.cwd).pipe(Effect.andThen(run(input))); return GitWorkflowService.of({ - isRepository: Effect.fn("GitWorkflowService.isRepository")(function (cwd) { + repositoryRoot: Effect.fn("GitWorkflowService.repositoryRoot")(function (cwd) { return registry.detect({ cwd }).pipe( - Effect.map((handle) => handle?.kind === "git"), + Effect.map((handle) => (handle?.kind === "git" ? handle.repository.rootPath : null)), Effect.mapError( (cause) => new GitManagerError({ - operation: "GitWorkflowService.isRepository", + operation: "GitWorkflowService.repositoryRoot", cwd, detail: "Failed to detect a VCS repository for this Git workflow.", cause, @@ -389,6 +391,19 @@ export const make = Effect.gen(function* () { Effect.andThen(git.refStatusLocal(input.cwd)), ); }), + headPath: Effect.fn("GitWorkflowService.headPath")(function (cwd) { + return git + .execute({ + operation: "GitWorkflowService.headPath", + cwd, + args: ["rev-parse", "--path-format=absolute", "--git-path", "HEAD"], + allowNonZeroExit: true, + }) + .pipe( + Effect.map((result) => (result.exitCode === 0 ? result.stdout.trim() || null : null)), + Effect.orElseSucceed(() => null), + ); + }), pullCurrentBranch: Effect.fn("GitWorkflowService.pullCurrentBranch")(function (cwd) { return ensureGitCommand("GitWorkflowService.pullCurrentBranch", cwd).pipe( Effect.andThen(git.pullCurrentBranch(cwd)), diff --git a/apps/server/src/mcp/McpToolAccess.ts b/apps/server/src/mcp/McpToolAccess.ts index b82d9bb5ae59..f96a38d6d734 100644 --- a/apps/server/src/mcp/McpToolAccess.ts +++ b/apps/server/src/mcp/McpToolAccess.ts @@ -46,6 +46,7 @@ const refuseOutsideConstruction = (token: symbol) => { let declare: ( handle: (params: P) => Effect.Effect, remote: RemoteCheck

| undefined, + forwarded?: Forwarded, ) => Declaration<(params: P) => Effect.Effect>; /** @@ -58,6 +59,16 @@ type RemoteCheck

= ( ) => Effect.Effect>; let remoteCheckOf: (declaration: Declaration) => RemoteCheck | undefined; +/** What a call answered in a linked environment does here before it returns. */ +type Forwarded = ( + params: P, + environmentId: EnvironmentId, + result: A, +) => Effect.Effect; +let forwardedOf: ( + declaration: Declaration, +) => Forwarded | undefined; + type CheckedHandlerOf = D extends Declaration ? Handler : never; interface CheckedHandler extends Struct.Lambda { @@ -74,19 +85,29 @@ let handlersLayer: , EX, RX>( export class Declaration { readonly #handle: Handler; readonly #remote: RemoteCheck | undefined; + readonly #forwarded: Forwarded | undefined; private constructor( token: typeof builtHere, handle: Handler, remote: RemoteCheck | undefined, + forwarded: Forwarded | undefined, ) { refuseOutsideConstruction(token); this.#handle = handle; this.#remote = remote; + this.#forwarded = forwarded; } static { - declare = (handle, remote) => new Declaration(builtHere, handle, remote); + declare = (handle, remote, forwarded) => + new Declaration( + builtHere, + handle, + remote, + forwarded as Forwarded | undefined, + ); checkedHandler = Struct.lambda((declaration) => declaration.#handle); remoteCheckOf = (declaration) => declaration.#remote; + forwardedOf = (declaration) => declaration.#forwarded; } } @@ -281,6 +302,20 @@ export const either = ( ); }; +/** + * `declaration`, whose result from a linked environment then goes through + * `forwarded` here, for what this environment records about the call. + */ +export const afterForwarding = ( + declaration: Declaration<(params: P) => Effect.Effect>, + forwarded: Forwarded, +) => + declare( + checkedHandler(declaration) as (params: P) => Effect.Effect, + remoteCheckOf(declaration) as RemoteCheck

| undefined, + forwarded, + ); + const fullAccessRequired = "Changing projects or environment settings needs a live full-access/default calling thread or a full-access client."; @@ -387,6 +422,9 @@ const routed = Effect.fn("McpToolAccess.routed")(function* , never> + | undefined; if (remote === undefined) return yield* Effect.die( `${name} takes a linked environment, but it acts as its calling thread, which is here.`, @@ -402,7 +440,11 @@ const routed = Effect.fn("McpToolAccess.routed")(function* )(params); if (Option.isNone(forwarding)) return yield* linksUnavailable; - return yield* forwarding.value.call(scope, tool, target, { ...params, environmentId }); + const result = yield* forwarding.value.call(scope, tool, target, { + ...params, + environmentId, + }); + return forwarded === undefined ? result : yield* forwarded(params, target, result); }); } return handlers as Toolkit.HandlersFrom; diff --git a/apps/server/src/mcp/OrchestratorMcpService.test.ts b/apps/server/src/mcp/OrchestratorMcpService.test.ts index dafd8003f221..6395d37e7b38 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.test.ts @@ -92,15 +92,6 @@ describe("OrchestratorMcpService", () => { const parentProjection = { thread: { id: parentThreadId }, runs: [], - contextTransfers: [ - { - id: resultTransferId, - type: "subagent_result", - sourceThreadId: childThreadId, - targetThreadId: parentThreadId, - sourcePoint: { runId: childRunId }, - }, - ], subagents: [ { id: taskId, @@ -132,7 +123,15 @@ describe("OrchestratorMcpService", () => { completedAt: DateTime.makeUnsafe("2026-10-03T10:05:00Z"), }, ], - contextTransfers: [], + contextTransfers: [ + { + id: resultTransferId, + type: "subagent_result", + sourceThreadId: childThreadId, + targetThreadId: parentThreadId, + sourcePoint: { runId: childRunId }, + }, + ], messages: [ { id: MessageId.make("message:mcp-ack-result"), @@ -161,6 +160,7 @@ describe("OrchestratorMcpService", () => { return hasNestedWork ? { ...childProjection, + contextTransfers: [], subagents: [ { ...parentProjection.subagents[0]!, status: "running" as const }, ], @@ -170,7 +170,6 @@ describe("OrchestratorMcpService", () => { const snapshot = hasNestedWork ? { ...parentProjection, - contextTransfers: [], subagents: parentProjection.subagents.map((task) => ({ ...task, result: null, @@ -596,6 +595,8 @@ describe("OrchestratorMcpService", () => { listAppOwnedChildThreadIds: () => Effect.succeed([]), delegatedTaskResultPending: () => Effect.succeed(false), stopDelegatedTasks: () => Effect.void, + cancelDelegatedTask: ({ command }) => + Ref.update(dispatched, (commands) => [...commands, command]), // The orchestrator's subagent.stop repair is covered in ThreadStop.test.ts. dispatch: (command) => Ref.update(dispatched, (commands) => [...commands, command]).pipe( @@ -699,6 +700,12 @@ describe("OrchestratorMcpService", () => { Ref.update(dispatched, (commands) => [...commands, command]).pipe( Effect.andThen(Effect.fail(new Error("simulated stop failure") as never)), ), + stopDelegatedTasks: () => Effect.void, + cancelDelegatedTask: ({ command, stopDescendants }) => + stopDescendants(() => Effect.die("No remote descendants in this test")).pipe( + Effect.andThen(Ref.update(dispatched, (commands) => [...commands, command])), + Effect.andThen(Effect.fail(new Error("simulated stop failure") as never)), + ), }), ServerSettings.layerTest(), Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }), @@ -788,6 +795,10 @@ describe("OrchestratorMcpService", () => { ), ), stopDelegatedTasks: () => Effect.void, + cancelDelegatedTask: ({ command, stopDescendants }) => + stopDescendants(() => Effect.die("No remote descendants in this test")).pipe( + Effect.andThen(Ref.update(dispatched, (commands) => [...commands, command])), + ), }), ServerSettings.layerTest(), Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }), @@ -878,6 +889,10 @@ describe("OrchestratorMcpService", () => { ), ), stopDelegatedTasks: () => Effect.void, + cancelDelegatedTask: ({ command, stopDescendants }) => + stopDescendants(() => Effect.die("No remote descendants in this test")).pipe( + Effect.andThen(Ref.update(dispatched, (commands) => [...commands, command])), + ), }), ServerSettings.layerTest(), Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }), @@ -1043,6 +1058,10 @@ describe("OrchestratorMcpService", () => { Ref.update(dispatched, (commands) => [...commands, command]).pipe( Effect.as({} as never), ), + cancelDelegatedTask: ({ command, stopDescendants }) => + stopDescendants(() => Effect.die("No remote descendants in this test")).pipe( + Effect.andThen(Ref.update(dispatched, (commands) => [...commands, command])), + ), stopDelegatedTasks: () => Effect.gen(function* () { const limit = yield* DispatchModeLimit; @@ -1784,6 +1803,96 @@ describe("OrchestratorMcpService provider resolution", () => { ); }); + it.effect.each(["already advertised", "newly advertised", "still unavailable"] as const)( + "rechecks a delegated model that is %s", + (availability) => + Effect.gen(function* () { + const museInstanceId = ProviderInstanceId.make("muse"); + const model = "muse-wonder-1.0"; + const provider = (advertisedModel: string) => + providerSnapshot({ + instanceId: museInstanceId, + driver: ProviderDriverKind.make("muse"), + model: advertisedModel, + }); + const commands: Array = []; + const refreshedInstances: Array = []; + const task = { + id: taskId, + threadId: parentThreadId, + origin: "app_owned", + childThreadId, + providerInstanceId: museInstanceId, + model, + status: "running", + result: null, + }; + const dependencies = Layer.mergeAll( + NodeServices.layer, + NodeCrypto.layer, + Layer.mock(ThreadManagementService.ThreadManagementService)({ + hasReceipt: () => Effect.succeed(false), + getThreadRecords: (threadId) => + Effect.succeed( + threadId === parentThreadId + ? parentProjection(commands.length > 0 ? [task] : []) + : childProjection, + ), + dispatch: (command) => + Effect.sync(() => { + commands.push(command); + return { + sequence: 1, + storedEvents: [{ event: { type: "subagent.updated", payload: task } }], + } as never; + }), + }), + ServerSettings.layerTest(), + Layer.mock(ProviderRegistry.ProviderRegistry)({ + getProviders: Effect.succeed([ + provider(availability === "already advertised" ? model : "muse-spark-1.3"), + ]), + refreshInstance: (instanceId) => + Effect.sync(() => { + refreshedInstances.push(instanceId); + return [provider(availability === "still unavailable" ? "muse-spark-1.3" : model)]; + }), + }), + adapterRegistryLayer([museInstanceId]), + Layer.mock(ProjectService.ProjectService)({}), + Layer.mock(SecretRequests.SecretRequests)({}), + Layer.mock(RemoteDelegation.RemoteDelegation)({}), + Layer.mock(ScheduledTaskService.ScheduledTaskService)({}), + ); + yield* Effect.gen(function* () { + const service = yield* OrchestratorMcpService.OrchestratorMcpService; + const delegation = service.delegateTask(scope, { + task: "Review the change.", + target: { providerInstanceId: museInstanceId, model }, + mode: "async", + clientRequestId: availability, + }); + if (availability === "still unavailable") { + const error = yield* delegation.pipe(Effect.flip); + assert.equal(error.code, "model_unavailable"); + assert.deepEqual(commands, []); + } else { + const result = yield* delegation; + assert.equal(result.model, model); + assert.equal(commands.length, 1); + assert.deepEqual((commands[0] as { modelSelection: unknown }).modelSelection, { + instanceId: museInstanceId, + model, + }); + } + assert.deepEqual( + refreshedInstances, + availability === "already advertised" ? [] : [museInstanceId], + ); + }).pipe(Effect.provide(layerOrchestratorService.pipe(Layer.provide(dependencies)))); + }), + ); + it.effect("rejects delegation to a provider without a registered adapter", () => Effect.gen(function* () { const forkOnlyInstanceId = ProviderInstanceId.make("forkOnly"); @@ -2243,6 +2352,157 @@ describe("OrchestratorMcpService provider resolution", () => { ), ); + it.effect.each(["local", "remote"] as const)( + "a blocking %s delegation cancelled mid-wait wakes its parent like an async task", + (placement) => + Effect.gen(function* () { + const driver = ProviderDriverKind.make("codex"); + const layerRegistry = ProviderAdapterRegistry.layerFromAdapters([ + { + instanceId: codexInstanceId, + driver, + getCapabilities: () => Effect.succeed(CodexProviderCapabilitiesV2), + planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" as const }), + openSession: () => Effect.die("The child never starts: the effect worker is off."), + }, + ]); + const layerDatabase = SqlitePersistence.layerMemory; + const layerOrchestrator = ProviderReplayHarness.layerWithRegistry( + { name: "mcp-wait-cancelled" }, + layerRegistry, + { databaseLayer: layerDatabase, runEffectWorker: false }, + ); + const layerSettings = ServerSettings.layerTest({ + providerInstances: { [codexInstanceId]: { driver: "codex", enabled: true } }, + }); + const layerThreads = ThreadManagementService.layer.pipe( + Layer.provideMerge(layerOrchestrator), + Layer.provide( + LegacyV1ThreadImporter.layer.pipe( + Layer.provide(layerOrchestrator), + Layer.provide(layerDatabase), + ), + ), + Layer.provide(CommandReceiptStore.layer.pipe(Layer.provide(layerDatabase))), + Layer.provideMerge(layerSettings), + ); + const waiting = yield* Deferred.make(); + const layerWatchedThreads = Layer.effect( + ThreadManagementService.ThreadManagementService, + Effect.gen(function* () { + const threads = yield* ThreadManagementService.ThreadManagementService; + return { + ...threads, + streamStoredEventsFrom: (input) => + Stream.unwrap( + Deferred.succeed(waiting, undefined).pipe( + Effect.as(threads.streamStoredEventsFrom(input)), + ), + ), + } satisfies ThreadManagementService.ThreadManagementService["Service"]; + }), + ).pipe(Layer.provideMerge(layerThreads)); + const remoteChild = { + environmentId: EnvironmentId.make("environment:mcp-providers-box"), + threadId: ThreadId.make("thread:mcp-providers-on-the-box"), + label: "Box", + }; + // Records the task as the real delegation does once the linked environment launched it. + const layerRemoteDelegation = Layer.unwrap( + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + return Layer.mock(RemoteDelegation.RemoteDelegation)({ + delegate: (_scope, input) => + Effect.gen(function* () { + const run = (yield* orchestrator.getThreadProjection(parentThreadId)).runs[0]!; + yield* orchestrator.dispatch({ + type: "delegated_task.remote.request", + commandId: CommandId.make("command:remote-delegate:wait-cancelled"), + parentThreadId, + parentRunId: run.id, + parentNodeId: run.rootNodeId!, + task: input.task, + driver: ProviderDriverKind.make("remote"), + modelSelection: run.modelSelection, + remoteChild, + completionWake: "settled_only", + }); + const parent = yield* orchestrator.getThreadProjection(parentThreadId); + return { taskId: parent.subagents[0]!.id }; + }).pipe(Effect.orDie), + }); + }), + ).pipe(Layer.provide(layerWatchedThreads)); + const layerTest = layerOrchestratorService.pipe( + Layer.provideMerge(layerWatchedThreads), + Layer.provide( + Layer.mergeAll( + NodeServices.layer, + layerRegistry, + layerSettings, + providerRegistryLayer([ + providerSnapshot({ instanceId: codexInstanceId, driver, model: "gpt-5.4" }), + ]), + Layer.mock(ProjectService.ProjectService)({}), + Layer.mock(SecretRequests.SecretRequests)({}), + layerRemoteDelegation, + Layer.mock(ScheduledTaskService.ScheduledTaskService)({}), + ), + ), + ); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const service = yield* OrchestratorMcpService.OrchestratorMcpService; + yield* orchestrator.dispatch({ + type: "thread.create", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make("parent:create"), + threadId: parentThreadId, + projectId, + title: "MCP parent", + modelSelection: { instanceId: codexInstanceId, model: "gpt-5.4" }, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make("parent:start"), + threadId: parentThreadId, + messageId: MessageId.make("parent:message"), + text: "Delegate a review and wait for it.", + attachments: [], + dispatchMode: { type: "start_immediately" }, + }); + const call = yield* service + .delegateTask(scope, { + task: "Review the change.", + ...(placement === "remote" + ? { target: { environmentId: remoteChild.environmentId } } + : {}), + mode: "wait", + clientRequestId: "delegate-wait-cancelled", + }) + .pipe(Effect.forkChild); + yield* Deferred.await(waiting); + // A steered Claude parent cancels the tools/call, which interrupts this fiber. + yield* Fiber.interrupt(call); + const parent = yield* orchestrator.getThreadProjection(parentThreadId); + const task = parent.subagents.find((subagent) => subagent.origin === "app_owned"); + assert.equal(task?.status, "running"); + assert.equal( + task?.completionWake, + "always", + "a cancelled blocking delegation left the task settled_only, so its result never reaches the still-live parent run", + ); + }).pipe(Effect.provide(layerTest)); + }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), + ); + it.effect( "inherits an available parent instance for driver-only targets and otherwise selects a healthy peer", () => @@ -2913,7 +3173,13 @@ describe("OrchestratorMcpService provider resolution", () => { Effect.gen(function* () { const saved = yield* Ref.make>([]); const mcp = yield* OrchestratorMcpService.OrchestratorMcpService.pipe( - Effect.provide(service(masterRepository, saved)), + Effect.provide( + service(masterRepository, saved, { + ...existing, + threadId: ThreadId.make("thread:scheduled-base"), + threadMode: "continue", + }), + ), ); yield* mcp.updateScheduledTask(client, { scheduledTaskId: existing.id, @@ -2925,6 +3191,20 @@ describe("OrchestratorMcpService provider resolution", () => { }), ); + it.effect("keeps the saved workspace when an unbound task stays unbound", () => + Effect.gen(function* () { + const saved = yield* Ref.make>([]); + const mcp = yield* OrchestratorMcpService.OrchestratorMcpService.pipe( + Effect.provide(service(masterRepository, saved)), + ); + yield* mcp.updateScheduledTask(client, { + scheduledTaskId: existing.id, + bindToCurrentThread: false, + }); + assert.deepEqual(yield* Ref.get(saved), [{ type: "root" }]); + }), + ); + it.effect.each([ { name: "no remote, only master", @@ -3143,6 +3423,17 @@ describe("OrchestratorMcpService provider resolution", () => { target: { options: {} }, expected: { ...inherited, options: [] }, }, + { + name: "same model by alias", + target: { model: "GPT" }, + expected: { ...inherited, model: "GPT" }, + }, + { + name: "parent named the same model by alias", + parent: { ...inherited, model: "gpt" }, + target: { model: inherited.model }, + expected: inherited, + }, { name: "change model", target: { model: "other-model" }, @@ -3187,7 +3478,10 @@ describe("OrchestratorMcpService provider resolution", () => { getThreadRecords: (threadId) => Effect.succeed( threadId === parentThreadId - ? parentProjection(commands.length > 0 ? [task] : [], inherited) + ? parentProjection( + commands.length > 0 ? [task] : [], + "parent" in testCase ? testCase.parent : inherited, + ) : childProjection, ), dispatch: (command) => @@ -3213,6 +3507,7 @@ describe("OrchestratorMcpService provider resolution", () => { name: model, isCustom: false, capabilities: null, + ...(model === inherited.model ? { aliases: ["gpt"] } : {}), })), { slug: "ultracode-model", @@ -3354,6 +3649,59 @@ describe("OrchestratorMcpService provider resolution", () => { }), ); + it.effect("validates options against the model a target names by alias", () => + Effect.gen(function* () { + const dependencies = Layer.mergeAll( + NodeServices.layer, + NodeCrypto.layer, + Layer.mock(ThreadManagementService.ThreadManagementService)({ + hasReceipt: () => Effect.succeed(false), + assertProviderInstanceEnabled: () => Effect.void, + getThreadRecords: (threadId) => + Effect.succeed(threadId === parentThreadId ? parentProjection([]) : childProjection), + dispatch: () => Effect.die("an invalid target must not dispatch"), + }), + ServerSettings.layerTest(), + providerRegistryLayer([ + { + ...providerSnapshot({ + instanceId: codexInstanceId, + driver: ProviderDriverKind.make("codex"), + }), + models: [ + { + slug: "gpt-5.4", + name: "gpt-5.4", + aliases: ["gpt"], + isCustom: false, + capabilities: { + optionDescriptors: [{ id: "fastMode", label: "Fast", type: "boolean" }], + }, + }, + ], + }, + ]), + adapterRegistryLayer([codexInstanceId]), + Layer.mock(ProjectService.ProjectService)({}), + Layer.mock(SecretRequests.SecretRequests)({}), + Layer.mock(RemoteDelegation.RemoteDelegation)({}), + Layer.mock(ScheduledTaskService.ScheduledTaskService)({}), + ); + const error = yield* Effect.gen(function* () { + const service = yield* OrchestratorMcpService.OrchestratorMcpService; + return yield* service + .delegateTask(scope, { + task: "Review the change.", + target: yield* decodeTarget({ model: "GPT", options: { bogus: "x" } }), + mode: "async", + }) + .pipe(Effect.flip); + }).pipe(Effect.provide(layerOrchestratorService.pipe(Layer.provide(dependencies)))); + assert.equal(error.code, "invalid_request"); + assert.include(error.message, "Unknown option bogus"); + }), + ); + describe("resolveRuntimeMode", () => { const muse = { modelSelection: { instanceId: ProviderInstanceId.make("muse"), model: "muse-model" }, @@ -3584,6 +3932,15 @@ describe("OrchestratorMcpService thread list", () => { ); }).pipe(Effect.provide(service)), ); + + it.effect("points a caller outside any thread at allProjects when it omits projectId", () => + Effect.gen(function* () { + const mcp = yield* OrchestratorMcpService.OrchestratorMcpService; + const error = yield* mcp.listThreads(client, {}).pipe(Effect.flip); + assert.equal(error.code, "target_required"); + assert.include(error.message, "allProjects"); + }).pipe(Effect.provide(service)), + ); }); describe("OrchestratorMcpService thread stop", () => { @@ -3673,9 +4030,17 @@ describe("OrchestratorMcpService thread stop", () => { }); it.effect.each([ - { name: "a running turn", queued: false }, - { name: "a running turn with a queued message", queued: true }, - ])("t3_thread_interrupt stop reports the run it interrupted on $name", ({ queued }) => + { + name: "a running turn", + queued: false, + stop: { reason: undefined, message: "Stopped by an agent" }, + }, + { + name: "a running turn with a queued message", + queued: true, + stop: { reason: "Make room", message: "Stopped by an agent: Make room" }, + }, + ])("t3_thread_interrupt stop reports the run it interrupted on $name", ({ queued, stop }) => Effect.scoped( Effect.gen(function* () { const cwd = yield* checkpointWorkspace("mcp-thread-stop"); @@ -3753,17 +4118,61 @@ describe("OrchestratorMcpService thread stop", () => { const before = yield* orchestrator.getThreadProjection(threadId); const runningRun = before.runs.find((run) => run.status === "running")!; assert.equal(before.runs.filter((run) => run.status === "queued").length, queued ? 1 : 0); + // A task the thread delegated, running when the thread is stopped. + const childRunning = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "provider-turn.updated" && + event.threadId !== threadId && + event.payload.status === "running", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + yield* orchestrator.dispatch({ + type: "delegated_task.request", + commandId: CommandId.make("delegate"), + parentThreadId: threadId, + parentRunId: runningRun.id, + parentNodeId: runningRun.rootNodeId!, + task: "child work", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + completionWake: "always", + createdBy: "agent", + creationSource: "mcp", + }); + yield* worker.drain(); + yield* Fiber.join(childRunning); + const childThreadId = (yield* orchestrator.getThreadProjection(threadId)).subagents[0]! + .childThreadId!; const stopped = yield* service.interruptThread(client, { threadId, stop: true, clientRequestId: "stop", + ...(stop.reason === undefined ? {} : { reason: stop.reason }), }); assert.deepEqual(stopped, { threadId, runId: runningRun.id, status: "interrupt_requested", }); + // An agent's Stop must not read as the user's, on its thread or a task it delegated. + const interruptMessages = (id: ThreadId) => + orchestrator + .getThreadProjection(id) + .pipe( + Effect.map(({ turnItems }) => + turnItems.flatMap((item) => + item.type === "run_interrupt_request" ? [item.message] : [], + ), + ), + ); + assert.deepEqual(yield* interruptMessages(threadId), [stop.message]); + assert.deepEqual(yield* interruptMessages(childThreadId), [stop.message]); }).pipe(Effect.provide(layerTest)); }), ), diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index a28ec1cd1674..f62d1da293a3 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -100,7 +100,7 @@ import type { ProjectionRecordField, ProjectionRecordFilter, } from "../orchestration-v2/ProjectionStore.ts"; -import { runOutcome } from "../orchestration-v2/RestartBackgroundNote.ts"; +import { delegatedTaskResultRun, runOutcome } from "../orchestration-v2/RestartBackgroundNote.ts"; import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts"; import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts"; import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; @@ -346,6 +346,23 @@ function capabilityModelOption(descriptor: ProviderOptionDescriptor): Orchestrat }; } +const advertisedModel = (provider: ServerProvider, model: string) => + provider.models.find((candidate) => candidate.slug === model) ?? + provider.models.find((candidate) => + candidate.aliases?.some((alias) => alias.toLowerCase() === model.toLowerCase()), + ); + +/** A provider that lists no models accepts any slug. */ +export const requireAdvertisedModel = (provider: ServerProvider, model: string) => + provider.models.length > 0 && advertisedModel(provider, model) === undefined + ? Effect.fail( + failure( + "model_unavailable", + `Model ${model} is not advertised by provider ${provider.instanceId}.`, + ), + ) + : Effect.void; + function providerConstraints( provider: ServerProvider | undefined, supportsOrchestrationV2: boolean, @@ -579,8 +596,9 @@ function directAppOwnedChildTasks( ); } +// The child's transfers include the results it sent, so the parent's transfers are never read. function taskResultTransfer( - parent: Pick, + parentThreadId: ThreadId, child: Pick, task: OrchestrationV2Subagent, ) { @@ -595,18 +613,18 @@ function taskResultTransfer( .map((run) => run.id), ); if (originalRun !== undefined) resultRunIds.add(originalRun.id); - return parent.contextTransfers.find( + return child.contextTransfers.find( (transfer) => transfer.type === "subagent_result" && transfer.sourceThreadId === task.childThreadId && - transfer.targetThreadId === parent.thread.id && + transfer.targetThreadId === parentThreadId && ((transfer.sourcePoint.runId !== undefined && resultRunIds.has(transfer.sourcePoint.runId)) || (transfer.sourcePoint.runId === undefined && originalRun?.delegatedTaskId === undefined)), ); } function pageIncludesTerminalTaskResult(input: { - readonly parent: Pick; + readonly parentThreadId: ThreadId; readonly page: ReadonlyArray; readonly task: OrchestrationV2Subagent; readonly target: Pick< @@ -615,7 +633,7 @@ function pageIncludesTerminalTaskResult(input: { >; readonly maxChars: number; }): boolean { - const transfer = taskResultTransfer(input.parent, input.target, input.task); + const transfer = taskResultTransfer(input.parentThreadId, input.target, input.task); if (transfer === undefined) return false; const run = transfer.sourcePoint.runId === undefined @@ -964,8 +982,8 @@ function turnItemText(item: OrchestrationV2TurnItem): string | null { return `Asked the user for ${item.label}: ${item.secretStatus}.`; case "link_request": return item.linkStatus === "linked" - ? `Linked ${item.linkedLabel ?? item.label ?? item.url ?? "an environment"} (${item.linkedAccess}).` - : `Asked the user to link ${item.label ?? item.url ?? "an environment"}: ${item.linkStatus}.`; + ? `Linked ${item.linkedLabel ?? item.url ?? "an environment"} (${item.linkedAccess}).` + : `Asked the user to link ${item.url ?? "an environment"}: ${item.linkStatus}.`; case "subagent": return item.result ?? item.progress ?? item.prompt; case "dynamic_tool": @@ -1088,20 +1106,11 @@ const make = Effect.gen(function* () { ), ); - const loadProjection = (threadId: ThreadId, includeUserMessages = true) => - loadThreadRecords( - threadId, - [ - "runs", - "messages", - "providerThreads", - "providerTurns", - "runtimeRequests", - "contextTransfers", - "turnItems", - ], - { turnItemTypes: [], messageRoles: includeUserMessages ? ["user"] : [] }, - ); + // Runs on every MCP call, and long threads hold thousands of records; callers need only active runs. + const loadCallerRecords = (threadId: ThreadId, fields: ReadonlyArray) => + loadThreadRecords(threadId, fields, { + runStatuses: ThreadManagementService.ACTIVE_RUN_STATUSES, + }); // A dispatcher thread holds thousands of tasks, each with its full prompt; read only the ones named. const loadTasks = (threadId: ThreadId, taskIds: ReadonlyArray) => @@ -1112,43 +1121,9 @@ const make = Effect.gen(function* () { ), ); - // Task waits read on every wake of a parent that may hold thousands of - // tasks, so they read only the task they watch and what its status needs. - const loadTaskParent = (threadId: ThreadId) => - threadManagement - .getThreadRecords(threadId, ["runs", "contextTransfers"]) - .pipe( - Effect.mapError((error) => - failure( - "orchestration_error", - `Unable to read thread ${threadId}: ${errorMessage(error)}`, - ), - ), - ); - - const loadProjectThread = ( - projectId: OrchestrationV2ThreadProjection["thread"]["projectId"], - threadId: ThreadId, - ) => - threadManagement - .getProjectThreadRecords( - { projectId, threadId }, - [ - "runs", - "messages", - "subagents", - "providerThreads", - "providerTurns", - "runtimeRequests", - "contextTransfers", - "turnItems", - ], - { turnItemTypes: [], messageRoles: ["user"] }, - ) - .pipe(Effect.mapError(threadManagementFailure)); - - const loadCaller = Effect.fn("OrchestratorMcpService.loadCaller")(function* ( + const loadCaller = Effect.fn("OrchestratorMcpService.loadCaller")(function* ( scope: McpInvocationScope, + fields: ReadonlyArray, ) { yield* requireCapability(scope); if (scope.thread === undefined) { @@ -1157,8 +1132,7 @@ const make = Effect.gen(function* () { limits: clientModeCeiling(scope.client), } as const; } - // No loadCaller consumer reads the caller's own prompts. - const parent = yield* loadProjection(scope.thread.threadId, false); + const parent = yield* loadCallerRecords(scope.thread.threadId, fields); return { parent, limits: { @@ -1174,28 +1148,23 @@ const make = Effect.gen(function* () { ) { yield* requireCapability(scope); const threadScope = yield* requireThreadScope(scope, operation); - const parent = yield* loadProjection(threadScope.thread.threadId); + const parent = yield* loadCallerRecords(threadScope.thread.threadId, ["runs"]); return { scope: threadScope, parent } as const; }); const resolveProjectTarget = ( parent: Pick | undefined, projectId: ProjectId | undefined, + missingMessage = "Pass projectId: this MCP client is not running inside a T3 thread.", ) => projectId !== undefined ? Effect.succeed(projectId) : parent !== undefined ? Effect.succeed(parent.thread.projectId) - : Effect.fail( - failure( - "target_required", - "Pass projectId: this MCP client is not running inside a T3 thread.", - ), - ); + : Effect.fail(failure("target_required", missingMessage)); const loadTargetThread = Effect.fn("OrchestratorMcpService.loadTargetThread")(function* ( threadId: ThreadId, - includeHistory = true, ) { const shell = yield* threadManagement .getThreadShell(threadId) @@ -1203,24 +1172,21 @@ const make = Effect.gen(function* () { if (shell === null || shell.deletedAt !== null) { return yield* threadNotFound(threadManagement, threadId, `Thread ${threadId} was not found.`); } - // Native wait owns its history reads; keep legacy import and project validation here. - return yield* includeHistory - ? loadProjectThread(shell.projectId, threadId) - : threadManagement - .getProjectThreadRecords({ projectId: shell.projectId, threadId }, []) - .pipe(Effect.mapError(threadManagementFailure)); + // Keeps legacy import and project validation; callers read only the thread row. + return yield* threadManagement + .getProjectThreadRecords({ projectId: shell.projectId, threadId }, []) + .pipe(Effect.mapError(threadManagementFailure)); }); - const loadScopedThread = Effect.fn("OrchestratorMcpService.loadScopedThread")(function* ( - scope: McpInvocationScope, - threadId: ThreadId, - includeHistory = true, - ) { - const caller = yield* loadCaller(scope); - const target = + const loadScopedThread = Effect.fn("OrchestratorMcpService.loadScopedThread")(function* < + K extends "runs", + >(scope: McpInvocationScope, threadId: ThreadId, callerFields: ReadonlyArray) { + const caller = yield* loadCaller(scope, callerFields); + // Only the thread row: the caller's own record holds just its active runs. + const target: Pick = caller.parent !== undefined && threadId === caller.parent.thread.id ? caller.parent - : yield* loadTargetThread(threadId, includeHistory); + : yield* loadTargetThread(threadId); return { ...caller, target } as const; }); @@ -1274,7 +1240,7 @@ const make = Effect.gen(function* () { scope: McpInvocationScope, threadId: ThreadId, ) { - const { parent } = yield* loadCaller(scope); + const { parent } = yield* loadCaller(scope, []); const shell = yield* threadManagement .getThreadShell(threadId) .pipe(Effect.mapError(threadManagementFailure)); @@ -1331,7 +1297,7 @@ const make = Effect.gen(function* () { if (instanceId === undefined) return resolved; return resolved.pipe( Effect.catchIf( - (error) => error.code === "provider_unavailable", + (error) => error.code === "provider_unavailable" || error.code === "model_unavailable", () => providerRegistry .refreshInstance(instanceId) @@ -1434,19 +1400,9 @@ const make = Effect.gen(function* () { `Provider ${instanceId} has no model available for inheritance.`, ); } - if ( - requestedModel !== undefined && - provider.models.length > 0 && - !provider.models.some((candidate) => candidate.slug === requestedModel) - ) { - return yield* failure( - "model_unavailable", - `Model ${requestedModel} is not advertised by provider ${instanceId}.`, - ); - } - - const descriptors = provider.models.find((candidate) => candidate.slug === model)?.capabilities - ?.optionDescriptors; + if (requestedModel !== undefined) yield* requireAdvertisedModel(provider, requestedModel); + const advertised = advertisedModel(provider, model); + const descriptors = advertised?.capabilities?.optionDescriptors; const requestedOptions = withLegacyUltracodeEffort(descriptors, input.target?.options); if (requestedOptions !== undefined) { const invalid = invalidOptionSelections(requestedOptions, descriptors); @@ -1459,7 +1415,9 @@ const make = Effect.gen(function* () { } const inheritsOptions = - instanceId === inheritedSelection.instanceId && model === inheritedSelection.model; + instanceId === inheritedSelection.instanceId && + (advertised?.slug ?? model) === + (advertisedModel(provider, inheritedSelection.model)?.slug ?? inheritedSelection.model); let options = inheritsOptions ? knownOptions( withLegacyUltracodeEffort(descriptors, inheritedSelection.options), @@ -1510,7 +1468,6 @@ const make = Effect.gen(function* () { if (task.childThreadId === null) { return yield* readRemoteTask(scope, task, waitTimedOut, acknowledgeTerminal); } - const parentProjection = yield* loadTaskParent(scope.thread.threadId); const childControls = yield* threadManagement .getThreadRecords( task.childThreadId, @@ -1533,9 +1490,7 @@ const make = Effect.gen(function* () { const progress = delegatedTaskProgress(childControls); const taskResultRun = childRun?.delegatedTaskId === task.id - ? childControls.runs - .filter((run) => run.delegatedTaskId === task.id) - .toSorted((a, b) => b.ordinal - a.ordinal)[0] + ? delegatedTaskResultRun(childControls.runs, task.id) : progress.resultRun; const taskWorkState = childRun?.delegatedTaskId === task.id @@ -1580,7 +1535,11 @@ const make = Effect.gen(function* () { : childRun, ) : workState === "result_available" - ? taskStatusForRun(taskResultRun ?? childRun) + ? // A cancelled task's finished turn settles it as interrupted. + task.cancelledRunId !== undefined && + taskStatusForRun(taskResultRun ?? childRun) === "completed" + ? "interrupted" + : taskStatusForRun(taskResultRun ?? childRun) : taskStatusForRun(taskResultRun ?? childRun) === "queued" ? "queued" : "running"; @@ -1590,7 +1549,7 @@ const make = Effect.gen(function* () { : taskResultRun !== undefined && isTerminalTaskStatus(status) ? subagentResultForRun(childProjection, taskResultRun).text : null; - const resultTransfers = parentProjection.contextTransfers.filter( + const resultTransfers = childControls.contextTransfers.filter( (transfer) => transfer.type === "subagent_result" && transfer.sourceThreadId === task.childThreadId && @@ -1604,7 +1563,7 @@ const make = Effect.gen(function* () { ? resultTransfers.find((transfer) => transfer.sourcePoint.runId === undefined) : undefined) ?? null); - const resultTransfer = taskResultTransfer(parentProjection, childProjection, task) ?? null; + const resultTransfer = taskResultTransfer(scope.thread.threadId, childProjection, task) ?? null; const terminalStatus = terminalRun === undefined ? null : taskStatusForRun(terminalRun); const latestTerminalSummary = canExposeTaskRunResult(terminalRun) ? terminalRun.id === childRun?.id @@ -1673,6 +1632,7 @@ const make = Effect.gen(function* () { task.completionDelivery?.state !== "acknowledged" && task.completionDelivery?.state !== "disposed" ) { + const parentProjection = yield* loadCallerRecords(scope.thread.threadId, ["runs"]); const observingRun = ThreadManagementService.latestActiveRun({ runs: parentProjection.runs.filter( (run) => run.providerInstanceId === scope.thread.providerInstanceId, @@ -1703,33 +1663,61 @@ const make = Effect.gen(function* () { return response; }); - /** `mode=wait` on a task already recorded, as `delegateTask` does for its own. */ + /** + * Blocks for `mode=wait`. A wait that ends without a terminal result, including an + * interrupted tools/call, upgrades the wake so a later result still reaches the parent. + */ const awaitTask = Effect.fn("OrchestratorMcpService.awaitTask")(function* ( scope: McpThreadInvocationScope, taskId: NodeId, + wakePolicyRequestKey: string, requestedTimeoutMs: number | undefined, ): Effect.fn.Return { const timeoutMs = Math.min( MAX_WAIT_TIMEOUT_MS, Math.max(1, requestedTimeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS), ); - const waited = yield* waitForTask(scope, taskId, timeoutMs); - if (Option.isSome(waited)) return waited.value; - // As for a local task: once the blocking wait ends, a later result wakes the parent. - yield* threadManagement + const upgradeWake = threadManagement .dispatch({ type: "delegated_task.wake-policy", commandId: stableCommandId({ scope, - requestKey: taskId, + requestKey: wakePolicyRequestKey, operation: "delegate-task-wake-policy", }), parentThreadId: scope.thread.threadId, taskId, completionWake: "always", }) - .pipe(Effect.ignoreCause({ log: true })); - return yield* readTask(scope, taskId, true, true); + .pipe( + Effect.catch((error) => + Effect.logWarning("orchestrator-mcp.delegate-task.wake-policy-failed", { + taskId, + outcome: + error._tag === "OrchestratorCommandPreviouslyRejectedError" + ? "previously_rejected" + : "dispatch_failed", + error, + }), + ), + Effect.catchCause((cause) => + Effect.logWarning("orchestrator-mcp.delegate-task.wake-policy-failed", { + taskId, + outcome: "defect", + cause, + }), + ), + ); + const delivered = (waited: Option.Option) => + Option.isSome(waited) && isTerminalTaskStatus(waited.value.status); + // The finalizer runs uninterruptibly, so a cancel right after the wait returns still upgrades. + const waited = yield* waitForTask(scope, taskId, timeoutMs).pipe( + Effect.onExit((exit) => + Exit.isSuccess(exit) && delivered(exit.value) ? Effect.void : upgradeWake, + ), + ); + if (Option.isSome(waited) && delivered(waited)) return waited.value; + return yield* readTask(scope, taskId, Option.isNone(waited), true); }); /** @@ -1758,17 +1746,7 @@ const make = Effect.gen(function* () { task.completionDelivery?.state !== "acknowledged" && task.completionDelivery?.state !== "disposed" ) { - const parentThreadId = scope.thread.threadId; - const parentProjection = yield* threadManagement - .getThreadRecords(parentThreadId, ["runs"]) - .pipe( - Effect.mapError((error) => - failure( - "orchestration_error", - `Unable to read thread ${parentThreadId}: ${errorMessage(error)}`, - ), - ), - ); + const parentProjection = yield* loadCallerRecords(scope.thread.threadId, ["runs"]); const observingRun = ThreadManagementService.latestActiveRun({ runs: parentProjection.runs.filter( (run) => run.providerInstanceId === scope.thread.providerInstanceId, @@ -2011,7 +1989,7 @@ const make = Effect.gen(function* () { return OrchestratorMcpService.of({ scheduleTask: Effect.fn("OrchestratorMcpService.scheduleTask")(function* (scope, input) { - const { parent, limits } = yield* loadCaller(scope); + const { parent, limits } = yield* loadCaller(scope, ["runs"]); const projectId = yield* resolveProjectTarget(parent, input.projectId); yield* assertLiveCallerForOtherProject(scope, parent, projectId); const project = yield* requireProject(projectId); @@ -2072,7 +2050,7 @@ const make = Effect.gen(function* () { }), listScheduledTasks: Effect.fn("OrchestratorMcpService.listScheduledTasks")( function* (scope, input) { - const { parent, limits } = yield* loadCaller(scope); + const { parent, limits } = yield* loadCaller(scope, ["runs"]); const projectId = input.projectId ?? parent?.thread.projectId; const { tasks } = yield* scheduledTasks .list() @@ -2091,7 +2069,7 @@ const make = Effect.gen(function* () { ), updateScheduledTask: Effect.fn("OrchestratorMcpService.updateScheduledTask")( function* (scope, input) { - const { parent, limits } = yield* loadCaller(scope); + const { parent, limits } = yield* loadCaller(scope, ["runs"]); const existing = yield* loadScheduledTask(input.scheduledTaskId, limits); yield* assertLiveCallerForOtherProject(scope, parent, existing.projectId); if ( @@ -2123,11 +2101,11 @@ const make = Effect.gen(function* () { : input.bindToCurrentThread && parent !== undefined ? parent.thread.id : null; - // Update workspace strategy on rebinding so unbound root tasks cannot run in the shared checkout. + // Reset the workspace only when the binding flips, so unbinding cannot leave runs in the shared checkout. const workspaceStrategy = - input.bindToCurrentThread === undefined + (threadId === null) === (existing.threadId === null) ? existing.workspaceStrategy - : yield* scheduledTaskWorkspaceStrategy(input.bindToCurrentThread, existing.projectId); + : yield* scheduledTaskWorkspaceStrategy(threadId !== null, existing.projectId); const upsertInput: ScheduledTaskUpsertInput = { id: existing.id, title: input.title ?? existing.title, @@ -2167,7 +2145,7 @@ const make = Effect.gen(function* () { ), deleteScheduledTask: Effect.fn("OrchestratorMcpService.deleteScheduledTask")( function* (scope, input) { - const { parent, limits } = yield* loadCaller(scope); + const { parent, limits } = yield* loadCaller(scope, ["runs"]); const existing = yield* loadScheduledTask(input.scheduledTaskId, limits); yield* assertLiveCallerForOtherProject(scope, parent, existing.projectId); yield* scheduledTasks @@ -2333,7 +2311,7 @@ const make = Effect.gen(function* () { }), capabilities: Effect.fn("OrchestratorMcpService.capabilities")(function* (scope, input = {}) { - const { parent, limits } = yield* loadCaller(scope); + const { parent, limits } = yield* loadCaller(scope, []); const providers = yield* loadProviders; const orchestrationCapableInstanceIds = yield* loadOrchestrationCapableInstanceIds(); const settings = yield* serverSettings.getSettings.pipe( @@ -2430,7 +2408,7 @@ const make = Effect.gen(function* () { target: { ...input.target, environmentId }, }); return yield* input.mode === "wait" - ? awaitTask(scope, taskId, input.timeoutMs) + ? awaitTask(scope, taskId, taskId, input.timeoutMs) : readTask(scope, taskId, false, true); } const { scope, parent } = yield* loadThreadCaller(callerScope, "delegate_task"); @@ -2528,8 +2506,8 @@ const make = Effect.gen(function* () { interactionMode, ...(workspaceStrategy === undefined ? {} : { workspaceStrategy }), // Async delegations wake the parent on every child terminal; wait - // delegations deliver through the blocking tool call, so a wake is - // only needed if the parent settled first (timeout, disconnect). + // delegations deliver through the blocking tool call until `awaitTask` + // gives up the wait (timeout, cancelled call) and upgrades the wake. completionWake: input.mode === "wait" ? "settled_only" : "always", }) .pipe( @@ -2569,51 +2547,9 @@ const make = Effect.gen(function* () { }); } - if (input.mode !== "wait") { - return yield* readTask(scope, taskId, false, true); - } - const timeoutMs = Math.min( - MAX_WAIT_TIMEOUT_MS, - Math.max(1, input.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS), - ); - const waited = yield* waitForTask(scope, taskId, timeoutMs); - if (Option.isSome(waited) && isTerminalTaskStatus(waited.value.status)) { - return waited.value; - } - // An incomplete wait gives up delivery: upgrade so a later terminal - // wakes the parent even mid-turn. - yield* threadManagement - .dispatch({ - type: "delegated_task.wake-policy", - commandId: stableCommandId({ - scope, - requestKey: key, - operation: "delegate-task-wake-policy", - }), - parentThreadId: scope.thread.threadId, - taskId, - completionWake: "always", - }) - .pipe( - Effect.catch((error) => - Effect.logWarning("orchestrator-mcp.delegate-task.wake-policy-failed", { - taskId, - outcome: - error._tag === "OrchestratorCommandPreviouslyRejectedError" - ? "previously_rejected" - : "dispatch_failed", - error, - }), - ), - Effect.catchCause((cause) => - Effect.logWarning("orchestrator-mcp.delegate-task.wake-policy-failed", { - taskId, - outcome: "defect", - cause, - }), - ), - ); - return yield* readTask(scope, taskId, Option.isNone(waited), true); + return yield* input.mode === "wait" + ? awaitTask(scope, taskId, key, input.timeoutMs) + : readTask(scope, taskId, false, true); }), taskStatus: Effect.fn("OrchestratorMcpService.taskStatus")(function* (callerScope, taskId) { const scope = yield* requireThreadScope(callerScope, "task_status"); @@ -2695,13 +2631,25 @@ const make = Effect.gen(function* () { const stopChild = Effect.gen(function* () { const commandId = stableCommandId({ scope, requestKey: key, operation: "cancel-task" }); const reason = agentStopReason(input.reason); + const command = { + type: "thread.stop" as const, + commandId, + threadId: childThreadId, + stopDelegatedTasks: false, + reason, + }; yield* threadManagement - .dispatch({ - type: "thread.stop", - commandId, - threadId: childThreadId, - stopDelegatedTasks: false, - reason, + .cancelDelegatedTask({ + parentThreadId: scope.thread.threadId, + taskId: input.taskId, + command, + stopDescendants: (remoteDispatch) => + threadManagement.stopDelegatedTasks({ + threadId: childThreadId, + commandId, + reason, + remoteDispatch, + }), }) .pipe( Effect.mapError((error) => @@ -2711,16 +2659,6 @@ const make = Effect.gen(function* () { ), ), ); - yield* threadManagement - .stopDelegatedTasks({ threadId: childThreadId, commandId, reason }) - .pipe( - Effect.mapError((error) => - failure( - "task_not_cancellable", - `Stopped delegated task ${input.taskId}, but not every task it delegated: ${errorMessage(error)}`, - ), - ), - ); const stopped = yield* threadManagement .getThreadRecords(childThreadId, ["runs"]) .pipe(Effect.mapError(threadManagementFailure)); @@ -2964,7 +2902,7 @@ const make = Effect.gen(function* () { ), ); } - const projection = yield* loadProjection(threadId); + const projection = yield* loadThreadRecords(threadId, ["runs"]); const run = projection.runs.at(-1); yield* threadManagement .dispatch({ @@ -3010,10 +2948,14 @@ const make = Effect.gen(function* () { }, ), listThreads: Effect.fn("OrchestratorMcpService.listThreads")(function* (scope, input) { - const { parent } = yield* loadCaller(scope); + const { parent } = yield* loadCaller(scope, []); const projectId = input.allProjects ? null - : yield* resolveProjectTarget(parent, input.projectId); + : yield* resolveProjectTarget( + parent, + input.projectId, + "Pass projectId, or allProjects: true for every project. This MCP client is not running inside a T3 thread.", + ); const projectThreads = yield* threadManagement .listProjectThreads({ projectId: projectId ?? undefined, @@ -3114,7 +3056,7 @@ const make = Effect.gen(function* () { (input.textOffset ?? 0) === 0 ) { for (const task of tasks) { - const transfer = taskResultTransfer(parent, target, task); + const transfer = taskResultTransfer(parent.thread.id, target, task); const resultRunId = transfer?.sourcePoint.runId ?? delegatedTaskRun(target, task)?.id; const resultRunIds = resultRunId === undefined ? [] : [resultRunId]; const resultRecords = yield* threadManagement @@ -3127,7 +3069,7 @@ const make = Effect.gen(function* () { .pipe(Effect.mapError(threadManagementFailure)); if ( pageIncludesTerminalTaskResult({ - parent, + parentThreadId: parent.thread.id, page, task, target: { ...target, ...resultRecords }, @@ -3169,7 +3111,7 @@ const make = Effect.gen(function* () { } satisfies OrchestratorMcpThreadReadResult; }), sendToThread: Effect.fn("OrchestratorMcpService.sendToThread")(function* (scope, input) { - const { parent, limits, target } = yield* loadScopedThread(scope, input.threadId); + const { parent, limits, target } = yield* loadScopedThread(scope, input.threadId, ["runs"]); yield* assertLiveCallerForOtherThread(scope, parent, target); yield* resolveRuntimeMode(limits.runtimeMode, target.thread.runtimeMode); yield* resolveInteractionMode(limits.interactionMode, target.thread.interactionMode); @@ -3220,7 +3162,7 @@ const make = Effect.gen(function* () { } satisfies OrchestratorMcpThreadSendResult; }), waitForThread: Effect.fn("OrchestratorMcpService.waitForThread")(function* (scope, input) { - const { parent, target } = yield* loadScopedThread(scope, input.threadId, false); + const { parent, target } = yield* loadScopedThread(scope, input.threadId, []); const result = yield* threadManagement .waitForThread({ projectId: target.thread.projectId, @@ -3270,7 +3212,7 @@ const make = Effect.gen(function* () { } satisfies OrchestratorMcpThreadWaitResult; }), interruptThread: Effect.fn("OrchestratorMcpService.interruptThread")(function* (scope, input) { - const { parent, limits, target } = yield* loadScopedThread(scope, input.threadId); + const { parent, limits, target } = yield* loadScopedThread(scope, input.threadId, ["runs"]); yield* assertLiveCallerForOtherThread(scope, parent, target); yield* resolveRuntimeMode(limits.runtimeMode, target.thread.runtimeMode); yield* resolveInteractionMode(limits.interactionMode, target.thread.interactionMode); @@ -3279,6 +3221,7 @@ const make = Effect.gen(function* () { // the run instead, as this call did then. if (input.stop === true) { const commandId = stableCommandId({ scope, requestKey: key, operation: "thread-stop" }); + const reason = agentStopReason(input.reason); // The cause goes to the log; the caller learns only that the stop failed. const stopFailed = (error: unknown) => Effect.logWarning("orchestrator-mcp.thread-stop-failed", { @@ -3296,12 +3239,12 @@ const make = Effect.gen(function* () { type: "thread.stop", commandId, threadId: input.threadId, - ...(input.reason === undefined ? {} : { reason: input.reason }), + reason, }) .pipe(Effect.catch(stopFailed)); // A retry with the same clientRequestId repeats only the stops that failed. yield* threadManagement - .stopDelegatedTasks({ threadId: input.threadId, commandId, reason: input.reason }) + .stopDelegatedTasks({ threadId: input.threadId, commandId, reason }) .pipe(Effect.catch(stopFailed)); // The run this stop interrupted, from its own events, so a retry // reports the same run even if another has started since. Stop also diff --git a/apps/server/src/mcp/TaskCancelNativeSubagent.integration.test.ts b/apps/server/src/mcp/TaskCancelNativeSubagent.integration.test.ts index 7d6f894e79ed..4d8ae1a2ad46 100644 --- a/apps/server/src/mcp/TaskCancelNativeSubagent.integration.test.ts +++ b/apps/server/src/mcp/TaskCancelNativeSubagent.integration.test.ts @@ -4,6 +4,7 @@ import { assert, it } from "@effect/vitest"; import { CommandId, EnvironmentId, + EventId, MessageId, NodeId, ProjectId, @@ -11,6 +12,8 @@ import { ProviderInstanceId, ProviderThreadId, ProviderTurnId, + RunId, + RuntimeRequestId, ThreadId, TurnItemId, type OrchestrationV2DomainEvent, @@ -18,13 +21,16 @@ import { import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; import { ClaudeProviderCapabilitiesV2 } from "../orchestration-v2/Adapters/ClaudeAdapterV2.ts"; import * as CommandReceiptStore from "../orchestration-v2/CommandReceiptStore.ts"; import * as EffectWorker from "../orchestration-v2/EffectWorker.ts"; +import * as EventSink from "../orchestration-v2/EventSink.ts"; import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import * as ProviderAdapterRegistry from "../orchestration-v2/ProviderAdapterRegistry.ts"; @@ -53,14 +59,25 @@ const driver = ProviderDriverKind.make("claudeAgent"); const instanceId = ProviderInstanceId.make("claudeAgent"); const modelSelection = { instanceId, model: "claude-test" }; -it.effect("task_cancel ends a delegated child's background native subagent", () => +// Cancellation races with turn completion and checkpoint capture; "recorded" restarts before the Stop outbox runs. +const cancelBackgroundOnlyTask = ( + background: "subagent" | "command", + { + cancelAt = "settled", + remote = false, + }: { + readonly remote?: boolean; + readonly cancelAt?: "settled" | "checkpoint" | "turn" | "recorded" | "monitor" | "refused"; + } = {}, +) => Effect.scoped( Effect.gen(function* () { - const cwd = yield* checkpointWorkspace("task-cancel-native-subagent"); + const cwd = yield* checkpointWorkspace(`task-cancel-background-${background}`); const turns: Array<{ readonly turn: ProviderAdapter.ProviderAdapterV2TurnInput; readonly events: Queue.Queue; }> = []; + const interrupted: Array | undefined> = []; const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = { instanceId, driver, @@ -126,7 +143,10 @@ it.effect("task_cancel ends a delegated child's background native subagent", () }); }), steerTurn: () => Effect.die("unused"), - interruptTurn: () => Effect.void, + interruptTurn: ({ providerThread }) => + Effect.sync(() => { + interrupted.push(providerThread.pendingBackgroundTasks?.map((t) => t.taskId)); + }), respondToRuntimeRequest: () => Effect.die("unused"), readThreadSnapshot: () => Effect.die("unused"), rollbackThread: () => Effect.die("unused"), @@ -135,14 +155,54 @@ it.effect("task_cancel ends a delegated child's background native subagent", () }), }; const layerAdapters = ProviderAdapterRegistry.layerSingle(adapter); - const layerOrchestrator = ProviderReplayHarness.layerWithRegistry( - { name: "task-cancel-native-subagent" }, - layerAdapters, - { runEffectWorker: false }, + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3-task-cancel-" }); + const databaseLayer = SqlitePersistence.layerFromPath( + path.join(directory, "state.sqlite"), + ).pipe(Layer.provide(NodeServices.layer)); + // Migrate the file first, as an existing install's database already is. + yield* Effect.provide(Effect.void, databaseLayer); + // Each build is a server on the same database; the second runs startup recovery. + let restarted = false; + const layerOrchestrator = Layer.suspend(() => + ProviderReplayHarness.layerWithRegistry( + { name: "task-cancel-native-subagent" }, + layerAdapters, + restarted + ? { databaseLayer, recoverOnStartup: true } + : { databaseLayer, runEffectWorker: false }, + ), ); const layerOrchestration = Layer.merge( layerOrchestrator, - ThreadManagementService.layer.pipe( + Layer.effect( + ThreadManagementService.ThreadManagementService, + Effect.map(ThreadManagementService.ThreadManagementService, (service) => + cancelAt === "refused" + ? { + ...service, + stopDelegatedTasks: () => + Effect.fail( + new Orchestrator.OrchestratorDispatchError({ + commandId: CommandId.make("refused-descendant-stop"), + commandType: "thread.stop", + cause: "descendant refused stop", + }), + ), + } + : service, + ), + ).pipe( + Layer.provide( + ThreadManagementService.layer.pipe( + Layer.provide(layerOrchestrator), + Layer.provide(ServerSettings.layerTest()), + Layer.provide( + CommandReceiptStore.layer.pipe(Layer.provide(SqlitePersistence.layerMemory)), + ), + ), + ), Layer.provide(layerOrchestrator), Layer.provide(ServerSettings.layerTest()), Layer.provide( @@ -167,17 +227,25 @@ it.effect("task_cancel ends a delegated child's background native subagent", () ), ); - yield* Effect.gen(function* () { + const stopped = yield* Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; const service = yield* OrchestratorMcpService.OrchestratorMcpService; + const eventSink = yield* EventSink.EventSinkV2; const parentThreadId = ThreadId.make("thread:task-cancel-parent"); + // Read the cursor before forking: provider events offered right after a watch can + // commit before a live-tail subscriber has read its starting sequence. const watch = (predicate: (event: OrchestrationV2DomainEvent) => boolean) => - orchestrator.streamDomainEvents.pipe( - Stream.filter(predicate), - Stream.take(1), - Stream.runDrain, - Effect.forkScoped, + eventSink.latestSequence().pipe( + Effect.flatMap((afterSequence) => + eventSink.stream({ afterSequence }).pipe( + Stream.map((stored) => stored.event), + Stream.filter(predicate), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ), + ), ); const turnRunning = () => watch( @@ -228,6 +296,41 @@ it.effect("task_cancel ends a delegated child's background native subagent", () createdBy: "agent", creationSource: "mcp", }); + if (cancelAt === "turn") { + const pendingTask = (yield* orchestrator.getThreadProjection(parentThreadId)) + .subagents[0]!; + const { runs } = yield* orchestrator.getThreadProjection(pendingTask.childThreadId!); + const now = yield* DateTime.now; + const current = runs[0]!; + const previous = { + ...current, + id: RunId.make("run:before-continuation"), + ordinal: 1, + status: "completed" as const, + startedAt: now, + completedAt: now, + }; + const at = { threadId: current.threadId, occurredAt: now }; + yield* eventSink.write({ + commandId: CommandId.make("command:runtime-reconcile:continuation"), + events: [ + { + ...at, + runId: current.id, + id: EventId.make("event:continuation"), + type: "run.updated", + payload: { ...current, ordinal: 2, restartContinuationOfRunId: previous.id }, + }, + { + ...at, + runId: previous.id, + id: EventId.make("event:previous"), + type: "run.updated", + payload: previous, + }, + ], + }); + } yield* worker.drain(); yield* Fiber.join(childRunning); const task = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents[0]!; @@ -259,7 +362,7 @@ it.effect("task_cancel ends a delegated child's background native subagent", () event.payload.id === child.turn.runId && event.payload.status === "waiting", ); - for (const event of [ + const nativeSubagentEvents = [ { type: "node.updated", driver, @@ -308,6 +411,71 @@ it.effect("task_cancel ends a delegated child's background native subagent", () subagentId, }, }, + ] satisfies ReadonlyArray; + const shellEvents = [ + { + type: "provider_thread.updated", + driver, + providerThread: { + ...child.turn.providerThread, + status: "active", + pendingBackgroundTasks: [ + { taskId: "bg-sleep", description: "sleep 300", kind: "command", awaited: true }, + ], + updatedAt: now, + }, + }, + ] satisfies ReadonlyArray; + const started = yield* watch((event) => + background === "subagent" + ? event.type === "subagent.updated" && event.payload.id === subagentId + : event.type === "provider-thread.updated" && + event.threadId === task.childThreadId && + (event.payload.pendingBackgroundTasks?.length ?? 0) > 0, + ); + // An async question the child asked during its turn, and one no run of this task owns. + const question = (id: string, nodeId: NodeId) => + ({ + type: "runtime_request.updated", + driver, + runtimeRequest: { + id: RuntimeRequestId.make(id), + nodeId, + providerTurnId: childTurn.id, + nativeRequestRef: null, + kind: "user_input", + status: "pending", + responseCapability: { type: "message" }, + createdAt: now, + resolvedAt: null, + }, + }) satisfies ProviderAdapter.ProviderAdapterV2Event; + for (const event of [ + question("request:owned", child.turn.rootNodeId), + question("request:unrelated", NodeId.make("node:unrelated")), + ...(background === "subagent" ? nativeSubagentEvents : shellEvents), + ]) { + yield* Queue.offer(child.events, event); + } + yield* Fiber.join(started); + if (remote) { + yield* orchestrator.dispatch({ + type: "delegated_task.remote.request", + commandId: CommandId.make("remote-descendant"), + parentThreadId: task.childThreadId!, + parentRunId: child.turn.runId, + parentNodeId: child.turn.rootNodeId, + task: "Work in a linked environment", + driver, + modelSelection, + remoteChild: { + environmentId: EnvironmentId.make("environment:remote-descendant"), + threadId: ThreadId.make("thread:remote-descendant"), + label: "Remote", + }, + }); + } + const turnEndEvents = [ { type: "provider_turn.updated", driver, @@ -323,11 +491,32 @@ it.effect("task_cancel ends a delegated child's background native subagent", () failure: null, threadDisposition: "reusable", }, - ] satisfies ReadonlyArray) { - yield* Queue.offer(child.events, event); + ] satisfies ReadonlyArray; + const endTurn = Effect.gen(function* () { + for (const event of turnEndEvents) yield* Queue.offer(child.events, event); + yield* Fiber.join(waiting); + }); + if (cancelAt !== "turn") yield* endTurn; + const turnSettled = + cancelAt !== "checkpoint" && cancelAt !== "turn" && cancelAt !== "recorded"; + if (turnSettled) yield* worker.drain(); + if (cancelAt === "monitor") { + // A monitor notification then ran and completed while the shell kept running. + const { runs, messages } = yield* orchestrator.getThreadProjection(task.childThreadId!); + const run = { ...runs[0]!, id: RunId.make("run:monitor"), ordinal: 2 }; + const source = { kind: "monitor" } as const; + const notification = { source, outcome: "updated", summary: "Ran" } as const; + const user = { ...messages[0]!, id: MessageId.make("message:monitor"), runId: run.id }; + const message = { ...user, notification }; + const at = { threadId: run.threadId, runId: run.id, occurredAt: now }; + yield* (yield* EventSink.EventSinkV2).write({ + commandId: CommandId.make("command:runtime-reconcile:monitor"), + events: [ + { ...at, id: EventId.make("event:m1"), type: "message.updated", payload: message }, + { ...at, id: EventId.make("event:m2"), type: "run.updated", payload: run }, + ], + }); } - yield* Fiber.join(waiting); - yield* worker.drain(); const scope: McpInvocationScope = { environmentId: EnvironmentId.make("environment:task-cancel"), @@ -343,33 +532,166 @@ it.effect("task_cancel ends a delegated child's background native subagent", () }; const before = yield* service.taskStatus(scope, task.id); assert.equal(before.status, "running"); - assert.equal(before.workState, "waiting_for_children"); + assert.equal(before.workState, turnSettled ? "waiting_for_children" : "working"); + const parentTask = Effect.map(orchestrator.getThreadProjection(parentThreadId), (parent) => + parent.subagents.find((entry) => entry.id === task.id), + ); + if (cancelAt === "recorded") { + const commandId = CommandId.make("stop-recorded"); + yield* orchestrator.cancelDelegatedTask({ + parentThreadId, + taskId: task.id, + command: { + type: "thread.stop", + commandId, + threadId: task.childThreadId!, + stopDelegatedTasks: false, + }, + stopDescendants: () => Effect.void, + }); + const committed = yield* eventSink.readByCommandId({ commandId }).pipe(Stream.runCollect); + assert.isTrue( + committed.some( + ({ event }) => + event.type === "subagent.updated" && event.payload.cancelledRunId !== undefined, + ), + ); + assert.isTrue( + committed.some( + ({ event }) => + event.type === "turn-item.updated" && + event.payload.type === "run_interrupt_request", + ), + ); + assert.equal((yield* parentTask)?.status, "running"); + return { scope, taskId: task.id, settled: undefined }; + } + if (cancelAt === "refused") { + yield* Effect.flip( + service.cancelTask(scope, { taskId: task.id, clientRequestId: "refused-cancel" }), + ); + const after = yield* parentTask; + assert.equal(after?.status, "running"); + assert.equal(after?.result, null); + assert.isUndefined(after?.cancelledRunId); + return undefined; + } + // Settling the task ends the async question it left, after its own row. + const settled = yield* watch( + (event) => + event.type === "runtime-request.updated" && + event.payload.id === "request:owned" && + event.payload.status === "resolved", + ); const cancelled = yield* service.cancelTask(scope, { taskId: task.id, clientRequestId: "cancel-task", }); assert.equal(cancelled.status, "cancel_requested"); + assert.equal((yield* parentTask)?.cancelledRunId, child.turn.runId); + // The turn completes normally after Stop, before its interrupt reaches the provider. + if (cancelAt === "turn") yield* endTurn; yield* worker.drain(); + const owned = Effect.map(orchestrator.getThreadProjection(task.childThreadId!), (child) => + child.runtimeRequests.find((request) => request.id === "request:owned"), + ); + // task_cancel settles a finished turn itself; a running one settles when it ends. + if ((yield* owned)?.status !== "resolved") yield* Fiber.join(settled); const childAfter = yield* orchestrator.getThreadProjection(task.childThreadId!); + const nativeStatus = background === "subagent" ? "interrupted" : undefined; assert.equal( childAfter.subagents.find((entry) => entry.id === subagentId)?.status, - "interrupted", + nativeStatus, ); assert.equal( childAfter.nodes.find((entry) => entry.id === subagentId)?.status, - "interrupted", + nativeStatus, ); assert.equal( - childAfter.turnItems.find((item) => item.type === "subagent")?.status, - "interrupted", + childAfter.turnItems.find( + (item) => item.type === "subagent" && item.subagentId === subagentId, + )?.status, + nativeStatus, + ); + if (remote) { + const descendant = childAfter.subagents.find((entry) => entry.remoteChild !== undefined); + assert.equal(descendant?.status, "interrupted"); + assert.notEqual(descendant?.remoteChild?.stoppedThere, true); + } + // Stop reached the provider turn that owns the shell. + if (background === "command") assert.deepEqual(interrupted, [["bg-sleep"]]); + assert.equal(childAfter.runs.length, cancelAt === "monitor" || cancelAt === "turn" ? 2 : 1); + assert.equal(childAfter.runs[0]!.status, "completed"); + assert.deepEqual( + childAfter.runtimeRequests.map((request) => [request.id, request.status]), + [ + ["request:owned", "resolved"], + ["request:unrelated", "pending"], + ], ); const after = yield* service.taskStatus(scope, task.id); - assert.equal(after.status, "completed"); - assert.notEqual(after.workState, "waiting_for_children"); + assert.equal(after.status, "interrupted"); + assert.equal(after.workState, "result_available"); + const settledTask = yield* parentTask; + assert.equal(settledTask?.status, "interrupted"); + assert.notEqual(settledTask?.completedAt, null); + assert.equal(settledTask?.cancelledRunId, child.turn.runId); + + return { scope, taskId: task.id, settled: settledTask }; + }).pipe(Effect.provide(layerTest)); + if (stopped === undefined) return; + restarted = true; + + // A server restarted on the same database runs startup recovery and keeps the outcome. + yield* Effect.gen(function* () { + const service = yield* OrchestratorMcpService.OrchestratorMcpService; + const orchestrator = yield* Orchestrator.OrchestratorV2; + yield* (yield* EffectWorker.OrchestrationEffectWorkerV2).drain(); + yield* orchestrator.recoverDelegatedTasks; + const task = (yield* orchestrator.getThreadProjection( + stopped.scope.thread!.threadId, + )).subagents.find((entry) => entry.id === stopped.taskId); + const status = yield* service.taskStatus(stopped.scope, stopped.taskId); + assert.equal(status.status, "interrupted"); + assert.equal(status.summary, task?.result); + assert.equal(task?.status, "interrupted"); + assert.notEqual(task?.completedAt, null); + if (stopped.settled !== undefined) assert.deepEqual(task, stopped.settled); }).pipe(Effect.provide(layerTest)); }), - ), + ).pipe(Effect.provide(NodeServices.layer)); + +it.effect("task_cancel ends a delegated child's background native subagent", () => + cancelBackgroundOnlyTask("subagent"), +); + +it.effect("task_cancel ends a delegated child's background shell command", () => + cancelBackgroundOnlyTask("command"), +); + +it.effect("task_cancel ends background work while the child's run captures its checkpoint", () => + cancelBackgroundOnlyTask("command", { cancelAt: "checkpoint" }), +); + +it.effect("task_cancel ends background work left by a turn that finishes after Stop", () => + cancelBackgroundOnlyTask("command", { cancelAt: "turn" }), +); + +it.effect("task_cancel ends the shell's question though a monitor ran after the turn", () => + cancelBackgroundOnlyTask("command", { cancelAt: "monitor" }), +); + +it.effect("cancellation and Stop commit together and recover before the outbox ran", () => + cancelBackgroundOnlyTask("subagent", { cancelAt: "recorded" }), +); + +it.effect("a refused descendant stop does not settle or dispose its parent task", () => + cancelBackgroundOnlyTask("command", { cancelAt: "refused" }), +); + +it.effect("task_cancel stops a remote descendant without reacquiring its parent lock", () => + cancelBackgroundOnlyTask("command", { remote: true }), ); diff --git a/apps/server/src/mcp/threadAccess.ts b/apps/server/src/mcp/threadAccess.ts index af421060dbcd..bf153c9d56ea 100644 --- a/apps/server/src/mcp/threadAccess.ts +++ b/apps/server/src/mcp/threadAccess.ts @@ -47,7 +47,8 @@ export const unavailable = () => export const dispatchFailure = ( error: OrchestratorV2Error | ThreadManagement.ThreadManagementProviderInstanceDisabledError, ) => - error._tag === "ThreadManagementProviderInstanceDisabledError" + error._tag === "ThreadManagementProviderInstanceDisabledError" || + error._tag === "OrchestratorProviderSelectionBusyError" ? new OrchestratorMcpFailure({ code: "invalid_request", message: error.message }) : (error._tag === "OrchestratorDispatchError" || error._tag === "OrchestratorCommandRejectedError") && diff --git a/apps/server/src/mcp/toolkits/orchestrator/handlers.ts b/apps/server/src/mcp/toolkits/orchestrator/handlers.ts index e3ffcc7f7bd9..80a8ad4d5843 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/handlers.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/handlers.ts @@ -3,6 +3,7 @@ import { OrchestratorToolkit } from "./tools.ts"; import * as Effect from "effect/Effect"; import * as PeerForwarding from "../../../peer/PeerForwarding.ts"; +import * as RemoteDelegation from "../../../peer/RemoteDelegation.ts"; import { readCaller } from "../../threadAccess.ts"; import * as McpInvocationContext from "../../McpInvocationContext.ts"; import * as McpToolAccess from "../../McpToolAccess.ts"; @@ -166,13 +167,22 @@ const handlers = { return yield* service.update(scope, input); }), ), - t3_thread_send: McpToolAccess.writesThreads( - (input) => [input.threadId], - (input) => + t3_thread_send: McpToolAccess.afterForwarding( + McpToolAccess.writesThreads( + (input) => [input.threadId], + (input) => + Effect.gen(function* () { + const scope = yield* McpInvocationContext.McpInvocationContext; + const service = yield* OrchestratorMcpService.OrchestratorMcpService; + return yield* service.sendToThread(scope, input); + }), + ), + // There the thread's parent is here, so only this side can make a follow-up a task. + (input, environmentId, sent) => Effect.gen(function* () { const scope = yield* McpInvocationContext.McpInvocationContext; - const service = yield* OrchestratorMcpService.OrchestratorMcpService; - return yield* service.sendToThread(scope, input); + const remote = yield* RemoteDelegation.RemoteDelegation; + return yield* remote.followUp(scope, environmentId, input, sent); }), ), t3_thread_wait: McpToolAccess.reads((input) => diff --git a/apps/server/src/mcp/toolkits/orchestrator/tools.ts b/apps/server/src/mcp/toolkits/orchestrator/tools.ts index 66caff8e422a..e29778a10cfd 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/tools.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/tools.ts @@ -40,6 +40,7 @@ import * as Struct from "effect/Struct"; import * as ThreadManagementService from "../../../orchestration-v2/ThreadManagementService.ts"; import * as PeerForwarding from "../../../peer/PeerForwarding.ts"; +import * as RemoteDelegation from "../../../peer/RemoteDelegation.ts"; import * as McpInvocationContext from "../../McpInvocationContext.ts"; import * as OrchestratorMcpService from "../../OrchestratorMcpService.ts"; import * as ThreadMetadataMcpService from "../../ThreadMetadataMcpService.ts"; @@ -51,6 +52,8 @@ const dependencies = [ ]; /** Listing this environment's links. */ const linkDependencies = [...dependencies, PeerForwarding.PeerForwarding]; +/** A send there may record a follow-up to a remote task. */ +const sendDependencies = [...dependencies, RemoteDelegation.RemoteDelegation]; const threadMetadataDependencies = [ McpInvocationContext.McpInvocationContext, ThreadManagementService.ThreadManagementService, @@ -142,7 +145,11 @@ const BackgroundWaitTool = Tool.make("t3_background_wait", { failure: OrchestratorMcpFailure, failureMode: "return", dependencies, -}).annotate(Tool.Title, "Wait on background work"); +}) + .annotate(Tool.Title, "Wait on background work") + // Only marks which tasks keep the thread Working; Claude Plan mode blocks tools without this. + .annotate(Tool.Readonly, true) + .annotate(Tool.Destructive, false); const ScheduleTaskTool = Tool.make("schedule_task", { description: @@ -263,7 +270,7 @@ const ThreadSendTool = Tool.make("t3_thread_send", { success: OrchestratorMcpThreadSendResult, failure: OrchestratorMcpFailure, failureMode: "return", - dependencies, + dependencies: sendDependencies, }) .annotate(Tool.Title, "Send to a T3 thread") .annotate(Tool.Destructive, true) diff --git a/apps/server/src/mcp/toolkits/project/handlers.test.ts b/apps/server/src/mcp/toolkits/project/handlers.test.ts index 5b24d2daeeed..c8330c7a3ab6 100644 --- a/apps/server/src/mcp/toolkits/project/handlers.test.ts +++ b/apps/server/src/mcp/toolkits/project/handlers.test.ts @@ -1,6 +1,7 @@ import { expect, it } from "@effect/vitest"; import { EnvironmentId, + GitCommandError, ProjectId, ProviderDriverKind, ProviderInstanceId, @@ -13,13 +14,17 @@ import { import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Cause from "effect/Cause"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Stream from "effect/Stream"; +import { ChildProcessSpawner } from "effect/process"; +import * as TestClock from "effect/testing/TestClock"; import * as DocumentText from "../../../provider/DocumentText.ts"; import * as ThreadLaunch from "../../../orchestration-v2/ThreadLaunchService.ts"; @@ -31,6 +36,8 @@ import * as Project from "../../../project/ProjectService.ts"; import * as ProjectCloneTracker from "../../../project/ProjectCloneTracker.ts"; import * as ManagedProjectFolders from "../../../project/ManagedProjectFolders.ts"; // Coder: no peer-link services to mock; `apps/server/src/peer` is not carried. +import * as SourceControlProviderRegistry from "../../../sourceControl/SourceControlProviderRegistry.ts"; +import * as Repositories from "../../../sourceControl/SourceControlRepositoryService.ts"; import * as GitVcsDriver from "../../../vcs/GitVcsDriver.ts"; import * as VcsProcess from "../../../vcs/VcsProcess.ts"; import * as McpInvocationContext from "../../McpInvocationContext.ts"; @@ -296,6 +303,74 @@ it.effect("a launch steps an inherited mode down to the provider's, and refuses }), ); +it.effect("a launch with a model its provider does not advertise creates no thread", () => + Effect.gen(function* () { + const launched: Array = []; + const providers: ServerProvider[] = ["claudeAgent", "codex"].map((driver) => ({ + instanceId: ProviderInstanceId.make(driver), + driver: ProviderDriverKind.make(driver), + enabled: true, + installed: true, + status: "ready", + version: null, + auth: { status: "authenticated" }, + checkedAt: "2026-01-01T00:00:00Z", + models: [ + driver === "claudeAgent" + ? { + slug: "claude-opus", + name: "Opus", + aliases: ["opus"], + isCustom: false, + capabilities: null, + } + : { slug: "gpt-5.4", name: "GPT", isCustom: false, capabilities: null }, + ], + slashCommands: [], + skills: [], + })); + const { projectId, dependencies } = clientLaunchHarness({ + runtimeModeCeiling: "auto", + launched, + providers, + }); + const toolkit = yield* ProjectToolkit.pipe( + Effect.provide( + McpToolAccess.HandlersLayer.layer(ProjectHandlers.layer).pipe(Layer.provide(dependencies)), + ), + ); + const launchWith = (instanceId: string, model: string) => + toolkit + .handle("t3_thread_launch", { + title: "Fix", + projectId, + modelSelection: { instanceId: ProviderInstanceId.make(instanceId), model }, + }) + .pipe( + Stream.unwrap, + Stream.runCollect, + Effect.map((result) => result.at(-1)?.result), + Effect.provide(dependencies), + ); + + for (const [instanceId, model] of [ + ["claudeAgent", "claude-unadvertised"], + ["codex", "opus"], + ] as const) { + expect(yield* launchWith(instanceId, model)).toMatchObject({ + code: "model_unavailable", + message: `Model ${model} is not advertised by provider ${instanceId}.`, + }); + } + expect(launched).toHaveLength(0); + + expect(yield* launchWith("claudeAgent", "Opus")).toMatchObject({ + modelSelection: { instanceId: "claudeAgent", model: "Opus" }, + }); + expect(launched).toHaveLength(1); + }), +); + it.effect("a launch retried with the same clientRequestId replays the first one", () => Effect.gen(function* () { const launched: Array = []; @@ -493,6 +568,44 @@ it.effect("a launch into a new worktree creates nothing when its base ref has no }), ); +it.effect("a launch into a new worktree creates nothing when its branch collides", () => + Effect.gen(function* () { + const launched: Array = []; + const { projectId, dependencies } = clientLaunchHarness({ + runtimeModeCeiling: "auto", + launched, + checkWorktreeBase: ({ projectId, workspaceStrategy }) => + Effect.fail( + new ThreadLaunch.ThreadLaunchBranchConflictError({ + projectId, + projectTitle: "Client target", + branch: workspaceStrategy.branch ?? "", + conflictingBranch: "werkstatt", + }), + ), + }); + const toolkit = yield* ProjectToolkit.pipe( + Effect.provide( + McpToolAccess.HandlersLayer.layer(ProjectHandlers.layer).pipe(Layer.provide(dependencies)), + ), + ); + const refused = (yield* toolkit + .handle("t3_thread_launch", { + title: "Fix", + projectId, + workspaceStrategy: { type: "worktree", baseRef: "main", branch: "werkstatt/baseline" }, + }) + .pipe(Stream.unwrap, Stream.runCollect, Effect.provide(dependencies))).at(-1)?.result; + + expect(refused).toMatchObject({ + code: "invalid_request", + message: + 'Branch "werkstatt/baseline" cannot be created in project "Client target": branch "werkstatt" exists. No thread was created. Use a name that does not nest under or contain an existing branch, such as "werkstatt-baseline".', + }); + expect(launched).toHaveLength(0); + }), +); + it.effect("a retried worktree launch replays after its base ref is gone", () => Effect.gen(function* () { const launched: Array = []; @@ -536,6 +649,76 @@ it.effect("a retried worktree launch replays after its base ref is gone", () => }), ); +it.effect("an agent's clone is not cut off while git is still transferring", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const parent = yield* fs.makeTempDirectoryScoped({ prefix: "t3-mcp-clone-" }); + const destinationPath = path.join(parent, "repo"); + const started = yield* Deferred.make(); + const transferred = yield* Deferred.make(); + const { dependencies } = clientLaunchHarness({ + runtimeModeCeiling: "full-access", + launched: [], + }); + const repositories = Repositories.layer.pipe( + Layer.provide( + Layer.mock(SourceControlProviderRegistry.SourceControlProviderRegistry)({ + resolveLink: () => undefined, + }), + ), + Layer.provide( + Layer.mock(GitVcsDriver.GitVcsDriver)({ + // Like the real driver, the clone fails once it outlives the deadline it was given. + execute: (input) => + Deferred.succeed(started, undefined).pipe( + Effect.andThen(Deferred.await(transferred)), + Effect.as({ + exitCode: ChildProcessSpawner.ExitCode(0), + stdout: "", + stderr: "", + stdoutTruncated: false, + stderrTruncated: false, + }), + Effect.timeoutOrElse({ + duration: input.timeoutMs ?? Infinity, + orElse: () => + Effect.fail( + new GitCommandError({ + operation: input.operation, + command: "git", + cwd: input.cwd, + detail: "Git command timed out.", + }), + ), + }), + ), + }), + ), + Layer.provideMerge(dependencies), + ); + const toolkit = yield* ProjectToolkit.pipe( + Effect.provide( + McpToolAccess.HandlersLayer.layer(ProjectHandlers.layer).pipe(Layer.provide(repositories)), + ), + ); + + const clone = yield* toolkit + // Coder: the GitLab-only repository service clones GitLab URLs. + .handle("t3_project_clone", { + remoteUrl: "https://gitlab.example/group/repo.git", + destinationPath, + }) + .pipe(Stream.unwrap, Stream.runCollect, Effect.provide(repositories), Effect.forkChild); + yield* Deferred.await(started); + // A large repository over a slow link; agents' MCP calls get a far longer budget. + yield* TestClock.adjust("10 minutes"); + yield* Deferred.succeed(transferred, undefined); + + expect((yield* Fiber.join(clone)).at(-1)?.result).toMatchObject({ cwd: destinationPath }); + }).pipe(Effect.provide(NodeServices.layer)), +); + it.effect("deleting a project through MCP also discards its clone record", () => Effect.gen(function* () { const discarded: Array = []; diff --git a/apps/server/src/mcp/toolkits/project/handlers.ts b/apps/server/src/mcp/toolkits/project/handlers.ts index 0ead9e1e5882..ce8aca13c795 100644 --- a/apps/server/src/mcp/toolkits/project/handlers.ts +++ b/apps/server/src/mcp/toolkits/project/handlers.ts @@ -25,6 +25,7 @@ import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; import { resolveRuntimeMode } from "../../delegatedTaskModes.ts"; import * as McpInvocationContext from "../../McpInvocationContext.ts"; import * as McpToolAccess from "../../McpToolAccess.ts"; +import * as OrchestrationMcp from "../../OrchestratorMcpService.ts"; import { newCommandId, readCaller, resolveProjectId, unavailable } from "../../threadAccess.ts"; import { ProjectToolkit } from "./tools.ts"; @@ -41,6 +42,28 @@ function projectFailure(error: Project.ProjectServiceError) { return new OrchestratorMcpFailure({ code: "invalid_request", message }); } +function worktreeCheckHint( + error: + | ThreadLaunch.ThreadLaunchBaseRefError + | ThreadLaunch.ThreadLaunchBranchConflictError + | ThreadLaunch.ThreadLaunchNotRepositoryError, + strategy: { readonly startFromOrigin?: boolean | undefined }, +) { + if (error._tag === "ThreadLaunchNotRepositoryError") + return `Use workspaceStrategy: {type: "root"}, or pass the projectId of a Git repository.`; + if (error._tag === "ThreadLaunchBranchConflictError") { + const sibling = error.branch.replaceAll("/", "-"); + return `Use a name that does not nest under or contain an existing branch${ + sibling === error.branch ? "" : `, such as "${sibling}"` + }.`; + } + return `Pass a branch, tag, or commit that exists there${ + strategy.startFromOrigin === true + ? "; the project has no usable remote to fetch it from." + : ", or set startFromOrigin:true if it exists only on a remote." + }`; +} + /** Require project worktrees so launches cannot target arbitrary host directories. */ const assertProjectWorktree = Effect.fn("mcp.assertProjectWorktree")(function* ( workspaceRoot: string, @@ -146,7 +169,14 @@ export const layer = McpToolAccess.toLayer(ProjectToolkit, { }); yield* assertProjectWorktree(project.workspaceRoot, input.workspaceStrategy.worktreePath); } - // Reject unresolved base refs before accepting a launch; worktree provisioning happens later. + if (!replaying && input.modelSelection !== undefined) { + const { instanceId, model } = input.modelSelection; + const providers = yield* (yield* ProviderRegistry.ProviderRegistry).getProviders; + const provider = providers.find((candidate) => candidate.instanceId === instanceId); + if (provider !== undefined) + yield* OrchestrationMcp.requireAdvertisedModel(provider, model); + } + // Reject unresolved base refs and blocked branch names before accepting a launch; worktree provisioning happens later. const strategy = input.workspaceStrategy; if (!replaying && strategy?.type === "worktree") yield* ThreadLaunch.ThreadLaunchService.pipe( @@ -157,11 +187,7 @@ export const layer = McpToolAccess.toLayer(ProjectToolkit, { (error) => new OrchestratorMcpFailure({ code: "invalid_request", - message: `${error.message} No thread was created. Pass a branch, tag, or commit that exists there${ - strategy.startFromOrigin === true - ? "; the project has no usable remote to fetch it from." - : ", or set startFromOrigin:true if it exists only on a remote." - }`, + message: `${error.message} No thread was created. ${worktreeCheckHint(error, strategy)}`, }), ), ); @@ -356,7 +382,8 @@ export const layer = McpToolAccess.toLayer(ProjectToolkit, { (input) => Effect.gen(function* () { const repositories = yield* Repositories.SourceControlRepositoryService; - return yield* repositories.cloneRepository(input).pipe( + // Agents' MCP calls have their own long budget, and an aborted call interrupts git. + return yield* repositories.cloneRepository(input, { timeoutMs: null }).pipe( Effect.mapError( (error) => new OrchestratorMcpFailure({ diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index b060bae05e21..6802870e0627 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -4,6 +4,7 @@ import * as NodePath from "node:path"; import type { ElicitationRequest, + ElicitationResult, Query as ClaudeQuery, SDKMessage, SDKResultMessage, @@ -48,6 +49,7 @@ import { } from "@t3tools/contracts"; import { mcpAppFromToolItem } from "@t3tools/shared/toolOutput"; import { assert, describe, it } from "@effect/vitest"; +import * as Cause from "effect/Cause"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as SecureRandom from "effect/crypto/SecureRandom"; @@ -65,6 +67,7 @@ import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; import { formatClaudeResumeCompactionQuestion } from "@t3tools/shared/claudeCompaction"; +import { backgroundWorkHoldsCompletion } from "@t3tools/shared/orchestrationV2PendingBackgroundWork"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { getModelSelectionStringOptionValue } from "@t3tools/shared/model"; @@ -83,6 +86,7 @@ import type * as ProviderContinuationRequests from "@t3tools/provider-core/serve import { makeProviderFailure } from "@t3tools/provider-core/server/failure"; import * as ClaudeAdapterV2 from "./ClaudeAdapterV2.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; +import * as ProviderInstallationGate from "../../provider/ProviderInstallationGate.ts"; import { ClaudeOrchestratorReplayHarness } from "./ClaudeAdapterV2.testkit.ts"; import { provideDeterministicTestRuntime } from "../testkit/DeterministicRuntime.ts"; import { runOrchestratorV2Scenario } from "../testkit/OrchestratorScenario.ts"; @@ -1062,6 +1066,50 @@ describe("ClaudeAdapterV2 append-system-prompt-file launch argument", () => { ), ); + // regression: #26631 `~/` launch argument paths resolved under the project cwd + it.effect.each([ + [ + "--settings ~/.claude/settings.json", + (options: ClaudeAdapterV2.ClaudeAgentSdkQueryOptions) => + assert.deepInclude(options.settings, { skillOverrides: { lint: "off" } }), + ], + [ + "--append-system-prompt-file ~/.claude/settings.json", + (options: ClaudeAdapterV2.ClaudeAgentSdkQueryOptions) => + assert.include(appendSystemPrompt(options), '{"skillOverrides":{"lint":"off"}}'), + ], + ] as const)("reads %s from the home directory", ([launchArgs, assertOptions]) => + Effect.scoped( + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const home = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-claude-home-" }); + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-claude-home-cwd-" }); + yield* fileSystem.makeDirectory(NodePath.join(home, ".claude")); + yield* fileSystem.writeFileString( + NodePath.join(home, ".claude", "settings.json"), + '{"skillOverrides":{"lint":"off"}}', + ); + const { exit, opened } = yield* startTurnWithLaunchArgs(launchArgs, cwd).pipe( + Effect.provideService(HostProcess.HomeDirectory, home), + ); + if (Exit.isFailure(exit)) { + assert.fail(makeProviderFailure({ cause: exit.cause, class: "provider_error" }).message); + } + assert.lengthOf(opened, 1); + assertOptions(opened[0]!); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ), + ); + // regression: #20413 malformed append-system-prompt launch args fail before a query opens it.effect.each([ [ @@ -1132,6 +1180,102 @@ describe("ClaudeAdapterV2 append-system-prompt-file launch argument", () => { ); }); +describe("ClaudeAdapterV2 provider installation gate", () => { + it.effect("opens each turn's Claude process inside the installation gate", () => + Effect.scoped( + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const log: Array = []; + const instanceId = ClaudeAdapterV2.CLAUDE_DEFAULT_INSTANCE_ID; + const adapter = yield* ClaudeAdapterV2.makeClaudeAdapterV2({ + instanceId, + settings: DEFAULT_CLAUDE_SETTINGS, + environment: {}, + attachmentsDir: yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-claude-gate-" }), + fileSystem, + path: yield* Path.Path, + random: yield* SecureRandom.SecureRandom, + idAllocator: yield* IdAllocator.IdAllocatorV2, + installationGate: ProviderInstallationGate.ProviderInstallationGate.of({ + execute: (gatedInstanceId, effect) => + Effect.sync(() => log.push(`enter ${gatedInstanceId}`)).pipe( + Effect.andThen(effect), + Effect.ensuring(Effect.sync(() => log.push("exit"))), + ), + update: () => Effect.die("unused"), + }), + queryRunner: { + allocateSessionId: Effect.succeed("native-thread-claude-gate"), + open: () => + Effect.sync(() => { + log.push("open query"); + return { + messages: Stream.never, + offer: () => Effect.void, + setModel: () => Effect.void, + applyFlagSettings: () => Effect.void, + setPermissionMode: () => Effect.void, + backgroundTasks: () => Effect.succeed(false), + mcpServerStatus: Effect.succeed([]), + readMcpResource: () => Effect.die("unexpected MCP resource read"), + callMcpTool: () => Effect.die("unexpected MCP tool call"), + listPermissionRules: Effect.die("unexpected permission rules read"), + stopTask: () => Effect.void, + interrupt: Effect.void, + close: Effect.void, + }; + }), + forkSession: () => Effect.die("unused"), + subagentLaunchToolUseId: () => Effect.succeed(null), + assertComplete: Effect.void, + }, + }); + const runtimePolicy = ProviderAdapter.ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-claude-gate-cwd-" }), + }); + const threadId = ThreadId.make("thread-claude-gate"); + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-claude-gate"), + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy, + }); + assert.deepStrictEqual(log, []); + + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("attempt-claude-gate"), + text: "Hello", + attachments: [], + runtimePolicy, + }), + ); + + assert.deepStrictEqual(log, [`enter ${instanceId}`, "open query", "exit"]); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ), + ); +}); + describe("ClaudeAdapterV2 approval cancellation", () => { it.effect("observes an approval signal that was already aborted", () => Effect.gen(function* () { @@ -2085,7 +2229,6 @@ describe("ClaudeAdapterV2 attachments", () => { "missing-eof", "escaped-name", "incremental", - "compressed-bomb", "indirect-count", "indirect-type", "indirect-kids", @@ -2142,9 +2285,7 @@ describe("ClaudeAdapterV2 attachments", () => { "%PDF-1.7\n1 0 obj\n<< /Type /Catalog /Pages 42 >>\nendobj\ntrailer\n<< /Root 1 0 R >>\n%%EOF", ); yield* fileSystem.writeFile(documentPath, malformedBytes); - } else if ( - ["object-stream", "encrypted", "cyclic-tree", "compressed-bomb"].includes(scenario) - ) { + } else if (["object-stream", "encrypted", "cyclic-tree"].includes(scenario)) { const suffix = scenario === "encrypted" ? "/Encrypt 9 0 R" @@ -2227,6 +2368,8 @@ describe("ClaudeAdapterV2 attachments", () => { ); }); +// Coder: upstream's root-change fork test (#22836) reads the transcript the SDK's `forkSession` +// copies; the CLI forks natively on the fork's first query instead, so the fork's cursor test stays. describe("ClaudeAdapterV2 native fork", () => { it.effect("forks at the source assistant cursor and resumes the forked session", () => Effect.scoped( @@ -2253,7 +2396,8 @@ describe("ClaudeAdapterV2 native fork", () => { Effect.sync(() => { openedQueries.push(input); return { - messages: Stream.empty, + // As upstream (#33008): a query whose stream ends before the turn starts fails it. + messages: Stream.never, offer: () => Effect.void, setModel: () => Effect.void, applyFlagSettings: () => Effect.void, @@ -2333,8 +2477,8 @@ describe("ClaudeAdapterV2 native fork", () => { assert.deepEqual(forkCalls, [ { sessionId: "source-native-session", + // As upstream (#22836): the fork no longer passes the thread's cwd. options: { - dir: "/workspace", upToMessageId: "assistant-message-cursor", }, environment: { CLAUDE_CONFIG_DIR: "/custom/claude-home" }, @@ -2604,7 +2748,7 @@ describe("ClaudeAdapterV2 native session identity", () => { Effect.sync(() => { openedQueries.push(input); return { - messages: Stream.empty, + messages: Stream.never, offer: () => Effect.void, setModel: () => Effect.void, applyFlagSettings: () => Effect.void, @@ -2960,6 +3104,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { readonly freshQueueOnReopen?: boolean; readonly forkSession?: ClaudeAdapterV2.ClaudeAgentSdkQueryRunnerShape["forkSession"]; readonly allocateSessionId?: Effect.Effect; + readonly offer?: (message: SDKUserMessage) => Effect.Effect; readonly promptSuggestions?: boolean; }) => Effect.gen(function* () { @@ -3092,7 +3237,10 @@ describe("ClaudeAdapterV2 background wake turns", () => { offer: (message) => Effect.sync(() => { offeredMessages.push(message); - }).pipe(Effect.andThen(changes.notify)), + }).pipe( + Effect.andThen(changes.notify), + Effect.andThen(options?.offer?.(message) ?? Effect.void), + ), setModel: (model) => Effect.sync(() => { liveSelectionChanges.push({ model }); @@ -4689,126 +4837,448 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); - // regression: #15475 MCP title arriving after the tool starts must survive to completion + // regression: #15708 Stop must not silently drop a steer Claude queued behind a foreground task it.effect.each([ - { isError: false, title: "Check weather" }, - { isError: true, title: "Check weather" }, - { isError: false, title: undefined }, - ])("keeps late MCP display metadata with %j", ({ isError, title }) => + { + readBy: "none", + echoes: true, + title: "warns that a steer Claude never read before Stop was not delivered", + }, + { + readBy: "none", + echoes: false, + title: "does not warn about a steer on a CLI that never echoes prompt uuids", + }, + { + readBy: "assistant", + echoes: true, + title: "does not warn about a steer Claude read before Stop", + }, + ] as const)("$title", ({ readBy, echoes }) => Effect.gen(function* () { - const harness = yield* makeWakeHarness; - yield* harness.runtime.startTurn( - makeClaudeTestTurnInput({ - threadId: harness.threadId, - providerThread: harness.providerThread, - now: yield* DateTime.now, - attemptId: RunAttemptId.make("mcp-presentation"), - text: "Check the weather", - attachments: [], - }), - ); - const toolName = "mcp__weather__get_weather"; - const id = "weather-call"; - const permission = yield* Effect.promise(() => - harness.getOpenedOptions()!.canUseTool!( - toolName, - { city: "Berlin" }, - { - signal: new AbortController().signal, - toolUseID: id, - requestId: "weather-request", - }, - ), - ).pipe(Effect.forkScoped); - const request = yield* Queue.take(harness.runtimeRequestReceipts); - yield* harness.runtime.respondToRuntimeRequest({ - requestId: request.runtimeRequest.id, - decision: "accept", + const harness = yield* makeWakeHarnessWithOptions({ + close: (sdkMessages) => Queue.shutdown(sdkMessages), + freshQueueOnReopen: true, + }); + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const firstInput = makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("attempt-claude-unread-steer-1"), + text: "Run the slow curl in the foreground.", + attachments: [], + }); + yield* harness.runtime.startTurn(firstInput); + const providerTurnId = idAllocator.derive.providerTurn({ + driver: ClaudeAdapterV2.CLAUDE_PROVIDER, + nativeTurnId: `turn:${firstInput.attemptId}`, }); - yield* Fiber.join(permission); yield* harness.offerAndWait( claudeSdkFrame({ type: "assistant", - uuid: "weather-assistant", + uuid: "00000000-0000-4000-8000-000000015701", session_id: WAKE_NATIVE_SESSION, parent_tool_use_id: null, + ...(echoes ? { user_message_uuids: [harness.promptUuid(0)] } : {}), message: { - id: "weather-message", + id: "msg_unread_steer_bash", type: "message", role: "assistant", model: "claude-sonnet-4-6", - content: [{ type: "tool_use", id, name: toolName, input: { city: "Berlin" } }], - }, - tool_use_meta: [ - { - id, - display_name: title, - server_display_name: "Weather", - icon_url: "https://example.com/weather.png", - }, - ], - }), - ); - yield* harness.offerAndWait( - claudeSdkFrame({ - type: "user", - uuid: "weather-result", - session_id: WAKE_NATIVE_SESSION, - parent_tool_use_id: null, - message: { - role: "user", content: [ { - type: "tool_result", - tool_use_id: id, - content: [ - { type: "text", text: "Weather result" }, - { - type: "image", - source: { type: "base64", media_type: "image/png", data: "iVBORw0KGgo=" }, - }, - ], - is_error: isError, + type: "tool_use", + id: "toolu-unread-steer-bash", + name: "Bash", + input: { command: "curl http://127.0.0.1:9/hang" }, }, ], }, - tool_use_result: - title === undefined ? { screenshot: { mimeType: "image/png" } } : undefined, }), ); - yield* Queue.offer( - harness.sdkMessages, - makeResultFrame({ uuid: "weather-terminal", result: "Weather checked" }), - ); - yield* Queue.take(harness.terminalReceipts); - const items = harness.events.flatMap((event) => - event.type === "turn_item.updated" && event.turnItem.type === "dynamic_tool" - ? [event.turnItem] - : [], + // Past task_started the CLI queues a "now" steer until the call returns. + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "task_started", + task_id: "task-unread-steer-bash", + tool_use_id: "toolu-unread-steer-bash", + description: "Slow curl", + is_backgrounded: false, + task_type: "local_bash", + uuid: "00000000-0000-4000-8000-000000015702", + session_id: WAKE_NATIVE_SESSION, + }), ); - assert.deepEqual( - items.map((item) => item.status), - ["running", "running", isError ? "failed" : "completed"], + const steerText = "Codeword: PELICAN. Remember it, I will ask for it later."; + yield* harness.runtime.steerTurn({ + threadId: harness.threadId, + runId: firstInput.runId, + providerThread: harness.providerThread, + providerTurnId, + message: { + createdBy: "user", + creationSource: "web", + messageId: MessageId.make("unread-steer"), + text: steerText, + attachments: [], + }, + }); + const steerUuid = harness.offeredMessages.at(-1)?.uuid; + assert.isDefined(steerUuid); + if (readBy === "assistant") { + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "assistant", + uuid: "00000000-0000-4000-8000-000000015703", + session_id: WAKE_NATIVE_SESSION, + parent_tool_use_id: null, + user_message_uuids: [steerUuid], + message: { + id: "msg_unread_steer_reply", + type: "message", + role: "assistant", + model: "claude-sonnet-4-6", + content: [{ type: "text", text: "Noted." }], + }, + }), + ); + } + const noticesBefore = harness.events.length; + yield* harness.runtime.interruptTurn({ + providerThread: harness.providerThread, + providerTurnId, + }); + assert.equal((yield* Queue.take(harness.terminalReceipts)).status, "interrupted"); + + const notices = harness.events + .slice(noticesBefore) + .flatMap((event) => + event.type === "turn_item.updated" && event.turnItem.type === "system_notice" + ? [event.turnItem.message] + : [], + ); + const undelivered = notices.find((message) => + /not (have )?(been )?delivered|never (reached|delivered)/i.test(message ?? ""), ); - assert.isNull(items[0]?.title); - for (const item of items.slice(1)) { - assert.equal(item.title, title ?? "get weather"); - assert.deepEqual(item.toolIcon, { - _tag: "themed-logo", - logoUrl: "https://example.com/weather.png", - }); - assert.deepEqual(item.toolSource, { - key: "mcp:weather", - name: "Weather", - kind: "integration", - icon: { _tag: "themed-logo", logoUrl: "https://example.com/weather.png" }, - }); - assert.deepEqual(item.input, { city: "Berlin" }); + if (readBy !== "none" || !echoes) { + assert.isUndefined(undelivered); + return; } - assert.equal(new Set(items.map((item) => item.id)).size, 1); - assert.deepEqual(toolOutputImages(items.at(-1)?.output), [ - { mimeType: "image/png", data: "iVBORw0KGgo=" }, - ]); + assert.isDefined( + undelivered, + `expected an undelivered-steer notice, got ${JSON.stringify(notices)}`, + ); + assert.match(undelivered ?? "", /resend|send it again|re-send/i); + assert.include(undelivered, "PELICAN"); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + + // regression: #23948 Claude holds a steer behind a running foreground tool + // call and goes silent, so each steer stays pending until Claude starts it. + it.effect("keeps a steer pending until Claude starts it", () => + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const attemptId = RunAttemptId.make("attempt-steer-waiting"); + const input = makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId, + text: "Run the slow build.", + attachments: [], + }); + yield* harness.runtime.startTurn(input); + const steer = (id: string) => + harness.runtime.steerTurn({ + threadId: harness.threadId, + runId: input.runId, + providerThread: harness.providerThread, + providerTurnId: idAllocator.derive.providerTurn({ + driver: ClaudeAdapterV2.CLAUDE_PROVIDER, + nativeTurnId: `turn:${attemptId}`, + }), + message: { + createdBy: "user", + creationSource: "web", + messageId: MessageId.make(id), + text: `Steer ${id}`, + attachments: [], + }, + }); + yield* steer("message-steer-read"); + yield* steer("message-steer-unread"); + const steerStatuses = () => { + const latest = new Map(); + for (const event of harness.events) { + if (event.type === "steer.updated") latest.set(event.messageId, event.status); + } + return Object.fromEntries(latest); + }; + const expectSteers = (expected: Record) => + harness + .until(() => JSON.stringify(steerStatuses()) === JSON.stringify(expected)) + .pipe(Effect.andThen(Effect.sync(() => assert.deepEqual(steerStatuses(), expected)))); + + // Claude keeps streaming the foreground tool's turn; once its text is + // recorded, anything the steers emitted before it is recorded too. + yield* harness.offerAndWait( + makeAssistantTextFrame({ + uuid: "00000000-0000-4000-8000-000000002392", + text: "Still building.", + }), + ); + yield* harness.until(() => + harness.events.some( + (event) => event.type === "message.updated" && event.message.text === "Still building.", + ), + ); + assert.deepEqual(steerStatuses(), { + "message-steer-read": "pending", + "message-steer-unread": "pending", + }); + + const lifecycle = (index: number, state: string, uuid: string) => + harness.offerAndWait( + claudeSdkFrame({ + type: "command_lifecycle", + command_uuid: harness.promptUuid(index), + state, + uuid, + session_id: WAKE_NATIVE_SESSION, + }), + ); + // Claude completes prompts it never started, so completed alone is not + // proof it read one. + yield* lifecycle(2, "completed", "00000000-0000-4000-8000-000000002393"); + yield* lifecycle(1, "started", "00000000-0000-4000-8000-000000002394"); + yield* expectSteers({ + "message-steer-read": "completed", + "message-steer-unread": "pending", + }); + + yield* harness.offerAndWait( + claudeSdkFrame({ + ...makeResultFrame({ uuid: "00000000-0000-4000-8000-000000002395", result: "Built." }), + user_message_uuid: harness.promptUuid(0), + }), + ); + yield* harness.awaitTerminal(1); + yield* expectSteers({ + "message-steer-read": "completed", + "message-steer-unread": "completed", + }); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + + // Claude can answer a steer while its offer is still being logged. + it.effect("settles a turn whose result echoes only a steer before the offer returns", () => + Effect.gen(function* () { + let settleDuringOffer: (message: SDKUserMessage) => Effect.Effect = () => Effect.void; + const harness = yield* makeWakeHarnessWithOptions({ + offer: (message) => + message.priority === "now" + ? Effect.suspend(() => settleDuringOffer(message)) + : Effect.void, + }); + settleDuringOffer = (message) => + harness + .offerAndWait( + claudeSdkFrame({ + ...makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000002396", + result: "Built.", + }), + user_message_uuid: message.uuid, + }), + ) + .pipe(Effect.andThen(harness.awaitTerminal(1))); + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const attemptId = RunAttemptId.make("attempt-steer-result"); + const input = makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId, + text: "Run the slow build.", + attachments: [], + }); + yield* harness.runtime.startTurn(input); + yield* harness.runtime.steerTurn({ + threadId: harness.threadId, + runId: input.runId, + providerThread: harness.providerThread, + providerTurnId: idAllocator.derive.providerTurn({ + driver: ClaudeAdapterV2.CLAUDE_PROVIDER, + nativeTurnId: `turn:${attemptId}`, + }), + message: { + createdBy: "user", + creationSource: "web", + messageId: MessageId.make("message-steer-result"), + text: "Steer", + attachments: [], + }, + }); + assert.deepEqual( + harness.events.flatMap((event) => (event.type === "steer.updated" ? [event.status] : [])), + ["pending", "completed"], + ); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + + // regression: #15475 MCP title arriving after the tool starts must survive to completion + it.effect.each([ + { isError: false, title: "Check weather" }, + { isError: true, title: "Check weather" }, + { isError: false, title: undefined }, + ])("keeps late MCP display metadata with %j", ({ isError, title }) => + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("mcp-presentation"), + text: "Check the weather", + attachments: [], + }), + ); + const toolName = "mcp__weather__get_weather"; + const id = "weather-call"; + const permission = yield* Effect.promise(() => + harness.getOpenedOptions()!.canUseTool!( + toolName, + { city: "Berlin" }, + { + signal: new AbortController().signal, + toolUseID: id, + requestId: "weather-request", + }, + ), + ).pipe(Effect.forkScoped); + const request = yield* Queue.take(harness.runtimeRequestReceipts); + yield* harness.runtime.respondToRuntimeRequest({ + requestId: request.runtimeRequest.id, + decision: "accept", + }); + yield* Fiber.join(permission); + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "assistant", + uuid: "weather-assistant", + session_id: WAKE_NATIVE_SESSION, + parent_tool_use_id: null, + message: { + id: "weather-message", + type: "message", + role: "assistant", + model: "claude-sonnet-4-6", + content: [{ type: "tool_use", id, name: toolName, input: { city: "Berlin" } }], + }, + tool_use_meta: [ + { + id, + display_name: title, + server_display_name: "Weather", + icon_url: "https://example.com/weather.png", + }, + ], + }), + ); + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "user", + uuid: "weather-result", + session_id: WAKE_NATIVE_SESSION, + parent_tool_use_id: null, + message: { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: id, + content: [ + { type: "text", text: "Weather result" }, + { + type: "image", + source: { type: "base64", media_type: "image/png", data: "iVBORw0KGgo=" }, + }, + ], + is_error: isError, + }, + ], + }, + tool_use_result: + title === undefined ? { screenshot: { mimeType: "image/png" } } : undefined, + }), + ); + yield* Queue.offer( + harness.sdkMessages, + makeResultFrame({ uuid: "weather-terminal", result: "Weather checked" }), + ); + yield* Queue.take(harness.terminalReceipts); + const items = harness.events.flatMap((event) => + event.type === "turn_item.updated" && event.turnItem.type === "dynamic_tool" + ? [event.turnItem] + : [], + ); + assert.deepEqual( + items.map((item) => item.status), + ["running", "running", isError ? "failed" : "completed"], + ); + assert.isNull(items[0]?.title); + for (const item of items.slice(1)) { + assert.equal(item.title, title ?? "get weather"); + assert.deepEqual(item.toolIcon, { + _tag: "themed-logo", + logoUrl: "https://example.com/weather.png", + }); + assert.deepEqual(item.toolSource, { + key: "mcp:weather", + name: "Weather", + kind: "integration", + icon: { _tag: "themed-logo", logoUrl: "https://example.com/weather.png" }, + }); + assert.deepEqual(item.input, { city: "Berlin" }); + } + assert.equal(new Set(items.map((item) => item.id)).size, 1); + assert.deepEqual(toolOutputImages(items.at(-1)?.output), [ + { mimeType: "image/png", data: "iVBORw0KGgo=" }, + ]); }).pipe( Effect.scoped, Effect.provide( @@ -5530,6 +6000,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { options: [ { decision: "cancel", label: "Cancel" }, { decision: "decline", label: "Decline" }, + { decision: "acceptForSession", label: "Always allow this session" }, { decision: "accept", label: "Approve" }, ], }); @@ -5559,16 +6030,41 @@ describe("ClaudeAdapterV2 background wake turns", () => { }).pipe(Effect.scoped, Effect.provide(testLayer)), ); - it.effect.each(["acceptForSession", "acceptAlways"] as const)( - "declines an unsupported %s decision", - (decision) => - Effect.gen(function* () { - const { harness, elicit, awaitCard } = yield* startElicitationTurn(); - const answer = yield* elicit(approvalOnly); - const request = yield* awaitCard(1); - yield* harness.runtime.respondToRuntimeRequest({ requestId: request.id, decision }); - assert.deepEqual(yield* Fiber.join(answer), { action: "decline" }); - }).pipe(Effect.scoped, Effect.provide(testLayer)), + it.effect("declines an unsupported acceptAlways decision", () => + Effect.gen(function* () { + const { harness, elicit, awaitCard } = yield* startElicitationTurn(); + const answer = yield* elicit(approvalOnly); + const request = yield* awaitCard(1); + yield* harness.runtime.respondToRuntimeRequest({ + requestId: request.id, + decision: "acceptAlways", + }); + assert.deepEqual(yield* Fiber.join(answer), { action: "decline" }); + }).pipe(Effect.scoped, Effect.provide(testLayer)), + ); + + it.effect("answers a repeated prompt from a session grant without a card", () => + Effect.gen(function* () { + const { harness, elicit, awaitCard, runtimeRequests } = yield* startElicitationTurn(); + const first = yield* elicit(approvalOnly); + const request = yield* awaitCard(1); + yield* harness.runtime.respondToRuntimeRequest({ + requestId: request.id, + decision: "acceptForSession", + }); + const granted = { action: "accept", content: {} } satisfies ElicitationResult; + assert.deepEqual(yield* Fiber.join(first), granted); + + assert.deepEqual(yield* Fiber.join(yield* elicit(approvalOnly)), granted); + assert.equal(runtimeRequests().length, 1); + + const otherMessage = yield* elicit({ ...approvalOnly, message: "Deploy api?" }); + yield* awaitCard(2); + const otherServer = yield* elicit({ ...approvalOnly, serverName: "builds" }); + yield* awaitCard(3); + assert.isUndefined(otherMessage.pollUnsafe()); + assert.isUndefined(otherServer.pollUnsafe()); + }).pipe(Effect.scoped, Effect.provide(testLayer)), ); it.effect("declines URL and value forms without a card", () => @@ -5744,11 +6240,125 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); - it.effect("re-sends the model on the turn after a live selection change fails", () => + // regression: #34549 a model change sent with a steer was shown but never applied to the turn. + it.effect("applies a model change sent with a steer to the running turn", () => Effect.scoped( Effect.gen(function* () { - let failFlags = true; - const harness = yield* makeWakeHarnessWithOptions({ + const harness = yield* makeWakeHarness; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const opus = { + instanceId: ClaudeAdapterV2.CLAUDE_DEFAULT_INSTANCE_ID, + model: "claude-opus-5-5", + } satisfies ModelSelection; + const input = makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("attempt-steer-model-change"), + text: "Run the slow build.", + attachments: [], + modelSelection: opus, + }); + yield* harness.runtime.startTurn(input); + yield* harness.runtime.steerTurn({ + threadId: harness.threadId, + runId: input.runId, + providerThread: harness.providerThread, + providerTurnId: idAllocator.derive.providerTurn({ + driver: ClaudeAdapterV2.CLAUDE_PROVIDER, + nativeTurnId: `turn:${input.attemptId}`, + }), + modelSelection: { ...opus, model: "claude-sonnet-4-6" }, + message: { + createdBy: "user", + creationSource: "web", + messageId: MessageId.make("steer-model-change"), + text: "Which model are you?", + attachments: [], + }, + }); + assert.deepEqual(harness.liveSelectionChanges, [ + { model: "claude-sonnet-4-6" }, + { effortLevel: "high", fastMode: null, ultracode: null }, + ]); + assert.lengthOf(harness.offeredMessages, 2); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ), + ); + + it.effect("fails a model-changing steer when the turn ends during the selection change", () => + Effect.scoped( + Effect.gen(function* () { + let duringFlags: Effect.Effect = Effect.void; + const harness = yield* makeWakeHarnessWithOptions({ + applyFlagSettings: () => Effect.suspend(() => duringFlags), + }); + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const opus = { + instanceId: ClaudeAdapterV2.CLAUDE_DEFAULT_INSTANCE_ID, + model: "claude-opus-5-5", + } satisfies ModelSelection; + const input = makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("attempt-steer-model-change-race"), + text: "Run the slow build.", + attachments: [], + modelSelection: opus, + }); + yield* harness.runtime.startTurn(input); + duringFlags = harness + .offerAndWait(turnOneResult) + .pipe(Effect.andThen(Queue.take(harness.terminalReceipts)), Effect.asVoid); + const steer = yield* harness.runtime + .steerTurn({ + threadId: harness.threadId, + runId: input.runId, + providerThread: harness.providerThread, + providerTurnId: idAllocator.derive.providerTurn({ + driver: ClaudeAdapterV2.CLAUDE_PROVIDER, + nativeTurnId: `turn:${input.attemptId}`, + }), + modelSelection: { ...opus, model: "claude-sonnet-4-6" }, + message: { + createdBy: "user", + creationSource: "web", + messageId: MessageId.make("steer-model-change-race"), + text: "Which model are you?", + attachments: [], + }, + }) + .pipe(Effect.result); + assert.equal(steer._tag, "Failure"); + assert.lengthOf(harness.offeredMessages, 1); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ), + ); + + it.effect("re-sends the model on the turn after a live selection change fails", () => + Effect.scoped( + Effect.gen(function* () { + let failFlags = true; + const harness = yield* makeWakeHarnessWithOptions({ applyFlagSettings: () => Effect.suspend(() => { if (!failFlags) return Effect.void; @@ -5936,7 +6546,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { ); // regression: #15892 non-execution metadata is carried through, not guessed from result text - it.effect.each(["cancelled", "denied", "permission_denied", undefined])( + it.effect.each(["cancelled", "interrupted", "denied", "permission_denied", undefined])( "preserves native tool non-execution metadata %s without inferring a denial from text", (kind) => Effect.scoped( @@ -5991,7 +6601,10 @@ describe("ClaudeAdapterV2 background wake turns", () => { : [], ); const failed = items.findLast((item) => item.nativeItemRef?.nativeId === "tool-error")!; - assert.equal(failed.status, kind === "cancelled" ? "cancelled" : "failed"); + assert.equal( + failed.status, + kind === "cancelled" || kind === "interrupted" ? kind : "failed", + ); assert.equal(failed.toolNonExecutionKind, kind); const ok = items.findLast((item) => item.nativeItemRef?.nativeId === "tool-ok")!; assert.equal(ok.status, "completed"); @@ -6125,6 +6738,15 @@ describe("ClaudeAdapterV2 background wake turns", () => { }); assert.equal(harness.offeredMessages[1]?.priority, "now"); } + const partialText = "The audit covers the `integ"; + if (!queryExits) + yield* Queue.offer( + harness.sdkMessages, + makeAssistantTextFrame({ + uuid: "00000000-0000-4000-8000-000000000900", + text: partialText, + }), + ); yield* harness.offerAndWait( makeResultFrame({ uuid: "00000000-0000-4000-8000-000000000901", @@ -6161,6 +6783,19 @@ describe("ClaudeAdapterV2 background wake turns", () => { terminal.status, queryExits ? "failed" : steered ? "completed" : "interrupted", ); + if (steered && !queryExits && terminalReason === "aborted_streaming") { + // The steer cut this reply off mid-stream; it must not read as a finished answer. + const partial = harness.events.findLast( + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "assistant_message" && + event.turnItem.text === partialText, + ); + assert.equal( + partial?.type === "turn_item.updated" ? partial.turnItem.status : undefined, + "interrupted", + ); + } if (steered) { if (!queryExits) assert.isTrue( @@ -6386,9 +7021,42 @@ describe("ClaudeAdapterV2 background wake turns", () => { expected: "Claude stopped: the prompt exceeds the model's context window. Run /compact or start a new thread.", }, + ...[ + { + name: "a model version error rewritten by a custom endpoint", + baseUrl: "https://gateway.example.test/v1", + installed: "2.1.322", + expected: + "The custom API endpoint gateway.example.test reported Claude Code 2.1.280, but this session runs Claude Code 2.1.322. Running claude update will not fix this; the endpoint or a proxy behind it rewrote the version.", + }, + { name: "a model version error without a custom endpoint", installed: "2.1.322" }, + { + name: "a model version error from a Claude Code still below the required version", + baseUrl: "https://gateway.example.test/v1", + installed: "2.1.285", + }, + ].map((row) => { + const result = + "API Error: 400 Claude Code 2.1.280 does not support this model; version 2.1.290 or newer is required. Run 'claude update', or update the Claude desktop app, then try again."; + return { + ...row, + isError: true, + result, + errors: undefined, + apiErrorStatus: undefined, + terminalReason: undefined, + environment: { + CLAUDE_CONFIG_DIR: "/synthetic/no-claude-config", + ...(row.baseUrl === undefined ? {} : { ANTHROPIC_BASE_URL: row.baseUrl }), + }, + expected: row.expected ?? result, + }; + }), ])("reports terminal API error text for $name", (input) => Effect.gen(function* () { - const harness = yield* makeWakeHarness; + const harness = yield* makeWakeHarnessWithOptions( + "environment" in input ? { environment: input.environment } : undefined, + ); const now = yield* DateTime.now; yield* harness.runtime.startTurn( makeClaudeTestTurnInput({ @@ -6401,6 +7069,17 @@ describe("ClaudeAdapterV2 background wake turns", () => { }), ); const apiErrorStatus = input.apiErrorStatus ?? 400; + if ("installed" in input) { + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "init", + uuid: "00000000-0000-4000-8000-000000000698", + session_id: WAKE_NATIVE_SESSION, + claude_code_version: input.installed, + }), + ); + } yield* Queue.offer( harness.sdkMessages, makeResultFrame({ @@ -8143,6 +8822,70 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + // The passive row is the frame Claude 2.1.303 emits for a Stop hook returning a systemMessage. + it.effect.each([ + { name: "hides a passive summary", level: "notice", visible: false }, + { name: "shows a blocking notice", level: "notice", preventContinuation: true, visible: true }, + { name: "shows a warning", level: "warning", visible: true }, + ])("Claude Stop-hook notices: $name", ({ level, preventContinuation, visible }) => + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("claude-stop-hook-notice-attempt"), + text: "Continue.", + attachments: [], + }), + ); + const content = "Stop says: 40 scans, 0 findings"; + yield* Queue.offer( + harness.sdkMessages, + claudeSdkFrame({ + type: "system", + subtype: "informational", + content, + level, + ...(preventContinuation === undefined + ? {} + : { prevent_continuation: preventContinuation }), + session_id: WAKE_NATIVE_SESSION, + uuid: "00000000-0000-4000-8000-000000000820", + }), + ); + yield* Queue.offer( + harness.sdkMessages, + makeResultFrame({ uuid: "00000000-0000-4000-8000-000000000821", result: content }), + ); + assert.equal((yield* Queue.take(harness.terminalReceipts)).status, "completed"); + assert.deepEqual( + harness.events.flatMap((event) => + event.type === "turn_item.updated" && event.turnItem.type === "system_notice" + ? [event.turnItem.message] + : [], + ), + visible ? [content] : [], + ); + assert.isFalse( + harness.events.some( + (event) => event.type === "message.updated" && event.message.role === "assistant", + ), + ); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + NodeCrypto.layer, + IdAllocator.layer, + McpProviderSessions.layer, + ), + ), + ), + ); + it.effect("shows a hook-dropped Claude prompt once and skips info-level lines", () => Effect.gen(function* () { const harness = yield* makeWakeHarness; @@ -9684,11 +10427,15 @@ describe("ClaudeAdapterV2 background wake turns", () => { retryDelayMs: 1_500, }); + // The stream starting again settles the retry before any assistant message completes. yield* Queue.offer( harness.sdkMessages, - makeAssistantTextFrame({ - uuid: "00000000-0000-4000-8000-000000000202", - text: "Opening GitHub.", + claudeSdkFrame({ + type: "stream_event", + event: { type: "message_start", message: { id: "api-retry-recovered-message" } }, + parent_tool_use_id: null, + session_id: WAKE_NATIVE_SESSION, + uuid: "00000000-0000-4000-8000-000000000204", }), ); yield* harness.until(() => retryItems().length === 2); @@ -9697,6 +10444,13 @@ describe("ClaudeAdapterV2 background wake turns", () => { assert.equal(recoveredRetry?.id, runningRetry?.id); assert.equal(recoveredRetry?.status, "completed"); assert.equal(recoveredRetry?.title, "Provider recovered"); + yield* Queue.offer( + harness.sdkMessages, + makeAssistantTextFrame({ + uuid: "00000000-0000-4000-8000-000000000202", + text: "Opening GitHub.", + }), + ); yield* Queue.offer( harness.sdkMessages, @@ -9881,6 +10635,32 @@ describe("ClaudeAdapterV2 background wake turns", () => { attachments: [], }), ); + yield* Queue.offer( + harness.sdkMessages, + claudeSdkFrame({ + type: "assistant", + message: { + model: "claude-sonnet-4-6", + id: "msg_background_command", + type: "message", + role: "assistant", + content: [ + { + type: "tool_use", + id: WAKE_TOOL_USE_ID, + name: "Bash", + input: { + command: "python3 audit.py > audit.log 2>&1", + run_in_background: true, + }, + }, + ], + }, + parent_tool_use_id: null, + uuid: "background-command-launch", + session_id: WAKE_NATIVE_SESSION, + }), + ); yield* Queue.offer(harness.sdkMessages, wakeTaskStarted); yield* Queue.offer(harness.sdkMessages, turnOneResult); yield* harness.awaitTerminal(1); @@ -9894,6 +10674,37 @@ describe("ClaudeAdapterV2 background wake turns", () => { WAKE_TASK_ID, ); + const command = harness.events.find( + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "command_execution" && + event.turnItem.nativeItemRef?.nativeId === WAKE_TOOL_USE_ID, + ); + assert.equal(command?.type, "turn_item.updated"); + if (command?.type !== "turn_item.updated") return; + const details = { + command: "python3 audit.py > audit.log 2>&1", + itemId: command.turnItem.id, + }; + assert.deepInclude( + afterStart.at(-1)?.providerThread.pendingBackgroundTasks?.[0], + details, + ); + yield* harness.offerAndWait( + claudeSdkFrame({ + type: "system", + subtype: "background_tasks_changed", + tasks: [{ task_id: WAKE_TASK_ID, task_type: "local_bash", description: "Audit" }], + uuid: "background-command-snapshot", + session_id: WAKE_NATIVE_SESSION, + }), + ); + assert.deepInclude( + providerThreadRosterEvents(harness.events).at(-1)?.providerThread + .pendingBackgroundTasks?.[0], + details, + ); + yield* Queue.offer( harness.sdkMessages, claudeSdkFrame({ @@ -10212,39 +11023,507 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); - it.effect.skipIf(process.getuid === undefined)( - "tails foreground Bash output under the physical cwd the query launched in", - () => - Effect.scoped( - Effect.gen(function* () { - const fileSystem = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fileSystem.makeTempDirectoryScoped({ - prefix: "t3-claude-task-output-cwd-", - }); - const physicalCwd = path.join(root, "real", "project"); - const launchCwd = path.join(root, "link"); - yield* fileSystem.makeDirectory(physicalCwd, { recursive: true }); - yield* fileSystem.symlink(physicalCwd, launchCwd); - // The session opens in /workspace; a relative TMPDIR puts Claude's - // task files under whichever cwd the CLI was launched in. Linux, - // because Claude Code's macOS build ignores TMPDIR. - const harness = yield* makeWakeHarnessWithOptions({ environment: { TMPDIR: "../tmp" } }); - const now = yield* DateTime.now; - const toolUseId = "toolu_foreground_bash"; - const taskId = "foreground-bash-task"; - const tasks = path.join( - root, - "real", - "tmp", - `claude-${process.getuid?.()}`, - "project", - WAKE_NATIVE_SESSION, - "tasks", - ); - yield* fileSystem.makeDirectory(tasks, { recursive: true }); - yield* fileSystem.writeFileString(path.join(tasks, `${taskId}.output`), "line 1\n"); - + const scheduleWakeupFrames = (input: { + readonly toolUseId: string; + readonly uuids: readonly [string, string]; + readonly toolInput: Record; + readonly scheduledFor: number; + }) => [ + claudeSdkFrame({ + type: "assistant", + message: { + model: "claude-sonnet-4-6", + id: `msg_${input.toolUseId}`, + type: "message", + role: "assistant", + content: [ + { type: "tool_use", id: input.toolUseId, name: "ScheduleWakeup", input: input.toolInput }, + ], + stop_reason: null, + stop_sequence: null, + usage: { input_tokens: 1, output_tokens: 1 }, + }, + parent_tool_use_id: null, + uuid: input.uuids[0], + session_id: WAKE_NATIVE_SESSION, + }), + claudeSdkFrame({ + type: "user", + message: { + role: "user", + content: [{ type: "tool_result", tool_use_id: input.toolUseId, content: "Scheduled." }], + }, + parent_tool_use_id: null, + uuid: input.uuids[1], + session_id: WAKE_NATIVE_SESSION, + tool_use_result: { + scheduledFor: input.scheduledFor, + clampedDelaySeconds: 60, + wasClamped: false, + }, + }), + ]; + + const startScheduleWakeupTurn = Effect.fnUntraced(function* ( + harness: Effect.Success, + result = makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000002103", + result: "Scheduled.", + }), + ) { + const now = yield* DateTime.now; + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-schedule-wakeup-1"), + text: "Wake up in a minute.", + attachments: [], + }), + ); + for (const frame of scheduleWakeupFrames({ + toolUseId: "toolu_schedule_wakeup_1", + uuids: ["00000000-0000-4000-8000-000000002101", "00000000-0000-4000-8000-000000002102"], + toolInput: { delaySeconds: 60, prompt: "say WOKE", reason: "wake", noop: false }, + scheduledFor: DateTime.toEpochMillis(now) + 60_000, + })) { + yield* Queue.offer(harness.sdkMessages, frame); + } + yield* Queue.offer(harness.sdkMessages, result); + yield* harness.awaitTerminal(1); + }); + + const publishedRoster = (harness: Effect.Success) => + providerThreadRosterEvents(harness.events).at(-1)?.providerThread.pendingBackgroundTasks ?? []; + + it.effect.each([ + { rescheduled: false, firedAfter: "60 seconds" }, + { rescheduled: true, firedAfter: "60 seconds" }, + // Claude fires one-shot wakeups early, so the wake init can precede scheduledFor. + { rescheduled: false, firedAfter: "30 seconds" }, + ] as const)( + "a fired ScheduleWakeup holds completion only while its wake turn reschedules (rescheduled=$rescheduled, fired after $firedAfter)", + ({ rescheduled, firedAfter }) => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + yield* startScheduleWakeupTurn(harness); + assert.isTrue(backgroundWorkHoldsCompletion(publishedRoster(harness))); + assert.isTrue( + yield* harness.hasPendingBackgroundWork, + "a wakeup past the idle timeout must keep the session from being released", + ); + + yield* TestClock.adjust(firedAfter); + yield* harness.offerAndWait(wakeTurnInit); + yield* harness.awaitContinuation(1); + const now = yield* DateTime.now; + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-schedule-wakeup-2"), + text: "say WOKE", + attachments: [], + providerTurnOrdinal: 2, + messageCreatedBy: "agent", + messageCreationSource: "provider", + }), + ); + if (rescheduled) { + for (const frame of scheduleWakeupFrames({ + toolUseId: "toolu_schedule_wakeup_2", + uuids: [ + "00000000-0000-4000-8000-000000002201", + "00000000-0000-4000-8000-000000002202", + ], + toolInput: { delaySeconds: 60, prompt: "say WOKE", reason: "loop", noop: false }, + scheduledFor: DateTime.toEpochMillis(now) + 60_000, + })) { + yield* harness.offerAndWait(frame); + } + } + yield* harness.offerAndWait( + makeAssistantTextFrame({ uuid: "00000000-0000-4000-8000-000000002203", text: "WOKE" }), + ); + yield* harness.offerAndWait( + makeResultFrame({ uuid: "00000000-0000-4000-8000-000000002204", result: "WOKE" }), + ); + yield* harness.awaitTerminal(2); + assert.equal(harness.terminalEvents()[1]?.status, "completed"); + assert.equal(backgroundWorkHoldsCompletion(publishedRoster(harness)), rescheduled); + assert.equal(yield* harness.hasPendingBackgroundWork, rescheduled); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ), + ); + + it.effect("a user turn before the wakeup fires keeps the pending wakeup's hold", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + yield* startScheduleWakeupTurn(harness); + yield* TestClock.adjust("10 seconds"); + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("attempt-claude-schedule-wakeup-user"), + text: "What time is it?", + attachments: [], + providerTurnOrdinal: 2, + }), + ); + yield* harness.offerAndWait(wakeTurnInit); + yield* harness.offerAndWait( + makeAssistantTextFrame({ uuid: "00000000-0000-4000-8000-000000002401", text: "Noon." }), + ); + yield* harness.offerAndWait( + makeResultFrame({ uuid: "00000000-0000-4000-8000-000000002402", result: "Noon." }), + ); + yield* harness.awaitTerminal(2); + assert.equal(harness.terminalEvents()[1]?.status, "completed"); + assert.isTrue(backgroundWorkHoldsCompletion(publishedRoster(harness))); + assert.isTrue(yield* harness.hasPendingBackgroundWork); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ), + ); + + it.effect("a task notification waking Claude just before the wakeup keeps its hold", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-schedule-wakeup-task"), + text: "Run the build in the background and check back in two minutes.", + attachments: [], + }), + ); + yield* Queue.offer(harness.sdkMessages, wakeTaskStarted); + for (const frame of scheduleWakeupFrames({ + toolUseId: "toolu_schedule_wakeup_fallback", + uuids: ["00000000-0000-4000-8000-000000002501", "00000000-0000-4000-8000-000000002502"], + toolInput: { delaySeconds: 120, prompt: "check the build", reason: "fallback" }, + scheduledFor: DateTime.toEpochMillis(now) + 120_000, + })) { + yield* Queue.offer(harness.sdkMessages, frame); + } + yield* Queue.offer(harness.sdkMessages, turnOneResult); + yield* harness.awaitTerminal(1); + + yield* TestClock.adjust("60 seconds"); + // Claude runs an empty turn for the notification and opens the real one right after, + // both before the wake run attaches. + yield* harness.offerAndWait(wakeNotification); + yield* Queue.offer(harness.sdkMessages, wakeTurnInit); + yield* Queue.offer( + harness.sdkMessages, + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000002503", + result: "", + numTurns: 0, + origin: { kind: "task-notification" }, + terminalReason: null, + }), + ); + yield* harness.offerAndWait( + claudeSdkFrame({ ...wakeTurnInit, uuid: "00000000-0000-4000-8000-000000002504" }), + ); + yield* harness.awaitContinuation(1); + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("attempt-claude-schedule-wakeup-task-2"), + text: "Background task completed.", + attachments: [], + providerTurnOrdinal: 2, + messageCreatedBy: "agent", + messageCreationSource: "provider", + }), + ); + yield* harness.offerAndWait(wakeAssistant); + yield* harness.offerAndWait(wakeResult); + yield* harness.awaitTerminal(2); + assert.equal(harness.terminalEvents()[1]?.status, "completed"); + assert.isTrue( + backgroundWorkHoldsCompletion(publishedRoster(harness)), + "the wakeup is still pending after the task notification's wake turn", + ); + assert.isTrue(yield* harness.hasPendingBackgroundWork); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ), + ); + + it.effect.each([{ emptyResult: true }, { emptyResult: false }])( + "an early wakeup after an idle task notification releases its hold (empty result: $emptyResult)", + ({ emptyResult }) => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-schedule-wakeup-notified"), + text: "Run the build in the background and check back in two minutes.", + attachments: [], + }), + ); + yield* Queue.offer(harness.sdkMessages, wakeTaskStarted); + for (const frame of scheduleWakeupFrames({ + toolUseId: "toolu_schedule_wakeup_notified", + uuids: ["00000000-0000-4000-8000-000000002601", "00000000-0000-4000-8000-000000002602"], + toolInput: { delaySeconds: 120, prompt: "check the build", reason: "fallback" }, + scheduledFor: DateTime.toEpochMillis(now) + 120_000, + })) { + yield* Queue.offer(harness.sdkMessages, frame); + } + yield* Queue.offer(harness.sdkMessages, turnOneResult); + yield* harness.awaitTerminal(1); + + yield* TestClock.adjust("10 seconds"); + yield* harness.offerAndWait(wakeNotification); + if (emptyResult) { + yield* harness.offerAndWait( + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000002603", + result: "", + numTurns: 0, + origin: { kind: "task-notification" }, + terminalReason: null, + }), + ); + } + + yield* TestClock.adjust("80 seconds"); + yield* harness.offerAndWait(wakeTurnInit); + yield* harness.awaitContinuation(1); + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("attempt-claude-schedule-wakeup-notified-2"), + text: "check the build", + attachments: [], + providerTurnOrdinal: 2, + messageCreatedBy: "agent", + messageCreationSource: "provider", + }), + ); + yield* harness.offerAndWait( + makeAssistantTextFrame({ + uuid: "00000000-0000-4000-8000-000000002604", + text: "Built.", + }), + ); + yield* harness.offerAndWait( + makeResultFrame({ uuid: "00000000-0000-4000-8000-000000002605", result: "Built." }), + ); + yield* harness.awaitTerminal(2); + assert.equal(harness.terminalEvents()[1]?.status, "completed"); + assert.isFalse(backgroundWorkHoldsCompletion(publishedRoster(harness))); + assert.isFalse(yield* harness.hasPendingBackgroundWork); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ), + ); + + it.effect("a failed turn on a live Claude process keeps the pending wakeup's hold", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + yield* startScheduleWakeupTurn( + harness, + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000002103", + result: "API Error: 529 overloaded", + subtype: "error_during_execution", + isError: true, + errors: ["API Error: 529 overloaded"], + terminalReason: "model_error", + }), + ); + assert.equal(harness.terminalEvents()[0]?.status, "failed"); + assert.isTrue(backgroundWorkHoldsCompletion(publishedRoster(harness))); + assert.isTrue(yield* harness.hasPendingBackgroundWork); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ), + ); + + it.effect("ScheduleWakeup stop:true releases the pending wakeup's hold", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-schedule-wakeup-stop"), + text: "Schedule a wakeup, then cancel it.", + attachments: [], + }), + ); + const frames = [ + ...scheduleWakeupFrames({ + toolUseId: "toolu_schedule_wakeup_set", + uuids: ["00000000-0000-4000-8000-000000002301", "00000000-0000-4000-8000-000000002302"], + toolInput: { delaySeconds: 60, prompt: "say WOKE", reason: "wake", noop: false }, + scheduledFor: DateTime.toEpochMillis(now) + 60_000, + }), + ...scheduleWakeupFrames({ + toolUseId: "toolu_schedule_wakeup_stop", + uuids: ["00000000-0000-4000-8000-000000002303", "00000000-0000-4000-8000-000000002304"], + toolInput: { stop: true }, + scheduledFor: 0, + }), + makeResultFrame({ uuid: "00000000-0000-4000-8000-000000002305", result: "Cancelled." }), + ]; + for (const frame of frames) yield* Queue.offer(harness.sdkMessages, frame); + yield* harness.awaitTerminal(1); + assert.isFalse(backgroundWorkHoldsCompletion(publishedRoster(harness))); + assert.isFalse(yield* harness.hasPendingBackgroundWork); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ), + ); + + it.effect("Stop on a thread waiting for a ScheduleWakeup closes Claude and drops the hold", () => + Effect.scoped( + Effect.gen(function* () { + let closes = 0; + const harness = yield* makeWakeHarnessWithOptions({ + close: (sdkMessages) => + Effect.sync(() => { + closes++; + }).pipe(Effect.andThen(Queue.shutdown(sdkMessages))), + }); + yield* startScheduleWakeupTurn(harness); + const settledThread = providerThreadRosterEvents(harness.events).at(-1)?.providerThread; + assert.isTrue(backgroundWorkHoldsCompletion(settledThread?.pendingBackgroundTasks ?? [])); + // The wakeup alone must not refuse a model or mode change. + assert.isFalse(yield* harness.runtime.hasPendingBackgroundWorkForThread!(settledThread!)); + + yield* harness.runtime.interruptTurn({ + providerThread: settledThread ?? harness.providerThread, + providerTurnId: harness.terminalEvents()[0]!.providerTurnId, + requestRuntimeRestart: true, + }); + + assert.equal(closes, 1, "the wakeup lives in the CLI process, so Stop must close it"); + yield* harness.until(() => publishedRoster(harness).length === 0); + assert.isFalse(yield* harness.hasPendingBackgroundWork); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ), + ); + + it.effect.skipIf(process.getuid === undefined)( + "tails foreground Bash output under the physical cwd the query launched in", + () => + Effect.scoped( + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-claude-task-output-cwd-", + }); + const physicalCwd = path.join(root, "real", "project"); + const launchCwd = path.join(root, "link"); + yield* fileSystem.makeDirectory(physicalCwd, { recursive: true }); + yield* fileSystem.symlink(physicalCwd, launchCwd); + // The session opens in /workspace; a relative TMPDIR puts Claude's + // task files under whichever cwd the CLI was launched in. Linux, + // because Claude Code's macOS build ignores TMPDIR. + const harness = yield* makeWakeHarnessWithOptions({ environment: { TMPDIR: "../tmp" } }); + const now = yield* DateTime.now; + const toolUseId = "toolu_foreground_bash"; + const taskId = "foreground-bash-task"; + const tasks = path.join( + root, + "real", + "tmp", + `claude-${process.getuid?.()}`, + "project", + WAKE_NATIVE_SESSION, + "tasks", + ); + yield* fileSystem.makeDirectory(tasks, { recursive: true }); + yield* fileSystem.writeFileString(path.join(tasks, `${taskId}.output`), "line 1\n"); + yield* harness.runtime.startTurn( makeClaudeTestTurnInput({ threadId: harness.threadId, @@ -10697,23 +11976,151 @@ describe("ClaudeAdapterV2 background wake turns", () => { result: "Started another build.", }), ); - yield* changes.until(() => terminals().length === 2); - yield* Queue.offer( - processQueues[1]!, - claudeSdkFrame({ - ...wakeNotification, - task_id: replacementTaskId, - uuid: "00000000-0000-4000-8000-000000000907", - }), + yield* changes.until(() => terminals().length === 2); + yield* Queue.offer( + processQueues[1]!, + claudeSdkFrame({ + ...wakeNotification, + task_id: replacementTaskId, + uuid: "00000000-0000-4000-8000-000000000907", + }), + ); + yield* Queue.offer( + processQueues[1]!, + makeAssistantTextFrame({ + uuid: "00000000-0000-4000-8000-000000000908", + text: WAKE_ASSISTANT_TEXT, + }), + ); + yield* changes.until(() => continuationRequests.length === 1); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ), + ); + + const openAdapterWhoseFirstQueryEndsBeforeItsFirstFrame = Effect.fnUntraced(function* ( + name: string, + ) { + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const attachmentsDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: `t3-claude-v2-${name}-`, + }); + const deadStreamEnded = yield* Deferred.make(); + let opened = 0; + const adapter = yield* ClaudeAdapterV2.makeClaudeAdapterV2({ + instanceId: ClaudeAdapterV2.CLAUDE_DEFAULT_INSTANCE_ID, + settings: DEFAULT_CLAUDE_SETTINGS, + environment: {}, + attachmentsDir, + // startTurn reads the auto-compact window from settings between opening the query and + // attaching the turn; holding that read until the CLI's stream has ended pins the race. + fileSystem: { + ...fileSystem, + readFileString: (filePath, encoding) => + opened === 1 && filePath.endsWith("settings.json") + ? Deferred.await(deadStreamEnded).pipe( + Effect.andThen(fileSystem.readFileString(filePath, encoding)), + ) + : fileSystem.readFileString(filePath, encoding), + }, + path: yield* Path.Path, + random: yield* SecureRandom.SecureRandom, + idAllocator, + continuationRequests: { offer: () => Effect.void }, + queryRunner: { + allocateSessionId: Effect.succeed(`native-${name}`), + open: () => + Effect.gen(function* () { + opened += 1; + const sdkMessages = yield* Queue.unbounded(); + return { + // The first CLI exits without writing a frame. + messages: + opened === 1 + ? Stream.empty.pipe(Stream.ensuring(Deferred.succeed(deadStreamEnded, undefined))) + : Stream.fromQueue(sdkMessages), + offer: () => Effect.void, + setModel: () => Effect.void, + applyFlagSettings: () => Effect.void, + setPermissionMode: () => Effect.void, + backgroundTasks: () => Effect.succeed(false), + mcpServerStatus: Effect.succeed([]), + readMcpResource: () => Effect.die("unexpected MCP resource read"), + callMcpTool: () => Effect.die("unexpected MCP tool call"), + listPermissionRules: Effect.die("unexpected permission rules read"), + stopTask: () => Effect.void, + interrupt: Effect.void, + close: Queue.shutdown(sdkMessages), + }; + }), + forkSession: () => Effect.die("unused forkSession"), + subagentLaunchToolUseId: () => Effect.succeed(null), + assertComplete: Effect.void, + }, + }); + const threadId = ThreadId.make(`thread-${name}`); + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make(`provider-session-${name}`), + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + const now = yield* DateTime.now; + const turnInput = (attempt: string, providerTurnOrdinal: number) => + makeClaudeTestTurnInput({ + threadId, + providerThread, + now, + attemptId: RunAttemptId.make(`attempt-${name}-${attempt}`), + text: "Continue with the plan.", + attachments: [], + providerTurnOrdinal, + }); + return { runtime, turnInput }; + }); + + it.effect("fails a turn whose Claude query ends before its first frame", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* openAdapterWhoseFirstQueryEndsBeforeItsFirstFrame( + "claude-query-dies-before-frame", + ); + const started = yield* harness.runtime + .startTurn(harness.turnInput("a", 1)) + .pipe(Effect.exit); + assert.isTrue( + Exit.isFailure(started), + "a turn whose Claude query ended before its first frame started running on the dead query", ); - yield* Queue.offer( - processQueues[1]!, - makeAssistantTextFrame({ - uuid: "00000000-0000-4000-8000-000000000908", - text: WAKE_ASSISTANT_TEXT, - }), + if (Exit.isFailure(started)) { + assert.include( + Cause.pretty(started.cause), + "Claude provider query ended before the turn could start.", + ); + } + const followUp = yield* harness.runtime + .startTurn(harness.turnInput("b", 2)) + .pipe(Effect.exit); + assert.isTrue( + Exit.isSuccess(followUp), + `the turn whose query ended before its first frame is still active: ${ + Exit.isFailure(followUp) ? Cause.pretty(followUp.cause) : "" + }`, ); - yield* changes.until(() => continuationRequests.length === 1); }).pipe( Effect.provide( Layer.mergeAll( @@ -12143,6 +13550,161 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + it.effect("offers a later wake after an earlier wake's continuation dispatch failed", () => + Effect.scoped( + Effect.gen(function* () { + const SECOND_TASK_ID = "b2ndwake"; + const SECOND_TOOL_USE_ID = "toolu_second_wake"; + const SECOND_SUMMARY = 'Background command "Second sleep" completed (exit code 0)'; + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-wake-dispatch-failed-1"), + text: "Run two builds in the background.", + attachments: [], + }), + ); + yield* Queue.offer(harness.sdkMessages, wakeTaskStarted); + yield* Queue.offer( + harness.sdkMessages, + claudeSdkFrame({ + ...wakeTaskStarted, + task_id: SECOND_TASK_ID, + tool_use_id: SECOND_TOOL_USE_ID, + description: "Second sleep", + uuid: "00000000-0000-4000-8000-000000000121", + }), + ); + yield* Queue.offer(harness.sdkMessages, turnOneResult); + yield* harness.awaitTerminal(1); + + yield* harness.offerAndWait(wakeNotification); + yield* harness.offerAndWait(wakeTurnInit); + assert.lengthOf(harness.continuationRequests, 1); + yield* harness.offerAndWait(wakeAssistant); + yield* harness.offerAndWait(wakeResult); + + // A failed dispatch never attaches the wake turn; clearIfCurrent is the only release. + const failed = harness.continuationRequests[0]!; + assert.isDefined(failed.clearIfCurrent); + yield* failed.clearIfCurrent!(); + + yield* harness.offerAndWait( + claudeSdkFrame({ + ...wakeNotification, + task_id: SECOND_TASK_ID, + tool_use_id: SECOND_TOOL_USE_ID, + output_file: `/tmp/claude-replay/tasks/${SECOND_TASK_ID}.output`, + summary: SECOND_SUMMARY, + uuid: "00000000-0000-4000-8000-000000000123", + }), + ); + yield* harness.offerAndWait( + claudeSdkFrame({ ...wakeTurnInit, uuid: "00000000-0000-4000-8000-000000000124" }), + ); + assert.lengthOf(harness.continuationRequests, 2); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ), + ); + + it.effect("keeps a newer wake latched when a stale continuation request is cleared", () => + Effect.scoped( + Effect.gen(function* () { + const SECOND_TASK_ID = "b2ndwake"; + const SECOND_TOOL_USE_ID = "toolu_second_wake"; + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-wake-stale-clear-1"), + text: "Run two builds in the background.", + attachments: [], + }), + ); + yield* Queue.offer(harness.sdkMessages, wakeTaskStarted); + yield* Queue.offer( + harness.sdkMessages, + claudeSdkFrame({ + ...wakeTaskStarted, + task_id: SECOND_TASK_ID, + tool_use_id: SECOND_TOOL_USE_ID, + description: "Second sleep", + uuid: "00000000-0000-4000-8000-000000000131", + }), + ); + yield* Queue.offer(harness.sdkMessages, turnOneResult); + yield* harness.awaitTerminal(1); + + yield* harness.offerAndWait(wakeNotification); + yield* harness.offerAndWait(wakeTurnInit); + assert.lengthOf(harness.continuationRequests, 1); + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-wake-stale-clear-2"), + text: "Background task completed.", + attachments: [], + providerTurnOrdinal: 2, + messageCreatedBy: "agent", + messageCreationSource: "provider", + }), + ); + yield* harness.offerAndWait(wakeAssistant); + yield* harness.offerAndWait(wakeResult); + yield* harness.awaitTerminal(2); + + yield* harness.offerAndWait( + claudeSdkFrame({ + ...wakeNotification, + task_id: SECOND_TASK_ID, + tool_use_id: SECOND_TOOL_USE_ID, + output_file: `/tmp/claude-replay/tasks/${SECOND_TASK_ID}.output`, + uuid: "00000000-0000-4000-8000-000000000133", + }), + ); + yield* harness.offerAndWait( + claudeSdkFrame({ ...wakeTurnInit, uuid: "00000000-0000-4000-8000-000000000134" }), + ); + assert.lengthOf(harness.continuationRequests, 2); + + yield* harness.continuationRequests[0]!.clearIfCurrent!(); + yield* harness.offerAndWait( + claudeSdkFrame({ ...wakeAssistant, uuid: "00000000-0000-4000-8000-000000000135" }), + ); + assert.lengthOf(harness.continuationRequests, 2); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ), + ); + it.effect.each([true, false])( "carries a rejected wake rate limit into the continuation failure (event=%s)", (standaloneEvent) => @@ -15282,11 +16844,11 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); - it.effect("gives a nested subagent its own effort, else the one its owner runs at", () => + it.effect("records only explicit effort for nested subagents", () => Effect.scoped( Effect.gen(function* () { const OUTER_TOOL_USE_ID = "toolu-effort-outer"; - const INHERITING_TOOL_USE_ID = "toolu-effort-nested-inheriting"; + const UNSPECIFIED_TOOL_USE_ID = "toolu-effort-nested-unspecified"; const OVERRIDING_TOOL_USE_ID = "toolu-effort-nested-overriding"; const harness = yield* makeWakeHarness; const now = yield* DateTime.now; @@ -15352,14 +16914,14 @@ describe("ClaudeAdapterV2 background wake turns", () => { type: "message", role: "assistant", content: [ - agentCall(INHERITING_TOOL_USE_ID), + agentCall(UNSPECIFIED_TOOL_USE_ID), agentCall(OVERRIDING_TOOL_USE_ID, "max"), ], }, uuid: "00000000-0000-4000-8000-000000000223", session_id: WAKE_NATIVE_SESSION, }), - taskStarted(INHERITING_TOOL_USE_ID, "00000000-0000-4000-8000-000000000224"), + taskStarted(UNSPECIFIED_TOOL_USE_ID, "00000000-0000-4000-8000-000000000224"), taskStarted(OVERRIDING_TOOL_USE_ID, "00000000-0000-4000-8000-000000000225"), makeResultFrame({ uuid: "00000000-0000-4000-8000-000000000226", @@ -15384,7 +16946,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { ); assert.deepEqual(Object.fromEntries(efforts), { [`task-${OUTER_TOOL_USE_ID}`]: "low", - [`task-${INHERITING_TOOL_USE_ID}`]: "low", + [`task-${UNSPECIFIED_TOOL_USE_ID}`]: undefined, [`task-${OVERRIDING_TOOL_USE_ID}`]: "max", }); const childEfforts = harness.events.flatMap((event) => @@ -15392,7 +16954,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { ? [getModelSelectionStringOptionValue(event.appThread.modelSelection, "effort")] : [], ); - assert.deepEqual(childEfforts, ["low", "low", "max"]); + assert.deepEqual(childEfforts, ["low", undefined, "max"]); }).pipe( Effect.provide( Layer.mergeAll( @@ -16763,6 +18325,132 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + // regression: #26082 a killed Claude process names its signal on the failed turn + it.effect("names the signal that killed the Claude process on the failed turn", () => + Effect.scoped( + Effect.gen(function* () { + const changes = yield* makeChangeSignal; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const attachmentsDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-claude-v2-process-signal-", + }); + const sdkMessages = yield* Queue.unbounded< + SDKMessage, + ClaudeAdapterV2.ClaudeAgentSdkQueryRunnerError + >(); + const events: Array = []; + const adapter = yield* ClaudeAdapterV2.makeClaudeAdapterV2({ + instanceId: ClaudeAdapterV2.CLAUDE_DEFAULT_INSTANCE_ID, + settings: DEFAULT_CLAUDE_SETTINGS, + environment: {}, + attachmentsDir, + fileSystem, + path: yield* Path.Path, + random: yield* SecureRandom.SecureRandom, + idAllocator, + queryRunner: { + allocateSessionId: Effect.succeed(WAKE_NATIVE_SESSION), + open: () => + Effect.succeed({ + messages: Stream.fromQueue(sdkMessages), + offer: () => Effect.void, + setModel: () => Effect.void, + applyFlagSettings: () => Effect.void, + setPermissionMode: () => Effect.void, + backgroundTasks: () => Effect.succeed(false), + mcpServerStatus: Effect.succeed([]), + readMcpResource: () => Effect.die("unexpected MCP resource read"), + callMcpTool: () => Effect.die("unexpected MCP tool call"), + listPermissionRules: Effect.die("unexpected permission rules read"), + stopTask: () => Effect.void, + interrupt: Effect.void, + close: Queue.shutdown(sdkMessages), + }), + forkSession: () => Effect.die("unused forkSession"), + subagentLaunchToolUseId: () => Effect.succeed(null), + assertComplete: Effect.void, + }, + }); + const threadId = ThreadId.make("thread-claude-process-signal"); + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-claude-process-signal"), + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + yield* runtime.events.pipe( + Stream.runForEach((event) => + Effect.sync(() => { + events.push(event); + }).pipe(Effect.andThen(changes.notify)), + ), + Effect.forkScoped, + ); + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("attempt-claude-process-signal"), + text: "Write a long reply.", + attachments: [], + }), + ); + yield* Queue.offer( + sdkMessages, + makeAssistantTextFrame({ + uuid: "00000000-0000-4000-8000-000000026082", + text: "Here is the first part", + }), + ); + // Mirrors ProcessTransport.getProcessExitError in @anthropic-ai/claude-agent-sdk. + const processExit = Object.assign( + new Error( + "Claude Code process terminated by signal SIGABRT. stderr: fork failed: Resource temporarily unavailable", + ), + { + telemetryMessage: "Claude Code process terminated by signal SIGABRT", + errorClass: "process_killed_by_signal", + signal: "SIGABRT", + }, + ); + yield* Queue.fail( + sdkMessages, + new ClaudeAdapterV2.ClaudeAgentSdkQueryRunnerError({ + method: "messages", + cause: processExit, + }), + ); + + yield* changes.until(() => events.some((event) => event.type === "turn.terminal")); + const terminal = events.find((event) => event.type === "turn.terminal"); + assert.equal(terminal?.status, "failed"); + if (terminal?.status !== "failed") return; + assert.include( + terminal.failure.message, + "Claude Code process terminated by signal SIGABRT", + ); + assert.notInclude(terminal.failure.message, "stderr"); + assert.equal(terminal.failure.code, "process_killed_by_signal"); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ), + ); + const WORKFLOW_TASK_ID = "task-workflow-run"; const WORKFLOW_TOOL_USE_ID = "toolu-workflow-run"; const makeWorkflowHarnessWithOptions = ( @@ -16866,7 +18554,10 @@ describe("ClaudeAdapterV2 background wake turns", () => { phaseTitle: "Alpha", model: input.model ?? "claude-opus-5[1m]", promptPreview: input.prompt ?? "Reply with exactly: A1", - ...(input.startedAt === undefined ? {} : { startedAt: input.startedAt }), + // Claude reports a scheduled member as `start` and only sets startedAt once it runs. + ...(input.startedAt === undefined && input.state !== "start" + ? {} + : { startedAt: input.startedAt ?? 0 }), ...(input.lastProgressAt === undefined ? {} : { lastProgressAt: input.lastProgressAt }), ...(input.durationMs === undefined ? {} : { durationMs: input.durationMs }), ...(input.attempt === undefined ? {} : { attempt: input.attempt }), @@ -16883,6 +18574,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { phaseIndex: 1, phaseTitle: "Alpha", promptPreview: "Reply with exactly: A2", + ...((input.secondState ?? input.state) === "start" ? { startedAt: 0 } : {}), ...(input.state === "done" ? { resultPreview: "A2 excerpt" } : {}), }, ], @@ -17376,6 +19068,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { uuid: "00000000-0000-4000-8000-000000001005", state: "start", secondState: "queued", + startedAt: DateTime.toEpochMillis(memberStartedAt), }), ); yield* harness.until(() => coordinatorTurnItems().length > turnItemsBeforeRepeat); @@ -17399,6 +19092,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { uuid: "00000000-0000-4000-8000-000000001017", state: "start", secondState: "queued", + startedAt: DateTime.toEpochMillis(memberStartedAt), label: "alpha:review", prompt: "Review the change", model: "claude-sonnet-4-6", diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.ts index 6fc79a5497d6..81b613d0a15d 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.ts @@ -33,6 +33,7 @@ import * as ServerConfig from "../../config.ts"; import * as ClaudeAdapterV2 from "./ClaudeAdapterV2.ts"; import { claudeTaskOutputRoot } from "./ClaudeTaskOutputTail.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; +import * as ProviderInstallationGate from "../../provider/ProviderInstallationGate.ts"; import { ProviderAdapterDriverCreateError } from "@t3tools/provider-core/server/adapterDriver"; import * as ProviderAdapterRegistry from "../ProviderAdapterRegistry.ts"; import { randomUuidV4 } from "@t3tools/provider-core/server/randomUuid"; @@ -1053,6 +1054,7 @@ function layerClaudeProviderAdapterRegistryReplay( Layer.mergeAll( layerClaudeAgentSdkReplay(transcript, options), IdAllocator.layer, + ProviderInstallationGate.layer, NodeServices.layer, NodeCrypto.layer, layerServerConfig, @@ -2166,15 +2168,11 @@ async function recordClaudeForkSessionQuery(input: { type: "session.fork", sessionId: input.sessionId, options: { - dir: sanitizedReplayCwd(input.scenario), upToMessageId, }, }, }); - const forked = await forkSession(input.sessionId, { - dir: input.cwd, - upToMessageId, - }); + const forked = await forkSession(input.sessionId, { upToMessageId }); forkedNativeSessionIds.push(forked.sessionId); input.entries.push({ type: "emit_inbound", diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 1494de3717f3..2ad2812e35ff 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -81,6 +81,7 @@ import { import { AgentScope } from "@t3tools/shared/AgentScope"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { applyClaudePromptEffortPrefix } from "@t3tools/shared/model"; +import { compareSemverVersions } from "@t3tools/shared/semver"; import { CLAUDE_RESUME_COMPACTION_COMPACT_ANSWER, CLAUDE_RESUME_COMPACTION_NEVER_ANSWER, @@ -93,6 +94,7 @@ import { TrimmedNonEmptyString, defaultInstanceIdForDriver, isCompactionCommandMessage, + type MessageId, type ModelSelection, type OrchestrationV2ConversationMessage, type OrchestrationV2ExecutionNode, @@ -141,6 +143,7 @@ import * as Effect from "effect/Effect"; import * as Equal from "effect/Equal"; import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; +import * as Schedule from "effect/Schedule"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; @@ -153,14 +156,17 @@ import * as Stream from "effect/Stream"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import { snapshotMcpApp } from "../../mcpApps/McpAppSnapshot.ts"; +import * as ProviderInstallationGate from "../../provider/ProviderInstallationGate.ts"; import { planClaudeSkillDispatch, type ClaudeSkillTarget, } from "../../provider/Drivers/ClaudeSkillDispatch.ts"; import { claudeLaunchArgDirectories, + customClaudeApiEndpointHost, discoverClaudeSkills, claudeSettingsPaths, + readClaudeSettingsEnv, } from "../../provider/Drivers/ClaudeSkills.ts"; import { compileClaudeModelSelection, @@ -168,6 +174,7 @@ import { } from "../../claudeModelOptions.ts"; import * as ServerConfig from "../../config.ts"; import * as PortScanner from "../../preview/PortScanner.ts"; +import * as ProcessRunner from "../../processRunner.ts"; import * as ServerSettings from "../../serverSettings.ts"; import { expandHomePath } from "@t3tools/provider-core/server/pathExpansion"; import { signalProcessGroup } from "@t3tools/provider-core/server/processGroup"; @@ -332,6 +339,7 @@ export const CLAUDE_DEFAULT_INSTANCE_ID = defaultInstanceIdForDriver(CLAUDE_PROV const CLAUDE_SCRIPT_EXTENSIONS = [".js", ".mjs", ".tsx", ".ts", ".jsx"]; // The SDK has no control-request deadline; the CLI waits up to 30 s per MCP server. const CLAUDE_MCP_REGISTRATION_TIMEOUT = "90 seconds"; +const CLAUDE_BOOT_TIMEOUT = "5 minutes"; // Claude can consume a subagent result before the launching turn ends, leaving // its notification with no native wake turn to answer the continuation. const CLAUDE_WAKE_TURN_START_DEADLINE = "2 minutes"; @@ -348,6 +356,8 @@ const CLAUDE_STOP_CONFIRM_TIMEOUT_MS = 10_000; const CLAUDE_STDERR_TAIL_LENGTH = 2_048; // The process group is killed on exit, so its stderr closes promptly. const CLAUDE_STDERR_DRAIN_TIMEOUT = Duration.seconds(1); +// How often the CLI's process tree is recorded when no agent scope will stop what it leaves. +const CLAUDE_DESCENDANT_POLL = "1 second"; // Claude Code's refusal of bypassPermissions as root, which it explains only on stderr. const CLAUDE_ROOT_REFUSAL = "--dangerously-skip-permissions cannot be used with root/sudo privileges for security reasons"; @@ -367,7 +377,7 @@ export function spawnClaudeProcessGroup( onExit?: () => void, onStderr: (text: string) => void = () => {}, onStderrClose: () => void = () => {}, -): ClaudeSpawnedProcess { +): ClaudeSpawnedProcess & Pick { const child = NodeChildProcess.spawn( "/bin/sh", ["-c", CLAUDE_GROUP_WATCH_SCRIPT, "t3-claude", options.command, ...options.args], @@ -388,7 +398,60 @@ export function spawnClaudeProcessGroup( onExit?.(); }); // Node's typings make stdin and stdout nullable once stdio has a fourth entry; both are pipes. - return child as ClaudeSpawnedProcess; + return child as ClaudeSpawnedProcess & Pick; +} + +interface ClaudeProcessRow { + readonly pid: number; + readonly ppid: number; + readonly pgid: number; + readonly started: string; +} + +/** Parses `ps -A -o pid=,ppid=,pgid=,lstart=`; pid plus start time tells a process from a reused pid. */ +function parseClaudeProcessTable(table: string): ReadonlyArray { + return table.split("\n").flatMap((line) => { + const [pid, ppid, pgid, ...started] = line.trim().split(/\s+/); + const row = { + pid: Number(pid), + ppid: Number(ppid), + pgid: Number(pgid), + started: started.join(" "), + }; + return Number.isSafeInteger(row.pid) && + Number.isSafeInteger(row.ppid) && + Number.isSafeInteger(row.pgid) && + row.started + ? [row] + : []; + }); +} + +// A job leading its own group takes the processes it started since with it. +function killClaudeDescendant(row: ClaudeProcessRow) { + try { + if (row.pgid === row.pid) signalProcessGroup(row.pid, "SIGKILL"); + else process.kill(row.pid, "SIGKILL"); + } catch {} +} + +function claudeDescendants( + rows: ReadonlyArray, + rootPids: ReadonlyArray, +): ReadonlyArray { + const children = Map.groupBy(rows, (row) => row.ppid); + const found: Array = []; + const seen = new Set(rootPids); + const pending = [...rootPids]; + for (let parent = pending.pop(); parent !== undefined; parent = pending.pop()) { + for (const child of children.get(parent) ?? []) { + if (seen.has(child.pid)) continue; + seen.add(child.pid); + found.push(child); + pending.push(child.pid); + } + } + return found; } export const ClaudeProviderCapabilitiesV2 = { @@ -614,10 +677,12 @@ export class ClaudeCodeRootRefusalError extends Schema.TaggedError()( "ClaudeMcpRegistrationTimeoutError", - {}, + { phase: Schema.Literals(["initialize", "setMcpServers"]) }, ) { override get message(): string { - return "Claude timed out while registering its MCP tools. Retry the turn; if it keeps failing, check Claude and the server logs."; + return this.phase === "initialize" + ? "Claude timed out while starting up. Retry the turn; if it keeps failing, check Claude and the server logs." + : "Claude timed out while registering its MCP tools. Retry the turn; if it keeps failing, check Claude and the server logs."; } } @@ -716,6 +781,26 @@ function queryRunnerError(cause: unknown, method: string): ClaudeAgentSdkQueryRu : new ClaudeAgentSdkQueryRunnerError({ cause, method }); } +const CLAUDE_PROCESS_EXIT_ERROR_CLASSES: ReadonlySet = new Set([ + "process_killed_by_signal", + "process_exited_nonzero", +]); + +// The SDK's exit Error appends Claude's stderr tail to `message`; `telemetryMessage` omits it. +function claudeProcessExitFailure( + cause: Cause.Cause, +): { readonly message: string; readonly code: string } | undefined { + const error = Cause.squash(cause); + const sdkError = isClaudeAgentSdkQueryRunnerError(error) ? error.cause : error; + if (typeof sdkError !== "object" || sdkError === null) return undefined; + const { errorClass, telemetryMessage } = sdkError as Record; + return typeof errorClass === "string" && + CLAUDE_PROCESS_EXIT_ERROR_CLASSES.has(errorClass) && + typeof telemetryMessage === "string" + ? { message: `${telemetryMessage}.`, code: errorClass } + : undefined; +} + function closeClaudeQuery(queryRuntime: ClaudeQuery) { return Effect.try({ try: () => queryRuntime.close(), @@ -1101,6 +1186,41 @@ export const layerQueryRunner: Layer.Layer< const runFork = Effect.runForkWith(yield* Effect.context()); const hostEnvironment = yield* HostProcess.Environment; const portDiscovery = yield* PortScanner.PortDiscovery; + const processRunner = yield* ProcessRunner.make(); + const readProcessTable = processRunner + .run({ command: "ps", args: ["-A", "-o", "pid=,ppid=,pgid=,lstart="], timeout: "5 seconds" }) + .pipe( + Effect.map((result) => (result.code === 0 ? parseClaudeProcessTable(result.stdout) : [])), + Effect.orElseSucceed(() => []), + ); + // Without an agent scope, a job that left the CLI's group escapes the group kill and is + // re-parented once the CLI exits. So the CLI's tree is recorded while it runs, and what is + // still the same process (pid and start time) is stopped after it exits. + const trackDescendants = (cliPid: number) => { + const tracked = new Map(); + const identity = (row: ClaudeProcessRow) => `${row.pid} ${row.started}`; + // After exit the CLI's pid may belong to an unrelated process, so only tracked jobs are roots. + const refresh = Effect.fn("ClaudeAgentSdkQueryRunner.trackDescendants")(function* ( + cliRunning: boolean, + ) { + const rows = yield* readProcessTable; + const running = new Set(rows.map(identity)); + const roots = [ + ...(cliRunning ? [cliPid] : []), + ...[...tracked.values()].flatMap((row) => (running.has(identity(row)) ? [row.pid] : [])), + ]; + for (const row of claudeDescendants(rows, roots)) tracked.set(identity(row), row); + return running; + }); + const stop = refresh(false).pipe( + Effect.map((running) => { + for (const row of tracked.values()) { + if (running.has(identity(row))) killClaudeDescendant(row); + } + }), + ); + return { refresh: refresh(true).pipe(Effect.asVoid), stop }; + }; return ClaudeAgentSdkQueryRunner.of({ allocateSessionId: random.uuidV4(), @@ -1165,6 +1285,7 @@ export const layerQueryRunner: Layer.Layer< if (answered) return; stderrTail = (stderrTail + text).slice(-CLAUDE_STDERR_TAIL_LENGTH); }; + let refreshDescendants: Effect.Effect = Effect.void; const spawnProcess = options.spawnClaudeCodeProcess ?? ((spawnOptions: ClaudeSpawnOptions) => { @@ -1172,7 +1293,23 @@ export const layerQueryRunner: Layer.Layer< // Node can report "exit" before it has read the stderr the CLI wrote. let stderrDone = () => {}; stderrClosed = new Promise((resolve) => (stderrDone = resolve)); - return spawnClaudeProcessGroup(spawnOptions, stopScope, appendStderr, stderrDone); + const child = spawnClaudeProcessGroup( + spawnOptions, + stopScope, + appendStderr, + stderrDone, + ); + if (unit === undefined && child.pid !== undefined) { + const tracker = trackDescendants(child.pid); + const polling = runFork( + tracker.refresh.pipe(Effect.repeat(Schedule.spaced(CLAUDE_DESCENDANT_POLL))), + ); + child.once("exit", () => + runFork(Fiber.interrupt(polling).pipe(Effect.andThen(tracker.stop))), + ); + refreshDescendants = tracker.refresh; + } + return child; } const child = NodeChildProcess.spawn(spawnOptions.command, spawnOptions.args, { cwd: spawnOptions.cwd, @@ -1220,6 +1357,31 @@ export const layerQueryRunner: Layer.Layer< restore( Effect.gen(function* () { if (mcpServers !== undefined) { + const timedOut = (phase: ClaudeMcpRegistrationTimeoutError["phase"]) => () => + Effect.logWarning("orchestration-v2.claude-mcp-registration-timeout", { + threadId: input.threadId, + providerSessionId: input.providerSessionId, + phase, + }).pipe( + Effect.andThen( + Effect.fail( + queryRunnerError( + new ClaudeMcpRegistrationTimeoutError({ phase }), + phase, + ), + ), + ), + ); + // The SDK leaves query() boot unbounded, and it takes about a minute on a host + // with CPUs 12x oversubscribed, so it gets its own, longer deadline. + yield* Effect.promise(() => + queryRuntime.initializationResult().catch(() => undefined), + ).pipe( + Effect.timeoutOrElse({ + duration: CLAUDE_BOOT_TIMEOUT, + orElse: timedOut("initialize"), + }), + ); // Errors and mcp_status can echo headers; log only server names. const failed = yield* Effect.tryPromise({ try: () => queryRuntime.setMcpServers(mcpServers), @@ -1229,20 +1391,7 @@ export const layerQueryRunner: Layer.Layer< Effect.orElseSucceed(() => Object.keys(mcpServers)), Effect.timeoutOrElse({ duration: CLAUDE_MCP_REGISTRATION_TIMEOUT, - orElse: () => - Effect.logWarning("orchestration-v2.claude-mcp-registration-timeout", { - threadId: input.threadId, - providerSessionId: input.providerSessionId, - }).pipe( - Effect.andThen( - Effect.fail( - queryRunnerError( - new ClaudeMcpRegistrationTimeoutError(), - "setMcpServers", - ), - ), - ), - ), + orElse: timedOut("setMcpServers"), }), ); if (failed.length > 0) { @@ -1467,7 +1616,9 @@ export const layerQueryRunner: Layer.Layer< }), ), ), - close: Queue.shutdown(promptQueue).pipe( + // A job started since the last poll is recorded before the CLI is told to exit. + close: Effect.suspend(() => (processExited ? Effect.void : refreshDescendants)).pipe( + Effect.andThen(Queue.shutdown(promptQueue)), Effect.andThen(closeClaudeQuery(queryRuntime)), Effect.tap(() => logProtocolEvent({ @@ -1825,10 +1976,10 @@ export function makeClaudeQueryOptions(input: { } /** - * Checks the append launch arguments the way the Claude CLI would and reads - * the file relative to the process cwd. makeClaudeQueryOptions folds these - * flags out of extraArgs, so a value the CLI would have rejected has to be - * rejected here or it silently disappears. + * Checks the append launch arguments the way the Claude CLI would and reads the + * file (leading ~ expanded) relative to the thread's cwd. makeClaudeQueryOptions + * folds these flags out of extraArgs, so a value the CLI would have rejected has + * to be rejected here or it silently disappears. */ const readClaudeAppendSystemPromptFile = Effect.fnUntraced(function* (input: { readonly settings: ClaudeSettings | undefined; @@ -1853,7 +2004,10 @@ const readClaudeAppendSystemPromptFile = Effect.fnUntraced(function* (input: { flag: "--append-system-prompt-file", }); } - const filePath = input.path.resolve(input.cwd ?? ".", file); + const filePath = input.path.resolve( + input.cwd ?? ".", + expandHomePath(file, yield* HostProcess.HomeDirectory), + ); return yield* input.fileSystem .readFileString(filePath) .pipe( @@ -1867,8 +2021,9 @@ const decodeClaudeLaunchArgSettings = Schema.decodeUnknownEffect( /** * Reads the --settings launch argument: a value starting with a brace is inline - * JSON, anything else is a file path relative to the cwd. makeClaudeQueryOptions - * merges the result beneath T3's own settings. + * JSON, anything else is a file path that expands a leading ~ and is otherwise + * relative to the thread's cwd (the project or its worktree). + * makeClaudeQueryOptions merges the result beneath T3's own settings. */ const readClaudeLaunchArgSettings = Effect.fn("ClaudeAdapterV2.readLaunchArgSettings")( function* (input: { @@ -1889,7 +2044,10 @@ const readClaudeLaunchArgSettings = Effect.fn("ClaudeAdapterV2.readLaunchArgSett Effect.mapError((cause) => new ClaudeSettingsLaunchArgError({ cause })), )) as ClaudeSdkSettings; } - const filePath = input.path.resolve(input.cwd ?? ".", trimmed); + const filePath = input.path.resolve( + input.cwd ?? ".", + expandHomePath(trimmed, yield* HostProcess.HomeDirectory), + ); return (yield* input.fileSystem.readFileString(filePath).pipe( Effect.flatMap(decodeClaudeLaunchArgSettings), Effect.mapError((cause) => new ClaudeSettingsLaunchArgError({ path: filePath, cause })), @@ -1903,6 +2061,7 @@ export const CLAUDE_T3_MCP_TOOL_WILDCARD = "mcp__t3-code__*"; export const CLAUDE_READ_ONLY_T3_MCP_ALLOWED_TOOLS: ReadonlyArray = [ "mcp__t3-code__orchestrator_capabilities", "mcp__t3-code__list_scheduled_tasks", + "mcp__t3-code__t3_background_wait", "mcp__t3-code__t3_thread_list", "mcp__t3-code__t3_thread_wait", "mcp__t3-code__t3_worktree_status", @@ -3134,6 +3293,17 @@ function isClaudeRootTurnFrame(message: SDKMessage): boolean { ); } +// One stable id per native thread: a new ScheduleWakeup replaces the pending one. +const CLAUDE_SCHEDULED_WAKEUP_TASK: OrchestrationV2PendingBackgroundTask = { + taskId: "claude-scheduled-wakeup", + kind: "background_task", + description: "Scheduled wakeup", +}; + +// Claude Code jitters one-shot wakeups early but never before the requested time, which is at +// most 60 s before the minute-rounded scheduledFor it reports; the rest is slack. +const CLAUDE_SCHEDULED_WAKEUP_EARLY_FIRE_MARGIN_MS = 120_000; + function claudePendingBackgroundTasksFromRoster( roster: ReadonlyMap, awaitedTaskIds: ReadonlySet, @@ -3704,11 +3874,12 @@ export const awaitClaudeApprovalDecision = Effect.fn("awaitClaudeApprovalDecisio return yield* Effect.raceFirst(Deferred.await(decision), cancellation); }); -// Web and mobile offer a session grant when options are omitted, and this -// path only ever answers once. +// MCP servers such as Computer Use ask on every call and expect the client to +// remember a session grant, as Codex does. The CLI drops a reply's `_meta.persist`. const CLAUDE_MCP_ELICITATION_OPTIONS: ReadonlyArray = [ { decision: "cancel", label: "Cancel" }, { decision: "decline", label: "Decline" }, + { decision: "acceptForSession", label: "Always allow this session" }, { decision: "accept", label: "Approve" }, ]; @@ -3927,10 +4098,13 @@ function claudePromptLifecycle( function isClaudeResultForOtherTurn(input: { readonly message: SDKResultMessage; readonly promptUuid: string; + readonly steerUuids: ReadonlySet; readonly promptEchoMode: ClaudeLiveQueryContext["promptEchoMode"]; }): boolean { const echoed = claudeEchoedPromptUuids(input.message); - if (echoed.length > 0) return !echoed.includes(input.promptUuid); + if (echoed.length > 0) { + return !echoed.some((uuid) => uuid === input.promptUuid || input.steerUuids.has(uuid)); + } return ( input.promptEchoMode !== "unknown" && input.message.origin !== undefined && @@ -3984,6 +4158,25 @@ function gatewayCooldownResetAt( : undefined; } +const CLAUDE_UNSUPPORTED_VERSION_ERROR = + /^API Error: 400 Claude Code (\d+\.\d+\.\d+) does not support this model; version (\d+\.\d+\.\d+) or newer is required\./; + +/** The version a gateway sent in place of the session's own, when the session itself meets the requirement. */ +function claudeRewrittenVersion( + message: SDKResultMessage, + sessionVersion: string | undefined, +): string | undefined { + if (message.subtype !== "success" || message.terminal_reason !== "api_error") return undefined; + const [, reported, required] = CLAUDE_UNSUPPORTED_VERSION_ERROR.exec(message.result.trim()) ?? []; + if (reported === undefined || required === undefined || sessionVersion === undefined) { + return undefined; + } + return compareSemverVersions(reported, sessionVersion) < 0 && + compareSemverVersions(sessionVersion, required) >= 0 + ? reported + : undefined; +} + function providerFailureFromResult( message: SDKResultMessage, receivedAt: DateTime.Utc, @@ -4084,6 +4277,7 @@ function buildAssistantArtifacts(input: { readonly nativeItemId: string; readonly text: string; readonly ordinal: number; + readonly status?: "completed" | "interrupted"; readonly startedAt: DateTime.Utc; readonly completedAt: DateTime.Utc; }): { @@ -4117,7 +4311,7 @@ function buildAssistantArtifacts(input: { parentNodeId: input.turnInput.rootNodeId, rootNodeId: input.turnInput.rootNodeId, kind: "assistant_message", - status: "completed", + status: input.status ?? "completed", countsForRun: false, providerThreadId: input.turnInput.providerThread.id, providerTurnId: input.providerTurnId, @@ -4151,7 +4345,7 @@ function buildAssistantArtifacts(input: { nativeItemRef, parentItemId: null, ordinal: input.ordinal, - status: "completed", + status: input.status ?? "completed", title: null, startedAt: input.startedAt, completedAt: input.completedAt, @@ -4221,6 +4415,12 @@ interface ActiveClaudeTurnContext { fallbackText: string; fallbackNativeItemId: string; emittedNativeItemIds: Set; + // The root reply the latest assistant frame carried; a steer's aborted_streaming cuts it off. + lastReply: { + readonly nativeItemId: string; + readonly text: string; + readonly startedAt: DateTime.Utc; + } | null; // The CLI repeats a notice that ends a prompt as the turn's result text. noticeTexts: Set; }; @@ -4267,6 +4467,9 @@ interface ActiveClaudeTurnContext { compactionFailure: { readonly message: string | undefined } | null; // Steers end segments whose usage is excluded from the final result; bill them too, oldest first. readonly steeredSegmentUsage: Array; + // Set as finalization starts: a steer admitted later would stay pending + // forever, so it fails and the orchestrator follows up instead. + steersClosed: boolean; // Ledger growth of the steered segments; undefined once any segment's growth is unknown. steeredLedgerUsage: ClaudeLedgerUsage | undefined; readonly rootModels: Set; @@ -4278,6 +4481,19 @@ interface ActiveClaudeTurnContext { nativeWakeStarted: boolean; gatedFramesBeforeEcho: number; readonly heldRootFrames: Array; + // Steers Claude has not started or echoed yet, by prompt uuid. Claude holds + // a steer behind a running foreground tool call without saying so, and Stop + // drops it unread. + readonly pendingSteers: Map< + string, + { + readonly messageId: MessageId; + readonly text: string; + readonly liveQuery: ClaudeLiveQueryContext; + } + >; + // Claude may settle a turn with a result that echoes only a steer's uuid. + readonly steerUuids: Set; // Claude can run another model than requested, such as when an organization disallows it. executedModel: string | undefined; } @@ -4324,6 +4540,7 @@ interface ClaudeLiveQueryContext { receivedInit: boolean; // From system/init; feature-detects control requests such as MCP Apps. capabilities: ReadonlySet; + claudeCodeVersion: string | undefined; permissionMode: PermissionMode; stopping: boolean; readonly subagentsFromEarlierProcesses: ReadonlySet; @@ -4742,6 +4959,7 @@ export interface ClaudeAdapterV2Options { request: ProviderContinuationRequests.ProviderContinuationRequest, ) => Effect.Effect; }; + readonly installationGate?: ProviderInstallationGate.ProviderInstallationGate["Service"]; } export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( @@ -5091,6 +5309,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const pendingRuntimeRequests = yield* Ref.make( new Map(), ); + const mcpElicitationSessionGrants = yield* Ref.make(new Set()); // Background local_bash tasks can trigger wake turns on the query stream with no active provider turn. const lastTurnRouteByNativeThread = yield* Ref.make( new Map< @@ -5227,6 +5446,75 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( return { ...current, calls }; }); }; + // ScheduleWakeup reports no background task: its tool result is the only sign that Claude + // will start a turn by itself, and Claude keeps one pending wakeup per session. It stays + // out of pendingBackgroundTasksByNativeThread because a wakeup must not refuse a model or + // mode change; replacing the process cancels it, and that clears this record too. + const scheduleWakeupCalls = new Set(); + const scheduledWakeupsByNativeThread = new Map(); + // Native threads whose current turn Claude started by itself inside the early-fire margin. + const possibleEarlyWakeNativeThreads = new Set(); + const trackClaudeScheduledWakeup = Effect.fnUntraced(function* ( + nativeThreadId: string, + message: SDKMessage, + ) { + if (isClaudeTurnStartMessage(message)) { + const scheduledFor = scheduledWakeupsByNativeThread.get(nativeThreadId); + if (scheduledFor === undefined) return; + const now = DateTime.toEpochMillis(yield* DateTime.now); + if (now >= scheduledFor) { + scheduledWakeupsByNativeThread.delete(nativeThreadId); + } else if ( + (yield* Ref.get(activeTurn)) === null && + now >= scheduledFor - CLAUDE_SCHEDULED_WAKEUP_EARLY_FIRE_MARGIN_MS + ) { + possibleEarlyWakeNativeThreads.add(nativeThreadId); + } + return; + } + // A task notification also starts a turn by itself, so an early turn is the wake only + // when its result is not a task notification's. + if (message.type === "result") { + if ( + possibleEarlyWakeNativeThreads.delete(nativeThreadId) && + !isClaudeTaskNotificationOriginResult(message) + ) { + scheduledWakeupsByNativeThread.delete(nativeThreadId); + } + return; + } + for (const toolUse of claudeToolUseBlocksFromAssistantMessage(message)) { + if (toolUse.name === "ScheduleWakeup") scheduleWakeupCalls.add(toolUse.id); + } + if (message.type !== "user") return; + const returned = claudeToolResultBlocksFromUserMessage(message).filter((toolResult) => + scheduleWakeupCalls.delete(toolResult.tool_use_id), + ); + if (returned.length === 0) return; + const result = message.tool_use_result; + const scheduledFor = + returned.length === 1 && typeof result === "object" && result !== null + ? Reflect.get(result, "scheduledFor") + : undefined; + // stop:true reports scheduledFor 0. A failed call also clears: losing the hold early + // is recoverable, a hold that outlives its wakeup is not. + possibleEarlyWakeNativeThreads.delete(nativeThreadId); + if (typeof scheduledFor === "number" && scheduledFor > 0) { + scheduledWakeupsByNativeThread.set(nativeThreadId, scheduledFor); + } else { + scheduledWakeupsByNativeThread.delete(nativeThreadId); + } + }); + const publishedPendingBackgroundTasks = ( + nativeThreadId: string | null, + roster: ReadonlyMap, + awaitedTaskIds: ReadonlySet, + ): ReadonlyArray => { + const tasks = claudePendingBackgroundTasksFromRoster(roster, awaitedTaskIds); + return nativeThreadId !== null && scheduledWakeupsByNativeThread.has(nativeThreadId) + ? [...tasks, CLAUDE_SCHEDULED_WAKEUP_TASK] + : tasks; + }; const isClaudeMonitorTask = (input: { readonly nativeThreadId: string; readonly taskId: string; @@ -5286,7 +5574,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( return { kind: "command", label, outcome }; } }); - const requestedContinuations = yield* Ref.make(new Set()); + const requestedContinuations = yield* Ref.make(new Map()); const absorbedWakes = yield* Ref.make(new Set()); const heldProposedPlansByToolUseId = new Map(); const runtimeContext = yield* Effect.context(); @@ -5586,7 +5874,11 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ...input.providerThread, providerSessionId: session.id, ...(input.status === undefined ? {} : { status: input.status }), - pendingBackgroundTasks: claudePendingBackgroundTasksFromRoster(roster, awaitedTaskIds), + pendingBackgroundTasks: publishedPendingBackgroundTasks( + input.nativeThreadId, + roster, + awaitedTaskIds, + ), ...(goalsByNativeThread.has(input.nativeThreadId) ? { goal: goalsByNativeThread.get(input.nativeThreadId) ?? null } : {}), @@ -5656,7 +5948,12 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( } else { updated.set( nativeThreadId, - new Map(tasks.map((task) => [task.taskId, task] as const)), + new Map( + tasks.map((task) => { + const previous = current.get(nativeThreadId)?.get(task.taskId); + return [task.taskId, { ...previous, ...task }] as const; + }), + ), ); } return updated; @@ -5760,7 +6057,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( if (!current.has(nativeThreadId)) { return current; } - const updated = new Set(current); + const updated = new Map(current); updated.delete(nativeThreadId); return updated; }); @@ -5806,6 +6103,8 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const preservedOpaqueTombstones = [...priorOpaqueTombstones].filter((taskId) => bufferedTaskNotificationIds.has(taskId), ); + scheduledWakeupsByNativeThread.delete(nativeThreadId); + possibleEarlyWakeNativeThreads.delete(nativeThreadId); yield* clearPendingBackgroundTasksForNativeThread(nativeThreadId); yield* clearNativeThreadTaskIdSet( wakeEligibleBackgroundTasksByNativeThread, @@ -6735,13 +7034,8 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( workflow === undefined ? undefined : withWorkflowMemberThreadIds(workflow, childThreadId); - const effort = - existingSubagent === undefined - ? (input.effort ?? - input.owner?.effort ?? - compileClaudeModelSelection(input.context.input.modelSelection, modelCatalog) - .effort) - : existingSubagent.effort; + // Unstated effort stays unknown: a custom agent definition may override the owner's. + const effort = existingSubagent === undefined ? input.effort : existingSubagent.effort; const model = input.model ?? priorTask?.model ?? null; const task = { ...(priorTask ?? { @@ -7460,8 +7754,8 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ordinal: toolCall.ordinal, output: input.output, status: - toolNonExecutionKind === "cancelled" - ? "cancelled" + toolNonExecutionKind === "cancelled" || toolNonExecutionKind === "interrupted" + ? toolNonExecutionKind : isClaudeToolResultError(input.toolResult) ? "failed" : "completed", @@ -7828,6 +8122,41 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( subagentToolCalls.clear(); }); + const emitSteerStatus = ( + context: ActiveClaudeTurnContext, + messageId: MessageId, + status: "pending" | "completed" | "cancelled", + ) => + emitProviderEvent({ + type: "steer.updated", + driver: CLAUDE_PROVIDER, + threadId: context.input.threadId, + runId: context.input.runId, + messageId, + status, + }); + + const markSteersAccepted = Effect.fnUntraced(function* ( + context: ActiveClaudeTurnContext, + message: SDKMessage, + ) { + // Completed alone is not proof of acceptance: Claude also completes + // prompts it never started. + const lifecycle = claudePromptLifecycle(message); + const accepted = + lifecycle === null + ? claudeEchoedPromptUuids(message) + : lifecycle.state === "started" + ? [lifecycle.uuid] + : []; + for (const uuid of accepted) { + const steer = context.pendingSteers.get(uuid); + if (steer === undefined) continue; + context.pendingSteers.delete(uuid); + yield* emitSteerStatus(context, steer.messageId, "completed"); + } + }); + const finalizeActiveTurn = Effect.fnUntraced(function* (input: { readonly context: ActiveClaudeTurnContext; readonly status: Extract< @@ -7839,7 +8168,18 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( readonly threadDisposition?: "reusable" | "broken"; readonly result?: SDKResultMessage; }) { + input.context.steersClosed = true; yield* reasoningDeltas.flushTurn(input.context.nativeTurnId); + // A turn that completes has taken in every steer; one that stops drops them unread. + const droppedSteers = [...input.context.pendingSteers]; + input.context.pendingSteers.clear(); + for (const [, { messageId }] of droppedSteers) { + yield* emitSteerStatus( + input.context, + messageId, + input.status === "completed" ? "completed" : "cancelled", + ); + } if (input.context.compaction?.status === "running") { const reason = input.result?.subtype === "success" ? input.result.result.trim() : ""; yield* emitProviderEvent({ @@ -7977,6 +8317,49 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }); } + if (input.status === "interrupted") { + for (const [uuid, { text, liveQuery }] of droppedSteers) { + // Only a CLI proven to echo prompt uuids can leave a steer unconfirmed. + if ( + liveQuery.promptEchoMode !== "early" && + liveQuery.promptEchoMode !== "result_only" + ) { + continue; + } + const nativeItemId = `unread-steer:${uuid}`; + const notice = `Claude did not confirm reading your steer before Stop. It may not have been delivered. Resend it if needed: "${text}"`; + yield* emitProviderEvent({ + type: "turn_item.updated", + driver: CLAUDE_PROVIDER, + turnItem: { + id: idAllocator.derive.turnItemFromProviderItem({ + driver: CLAUDE_PROVIDER, + nativeItemId, + }), + threadId: input.context.input.threadId, + runId: input.context.input.runId, + nodeId: input.context.input.rootNodeId, + providerThreadId: input.context.input.providerThread.id, + providerTurnId: input.context.providerTurnId, + nativeItemRef: { + driver: CLAUDE_PROVIDER, + nativeId: nativeItemId, + strength: "weak", + }, + parentItemId: null, + ordinal: yield* resolveItemOrdinal(input.context, nativeItemId), + type: "system_notice", + status: "completed", + title: "Steer delivery not confirmed", + message: notice, + startedAt: input.completedAt, + completedAt: input.completedAt, + updatedAt: input.completedAt, + }, + }); + } + } + const threadDisposition = input.threadDisposition ?? "reusable"; const terminalEvent: ProviderAdapter.ProviderAdapterV2Event = input.status === "failed" @@ -8047,6 +8430,11 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( input.context.input.providerThread.nativeThreadRef?.nativeId ?? null; if (nativeThreadId !== null) { if (input.status !== "completed") { + // Stop cancels Claude's wakeup and an exited process loses it; a failed + // turn on a live process leaves it pending. Process exits null the query first. + if (input.status !== "failed" || (yield* Ref.get(queryContext)) === null) { + scheduledWakeupsByNativeThread.delete(nativeThreadId); + } yield* clearPendingBackgroundTasksForNativeThread(nativeThreadId); yield* clearNativeThreadTaskIdSet( wakeEligibleBackgroundTasksByNativeThread, @@ -8110,7 +8498,8 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( input.context.input.providerThread.firstRunOrdinal ?? input.context.input.runOrdinal, lastRunOrdinal: input.context.input.runOrdinal, - pendingBackgroundTasks: claudePendingBackgroundTasksFromRoster( + pendingBackgroundTasks: publishedPendingBackgroundTasks( + nativeThreadId, roster, awaitedTaskIds, ), @@ -8143,16 +8532,14 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }); }); - const emitAssistantTextArtifacts = Effect.fnUntraced(function* (input: { + const publishAssistantArtifacts = Effect.fnUntraced(function* (input: { readonly context: ActiveClaudeTurnContext; readonly nativeItemId: string; readonly text: string; + readonly status: "completed" | "interrupted"; + readonly startedAt: DateTime.Utc; + readonly completedAt: DateTime.Utc; }) { - if (input.context.assistant.emittedNativeItemIds.has(input.nativeItemId)) { - return; - } - input.context.assistant.emittedNativeItemIds.add(input.nativeItemId); - const now = yield* DateTime.now; const ordinal = yield* resolveItemOrdinal(input.context, input.nativeItemId); const artifacts = buildAssistantArtifacts({ idAllocator, @@ -8161,8 +8548,9 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( nativeItemId: input.nativeItemId, text: input.text, ordinal, - startedAt: now, - completedAt: now, + status: input.status, + startedAt: input.startedAt, + completedAt: input.completedAt, }); yield* Effect.all( [ @@ -8186,6 +8574,25 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ); }); + const emitAssistantTextArtifacts = Effect.fnUntraced(function* (input: { + readonly context: ActiveClaudeTurnContext; + readonly nativeItemId: string; + readonly text: string; + }) { + if (input.context.assistant.emittedNativeItemIds.has(input.nativeItemId)) { + return undefined; + } + input.context.assistant.emittedNativeItemIds.add(input.nativeItemId); + const now = yield* DateTime.now; + yield* publishAssistantArtifacts({ + ...input, + status: "completed", + startedAt: now, + completedAt: now, + }); + return now; + }); + const endSubagentsOfClosedQuery = Effect.fnUntraced(function* (input: { readonly closedQuery: ClaudeLiveQueryContext; readonly context: ActiveClaudeTurnContext; @@ -8272,6 +8679,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( : { failure: makeProviderFailure({ cause: cause === undefined ? undefined : Cause.squash(cause), + ...(cause === undefined ? {} : claudeProcessExitFailure(cause)), class: "transport_error", }), }), @@ -8508,12 +8916,14 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }); return; } + const nativeThreadId = wakeInput.nativeThreadId; + const latch = Symbol(nativeThreadId); const shouldOffer = yield* Ref.modify(requestedContinuations, (current) => { - if (current.has(wakeInput.nativeThreadId)) { + if (current.has(nativeThreadId)) { return [false, current] as const; } - const updated = new Set(current); - updated.add(wakeInput.nativeThreadId); + const updated = new Map(current); + updated.set(nativeThreadId, latch); return [true, updated] as const; }); if (!shouldOffer) { @@ -8536,6 +8946,16 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( driver: CLAUDE_PROVIDER, detail, ...(notification === null ? {} : { notification }), + // A process reset can clear and re-latch this thread; a stale failure must not release the newer wake. + clearIfCurrent: () => + Ref.update(requestedContinuations, (current) => { + if (current.get(nativeThreadId) !== latch) { + return current; + } + const updated = new Map(current); + updated.delete(nativeThreadId); + return updated; + }), }); }); @@ -8598,9 +9018,26 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( if (!isClaudeNonSubagentTask(message) || message.is_backgrounded === false) { return false; } - yield* upsertPendingBackgroundTask( - input.nativeThreadId, - claudePendingBackgroundTask({ + const launch = + message.tool_use_id === undefined || input.activeContext === null + ? undefined + : findToolCall(input.activeContext, message.tool_use_id); + const command = + launch?.classification.itemType === "command_execution" + ? firstStringInputField(launch.input, ["command"]) + : undefined; + yield* upsertPendingBackgroundTask(input.nativeThreadId, { + ...(launch === undefined || command === undefined + ? {} + : { + itemId: idAllocator.derive.turnItemFromProviderItem({ + driver: CLAUDE_PROVIDER, + nativeItemId: launch.nativeItemId, + }), + sourceThreadId: launch.threadId, + }), + ...(command === undefined ? {} : { command }), + ...claudePendingBackgroundTask({ taskId: message.task_id, taskType: claudeTaskTypeFromSdkMessage(message), startedByMonitor: yield* isClaudeMonitorTask({ @@ -8612,7 +9049,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( typeof message.description === "string" ? message.description : undefined, parentTaskId: Reflect.get(message, "parent_task_id"), }), - ); + }); rosterChanged = true; } else if (message.type === "system" && message.subtype === "task_notification") { yield* endClaudeMonitorTasks((taskId) => taskId === message.task_id); @@ -8734,6 +9171,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( } // Track before routing: idle-wake Monitors report tasks before replaying their tool calls. yield* trackClaudeMonitorCalls(message); + yield* trackClaudeScheduledWakeup(liveQuery.nativeThreadId, message); // Update before routing because the idle wake gate reads this. if ( message.type === "system" && @@ -9193,9 +9631,19 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( (message.subtype === "model_refusal_fallback" || (message.subtype === "informational" && message.level !== "info")) ) { + context.assistant.noticeTexts.add(message.content); + // A Stop hook's systemMessage arrives as a passive notice after every turn. + // Warnings and notices that prevent continuation explain a blocked turn. + if ( + message.subtype === "informational" && + message.level === "notice" && + message.content.startsWith("Stop says: ") && + message.prevent_continuation !== true + ) { + return; + } const now = yield* DateTime.now; const nativeItemId = message.uuid; - context.assistant.noticeTexts.add(message.content); yield* emitProviderEvent({ type: "turn_item.updated", driver: CLAUDE_PROVIDER, @@ -9785,6 +10233,9 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( } } const assistantText = assistantTextFromSdkMessage(message); + if (message.type === "assistant" && assistantParentToolUseId === null) { + context.assistant.lastReply = null; + } if ( assistantText !== null && assistantText.text.length > 0 && @@ -9833,11 +10284,14 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( return; } if (assistantText !== null && assistantText.text.length > 0) { - yield* emitAssistantTextArtifacts({ + const startedAt = yield* emitAssistantTextArtifacts({ context, nativeItemId: assistantText.nativeItemId, text: assistantText.text, }); + if (startedAt !== undefined) { + context.assistant.lastReply = { ...assistantText, startedAt }; + } return; } @@ -9910,6 +10364,16 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const interrupted = (yield* Ref.get(interruptedTurns)).has(context.providerTurnId); const wasSteered = (yield* Ref.get(steeredTurns)).has(context.providerTurnId); if (!interrupted && wasSteered && isClaudeActiveSteeringAbortResult(message)) { + const cutOffReply = context.assistant.lastReply; + context.assistant.lastReply = null; + if (message.terminal_reason === "aborted_streaming" && cutOffReply !== null) { + yield* publishAssistantArtifacts({ + context, + ...cutOffReply, + status: "interrupted", + completedAt, + }); + } context.steeredSegmentUsage.push(message.usage); context.steeredLedgerUsage = addClaudeLedgerUsage( context.steeredLedgerUsage, @@ -9938,12 +10402,30 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( // The CLI also labels 403 authentication_failed; show its text because login cannot grant access. const permissionDenied = message.subtype === "success" && message.api_error_status === 403; + const rewrittenVersion = claudeRewrittenVersion(message, liveQuery.claudeCodeVersion); + const endpointHost = + rewrittenVersion === undefined + ? undefined + : customClaudeApiEndpointHost( + (yield* readClaudeSettingsEnv( + adapterOptions.settings, + context.input.runtimePolicy.cwd ?? undefined, + adapterOptions.environment, + ).pipe( + Effect.provideService(FileSystem.FileSystem, fileSystem), + Effect.provideService(Path.Path, path), + )).ANTHROPIC_BASE_URL ?? adapterOptions.environment.ANTHROPIC_BASE_URL, + ); + const rewrittenVersionError = + endpointHost === undefined + ? undefined + : `The custom API endpoint ${endpointHost} reported Claude Code ${rewrittenVersion}, but this session runs Claude Code ${liveQuery.claudeCodeVersion}. Running claude update will not fix this; the endpoint or a proxy behind it rewrote the version.`; const failureHint = permissionDenied ? message.result : (context.authenticationFailureMessage ?? (usageLimited ? "Claude usage limit reached. Send the message again once the limit resets." - : undefined)); + : rewrittenVersionError)); // Usage-limited /compact reports success; fail it so queued messages wait for the limit to clear. const limitedCompaction = usageLimited && message.subtype === "success" && message.local_command === "compact" @@ -10065,6 +10547,23 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const message = input.message; const context = yield* Ref.get(activeTurn); const liveQuery = yield* Ref.get(queryContext); + // The API answering again is recovery; a long text or tool input block emits nothing + // else. Judged here, in arrival order: held and buffered frames are handled later. + if ( + context !== null && + liveQuery?.query === input.query && + message.type === "stream_event" && + message.event.type === "message_start" + ) { + yield* completeProviderRetry(context, yield* DateTime.now); + } + if ( + context !== null && + context.pendingSteers.size > 0 && + liveQuery?.query === input.query + ) { + yield* markSteersAccepted(context, message); + } if ( context?.promptUuid === null && liveQuery?.query === input.query && @@ -10138,6 +10637,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( isClaudeResultForOtherTurn({ message, promptUuid: context.promptUuid, + steerUuids: context.steerUuids, promptEchoMode: liveQuery.promptEchoMode, }) ) { @@ -10476,6 +10976,10 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ); return { action: "decline" } satisfies ElicitationResult; } + const grantKey = `${request.serverName}\u0000${request.message}`; + if ((yield* Ref.get(mcpElicitationSessionGrants)).has(grantKey)) { + return acceptance; + } // The CLI waits on this answer, so a turn held for its prompt echo // must show its output now, as canUseTool does. if (context.heldRootFrames.length > 0) { @@ -10562,8 +11066,14 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( runtimeRequest: { ...artifacts.request, status: "cancelled", resolvedAt: now }, }); } + if (resolved === "acceptForSession") { + yield* Ref.update(mcpElicitationSessionGrants, (current) => + new Set(current).add(grantKey), + ); + return acceptance; + } // Never null: the SDK reads null as answered out of band and leaves - // the server waiting. Every accept is one-time; no grant is offered. + // the server waiting. return resolved === "accept" ? acceptance : ({ @@ -10698,6 +11208,29 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( return false; }); + const applySelectionLive = Effect.fnUntraced(function* ( + live: ClaudeLiveQueryContext, + selection: CompiledClaudeModelSelection, + ) { + // Preserve launcher-selected models unless the model changed or a failed update left it unknown. + yield* ( + live.selection.queryIdentity === "" || + live.selection.apiModelId !== selection.apiModelId + ? live.query.setModel(selection.apiModelId) + : Effect.void + ).pipe( + Effect.andThen( + live.query.applyFlagSettings(claudeLiveSelectionFlagSettings(selection)), + ), + Effect.tapError(() => + Effect.sync(() => { + live.selection = { ...live.selection, queryIdentity: "" }; + }), + ), + ); + live.selection = selection; + }); + const openQuery = Effect.fnUntraced(function* ( turnInput: ProviderAdapter.ProviderAdapterV2TurnInput, nativeThreadId: string, @@ -10738,24 +11271,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( (existing.selection.apiModelId === compiledSelection.apiModelId || existing.openedPermissionMode === queryPolicy.permissionMode) ) { - const live = existing; - // Preserve launcher-selected models unless the model changed or a failed update left it unknown. - yield* ( - live.selection.queryIdentity === "" || - live.selection.apiModelId !== compiledSelection.apiModelId - ? live.query.setModel(compiledSelection.apiModelId) - : Effect.void - ).pipe( - Effect.andThen( - live.query.applyFlagSettings(claudeLiveSelectionFlagSettings(compiledSelection)), - ), - Effect.tapError(() => - Effect.sync(() => { - live.selection = { ...live.selection, queryIdentity: "" }; - }), - ), - ); - live.selection = compiledSelection; + yield* applySelectionLive(existing, compiledSelection); } if ( existing !== null && @@ -10876,24 +11392,28 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const resolvedTurnCwd = yield* fileSystem .realPath(turnInput.runtimePolicy.cwd ?? ".") .pipe(Effect.orElseSucceed(() => turnInput.runtimePolicy.cwd)); - const querySession = yield* queryRunner - .open({ - threadId: turnInput.threadId, - providerSessionId: input.providerSessionId, - options: queryOptions, - }) - .pipe( - Effect.onError(() => - closedExistingNativeThreadId === nativeThreadId - ? Effect.gen(function* () { - yield* clearWakeStateForNativeThread(nativeThreadId); - yield* resetBackgroundTaskStateForNativeThreadProcess(nativeThreadId, { - status: "idle", - }); - }) - : Effect.void, - ), - ); + const openQueryRuntime = queryRunner.open({ + threadId: turnInput.threadId, + providerSessionId: input.providerSessionId, + options: queryOptions, + }); + // The SDK spawns the CLI per turn, so only the open waits for a provider update. + const querySession = yield* ( + adapterOptions.installationGate === undefined + ? openQueryRuntime + : adapterOptions.installationGate.execute(adapterOptions.instanceId, openQueryRuntime) + ).pipe( + Effect.onError(() => + closedExistingNativeThreadId === nativeThreadId + ? Effect.gen(function* () { + yield* clearWakeStateForNativeThread(nativeThreadId); + yield* resetBackgroundTaskStateForNativeThreadProcess(nativeThreadId, { + status: "idle", + }); + }) + : Effect.void, + ), + ); // Mark only successful opens; retries must not resume sessions that failed to create. yield* Ref.update(openedNativeThreads, (current) => { if (current.has(nativeThreadId)) { @@ -10921,6 +11441,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( openedPermissionMode: queryOptions.permissionMode, receivedInit: false, capabilities: new Set(), + claudeCodeVersion: undefined, permissionMode: queryOptions.permissionMode, stopping: false, handedOffSubagents: new Set(), @@ -10961,6 +11482,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( message.permissionMode ?? context.openedPermissionMode; context.receivedInit = true; context.capabilities = new Set(message.capabilities ?? []); + context.claudeCodeVersion = message.claude_code_version; } if ( message.type === "system" && @@ -11119,6 +11641,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( fallbackText: "", fallbackNativeItemId: `assistant:${turnInput.runId}`, emittedNativeItemIds: new Set(), + lastReply: null, noticeTexts: new Set(), }, reasoning: { @@ -11142,6 +11665,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( latestAssistantRateLimit: null, compactionFailure: null, steeredSegmentUsage: [], + steersClosed: false, steeredLedgerUsage: { inputTokens: 0, outputTokens: 0 }, rootModels: new Set(), subagentsByTaskId: new Map(), @@ -11152,11 +11676,27 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( nativeWakeStarted: false, gatedFramesBeforeEcho: 0, heldRootFrames: [], + pendingSteers: new Map(), + steerUuids: new Set(), executedModel: undefined, }; const attachTurn = Effect.gen(function* () { - querySession.lastTurn = context; + // Set before claiming: a query that exits after the claim finalizes this turn; one that exits before it fails the start. yield* Ref.set(activeTurn, context); + const attached = yield* Ref.modify(queryContext, (current) => { + if (current?.query !== querySession.query || querySession.stopping) { + return [false, current]; + } + querySession.lastTurn = context; + return [true, current]; + }); + if (!attached) { + yield* Ref.update(activeTurn, (current) => (current === context ? null : current)); + return yield* new ProviderAdapter.ProviderAdapterProtocolError({ + driver: CLAUDE_PROVIDER, + detail: "Claude provider query ended before the turn could start.", + }); + } if ( !isClaudeProviderContinuationTurn(turnInput) && permissionModeForClaudeRuntimePolicy(turnInput.runtimePolicy) === @@ -11233,19 +11773,10 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( // Transcript reads suspend replay; hold SDK frames until the buffer is applied to preserve order. yield* sdkMessageLock.withPermits(1)( Effect.gen(function* () { - if ( - (yield* Ref.get(queryContext))?.query !== querySession.query || - querySession.stopping - ) { - return yield* new ProviderAdapter.ProviderAdapterProtocolError({ - driver: CLAUDE_PROVIDER, - detail: "Claude provider query ended before the continuation could start.", - }); - } yield* attachTurn; const drained = (yield* Ref.get(wakeBuffers)).get(nativeThreadId)?.messages ?? []; yield* Ref.update(requestedContinuations, (current) => { - const updated = new Set(current); + const updated = new Map(current); updated.delete(nativeThreadId); return updated; }); @@ -11532,6 +12063,18 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( detail: `Claude provider turn ${turnInput.providerTurnId} is not the active turn.`, }); } + // Claude echoes this uuid once it accepts the steer. + const steerUuid = yield* random.uuidV4().pipe( + Effect.filterOrFail(isClaudePromptUuid), + Effect.mapError( + (cause) => + new ProviderAdapter.ProviderAdapterProtocolError({ + driver: CLAUDE_PROVIDER, + detail: "Failed to allocate a Claude prompt uuid.", + payload: cause, + }), + ), + ); const pdfReservation = yield* reserveClaudePdfBudget( pdfBudgets.get(existing.nativeThreadId) ?? makeClaudeAttachmentBudget({ @@ -11540,14 +12083,30 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }), persistAttachmentBudget(existing.nativeThreadId), ); + // A selection sent with the steer applies to the running turn when the live process can switch to it. + const steerSelection = + turnInput.modelSelection === undefined + ? undefined + : compileClaudeModelSelection(turnInput.modelSelection, modelCatalog); + const liveSteerSelection = + steerSelection !== undefined && + existing.selection.queryIdentity !== steerSelection.queryIdentity && + canApplyClaudeSelectionLive(existing.selection, steerSelection) && + (existing.selection.apiModelId === steerSelection.apiModelId || + existing.openedPermissionMode === currentTurn.queryPolicy.permissionMode) + ? steerSelection + : undefined; const userMessage = yield* makeClaudeUserMessageWithAttachments({ text: applyClaudePromptEffortPrefix( turnInput.message.text, - compileClaudeModelSelection(currentTurn.input.modelSelection, modelCatalog) - .promptEffort, + ( + liveSteerSelection ?? + compileClaudeModelSelection(currentTurn.input.modelSelection, modelCatalog) + ).promptEffort, ), attachments: turnInput.message.attachments, priority: "now", + uuid: steerUuid, attachmentsDir, fileSystem, pdfReservation, @@ -11557,23 +12116,37 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( Effect.provideService(Path.Path, path), ); yield* backgroundForegroundSubagents(existing, backgroundDeadline); - // The turn can end while the moves are answered. A steer offered - // then would start a new turn the orchestrator never asked for, so - // the steer fails and the orchestrator starts the follow-up turn. + if (liveSteerSelection !== undefined) { + yield* applySelectionLive(existing, liveSteerSelection); + } + // The turn can end while the moves or the selection change are answered. + // A steer offered then would start a new turn the orchestrator never + // asked for, so the steer fails and the orchestrator starts the follow-up turn. if ( (yield* Ref.get(activeTurn))?.providerTurnId !== turnInput.providerTurnId || - (yield* Ref.get(queryContext)) !== existing + (yield* Ref.get(queryContext)) !== existing || + // No yield between this check and the set below. + currentTurn.steersClosed ) { return yield* new ProviderAdapter.ProviderAdapterProtocolError({ driver: CLAUDE_PROVIDER, detail: `Claude provider turn ${turnInput.providerTurnId} ended before the steer was sent.`, }); } + // Register before offering: Claude can start the steer and end the turn before offer returns. + const messageId = turnInput.message.messageId; + currentTurn.steerUuids.add(steerUuid); + currentTurn.pendingSteers.set(steerUuid, { + messageId, + text: turnInput.message.text, + liveQuery: existing, + }); yield* Ref.update(steeredTurns, (current) => { const next = new Set(current); next.add(turnInput.providerTurnId); return next; }); + yield* emitSteerStatus(currentTurn, messageId, "pending"); yield* Effect.uninterruptibleMask((restore) => restore( existing.query.offer(userMessage, () => { @@ -11586,6 +12159,12 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }), ), ), + ).pipe( + Effect.onError(() => + currentTurn.pendingSteers.delete(steerUuid) + ? emitSteerStatus(currentTurn, messageId, "completed") + : Effect.void, + ), ); }, (effect, turnInput) => @@ -11703,6 +12282,10 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( events: Stream.fromEffectRepeat(Queue.take(events)), backgroundTasksEndWithProcess: true, hasPendingBackgroundWork: Effect.gen(function* () { + // A wakeup can be up to an hour out, past the idle timeout, and dies with the process. + if (scheduledWakeupsByNativeThread.size > 0) { + return true; + } for (const roster of (yield* Ref.get(pendingBackgroundTasksByNativeThread)).values()) { if (roster.size > 0) { return true; @@ -11909,6 +12492,30 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( ); }), }, + applyModelSelection: Effect.fn("ClaudeAdapterV2.applyModelSelection")(function* ({ + providerThread, + modelSelection, + }) { + const live = yield* Ref.get(queryContext); + const selection = compileClaudeModelSelection(modelSelection, modelCatalog); + // A model change still waits for the next turn, which checks it against the turn's permission mode. + if ( + live === null || + live.stopping || + live.nativeThreadId !== providerThread.nativeThreadRef?.nativeId || + live.selection.queryIdentity === selection.queryIdentity || + live.selection.apiModelId !== selection.apiModelId || + !canApplyClaudeSelectionLive(live.selection, selection) + ) { + return; + } + // A failed update clears the live identity, so the next turn applies the selection again. + yield* applySelectionLive(live, selection).pipe( + Effect.catch((cause) => + Effect.logWarning("orchestration-v2.claude-live-selection-failed", { cause }), + ), + ); + }), stopTask: Effect.fn("ClaudeAdapterV2.stopTask")(function* ({ providerThread, taskId }) { const liveQuery = yield* Ref.get(queryContext); const task = (yield* Ref.get(sessionSubagentsByTaskId)).get(taskId); @@ -12181,10 +12788,10 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( const sourceNativeThreadId = yield* getNativeThreadId(forkInput.sourceProviderThread); yield* closeLiveQueryForNativeThread(sourceNativeThreadId); const upToMessageId = yield* resolveClaudeForkUpToMessageId(forkInput); - const forkOptions: ForkSessionOptions = { - ...(input.runtimePolicy.cwd === null ? {} : { dir: input.runtimePolicy.cwd }), - ...(upToMessageId === undefined ? {} : { upToMessageId }), - }; + // A moved project still has its transcript under the original cwd. + // Let the SDK find the session by ID within this Claude home. + const forkOptions: ForkSessionOptions = + upToMessageId === undefined ? {} : { upToMessageId }; const forked = yield* queryRunner.forkSession({ sessionId: sourceNativeThreadId, options: forkOptions, @@ -12261,6 +12868,7 @@ export type ClaudeAdapterV2DriverEnv = | IdAllocator.IdAllocatorV2 | McpProviderSessions.McpProviderSessions | Path.Path + | ProviderInstallationGate.ProviderInstallationGate | ServerConfig.ServerConfig; export const createClaudeAdapterV2 = Effect.fn("ClaudeAdapterV2Driver.create")( @@ -12275,6 +12883,7 @@ export const createClaudeAdapterV2 = Effect.fn("ClaudeAdapterV2Driver.create")( const queryRunner = yield* ClaudeAgentSdkQueryRunner; const serverConfig = yield* ServerConfig.ServerConfig; const continuationRequests = yield* ProviderContinuationRequests.ProviderContinuationRequests; + const installationGate = yield* ProviderInstallationGate.ProviderInstallationGate; const baseEnvironment = yield* mergeClaudeProcessEnvironment( config, environment, @@ -12296,6 +12905,7 @@ export const createClaudeAdapterV2 = Effect.fn("ClaudeAdapterV2Driver.create")( idAllocator, queryRunner, continuationRequests, + installationGate, ...hooks, }); }, diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAgentSdkQueryRunner.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAgentSdkQueryRunner.test.ts index 64423fd2e4d4..eaeef03339a4 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAgentSdkQueryRunner.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAgentSdkQueryRunner.test.ts @@ -94,6 +94,7 @@ it.effect("reopens a stalled MCP registration before delivering the resumed prom if (typeof prompt === "string") throw new Error("Expected streaming input"); prompts.push(prompt); return { + initializationResult: async () => ({}), setMcpServers: prompts.length === 1 ? firstRegistration : secondRegistration, close: prompts.length === 1 ? firstClose : secondClose, [Symbol.asyncDispose]: async () => { @@ -200,6 +201,7 @@ it.effect.each([true, false])("only retries after confirmed process exit: %s", ( cleanupProgress.resolve("retry"); } return { + initializationResult: async () => ({}), setMcpServers: () => { registrationStarted.resolve(); return first @@ -250,6 +252,7 @@ it.effect("stops retrying MCP registration after two timeouts and reports the ca const querySpy = vi.spyOn(ClaudeAgentSdk, "query").mockImplementation(() => { const started = querySpy.mock.calls.length === 1 ? firstRegistration : secondRegistration; return { + initializationResult: async () => ({}), setMcpServers: () => { started.resolve(); return new Promise(() => {}); @@ -290,6 +293,7 @@ it.effect("cancels a stalled registration without retrying or retaining its prom const querySpy = vi.spyOn(ClaudeAgentSdk, "query").mockImplementation( () => ({ + initializationResult: async () => ({}), setMcpServers: () => { registrationStarted.resolve(); return new Promise(() => {}); @@ -320,6 +324,7 @@ it.effect("does not reuse a fresh session ID after MCP registration times out", const querySpy = vi.spyOn(ClaudeAgentSdk, "query").mockImplementation( () => ({ + initializationResult: async () => ({}), setMcpServers: () => { registrationStarted.resolve(); return new Promise(() => {}); @@ -347,6 +352,56 @@ it.effect("does not reuse a fresh session ID after MCP registration times out", ), ); +it.effect.each([ + ["opens after a slow CLI boot", "120 seconds", true], + ["times out a CLI that never boots", "5 minutes", false], +] as const)("%s", ([, bootWait, boots]) => + Effect.scoped( + Effect.gen(function* () { + const waiting = Promise.withResolvers(); + const booted = Promise.withResolvers(); + // Coder: the runner calls the CLI transport's `query`, as this file's other cases spy. + const querySpy = vi.spyOn(ClaudeAgentSdk, "query").mockImplementation( + () => + ({ + initializationResult: () => { + waiting.resolve(); + return booted.promise; + }, + // The CLI answers control requests only after it has booted. + setMcpServers: async () => { + waiting.resolve(); + await booted.promise; + return { added: ["t3-code"], removed: [], errors: {} }; + }, + close: vi.fn(), + [Symbol.asyncDispose]: async () => {}, + }) as unknown as ClaudeSdk.Query, + ); + yield* Effect.addFinalizer(() => Effect.sync(() => querySpy.mockRestore())); + const runner = yield* ClaudeAgentSdkQueryRunner; + const { resume, ...options } = resumedMcpQueryInput.options; + const opening = yield* runner + .open({ ...resumedMcpQueryInput, options: { ...options, sessionId: resume } }) + .pipe(Effect.result, Effect.forkChild); + yield* Effect.promise(() => waiting.promise); + yield* TestClock.adjust(bootWait); + if (boots) booted.resolve(); + const result = yield* Fiber.join(opening); + + assert.equal(result._tag, boots ? "Success" : "Failure"); + if (result._tag === "Success") yield* result.success.close; + else + assert.equal( + makeProviderFailure({ + cause: { _tag: "ProviderAdapterTurnStartError", cause: result.failure }, + }).message, + "Claude timed out while starting up. Retry the turn; if it keeps failing, check Claude and the server logs.", + ); + }).pipe(Effect.provide(realQueryRunner)), + ), +); + function fakeClaudeCli(script: string): string { const dir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-claude-cli-")); const cli = NodePath.join(dir, "claude"); @@ -419,6 +474,69 @@ describe.skipIf(HostProcess.Platform.defaultValue() === "win32")("Claude CLI pro }), ); + // The default AgentScope is what macOS gets: no systemd scope to catch a job outside the CLI's group. + it.effect("stops a job the CLI started in its own process group when the session closes", () => + Effect.gen(function* () { + const cli = fakeClaudeCli( + [ + 'dir=$(dirname "$0")', + `perl -e 'setpgrp(0, 0); exec @ARGV' sleep 600 > "$dir/alive" &`, + 'echo $! > "$dir/leftover.pid"', + "cat > /dev/null", + ].join("\n"), + ); + const dir = NodePath.dirname(cli); + NodeChildProcess.execFileSync("mkfifo", [NodePath.join(dir, "alive")]); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + try { + process.kill(Number(NodeFS.readFileSync(NodePath.join(dir, "leftover.pid"), "utf8"))); + } catch {} + NodeFS.rmSync(dir, { recursive: true, force: true }); + }), + ); + const alive = NodeFS.createReadStream(NodePath.join(dir, "alive")); + const leftoverStarted = new Promise((resolve) => alive.once("open", () => resolve())); + const leftoverStopped = new Promise((resolve) => alive.once("close", () => resolve())); + alive.resume(); + + const session = yield* openFakeCli(cli).pipe(Effect.provide(realQueryRunner)); + yield* Effect.promise(() => leftoverStarted); + yield* session.close; + yield* Effect.promise(() => leftoverStopped); + }), + ); + + // Live clock: the job is found by polling while the CLI runs, and the CLI exits on its own. + it.live("stops a job the CLI started in its own process group when the CLI exits", () => + Effect.gen(function* () { + const cli = fakeClaudeCli( + [ + 'dir=$(dirname "$0")', + `perl -e 'setpgrp(0, 0); exec @ARGV' sleep 600 > "$dir/alive" &`, + 'echo $! > "$dir/leftover.pid"', + "sleep 3", + ].join("\n"), + ); + const dir = NodePath.dirname(cli); + NodeChildProcess.execFileSync("mkfifo", [NodePath.join(dir, "alive")]); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + try { + process.kill(Number(NodeFS.readFileSync(NodePath.join(dir, "leftover.pid"), "utf8"))); + } catch {} + NodeFS.rmSync(dir, { recursive: true, force: true }); + }), + ); + const alive = NodeFS.createReadStream(NodePath.join(dir, "alive")); + const leftoverStopped = new Promise((resolve) => alive.once("close", () => resolve())); + alive.resume(); + + yield* openFakeCli(cli).pipe(Effect.provide(realQueryRunner)); + yield* Effect.promise(() => leftoverStopped); + }), + ); + const rootRefusal = "--dangerously-skip-permissions cannot be used with root/sudo privileges for security reasons"; @@ -436,14 +554,6 @@ describe.skipIf(HostProcess.Platform.defaultValue() === "win32")("Claude CLI pro : ""; }); - // Claude Code refuses bypassPermissions as root and explains why only on - // stderr, so the turn failure has to carry that line. - it.effect("reports the CLI's root refusal when it exits before answering", () => - Effect.gen(function* () { - assert.include(yield* startupFailureMessage(rootRefusal), rootRefusal); - }).pipe(Effect.provide(realQueryRunner)), - ); - // Startup stderr can carry paths, tokens, or settings, so only the known // refusal reaches the failure the client shows. it.effect("keeps other startup stderr out of the turn failure", () => diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeTaskOutputTail.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeTaskOutputTail.test.ts index b70b7322d5e6..a1a421232686 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeTaskOutputTail.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeTaskOutputTail.test.ts @@ -91,11 +91,12 @@ describe("tailClaudeTaskOutput", () => { }).pipe(Effect.forkScoped); assert.equal(yield* Queue.take(chunks), "start\n"); - yield* fileSystem.writeFileString(file, `start\n${"y\n".repeat(1_000_000)}end\n`); + // The skip lands on the second byte of a two-byte é. + yield* fileSystem.writeFileString(file, `start\n${"é\n".repeat(1_000_000)}end.\n`); const flooded = yield* Queue.take(chunks); assert.isAtMost(flooded.length, 256 * 1024 + 1); - assert.isTrue(flooded.startsWith("\n")); - assert.isTrue(flooded.endsWith("y\nend\n")); + assert.equal(flooded.slice(0, 3), "\n\né"); + assert.isTrue(flooded.endsWith("é\nend.\n")); }), ).pipe(Effect.provide(NodeServices.layer)), ); diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeTaskOutputTail.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeTaskOutputTail.ts index 77505d24a5bb..7eac250d7afd 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeTaskOutputTail.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeTaskOutputTail.ts @@ -104,10 +104,15 @@ export const tailClaudeTaskOutput = Effect.fn("ClaudeTaskOutputTail.tail")(funct if (Option.isSome(bytes)) { // Skipped output must not splice two unrelated lines together. const skipped = start > offset; - // Reset partial UTF-8 bytes when skipping output. - if (skipped) decoder = new TextDecoder("utf-8", { fatal: false }); + let first = 0; + if (skipped) { + // Reset partial UTF-8 bytes, and skip the rest of a code point the cut landed inside. + decoder = new TextDecoder("utf-8", { fatal: false }); + while (((bytes.value[first] ?? 0) & 0xc0) === 0x80) first += 1; + } offset = start + bytes.value.byteLength; - const text = (skipped ? "\n" : "") + decoder.decode(bytes.value, { stream: true }); + const text = + (skipped ? "\n" : "") + decoder.decode(bytes.value.subarray(first), { stream: true }); if (text.length > 0) yield* input.onChunk(text); } } diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index a8ce0d5c7ce9..134bcfc991db 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -52,6 +52,7 @@ import * as PubSub from "effect/PubSub"; import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; +import * as Option from "effect/Option"; import * as Stream from "effect/Stream"; import * as Sink from "effect/Sink"; import * as Stdio from "effect/Stdio"; @@ -60,6 +61,8 @@ import * as Cause from "effect/Cause"; import * as PlatformError from "effect/PlatformError"; import { TestClock } from "effect/testing"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; +import * as Scope from "effect/Scope"; +import * as HostProcess from "@t3tools/shared/HostProcess"; import packageJson from "../../../package.json" with { type: "json" }; import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions"; @@ -1278,6 +1281,7 @@ describe("CodexAdapterV2 post-settle continuation", () => { beforeEmitInbound?: CodexReplay.CodexAppServerReplayDriver["beforeEmitInbound"], onUsageLimits?: CodexAdapterV2.CodexAdapterV2Options["onUsageLimits"], promptSuggestions: Effect.Effect = Effect.succeed(false), + onNotification: (method: string) => Effect.Effect = () => Effect.void, ) => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; @@ -1316,6 +1320,10 @@ describe("CodexAdapterV2 post-settle continuation", () => { onRequest(method, params).pipe( Effect.andThen(client.request(method, params)), ), + handleServerNotification: (method, handler) => + client.handleServerNotification(method, (payload) => + onNotification(method).pipe(Effect.andThen(handler(payload))), + ), raw: { ...client.raw, request: (method, params) => @@ -1414,6 +1422,74 @@ describe("CodexAdapterV2 post-settle continuation", () => { }; }); + it.effect("keeps native image views available to the clients", () => + Effect.gen(function* () { + const nativeThreadId = "image-view-thread"; + const nativeTurnId = "image-view-turn"; + const imagePath = "file:///workspace/reference.png"; + const harness = yield* makeCodexReplayHarness( + makeCodexReplayTranscript({ + scenario: "image-view", + entries: [ + ...codexReplayPreamble({ nativeThreadId, nativeTurnId, prompt: "Show the reference." }), + ...(["started", "completed"] as const).map((phase) => ({ + type: "emit_inbound" as const, + label: `image ${phase}`, + frame: { + method: `item/${phase}`, + params: { + threadId: nativeThreadId, + turnId: nativeTurnId, + item: { type: "imageView", id: "image-view", path: imagePath }, + ...(phase === "started" ? { startedAtMs: 1 } : { completedAtMs: 2 }), + }, + }, + })), + { + type: "emit_inbound", + label: "turn completed", + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: nativeTurnId, status: "completed" }), + }, + }, + }, + ], + }), + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("image-view-attempt"), + text: "Show the reference.", + }), + ); + yield* harness.firstTerminal; + const images = harness.events.flatMap((event) => + event.type === "turn_item.updated" && event.turnItem.type === "dynamic_tool" + ? [event.turnItem] + : [], + ); + assert.lengthOf(images, 1); + assert.equal(images[0]?.viewedImagePath, imagePath); + assert.equal(images[0]?.status, "completed"); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + it.effect.each(["before", "after"] as const)( "rejects a previous run's native turn when it completes %s the new start response", (completionOrder) => @@ -2218,6 +2294,74 @@ describe("CodexAdapterV2 post-settle continuation", () => { ), ); + it.effect("records the model Codex reports on the root provider turn", () => + Effect.scoped( + Effect.gen(function* () { + const nativeThreadId = "reported-model-thread"; + const nativeTurnId = "reported-model-turn"; + const transcript = makeCodexReplayTranscript({ + scenario: "codex-reported-model", + entries: [ + ...codexReplayPreamble({ nativeThreadId, nativeTurnId, prompt: "say hi" }), + { + type: "emit_inbound", + label: "rerouted", + frame: { + method: "model/rerouted", + params: { + threadId: nativeThreadId, + turnId: nativeTurnId, + fromModel: "gpt-6-astra", + toModel: "gpt-5.6-sol", + reason: "highRiskCyberActivity", + }, + }, + }, + { + type: "emit_inbound", + label: "complete", + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: nativeTurnId, status: "completed" }), + }, + }, + }, + ], + }); + const harness = yield* makeCodexReplayHarness(transcript); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("reported-model-attempt"), + text: "say hi", + }), + ); + yield* harness.firstTerminal; + const completed = harness.events.find( + (event) => + event.type === "provider_turn.updated" && event.providerTurn.status === "completed", + ); + assert.equal(completed?.type, "provider_turn.updated"); + if (completed?.type !== "provider_turn.updated") return; + assert.deepEqual(completed.providerTurn.turnTokenUsage?.models, ["gpt-5.6-sol"]); + assert.equal(completed.providerTurn.model, "gpt-5.6-sol"); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ), + ); + it.effect("stops asking for commands the user allowed for the session", () => Effect.scoped( Effect.gen(function* () { @@ -2479,6 +2623,17 @@ describe("CodexAdapterV2 post-settle continuation", () => { }, }, }, + { + type: "emit_inbound", + label: "done", + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: nativeTurnId, status: "completed" }), + }, + }, + }, { type: "expect_outbound", label: "restore context", @@ -2496,20 +2651,30 @@ describe("CodexAdapterV2 post-settle continuation", () => { }, }, { type: "emit_inbound", label: "restored", frame: { id: 4, result: {} } }, - { - type: "emit_inbound", - label: "done", - frame: { - method: "turn/completed", - params: { - threadId: nativeThreadId, - turn: makeCodexReplayTurn({ id: nativeTurnId, status: "completed" }), - }, - }, - }, ], }); - const harness = yield* makeCodexReplayHarness(transcript); + // Codex queues items injected mid-turn as pending input and samples again. + const wireOrder: Array = []; + const restoreRequested = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness( + transcript, + undefined, + (method) => + method === "thread/inject_items" + ? Effect.sync(() => wireOrder.push(method)).pipe( + Effect.andThen(Deferred.succeed(restoreRequested, undefined)), + ) + : Effect.void, + undefined, + undefined, + undefined, + undefined, + (method) => + Effect.sync(() => { + if (method === "item/completed" || method === "turn/completed") + wireOrder.push(method); + }), + ); yield* (yield* McpProviderSessions.McpProviderSessions).set({ environmentId: EnvironmentId.make("test"), threadId: harness.threadId, @@ -2530,7 +2695,13 @@ describe("CodexAdapterV2 post-settle continuation", () => { }), ); yield* harness.firstTerminal; + yield* Deferred.await(restoreRequested); assert.equal(harness.terminalEvents()[0]?.status, "completed"); + assert.deepEqual( + wireOrder, + ["item/completed", "turn/completed", "thread/inject_items"], + "restored T3 context while the compacting native turn was still running", + ); }).pipe( Effect.provide( Layer.mergeAll( @@ -3133,6 +3304,109 @@ describe("CodexAdapterV2 post-settle continuation", () => { ), ); + // regression: #33747 Codex refuses thread/resume until an unsubscribed thread finishes closing + it.effect.each([ + { name: "waits for thread/closed after a closing error", closedFirst: false }, + { name: "retries at once when the thread already closed", closedFirst: true }, + ])("$name", ({ name, closedFirst }) => + Effect.scoped( + Effect.gen(function* () { + const nativeThreadId = "closing-thread"; + const params = { + threadId: nativeThreadId, + excludeTurns: true, + cwd: CODEX_TEST_RUNTIME_POLICY.cwd, + model: CODEX_TEST_MODEL_SELECTION.model, + config: CodexAdapterV2.CODEX_THREAD_CONFIG, + }; + const closed: CodexReplay.CodexAppServerReplayEntry = { + type: "emit_inbound", + label: "thread closed", + frame: { method: "thread/closed", params: { threadId: nativeThreadId } }, + }; + const closingError: CodexReplay.CodexAppServerReplayEntry = { + type: "emit_inbound", + label: "resume closing error", + frame: { + id: 4, + error: { + code: -32600, + message: `thread ${nativeThreadId} is closing; retry thread/resume after the thread is closed`, + }, + }, + }; + const harness = yield* makeCodexReplayHarness( + makeCodexReplayTranscript({ + scenario: name, + entries: [ + ...codexReplayPreamble({ + nativeThreadId, + nativeTurnId: "unused-turn", + prompt: "unused-prompt", + }).slice(0, 5), + { + type: "expect_outbound", + label: "unsubscribe", + frame: { + id: 3, + method: "thread/unsubscribe", + params: { threadId: nativeThreadId }, + }, + }, + { + type: "emit_inbound", + label: "unsubscribe result", + frame: { id: 3, result: { status: "unsubscribed" } }, + }, + { + type: "expect_outbound", + label: "resume closing thread", + frame: { id: 4, method: "thread/resume", params }, + }, + ...(closedFirst ? [closed, closingError] : [closingError, closed]), + { + type: "expect_outbound", + label: "retry identical resume", + frame: { id: 5, method: "thread/resume", params }, + }, + { + type: "emit_inbound", + label: "retry result", + frame: { + id: 5, + result: { + thread: { + id: nativeThreadId, + updatedAt: 1782622450, + turns: [{ type: "unknown-history-item" }], + }, + }, + }, + }, + ], + }), + ); + yield* harness.runtime.unloadThread!({ providerThread: harness.providerThread }); + const resumed = yield* harness.runtime.resumeThread({ + providerThread: harness.providerThread, + modelSelection: CODEX_TEST_MODEL_SELECTION, + runtimePolicy: CODEX_TEST_RUNTIME_POLICY, + }); + assert.equal(resumed.nativeThreadRef?.nativeId, nativeThreadId); + assert.equal(resumed.status, "idle"); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ), + ); + it.effect.each([ "recover", "completion-only", @@ -3673,6 +3947,25 @@ describe("CodexAdapterV2 post-settle continuation", () => { }, }, }, + { + type: "emit_inbound", + label: "error/retry/waiting-for-network", + frame: { + method: "error", + params: { + threadId: nativeThreadId, + turnId: nativeTurnId, + willRetry: true, + error: { + message: "Reconnecting... waiting for network", + additionalDetails: "Connection failed: error sending request", + codexErrorInfo: { + responseStreamDisconnected: { httpStatusCode: null }, + }, + }, + }, + }, + }, { type: "emit_inbound", label: "item/started/after-retry", @@ -3756,7 +4049,7 @@ describe("CodexAdapterV2 post-settle continuation", () => { ? [event.turnItem] : [], ); - assert.lengthOf(retryItems, 2); + assert.lengthOf(retryItems, 3); assert.equal(retryItems[0]?.status, "running"); assert.equal(retryItems[0]?.failure.code, "responseStreamDisconnected"); assert.deepEqual(retryItems[0]?.retry, { @@ -3764,9 +4057,14 @@ describe("CodexAdapterV2 post-settle continuation", () => { maxAttempts: 5, retryDelayMs: null, }); - assert.equal(retryItems[1]?.id, retryItems[0]?.id); - assert.equal(retryItems[1]?.status, "completed"); - assert.equal(retryItems[1]?.title, "Provider recovered"); + assert.deepEqual(retryItems[1]?.retry, { + attempt: 3, + maxAttempts: null, + retryDelayMs: null, + }); + assert.equal(retryItems[2]?.id, retryItems[0]?.id); + assert.equal(retryItems[2]?.status, "completed"); + assert.equal(retryItems[2]?.title, "Provider recovered"); const recoveredIndex = harness.events.findIndex( (event) => @@ -3797,6 +4095,106 @@ describe("CodexAdapterV2 post-settle continuation", () => { ), ); + it.effect("fails a stalled 429 provider retry as a usage limit and stops the Codex turn", () => + Effect.scoped( + Effect.gen(function* () { + const scenario = "codex-provider-retry-stall"; + const nativeThreadId = `native-${scenario}-thread`; + const nativeTurnId = `native-${scenario}-turn`; + const transcript = makeCodexReplayTranscript({ + scenario, + entries: [ + ...codexReplayPreamble({ nativeThreadId, nativeTurnId, prompt: "Continue." }), + { + type: "emit_inbound", + label: "error/retry", + frame: { + method: "error", + params: { + threadId: nativeThreadId, + turnId: nativeTurnId, + willRetry: true, + error: { + message: "Reconnecting... 1/5", + additionalDetails: "exceeded retry limit, last status: 429 Too Many Requests", + codexErrorInfo: { responseStreamDisconnected: { httpStatusCode: 429 } }, + }, + }, + }, + }, + { + type: "expect_outbound", + label: "turn/interrupt", + frame: { + id: 4, + method: "turn/interrupt", + params: { threadId: nativeThreadId, turnId: nativeTurnId }, + }, + }, + { type: "emit_inbound", label: "turn/interrupt", frame: { id: 4, result: {} } }, + { + type: "emit_inbound", + label: "turn/completed", + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: nativeTurnId, status: "interrupted" }), + }, + }, + }, + ], + }); + const retryShown = yield* Deferred.make(); + const interruptSent = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness( + transcript, + (event) => + event.type === "turn_item.updated" && event.turnItem.type === "error" + ? Deferred.succeed(retryShown, undefined) + : Effect.void, + (method) => + method === "turn/interrupt" ? Deferred.succeed(interruptSent, undefined) : Effect.void, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + text: "Continue.", + attemptId: RunAttemptId.make(`attempt-${scenario}`), + }), + ); + yield* Deferred.await(retryShown); + yield* TestClock.adjust("9 minutes"); + assert.isEmpty(harness.terminalEvents()); + yield* TestClock.adjust("1 minute"); + yield* harness.firstTerminal; + yield* Deferred.await(interruptSent); + const terminal = harness.terminalEvents()[0]; + assert.equal(terminal?.status, "failed"); + if (terminal?.status !== "failed") return; + assert.equal(terminal.failure.class, "usage_limit"); + assert.equal(terminal.failure.code, "responseStreamDisconnected"); + assert.equal( + terminal.failure.message, + "exceeded retry limit, last status: 429 Too Many Requests", + ); + assert.equal(terminal.retry?.attempt, 1); + assert.equal(terminal.threadDisposition, "reusable"); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ), + ); + it.effect.each(["completed", "interrupted"] as const)( "retains Codex reasoning parts when the turn is %s", (terminalStatus) => @@ -4076,10 +4474,50 @@ describe("CodexAdapterV2 post-settle continuation", () => { Effect.scoped( Effect.gen(function* () { const transcript = finalAnswerTranscript("codex-partial-then-final", [ - { id: "partial", text: "CODEX_RECOVERY_OK", phase: "partial_answer", streamed }, - { id: "final", text: "CODEX_RECOVERY_OK", phase: "final_answer", streamed }, + { + id: "status", + text: "Checking the files", + phase: "commentary", + streamed, + }, + { + id: "partial", + text: "CODEX_RECOVERY_OK", + phase: "partial_answer", + streamed, + }, + { + id: "final", + text: "CODEX_RECOVERY_OK", + phase: "final_answer", + streamed, + }, ]); - const harness = yield* makeCodexReplayHarness(transcript); + const gates = yield* Effect.forEach(["status", "partial", "final"], (id) => + Effect.gen(function* () { + return { + id, + ready: yield* Deferred.make(), + release: yield* Deferred.make(), + }; + }), + ); + const harness = yield* makeCodexReplayHarness( + transcript, + undefined, + undefined, + undefined, + (entry) => { + const gate = streamed + ? gates.find((gate) => entry.label === `item/completed/${gate.id}`) + : undefined; + return gate === undefined + ? Effect.void + : Deferred.succeed(gate.ready, undefined).pipe( + Effect.andThen(Deferred.await(gate.release)), + ); + }, + ); const now = yield* DateTime.now; yield* harness.runtime.startTurn( makeCodexTestTurnInput({ @@ -4090,11 +4528,41 @@ describe("CodexAdapterV2 post-settle continuation", () => { text: "Reply with the requested recovery marker.", }), ); + if (streamed) { + for (const gate of gates) { + yield* Deferred.await(gate.ready); + yield* TestClock.adjust("50 millis"); + yield* harness.until(() => + harness.events.some( + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "assistant_message" && + event.turnItem.nativeItemRef?.nativeId === gate.id && + event.turnItem.streaming, + ), + ); + yield* Deferred.succeed(gate.release, undefined); + } + } yield* harness.awaitTerminal(1); assert.deepEqual( assistantMessages(harness.events).map((event) => event.message.text), - ["CODEX_RECOVERY_OK", "CODEX_RECOVERY_OK"], + ["Checking the files", "CODEX_RECOVERY_OK", "CODEX_RECOVERY_OK"], + ); + const updates = harness.events.flatMap((event) => + event.type === "turn_item.updated" && event.turnItem.type === "assistant_message" + ? [event.turnItem] + : [], ); + for (const [text, phase] of [ + ["Checking the files", "commentary"], + ["CODEX_RECOVERY_OK", "partial_answer"], + ["CODEX_RECOVERY_OK", "final_answer"], + ]) { + const matching = updates.filter((item) => item.text === text && item.phase === phase); + assert.isTrue(matching.some((item) => !item.streaming)); + if (streamed) assert.isTrue(matching.some((item) => item.streaming)); + } }).pipe( Effect.provide( Layer.mergeAll( @@ -4155,6 +4623,8 @@ describe("CodexAdapterV2 post-settle continuation", () => { id: `image-${index}`, status: "completed", result, + revisedPrompt: "A tree shaped like T3", + transparentBackground: true, ...image, }, }, @@ -4175,6 +4645,42 @@ describe("CodexAdapterV2 post-settle continuation", () => { ); yield* harness.firstTerminal; + const imageItems = harness.events + .flatMap((event) => (event.type === "turn_item.updated" ? [event.turnItem] : [])) + .filter((item) => item.type === "dynamic_tool" && item.toolName === "image_gen.imagegen"); + const latestItems = new Map( + harness.events.flatMap((event) => + event.type === "turn_item.updated" ? [[event.turnItem.id, event.turnItem] as const] : [], + ), + ); + assert.strictEqual( + [...latestItems.values()].filter((item) => item.type === "dynamic_tool").length, + images.length, + ); + assert.strictEqual(imageItems.length, images.length); + assert.deepEqual( + imageItems.map((item) => item.status), + images.map((image) => + ("status" in image && image.status === "failed") || "failure" in image + ? "failed" + : "completed", + ), + ); + for (const [index, item] of imageItems.entries()) { + assert.strictEqual(item.title, "Generate image"); + if (item.type !== "dynamic_tool") continue; + assert.deepEqual(item.input, { + prompt: "A tree shaped like T3", + transparent_background: true, + }); + const image = images[index]!; + assert.deepEqual( + item.output, + Object.fromEntries( + Object.entries({ status: "completed", ...image }).filter(([, value]) => value != null), + ), + ); + } const messages = assistantMessages(harness.events); assert.strictEqual(messages.at(-1)?.message.text, expected); assert.strictEqual(messages.at(-1)?.message.streaming, false); @@ -4218,6 +4724,8 @@ describe("CodexAdapterV2 post-settle continuation", () => { item: { type: "imageGeneration", id: image, + revisedPrompt: "A friendly green tree", + transparentBackground: true, status: phase === "started" ? "inProgress" : "completed", result: "", ...(phase === "completed" ? { savedPath: `/tmp/${image}.png` } : {}), @@ -4288,6 +4796,31 @@ describe("CodexAdapterV2 post-settle continuation", () => { yield* Deferred.await(drained); assert.deepEqual(yield* pending, [false, false]); assert.lengthOf(harness.terminalEvents(), 1); + const imageItems = harness.events.flatMap((event) => + event.type === "turn_item.updated" && + event.turnItem.type === "dynamic_tool" && + event.turnItem.toolName === "image_gen.imagegen" + ? [event.turnItem] + : [], + ); + assert.deepEqual( + imageItems.slice(0, 2).map((item) => item.status), + ["running", "running"], + ); + for (const id of new Set(imageItems.map((item) => item.id))) { + const updates = imageItems.filter((item) => item.id === id); + assert.lengthOf(updates, 2); + assert.strictEqual( + updates[1]?.status, + outcome === "complete" ? "completed" : "cancelled", + ); + assert.strictEqual(updates[1]?.ordinal, updates[0]?.ordinal); + assert.deepEqual(updates[1]?.input, updates[0]?.input); + assert.deepEqual(updates[1]?.input, { + prompt: "A friendly green tree", + transparent_background: true, + }); + } // Run ingestion follows these updates, so they must bracket the terminal and the images. const indexOf = (predicate: (event: ProviderAdapter.ProviderAdapterV2Event) => boolean) => harness.events.findIndex(predicate); @@ -4449,6 +4982,52 @@ describe("CodexAdapterV2 post-settle continuation", () => { ], }); + it.effect("marks a root command it wakes the agent for once the command finishes", () => + Effect.gen(function* () { + const harness = yield* makeCodexReplayHarness({ + ...backgroundExecTranscript, + entries: backgroundExecTranscript.entries.map((entry) => + entry.type === "emit_inbound" && entry.label === "item/completed/command-late" + ? { ...entry, afterMs: 0 } + : entry, + ), + }); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("attempt-command-completion-wake"), + text: BG_PROMPT, + }), + ); + const commands = () => + harness.events.flatMap((event) => + event.type === "turn_item.updated" && event.turnItem.type === "command_execution" + ? [[event.turnItem.status, event.turnItem.completionWake]] + : [], + ); + yield* harness.until( + () => harness.continuationRequests.length === 1 && commands().length === 2, + ); + // Restart recovery reads this to keep the wake the restart cut. + assert.deepEqual(commands(), [ + ["running", "always"], + ["completed", "always"], + ]); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + it.effect("wakes with well-formed text when the cuts split an emoji", () => Effect.scoped( Effect.gen(function* () { @@ -5813,8 +6392,15 @@ describe("CodexAdapterV2 post-settle continuation", () => { ); it.effect.each([ + // regression: #27773 a first-request 429 only notifies the reset-less premium limit { - name: "usage", + name: "first-request-reset", + code: "usageLimitExceeded", + notification: false, + expectedClass: "usage_limit", + }, + { + name: "newer-reset-during-read", code: "usageLimitExceeded", notification: false, expectedClass: "usage_limit", @@ -5879,6 +6465,37 @@ describe("CodexAdapterV2 post-settle continuation", () => { }, }, }; + const readsReset = + scenario.name === "first-request-reset" || scenario.name === "newer-reset-during-read"; + // Usage limits with no known reset read the account before publishing their terminal. + const readsAccount = + scenario.expectedClass === "usage_limit" && + !("expectedCode" in scenario) && + scenario.name !== "deferred-reset"; + const premium = { + ...snapshot, + frame: { + method: "account/rateLimits/updated", + params: { rateLimits: { limitId: "premium", primary: null, secondary: null } }, + }, + }; + const newerSnapshot = { + ...snapshot, + frame: { + method: "account/rateLimits/updated", + params: { + rateLimits: { + limitId: "codex", + primary: { usedPercent: 100, resetsAt: 2000200000, windowDurationMins: 300 }, + }, + }, + }, + }; + const readRequest = { + type: "expect_outbound" as const, + label: "account/rateLimits/read", + frame: { id: 4, method: "account/rateLimits/read", params: null }, + }; const transcript = makeCodexReplayTranscript({ scenario: `codex-limit-${scenario.name}`, entries: [ @@ -5903,6 +6520,7 @@ describe("CodexAdapterV2 post-settle continuation", () => { }, ] : []), + ...(readsReset ? [premium] : []), ...(scenario.name === "deferred-reset" ? [snapshot] : []), ...(scenario.name === "deferred-reset" ? [ @@ -5983,7 +6601,36 @@ describe("CodexAdapterV2 post-settle continuation", () => { }, }, }, - ...(scenario.name === "late-reset" || "resetAfter" in scenario ? [snapshot] : []), + ...(readsAccount ? [readRequest] : []), + ...(scenario.name === "newer-reset-during-read" ? [newerSnapshot] : []), + ...(readsReset + ? [ + { + type: "emit_inbound" as const, + label: "account/rateLimits/read", + frame: { + id: 4, + result: { + ...premium.frame.params, + rateLimitsByLimitId: { codex: snapshot.frame.params.rateLimits }, + }, + }, + }, + ] + : []), + ...(readsAccount && !readsReset + ? [ + { + type: "emit_inbound" as const, + label: "account/rateLimits/read", + frame: { id: 4, error: { code: -32600, message: "Not signed in." } }, + }, + ] + : []), + ...(scenario.name === "late-reset" + ? [{ ...snapshot, label: "account/rateLimits/updated/late" }] + : []), + ...("resetAfter" in scenario ? [snapshot] : []), ...(scenario.name === "deferred-reset" ? [ { @@ -6022,17 +6669,24 @@ describe("CodexAdapterV2 post-settle continuation", () => { }); const resetReceipt = yield* Deferred.make(); const snapshotReceipt = yield* Deferred.make(); + const terminalReceipt = yield* Deferred.make(); const harness = yield* makeCodexReplayHarness( transcript, (event) => - event.type === "turn_item.updated" && - event.turnItem.type === "error" && - event.turnItem.failure.resetAt === resetAt - ? Deferred.succeed(resetReceipt, undefined) - : Effect.void, - undefined, + event.type === "turn.terminal" + ? Deferred.succeed(terminalReceipt, undefined) + : event.type === "turn_item.updated" && + event.turnItem.type === "error" && + event.turnItem.failure.resetAt === resetAt + ? Deferred.succeed(resetReceipt, undefined) + : Effect.void, undefined, undefined, + // The late reset must land after the terminal its failed read released. + (entry) => + entry.label === "account/rateLimits/updated/late" + ? Deferred.await(terminalReceipt) + : Effect.void, () => Deferred.succeed(snapshotReceipt, undefined), ); yield* harness.runtime.startTurn( @@ -6050,7 +6704,10 @@ describe("CodexAdapterV2 post-settle continuation", () => { if (terminal?.status !== "failed") return; assert.equal(terminal.failure.class, scenario.expectedClass); assert.equal(terminal.threadDisposition, "reusable"); - if (scenario.name === "deferred-reset") assert.equal(terminal.failure.resetAt, resetAt); + if (scenario.name === "deferred-reset" || scenario.name === "first-request-reset") + assert.equal(terminal.failure.resetAt, resetAt); + if (scenario.name === "newer-reset-during-read") + assert.equal(terminal.failure.resetAt, "2033-05-20T11:06:40.000Z"); if (scenario.name === "matching-details") assert.equal(terminal.failure.message, "Detailed provider allowance explanation."); if ("resetAfter" in scenario) { @@ -8387,7 +9044,11 @@ describe("CodexAdapterV2 post-settle continuation", () => { frame: { id: 3, method: "thread/fork", - params: { threadId: nativeThreadId, config: CodexAdapterV2.CODEX_THREAD_CONFIG }, + params: { + threadId: nativeThreadId, + excludeTurns: true, + config: CodexAdapterV2.CODEX_THREAD_CONFIG, + }, }, }, { @@ -8525,6 +9186,7 @@ describe("CodexAdapterV2 post-settle continuation", () => { const forkParams = { threadId: nativeThreadId, lastTurnId: "native-turn-first", + excludeTurns: true, config: CodexAdapterV2.CODEX_THREAD_CONFIG, }; const exchange = ( @@ -9799,3 +10461,50 @@ describe("codexDynamicToolOutput", () => { ]); }); }); + +describe("CodexAdapterV2 app-server process", () => { + it.live.skipIf(HostProcess.Platform.defaultValue() === "win32")( + "stops an app-server that ignores SIGTERM within its kill deadline", + () => + Effect.gen(function* () { + const command = yield* CodexAdapterV2.makeCodexAppServerSpawnCommand({ + command: "/bin/sh", + args: ["-c", "trap '' TERM; echo ready; while :; do sleep 1; done"], + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const scope = yield* Scope.make(); + const child = yield* spawner.spawn(command).pipe(Scope.provide(scope)); + const pid = Number(child.pid); + const killGroup = () => { + try { + process.kill(-pid, "SIGKILL"); + } catch {} + }; + yield* Effect.addFinalizer(() => Effect.sync(killGroup)); + // Stop it only once its SIGTERM trap is set; an early exit would pass without testing it. + const firstLine = yield* child.stdout.pipe( + Stream.decodeText(), + Stream.splitLines, + Stream.runHead, + ); + assert.deepStrictEqual(firstLine, Option.some("ready")); + // Scope.close cannot be interrupted, so a regression is ended by this rescue instead of + // hanging the test; the deadline under test must make it unnecessary. + let rescued = false; + // oxlint-disable-next-line t3code/no-real-time-wait-in-tests -- a watchdog for a hung child, not a wait for a milestone; it must never fire. + const rescue = yield* Effect.sleep("10 seconds").pipe( + Effect.andThen( + Effect.sync(() => { + rescued = true; + killGroup(); + }), + ), + Effect.forkChild, + ); + yield* Scope.close(scope, Exit.void); + yield* Fiber.interrupt(rescue); + assert.isFalse(rescued, "the app-server was stopped by its own kill deadline"); + assert.isFalse(yield* child.isRunning.pipe(Effect.orElseSucceed(() => true))); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +}); diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index c12fc79ca41f..f52db76f449c 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -157,6 +157,8 @@ import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter" const CODEX_PROVIDER = ProviderDriverKind.make("codex"); const CODEX_CAPACITY_MAX_RETRIES = 5; +// Codex can announce a retry and never resume. Recovered retries resume model output within ~75s. +const CODEX_PROVIDER_RETRY_STALL_TIMEOUT = "10 minutes"; export const CODEX_DRIVER_KIND = CODEX_PROVIDER; export const CODEX_DEFAULT_INSTANCE_ID = defaultInstanceIdForDriver(CODEX_DRIVER_KIND); @@ -185,6 +187,13 @@ function codexFileChangeApprovalPrompt(input: { return input.grantRoot?.trim() || undefined; } +export function normalizeCodexThreadLoadModelSelection(selection: ModelSelection): ModelSelection { + // Codex applies effort on turn/start; resume discards its pinned request effort. + const { options, ...rest } = selection; + const loadOptions = options?.filter((option) => option.id !== "reasoningEffort"); + return { ...rest, ...(loadOptions?.length ? { options: loadOptions } : {}) }; +} + // Reasoning-effort changes preserve the tokenizer and context window; other changes need fresh telemetry. export function canReuseCodexContextUsage(previous: ModelSelection, next: ModelSelection): boolean { return modelSelectionsEqual( @@ -264,6 +273,7 @@ export const CodexProviderCapabilitiesV2 = { emitsTurnCompleted: true, supportsInterrupt: true, supportsActiveSteering: true, + activeSteeringDuringCompaction: true, supportsSteeringByInterruptRestart: true, supportsQueuedMessages: true, terminalStatusQuality: "strong", @@ -1142,6 +1152,17 @@ function codexExhaustionHttpStatus(errorInfo: CodexErrorInfo | null | undefined) : null; } +function codexErrorHttpStatus(errorInfo: CodexErrorInfo | null | undefined): number | null { + const detail = + typeof errorInfo === "object" && errorInfo !== null ? Object.values(errorInfo)[0] : undefined; + return typeof detail === "object" && + detail !== null && + "httpStatusCode" in detail && + typeof detail.httpStatusCode === "number" + ? detail.httpStatusCode + : null; +} + function codexUsageLimitError( code: string | null, errorInfo: CodexErrorInfo | null | undefined, @@ -1181,6 +1202,7 @@ interface ActiveCodexTurnContext { readonly runningCompactionItemIds: Set; // Modern approval requests omit paths; keep the kinds without retaining patch bodies. readonly fileChangeKindsByItem: Map; + readonly messagePhases: Map; readonly itemPositions: Map< string, { readonly ordinal: number; readonly startedAt: DateTime.Utc } @@ -1415,6 +1437,9 @@ const decodeCodexChildThread = Schema.decodeUnknownEffect( }), ); +// Without a deadline the spawner waits indefinitely for a process that ignores SIGTERM. +const CODEX_APP_SERVER_FORCE_KILL_AFTER = "2 seconds"; + export const makeCodexAppServerSpawnCommand = Effect.fn( "CodexAdapterV2.makeCodexAppServerSpawnCommand", )(function* (input: { @@ -1442,6 +1467,7 @@ export const makeCodexAppServerSpawnCommand = Effect.fn( ...(input.env === undefined ? {} : { env: input.env }), ...(input.extendEnv === undefined ? {} : { extendEnv: input.extendEnv }), shell: spawnCommand.shell, + forceKillAfter: CODEX_APP_SERVER_FORCE_KILL_AFTER, }); }); @@ -1713,8 +1739,9 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( NonNullable >(), ); - // Codex drops client developer messages during compaction but only - // resends additionalContext when it changes. + const threadsAwaitingContextRestore = yield* Ref.make(new Set()); + // Codex drops client developer messages during compaction but only resends additionalContext + // when it changes. Restore after turn/completed: items injected mid-turn make the turn sample again. const restoreAdditionalContext = Effect.fn("CodexAdapterV2.restoreAdditionalContext")( function* (threadId: string) { const context = (yield* Ref.get(additionalContextByThread)).get(threadId); @@ -1880,6 +1907,8 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( const pendingRuntimeRequests = yield* Ref.make( new Map(), ); + /** Completed by thread/closed after this connection unsubscribes from the native thread. */ + const threadClosings = new Map>(); /** Codex acceptForSession covers only the exact command or patch, not future commands. */ const sessionApprovals = new Map< string, @@ -1926,7 +1955,13 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( const imageTurns = new Map< string, { - readonly running: Set; + readonly running: Map< + string, + { + readonly revisedPrompt: string | null; + readonly transparentBackground: boolean | null; + } + >; readonly saved: Map; pending: boolean; answer?: { readonly id: string; readonly text: string }; @@ -1935,7 +1970,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( const imageTurn = (nativeTurnId: string) => { let state = imageTurns.get(nativeTurnId); if (state === undefined) { - state = { running: new Set(), saved: new Map(), pending: false }; + state = { running: new Map(), saved: new Map(), pending: false }; imageTurns.set(nativeTurnId, state); } return state; @@ -2020,6 +2055,19 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( const clearImageWork = Effect.fn("CodexAdapterV2.clearImageWork")(function* ( context: ActiveCodexTurnContext, ) { + for (const [id, input] of imageTurns.get(context.nativeTurnId)?.running ?? []) { + yield* emitImageGenerationItem( + context, + { + type: "imageGeneration", + id, + ...input, + status: "cancelled", + result: "", + }, + true, + ); + } yield* emitImageWorkState(context, false); imageTurns.delete(context.nativeTurnId); }); @@ -2118,6 +2166,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( startedAt: input.startedAt, itemPositions: new Map(), fileChangeKindsByItem: new Map(), + messagePhases: new Map(), runningCompactionItemIds: new Set(), }; const nativeThreadId = input.turnInput.providerThread.nativeThreadRef?.nativeId; @@ -2724,6 +2773,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( startedAt: turn.startedAt, itemPositions: new Map(), fileChangeKindsByItem: new Map(), + messagePhases: new Map(), runningCompactionItemIds: new Set(), }; beginTurnTokenUsage(activeContext); @@ -3401,6 +3451,9 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( completedAt, updatedAt, type: "assistant_message", + ...(context.messagePhases.has(item.id) + ? { phase: context.messagePhases.get(item.id) } + : {}), messageId, text: item.text, streaming: !completed, @@ -3683,6 +3736,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( ...(item.exitCode === null || item.exitCode === undefined ? {} : { exitCode: item.exitCode }), + // Matches the continuation offered when it completes after the turn. + ...(context.subagent === null && continuationRequests !== undefined + ? { completionWake: "always" as const } + : {}), }; return { node, turnItem }; }); @@ -3871,6 +3928,56 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }, ); + const emitImageGenerationItem = Effect.fn("CodexAdapterV2.emitImageGenerationItem")( + function* ( + context: ActiveCodexTurnContext, + item: Extract< + CodexSchema.V2ItemCompletedNotification__ThreadItem, + { type: "imageGeneration" } + >, + completed: boolean, + ) { + const cancelled = completed && item.status === "cancelled"; + const failed = item.status === "failed" || item.failure != null; + const artifacts = yield* buildDynamicToolArtifacts(context, { + type: "dynamicToolCall", + id: item.id, + namespace: "image_gen", + tool: "imagegen", + status: completed ? (failed ? "failed" : "completed") : "inProgress", + arguments: { + ...(item.revisedPrompt == null ? {} : { prompt: item.revisedPrompt }), + ...(item.transparentBackground == null + ? {} + : { transparent_background: item.transparentBackground }), + }, + }); + yield* emitProviderEvent({ + type: "node.updated", + driver: CODEX_PROVIDER, + node: { ...artifacts.node, ...(cancelled ? { status: "cancelled" } : {}) }, + }); + yield* emitProviderEvent({ + type: "turn_item.updated", + driver: CODEX_PROVIDER, + turnItem: { + ...artifacts.turnItem, + title: "Generate image", + ...(cancelled ? { status: "cancelled" } : {}), + ...(completed + ? { + output: { + status: item.status, + ...(item.savedPath == null ? {} : { savedPath: item.savedPath }), + ...(item.failure == null ? {} : { failure: item.failure }), + }, + } + : {}), + }, + }); + }, + ); + /** * The item stays running until then, which keeps the run's event * ingestion open past turn/completed; it runs off the notification @@ -4514,6 +4621,26 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }).pipe(Effect.orDie), ); + // A run's first request can 429 before Codex pushes a codex snapshot. + const readUsageLimitReset = Effect.gen(function* () { + const sent = yield* Ref.get(rateLimitSnapshot); + const response = yield* client + .request("account/rateLimits/read", null) + .pipe(Effect.timeout("10 seconds")); + const read = response.rateLimitsByLimitId?.codex ?? response.rateLimits; + // A notification that landed during the read is newer than it. + yield* Ref.update(rateLimitSnapshot, (current) => + current === sent ? mergeCodexRateLimits(current, read) : current, + ); + }).pipe( + // The terminal waits on this read, so no failure may skip it; interruption ends both. + Effect.catchCauseIf( + (cause) => !Cause.hasInterrupts(cause), + (cause) => + Effect.logWarning("orchestration-v2.codex-usage-limit-reset-read-failed", { cause }), + ), + ); + yield* client.handleServerNotification("thread/tokenUsage/updated", (payload) => Effect.gen(function* () { accumulateCodexTurnTokenUsage( @@ -4716,7 +4843,8 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( const progress = parseCodexRetryProgress(payload.error.message); const retry: OrchestrationV2ProviderRetry = { attempt: progress?.attempt ?? (previous?.retry.attempt ?? 0) + 1, - maxAttempts: progress?.maxAttempts ?? previous?.retry.maxAttempts ?? null, + // Codex drops the count once it starts waiting for the network, which has no limit. + maxAttempts: progress?.maxAttempts ?? null, retryDelayMs: null, }; const code = codexErrorInfoCode(payload.error.codexErrorInfo); @@ -4751,6 +4879,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( updated.set(context.providerTurnId, state); return updated; }); + yield* failStalledProviderRetry(context, state, payload.error.codexErrorInfo).pipe( + Effect.delay(CODEX_PROVIDER_RETRY_STALL_TIMEOUT), + Effect.forkIn(scope, { startImmediately: true }), + ); yield* emitProviderEvent({ type: "turn_item.updated", driver: CODEX_PROVIDER, @@ -4810,6 +4942,9 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( // their command items, while the shared provider session stays alive. yield* client.handleServerNotification("thread/closed", (payload) => Effect.gen(function* () { + const closing = threadClosings.get(payload.threadId); + threadClosings.delete(payload.threadId); + if (closing !== undefined) yield* Deferred.succeed(closing, undefined); const completedAt = yield* DateTime.now; for (const context of (yield* Ref.get(settledTurns)).values()) { if (context.providerThread.nativeThreadRef?.nativeId !== payload.threadId) continue; @@ -4888,6 +5023,8 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( } if (payload.item.type === "agentMessage") { + if (payload.item.phase != null) + context.messagePhases.set(payload.item.id, payload.item.phase); if (payload.item.phase == null || payload.item.phase === "final_answer") { yield* Ref.update(finalAnswerItemIdsByTurn, (current) => { const updated = new Map(current); @@ -4902,7 +5039,11 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( if (payload.item.type === "imageGeneration") { if (!(yield* Ref.get(interruptingNativeTurns)).has(payload.turnId)) { - imageTurn(payload.turnId).running.add(payload.item.id); + imageTurn(payload.turnId).running.set(payload.item.id, { + revisedPrompt: payload.item.revisedPrompt ?? null, + transparentBackground: payload.item.transparentBackground ?? null, + }); + yield* emitImageGenerationItem(context, payload.item, false); yield* emitImageWorkState(context, true); } return; @@ -4979,9 +5120,8 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( yield* client.handleServerNotification("item/completed", (payload) => Effect.gen(function* () { if (payload.item.type === "contextCompaction") - // Fork requests off the notification reader so it can read their responses. - yield* restoreAdditionalContext(payload.threadId).pipe( - Effect.forkIn(scope, { startImmediately: true }), + yield* Ref.update(threadsAwaitingContextRestore, (current) => + new Set(current).add(payload.threadId), ); if ( (payload.item.type === "subAgentActivity" || @@ -5188,6 +5328,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( if (payload.item.type === "imageGeneration") { if ((yield* Ref.get(interruptingNativeTurns)).has(payload.turnId)) return; + yield* emitImageGenerationItem(context, payload.item, true); const { id, savedPath, status, failure } = payload.item; const state = imageTurn(payload.turnId); if (savedPath?.trim() && status === "completed" && failure == null) { @@ -5205,6 +5346,27 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( return; } + if (payload.item.type === "imageView") { + const artifacts = yield* buildDynamicToolArtifacts(context, { + type: "dynamicToolCall", + id: payload.item.id, + tool: "view_image", + arguments: { path: payload.item.path }, + status: "completed", + }); + yield* emitProviderEvent({ + type: "node.updated", + driver: CODEX_PROVIDER, + node: artifacts.node, + }); + yield* emitProviderEvent({ + type: "turn_item.updated", + driver: CODEX_PROVIDER, + turnItem: { ...artifacts.turnItem, viewedImagePath: payload.item.path }, + }); + return; + } + if (payload.item.type !== "agentMessage") { return; } @@ -5251,6 +5413,8 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( return; } + if (payload.item.phase != null) + context.messagePhases.set(payload.item.id, payload.item.phase); const finalAnswer = payload.item.phase == null || payload.item.phase === "final_answer"; if (finalAnswer) { yield* Ref.update(finalAnswerItemIdsByTurn, (current) => { @@ -5855,7 +6019,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }, ); - const emitRootTerminal = Effect.fnUntraced(function* ( + const publishRootTerminal = Effect.fnUntraced(function* ( context: ActiveCodexTurnContext, event: CodexRootTerminalEvent, ) { @@ -5872,8 +6036,38 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( } : event; yield* emitProviderEvent(current); + if (current.status === "failed" && current.failure.class === "usage_limit") { + const item = makeProviderFailureTurnItem({ + driver: CODEX_PROVIDER, + threadId: context.input.threadId, + runId: context.input.runId, + nodeId: context.input.rootNodeId, + providerThreadId: current.providerThreadId, + providerTurnId: current.providerTurnId, + itemOrdinal: current.failureItemOrdinal, + failure: current.failure, + occurredAt: yield* DateTime.now, + }); + yield* Ref.update(limitedTurnItems, (items) => + new Map(items).set(context.providerThread.id, item), + ); + } + }); + + const emitRootTerminal = Effect.fnUntraced(function* ( + context: ActiveCodexTurnContext, + event: CodexRootTerminalEvent, + ) { + const awaitsReset = + event.status === "failed" && + event.failure.class === "usage_limit" && + event.failure.resetAt == null && + // Coder: managed Codex sharing caps (`CodexManagedErrors`) are not carried, so every + // usage limit without a reset waits for the account read. + codexUsageLimitResetAt(yield* Ref.get(rateLimitSnapshot)) === null; + if (!awaitsReset) yield* publishRootTerminal(context, event); const continued = continuedRuns.get(context.providerTurnId) ?? [context]; - if (current.status !== "completed") { + if (event.status !== "completed") { yield* Ref.update( terminalizedNonCompletedNativeTurns, (nativeTurns) => @@ -5889,20 +6083,13 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( if (nativeThreadId != null && rootProviderThread !== undefined) { rootProviderThreads.set(nativeThreadId, { ...rootProviderThread, status: "idle" }); } - if (current.status === "failed" && current.failure.class === "usage_limit") { - const item = makeProviderFailureTurnItem({ - driver: CODEX_PROVIDER, - threadId: context.input.threadId, - runId: context.input.runId, - nodeId: context.input.rootNodeId, - providerThreadId: current.providerThreadId, - providerTurnId: current.providerTurnId, - itemOrdinal: current.failureItemOrdinal, - failure: current.failure, - occurredAt: yield* DateTime.now, - }); - yield* Ref.update(limitedTurnItems, (items) => - new Map(items).set(context.providerThread.id, item), + // Notification handlers block response routing, so the read and terminal get a fiber. + if (awaitsReset) { + yield* readUsageLimitReset.pipe( + Effect.andThen( + turnTerminalizationPermit.withPermits(1)(publishRootTerminal(context, event)), + ), + Effect.forkIn(scope, { startImmediately: true }), ); } }); @@ -6078,7 +6265,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( state.saved.clear(); return; } - const message = { id: finalAnswer?.id ?? firstImageItemId, text }; + const message = { + id: finalAnswer?.id ?? `generated-image-message:${firstImageItemId}`, + text, + }; state.answer = message; const artifacts = yield* buildAgentMessageArtifacts(context, message, true); yield* emitProviderEvent({ @@ -6108,6 +6298,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( readonly failureMessage?: string; readonly failureErrorInfo?: CodexErrorInfo; readonly continueRun?: boolean; + readonly stalledRetry?: { + readonly state: ActiveCodexProviderRetry; + readonly failure: NonNullable; + }; }) => // Capacity continuation already holds the permit across finalization and restart. (input.continueRun ? Function.identity : turnTerminalizationPermit.withPermits(1))( @@ -6116,6 +6310,16 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( if (current !== input.context) { return false; } + if (input.stalledRetry !== undefined) { + // Checked under the permit so item/started cannot clear the retry mid-finalize. + const retries = yield* Ref.get(providerRetries); + if ( + retries.get(input.context.providerTurnId) !== input.stalledRetry.state || + (yield* Ref.get(interruptingNativeTurns)).has(input.nativeTurnId) + ) + return false; + input.context.latestProviderFailure ??= input.stalledRetry.failure; + } input.context.fileChangeKindsByItem.clear(); const providerRetry = yield* Ref.modify(providerRetries, (current) => { const retry = current.get(input.context.providerTurnId); @@ -6191,6 +6395,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( nativeThreadId != null && (goalsByNativeThread.get(nativeThreadId)?.status === "active" || (activation !== undefined && !activation.stopped)); + const executedModel = reportedTurnModel(nativeThreadId ?? "", input.nativeTurnId); const completedTurn: OrchestrationV2ProviderTurn = { id: input.context.providerTurnId, providerThreadId: input.context.providerThread.id, @@ -6213,11 +6418,9 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( ), input.nativeTurnId, input.status === "completed", - reportedTurnModel( - input.context.providerThread.nativeThreadRef?.nativeId ?? "", - input.nativeTurnId, - ), + executedModel, ), + ...(executedModel === undefined ? {} : { model: executedModel }), }; if (!holdsForGoal) { yield* emitProviderEvent({ @@ -6434,6 +6637,48 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }), ); + // A retry that ends on 429 is the same exhaustion responseTooManyFailedAttempts reports. + const failStalledProviderRetry = Effect.fn("CodexAdapterV2.failStalledProviderRetry")( + function* ( + context: ActiveCodexTurnContext, + state: ActiveCodexProviderRetry, + errorInfo: CodexErrorInfo | null | undefined, + ) { + const finalized = yield* finalizeCodexTurn({ + context, + nativeTurnId: context.nativeTurnId, + status: "failed", + completedAt: yield* DateTime.now, + stalledRetry: { + state, + failure: { + nativeMessage: state.nativeMessage, + nativeCode: state.nativeCode, + failure: { + ...state.failure, + retryable: false, + ...(codexErrorHttpStatus(errorInfo) === 429 ? { class: "usage_limit" } : {}), + }, + }, + }, + }); + if (!finalized) return; + const nativeThreadId = yield* getNativeThreadId(context.providerThread); + yield* client + .request("turn/interrupt", { threadId: nativeThreadId, turnId: context.nativeTurnId }) + .pipe( + Effect.timeout("10 seconds"), + Effect.catch((cause) => + Effect.logWarning("orchestration-v2.codex-stalled-retry-interrupt-failed", { + nativeThreadId, + nativeTurnId: context.nativeTurnId, + cause, + }), + ), + ); + }, + ); + // Codex can end an overloaded turn without retry advice. Continue its // persisted history with empty input instead of resending the user message. const retryCapacityFailure = Effect.fn("CodexAdapterV2.retryCapacityFailure")(function* ( @@ -6571,6 +6816,17 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( yield* client.handleServerNotification("turn/completed", (payload) => Effect.gen(function* () { + const compacted = yield* Ref.modify(threadsAwaitingContextRestore, (current) => { + if (!current.has(payload.threadId)) return [false, current] as const; + const next = new Set(current); + next.delete(payload.threadId); + return [true, next] as const; + }); + if (compacted) + // Fork requests off the notification reader so it can read their responses. + yield* restoreAdditionalContext(payload.threadId).pipe( + Effect.forkIn(scope, { startImmediately: true }), + ); const context = (yield* Ref.get(activeTurns)).get(payload.turn.id); if (context === undefined) { return; @@ -6626,6 +6882,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( startedAt: now, itemPositions: new Map(), fileChangeKindsByItem: new Map(), + messagePhases: new Map(), runningCompactionItemIds: new Set(), }; const providerTurn = { @@ -6765,7 +7022,8 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( ); }); - // Codex refuses to resume a session archived elsewhere until it is unarchived. + // Codex refuses to resume a session archived elsewhere until it is unarchived, and + // one it is still unloading after thread/unsubscribe until it emits thread/closed. const resumeNativeThread = Effect.fn("CodexAdapterV2.resumeNativeThread")(function* ( threadId: string, params: Record, @@ -6773,12 +7031,22 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( const resume = client.raw.request("thread/resume", { threadId, ...params }); return yield* resume.pipe( Effect.catchTags({ - CodexAppServerRequestError: (cause) => - /\bsession \S+ is archived\b|\bcodex unarchive\b/i.test(cause.errorMessage) - ? client.raw - .request("thread/unarchive", { threadId }) - .pipe(Effect.andThen(resume)) - : Effect.fail(cause), + CodexAppServerRequestError: (cause) => { + if (/\bsession \S+ is archived\b|\bcodex unarchive\b/i.test(cause.errorMessage)) { + return client.raw + .request("thread/unarchive", { threadId }) + .pipe(Effect.andThen(resume)); + } + if (/\bthread \S+ is closing\b/i.test(cause.errorMessage)) { + const closing = threadClosings.get(threadId); + return ( + closing === undefined + ? Effect.void + : Deferred.await(closing).pipe(Effect.timeoutOption("10 seconds")) + ).pipe(Effect.andThen(resume)); + } + return Effect.fail(cause); + }, }), ); }); @@ -7058,6 +7326,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( providerSession: session, events: Stream.fromEffectRepeat(Queue.take(events)), canReuseContextUsage: canReuseCodexContextUsage, + normalizeThreadLoadModelSelection: normalizeCodexThreadLoadModelSelection, // Idle release can beat delayed subagent resumes: Codex reports them completed between turns with no resume-expected signal. hasPendingBackgroundWork: Effect.gen(function* () { for (const items of (yield* Ref.get(runningCommandItemsByTurn)).values()) { @@ -7325,6 +7594,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( function* (unloadInput) { const nativeThreadId = yield* getNativeThreadId(unloadInput.providerThread); sessionApprovals.delete(nativeThreadId); + // A resume may already wait on the pending entry; replacing it would strand that wait. + if (!threadClosings.has(nativeThreadId)) { + threadClosings.set(nativeThreadId, yield* Deferred.make()); + } yield* client.request("thread/unsubscribe", { threadId: nativeThreadId }); }, (effect, unloadInput) => @@ -7965,6 +8238,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( const boundary = yield* resolveCodexForkBoundary(threadInput); const fork = client.request("thread/fork", { threadId, + excludeTurns: true, ...(boundary.lastTurnId === undefined ? {} : { lastTurnId: boundary.lastTurnId }), ...codexThreadRuntimeParams({ threadId: threadInput.targetThreadId, diff --git a/apps/server/src/orchestration-v2/Adapters/claudeWorkflowProgress.test.ts b/apps/server/src/orchestration-v2/Adapters/claudeWorkflowProgress.test.ts index 360a21c068f2..451035d669a3 100644 --- a/apps/server/src/orchestration-v2/Adapters/claudeWorkflowProgress.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/claudeWorkflowProgress.test.ts @@ -74,6 +74,19 @@ describe("mergeClaudeWorkflowProgress", () => { expect(workflow?.agents[2]).not.toHaveProperty("completedAt"); }); + it("keeps a started member queued until the provider reports its startedAt", () => { + const scheduled = mergeClaudeWorkflowProgress({ + previous: undefined, + message: snapshotFrame([agent({ state: "start", queuedAt: 1000 })]), + }); + expect(scheduled?.agents[0]?.state).toBe("queued"); + const started = mergeClaudeWorkflowProgress({ + previous: scheduled, + message: snapshotFrame([agent({ state: "start", queuedAt: 1000, startedAt: 21_000 })]), + }); + expect(started?.agents[0]).toMatchObject({ state: "running", startedAt: 21_000 }); + }); + it.each(["something-new", "toString", "constructor", "__proto__"])( "treats unrecognised state %s as still running rather than settled", (state) => { @@ -133,7 +146,14 @@ describe("mergeClaudeWorkflowProgress", () => { agents: first.agents.map((member) => ({ ...member, effort: "max", speed: "standard" })), }, message: snapshotFrame([ - { type: "workflow_agent", index: 1, label: "alpha:one", state: "start", attempt: 2 }, + { + type: "workflow_agent", + index: 1, + label: "alpha:one", + state: "start", + attempt: 2, + startedAt: 3000, + }, ]), }); expect(second?.agents).toEqual([ @@ -142,6 +162,7 @@ describe("mergeClaudeWorkflowProgress", () => { label: "alpha:one", state: "running", attempt: 2, + startedAt: 3000, phaseIndex: 1, phaseTitle: "Alpha", }, @@ -171,12 +192,14 @@ describe("mergeClaudeWorkflowProgress", () => { ])("keeps member state when the same attempt regresses from %s to %s", (state, staleState) => { const first = mergeClaudeWorkflowProgress({ previous: undefined, - message: snapshotFrame([agent({ state, attempt: 2, resultPreview: "Current result" })]), + message: snapshotFrame([ + agent({ state, attempt: 2, startedAt: 1010, resultPreview: "Current result" }), + ]), }); const second = mergeClaudeWorkflowProgress({ previous: first, message: snapshotFrame([ - agent({ state: staleState, attempt: 2, resultPreview: "Old result" }), + agent({ state: staleState, attempt: 2, startedAt: 1010, resultPreview: "Old result" }), ]), }); expect(second?.agents).toEqual(first?.agents); diff --git a/apps/server/src/orchestration-v2/Adapters/claudeWorkflowProgress.ts b/apps/server/src/orchestration-v2/Adapters/claudeWorkflowProgress.ts index 1653e5b42f1d..2db512e82501 100644 --- a/apps/server/src/orchestration-v2/Adapters/claudeWorkflowProgress.ts +++ b/apps/server/src/orchestration-v2/Adapters/claudeWorkflowProgress.ts @@ -53,9 +53,14 @@ function parseAgent(entry: unknown): OrchestrationV2WorkflowAgent | null { const label = text(field(entry, "label")); if (index === undefined || label === undefined) return null; const reportedState = text(field(entry, "state")) ?? ""; - const state = Object.hasOwn(AGENT_STATES, reportedState) - ? (AGENT_STATES[reportedState] ?? "running") - : "running"; + const startedAt = timestamp(field(entry, "startedAt")); + // Claude marks members `start` when scheduled; they only run once a concurrency slot sets startedAt. + const state = + reportedState === "start" && startedAt === undefined + ? "queued" + : Object.hasOwn(AGENT_STATES, reportedState) + ? (AGENT_STATES[reportedState] ?? "running") + : "running"; return { index, label, @@ -69,7 +74,7 @@ function parseAgent(entry: unknown): OrchestrationV2WorkflowAgent | null { ...optional("toolCalls", count(field(entry, "toolCalls"))), ...optional("durationMs", count(field(entry, "durationMs"))), ...optional("queuedAt", timestamp(field(entry, "queuedAt"))), - ...optional("startedAt", timestamp(field(entry, "startedAt"))), + ...optional("startedAt", startedAt), // Execution duration excludes startup waits; terminal progress carries the end instant. ...optional( "completedAt", diff --git a/apps/server/src/orchestration-v2/CheckpointCaptureService.test.ts b/apps/server/src/orchestration-v2/CheckpointCaptureService.test.ts index 9d34049c3b16..828d438646cd 100644 --- a/apps/server/src/orchestration-v2/CheckpointCaptureService.test.ts +++ b/apps/server/src/orchestration-v2/CheckpointCaptureService.test.ts @@ -914,7 +914,24 @@ it.layer(layerProjectionStoreTest)("CheckpointCaptureServiceV2", (it) => { DateTime.formatIso(cancelledAt), ); assert.equal(projected.rootNode?.status, "cancelled"); - assert.deepEqual([...projected.readyCheckpointOrdinals].toSorted(), [0, 1]); + assert.deepEqual([...projected.preservedCheckpointOrdinals].toSorted(), [0, 1]); + + // A rewind leaves the run's checkpoint stale; the next capture must rebaseline it. + yield* projectionStore.apply({ + id: EventId.make("event:checkpoint-capture-cancelled:captured-stale"), + type: "checkpoint.captured", + threadId: cancelledThreadId, + runId: cancelledRunId, + nodeId: cancelledRootNodeId, + providerInstanceId, + occurredAt: cancelledAt, + payload: { ...captured, status: "stale" }, + }); + const afterRewind = yield* projectionStore.getCheckpointCaptureContext(cancelledThreadId, { + runId: cancelledRunId, + scopeId: cancelledScopeId, + }); + assert.deepEqual(afterRewind.preservedCheckpointOrdinals, [0]); }), ); diff --git a/apps/server/src/orchestration-v2/CheckpointCaptureService.ts b/apps/server/src/orchestration-v2/CheckpointCaptureService.ts index 6a2d9b2062b3..43cdae25d4bc 100644 --- a/apps/server/src/orchestration-v2/CheckpointCaptureService.ts +++ b/apps/server/src/orchestration-v2/CheckpointCaptureService.ts @@ -65,7 +65,7 @@ export const layer: Layer.Layer< readonly runId: RunId; readonly scopeId: CheckpointScopeId; }) { - const { run, rootNode, scope, providerThread, readyCheckpointOrdinals, sideChat } = + const { run, rootNode, scope, providerThread, preservedCheckpointOrdinals, sideChat } = yield* projections.getCheckpointCaptureContext(input.threadId, input); // Stopped runs remain terminal; their checkpoint is the next message's rollback point. const stopped = run?.status === "interrupted" || run?.status === "cancelled"; @@ -101,8 +101,8 @@ export const layer: Layer.Layer< const capturedAt = yield* DateTime.now; const baselineOrdinalWithinScope = Math.max(0, run.ordinal - 1); - const hasReadyCheckpoint = (ordinalWithinScope: number) => - readyCheckpointOrdinals.includes(ordinalWithinScope); + const isPreserved = (ordinalWithinScope: number) => + preservedCheckpointOrdinals.includes(ordinalWithinScope); // The baselines this capture materializes are attributed to the run materializing them. The // thread's scope may already be a later run's: one queued meanwhile, or sent after Stop. const materializeBaseline = (ordinalWithinScope: number) => @@ -114,10 +114,8 @@ export const layer: Layer.Layer< forceMissing: sideChat, }); const threadStartCheckpoint = - baselineOrdinalWithinScope === 0 || hasReadyCheckpoint(0) - ? null - : yield* materializeBaseline(0); - const baselineCheckpoint = hasReadyCheckpoint(baselineOrdinalWithinScope) + baselineOrdinalWithinScope === 0 || isPreserved(0) ? null : yield* materializeBaseline(0); + const baselineCheckpoint = isPreserved(baselineOrdinalWithinScope) ? null : yield* materializeBaseline(baselineOrdinalWithinScope); const checkpoint = yield* checkpoints.capture({ diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts index 120efef2eaee..3780bba77cbb 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts @@ -150,7 +150,9 @@ export const layer: Layer.Layer< checkpoint === undefined || scope === undefined || checkpoint.scopeId !== scope.id || - checkpoint.status !== "ready" + // Missing snapshots support conversation rewind only. + (checkpoint.status !== "ready" && + !(input.restoreFiles === false && checkpoint.status === "missing")) ) { return yield* new CheckpointRollbackExecutionError({ reason: "rollback-target-invalid", diff --git a/apps/server/src/orchestration-v2/CheckpointService.test.ts b/apps/server/src/orchestration-v2/CheckpointService.test.ts index 250a079c08b7..e04c473e7e00 100644 --- a/apps/server/src/orchestration-v2/CheckpointService.test.ts +++ b/apps/server/src/orchestration-v2/CheckpointService.test.ts @@ -7,6 +7,7 @@ import { RunId, ThreadId, type OrchestrationV2CheckpointScope, + VcsProcessExitError, VcsProcessTimeoutError, VcsUnsupportedOperationError, } from "@t3tools/contracts"; @@ -96,3 +97,62 @@ it.effect( } }), ); + +it.effect("records a repository probe failure as an error checkpoint, not a missing one", () => + Effect.gen(function* () { + const scope: OrchestrationV2CheckpointScope = { + id: CheckpointScopeId.make("checkpoint-scope:probe-failure"), + threadId: ThreadId.make("thread:probe-failure"), + runId: RunId.make("run:probe-failure:1"), + nodeId: NodeId.make("node:probe-failure:1"), + parentScopeId: null, + providerThreadId: ProviderThreadId.make("provider-thread:probe-failure"), + kind: "root_run", + ordinalWithinParent: 0, + advancesAppRunCount: true, + cwd: "/repo/bad-config", + createdAt: DateTime.makeUnsafe("2026-07-28T00:00:00.000Z"), + }; + const layerTest = CheckpointService.layer.pipe( + Layer.provide( + Layer.mergeAll( + IdAllocator.layer, + Layer.mock(CheckpointStore.CheckpointStore)({ + isGitRepository: () => + Effect.fail( + new VcsProcessExitError({ + operation: "GitVcsDriver.detectRepository", + command: "git", + cwd: scope.cwd, + exitCode: 128, + detail: "fatal: bad config line 1 in file .git/config", + }), + ), + }), + ), + ), + Layer.provideMerge(NodeCrypto.layer), + ); + + const statuses = yield* Effect.gen(function* () { + const checkpoints = yield* CheckpointService.CheckpointServiceV2; + const baseline = yield* checkpoints.materializeBaselineCheckpoint({ + scope, + ordinalWithinScope: 0, + nodeId: scope.nodeId, + capturedAt: scope.createdAt, + }); + const turn = yield* checkpoints.capture({ + scope, + ordinalWithinScope: 1, + runId: scope.runId!, + nodeId: scope.nodeId!, + appRunOrdinal: 1, + capturedAt: scope.createdAt, + }); + return { baseline: baseline.status, turn: turn.status }; + }).pipe(Effect.provide(layerTest)); + + assert.deepEqual(statuses, { baseline: "error", turn: "error" }); + }), +); diff --git a/apps/server/src/orchestration-v2/CheckpointService.ts b/apps/server/src/orchestration-v2/CheckpointService.ts index 09d6f83faef1..6dc489a68b31 100644 --- a/apps/server/src/orchestration-v2/CheckpointService.ts +++ b/apps/server/src/orchestration-v2/CheckpointService.ts @@ -247,14 +247,25 @@ export const layer: Layer.Layer< const withWorkspaceLock = (cwd: string, effect: Effect.Effect) => workspaceLocks.withLock(cwd, effect); - const isGitCheckpointable = (cwd: string) => - checkpointStore.isGitRepository(cwd).pipe(Effect.orElseSucceed(() => false)); + // A failed probe is an operational error, not proof that the workspace has no repository. + const probeRepository = (scope: OrchestrationV2CheckpointScope) => + checkpointStore.isGitRepository(scope.cwd).pipe( + Effect.map((isRepository) => + isRepository ? ("repository" as const) : ("missing" as const), + ), + Effect.catch((cause) => + Effect.logWarning("orchestration V2 checkpoint repository probe failed", { + scopeId: scope.id, + cause: String(cause), + }).pipe(Effect.as("error" as const)), + ), + ); const captureBaseline: CheckpointServiceV2Shape["captureBaseline"] = (input) => withWorkspaceLock( input.scope.cwd, Effect.gen(function* () { - if (!(yield* isGitCheckpointable(input.scope.cwd))) { + if (!(yield* checkpointStore.isGitRepository(input.scope.cwd))) { return; } @@ -305,24 +316,26 @@ export const layer: Layer.Layer< scopeId: input.scope.id, ordinalWithinScope: input.ordinalWithinScope, }); - const checkpointable = - input.forceMissing !== true && (yield* isGitCheckpointable(input.scope.cwd)); - const available = checkpointable - ? yield* checkpointStore - .hasCheckpointRef({ - cwd: input.scope.cwd, - checkpointRef, - }) - .pipe( - Effect.catch((cause) => - Effect.logWarning("orchestration V2 baseline ref lookup failed", { - scopeId: input.scope.id, - checkpointRef, - cause: String(cause), - }).pipe(Effect.as(false)), - ), - ) - : false; + const repository = + input.forceMissing === true ? "missing" : yield* probeRepository(input.scope); + const status = + repository === "repository" + ? yield* checkpointStore + .hasCheckpointRef({ + cwd: input.scope.cwd, + checkpointRef, + }) + .pipe( + Effect.map((exists) => (exists ? ("ready" as const) : ("missing" as const))), + Effect.catch((cause) => + Effect.logWarning("orchestration V2 baseline ref lookup failed", { + scopeId: input.scope.id, + checkpointRef, + cause: String(cause), + }).pipe(Effect.as("error" as const)), + ), + ) + : repository; return makeCheckpoint({ id: checkpointId, scope: input.scope, @@ -332,7 +345,7 @@ export const layer: Layer.Layer< ordinalWithinScope: input.ordinalWithinScope, appRunOrdinal: null, ref: checkpointRef, - status: available ? "ready" : "missing", + status, files: [], capturedAt: input.capturedAt, }); @@ -371,7 +384,9 @@ export const layer: Layer.Layer< ordinalWithinScope: Math.max(0, input.ordinalWithinScope - 1), }); - if (input.forceMissing === true || !(yield* isGitCheckpointable(input.scope.cwd))) { + const repository = + input.forceMissing === true ? "missing" : yield* probeRepository(input.scope); + if (repository !== "repository") { return makeCheckpoint({ id: checkpointId, scope: input.scope, @@ -381,7 +396,7 @@ export const layer: Layer.Layer< ordinalWithinScope: input.ordinalWithinScope, appRunOrdinal: input.appRunOrdinal, ref: checkpointRef, - status: "missing", + status: repository, files: [], capturedAt: input.capturedAt, }); diff --git a/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts b/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts index f1810305b43b..afcaa18d37eb 100644 --- a/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts +++ b/apps/server/src/orchestration-v2/ClaudeAutomaticDelivery.integration.test.ts @@ -492,3 +492,119 @@ it.effect.each(deliveries)( ), ), ); + +it.effect("applies an effort change to the running Claude query without waiting for a turn", () => + Effect.scoped( + Effect.gen(function* () { + const cwd = yield* checkpointWorkspace("claude-live-effort"); + const sdkMessages = yield* Queue.unbounded(); + const flagSettings: unknown[] = []; + const adapter = yield* ClaudeAdapterV2.makeClaudeAdapterV2({ + instanceId: modelSelection.instanceId, + settings, + environment: {}, + attachmentsDir: cwd, + fileSystem: yield* FileSystem.FileSystem, + path: yield* Path.Path, + random: yield* SecureRandom.SecureRandom, + idAllocator: yield* IdAllocator.IdAllocatorV2, + queryRunner: { + allocateSessionId: Effect.succeed(sessionId), + open: () => + Effect.succeed({ + messages: Stream.fromQueue(sdkMessages), + offer: () => Effect.void, + setModel: () => Effect.void, + applyFlagSettings: (flags) => + Effect.sync(() => { + flagSettings.push(flags); + }), + setPermissionMode: () => Effect.void, + backgroundTasks: () => Effect.succeed(false), + mcpServerStatus: Effect.succeed([]), + readMcpResource: () => Effect.die("unexpected MCP resource read"), + callMcpTool: () => Effect.die("unexpected MCP tool call"), + listPermissionRules: Effect.die("unexpected permission rules read"), + stopTask: () => Effect.void, + interrupt: Effect.die("an effort change must not interrupt the turn"), + close: Effect.void, + }), + forkSession: () => Effect.die("unused"), + subagentLaunchToolUseId: () => Effect.succeed(null), + assertComplete: Effect.void, + }, + }); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const threadId = ThreadId.make("thread:live-effort"); + const high = { ...modelSelection, options: [{ id: "effort", value: "high" }] }; + const running = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => event.type === "provider-turn.updated" && event.payload.status === "running", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("create"), + threadId, + projectId: ProjectId.make("project:live-effort"), + title: "Live effort", + modelSelection: high, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: cwd, + createdBy: "user", + creationSource: "web", + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("first"), + threadId, + messageId: MessageId.make("first"), + text: "Keep working.", + attachments: [], + dispatchMode: { type: "start_immediately" }, + createdBy: "user", + creationSource: "web", + }); + yield* worker.drain(); + yield* Fiber.join(running); + assert.deepEqual(flagSettings, []); + + yield* orchestrator.dispatch({ + type: "thread.model-selection.set", + commandId: CommandId.make("medium"), + threadId, + modelSelection: { ...modelSelection, options: [{ id: "effort", value: "medium" }] }, + }); + yield* worker.drain(); + + assert.deepEqual(flagSettings, [ + { effortLevel: "medium", fastMode: null, ultracode: null }, + ]); + }).pipe( + Effect.provide( + ProviderReplayHarness.layerWithRegistry( + { name: "claude-live-effort" }, + ProviderAdapterRegistry.layerSingle(adapter), + { runEffectWorker: false }, + ), + ), + ); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ), +); diff --git a/apps/server/src/orchestration-v2/CommandPolicy.ts b/apps/server/src/orchestration-v2/CommandPolicy.ts index 02f329e536fe..cd56460369bd 100644 --- a/apps/server/src/orchestration-v2/CommandPolicy.ts +++ b/apps/server/src/orchestration-v2/CommandPolicy.ts @@ -10,6 +10,7 @@ import { ProviderInstanceId, ThreadId, } from "@t3tools/contracts"; +import { latestExecutedRun } from "@t3tools/shared/orchestrationV2ThreadError"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; @@ -125,6 +126,15 @@ export function isActiveRun(run: OrchestrationV2Run): boolean { ); } +/** The run a follow-up to a finished child binds to: the latest active one, else the last that ran. */ +export function followUpOwnerRun( + runs: ReadonlyArray, +): OrchestrationV2Run | null { + return ( + runs.filter(isActiveRun).toSorted((a, b) => b.ordinal - a.ordinal)[0] ?? latestExecutedRun(runs) + ); +} + function providerTurnCapabilities( projection: OrchestrationV2ThreadProjection, run: OrchestrationV2Run, diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index 64ed215ebed5..3d4404bfdfde 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -143,6 +143,7 @@ const layerTest = Layer.mergeAll( repositoryIdentityResolved: false, }), invalidate: () => Effect.void, + refreshRepositoryIdentity: () => Effect.void, subscribeChanges: Effect.never, }), ), @@ -754,6 +755,67 @@ it.layer(layerTest)("delegated completion delivery repairs", (it) => { }), ); + it.effect.each([ + { name: "no result is waiting", delivery: "none", compacts: true }, + { name: "a result is waiting to be sent", delivery: "pending", compacts: false }, + { name: "a result was just sent", delivery: "sent", compacts: false }, + ] as const)("compacts before the cache expires only when $name", ({ delivery, compacts }) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make(`thread:idle-compaction-delivery-${delivery}`); + const runId = RunId.make(`run:idle-compaction-delivery-${delivery}`); + const taskId = NodeId.make(`node:idle-compaction-delivery-${delivery}-task`); + + yield* seedParentWithTerminalTask({ + threadId, + projectId: ProjectId.make(`project:idle-compaction-delivery-${delivery}`), + runId, + rootNodeId: NodeId.make(`node:idle-compaction-delivery-${delivery}-root`), + taskId, + deliveryState: delivery === "none" ? "delivered" : "claimed", + completionWake: "always", + parentRunStatus: "completed", + ...(delivery === "none" ? {} : { deliveryTaskIds: [taskId] }), + now, + }); + if (delivery === "sent") { + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`command:idle-compaction-delivery-${delivery}`), + threadId, + messageId: MessageId.make(`message:delegated-delivery:${threadId}`), + text: `Delegated task ${taskId} reached a terminal state.`, + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "agent", + creationSource: "server", + delegatedCompletion: { parentRunId: runId, generation: 1, taskIds: [taskId] }, + }); + } + + const compactionId = MessageId.make(`idle-compact:${threadId}:${runId}`); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`server:idle-compact:${threadId}:${runId}`), + threadId, + messageId: compactionId, + text: "/compact", + attachments: [], + createdBy: "agent", + creationSource: "server", + idleCompaction: "beforeCacheExpires", + dispatchMode: { type: "start_immediately" }, + }); + + const projection = yield* orchestrator.getThreadProjection(threadId); + assert.equal( + projection.messages.some((message) => message.id === compactionId), + compacts, + ); + }), + ); + it.effect.each([ { name: "oversized UTF-8 result", @@ -1055,6 +1117,73 @@ it.layer(layerTest)("delegated completion delivery repairs", (it) => { }), ); + it.effect("acknowledging a queued wake's results rewrites it, then cancels it", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("queued-wake-ack-parent"); + const runId = RunId.make("queued-wake-ack-origin"); + const taskId = NodeId.make("queued-wake-ack-first"); + const siblingId = NodeId.make("queued-wake-ack-sibling"); + yield* seedParentWithTerminalTask({ + threadId, + projectId: ProjectId.make("queued-wake-ack-project"), + runId, + rootNodeId: NodeId.make("queued-wake-ack-root"), + taskId, + deliveryState: "claimed", + deliveryTaskIds: [taskId, siblingId], + siblingTasks: [{ id: siblingId, deliveryState: "claimed" }], + completionWake: "always", + now, + }); + const delivery = (yield* orchestrator.getThreadProjection(threadId)).runs[0]! + .delegatedCompletion!.delivery!; + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "agent", + creationSource: "server", + commandId: CommandId.make("queued-wake-ack-wake"), + threadId, + messageId: delivery.messageId, + text: "Delegated tasks completed", + attachments: [], + modelSelection, + dispatchMode: { type: "queue_after_active" }, + delegatedCompletion: { + parentRunId: runId, + generation: delivery.generation, + taskIds: delivery.taskIds, + }, + }); + const acknowledge = (id: NodeId) => + orchestrator.dispatch({ + type: "delegated_task.completion-delivery.acknowledge", + commandId: CommandId.make(`queued-wake-ack:${id}`), + parentThreadId: threadId, + taskId: id, + observedByRunId: runId, + }); + + yield* acknowledge(taskId); + const rewritten = yield* orchestrator.getThreadProjection(threadId); + const wake = rewritten.runs.find((run) => run.userMessageId === delivery.messageId)!; + assert.equal(wake.status, "queued"); + const message = rewritten.messages.find((row) => row.id === delivery.messageId); + assert.deepEqual(message?.delegatedCompletion?.taskIds, [siblingId]); + assert.equal(message?.text, `Delegated task ${siblingId} [completed]:\nchild finished`); + + yield* acknowledge(siblingId); + const cancelled = yield* orchestrator.getThreadProjection(threadId); + assert.equal(cancelled.runs.find((run) => run.id === wake.id)?.status, "cancelled"); + assert.equal( + cancelled.nodes.find((node) => node.id === wake.rootNodeId)?.status, + "cancelled", + ); + assert.isNull(cancelled.runs.find((run) => run.id === runId)?.delegatedCompletion?.delivery); + }), + ); + it.effect.each([ { command: "acknowledge", before: "running", after: "failed", capture: false }, { command: "acknowledge", before: "waiting", after: "completed", capture: true }, @@ -1380,6 +1509,145 @@ const seedRestartCancelledChild = (input: { return { taskId, childThreadId, childRunId }; }); +// Parent spawned two always-wake tasks: the incoming one's completion wake is queued behind a busy +// run while the sibling's child run is still going. +const queueCompletionWakeBesideRunningSibling = (key: string) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const sink = yield* EventSink.EventSinkV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make(`${key}-parent`); + const originRunId = RunId.make(`${key}-origin`); + const originRootId = NodeId.make(`${key}-origin-root`); + const incomingTaskId = NodeId.make(`${key}-incoming`); + yield* seedParentWithTerminalTask({ + threadId, + projectId: ProjectId.make(`${key}-project`), + runId: originRunId, + rootNodeId: originRootId, + taskId: incomingTaskId, + deliveryState: "claimed", + deliveryTaskIds: [incomingTaskId], + completionWake: "always", + now, + }); + yield* sink.write({ + events: [ + { + id: EventId.make(`event:${key}-origin-root`), + type: "node.updated", + threadId, + runId: originRunId, + nodeId: originRootId, + occurredAt: now, + payload: { + id: originRootId, + threadId, + runId: originRunId, + parentNodeId: null, + rootNodeId: originRootId, + kind: "root_turn", + status: "running", + countsForRun: true, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: now, + completedAt: null, + }, + }, + ], + }); + yield* orchestrator.dispatch({ + type: "delegated_task.request", + createdBy: "agent", + creationSource: "mcp", + commandId: CommandId.make(`${key}-sibling`), + parentThreadId: threadId, + parentRunId: originRunId, + parentNodeId: originRootId, + task: "Keep working", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + completionWake: "always", + }); + const seeded = yield* orchestrator.getThreadProjection(threadId); + const origin = seeded.runs.find((run) => run.id === originRunId)!; + const sibling = seeded.subagents.find((task) => task.id !== incomingTaskId)!; + // The coordinator finished its spawning turn but is busy with another run, so the wake queues. + yield* sink.write({ + events: [ + { + ...runEvent({ threadId, runId: originRunId, ordinal: 1, status: "completed", now }), + payload: { ...origin, status: "completed", completedAt: now }, + }, + runEvent({ + threadId, + runId: RunId.make(`${key}-busy`), + ordinal: 2, + status: "running", + now, + }), + ], + }); + const delivery = origin.delegatedCompletion!.delivery!; + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "agent", + creationSource: "server", + commandId: CommandId.make(`${key}-wake`), + threadId, + messageId: delivery.messageId, + text: "Delegated task completed", + attachments: [], + modelSelection, + dispatchMode: { type: "queue_after_active" }, + delegatedCompletion: { + parentRunId: originRunId, + generation: delivery.generation, + taskIds: delivery.taskIds, + }, + }); + const wake = (yield* orchestrator.getThreadProjection(threadId)).runs.find( + (run) => run.userMessageId === delivery.messageId, + )!; + assert.equal(wake.status, "queued"); + const completeSibling = Effect.gen(function* () { + const childRun = (yield* orchestrator.getThreadProjection(sibling.childThreadId!)).runs[0]!; + const afterSequence = yield* sink.latestSequence(); + yield* sink.write({ + events: [ + { + ...runEvent({ + threadId: sibling.childThreadId!, + runId: childRun.id, + ordinal: childRun.ordinal, + status: "completed", + now, + }), + payload: { ...childRun, status: "completed", completedAt: now }, + }, + ], + }); + return afterSequence; + }); + return { + orchestrator, + sink, + now, + threadId, + originRunId, + incomingTaskId, + sibling, + delivery, + wake, + completeSibling, + }; + }); + it.layer(layerContinuationWorkerTest)("interrupted completion wakes", (it) => { it.effect.each([ { name: "an interrupt keeps", holdQueue: false, siblingFinishesFirst: false }, @@ -1623,6 +1891,137 @@ it.layer(layerContinuationWorkerTest)("interrupted completion wakes", (it) => { ); }), ); + + it.effect("cancelling a queued completion wake keeps its other siblings", () => + Effect.gen(function* () { + const { + orchestrator, + sink, + threadId, + originRunId, + incomingTaskId, + sibling, + delivery, + wake, + completeSibling, + } = yield* queueCompletionWakeBesideRunningSibling("sibling-queued-cancel"); + const cancelled = yield* orchestrator.dispatch({ + type: "queued-run.cancel", + commandId: CommandId.make("sibling-queued-cancel-cancel"), + threadId, + runId: wake.id, + }); + const cancelDelivery = new Map( + cancelled.storedEvents.flatMap(({ event }) => + event.type === "subagent.updated" + ? [[event.payload.id, event.payload.completionDelivery?.state] as const] + : [], + ), + ); + assert.equal(cancelDelivery.get(incomingTaskId), "disposed"); + assert.notEqual( + cancelDelivery.get(sibling.id), + "disposed", + "cancelling the wake must not dispose a still-running sibling it did not carry", + ); + const afterCancel = yield* orchestrator.getThreadProjection(threadId); + assert.equal( + afterCancel.runs.find((run) => run.id === originRunId)?.delegatedCompletion?.disposition, + "open", + ); + + const afterSequence = yield* completeSibling; + yield* sink.stream({ afterSequence, eventType: "subagent.updated" }).pipe( + Stream.filter( + (stored) => + stored.event.type === "subagent.updated" && + stored.event.payload.id === sibling.id && + stored.event.payload.completionDelivery?.state === "claimed", + ), + Stream.take(1), + Stream.runDrain, + ); + const successor = (yield* orchestrator.getThreadProjection(threadId)).runs.find( + (run) => run.id === originRunId, + )?.delegatedCompletion?.delivery; + assert.deepEqual(successor?.taskIds, [sibling.id]); + assert.notEqual(successor?.messageId, delivery.messageId); + }), + ); + + it.effect("settling with a queued completion wake disposes its running siblings", () => + Effect.gen(function* () { + const { orchestrator, sink, now, threadId, originRunId, sibling, wake, completeSibling } = + yield* queueCompletionWakeBesideRunningSibling("sibling-settle"); + const busy = (yield* orchestrator.getThreadProjection(threadId)).runs.find( + (run) => run.status === "running", + )!; + yield* sink.write({ + events: [ + { + ...runEvent({ + threadId, + runId: busy.id, + ordinal: busy.ordinal, + status: "cancelled", + now, + }), + payload: { ...busy, status: "cancelled", completedAt: now }, + }, + { + ...runEvent({ threadId, runId: wake.id, ordinal: wake.ordinal, status: "queued", now }), + payload: { ...wake, queueHeld: true }, + }, + ], + }); + + const settled = yield* orchestrator.dispatch({ + type: "thread.settle", + commandId: CommandId.make("sibling-settle-settle"), + threadId, + }); + assert.deepEqual( + settled.storedEvents.flatMap(({ event }) => + event.type === "run.updated" && event.payload.id === wake.id + ? [event.payload.status] + : [], + ), + ["cancelled"], + ); + const afterSettle = yield* orchestrator.getThreadProjection(threadId); + assert.equal( + afterSettle.subagents.find((task) => task.id === sibling.id)?.completionDelivery?.state, + "disposed", + ); + + const afterSequence = yield* completeSibling; + yield* sink.stream({ afterSequence, eventType: "subagent.updated" }).pipe( + Stream.filter( + (stored) => + stored.event.type === "subagent.updated" && + stored.event.payload.id === sibling.id && + stored.event.payload.status === "completed", + ), + Stream.take(1), + Stream.runDrain, + ); + const afterLateResult = yield* orchestrator.getThreadProjection(threadId); + assert.equal( + afterLateResult.subagents.find((task) => task.id === sibling.id)?.completionDelivery?.state, + "disposed", + ); + assert.deepEqual( + afterLateResult.runs.map((run) => run.id), + afterSettle.runs.map((run) => run.id), + ); + assert.equal( + afterLateResult.runs.find((run) => run.id === originRunId)?.delegatedCompletion + ?.disposition, + "disposed", + ); + assert.equal(afterLateResult.thread.settledOverride, "settled"); + }), + ); }); it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))( @@ -2161,6 +2560,287 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))( }), ); + // The child's follow-up run sends `t3_thread_send` to its own thread before it ends. + const startSelfQueuedFollowup = (name: string) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; + const now = yield* DateTime.now; + const parentThreadId = ThreadId.make(`thread:${name}-parent`); + const projectId = ProjectId.make(`project:${name}`); + const parentRunId = RunId.make(`run:${name}-parent`); + const rootNodeId = NodeId.make(`node:${name}-parent`); + yield* seedParentWithTerminalTask({ + threadId: parentThreadId, + projectId, + runId: parentRunId, + rootNodeId, + taskId: NodeId.make(`node:${name}-seed`), + deliveryState: "acknowledged", + now, + }); + const child = yield* seedRestartCancelledChild({ + parentThreadId, + projectId, + parentRunId, + rootNodeId, + name: `${name}-child`, + completionWake: "always", + continuationPending: false, + runStatus: "completed", + now, + }); + yield* orchestrator.recoverDelegatedTasks; + let firstWake = yield* requests.take; + while (firstWake.threadId !== parentThreadId) firstWake = yield* requests.take; + yield* orchestrator.dispatch({ + type: "delegated_task.completion-delivery.acknowledge", + commandId: CommandId.make(`command:${name}-ack`), + parentThreadId, + taskId: child.taskId, + observedByRunId: parentRunId, + }); + const parentRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs.find( + (run) => run.id === parentRunId, + )!; + yield* eventSink.write({ + commandId: reconcileCommandId(`${name}-parent-finished`), + events: [ + { + id: EventId.make(`event:${name}-parent-finished`), + type: "run.updated", + threadId: parentThreadId, + runId: parentRunId, + occurredAt: now, + payload: { ...parentRun, status: "completed", completedAt: now }, + }, + ], + }); + const followupMessageId = MessageId.make(`message:${name}`); + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "agent", + creationSource: "mcp", + commandId: CommandId.make(`command:${name}`), + threadId: child.childThreadId, + senderThreadId: parentThreadId, + messageId: followupMessageId, + text: "Check the same work again.", + attachments: [], + dispatchMode: { type: "start_immediately" }, + }); + const queuedRun = (yield* orchestrator.getThreadProjection(child.childThreadId)).runs.find( + (row) => row.userMessageId === followupMessageId, + )!; + const followupTaskId = queuedRun.delegatedTaskId; + assert.isDefined(followupTaskId); + const followupRun = { ...queuedRun, status: "running" as const, startedAt: now }; + yield* eventSink.write({ + commandId: reconcileCommandId(`${name}-running`), + events: [ + { + id: EventId.make(`event:${name}-running`), + type: "run.updated", + threadId: child.childThreadId, + runId: followupRun.id, + occurredAt: now, + payload: followupRun, + }, + ], + }); + const selfQueuedMessageId = MessageId.make(`message:${name}-self-send`); + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "agent", + creationSource: "mcp", + commandId: CommandId.make(`command:${name}-self-send`), + threadId: child.childThreadId, + senderThreadId: child.childThreadId, + messageId: selfQueuedMessageId, + text: "Finish the work after this turn.", + attachments: [], + dispatchMode: { type: "queue_after_active" }, + }); + const selfQueuedRun = (yield* orchestrator.getThreadProjection( + child.childThreadId, + )).runs.find((row) => row.userMessageId === selfQueuedMessageId)!; + assert.equal(selfQueuedRun.status, "queued"); + const finishRun = (step: string, run: OrchestrationV2Run, text: string) => + eventSink.write({ + commandId: reconcileCommandId(`${name}-${step}`), + events: [ + { + id: EventId.make(`event:${name}-${step}:result`), + type: "message.updated", + threadId: child.childThreadId, + runId: run.id, + occurredAt: now, + payload: { + id: MessageId.make(`message:${name}-${step}:result`), + threadId: child.childThreadId, + runId: run.id, + nodeId: null, + role: "assistant", + text, + attachments: [], + streaming: false, + createdBy: "agent", + creationSource: "server", + createdAt: now, + updatedAt: now, + }, + }, + { + id: EventId.make(`event:${name}-${step}:finished`), + type: "run.updated", + threadId: child.childThreadId, + runId: run.id, + occurredAt: now, + payload: { ...run, status: "completed", startedAt: now, completedAt: now }, + }, + ], + }); + const removeSelfQueuedRun = orchestrator.dispatch({ + type: "queued-run.cancel", + commandId: CommandId.make(`command:${name}-remove-self-send`), + threadId: child.childThreadId, + runId: selfQueuedRun.id, + }); + const readFollowup = Effect.gen(function* () { + yield* orchestrator.recoverDelegatedTasks; + return (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find( + (task) => task.id === followupTaskId, + )!; + }); + return { + parentThreadId, + parentRunId, + childThreadId: child.childThreadId, + firstWake, + followupRun, + selfQueuedRun, + finishRun, + removeSelfQueuedRun, + readFollowup, + }; + }); + + it.effect("waits for a run the follow-up's child queued on its own thread", () => + Effect.gen(function* () { + const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; + const followup = yield* startSelfQueuedFollowup("self-queued-followup"); + yield* followup.finishRun("handoff", followup.followupRun, "HANDOFF-C"); + const waiting = yield* followup.readFollowup; + assert.isNull( + waiting.result, + "the follow-up task settled on its handoff turn while the run its child queued was pending", + ); + assert.notEqual(waiting.status, "completed"); + + yield* followup.finishRun("final", followup.selfQueuedRun, "FINAL-D"); + const finished = yield* followup.readFollowup; + assert.equal(finished.status, "completed"); + assert.equal(finished.result, "FINAL-D"); + assert.equal(finished.completionDelivery?.state, "claimed"); + let wake = yield* requests.take; + while ( + wake.threadId !== followup.parentThreadId || + wake.delegatedCompletion?.messageId === followup.firstWake.delegatedCompletion?.messageId + ) { + wake = yield* requests.take; + } + assert.equal(wake.delegatedCompletion?.parentRunId, followup.parentRunId); + }), + ); + + it.effect("keeps a follow-up running when the run its child queued is removed early", () => + Effect.gen(function* () { + const followup = yield* startSelfQueuedFollowup("self-queued-removed-early-followup"); + yield* followup.removeSelfQueuedRun; + const running = yield* followup.readFollowup; + assert.isNull(running.result); + assert.equal(running.status, "running"); + + yield* followup.finishRun("handoff", followup.followupRun, "HANDOFF-C"); + const finished = yield* followup.readFollowup; + assert.equal(finished.status, "completed"); + assert.equal(finished.result, "HANDOFF-C"); + }), + ); + + it.effect.each(["removed", "held"] as const)( + "settles a follow-up on its last turn when the run it queued is %s", + (ending) => + Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const now = yield* DateTime.now; + const followup = yield* startSelfQueuedFollowup(`self-queued-${ending}-followup`); + if (ending === "held") { + yield* eventSink.write({ + commandId: reconcileCommandId("self-queued-held-followup-hold"), + events: [ + { + id: EventId.make("event:self-queued-held-followup-hold"), + type: "run.updated", + threadId: followup.childThreadId, + runId: followup.selfQueuedRun.id, + occurredAt: now, + payload: { ...followup.selfQueuedRun, queueHeld: true }, + }, + ], + }); + } + yield* followup.finishRun("handoff", followup.followupRun, "HANDOFF-C"); + if (ending === "removed") yield* followup.removeSelfQueuedRun; + const finished = yield* followup.readFollowup; + assert.equal(finished.status, "completed"); + assert.equal(finished.result, "HANDOFF-C"); + }), + ); + + it.effect("waits for every run its child queued when a later one is moved ahead", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const followup = yield* startSelfQueuedFollowup("self-queued-reordered-followup"); + const laterMessageId = MessageId.make("message:self-queued-reordered-followup-later"); + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "agent", + creationSource: "mcp", + commandId: CommandId.make("command:self-queued-reordered-followup-later"), + threadId: followup.childThreadId, + senderThreadId: followup.childThreadId, + messageId: laterMessageId, + text: "Do this part first.", + attachments: [], + dispatchMode: { type: "queue_after_active" }, + }); + const later = (yield* orchestrator.getThreadProjection(followup.childThreadId)).runs.find( + (run) => run.userMessageId === laterMessageId, + )!; + yield* orchestrator.dispatch({ + type: "queued-run.reorder", + commandId: CommandId.make("command:self-queued-reordered-followup-reorder"), + threadId: followup.childThreadId, + runId: later.id, + beforeRunId: followup.selfQueuedRun.id, + }); + assert.equal((yield* followup.readFollowup).status, "running"); + yield* followup.finishRun("handoff", followup.followupRun, "HANDOFF-C"); + yield* followup.finishRun("later", later, "PART-ONE"); + const waiting = yield* followup.readFollowup; + assert.isNull(waiting.result, "the follow-up settled while an earlier queued run waited"); + + // Promotion moves a queued run after the runs that started ahead of it. + const promoted = { ...followup.selfQueuedRun, ordinal: later.ordinal + 1 }; + yield* followup.finishRun("final", promoted, "FINAL-D"); + const finished = yield* followup.readFollowup; + assert.equal(finished.status, "completed"); + assert.equal(finished.result, "FINAL-D"); + }), + ); + const outcomes = [ "completed", "failed", @@ -2168,6 +2848,8 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))( "restarted", "stopped", "overlapping", + "after-cancel", + "cancelled", ] as const; it.effect.each(outcomes)( "delivers a %s follow-up independently of the original task", @@ -2212,6 +2894,26 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))( taskId: child.taskId, observedByRunId: parentRunId, }); + if (outcome === "after-cancel") { + // A follow-up is its own task, even after task_cancel reached the original. + const settled = (yield* orchestrator.getThreadProjection( + parentThreadId, + )).subagents.find((task) => task.id === child.taskId)!; + // The follow-up starts in the same millisecond the cancel was recorded. + yield* eventSink.write({ + commandId: reconcileCommandId("followup-original-cancelled"), + events: [ + { + id: EventId.make("event:followup-original-cancelled"), + type: "subagent.updated", + threadId: parentThreadId, + nodeId: settled.id, + occurredAt: now, + payload: { ...settled, cancelledRunId: child.childRunId }, + }, + ], + }); + } const original = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find( (task) => task.id === child.taskId, )!; @@ -2238,7 +2940,8 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))( creationSource: outcome === "failed" ? "web" : "mcp", commandId: CommandId.make(`command:followup-send-${outcome}`), threadId: child.childThreadId, - senderThreadId: parentThreadId, + // `t3 thread send` and unscoped MCP sessions carry no sender thread. + ...(outcome === "completed" ? {} : { senderThreadId: parentThreadId }), messageId, text: "Check the same work again.", attachments: [], @@ -2321,9 +3024,54 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))( commandId: CommandId.make(`command:followup-stop-${outcome}`), }); } + if (outcome === "cancelled") { + // task_cancel reaches the follow-up after its turn ended, while background work runs. + yield* eventSink.write({ + commandId: reconcileCommandId("followup-waiting"), + events: [ + { + id: EventId.make("event:followup-waiting"), + type: "run.updated", + threadId: child.childThreadId, + runId: run.id, + occurredAt: now, + payload: { ...run, status: "waiting", startedAt: now }, + }, + ], + }); + yield* orchestrator.cancelDelegatedTask({ + parentThreadId, + taskId: run.delegatedTaskId!, + command: { + type: "thread.stop", + commandId: CommandId.make(`cancel-followup-${outcome}`), + threadId: child.childThreadId, + stopDelegatedTasks: false, + }, + stopDescendants: () => Effect.void, + }); + } yield* eventSink.write({ commandId: reconcileCommandId(`followup-finish-${outcome}`), events: [ + { + id: EventId.make(`event:followup-question-${outcome}`), + type: "runtime-request.updated", + threadId: child.childThreadId, + runId: run.id, + occurredAt: now, + payload: { + id: RuntimeRequestId.make(`request:followup-${outcome}`), + nodeId: run.rootNodeId!, + providerTurnId: null, + nativeRequestRef: null, + kind: "user_input", + status: "pending", + responseCapability: { type: "message" }, + createdAt: now, + resolvedAt: null, + }, + }, { id: EventId.make(`event:followup-finish-${outcome}`), type: "run.updated", @@ -2335,7 +3083,10 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))( status: outcome === "stopped" ? "interrupted" - : outcome === "restarted" || outcome === "overlapping" + : outcome === "restarted" || + outcome === "overlapping" || + outcome === "after-cancel" || + outcome === "cancelled" ? "completed" : outcome, startedAt: now, @@ -2345,6 +3096,12 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))( ], }); yield* orchestrator.recoverDelegatedTasks; + // Only an interrupted or cancelled follow-up ends its question, even after the original + // was cancelled. + const ended = + outcome === "stopped" || outcome === "interrupted" || outcome === "cancelled"; + const { runtimeRequests } = yield* orchestrator.getThreadProjection(child.childThreadId); + assert.equal(runtimeRequests[0]?.status, ended ? "resolved" : "pending"); const finished = yield* orchestrator.getThreadProjection(parentThreadId); assert.deepEqual( finished.subagents.find((task) => task.id === child.taskId), @@ -2363,17 +3120,18 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))( } assert.equal( followup.status, - outcome === "stopped" + outcome === "stopped" || outcome === "cancelled" ? "interrupted" - : outcome === "restarted" || outcome === "overlapping" + : outcome === "restarted" || outcome === "overlapping" || outcome === "after-cancel" ? "completed" : outcome, ); assert.equal( followup.completionDelivery?.state, - outcome === "stopped" ? "disposed" : "claimed", + outcome === "stopped" || outcome === "cancelled" ? "disposed" : "claimed", ); - if (outcome !== "stopped") { + assert.equal(followup.cancelledRunId, outcome === "cancelled" ? run.id : undefined); + if (outcome !== "stopped" && outcome !== "cancelled") { let wake = yield* requests.take; while ( wake.threadId !== parentThreadId || @@ -3141,6 +3899,54 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => { }), ); + it.effect("cancelling a task whose child already finished keeps its completed result", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread:cancel-finished-parent"); + const projectId = ProjectId.make("project:cancel-finished-parent"); + const runId = RunId.make("run:cancel-finished-parent"); + const rootNodeId = NodeId.make("node:cancel-finished-parent-root"); + yield* seedParentWithTerminalTask({ + threadId, + projectId, + runId, + rootNodeId, + taskId: NodeId.make("node:cancel-finished-parent-settled"), + deliveryState: "delivered", + now, + }); + // The child run finished, but its parent row has not been finalized yet. + const child = yield* seedRestartCancelledChild({ + parentThreadId: threadId, + projectId, + parentRunId: runId, + rootNodeId, + name: "cancel-finished-child", + completionWake: "always", + continuationPending: false, + runStatus: "completed", + now, + }); + yield* orchestrator.cancelDelegatedTask({ + parentThreadId: threadId, + taskId: child.taskId, + command: { + type: "thread.stop", + commandId: CommandId.make("cancel-finished-child"), + threadId: child.childThreadId, + stopDelegatedTasks: false, + }, + stopDescendants: () => Effect.void, + }); + const task = (yield* orchestrator.getThreadProjection(threadId)).subagents.find( + (row) => row.id === child.taskId, + ); + assert.equal(task?.status, "completed"); + assert.isUndefined(task?.cancelledRunId); + }), + ); + // Running does not prove prompt consumption in Claude; re-offer cut deliveries even when resuming the turn. it.effect("re-offers a cut delivery even when a continuation resumes its turn", () => Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/DelegatedTaskQuestionReactor.ts b/apps/server/src/orchestration-v2/DelegatedTaskQuestionReactor.ts index f6eeb9a00ede..9fd17580e333 100644 --- a/apps/server/src/orchestration-v2/DelegatedTaskQuestionReactor.ts +++ b/apps/server/src/orchestration-v2/DelegatedTaskQuestionReactor.ts @@ -4,6 +4,8 @@ import { type OrchestrationV2DomainEvent, type OrchestrationV2RuntimeRequest, type OrchestrationV2Subagent, + type RuntimeRequestId, + type ThreadId, } from "@t3tools/contracts"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; @@ -58,6 +60,37 @@ const make = Effect.gen(function* () { }); }); + const cancelQueuedWakes = Effect.fnUntraced(function* ( + parentThreadId: ThreadId, + requestIds: ReadonlyArray, + ) { + const parent = yield* projections.getThreadRecords(parentThreadId, ["runs"]); + for (const requestId of requestIds) { + const key = `${parentThreadId}:${requestId}`; + const messageId = MessageId.make(`message:delegated-task-question:${key}`); + const queued = parent.runs.find( + (run) => run.status === "queued" && run.userMessageId === messageId, + ); + if (queued === undefined) continue; + yield* engine + .dispatch({ + type: "queued-run.cancel", + commandId: CommandId.make(`server:delegated-task-question-cancel:${key}`), + threadId: parentThreadId, + runId: queued.id, + }) + .pipe( + Effect.catchCause((cause) => + Effect.logWarning("cancelling a delegated task question wake failed", { + threadId: parentThreadId, + requestId, + cause, + }), + ), + ); + } + }); + // Runs for every domain event; most events return before any work is needed. const handle: DelegatedTaskQuestionReactor["Service"]["handle"] = Effect.fnUntraced( function* (event) { @@ -100,6 +133,46 @@ const make = Effect.gen(function* () { } return; } + if (event.type === "subagent.updated") { + const task = event.payload; + if ( + task.origin !== "app_owned" || + task.childThreadId === null || + !["completed", "failed", "cancelled", "interrupted"].includes(task.status) + ) { + return; + } + // A settled task can no longer use an answer to its still-pending questions. + const { runtimeRequests } = yield* projections.getThreadRecords(task.childThreadId, [ + "runtimeRequests", + ]); + const questions = runtimeRequests.filter((request) => request.kind === "user_input"); + if (questions.length === 0) return; + const child = yield* projections.getThreadRecords(task.childThreadId, ["runs", "nodes"]); + const taskRunIds = new Set( + child.runs + .filter( + (run) => + run.delegatedTaskId === task.id || + (run.delegatedTaskId === undefined && + child.thread.forkedFrom?.type === "node" && + child.thread.forkedFrom.nodeId === task.id), + ) + .map((run) => run.id), + ); + const taskNodeIds = new Set( + child.nodes + .filter((node) => node.runId !== null && taskRunIds.has(node.runId)) + .map((node) => node.id), + ); + yield* cancelQueuedWakes( + task.threadId, + questions + .filter((request) => taskNodeIds.has(request.nodeId)) + .map((request) => request.id), + ); + return; + } if (event.type !== "runtime-request.updated" || event.payload.kind !== "user_input") { return; } @@ -107,21 +180,8 @@ const make = Effect.gen(function* () { const parentThreadId = child.lineage.parentThreadId; if (parentThreadId === null) return; const request = yield* projections.getRuntimeRequest(event.threadId, event.payload.id); - const key = `${parentThreadId}:${event.payload.id}`; - const messageId = MessageId.make(`message:delegated-task-question:${key}`); if (event.payload.status !== "pending" || request?.status !== "pending") { - const parent = yield* projections.getThreadRecords(parentThreadId, ["runs"]); - const queued = parent.runs.find( - (run) => run.status === "queued" && run.userMessageId === messageId, - ); - if (queued !== undefined) { - yield* engine.dispatch({ - type: "queued-run.cancel", - commandId: CommandId.make(`server:delegated-task-question-cancel:${key}`), - threadId: parentThreadId, - runId: queued.id, - }); - } + yield* cancelQueuedWakes(parentThreadId, [event.payload.id]); return; } const parent = yield* projections.getThreadRecords(parentThreadId, ["subagents", "runs"]); diff --git a/apps/server/src/orchestration-v2/EffectOutbox.ts b/apps/server/src/orchestration-v2/EffectOutbox.ts index 92e1e01ad43f..58aa3ea97489 100644 --- a/apps/server/src/orchestration-v2/EffectOutbox.ts +++ b/apps/server/src/orchestration-v2/EffectOutbox.ts @@ -3,6 +3,7 @@ import { CheckpointScopeId, CommandId, MessageId, + ModelSelection, NodeId, ProviderSessionId, RunAttemptId, @@ -40,6 +41,12 @@ export const OrchestrationEffectRequestV2 = Schema.Union([ detail: Schema.optional(Schema.String), revokeMcpCredential: Schema.optional(Schema.Boolean), }), + Schema.Struct({ + type: Schema.Literal("provider-session.apply-model-selection"), + providerSessionId: ProviderSessionId, + providerThreadId: ProviderThreadId, + modelSelection: ModelSelection, + }), Schema.Struct({ type: Schema.Literal("provider-turn.start"), runId: RunId, @@ -57,6 +64,7 @@ export const OrchestrationEffectRequestV2 = Schema.Union([ providerThreadId: ProviderThreadId, providerTurnId: ProviderTurnId, messageId: MessageId, + modelSelection: Schema.optional(ModelSelection), }), Schema.Struct({ type: Schema.Literal("provider-turn.restart"), @@ -136,6 +144,7 @@ export const REPLAY_SAFE_EFFECT_TYPES_AFTER_PROCESS_LOSS = [ ] as const satisfies ReadonlyArray; export const PROCESS_BOUND_EFFECT_TYPES = [ + "provider-session.apply-model-selection", "provider-turn.start", "provider-turn.interrupt", "provider-turn.steer", @@ -224,8 +233,11 @@ export interface EffectOutboxV2Shape { readonly claimNext: (input: { readonly workerId: string; readonly excludeRestartContinuations?: boolean; + readonly maxRunningTitleGenerations?: number; }) => Effect.Effect, EffectOutboxError>; - readonly nextClaimableAt: Effect.Effect, EffectOutboxError>; + readonly nextClaimableAt: (input?: { + readonly maxRunningTitleGenerations?: number; + }) => Effect.Effect, EffectOutboxError>; readonly succeed: (input: { readonly effectId: string; readonly workerId: string; @@ -314,6 +326,7 @@ export const layer: Layer.Layer = La const claimableCandidatePredicate = ( availableBefore?: string, excludeRestartContinuations = false, + maxRunningTitleGenerations?: number, ) => sql` ${ @@ -350,6 +363,19 @@ export const layer: Layer.Layer = La ) ) ) + AND ${ + maxRunningTitleGenerations === undefined + ? sql`1 = 1` + : sql`( + candidate.effect_type != 'thread-title.generate' + OR ( + SELECT COUNT(*) + FROM orchestration_v2_effect_outbox AS running_title + WHERE running_title.status = 'running' + AND running_title.effect_type = 'thread-title.generate' + ) < ${maxRunningTitleGenerations} + )` + } `; const cancellationSignal = (effectId: string) => { @@ -519,7 +545,7 @@ export const layer: Layer.Layer = La ), // The worker polls for the next effect even when there is no work to claim. claimNext: Effect.fnUntraced( - function* ({ workerId, excludeRestartContinuations = false }) { + function* ({ workerId, excludeRestartContinuations = false, maxRunningTitleGenerations }) { const now = yield* DateTime.now; const nowIso = DateTime.formatIso(now); // Idle claims return empty; tracing their full SQL on every poll overwhelms useful traces. @@ -534,7 +560,11 @@ export const layer: Layer.Layer = La WHERE effect_id = ( SELECT candidate.effect_id FROM orchestration_v2_effect_outbox AS candidate - WHERE ${claimableCandidatePredicate(nowIso, excludeRestartContinuations)} + WHERE ${claimableCandidatePredicate( + nowIso, + excludeRestartContinuations, + maxRunningTitleGenerations, + )} AND ${excludeRestartContinuations ? sql`candidate.effect_type != 'provider-runtime.continue'` : sql`1 = 1`} ORDER BY candidate.available_at ASC, candidate.created_at ASC, candidate.effect_id ASC LIMIT 1 @@ -549,23 +579,26 @@ export const layer: Layer.Layer = La }, Effect.mapError((cause) => new EffectOutboxError({ operation: "claim", cause })), ), - nextClaimableAt: Effect.gen(function* () { - const rows = yield* sql<{ readonly available_at: string | null }>` - SELECT MIN(candidate.available_at) AS available_at - FROM orchestration_v2_effect_outbox AS candidate - WHERE ${claimableCandidatePredicate()} - `.pipe(NodeSqliteClient.readOnly, Effect.withTracerEnabled(false)); - const availableAt = rows[0]?.available_at; - if (availableAt === undefined || availableAt === null) return Option.none(); - const parsed = DateTime.make(availableAt); - if (Option.isNone(parsed)) { - return yield* new EffectOutboxError({ - operation: "next-claimable", - cause: `Invalid available_at timestamp: ${availableAt}`, - }); - } - return parsed; - }).pipe( + nextClaimableAt: Effect.fnUntraced( + function* ({ + maxRunningTitleGenerations, + }: { readonly maxRunningTitleGenerations?: number } = {}) { + const rows = yield* sql<{ readonly available_at: string | null }>` + SELECT MIN(candidate.available_at) AS available_at + FROM orchestration_v2_effect_outbox AS candidate + WHERE ${claimableCandidatePredicate(undefined, false, maxRunningTitleGenerations)} + `.pipe(NodeSqliteClient.readOnly, Effect.withTracerEnabled(false)); + const availableAt = rows[0]?.available_at; + if (availableAt === undefined || availableAt === null) return Option.none(); + const parsed = DateTime.make(availableAt); + if (Option.isNone(parsed)) { + return yield* new EffectOutboxError({ + operation: "next-claimable", + cause: `Invalid available_at timestamp: ${availableAt}`, + }); + } + return parsed; + }, Effect.mapError((cause) => isEffectOutboxError(cause) ? cause diff --git a/apps/server/src/orchestration-v2/EffectWorker.test.ts b/apps/server/src/orchestration-v2/EffectWorker.test.ts index 3be77bba74c7..cc43601543ee 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.test.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.test.ts @@ -93,6 +93,7 @@ function layerExecutorFor(input: { ProviderTurnControlService.ProviderTurnControlServiceV2.of({ interrupt: input.interrupt ?? (() => Effect.void), steer: () => Effect.void, + applyModelSelection: () => Effect.void, interruptAndAwaitTerminal: (request) => record( request.replacementProviderSessionId === undefined @@ -376,7 +377,7 @@ it.effect("requeues a claim when a pre-execution worker check fails", () => ); const exit = yield* EffectWorker.OrchestrationEffectWorkerV2.pipe( - Effect.flatMap((worker) => worker.runOnce), + Effect.flatMap((worker) => worker.runOnce()), Effect.provide(layerWorker), Effect.exit, ); @@ -446,7 +447,7 @@ it.effect("arms cancellation before the durable pre-execution check", () => ); const exit = yield* EffectWorker.OrchestrationEffectWorkerV2.pipe( - Effect.flatMap((worker) => worker.runOnce), + Effect.flatMap((worker) => worker.runOnce()), Effect.provide(layerWorker), Effect.exit, ); @@ -512,7 +513,7 @@ it.effect("terminalizes a process-bound claim when success settlement fails", () ); const exit = yield* EffectWorker.OrchestrationEffectWorkerV2.pipe( - Effect.flatMap((worker) => worker.runOnce), + Effect.flatMap((worker) => worker.runOnce()), Effect.provide(layerWorker), Effect.exit, ); diff --git a/apps/server/src/orchestration-v2/EffectWorker.ts b/apps/server/src/orchestration-v2/EffectWorker.ts index 7d210f767867..8177a4a4b0ee 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.ts @@ -131,6 +131,24 @@ export const layerExecutor: Layer.Layer< }), ), ); + case "provider-session.apply-model-selection": + return providerTurnControl + .applyModelSelection({ + threadId: effect.threadId, + providerSessionId: effect.request.providerSessionId, + providerThreadId: effect.request.providerThreadId, + modelSelection: effect.request.modelSelection, + }) + .pipe( + Effect.mapError( + (cause) => + new OrchestrationEffectExecutionError({ + effectId: effect.id, + effectType: effect.request.type, + cause, + }), + ), + ); case "provider-turn.start": return providerTurnStart .start({ threadId: effect.threadId, runId: effect.request.runId, willRetry }) @@ -232,6 +250,9 @@ export const layerExecutor: Layer.Layer< providerThreadId: effect.request.providerThreadId, providerTurnId: effect.request.providerTurnId, messageId: effect.request.messageId, + ...(effect.request.modelSelection === undefined + ? {} + : { modelSelection: effect.request.modelSelection }), }) .pipe( Effect.tap(() => @@ -498,8 +519,19 @@ export const layerExecutor: Layer.Layer< }), ), ); + // Archive and delete queue this one per-thread teardown, so it also releases the + // thread's device claims; the devices stay booted for other threads. case "terminal.cleanup": - return resourceCleanup.cleanupTerminals(effect.threadId).pipe( + return Effect.partition( + [ + resourceCleanup.cleanupTerminals(effect.threadId), + resourceCleanup.cleanupDevices(effect.threadId), + ], + (cleanup) => cleanup, + ).pipe( + Effect.flatMap(([, [failure]]) => + failure === undefined ? Effect.void : Effect.fail(failure), + ), Effect.mapError( (cause) => new OrchestrationEffectExecutionError({ @@ -605,14 +637,17 @@ export class OrchestrationEffectWorkerError extends Schema.TaggedError[0], "workerId">; + export interface OrchestrationEffectWorkerV2Shape { readonly awaitWork: Effect.Effect; - readonly runOnce: Effect.Effect; + readonly runOnce: (input?: { + readonly maxRunningTitleGenerations?: number; + }) => Effect.Effect; readonly runRecoveryOnce: Effect.Effect; - readonly nextClaimableAt: Effect.Effect< - Option.Option, - OrchestrationEffectWorkerError - >; + readonly nextClaimableAt: (input?: { + readonly maxRunningTitleGenerations?: number; + }) => Effect.Effect, OrchestrationEffectWorkerError>; readonly drain: (maxEffects?: number) => Effect.Effect; } @@ -732,10 +767,8 @@ export const layerWithOptions = ( // The worker probes for work repeatedly, including when the outbox is empty. const runOnce = Effect.fnUntraced( - function* (excludeRestartContinuations = false) { - const claimExit = yield* Effect.exit( - outbox.claimNext({ workerId, excludeRestartContinuations }), - ); + function* (filter: ClaimFilter = {}) { + const claimExit = yield* Effect.exit(outbox.claimNext({ workerId, ...filter })); // Coder: no orchestration metrics are recorded. if (Exit.isFailure(claimExit)) return yield* Effect.failCause(claimExit.cause); const claimed = claimExit.value; @@ -816,7 +849,7 @@ export const layerWithOptions = ( } return true; }, - (effect, _excludeRestartContinuations?: boolean) => + (effect, _filter?: ClaimFilter) => effect.pipe( Effect.mapError((cause) => isOrchestrationEffectWorkerError(cause) @@ -828,9 +861,13 @@ export const layerWithOptions = ( return OrchestrationEffectWorkerV2.of({ awaitWork: outbox.awaitAvailable, - runOnce: runOnce(), - runRecoveryOnce: runOnce(true), - nextClaimableAt: outbox.nextClaimableAt.pipe( + runOnce, + runRecoveryOnce: runOnce({ excludeRestartContinuations: true }), + // Idle workers poll this; a span per poll would bury useful traces. + nextClaimableAt: Effect.fnUntraced( + function* (input) { + return yield* outbox.nextClaimableAt(input); + }, Effect.mapError( (cause) => new OrchestrationEffectWorkerError({ @@ -873,10 +910,12 @@ export const runDaemonWithOptions = (options: OrchestrationEffectDaemonOptions = const livenessPollIntervalMs = Number.isFinite(requestedLivenessPollIntervalMs) ? Math.max(1, Math.floor(requestedLivenessPollIntervalMs)) : DEFAULT_EFFECT_WORKER_LIVENESS_POLL_INTERVAL_MS; + // Title generation can run for minutes; leave at least one worker free for turn work. + const claimLimits = concurrency > 1 ? { maxRunningTitleGenerations: concurrency - 1 } : {}; // Notifications wake promptly; availableAt persists retry timing, and liveness polling recovers missed wakeups. const runWorker = Effect.gen(function* () { while (true) { - const outcome = yield* worker.runOnce.pipe( + const outcome = yield* worker.runOnce(claimLimits).pipe( Effect.map((worked) => (worked ? ("worked" as const) : ("idle" as const))), Effect.catchCause((cause) => Effect.logWarning("Orchestration effect worker failed", cause).pipe( @@ -894,14 +933,16 @@ export const runDaemonWithOptions = (options: OrchestrationEffectDaemonOptions = continue; } - const nextClaimableAt = yield* worker.nextClaimableAt.pipe( - Effect.catchCause((cause) => - Effect.logWarning( - "Failed to read the next orchestration effect deadline", - cause, - ).pipe(Effect.as(Option.none())), - ), - ); + const nextClaimableAt = yield* worker + .nextClaimableAt(claimLimits) + .pipe( + Effect.catchCause((cause) => + Effect.logWarning( + "Failed to read the next orchestration effect deadline", + cause, + ).pipe(Effect.as(Option.none())), + ), + ); const now = DateTime.toEpochMillis(yield* DateTime.now); const sleepMs = Option.match(nextClaimableAt, { onNone: () => livenessPollIntervalMs, diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index 720a95998e56..0e4c1f9eafbb 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -3,6 +3,7 @@ import { type OrchestrationV2Run, OrchestrationV2DomainEvent, type OrchestrationV2ProviderThread, + type OrchestrationV2TurnItem, OrchestrationV2StoredEvent, ProviderThreadId, RunAttemptId, @@ -12,7 +13,6 @@ import { type ProjectId, ThreadId, } from "@t3tools/contracts"; -import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import * as ProviderCommitBatch from "./ProviderCommitBatch.ts"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; @@ -78,9 +78,10 @@ const CurrentProviderBatch = Context.Reference( - source: Stream.Stream, - process: (event: ProviderAdapter.ProviderAdapterV2Event) => Effect.Effect, + /** Items that are not provider snapshots, such as a consumer's own control items, never batch. */ + readonly consumeProviderEvents: ( + source: Stream.Stream, + process: (event: A) => Effect.Effect, after: Effect.Effect, canBatch: Effect.Effect, ) => Effect.Effect< @@ -126,6 +127,11 @@ export interface EventSinkV2Shape { readonly runId: RunId; readonly activeAttemptId: RunAttemptId; readonly expectedStatus: OrchestrationV2Run["status"]; + /** Also refuse while this run interrupt request is stored without its result. */ + readonly unlessInterruptRequested?: { + readonly requestId: OrchestrationV2TurnItem["id"]; + readonly resultId: OrchestrationV2TurnItem["id"]; + }; readonly events: ReadonlyArray; readonly effects?: ReadonlyArray; }) => Effect.Effect< @@ -507,6 +513,24 @@ const layerBase: Layer.Layer< storedEvents: [] as ReadonlyArray, }; } + if (input.unlessInterruptRequested !== undefined) { + const { requestId, resultId } = input.unlessInterruptRequested; + const pending = yield* sql<{ readonly pending: number }>` + SELECT EXISTS ( + SELECT 1 FROM orchestration_v2_projection_turn_items + WHERE thread_id = ${input.threadId} AND turn_item_id = ${requestId} + ) AND NOT EXISTS ( + SELECT 1 FROM orchestration_v2_projection_turn_items + WHERE thread_id = ${input.threadId} AND turn_item_id = ${resultId} + ) AS pending + `; + if (pending[0]?.pending === 1) { + return { + committed: false as const, + storedEvents: [] as ReadonlyArray, + }; + } + } const normalized = yield* normalizeEvents( input.guardPendingUserInputCancellations === true diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index acf3cc27f621..796c10b41194 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -663,8 +663,8 @@ it.effect.each([false, true])("groups shutdown commits with rollback=%s", (fail) EffectWorker.OrchestrationEffectWorkerV2.of({ awaitWork: Effect.void, runRecoveryOnce: Effect.succeed(false), - runOnce: Effect.succeed(false), - nextClaimableAt: Effect.succeedNone, + runOnce: () => Effect.succeed(false), + nextClaimableAt: () => Effect.succeedNone, drain: () => Effect.succeed(0), }), ), @@ -1839,8 +1839,8 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { ); yield* Effect.gen(function* () { const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; - assert.isTrue(yield* worker.runOnce); - assert.isFalse(yield* worker.runOnce); + assert.isTrue(yield* worker.runOnce()); + assert.isFalse(yield* worker.runOnce()); }).pipe(Effect.provide(layerWorker)); assert.equal(yield* Ref.get(executionCount), 1); @@ -1984,6 +1984,107 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { }).pipe(Effect.provide(Layer.fresh(layerTest))), ); + it.effect("refuses a run-current write once an unpaired interrupt request is stored", () => + Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread:foundation-interrupt-guard"); + const runId = RunId.make("run:foundation-interrupt-guard"); + const attemptId = RunAttemptId.make("run-attempt:foundation-interrupt-guard"); + const requestId = TurnItemId.make("turn-item:foundation-interrupt-guard:request"); + const resultId = TurnItemId.make("turn-item:foundation-interrupt-guard:result"); + const run: OrchestrationV2Run = { + id: runId, + threadId, + ordinal: 1, + providerInstanceId, + modelSelection, + providerThreadId: null, + userMessageId: MessageId.make("message:foundation-interrupt-guard"), + rootNodeId: null, + activeAttemptId: attemptId, + status: "running", + queuePosition: null, + requestedAt: now, + startedAt: now, + completedAt: null, + checkpointId: null, + contextHandoffId: null, + }; + yield* eventSink.write({ + events: [ + threadCreatedEvent({ + id: "event:foundation-interrupt-guard:thread", + thread: makeThread(threadId, now), + now, + }), + { + id: EventId.make("event:foundation-interrupt-guard:run"), + type: "run.created", + threadId, + runId, + providerInstanceId, + occurredAt: now, + payload: run, + }, + ], + }); + const signalItem = (id: TurnItemId, type: "run_interrupt_request" | "run_interrupt_result") => + ({ + id: EventId.make(`event:${id}`), + type: "turn-item.updated", + threadId, + runId, + occurredAt: now, + payload: { + id, + threadId, + runId, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 1, + status: "completed", + title: "Interrupt", + startedAt: now, + completedAt: now, + updatedAt: now, + type, + message: "Stopped", + }, + }) as OrchestrationV2DomainEvent; + const completeRun = (suffix: string) => + eventSink.writeIfRunCurrent({ + threadId, + runId, + activeAttemptId: attemptId, + expectedStatus: "running", + unlessInterruptRequested: { requestId, resultId }, + events: [ + { + id: EventId.make(`event:foundation-interrupt-guard:waiting:${suffix}`), + type: "run.updated", + threadId, + runId, + providerInstanceId, + occurredAt: now, + payload: { ...run, status: "waiting" }, + }, + ], + }); + + yield* eventSink.write({ events: [signalItem(requestId, "run_interrupt_request")] }); + const refused = yield* completeRun("refused"); + assert.isFalse(refused.committed); + assert.deepEqual(refused.storedEvents, []); + + yield* eventSink.write({ events: [signalItem(resultId, "run_interrupt_result")] }); + assert.isTrue((yield* completeRun("paired")).committed); + }).pipe(Effect.provide(Layer.fresh(layerTest))), + ); + it.effect("guards post-terminal provider-thread writes by attempt and run ordinal", () => Effect.gen(function* () { const eventSink = yield* EventSink.EventSinkV2; @@ -2249,7 +2350,7 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { yield* Effect.gen(function* () { const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; - const workerFiber = yield* worker.runOnce.pipe(Effect.forkChild); + const workerFiber = yield* worker.runOnce().pipe(Effect.forkChild); yield* Deferred.await(started); const cancelledEffectIds = yield* outbox.cancelUnsettled({ threadId, @@ -2372,7 +2473,7 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { assert.isTrue( yield* EffectWorker.OrchestrationEffectWorkerV2.pipe( - Effect.flatMap((worker) => worker.runOnce), + Effect.flatMap((worker) => worker.runOnce()), Effect.provide(layerWorker), ), ); @@ -2578,7 +2679,7 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { if (Option.isNone(claimed)) return; assert.equal(claimed.value.id, "effect:foundation-next-claimable:a1"); - const whileBlocked = yield* outbox.nextClaimableAt; + const whileBlocked = yield* outbox.nextClaimableAt(); assert.isTrue(Option.isSome(whileBlocked)); if (Option.isSome(whileBlocked)) { assert.equal(DateTime.toEpochMillis(whileBlocked.value), DateTime.toEpochMillis(future)); @@ -2588,7 +2689,7 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { effectId: claimed.value.id, workerId: "next-claimable-worker", }); - const afterCompletion = yield* outbox.nextClaimableAt; + const afterCompletion = yield* outbox.nextClaimableAt(); assert.isTrue(Option.isSome(afterCompletion)); if (Option.isSome(afterCompletion)) { assert.isAtMost(DateTime.toEpochMillis(afterCompletion.value), DateTime.toEpochMillis(now)); @@ -2765,7 +2866,7 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { assert.equal(Option.getOrUndefined(title)?.id, "effect:foundation-retry-order:b-title"); const blocked = yield* outbox.claimNext({ workerId }); assert.isTrue(Option.isNone(blocked)); - const nextClaimable = yield* outbox.nextClaimableAt; + const nextClaimable = yield* outbox.nextClaimableAt(); assert.isTrue(Option.isSome(nextClaimable)); if (Option.isSome(nextClaimable)) { assert.equal( @@ -2897,6 +2998,108 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { }), ); + it.effect("starts a provider turn while slow title generations are running", () => + Effect.gen(function* () { + const outbox = yield* EffectOutbox.EffectOutboxV2; + const commandId = CommandId.make("command:foundation-title-starvation"); + const titleThreads = ["a", "b", "c"].map((suffix) => + ThreadId.make(`thread:foundation-title-starvation:${suffix}`), + ); + const turnStarted = yield* Deferred.make(); + const releaseTitles = yield* Deferred.make(); + const layerWorker = EffectWorker.layerWithOptions({ + workerId: "title-starvation-worker", + }).pipe( + Layer.provide( + Layer.merge( + Layer.succeed(EffectOutbox.EffectOutboxV2, outbox), + Layer.succeed( + EffectWorker.OrchestrationEffectExecutorV2, + EffectWorker.OrchestrationEffectExecutorV2.of({ + execute: (effect) => + effect.request.type === "thread-title.generate" + ? Deferred.await(releaseTitles) + : Deferred.succeed(turnStarted, undefined).pipe(Effect.asVoid), + }), + ), + ), + ), + ); + + yield* Effect.gen(function* () { + yield* EffectWorker.runDaemonWithOptions({ concurrency: 2 }).pipe(Effect.forkScoped); + yield* Effect.yieldNow; + yield* outbox.enqueue([ + ...titleThreads.map((threadId, index) => ({ + id: `effect:foundation-title-starvation:${index}-title`, + commandId, + threadId, + request: { + type: "thread-title.generate" as const, + kind: { type: "regenerate" as const }, + }, + })), + { + id: "effect:foundation-title-starvation:turn-start", + commandId, + threadId: titleThreads[0]!, + request: { + type: "provider-turn.start", + runId: RunId.make("run:foundation-title-starvation"), + }, + }, + ]); + yield* outbox.notifyAvailable(titleThreads.length + 1); + + const started = yield* Deferred.await(turnStarted).pipe( + Effect.timeoutOption(Duration.seconds(1)), + Effect.forkScoped, + ); + yield* TestClock.adjust(Duration.seconds(1)); + assert.isTrue( + Option.isSome(yield* Fiber.join(started)), + "provider-turn.start was never claimed: thread title generations occupied every effect worker", + ); + assert.isFalse(yield* Deferred.isDone(releaseTitles)); + + yield* Deferred.succeed(releaseTitles, undefined); + yield* Effect.forEach(titleThreads, (_, index) => + outbox.awaitSettled(`effect:foundation-title-starvation:${index}-title`), + ); + }).pipe(Effect.provide(layerWorker), Effect.scoped); + }), + ); + + it.effect("holds pending titles at the running-title cap without waking idle workers", () => + Effect.gen(function* () { + const outbox = yield* EffectOutbox.EffectOutboxV2; + const cap = { maxRunningTitleGenerations: 1 }; + const title = (name: string) => ({ + id: `effect:foundation-title-cap:${name}`, + commandId: CommandId.make(`command:foundation-title-cap:${name}`), + threadId: ThreadId.make(`thread:foundation-title-cap:${name}`), + request: { type: "thread-title.generate" as const, kind: { type: "regenerate" as const } }, + }); + yield* outbox.enqueue([title("first"), title("second")]); + const first = yield* outbox.claimNext({ workerId: "title-cap-worker", ...cap }); + assert.strictEqual(Option.getOrUndefined(first)?.id, title("first").id); + + assert.isTrue( + Option.isNone(yield* outbox.claimNext({ workerId: "title-cap-worker", ...cap })), + ); + assert.isTrue( + Option.isNone(yield* outbox.nextClaimableAt(cap)), + "a capped title must not report a past deadline, or idle workers re-poll every 25 ms", + ); + assert.isTrue(Option.isSome(yield* outbox.nextClaimableAt())); + + yield* outbox.succeed({ effectId: title("first").id, workerId: "title-cap-worker" }); + const second = yield* outbox.claimNext({ workerId: "title-cap-worker", ...cap }); + assert.strictEqual(Option.getOrUndefined(second)?.id, title("second").id); + yield* outbox.succeed({ effectId: title("second").id, workerId: "title-cap-worker" }); + }), + ); + it.effect("does not reclaim a running effect after its process-local lease expires", () => Effect.gen(function* () { const outbox = yield* EffectOutbox.EffectOutboxV2; @@ -2948,7 +3151,7 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { yield* Effect.gen(function* () { const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; - const firstFiber = yield* worker.runOnce.pipe(Effect.forkChild); + const firstFiber = yield* worker.runOnce().pipe(Effect.forkChild); yield* Deferred.await(firstStarted); yield* sql` UPDATE orchestration_v2_effect_outbox @@ -2956,12 +3159,12 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { WHERE effect_id = ${firstEffectId} `; - assert.isFalse(yield* worker.runOnce); + assert.isFalse(yield* worker.runOnce()); assert.deepEqual(yield* Ref.get(executions), [firstEffectId]); yield* Deferred.succeed(releaseFirst, undefined); assert.isTrue(yield* Fiber.join(firstFiber)); - assert.isTrue(yield* worker.runOnce); + assert.isTrue(yield* worker.runOnce()); assert.deepEqual(yield* Ref.get(executions), [firstEffectId, secondEffectId]); }).pipe(Effect.provide(layerWorker)); }), @@ -3290,8 +3493,8 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { EffectWorker.OrchestrationEffectWorkerV2.of({ awaitWork: Effect.void, runRecoveryOnce: Effect.succeed(false), - runOnce: Effect.succeed(false), - nextClaimableAt: Effect.succeedNone, + runOnce: () => Effect.succeed(false), + nextClaimableAt: () => Effect.succeedNone, drain: () => Effect.succeed(0), }), ), @@ -3507,8 +3710,8 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => { EffectWorker.OrchestrationEffectWorkerV2.of({ awaitWork: Effect.void, runRecoveryOnce: Effect.succeed(false), - runOnce: Effect.succeed(false), - nextClaimableAt: Effect.succeedNone, + runOnce: () => Effect.succeed(false), + nextClaimableAt: () => Effect.succeedNone, drain: () => Effect.succeed(0), }), ), diff --git a/apps/server/src/orchestration-v2/NotificationMailbox.ts b/apps/server/src/orchestration-v2/NotificationMailbox.ts index 29ba40c5777f..79458677c1af 100644 --- a/apps/server/src/orchestration-v2/NotificationMailbox.ts +++ b/apps/server/src/orchestration-v2/NotificationMailbox.ts @@ -6,7 +6,10 @@ import { } from "@t3tools/contracts"; export function isBufferedProviderNotification( - message: OrchestrationV2ConversationMessage, + message: Pick< + OrchestrationV2ConversationMessage, + "createdBy" | "creationSource" | "notification" | "delegatedCompletion" + >, ): boolean { return ( message.createdBy === "agent" && diff --git a/apps/server/src/orchestration-v2/Orchestrator.control-reads.test.ts b/apps/server/src/orchestration-v2/Orchestrator.control-reads.test.ts index 7ccf402c229e..f0a933040b7b 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.control-reads.test.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.control-reads.test.ts @@ -19,7 +19,9 @@ import { TurnItemId, } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Result from "effect/Result"; import * as Stream from "effect/Stream"; @@ -57,6 +59,194 @@ const layerTest = Layer.mergeAll( ), ); +it.effect.each( + (["provider.switch", "thread.model-selection.set"] as const).flatMap((type) => + [ + { active: true, selectionChange: "instance" }, + { active: false, selectionChange: "instance" }, + { active: true, selectionChange: "next-turn" }, + { active: true, selectionChange: "restart" }, + ].map((scenario) => ({ type, ...scenario })), + ), +)("$type $selectionChange ($active)", ({ type, active, selectionChange }) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const threadId = ThreadId.make("thread:instance-switch"); + const now = yield* DateTime.now; + const sessionId = ProviderSessionId.make("session:instance-switch"); + const providerThreadId = ProviderThreadId.make("provider-thread:instance-switch"); + const runId = RunId.make("run:instance-switch"); + const nodeId = NodeId.make("node:instance-switch"); + const attemptId = RunAttemptId.make("attempt:instance-switch"); + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("create-instance-switch"), + threadId, + projectId: ProjectId.make("project:instance-switch"), + title: "Instance switch", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + createdBy: "user", + creationSource: "web", + }); + yield* projections.apply({ + id: EventId.make("instance-switch:session"), + type: "provider-session.attached", + threadId, + occurredAt: now, + payload: { + id: sessionId, + driver: adapter.driver, + providerInstanceId: instanceId, + status: "ready", + cwd: "/repo", + model: modelSelection.model, + capabilities: CodexProviderCapabilitiesV2, + createdAt: now, + updatedAt: now, + lastError: null, + }, + }); + yield* projections.apply({ + id: EventId.make("instance-switch:provider-thread"), + type: "provider-thread.updated", + threadId, + occurredAt: now, + payload: { + id: providerThreadId, + driver: adapter.driver, + providerInstanceId: instanceId, + providerSessionId: sessionId, + appThreadId: threadId, + ownerNodeId: null, + nativeThreadRef: { + driver: adapter.driver, + nativeId: "native:instance-switch", + strength: "strong", + }, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: 1, + lastRunOrdinal: 1, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }, + }); + if (active) { + yield* projections.apply({ + id: EventId.make("instance-switch:run"), + type: "run.created", + threadId, + occurredAt: now, + payload: { + id: runId, + threadId, + ordinal: 1, + providerInstanceId: instanceId, + modelSelection, + providerThreadId, + userMessageId: MessageId.make("message:instance-switch"), + rootNodeId: nodeId, + activeAttemptId: attemptId, + status: "running", + requestedAt: now, + startedAt: now, + completedAt: null, + checkpointId: null, + contextHandoffId: null, + }, + }); + yield* projections.apply({ + id: EventId.make("instance-switch:turn"), + type: "provider-turn.updated", + threadId, + runId, + occurredAt: now, + payload: { + id: ProviderTurnId.make("provider-turn:instance-switch"), + providerThreadId, + nodeId, + runAttemptId: attemptId, + nativeTurnRef: { driver: adapter.driver, nativeId: "native:turn", strength: "strong" }, + ordinal: 1, + status: "running", + startedAt: now, + completedAt: null, + }, + }); + } + const before = yield* projections.getThreadProjection(threadId); + const sequence = yield* orchestrator.getThreadEventSequence(threadId); + const commandId = CommandId.make("switch-instance"); + const target = { + instanceId: + selectionChange === "instance" ? ProviderInstanceId.make("codex-other") : instanceId, + model: selectionChange === "instance" ? modelSelection.model : "gpt-6", + }; + const outcome = yield* orchestrator + .dispatch({ + type, + commandId, + threadId, + modelSelection: target, + }) + .pipe(Effect.result); + const after = yield* projections.getThreadProjection(threadId); + if (active && selectionChange !== "next-turn") { + assert.deepEqual( + after, + before, + "a refused switch must preserve the turn, session and selection", + ); + assert.deepEqual( + yield* outbox.listByCommandId(commandId), + [], + "a refused switch must not release the provider session", + ); + assert.equal(yield* orchestrator.getThreadEventSequence(threadId), sequence); + assert.ok(Result.isFailure(outcome)); + assert.instanceOf(outcome.failure, Orchestrator.OrchestratorProviderSelectionBusyError); + } else { + assert.ok(Result.isSuccess(outcome)); + assert.deepEqual(after.thread.modelSelection, target); + assert.equal(after.thread.providerInstanceId, target.instanceId); + assert.deepEqual(after.runs, before.runs); + assert.deepEqual(after.providerTurns, before.providerTurns); + } + }).pipe( + Effect.provide( + Layer.mergeAll( + layerDatabase, + ProjectionStore.layer.pipe(Layer.provide(layerDatabase)), + EffectOutbox.layer.pipe(Layer.provide(layerDatabase)), + ProviderReplayHarness.layerWithRegistry( + { name: "instance-switch" }, + ProviderAdapterRegistry.layerFromAdapters([ + { + ...adapter, + planSelectionTransition: () => + Effect.succeed( + selectionChange === "restart" + ? { type: "restart_session" as const } + : { type: "apply_on_next_turn" as const }, + ), + }, + { ...adapter, instanceId: ProviderInstanceId.make("codex-other") }, + ]), + { databaseLayer: layerDatabase, runEffectWorker: false }, + ), + ), + ), + ), +); + it.effect( "records approval feedback and targets only its runtime request without dispatching a parent turn", () => @@ -1381,6 +1571,234 @@ it.effect("wakes the parent once when its delegated task asks a question", () => }).pipe(Effect.provide(DelegatedTaskQuestionReactor.layer.pipe(Layer.provideMerge(layerTest)))), ); +it.effect.each([false, true])( + "drops a stale question wake (missed terminal event: %s)", + (missedEvent) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const reactor = yield* DelegatedTaskQuestionReactor.DelegatedTaskQuestionReactor; + const eventSink = yield* EventSink.EventSinkV2; + const parentThreadId = ThreadId.make("thread:stale-question-parent"); + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("create-stale-question-parent"), + threadId: parentThreadId, + projectId: ProjectId.make("project:stale-question"), + title: "Parent", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + createdBy: "user", + creationSource: "web", + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("start-stale-question-parent"), + threadId: parentThreadId, + messageId: MessageId.make("message:stale-question-parent"), + text: "Delegate the proof", + attachments: [], + dispatchMode: { type: "start_immediately" }, + createdBy: "user", + creationSource: "web", + }); + const parentRun = (yield* projections.getThreadProjection(parentThreadId)).runs[0]!; + yield* orchestrator.dispatch({ + type: "delegated_task.request", + commandId: CommandId.make("delegate-stale-question-task"), + parentThreadId, + parentRunId: parentRun.id, + parentNodeId: parentRun.rootNodeId!, + task: "Capture the proof", + completionWake: "settled_only", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + createdBy: "agent", + creationSource: "mcp", + }); + const task = (yield* projections.getThreadProjection(parentThreadId)).subagents[0]!; + const childRun = (yield* projections.getThreadRecords(task.childThreadId!, ["runs"])) + .runs[0]!; + const now = yield* DateTime.now; + const question = { + id: EventId.make("event:stale-question"), + type: "runtime-request.updated", + threadId: task.childThreadId!, + runId: childRun.id, + occurredAt: now, + payload: { + id: RuntimeRequestId.make("request:stale-question"), + nodeId: childRun.rootNodeId!, + providerTurnId: null, + nativeRequestRef: null, + kind: "user_input", + status: "pending", + responseCapability: { type: "message" }, + createdAt: now, + resolvedAt: null, + }, + } satisfies OrchestrationV2DomainEvent; + yield* eventSink.write({ events: [question] }); + yield* reactor.handle(question); + const wake = (yield* projections.getThreadRecords(parentThreadId, [ + "messages", + ])).messages.find((message) => message.notification !== undefined)!; + const wakeRun = () => + projections + .getThreadRecords(parentThreadId, ["runs"]) + .pipe(Effect.map(({ runs }) => runs.find((run) => run.userMessageId === wake.id))); + assert.equal((yield* wakeRun())?.status, "queued"); + + // The child finishes without ever resolving its message-capability question. + const childCompleted = { + id: EventId.make("event:stale-question-child-completed"), + type: "run.updated", + threadId: task.childThreadId!, + runId: childRun.id, + occurredAt: now, + payload: { ...childRun, status: "completed", completedAt: now }, + } satisfies OrchestrationV2DomainEvent; + const taskCompleted = { + id: EventId.make("event:stale-question-task-completed"), + type: "subagent.updated", + threadId: parentThreadId, + occurredAt: now, + payload: { ...task, status: "completed", result: "Proof captured.", completedAt: now }, + } satisfies OrchestrationV2DomainEvent; + yield* eventSink.write({ + commandId: CommandId.make("command:runtime-reconcile:stale-question"), + events: [childCompleted, taskCompleted], + }); + yield* reactor.handle(childCompleted); + if (missedEvent) { + // An acknowledged result drops the task from the parent's command projection. + yield* orchestrator.dispatch({ + type: "delegated_task.completion-delivery.acknowledge", + commandId: CommandId.make("ack-stale-question-task"), + parentThreadId, + taskId: task.id, + observedByRunId: parentRun.id, + }); + // Persisted terminal state can predate the reactor subscription after restart. + yield* projections.apply({ + id: EventId.make("event:stale-question-parent-finished"), + type: "run.updated", + threadId: parentThreadId, + occurredAt: now, + payload: { ...parentRun, status: "completed", completedAt: now }, + }); + yield* orchestrator.dispatch({ + type: "queue.resume", + commandId: CommandId.make("resume-stale-question"), + threadId: parentThreadId, + }); + } else { + yield* reactor.handle(taskCompleted); + } + assert.equal((yield* wakeRun())?.status, "cancelled"); + if (missedEvent) { + const before = (yield* projections.getThreadRecords(parentThreadId, ["runs"])).runs.length; + const delayedQuestion = { + ...question, + id: EventId.make("event:delayed-question"), + payload: { ...question.payload, id: RuntimeRequestId.make("request:delayed-question") }, + }; + yield* eventSink.write({ events: [delayedQuestion] }); + yield* reactor.handle(delayedQuestion); + // A stale delivery is accepted without starting work, not rejected. + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("stale-question-redelivery"), + threadId: parentThreadId, + messageId: MessageId.make("message:stale-question-redelivery"), + text: "Answer the delegated question", + notification: { + source: { + kind: "delegated_task", + taskIds: [task.id], + childThreadId: task.childThreadId!, + }, + outcome: "updated", + summary: "Delegated task has a question", + }, + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "agent", + creationSource: "server", + }); + assert.equal( + (yield* projections.getThreadRecords(parentThreadId, ["runs"])).runs.length, + before, + ); + return; + } + + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("stale-question-followup"), + threadId: task.childThreadId!, + messageId: MessageId.make("message:stale-question-followup"), + text: "Capture another proof", + attachments: [], + dispatchMode: { type: "start_immediately" }, + createdBy: "user", + creationSource: "web", + }); + const followup = (yield* projections.getThreadRecords(task.childThreadId!, [ + "runs", + ])).runs.find((run) => run.userMessageId === "message:stale-question-followup")!; + const followupQuestion = { + ...question, + id: EventId.make("event:stale-question-followup"), + runId: followup.id, + payload: { + ...question.payload, + id: RuntimeRequestId.make("request:stale-question-followup"), + nodeId: followup.rootNodeId!, + }, + }; + yield* eventSink.write({ events: [followupQuestion] }); + yield* reactor.handle(followupQuestion); + const followupWake = (yield* projections.getThreadRecords(parentThreadId, [ + "runs", + ])).runs.find( + (run) => + run.userMessageId === + `message:delegated-task-question:${parentThreadId}:${followupQuestion.payload.id}`, + )!; + assert.equal(followupWake.status, "queued"); + yield* reactor.handle(taskCompleted); + assert.equal( + (yield* projections.getThreadRecords(parentThreadId, ["runs"])).runs.find( + (run) => run.id === followupWake.id, + )?.status, + "queued", + ); + yield* projections.apply({ + id: EventId.make("event:followup-question-parent-finished"), + type: "run.updated", + threadId: parentThreadId, + occurredAt: now, + payload: { ...parentRun, status: "completed", completedAt: now }, + }); + yield* orchestrator.dispatch({ + type: "queue.resume", + commandId: CommandId.make("resume-followup-question"), + threadId: parentThreadId, + }); + assert.equal( + (yield* projections.getThreadRecords(parentThreadId, ["runs"])).runs.find( + (run) => run.id === followupWake.id, + )?.status, + "starting", + ); + }).pipe(Effect.provide(DelegatedTaskQuestionReactor.layer.pipe(Layer.provideMerge(layerTest)))), +); + it.effect.each([false, true])("rejects thread.create for an existing id (deleted: %s)", (deleted) => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; @@ -1628,3 +2046,76 @@ it.effect("keeps a snooze when a live question is cancelled or an approval is an } }).pipe(Effect.provide(layerTest)), ); + +it.effect("a user message dispatched first stays run 1 when its lineage read is slow", () => { + const threadId = ThreadId.make("thread:dispatch-order"); + // A file database serves projection reads from a reader worker, so the lineage read a user + // dispatch makes before taking the thread lock can yield. This gate holds it open instead. + const lineageReadEntered = Deferred.makeUnsafe(); + const lineageReadRelease = Deferred.makeUnsafe(); + let gateArmed = false; + const layerGatedLineageRead = ProviderReplayHarness.layerWithRegistry( + { name: "dispatch-order" }, + ProviderAdapterRegistry.layerFromAdapters([adapter]), + { + runEffectWorker: false, + wrapProjectionStore: (store) => ({ + ...store, + getThread: (id) => { + if (!gateArmed || id !== threadId) return store.getThread(id); + gateArmed = false; + return Deferred.succeed(lineageReadEntered, undefined).pipe( + Effect.andThen(Deferred.await(lineageReadRelease)), + Effect.andThen(store.getThread(id)), + ); + }, + }), + }, + ); + return Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("create-dispatch-order"), + threadId, + projectId: ProjectId.make("project:dispatch-order"), + title: "Dispatch order", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + createdBy: "user", + creationSource: "web", + }); + const message = (id: string, createdBy: "user" | "agent") => + orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(id), + threadId, + messageId: MessageId.make(id), + text: id, + attachments: [], + dispatchMode: { type: "defer_start" }, + createdBy, + creationSource: createdBy === "user" ? "web" : "server", + }); + + gateArmed = true; + const user = yield* message("message:user-first", "user").pipe(Effect.forkChild); + yield* Deferred.await(lineageReadEntered); + // Agent messages skip the lineage read, so this one would take a free thread lock first. + const agent = yield* message("message:agent-second", "agent").pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.succeed(lineageReadRelease, undefined); + yield* Fiber.join(agent); + yield* Fiber.join(user); + + const { runs } = yield* orchestrator.getThreadProjection(threadId); + assert.deepEqual( + runs.toSorted((left, right) => left.ordinal - right.ordinal).map((run) => run.userMessageId), + [MessageId.make("message:user-first"), MessageId.make("message:agent-second")], + ); + }).pipe(Effect.provide(layerGatedLineageRead)); +}); diff --git a/apps/server/src/orchestration-v2/Orchestrator.migration.test.ts b/apps/server/src/orchestration-v2/Orchestrator.migration.test.ts index 911c1b8de7df..7f2a3cb3a753 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.migration.test.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.migration.test.ts @@ -10,6 +10,14 @@ it("reissues imported context until a V2 run completes", () => { legacyImportItemCount: 2, }), ); + assert.isFalse( + shouldPrepareLegacyImportHandoff({ + historyOrigin: "v1_import", + hasCompletedRun: false, + legacyImportItemCount: 2, + hasNativeHistory: true, + }), + ); assert.isFalse( shouldPrepareLegacyImportHandoff({ historyOrigin: "v1_import", diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 661ec9130acf..8c564fe6d43e 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -75,6 +75,7 @@ import { TurnItemId, TrimmedNonEmptyString, } from "@t3tools/contracts"; +import * as KeyedLock from "@t3tools/shared/KeyedLock"; import { modelSelectionsEqual } from "@t3tools/shared/model"; import { modelSwitchSince } from "./modelSwitch.ts"; import { recordThreadOpen } from "@t3tools/shared/paletteSearch"; @@ -104,6 +105,7 @@ import { import { CheckpointServiceV2 } from "./CheckpointService.ts"; import { CommandPolicyV2, + followUpOwnerRun, goalCommandStopsActiveRun, isActiveRun, isNativeMaintenanceCommand, @@ -122,6 +124,7 @@ import { cancelledBeforeStart, continuationFailedBeforePrompt, continuationStartRetryDelayMs, + delegatedTaskResultRun, isRestartContinuationSource, runOutcome, } from "./RestartBackgroundNote.ts"; @@ -281,6 +284,15 @@ export class OrchestratorThreadAboveModeLimitError extends Schema.TaggedError()( + "OrchestratorProviderSelectionBusyError", + { commandId: CommandId, threadId: ThreadId }, +) { + override get message(): string { + return `Thread ${this.threadId} has an active turn. Wait for it to finish, then retry the provider selection change.`; + } +} + export class OrchestratorCommandPreviouslyRejectedError extends Schema.TaggedError()( "OrchestratorCommandPreviouslyRejectedError", { @@ -319,6 +331,7 @@ export const OrchestratorV2Error = Schema.Union([ OrchestratorCommandIdConflictError, OrchestratorSubagentThreadReadOnlyError, OrchestratorThreadAboveModeLimitError, + OrchestratorProviderSelectionBusyError, ]); export type OrchestratorV2Error = typeof OrchestratorV2Error.Type; @@ -333,6 +346,15 @@ export interface OrchestratorV2Shape { readonly resumeQueuedRuns: Effect.Effect; readonly recoverDelegatedTasks: Effect.Effect; readonly recoverDelegatedTask: (threadId: ThreadId, sourceRunId: RunId) => Effect.Effect; + /** Commits cancellation with the child's Stop after descendants accept stopping. */ + readonly cancelDelegatedTask: (input: { + readonly parentThreadId: ThreadId; + readonly taskId: NodeId; + readonly command: Extract; + readonly stopDescendants: ( + dispatch: OrchestratorV2Shape["dispatch"], + ) => Effect.Effect; + }) => Effect.Effect; readonly delegatedTaskResultPending: ( childThreadId: ThreadId, taskId?: NodeId, @@ -471,23 +493,41 @@ function nextRunOrdinal(projection: Pick Math.max(max, run.ordinal), 0) + 1; } -/** Stamp at wake start; queued prompts lack start times and delegated results jump the queue. */ +/** + * Stamp at wake start; queued prompts lack start times and delegated results jump the queue. + * Auto-drained agent messages continue active work; explicit queue starts reset it. + */ function wakeWorkStartedAt( - runs: ReadonlyArray, - trigger: { + projection: Pick, + trigger: Pick< + OrchestrationV2ConversationMessage, + "createdBy" | "creationSource" | "senderThreadId" + > & { readonly notification?: unknown; readonly delegatedCompletion?: unknown; readonly restartContinuationOfRunId?: RunId | undefined; }, + delivery: "immediate" | "automatic-queue" | "explicit-queue" = "immediate", ): Pick { if ( trigger.notification === undefined && trigger.delegatedCompletion === undefined && - trigger.restartContinuationOfRunId === undefined + trigger.restartContinuationOfRunId === undefined && + !( + trigger.createdBy === "agent" && + trigger.creationSource === "mcp" && + trigger.senderThreadId !== undefined && + (delivery === "automatic-queue" || + (delivery === "immediate" && + projection.subagents.some( + (subagent) => + subagent.origin === "app_owned" && subagent.childThreadId === trigger.senderThreadId, + ))) + ) ) { return {}; } - const previous = runs + const previous = projection.runs .flatMap((run) => (run.startedAt === null ? [] : [{ run, startedAt: run.startedAt }])) .toSorted( (left, right) => @@ -796,7 +836,12 @@ function nextQueuedRun( } function followsCurrentConversation( - message: OrchestrationV2ConversationMessage | undefined, + message: + | Pick< + OrchestrationV2ConversationMessage, + "createdBy" | "creationSource" | "notification" | "delegatedCompletion" + > + | undefined, ): boolean { return ( message?.delegatedCompletion !== undefined || @@ -814,9 +859,29 @@ function queuedRunModelSelection( : run.modelSelection; } +/** The selection the thread will be on once every queued run has started. */ +function queueTailModelSelection( + projection: Pick, +): ModelSelection { + return ( + queuedRunsInDeliveryOrder(projection).findLast( + (run) => + !followsCurrentConversation( + projection.messages.find((message) => message.id === run.userMessageId), + ), + )?.modelSelection ?? projection.thread.modelSelection + ); +} + +/** + * The provider's latest run stopped on its usage limit and nothing ran after it. + * With `now` (automatic starts), a run on another provider since does not lift + * a reported reset still in the future; an unknown reset is lifted, as transient 429s are. + */ function usageLimitHoldsQueue( projection: Pick, providerInstanceId: ModelSelection["instanceId"], + now?: DateTime.Utc, ): boolean { let latestSession: OrchestrationV2ThreadProjection["providerSessions"][number] | undefined; for (const session of projection.providerSessions) { @@ -828,12 +893,118 @@ function usageLimitHoldsQueue( latestSession = session; } } + const limited = usageLimitBlockedRun( + projection.runs.filter((run) => run.providerInstanceId === providerInstanceId), + projection.turnItems, + latestSession?.lastError ?? null, + ); + if (limited === null) return false; + if (latestExecutedRun(projection.runs)?.id === limited.id) return true; + if (now === undefined) return false; + const resetAt = latestRootProviderFailure(limited, projection.turnItems)?.resetAt; + return resetAt != null && Date.parse(resetAt) > DateTime.toEpochMillis(now); +} + +function pullRequestSnapshotEndsWatch( + snapshot: ThreadPullRequestLink["snapshot"], + watchStartedAt: ThreadPullRequestWatch["startedAt"], +): boolean { + return ( + snapshot?.state === "merged" || + // A restarted watch can see a closed snapshot from before the PR reopened. + (snapshot?.state === "closed" && + DateTime.isGreaterThanOrEqualTo( + DateTime.makeUnsafe(snapshot.syncedAt), + DateTime.makeUnsafe(watchStartedAt), + )) + ); +} + +function isStalePullRequestWatchWake( + projection: Pick, + run: OrchestrationV2Run, +): boolean { + const source = projection.messages.find((message) => message.id === run.userMessageId) + ?.notification?.source; + if (source?.kind !== "monitor" || source.pullRequest === undefined) return false; + const stamp = source.pullRequest; + const link = threadPullRequestsOf(projection.thread).find( + (candidate) => + candidate.source !== "stack-dismissed" && threadPullRequestKeysEqual(candidate, stamp), + ); + if (link === undefined) return true; + if (stamp.watchEnded === true) { + // The final notice outlives its watch, but a replacement watch supersedes it. + const latestWatch = link.watch ?? link.endedWatch; + return latestWatch?.startedAt !== stamp.watchStartedAt; + } + if (link.watch === undefined) { + // A watch the server ended with a final notice still owes the wakes it queued before it. + const endedWithNotice = projection.messages.some((message) => { + const ending = message.notification?.source; + return ( + ending?.kind === "monitor" && + ending.pullRequest?.watchEnded === true && + ending.pullRequest.watchStartedAt === stamp.watchStartedAt && + threadPullRequestKeysEqual(ending.pullRequest, stamp) + ); + }); + return ( + !endedWithNotice || + link.endedWatch?.startedAt !== stamp.watchStartedAt || + pullRequestSnapshotEndsWatch(link.snapshot, stamp.watchStartedAt) + ); + } return ( - usageLimitBlockedRun(projection.runs, projection.turnItems, latestSession?.lastError ?? null) - ?.providerInstanceId === providerInstanceId + link.watch.startedAt !== stamp.watchStartedAt || + (stamp.headSha !== null && link.watch.headSha !== stamp.headSha) || + pullRequestSnapshotEndsWatch(link.snapshot, link.watch.startedAt) ); } +function queuedRunCancellationEvents( + projection: Pick, + run: OrchestrationV2Run, + now: DateTime.Utc, +): ReadonlyArray> { + const rootNode = projection.nodes.find((node) => node.id === run.rootNodeId); + const attempt = projection.attempts.find((candidate) => candidate.id === run.activeAttemptId); + const events: Array> = [ + { + type: "run.updated", + threadId: run.threadId, + runId: run.id, + ...(run.rootNodeId === null ? {} : { nodeId: run.rootNodeId }), + providerInstanceId: run.providerInstanceId, + occurredAt: now, + payload: { ...run, status: "cancelled", queuePosition: null, completedAt: now }, + }, + ]; + if (attempt !== undefined && rootNode !== undefined) { + events.push({ + type: "run-attempt.updated", + threadId: run.threadId, + runId: run.id, + nodeId: rootNode.id, + providerInstanceId: run.providerInstanceId, + occurredAt: now, + payload: { ...attempt, status: "cancelled", completedAt: now }, + }); + } + if (rootNode !== undefined) { + events.push({ + type: "node.updated", + threadId: run.threadId, + runId: run.id, + nodeId: rootNode.id, + providerInstanceId: run.providerInstanceId, + occurredAt: now, + payload: { ...rootNode, status: "cancelled", completedAt: now }, + }); + } + return events; +} + function latestStableRun( projection: Pick, ): OrchestrationV2Run | null { @@ -951,12 +1122,16 @@ function latestContextTransfer( } export function shouldPrepareLegacyImportHandoff(input: { + readonly hasNativeHistory?: boolean; readonly hasCompletedRun: boolean; readonly historyOrigin: OrchestrationV2AppThread["historyOrigin"]; readonly legacyImportItemCount: number; }): boolean { return ( - input.historyOrigin === "v1_import" && !input.hasCompletedRun && input.legacyImportItemCount > 0 + input.historyOrigin === "v1_import" && + !input.hasNativeHistory && + !input.hasCompletedRun && + input.legacyImportItemCount > 0 ); } @@ -1045,7 +1220,13 @@ function rootProviderThreadsForProvider( } function isHandoffSourceRun(run: OrchestrationV2Run): boolean { - return run.status === "completed" || run.status === "failed" || run.status === "interrupted"; + return ( + run.status === "completed" || + run.status === "failed" || + run.status === "interrupted" || + // A queued request cancelled before it started never reached any provider. + (run.status === "cancelled" && run.startedAt !== null) + ); } function lastDeliveredRunForProviderThread( @@ -1097,15 +1278,19 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const runtimePolicy = yield* RuntimePolicyV2; const threadForkService = yield* ThreadForkServiceV2; const threadDispatch = yield* ThreadCommandExecutor.ThreadCommandExecutor; + const messageIntake = yield* KeyedLock.make(); /** The driver when this provider runs `/goal` natively, else undefined. */ const goalDriverFor = Effect.fn("Orchestrator.goalDriverFor")(function* ( command: OrchestrationV2ServerCommand, providerInstanceId: ProviderInstanceId, ) { - const { driver } = yield* providerAdapters - .get(providerInstanceId) - .pipe(mapDispatchError(command)); + const adapter = yield* providerAdapters.get(providerInstanceId).pipe(mapDispatchError(command)); + const { driver } = adapter; + if (driver === "opencode") { + const capabilities = yield* adapter.getCapabilities().pipe(mapDispatchError(command)); + return capabilities.turns.supportsNativeGoals === true ? driver : undefined; + } return driver === "codex" || driver === "claudeAgent" ? driver : undefined; }); @@ -1535,6 +1720,104 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } }); + /** Marks a wake the provider received delivered and reserves the next one for pending siblings. */ + const acceptDelegatedCompletionDelivery = Effect.fn( + "Orchestrator.acceptDelegatedCompletionDelivery", + )(function* (input: { + readonly command: OrchestrationV2ServerCommand; + readonly events: Ref.Ref>; + readonly projection: Pick< + OrchestrationV2ThreadProjection, + "thread" | "runs" | "messages" | "subagents" + >; + readonly messageId: OrchestrationV2ConversationMessage["id"]; + readonly now: DateTime.Utc; + }) { + const { projection, now } = input; + const message = projection.messages.find((row) => row.id === input.messageId); + const ownership = message?.delegatedCompletion; + const parentRun = projection.runs.find((run) => run.id === ownership?.parentRunId); + const cohort = parentRun?.delegatedCompletion; + const delivery = cohort?.delivery; + if ( + parentRun === undefined || + cohort === undefined || + cohort.disposition !== "open" || + delivery == null || + delivery.messageId !== input.messageId || + delivery.generation !== ownership?.generation + ) { + return undefined; + } + const threadId = projection.thread.id; + const parentIsLive = hasLiveRun({ runs: [parentRun] }); + const pendingTaskIds = + projection.thread.archivedAt === null && projection.thread.deletedAt === null + ? projection.subagents + .filter( + (task) => + task.origin === "app_owned" && + task.runId === parentRun.id && + task.completionDelivery?.state === "pending" && + isTerminalDelegatedTaskStatus(task.status) && + (!parentIsLive || task.completionWake === "always"), + ) + .map((task) => task.id) + : []; + const nextDelivery = + pendingTaskIds.length === 0 + ? null + : { + generation: cohort.nextGeneration, + messageId: yield* mapDelegatedCompletionError( + idAllocator.allocate.message({ + threadId, + ordinal: + (yield* mapDelegatedCompletionError(projectionStore.getMessageCount(threadId))) + + 1, + }), + ), + taskIds: pendingTaskIds, + }; + const emitEvent = emit(input.events, input.command); + const acceptedIds = new Set(delivery.taskIds); + const pendingIds = new Set(pendingTaskIds); + for (const task of projection.subagents) { + const state = pendingIds.has(task.id) + ? "claimed" + : acceptedIds.has(task.id) && task.completionDelivery?.state === "claimed" + ? "delivered" + : undefined; + if (state === undefined) continue; + yield* emitEvent({ + type: "subagent.updated", + threadId, + runId: parentRun.id, + nodeId: task.id, + driver: task.driver, + providerInstanceId: task.providerInstanceId, + occurredAt: now, + payload: { ...task, completionDelivery: { state, observedByRunId: null }, updatedAt: now }, + }); + } + yield* emitEvent({ + type: "run.updated", + threadId, + runId: parentRun.id, + providerInstanceId: parentRun.providerInstanceId, + occurredAt: now, + payload: { + ...parentRun, + delegatedCompletion: { + ...cohort, + nextGeneration: cohort.nextGeneration + (nextDelivery === null ? 0 : 1), + delivery: nextDelivery, + }, + }, + }); + return parentRun.id; + }); + const emitQueuedRunCancellation = Effect.fn("Orchestrator.emitQueuedRunCancellation")( function* (input: { readonly command: OrchestrationV2ServerCommand; @@ -1543,60 +1826,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio readonly run: OrchestrationV2Run; readonly now: DateTime.Utc; }) { - const rootNode = - input.run.rootNodeId === null - ? undefined - : input.projection.nodes.find((candidate) => candidate.id === input.run.rootNodeId); - const attempt = - input.run.activeAttemptId === null - ? undefined - : input.projection.attempts.find( - (candidate) => candidate.id === input.run.activeAttemptId, - ); const emitEvent = emit(input.events, input.command); - yield* emitEvent({ - type: "run.updated", - threadId: input.run.threadId, - runId: input.run.id, - ...(input.run.rootNodeId === null ? {} : { nodeId: input.run.rootNodeId }), - providerInstanceId: input.run.providerInstanceId, - occurredAt: input.now, - payload: { - ...input.run, - status: "cancelled", - queuePosition: null, - completedAt: input.now, - }, - }); - if (attempt !== undefined && rootNode !== undefined) { - yield* emitEvent({ - type: "run-attempt.updated", - threadId: input.run.threadId, - runId: input.run.id, - nodeId: rootNode.id, - providerInstanceId: input.run.providerInstanceId, - occurredAt: input.now, - payload: { - ...attempt, - status: "cancelled", - completedAt: input.now, - }, - }); - } - if (rootNode !== undefined) { - yield* emitEvent({ - type: "node.updated", - threadId: input.run.threadId, - runId: input.run.id, - nodeId: rootNode.id, - providerInstanceId: input.run.providerInstanceId, - occurredAt: input.now, - payload: { - ...rootNode, - status: "cancelled", - completedAt: input.now, - }, - }); + for (const event of queuedRunCancellationEvents(input.projection, input.run, input.now)) { + yield* emitEvent(event); } }, ); @@ -1712,6 +1944,85 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ]); }); + const isStaleQuestionWake = Effect.fn("Orchestrator.isStaleQuestionWake")(function* ( + projection: Pick, + messageId: MessageId, + notification: OrchestrationV2ThreadProjection["messages"][number]["notification"], + ) { + if (notification?.source.kind !== "delegated_task" || notification.outcome !== "updated") + return false; + const source = notification.source; + const threadId = projection.thread.id; + const task = + projection.subagents.find((candidate) => source.taskIds.includes(candidate.id)) ?? + // Command projections omit settled tasks whose result was already handed over. + (yield* projectionStore + .getThreadRecords(threadId, ["subagents"], { subagentIds: source.taskIds }) + .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })))) + .subagents[0]; + // An unknown task gives no evidence the question is obsolete. + if (task === undefined) return false; + const childThreadId = task.childThreadId; + if ( + childThreadId === null || + ["completed", "failed", "cancelled", "interrupted"].includes(task.status) + ) + return true; + const child = yield* projectionStore + .getThreadRecords(childThreadId, ["runtimeRequests"]) + .pipe( + Effect.mapError( + (cause) => new OrchestratorProjectionError({ threadId: childThreadId, cause }), + ), + ); + return !child.runtimeRequests.some( + (request) => + request.kind === "user_input" && + request.status === "pending" && + messageId === `message:delegated-task-question:${threadId}:${request.id}`, + ); + }); + + const cancelStaleQuestionWakes = Effect.fn("Orchestrator.cancelStaleQuestionWakes")(function* ( + projection: OrchestrationV2ThreadProjection, + ) { + const events = yield* Ref.make>([]); + for (const run of projection.runs) { + if (run.status !== "queued") continue; + const message = projection.messages.find((candidate) => candidate.id === run.userMessageId); + if ( + message === undefined || + !(yield* isStaleQuestionWake(projection, message.id, message.notification).pipe( + // A child thread read must not fail the parent's unrelated next queued run. + Effect.catchTags({ + OrchestratorProjectionError: (cause) => + Effect.logWarning("Failed to check a queued task question wake", { + runId: run.id, + cause, + }).pipe(Effect.as(false)), + }), + )) + ) + continue; + yield* emitQueuedRunCancellation({ + command: { + type: "queued-run.cancel", + commandId: CommandId.make(`command:system:stale-question:${run.id}`), + threadId: projection.thread.id, + runId: run.id, + }, + events, + projection, + run, + now: yield* DateTime.now, + }); + } + const cancellations = yield* Ref.get(events); + if (cancellations.length === 0) return false; + yield* writeSystemEvents(cancellations); + return true; + }); + const startNextQueuedRun = Effect.fn("Orchestrator.startNextQueuedRun")( function* ( threadId: ThreadId, @@ -1719,10 +2030,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio readonly failedRunId?: RunId; readonly endedRunId?: RunId; readonly queuedRunId?: RunId; + readonly userStarted?: boolean; }, ) { if (!(yield* projectionStore.canStartQueuedRun(threadId))) return; - const projection = yield* readCommandProjection(threadId); + let projection = yield* readCommandProjection(threadId); if ( projection.thread.archivedAt !== null || projection.thread.deletedAt !== null || @@ -1731,12 +2043,25 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio return; } - const storedQueuedRun = + if (yield* cancelStaleQuestionWakes(projection)) { + projection = yield* readCommandProjection(threadId); + } + let storedQueuedRun = options?.queuedRunId === undefined ? nextQueuedRun(projection) : projection.runs.find( (run) => run.id === options.queuedRunId && run.status === "queued", ); + while ( + storedQueuedRun !== undefined && + isStalePullRequestWatchWake(projection, storedQueuedRun) + ) { + yield* writeSystemEvents( + queuedRunCancellationEvents(projection, storedQueuedRun, yield* DateTime.now), + ); + projection = yield* readCommandProjection(threadId); + storedQueuedRun = nextQueuedRun(projection); + } if (storedQueuedRun === undefined) { return; } @@ -1750,7 +2075,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio providerInstanceId: modelSelection.instanceId, }; // A limited provider would fail and consume every waiting message. - if (usageLimitHoldsQueue(projection, queuedRun.providerInstanceId)) { + if ( + usageLimitHoldsQueue( + projection, + queuedRun.providerInstanceId, + options?.userStarted === true ? undefined : yield* DateTime.now, + ) + ) { return; } // Failed worktree creation would run queued work in the project checkout; wait for the user. @@ -2196,6 +2527,22 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ), updatedAt: now, }; + // Resume can clear the hold before the stopped run ends; its Stop marker still makes the drain explicit. + const endedRunId = options?.endedRunId; + const queueDelivery = + endedRunId === undefined || + queuedRun.queueHeld === true || + (yield* projectionStore + .getThreadRecords(threadId, ["turnItems"], { + turnItemTypes: ["run_interrupt_request"], + turnItemRunIds: [endedRunId], + }) + .pipe( + Effect.map(({ turnItems }) => turnItems.length > 0), + Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })), + )) + ? "explicit-queue" + : "automatic-queue"; const startingRun: OrchestrationV2Run = { ...queuedRun, providerThreadId: queuedProviderThread.id, @@ -2204,11 +2551,18 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio queuePosition: null, startedAt: null, contextHandoffId: activeHandoff?.id ?? null, - ...wakeWorkStartedAt(projection.runs, { - notification: queuedMessage.notification, - delegatedCompletion: queuedMessage.delegatedCompletion, - restartContinuationOfRunId: queuedRun.restartContinuationOfRunId, - }), + ...wakeWorkStartedAt( + projection, + { + createdBy: queuedMessage.createdBy, + creationSource: queuedMessage.creationSource, + senderThreadId: queuedMessage.senderThreadId, + notification: queuedMessage.notification, + delegatedCompletion: queuedMessage.delegatedCompletion, + restartContinuationOfRunId: queuedRun.restartContinuationOfRunId, + }, + queueDelivery, + ), }; const userTurnItem: OrchestrationV2TurnItem = { ...(legacyQueuedTurnItem ?? { @@ -2616,21 +2970,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio >, events: Ref.Ref>, ) { - const projection = yield* projectionStore - .getThreadRecords( - command.parentThreadId, - ["subagents", "runs", "nodes", "attempts", "turnItems"], - { turnItemTypes: [] }, - ) - .pipe( - Effect.mapError( - (cause) => - new OrchestratorProjectionError({ - threadId: command.parentThreadId, - cause, - }), - ), - ); + const projection = yield* loadProjectionForCommand(command, ["subagents"], { + subagentIds: [command.taskId], + }); const task = projection.subagents.find( (candidate) => candidate.id === command.taskId && candidate.origin === "app_owned", ); @@ -2647,6 +2989,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio : "disposed"; const now = yield* DateTime.now; const emitEvent = emit(events, command); + const { + runs: [parentRun], + } = yield* loadProjectionForCommand(command, ["runs"], { + runIds: task.runId === null ? [] : [task.runId], + }); // A late wait can arrive after finalization held the result for its still-live spawning run. if (isUnobservedAcknowledgement(command)) { if ( @@ -2654,23 +3001,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio task.completionDelivery?.state !== "pending" ) return; - const parentRun = projection.runs.find((run) => run.id === task.runId); if (parentRun !== undefined && isLiveRun(parentRun)) return; - const records = yield* projectionStore - .getThreadRecords(command.parentThreadId, ["messages", "providerTurns"], { - messageRoles: ["user"], - }) - .pipe( - Effect.mapError( - (cause) => - new OrchestratorProjectionError({ - threadId: command.parentThreadId, - cause, - }), - ), - ); + const records = yield* loadProjectionForCommand( + command, + ["runs", "subagents", "messages", "providerTurns"], + { messageRoles: ["user"] }, + ); const plan = yield* planDelegatedCompletionDelivery({ - parentProjection: { ...projection, ...records }, + parentProjection: records, parentRun, task, updatedTask: task, @@ -2740,10 +3078,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio payload: updatedTask, }); - const parentRun = - task.runId === null - ? undefined - : projection.runs.find((candidate) => candidate.id === task.runId); const cohort = parentRun?.delegatedCompletion; const delivery = cohort?.delivery; if ( @@ -2758,17 +3092,15 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio return; } - const { messages } = yield* projectionStore - .getThreadRecords(command.parentThreadId, ["messages"], { - messageRoles: ["user"], - messageIds: [delivery.messageId], - }) - .pipe( - Effect.mapError( - (cause) => new OrchestratorProjectionError({ threadId: command.parentThreadId, cause }), - ), - ); - const deliveryProjection = { ...projection, messages }; + const { messages } = yield* loadProjectionForCommand(command, ["messages"], { + messageRoles: ["user"], + messageIds: [delivery.messageId], + }); + const deliveryRunId = completionDeliveryMessage({ messages }, delivery)?.runId ?? null; + const { runs } = yield* loadProjectionForCommand(command, ["runs"], { + runIds: deliveryRunId === null ? [] : [deliveryRunId], + }); + const deliveryProjection = { messages, runs }; const remainingTaskIds = delivery.taskIds.filter((taskId) => taskId !== task.id); const deliveryRun = completionDeliveryRun(deliveryProjection, delivery); const clearDelivery = @@ -2798,10 +3130,20 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio if (deliveryRun?.status === "queued") { if (remainingTaskIds.length === 0) { + const rootNodeFilter = { + nodeIds: deliveryRun.rootNodeId === null ? [] : [deliveryRun.rootNodeId], + }; yield* emitQueuedRunCancellation({ command, events, - projection, + // Attempts have no id filter; a wake that never started has no attempt to cancel. + projection: + deliveryRun.activeAttemptId === null + ? { + attempts: [], + ...(yield* loadProjectionForCommand(command, ["nodes"], rootNodeFilter)), + } + : yield* loadProjectionForCommand(command, ["nodes", "attempts"], rootNodeFilter), run: deliveryRun, now, }); @@ -2809,6 +3151,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } const message = completionDeliveryMessage(deliveryProjection, delivery); if (message !== undefined) { + const { subagents: remainingTasks } = yield* loadProjectionForCommand( + command, + ["subagents"], + { subagentIds: remainingTaskIds }, + ); yield* emitEvent({ type: "message.updated", threadId: command.parentThreadId, @@ -2818,7 +3165,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio occurredAt: now, payload: { ...message, - text: delegatedCompletionWakeDetail(remainingTaskIds, projection.subagents), + text: delegatedCompletionWakeDetail(remainingTaskIds, remainingTasks), delegatedCompletion: { parentRunId: parentRun.id, generation: delivery.generation, @@ -2845,8 +3192,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio readonly disposition: "stopped" | "disposed"; readonly now: DateTime.Utc; readonly cancelQueuedDelivery?: boolean; - // An interrupted wake disposes only the results it carried. The cohort stays open, and its - // delivery stays the wake's, so the wake's terminal reserves any sibling's result. + // An interrupted or cancelled wake disposes only the results it carried. The cohort stays + // open, and its delivery stays the wake's, so the wake's terminal reserves any sibling's result. readonly interruptedWakeTaskIds?: ReadonlyArray; }) { const parentRun = input.projection.runs.find((candidate) => candidate.id === input.parentRunId); @@ -3164,6 +3511,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } yield* dispatchThreadMutation(command, events, effects); if (command.wake === undefined) return; + // An idle thread dispatches immediately, bypassing queued-wake validation. + if ( + command.wake.notification.source.kind === "monitor" && + command.watch !== null && + pullRequestSnapshotEndsWatch(link.snapshot, command.startedAt) + ) { + return; + } yield* dispatchMessage( { type: "message.dispatch", @@ -3171,7 +3526,24 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio threadId: command.threadId, messageId: command.wake.messageId, text: command.wake.text, - notification: command.wake.notification, + notification: { + ...command.wake.notification, + source: + command.wake.notification.source.kind === "monitor" + ? { + kind: "monitor", + pullRequest: { + ...key, + watchStartedAt: command.startedAt, + headSha: + command.wake.includesRemarks === true + ? null + : (command.watch?.headSha ?? null), + ...(command.watch === null ? { watchEnded: true } : {}), + }, + } + : command.wake.notification.source, + }, attachments: [], dispatchMode: { type: "queue_after_active" }, createdBy: "agent", @@ -3585,6 +3957,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio runId: run.id, }, events, + "cohort", ); } } @@ -3621,12 +3994,24 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }), ), ); - return yield* providerSwitchService + const plan = yield* providerSwitchService .plan({ projection: providerContext!, targetModelSelection: command.modelSelection, }) .pipe(mapDispatchError(command)); + if (plan.instanceChanged || plan.releaseProviderSessionIds.length > 0) { + const projection = yield* loadProjectionForCommand(command, ["runs"], { + turnItemTypes: [], + }); + if (projection.runs.some(isActiveRun)) { + return yield* new OrchestratorProviderSelectionBusyError({ + commandId: command.commandId, + threadId: command.threadId, + }); + } + } + return plan; }) : null; @@ -4108,6 +4493,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ...(existing.endedWatch?.owedChecks !== undefined && { owedChecks: existing.endedWatch.owedChecks, }), + ...(existing.endedWatch?.awaitingApproval !== undefined && { + awaitingApproval: existing.endedWatch.awaitingApproval, + }), + ...(existing.endedWatch?.approvalAnnounced !== undefined && { + approvalAnnounced: existing.endedWatch.approvalAnnounced, + }), ...(existing.endedWatch?.checksAwaitedSince !== undefined && { checksAwaitedSince: existing.endedWatch.checksAwaitedSince, }), @@ -4268,6 +4659,46 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio // Archiving ends the guest share, so its waiting files go too. yield* releaseGuestAttachments(command, thread.guestShare, updatedThread.guestShare, effects); + if ( + command.type === "thread.model-selection.set" && + providerSwitchPlan !== null && + !providerSwitchPlan.instanceChanged && + providerSwitchPlan.releaseProviderSessionIds.length === 0 + ) { + const { providerThread, providerTurn } = yield* projectionStore + .getRunningTurnContext(command.threadId) + .pipe(mapDispatchError(command)); + const providerSessionId = providerThread?.providerSessionId; + if (providerThread !== undefined && providerTurn !== undefined && providerSessionId) { + const session = yield* providerSessions.get(providerSessionId).pipe( + Effect.mapError( + (cause) => + new OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause, + }), + ), + ); + if (Option.isSome(session) && session.value.applyModelSelection !== undefined) { + yield* Ref.update(effects, (existing) => [ + ...existing, + { + id: `effect:${command.commandId}:provider-session.apply-model-selection`, + commandId: command.commandId, + threadId: command.threadId, + request: { + type: "provider-session.apply-model-selection", + providerSessionId, + providerThreadId: providerThread.id, + modelSelection: command.modelSelection, + }, + } satisfies PendingOrchestrationEffectV2, + ]); + } + } + } + if (command.type === "thread.metadata.update" && command.regenerateTitle === true) { yield* Ref.update(effects, (existing) => [ ...existing, @@ -4891,6 +5322,33 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }); }); + const ensureSteeringMessage = Effect.fn("Orchestrator.ensureSteeringMessage")(function* ( + command: Extract< + OrchestrationV2Command, + { readonly type: "message.dispatch" | "queued-message.promote-to-steer" } + >, + message: Pick, + ) { + if (isNativeMaintenanceCommand(message)) { + return yield* new OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: + message.text.trim().toLowerCase() === "/compact" + ? "Context compaction must run as a separate turn. Queue it or wait for the active turn to finish." + : "Signing out must run as a separate turn. Queue it or wait for the active turn to finish.", + }); + } + if (parseGoalCommand(message) !== null) { + return yield* new OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: + "Goal commands must run as a separate turn. Queue it or wait for the active turn to finish.", + }); + } + }); + const dispatchSteerIntoRun = Effect.fn("Orchestrator.dispatchSteerIntoRun")(function* (input: { readonly command: Extract< OrchestrationV2Command, @@ -4934,24 +5392,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio cause: `Target run ${input.targetRunId} was not found.`, }); } - if (isNativeMaintenanceCommand(input)) { - return yield* new OrchestratorDispatchError({ - commandId: input.command.commandId, - commandType: input.command.type, - cause: - input.text.trim().toLowerCase() === "/compact" - ? "Context compaction must run as a separate turn. Queue it or wait for the active turn to finish." - : "Signing out must run as a separate turn. Queue it or wait for the active turn to finish.", - }); - } - if (parseGoalCommand(input) !== null) { - return yield* new OrchestratorDispatchError({ - commandId: input.command.commandId, - commandType: input.command.type, - cause: - "Goal commands must run as a separate turn. Queue it or wait for the active turn to finish.", - }); - } + yield* ensureSteeringMessage(input.command, input); const targetMessage = input.projection.messages.find( (message) => message.id === targetRun.userMessageId, ); @@ -5177,9 +5618,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio if (steeringPolicy === "active_steering") { const instanceChanged = input.projection.thread.providerInstanceId !== input.modelSelection.instanceId; + // Only a user's steer carries the picker's selection; the adapter applies it to the live turn. + const steerCarriesSelection = + input.delegatedCompletion === undefined && input.notification === undefined; if ( - input.delegatedCompletion === undefined && - input.notification === undefined && + steerCarriesSelection && (instanceChanged || !modelSelectionsEqual(input.projection.thread.modelSelection, input.modelSelection)) ) { @@ -5215,6 +5658,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio providerThreadId: providerThread.id, providerTurnId: providerTurn.id, messageId: input.messageId, + ...(steerCarriesSelection ? { modelSelection: input.modelSelection } : {}), }, } satisfies PendingOrchestrationEffectV2, ]); @@ -5595,6 +6039,15 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio occurredAt: now, payload: checkpointScope, }); + // The restart replaces the run's input, so the wake it was running can no longer be + // settled at its terminal. Its provider turn was running, so the wake already landed. + yield* acceptDelegatedCompletionDelivery({ + command: input.command, + events: input.events, + projection: input.projection, + messageId: targetRun.userMessageId, + now, + }); yield* appendSteeringMessage({ runId: targetRun.id, nodeId: nextRootNodeId, @@ -5639,6 +6092,18 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio effects: Ref.Ref>, ) { let projection = yield* getProjectionWithPendingEvents(command.threadId, events); + if (yield* isStaleQuestionWake(projection, command.messageId, command.notification)) { + yield* emit( + events, + command, + )({ + type: "thread.metadata-updated", + threadId: command.threadId, + occurredAt: yield* DateTime.now, + payload: projection.thread, + }); + return; + } if (command.manualContinuationOfRunId !== undefined) { const source = projection.runs.find((run) => run.id === command.manualContinuationOfRunId); const limited = latestRootProviderFailure(source ?? null, projection.turnItems); @@ -5738,6 +6203,33 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } } + // The service checked idleness before this command; a run or a delegated result that landed + // since makes the thread busy, and that turn refreshes the cache anyway. + if ( + command.idleCompaction === "beforeCacheExpires" && + projection.runs.some( + (run) => + run.status === "queued" || + isActiveRun(run) || + (run.delegatedCompletion?.disposition === "open" && + run.delegatedCompletion.delivery !== null && + !projection.messages.some( + (message) => message.id === run.delegatedCompletion?.delivery?.messageId, + )), + ) + ) { + yield* emit( + events, + command, + )({ + type: "thread.metadata-updated", + threadId: command.threadId, + occurredAt: yield* DateTime.now, + payload: projection.thread, + }); + return; + } + if (projection.thread.lastVisitedAt === null && !isNativeMaintenanceCommand(command)) { const now = yield* DateTime.now; // Null keeps imported history read. The first fresh dispatch establishes @@ -5759,8 +6251,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio projection = applyToProjection(projection, event); } - const continuedTaskId = projection.runs.find( - (run) => run.id === command.restartContinuationOfRunId, + // A run the thread queues for itself continues the task of the run that sent it. + const continuedTaskId = projection.runs.find((run) => + command.restartContinuationOfRunId === undefined + ? command.senderThreadId === command.threadId && isActiveRun(run) + : run.id === command.restartContinuationOfRunId, )?.delegatedTaskId; if (projection.thread.settledOverride !== null) { @@ -5848,13 +6343,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ]); projection = applyToProjection(projection, event); } - const modelSelection = command.modelSelection ?? projection.thread.modelSelection; + let modelSelection = command.modelSelection ?? projection.thread.modelSelection; let dispatchMode = resolveMessageDispatchIntent( projection, command.dispatchMode, command.deliveryIntent, ); - if (dispatchMode.type === "steer_active") { + if (dispatchMode.type === "steer_active" || dispatchMode.type === "restart_active") { const targetRunId = dispatchMode.targetRunId; const target = projection.runs.find((run) => run.id === targetRunId); const turn = latestProviderTurnForAttempt( @@ -5866,9 +6361,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio if ( target !== undefined && (target.status === "completed" || + (dispatchMode.type === "restart_active" && !isActiveRun(target)) || ((target.status === "running" || target.status === "waiting") && turn !== undefined && - turn.status === "completed")) + (turn.status === "completed" || + (dispatchMode.type === "restart_active" && + turn.status !== "pending" && + turn.status !== "running")))) ) { dispatchMode = { type: "start_immediately" }; } @@ -5876,7 +6375,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio // Stop leaves the run reading as running until its provider stops. A plain // send in that window queues behind the stopped queue instead of steering // text the agent never receives, and like any user send it releases the - // hold; an explicit steer or restart still fails below. + // hold; an explicit steer still fails below. if ( command.deliveryIntent === "auto" && (dispatchMode.type === "steer_active" || dispatchMode.type === "restart_active") && @@ -5884,6 +6383,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ) { dispatchMode = { type: "queue_after_active" }; } + // An explicit restart that Stop reached has only its start left, so it + // queues behind the stopping run like any other start. + if ( + dispatchMode.type === "restart_active" && + (yield* stopReachedRun(command, command.threadId, dispatchMode.targetRunId)) + ) { + dispatchMode = { type: "start_immediately" }; + } const freshContext = command.freshContext === true; if ( freshContext && @@ -5891,16 +6398,17 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ) { dispatchMode = { type: "queue_after_active" }; } - // Codex and Claude own /goal; elsewhere it is ordinary text that cannot steer. + // Native goal commands must not steer an existing turn. const parsedGoalCommand = parseGoalCommand(command); const goalCommand = parsedGoalCommand === null || (yield* goalDriverFor(command, modelSelection.instanceId)) === undefined ? null : parsedGoalCommand; - let goalStopRunId: RunId | undefined; + let queuedRunStop: + | { readonly runId: RunId; readonly reason: string; readonly skipsHeldQueue?: true } + | undefined; if (goalCommand !== null) { - yield* ensureGoalCommandSupported(command, command, modelSelection.instanceId); // A goal command never joins a running turn. One that starts or ends goal // work stops the run and goes first in the queue; the rest wait. A send // that asked to be queued (no delivery intent) only queues, and a run @@ -5925,7 +6433,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio (targetMessage === undefined || !isNativeMaintenanceCommand(targetMessage)) && !(yield* stopReachedRun(command, command.threadId, target.id)) ) { - goalStopRunId = target.id; + queuedRunStop = { runId: target.id, reason: "Stopped to apply /goal" }; } dispatchMode = { type: "queue_after_active" }; } @@ -5985,6 +6493,37 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio taskIds: delivery.taskIds, }; } + // A provider that can stop a turn but not restart it with new input runs a + // restart as the next turn: first in the queue, with the running turn stopped. + if (dispatchMode.type === "restart_active" && !isNativeMaintenanceCommand(command)) { + const targetRunId = dispatchMode.targetRunId; + const target = projection.runs.find((run) => run.id === targetRunId); + const providerSessionId = projection.providerThreads.find( + (candidate) => candidate.id === target?.providerThreadId, + )?.providerSessionId; + const turns = projection.providerSessions.find((session) => session.id === providerSessionId) + ?.capabilities.turns; + const targetMessage = projection.messages.find( + (message) => message.id === target?.userMessageId, + ); + if ( + target?.status === "running" && + turns !== undefined && + turns.supportsInterrupt && + turns.supportsQueuedMessages && + !turns.supportsSteeringByInterruptRestart && + (targetMessage === undefined || !isNativeMaintenanceCommand(targetMessage)) && + !(yield* stopReachedRun(command, command.threadId, target.id)) + ) { + // The restart replaces the stopped turn, so an older held queue does not hold it. + queuedRunStop = { + runId: target.id, + reason: "Stopped to send a new message", + skipsHeldQueue: true, + }; + dispatchMode = { type: "queue_after_active" }; + } + } // Steer mailbox deliveries, delegated task questions, and command notices // into an active goal under the thread lock: queued behind a long turn or a // goal run, they are never seen. Never interrupt/restart a turn for one. @@ -6083,7 +6622,38 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio payload: { ...sourcePlan, status: "completed" }, }); + let steerWhenReadyRunId: RunId | undefined; + if (dispatchMode.type === "steer_active") { + const targetRunId = dispatchMode.targetRunId; + const target = projection.runs.find((run) => run.id === targetRunId); + if ( + (target?.status === "preparing" || + target?.status === "starting" || + (target?.status === "running" && + !projection.providerTurns.some( + (turn) => turn.runAttemptId === target.activeAttemptId && turn.status === "running", + ))) && + !(yield* stopReachedRun(command, command.threadId, target.id)) + ) { + const adapter = yield* providerAdapters + .get(target.providerInstanceId) + .pipe(mapDispatchError(command)); + const capabilities = yield* adapter.getCapabilities().pipe(mapDispatchError(command)); + yield* commandPolicy + .decideSteeringExecution({ + commandId: command.commandId, + threadId: command.threadId, + providerInstanceId: target.providerInstanceId, + capabilities, + }) + .pipe(mapDispatchError(command)); + yield* ensureSteeringMessage(command, command); + steerWhenReadyRunId = target.id; + dispatchMode = { type: "queue_after_active" }; + } + } if (dispatchMode.type === "steer_active" || dispatchMode.type === "restart_active") { + yield* ensureGoalCommandSupported(command, command, modelSelection.instanceId); yield* dispatchSteerIntoRun({ command, events, @@ -6136,6 +6706,21 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio dispatchMode.type === "start_immediately" || dispatchMode.type === "queue_after_active"; const shouldQueue = queueBehindRun !== undefined && startsOrQueuesRun; + // An explicit pick queued ahead reaches the thread only when its run starts. A goal or + // restart command that stops the active run goes first, so nothing queued applies before it. + // Buffered provider wakes drain output their own provider already produced. + const incomingMessage = { ...command, delegatedCompletion }; + if ( + shouldQueue && + steerWhenReadyRunId === undefined && + command.modelSelection === undefined && + queuedRunStop === undefined && + !followsCurrentConversation(incomingMessage) && + !isBufferedProviderNotification(incomingMessage) + ) { + modelSelection = queueTailModelSelection(projection); + } + yield* ensureGoalCommandSupported(command, command, modelSelection.instanceId); // Client messages resume held queues; server continuations and scheduled tasks do not. // Provider failure or usage limits re-hold the queue. const worktreeSetupUnresolved = worktreeSetupPending(projection); @@ -6282,12 +6867,16 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio rootNodeId, activeAttemptId: attemptId, status: "queued", + ...(steerWhenReadyRunId === undefined ? {} : { steerWhenReadyRunId }), ...(worktreeMissingRun !== undefined || - (heldRunsToRelease.length === 0 && queueIsHeld && !wakeSkipsHold) + (heldRunsToRelease.length === 0 && + queueIsHeld && + !wakeSkipsHold && + queuedRunStop?.skipsHeldQueue !== true) ? { queueHeld: true } : {}), queuePosition: - goalStopRunId !== undefined + queuedRunStop !== undefined ? 1 : Math.max( 0, @@ -6445,15 +7034,15 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }, }); } - if (goalStopRunId !== undefined) { + if (queuedRunStop !== undefined) { return yield* stopRunForQueuedCommand({ command, events, effects, projection: yield* getProjectionWithPendingEvents(command.threadId, events), queuedRunId: runId, - targetRunId: goalStopRunId, - reason: "Stopped to apply /goal", + targetRunId: queuedRunStop.runId, + reason: queuedRunStop.reason, now, }); } @@ -6597,6 +7186,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio nativeThreadId: `pending:${runId}`, }); const legacyImportHandoff = shouldPrepareLegacyImportHandoff({ + hasNativeHistory: + activeProviderThread?.driver === "acpRegistry" && + activeProviderThread.nativeThreadRef !== null, historyOrigin: projection.thread.historyOrigin, hasCompletedRun: latestCompletedRun !== undefined, legacyImportItemCount: legacyImportItems.length, @@ -6694,7 +7286,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ? { workspacePreparation: dispatchMode.workspaceStrategy } : {}), ...(continuedTaskId === undefined ? {} : { delegatedTaskId: continuedTaskId }), - ...wakeWorkStartedAt(projection.runs, command), + ...wakeWorkStartedAt(projection, command), ...(freshContext ? { freshContext: true } : {}), }; const attempt: OrchestrationV2RunAttempt = { @@ -7480,7 +8072,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ? {} : { usageLimitContinuationOfRunId: command.usageLimitContinuationOfRunId }), ...(continuedTaskId === undefined ? {} : { delegatedTaskId: continuedTaskId }), - ...wakeWorkStartedAt(projection.runs, command), + ...wakeWorkStartedAt(projection, command), }; const attempt: OrchestrationV2RunAttempt = { id: attemptId, @@ -8247,12 +8839,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio if (!startsAttempt) return; const child = yield* projectionStore.getThread(command.threadId); const parentThreadId = child.lineage.parentThreadId; + // A send with no sender thread comes from a client acting for the owner, like a user send. if ( child.lineage.relationshipToParent !== "subagent" || parentThreadId === null || (command.notification === undefined && command.createdBy !== "user" && command.usageLimitContinuationOfRunId === undefined && + command.senderThreadId !== undefined && command.senderThreadId !== parentThreadId) ) return; @@ -8281,9 +8875,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio task.id === (child.forkedFrom?.type === "node" ? child.forkedFrom.nodeId : undefined), ); if (original === undefined || original.result === null) return; - const owner = - parent.runs.filter(isActiveRun).toSorted((a, b) => b.ordinal - a.ordinal)[0] ?? - latestExecutedRun(parent.runs); + const owner = followUpOwnerRun(parent.runs); if (owner === null || owner.rootNodeId === null) return; const now = message.createdAt; const adapter = yield* providerAdapters.get(run.providerInstanceId); @@ -8305,6 +8897,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio completionDelivery: { state: "pending", observedByRunId: null }, status: run.status === "queued" ? (run.queueHeld === true ? "idle" : "pending") : "running", progress: undefined, + cancelledRunId: undefined, result: null, startedAt: now, completedAt: null, @@ -8400,7 +8993,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio events: Ref.Ref>, ) { const parentProjection = yield* projectionStore - .getThreadRecords(command.parentThreadId, ["runs", "providerTurns", "attempts"]) + .getThreadRecords(command.parentThreadId, ["runs", "providerTurns", "attempts", "subagents"]) .pipe( Effect.mapError( (cause) => new OrchestratorProjectionError({ threadId: command.parentThreadId, cause }), @@ -8409,7 +9002,20 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const parentRun = parentProjection.runs.find( (candidate) => candidate.id === command.parentRunId, ); - if (parentRun === undefined || !isActiveRun(parentRun) || parentRun.rootNodeId === null) { + // A follow-up to a finished task there binds as one to a child here does. + const followsUp = parentProjection.subagents.some( + (task) => + task.result !== null && + task.remoteChild?.environmentId === command.remoteChild.environmentId && + task.remoteChild.threadId === command.remoteChild.threadId, + ); + if ( + parentRun === undefined || + (followsUp + ? parentRun.id !== followUpOwnerRun(parentProjection.runs)?.id + : !isActiveRun(parentRun)) || + parentRun.rootNodeId === null + ) { return yield* new OrchestratorDispatchError({ commandId: command.commandId, commandType: command.type, @@ -9909,6 +10515,39 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio now, }); } + if ( + (target?.status === "preparing" || + target?.status === "starting" || + (target?.status === "running" && + !projection.providerTurns.some( + (turn) => turn.runAttemptId === target.activeAttemptId && turn.status === "running", + ))) && + !(yield* stopReachedRun(command, command.threadId, target.id)) + ) { + yield* ensureSteeringMessage(command, queuedMessage); + const adapter = yield* providerAdapters + .get(target.providerInstanceId) + .pipe(mapDispatchError(command)); + const capabilities = yield* adapter.getCapabilities().pipe(mapDispatchError(command)); + yield* commandPolicy + .decideSteeringExecution({ + commandId: command.commandId, + threadId: command.threadId, + providerInstanceId: target.providerInstanceId, + capabilities, + }) + .pipe(mapDispatchError(command)); + yield* emitEvent({ + type: "run.updated", + threadId: command.threadId, + runId: queuedRun.id, + nodeId: queuedRootNode.id, + providerInstanceId: queuedRun.providerInstanceId, + occurredAt: now, + payload: { ...queuedRun, steerWhenReadyRunId: target.id }, + }); + return; + } yield* emitEvent({ type: "run.updated", threadId: command.threadId, @@ -10065,6 +10704,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const dispatchQueuedRunCancel = Effect.fn("Orchestrator.dispatchQueuedRunCancel")(function* ( command: Extract, events: Ref.Ref>, + // Settling disposes a cancelled wake's whole cohort, so a sibling's late result cannot unsettle it. + disposes: "wake" | "cohort" = "wake", ) { const projection = yield* loadProjectionForCommand( command, @@ -10082,8 +10723,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const queuedMessage = projection.messages.find( (candidate) => candidate.id === queuedRun.userMessageId, ); + const now = yield* DateTime.now; if (queuedMessage?.delegatedCompletion !== undefined) { - const now = yield* DateTime.now; yield* disposeDelegatedCompletionCohort({ command, events, @@ -10091,64 +10732,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio parentRunId: queuedMessage.delegatedCompletion.parentRunId, disposition: "disposed", now, + cancelQueuedDelivery: false, + ...(disposes === "wake" + ? { interruptedWakeTaskIds: queuedMessage.delegatedCompletion.taskIds } + : {}), }); - return; } - const queuedRootNode = - queuedRun.rootNodeId === null - ? undefined - : projection.nodes.find((candidate) => candidate.id === queuedRun.rootNodeId); - const queuedAttempt = - queuedRun.activeAttemptId === null - ? undefined - : projection.attempts.find((candidate) => candidate.id === queuedRun.activeAttemptId); - - const now = yield* DateTime.now; + yield* emitQueuedRunCancellation({ command, events, projection, run: queuedRun, now }); const emitEvent = emit(events, command); - yield* emitEvent({ - type: "run.updated", - threadId: command.threadId, - runId: queuedRun.id, - ...(queuedRun.rootNodeId === null ? {} : { nodeId: queuedRun.rootNodeId }), - providerInstanceId: queuedRun.providerInstanceId, - occurredAt: now, - payload: { - ...queuedRun, - status: "cancelled", - queuePosition: null, - completedAt: now, - }, - }); - if (queuedAttempt !== undefined && queuedRootNode !== undefined) { - yield* emitEvent({ - type: "run-attempt.updated", - threadId: command.threadId, - runId: queuedRun.id, - nodeId: queuedRootNode.id, - providerInstanceId: queuedRun.providerInstanceId, - occurredAt: now, - payload: { - ...queuedAttempt, - status: "cancelled", - completedAt: now, - }, - }); - } - if (queuedRootNode !== undefined) { - yield* emitEvent({ - type: "node.updated", - threadId: command.threadId, - runId: queuedRun.id, - nodeId: queuedRootNode.id, - providerInstanceId: queuedRun.providerInstanceId, - occurredAt: now, - payload: { - ...queuedRootNode, - status: "cancelled", - completedAt: now, - }, - }); - } const guest = queuedMessage?.context?.guest; if (guest !== undefined) { const thread = guestShareAfterCancelledSend(projection.thread, guest, now); @@ -12333,7 +12924,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio cause: `Checkpoint ${command.checkpointId} was not found.`, }); } - if (targetCheckpoint.status !== "ready") { + // Missing snapshots support conversation rewind only. + if ( + targetCheckpoint.status !== "ready" && + !(command.restoreFiles === false && targetCheckpoint.status === "missing") + ) { return yield* new OrchestratorDispatchError({ commandId: command.commandId, commandType: command.type, @@ -12386,7 +12981,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const targetRun = projection.runs.find((run) => run.ordinal === targetOrdinal); const targetProviderTurn = targetRun === undefined ? undefined : providerTurnForRun(projection, targetRun); - if (targetRun === undefined || targetProviderTurn === undefined) { + if ( + targetRun === undefined || + targetRun.status === "rolled_back" || + targetProviderTurn === undefined + ) { return yield* new OrchestratorDispatchError({ commandId: command.commandId, commandType: command.type, @@ -12737,10 +13336,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ? options.task.id === forkedFrom.nodeId ? options.resultRun : undefined - : taskRuns.toSorted((a, b) => b.ordinal - a.ordinal)[0]; + : delegatedTaskResultRun(taskRuns, options.task.id); if (childRun === undefined) return; - const outcome = runOutcome(childRun); - const terminalStatus = delegatedTaskTerminalStatus(outcome); + let outcome = runOutcome(childRun); + let terminalStatus = delegatedTaskTerminalStatus(outcome); if (terminalStatus === null) { if (childRun.delegatedTaskId === undefined || childRun.id !== options.syncRunId) return; const task = options.task; @@ -12846,6 +13445,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio if (task === undefined) { return; } + // A cancelled task's completed run ends it interrupted. + const cancelled = task.cancelledRunId === childRun.id; + if (cancelled && terminalStatus === "completed") outcome = terminalStatus = "interrupted"; const existingResultTransfer = parentProjection.contextTransfers.find( (transfer) => transfer.type === "subagent_result" && @@ -12875,6 +13477,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio result: result.text, completedAt: now, updatedAt: now, + // The cancelling parent gets no wake, and the task records the run it settled from. + ...(cancelled + ? { + cancelledRunId: childRun.id, + completionDelivery: { state: "disposed" as const, observedByRunId: null }, + } + : {}), }; const completionPlan = yield* planDelegatedCompletionDelivery({ parentProjection, @@ -12981,6 +13590,15 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }, ); + // A waiting run has ended its turn, so its task waits only on the work it left. + const progressAfterTurn = (child: Parameters[0]) => + delegatedTaskProgress({ + ...child, + runs: child.runs.map((run) => + run.status === "waiting" ? { ...run, status: "completed" as const } : run, + ), + }); + const finalizeAppOwnedSubagent = Effect.fn("Orchestrator.finalizeAppOwnedSubagent")(function* ( childThreadId: ThreadId, options?: { readonly settledContinuationOf?: RunId; readonly syncRunId?: RunId }, @@ -13003,11 +13621,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio "contextTransfers", ]); for (const task of parent.subagents) { + // A cancelled task does not wait for the work its cancel is stopping. + const cancelled = task.cancelledRunId !== undefined; if ( task.origin !== "app_owned" || task.childThreadId !== childThreadId || (task.id === childControls.thread.forkedFrom.nodeId && - progress.state !== "result_available") || + (cancelled ? progress.state === "working" : progress.state !== "result_available")) || (task.result !== null && parent.contextTransfers.some( (transfer) => @@ -13022,10 +13642,20 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }); } + // Only the run a cancelled task settled from ends with it. The row read above has no + // such run yet when this call settled the task, from the same result run. const { resultRun } = progress; - if (progress.state !== "result_available" || resultRun === undefined) return; + const resultTaskId = resultRun?.delegatedTaskId ?? childControls.thread.forkedFrom.nodeId; + const resultTask = parent.subagents.find((task) => task.id === resultTaskId); + const taskCancelled = + resultTask?.cancelledRunId !== undefined && resultTask.cancelledRunId === resultRun?.id; + if ( + resultRun === undefined || + (taskCancelled ? progress.state === "working" : progress.state !== "result_available") + ) + return; const status = delegatedTaskTerminalStatus(runOutcome(resultRun)); - if (status !== "cancelled" && status !== "interrupted") return; + if (status !== "interrupted" && status !== "cancelled" && !taskCancelled) return; if ( yield* childAwaitsRestartContinuation( childControls.runs, @@ -13230,95 +13860,26 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio events: Ref.Ref>, ) { const projection = yield* getProjectionWithPendingEvents(command.threadId, events); - const message = projection.messages.find((row) => row.id === command.messageId); - const ownership = message?.delegatedCompletion; - const parentRun = projection.runs.find((run) => run.id === ownership?.parentRunId); - const cohort = parentRun?.delegatedCompletion; - const delivery = cohort?.delivery; const now = yield* DateTime.now; - const emitEvent = emit(events, command); - if ( - parentRun === undefined || - cohort === undefined || - cohort.disposition !== "open" || - delivery == null || - delivery.messageId !== command.messageId || - delivery.generation !== ownership?.generation - ) { - yield* emitEvent({ + const parentRunId = yield* acceptDelegatedCompletionDelivery({ + command, + events, + projection, + messageId: command.messageId, + now, + }); + if (parentRunId === undefined) { + yield* emit( + events, + command, + )({ type: "thread.metadata-updated", threadId: command.threadId, occurredAt: now, payload: projection.thread, }); - return; - } - const parentIsLive = hasLiveRun({ runs: [parentRun] }); - const pendingTaskIds = - projection.thread.archivedAt === null && projection.thread.deletedAt === null - ? projection.subagents - .filter( - (task) => - task.origin === "app_owned" && - task.runId === parentRun.id && - task.completionDelivery?.state === "pending" && - isTerminalDelegatedTaskStatus(task.status) && - (!parentIsLive || task.completionWake === "always"), - ) - .map((task) => task.id) - : []; - const nextDelivery = - pendingTaskIds.length === 0 - ? null - : { - generation: cohort.nextGeneration, - messageId: yield* mapDelegatedCompletionError( - idAllocator.allocate.message({ - threadId: command.threadId, - ordinal: - (yield* mapDelegatedCompletionError( - projectionStore.getMessageCount(projection.thread.id), - )) + 1, - }), - ), - taskIds: pendingTaskIds, - }; - const acceptedIds = new Set(delivery.taskIds); - const pendingIds = new Set(pendingTaskIds); - for (const task of projection.subagents) { - const state = pendingIds.has(task.id) - ? "claimed" - : acceptedIds.has(task.id) && task.completionDelivery?.state === "claimed" - ? "delivered" - : undefined; - if (state === undefined) continue; - yield* emitEvent({ - type: "subagent.updated", - threadId: command.threadId, - runId: parentRun.id, - nodeId: task.id, - driver: task.driver, - providerInstanceId: task.providerInstanceId, - occurredAt: now, - payload: { ...task, completionDelivery: { state, observedByRunId: null }, updatedAt: now }, - }); } - yield* emitEvent({ - type: "run.updated", - threadId: command.threadId, - runId: parentRun.id, - providerInstanceId: parentRun.providerInstanceId, - occurredAt: now, - payload: { - ...parentRun, - delegatedCompletion: { - ...cohort, - nextGeneration: cohort.nextGeneration + (nextDelivery === null ? 0 : 1), - delivery: nextDelivery, - }, - }, - }); - return parentRun.id; + return parentRunId; }); const dispatchUnsupported = (command: OrchestrationV2ServerCommand) => @@ -13782,10 +14343,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ); if (command.type === "queue.resume" || command.type === "queued-run.start") { yield* mapDispatchError(command)( - startNextQueuedRun( - command.threadId, - command.type === "queued-run.start" ? { queuedRunId: command.runId } : undefined, - ), + startNextQueuedRun(command.threadId, { + userStarted: true, + ...(command.type === "queued-run.start" ? { queuedRunId: command.runId } : {}), + }), ); } return { @@ -13799,6 +14360,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const dispatchWithReceiptEffect = Effect.fn("orchestrationV2.dispatch.withReceipt")(function* ( command: OrchestrationV2ServerCommand, sideChatIds: ReadonlyArray = [], + cancelledTask?: OrchestrationV2Subagent, ): Effect.fn.Return { yield* Effect.annotateCurrentSpan({ "orchestration_v2.command_id": command.commandId, @@ -13889,7 +14451,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ), ); - if (plan.events.length === 0 && plan.effects.length === 0) { + if (plan.events.length === 0 && plan.effects.length === 0 && cancelledTask === undefined) { // A settle that ended nothing still records its receipt: a replayed Stop // effect then finds it instead of settling work that appeared since. const { resultSequence, inserted } = yield* Effect.gen(function* () { @@ -13939,7 +14501,19 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio threadId: commandThreadId(command), commandType: command.type, acceptedAt, - events: plan.events, + events: + cancelledTask === undefined + ? plan.events + : [ + ...plan.events, + yield* makeSystemEvent({ + type: "subagent.updated", + threadId: cancelledTask.threadId, + nodeId: cancelledTask.id, + occurredAt: cancelledTask.updatedAt, + payload: cancelledTask, + }).pipe(mapDispatchError(command)), + ], effects: plan.effects, ...(plan.cancelUnsettledEffects === undefined ? {} @@ -13978,13 +14552,24 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } if (command.type === "queue.resume" || command.type === "queued-run.start") { yield* mapDispatchError(command)( - startNextQueuedRun( - command.threadId, - command.type === "queued-run.start" ? { queuedRunId: command.runId } : undefined, - ), + startNextQueuedRun(command.threadId, { + // Coder: without the managed-policy runtime-mode branch, only a user starts these runs. + userStarted: true, + ...(command.type === "queued-run.start" ? { queuedRunId: command.runId } : {}), + }), ); } - if (command.type === "notification.delivery.accept") { + if ( + command.type === "notification.delivery.accept" || + // A steer restart that settled a wake can reserve the next one for its siblings. + ((command.type === "message.dispatch" || + command.type === "queued-message.promote-to-steer") && + committed.storedEvents.some( + ({ event }) => + event.type === "subagent.updated" && + event.payload.completionDelivery?.state === "claimed", + )) + ) { yield* mapDispatchError(command)(offerDelegatedCompletionDeliveries(command.threadId)); } if ( @@ -14003,6 +14588,15 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio // Coder: orchestration metrics are not carried, so dispatch has no `withCommandMetrics` wrapper. }); + // Follow-up intake and Stop serialize on the parent before taking the child lock. + const withSubagentParentLock = (threadId: ThreadId, dispatch: Effect.Effect) => + appOwnedSubagentParentThreadId(threadId).pipe( + Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })), + Effect.flatMap((parentThreadId) => + parentThreadId === undefined ? dispatch : threadDispatch.withLock(parentThreadId, dispatch), + ), + ); + // dispatchWithReceiptEffect owns the dispatch span; this wrapper selects the thread locks. const dispatchWithReceipt = Effect.fnUntraced(function* (command: OrchestrationV2ServerCommand) { const dispatch = threadDispatch.withLock( @@ -14035,21 +14629,19 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ), ), ); + // Taken before the lineage read, so a message whose read yields keeps its + // place ahead of later messages to the same thread. + if (command.type === "message.dispatch") + return yield* messageIntake.withLock( + command.threadId, + mayStartSubagentFollowup(command) + ? withSubagentParentLock(command.threadId, dispatch) + : dispatch, + ); // Answering a message-mode question can also start a follow-up. - if ( - command.type !== "runtime-request.respond" && - (command.type !== "message.dispatch" || !mayStartSubagentFollowup(command)) - ) - return yield* dispatch; - const parentThreadId = yield* appOwnedSubagentParentThreadId(command.threadId).pipe( - Effect.mapError( - (cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }), - ), - ); - // Follow-up intake and Stop serialize on the parent before taking the child lock. - return yield* parentThreadId === undefined - ? dispatch - : threadDispatch.withLock(parentThreadId, dispatch); + return yield* command.type === "runtime-request.respond" + ? withSubagentParentLock(command.threadId, dispatch) + : dispatch; }); const handleRunUpdate = Effect.fn("Orchestrator.handleRunUpdate")( @@ -14141,6 +14733,63 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio Effect.forkScoped, ); + yield* eventSink + .stream({ afterSequence: terminalEventsAfterSequence, eventType: "provider-turn.updated" }) + .pipe( + Stream.filter( + (stored) => + stored.event.type === "provider-turn.updated" && + stored.event.payload.status === "running", + ), + Stream.runForEach( + Effect.fn("Orchestrator.deliverDeferredSteers")( + function* (stored) { + if (stored.event.type !== "provider-turn.updated") return; + const threadId = stored.event.threadId; + const turn = stored.event.payload; + const projection = yield* projectionStore.getThreadRecords(threadId, ["runs"]); + const run = projection.runs.find( + (candidate) => + candidate.activeAttemptId === turn.runAttemptId && candidate.status === "running", + ); + if (run === undefined) return; + const pending = projection.runs + .filter( + (queued) => queued.status === "queued" && queued.steerWhenReadyRunId === run.id, + ) + .toSorted( + (left, right) => + (left.queuePosition ?? left.ordinal) - (right.queuePosition ?? right.ordinal), + ); + for (const queued of pending) { + yield* dispatchWithReceiptEffect({ + type: "queued-message.promote-to-steer", + commandId: CommandId.make( + `command:deferred-steer:${queued.id}:${run.id}:${turn.runAttemptId}`, + ), + threadId, + queuedRunId: queued.id, + targetRunId: run.id, + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Failed to deliver a deferred steer", { + threadId, + runId: queued.id, + cause, + }), + ), + ); + } + }, + (effect, stored) => threadDispatch.withLock(stored.event.threadId, effect), + Effect.catchCause((cause) => + Effect.logWarning("Failed to deliver deferred steering", { cause }), + ), + ), + ), + Effect.forkScoped, + ); + // A child whose run ends while its background tasks or its own subagents still run has no // result yet. When that work settles there is no run update to finalize it, so these events // do it. A background roster counts once it stops holding after it was seen holding. @@ -14287,7 +14936,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ); if (taskId !== undefined) { const runs = child.runs.filter((run) => run.delegatedTaskId === taskId); - const run = runs.toSorted((a, b) => b.ordinal - a.ordinal)[0]; + const run = delegatedTaskResultRun(runs, taskId); if (run === undefined || delegatedTaskTerminalStatus(run.status) === null) return true; return yield* childAwaitsRestartContinuation(runs, run); } @@ -14324,6 +14973,81 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ), ); + const cancelDelegatedTask: OrchestratorV2Shape["cancelDelegatedTask"] = Effect.fn( + "Orchestrator.cancelDelegatedTask", + )(function* (input) { + const { command } = input; + return yield* threadDispatch.withLock( + input.parentThreadId, + Effect.gen(function* () { + const { subagents } = yield* projectionStore + .getThreadRecords(input.parentThreadId, ["subagents"]) + .pipe(mapDispatchError(command)); + const task = subagents.find( + (candidate) => + candidate.id === input.taskId && + candidate.origin === "app_owned" && + candidate.childThreadId === command.threadId && + isOrchestrationV2WorkActive(candidate.status), + ); + const child = yield* projectionStore + .getThreadRecords( + command.threadId, + ["runs", "messages", "subagents", "providerThreads"], + { messageRoles: ["user"] }, + ) + .pipe(mapDispatchError(command)); + const followUpRun = child.runs + .filter((run) => run.delegatedTaskId === input.taskId) + .toSorted((a, b) => b.ordinal - a.ordinal)[0]; + const unsettled = + followUpRun === undefined + ? child.thread.forkedFrom?.type === "node" && + child.thread.forkedFrom.nodeId === input.taskId && + progressAfterTurn(child).state !== "result_available" + : delegatedTaskTerminalStatus(runOutcome(followUpRun)) === null; + yield* threadDispatch.withLock( + command.threadId, + Effect.gen(function* () { + // Hold intake closed until descendants stop and this thread's Stop commits. + yield* input.stopDescendants((command) => + commandThreadId(command) === input.command.threadId + ? dispatchWithReceiptEffect(command) + : dispatchWithReceipt(command), + ); + const child = yield* projectionStore + .getThreadRecords( + command.threadId, + ["runs", "messages", "subagents", "providerThreads"], + { messageRoles: ["user"] }, + ) + .pipe(mapDispatchError(command)); + const followUpRun = child.runs + .filter((run) => run.delegatedTaskId === input.taskId) + .toSorted((a, b) => b.ordinal - a.ordinal)[0]; + const monitorRunIds = new Set( + child.messages + .filter((message) => message.notification?.source.kind === "monitor") + .map((message) => message.runId), + ); + const activeRun = child.runs + .filter((run) => isLiveRun(run) && !monitorRunIds.has(run.id)) + .toSorted((a, b) => b.ordinal - a.ordinal)[0]; + const cancelledRun = followUpRun ?? activeRun ?? progressAfterTurn(child).resultRun; + const now = yield* DateTime.now; + const cancelledTask = + task !== undefined && unsettled && cancelledRun !== undefined + ? { ...task, cancelledRunId: cancelledRun.id, updatedAt: now } + : undefined; + // The marker, Stop receipt and interrupt outbox commit together under both locks. + yield* dispatchWithReceiptEffect(command, [], cancelledTask); + }), + ); + yield* finalizeAppOwnedSubagent(command.threadId).pipe(mapDispatchError(command)); + }), + ); + }); + const shellProjectionError = (cause: unknown) => new OrchestratorProjectionError({ threadId: ThreadId.make("thread:shell"), cause }); @@ -14331,6 +15055,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio resumeQueuedRuns, recoverDelegatedTasks, recoverDelegatedTask, + cancelDelegatedTask, delegatedTaskResultPending, restartContinuationPending: Effect.fn("OrchestratorV2.restartContinuationPending")( function (run) { diff --git a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts index 825bc5d7ce53..86a4c6c77375 100644 --- a/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionSettlement.test.ts @@ -478,3 +478,45 @@ it.effect("sql: an unsettled-only shell read skips settled threads", () => assert.equal(all.threads.length, 4); }).pipe(Effect.provide(layerSql)), ); + +it.effect("sql: a subagent thread becomes a candidate once its parent is done", () => + Effect.gen(function* () { + const store = yield* ProjectionStore.ProjectionStoreV2; + const parent = yield* createThread("subagent-parent"); + const child = yield* createThread("subagent-child", { + lineage: { parentThreadId: parent, relationshipToParent: "subagent", rootThreadId: parent }, + }); + const ids = () => + store.getSettlementCandidates().pipe(Effect.map((rows) => rows.map((row) => row.id))); + assert.deepEqual(yield* ids(), [parent]); + + const parentThread = yield* store.getThread(parent); + for (const [type, overrides] of [ + ["thread.settled", { settledOverride: "settled", settledAt: now }], + ["thread.archived", { archivedAt: now }], + ["thread.deleted", { deletedAt: now }], + ] satisfies ReadonlyArray< + readonly [ + "thread.settled" | "thread.archived" | "thread.deleted", + Partial, + ] + >) { + yield* store.apply({ + id: EventId.make(`event:settlement:subagent-parent:${type}`), + type, + threadId: parent, + occurredAt: now, + payload: { ...parentThread, ...overrides, updatedAt: now }, + }); + assert.deepEqual(yield* ids(), [child]); + } + const orphan = yield* createThread("orphan", { + lineage: { + parentThreadId: ThreadId.make("missing-parent"), + relationshipToParent: "subagent", + rootThreadId: parent, + }, + }); + assert.deepEqual(new Set(yield* ids()), new Set([child, orphan])); + }).pipe(Effect.provide(layerSql)), +); diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index 8637074cfa52..9d37fc0d7ad1 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -1,3 +1,5 @@ +import * as NodeSqlite from "node:sqlite"; + import { assert, it, vi } from "@effect/vitest"; import { EnvironmentId, @@ -57,6 +59,17 @@ import { limitRecoveryCommand } from "./UsageLimitRecoveryWorker.ts"; import { layerReaderDatabase, traceSqlRouting } from "../persistence/testkit/SqliteRouting.ts"; +declare module "node:sqlite" { + interface StatementSync { + // Node 26 has this; @types/node 24 does not declare it yet. + stat(counter: "vmStep"): number; + } +} + +// Coder: the fork runs on Node 24 (`engines.node`), whose `StatementSync` has no `stat`, so the +// VM-step comparisons run only where Node 26 provides it. +const hasVmStepCounter = typeof NodeSqlite.StatementSync.prototype.stat === "function"; + const layerTest = Layer.mergeAll( ProjectionStore.layer.pipe(Layer.provideMerge(SqlitePersistence.layerMemory)), SqlitePersistence.layerMemory, @@ -685,6 +698,53 @@ it.effect("idle compaction filters keep the latest run, live work and the newest records.runs.map((run) => run.id), ["run:idle-compaction:rolled-back", "run:idle-compaction:4"], ); + const { statements, tracer } = traceSqlStatements(); + const statusOnly = yield* store + .getThreadRecords(threadId, ["runs"], { + runStatuses: ["preparing", "starting", "running", "rolled_back"], + }) + .pipe(Effect.withTracer(tracer)); + assert.deepEqual( + statusOnly.runs.map((run) => run.id), + ["run:idle-compaction:rolled-back"], + ); + const runQuery = statements.find((statement) => + statement.includes("FROM orchestration_v2_projection_runs"), + ); + assert.isDefined(runQuery); + const sql = yield* SqlClient.SqlClient; + // Read only the matching runs, not every run in ordinal order. + const runPlan = (yield* sql.unsafe<{ readonly detail: string }>( + `EXPLAIN QUERY PLAN ${runQuery}`, + )) + .map((row) => row.detail) + .join("\n"); + assert.notInclude(runPlan, "orchestration_v2_projection_runs_thread_ordinal_idx"); + assert.match(runPlan, /SEARCH orchestration_v2_projection_runs USING INDEX \w*status\w*/); + // An id filter reads by primary key, not every row of the thread. + const byId = traceSqlStatements(); + const found = yield* store + .getThreadRecords(threadId, ["runs", "nodes", "subagents", "messages"], { + runIds: [RunId.make("run:idle-compaction:4")], + nodeIds: [nodeId], + subagentIds: [nodeId], + messageIds: [MessageId.make("message:idle-compaction:4")], + }) + .pipe(Effect.withTracer(byId.tracer)); + assert.deepEqual( + found.runs.map((run) => run.id), + ["run:idle-compaction:4"], + ); + for (const table of ["runs", "nodes", "subagents", "messages"]) { + const query = byId.statements.find((statement) => + statement.includes(`FROM orchestration_v2_projection_${table}`), + ); + assert.isDefined(query); + const plan = (yield* sql.unsafe<{ readonly detail: string }>(`EXPLAIN QUERY PLAN ${query}`)) + .map((row) => row.detail) + .join("\n"); + assert.include(plan, `USING INDEX sqlite_autoindex_orchestration_v2_projection_${table}_1`); + } assert.deepEqual( records.providerTurns.map((turn) => turn.id), ["turn:idle-newest"], @@ -755,6 +815,23 @@ it.effect("record reads filter notification sources without dropping matching me matching.messages.map((message) => [message.id, message.text]), [["monitor", "Complete body for monitor"]], ); + const { statements, tracer } = traceSqlStatements(); + const userMessages = yield* store + .getThreadRecords(threadId, ["messages"], { messageRoles: ["user"] }) + .pipe(Effect.withTracer(tracer)); + assert.lengthOf(userMessages.messages, 3); + const ordinalLookup = statements.find((statement) => statement.includes("MIN(ordinal)")); + assert.isDefined(ordinalLookup); + const sql = yield* SqlClient.SqlClient; + const plan = (yield* sql.unsafe<{ readonly detail: string }>( + `EXPLAIN QUERY PLAN ${ordinalLookup}`, + [threadId, "[]"], + )).map((row) => row.detail); + // User-only reads must stay on the partial index instead of scanning every turn item. + assert.isTrue( + plan.some((detail) => detail.includes("turn_items_user_message_idx")), + plan.join("\n"), + ); assert.lengthOf((yield* store.getThreadRecords(threadId, ["messages"])).messages, 4); assert.lengthOf( (yield* store.getThreadRecords(threadId, ["messages"], { @@ -771,6 +848,105 @@ it.effect("record reads filter notification sources without dropping matching me }).pipe(Effect.provide(layerTest)), ); +it.effect("shells flag a latest run triggered by a provider background wake", () => + Effect.gen(function* () { + const store = yield* ProjectionStore.ProjectionStoreV2; + const threadId = yield* addRolledBackRecoveryCandidate("background-wake"); + const initial = yield* store.getThreadProjection(threadId); + const at = initial.updatedAt; + let ordinal = 1; + for (const [id, source, outcome, expected] of [ + ["background-task", { kind: "background_task" }, "updated", true], + ["monitor", { kind: "monitor" }, "updated", true], + ["command", { kind: "command" }, "updated", true], + ["command-finished", { kind: "command" }, "completed", false], + ["monitor-stopped", { kind: "monitor" }, "cancelled", false], + ["subagent", { kind: "subagent" }, "updated", false], + ["delegated", { kind: "delegated_task", taskIds: [] }, "updated", false], + ["steered", { kind: "background_task" }, "updated", false], + ] as const) { + const messageId = MessageId.make(`message:background-wake:${id}`); + yield* store.apply({ + id: EventId.make(`event:background-wake:${id}:message`), + type: "message.updated", + threadId, + occurredAt: at, + payload: { + id: messageId, + threadId, + runId: null, + nodeId: null, + createdBy: "agent", + creationSource: "server", + role: "user", + text: id, + attachments: [], + streaming: false, + createdAt: at, + updatedAt: at, + notification: { source, outcome, summary: "Background activity updated" }, + ...(id === "delegated" + ? { + delegatedCompletion: { + parentRunId: initial.runs[0]!.id, + generation: 1, + taskIds: [], + }, + } + : {}), + }, + }); + ordinal += 1; + yield* store.apply({ + id: EventId.make(`event:background-wake:${id}:run`), + type: "run.created", + threadId, + occurredAt: at, + payload: { + ...initial.runs[0]!, + id: RunId.make(`run:background-wake:${id}`), + ordinal, + userMessageId: messageId, + status: "completed", + completedAt: at, + }, + }); + if (id === "steered") { + yield* store.apply({ + id: EventId.make("event:background-wake:steered:follow-up"), + type: "message.updated", + threadId, + occurredAt: at, + payload: { + id: MessageId.make("message:background-wake:steered:follow-up"), + threadId, + runId: RunId.make("run:background-wake:steered"), + nodeId: null, + createdBy: "user", + creationSource: "web", + role: "user", + text: "also fix X", + attachments: [], + streaming: false, + createdAt: at, + updatedAt: at, + }, + }); + } + const shells = [ + yield* store.getThreadShell(threadId), + ProjectionStore.threadShellFromProjection(yield* store.getThreadProjection(threadId)), + ]; + assert.deepEqual( + shells.map((shell) => shell?.latestRunBackgroundWake), + [expected, expected], + id, + ); + yield* assertShellIndexCoversShells(); + } + }).pipe(Effect.provide(layerTest)), +); + it.effect("SQL queue readiness skips held runs without bypassing active work", () => Effect.gen(function* () { const store = yield* ProjectionStore.ProjectionStoreV2; @@ -1390,7 +1566,9 @@ const commandReadsSkipSettledLeafNodes = Effect.gen(function* () { ["settled-reasoning", settledRun, "reasoning", "cancelled"], ["settled-subagent", settledRun, "subagent", "completed"], ["settled-run-live-tool", settledRun, "tool_call", "running"], + ["settled-run-todo", settledRun, "todo_list", "running"], ["active-run-done-tool", activeRun, "tool_call", "completed"], + ["active-run-todo", activeRun, "todo_list", "running"], ] as const; for (const [name, run, kind, status] of nodes) { const id = NodeId.make(`node:settled-leaf-nodes:${name}`); @@ -1420,7 +1598,49 @@ const commandReadsSkipSettledLeafNodes = Effect.gen(function* () { completedAt: status === "running" ? null : now, }, }); + if (kind === "todo_list") { + yield* store.apply({ + id: EventId.make(`event:settled-leaf-nodes:${name}:plan`), + type: "plan.updated", + threadId, + runId: run.id, + nodeId: id, + driver, + occurredAt: now, + payload: { + id: PlanId.make(`plan:settled-leaf-nodes:${name}`), + threadId, + runId: run.id, + nodeId: id, + status: "active", + kind: "todo_list", + steps: [], + }, + }); + } } + // No row or node of this run is in the window. + const settledCheckpointId = CheckpointId.make("checkpoint:settled-leaf-nodes:earlier-run"); + yield* store.apply({ + id: EventId.make("event:settled-leaf-nodes:earlier-run-checkpoint"), + type: "checkpoint.captured", + threadId, + occurredAt: now, + payload: { + id: settledCheckpointId, + threadId, + scopeId: CheckpointScopeId.make("scope:settled-leaf-nodes:root"), + runId: RunId.make("run:settled-leaf-nodes:earlier"), + nodeId: NodeId.make("node:settled-leaf-nodes:earlier-root"), + parentCheckpointId: null, + ordinalWithinScope: 1, + appRunOrdinal: 1, + ref: CheckpointRef.make("refs/t3/settled-leaf-nodes/1"), + status: "ready", + files: [], + capturedAt: now, + }, + }); const nodeNames = (records: { readonly nodes: ReadonlyArray<{ readonly id: string }> }) => records.nodes .map((node) => node.id.replace("node:settled-leaf-nodes:", "")) @@ -1429,12 +1649,32 @@ const commandReadsSkipSettledLeafNodes = Effect.gen(function* () { assert.deepEqual( nodeNames(yield* store.getThreadRecords(threadId, ["nodes"], { omitSettledLeafNodes: true })), - ["active-run-done-tool", "settled-root", "settled-run-live-tool", "settled-subagent"], + [ + "active-run-done-tool", + "active-run-todo", + "settled-root", + "settled-run-live-tool", + "settled-run-todo", + "settled-subagent", + ], ); assert.deepEqual( nodeNames(yield* store.getThreadRecords(threadId, ["nodes"])), nodes.map(([name]) => name).toSorted(), ); + // A todo list stays running after its run ends. Counted as live work, every old run of a + // long thread would ride along with each window, with its nodes and subagents. + const liveWindow = (yield* store.getThreadSnapshotWindow(threadId, { rowLimit: 1 })).projection; + assert.deepEqual(nodeNames(liveWindow), ["active-run-todo", "settled-run-live-tool"]); + assert.deepEqual( + liveWindow.plans.map((plan) => plan.id), + [PlanId.make("plan:settled-leaf-nodes:active-run-todo")], + ); + // Clients show the diff of the last finished turn from this record. + assert.deepEqual( + liveWindow.checkpoints.map((checkpoint) => checkpoint.id), + [settledCheckpointId], + ); const subagents = [ ["delivered", settledRun, "completed", "delivered"], @@ -2579,18 +2819,6 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { "command reads skip finished leaf nodes of finished runs", () => commandReadsSkipSettledLeafNodes, ); - it.effect("limits turn-start history to the requested runs, including an empty selection", () => - Effect.gen(function* () { - const store = yield* ProjectionStore.ProjectionStoreV2; - const threadId = yield* addRolledBackRecoveryCandidate("selected-turn-start-history"); - const runId = (yield* store.getThreadProjection(threadId)).runs[0]!.id; - const history = yield* store.getTurnStartHistory(threadId); - assert.isNotEmpty(history); - assert.deepEqual(yield* store.getTurnStartHistory(threadId, [runId]), history); - assert.deepEqual(yield* store.getTurnStartHistory(threadId, []), []); - assert.deepEqual(yield* store.getTurnStartHistory(threadId, [RunId.make("run:other")]), []); - }), - ); it.effect("preserves stored provider usage when a terminal update omits it", () => Effect.gen(function* () { const projectionStore = yield* ProjectionStore.ProjectionStoreV2; @@ -2705,6 +2933,16 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { const now = yield* DateTime.now; const nowIso = DateTime.formatIso(now); const threadId = ThreadId.make("thread:bounded-sql-history"); + // The raw SQL edits below stand in for committed writes, which always append a thread event. + let revision = 0; + const historyPage = (...args: Parameters) => + sql`INSERT INTO orchestration_events ( + event_id, aggregate_kind, stream_id, stream_version, event_type, + occurred_at, actor_kind, payload_json, metadata_json, application_event_version + ) VALUES ( + ${`event:bounded-sql-history:revision:${++revision}`}, 'thread', ${threadId}, ${revision}, + 'turn-item.updated', ${nowIso}, 'user', '{}', '{}', 2 + )`.pipe(Effect.andThen(projectionStore.getThreadHistoryPage(...args))); yield* projectionStore.apply({ id: EventId.make("event:bounded-sql-history:thread"), type: "thread.created", @@ -2928,7 +3166,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { sourceItemId: TurnItemId.make("turn-item:bounded-sql-history:71"), position: 70, }); - const bytePage = yield* projectionStore.getThreadHistoryPage(threadId, byteCursor); + const bytePage = yield* historyPage(threadId, byteCursor); assert.isAbove(bytePage.items.length, 0); assert.isBelow(bytePage.items.length, 70); assert.isAtMost( @@ -2939,7 +3177,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { let byteNext = bytePage.nextCursor; for (let pages = 0; byteNext !== null; pages += 1) { assert.isBelow(pages, 70); - const page = yield* projectionStore.getThreadHistoryPage(threadId, byteNext); + const page = yield* historyPage(threadId, byteNext); assert.isAbove(page.items.length, 0); byteIds.unshift(...page.items.map((row) => String(row.sourceItemId))); byteNext = page.nextCursor; @@ -2964,7 +3202,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { parent_item_id, ordinal, type, status, updated_at, payload_json ) VALUES ( ${`turn-item:bounded-sql-history:ordinal-${suffix}`}, ${itemThreadId}, NULL, NULL, NULL, - NULL, NULL, ${ordinal}, 'assistant_message', 'completed', ${nowIso}, + NULL, NULL, ${ordinal}, 'user_message', 'completed', ${nowIso}, ${encodeUnknownJsonString({ messageId })} ) `; @@ -3045,7 +3283,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { '$.attachments', json('[]')) WHERE thread_id = ${threadId} AND ordinal BETWEEN 600 AND 624 `; - const targetedPage = yield* projectionStore.getThreadHistoryPage( + const targetedPage = yield* historyPage( threadId, encodeThreadHistoryCursor({ snapshotSequence: 0, @@ -3075,7 +3313,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { '$.type', CASE ordinal WHEN 500 THEN 'run_interrupt_request' ELSE 'run_interrupt_result' END) WHERE thread_id = ${threadId} AND ordinal IN (500, 580) `; - const interruptPage = yield* projectionStore.getThreadHistoryPage( + const interruptPage = yield* historyPage( threadId, encodeThreadHistoryCursor({ snapshotSequence: 0, @@ -3222,7 +3460,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { let workCursor: string | null = anchor.item.deferredWork!.cursor; for (let pages = 0; workCursor !== null; pages += 1) { assert.isBelow(pages, 20); - const page: OrchestrationV2ThreadHistoryPage = yield* projectionStore.getThreadHistoryPage( + const page: OrchestrationV2ThreadHistoryPage = yield* historyPage( threadId, workCursor, anchor.sourceItemId, @@ -3251,7 +3489,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { window.projection.visibleTurnItems.find((row) => row.item.deferredWork)?.item.ordinal, 3, ); - const page = yield* projectionStore.getThreadHistoryPage( + const page = yield* historyPage( threadId, encodeThreadHistoryCursor({ snapshotSequence: 0, @@ -3308,14 +3546,13 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { const loaded = new Set(); for (let pages = 0; cursor !== null; pages += 1) { assert.isBelow(pages, 20); - const page: OrchestrationV2ThreadHistoryPage = - yield* projectionStore.getThreadHistoryPage( - threadId, - cursor, - oldWork!.sourceItemId, - false, - "work", - ); + const page: OrchestrationV2ThreadHistoryPage = yield* historyPage( + threadId, + cursor, + oldWork!.sourceItemId, + false, + "work", + ); for (const row of page.items) loaded.add(row.sourceItemId); cursor = page.nextCursor; } @@ -3356,6 +3593,137 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { yield* sql`UPDATE orchestration_v2_projection_turn_items SET ordinal = 999, payload_json = json_set(payload_json, '$.ordinal', 999) WHERE turn_item_id = 'turn-item:bounded-sql-history:999'`; + // Work folded below the newest 502 visible rows still counts its whole group, so the + // 100 commands after the messages stay hidden in the 398-item group. + yield* sql`UPDATE orchestration_v2_projection_turn_items SET type = 'assistant_message', + payload_json = json_set(payload_json, '$.type', 'assistant_message', '$.text', 'Working', + '$.messageId', turn_item_id, '$.streaming', json('false')) + WHERE thread_id = ${threadId} AND ordinal BETWEEN 300 AND 899`; + const straddledWindow = yield* projectionStore.getThreadSnapshotWindow(threadId, { + rowLimit: sqlPageLimit, + userTurnLimit: 10, + deferWork: true, + }); + const straddledOrdinals = [ + ...Array.from({ length: sqlPageLimit }, (_, index) => 900 - sqlPageLimit + index), + 1000, + ]; + assert.deepEqual( + straddledWindow.projection.turnItems.map((item) => item.ordinal), + straddledOrdinals, + ); + // A run that starts with work (a wake) has no segment start, so the group begins at its first item. + const moveFirstItemToRun = (runId: string) => sql` + UPDATE orchestration_v2_projection_turn_items SET run_id = ${runId}, + payload_json = json_set(payload_json, '$.runId', ${runId}) + WHERE thread_id = ${threadId} AND ordinal = 1`; + yield* moveFirstItemToRun("run:bounded-sql-history:2"); + const wakeWindow = yield* projectionStore.getThreadSnapshotWindow(threadId, { + rowLimit: sqlPageLimit, + userTurnLimit: 10, + deferWork: true, + }); + assert.deepEqual( + wakeWindow.projection.turnItems.map((item) => item.ordinal), + straddledOrdinals, + ); + yield* moveFirstItemToRun(sharedRunId); + // A failed error below the tail still blocks folding for its run. Only the SQL columns + // change; the row sits below the window, so its payload is never decoded. + // A steer at 100 starts a later segment, so the error sits below the scan start too. + const setItem = (ordinal: number, type: string, status: string) => sql` + UPDATE orchestration_v2_projection_turn_items SET type = ${type}, status = ${status} + WHERE thread_id = ${threadId} AND ordinal = ${ordinal}`; + yield* setItem(100, "user_message", "completed"); + yield* setItem(2, "error", "failed"); + const blockedWindow = yield* projectionStore.getThreadSnapshotWindow(threadId, { + rowLimit: sqlPageLimit, + userTurnLimit: 10, + deferWork: true, + }); + assert.isFalse( + blockedWindow.projection.visibleTurnItems.some((row) => row.item.deferredWork), + ); + assert.deepEqual( + blockedWindow.projection.turnItems.map((item) => item.ordinal), + Array.from({ length: sqlPageLimit }, (_, index) => 1001 - sqlPageLimit + index), + ); + yield* setItem(2, "command_execution", "completed"); + yield* setItem(100, "command_execution", "completed"); + yield* sql`UPDATE orchestration_v2_projection_turn_items SET type = 'command_execution', + payload_json = json_set(payload_json, '$.type', 'command_execution') + WHERE thread_id = ${threadId} AND ordinal BETWEEN 300 AND 899`; + + // Runless rows below a run that starts after the newest 502 visible rows stay in the window. + yield* sql`UPDATE orchestration_v2_projection_turn_items + SET run_id = CASE WHEN ordinal > 990 THEN 'run:bounded-sql-history:991' END, + type = CASE WHEN ordinal = 991 THEN 'user_message' ELSE 'assistant_message' END, + payload_json = CASE WHEN ordinal = 991 + THEN json_set(payload_json, '$.type', 'user_message', '$.runId', 'run:bounded-sql-history:991', + '$.createdBy', 'user', '$.creationSource', 'web', '$.inputIntent', 'turn_start', + '$.messageId', turn_item_id, '$.text', 'Turn 991', '$.attachments', json('[]')) + ELSE json_set(payload_json, '$.type', 'assistant_message', + '$.runId', CASE WHEN ordinal > 990 THEN 'run:bounded-sql-history:991' END, + '$.text', 'Working', '$.messageId', turn_item_id, '$.streaming', json('false')) + END + WHERE thread_id = ${threadId} AND ordinal > 1`; + const windowStatements: Array]> = []; + const runlessWindow = yield* projectionStore + .getThreadSnapshotWindow(threadId, { + rowLimit: sqlPageLimit, + userTurnLimit: 10, + deferWork: true, + }) + .pipe( + Effect.provideService(Statement.CurrentTransformer, (statement) => { + windowStatements.push(statement.compile()); + return Effect.succeed(statement); + }), + ); + // The window read must not grow with history below the newest visible rows. VM steps are + // deterministic; reading the whole history here cost 65% more steps. + const copyPath = "file:/thread-window-steps?vfs=memdb"; + if (hasVmStepCounter) { + using copy = new NodeSqlite.DatabaseSync(copyPath); + yield* sql`VACUUM INTO ${copyPath}`; + assert.isNotEmpty(windowStatements); + const windowSteps = () => + windowStatements.reduce((steps, [query, params]) => { + const statement = copy.prepare(query); + statement.all(...(params as ReadonlyArray)); + return steps + statement.stat("vmStep"); + }, 0); + const fullSteps = windowSteps(); + assert.isAbove(fullSteps, 0); + copy.exec( + "DELETE FROM orchestration_v2_projection_turn_items WHERE ordinal BETWEEN 2 AND 498", + ); + assert.isAtMost(fullSteps, windowSteps() * 1.1); + } + assert.deepEqual( + runlessWindow.projection.turnItems.map((item) => item.ordinal), + Array.from({ length: THREAD_HISTORY_TURN_SNAPSHOT_ROW_LIMIT }, (_, index) => 499 + index), + ); + const runlessPage = buildBoundedThreadProjection({ + projection: runlessWindow.projection, + snapshotSequence: runlessWindow.snapshotSequence, + }); + assert.isTrue(runlessPage.hasMoreHistory); + assert.isNotNull(runlessPage.historyCursor); + // Limits SQLite reads differently from `tailRows` skip the tail bound and match main. + const windowOrdinalCount = (rowLimit: number) => + projectionStore + .getThreadSnapshotWindow(threadId, { rowLimit, userTurnLimit: 10, deferWork: true }) + .pipe(Effect.map((window) => window.projection.turnItems.length)); + assert.strictEqual(yield* windowOrdinalCount(502.5), THREAD_HISTORY_TURN_SNAPSHOT_ROW_LIMIT); + yield* sql`UPDATE orchestration_v2_projection_turn_items + SET payload_json = json_set(payload_json, '$.inputIntent', 'steer') + WHERE thread_id = ${threadId} AND ordinal = 991`; + assert.strictEqual(yield* windowOrdinalCount(-1), 1000); + yield* sql`UPDATE orchestration_v2_projection_turn_items SET run_id = ${sharedRunId}, + type = 'command_execution', + payload_json = json_set(payload_json, '$.type', 'command_execution', '$.runId', ${sharedRunId}) + WHERE thread_id = ${threadId} AND ordinal > 1`; yield* sql` UPDATE orchestration_v2_projection_turn_items SET type = 'command_execution', payload_json = json_set(payload_json, '$.type', 'command_execution') @@ -3372,6 +3740,92 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { SET type = 'user_message', payload_json = json_set(payload_json, '$.type', 'user_message') WHERE thread_id = ${threadId} AND ordinal = 1 `; + // A running run folds nothing, so its newest rows are the window. Reading them must not + // walk the rest of the turn: 999 commands here, tens of thousands after a day of work. + const setSharedRunStatus = (status: "running" | "completed") => sql` + UPDATE orchestration_v2_projection_runs + SET status = ${status}, payload_json = json_set(payload_json, '$.status', ${status}) + WHERE run_id = ${sharedRunId}`; + yield* setSharedRunStatus("running"); + // Another thread's rows that carry this run's id must not count toward the tail. + const foreignThreadId = "thread:bounded-sql-history:foreign"; + yield* sql` + INSERT INTO orchestration_v2_projection_turn_items ( + turn_item_id, thread_id, run_id, node_id, provider_thread_id, provider_turn_id, + parent_item_id, ordinal, type, status, updated_at, payload_json + ) + SELECT turn_item_id || ':foreign', ${foreignThreadId}, run_id, NULL, NULL, NULL, + NULL, ordinal + 5000, type, status, updated_at, payload_json + FROM orchestration_v2_projection_turn_items + WHERE thread_id = ${threadId} AND ordinal > 400`; + const liveStatements: Array]> = []; + const liveWindow = yield* projectionStore + .getThreadSnapshotWindow(threadId, { + rowLimit: sqlPageLimit, + userTurnLimit: 10, + deferWork: true, + }) + .pipe( + Effect.provideService(Statement.CurrentTransformer, (statement) => { + liveStatements.push(statement.compile()); + return Effect.succeed(statement); + }), + ); + assert.deepEqual( + liveWindow.projection.turnItems.map((item) => item.ordinal), + Array.from({ length: sqlPageLimit }, (_, index) => 1001 - sqlPageLimit + index), + ); + if (hasVmStepCounter) { + const livePath = "file:/thread-window-live-steps?vfs=memdb"; + using copy = new NodeSqlite.DatabaseSync(livePath); + yield* sql`VACUUM INTO ${livePath}`; + const liveSteps = (statements = liveStatements) => + statements.reduce((steps, [query, params]) => { + const statement = copy.prepare(query); + statement.all(...(params as ReadonlyArray)); + return steps + statement.stat("vmStep"); + }, 0); + const deleteOrdinals = (from: number, to: number) => + copy.exec( + `DELETE FROM orchestration_v2_projection_turn_items WHERE ordinal BETWEEN ${from} AND ${to}`, + ); + // Nothing below the newest 502 rows is read to find the tail. + const fullSteps = liveSteps(); + assert.isAbove(fullSteps, 0); + deleteOrdinals(2, 498); + assert.isAtMost(fullSteps, liveSteps() * 1.1); + // The rows themselves are read only from the newest page, not from all 502. + const rowStatements = liveStatements.filter(([query]) => query.includes("work_segments")); + assert.lengthOf(rowStatements, 1); + const pageSteps = liveSteps(rowStatements); + deleteOrdinals(499, 900); + assert.isAtMost(pageSteps, liveSteps(rowStatements) * 1.1); + // The tail search reads conversation rows from their own index, not past the work rows. + const tailSearch = liveStatements.find(([query]) => query.includes("visible_tail")); + assert.isDefined(tailSearch); + const plan = copy + .prepare(`EXPLAIN QUERY PLAN ${tailSearch![0]}`) + .all(...(tailSearch![1] as ReadonlyArray)) + .map((row) => String(row.detail)); + assert.isTrue( + plan.some((detail) => detail.includes("turn_items_conversation_idx")), + plan.join("\n"), + ); + } + const olderLivePage = yield* projectionStore.getThreadSnapshotWindow(threadId, { + rowLimit: sqlPageLimit, + userTurnLimit: 10, + deferWork: true, + anchorItemId: TurnItemId.make("turn-item:bounded-sql-history:600"), + }); + // Rows above the anchor must not set the tail, or the page would start above its end. + assert.deepEqual( + olderLivePage.projection.turnItems.map((item) => item.ordinal), + Array.from({ length: sqlPageLimit }, (_, index) => 601 - sqlPageLimit + index), + ); + yield* sql`DELETE FROM orchestration_v2_projection_turn_items + WHERE thread_id = ${foreignThreadId}`; + yield* setSharedRunStatus("completed"); yield* sql` UPDATE orchestration_v2_projection_turn_items SET run_id = 'run:bounded-sql-history:' || ordinal, @@ -3517,7 +3971,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { activeAttemptId: null, status: "cancelled", requestedAt: nowIso, - startedAt: nowIso, + startedAt: null, completedAt: nowIso, checkpointId: null, contextHandoffId: null, @@ -3646,6 +4100,48 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { ), ); + // A steer cancels the queued run before it starts while the steered run keeps writing, so + // the queued message leaves the window but its run is still the shell's latest run. + const steeredItemId = "turn-item:bounded-sql-history:steered-output"; + const steeredRunId = RunId.make("run:bounded-sql-history:1000"); + yield* sql` + INSERT INTO orchestration_v2_projection_turn_items ( + turn_item_id, thread_id, run_id, node_id, provider_thread_id, provider_turn_id, + parent_item_id, ordinal, type, status, updated_at, payload_json + ) VALUES ( + ${steeredItemId}, ${threadId}, ${steeredRunId}, NULL, NULL, NULL, NULL, 1203, + 'command_execution', 'completed', ${nowIso}, + ${encodeUnknownJsonString({ + id: steeredItemId, + threadId, + runId: steeredRunId, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 1203, + status: "completed", + title: "steered output", + input: "echo steered", + output: "steered", + exitCode: 0, + startedAt: nowIso, + completedAt: nowIso, + updatedAt: nowIso, + type: "command_execution", + })} + ) + `; + const steeredSnapshot = yield* projectionStore.getThreadSnapshotWindow(threadId, { + rowLimit: sqlPageLimit, + }); + assert.isFalse( + steeredSnapshot.projection.turnItems.some((item) => item.runId === cancelledRunId), + ); + assert.isTrue(steeredSnapshot.projection.runs.some((run) => run.id === cancelledRunId)); + yield* sql`DELETE FROM orchestration_v2_projection_turn_items WHERE turn_item_id = ${steeredItemId}`; + const interruptRunId = RunId.make("run:bounded-sql-history:interrupt"); const interruptNodeId = NodeId.make("node:bounded-sql-history:interrupt"); const interruptRequestId = TurnItemId.make("turn-item:bounded-sql-history:interrupt-request"); @@ -3987,6 +4483,276 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { }), ); + it.effect("pages a whole thread without re-reading its full timeline for every page", () => + Effect.gen(function* () { + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const sql = yield* SqlClient.SqlClient; + const now = yield* DateTime.now; + const nowIso = DateTime.formatIso(now); + const threadId = ThreadId.make("thread:paged-scrollback"); + const runId = RunId.make("run:paged-scrollback"); + const itemCount = 600; + yield* projectionStore.apply({ + id: EventId.make("event:paged-scrollback:thread"), + type: "thread.created", + threadId, + occurredAt: now, + payload: { + createdBy: "user", + creationSource: "web", + id: threadId, + projectId: ProjectId.make("project:paged-scrollback"), + title: "Paged scrollback", + providerInstanceId, + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + activeProviderThreadId: null, + lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId }, + forkedFrom: null, + createdAt: now, + updatedAt: now, + archivedAt: null, + settledOverride: null, + settledAt: null, + lastVisitedAt: null, + deletedAt: null, + }, + }); + yield* sql` + INSERT INTO orchestration_v2_projection_runs ( + run_id, thread_id, ordinal, provider, provider_thread_id, status, + requested_at, completed_at, payload_json + ) VALUES ( + ${runId}, ${threadId}, 1, 'codex', NULL, 'completed', ${nowIso}, ${nowIso}, + ${encodeUnknownJsonString({ + id: runId, + threadId, + ordinal: 1, + providerInstanceId, + modelSelection, + providerThreadId: null, + userMessageId: "message:paged-scrollback", + rootNodeId: "node:paged-scrollback", + activeAttemptId: null, + status: "completed", + requestedAt: nowIso, + startedAt: nowIso, + completedAt: nowIso, + checkpointId: null, + contextHandoffId: null, + })} + ) + `; + for (let ordinal = 0; ordinal < itemCount; ordinal += 1) { + const id = `turn-item:paged-scrollback:${ordinal}`; + const message = ordinal % 9 === 0; + yield* sql` + INSERT INTO orchestration_v2_projection_turn_items ( + turn_item_id, thread_id, run_id, node_id, provider_thread_id, provider_turn_id, + parent_item_id, ordinal, type, status, updated_at, payload_json + ) VALUES ( + ${id}, ${threadId}, ${runId}, NULL, NULL, NULL, NULL, ${ordinal}, + ${message ? "user_message" : "command_execution"}, 'completed', ${nowIso}, + ${encodeUnknownJsonString({ + id, + threadId, + runId, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal, + status: "completed", + title: `item ${ordinal}`, + startedAt: nowIso, + completedAt: nowIso, + updatedAt: nowIso, + ...(message + ? { + type: "user_message", + createdBy: "user", + creationSource: "web", + messageId: `message:paged-scrollback:${ordinal}`, + inputIntent: "turn_start", + text: `Turn ${ordinal}`, + attachments: [], + } + : { type: "command_execution", input: "echo", output: "ok", exitCode: 0 }), + })} + ) + `; + } + // An unbounded read of every turn item in the thread costs O(thread) per page, so a full + // scrollback that repeats it on each page is quadratic in thread length. + const wholeThreadReads = (statements: ReadonlyArray) => + statements.filter( + (statement) => + /FROM orchestration_v2_projection_turn_items(\s+AS item)?\s+WHERE (item\.)?thread_id = \?\s*(ORDER BY|$)/.test( + statement, + ) && !/\bLIMIT\b/.test(statement), + ).length; + const snapshot = yield* projectionStore.getThreadSnapshot(threadId); + + const history = traceSqlStatements(); + let cursor: string | null = encodeThreadHistoryCursor({ + snapshotSequence: snapshot.snapshotSequence, + sourceThreadId: threadId, + sourceItemId: "absent", + position: itemCount, + }); + const historyIds: Array = []; + let historyPages = 0; + while (cursor !== null) { + const page: OrchestrationV2ThreadHistoryPage = yield* projectionStore + .getThreadHistoryPage(threadId, cursor) + .pipe(Effect.withTracer(history.tracer)); + historyIds.push(...page.items.map((row) => String(row.sourceItemId))); + historyPages += 1; + cursor = page.hasMoreHistory ? page.nextCursor : null; + } + assert.strictEqual(new Set(historyIds).size, itemCount); + assert.isAtLeast(historyPages, 4); + + const timeline = traceSqlStatements(); + const timelineIds: Array = []; + let afterPosition = -1; + let timelinePages = 0; + for (let hasMore = true; hasMore;) { + const page = yield* projectionStore + .getTimelinePage(threadId, { view: "activity", afterPosition, limit: 100 }) + .pipe(Effect.withTracer(timeline.tracer)); + timelineIds.push(...page.items.map((row) => String(row.sourceItemId))); + afterPosition = page.items.at(-1)?.position ?? afterPosition; + timelinePages += 1; + hasMore = page.hasMore; + } + assert.strictEqual(new Set(timelineIds).size, itemCount); + assert.isAtLeast(timelinePages, 4); + + assert.isAtMost( + wholeThreadReads(history.statements), + 1, + `getThreadHistoryPage read the whole thread on each of ${historyPages} pages`, + ); + assert.isAtMost( + wholeThreadReads(timeline.statements), + 1, + `getTimelinePage read the whole thread on each of ${timelinePages} pages`, + ); + + // A write to the thread a fork inherits from must reach the fork's next page. + const forkThreadId = ThreadId.make("thread:paged-scrollback:fork"); + yield* projectionStore.apply({ + id: EventId.make("event:paged-scrollback:fork"), + type: "thread.created", + threadId: forkThreadId, + occurredAt: now, + payload: { + createdBy: "user", + creationSource: "web", + id: forkThreadId, + projectId: ProjectId.make("project:paged-scrollback"), + title: "Paged scrollback fork", + providerInstanceId, + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + activeProviderThreadId: null, + lineage: { + parentThreadId: threadId, + relationshipToParent: "fork", + rootThreadId: threadId, + }, + forkedFrom: { type: "run", threadId, runId }, + createdAt: now, + updatedAt: now, + archivedAt: null, + settledOverride: null, + settledAt: null, + lastVisitedAt: null, + deletedAt: null, + }, + }); + const forkPage = () => + projectionStore.getTimelinePage(forkThreadId, { view: "activity", limit: 1 }); + const forkTotal = (yield* forkPage()).totalItems; + assert.strictEqual((yield* forkPage()).totalItems, forkTotal); + yield* projectionStore.apply({ + id: EventId.make("event:paged-scrollback:late-item"), + type: "turn-item.updated", + threadId, + runId, + driver, + occurredAt: now, + payload: { + id: TurnItemId.make("turn-item:paged-scrollback:late"), + threadId, + runId, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: itemCount, + status: "completed", + title: "late", + startedAt: now, + completedAt: now, + updatedAt: now, + type: "command_execution", + input: "echo", + }, + }); + assert.strictEqual((yield* forkPage()).totalItems, forkTotal + 1); + + // Another store instance's write reaches this one only through the committed thread event. + const directId = "turn-item:paged-scrollback:direct"; + yield* sql` + INSERT INTO orchestration_v2_projection_turn_items ( + turn_item_id, thread_id, run_id, node_id, provider_thread_id, provider_turn_id, + parent_item_id, ordinal, type, status, updated_at, payload_json + ) VALUES ( + ${directId}, ${threadId}, ${runId}, NULL, NULL, NULL, NULL, ${itemCount + 1}, + 'command_execution', 'completed', ${nowIso}, + ${encodeUnknownJsonString({ + id: directId, + threadId, + runId, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: itemCount + 1, + status: "completed", + title: "direct", + startedAt: nowIso, + completedAt: nowIso, + updatedAt: nowIso, + type: "command_execution", + input: "echo", + })} + ) + `; + yield* sql` + INSERT INTO orchestration_events ( + event_id, aggregate_kind, stream_id, stream_version, event_type, + occurred_at, actor_kind, payload_json, metadata_json, application_event_version + ) VALUES ( + 'event:paged-scrollback:direct', 'thread', ${threadId}, 1, + 'turn-item.updated', ${nowIso}, 'user', '{}', '{}', 2 + ) + `; + assert.strictEqual((yield* forkPage()).totalItems, forkTotal + 2); + }), + ); + it.effect("scopes actionable provider state to the requested bounded thread", () => Effect.gen(function* () { const projectionStore = yield* ProjectionStore.ProjectionStoreV2; @@ -4298,6 +5064,8 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { number: 42, url: "https://github.com/acme/project/pull/42", })})) WHERE thread_id = ${readThreadId}`; + const [latest] = yield* sql<{ readonly sequence: number }>` + SELECT COALESCE(MAX(sequence), 0) AS sequence FROM orchestration_events`; const trace = traceSqlStatements(); const discovery = yield* projectionStore .getPullRequestSnapshot({}) @@ -4308,7 +5076,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { .filter((id) => id === readThreadId || id === laterThreadId), [readThreadId], ); - assert.strictEqual(discovery.snapshotSequence, 0); + assert.strictEqual(discovery.snapshotSequence, latest?.sequence); assert.strictEqual(trace.queryTransactions.length, 2); assert.isNotNull(trace.queryTransactions[0]); assert.strictEqual(new Set(trace.queryTransactions).size, 1); @@ -4331,7 +5099,8 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { ); const settledPayload = (settledAt: string | null, archivedAt: string | null) => sql` UPDATE orchestration_v2_projection_threads - SET payload_json = json_set(payload_json, '$.settledAt', ${settledAt}, '$.archivedAt', ${archivedAt}) + SET payload_json = json_set(payload_json, '$.settledAt', ${settledAt}, '$.archivedAt', ${archivedAt}), + archived_at = ${archivedAt} WHERE thread_id = ${readThreadId} `; yield* settledPayload(DateTime.formatIso(now), null); diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 90bd54a73d58..e113449c34f2 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -96,11 +96,13 @@ import type * as Statement from "effect/sql/Statement"; import { MCP_APP_OUTPUT_KEY } from "@t3tools/shared/mcpApp"; import { threadHtmlRenderAttachmentIds } from "../attachmentStore.ts"; +import { readableSearchText } from "./messageSearchIndex.ts"; import { isThreadHistoryUserTurn, isConversationHistoryItem, THREAD_HISTORY_MAX_RAW_TURNS, missingInterruptRequests, + DEFERRED_WORK_ITEM_TYPES, deferWorkHistoryRows, selectDeferredWorkPage, encodeThreadHistoryCursor, @@ -200,6 +202,7 @@ export type ProjectionSettlementCandidate = Pick< | "projectId" | "branch" | "worktreePath" + | "lineage" | "pullRequests" | "linkedPullRequest" | "branchPullRequest" @@ -260,7 +263,11 @@ export interface ProjectionCheckpointCaptureContext { readonly rootNode: OrchestrationV2ExecutionNode | undefined; readonly scope: OrchestrationV2CheckpointScope | undefined; readonly providerThread: OrchestrationV2ProviderThread | undefined; - readonly readyCheckpointOrdinals: ReadonlyArray; + /** + * Ordinals a runless baseline must not overwrite: ready snapshots, and missing checkpoints pinning + * a run. Without file snapshots that run link is what rewind uses. Stale rows are rebaselined. + */ + readonly preservedCheckpointOrdinals: ReadonlyArray; /** Side chats share their parent's worktree and never snapshot it. */ readonly sideChat: boolean; } @@ -344,6 +351,7 @@ export interface ProjectionRecordFilter { readonly turnItemRunIds?: ReadonlyArray; readonly turnItemNodeIds?: ReadonlyArray; readonly runIds?: ReadonlyArray; + /** Without `includeLatestRun`, reads through a status index and sorts the matches, so keep it to statuses few runs hold. */ readonly runStatuses?: ReadonlyArray; /** Keeps the thread's highest-ordinal run even when `runStatuses` excludes it. */ readonly includeLatestRun?: true; @@ -676,6 +684,10 @@ export interface ProjectionStoreV2Shape { threadId: ThreadId, runId: RunId, ) => Effect.Effect; + readonly getRun: ( + threadId: ThreadId, + runId: RunId, + ) => Effect.Effect; readonly canStartQueuedRun: ( threadId: ThreadId, ) => Effect.Effect; @@ -1030,6 +1042,7 @@ type ShellThreadRow = { readonly latest_run_started_at: string | null; readonly latest_run_completed_at: string | null; readonly latest_run_stop_requested: number; + readonly latest_run_background_wake: number; readonly active_run_id: string | null; readonly active_run_status: string | null; readonly activity_run_status: string | null; @@ -1041,6 +1054,7 @@ type ShellThreadRow = { readonly blocking_run_started_at: string | null; readonly blocking_run_completed_at: string | null; readonly blocking_run_stop_requested: number; + readonly blocking_run_background_wake: number; readonly blocking_failure_payload_json: string | null; readonly pending_request_payload_json: string | null; readonly pending_secret_request_payload_json: string | null; @@ -1152,6 +1166,7 @@ const PullRequestThread = OrchestrationV2AppThreadJsonSchema.mapFields( id, projectId, lineage, + createdAt, branch, worktreePath, archivedAt, @@ -1164,6 +1179,7 @@ const PullRequestThread = OrchestrationV2AppThreadJsonSchema.mapFields( id, projectId, lineage, + createdAt, branch, worktreePath, archivedAt, @@ -1656,6 +1672,8 @@ export function threadShellFromProjection( projection.turnItems.some( (item) => item.runId === latestRun.id && item.type === "run_interrupt_request", ), + latestRunBackgroundWake: + latestRun != null && isBackgroundWakeRunForShell(latestRun, projection), activeRunId: activeRun?.id ?? null, blockedOnThreadWait: deriveBlockedOnThreadWait({ activeRun, turnItems: projection.turnItems }), activityRunStatus: activityRun?.status ?? null, @@ -1751,6 +1769,25 @@ function isSilentRunForShell( ); } +const BACKGROUND_WAKE_SOURCE_KINDS = new Set(["background_task", "command", "monitor"]); + +function isBackgroundWakeRunForShell( + run: OrchestrationV2ThreadProjection["runs"][number], + projection: Pick, +): boolean { + const message = projection.messages.find((candidate) => candidate.id === run.userMessageId); + return ( + message?.delegatedCompletion === undefined && + // Only progress wakes; a terminal one ("Command … finished") is the work reporting its end. + message?.notification?.outcome === "updated" && + BACKGROUND_WAKE_SOURCE_KINDS.has(message.notification.source.kind) && + // A follow-up the user steered into the wake makes the run theirs. + !projection.messages.some( + (candidate) => candidate.runId === run.id && candidate.createdBy === "user", + ) + ); +} + function isInterruptibleRunForShell(run: OrchestrationV2ThreadProjection["runs"][number]): boolean { return run.status === "preparing" || run.status === "starting" || run.status === "running"; } @@ -1773,6 +1810,7 @@ type ShellThreadState = { readonly latestRunStartedAt: DateTime.Utc | null; readonly latestRunCompletedAt: DateTime.Utc | null; readonly latestRunStopRequested: boolean; + readonly latestRunBackgroundWake: boolean; readonly activeRunId: RunId | null; readonly blockedOnThreadWait: boolean; readonly activityRunStatus: ShellActivityRunStatus | null; @@ -1933,6 +1971,7 @@ function shellFromState(input: { latestRunStartedAt: input.state.latestRunStartedAt, latestRunCompletedAt: input.state.latestRunCompletedAt, latestRunStopRequested: input.state.latestRunStopRequested, + latestRunBackgroundWake: input.state.latestRunBackgroundWake, activeRunId: input.state.activeRunId, blockedOnThreadWait: input.state.blockedOnThreadWait, activityRunStatus: input.state.activityRunStatus, @@ -1983,6 +2022,20 @@ export const layer: Layer.Layer = OR item.payload_json LIKE '%"viewedImagePath"%' OR item.payload_json LIKE '%"toolName":"%delegate_task"%')`; + // SQLite uses the partial conversation index of migration 082 only for this exact predicate. + const CONVERSATION_INDEX_EXCLUDED_TYPES = sql.literal( + `(${DEFERRED_WORK_ITEM_TYPES.map((type) => `'${type}'`).join(", ")})`, + ); + + // A todo list is never closed when its run ends, so a long thread holds an active one per + // old run. Counting those as live work drags each run's nodes and subagents into every window. + const todoListOfSettledRun = (row: Statement.Fragment) => sql`( + ${row}.kind = 'todo_list' AND ${row}.run_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM orchestration_v2_projection_runs AS todo_run + WHERE todo_run.run_id = ${row}.run_id + AND todo_run.status IN ('queued','preparing','starting','running','waiting') + ))`; + // Start from the few owned provider threads; a thread can have a large settled node history. const nodesOwningProviderThreads = (threadId: ThreadId) => sql` SELECT owner.node_id @@ -2006,10 +2059,50 @@ export const layer: Layer.Layer = END `; + type TimelineIndexItem = Pick< + OrchestrationV2TurnItem, + "id" | "threadId" | "runId" | "nodeId" | "parentItemId" | "type" | "status" | "ordinal" + > & { + readonly inputIntent?: Extract< + OrchestrationV2TurnItem, + { type: "user_message" } + >["inputIntent"]; + readonly createdBy?: "user" | "agent" | "system"; + readonly messageId?: MessageId; + readonly runStatus?: string | undefined; + readonly attemptId?: string | null; + readonly attemptStatus?: string | null; + readonly persistentOutput?: boolean; + readonly deferredWork?: OrchestrationV2TurnItem["deferredWork"]; + }; + type TimelineIndexRow = { + readonly sourceThreadId: ThreadId; + readonly sourceItemId: TurnItemId; + readonly visibility: OrchestrationV2ProjectedTurnItem["visibility"]; + readonly item: TimelineIndexItem; + readonly synthetic?: OrchestrationV2TurnItem; + }; + type TimelineIndex = { + readonly records: ProjectionRecords<"runs" | "attempts">; + /** The thread and every fork source the index read. */ + readonly threadIds: ReadonlyArray; + readonly local: ReadonlyArray; + readonly visible: ReadonlyArray; + }; + // Least recently used first; a few cover clients scrolling side by side without pinning every thread. + const TIMELINE_INDEX_CACHE_LIMIT = 8; + const timelineIndexes = new Map< + string, + { readonly sequence: number; readonly index: TimelineIndex } + >(); + const apply: ProjectionStoreV2Shape["apply"] = /* This helper runs for individual orchestration events; the enclosing operation owns the span. */ Effect.fnUntraced( function* (event) { + for (const [key, cached] of timelineIndexes) { + if (cached.index.threadIds.includes(event.threadId)) timelineIndexes.delete(key); + } switch (event.type) { case "thread.pull-request-link-updated": { const thread = yield* getThread(event.threadId); @@ -2566,7 +2659,8 @@ export const layer: Layer.Layer = streaming, created_at, updated_at, - payload_json + payload_json, + search_text ) VALUES ( ${event.payload.id}, @@ -2577,7 +2671,8 @@ export const layer: Layer.Layer = ${booleanInt(event.payload.streaming)}, ${createdAt}, ${updatedAt}, - ${payloadJson} + ${payloadJson}, + ${readableSearchText(event.payload.text)} ) ON CONFLICT(message_id) DO UPDATE SET @@ -2588,7 +2683,8 @@ export const layer: Layer.Layer = streaming = excluded.streaming, created_at = excluded.created_at, updated_at = excluded.updated_at, - payload_json = excluded.payload_json + payload_json = excluded.payload_json, + search_text = excluded.search_text `; break; } @@ -3031,14 +3127,124 @@ export const layer: Layer.Layer = SELECT ordinal, (SELECT MAX(ordinal) FROM user_anchors) AS latest FROM boundary `; const boundary = anchors[0]!; + // Folding needs whole segments, so the window would walk every item since the boundary. + // It stays above the newest `tailRows` rows that are sure to keep their own row. Start at + // the latest segment start of each run that reaches that point, clamped to it so runless + // rows below stay in the scan. + const tailRows = Math.max(window.rowLimit, THREAD_HISTORY_TURN_SNAPSHOT_ROW_LIMIT); + // The newest rows sure to keep their own row: conversation rows, and every row of the + // running run, which folds nothing. More of them than a whole turn may hold means the + // window pages from inside the newest turn with the row limit, so the tail shrinks to + // that limit. A turn of thousands of rows is then not walked to return the newest few. + const tailWorkStart = sql<{ + readonly ordinal: number | null; + readonly long_turn: number; + }>` + ${eligible}, live_run AS ( + -- The oldest live run is the one at work; queued runs follow it. + SELECT live.run_id FROM orchestration_v2_projection_runs AS live + WHERE live.thread_id = ${threadId} + AND live.status IN ('queued','preparing','starting','running','waiting') + AND ${window.requiredRunId ?? null} IS NULL + AND NOT EXISTS ( + SELECT 1 FROM orchestration_v2_projection_run_attempts + WHERE run_id = live.run_id AND status = 'superseded' + ) + ORDER BY +live.ordinal ASC LIMIT 1 + ), visible_tail AS ( + SELECT ordinal FROM ( + SELECT ordinal FROM eligible + WHERE ordinal >= ${boundary.ordinal} + AND type NOT IN ${CONVERSATION_INDEX_EXCLUDED_TYPES} + ORDER BY ordinal DESC + LIMIT ${tailRows} + ) + UNION + SELECT ordinal FROM ( + SELECT live_item.ordinal + FROM orchestration_v2_projection_turn_items AS live_item + -- The unary plus keeps this on the run index, whose ordinal order ends + -- the walk at the limit. + WHERE live_item.run_id = (SELECT run_id FROM live_run) + AND +live_item.thread_id = ${threadId} + AND live_item.ordinal >= ${boundary.ordinal} + AND live_item.ordinal <= COALESCE( + ( + SELECT ordinal FROM orchestration_v2_projection_turn_items + WHERE thread_id = ${threadId} + AND turn_item_id = ${window.anchorItemId ?? null} + LIMIT 1 + ), + 9223372036854775807 + ) + ORDER BY live_item.ordinal DESC + LIMIT ${tailRows} + ) + ORDER BY ordinal DESC + LIMIT ${tailRows} + ), turn AS ( + SELECT ( + SELECT COUNT(*) FROM visible_tail WHERE ordinal >= ${boundary.latest} + ) > ${THREAD_HISTORY_PAGE_POLICY.maxTurnItems} AS long_turn + ), tail AS ( + SELECT visible_tail.ordinal FROM visible_tail + ORDER BY visible_tail.ordinal DESC + LIMIT 1 OFFSET ( + SELECT MAX(CASE WHEN long_turn THEN ${window.rowLimit} ELSE ${tailRows} END, 1) - 1 + FROM turn + ) + ), tail_runs AS ( + SELECT DISTINCT item.run_id + FROM tail CROSS JOIN eligible AS item + WHERE item.ordinal >= tail.ordinal AND item.run_id IS NOT NULL + ) + SELECT MIN(MIN(CASE + -- Only a completed run or a superseded attempt folds work, and folding is + -- what needs the whole segment. Any other run keeps every row. + WHEN ( + SELECT status FROM orchestration_v2_projection_runs + WHERE run_id = tail_runs.run_id + ) IS NOT 'completed' AND NOT EXISTS ( + SELECT 1 FROM orchestration_v2_projection_run_attempts + WHERE run_id = tail_runs.run_id AND status = 'superseded' + ) THEN tail.ordinal + ELSE COALESCE( + ( + SELECT start.ordinal FROM eligible AS start + WHERE start.run_id = tail_runs.run_id AND start.type = 'user_message' + AND start.ordinal <= tail.ordinal + ORDER BY start.ordinal DESC LIMIT 1 + ), + ( + SELECT MIN(first.ordinal) FROM orchestration_v2_projection_turn_items AS first + WHERE first.thread_id = ${threadId} AND first.run_id = tail_runs.run_id + ), + tail.ordinal + ) + END, tail.ordinal)) AS ordinal, (SELECT long_turn FROM turn) AS long_turn + FROM tail LEFT JOIN tail_runs + `; + // A negative or fractional limit makes SQLite's LIMIT disagree with `tailRows`; keep the old start. + const tail = + window.deferWork === true && + boundary.latest !== null && + Number.isSafeInteger(window.rowLimit) && + window.rowLimit >= 0 + ? (yield* tailWorkStart)[0] + : undefined; + const longTurn = tail?.long_turn === 1; + const workStart = Math.max(boundary.ordinal, tail?.ordinal ?? 0); return sql` ${eligible}, blocked_runs AS MATERIALIZED ( SELECT DISTINCT run_id FROM orchestration_v2_projection_turn_items WHERE thread_id = ${threadId} AND run_id IS NOT NULL AND ${window.deferWork ? 1 : 0} = 1 + -- Only a run that can fold asks whether it is blocked. A live run cannot, + -- and this check would read every row it ever wrote. AND run_id IN ( SELECT run_id FROM eligible - WHERE ordinal >= ${boundary.ordinal} AND run_id IS NOT NULL + WHERE ordinal >= ${workStart} AND run_id IS NOT NULL + AND (run_status = 'completed' OR attempt_status = 'superseded') ) AND (type IN ('run_interrupt_request', 'run_interrupt_result') OR (type = 'error' AND status = 'failed' AND parent_item_id IS NULL)) @@ -3047,14 +3253,14 @@ export const layer: Layer.Layer = MAX(CASE WHEN type = 'user_message' THEN ordinal ELSE 0 END) OVER (PARTITION BY run_id ORDER BY ordinal) AS segment, CASE WHEN status = 'completed' AND (run_status = 'completed' OR attempt_status = 'superseded') - AND type IN ('command_execution', 'reasoning', 'file_change', 'file_search', 'web_search', 'dynamic_tool') + AND type IN ${sql.in(DEFERRED_WORK_ITEM_TYPES)} AND NOT (${inlineWorkOutput}) AND run_id NOT IN (SELECT run_id FROM blocked_runs) THEN 1 ELSE 0 END AS can_defer FROM eligible AS item WHERE ${window.deferWork ? 1 : 0} = 1 AND ${boundary.latest === null ? 0 : 1} = 1 - AND ordinal >= ${boundary.ordinal} + AND ordinal >= ${workStart} ), deferred_work AS ( SELECT run_id, segment, attempt_id, MIN(ordinal) AS first_ordinal, MAX(ordinal) AS last_ordinal, COUNT(*) AS item_count @@ -3089,8 +3295,10 @@ export const layer: Layer.Layer = FROM window_items WHERE ordinal >= ${boundary.ordinal} ORDER BY ordinal DESC, turn_item_id DESC + -- Every LIMIT here and in latest_user_turn must stay <= tailRows, or rows below workStart go missing. LIMIT CASE WHEN ${window.rowLimit} = 0 THEN 0 + WHEN ${longTurn ? 1 : 0} = 1 THEN ${window.rowLimit} WHEN (SELECT row_count FROM latest_user_turn) BETWEEN 1 AND ${THREAD_HISTORY_PAGE_POLICY.maxTurnItems} THEN ${THREAD_HISTORY_TURN_SNAPSHOT_ROW_LIMIT} @@ -3122,6 +3330,11 @@ export const layer: Layer.Layer = `; }); + // An id filter reads by primary key. `+` keeps SQLite off the thread index, which walks + // every row of the thread to find the few ids. + const threadColumn = (ids: ReadonlyArray | undefined, column = "thread_id") => + sql.literal(ids === undefined ? column : `+${column}`); + const readCanonicalProjection = Effect.fn("ProjectionStore.readCanonicalProjection")( function* ( threadId: ThreadId, @@ -3235,8 +3448,9 @@ export const layer: Layer.Layer = WITH RECURSIVE retained(node_id) AS ( SELECT value FROM json_each(${cohortJson("nodeId")}) UNION - SELECT node_id FROM orchestration_v2_projection_nodes + SELECT node_id FROM orchestration_v2_projection_nodes AS live WHERE thread_id = ${threadId} AND status IN ('pending','starting','running','waiting') + AND NOT ${todoListOfSettledRun(sql`live`)} UNION SELECT root_node_id FROM orchestration_v2_projection_run_attempts WHERE thread_id = ${threadId} AND ( @@ -3355,21 +3569,26 @@ export const layer: Layer.Layer = ? sql` SELECT payload_json FROM orchestration_v2_projection_runs - WHERE thread_id = ${threadId} + WHERE ${threadColumn(filter?.runIds)} = ${threadId} ${filter?.runIds === undefined ? sql`` : sql`AND run_id IN (SELECT value FROM json_each(${encodeIdList(filter.runIds)}))`} ${ filter?.runStatuses === undefined - ? sql`` - : sql`AND (status IN (SELECT value FROM json_each(${encodeIdList(filter.runStatuses)})) - OR (${filter.includeLatestRun === true ? 1 : 0} = 1 AND run_id = ( + ? sql`ORDER BY ordinal ASC` + : filter.includeLatestRun !== true + ? // `+ordinal` stops SQLite from walking every run in the (thread_id, ordinal) + // index to skip a sort; a status index reads only matching runs. + sql`AND status IN (SELECT value FROM json_each(${encodeIdList(filter.runStatuses)})) + ORDER BY +ordinal ASC` + : sql`AND (status IN (SELECT value FROM json_each(${encodeIdList(filter.runStatuses)})) + OR run_id = ( SELECT run_id FROM orchestration_v2_projection_runs WHERE thread_id = ${threadId} AND (status <> 'cancelled' OR json_extract(payload_json, '$.startedAt') IS NOT NULL) ORDER BY ordinal DESC LIMIT 1 - )))` + )) + ORDER BY ordinal ASC` } - ORDER BY ordinal ASC ` : sql` SELECT payload_json FROM orchestration_v2_projection_runs @@ -3377,6 +3596,12 @@ export const layer: Layer.Layer = AND (status IN ('queued','preparing','starting','running','waiting') OR run_id IN (SELECT value FROM json_each(${cohortRunIds})) OR run_id = ${window.requiredRunId ?? null} + ${ + // The shell's latest run, even a steer-cancelled one outside the newest window. + window.anchorItemId === undefined && window.requiredRunId === undefined + ? sql`OR run_id = ${presentedShellRunId(sql`${threadId}`)}` + : sql`` + } -- Queued or cancelled turns must not hide the latest actual session. OR run_id = ${latestTodoList?.run_id ?? null} OR run_id = ( @@ -3413,7 +3638,7 @@ export const layer: Layer.Layer = ? sql` SELECT payload_json FROM orchestration_v2_projection_nodes AS node - WHERE node.thread_id = ${threadId} + WHERE ${threadColumn(filter?.nodeIds, "node.thread_id")} = ${threadId} ${filter?.nodeIds === undefined ? sql`` : sql`AND node.node_id IN (SELECT value FROM json_each(${encodeIdList(filter.nodeIds)}))`} ${ filter?.omitSettledLeafNodes === true @@ -3452,7 +3677,7 @@ export const layer: Layer.Layer = ? sql` SELECT payload_json FROM orchestration_v2_projection_subagents AS subagent - WHERE thread_id = ${threadId} + WHERE ${threadColumn(filter?.subagentIds)} = ${threadId} ${filter?.subagentIds === undefined ? sql`` : sql`AND subagent_id IN (SELECT value FROM json_each(${encodeIdList(filter.subagentIds)}))`} ${filter?.subagentRunIds === undefined ? sql`` : sql`AND run_id IN (SELECT value FROM json_each(${encodeIdList(filter.subagentRunIds)}))`} ${ @@ -3587,7 +3812,7 @@ export const layer: Layer.Layer = : window === undefined ? sql` SELECT payload_json FROM orchestration_v2_projection_messages - WHERE thread_id = ${threadId} + WHERE ${threadColumn(filter?.messageIds)} = ${threadId} ${filter?.messageIds === undefined ? sql`` : sql`AND message_id IN (SELECT value FROM json_each(${encodeIdList(filter.messageIds)}))`} ${filter?.messageRoles === undefined ? sql`` : sql`AND role IN (SELECT value FROM json_each(${encodeIdList(filter.messageRoles)}))`} ${filter?.monitorNotificationsOnly === true ? sql`AND json_extract(payload_json, '$.notification.source.kind') = 'monitor'` : sql``} @@ -3595,18 +3820,23 @@ export const layer: Layer.Layer = ORDER BY created_at ASC, message_id ASC ` : sql` - SELECT payload_json FROM orchestration_v2_projection_messages AS message - WHERE message.thread_id = ${threadId} - AND ( - message.message_id IN ( - SELECT value FROM json_each(${cohortMessageIds}) - ) - OR message.run_id IN ( - SELECT run_id FROM orchestration_v2_projection_runs - WHERE thread_id = ${threadId} - AND status IN ('queued', 'preparing', 'starting', 'running', 'waiting') - ) - ) + -- Look the window's messages up by key and the live runs' by run. One filter + -- over the thread reads every message the thread ever had. + SELECT payload_json FROM ( + SELECT message.payload_json, message.created_at, message.message_id + FROM json_each(${cohortMessageIds}) AS wanted + CROSS JOIN orchestration_v2_projection_messages AS message + ON message.message_id = wanted.value + WHERE message.thread_id = ${threadId} + UNION + SELECT message.payload_json, message.created_at, message.message_id + FROM orchestration_v2_projection_runs AS run + CROSS JOIN orchestration_v2_projection_messages AS message + ON message.run_id = run.run_id + WHERE run.thread_id = ${threadId} + AND run.status IN ('queued', 'preparing', 'starting', 'running', 'waiting') + AND message.thread_id = ${threadId} + ) ORDER BY created_at ASC, message_id ASC `, fields !== undefined && !fields.includes("plans") @@ -3620,7 +3850,7 @@ export const layer: Layer.Layer = SELECT payload_json FROM orchestration_v2_projection_plans AS plan WHERE plan.thread_id = ${threadId} AND ( - plan.status = 'active' + (plan.status = 'active' AND NOT ${todoListOfSettledRun(sql`plan`)}) OR plan.plan_id = ${latestTodoList?.plan_id ?? null} OR plan.plan_id IN ( SELECT value FROM json_each(${cohortPlanIds}) @@ -3661,6 +3891,13 @@ export const layer: Layer.Layer = WHERE thread_id = ${threadId} AND (status IN ('pending','capturing') OR (run_id IS NULL AND ordinal_within_scope = 0 AND status = 'ready') + -- The diff of the last finished turn stays in reach while a long run is at work. + OR checkpoint_id = ( + SELECT checkpoint_id FROM orchestration_v2_projection_checkpoints + WHERE thread_id = ${threadId} AND status = 'ready' + AND app_run_ordinal IS NOT NULL + ORDER BY app_run_ordinal DESC LIMIT 1 + ) OR checkpoint_id IN (SELECT value FROM json_each(${cohortCheckpointIds})) OR run_id IN (SELECT value FROM json_each(${cohortRunIds})) OR node_id IN (SELECT value FROM json_each(${cohortNodeIds}))) @@ -3747,10 +3984,13 @@ export const layer: Layer.Layer = (filter.messageRoles.includes("user") && filter.messageRoles.includes("assistant"))) ? undefined : messages.map((message) => message.id); + // The literal `type = 'user_message'` lets SQLite pick the partial user-message index. + const userOnly = messages.every((message) => message.role === "user"); const rows = yield* sql<{ message_id: string; ordinal: number }>` SELECT json_extract(payload_json, '$.messageId') AS message_id, MIN(ordinal) AS ordinal FROM orchestration_v2_projection_turn_items - WHERE thread_id = ${threadId} AND type IN ('user_message', 'assistant_message') + WHERE thread_id = ${threadId} + AND ${userOnly ? sql`type = 'user_message'` : sql`type IN ('user_message', 'assistant_message')`} ${loadedMessageIds === undefined ? sql`` : sql`AND json_extract(payload_json, '$.messageId') IN (SELECT value FROM json_each(${encodeIdList(loadedMessageIds)}))`} GROUP BY json_extract(payload_json, '$.messageId') `; @@ -5327,7 +5567,7 @@ export const layer: Layer.Layer = runRows[0] === undefined ? undefined : yield* decodeRunPayload(runRows[0].payload_json); - const [nodeRows, scopeRows, providerRows, readyRows] = yield* Effect.all([ + const [nodeRows, scopeRows, providerRows, preservedRows] = yield* Effect.all([ sql`SELECT payload_json FROM orchestration_v2_projection_nodes WHERE thread_id = ${threadId} AND node_id = ${run?.rootNodeId ?? null}`, sql`SELECT payload_json FROM orchestration_v2_projection_checkpoint_scopes @@ -5339,7 +5579,8 @@ export const layer: Layer.Layer = OR provider_thread_id IN (SELECT provider_thread_id FROM orchestration_v2_projection_subagents WHERE thread_id = ${threadId}))`, sql<{ readonly ordinal: number }>`SELECT ordinal_within_scope AS ordinal FROM orchestration_v2_projection_checkpoints - WHERE thread_id = ${threadId} AND scope_id = ${target.scopeId} AND status = 'ready'`, + WHERE thread_id = ${threadId} AND scope_id = ${target.scopeId} + AND (status = 'ready' OR (status = 'missing' AND run_id IS NOT NULL))`, ]); return { run, @@ -5355,7 +5596,7 @@ export const layer: Layer.Layer = providerRows[0] === undefined ? undefined : yield* decodeProviderThreadPayload(providerRows[0].payload_json), - readyCheckpointOrdinals: readyRows.map(({ ordinal }) => ordinal), + preservedCheckpointOrdinals: preservedRows.map(({ ordinal }) => ordinal), sideChat: threadRows[0].side_chat === 1, }; }), @@ -5412,7 +5653,7 @@ export const layer: Layer.Layer = checkpoint === undefined || scope === undefined || checkpoint.scopeId !== scope.id || - checkpoint.status !== "ready" || + (checkpoint.status !== "ready" && checkpoint.status !== "missing") || providerThread.id !== thread.activeProviderThreadId || providerThread.providerInstanceId !== thread.modelSelection.instanceId ) @@ -5426,7 +5667,7 @@ export const layer: Layer.Layer = JOIN orchestration_v2_projection_runs AS run ON attempt.attempt_id = json_extract(run.payload_json, '$.activeAttemptId') WHERE run.thread_id = ${threadId} AND run.ordinal = ${targetOrdinal} - AND attempt.thread_id = ${threadId}`, + AND run.status != 'rolled_back' AND attempt.thread_id = ${threadId}`, sql`SELECT payload_json FROM orchestration_v2_projection_runs WHERE thread_id = ${threadId} AND ordinal > ${targetOrdinal} AND status IN ('completed', 'interrupted', 'failed', 'cancelled') @@ -5498,6 +5739,21 @@ export const layer: Layer.Layer = ) .pipe(Effect.mapError(controlReadError(threadId))); + const getRun: ProjectionStoreV2Shape["getRun"] = (threadId, runId) => + sql + .withTransaction( + Effect.gen(function* () { + yield* requireThread(threadId); + const rows = yield* sql` + SELECT payload_json FROM orchestration_v2_projection_runs + WHERE run_id = ${runId} AND thread_id = ${threadId}`; + return rows[0] === undefined + ? undefined + : yield* decodeRunPayload(rows[0].payload_json); + }), + ) + .pipe(Effect.mapError(controlReadError(threadId))); + const canStartQueuedRun: ProjectionStoreV2Shape["canStartQueuedRun"] = (threadId) => sql .withTransaction( @@ -5685,41 +5941,13 @@ export const layer: Layer.Layer = ), ); - type TimelineIndexItem = Pick< - OrchestrationV2TurnItem, - "id" | "threadId" | "runId" | "nodeId" | "parentItemId" | "type" | "status" | "ordinal" - > & { - readonly inputIntent?: Extract< - OrchestrationV2TurnItem, - { type: "user_message" } - >["inputIntent"]; - readonly createdBy?: "user" | "agent" | "system"; - readonly messageId?: MessageId; - readonly runStatus?: string | undefined; - readonly attemptId?: string | null; - readonly attemptStatus?: string | null; - readonly persistentOutput?: boolean; - readonly deferredWork?: OrchestrationV2TurnItem["deferredWork"]; - }; - type TimelineIndexRow = { - readonly sourceThreadId: ThreadId; - readonly sourceItemId: TurnItemId; - readonly visibility: OrchestrationV2ProjectedTurnItem["visibility"]; - readonly item: TimelineIndexItem; - readonly synthetic?: OrchestrationV2TurnItem; - }; const readTimelineIndex: ( threadId: ThreadId, seen: ReadonlySet, includePersistentOutput?: boolean, - ) => Effect.Effect< - { - readonly records: ProjectionRecords<"runs" | "attempts">; - readonly local: ReadonlyArray; - readonly visible: ReadonlyArray; - }, - ProjectionStoreV2Error - > = Effect.fn("ProjectionStore.readTimelineIndex")( + ) => Effect.Effect = Effect.fn( + "ProjectionStore.readTimelineIndex", + )( function* (threadId: ThreadId, seen: ReadonlySet, includePersistentOutput = false) { const records = yield* getThreadRecords(threadId, ["runs", "attempts"]); const rows = yield* sql<{ @@ -5789,7 +6017,8 @@ export const layer: Layer.Layer = }); const visible = local.filter((row) => isVisible(row.item)); const fork = inheritedRunFork(records.thread); - if (fork === null || seen.has(fork.threadId)) return { records, local, visible }; + if (fork === null || seen.has(fork.threadId)) + return { records, threadIds: [threadId], local, visible }; const source = yield* readTimelineIndex( fork.threadId, new Set([...seen, threadId]), @@ -5827,6 +6056,7 @@ export const layer: Layer.Layer = }); return { records, + threadIds: [threadId, ...source.threadIds], local, visible: [ ...inherited, @@ -5851,6 +6081,39 @@ export const layer: Layer.Layer = ), ); + // Every committed projection write appends a thread event, so the newest event across the + // index's threads identifies its snapshot; apply also drops entries for replayed writes. Call + // inside the read transaction. Without the index hint SQLite walks every newer V2 event. + const latestThreadEventSequence = (threadId: ThreadId, threadIds: ReadonlyArray) => + sql<{ sequence: number }>`SELECT COALESCE(MAX(sequence), 0) AS sequence + FROM orchestration_events INDEXED BY idx_orchestration_events_agent_stream_sequence + WHERE application_event_version = 2 + AND aggregate_kind = 'thread' AND stream_id IN ${sql.in(threadIds)}`.pipe( + Effect.map((rows) => rows[0]?.sequence ?? 0), + Effect.mapError((cause) => new ProjectionStoreReadError({ threadId, cause })), + ); + const readSnapshotTimelineIndex = Effect.fn("ProjectionStore.readSnapshotTimelineIndex")( + function* (threadId: ThreadId, includePersistentOutput: boolean) { + const key = `${includePersistentOutput ? "work" : "plain"}:${threadId}`; + const cached = timelineIndexes.get(key); + const index = + cached !== undefined && + (yield* latestThreadEventSequence(threadId, cached.index.threadIds)) === cached.sequence + ? cached.index + : yield* readTimelineIndex(threadId, new Set(), includePersistentOutput); + const sequence = + index === cached?.index + ? cached.sequence + : yield* latestThreadEventSequence(threadId, index.threadIds); + timelineIndexes.delete(key); + timelineIndexes.set(key, { sequence, index }); + const oldest = timelineIndexes.keys().next(); + if (timelineIndexes.size > TIMELINE_INDEX_CACHE_LIMIT && oldest.done !== true) + timelineIndexes.delete(oldest.value); + return index; + }, + ); + const { searchThread, searchThreadStream } = yield* makeThreadFind((threadId) => readTimelineIndex(threadId, new Set()).pipe(Effect.map((index) => index.visible)), ); @@ -5865,7 +6128,7 @@ export const layer: Layer.Layer = sql .withTransaction( Effect.gen(function* () { - const index = yield* readTimelineIndex(threadId, new Set(), workMode !== undefined); + const index = yield* readSnapshotTimelineIndex(threadId, workMode !== undefined); if (index.records.thread.deletedAt !== null) return yield* new ProjectionStoreThreadNotFoundError({ threadId }); const revisions = yield* sql<{ @@ -5982,7 +6245,7 @@ export const layer: Layer.Layer = sql .withTransaction( Effect.gen(function* () { - const index = yield* readTimelineIndex(threadId, new Set()); + const index = yield* readSnapshotTimelineIndex(threadId, false); const attachmentOwners = options.attachmentId === undefined ? undefined @@ -6191,6 +6454,38 @@ export const layer: Layer.Layer = ) )`; }; + // The newest run not waiting in a held queue, matching latestUnheldRun. + const presentedShellRunId = (threadId: Statement.Fragment) => sql`( + SELECT candidate.run_id + FROM orchestration_v2_projection_runs candidate + WHERE candidate.thread_id = ${threadId} + AND NOT ( + candidate.status = 'queued' + AND json_extract(candidate.payload_json, '$.queueHeld') IS 1 + ) + AND NOT ${isSilentShellRun("candidate")} + ORDER BY candidate.ordinal DESC, candidate.run_id DESC + LIMIT 1 + )`; + + // Mirrors isBackgroundWakeRunForShell. Stored sources keep the legacy encoding, where a + // command is background_command and a subagent is background_task with work "subagent". + const isBackgroundWakeShellRun = (alias: string) => { + const run = sql.literal(alias); + return sql`EXISTS ( + SELECT 1 FROM orchestration_v2_projection_messages wake + WHERE wake.message_id = json_extract(${run}.payload_json, '$.userMessageId') + AND json_extract(wake.payload_json, '$.notification.source.kind') + IN ('background_task', 'background_command', 'command', 'monitor') + AND json_extract(wake.payload_json, '$.notification.source.work') IS NULL + AND json_extract(wake.payload_json, '$.notification.outcome') = 'updated' + AND json_extract(wake.payload_json, '$.delegatedCompletion') IS NULL + ) AND NOT EXISTS ( + SELECT 1 FROM orchestration_v2_projection_messages steer + WHERE steer.run_id = ${run}.run_id + AND json_extract(steer.payload_json, '$.createdBy') = 'user' + )`; + }; const selectShellThreadRows = ( threadId?: ThreadId, @@ -6224,6 +6519,7 @@ export const layer: Layer.Layer = WHERE stop_request.thread_id = t.thread_id AND stop_request.run_id = presented.run_id AND stop_request.type = 'run_interrupt_request' ) AS latest_run_stop_requested, + ${isBackgroundWakeShellRun("presented")} AS latest_run_background_wake, active.run_id AS active_run_id, active.status AS active_run_status, ( @@ -6281,6 +6577,7 @@ export const layer: Layer.Layer = WHERE stop_request.thread_id = t.thread_id AND stop_request.run_id = blocked.run_id AND stop_request.type = 'run_interrupt_request' ) AS blocking_run_stop_requested, + ${isBackgroundWakeShellRun("blocked")} AS blocking_run_background_wake, ( SELECT item.payload_json FROM orchestration_v2_projection_turn_items item @@ -6358,19 +6655,8 @@ export const layer: Layer.Layer = ORDER BY r.ordinal DESC, r.run_id DESC LIMIT 1 ) - -- The newest run not waiting in a held queue, matching latestUnheldRun. - LEFT JOIN orchestration_v2_projection_runs presented ON presented.run_id = ( - SELECT candidate.run_id - FROM orchestration_v2_projection_runs candidate - WHERE candidate.thread_id = t.thread_id - AND NOT ( - candidate.status = 'queued' - AND json_extract(candidate.payload_json, '$.queueHeld') IS 1 - ) - AND NOT ${isSilentShellRun("candidate")} - ORDER BY candidate.ordinal DESC, candidate.run_id DESC - LIMIT 1 - ) + LEFT JOIN orchestration_v2_projection_runs presented + ON presented.run_id = ${presentedShellRunId(sql.literal("t.thread_id"))} LEFT JOIN orchestration_v2_projection_runs blocked ON blocked.run_id = ( SELECT candidate.run_id FROM orchestration_v2_projection_runs candidate @@ -6614,6 +6900,16 @@ export const layer: Layer.Layer = AND json_extract(t.payload_json, '$.settledOverride') IS NULL AND json_extract(t.payload_json, '$.pinnedAt') IS NULL AND json_extract(t.payload_json, '$.autoSettleDisabledAt') IS NULL + AND ( + json_extract(t.payload_json, '$.lineage.relationshipToParent') IS NOT 'subagent' + OR NOT EXISTS ( + SELECT 1 FROM orchestration_v2_projection_threads parent + WHERE parent.thread_id = json_extract(t.payload_json, '$.lineage.parentThreadId') + AND parent.deleted_at IS NULL + AND json_extract(parent.payload_json, '$.archivedAt') IS NULL + AND json_extract(parent.payload_json, '$.settledOverride') IS NOT 'settled' + ) + ) AND NOT EXISTS ( SELECT 1 FROM orchestration_v2_projection_runs active WHERE active.thread_id = t.thread_id @@ -6864,6 +7160,7 @@ export const layer: Layer.Layer = ? null : DateTime.makeUnsafe(row.latest_run_completed_at); let latestRunStopRequested = row.latest_run_stop_requested === 1; + let latestRunBackgroundWake = row.latest_run_background_wake === 1; if (row.blocking_run_id !== null && row.blocking_failure_payload_json !== null) { const blockingFailure = yield* decodeTurnItemPayload( row.blocking_failure_payload_json, @@ -6877,6 +7174,7 @@ export const layer: Layer.Layer = terminalFailureItem = blockingFailure; latestRunId = RunId.make(row.blocking_run_id); latestRunStopRequested = row.blocking_run_stop_requested === 1; + latestRunBackgroundWake = row.blocking_run_background_wake === 1; latestRunStatus = "failed"; latestRunRequestedAt = row.blocking_run_requested_at === null @@ -6917,6 +7215,7 @@ export const layer: Layer.Layer = latestRunStartedAt, latestRunCompletedAt, latestRunStopRequested, + latestRunBackgroundWake, activeRunId: row.active_run_id === null ? null : RunId.make(row.active_run_id), blockedOnThreadWait: deriveBlockedOnThreadWait({ activeRun: @@ -7102,7 +7401,7 @@ export const layer: Layer.Layer = SELECT t.payload_json FROM orchestration_v2_projection_threads t WHERE t.deleted_at IS NULL${ threadId === undefined - ? sql` AND json_extract(t.payload_json, '$.archivedAt') IS NULL` + ? sql` AND t.archived_at IS NULL` : sql` AND t.thread_id = ${threadId}` }${ unsettledOnly @@ -7187,6 +7486,7 @@ export const layer: Layer.Layer = activityRunStatus: state.activityRunStatus, latestRunCompletedAt: state.latestRunCompletedAt, latestRunStopRequested: state.latestRunStopRequested, + latestRunBackgroundWake: state.latestRunBackgroundWake, pendingRuntimeRequest: state.pendingRuntimeRequest, pendingBackgroundTasks: state.pendingBackgroundTasks, })), @@ -7522,6 +7822,7 @@ export const layer: Layer.Layer = getCheckpointCaptureContext: readOnReader(getCheckpointCaptureContext), getCheckpointRollbackContext: readOnReader(getCheckpointRollbackContext), getRunMessage: readOnReader(getRunMessage), + getRun: readOnReader(getRun), canStartQueuedRun: readOnReader(canStartQueuedRun), getHeldQueuedRuns: readOnReader(getHeldQueuedRuns), getPendingNativeUserInputs: readOnReader(getPendingNativeUserInputs), diff --git a/apps/server/src/orchestration-v2/ProviderCommitBatch.ts b/apps/server/src/orchestration-v2/ProviderCommitBatch.ts index 60cee2b632f6..1a6ca8d28a60 100644 --- a/apps/server/src/orchestration-v2/ProviderCommitBatch.ts +++ b/apps/server/src/orchestration-v2/ProviderCommitBatch.ts @@ -1,4 +1,3 @@ -import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as Config from "effect/Config"; @@ -19,8 +18,7 @@ export const enabled = Config.Boolean("T3CODE_PROVIDER_COMMIT_BATCHING").pipe( Effect.orDie, ); -type Event = ProviderAdapter.ProviderAdapterV2Event; -const isSnapshot = (event: Event) => +const isSnapshot = (event: { readonly type: string }) => event.type === "node.updated" || event.type === "message.updated" || event.type === "turn_item.updated" || @@ -232,9 +230,9 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( ), ); }); - const consume = Effect.fnUntraced(function* ( - source: Stream.Stream, - process: (event: Event) => Effect.Effect, + const consume = Effect.fnUntraced(function* ( + source: Stream.Stream, + process: (event: A) => Effect.Effect, after: Effect.Effect, canBatch: Effect.Effect = Effect.succeed(true), ) { diff --git a/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts b/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts index 2d52bf8b15b5..1827700ca16c 100644 --- a/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts @@ -1,5 +1,6 @@ import { assert, describe, it } from "@effect/vitest"; import { + CommandId, MessageId, ProjectId, ProviderDriverKind, @@ -22,6 +23,7 @@ import * as TestClock from "effect/testing/TestClock"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests"; +import * as Orchestrator from "./Orchestrator.ts"; import * as ProviderContinuationService from "./ProviderContinuationService.ts"; import * as ThreadManagementService from "./ThreadManagementService.ts"; @@ -418,6 +420,43 @@ describe("ProviderContinuationService", () => { }, ); + it.effect( + "clears a sticky continuation offer when a non-delegated wake dispatch fails generically", + () => { + return Effect.gen(function* () { + const cleared = yield* Deferred.make(); + const layerThreads = Layer.mock(ThreadManagementService.ThreadManagementService)({ + getThreadRecords: () => Effect.succeed(projection), + dispatch: (command) => + Effect.fail( + new Orchestrator.OrchestratorDispatchError({ + commandId: CommandId.make(String(command.commandId)), + commandType: command.type, + cause: new Error("SQLITE_FULL: database or disk is full"), + }), + ), + }); + const layerWorker = ProviderContinuationService.layer.pipe( + Layer.provide( + Layer.mergeAll(IdAllocator.layer, ProviderContinuationRequests.layer, layerThreads), + ), + ); + + yield* Effect.gen(function* () { + const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; + yield* requests.offer({ + ...request(), + clearIfCurrent: () => Deferred.succeed(cleared, undefined), + }); + yield* Deferred.await(cleared); + }).pipe( + Effect.provide(Layer.merge(ProviderContinuationRequests.layer, layerWorker)), + Effect.scoped, + ); + }); + }, + ); + it.effect( "drops stopped and disposed delegated completions instead of reviving them after recovery", () => { diff --git a/apps/server/src/orchestration-v2/ProviderContinuationService.ts b/apps/server/src/orchestration-v2/ProviderContinuationService.ts index 1fe30ee0645c..ac414413ca3b 100644 --- a/apps/server/src/orchestration-v2/ProviderContinuationService.ts +++ b/apps/server/src/orchestration-v2/ProviderContinuationService.ts @@ -274,6 +274,8 @@ export const layer = Layer.effectDiscard( ), Effect.forkScoped, ); + } else if (request.clearIfCurrent !== undefined) { + yield* request.clearIfCurrent(); } }), ), diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts index 9814e3030683..7cf409ab56ed 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts @@ -1641,6 +1641,73 @@ layer("ProviderEventIngestorV2", (it) => { }), ); + it.effect("keeps a steered message pending until the provider reports it read", () => + Effect.gen(function* () { + const now = yield* DateTime.now; + const eventSink = yield* EventSink.EventSinkV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const threadEvent = yield* threadCreatedEvent(now); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId: threadEvent.threadId, + }); + const runId = RunId.make("run:steer-read"); + const messageId = MessageId.make("message:steer-read"); + const steer: OrchestrationV2TurnItem = { + id: idAllocator.derive.userTurnItem({ messageId }), + threadId: threadEvent.threadId, + runId, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 1, + status: "completed", + title: null, + startedAt: now, + completedAt: now, + updatedAt: now, + type: "user_message", + createdBy: "user", + creationSource: "web", + messageId, + inputIntent: "steer", + text: "Steer behind a command", + attachments: [], + }; + yield* eventSink.write({ events: [threadEvent] }); + const ingest = (event: ProviderAdapter.ProviderAdapterV2Event) => + ingestor.ingestNormalized({ + providerSessionId, + providerInstanceId: modelSelection.instanceId, + threadId: threadEvent.threadId, + event, + }); + const projected = projectionStore.getTurnItem({ + threadId: threadEvent.threadId, + itemId: steer.id, + }); + const steerUpdated = (status: "pending" | "completed") => + ingest({ + type: "steer.updated", + driver: CODEX_DRIVER, + threadId: threadEvent.threadId, + runId, + messageId, + status, + }); + + yield* ingest({ type: "turn_item.updated", driver: CODEX_DRIVER, turnItem: steer }); + yield* steerUpdated("pending"); + assert.deepInclude(yield* projected, { status: "pending", completedAt: null }); + yield* steerUpdated("completed"); + assert.equal((yield* projected)?.status, "completed"); + }), + ); + it.effect("stores a bounded copy of a large file-change diff in both SQLite tables", () => Effect.gen(function* () { const now = yield* DateTime.now; diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts index 40a858bbeb79..623fddb49110 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts @@ -616,6 +616,35 @@ export const layer: Layer.Layer< }), ]; } + case "steer.updated": { + const item = yield* projections.getTurnItem({ + threadId: input.event.threadId, + itemId: idAllocator.derive.userTurnItem({ messageId: input.event.messageId }), + }); + if ( + item?.type !== "user_message" || + item.status === input.event.status || + (item.status !== "pending" && item.status !== "completed") + ) { + return []; + } + const occurredAt = yield* DateTime.now; + return [ + yield* makeDomainEvent(input, { + type: "turn-item.updated", + threadId: item.threadId, + payload: { + ...item, + status: input.event.status, + completedAt: input.event.status === "pending" ? null : occurredAt, + updatedAt: occurredAt, + }, + runId: item.runId, + nodeId: item.nodeId, + occurredAt, + }), + ]; + } case "background_work.updated": return []; case "command_output.delta": diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts index 642a642a80df..0197a10fae2e 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts @@ -771,6 +771,7 @@ it.effect( assert.equal(summary.stoppedSessions, 1); assert.equal(summary.retiredEffects, 2); assert.deepEqual(committedInput?.cancelUnsettledEffects?.effectTypes, [ + "provider-session.apply-model-selection", "provider-turn.start", "provider-turn.interrupt", "provider-turn.steer", @@ -793,6 +794,7 @@ it.effect( ["turn-item.updated", "cancelled"], ["turn-item.updated", "cancelled"], ["turn-item.updated", "cancelled"], + ["turn-item.updated", "completed"], ["provider-thread.updated", "idle"], ["provider-session.updated", "stopped"], ], diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts index 89e43e5b1ede..46b739bec573 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts @@ -634,6 +634,36 @@ export const make = Effect.gen(function* () { payload: cancelledItem(item, now), }); } + // Queued runs and unsent resumes leave no visible turn behind, so there is nothing to explain. + if (run.status === "queued" || unsentResumeRunIds.includes(run.id)) continue; + events.push({ + id: yield* allocateEventId(), + type: "turn-item.updated", + threadId: projection.thread.id, + runId: run.id, + ...(run.rootNodeId === null ? {} : { nodeId: run.rootNodeId }), + providerInstanceId: run.providerInstanceId, + occurredAt: now, + payload: { + id: ids.derive.runSignalTurnItem({ runId: run.id, signal: "server-recovery" }), + threadId: projection.thread.id, + runId: run.id, + nodeId: run.rootNodeId, + providerThreadId: run.providerThreadId, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + // EventSink allocates the real position, after the run's existing items. + ordinal: 0, + status: "completed", + title: trigger === "startup" ? "Server restarted" : "Server shut down", + startedAt: now, + completedAt: now, + updatedAt: now, + type: "system_notice", + message: detail, + }, + }); } // Blocking waits die with their turn; let child results wake the parent continuation as async tasks. const cutRunIds = new Set(runs.map((run) => run.id)); diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index 40bc51492a46..b3590ddc3dd9 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -42,19 +42,27 @@ import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderS import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; import * as ServerSettings from "../serverSettings.ts"; -import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; +import { + CodexProviderCapabilitiesV2, + normalizeCodexThreadLoadModelSelection, +} from "./Adapters/CodexAdapterV2.ts"; import * as EventSink from "./EventSink.ts"; import * as EventStore from "./EventStore.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; import * as ProviderEventIngestor from "./ProviderEventIngestor.ts"; +import * as ProviderInstallationGate from "../provider/ProviderInstallationGate.ts"; import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts"; import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts"; import * as ProviderSessionManager from "./ProviderSessionManager.ts"; import * as CommandOutputHub from "./CommandOutputHub.ts"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import { + makeProviderFailure, + makeProviderRetryTurnItem, +} from "@t3tools/provider-core/server/failure"; const layerTestDatabase = SqlitePersistence.layerMemory; const layerTestStores = Layer.merge(EventStore.layer, ProjectionStore.layer).pipe( @@ -423,6 +431,7 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( state: Ref.Ref, options: { readonly failEventStream?: boolean; + readonly normalizeThreadLoadModelSelection?: (selection: ModelSelection) => ModelSelection; readonly capabilities?: OrchestrationV2ProviderCapabilities; readonly extensionStatuses?: OrchestrationV2ProviderSession["extensionStatuses"]; readonly mcpConfigs?: Ref.Ref< @@ -509,6 +518,9 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( instanceId: ProviderInstanceId.make("codex"), driver: CODEX_DRIVER, providerSessionId: input.providerSessionId, + ...(options.normalizeThreadLoadModelSelection === undefined + ? {} + : { normalizeThreadLoadModelSelection: options.normalizeThreadLoadModelSelection }), get providerSession() { return session; }, @@ -577,6 +589,7 @@ function layerTest(input: { readonly idleTimeoutMs: number; readonly maxIdlePinMs?: number; readonly failEventStream?: boolean; + readonly normalizeThreadLoadModelSelection?: (selection: ModelSelection) => ModelSelection; readonly capabilities?: OrchestrationV2ProviderCapabilities; readonly extensionStatuses?: OrchestrationV2ProviderSession["extensionStatuses"]; readonly mcpConfigs?: Ref.Ref< @@ -605,6 +618,7 @@ function layerTest(input: { readonly scopeCloseReached?: Deferred.Deferred; readonly serverSettingsLayer?: ReturnType; readonly projectServiceLayer?: Layer.Layer; + readonly installationGateLayer?: Layer.Layer; }) { const layerConfiguredEventSink = input.flakyReleaseWrites !== undefined @@ -619,6 +633,9 @@ function layerTest(input: { const layerRegistry = Layer.unwrap( makeProviderAdapter(input.state, { failEventStream: input.failEventStream ?? false, + ...(input.normalizeThreadLoadModelSelection === undefined + ? {} + : { normalizeThreadLoadModelSelection: input.normalizeThreadLoadModelSelection }), ...(input.capabilities === undefined ? {} : { capabilities: input.capabilities }), ...(input.extensionStatuses === undefined ? {} @@ -687,6 +704,7 @@ function layerTest(input: { layerProviderEventIngestorTest, layerConfiguredMcpRegistry, layerTestStores, + input.installationGateLayer ?? ProviderInstallationGate.layer, ...(input.serverSettingsLayer === undefined ? [] : [input.serverSettingsLayer]), ...(input.projectServiceLayer === undefined ? [] : [input.projectServiceLayer]), ), @@ -1100,6 +1118,53 @@ it.effect("ProviderSessionManagerV2 closes every live session for a provider ins }), ); +it.effect("ProviderSessionManagerV2 opens provider sessions inside the installation gate", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const log: Array = []; + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const threadId = ThreadId.make("thread-provider-session-manager-installation-gate"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + yield* eventSink.write({ + events: [ + yield* makeThreadCreatedEvent({ idAllocator, threadId, now: yield* DateTime.now }), + ], + }); + + yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); + + assert.deepStrictEqual(log, [`enter ${modelSelection.instanceId}`, "openSession", "exit"]); + }); + + yield* effect.pipe( + Effect.provide( + layerTest({ + state, + idleTimeoutMs: 60_000, + beforeOpen: () => Effect.sync(() => log.push("openSession")), + installationGateLayer: Layer.succeed( + ProviderInstallationGate.ProviderInstallationGate, + ProviderInstallationGate.ProviderInstallationGate.of({ + execute: (instanceId, gated) => + Effect.sync(() => log.push(`enter ${instanceId}`)).pipe( + Effect.andThen(gated), + Effect.ensuring(Effect.sync(() => log.push("exit"))), + ), + update: () => Effect.die("unused"), + }), + ), + }), + ), + ); + }), +); + it.effect("ProviderSessionManagerV2 opens a duplicate session only once", () => Effect.gen(function* () { const state = yield* Ref.make(emptyState); @@ -2034,6 +2099,195 @@ it.effect("ProviderSessionManagerV2 drains subscribers when the provider stops", }), ); +it.effect("ProviderSessionManagerV2 fails a turn whose provider retry never resumes", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const interrupted = yield* Deferred.make(); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-provider-session-manager-retry-stall"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + const providerThread = makeProviderThread({ idAllocator, threadId, providerSessionId, now }); + yield* eventSink.write({ + events: [ + yield* makeThreadCreatedEvent({ idAllocator, threadId, now }), + { + id: yield* idAllocator.allocate.event({ threadId }), + type: "provider-thread.updated", + threadId, + occurredAt: now, + payload: providerThread, + }, + ], + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const subscription = yield* runtime.subscribeEvents!; + const published = yield* Queue.unbounded(); + // The pump updates the stall watch before it publishes a turn item. + const collected = yield* subscription.events.pipe( + Stream.tap((event) => + event.type === "turn_item.updated" ? Queue.offer(published, undefined) : Effect.void, + ), + Stream.runCollect, + Effect.forkScoped, + ); + const adapterQueue = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId)); + assert.isDefined(adapterQueue); + const providerTurnId = idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: "retry-stall-turn", + }); + const failure = makeProviderFailure({ + message: "Rate limit exceeded", + code: "rate_limited", + class: "usage_limit", + retryable: true, + }); + const retry = { attempt: 2, maxAttempts: 5, retryDelayMs: 60_000 }; + const retryItem = (status: "running" | "interrupted") => + makeProviderRetryTurnItem({ + driver: CODEX_DRIVER, + threadId, + runId: null, + nodeId: null, + providerThreadId: providerThread.id, + providerTurnId, + itemOrdinal: 3, + failure, + retry, + status, + startedAt: now, + updatedAt: now, + }); + yield* Queue.offer(adapterQueue!, { + type: "turn_item.updated", + driver: CODEX_DRIVER, + turnItem: retryItem("running"), + }); + yield* Queue.take(published); + const offerOutput = (outputThreadId: ThreadId) => + Queue.offer(adapterQueue!, { + type: "turn_item.updated", + driver: CODEX_DRIVER, + turnItem: { + id: idAllocator.derive.turnItemFromProviderItem({ + driver: CODEX_DRIVER, + nativeItemId: `retry-stall-output-${outputThreadId}`, + }), + threadId: outputThreadId, + runId: null, + nodeId: null, + providerThreadId: providerThread.id, + providerTurnId, + nativeItemRef: null, + parentItemId: null, + ordinal: 4, + type: "compaction", + driver: CODEX_DRIVER, + status: "running", + title: "Compacting context", + startedAt: now, + completedAt: null, + updatedAt: now, + }, + }).pipe(Effect.andThen(Queue.take(published))); + // The stall window is 15 minutes plus the provider's announced retry delay. + yield* TestClock.adjust("15 minutes"); + assert.equal((yield* Ref.get(state)).interruptCount, 0); + // Output on the retry's thread proves the provider is alive, so a new window starts. + yield* offerOutput(threadId); + yield* TestClock.adjust("1 minute"); + // Another thread's output lets the timer start its next wait and does not count. + yield* offerOutput(ThreadId.make("thread-provider-session-manager-retry-stall-other")); + yield* TestClock.adjust("15 minutes"); + assert.equal((yield* Ref.get(state)).interruptCount, 0); + yield* TestClock.adjust("1 minute"); + yield* Deferred.await(interrupted); + yield* Queue.offer(adapterQueue!, { + type: "turn_item.updated", + driver: CODEX_DRIVER, + turnItem: retryItem("interrupted"), + }); + yield* Queue.offer(adapterQueue!, { + type: "provider_turn.updated", + driver: CODEX_DRIVER, + threadId, + providerTurn: { + id: providerTurnId, + providerThreadId: providerThread.id, + nodeId: idAllocator.derive.rootNode({ + runId: idAllocator.derive.run({ threadId, ordinal: 1 }), + }), + runAttemptId: null, + nativeTurnRef: null, + ordinal: 1, + status: "interrupted", + startedAt: now, + completedAt: now, + }, + }); + yield* Queue.offer(adapterQueue!, { + type: "turn.terminal", + driver: CODEX_DRIVER, + providerThreadId: providerThread.id, + providerTurnId, + runOrdinal: 1, + status: "interrupted", + failure: null, + threadDisposition: "reusable", + }); + yield* Queue.offer(adapterQueue!, { + type: "provider_session.updated", + driver: CODEX_DRIVER, + providerSession: { ...runtime.providerSession, status: "stopped", updatedAt: now }, + }); + yield* Queue.end(adapterQueue!); + + const events = Array.from(yield* Fiber.join(collected)); + const terminal = events.find((event) => event.type === "turn.terminal"); + const providerTurn = events.find((event) => event.type === "provider_turn.updated"); + assert.equal( + providerTurn?.type === "provider_turn.updated" ? providerTurn.providerTurn.status : null, + "failed", + ); + assert.deepEqual(terminal, { + type: "turn.terminal", + driver: CODEX_DRIVER, + providerThreadId: providerThread.id, + providerTurnId, + runOrdinal: 1, + failureItemOrdinal: 3, + status: "failed", + failure: { ...failure, retryable: false }, + retry, + retryStartedAt: now, + threadDisposition: "reusable", + }); + }); + + yield* effect.pipe( + Effect.provide( + layerTest({ + state, + idleTimeoutMs: 60 * 60_000, + interruptTurn: Deferred.succeed(interrupted, undefined).pipe(Effect.asVoid), + }), + ), + ); + }), +); + it.effect( "ProviderSessionManagerV2 issues MCP credentials before opening and revokes them on close", () => @@ -3619,94 +3873,277 @@ it.effect.each(["manual_shutdown", "server_shutdown"] as const)( }), ); -it.effect("ProviderSessionManagerV2 persists session-scoped runtime requests without a run", () => - Effect.gen(function* () { - const state = yield* Ref.make(emptyState); - const effect = Effect.gen(function* () { - const eventSink = yield* EventSink.EventSinkV2; - const idAllocator = yield* IdAllocator.IdAllocatorV2; - const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; - const projectionStore = yield* ProjectionStore.ProjectionStoreV2; - const now = yield* DateTime.now; - const projectId = yield* idAllocator.allocate.project({ - fixtureName: "provider-session-manager-session-request", - }); - const threadId = yield* idAllocator.allocate.thread({ - fixtureName: "provider-session-manager-session-request", - projectId, - }); - const providerSessionId = yield* idAllocator.allocate.providerSession({ - providerInstanceId: modelSelection.instanceId, - threadId, - }); - const providerThread = makeProviderThread({ - idAllocator, - threadId, - providerSessionId, - now, - }); +it.effect( + "ProviderSessionManagerV2 cancels a pending approval when a detach releases its session", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const projectId = yield* idAllocator.allocate.project({ + fixtureName: "provider-session-manager-detach-request", + }); + const threadId = yield* idAllocator.allocate.thread({ + fixtureName: "provider-session-manager-detach-request", + projectId, + }); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); - yield* eventSink.write({ - events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], - }); - yield* manager.open({ - threadId, - providerSessionId, - modelSelection, - runtimePolicy, - }); - const pendingRequest = yield* makePendingRuntimeRequestEvents({ - idAllocator, - threadId, - providerSessionId, - providerThread, - now, + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const pendingRequest = yield* makePendingRuntimeRequestEvents({ + idAllocator, + threadId, + providerSessionId, + providerThread, + now, + }); + yield* eventSink.write({ events: pendingRequest.events }); + yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); + + yield* manager.detach({ providerSessionId, threadId, detail: "Workspace changed." }); + + assert.isTrue(Option.isNone(yield* manager.get(providerSessionId))); + const projection = yield* projectionStore.getThreadProjection(threadId); + const request = projection.runtimeRequests.find( + (candidate) => candidate.id === pendingRequest.requestId, + ); + assert.equal(request?.status, "cancelled"); + assert.equal(request?.responseCapability.type, "not_resumable"); }); - const afterSequence = yield* eventSink.latestSequence({ threadId }); - const persistedFiber = yield* eventSink.stream({ threadId, afterSequence }).pipe( - Stream.filter( - (stored) => - stored.event.type === "runtime-request.updated" || - stored.event.type === "node.updated" || - stored.event.type === "turn-item.updated", - ), - Stream.take(3), - Stream.runCollect, - Effect.forkScoped, - ); - const adapterEvents = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId)); - assert.isDefined(adapterEvents); - yield* Queue.offerAll(adapterEvents!, pendingRequest.providerEvents); - const persisted = Array.from(yield* Fiber.join(persistedFiber)); - assert.sameMembers( - persisted.map((stored) => stored.event.type), - ["runtime-request.updated", "node.updated", "turn-item.updated"], - ); - const projection = yield* projectionStore.getThreadProjection(threadId); - const request = projection.runtimeRequests.find( - (candidate) => candidate.id === pendingRequest.requestId, - ); - const node = projection.nodes.find((candidate) => candidate.id === pendingRequest.nodeId); - const turnItem = projection.turnItems.find( - (candidate) => - candidate.type === "approval_request" && candidate.requestId === pendingRequest.requestId, + yield* effect.pipe( + Effect.provide( + layerTest({ state, idleTimeoutMs: 60_000, capabilities: ExclusiveCapabilities }), + ), ); - assert.equal(request?.status, "pending"); - assert.equal(request?.providerTurnId, null); - assert.equal(node?.runId, null); - assert.equal(node?.status, "waiting"); - assert.equal(turnItem?.runId, null); - assert.equal(turnItem?.status, "waiting"); - }); - - yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 1000 }))); - }), + }), ); it.effect( - "ProviderSessionManagerV2 opens one shared runtime, broadcasts events, and detaches threads independently", + "ProviderSessionManagerV2 persists session-scoped requests and recovered commands without a run", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const effect = Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const projectId = yield* idAllocator.allocate.project({ + fixtureName: "provider-session-manager-session-request", + }); + const threadId = yield* idAllocator.allocate.thread({ + fixtureName: "provider-session-manager-session-request", + projectId, + }); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + }); + + const otherThreadId = yield* idAllocator.allocate.thread({ + fixtureName: "unattached-shell-owner", + projectId, + }); + yield* eventSink.write({ + events: [ + yield* makeThreadCreatedEvent({ idAllocator, threadId, now }), + yield* makeThreadCreatedEvent({ idAllocator, threadId: otherThreadId, now }), + ], + }); + yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const pendingRequest = yield* makePendingRuntimeRequestEvents({ + idAllocator, + threadId, + providerSessionId, + providerThread, + now, + }); + const shellId = "sh_recovered"; + const command = { + id: idAllocator.derive.turnItemFromProviderItem({ + driver: CODEX_DRIVER, + nativeItemId: shellId, + }), + threadId, + runId: null, + nodeId: null, + providerThreadId: providerThread.id, + providerTurnId: null, + nativeItemRef: { driver: CODEX_DRIVER, nativeId: shellId, strength: "strong" as const }, + parentItemId: null, + ordinal: 0, + status: "running" as const, + title: null, + startedAt: now, + completedAt: null, + updatedAt: now, + type: "command_execution" as const, + input: "echo retained", + output: "retained", + }; + const anchored = { + ...command, + id: idAllocator.derive.turnItemFromProviderItem({ + driver: CODEX_DRIVER, + nativeItemId: "sh_anchored", + }), + runId: idAllocator.derive.run({ threadId, ordinal: 1 }), + }; + const commandEvents = [ + { + type: "turn_item.updated" as const, + driver: CODEX_DRIVER, + turnItem: anchored, + sessionScoped: true, + }, + { + type: "turn_item.updated" as const, + driver: CODEX_DRIVER, + turnItem: command, + sessionScoped: true, + }, + { + type: "provider_thread.updated" as const, + driver: CODEX_DRIVER, + providerThread: { + ...providerThread, + pendingBackgroundTasks: [{ taskId: shellId, kind: "command" as const }], + }, + sessionScoped: true, + }, + { + type: "turn_item.updated" as const, + driver: CODEX_DRIVER, + sessionScoped: true, + turnItem: { + ...command, + status: "completed" as const, + output: "retained\nfinished", + completedAt: now, + }, + }, + { + type: "provider_thread.updated" as const, + driver: CODEX_DRIVER, + providerThread: { ...providerThread, pendingBackgroundTasks: [] }, + sessionScoped: true, + }, + ]; + const afterSequence = yield* eventSink.latestSequence({ threadId }); + const persistedFiber = yield* eventSink.stream({ threadId, afterSequence }).pipe( + Stream.filter( + (stored) => + stored.event.type === "runtime-request.updated" || + stored.event.type === "node.updated" || + (stored.event.type === "turn-item.updated" && + stored.event.payload.type === "approval_request"), + ), + Stream.take(3), + Stream.runCollect, + Effect.forkScoped, + ); + const adapterEvents = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId)); + assert.isDefined(adapterEvents); + yield* Queue.offerAll(adapterEvents!, [ + ...commandEvents, + { + type: "turn_item.updated", + driver: CODEX_DRIVER, + sessionScoped: true, + turnItem: { ...command, threadId: otherThreadId }, + }, + { + type: "provider_thread.updated", + driver: CODEX_DRIVER, + providerThread: { ...providerThread, appThreadId: otherThreadId }, + sessionScoped: true, + }, + ...pendingRequest.providerEvents, + ]); + const persisted = Array.from(yield* Fiber.join(persistedFiber)); + + assert.sameMembers( + persisted.map((stored) => stored.event.type), + ["runtime-request.updated", "node.updated", "turn-item.updated"], + ); + const projection = yield* projectionStore.getThreadProjection(threadId); + const request = projection.runtimeRequests.find( + (candidate) => candidate.id === pendingRequest.requestId, + ); + const node = projection.nodes.find((candidate) => candidate.id === pendingRequest.nodeId); + const turnItem = projection.turnItems.find( + (candidate) => + candidate.type === "approval_request" && + candidate.requestId === pendingRequest.requestId, + ); + const recovered = projection.turnItems.find((item) => item.id === command.id); + assert.deepInclude(recovered, { + runId: null, + status: "completed", + output: "retained\nfinished", + }); + assert.deepEqual( + projection.providerThreads.find((item) => item.id === providerThread.id) + ?.pendingBackgroundTasks, + [], + ); + assert.deepInclude( + projection.turnItems.find((item) => item.id === anchored.id), + { runId: anchored.runId, output: "retained" }, + ); + assert.isEmpty(projection.runs); + const unrelated = yield* projectionStore.getThreadProjection(otherThreadId); + assert.isEmpty(unrelated.turnItems); + assert.isEmpty(unrelated.providerThreads); + assert.equal(request?.status, "pending"); + assert.equal(request?.providerTurnId, null); + assert.equal(node?.runId, null); + assert.equal(node?.status, "waiting"); + assert.equal(turnItem?.runId, null); + assert.equal(turnItem?.status, "waiting"); + }); + + yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 1000 }))); + }), +); + +it.effect.each([ + { name: "default load key", normalizeThreadLoadModelSelection: undefined }, + { + name: "Codex load key", + normalizeThreadLoadModelSelection: normalizeCodexThreadLoadModelSelection, + }, +])( + "ProviderSessionManagerV2 opens one shared runtime, broadcasts events, and detaches threads independently ($name)", + ({ normalizeThreadLoadModelSelection }) => Effect.gen(function* () { const state = yield* Ref.make(emptyState); const settings = yield* ServerSettings.ServerSettingsService.pipe( @@ -3803,25 +4240,91 @@ it.effect( const resumeSecondThread = secondRuntime.resumeThread({ providerThread: secondProviderThread, threadId: secondThreadId, - modelSelection, + modelSelection: { + ...modelSelection, + options: [{ id: "reasoningEffort", value: "medium" }], + }, runtimePolicy, }); yield* resumeSecondThread; yield* resumeSecondThread; assert.equal((yield* Ref.get(state)).resumeCount, 1); + let resumeCount = 1; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const appThread = (yield* projectionStore.getThreadProjection(secondThreadId)).thread; + const turnEvents = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId)); + assert.isDefined(turnEvents); + for (const [index, effort] of ["medium", "high", "medium"].entries()) { + const selection = { + ...modelSelection, + options: [{ id: "reasoningEffort", value: effort }], + } satisfies ModelSelection; + yield* secondRuntime.resumeThread({ + providerThread: secondProviderThread, + threadId: secondThreadId, + modelSelection: selection, + runtimePolicy, + }); + const ordinal = index + 1; + const runId = idAllocator.derive.run({ threadId: secondThreadId, ordinal }); + yield* secondRuntime.startTurn({ + appThread, + threadId: secondThreadId, + runId, + runOrdinal: ordinal, + providerTurnOrdinal: ordinal, + attemptId: idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }), + rootNodeId: idAllocator.derive.rootNode({ runId }), + providerThread: secondProviderThread, + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* idAllocator.allocate.message({ + threadId: secondThreadId, + ordinal, + }), + text: "turn", + attachments: [], + }, + modelSelection: selection, + runtimePolicy, + }); + const subscription = yield* secondRuntime.subscribeEvents!; + const terminal = yield* subscription.events.pipe( + Stream.filter((event) => event.type === "turn.terminal"), + Stream.runHead, + Effect.forkScoped, + ); + yield* Queue.offer(turnEvents!, { + type: "turn.terminal", + driver: CODEX_DRIVER, + providerThreadId: secondProviderThread.id, + providerTurnId: idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: `native-effort-turn-${ordinal}`, + }), + runOrdinal: ordinal, + status: "completed", + failure: null, + threadDisposition: "reusable", + }); + yield* Fiber.join(terminal); + if (normalizeThreadLoadModelSelection === undefined && index > 0) resumeCount++; + assert.equal((yield* Ref.get(state)).resumeCount, resumeCount, `${effort} effort reload`); + } yield* secondRuntime.resumeThread({ providerThread: secondProviderThread, threadId: secondThreadId, modelSelection: { ...modelSelection, model: "gpt-5.4-mini" }, runtimePolicy, }); - assert.equal((yield* Ref.get(state)).resumeCount, 2); + assert.equal((yield* Ref.get(state)).resumeCount, ++resumeCount); yield* resumeSecondThread; - assert.equal((yield* Ref.get(state)).resumeCount, 3); + assert.equal((yield* Ref.get(state)).resumeCount, ++resumeCount); // regression: #26722 a Linked environments change reloads only the idle thread's tools. yield* settings.updateSettings({ linkedEnvironmentsEnabled: true }); yield* resumeSecondThread; - assert.equal((yield* Ref.get(state)).resumeCount, 4); + assert.equal((yield* Ref.get(state)).resumeCount, ++resumeCount); assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, ["native-thread"]); const subscribe = firstRuntime.subscribeEvents; assert.isDefined(subscribe); @@ -3854,10 +4357,43 @@ it.effect( runtimePolicy, }); yield* resumeSecondThread; - assert.equal((yield* Ref.get(state)).resumeCount, 5); + assert.equal((yield* Ref.get(state)).resumeCount, ++resumeCount); assert.deepEqual((yield* Ref.get(state)).unloadedNativeThreadIds, ["native-thread"]); + const reloadCases = [ + { + name: "other model options", + providerThread: secondProviderThread, + selection: { + ...modelSelection, + options: [{ id: "serviceTier", value: "priority" }], + }, + policy: runtimePolicy, + }, + { + name: "runtime policy", + providerThread: secondProviderThread, + selection: modelSelection, + policy: { ...runtimePolicy, interactionMode: "plan" }, + }, + ] satisfies ReadonlyArray<{ + name: string; + providerThread: OrchestrationV2ProviderThread; + selection: ModelSelection; + policy: ProviderAdapter.ProviderAdapterV2RuntimePolicy; + }>; + for (const { name, providerThread, selection, policy } of reloadCases) { + const before = (yield* Ref.get(state)).resumeCount; + yield* secondRuntime.resumeThread({ + providerThread, + threadId: secondThreadId, + modelSelection: selection, + runtimePolicy: policy, + }); + assert.equal((yield* Ref.get(state)).resumeCount, before + 1, `${name} did not reload`); + } + yield* manager.detach({ providerSessionId, threadId: firstThreadId }); assert.isTrue(Option.isSome(yield* manager.get(providerSessionId))); assert.equal((yield* Ref.get(state)).closeCount, 0); @@ -3878,6 +4414,9 @@ it.effect( layerTest({ state, idleTimeoutMs: 1000, + ...(normalizeThreadLoadModelSelection === undefined + ? {} + : { normalizeThreadLoadModelSelection }), serverSettingsLayer: Layer.succeed(ServerSettings.ServerSettingsService, settings), }), ), diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index c4b8e94542d7..7add110f3a8f 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -12,7 +12,9 @@ import { ProviderInstanceId, ProviderSessionId, ThreadId, + type OrchestrationV2TurnItem, type ProviderThreadId, + type ProviderTurnId, } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; @@ -37,6 +39,7 @@ import * as SubscriptionRef from "effect/SubscriptionRef"; // Coder: no provider session or turn metrics (`withMetrics`); metrics are not carried. import { ProviderWorkspaceMissingError } from "../provider/Errors.ts"; +import * as ProviderInstallationGate from "../provider/ProviderInstallationGate.ts"; import * as ProjectService from "../project/ProjectService.ts"; import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions"; import * as ServerSettings from "../serverSettings.ts"; @@ -60,6 +63,14 @@ const busyTurnPrefix = (providerThreadId: ProviderThreadId) => `${providerThread const busyTurnKey = (providerThreadId: ProviderThreadId, runOrdinal: number) => `${busyTurnPrefix(providerThreadId)}${runOrdinal}`; const UNLOAD_THREAD_TIMEOUT_MS = 10 * 1000; +// Providers can announce a retry and never resume. Longer than any adapter's own stall handling. +const PROVIDER_RETRY_STALL_TIMEOUT = Duration.minutes(15); +type ProviderErrorItem = Extract; +type ProviderRetryItem = ProviderErrorItem & { + readonly providerTurnId: ProviderTurnId; + readonly providerThreadId: ProviderThreadId; + readonly retry: NonNullable; +}; /** Suggestions outlive their sessions, so keep only the most recently updated threads. */ const MAX_PROMPT_SUGGESTIONS = 256; const WORKING_RUN_STATUSES: ReadonlySet = new Set([ @@ -275,10 +286,14 @@ function sessionKey(providerSessionId: ProviderSessionId): string { return String(providerSessionId); } -function sessionScopedRuntimeRequestThreadId( +function sessionScopedArtifactThreadId( event: ProviderAdapter.ProviderAdapterV2Event, ): ThreadId | undefined { switch (event.type) { + case "provider_thread.updated": + return event.sessionScoped === true + ? (event.providerThread.appThreadId ?? undefined) + : undefined; case "runtime_request.updated": return event.runtimeRequest.providerTurnId === null ? event.threadId : undefined; case "node.updated": @@ -286,6 +301,8 @@ function sessionScopedRuntimeRequestThreadId( ? event.node.threadId : undefined; case "turn_item.updated": + if (event.turnItem.type === "command_execution") + return event.sessionScoped === true ? event.turnItem.threadId : undefined; return event.turnItem.runId === null && (event.turnItem.type === "approval_request" || event.turnItem.type === "user_input_request") ? event.turnItem.threadId @@ -307,15 +324,21 @@ function providerThreadRuntimeKey( } function providerThreadLoadKey(input: { + readonly runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime; readonly providerThread: Parameters< ProviderAdapter.ProviderAdapterV2SessionRuntime["resumeThread"] >[0]["providerThread"]; readonly modelSelection?: ModelSelection; readonly runtimePolicy?: ProviderAdapter.ProviderAdapterV2RuntimePolicy; }): string { + const modelSelection = + input.modelSelection === undefined + ? undefined + : (input.runtime.normalizeThreadLoadModelSelection?.(input.modelSelection) ?? + input.modelSelection); return JSON.stringify({ providerThread: providerThreadRuntimeKey(input.providerThread), - modelSelection: input.modelSelection ?? null, + modelSelection: modelSelection ?? null, runtimePolicy: input.runtimePolicy ?? null, }); } @@ -334,6 +357,7 @@ export const layerWithOptions = ( | ProjectionStore.ProjectionStoreV2 | ProviderEventIngestor.ProviderEventIngestorV2 | ProviderAdapterRegistry.ProviderAdapterRegistryV2 + | ProviderInstallationGate.ProviderInstallationGate > => Layer.effect( ProviderSessionManagerV2, @@ -354,6 +378,7 @@ export const layerWithOptions = ( const providerEventIngestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; const projectionStore = yield* ProjectionStore.ProjectionStoreV2; const agentScope = yield* AgentScope; + const installationGate = yield* ProviderInstallationGate.ProviderInstallationGate; // Labs flag, read live: Codex per turn (its shared session outlives a toggle), Claude per query. // Unreadable settings keep it off. const promptSuggestionsEnabled = Option.isNone(serverSettings) @@ -659,12 +684,23 @@ export const layerWithOptions = ( }); }); + // A detach removes its thread before releasing, but that thread's records still need releasing. + const releasedThreadIds = (input: { + readonly entry: LiveSessionEntry; + readonly detachedThreadId?: ThreadId; + }) => { + const threadIds = new Set(input.entry.attachedThreadIds); + if (input.detachedThreadId !== undefined) threadIds.add(input.detachedThreadId); + return threadIds; + }; + const writeReleasedRuntimeRequestEvents = Effect.fn( "ProviderSessionManager.writeReleasedRuntimeRequestEvents", )(function* (input: { readonly entry: LiveSessionEntry; readonly reason: ProviderSessionReleaseReason; readonly releasedAt: DateTime.Utc; + readonly detachedThreadId?: ThreadId; }) { const providerSessionId = input.entry.runtime.providerSessionId; const now = yield* DateTime.now; @@ -675,7 +711,7 @@ export const layerWithOptions = ( : "Provider session was closed before this runtime request was resolved."; const events: Array = []; - for (const threadId of input.entry.attachedThreadIds) { + for (const threadId of releasedThreadIds(input)) { const projection = yield* projectionStore.getThreadRecords( threadId, ["runtimeRequests"], @@ -794,10 +830,8 @@ export const layerWithOptions = ( return []; } const providerSessionId = input.entry.runtime.providerSessionId; - const threadIds = new Set(input.entry.attachedThreadIds); - if (input.detachedThreadId !== undefined) threadIds.add(input.detachedThreadId); const released = []; - for (const threadId of threadIds) { + for (const threadId of releasedThreadIds(input)) { const projection = yield* projectionStore.getThreadRecords(threadId, [ "providerThreads", ]); @@ -886,6 +920,7 @@ export const layerWithOptions = ( readonly detail?: string; readonly releasedBackgroundTasks: ReadonlyArray; readonly releasedAt: DateTime.Utc; + readonly detachedThreadId?: ThreadId; readonly replaced: boolean; }) => Effect.all( @@ -1064,12 +1099,14 @@ export const layerWithOptions = ( // @effect-diagnostics-next-line effectFnOpportunity:off onSome: (entry) => Effect.gen(function* () { + const detachedThread = + input.detachedThreadId === undefined + ? {} + : { detachedThreadId: input.detachedThreadId }; const releasedBackgroundTasks = yield* readReleasedBackgroundTasks({ entry, reason: input.reason, - ...(input.detachedThreadId === undefined - ? {} - : { detachedThreadId: input.detachedThreadId }), + ...detachedThread, }); if (input.cancelIdleFiber !== false) { yield* cancelIdleFiber(entry.idleFiber); @@ -1094,6 +1131,7 @@ export const layerWithOptions = ( ...(input.detail === undefined ? {} : { detail: input.detail }), releasedBackgroundTasks, releasedAt, + ...detachedThread, }; const recorded = yield* Effect.exit( writeReleaseRecords({ ...records, replaced: false }), @@ -1932,6 +1970,7 @@ export const layerWithOptions = ( providerSessionId, threadId: input.threadId, providerThreadKey: providerThreadLoadKey({ + runtime, providerThread, modelSelection: input.modelSelection, runtimePolicy: input.runtimePolicy, @@ -1947,6 +1986,7 @@ export const layerWithOptions = ( return runtime.resumeThread(input); } const providerThreadKey = providerThreadLoadKey({ + runtime, providerThread: input.providerThread, ...(input.modelSelection === undefined ? {} @@ -1991,6 +2031,7 @@ export const layerWithOptions = ( providerSessionId, threadId, providerThreadKey: providerThreadLoadKey({ + runtime, providerThread, ...(input.modelSelection === undefined ? {} @@ -2020,6 +2061,7 @@ export const layerWithOptions = ( providerSessionId, threadId: input.targetThreadId, providerThreadKey: providerThreadLoadKey({ + runtime, providerThread, ...(input.modelSelection === undefined ? {} @@ -2155,21 +2197,198 @@ export const layerWithOptions = ( ), ); + const makeRetryStallWatch = (entry: LiveSessionEntry) => { + const runningRetries = new Map(); + const stalledRetries = new Map(); + const stallTimers = new Map>(); + // Claude and Pi keep the retry row running while a recovered reply streams, so any + // event on the retry's thread counts as the provider being alive. + const activeSinceCheck = new Set(); + const noteThreadActivity = (event: ProviderAdapter.ProviderAdapterV2Event) => { + if (runningRetries.size === 0) return; + const threadId = + event.type === "turn_item.updated" + ? event.turnItem.threadId + : event.type === "message.updated" + ? event.message.threadId + : event.type === "node.updated" + ? event.node.threadId + : event.type === "plan.updated" + ? event.plan.threadId + : undefined; + if (threadId === undefined) return; + for (const item of runningRetries.values()) { + if (item.threadId === threadId) activeSinceCheck.add(item.providerTurnId); + } + }; + // Stops the stalled turn; its interrupted terminal is then reported as the retry's failure. + const stopStalledRetry = Effect.fn("ProviderSessionManager.stopStalledRetry")( + function* (item: ProviderRetryItem) { + if (runningRetries.get(item.providerTurnId) !== item) return; + // The terminal this stop causes must not interrupt the stop itself. + stallTimers.delete(item.providerTurnId); + const current = (yield* Ref.get(sessions)).get( + sessionKey(entry.runtime.providerSessionId), + ); + if (current?.runtime !== entry.runtime) return; + const { providerThread } = yield* projectionStore.getProviderControlContext( + item.threadId, + { providerThreadId: item.providerThreadId, providerTurnId: item.providerTurnId }, + ); + if (providerThread === undefined || runningRetries.get(item.providerTurnId) !== item) + return; + stalledRetries.set(item.providerTurnId, item); + yield* entry.exposedRuntime + .interruptTurn({ providerThread, providerTurnId: item.providerTurnId }) + .pipe( + // A slow stop can still end the turn later, so that terminal keeps the retry failure. + Effect.catchTags({ + ProviderAdapterInterruptTimeoutError: (cause) => + Effect.logWarning("orchestration-v2.provider-retry-stall-stop-timed-out", { + providerSessionId: entry.runtime.providerSessionId, + cause, + }), + }), + ); + }, + (effect, item) => + effect.pipe( + Effect.catchCause((cause) => + Effect.sync(() => stalledRetries.delete(item.providerTurnId)).pipe( + Effect.andThen( + Effect.logWarning("orchestration-v2.provider-retry-stall-stop-failed", { + providerSessionId: entry.runtime.providerSessionId, + cause, + }), + ), + ), + ), + ), + ); + const stopStallTimer = (providerTurnId: ProviderTurnId) => { + const fiber = stallTimers.get(providerTurnId); + stallTimers.delete(providerTurnId); + activeSinceCheck.delete(providerTurnId); + return fiber === undefined ? Effect.void : Fiber.interrupt(fiber); + }; + const trackProviderRetry = ( + event: ProviderAdapter.ProviderAdapterV2Event, + ): { + readonly event: ProviderAdapter.ProviderAdapterV2Event; + readonly armed?: ProviderRetryItem; + readonly settled?: ProviderTurnId; + } => { + noteThreadActivity(event); + if (event.type === "turn_item.updated") { + const item = event.turnItem; + if ( + item.type !== "error" || + item.retry === undefined || + item.providerTurnId === null || + item.providerThreadId === null + ) + return { event }; + const retryItem = { + ...item, + providerTurnId: item.providerTurnId, + providerThreadId: item.providerThreadId, + retry: item.retry, + }; + if (item.status === "running") { + runningRetries.set(retryItem.providerTurnId, retryItem); + return { event, armed: retryItem }; + } + // A stall marker is set only once the stop is sent, so only the terminal clears it. + runningRetries.delete(retryItem.providerTurnId); + return { event, settled: retryItem.providerTurnId }; + } + if (event.type === "provider_turn.updated") { + if (event.providerTurn.status === "running") return { event }; + runningRetries.delete(event.providerTurn.id); + const settled = event.providerTurn.id; + return stalledRetries.has(settled) && + (event.providerTurn.status === "interrupted" || + event.providerTurn.status === "cancelled") + ? { + event: { ...event, providerTurn: { ...event.providerTurn, status: "failed" } }, + settled, + } + : { event, settled }; + } + if (event.type !== "turn.terminal") return { event }; + runningRetries.delete(event.providerTurnId); + const stalled = stalledRetries.get(event.providerTurnId); + stalledRetries.delete(event.providerTurnId); + const settled = event.providerTurnId; + if ( + stalled === undefined || + (event.status !== "interrupted" && event.status !== "cancelled") + ) + return { event, settled }; + return { + settled, + event: { + type: "turn.terminal", + driver: event.driver, + providerThreadId: event.providerThreadId, + providerTurnId: event.providerTurnId, + runOrdinal: event.runOrdinal, + failureItemOrdinal: stalled.ordinal, + status: "failed", + failure: { ...stalled.failure, retryable: false }, + retry: stalled.retry, + ...(stalled.startedAt === null ? {} : { retryStartedAt: stalled.startedAt }), + threadDisposition: event.threadDisposition, + }, + }; + }; + return { + track: trackProviderRetry, + // Runs for every provider event; spans would be per-event noise. Timers are children + // of the pump fiber, so a released session takes them down. + update: Effect.fnUntraced(function* (tracked: ReturnType) { + const item = tracked.armed; + if (item === undefined) { + if (tracked.settled !== undefined) yield* stopStallTimer(tracked.settled); + return; + } + yield* stopStallTimer(item.providerTurnId); + const window = Duration.sum( + PROVIDER_RETRY_STALL_TIMEOUT, + Duration.millis(item.retry.retryDelayMs ?? 0), + ); + const timer = yield* Effect.gen(function* () { + // Only a thread that stayed silent for a whole window has stalled. + do { + activeSinceCheck.delete(item.providerTurnId); + yield* Effect.sleep(window); + } while (activeSinceCheck.has(item.providerTurnId)); + yield* stopStalledRetry(item); + }).pipe(Effect.forkChild({ startImmediately: true })); + stallTimers.set(item.providerTurnId, timer); + }), + }; + }; + const startEventPump = (entry: LiveSessionEntry) => { let stoppedByProvider = false; let lastSessionStatus = entry.runtime.providerSession.status; + const retryStalls = makeRetryStallWatch(entry); return entry.runtime.events.pipe( - Stream.runForEach((event) => { + Stream.runForEach((nativeEvent) => { if (shutdownSignal.received) return Effect.void; // Live output bypasses persistence and subscribers so floods cannot delay domain events. - if (event.type === "command_output.delta") { - return commandOutput.append(event); + if (nativeEvent.type === "command_output.delta") { + return commandOutput.append(nativeEvent); } - if (event.type === "prompt_suggestion.updated") { + if (nativeEvent.type === "prompt_suggestion.updated") { return SubscriptionRef.update(promptSuggestionStore, (current) => { const next = new Map(current); - next.delete(event.threadId); - next.set(event.threadId, { runId: event.runId, text: event.text }); + next.delete(nativeEvent.threadId); + next.set(nativeEvent.threadId, { + runId: nativeEvent.runId, + text: nativeEvent.text, + }); for (const threadId of next.keys()) { if (next.size <= MAX_PROMPT_SUGGESTIONS) break; next.delete(threadId); @@ -2177,6 +2396,9 @@ export const layerWithOptions = ( return next; }); } + const tracked = retryStalls.track(nativeEvent); + const event = tracked.event; + const armRetryStall = retryStalls.update(tracked); const releaseLiveOutput = event.type === "turn_item.updated" && event.turnItem.type === "command_execution" && @@ -2206,6 +2428,7 @@ export const layerWithOptions = ( ? Effect.void : touchActivity(entry.runtime.providerSessionId, { providerActivity: true }), ).pipe( + Effect.andThen(armRetryStall), Effect.andThen( event.type === "provider_session.updated" ? persistProviderSessionUpdate(entry, event) @@ -2232,17 +2455,21 @@ export const layerWithOptions = ( }); }); } - // Some providers can block before a run subscriber exists - // (project trust, login, or session-switch hooks). Persist - // their runless request artifacts directly so the normal T3 - // request UI can answer them and unblock session setup. - const threadId = sessionScopedRuntimeRequestThreadId(event); + // Requests and recovered background commands can arrive before a run subscribes. + const threadId = sessionScopedArtifactThreadId(event); if (threadId !== undefined) { yield* Effect.gen(function* () { const current = (yield* Ref.get(sessions)).get( sessionKey(entry.runtime.providerSessionId), ); - if (current?.runtime !== entry.runtime) return; + if ( + current?.runtime !== entry.runtime || + ((event.type === "provider_thread.updated" || + (event.type === "turn_item.updated" && + event.turnItem.type === "command_execution")) && + !current.attachedThreadIds.has(threadId)) + ) + return; yield* providerEventIngestor .ingestNormalized({ providerSessionId: entry.runtime.providerSessionId, @@ -2261,6 +2488,7 @@ export const layerWithOptions = ( ), ); }).pipe(entry.requestEventPermit.withPermits(1)); + yield* releaseLiveOutput; return; } yield* publishToSubscribers(entry.eventSubscribers, { type: "event", event }); @@ -2414,26 +2642,29 @@ export const layerWithOptions = ( }); const sessionScope = yield* Scope.fork(sessionScopes); yield* agentScope.clear(input.threadId); - const runtime = yield* adapter - .openSession({ - threadId: input.threadId, - providerSessionId: input.providerSessionId, - modelSelection: input.modelSelection, - runtimePolicy: input.runtimePolicy, - promptSuggestions: promptSuggestionsEnabled, - ...(input.resumeFromSession === undefined - ? {} - : { resumeFromSession: input.resumeFromSession }), - ...(input.initialNativeThreadId === undefined - ? {} - : { initialNativeThreadId: input.initialNativeThreadId }), - ...(input.initialProviderItemIdentityVersion === undefined - ? {} - : { - initialProviderItemIdentityVersion: - input.initialProviderItemIdentityVersion, - }), - }) + const runtime = yield* installationGate + .execute( + input.modelSelection.instanceId, + adapter.openSession({ + threadId: input.threadId, + providerSessionId: input.providerSessionId, + modelSelection: input.modelSelection, + runtimePolicy: input.runtimePolicy, + promptSuggestions: promptSuggestionsEnabled, + ...(input.resumeFromSession === undefined + ? {} + : { resumeFromSession: input.resumeFromSession }), + ...(input.initialNativeThreadId === undefined + ? {} + : { initialNativeThreadId: input.initialNativeThreadId }), + ...(input.initialProviderItemIdentityVersion === undefined + ? {} + : { + initialProviderItemIdentityVersion: + input.initialProviderItemIdentityVersion, + }), + }), + ) .pipe( Effect.provideService(Scope.Scope, sessionScope), // Bound scope close after cleanup; a wedged finalizer must not hold the open lock or interrupter. diff --git a/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts index 8b30a961c099..25daa45a847f 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts @@ -268,6 +268,7 @@ it.effect( getCheckpointRollbackContext: () => Effect.die("unused rollback context"), getCheckpointCaptureContext: () => Effect.die("not used"), getRunMessage: () => Effect.die("not used"), + getRun: () => Effect.die("not used"), isNodeStored: () => Effect.die("not used"), canStartQueuedRun: () => Effect.die("not used"), getHeldQueuedRuns: () => Effect.die("not used"), diff --git a/apps/server/src/orchestration-v2/ProviderTurnControlService.ts b/apps/server/src/orchestration-v2/ProviderTurnControlService.ts index bd225bb41773..00f299cc0313 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnControlService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnControlService.ts @@ -1,6 +1,7 @@ import { projectComposerContextForProvider } from "@t3tools/shared/composerContextReferences"; import { MessageId, + type ModelSelection, type NodeId, ProviderSessionId, ProviderThreadId, @@ -67,7 +68,20 @@ export interface ProviderTurnControlServiceV2Shape { readonly providerThreadId: ProviderThreadId; readonly providerTurnId: ProviderTurnId; readonly messageId: MessageId; + readonly modelSelection?: ModelSelection; }) => Effect.Effect; + /** Best effort: a turn that ended or moved sessions picks the selection up when it next starts. */ + readonly applyModelSelection: (input: { + readonly threadId: ThreadId; + readonly providerSessionId: ProviderSessionId; + readonly providerThreadId: ProviderThreadId; + readonly modelSelection: ModelSelection; + }) => Effect.Effect< + void, + | ProjectionStore.ProjectionStoreV2Error + | ProviderSessionManager.ProviderSessionManagerV2Error + | ProviderAdapter.ProviderAdapterV2Error + >; readonly interruptAndAwaitTerminal: (input: { readonly threadId: ThreadId; readonly providerSessionId: ProviderSessionId; @@ -185,6 +199,26 @@ export const layer: Layer.Layer< }); return ProviderTurnControlServiceV2.of({ + applyModelSelection: Effect.fn("ProviderTurnControlService.applyModelSelection")( + function* (input) { + const { providerThread, providerTurn } = yield* projections.getRunningTurnContext( + input.threadId, + ); + if ( + providerThread?.id !== input.providerThreadId || + providerThread.providerSessionId !== input.providerSessionId || + providerTurn === undefined + ) { + return; + } + const session = yield* sessions.get(input.providerSessionId); + if (Option.isNone(session) || session.value.applyModelSelection === undefined) return; + yield* session.value.applyModelSelection({ + providerThread, + modelSelection: input.modelSelection, + }); + }, + ), interrupt: Effect.fn("ProviderTurnControlService.interrupt")( function* (input) { const loaded = yield* load({ ...input, operation: "interrupt" }); @@ -372,6 +406,9 @@ export const layer: Layer.Layer< runId: run.id, providerThread: loaded.providerThread, providerTurnId: loaded.providerTurn.id, + ...(input.modelSelection === undefined + ? {} + : { modelSelection: input.modelSelection }), message: { messageId: message.id, text: projectComposerContextForProvider({ diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts index d1b516825887..23e768950487 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts @@ -54,6 +54,7 @@ import * as ProviderTurnStart from "./ProviderTurnStartService.ts"; import * as RunExecutionService from "./RunExecutionService.ts"; import * as ProviderUsageAnalytics from "../telemetry/ProviderUsageAnalytics.ts"; import * as RuntimePolicy from "./RuntimePolicy.ts"; +import * as TurnItemPositionStore from "./TurnItemPositionStore.ts"; const isDomainEvent = Schema.is(OrchestrationV2DomainEvent); @@ -137,6 +138,7 @@ function startWithMissingWorktree( Layer.mock(ContextHandoffService.ContextHandoffServiceV2)({}), Layer.mock(EventSink.EventSinkV2)({ writeIfRunCurrent }), IdAllocator.layer, + Layer.mock(TurnItemPositionStore.TurnItemPositionStoreV2)({}), Layer.succeed(FileSystem.FileSystem, { exists: () => Effect.succeed(false), realPath: (target: string) => @@ -269,6 +271,7 @@ it.each([ function makeLocalCommandHarness(input: { readonly text: string; + readonly nativeGoals?: boolean; readonly worktreePath?: string; readonly worktreesDir?: string; readonly workspaceRoot?: string; @@ -337,7 +340,7 @@ function makeLocalCommandHarness(input: { }; const providerThread: OrchestrationV2ThreadProjection["providerThreads"][number] = { id: providerThreadId, - driver: ProviderDriverKind.make("codex"), + driver: ProviderDriverKind.make(input.nativeGoals === undefined ? "codex" : "opencode"), providerInstanceId: newInstanceId, providerSessionId, appThreadId: threadId, @@ -778,7 +781,13 @@ function makeLocalCommandHarness(input: { status: "ready", cwd: "/tmp/native-account-command", model: null, - capabilities: CodexProviderCapabilitiesV2, + capabilities: { + ...CodexProviderCapabilitiesV2, + turns: { + ...CodexProviderCapabilitiesV2.turns, + ...(input.nativeGoals === undefined ? {} : { supportsNativeGoals: input.nativeGoals }), + }, + }, createdAt: now, updatedAt: now, lastError: null, @@ -912,6 +921,18 @@ function makeLocalCommandHarness(input: { ), }), ); + const getRuntimeRecoveryProjection = vi.fn< + ProjectionStore.ProjectionStoreV2Shape["getRuntimeRecoveryProjection"] + >(() => + Effect.as(failReadIfRunning, { + ...projection, + hasConversation: projection.messages.some( + (m) => + m.role === "user" && + (m.text.trim().toLowerCase() !== "/compact" || m.attachments.length > 0), + ), + }), + ); const layer = ProviderTurnStart.layer.pipe( Layer.provide( Layer.mergeAll( @@ -941,6 +962,7 @@ function makeLocalCommandHarness(input: { }), }), IdAllocator.layer, + Layer.mock(TurnItemPositionStore.TurnItemPositionStoreV2)({}), layerWorktreeLocation(input.worktreesDir), input.fileSystem === undefined ? FileSystem.layerNoop({ exists: () => Effect.succeed(true) }) @@ -961,15 +983,12 @@ function makeLocalCommandHarness(input: { Layer.mock(ProjectionStore.ProjectionStoreV2)({ getThreadRecords, getTurnStartContext, - getRuntimeRecoveryProjection: () => - Effect.as(failReadIfRunning, { - ...projection, - hasConversation: projection.messages.some( - (m) => - m.role === "user" && - (m.text.trim().toLowerCase() !== "/compact" || m.attachments.length > 0), - ), - }), + getRuntimeRecoveryProjection, + getRun: (_threadId, id) => + Effect.as( + failReadIfRunning, + projection.runs.find((candidate) => candidate.id === id), + ), getTurnStartHistory: () => input.sideChat !== undefined ? Effect.succeed([]) @@ -1010,6 +1029,7 @@ function makeLocalCommandHarness(input: { settledRuns, getThreadRecords, getTurnStartContext, + getRuntimeRecoveryProjection, setProjection: (next: OrchestrationV2ThreadProjection) => { projection = next; }, @@ -1338,6 +1358,23 @@ effectIt.effect.each([ }), ); +effectIt.effect.each([false, true])( + "OpenCode goal prompts defer switched-provider history only with native support: %s", + (nativeGoals) => + Effect.gen(function* () { + const harness = makeLocalCommandHarness({ + text: "/goal all tests pass", + nativeGoals, + sideChat: { contextRun: "failed", switchedProvider: true }, + }); + harness.tryHandlePromptCommand.mockReturnValue(Effect.succeed(false)); + yield* harness.start; + expect(harness.startTurn.mock.calls[0]?.[0].message.text.includes("SIDE_CHAT_CONTEXT")).toBe( + !nativeGoals, + ); + }), +); + effectIt.effect.each([ { runStatus: "failed", turnStatus: "completed" }, { runStatus: "completed", turnStatus: "completed" }, @@ -1740,6 +1777,29 @@ effectIt.effect("does not overwrite a run interrupted while its provider session }), ); +effectIt.effect("checks the run's status without loading the thread's history", () => + Effect.gen(function* () { + const harness = makeLocalCommandHarness({ + text: "Continue", + previousTurn: { runStatus: "completed", turnStatus: "completed" }, + }); + + yield* harness.start; + + const controls = harness.startRootRun.mock.calls[0]?.[0]; + expect(controls).toBeDefined(); + if (controls === undefined) return; + expect(yield* controls.shouldStartProviderTurn!()).toBe(true); + expect(yield* controls.shouldFinalizeRun!()).toBe(true); + // Each check needs one run row; the recovery projection reads every run and open turn item. + // The one read left is the start's inherited background items snapshot, not a status check. + expect( + harness.getRuntimeRecoveryProjection.mock.calls.length, + "recovery projection reads beyond the inherited background items snapshot", + ).toBe(1); + }), +); + effectIt.effect("fails the run instead of replacing a native thread another client holds", () => Effect.gen(function* () { const detail = "This conversation is open in another Codex client."; diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 66a268f8f732..ded020002cd9 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -57,6 +57,7 @@ import { } from "@t3tools/provider-core/server/failure"; import * as RunExecutionService from "./RunExecutionService.ts"; import * as RuntimePolicy from "./RuntimePolicy.ts"; +import * as TurnItemPositionStore from "./TurnItemPositionStore.ts"; import { isRestartNoteContinuation, pendingRestartCancelledBackgroundWork, @@ -115,6 +116,18 @@ const refusedBeforePrompt = (error: unknown): boolean => PROMPT_REFUSAL_TAGS.some((tag) => Predicate.isTagged(error, tag)) || (Predicate.hasProperty(error, "cause") && refusedBeforePrompt(error.cause)); +const startFailureMessage = (error: Error): string => { + const nestedCause = "cause" in error ? error.cause : undefined; + return ( + knownProviderFailureMessage(error) ?? + (nestedCause instanceof Error + ? nestedCause.message + : typeof nestedCause === "string" + ? nestedCause + : error.message) + ); +}; + export interface ProviderTurnStartServiceV2Shape { readonly start: (input: { readonly threadId: ThreadId; @@ -145,6 +158,7 @@ export const layer: Layer.Layer< | ProviderSessionManager.ProviderSessionManagerV2 | RunExecutionService.RunExecutionServiceV2 | RuntimePolicy.RuntimePolicyV2 + | TurnItemPositionStore.TurnItemPositionStoreV2 > = Layer.effect( ProviderTurnStartServiceV2, Effect.gen(function* () { @@ -161,6 +175,7 @@ export const layer: Layer.Layer< const providerSessions = yield* ProviderSessionManager.ProviderSessionManagerV2; const runExecution = yield* RunExecutionService.RunExecutionServiceV2; const runtimePolicy = yield* RuntimePolicy.RuntimePolicyV2; + const turnItemPositions = yield* TurnItemPositionStore.TurnItemPositionStoreV2; const isManagedWorktreePath = yield* makeIsManagedWorktreePath; const resolveRecreateBase = Effect.fn("ProviderTurnStartService.resolveRecreateBase")( @@ -233,17 +248,21 @@ export const layer: Layer.Layer< readonly attemptId: OrchestrationV2RunAttempt["id"]; readonly providerThreadId: OrchestrationV2ProviderThread["id"]; readonly runOrdinal: number; + readonly worktreePath: string | null; readonly inheritedBackgroundTurnItems: ReturnType< typeof RunExecutionService.selectInheritedBackgroundTurnItems >; }) => { + const isCurrentAttemptIn = (statuses: ReadonlyArray) => + projectionStore + .getRun(input.threadId, input.runId) + .pipe( + Effect.map( + (run) => run?.activeAttemptId === input.attemptId && statuses.includes(run.status), + ), + ); const isCurrentAttemptInStatus = (expectedStatus: OrchestrationV2Run["status"]) => - projectionStore.getRuntimeRecoveryProjection(input.threadId).pipe( - Effect.map((current) => { - const run = current.runs.find((candidate) => candidate.id === input.runId); - return run?.activeAttemptId === input.attemptId && run.status === expectedStatus; - }), - ); + isCurrentAttemptIn([expectedStatus]); return { isCurrentAttemptInStatus, loadInheritedBackgroundTurnItems: () => @@ -260,16 +279,7 @@ export const layer: Layer.Layer< Effect.catchCause(() => Effect.succeed(input.inheritedBackgroundTurnItems)), ), shouldStartProviderTurn: () => isCurrentAttemptInStatus("running"), - shouldFinalizeRun: () => - projectionStore.getRuntimeRecoveryProjection(input.threadId).pipe( - Effect.map((current) => { - const run = current.runs.find((candidate) => candidate.id === input.runId); - return ( - run?.activeAttemptId === input.attemptId && - (run.status === "starting" || run.status === "running") - ); - }), - ), + shouldFinalizeRun: () => isCurrentAttemptIn(["starting", "running"]), hasUnpairedRunInterruptRequest: () => projectionStore .hasUnpairedRunInterruptRequest( @@ -284,6 +294,11 @@ export const layer: Layer.Layer< }), ) .pipe(Effect.catchCause(() => Effect.succeed(false))), + hasWorkspaceMoved: () => + projectionStore.getRuntimeRecoveryProjection(input.threadId).pipe( + Effect.map((current) => current.thread.worktreePath !== input.worktreePath), + Effect.catchCause(() => Effect.succeed(false)), + ), loadRunInterruptReason: () => projectionStore .getThreadRecords(input.threadId, ["turnItems"], { @@ -522,7 +537,6 @@ export const layer: Layer.Layer< readonly error: Error; readonly message?: string; }) { - const nestedCause = "cause" in failed.error ? failed.error.cause : undefined; yield* settleRunBeforeStart({ signal: failed.signal, status: "failed", @@ -534,14 +548,7 @@ export const layer: Layer.Layer< title: failed.title, failure: makeProviderFailure({ cause: failed.error, - message: - failed.message ?? - knownProviderFailureMessage(failed.error) ?? - (nestedCause instanceof Error - ? nestedCause.message - : typeof nestedCause === "string" - ? nestedCause - : failed.error.message), + message: failed.message ?? startFailureMessage(failed.error), class: "provider_error", }), }, @@ -695,6 +702,7 @@ export const layer: Layer.Layer< attemptId: attempt.id, providerThreadId: providerThread.id, runOrdinal: run.ordinal, + worktreePath: projection.thread.worktreePath, inheritedBackgroundTurnItems, }); const { isCurrentAttemptInStatus } = runControls; @@ -827,8 +835,12 @@ export const layer: Layer.Layer< text: message.text, records: message.context?.records ?? [], }); - // Codex and Claude parse /goal from the prompt, so it goes verbatim and history waits a turn. - const goalDriver = session.driver === "codex" || session.driver === "claudeAgent"; + // Native goal commands go verbatim; handoff history waits a turn. + const goalDriver = + session.driver === "codex" || + session.driver === "claudeAgent" || + (session.driver === "opencode" && + session.providerSession.capabilities.turns.supportsNativeGoals === true); const goalCommand = goalDriver && parseGoalCommand(message) !== null; // The provider turn marks cancellation work delivered; restart continuations use their own text or resume natively. const noteContinuation = isRestartNoteContinuation( @@ -1097,6 +1109,7 @@ export const layer: Layer.Layer< let resumedRetryHandoff: ReadonlyArray | undefined; let resumedHandoffContext: ReturnType | undefined; let portableForkFallback: OrchestrationV2ContextHandoff | undefined; + let recoveryTurnItem: OrchestrationV2TurnItem | undefined; const loadedProviderThread = yield* Effect.gen(function* () { if (nativeForkTransfer !== undefined) { const sourceProjection = yield* projectionStore.getThreadRecords( @@ -1128,6 +1141,7 @@ export const layer: Layer.Layer< const sourceBusy = sourceProjection.runs.some( (candidate) => candidate.ordinal > sourceRun.ordinal && isActiveRun(candidate), ); + let nativeForkFailure: string | undefined; if (!sourceBusy) { // Rolled-back turns are no longer in the source conversation, so they cannot bound a fork. const rolledBackAttemptIds = new Set( @@ -1140,7 +1154,7 @@ export const layer: Layer.Layer< ) .map((attempt) => attempt.id), ); - return yield* loadFromProvider( + const forked = yield* Effect.result( session.forkThread({ sourceProviderThread, sourceProviderTurns: sourceProjection.providerTurns.filter( @@ -1155,7 +1169,24 @@ export const layer: Layer.Layer< : { providerTurnId: sourceProviderTurn.id }), }), ); + if (forked._tag === "Success") return forked.success; + if (input.willRetry === true) return yield* forked.failure; + if (!(yield* isCurrentAttemptInStatus("starting"))) return undefined; + // Retries are spent (e.g. the source's native session is gone): start fresh with portable history. + yield* Effect.logWarning("Starting a native fork in a fresh native session", { + driver: session.driver, + providerThreadId: providerThread.id, + sourceProviderThreadId: sourceProviderThread.id, + runId, + errorTag: forked.failure._tag, + }); + nativeForkFailure = makeProviderFailure({ + cause: forked.failure, + message: startFailureMessage(forked.failure), + class: "provider_error", + }).message; } + const nativeForkFailed = nativeForkFailure !== undefined; const sourceItems = yield* ContextHandoffService.readForkHistoryItems( projectionStore, sourceProjection, @@ -1186,7 +1217,11 @@ export const layer: Layer.Layer< modelSelection: run.modelSelection, runtimePolicy: resolvedRuntimePolicy, providerSessionId, - existingProviderThread: { ...providerThread, forkedFrom: null }, + existingProviderThread: nativeForkFailed + ? { ...providerThread, nativeThreadRef: null, forkedFrom: null } + : { ...providerThread, forkedFrom: null }, + // The failed fork may have left the adapter's session half-switched. + ...(nativeForkFailed ? { forceNewNativeThread: true } : {}), }), ); if (loaded === undefined) return undefined; @@ -1219,6 +1254,45 @@ export const layer: Layer.Layer< updatedAt: createdAt, }, }, + ...(nativeForkFailure === undefined + ? [] + : [ + { + id: yield* idAllocator.allocate.event({ threadId: projection.thread.id }), + type: "turn-item.updated" as const, + threadId: projection.thread.id, + runId: run.id, + providerInstanceId: run.providerInstanceId, + occurredAt: createdAt, + payload: { + id: idAllocator.derive.runSignalTurnItem({ + runId, + signal: "native-fork-fallback", + }), + threadId: projection.thread.id, + runId, + nodeId: rootNode.id, + providerThreadId: providerThread.id, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: + Math.max( + 0, + ...projection.turnItems + .filter((item) => item.runId === runId) + .map((item) => item.ordinal), + ) + 1, + status: "completed" as const, + title: "Native fork failed", + startedAt: createdAt, + completedAt: createdAt, + updatedAt: createdAt, + type: "system_notice" as const, + message: `Could not fork the source conversation natively (${nativeForkFailure}). Started a fresh session with its history.`, + }, + }, + ]), ], }); return loaded; @@ -1351,6 +1425,50 @@ export const layer: Layer.Layer< const { handoff, transferId, createdAt } = yield* recoveryHandoff; effectiveHandoffs = withRecoveryHandoff(handoff); resumeFellBack = true; + const previousRun = projection.runs.findLast( + (candidate) => + candidate.providerThreadId === providerThread.id && + candidate.ordinal < run.ordinal && + candidate.startedAt !== null, + ); + // Stable per run so a retried start reuses the reserved run-start position. + const handoffTurnItemId = idAllocator.derive.runSignalTurnItem({ + runId, + signal: "resume-fallback-handoff", + }); + const handoffTurnItem: OrchestrationV2TurnItem = { + id: handoffTurnItemId, + threadId: projection.thread.id, + runId, + nodeId: rootNode.id, + providerThreadId: providerThread.id, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + // The summary is delivered ahead of this run's own handoffs and message. + ordinal: yield* turnItemPositions.allocate({ + threadId: projection.thread.id, + turnItemId: handoffTurnItemId, + runId, + runOrdinal: run.ordinal, + placement: "run_start", + }), + status: "completed", + title: "Recovered context", + startedAt: createdAt, + completedAt: createdAt, + updatedAt: createdAt, + type: "handoff", + contextHandoffId: handoff.id, + fromProviderThreadIds: handoff.fromProviderThreadIds, + toProviderThreadId: handoff.toProviderThreadId, + fromProviderInstanceIds: [providerThread.providerInstanceId], + toProviderInstanceId: run.providerInstanceId, + fromModelSelections: previousRun === undefined ? [] : [previousRun.modelSelection], + toModel: run.modelSelection.model, + strategy: handoff.strategy, + summary: handoff.summaryText, + }; yield* eventSink.write({ events: [ { @@ -1390,6 +1508,7 @@ export const layer: Layer.Layer< }, ], }); + recoveryTurnItem = handoffTurnItem; return replacement; }); if (loadedProviderThread === undefined) return; @@ -1496,6 +1615,20 @@ export const layer: Layer.Layer< }, }, ]), + // Shown only once the replacement session commits with the running transition. + ...(recoveryTurnItem === undefined + ? [] + : [ + { + id: yield* idAllocator.allocate.event({ threadId: projection.thread.id }), + type: "turn-item.updated" as const, + threadId: projection.thread.id, + runId: run.id, + providerInstanceId: run.providerInstanceId, + occurredAt: now, + payload: recoveryTurnItem, + }, + ]), { id: yield* idAllocator.allocate.event({ threadId: projection.thread.id }), type: "run.updated", @@ -1691,6 +1824,7 @@ export const layer: Layer.Layer< shouldFinalizeRun: runControls.shouldFinalizeRun, hasUnpairedRunInterruptRequest: runControls.hasUnpairedRunInterruptRequest, loadRunInterruptReason: runControls.loadRunInterruptReason, + hasWorkspaceMoved: runControls.hasWorkspaceMoved, message: { messageId: message.id, text: userText, diff --git a/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts b/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts index 51c11e24bc36..7a93f2400299 100644 --- a/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts +++ b/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts @@ -41,7 +41,7 @@ import * as ServerActivation from "../serverActivation.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as PullRequestSyncReactor from "./PullRequestSyncReactor.ts"; -import type { PullRequestTestThread } from "./testkit/pullRequestFixtures.ts"; +import { type PullRequestTestThread, v2PullRequestThread } from "./testkit/pullRequestFixtures.ts"; const NOW = "2026-08-28T12:00:00.000Z"; const PROJECT_ID = ProjectId.make("sync-project"); @@ -161,6 +161,14 @@ function makeSummary( }; } +const missingSummary = Effect.fail( + new PullRequestOperationError({ + operation: "summary", + detail: "Pull request not found", + reason: "not-found", + }), +); + interface HarnessOptions { readonly invalidate?: PullRequestService.PullRequestService["Service"]["invalidate"]; readonly snapshot: TestShellSnapshot; @@ -1229,7 +1237,7 @@ describe("PullRequestSyncReactor", () => { ), ); - it.effect("leaves a settled thread's links unread until the thread is unsettled", () => + it.effect("reads a settled thread's links once when it settles, then until unsettled", () => Effect.scoped( Effect.gen(function* () { yield* TestClock.setTime(Date.parse(NOW)); @@ -1238,13 +1246,57 @@ describe("PullRequestSyncReactor", () => { makeThread("settled", { settledOverride: "settled", settledAt: "2026-08-21T00:00:00.000Z", - pullRequests: [makeLink(5, { state: "open" }), makeLink(6, { state: "closed" })], + pullRequests: [ + makeLink(5, { state: "open" }), + makeLink(6, { state: "closed" }), + makeLink(7), + ], }), ]), }); yield* Effect.gen(function* () { const reactor = yield* startAndSweep(fixture); + assert.deepStrictEqual(yield* Ref.get(fixture.summaryCalls), []); + + const threadId = ThreadId.make("settled"); + const at = DateTime.makeUnsafe(NOW); + yield* Queue.offer(fixture.domainEvents, { + type: "thread.settled", + id: EventId.make("event:settled"), + threadId, + occurredAt: at, + payload: { + createdBy: "user", + creationSource: "web", + id: threadId, + projectId: PROJECT_ID, + title: "settled", + providerInstanceId: ProviderInstanceId.make("codex"), + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5" }, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + activeProviderThreadId: null, + lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId }, + forkedFrom: null, + createdAt: at, + updatedAt: at, + archivedAt: null, + settledOverride: "settled", + settledAt: at, + lastVisitedAt: null, + deletedAt: null, + }, + }); + yield* Queue.take(fixture.snapshotReads); + yield* Queue.take(fixture.snapshotReads); + yield* reactor.drain; + assert.deepStrictEqual( + (yield* Ref.getAndSet(fixture.summaryCalls, [])).map((call) => call.number).toSorted(), + [5, 6], + ); for (let index = 0; index < 15; index += 1) yield* sweepAgain(fixture, reactor); assert.deepStrictEqual(yield* Ref.get(fixture.summaryCalls), []); @@ -1259,7 +1311,7 @@ describe("PullRequestSyncReactor", () => { yield* sweepAgain(fixture, reactor); assert.deepStrictEqual( (yield* Ref.get(fixture.summaryCalls)).map((call) => call.number).toSorted(), - [5, 6], + [5, 6, 7], ); }).pipe(Effect.provide(fixture.layer)); }), @@ -1436,6 +1488,152 @@ describe("PullRequestSyncReactor", () => { ), ); + it.effect( + "retries missing links every fifteen minutes and resumes normal reads after recovery", + () => + Effect.scoped( + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse(NOW)); + const found = yield* Ref.make(false); + const fixture = yield* makeHarness({ + snapshot: makeSnapshot([ + makeThread("missing", { pullRequests: [makeLink(9), makeLink(10, {})] }), + ]), + summary: (input) => + Ref.get(found).pipe( + Effect.flatMap((available) => + available ? Effect.succeed(makeSummary(input)) : missingSummary, + ), + ), + }); + yield* Effect.gen(function* () { + const reactor = yield* startAndSweep(fixture); + assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, 2); + for (let index = 0; index < 14; index += 1) yield* sweepAgain(fixture, reactor); + assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, 2); + yield* sweepAgain(fixture, reactor); + assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, 4); + assert.deepStrictEqual(yield* Ref.get(fixture.syncCommands), []); + + yield* Ref.set(found, true); + for (let index = 0; index < 14; index += 1) yield* sweepAgain(fixture, reactor); + assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, 4); + yield* sweepAgain(fixture, reactor); + const commands = yield* Ref.get(fixture.syncCommands); + assert.deepStrictEqual( + commands.map((command) => command.number), + [9], + ); + yield* Ref.update(fixture.snapshots, (snapshot) => applySync(snapshot, commands)); + // Recovered quiet open pull requests return to the five-minute cadence. + for (let index = 0; index < 5; index += 1) yield* sweepAgain(fixture, reactor); + assert.deepStrictEqual( + (yield* Ref.get(fixture.summaryCalls)) + .map((call) => call.number) + .toSorted((left, right) => left - right), + [9, 9, 9, 9, 10, 10, 10, 10], + ); + }).pipe(Effect.provide(fixture.layer)); + }), + ), + ); + + it.effect( + "holds missing links through sibling and link updates but honors a reader refresh", + () => + Effect.scoped( + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse(NOW)); + const fixture = yield* makeHarness({ + snapshot: makeSnapshot([makeThread("missing", { pullRequests: [makeLink(9)] })]), + summary: (input) => + input.number === 9 ? missingSummary : Effect.succeed(makeSummary(input)), + }); + const missingReads = Ref.get(fixture.summaryCalls).pipe( + Effect.map((calls) => calls.filter((call) => call.number === 9).length), + ); + yield* Effect.gen(function* () { + const reactor = yield* startAndSweep(fixture); + const linked = makeThread("missing", { pullRequests: [makeLink(9), makeLink(8)] }); + yield* Ref.set(fixture.snapshots, makeSnapshot([linked])); + yield* Queue.offer(fixture.domainEvents, { + type: "thread.pull-request-synced", + id: EventId.make("sibling-synced"), + threadId: linked.id, + occurredAt: DateTime.makeUnsafe(NOW), + payload: { ...v2PullRequestThread(linked), lastVisitedAt: null }, + }); + yield* Queue.take(fixture.snapshotReads); + yield* reactor.drain; + const afterSiblingSync = yield* missingReads; + assert.deepStrictEqual( + (yield* Ref.get(fixture.syncCommands)).map((command) => command.number), + [8], + ); + + yield* Queue.offer(fixture.domainEvents, { + type: "thread.pull-request-link-updated", + id: EventId.make("missing-link-updated"), + threadId: linked.id, + occurredAt: DateTime.makeUnsafe(NOW), + payload: makeLink(9), + }); + // Let the event consumer finish even when it correctly queues no sweep. + yield* TestClock.adjust(0); + yield* reactor.drain; + const afterLinkUpdate = yield* missingReads; + + yield* Queue.offer(fixture.stateChanges, makeLink(9)); + yield* TestClock.adjust(0); + yield* reactor.drain; + const afterReaderRefresh = yield* missingReads; + yield* sweepAgain(fixture, reactor); + assert.deepStrictEqual( + { + afterSiblingSync, + afterLinkUpdate, + afterReaderRefresh, + afterSweep: yield* missingReads, + }, + { afterSiblingSync: 1, afterLinkUpdate: 1, afterReaderRefresh: 2, afterSweep: 2 }, + ); + }).pipe(Effect.provide(fixture.layer)); + }), + ), + ); + + it.effect("a pending stack retry respects the missing summary backoff", () => + Effect.scoped( + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse(NOW)); + const found = yield* Ref.make(true); + const fixture = yield* makeHarness({ + snapshot: makeSnapshot([makeThread("missing", { pullRequests: [makeLink(4)] })]), + summary: (input) => + Ref.get(found).pipe( + Effect.flatMap((available) => + available ? Effect.succeed(makeSummary(input)) : missingSummary, + ), + ), + stack: () => + Effect.fail(new PullRequestOperationError({ operation: "stack", detail: "host down" })), + }); + yield* Effect.gen(function* () { + const reactor = yield* startAndSweep(fixture); + assert.strictEqual((yield* Ref.get(fixture.stackCalls)).length, 1); + yield* Ref.set(found, false); + yield* sweepAgain(fixture, reactor); + assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, 2); + for (let index = 0; index < 14; index += 1) yield* sweepAgain(fixture, reactor); + assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, 2); + yield* sweepAgain(fixture, reactor); + assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, 3); + assert.strictEqual((yield* Ref.get(fixture.stackCalls)).length, 1); + }).pipe(Effect.provide(fixture.layer)); + }), + ), + ); + // Coder: GitLab is the only host definition, so only its close command marks a run's links. it.effect.each(["glab mr merge 7 --squash 2>&1 | tail -3", "glab mr close 7"])( "reads open links fresh only for a thread that ran `%s`", diff --git a/apps/server/src/orchestration-v2/PullRequestSyncReactor.ts b/apps/server/src/orchestration-v2/PullRequestSyncReactor.ts index 9dffe4babb42..5590a20ce019 100644 --- a/apps/server/src/orchestration-v2/PullRequestSyncReactor.ts +++ b/apps/server/src/orchestration-v2/PullRequestSyncReactor.ts @@ -346,13 +346,19 @@ export const make = Effect.gen(function* () { // linking dozens of pull requests) is read together and shares the summary batches. let requestedSweepQueued = false; const retryStacks = new Set(); + // Group keys the host answered not-found for; they are read on the slow cadence until found. + const missing = new Set(); // Pause per project and host because credentials are project-specific; leave paused PRs due until reset. const pausedUntil = new Map(); const isDue = (key: string, entries: ReadonlyArray, nowMs: number): boolean => { - if (requested.has(key) || retryStacks.has(key)) return true; - if (entries.some((entry) => entry.link.snapshot === null)) return true; + if (requested.has(key)) return true; const active = entries.filter((entry) => isUnsettled(entry.thread)); + if (active.length === 0) return false; + const last = lastSyncedAt.get(key); + const sinceLastMs = last === undefined ? Number.POSITIVE_INFINITY : nowMs - last; + if (missing.has(key)) return sinceLastMs >= SLOW_SYNC_INTERVAL_MS; + if (retryStacks.has(key) || active.some((entry) => entry.link.snapshot === null)) return true; // Merged snapshots without mergedAt cannot settle threads; reread slowly until supplied. const isFinal = ({ link: { snapshot } }: LinkEntry) => snapshot?.state === "merged" @@ -360,8 +366,6 @@ export const make = Effect.gen(function* () { : snapshot?.state === "closed" && nowMs - Date.parse(snapshot.closedAt ?? "") >= CLOSED_FINAL_AFTER_MS; if (active.every(isFinal)) return false; - const last = lastSyncedAt.get(key); - const sinceLastMs = last === undefined ? Number.POSITIVE_INFINITY : nowMs - last; const open = active.filter((entry) => entry.link.snapshot?.state === "open"); if (open.length > 0) { const quiet = open.every((entry) => { @@ -403,6 +407,7 @@ export const make = Effect.gen(function* () { for (const key of lastSyncedAt.keys()) if (!groups.has(key)) lastSyncedAt.delete(key); for (const key of retryStacks) if (!groups.has(key)) retryStacks.delete(key); + for (const key of missing) if (!groups.has(key)) missing.delete(key); for (const key of requested.keys()) if (!groups.has(key)) requested.delete(key); const linkedThisSweep = new Set(); @@ -488,7 +493,19 @@ export const make = Effect.gen(function* () { // A stack retry reads past the cache, or it could get back the layer list it is retrying. const invalidated = refreshRequested || missingMergedAt || retryStacks.has(key); if (invalidated) yield* pullRequests.invalidate({ reference: ref }); - const summary = yield* pullRequests.summary(ref, { recoverTransientFailure: false }); + const summary = yield* pullRequests.summary(ref, { recoverTransientFailure: false }).pipe( + Effect.tapError((error) => + error.reason === "not-found" + ? Effect.sync(() => { + // Not-found is a host answer: it consumes the refresh and starts the backoff. + missing.add(key); + lastSyncedAt.set(key, nowMs); + requested.delete(key); + }) + : Effect.void, + ), + ); + missing.delete(key); const fields = snapshotFieldsOf(summary); const needsStack = refreshRequested || @@ -645,6 +662,27 @@ export const make = Effect.gen(function* () { Effect.catchCause(logSkipped("pull request refresh after run skipped", { threadId })), ); + // A thread's sibling syncs and relinks must not re-read a pull request its project's host + // could not find; the slow cadence and explicit refreshes still do. + const requestUnsyncedLinks = Effect.fnUntraced( + function* (threadId: ThreadId, links: ReadonlyArray) { + const unsynced = links.filter((link) => link.snapshot === null); + if (unsynced.length === 0) return; + const [thread] = + missing.size === 0 ? [] : yield* projections.getThreadsWithPullRequests(threadId); + for (const link of unsynced) { + if ( + thread !== undefined && + missing.has(`${thread.projectId}\0${threadPullRequestKeyOf(link)}`) + ) { + continue; + } + yield* requestSync(link); + } + }, + Effect.catchCause(logSkipped("pull request refresh after link update skipped", {})), + ); + // An agent that opens a pull request from a shell gets it linked as link_pull_request would. const linkCreatedChangeRequests = Effect.fn("PullRequestSyncReactor.linkCreatedChangeRequests")( function* (threadId: ThreadId, item: CommandExecution) { @@ -690,14 +728,11 @@ export const make = Effect.gen(function* () { Stream.runForEach(events, (event) => { switch (event.type) { case "thread.pull-request-link-updated": - return event.payload.snapshot === null ? requestSync(event.payload) : Effect.void; + return requestUnsyncedLinks(event.threadId, [event.payload]); case "thread.pull-request-synced": - return Effect.forEach( - visibleThreadPullRequests(event.payload.pullRequests ?? []).filter( - (link) => link.snapshot === null, - ), - requestSync, - { discard: true }, + return requestUnsyncedLinks( + event.threadId, + visibleThreadPullRequests(event.payload.pullRequests ?? []), ); // An agent can merge or close its pull request from a shell with its host's CLI, which // sends no merge notification. Read the thread's unmerged links fresh once that command @@ -714,6 +749,9 @@ export const make = Effect.gen(function* () { ended.add(item.id); return Effect.andThen(linked, refreshUnmergedLinks(event.threadId)); } + // Settled threads are not swept, so read their links once to settle with fresh status. + case "thread.settled": + return refreshUnmergedLinks(event.threadId); case "run.updated": return isTerminalRunStatus(event.payload.status) && closeCommands.delete(event.threadId) ? refreshUnmergedLinks(event.threadId) diff --git a/apps/server/src/orchestration-v2/PullRequestWatchReactor.ts b/apps/server/src/orchestration-v2/PullRequestWatchReactor.ts index d6a15eb30161..51ec02e93c3e 100644 --- a/apps/server/src/orchestration-v2/PullRequestWatchReactor.ts +++ b/apps/server/src/orchestration-v2/PullRequestWatchReactor.ts @@ -178,6 +178,8 @@ function watchesEqual(left: ThreadPullRequestWatch, right: ThreadPullRequestWatc left.passed === right.passed && left.passedChecks.join("\n") === right.passedChecks.join("\n") && (left.owedChecks ?? []).join("\n") === (right.owedChecks ?? []).join("\n") && + (left.awaitingApproval ?? []).join("\n") === (right.awaitingApproval ?? []).join("\n") && + (left.approvalAnnounced ?? false) === (right.approvalAnnounced ?? false) && left.remarksThrough === right.remarksThrough && left.remarkIds.join("\n") === right.remarkIds.join("\n") && left.conflicting === right.conflicting && @@ -276,7 +278,11 @@ export const make = Effect.gen(function* () { const record = Effect.fn("PullRequestWatchReactor.record")(function* ( target: WatchTarget, next: ThreadPullRequestWatch | null, - wake?: { readonly text: string; readonly notification: OrchestrationV2Notification }, + wake?: { + readonly text: string; + readonly notification: OrchestrationV2Notification; + readonly includesRemarks?: boolean; + }, endedWatch?: ThreadPullRequestWatch, ) { const uuid = yield* random.uuidV4(); @@ -296,7 +302,7 @@ export const make = Effect.gen(function* () { const giveUp = (target: WatchTarget) => record(target, null, { - text: `T3 Coder stopped watching merge request !${target.link.number} (${target.link.url}) because it failed to read it from the host ${READ_FAILURE_LIMIT} times in a row. Check it yourself, and call watch_pull_request to watch it again.`, + text: `T3 Coder stopped watching merge request !${target.link.number} (${target.link.url}) because it failed to read it from the host ${READ_FAILURE_LIMIT} times in a row. Check it yourself, and call watch_pull_request only if it is still open.`, notification: { source: { kind: "monitor" }, outcome: "failed", @@ -489,11 +495,14 @@ export const make = Effect.gen(function* () { { concurrency: 2 }, ); // Activity failures must not cancel or hide a permanent detail failure. + // Activity is irrelevant once the pull request is closed or merged, so it cannot block ending. const read = Exit.isFailure(detailRead) ? Exit.failCause(detailRead.cause) - : Exit.isFailure(activityRead) - ? Exit.failCause(activityRead.cause) - : Exit.succeed([detailRead.value, activityRead.value] as const); + : detailRead.value.state !== "open" + ? Exit.succeed([detailRead.value, Option.none()] as const) + : Exit.isFailure(activityRead) + ? Exit.failCause(activityRead.cause) + : Exit.succeed([detailRead.value, activityRead.value] as const); if (Exit.isFailure(read)) { if (Cause.hasInterruptsOnly(read.cause)) return yield* Effect.failCause(read.cause); lastReads.delete(group.key); @@ -562,13 +571,16 @@ export const make = Effect.gen(function* () { yield* eachTarget(group, (target) => { const report = evaluatePullRequestWatch(target.watch, detail, remarks, now); if (report.changes.length > 0) { - const wake = pullRequestWatchMessage({ - number: target.link.number, - url: target.link.url, - baseBranch: detail.baseBranch, - headSha: report.next.headSha, - report, - }); + const wake = { + ...pullRequestWatchMessage({ + number: target.link.number, + url: target.link.url, + baseBranch: detail.baseBranch, + headSha: report.next.headSha, + report, + }), + includesRemarks: report.changes.some((change) => change.kind === "remarks"), + }; // An ending watch keeps what this pass saw, so watching again starts from it. return ( report.exhausted diff --git a/apps/server/src/orchestration-v2/ResourceCleanupService.ts b/apps/server/src/orchestration-v2/ResourceCleanupService.ts index e9e4cdbc6a4e..387b8b296648 100644 --- a/apps/server/src/orchestration-v2/ResourceCleanupService.ts +++ b/apps/server/src/orchestration-v2/ResourceCleanupService.ts @@ -10,7 +10,7 @@ import * as TerminalManager from "../terminal/Manager.ts"; export class ResourceCleanupError extends Schema.TaggedError()( "ResourceCleanupError", { - operation: Schema.Literals(["terminal", "preview", "attachment", "scheduled-tasks"]), + operation: Schema.Literals(["terminal", "preview", "device", "attachment", "scheduled-tasks"]), threadId: Schema.optional(Schema.String), attachmentId: Schema.optional(Schema.String), cause: Schema.Defect(), @@ -21,6 +21,8 @@ export class ResourceCleanupService extends Context.Reference<{ readonly cleanupScheduledTasks: (threadId: ThreadId) => Effect.Effect; readonly cleanupTerminals: (threadId: string) => Effect.Effect; readonly cleanupPreviews: (threadId: string) => Effect.Effect; + /** Releases the thread's device claims, as `device_close` does; the devices stay booted. */ + readonly cleanupDevices: (threadId: ThreadId) => Effect.Effect; readonly cleanupAttachments: ( attachmentIds: ReadonlyArray, ) => Effect.Effect; @@ -29,6 +31,7 @@ export class ResourceCleanupService extends Context.Reference<{ cleanupScheduledTasks: () => Effect.void, cleanupTerminals: () => Effect.void, cleanupPreviews: () => Effect.void, + cleanupDevices: () => Effect.void, cleanupAttachments: () => Effect.void, }), }) {} @@ -64,6 +67,8 @@ export const layer = Layer.effect( }), // Coder: there is no browser preview, so a thread has no preview sessions to close. cleanupPreviews: Effect.fn("ResourceCleanupService.cleanupPreviews")(function* () {}), + // Coder: there are no device hosts, so a thread holds no device sessions to release. + cleanupDevices: Effect.fn("ResourceCleanupService.cleanupDevices")(function* () {}), // Coder: deleting or reverting a thread never deletes its attachments. cleanupAttachments: Effect.fn("ResourceCleanupService.cleanupAttachments")(function* () {}), }; diff --git a/apps/server/src/orchestration-v2/RestartBackgroundNote.test.ts b/apps/server/src/orchestration-v2/RestartBackgroundNote.test.ts index 954ba2f31a3e..90da48e32de1 100644 --- a/apps/server/src/orchestration-v2/RestartBackgroundNote.test.ts +++ b/apps/server/src/orchestration-v2/RestartBackgroundNote.test.ts @@ -128,6 +128,11 @@ it("names a lost subagent by the id the agent resumes it with", () => { assert.isTrue( isAwaitedWork(cancelledRosterTaskWork({ kind: "monitor", taskId: "ci", awaited: true })), ); + const codexCommand = { id: "item-2", type: "command_execution", input: "sleep 180" }; + assert.isFalse(isAwaitedWork(cancelledTurnItemWork(codexCommand as never)!)); + assert.isTrue( + isAwaitedWork(cancelledTurnItemWork({ ...codexCommand, completionWake: "always" } as never)!), + ); assert.equal( cancelledTurnItemWork({ id: "item-1", diff --git a/apps/server/src/orchestration-v2/RestartBackgroundNote.ts b/apps/server/src/orchestration-v2/RestartBackgroundNote.ts index c495290ad081..48029bdc75fd 100644 --- a/apps/server/src/orchestration-v2/RestartBackgroundNote.ts +++ b/apps/server/src/orchestration-v2/RestartBackgroundNote.ts @@ -1,4 +1,5 @@ import type { + NodeId, OrchestrationV2PendingBackgroundTask, OrchestrationV2ProviderTurn, OrchestrationV2RestartCancelledBackgroundWork, @@ -55,6 +56,7 @@ export function cancelledTurnItemWork( kind: item.monitor === true ? "monitor" : "shell", label: compactLabel(item.input) ?? compactLabel(item.title) ?? "background command", id: item.id, + ...(item.completionWake === "always" ? { awaited: true as const } : {}), }; case "dynamic_tool": { const monitor = @@ -185,6 +187,24 @@ export function cancelledBeforeStart( return run.status === "cancelled" && neverStarted(run); } +/** A follow-up task's latest run, passing over queued work it no longer waits on, as the original task does. */ +export function delegatedTaskResultRun( + runs: ReadonlyArray, + taskId: NodeId, +): OrchestrationV2Run | undefined { + const taskRuns = runs + .filter((run) => run.delegatedTaskId === taskId) + .toSorted((a, b) => b.ordinal - a.ordinal); + const awaited = taskRuns.filter( + (run) => !cancelledBeforeStart(run) && !(run.status === "queued" && run.queueHeld === true), + ); + // Executing work reports the task's state; a reordered queue can finish a newer run before an older one. + const executing = awaited.find((run) => + ["preparing", "starting", "running", "waiting"].includes(run.status), + ); + return executing ?? awaited.find((run) => run.status === "queued") ?? awaited[0] ?? taskRuns[0]; +} + /** The provider never accepted this continuation's prompt, so starting it again repeats nothing. */ export function continuationFailedBeforePrompt( run: Pick, diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 4bcb8970f121..6297810e3e11 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -14,6 +14,7 @@ import { type OrchestrationV2ProviderTurn, type OrchestrationV2Run, type OrchestrationV2RunAttempt, + type OrchestrationV2StoredEvent, type OrchestrationV2Subagent, type OrchestrationV2TurnItem, ProviderDriverKind, @@ -28,14 +29,19 @@ import { TurnItemId, } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; +import * as Clock from "effect/Clock"; import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; import { CodexAppServerRequestError } from "effect-codex-app-server/errors"; import { SqlError, classifySqliteError } from "effect/sql/SqlError"; @@ -47,6 +53,7 @@ import * as EventSink from "./EventSink.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import * as ProjectionStore from "./ProjectionStore.ts"; +import * as ProviderCommitBatch from "./ProviderCommitBatch.ts"; import * as ProviderEventIngestor from "./ProviderEventIngestor.ts"; import * as ProviderUsageAnalytics from "../telemetry/ProviderUsageAnalytics.ts"; import * as McpAppModelContext from "../mcpApps/McpAppModelContext.ts"; @@ -1877,6 +1884,40 @@ it.effect("keeps ingesting until queued background work clears after the probe g }), ); +it.effect("keeps ingesting while the last stored root snapshot reports pending work", () => + Effect.gen(function* () { + // The live probe is already idle, as when it reads adapter state ahead of the queued events. + const consumed = yield* Deferred.make(); + const result = yield* Effect.raceFirst( + runBackgroundItemScenario( + "root-snapshot-pending-work", + (ids) => { + const snapshot = (pendingBackgroundWork: boolean) => + ({ + type: "provider_thread.updated", + driver, + providerThread: { id: ids.providerThreadId, appThreadId: ids.threadId }, + pendingBackgroundWork, + }) as ProviderAdapter.ProviderAdapterV2Event; + return [ + snapshot(true), + rootTerminalEvent(ids, "completed"), + backgroundTurnItemEvent(ids, "dynamic_tool", "completed", 1), + snapshot(false), + ]; + }, + { + keepEventStreamOpen: true, + hasPendingBackgroundWorkForThread: () => Effect.succeed(false), + onEventsConsumed: Deferred.succeed(consumed, undefined).pipe(Effect.asVoid), + }, + ).pipe(Effect.asSome), + Deferred.await(consumed).pipe(Effect.as(Option.none())), + ); + assert.deepEqual(result, Option.some(["root-finalized", "turn_item:completed"])); + }), +); + it.effect("keeps ingesting a child thread's late background item completion", () => Effect.gen(function* () { const observed = yield* runBackgroundItemScenario("bg-child-item", (ids) => [ @@ -2491,6 +2532,142 @@ it.effect( }), ); +it.effect("settles native subagents when a workspace handoff completes the root run", () => + Effect.gen(function* () { + const ids = backgroundScenarioIds("subagent-workspace-handoff"); + const providerInstanceId = ProviderInstanceId.make("codex"); + const written = yield* Ref.make>([]); + const ingestionDone = yield* Deferred.make(); + const layerTest = RunExecutionService.layer.pipe( + Layer.provide( + Layer.mergeAll( + McpAppModelContext.layerEmpty, + Layer.mock(CheckpointService.CheckpointServiceV2)({ + captureBaseline: () => Effect.void, + }), + layerTurnItemPositionsTest, + mockEventSink({ + write: () => Effect.succeed([]), + writeWithEffects: (input) => + Ref.update(written, (current) => [...current, ...input.events]).pipe(Effect.as([])), + writeIfRunCurrent: (input) => + Ref.update(written, (current) => [...current, ...input.events]).pipe( + Effect.as({ committed: true, storedEvents: [] }), + ), + }), + IdAllocator.layer, + Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({ + ingestNormalized: () => Effect.succeed([]), + }), + ServerSettings.layerTest(), + ), + ), + ); + const fixture = { + ids, + providerInstanceId, + childThreadId: ids.childThreadId, + driver, + status: "running", + } as const; + const runningSubagent = makeRunOwnedSubagentFixture(fixture); + const runningTurnItem = makeRunOwnedSubagentTurnItemFixture(fixture); + const runningNode = makeRunOwnedSubagentNodeFixture({ ids, status: "running" }); + const runningChildNode = makeRunOwnedSubagentChildNodeFixture({ ids, status: "running" }); + + yield* Effect.gen(function* () { + const runExecution = yield* RunExecutionService.RunExecutionServiceV2; + yield* runExecution.startRootRun({ + commandId: CommandId.make("command:subagent-workspace-handoff"), + appThread: rootAppThread(ids.threadId), + providerSessionId: ProviderSessionId.make("session:subagent-workspace-handoff"), + session: { + events: Stream.empty, + subscribeEvents: Effect.succeed({ + // The provider event stream stays open: only settled children may end ingestion. + events: Stream.concat( + Stream.fromIterable([ + childThreadCreatedEvent(ids), + { type: "subagent.updated", driver, subagent: runningSubagent }, + { type: "node.updated", driver, node: runningNode }, + { type: "node.updated", driver, node: runningChildNode }, + { type: "turn_item.updated", driver, turnItem: runningTurnItem }, + rootTerminalEvent(ids, "interrupted"), + ] satisfies ReadonlyArray), + Stream.never, + ), + close: Deferred.succeed(ingestionDone, undefined), + }), + startTurn: () => Effect.void, + } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime, + run: { + id: ids.runId, + threadId: ids.threadId, + ordinal: 1, + providerInstanceId, + } as OrchestrationV2Run, + rootNode: { id: ids.rootNodeId } as OrchestrationV2ExecutionNode, + checkpointScope: { + id: CheckpointScopeId.make("checkpoint-scope:subagent-workspace-handoff"), + } as OrchestrationV2CheckpointScope, + providerThread: { id: ids.providerThreadId, driver } as OrchestrationV2ProviderThread, + attempt: { + id: ids.attemptId, + providerTurnId: ids.rootProviderTurnId, + } as OrchestrationV2RunAttempt, + attemptId: ids.attemptId, + providerTurnOrdinal: 1, + shouldFinalizeRun: () => Effect.succeed(true), + hasWorkspaceMoved: () => Effect.succeed(true), + message: { + messageId: MessageId.make("message:subagent-workspace-handoff:user"), + text: "Spawn a subagent then move to a worktree.", + attachments: [], + createdBy: "user", + creationSource: "web", + }, + modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" }, + runtimePolicy: { + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + approvalPolicy: "never", + sandboxPolicy: { + type: "readOnly", + access: { type: "fullAccess" }, + networkAccess: false, + }, + }, + }); + }).pipe(Effect.provide(layerTest)); + + const closed = yield* Deferred.await(ingestionDone).pipe(Effect.timeoutOption("2 seconds")); + assert.isTrue(Option.isSome(closed), "settled children must end event ingestion"); + + const events = yield* Ref.get(written); + const statusOf = (type: OrchestrationV2DomainEvent["type"], id: string) => + events.flatMap((event) => { + if (event.type !== type) return []; + const payload = ( + event as { readonly payload: { readonly id: string; readonly status: string } } + ).payload; + return payload.id === id ? [payload.status] : []; + }); + assert.deepEqual(statusOf("subagent.updated", runningSubagent.id), ["interrupted"]); + assert.deepEqual(statusOf("turn-item.updated", runningTurnItem.id), ["interrupted"]); + assert.deepEqual(statusOf("node.updated", runningNode.id), ["interrupted"]); + assert.deepEqual(statusOf("node.updated", runningChildNode.id), ["interrupted"]); + assert.deepEqual(statusOf("run.updated", ids.runId), ["waiting"]); + assert.deepEqual(statusOf("run-attempt.updated", ids.attemptId), ["completed"]); + assert.isFalse( + events.some( + (event) => + event.type === "turn-item.updated" && event.payload.type === "run_interrupt_result", + ), + ); + }), +); + it.effect("omits interrupt results and subagent cascade for a superseded attempt", () => Effect.gen(function* () { const { written, observed } = yield* captureRootRunTermination({ @@ -2530,6 +2707,37 @@ it.effect("emits run_interrupt_result when superseded attempt still has a hard-s }), ); +it.effect("keeps a Stop that commits while a workspace handoff completes the run", () => + Effect.gen(function* () { + const stopCommitted = yield* Ref.make(false); + const { written, finalEvents, committedEffects } = yield* captureRootRunTermination({ + key: "stop-races-workspace-handoff", + shouldFinalizeRun: () => Effect.succeed(true), + // Stop commits after the handoff is detected and before the terminal write. + hasWorkspaceMoved: () => Ref.set(stopCommitted, true).pipe(Effect.as(true)), + // The sink refuses a guarded write once the Stop request is stored. + beforeTerminalWrite: (request) => + Ref.get(stopCommitted).pipe( + Effect.map((stopped) => !(stopped && request.unlessInterruptRequested !== undefined)), + ), + }); + const statuses = (type: OrchestrationV2DomainEvent["type"]) => + finalEvents.flatMap((event) => + event.type === type ? [(event.payload as { readonly status: string }).status] : [], + ); + assert.deepEqual(statuses("run.updated"), ["interrupted"]); + assert.deepEqual(statuses("run-attempt.updated"), ["interrupted"]); + assert.deepEqual( + written.map((item) => item.type), + ["run_interrupt_result"], + ); + assert.deepEqual( + committedEffects.map((effect) => effect.request.type), + ["checkpoint.capture"], + ); + }), +); + it.effect("does not overwrite Stop when ownership changes after the finalization read", () => Effect.gen(function* () { const { written, observed, submittedEffects, committedEffects } = @@ -2679,6 +2887,20 @@ it.effect("records a provider's input rejection at turn start in the terminal er }), ); +it.effect("tells the user why /compact failed on a provider without compaction", () => + Effect.gen(function* () { + const { written } = yield* captureRootRunTermination({ + key: "start-failure:compact-unsupported", + shouldFinalizeRun: () => Effect.succeed(true), + events: () => Stream.never, + afterStart: (closed) => Deferred.succeed(closed, undefined), + messageText: "/compact", + }); + const error = written.find((item) => item.type === "error"); + assert.include(error?.failure.message ?? "", "does not support /compact"); + }), +); + it.effect.each(["failure", "defect", "interrupt-failure", "ownership-lost"] as const)( "stops the live provider and background work before settling an ingestion %s", (failureMode) => @@ -3002,6 +3224,7 @@ function captureRootRunTermination(input: { readonly shouldFinalizeRun: () => Effect.Effect; readonly rejectTerminalWrite?: boolean; readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect; + readonly hasWorkspaceMoved?: () => Effect.Effect; readonly loadRunInterruptReason?: () => Effect.Effect; readonly hasPendingBackgroundWorkForThread?: ProviderAdapter.ProviderAdapterV2SessionRuntime["hasPendingBackgroundWorkForThread"]; readonly seedOpenSubagent?: boolean; @@ -3013,6 +3236,7 @@ function captureRootRunTermination(input: { >; readonly startTurn?: ProviderAdapter.ProviderAdapterV2SessionRuntime["startTurn"]; readonly interruptTurn?: ProviderAdapter.ProviderAdapterV2SessionRuntime["interruptTurn"]; + readonly messageText?: string; readonly initialProviderTurnId?: ProviderTurnId | null; readonly onStartFailure?: ( error: RunExecutionService.RunExecutionServiceV2Error, @@ -3121,6 +3345,7 @@ function captureRootRunTermination(input: { appThread: rootAppThread(ids.threadId), providerSessionId: ProviderSessionId.make(`session:${input.key}`), session: { + driver, events: Stream.empty, subscribeEvents: Effect.succeed({ events: @@ -3194,9 +3419,12 @@ function captureRootRunTermination(input: { : { hasUnpairedRunInterruptRequest: input.hasUnpairedRunInterruptRequest, }), + ...(input.hasWorkspaceMoved === undefined + ? {} + : { hasWorkspaceMoved: input.hasWorkspaceMoved }), message: { messageId: MessageId.make(`message:${input.key}`), - text: "interrupt projection", + text: input.messageText ?? "interrupt projection", attachments: [], createdBy: "user", creationSource: "web", @@ -3548,8 +3776,18 @@ function rootTerminalEvent( function runBackgroundItemScenario( key: string, - makeEvents: (ids: BackgroundScenarioIds) => ReadonlyArray, + makeEvents: ( + ids: BackgroundScenarioIds, + ) => + | ReadonlyArray + | Stream.Stream, options?: { + readonly onIngest?: ( + event: ProviderAdapter.ProviderAdapterV2Event, + write: { readonly inSharedCommit: boolean }, + ) => Effect.Effect; + // Consumes through the real commit coordinator, as the event sink does. + readonly batched?: boolean; readonly keepEventStreamOpen?: boolean; readonly hasPendingBackgroundWorkForThread?: ( providerThread: OrchestrationV2ProviderThread, @@ -3569,6 +3807,24 @@ function runBackgroundItemScenario( const providerInstanceId = ProviderInstanceId.make("codex"); const observed = yield* Ref.make>([]); const ingestionDone = yield* Deferred.make(); + let inSharedCommit = false; + const coordinator = + options?.batched === true + ? yield* ProviderCommitBatch.make( + (work) => + Effect.suspend(() => { + inSharedCommit = true; + return work; + }).pipe( + Effect.ensuring( + Effect.sync(() => { + inSharedCommit = false; + }), + ), + ), + true, + ) + : undefined; const layerTest = RunExecutionService.layer.pipe( Layer.provide( Layer.mergeAll( @@ -3576,6 +3832,7 @@ function runBackgroundItemScenario( Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }), layerTurnItemPositionsTest, mockEventSink({ + ...(coordinator === undefined ? {} : { consumeProviderEvents: coordinator.consume }), write: () => Effect.succeed([]), writeWithEffects: (input) => Effect.gen(function* () { @@ -3593,9 +3850,10 @@ function runBackgroundItemScenario( }), IdAllocator.layer, Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({ - ingestNormalized: (input) => - Effect.gen(function* () { + ingestNormalized: (input) => { + const write = Effect.gen(function* () { const event = input.event; + yield* options?.onIngest?.(event, { inSharedCommit }) ?? Effect.void; if (event.type === "turn_item.updated") { yield* Ref.update(observed, (current) => [ ...current, @@ -3608,8 +3866,14 @@ function runBackgroundItemScenario( `subagent:${event.subagent.status}`, ]); } - return []; - }), + // Only the count of stored events matters to the run. + return event.type === "provider_thread.updated" + ? [{} as OrchestrationV2StoredEvent] + : []; + }); + // A write outside a batch waits for queued snapshots, as the event sink's does. + return coordinator === undefined ? write : coordinator.barrier(write); + }, }), ServerSettings.layerTest(), ), @@ -3629,7 +3893,8 @@ function runBackgroundItemScenario( : { hasPendingBackgroundWorkForThread: options.hasPendingBackgroundWorkForThread }), subscribeEvents: Effect.gen(function* () { yield* options?.onSubscribe ?? Effect.void; - const events = Stream.fromIterable(makeEvents(ids)).pipe( + const source = makeEvents(ids); + const events = (Stream.isStream(source) ? source : Stream.fromIterable(source)).pipe( Stream.concat( Stream.fromEffect(options?.onEventsConsumed ?? Effect.void).pipe(Stream.drain), ), @@ -3691,8 +3956,9 @@ function runBackgroundItemScenario( }); }).pipe(Effect.provide(layerTest)); - const closed = yield* Deferred.await(ingestionDone).pipe(Effect.timeoutOption("2 seconds")); - assert.isTrue(Option.isSome(closed), "event ingestion fiber did not finish"); + // No timeout: tests advance the TestClock, which would fire it early. + yield* Deferred.await(ingestionDone); + return yield* Ref.get(observed); }); } @@ -3744,3 +4010,395 @@ it.effect("keeps a failed root's still-running native children live while the ru ); }), ); + +it.effect.each([ + { name: "on time", wakesEarly: false }, + // Timers run on the monotonic clock, so one can fire while the wall clock still reads a + // millisecond before its deadline. + { name: "before the wall clock reaches the deadline", wakesEarly: true }, +])("delivers held text while an open-prompt provider idles, waking $name", ({ wakesEarly }) => + Effect.gen(function* () { + const key = `idle-paragraph-${wakesEarly}`; + const ids = backgroundScenarioIds(key); + const now = yield* DateTime.now; + const providerEvents = yield* Queue.unbounded(); + const delivered = yield* Ref.make>([]); + const paragraphDelivered = yield* Deferred.make(); + const nextParagraphDelivered = yield* Deferred.make(); + const clock = yield* Clock.Clock; + const assistantText = (text: string, streaming: boolean) => + [ + { + type: "message.updated", + driver, + message: { + createdBy: "agent", + creationSource: "provider", + id: MessageId.make(`message:${key}`), + threadId: ids.threadId, + runId: ids.runId, + nodeId: ids.rootNodeId, + role: "assistant", + text, + attachments: [], + streaming, + createdAt: now, + updatedAt: now, + }, + }, + { + type: "turn_item.updated", + driver, + turnItem: { + ...makeLinkedChildTurnItemFixture({ ids, driver, type: "assistant_message" }), + id: ids.itemId, + threadId: ids.threadId, + runId: ids.runId, + nodeId: ids.rootNodeId, + providerTurnId: ids.rootProviderTurnId, + text, + streaming, + }, + }, + ] as ReadonlyArray; + + const blockerStarted = yield* Deferred.make(); + const releaseBlocker = yield* Deferred.make(); + const blocker = streamedMessageEvent({ + id: `message:${key}:blocker`, + threadId: ids.threadId, + runId: ids.runId, + role: "assistant", + text: "Other", + streaming: false, + }); + const scenario = yield* runBackgroundItemScenario(key, () => Stream.fromQueue(providerEvents), { + batched: true, + onIngest: (event, write) => + event.type === "message.updated" && event.message.text === "Other" + ? Deferred.succeed(blockerStarted, undefined).pipe( + Effect.andThen(Deferred.await(releaseBlocker)), + ) + : event.type === "message.updated" || event.type === "turn_item.updated" + ? Effect.gen(function* () { + const message = event.type === "message.updated" ? event.message : event.turnItem; + const text = "text" in message ? message.text : ""; + const streaming = "streaming" in message && message.streaming; + yield* Ref.update(delivered, (current) => [ + ...current, + `${event.type}:${streaming ? "streaming" : "final"}:${JSON.stringify(text)}${ + write.inSharedCommit ? " in commit" : "" + }`, + ]); + if (event.type === "turn_item.updated") { + yield* Deferred.succeed(paragraphDelivered, undefined); + if (text === "First.\n\nWaiting on the build.\n\n") + yield* Deferred.succeed(nextParagraphDelivered, undefined); + } + }) + : Effect.void, + }).pipe( + Effect.provideService(Clock.Clock, { + ...clock, + sleep: (duration) => + clock.sleep(wakesEarly ? Duration.subtract(duration, Duration.millis(1)) : duration), + }), + Effect.forkChild, + ); + // Waking a sleep runs ingestion until it suspends, so an idle write has landed by now. + const lastShown = Ref.get(delivered).pipe(Effect.map((shown) => shown.at(-1))); + + yield* Queue.offerAll(providerEvents, assistantText("First.\n\nWaiting on the build", true)); + yield* Deferred.await(paragraphDelivered); + // The provider keeps its prompt open while a backgrounded command runs. + yield* TestClock.adjust("2 seconds"); + assert.strictEqual( + yield* lastShown, + 'turn_item.updated:streaming:"First.\\n\\nWaiting on the build"', + ); + // The next paragraph's tail still flushes on a later pause. + yield* Queue.offerAll( + providerEvents, + assistantText("First.\n\nWaiting on the build.\n\nRunning", true), + ); + yield* Deferred.await(nextParagraphDelivered); + yield* TestClock.adjust("1 second"); + assert.strictEqual( + yield* lastShown, + 'turn_item.updated:streaming:"First.\\n\\nWaiting on the build.\\n\\nRunning"', + ); + yield* Queue.offerAll(providerEvents, [ + ...assistantText("First.\n\nWaiting on the build.\n\nRunning tests", true), + blocker, + ...assistantText("First.\n\nWaiting on the build.\n\nRunning tests passed.", false), + ]); + yield* Deferred.await(blockerStarted); + // The appended text comes due while its final snapshots wait behind the blocked write. + yield* TestClock.adjust("1 second"); + yield* Deferred.succeed(releaseBlocker, undefined); + yield* Queue.offer(providerEvents, rootTerminalEvent(ids, "completed")); + yield* Fiber.join(scenario); + + // Snapshots share the batch commit; the idle write commits alone, before it counts as shown. + // The final snapshots win over the stale append that came due behind them. + assert.deepEqual(yield* Ref.get(delivered), [ + `message.updated:streaming:"First.\\n\\n" in commit`, + `turn_item.updated:streaming:"First.\\n\\n" in commit`, + 'message.updated:streaming:"First.\\n\\nWaiting on the build"', + 'turn_item.updated:streaming:"First.\\n\\nWaiting on the build"', + `message.updated:streaming:"First.\\n\\nWaiting on the build.\\n\\n" in commit`, + `turn_item.updated:streaming:"First.\\n\\nWaiting on the build.\\n\\n" in commit`, + 'message.updated:streaming:"First.\\n\\nWaiting on the build.\\n\\nRunning"', + 'turn_item.updated:streaming:"First.\\n\\nWaiting on the build.\\n\\nRunning"', + `message.updated:final:"First.\\n\\nWaiting on the build.\\n\\nRunning tests passed." in commit`, + `turn_item.updated:final:"First.\\n\\nWaiting on the build.\\n\\nRunning tests passed." in commit`, + ]); + }), +); + +const heldChildText = "partial response"; + +function childAssistantSnapshot( + ids: BackgroundScenarioIds, +): ProviderAdapter.ProviderAdapterV2Event { + return { + type: "turn_item.updated", + driver, + turnItem: { + ...makeLinkedChildTurnItemFixture({ ids, driver, type: "assistant_message" }), + text: heldChildText, + }, + } as ProviderAdapter.ProviderAdapterV2Event; +} + +function linkedSubagentEvent( + ids: BackgroundScenarioIds, + status: "running" | "completed", +): ProviderAdapter.ProviderAdapterV2Event { + return { + type: "subagent.updated", + driver, + subagent: { + ...makeRunOwnedSubagentFixture({ + ids, + providerInstanceId: ProviderInstanceId.make("codex"), + childThreadId: ids.childThreadId, + driver, + status: "running", + }), + status, + }, + }; +} + +const isHeldChildSnapshot = (event: ProviderAdapter.ProviderAdapterV2Event) => + event.type === "turn_item.updated" && + "text" in event.turnItem && + event.turnItem.text === heldChildText; + +// Feeds a held child snapshot, then waits until ingestion has read past it. +const holdChildSnapshot = Effect.fn(function* ( + ids: BackgroundScenarioIds, + events: Queue.Enqueue, + subagentIngested: Deferred.Deferred, +) { + yield* Queue.offerAll(events, [ + childThreadCreatedEvent(ids), + childAssistantSnapshot(ids), + linkedSubagentEvent(ids, "running"), + ]); + yield* Deferred.await(subagentIngested); +}); + +it.effect("settles a child message only an idle flush wrote when the root is interrupted", () => + Effect.gen(function* () { + const ids = backgroundScenarioIds("idle-child-interrupt"); + const events = yield* Queue.unbounded(); + const subagentIngested = yield* Deferred.make(); + const flushed = yield* Deferred.make(); + const finalized = yield* Ref.make>([]); + const scenario = yield* runBackgroundItemScenario( + "idle-child-interrupt", + () => Stream.fromQueue(events), + { + onIngest: (event) => + event.type === "subagent.updated" + ? Deferred.succeed(subagentIngested, undefined) + : isHeldChildSnapshot(event) + ? Deferred.succeed(flushed, undefined) + : Effect.void, + onFinalized: (written) => Ref.set(finalized, written), + }, + ).pipe(Effect.forkChild); + + yield* holdChildSnapshot(ids, events, subagentIngested); + yield* TestClock.adjust("1 second"); + yield* Deferred.await(flushed); + yield* Queue.offer(events, rootTerminalEvent(ids, "interrupted")); + yield* Fiber.join(scenario); + + const settled = (yield* Ref.get(finalized)).flatMap((event) => + event.type === "turn-item.updated" && event.payload.id === ids.childItemId + ? [event.payload] + : [], + ); + assert.equal(settled.length, 1); + assert.equal(settled[0]?.status, "interrupted"); + assert.equal("text" in settled[0]! ? settled[0].text : null, heldChildText); + assert.isFalse("streaming" in settled[0]! && settled[0].streaming); + }), +); + +it.effect.each([ + { name: "completed", terminalStatus: "completed" as const }, + { name: "failed with its children continuing", terminalStatus: "failed" as const }, +])("idle-flushes a continuing child's text after the root $name", ({ terminalStatus }) => + Effect.gen(function* () { + const key = `idle-child-after-${terminalStatus}`; + const ids = backgroundScenarioIds(key); + const events = yield* Queue.unbounded(); + const subagentIngested = yield* Deferred.make(); + const rootFinalized = yield* Deferred.make(); + const flushed = yield* Deferred.make(); + const rootWrites = yield* Ref.make>([]); + const scenario = yield* runBackgroundItemScenario(key, () => Stream.fromQueue(events), { + onIngest: (event) => + event.type === "subagent.updated" + ? Deferred.succeed(subagentIngested, undefined) + : isHeldChildSnapshot(event) + ? Deferred.succeed(flushed, undefined) + : event.type === "message.updated" && event.message.threadId === ids.threadId + ? Ref.update(rootWrites, (current) => [...current, event.message.text]) + : Effect.void, + onFinalized: () => Deferred.succeed(rootFinalized, undefined), + }).pipe(Effect.forkChild); + + yield* holdChildSnapshot(ids, events, subagentIngested); + // The root ends mid-paragraph: its tail is held, and the run settles without it. + yield* Queue.offer( + events, + streamedMessageEvent({ + id: `message:${key}:root`, + threadId: ids.threadId, + runId: ids.runId, + role: "assistant", + text: "Root.\n\nhalf", + streaming: true, + }), + ); + const terminal = rootTerminalEvent(ids, terminalStatus); + yield* Queue.offer( + events, + terminalStatus === "failed" + ? ({ ...terminal, childTurnsContinue: true } as ProviderAdapter.ProviderAdapterV2Event) + : terminal, + ); + yield* Deferred.await(rootFinalized); + yield* TestClock.adjust("1 second"); + yield* Deferred.await(flushed); + yield* Queue.offer(events, linkedSubagentEvent(ids, "completed")); + const observed = yield* Fiber.join(scenario); + + assert.deepEqual(observed.slice(-3), [ + "root-finalized", + "turn_item:running", + "subagent:completed", + ]); + // A streaming write after the run settled would reopen the root's message. + assert.deepEqual(yield* Ref.get(rootWrites), ["Root.\n\n"]); + }), +); + +function streamedMessageEvent(input: { + readonly id: string; + readonly threadId: ThreadId; + readonly runId: RunId | null; + readonly role: "assistant" | "user"; + readonly text: string; + readonly streaming: boolean; +}): ProviderAdapter.ProviderAdapterV2Event { + const now = DateTime.makeUnsafe("2026-07-21T12:00:00.000Z"); + return { + type: "message.updated", + driver, + message: { + createdBy: input.role === "user" ? "user" : "agent", + creationSource: "provider", + id: MessageId.make(input.id), + threadId: input.threadId, + runId: input.runId, + nodeId: null, + role: input.role, + text: input.text, + attachments: [], + streaming: input.streaming, + createdAt: now, + updatedAt: now, + }, + } as ProviderAdapter.ProviderAdapterV2Event; +} + +it.effect("drops a child's held text once the child is handed to another run", () => + Effect.gen(function* () { + const key = "idle-child-handoff"; + const ids = backgroundScenarioIds(key); + const events = yield* Queue.unbounded(); + const subagentIngested = yield* Deferred.make(); + const handoffRead = yield* Deferred.make(); + const rootFlushed = yield* Deferred.make(); + const staleWrites = yield* Ref.make>([]); + const scenario = yield* runBackgroundItemScenario(key, () => Stream.fromQueue(events), { + onIngest: (event) => { + if (event.type === "subagent.updated") return Deferred.succeed(subagentIngested, undefined); + if (event.type !== "message.updated" && event.type !== "turn_item.updated") + return Effect.void; + const message = event.type === "message.updated" ? event.message : event.turnItem; + if (message.threadId === ids.childThreadId) + return Ref.update(staleWrites, (current) => [...current, event.type]); + if (event.type === "message.updated" && event.message.role === "user") + return Deferred.succeed(handoffRead, undefined); + return Deferred.succeed(rootFlushed, undefined); + }, + }).pipe(Effect.forkChild); + + yield* holdChildSnapshot(ids, events, subagentIngested); + const handoff = linkedSubagentEvent(ids, "running"); + yield* Queue.offerAll(events, [ + streamedMessageEvent({ + id: `message:${key}:child`, + threadId: ids.childThreadId, + runId: null, + role: "assistant", + text: heldChildText, + streaming: true, + }), + handoff.type === "subagent.updated" + ? { ...handoff, subagent: { ...handoff.subagent, runId: RunId.make(`run:${key}:next`) } } + : handoff, + // Due in the same flush as the child's text, after it. + streamedMessageEvent({ + id: `message:${key}:root`, + threadId: ids.threadId, + runId: ids.runId, + role: "assistant", + text: "Root partial", + streaming: true, + }), + streamedMessageEvent({ + id: `message:${key}:marker`, + threadId: ids.threadId, + runId: ids.runId, + role: "user", + text: "marker", + streaming: false, + }), + ]); + yield* Deferred.await(handoffRead); + yield* TestClock.adjust("1 second"); + yield* Deferred.await(rootFlushed); + yield* Queue.end(events); + yield* Fiber.join(scenario); + + assert.deepEqual(yield* Ref.get(staleWrites), []); + }), +); diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index e97d7e43e1fa..9681b84027a3 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -32,9 +32,11 @@ import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; +import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import * as Schedule from "effect/Schedule"; import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts"; import * as McpAppModelContext from "../mcpApps/McpAppModelContext.ts"; @@ -173,6 +175,12 @@ function isRunOwnedSubagentTerminalStatus( return status === "interrupted" || status === "failed" || status === "cancelled"; } +function streamedThreadId(event: ProviderAdapter.ProviderAdapterV2Event): ThreadId | null { + if (event.type === "message.updated") return event.message.threadId; + if (event.type === "turn_item.updated") return event.turnItem.threadId; + return null; +} + function childTurnsContinue(terminal: ProviderTerminalEvent): boolean { return terminal.status === "failed" && terminal.childTurnsContinue === true; } @@ -479,6 +487,8 @@ export function routeProviderEvent( } case "message.updated": return [ownsRun(event.message.runId) || ownsChildThread(event.message.threadId), state]; + case "steer.updated": + return [ownsRun(event.runId), state]; case "turn_item.updated": { if (ownsRun(event.turnItem.runId) || ownsChildThread(event.turnItem.threadId)) { return [true, state]; @@ -575,6 +585,7 @@ export interface RunExecutionServiceV2StartRootRunInput { readonly shouldFinalizeRun?: () => Effect.Effect; readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect; readonly loadRunInterruptReason?: () => Effect.Effect; + readonly hasWorkspaceMoved?: () => Effect.Effect; readonly message: ProviderAdapter.ProviderAdapterV2TurnMessage; readonly modelSelection: ModelSelection; readonly runtimePolicy: ProviderAdapter.ProviderAdapterV2RuntimePolicy; @@ -626,6 +637,7 @@ export const layer: Layer.Layer< readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect; readonly loadRunInterruptReason?: () => Effect.Effect; readonly openRunOwnedSubagents?: OpenRunOwnedSubagentProjection; + readonly openRunOwnedSubagentsTerminal?: ProviderTerminalEvent; // Root turn snapshot written with the run when T3 ends the run itself. readonly abandonedProviderTurn?: OrchestrationV2ProviderTurn; readonly terminal: ProviderTerminalEvent; @@ -635,6 +647,7 @@ export const layer: Layer.Layer< readonly writeIfRunCurrent?: { readonly activeAttemptId: RunAttemptId; readonly expectedStatus: OrchestrationV2Run["status"]; + readonly unlessInterruptRequested?: boolean; }; }) { const completedAt = input.completedAt ?? (yield* DateTime.now); @@ -681,7 +694,7 @@ export const layer: Layer.Layer< yield* input.refreshAfterTurn; } } - return; + return false; } const allocateEventId = () => idAllocator.allocate.event({ threadId: input.run.threadId }); const reason = @@ -696,14 +709,15 @@ export const layer: Layer.Layer< open.turnItems.size > 0 || open.childTurnItems.size > 0 || open.nodes.size > 0; + const subagentTerminal = input.openRunOwnedSubagentsTerminal ?? input.terminal; const cascadedSubagentEvents = - isRunOwnedSubagentTerminalStatus(input.terminal.status) && - !childTurnsContinue(input.terminal) && + isRunOwnedSubagentTerminalStatus(subagentTerminal.status) && + !childTurnsContinue(subagentTerminal) && hasOpenSubagentProjection ? yield* cascadeTerminalizeRunOwnedSubagents({ run: input.run, open, - status: input.terminal.status, + status: subagentTerminal.status, completedAt, allocateEventId, }) @@ -861,11 +875,25 @@ export const layer: Layer.Layer< runId: input.run.id, activeAttemptId: input.writeIfRunCurrent.activeAttemptId, expectedStatus: input.writeIfRunCurrent.expectedStatus, + ...(input.writeIfRunCurrent.unlessInterruptRequested === true + ? { + unlessInterruptRequested: { + requestId: idAllocator.derive.runSignalTurnItem({ + runId: input.run.id, + signal: "interrupt-request", + }), + resultId: idAllocator.derive.runSignalTurnItem({ + runId: input.run.id, + signal: "interrupt-result", + }), + }, + } + : {}), events: finalization.events, effects: finalization.effects, }); if (!result.committed) { - return; + return false; } } else { yield* eventSink.writeWithEffects(finalization); @@ -877,6 +905,7 @@ export const layer: Layer.Layer< childrenRunning: hasOpenSubagentProjection && cascadedSubagentEvents.length === 0, }); yield* input.refreshAfterTurn; + return true; }, ); @@ -1070,6 +1099,12 @@ export const layer: Layer.Layer< ReadonlySet >(new Set(inheritedBackgroundTurnItemsById.keys())); const openRunOwnedSubagents = yield* Ref.make(emptyOpenRunOwnedSubagentProjection()); + // The last stored root snapshot's pendingBackgroundWork; adapters that omit it leave it false. + let rootSnapshotPendingWork = false; + const assistantStreaming = makeAssistantStreamingFilter(responseStreamingMode); + // Set when ingestion ends or finalization settles the run's open items, which a later + // streaming snapshot must not reopen. + let idleFlushClosed = false; const finalizeRootRun = Effect.fn("RunExecutionService.finalizeRootRun")(function* ( terminal: ProviderTerminalEvent, ) { @@ -1078,40 +1113,58 @@ export const layer: Layer.Layer< } const providerThread = yield* Ref.get(latestProviderThread); const openSubagents = yield* Ref.get(openRunOwnedSubagents); - yield* writeFinalRunEvents({ - run: input.run, - rootNode: input.rootNode, - checkpointScope: input.checkpointScope, - providerThread, - attempt: input.attempt, - ...(input.shouldFinalizeRun === undefined - ? {} - : { - shouldFinalizeRun: input.shouldFinalizeRun, - // Stop may commit between the ownership read and this - // terminal write. Gate the events and checkpoint together. - writeIfRunCurrent: { - activeAttemptId: input.attempt.id, - expectedStatus: "running" as const, - }, - }), - ...(input.hasUnpairedRunInterruptRequest === undefined - ? {} - : { - hasUnpairedRunInterruptRequest: input.hasUnpairedRunInterruptRequest, - }), - ...(input.loadRunInterruptReason === undefined - ? {} - : { loadRunInterruptReason: input.loadRunInterruptReason }), - openRunOwnedSubagents: openSubagents, - terminal, - failureItemPersisted: terminal.status === "failed", - refreshAfterTurn, - }).pipe( - Effect.mapError((cause) => new RunExecutionIngestError({ runId: input.run.id, cause })), - ); + const writeTerminal = (settledTerminal: ProviderTerminalEvent) => + writeFinalRunEvents({ + run: input.run, + rootNode: input.rootNode, + checkpointScope: input.checkpointScope, + providerThread, + attempt: input.attempt, + ...(input.shouldFinalizeRun === undefined + ? {} + : { + shouldFinalizeRun: input.shouldFinalizeRun, + // Stop may commit between the ownership read and this + // terminal write. Gate the events and checkpoint together. + writeIfRunCurrent: { + activeAttemptId: input.attempt.id, + expectedStatus: "running" as const, + unlessInterruptRequested: settledTerminal !== terminal, + }, + }), + ...(input.hasUnpairedRunInterruptRequest === undefined + ? {} + : { + hasUnpairedRunInterruptRequest: input.hasUnpairedRunInterruptRequest, + }), + ...(input.loadRunInterruptReason === undefined + ? {} + : { loadRunInterruptReason: input.loadRunInterruptReason }), + openRunOwnedSubagents: openSubagents, + openRunOwnedSubagentsTerminal: terminal, + terminal: settledTerminal, + failureItemPersisted: settledTerminal.status === "failed", + refreshAfterTurn, + }).pipe( + Effect.mapError( + (cause) => new RunExecutionIngestError({ runId: input.run.id, cause }), + ), + ); + // A workspace handoff detaches the session and stops the turn that asked + // for it: the run completes, while its native subagents still settle + // as interrupted. The completed write refuses in the same transaction + // as a stored Stop request, so a racing Stop stays authoritative. + const handedOff = + terminal.status === "interrupted" && + input.shouldFinalizeRun !== undefined && + input.hasWorkspaceMoved !== undefined && + (yield* input.hasWorkspaceMoved()); + if (!handedOff || !(yield* writeTerminal({ ...terminal, status: "completed" }))) { + yield* writeTerminal(terminal); + } if (isRunOwnedSubagentTerminalStatus(terminal.status) && !childTurnsContinue(terminal)) { yield* Ref.set(openRunOwnedSubagents, emptyOpenRunOwnedSubagentProjection()); + idleFlushClosed = true; } yield* Ref.set(rootRunFinalized, true); }); @@ -1298,6 +1351,10 @@ export const layer: Layer.Layer< if (yield* Ref.get(providerThreadOwnerLost)) { return true; } + // The live probe reads adapter state, which can be ahead of the events consumed here. + if (rootSnapshotPendingWork) { + return false; + } // Claude background Bash has no turn-item projection. Use only the // thread-scoped probe: session-wide pending work (siblings, wake // buffers, session subagents) must not pin this root subscription. @@ -1312,7 +1369,6 @@ export const layer: Layer.Layer< } return true; }); - const filterAssistantEvent = makeAssistantStreamingFilter(responseStreamingMode); // Held text writes nothing, so its item would take a position only // when finally written, behind every item that began after it. const reservedHeldTurnItemIds = new Set(); @@ -1393,6 +1449,121 @@ export const layer: Layer.Layer< } return { event, accepted }; }); + // Per-event ingestion avoids adding a span for every provider snapshot. + const ingestDelivered = Effect.fnUntraced(function* ( + event: ProviderAdapter.ProviderAdapterV2Event, + deliveredEvent: ProviderAdapter.ProviderAdapterV2Event, + ) { + const rootTerminalAlreadySeen = yield* Ref.get(rootTerminalSeen); + const isRootProviderThreadUpdate = + event.type === "provider_thread.updated" && + event.providerThread.id === input.providerThread.id; + const nativeChildTurn = + event.type === "provider_turn.updated" && + event.providerTurn.providerThreadId !== input.providerThread.id; + if (event.type === "provider_turn.updated" && !nativeChildTurn) { + firstRootProviderTurnId ??= event.providerTurn.id; + } + const storedEvents = yield* providerEventIngestor.ingestNormalized({ + analyticsContext: nativeChildTurn + ? { + ...analyticsContext, + origin: NATIVE_SUBAGENT_TURN_ORIGIN, + segment: "native_child", + } + : { + ...analyticsContext, + segment: + event.type === "provider_turn.updated" && + event.providerTurn.id !== firstRootProviderTurnId + ? "continuation" + : input.attempt.reason, + }, + providerSessionId: input.providerSessionId, + providerInstanceId: input.run.providerInstanceId, + threadId: input.run.threadId, + runId: input.run.id, + nodeId: input.rootNode.id, + event: deliveredEvent, + ...(isRootProviderThreadUpdate + ? rootTerminalAlreadySeen + ? { + writeIfProviderThreadOwner: { + providerThreadId: input.providerThread.id, + runId: input.run.id, + activeAttemptId: input.attempt.id, + expectedLastRunOrdinal: input.run.ordinal, + }, + } + : { + writeIfRunCurrent: { + runId: input.run.id, + activeAttemptId: input.attempt.id, + expectedStatus: "running" as const, + }, + } + : {}), + }); + if (isRootProviderThreadUpdate && rootTerminalAlreadySeen && storedEvents.length === 0) { + yield* Ref.set(providerThreadOwnerLost, true); + } + return storedEvents.length; + }); + // The timer only wakes ingestion. The flush runs as an ingestion step outside any provider + // batch, so it stays ordered with routing and snapshots, and a write has committed when + // it returns. + const idleFlushTicks = yield* Queue.dropping<{ + readonly type: "idle_flush"; + readonly at: number; + }>(1); + let idleFlushWake: { readonly at: number; readonly fiber: Fiber.Fiber } | undefined; + const scheduleIdleFlush = Effect.suspend(() => { + const at = assistantStreaming.nextIdleFlushAt(); + if ( + idleFlushClosed || + at === null || + (idleFlushWake !== undefined && idleFlushWake.at <= at) + ) + return Effect.void; + const previous = idleFlushWake?.fiber; + return Effect.gen(function* () { + if (previous !== undefined) yield* Fiber.interrupt(previous); + const now = DateTime.toEpochMillis(yield* DateTime.now); + const fiber = yield* Effect.sleep(Duration.millis(Math.max(0, at - now))).pipe( + Effect.andThen(Queue.offer(idleFlushTicks, { type: "idle_flush", at })), + Effect.asVoid, + Effect.forkDetach, + ); + idleFlushWake = { at, fiber }; + }); + }); + const closeIdleFlush = Effect.suspend(() => { + idleFlushClosed = true; + const wake = idleFlushWake; + idleFlushWake = undefined; + return wake === undefined ? Effect.void : Fiber.interrupt(wake.fiber); + }); + // Per-tick ingestion avoids adding a span for every idle wake. + const flushIdleAssistantText = Effect.fnUntraced(function* (wokeAt: number) { + // A tick means its timer slept until `wokeAt`, but the wall clock can still read up + // to a millisecond earlier; trusting it would keep the finished wake and never re-arm. + const now = Math.max(DateTime.toEpochMillis(yield* DateTime.now), wokeAt); + // A wake due by now has fired; a later deadline needs a new one. + if (idleFlushWake !== undefined && idleFlushWake.at <= now) idleFlushWake = undefined; + if (idleFlushClosed) return; + const rootSettled = yield* Ref.get(rootRunFinalized); + for (const event of assistantStreaming.takeIdleFlushes(now)) { + // Settling the root run closed its messages; only children may still stream. + if (rootSettled && streamedThreadId(event) === input.run.threadId) continue; + // The held text's child may have been handed to another run since it arrived. + if (!routeProviderEvent(event, routeIdentity, yield* Ref.get(eventRouting))[0]) + continue; + // A failed write fails ingestion like any provider write. + yield* ingestDelivered(event, event); + yield* trackChildLifecycle(event, true); + } + yield* scheduleIdleFlush; + }); const ingestEvent = Effect.fnUntraced(function* ({ event, accepted, @@ -1409,65 +1580,12 @@ export const layer: Layer.Layer< yield* Ref.set(latestRootProviderTurn, event.providerTurn); } let storedEventCount = 0; - const deliveredEvent = filterAssistantEvent( + const deliveredEvent = assistantStreaming.filter( event, DateTime.toEpochMillis(yield* DateTime.now), ); if (deliveredEvent) { - const rootTerminalAlreadySeen = yield* Ref.get(rootTerminalSeen); - const isRootProviderThreadUpdate = - event.type === "provider_thread.updated" && - event.providerThread.id === input.providerThread.id; - const nativeChildTurn = - event.type === "provider_turn.updated" && - event.providerTurn.providerThreadId !== input.providerThread.id; - if (event.type === "provider_turn.updated" && !nativeChildTurn) { - firstRootProviderTurnId ??= event.providerTurn.id; - } - const storedEvents = yield* providerEventIngestor.ingestNormalized({ - analyticsContext: nativeChildTurn - ? { - ...analyticsContext, - origin: NATIVE_SUBAGENT_TURN_ORIGIN, - segment: "native_child", - } - : { - ...analyticsContext, - segment: - event.type === "provider_turn.updated" && - event.providerTurn.id !== firstRootProviderTurnId - ? "continuation" - : input.attempt.reason, - }, - providerSessionId: input.providerSessionId, - providerInstanceId: input.run.providerInstanceId, - threadId: input.run.threadId, - runId: input.run.id, - nodeId: input.rootNode.id, - event: deliveredEvent, - ...(isRootProviderThreadUpdate - ? rootTerminalAlreadySeen - ? { - writeIfProviderThreadOwner: { - providerThreadId: input.providerThread.id, - runId: input.run.id, - activeAttemptId: input.attempt.id, - expectedLastRunOrdinal: input.run.ordinal, - }, - } - : { - writeIfRunCurrent: { - runId: input.run.id, - activeAttemptId: input.attempt.id, - expectedStatus: "running" as const, - }, - } - : {}), - }); - storedEventCount = storedEvents.length; - if (isRootProviderThreadUpdate && rootTerminalAlreadySeen && storedEventCount === 0) { - yield* Ref.set(providerThreadOwnerLost, true); - } + storedEventCount = yield* ingestDelivered(event, deliveredEvent); } else if ( event.type === "turn_item.updated" && (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning") && @@ -1484,6 +1602,7 @@ export const layer: Layer.Layer< if (event.type === "provider_thread.updated") { if (event.providerThread.id === input.providerThread.id && storedEventCount > 0) { yield* Ref.set(latestProviderThread, event.providerThread); + rootSnapshotPendingWork = event.pendingBackgroundWork === true; } } if ( @@ -1507,18 +1626,24 @@ export const layer: Layer.Layer< } } yield* trackChildLifecycle(event, deliveredEvent !== null); + yield* scheduleIdleFlush; }); const providerEventFiber = yield* eventSink .consumeProviderEvents( - eventSubscription.events, + eventSubscription.events.pipe( + Stream.merge(Stream.fromQueue(idleFlushTicks), { haltStrategy: "left" }), + ), (event) => - Effect.gen(function* () { - yield* ingestEvent(yield* routeEvent(event)); - }), + event.type === "idle_flush" + ? flushIdleAssistantText(event.at) + : Effect.gen(function* () { + yield* ingestEvent(yield* routeEvent(event)); + }), shouldStopProviderEventIngestion, Ref.get(rootTerminalSeen).pipe(Effect.map((seen) => !seen)), ) .pipe( + Effect.ensuring(closeIdleFlush), Effect.mapError((cause) => new RunExecutionIngestError({ runId: input.run.id, cause })), Effect.flatMap(() => Effect.gen(function* () { @@ -1727,7 +1852,10 @@ export const layer: Layer.Layer< threadId: input.run.threadId, providerThreadId: input.providerThread.id, runId: input.run.id, - cause: "This provider does not support context compaction.", + cause: new ProviderAdapter.ProviderCompactionUnsupportedError({ + driver: input.session.driver, + providerThreadId: input.providerThread.id, + }), }), )) : input.session.startTurn(turnInput); diff --git a/apps/server/src/orchestration-v2/RunFinalizationQueue.test.ts b/apps/server/src/orchestration-v2/RunFinalizationQueue.test.ts index c2b9e0aabcd5..c0f62b93ecca 100644 --- a/apps/server/src/orchestration-v2/RunFinalizationQueue.test.ts +++ b/apps/server/src/orchestration-v2/RunFinalizationQueue.test.ts @@ -5,6 +5,8 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as ProjectEnrichment from "../project/ProjectEnrichmentService.ts"; +import * as ProjectService from "../project/ProjectService.ts"; import * as PullRequestService from "../pullRequest/PullRequestService.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as VcsStatusBroadcaster from "../vcs/VcsStatusBroadcaster.ts"; @@ -43,11 +45,14 @@ it.effect.each([ checkpointScopes: [{ id: scopeId, runId, kind: "root_run" as const, cwd: "/repo" }], checkpoints: [], }), + getThreadShell: () => Effect.succeed(null), }); const observer = RunFinalization.layerObserver.pipe( Layer.provide( Layer.mergeAll( projections, + Layer.mock(ProjectService.ProjectService)({}), + Layer.mock(ProjectEnrichment.ProjectEnrichmentService)({}), Layer.mock(WorkspaceEntries.WorkspaceEntries)({ refresh: () => Effect.void }), Layer.mock(PullRequestService.PullRequestService)({ refreshAfterTurn: () => Effect.void }), Layer.mock(VcsStatusBroadcaster.VcsStatusBroadcaster)({ diff --git a/apps/server/src/orchestration-v2/RunFinalizationService.test.ts b/apps/server/src/orchestration-v2/RunFinalizationService.test.ts index b51ec8fb80de..9234add511d2 100644 --- a/apps/server/src/orchestration-v2/RunFinalizationService.test.ts +++ b/apps/server/src/orchestration-v2/RunFinalizationService.test.ts @@ -1,17 +1,28 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; import { CheckpointScopeId, + ProjectId, RunId, ThreadId, type OrchestrationV2ThreadShell, + type Project, + type RepositoryIdentity, } from "@t3tools/contracts"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as PubSub from "effect/PubSub"; import * as TestClock from "effect/testing/TestClock"; +import * as ProcessRunner from "../processRunner.ts"; +import * as ProjectEnrichment from "../project/ProjectEnrichmentService.ts"; +import * as ProjectFaviconResolver from "../project/ProjectFaviconResolver.ts"; +import * as ProjectService from "../project/ProjectService.ts"; +import * as RepositoryIdentityResolver from "../project/RepositoryIdentityResolver.ts"; import * as PullRequestService from "../pullRequest/PullRequestService.ts"; import * as VcsStatusBroadcaster from "../vcs/VcsStatusBroadcaster.ts"; import * as WorkspaceEntries from "../workspace/WorkspaceEntries.ts"; @@ -19,6 +30,12 @@ import * as CheckpointCapture from "./CheckpointCaptureService.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as RunFinalization from "./RunFinalizationService.ts"; +// Without a project the observer never touches enrichment, so its methods stay unimplemented. +const layerNoProject = Layer.mergeAll( + Layer.mock(ProjectService.ProjectService)({ getById: () => Effect.succeedNone }), + Layer.mock(ProjectEnrichment.ProjectEnrichmentService)({}), +); + it.effect("completes finalization when the post-checkpoint workspace refresh fails", () => { const threadId = ThreadId.make("thread_refresh_failure"); const runId = RunId.make("run_refresh_failure"); @@ -103,6 +120,7 @@ it.effect.each( const layer = RunFinalization.layerObserver.pipe( Layer.provide( Layer.mergeAll( + layerNoProject, Layer.mock(WorkspaceEntries.WorkspaceEntries)({ refresh: () => Effect.void }), Layer.mock(PullRequestService.PullRequestService)({ refreshAfterTurn: () => Effect.void, @@ -150,6 +168,7 @@ it.effect("returns from the post-turn refresh without waiting for the PR lookup" const layer = RunFinalization.layerObserver.pipe( Layer.provide( Layer.mergeAll( + layerNoProject, Layer.mock(WorkspaceEntries.WorkspaceEntries)({ refresh: () => Effect.void }), Layer.mock(PullRequestService.PullRequestService)({ refreshAfterTurn: () => Effect.void, @@ -200,6 +219,7 @@ it.effect("replaces a still-running PR lookup when a newer turn finishes", () => const layer = RunFinalization.layerObserver.pipe( Layer.provide( Layer.mergeAll( + layerNoProject, Layer.mock(WorkspaceEntries.WorkspaceEntries)({ refresh: () => Effect.void }), Layer.mock(PullRequestService.PullRequestService)({ refreshAfterTurn: () => Effect.void, @@ -241,3 +261,138 @@ it.effect("replaces a still-running PR lookup when a newer turn finishes", () => assert.strictEqual(lookups, 2); }).pipe(Effect.provide(layer)); }); + +const identityProjectId = ProjectId.make("project-identity-refresh"); +const identityThreadId = ThreadId.make("thread-identity-refresh"); +const identityRunId = RunId.make("completed-run"); + +const layerIdentityRefresh = ( + workspaceRoot: string, + resolver: Layer.Layer, +) => + RunFinalization.layerObserver.pipe( + Layer.provideMerge( + Layer.mergeAll( + Layer.effect(ProjectEnrichment.ProjectEnrichmentService, ProjectEnrichment.make()).pipe( + Layer.provide( + Layer.mergeAll( + resolver, + Layer.succeed(ProjectFaviconResolver.ProjectFaviconResolver, { + invalidate: () => Effect.void, + resolvePath: () => Effect.succeed(null), + }), + ), + ), + ), + Layer.mock(WorkspaceEntries.WorkspaceEntries)({ refresh: () => Effect.void }), + Layer.mock(PullRequestService.PullRequestService)({ refreshAfterTurn: () => Effect.void }), + Layer.mock(VcsStatusBroadcaster.VcsStatusBroadcaster)({ + refreshLocalStatus: () => + Effect.succeed({ + isRepo: true, + hasPrimaryRemote: true, + isDefaultRef: true, + refName: "main", + hasWorkingTreeChanges: false, + workingTree: { files: [], insertions: 0, deletions: 0 }, + }), + }), + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getThreadShell: () => + Effect.succeed({ + id: identityThreadId, + projectId: identityProjectId, + branch: "main", + activeRunId: null, + } as OrchestrationV2ThreadShell), + }), + Layer.mock(ProjectService.ProjectService)({ + getById: () => Effect.succeedSome({ id: identityProjectId, workspaceRoot } as Project), + }), + ), + ), + ); + +it.effect("resolves the identity of a project whose remote was added during the turn", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const processRunner = yield* ProcessRunner.ProcessRunner; + const workspaceRoot = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-run-finalization-identity-test-", + }); + const git = (args: ReadonlyArray) => + processRunner.run({ command: "git", args: ["-C", workspaceRoot, ...args] }); + yield* git(["init"]); + + yield* Effect.gen(function* () { + const enrichment = yield* ProjectEnrichment.ProjectEnrichmentService; + // A shell read before the turn caches the folder as having no repository. + assert.isNull(yield* enrichment.readRepositoryIdentity(workspaceRoot)); + + yield* git(["remote", "add", "origin", "https://github.com/t3-triage/example-29929.git"]); + const changes = yield* enrichment.subscribeChanges; + const observer = yield* RunFinalization.RunFinalizationObserver; + yield* observer.refresh({ + cwd: workspaceRoot, + threadId: identityThreadId, + runId: identityRunId, + }); + + const change = yield* PubSub.take(changes); + assert.strictEqual( + change.enrichment.repositoryIdentity?.locator.remoteUrl, + "https://github.com/t3-triage/example-29929.git", + "the project's repository identity is still null after the turn added a remote", + ); + }).pipe( + Effect.scoped, + Effect.provide(layerIdentityRefresh(workspaceRoot, RepositoryIdentityResolver.layer)), + ); + }).pipe( + Effect.scoped, + Effect.provide(ProcessRunner.layer.pipe(Layer.provideMerge(NodeServices.layer))), + ), +); + +it.effect("returns from the post-turn refresh while the identity resolver is still running", () => { + const workspaceRoot = "/repo-stubbed-identity"; + const identity: RepositoryIdentity = { + canonicalKey: "example.test/stubbed-identity", + locator: { + source: "git-remote", + remoteName: "origin", + remoteUrl: "https://example.test/stubbed-identity.git", + }, + }; + const releaseResolver = Deferred.makeUnsafe(); + let remoteAdded = false; + const resolver = Layer.succeed(RepositoryIdentityResolver.RepositoryIdentityResolver, { + resolve: (_cwd, options) => + options?.refresh + ? Deferred.await(releaseResolver).pipe( + Effect.map(() => { + remoteAdded = true; + return identity; + }), + ) + : Effect.sync(() => (remoteAdded ? identity : null)), + }); + return Effect.gen(function* () { + const enrichment = yield* ProjectEnrichment.ProjectEnrichmentService; + assert.isNull(yield* enrichment.readRepositoryIdentity(workspaceRoot)); + const changes = yield* enrichment.subscribeChanges; + const observer = yield* RunFinalization.RunFinalizationObserver; + const refresh = yield* observer + .refresh({ cwd: workspaceRoot, threadId: identityThreadId, runId: identityRunId }) + .pipe(Effect.timeoutOption("1 minute"), Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust("1 minute"); + assert.isTrue( + Option.isSome(yield* Fiber.join(refresh)), + "turn finalization waited on the identity resolver, blocking the thread's next turn", + ); + + yield* Deferred.succeed(releaseResolver, undefined); + const change = yield* PubSub.take(changes); + assert.strictEqual(change.enrichment.repositoryIdentity?.canonicalKey, identity.canonicalKey); + }).pipe(Effect.scoped, Effect.provide(layerIdentityRefresh(workspaceRoot, resolver))); +}); diff --git a/apps/server/src/orchestration-v2/RunFinalizationService.ts b/apps/server/src/orchestration-v2/RunFinalizationService.ts index 3775e63f0e73..6da6833ac793 100644 --- a/apps/server/src/orchestration-v2/RunFinalizationService.ts +++ b/apps/server/src/orchestration-v2/RunFinalizationService.ts @@ -3,8 +3,11 @@ import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as FiberMap from "effect/FiberMap"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; +import * as ProjectEnrichment from "../project/ProjectEnrichmentService.ts"; +import * as ProjectService from "../project/ProjectService.ts"; import * as PullRequestService from "../pullRequest/PullRequestService.ts"; import * as VcsStatusBroadcaster from "../vcs/VcsStatusBroadcaster.ts"; import * as WorkspaceEntries from "../workspace/WorkspaceEntries.ts"; @@ -97,7 +100,32 @@ export const layerObserver = Layer.effect( const vcsStatus = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; const projections = yield* ProjectionStore.ProjectionStoreV2; const pullRequests = yield* PullRequestService.PullRequestService; + const projects = yield* ProjectService.ProjectService; + const enrichment = yield* ProjectEnrichment.ProjectEnrichmentService; const pullRequestLookups = yield* FiberMap.make(); + const identityRefreshes = yield* FiberMap.make(); + // A turn can add the first remote (`git remote add`, `gh repo create --push`), which the + // enrichment and resolver caches would otherwise keep reporting as missing for a minute. + const refreshMissingRepositoryIdentity = Effect.fn( + "RunFinalizationService.refreshMissingRepositoryIdentity", + )( + function* (projectId: ProjectId) { + const project = yield* projects.getById(projectId); + if (Option.isNone(project)) return; + const { workspaceRoot } = project.value; + if ((yield* enrichment.readRepositoryIdentity(workspaceRoot)) !== null) return; + yield* enrichment.refreshRepositoryIdentity(workspaceRoot); + }, + (effect, projectId) => + effect.pipe( + Effect.catch((error) => + Effect.logWarning("failed to refresh repository identity after run completion", { + projectId, + detail: error.message, + }), + ), + ), + ); return { refreshAfterTurn: pullRequests.refreshAfterTurn, refresh: Effect.fn("RunFinalizationService.refresh")( @@ -106,9 +134,14 @@ export const layerObserver = Layer.effect( [workspaceEntries.refresh(cwd), vcsStatus.refreshLocalStatus(cwd)], { concurrency: "unbounded" }, ); - if (local.refName === null || local.isDefaultRef) return; const thread = yield* projections.getThreadShell(threadId); - if (!thread || thread.branch !== local.refName) return; + if (!thread) return; + // Forge discovery probes can take seconds; like the PR lookup, the next turn must not wait. + yield* refreshMissingRepositoryIdentity(thread.projectId).pipe( + FiberMap.run(identityRefreshes, thread.projectId), + ); + if (local.refName === null || local.isDefaultRef) return; + if (thread.branch !== local.refName) return; if (thread.activeRunId !== null && thread.activeRunId !== runId) return; // The forge CLI can take minutes, so the next turn must not wait for it. A newer // turn's lookup replaces a running one instead of queueing behind the cwd's lock. diff --git a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts index 4352d8b2d647..ba49f0bcfba5 100644 --- a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts +++ b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts @@ -446,6 +446,119 @@ it.live("retries a failed selection restart session open before completing", () ), ); +it.live("a workspace handoff during a run does not leave the run interrupted", () => + Effect.scoped( + Effect.gen(function* () { + const cwd = yield* checkpointWorkspace("workspace-handoff-lifecycle"); + const handoffCwd = yield* checkpointWorkspace("workspace-handoff-lifecycle-target"); + const threadId = ThreadId.make("thread:workspace-handoff-lifecycle"); + const state = yield* Ref.make({ + activeTurn: null, + opened: [], + started: [], + closedSessionCount: 0, + failedReplacementOpen: false, + }); + const layerRegistry = ProviderAdapterRegistry.layerSingle(makeRestartAdapter(state)); + + const projection = yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const eventSink = yield* EventSink.EventSinkV2; + + yield* orchestrator.dispatch({ + type: "thread.create", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make("command:workspace-handoff:create"), + threadId, + projectId: ProjectId.make("project:workspace-handoff"), + title: "Workspace handoff", + modelSelection: initialSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: cwd, + }); + const running = yield* eventSink + .stream({ + threadId, + afterSequence: yield* eventSink.latestSequence({ threadId }), + }) + .pipe( + Stream.map((stored) => stored.event), + Stream.filter( + (event) => + event.type === "provider-turn.updated" && event.payload.status === "running", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make("command:workspace-handoff:first"), + threadId, + messageId: MessageId.make("message:workspace-handoff:first"), + text: "move to a worktree", + attachments: [], + modelSelection: initialSelection, + dispatchMode: { type: "start_immediately" }, + }); + yield* Fiber.join(running); + yield* worker.drain(); + + const settled = yield* eventSink + .stream({ + threadId, + afterSequence: yield* eventSink.latestSequence({ threadId }), + }) + .pipe( + Stream.map((stored) => stored.event), + Stream.filter( + (event) => + event.type === "run.updated" && + event.payload.status !== "running" && + event.payload.status !== "queued" && + event.payload.status !== "starting", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + // What t3_worktree_handoff dispatches while the calling turn is still running. + yield* orchestrator.dispatch({ + type: "thread.metadata.update", + commandId: CommandId.make("command:workspace-handoff:rebind"), + threadId, + branch: "repro-handoff", + worktreePath: handoffCwd, + }); + yield* Fiber.join(settled); + yield* worker.drain(); + return yield* orchestrator.getThreadProjection(threadId); + }).pipe( + Effect.provide( + ProviderReplayHarness.layerWithRegistry( + { name: "workspace-handoff-lifecycle" }, + layerRegistry, + ), + ), + ); + + assert.equal(projection.thread.worktreePath, handoffCwd); + assert.equal(projection.runs[0]?.status, "completed"); + assert.isNotNull(projection.runs[0]?.checkpointId ?? null); + assert.isFalse( + projection.turnItems.some((item) => item.type === "run_interrupt_result"), + "an intentional workspace handoff must not project an interrupt result", + ); + }), + ), +); + it.live("Stop during a restart interrupts the turn the restart could not stop", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts index 2c311f256975..e39c9bb07e93 100644 --- a/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts +++ b/apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts @@ -2,6 +2,7 @@ import { assert, it } from "@effect/vitest"; import { ChatAttachmentId, CommandId, + EnvironmentId, MessageId, EventId, NodeId, @@ -19,6 +20,7 @@ import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; @@ -27,8 +29,13 @@ import * as EventSink from "./EventSink.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts"; +import * as ThreadManagementService from "./ThreadManagementService.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as CommandReceiptStore from "./CommandReceiptStore.ts"; import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests"; const driver = ProviderDriverKind.make("codex"); const instanceId = ProviderInstanceId.make("codex"); @@ -339,7 +346,7 @@ it.effect.each( } if (timing === "after delivery") yield* worker.drain(); const delivery = - timing === "during delivery" ? yield* worker.runOnce.pipe(Effect.forkScoped) : null; + timing === "during delivery" ? yield* worker.runOnce().pipe(Effect.forkScoped) : null; if (delivery !== null) yield* Deferred.await(steerEntered); const completed = yield* watch( (event) => @@ -485,11 +492,14 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* canSteer = true, interruptTurn: Effect.Effect = Effect.void, steeringAcceptsAttachments = true, + nativeGoals?: boolean, ) { + const driver = ProviderDriverKind.make(nativeGoals === undefined ? "codex" : "opencode"); const cwd = yield* checkpointWorkspace(name); const events = yield* Queue.unbounded(); const started: ProviderAdapter.ProviderAdapterV2TurnInput[] = []; const steered: string[] = []; + const steeredSelections: Array = []; const interrupted: ProviderTurnId[] = []; const capabilities = { ...CodexProviderCapabilitiesV2, @@ -498,6 +508,7 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* supportsActiveSteering: canSteer, supportsSteeringByInterruptRestart: canRestart, activeSteeringAcceptsAttachments: steeringAcceptsAttachments, + ...(nativeGoals === undefined ? {} : { supportsNativeGoals: nativeGoals }), }, }; const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = { @@ -577,6 +588,7 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* steerTurn: (turn) => Effect.sync(() => { steered.push(turn.message.text); + steeredSelections.push(turn.modelSelection); }), interruptTurn: ({ providerTurnId }) => Effect.sync(() => { @@ -648,7 +660,7 @@ const nextTurnSelectionHarness = Effect.fn("nextTurnSelectionHarness")(function* yield* Fiber.join(running); return threadId; }); - return { cwd, events, started, steered, interrupted, layer, startFirstTurn }; + return { cwd, events, started, steered, steeredSelections, interrupted, layer, startFirstTurn }; }); it.effect.each([ @@ -837,6 +849,230 @@ it.effect("sends a selected held message once and preserves the remaining queue ), ); +it.effect.each( + (["preparing", "starting", "running"] as const).flatMap((status) => + [false, true].map((direct) => ({ status, direct })), + ), +)("delivers a deferred steer for a $status run (direct: $direct)", ({ status, direct }) => + Effect.scoped( + Effect.gen(function* () { + const restarting = status === "starting" && !direct; + const { started, steered, layer, startFirstTurn, events } = yield* nextTurnSelectionHarness( + `deferred-steer-${status}-${direct}`, + restarting, + !restarting, + ); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const threadId = yield* startFirstTurn; + const before = yield* orchestrator.getThreadProjection(threadId); + const run = before.runs[0]!; + const turn = before.providerTurns[0]!; + const now = yield* DateTime.now; + yield* (yield* EventSink.EventSinkV2).write({ + events: [ + { + id: EventId.make("setup-run"), + type: "run.updated", + threadId, + occurredAt: now, + payload: { ...run, status }, + }, + { + id: EventId.make("setup-turn"), + type: "provider-turn.updated", + threadId, + occurredAt: now, + payload: { ...turn, status: "pending" }, + }, + ], + }); + if (status === "preparing" && !direct) { + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("edited-steer"), + threadId, + messageId: MessageId.make("message:edited-steer"), + text: "edit this steer", + attachments: [], + deliveryIntent: "steer", + dispatchMode: { type: "queue_after_active" }, + createdBy: "user", + creationSource: "web", + }); + const edited = (yield* orchestrator.getThreadProjection(threadId)).runs.find( + (candidate) => candidate.userMessageId === MessageId.make("message:edited-steer"), + )!; + yield* orchestrator.dispatch({ + type: "queued-run.edit", + commandId: CommandId.make("edit-steer-to-compact"), + threadId, + runId: edited.id, + text: "/compact", + }); + } + if (direct) { + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("earlier-queue"), + threadId, + messageId: MessageId.make("message:earlier-queue"), + text: "earlier queued selection", + modelSelection: composerSelection, + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "user", + creationSource: "web", + }); + } + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("queue"), + threadId, + messageId: MessageId.make("message:steer-after-setup"), + text: "steer after setup", + attachments: [], + dispatchMode: { type: "queue_after_active" }, + ...(direct ? { deliveryIntent: "steer" as const } : {}), + createdBy: "user", + creationSource: "web", + }); + const queued = (yield* orchestrator.getThreadProjection(threadId)).runs.find( + (candidate) => candidate.userMessageId === MessageId.make("message:steer-after-setup"), + )!; + assert.deepEqual(queued.modelSelection, runSelection); + for (const text of ["/compact", "/logout"]) { + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`queue-${text}`), + threadId, + messageId: MessageId.make(`message:${text}`), + text, + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "user", + creationSource: "web", + }); + const maintenance = (yield* orchestrator.getThreadProjection(threadId)).runs.find( + (candidate) => candidate.userMessageId === MessageId.make(`message:${text}`), + )!; + const error = yield* orchestrator + .dispatch({ + type: "queued-message.promote-to-steer", + commandId: CommandId.make(`promote-${text}`), + threadId, + queuedRunId: maintenance.id, + targetRunId: run.id, + }) + .pipe(Effect.flip); + assert.equal(error._tag, "OrchestratorDispatchError"); + } + const command = { + type: "queued-message.promote-to-steer" as const, + commandId: CommandId.make("promote"), + threadId, + queuedRunId: queued.id, + targetRunId: run.id, + }; + if (!direct) { + yield* orchestrator.dispatch(command); + yield* orchestrator.dispatch(command); + } + let secondQueuedId: typeof queued.id | undefined; + if (restarting) { + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("second"), + threadId, + messageId: MessageId.make("message:second"), + text: "second steer", + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "user", + creationSource: "web", + }); + secondQueuedId = (yield* orchestrator.getThreadProjection(threadId)).runs.find( + (candidate) => candidate.userMessageId === MessageId.make("message:second"), + )!.id; + yield* orchestrator.dispatch({ + ...command, + commandId: CommandId.make("promote-second"), + queuedRunId: secondQueuedId, + }); + } + const secondReDeferred = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "run.updated" && + event.payload.id === secondQueuedId && + event.payload.status === "queued", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + const secondPromoted = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "run.updated" && + event.payload.id === secondQueuedId && + event.payload.status === "cancelled", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + yield* worker.drain(); + assert.deepEqual(steered, []); + const promoted = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "run.updated" && + event.payload.id === queued.id && + event.payload.status === "cancelled", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + yield* (yield* EventSink.EventSinkV2).write({ + events: [ + { + id: EventId.make("ready-run"), + type: "run.updated", + threadId, + occurredAt: now, + payload: run, + }, + ], + }); + yield* Queue.offer(events, { type: "provider_turn.updated", driver, providerTurn: turn }); + yield* Fiber.join(promoted); + if (restarting) yield* Fiber.join(secondReDeferred); + yield* worker.drain(); + if (restarting) { + yield* Fiber.join(secondPromoted); + yield* worker.drain(); + assert.deepEqual( + started.map((turn) => turn.message.text), + ["first", "steer after setup", "second steer"], + ); + } else { + assert.deepEqual(steered, ["steer after setup"]); + assert.lengthOf(started, 1); + } + const after = yield* orchestrator.getThreadProjection(threadId); + assert.equal( + after.messages.find((message) => message.id === queued.userMessageId)?.runId, + run.id, + ); + assert.equal(after.runs.find((candidate) => candidate.id === run.id)?.status, "running"); + }).pipe(Effect.provide(layer)); + }), + ), +); + it.effect("force sends multiple queued messages and can force send a requeued message again", () => Effect.scoped( Effect.gen(function* () { @@ -955,10 +1191,8 @@ it.effect( () => Effect.scoped( Effect.gen(function* () { - const { started, steered, layer, startFirstTurn } = yield* nextTurnSelectionHarness( - "steering-selection-change", - true, - ); + const { started, steered, steeredSelections, layer, startFirstTurn } = + yield* nextTurnSelectionHarness("steering-selection-change", true); yield* Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; @@ -1018,6 +1252,8 @@ it.effect( yield* steer("steer-back", runSelection); const back = yield* orchestrator.getThreadProjection(threadId); assert.deepEqual(steered, ["steer-changed", "steer-reverted", "steer-back"]); + // The adapter gets each steer's selection so it can switch the live turn, and back. + assert.deepEqual(steeredSelections, [composerSelection, runSelection, runSelection]); assert.equal(back.thread.providerInstanceId, instanceId); assert.deepEqual(back.thread.modelSelection, runSelection); }).pipe(Effect.provide(layer)); @@ -1215,6 +1451,301 @@ it.effect("queues a steer until the turn ends when the provider cannot steer it" ), ); +// regression: #33454 a steer that restarts a completion wake run left its cohort claimed forever +const restartCompletionWake = Effect.fn("restartCompletionWake")(function* ( + harness: Pick< + Effect.Success>, + "events" | "started" | "startFirstTurn" + >, + siblingFinishesFirst: boolean, +) { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const sink = yield* EventSink.EventSinkV2; + const threadId = yield* harness.startFirstTurn; + const watchRun = (runId: string, status: string) => + orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "run.updated" && + event.payload.id === runId && + event.payload.status === status, + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + const watchTurnRunning = (ordinal: number) => + orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "provider-turn.updated" && + event.payload.status === "running" && + event.payload.ordinal === ordinal, + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + const finishTurn = (turn: ProviderAdapter.ProviderAdapterV2TurnInput) => + Effect.gen(function* () { + const providerTurn = (yield* orchestrator.getThreadProjection(threadId)).providerTurns.find( + (candidate) => candidate.runAttemptId === turn.attemptId, + )!; + const settled = yield* watchRun(turn.runId, "waiting"); + yield* Queue.offer(harness.events, { + type: "provider_turn.updated", + driver, + providerTurn: { + ...providerTurn, + status: "completed", + completedAt: yield* DateTime.now, + }, + }); + yield* Queue.offer(harness.events, { + type: "turn.terminal", + driver, + providerThreadId: providerTurn.providerThreadId, + providerTurnId: providerTurn.id, + runOrdinal: turn.runOrdinal, + status: "completed", + failure: null, + threadDisposition: "reusable", + }); + yield* Fiber.join(settled); + yield* worker.drain(); + }); + + const origin = harness.started[0]!; + const firstTaskId = NodeId.make("task:steered-wake-first"); + const siblingTaskId = NodeId.make("task:steered-wake-sibling"); + const now = yield* DateTime.now; + yield* sink.write({ + events: [firstTaskId, siblingTaskId].map((taskId) => ({ + id: EventId.make(`event:seed:${taskId}`), + type: "subagent.updated" as const, + threadId, + runId: origin.runId, + nodeId: taskId, + occurredAt: now, + payload: { + id: taskId, + threadId, + runId: origin.runId, + parentNodeId: origin.rootNodeId, + origin: "app_owned" as const, + createdBy: "agent" as const, + driver, + providerInstanceId: instanceId, + providerThreadId: null, + childThreadId: null, + remoteChild: { + environmentId: EnvironmentId.make("environment:box"), + threadId: ThreadId.make(`thread:remote:${taskId}`), + label: "box", + }, + nativeTaskRef: null, + prompt: "Do background work", + title: null, + model: null, + completionWake: "always" as const, + completionDelivery: { state: "pending" as const, observedByRunId: null }, + status: "running" as const, + result: null, + startedAt: now, + completedAt: null, + updatedAt: now, + }, + })), + }); + yield* finishTurn(origin); + + const completeTask = (taskId: NodeId) => + orchestrator.dispatch({ + type: "delegated_task.remote.complete", + commandId: CommandId.make(`command:remote-complete:${taskId}`), + parentThreadId: threadId, + taskId, + status: "completed", + result: `${taskId} finished`, + stoppedThere: "ran-out", + }); + yield* completeTask(firstTaskId); + const firstDelivery = (yield* orchestrator.getThreadProjection(threadId)).runs.find( + (run) => run.id === origin.runId, + )!.delegatedCompletion!.delivery!; + assert.deepEqual(firstDelivery.taskIds, [firstTaskId]); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("command:first-wake"), + threadId, + messageId: firstDelivery.messageId, + text: "Delegated task completed", + attachments: [], + modelSelection: runSelection, + dispatchMode: { type: "queue_after_active" }, + delegatedCompletion: { + parentRunId: origin.runId, + generation: firstDelivery.generation, + taskIds: firstDelivery.taskIds, + }, + createdBy: "agent", + creationSource: "server", + }); + const wakeRunning = yield* watchTurnRunning(2); + yield* orchestrator.resumeQueuedRuns; + yield* worker.drain(); + yield* Fiber.join(wakeRunning); + const wake = harness.started[1]!; + assert.equal(wake.message.messageId, firstDelivery.messageId); + + if (siblingFinishesFirst) { + yield* completeTask(siblingTaskId); + } + + // The provider cannot steer natively, so the steer restarts the wake run. + const steeredRunning = yield* watchTurnRunning(3); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("command:steer-wake"), + threadId, + messageId: MessageId.make("message:steer-wake"), + text: "also check the logs", + attachments: [], + modelSelection: runSelection, + dispatchMode: { type: "steer_active", targetRunId: wake.runId }, + createdBy: "user", + creationSource: "web", + }); + yield* worker.drain(); + yield* Fiber.join(steeredRunning); + const steered = harness.started[2]!; + assert.equal(steered.runId, wake.runId); + assert.equal( + (yield* orchestrator.getThreadProjection(threadId)).attempts.find( + (attempt) => attempt.id === steered.attemptId, + )?.reason, + "steering_restart", + ); + return { + threadId, + origin, + wake, + steered, + firstTaskId, + siblingTaskId, + watchRun, + finishTurn, + completeTask, + }; +}); + +it.effect("wakes the parent for a sibling result after a steer restarts the completion wake", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* nextTurnSelectionHarness("steered-completion-wake", true, false); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const { + threadId, + origin, + wake, + steered, + firstTaskId, + siblingTaskId, + watchRun, + finishTurn, + completeTask, + } = yield* restartCompletionWake(harness, false); + + // A follow-up queued behind the wake starts once the terminal-run listener has settled it. + const followUp = MessageId.make("message:follow-up"); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("command:follow-up"), + threadId, + messageId: followUp, + text: "follow up", + attachments: [], + modelSelection: runSelection, + dispatchMode: { type: "queue_after_active" }, + createdBy: "user", + creationSource: "web", + }); + const followUpRun = (yield* orchestrator.getThreadProjection(threadId)).runs.find( + (run) => run.userMessageId === followUp, + )!; + const followUpStarted = yield* watchRun(followUpRun.id, "starting"); + yield* finishTurn(steered); + yield* Fiber.join(followUpStarted); + assert.equal( + (yield* orchestrator.getThreadProjection(threadId)).runs.find( + (run) => run.id === wake.runId, + )?.status, + "completed", + ); + + yield* completeTask(siblingTaskId); + const final = yield* orchestrator.getThreadProjection(threadId); + const tasks = new Map( + final.subagents.map((task) => [task.id, task.completionDelivery?.state]), + ); + assert.deepInclude( + final.runs.find((run) => run.id === origin.runId)?.delegatedCompletion?.delivery ?? {}, + { generation: 2, taskIds: [siblingTaskId] }, + "the sibling finished after the steered wake but no new completion wake was reserved", + ); + assert.equal(tasks.get(siblingTaskId), "claimed"); + assert.equal( + tasks.get(firstTaskId), + "delivered", + "the steered wake run's own task stays claimed after that run completed", + ); + }).pipe(Effect.provide(harness.layer)); + }), + ), +); + +it.effect( + "offers the next completion wake when a steer restarts a wake a sibling finished behind", + () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* nextTurnSelectionHarness( + "steered-completion-wake-behind", + true, + false, + ); + const offers: ProviderContinuationRequests.ProviderContinuationRequest[] = []; + const requests = Layer.succeed(ProviderContinuationRequests.ProviderContinuationRequests, { + offer: (request) => Effect.sync(() => void offers.push(request)), + take: Effect.never, + }); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const { threadId, origin, firstTaskId, siblingTaskId } = yield* restartCompletionWake( + harness, + true, + ); + const projection = yield* orchestrator.getThreadProjection(threadId); + const delivery = projection.runs.find((run) => run.id === origin.runId) + ?.delegatedCompletion?.delivery; + assert.deepInclude(delivery ?? {}, { generation: 2, taskIds: [siblingTaskId] }); + const tasks = new Map( + projection.subagents.map((task) => [task.id, task.completionDelivery?.state]), + ); + assert.equal(tasks.get(firstTaskId), "delivered"); + assert.equal(tasks.get(siblingTaskId), "claimed"); + assert.deepEqual(offers.at(-1)?.delegatedCompletion, { + parentRunId: origin.runId, + generation: 2, + messageId: delivery!.messageId, + }); + }).pipe(Effect.provide(harness.layer.pipe(Layer.provide(requests)))); + }), + ), +); + // Codex flushes held drafts after the final answer; reserve their pre-steer position before turn end. it.effect("keeps a held commentary draft before the steer when turn end flushes it", () => Effect.scoped( @@ -1475,6 +2006,254 @@ it.effect("keeps a queued message queued when promoted into a run that Stop reac ), ); +// regression: #32891 a restart that Stop already reached is lost +it.effect.each([false, true])( + "refuses a steer after Stop (provider turn ready: %s)", + (providerTurnReady) => + Effect.scoped( + Effect.gen(function* () { + const { started, steered, layer, startFirstTurn } = + yield* nextTurnSelectionHarness("restart-after-stop"); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const threadId = yield* startFirstTurn; + const first = started[0]!; + // The worker has not delivered the interrupt yet, so the run still reads as running. + yield* orchestrator.dispatch({ + type: "run.interrupt", + commandId: CommandId.make("stop"), + threadId, + runId: first.runId, + }); + const turn = (yield* orchestrator.getThreadProjection(threadId)).providerTurns[0]!; + if (!providerTurnReady) { + yield* (yield* EventSink.EventSinkV2).write({ + events: [ + { + id: EventId.make("turn-not-ready"), + type: "provider-turn.updated", + threadId, + occurredAt: yield* DateTime.now, + payload: { ...turn, status: "pending" }, + }, + ], + }); + } + const steerError = yield* orchestrator + .dispatch({ + type: "message.dispatch", + commandId: CommandId.make("steer"), + threadId, + messageId: MessageId.make("message:steer"), + text: "steer", + attachments: [], + modelSelection: runSelection, + dispatchMode: { type: "steer_active", targetRunId: first.runId }, + createdBy: "user", + creationSource: "web", + }) + .pipe(Effect.flip); + assert.equal(steerError._tag, "OrchestratorDispatchError"); + assert.include(String(steerError.cause), "is stopping and cannot be steered"); + if (!providerTurnReady) { + yield* (yield* EventSink.EventSinkV2).write({ + events: [ + { + id: EventId.make("restore-ready-turn"), + type: "provider-turn.updated", + threadId, + occurredAt: yield* DateTime.now, + payload: turn, + }, + ], + }); + } + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("restart"), + threadId, + messageId: MessageId.make("message:restart"), + text: "restart", + attachments: [], + modelSelection: runSelection, + dispatchMode: { type: "restart_active", targetRunId: first.runId }, + createdBy: "user", + creationSource: "web", + }); + yield* worker.drain(); + + assert.deepEqual(steered, []); + assert.deepEqual( + started.map((turn) => turn.message.text), + ["first", "restart"], + ); + }).pipe(Effect.provide(layer)); + }), + ), +); + +// regression: #32891 a restart that arrives after the turn ended is lost +it.effect.each([ + { status: "completed", settled: true }, + { status: "completed", settled: false }, + { status: "interrupted", settled: true }, + { status: "interrupted", settled: false }, + { status: "cancelled", settled: false }, +] as const)( + "starts a restart that missed the turn ($status, run settled: $settled) as a new turn", + ({ status, settled: runSettled }) => + Effect.scoped( + Effect.gen(function* () { + const { events, started, steered, layer, startFirstTurn } = yield* nextTurnSelectionHarness( + `restart-after-${status}-${runSettled ? "settled" : "unsettled"}`, + ); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const threadId = yield* startFirstTurn; + const first = started[0]!; + const turn = (yield* orchestrator.getThreadProjection(threadId)).providerTurns[0]!; + const turnEnded = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "provider-turn.updated" && + event.payload.id === turn.id && + event.payload.status === status, + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + yield* Queue.offer(events, { + type: "provider_turn.updated", + driver, + providerTurn: { ...turn, status, completedAt: yield* DateTime.now }, + }); + yield* Fiber.join(turnEnded); + const settleRun = Effect.gen(function* () { + const settled = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "run.updated" && + event.payload.id === first.runId && + event.payload.status !== "running", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + yield* Queue.offer(events, { + type: "turn.terminal", + driver, + providerThreadId: turn.providerThreadId, + providerTurnId: turn.id, + runOrdinal: first.runOrdinal, + status, + failure: null, + threadDisposition: "reusable", + }); + yield* Fiber.join(settled); + yield* worker.drain(); + }); + if (runSettled) yield* settleRun; + + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("restart"), + threadId, + messageId: MessageId.make("message:restart"), + text: "restart", + attachments: [], + modelSelection: runSelection, + dispatchMode: { type: "restart_active", targetRunId: first.runId }, + createdBy: "user", + creationSource: "web", + }); + yield* worker.drain(); + if (!runSettled) { + const restartLeftQueue = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "run.updated" && + event.payload.userMessageId === "message:restart" && + event.payload.status !== "queued", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + yield* settleRun; + yield* Fiber.join(restartLeftQueue); + yield* worker.drain(); + } + + assert.deepEqual(steered, []); + assert.deepEqual( + started.map((turn) => turn.message.text), + ["first", "restart"], + ); + }).pipe(Effect.provide(layer)); + }), + ), +); + +// regression: #32891 a restart into a failed provider turn is lost +it.effect("queues a restart sent into a provider turn that failed before its run settled", () => + Effect.scoped( + Effect.gen(function* () { + const { events, started, steered, layer, startFirstTurn } = yield* nextTurnSelectionHarness( + "restart-after-failed-unsettled", + ); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const threadId = yield* startFirstTurn; + const first = started[0]!; + const turn = (yield* orchestrator.getThreadProjection(threadId)).providerTurns[0]!; + const turnFailed = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "provider-turn.updated" && + event.payload.id === turn.id && + event.payload.status === "failed", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + yield* Queue.offer(events, { + type: "provider_turn.updated", + driver, + providerTurn: { ...turn, status: "failed", completedAt: yield* DateTime.now }, + }); + yield* Fiber.join(turnFailed); + + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("restart"), + threadId, + messageId: MessageId.make("message:restart"), + text: "restart", + attachments: [], + modelSelection: runSelection, + dispatchMode: { type: "restart_active", targetRunId: first.runId }, + createdBy: "user", + creationSource: "web", + }); + yield* worker.drain(); + + assert.deepEqual(steered, []); + const projection = yield* orchestrator.getThreadProjection(threadId); + assert.equal( + projection.runs.find((run) => run.userMessageId === "message:restart")?.status, + "queued", + ); + }).pipe(Effect.provide(layer)); + }), + ), +); + it.effect("queues a plain send behind the stopped queue and releases its hold", () => Effect.scoped( Effect.gen(function* () { @@ -1543,8 +2322,15 @@ it.effect("queues a plain send behind the stopped queue and releases its hold", ), ); -const goalHarness = Effect.fn("goalHarness")(function* (name: string) { - const harness = yield* nextTurnSelectionHarness(name); +const goalHarness = Effect.fn("goalHarness")(function* (name: string, nativeGoals?: boolean) { + const harness = yield* nextTurnSelectionHarness( + name, + false, + true, + Effect.void, + true, + nativeGoals, + ); const send = (id: string, text: string, delivery: "queue" | "auto", freshContext = false) => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; @@ -1584,21 +2370,105 @@ const goalHarness = Effect.fn("goalHarness")(function* (name: string) { return { ...harness, send, read }; }); -it.effect("stops the running turn and starts a goal sent into it next", () => +it.effect.each([undefined, false, true])( + "goal dispatch stops the active turn only with native support: %s", + (nativeGoals) => + Effect.scoped( + Effect.gen(function* () { + const { steered, layer, startFirstTurn, send, read } = yield* goalHarness( + "goal-steer", + nativeGoals, + ); + yield* Effect.gen(function* () { + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const threadId = yield* startFirstTurn; + yield* send("queued", "queued", "queue"); + const sending = send("goal", "/goal all tests pass", "auto"); + if (nativeGoals === false) { + yield* Effect.flip(sending); + } else yield* sending; + yield* worker.drain(); + + assert.deepEqual(steered, []); + assert.deepEqual(yield* read(threadId), { + queue: ["queued"], + started: nativeGoals === false ? ["first"] : ["first", "/goal all tests pass"], + stopRequested: nativeGoals !== false, + }); + }).pipe(Effect.provide(layer)); + }), + ), +); + +it.effect("starts a goal that names no model on the thread's model ahead of a queued pick", () => + Effect.scoped( + Effect.gen(function* () { + const { started, layer, startFirstTurn, read } = yield* goalHarness("goal-ahead-of-pick"); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const threadId = yield* startFirstTurn; + for (const [id, text, modelSelection] of [ + ["picked", "picked", composerSelection], + ["goal", "/goal all tests pass", undefined], + ] as const) { + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(id), + threadId, + messageId: MessageId.make(`message:${id}`), + text, + attachments: [], + ...(modelSelection === undefined ? {} : { modelSelection }), + dispatchMode: + id === "picked" ? { type: "queue_after_active" } : { type: "start_immediately" }, + ...(id === "picked" ? {} : { deliveryIntent: "auto" as const }), + createdBy: "user", + creationSource: "web", + }); + } + yield* worker.drain(); + + assert.deepEqual(yield* read(threadId), { + queue: ["picked"], + started: ["first", "/goal all tests pass"], + stopRequested: true, + }); + assert.deepEqual(started[1]?.modelSelection, runSelection); + }).pipe(Effect.provide(layer)); + }), + ), +); + +it.effect("restarts by stopping the running turn when the provider cannot interrupt-restart", () => Effect.scoped( Effect.gen(function* () { - const { steered, layer, startFirstTurn, send, read } = yield* goalHarness("goal-steer"); + const { started, steered, layer, startFirstTurn, send, read } = yield* goalHarness( + "restart-without-interrupt-restart", + ); yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; const threadId = yield* startFirstTurn; yield* send("queued", "queued", "queue"); - yield* send("goal", "/goal all tests pass", "auto"); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("restart"), + threadId, + messageId: MessageId.make("message:restart"), + text: "do this instead", + attachments: [], + modelSelection: runSelection, + dispatchMode: { type: "restart_active", targetRunId: started[0]!.runId }, + createdBy: "agent", + creationSource: "server", + }); yield* worker.drain(); assert.deepEqual(steered, []); assert.deepEqual(yield* read(threadId), { queue: ["queued"], - started: ["first", "/goal all tests pass"], + started: ["first", "do this instead"], stopRequested: true, }); }).pipe(Effect.provide(layer)); @@ -1606,6 +2476,111 @@ it.effect("stops the running turn and starts a goal sent into it next", () => ), ); +it.effect("starts a restart that stops the running turn ahead of an older held queue", () => + Effect.scoped( + Effect.gen(function* () { + const { started, layer, startFirstTurn, send, read } = + yield* goalHarness("restart-past-held-queue"); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const threadId = yield* startFirstTurn; + yield* send("queued", "queued", "queue"); + yield* orchestrator.dispatch({ + type: "thread.stop", + commandId: CommandId.make("stop"), + threadId, + }); + yield* worker.drain(); + const secondRunning = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => event.type === "provider-turn.updated" && event.payload.status === "running", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("second"), + threadId, + messageId: MessageId.make("message:second"), + text: "second", + attachments: [], + dispatchMode: { type: "start_immediately" }, + createdBy: "agent", + creationSource: "mcp", + }); + yield* worker.drain(); + yield* Fiber.join(secondRunning); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("restart"), + threadId, + messageId: MessageId.make("message:restart"), + text: "do this instead", + attachments: [], + dispatchMode: { type: "restart_active", targetRunId: started[1]!.runId }, + createdBy: "agent", + creationSource: "mcp", + }); + yield* worker.drain(); + + const { queue, started: startedTexts } = yield* read(threadId); + assert.deepEqual( + { queue, started: startedTexts }, + { queue: ["queued"], started: ["first", "second", "do this instead"] }, + ); + }).pipe(Effect.provide(layer)); + }), + ), +); + +it.effect("reports a restart that stops the running turn as restarted", () => + Effect.scoped( + Effect.gen(function* () { + const { started, layer, startFirstTurn } = yield* goalHarness("restart-send-delivery"); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const threadManagement = yield* ThreadManagementService.ThreadManagementService; + const threadId = yield* startFirstTurn; + const input = { + projectId: ProjectId.make("project:restart-send-delivery"), + commandId: CommandId.make("restart"), + threadId, + messageId: MessageId.make("message:restart"), + text: "do this instead", + attachments: [], + mode: "restart", + createdBy: "agent", + creationSource: "mcp", + } as const; + const sent = yield* threadManagement.sendToThread(input); + // Stop holds the replacement before it starts; a replay must still report the restart. + yield* orchestrator.dispatch({ + type: "thread.stop", + commandId: CommandId.make("stop-after-restart"), + threadId, + }); + const replayed = yield* threadManagement.sendToThread(input); + + assert.equal(started.length, 1); + assert.deepEqual([sent.delivery, replayed.delivery], ["restarted", "restarted"]); + }).pipe( + Effect.provide( + ThreadManagementService.layer.pipe( + Layer.provideMerge(layer), + Layer.provide(ServerSettings.layerTest()), + Layer.provide( + CommandReceiptStore.layer.pipe(Layer.provide(SqlitePersistence.layerMemory)), + ), + ), + ), + ); + }), + ), +); + it.effect("queues a goal clear without stopping work when no goal drives it", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/SubagentProjection.test.ts b/apps/server/src/orchestration-v2/SubagentProjection.test.ts index 87ac8a2ad175..9b3ebfc02f62 100644 --- a/apps/server/src/orchestration-v2/SubagentProjection.test.ts +++ b/apps/server/src/orchestration-v2/SubagentProjection.test.ts @@ -126,7 +126,7 @@ it("keeps a subagent child awake when its parent thread is snoozed", () => { }); it.effect.each(["provider", "mcp"] as const)( - "a %s child does not inherit parent settlement protections and settles once idle", + "a %s child does not inherit parent settlement protections and settles once its parent settles", (creationSource) => Effect.gen(function* () { const store = yield* ProjectionStore.ProjectionStoreV2; @@ -168,6 +168,21 @@ it.effect.each(["provider", "mcp"] as const)( assert.deepEqual(savedParent.pinnedAt, parentCreatedAt); assert.equal(savedParent.pinOrderKey, "a0"); assert.deepEqual(savedParent.autoSettleDisabledAt, parentCreatedAt); + // A subagent child waits for its parent, so settle the pinned parent first. + assert.deepEqual(yield* store.getSettlementCandidates(), []); + yield* store.apply({ + id: EventId.make(`event:${parentThreadId}:settled`), + type: "thread.settled", + threadId: parentThreadId, + occurredAt: childCreatedAt, + payload: { + ...savedParent, + settledOverride: "settled", + settledAt: childCreatedAt, + pinnedAt: null, + pinOrderKey: null, + }, + }); const candidates = yield* store.getSettlementCandidates(); assert.deepEqual( candidates.map((candidate) => candidate.id), diff --git a/apps/server/src/orchestration-v2/ThreadDeletion.test.ts b/apps/server/src/orchestration-v2/ThreadDeletion.test.ts index ef7c23bf7512..a9f20246eb00 100644 --- a/apps/server/src/orchestration-v2/ThreadDeletion.test.ts +++ b/apps/server/src/orchestration-v2/ThreadDeletion.test.ts @@ -279,7 +279,7 @@ it.effect("queues provider and resource cleanup and preserves an earlier deletio assert.deepEqual(deleted.thread.deletedAt, createdAt); assert.deepEqual( deleted.providerSessions.map((session) => session.status), - ["stopped", "error"], + ["stopped"], ); assert.deepEqual( plan.effects.map((effect) => effect.request), @@ -290,6 +290,12 @@ it.effect("queues provider and resource cleanup and preserves an earlier deletio detail: "Thread deleted.", revokeMcpCredential: true, }, + { + type: "provider-session.detach", + providerSessionId: ProviderSessionId.make("session:delete-plan:error"), + detail: "Thread deleted.", + revokeMcpCredential: true, + }, { type: "scheduled-tasks.cleanup" }, { type: "terminal.cleanup" }, { type: "preview.cleanup" }, diff --git a/apps/server/src/orchestration-v2/ThreadDeletion.ts b/apps/server/src/orchestration-v2/ThreadDeletion.ts index 67a27fcf7696..265f59008d87 100644 --- a/apps/server/src/orchestration-v2/ThreadDeletion.ts +++ b/apps/server/src/orchestration-v2/ThreadDeletion.ts @@ -183,7 +183,8 @@ export const planThreadDeletion = Effect.fn("ThreadDeletion.planThreadDeletion") } for (const session of projection.providerSessions) { - if (session.status === "stopped" || session.status === "error") continue; + // An errored session can still own a reusable live runtime. + if (session.status === "stopped") continue; yield* emitEvent({ type: "provider-session.detached", threadId: command.threadId, diff --git a/apps/server/src/orchestration-v2/ThreadFind.test.ts b/apps/server/src/orchestration-v2/ThreadFind.test.ts index 6793421a6433..c3464eecea6f 100644 --- a/apps/server/src/orchestration-v2/ThreadFind.test.ts +++ b/apps/server/src/orchestration-v2/ThreadFind.test.ts @@ -1,4 +1,5 @@ import { assert, it, describe } from "@effect/vitest"; +import { searchableMessageSegments } from "@t3tools/shared/threadFindText"; import { EventId, MessageId, @@ -19,6 +20,8 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as SqlClient from "effect/sql/SqlClient"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; +import { vi } from "vite-plus/test"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; @@ -31,6 +34,8 @@ import { } from "./threadHistoryPaging.ts"; import { projectTurnItemForWire } from "./WireProjection.ts"; +vi.mock("@t3tools/shared/threadFindText", { spy: true }); + const layerTest = Layer.mergeAll(EventStore.layer, ProjectionStore.layer, ProjectStore.layer).pipe( Layer.provideMerge(SqlitePersistence.layerMemory), ); @@ -460,6 +465,40 @@ describe("V2 thread find", () => { }).pipe(Effect.provide(layerTest)), ); + it.effect("keeps its caches while Find is in use and drops them after one idle minute", () => + Effect.gen(function* () { + const projection = yield* setup; + yield* putItems([item("one", 1, "idle needle one"), item("two", 2, "idle needle two")]); + const parses = () => + vi + .mocked(searchableMessageSegments) + .mock.calls.filter(([message]) => message.text.startsWith("idle needle")).length; + const search = (query: string) => projection.searchThread({ threadId, query }); + // One idle minute passes inside this search. The later counts fail if it let the caches go. + yield* projection.searchThreadStream({ threadId, query: "needle" }).pipe( + Stream.tap((frame) => + frame.complete === false ? TestClock.adjust("1 minute") : Effect.void, + ), + Stream.runDrain, + ); + assert.equal(parses(), 2); + yield* TestClock.adjust("40 seconds"); + yield* search("idle"); + yield* TestClock.adjust("40 seconds"); + // A repeated query is answered from the completed scan and still counts as use. + const repeated = yield* projection + .searchThreadStream({ threadId, query: "needle" }) + .pipe(Stream.runCollect); + assert.equal(repeated.length, 1); + yield* TestClock.adjust("40 seconds"); + yield* search("one"); + assert.equal(parses(), 2); + yield* TestClock.adjust("1 minute"); + yield* search("needle"); + assert.equal(parses(), 4); + }).pipe(Effect.provide(layerTest)), + ); + it.effect("invalidates cached counts when streamed text changes or a run rolls back", () => Effect.gen(function* () { const projection = yield* setup; @@ -524,6 +563,50 @@ describe("V2 thread find", () => { }).pipe(Effect.provide(layerTest)), ); + it.effect("finds an async answer that started a later run", () => + Effect.gen(function* () { + yield* createProject(projectId); + const projection = yield* ProjectionStore.ProjectionStoreV2; + const queuedRun = RunId.make("run:queued"); + const answerRun = RunId.make("run:answer"); + yield* commit([ + thread(threadId, projectId), + run(), + run(threadId, queuedRun, 2), + run(threadId, answerRun, 3), + ]); + const requestId = RuntimeRequestId.make("late"); + const question: OrchestrationV2TurnItem = { + ...item("question", 1, ""), + type: "user_input_request", + requestId, + questions: [], + questionAnswer: { requestId, answers: {}, attachmentsByQuestionId: {} }, + }; + const items = [ + question, + { ...item("queued", 2, "queued follow-up", "user_message"), runId: queuedRun }, + { ...item(`async-answer:${requestId}`, 3, "needle", "user_message"), runId: answerRun }, + ]; + yield* commit( + items.map((payload) => ({ + id: EventId.make(`${payload.id}:initial`), + threadId, + occurredAt: at(1), + type: "turn-item.updated" as const, + payload, + })), + ); + const result = yield* projection.searchThread({ threadId, query: "needle" }); + assert.equal(result.totalMatches, 1); + assert.deepEqual(result.match, { + entryId: `async-answer:${requestId}`, + runId: answerRun, + occurrence: 0, + }); + }).pipe(Effect.provide(layerTest)), + ); + it.effect( "searches inherited fork history through the fork boundary, using source item identities", () => diff --git a/apps/server/src/orchestration-v2/ThreadFind.ts b/apps/server/src/orchestration-v2/ThreadFind.ts index ead68680a21c..44099f4ebbc1 100644 --- a/apps/server/src/orchestration-v2/ThreadFind.ts +++ b/apps/server/src/orchestration-v2/ThreadFind.ts @@ -12,6 +12,7 @@ import { countThreadSearchOccurrences } from "@t3tools/shared/threadSearch"; import * as Cache from "effect/Cache"; import * as Data from "effect/Data"; import * as Effect from "effect/Effect"; +import * as RcRef from "effect/RcRef"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; @@ -31,6 +32,7 @@ interface FindRow { interface FindPayload extends FindRow { readonly payload: string; readonly entryId: string; + readonly runId: OrchestrationV2TurnItem["runId"]; readonly type: OrchestrationV2TurnItem["type"]; } interface FindDocument extends FindRow { @@ -38,6 +40,14 @@ interface FindDocument extends FindRow { readonly runId: OrchestrationV2TurnItem["runId"]; readonly count: number; } +type CompletedScans = Map< + string, + { + sequence: number; + documents: readonly FindDocument[]; + sourceThreadIds: readonly ThreadId[]; + } +>; class TextKey extends Data.Class<{ text: string; role: "user" | "assistant" | "plan"; @@ -204,10 +214,12 @@ export const makeThreadFind = Effect.fn("makeThreadFind")(function* ( turn_item_id: string; payload_json: string; entry_id: string; + run_id: OrchestrationV2TurnItem["runId"]; item_type: OrchestrationV2TurnItem["type"]; }>` SELECT item.thread_id, item.turn_item_id, item.payload_json, COALESCE(json_extract(item.payload_json, '$.messageId'), item.turn_item_id) AS entry_id, + json_extract(item.payload_json, '$.runId') AS run_id, json_extract(item.payload_json, '$.type') AS item_type FROM orchestration_v2_projection_turn_items AS item JOIN json_each(${sources}) AS wanted @@ -228,30 +240,26 @@ export const makeThreadFind = Effect.fn("makeThreadFind")(function* ( ...row, payload: payload.payload_json, entryId: payload.entry_id, + runId: payload.run_id, type: payload.item_type, }); }); }); - // Scans visit every message; a cache smaller than the thread churns without hits. - const textCache = yield* Cache.make({ - capacity: 8192, - timeToLive: "1 minute", - lookup: (key: TextKey) => Effect.sync(() => parseText(key)), + const makeTextCache = (capacity: number) => + Cache.make({ capacity, lookup: (key: TextKey) => Effect.sync(() => parseText(key)) }); + // Text keys are message content, so entries never go stale. Cache TTL only expires on access + // and would keep the last searched threads forever. Each search borrows this for its whole + // stream, and all of it is dropped once Find is idle. + const findCaches = yield* RcRef.make({ + acquire: Effect.all({ + // Scans visit every message; a cache smaller than the thread churns without hits. + text: makeTextCache(8192), + // Bounded separately because each entry can hold a large body; typing reuses the parse. + largeText: makeTextCache(32), + scans: Effect.sync((): CompletedScans => new Map()), + }), + idleTimeToLive: "1 minute", }); - // Bounded separately because each entry can hold a large body; typing reuses the parse. - const largeTextCache = yield* Cache.make({ - capacity: 32, - timeToLive: "1 minute", - lookup: (key: TextKey) => Effect.sync(() => parseText(key)), - }); - const scans = new Map< - string, - { - sequence: number; - documents: readonly FindDocument[]; - sourceThreadIds: readonly ThreadId[]; - } - >(); const readRevision = Effect.fn("ThreadFind.readRevision")(function* ( threadIds: readonly ThreadId[], ) { @@ -275,7 +283,7 @@ export const makeThreadFind = Effect.fn("makeThreadFind")(function* ( return revision[0]?.sequence ?? 0; }); const snapshot = Effect.fn("ThreadFind.snapshot")( - function* (input: OrchestrationV2SearchThreadInput) { + function* (input: OrchestrationV2SearchThreadInput, scans: CompletedScans) { const { threadId } = input; const active = yield* sql<{ cwd: string | null }>` SELECT COALESCE(json_extract(t.payload_json, '$.worktreePath'), p.workspace_root) AS cwd @@ -326,8 +334,9 @@ export const makeThreadFind = Effect.fn("makeThreadFind")(function* ( const searchThreadStream = (input: OrchestrationV2SearchThreadInput) => Stream.unwrap( Effect.gen(function* () { + const { scans, ...textCache } = yield* RcRef.get(findCaches); // Release SQLite before decoding and parsing. Both frames describe one snapshot. - const data = yield* snapshot(input); + const data = yield* snapshot(input, scans); if (data.result) return Stream.succeed(data.result); const { payloads, cwd, sequence, cacheKey, sourceThreadIds } = data; const decode = (row: FindPayload) => @@ -338,17 +347,17 @@ export const makeThreadFind = Effect.fn("makeThreadFind")(function* ( ), ); // Answers rendered inside question cards must not appear as separate matches. - const foldedAnswers = new Set(); + const foldedAnswerRunIds = new Map(); for (const payload of payloads) { if (payload.type !== "user_input_request") continue; const { item } = yield* decode(payload); if (item.type === "user_input_request" && item.questionAnswer) - foldedAnswers.add(`async-answer:${item.questionAnswer.requestId}`); + foldedAnswerRunIds.set(`async-answer:${item.questionAnswer.requestId}`, item.runId); } const rows = payloads.filter( (row) => row.type !== "user_input_request" && - !(row.type === "user_message" && foldedAnswers.has(row.entryId)), + !(row.type === "user_message" && foldedAnswerRunIds.get(row.entryId) === row.runId), ); const documents = new Map(); const count = Effect.fnUntraced(function* (row: FindPayload) { @@ -357,7 +366,7 @@ export const makeThreadFind = Effect.fn("makeThreadFind")(function* ( const decoded = yield* decode(row); const key = textKey(decoded.item, cwd, input.skills ?? []); const segments = yield* Cache.get( - key.text.length <= 32_768 ? textCache : largeTextCache, + key.text.length <= 32_768 ? textCache.text : textCache.largeText, key, ); const document = documentFor(decoded, countSegments(segments, input.query)); diff --git a/apps/server/src/orchestration-v2/ThreadFork.execution.test.ts b/apps/server/src/orchestration-v2/ThreadFork.execution.test.ts index 41a1120620f8..fa6fcea91138 100644 --- a/apps/server/src/orchestration-v2/ThreadFork.execution.test.ts +++ b/apps/server/src/orchestration-v2/ThreadFork.execution.test.ts @@ -18,8 +18,10 @@ import { TurnItemId, } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Statement from "effect/sql/Statement"; +import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; @@ -1311,3 +1313,244 @@ it.effect("merge-back rejects a second fork's context until the first is consume ); }).pipe(Effect.provide(forkCases[0]!.layer)), ); + +const runForkWithFailingNativeSource = (forkFailures: number) => { + const driver = ProviderDriverKind.make("claudeAgent"); + const instanceId = ProviderInstanceId.make("claudeAgent"); + const modelSelection = { instanceId, model: "test-model" }; + const started = Deferred.makeUnsafe(); + let forkCalls = 0; + const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = { + instanceId, + driver, + getCapabilities: () => Effect.succeed(ClaudeProviderCapabilitiesV2), + planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }), + openSession: (input) => + Effect.gen(function* () { + const now = yield* DateTime.now; + return { + instanceId, + driver, + providerSessionId: input.providerSessionId, + providerSession: { + id: input.providerSessionId, + driver, + providerInstanceId: instanceId, + status: "ready", + cwd: "/tmp/fork-missing-native-source", + model: modelSelection.model, + capabilities: ClaudeProviderCapabilitiesV2, + createdAt: now, + updatedAt: now, + lastError: null, + }, + events: Stream.never, + ensureThread: ({ threadId, existingProviderThread }) => + Effect.succeed({ + ...existingProviderThread!, + appThreadId: threadId, + nativeThreadRef: { driver, nativeId: "native-fresh-fork", strength: "strong" }, + status: "idle", + updatedAt: now, + }), + resumeThread: ({ providerThread }) => Effect.succeed(providerThread), + startTurn: (turn) => Deferred.succeed(started, turn).pipe(Effect.asVoid), + steerTurn: () => Effect.void, + interruptTurn: () => Effect.void, + respondToRuntimeRequest: () => Effect.void, + readThreadSnapshot: () => Effect.die("unused"), + rollbackThread: () => Effect.die("unused"), + forkThread: ({ sourceProviderThread, targetThreadId }) => + ++forkCalls <= forkFailures + ? Effect.fail( + new ProviderAdapter.ProviderAdapterForkThreadError({ + driver, + providerThreadId: sourceProviderThread.id, + cause: + "Session not found: native-deleted-source; authorization=BearerSyntheticSecret", + }), + ) + : Effect.succeed({ + ...sourceProviderThread, + appThreadId: targetThreadId, + nativeThreadRef: { driver, nativeId: "native-forked", strength: "strong" }, + updatedAt: now, + }), + } satisfies ProviderAdapter.ProviderAdapterV2SessionRuntime; + }), + }; + return Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const now = yield* DateTime.now; + const sourceThreadId = ThreadId.make("fork-missing-source"); + const targetThreadId = ThreadId.make("fork-missing-target"); + const providerThreadId = ProviderThreadId.make("fork-missing-native-thread"); + const sourceRunId = RunId.make("fork-missing-source-run"); + const userMessageId = MessageId.make("fork-missing-source-message"); + + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("fork-missing-create-source"), + threadId: sourceThreadId, + projectId: ProjectId.make("fork-missing-project"), + title: "Fork missing source", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + createdBy: "user", + creationSource: "web", + }); + yield* eventSink.write({ + events: [ + { + id: EventId.make("fork-missing-provider-thread"), + type: "provider-thread.updated", + threadId: sourceThreadId, + occurredAt: now, + payload: { + id: providerThreadId, + driver, + providerInstanceId: instanceId, + providerSessionId: null, + appThreadId: sourceThreadId, + ownerNodeId: null, + nativeThreadRef: { driver, nativeId: "native-deleted-source", strength: "strong" }, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: 1, + lastRunOrdinal: 1, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }, + }, + { + id: EventId.make("fork-missing-source-run"), + type: "run.created", + threadId: sourceThreadId, + runId: sourceRunId, + occurredAt: now, + payload: { + id: sourceRunId, + threadId: sourceThreadId, + ordinal: 1, + providerInstanceId: instanceId, + modelSelection, + providerThreadId, + userMessageId, + rootNodeId: null, + activeAttemptId: null, + status: "completed", + queuePosition: null, + requestedAt: now, + startedAt: now, + completedAt: now, + checkpointId: null, + contextHandoffId: null, + }, + }, + { + id: EventId.make("fork-missing-user-item"), + type: "turn-item.updated", + threadId: sourceThreadId, + runId: sourceRunId, + occurredAt: now, + payload: { + id: TurnItemId.make("fork-missing-user-item"), + threadId: sourceThreadId, + runId: sourceRunId, + nodeId: null, + providerThreadId, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 1, + status: "completed", + title: null, + startedAt: now, + completedAt: now, + updatedAt: now, + type: "user_message", + createdBy: "user", + creationSource: "web", + inputIntent: "turn_start", + messageId: userMessageId, + text: "FORKED_SOURCE_MARKER", + attachments: [], + }, + }, + ], + }); + yield* orchestrator.dispatch({ + type: "thread.fork", + commandId: CommandId.make("fork-missing-fork"), + sourceThreadId, + targetThreadId, + sourcePoint: { type: "run", runId: sourceRunId }, + createdBy: "user", + creationSource: "web", + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("fork-missing-continue"), + threadId: targetThreadId, + messageId: MessageId.make("fork-missing-continue"), + text: "Continue the fork", + attachments: [], + modelSelection, + dispatchMode: { type: "start_immediately" }, + createdBy: "user", + creationSource: "web", + }); + // Run every start attempt so the last one runs with willRetry false. + for (let attempt = 0; attempt < 5; attempt++) { + yield* worker.drain(); + yield* TestClock.adjust("1 second"); + } + + const projection = yield* orchestrator.getThreadProjection(targetThreadId); + assert.deepEqual( + projection.turnItems.flatMap((item) => (item.type === "error" ? [item.title] : [])), + [], + ); + assert.equal(projection.runs[0]?.status, "running"); + return { projection, turn: yield* Deferred.await(started), forkCalls }; + }).pipe( + Effect.provide( + ProviderReplayHarness.layerWithRegistry( + { name: "fork-missing-native-source" }, + ProviderAdapterRegistry.layerFromAdapters([adapter]), + ), + ), + ); +}; + +it.effect("a native fork whose source session is gone starts fresh with the source history", () => + Effect.gen(function* () { + const { projection, turn } = yield* runForkWithFailingNativeSource(Number.POSITIVE_INFINITY); + // Resolving the fork transfer stops later sends from retrying the native fork. + assert.equal(projection.contextTransfers[0]?.status, "resolved_portable"); + assert.isNull(projection.providerThreads[0]?.forkedFrom); + assert.include(JSON.stringify(turn), "FORKED_SOURCE_MARKER"); + const notices = projection.turnItems.flatMap((item) => + item.type === "system_notice" ? [item.message] : [], + ); + assert.lengthOf(notices, 1); + assert.include(notices[0], "Session not found: native-deleted-source"); + assert.notInclude(notices[0], "BearerSyntheticSecret"); + }), +); + +it.effect("a native fork that fails before its last attempt still forks natively", () => + Effect.gen(function* () { + const { projection, forkCalls } = yield* runForkWithFailingNativeSource(1); + assert.equal(forkCalls, 2); + assert.equal(projection.contextTransfers[0]?.status, "consumed"); + assert.equal(projection.contextTransfers[0]?.resolution?.strategy, "native_fork"); + }), +); diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts index 76b9ae316379..e4f61a2e76a4 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts @@ -200,11 +200,13 @@ function makeHarness(options: HarnessOptions = {}) { : Option.none(), ), getByWorkspaceRoot: () => Effect.succeedSome(project), + workspaceScanDirectory: () => Effect.die("unused"), snapshot: Effect.die("unused"), getShell: () => Effect.die("unused"), listShells: () => Effect.die("unused"), }), Layer.mock(GitWorkflow.GitWorkflowService)({ + repositoryRoot: () => Effect.succeed(project.workspaceRoot), createWorktree, renameBranch, fetchWorktreeBaseCommit: @@ -214,6 +216,7 @@ function makeHarness(options: HarnessOptions = {}) { resolveWorktreeBaseRemote: () => Effect.succeed({ remoteName: "origin", branchName: "main" }), remoteExists: options.remoteExists ?? (() => Effect.succeed(true)), remoteBranchExists: () => Effect.succeed(true), + listLocalBranchNames: () => Effect.succeed([]), removeWorktree, deleteLocalBranch, ...options.git, @@ -909,6 +912,7 @@ function scratchHarness() { return Option.some(`/scratch/folder-${claimed.length}`); }), }), + git: { repositoryRoot: (cwd) => Effect.succeed(cwd === scratchRoot ? null : cwd) }, remoteExists: (input) => input.cwd === scratchRoot ? Effect.fail(notARepository("GitWorkflowService.remoteExists", input.cwd)) @@ -979,6 +983,7 @@ it.effect("launches a worktree request for Scratch at the root, in its own folde const harness = scratchHarness(); return Effect.gen(function* () { const launches = yield* ThreadLaunch.ThreadLaunchService; + yield* launches.checkWorktreeBase({ projectId, workspaceStrategy: worktreeStrategy }); const launched = yield* launches.launch( launchInput({ command: "command:launch:scratch-worktree", @@ -2403,6 +2408,64 @@ it.effect.each([0, 1])("releases an async setup before its completion with exit }), ); +it.effect.each(["origin/master", ":/initial"])( + "refuses a worktree in a non-Git project before checking base %s", + (baseRef) => + Effect.gen(function* () { + const harness = makeHarness({ + git: { repositoryRoot: () => Effect.succeed(null) }, + }); + yield* Effect.gen(function* () { + const launches = yield* ThreadLaunch.ThreadLaunchService; + const result = yield* launches + .checkWorktreeBase({ + projectId, + workspaceStrategy: { type: "worktree", baseRef, startFromOrigin: true }, + }) + .pipe(Effect.result); + assert.isTrue(Result.isFailure(result)); + if (Result.isFailure(result)) { + assert.instanceOf(result.failure, ThreadLaunch.ThreadLaunchNotRepositoryError); + assert.deepInclude(result.failure, { + projectId, + projectTitle: project.title, + workspaceRoot: project.workspaceRoot, + }); + } + yield* launches.checkWorktreeBase({ projectId, workspaceStrategy: { type: "root" } }); + yield* launches.checkWorktreeBase({ + projectId, + workspaceStrategy: { type: "existing_worktree", worktreePath: "/existing-worktree" }, + }); + }).pipe(Effect.provide(harness.layer)); + }), +); + +it.effect("leaves uncertain Git detection failures to workspace preparation", () => + Effect.gen(function* () { + const harness = makeHarness({ + git: { + repositoryRoot: () => + Effect.fail( + new GitCommandError({ + operation: "detect-repository", + command: "git", + cwd: project.workspaceRoot, + detail: "Detection failed.", + }), + ), + }, + hasCommit: () => Effect.succeed(true), + }); + yield* ThreadLaunch.ThreadLaunchService.pipe( + Effect.flatMap((launches) => + launches.checkWorktreeBase({ projectId, workspaceStrategy: worktreeStrategy }), + ), + Effect.provide(harness.layer), + ); + }), +); + it.effect.each([ { name: "is missing", baseRef: "t3/renamed", startFromOrigin: false, origin: true }, { @@ -2452,6 +2515,40 @@ it.effect.each([ }), ); +it.effect.each([ + { requested: "werkstatt/baseline", blocking: "werkstatt" }, + { requested: "foo", blocking: "foo/bar" }, +])("refuses a new worktree branch $requested that collides with branch $blocking", (testCase) => + Effect.gen(function* () { + const harness = makeHarness({ + hasCommit: () => Effect.succeed(true), + git: { listLocalBranchNames: () => Effect.succeed(["main", "werkstatt", "foo/bar"]) }, + }); + yield* Effect.gen(function* () { + const launches = yield* ThreadLaunch.ThreadLaunchService; + const workspace = { type: "worktree" as const, baseRef: "main", branch: testCase.requested }; + const commandId = CommandId.make("launch:branch-conflict"); + const check = (branch: string) => + launches.checkWorktreeBase({ + commandId, + projectId, + workspaceStrategy: { ...workspace, branch }, + }); + yield* check("werkstatt-baseline"); + const error = yield* check(testCase.requested).pipe(Effect.flip); + assert.deepInclude(error, { + _tag: "ThreadLaunchBranchConflictError", + branch: testCase.requested, + conflictingBranch: testCase.blocking, + }); + yield* launches.launch( + launchInput({ command: commandId, thread: "thread:branch-conflict", workspace }), + ); + yield* check(testCase.requested); + }).pipe(Effect.provide(harness.layer)); + }), +); + it.effect("replays an accepted send after the project disables the instance", () => { const harness = makeHarness(); const layerFiles = ServerConfig.layerTest(process.cwd(), { prefix: "t3-send-retry-" }).pipe( diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.ts index 7e7829ba947c..65d2c3b8d8e9 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.ts @@ -166,6 +166,33 @@ export class ThreadLaunchBaseRefError extends Schema.TaggedError()( + "ThreadLaunchBranchConflictError", + { + projectId: ProjectId, + projectTitle: Schema.String, + branch: Schema.String, + conflictingBranch: Schema.String, + }, +) { + override get message(): string { + return `Branch "${this.branch}" cannot be created in project "${this.projectTitle}": branch "${this.conflictingBranch}" exists.`; + } +} + +export class ThreadLaunchNotRepositoryError extends Schema.TaggedError()( + "ThreadLaunchNotRepositoryError", + { + projectId: ProjectId, + projectTitle: Schema.String, + workspaceRoot: Schema.String, + }, +) { + override get message(): string { + return `Project "${this.projectTitle}" (${this.workspaceRoot}) is not a Git repository.`; + } +} + export class ThreadLaunchService extends Context.Service< ThreadLaunchService, { @@ -175,10 +202,16 @@ export class ThreadLaunchService extends Context.Service< ThreadLaunchResult, ThreadLaunchError | ThreadManagement.ThreadManagementProviderInstanceDisabledError >; - /** Check before creation for early errors; launch otherwise accepts and reports base failures inside the thread. */ + /** Check a new worktree's base ref and branch name before creation for early errors; launch otherwise accepts and reports these failures inside the thread. */ readonly checkWorktreeBase: ( - input: Pick, - ) => Effect.Effect; + input: Pick & { + /** Skips the check when this launch was already accepted, so its retry replays. */ + readonly commandId?: CommandId; + }, + ) => Effect.Effect< + void, + ThreadLaunchBaseRefError | ThreadLaunchBranchConflictError | ThreadLaunchNotRepositoryError + >; readonly retryPreparation: ( input: ThreadLaunchPreparationInput, ) => Effect.Effect< @@ -263,32 +296,67 @@ const make = Effect.gen(function* () { "ThreadLaunchService.checkWorktreeBase", )(function* (input) { const strategy = input.workspaceStrategy; - // `:/text` is a commit search, which cannot be asked for a commit type. - if (strategy.type !== "worktree" || strategy.baseRef.startsWith(":")) return; - const { baseRef } = strategy; - const unresolved = yield* Effect.gen(function* () { - const project = Option.getOrUndefined(yield* projects.getById(input.projectId)); - if (project === undefined) return undefined; - const cwd = project.workspaceRoot; - // `git worktree add` reads `-` as the previous checkout. - if (yield* git.hasCommit({ cwd, refName: baseRef === "-" ? "@{-1}" : baseRef })) - return undefined; - if ( - strategy.startFromOrigin === true && - (yield* git.resolveWorktreeBaseRemote({ cwd, refName: baseRef })) !== null - ) - return undefined; - // git worktree add accepts remote-tracking branches; let Git decide when any matching ref exists. - if (yield* git.hasRefNamed({ cwd, refName: baseRef })) return undefined; - return (yield* git.hasCommit({ cwd, refName: "HEAD" })) ? project : undefined; - }).pipe(Effect.orElseSucceed(() => undefined)); - if (unresolved !== undefined) + // launch() turns a Scratch worktree request into a root launch. + if (strategy.type !== "worktree" || (yield* managedFolders.isScratchProject(input.projectId))) + return; + if (input.commandId !== undefined) { + const receipt = yield* receipts + .getByCommandId(input.commandId) + .pipe(Effect.orElseSucceed(() => Option.none())); + if (Option.isSome(receipt) && receipt.value.status === "accepted") return; + } + const project = Option.getOrUndefined( + yield* projects.getById(input.projectId).pipe(Effect.orElseSucceed(() => Option.none())), + ); + if (project === undefined) return; + const cwd = project.workspaceRoot; + const repositoryRoot = yield* git + .repositoryRoot(cwd) + .pipe(Effect.orElseSucceed(() => undefined)); + if (repositoryRoot === null) + return yield* new ThreadLaunchNotRepositoryError({ + projectId: project.id, + projectTitle: project.title, + workspaceRoot: cwd, + }); + const { baseRef, branch } = strategy; + const baseResolves = + // `:/text` is a commit search, which cannot be asked for a commit type. + baseRef.startsWith(":") || + (yield* Effect.gen(function* () { + // `git worktree add` reads `-` as the previous checkout. + if (yield* git.hasCommit({ cwd, refName: baseRef === "-" ? "@{-1}" : baseRef })) + return true; + if ( + strategy.startFromOrigin === true && + (yield* git.resolveWorktreeBaseRemote({ cwd, refName: baseRef })) !== null + ) + return true; + // git worktree add accepts remote-tracking branches; let Git decide when any matching ref exists. + if (yield* git.hasRefNamed({ cwd, refName: baseRef })) return true; + return !(yield* git.hasCommit({ cwd, refName: "HEAD" })); + }).pipe(Effect.orElseSucceed(() => true))); + if (!baseResolves) return yield* new ThreadLaunchBaseRefError({ - projectId: unresolved.id, - projectTitle: unresolved.title, - workspaceRoot: unresolved.workspaceRoot, + projectId: project.id, + projectTitle: project.title, + workspaceRoot: project.workspaceRoot, baseRef, }); + // Provisioning flattens temporary names that collide, so only explicit names are refused. + if (branch === undefined || isTemporaryWorktreeBranch(branch)) return; + const conflictingBranch = (yield* git + .listLocalBranchNames(cwd) + .pipe(Effect.orElseSucceed((): ReadonlyArray => []))).find( + (existing) => existing.startsWith(`${branch}/`) || branch.startsWith(`${existing}/`), + ); + if (conflictingBranch !== undefined) + return yield* new ThreadLaunchBranchConflictError({ + projectId: project.id, + projectTitle: project.title, + branch, + conflictingBranch, + }); }); const prepareInBackground = Effect.fn("ThreadLaunchService.prepareInBackground")(function* ( @@ -361,6 +429,7 @@ const make = Effect.gen(function* () { attachments: message.attachments, ...(message.context ? { context: message.context } : {}), modelSelection, + projectId: input.projectId, }) .pipe( Effect.map((result) => ({ diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.test.ts b/apps/server/src/orchestration-v2/ThreadManagementService.test.ts index 742616f27bd3..f60e36b530db 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.test.ts @@ -29,6 +29,8 @@ import * as TestClock from "effect/testing/TestClock"; import { AgentScope } from "@t3tools/shared/AgentScope"; +// Coder: the identity service lives in `coderEnvironment.ts` (upstream's `ServerEnvironment.ts` is cut). +import * as ServerEnvironment from "../coderEnvironment.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; import * as CommandReceiptStore from "./CommandReceiptStore.ts"; @@ -873,3 +875,76 @@ it.effect("Force stop refuses once a newer run has started, before killing anyth expect(killed).toBe(false); }), ); + +it.effect( + "admits two transcript exports at a time and frees a slot on failure or interruption", + () => + Effect.gen(function* () { + const started = yield* Queue.unbounded(); + const gates = new Map( + ["a", "b", "c", "d", "e", "f"].map((id) => [ + id, + Deferred.makeUnsafe(), + ]), + ); + const layerTest = ThreadManagementService.layer.pipe( + Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})), + Layer.provide( + Layer.mock(Orchestrator.OrchestratorV2)({ + getThreadSnapshot: (threadId) => + Queue.offer(started, threadId).pipe( + Effect.andThen(Deferred.await(gates.get(threadId)!)), + Effect.as({ + schemaVersion: 1, + snapshotSequence: 1, + projection: { + thread: { id: threadId, title: threadId }, + updatedAt: DateTime.makeUnsafe(0), + visibleTurnItems: [], + } as unknown as OrchestrationV2ThreadProjection, + }), + ), + }), + ), + Layer.provide(ServerSettings.layerTest()), + Layer.provide(layerReceiptsTest), + ); + + yield* Effect.gen(function* () { + const threads = yield* ThreadManagementService.ThreadManagementService; + const exportThread = (id: string) => + threads.getThreadTranscript(ThreadId.make(id)).pipe( + Effect.provideService(ServerEnvironment.ServerEnvironmentIdentity, { + getEnvironmentId: Effect.succeed(EnvironmentId.make("environment:export")), + }), + Effect.forkChild({ startImmediately: true }), + ); + const a = yield* exportThread("a"); + const b = yield* exportThread("b"); + const c = yield* exportThread("c"); + const d = yield* exportThread("d"); + const e = yield* exportThread("e"); + const f = yield* exportThread("f"); + expect(yield* Queue.takeAll(started)).toEqual(["a", "b"]); + + yield* Fiber.interrupt(c); + yield* Deferred.fail( + gates.get("a")!, + new Orchestrator.OrchestratorProjectionError({ threadId: ThreadId.make("a"), cause: "" }), + ); + expect(yield* Queue.take(started)).toBe("d"); + yield* Fiber.interrupt(b); + expect(yield* Queue.take(started)).toBe("e"); + yield* Deferred.succeed(gates.get("d")!, undefined); + expect(yield* Queue.take(started)).toBe("f"); + yield* Deferred.succeed(gates.get("e")!, undefined); + yield* Deferred.succeed(gates.get("f")!, undefined); + + expect((yield* Fiber.await(a))._tag).toBe("Failure"); + expect((yield* Fiber.join(d)).threadId).toBe("d"); + expect((yield* Fiber.join(e)).threadId).toBe("e"); + expect((yield* Fiber.join(f)).threadId).toBe("f"); + expect(yield* Queue.size(started)).toBe(0); + }).pipe(Effect.provide(layerTest)); + }), +); diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index d060943ec7f2..5e0fff866c33 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -54,6 +54,7 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; import * as CommandReceiptStore from "./CommandReceiptStore.ts"; @@ -485,6 +486,7 @@ export interface ThreadManagementServiceShape { ThreadManagementBackgroundWaitError >; readonly stopDelegatedTasks: (input: { + readonly remoteDispatch?: Orchestrator.OrchestratorV2["Service"]["dispatch"]; readonly threadId: ThreadId; readonly commandId: CommandId; readonly reason?: string | undefined; @@ -494,6 +496,7 @@ export interface ThreadManagementServiceShape { >; readonly getThreadEventSequence: Orchestrator.OrchestratorV2["Service"]["getThreadEventSequence"]; readonly recoverDelegatedTask: Orchestrator.OrchestratorV2["Service"]["recoverDelegatedTask"]; + readonly cancelDelegatedTask: Orchestrator.OrchestratorV2["Service"]["cancelDelegatedTask"]; readonly delegatedTaskResultPending: Orchestrator.OrchestratorV2["Service"]["delegatedTaskResultPending"]; readonly streamStoredEvents: Orchestrator.OrchestratorV2["Service"]["streamStoredEvents"]; readonly streamStoredEventsFrom: Orchestrator.OrchestratorV2["Service"]["streamStoredEventsFrom"]; @@ -505,13 +508,15 @@ export class ThreadManagementService extends Context.Service< ThreadManagementServiceShape >()("t3/orchestration-v2/ThreadManagementService") {} +export const ACTIVE_RUN_STATUSES = [ + "preparing", + "starting", + "running", + "waiting", +] as const satisfies ReadonlyArray; + export function isActiveRun(run: OrchestrationV2Run): boolean { - return ( - run.status === "preparing" || - run.status === "starting" || - run.status === "running" || - run.status === "waiting" - ); + return (ACTIVE_RUN_STATUSES as ReadonlyArray).includes(run.status); } export function isTerminalRunStatus( @@ -676,6 +681,8 @@ const make = Effect.gen(function* () { ensureProjectionTranscript(threadId).pipe( Effect.andThen(orchestrator.getThreadSnapshot(threadId)), ); + // Full-thread exports can otherwise hold every SQLite reader and stall ordinary thread opens. + const transcriptExportPermits = yield* Semaphore.make(2); const getThreadTranscript = Effect.fn("orchestrationV2.threadManagement.getThreadTranscript")( function* (threadId: ThreadId) { const { projection } = yield* getThreadSnapshot(threadId); @@ -701,6 +708,7 @@ const make = Effect.gen(function* () { } return transcript; }, + transcriptExportPermits.withPermits(1), ); const getThreadSnapshotWindow: ThreadManagementServiceShape["getThreadSnapshotWindow"] = ( threadId, @@ -864,6 +872,7 @@ const make = Effect.gen(function* () { ...(input.modelSelection === undefined ? {} : { modelSelection: input.modelSelection }), ...(input.freshContext ? { freshContext: true } : {}), dispatchMode, + ...(mode === "auto" ? { deliveryIntent: "auto" } : {}), createdBy: input.createdBy, creationSource: input.creationSource, }); @@ -893,8 +902,19 @@ const make = Effect.gen(function* () { messageId: input.messageId, }); } - const delivery: ThreadManagementSendResult["delivery"] = - turnItem === null || turnItem.inputIntent === "queued_turn" + // A provider without interrupt-and-restart stops the turn and runs the + // restart as the next turn. Read from this command's own events, so a replay agrees. + const restartedAsNextTurn = + mode === "restart" && + dispatched.storedEvents.some( + ({ event }) => + event.type === "turn-item.updated" && + event.payload.type === "run_interrupt_request" && + event.payload.runId !== run.id, + ); + const delivery: ThreadManagementSendResult["delivery"] = restartedAsNextTurn + ? "restarted" + : turnItem === null || turnItem.inputIntent === "queued_turn" ? "queued" : turnItem.inputIntent === "turn_start" ? "started" @@ -1481,7 +1501,7 @@ const make = Effect.gen(function* () { // One with its result keeps it, but its thread there may run follow-up work. if (task.remoteChild === undefined) continue; const commandId = CommandId.make(`${input.commandId}:stop:${task.id}`); - yield* dispatch( + yield* (input.remoteDispatch ?? dispatch)( task.result === null ? { type: "delegated_task.remote.complete", @@ -1522,7 +1542,9 @@ const make = Effect.gen(function* () { stopDelegatedTasks: false, ...(input.reason === undefined ? {} : { reason: input.reason }), }).pipe( - Effect.andThen(stopDelegatedTasks({ ...input, threadId })), + Effect.andThen( + stopDelegatedTasks({ threadId, commandId: input.commandId, reason: input.reason }), + ), Effect.catch((error) => Effect.logWarning("Unable to stop a delegated task", { parentThreadId: input.threadId, @@ -1627,6 +1649,7 @@ const make = Effect.gen(function* () { awaitBackgroundTasks, getThreadEventSequence: orchestrator.getThreadEventSequence, recoverDelegatedTask: orchestrator.recoverDelegatedTask, + cancelDelegatedTask: orchestrator.cancelDelegatedTask, delegatedTaskResultPending: orchestrator.delegatedTaskResultPending, streamStoredEvents: orchestrator.streamStoredEvents, streamStoredEventsFrom: orchestrator.streamStoredEventsFrom, diff --git a/apps/server/src/orchestration-v2/ThreadMessageIntake.ts b/apps/server/src/orchestration-v2/ThreadMessageIntake.ts index 987a1deb246b..2a390ad339f6 100644 --- a/apps/server/src/orchestration-v2/ThreadMessageIntake.ts +++ b/apps/server/src/orchestration-v2/ThreadMessageIntake.ts @@ -24,6 +24,7 @@ function dispatchWasNotAccepted( case "OrchestratorCommandIdConflictError": case "OrchestratorSubagentThreadReadOnlyError": case "OrchestratorThreadAboveModeLimitError": + case "OrchestratorProviderSelectionBusyError": case "ThreadManagementProviderInstanceDisabledError": return true; default: diff --git a/apps/server/src/orchestration-v2/ThreadPullRequestService.test.ts b/apps/server/src/orchestration-v2/ThreadPullRequestService.test.ts index 6aaa22b43859..c53e1700518d 100644 --- a/apps/server/src/orchestration-v2/ThreadPullRequestService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadPullRequestService.test.ts @@ -181,6 +181,139 @@ describe("ThreadPullRequestServiceV2 reads", () => { ), ); + it.effect("looks up a finished turn's branch once for all its turn-end events", () => + Effect.scoped( + Effect.gen(function* () { + const thread = { ...threadShell("turn-end-thread"), branch: "t3/turn-end" }; + const otherThread = { + ...threadShell("other-turn-end-thread"), + branch: "t3/other-turn-end", + }; + const project: OrchestrationProjectShell = { + id: thread.projectId, + title: "Project", + workspaceRoot: "/workspace/project", + defaultModelSelection: null, + scripts: [], + createdAt: "2026-08-01T00:00:00.000Z", + updatedAt: "2026-08-01T00:00:00.000Z", + }; + // A turn end commits its checkpoints and the run's end together. The queued + // non-refresh request ahead of them must still become a refresh when they join it. + const turnEnd = [ + { type: "thread.metadata-updated", threadId: thread.id }, + { type: "checkpoint.captured", threadId: thread.id }, + { type: "checkpoint.captured", threadId: thread.id }, + { type: "run.updated", threadId: thread.id, payload: { status: "completed" } }, + { type: "checkpoint.captured", threadId: otherThread.id }, + { type: "run.updated", threadId: otherThread.id, payload: { status: "completed" } }, + ] as Array; + // Arrives during the thread's own lookup, which may already have read the old host state. + const duringLookup = [ + { type: "checkpoint.captured", threadId: thread.id }, + { type: "thread.metadata-updated", threadId: thread.id }, + ] as Array; + const activation = yield* Deferred.make(); + const sweepStarted = yield* Deferred.make(); + const turnEndQueued = yield* Deferred.make(); + const lookupStarted = yield* Deferred.make(); + const followUpQueued = yield* Deferred.make(); + const lookups: Array<{ readonly branch: string; readonly refresh: boolean }> = []; + const dependencies = Layer.mergeAll( + Layer.mock(Orchestrator.OrchestratorV2)({ + // The batch arrives while the worker is busy with the startup sweep. + streamDomainEvents: Stream.fromEffect(Deferred.await(sweepStarted)).pipe( + Stream.drain, + Stream.concat(Stream.fromIterable(turnEnd)), + Stream.concat( + Stream.fromEffect(Deferred.succeed(turnEndQueued, undefined)).pipe(Stream.drain), + ), + Stream.concat(Stream.fromEffect(Deferred.await(lookupStarted)).pipe(Stream.drain)), + Stream.concat(Stream.fromIterable(duringLookup)), + Stream.concat( + Stream.fromEffect(Deferred.succeed(followUpQueued, undefined)).pipe(Stream.drain), + ), + Stream.concat(Stream.never), + ), + getPullRequestSnapshot: ({ threadId }) => + Effect.succeed({ + schemaVersion: 2, + snapshotSequence: 1, + threads: [thread, otherThread].filter( + (candidate) => threadId === undefined || candidate.id === threadId, + ), + archivedThreads: [], + }), + }), + Layer.mock(ProjectStore.ProjectStoreV2)({ + listShells: () => Effect.succeed([project]), + getShell: () => Effect.succeedSome(project), + }), + Layer.mock(GitManager.GitManager)({ + branchPullRequest: (input, options) => + Effect.gen(function* () { + lookups.push({ branch: input.branch, refresh: options?.refresh ?? false }); + if (!(yield* Deferred.isDone(turnEndQueued))) { + yield* Deferred.succeed(sweepStarted, undefined); + yield* Deferred.await(turnEndQueued); + } else if ( + input.branch === thread.branch && + !(yield* Deferred.isDone(followUpQueued)) + ) { + yield* Deferred.succeed(lookupStarted, undefined); + yield* Deferred.await(followUpQueued); + } + return null; + }), + }), + Layer.mock(PullRequestService.PullRequestService)({}), + Layer.mock(RepositoryIdentityResolver.RepositoryIdentityResolver)({ + resolve: () => + Effect.succeed({ + canonicalKey: "github.com/acme/project", + locator: { + source: "git-remote" as const, + remoteName: "origin", + remoteUrl: "git@github.com:acme/project.git", + }, + provider: SourceControlProviderKind.make("github"), + owner: "acme", + name: "project", + }), + }), + Layer.mock(BackgroundPolicy.BackgroundPolicy)({ + shouldRunScopeWork: () => Effect.succeed(false), + shouldRunOpportunisticWork: Effect.succeed(false), + }), + Layer.succeed(ServerActivation.ServerActivation, Deferred.await(activation)), + Layer.succeed( + SecureRandom.SecureRandom, + SecureRandom.make((size) => new Uint8Array(size).fill(1)), + ), + FileSystem.layerNoop({}), + Path.layer, + ); + yield* Effect.gen(function* () { + const service = yield* ThreadPullRequestService.make; + yield* service.start(); + yield* Deferred.succeed(activation, undefined); + yield* Deferred.await(followUpQueued); + yield* service.drain; + }).pipe(Effect.provide(dependencies)); + + // The startup sweep, one refresh lookup per thread for its turn-end batch, then one + // follow-up refresh for the events that arrived during the thread's lookup. + expect(lookups).toEqual([ + { branch: thread.branch, refresh: false }, + { branch: otherThread.branch, refresh: false }, + { branch: thread.branch, refresh: true }, + { branch: otherThread.branch, refresh: true }, + { branch: thread.branch, refresh: true }, + ]); + }), + ), + ); + it.effect.each([ { worktree: "a linked worktree", @@ -423,6 +556,132 @@ describe("ThreadPullRequestServiceV2 reads", () => { ), ); + it.effect.each([ + { state: "open", checkedOut: false }, + { state: "merged", checkedOut: false }, + { state: "closed", checkedOut: false }, + { state: "merged", checkedOut: true }, + ] as const)( + "associates a $state branch PR (checkedOut=$checkedOut) only with threads that predate its completion or saved it", + ({ state, checkedOut }) => + Effect.scoped( + Effect.gen(function* () { + const reference = { + projectId: ProjectId.make("project-1"), + repository: "acme/project", + number: 42, + url: "https://github.com/acme/project/pull/42", + }; + const terminalAt = "2026-09-19T00:00:00.000Z"; + const detected = { + number: 42, + url: reference.url, + title: "Previous staging work", + baseRef: "main", + headRef: "staging", + state, + updatedAt: terminalAt, + closedAt: state === "closed" ? terminalAt : null, + mergedAt: state === "merged" ? terminalAt : null, + repositoryKey: "github.com/acme/project", + }; + const threads = [ + { ...threadShell("older"), createdAt: DateTime.makeUnsafe("2026-09-18T00:00:00Z") }, + { ...threadShell("same-time"), createdAt: DateTime.makeUnsafe(terminalAt) }, + threadShell("new"), + { ...threadShell("saved"), branchPullRequest: reference }, + { ...threadShell("explicit"), linkedPullRequest: reference }, + ].map((thread) => ({ ...thread, branch: "staging" })); + const project: OrchestrationProjectShell = { + id: reference.projectId, + title: "Project", + workspaceRoot: "/workspace/project", + defaultModelSelection: null, + scripts: [], + createdAt: "2026-08-01T00:00:00.000Z", + updatedAt: "2026-08-01T00:00:00.000Z", + }; + const activation = yield* Deferred.make(); + const sweepStarted = yield* Deferred.make(); + const commands: Array = []; + const dependencies = Layer.mergeAll( + Layer.mock(Orchestrator.OrchestratorV2)({ + streamDomainEvents: Stream.never, + getPullRequestSnapshot: () => + Deferred.succeed(sweepStarted, undefined).pipe( + Effect.as({ snapshotSequence: 1, threads }), + ), + dispatch: (command) => + Effect.sync(() => { + commands.push(command); + return { sequence: 2, storedEvents: [] }; + }), + }), + Layer.mock(ProjectStore.ProjectStoreV2)({ + listShells: () => Effect.succeed([project]), + getShell: () => Effect.succeedSome(project), + }), + Layer.mock(GitManager.GitManager)({ + branchPullRequest: () => Effect.succeed(detected), + checkedOutPullRequest: () => Effect.succeed(checkedOut), + }), + Layer.mock(PullRequestService.PullRequestService)({ + summary: () => + Effect.succeed({ + ...detected, + ...reference, + provider: SourceControlProviderKind.make("github"), + baseBranch: detected.baseRef, + headBranch: detected.headRef, + }), + }), + Layer.mock(RepositoryIdentityResolver.RepositoryIdentityResolver)({ + resolve: () => + Effect.succeed({ + canonicalKey: "github.com/acme/project", + locator: { + source: "git-remote" as const, + remoteName: "origin", + remoteUrl: "git@github.com:acme/project.git", + }, + provider: SourceControlProviderKind.make("github"), + owner: "acme", + name: "project", + }), + }), + Layer.mock(BackgroundPolicy.BackgroundPolicy)({}), + Layer.succeed(ServerActivation.ServerActivation, Deferred.await(activation)), + Layer.succeed( + SecureRandom.SecureRandom, + SecureRandom.make((size) => new Uint8Array(size).fill(1)), + ), + FileSystem.layerNoop({}), + Path.layer, + ); + yield* Effect.gen(function* () { + const service = yield* ThreadPullRequestService.make; + yield* service.start(); + yield* Deferred.succeed(activation, undefined); + yield* Deferred.await(sweepStarted); + yield* service.drain; + }).pipe(Effect.provide(dependencies)); + + expect( + commands.map((command) => { + if (command.type !== "thread.pull-request.sync") throw new Error(command.type); + expect(command).not.toHaveProperty("linkedPullRequest"); + return [command.threadId, command.branchPullRequest]; + }), + ).toEqual( + (state === "open" || checkedOut + ? ["older", "same-time", "new", "explicit"] + : ["older", "same-time"] + ).map((id) => [id, reference]), + ); + }), + ), + ); + it.effect.each([ { state: "closed", sameBranch: false }, { state: "merged", sameBranch: false }, @@ -457,6 +716,7 @@ describe("ThreadPullRequestServiceV2 reads", () => { const activation = yield* Deferred.make(); const sweepStarted = yield* Deferred.make(); const commands: Array = []; + let snapshotThread: unknown; const dependencies = Layer.mergeAll( Layer.mock(Orchestrator.OrchestratorV2)({ streamDomainEvents: Stream.never, @@ -484,6 +744,7 @@ describe("ThreadPullRequestServiceV2 reads", () => { ]) yield* sql.unsafe(`DROP TABLE ${table}`); const snapshot = yield* store.getPullRequestSnapshot(options); + snapshotThread = snapshot.threads[0]; yield* Deferred.succeed(sweepStarted, undefined); return snapshot; }).pipe( @@ -548,6 +809,7 @@ describe("ThreadPullRequestServiceV2 reads", () => { yield* service.drain; }).pipe(Effect.provide(dependencies)); + expect(snapshotThread).toHaveProperty("createdAt", NOW); if (sameBranch) { expect(commands).toEqual([]); } else { diff --git a/apps/server/src/orchestration-v2/ThreadPullRequestService.ts b/apps/server/src/orchestration-v2/ThreadPullRequestService.ts index d84c532d06ec..69fbd1fd918b 100644 --- a/apps/server/src/orchestration-v2/ThreadPullRequestService.ts +++ b/apps/server/src/orchestration-v2/ThreadPullRequestService.ts @@ -3,6 +3,7 @@ import { type OrchestrationProjectShell, type OrchestrationV2DomainEvent, type OrchestrationV2ThreadShell, + type PullRequestSummary, type ThreadId, type ThreadLinkedPullRequest, } from "@t3tools/contracts"; @@ -10,6 +11,7 @@ import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker"; import * as Cause from "effect/Cause"; import * as Context from "effect/Context"; import * as SecureRandom from "effect/crypto/SecureRandom"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Option from "effect/Option"; @@ -49,6 +51,19 @@ function samePullRequest( ); } +function pullRequestPredatesThread( + pullRequest: Pick, + createdAt: DateTime.Utc, +): boolean { + const terminalAt = + pullRequest.state === "merged" + ? pullRequest.mergedAt + : pullRequest.state === "closed" + ? pullRequest.closedAt + : null; + return terminalAt != null && Date.parse(terminalAt) < DateTime.toEpochMillis(createdAt); +} + function pullRequestMatchesProject( pullRequest: GitManager.GitBranchPullRequest, project: OrchestrationProjectShell, @@ -209,9 +224,22 @@ export const make = Effect.gen(function* () { } : null; + // Only a new association is refused: a thread keeps a PR it already saved, and a PR the + // user checked out stays with the threads made for it, however long ago it closed. + const isHistoricalFor = (thread: (typeof group)[number]) => + detected !== null && + pullRequestPredatesThread(detected, thread.createdAt) && + !samePullRequest(thread.branchPullRequest, detectedReference); + const dropsHistorical = + detected !== null && + first.branch !== null && + group.some(isHistoricalFor) && + !(yield* git.checkedOutPullRequest({ cwd, branch: first.branch, url: detected.url })); + const plans = yield* Effect.forEach(group, (thread) => Effect.gen(function* () { - let branchPullRequest = detectedReference; + let branchPullRequest = + dropsHistorical && isHistoricalFor(thread) ? null : detectedReference; if ( branchPullRequest === null && thread.branch !== null && @@ -346,8 +374,15 @@ export const make = Effect.gen(function* () { ); }); + // At most one queued request per thread, so a turn end's events share one lookup. + // A request after that lookup starts queues another, so later host changes are still read. + const queuedThreadRequests = new Map(); + const worker = yield* makeDrainableWorker((request: RefreshRequest) => - synchronize(request).pipe( + Effect.suspend(() => { + if (request.threadId !== null) queuedThreadRequests.delete(request.threadId); + return synchronize(request); + }).pipe( Effect.provideService(GitVcsDriver.BackgroundGitWork, true), Effect.catchCauseIf( (cause) => !Cause.hasInterruptsOnly(cause), @@ -359,15 +394,27 @@ export const make = Effect.gen(function* () { ), ); + const enqueueThread = (threadId: ThreadId, refresh: boolean) => + Effect.suspend(() => { + const queued = queuedThreadRequests.get(threadId); + if (queued !== undefined) { + queued.refresh ||= refresh; + return Effect.void; + } + const request = { threadId, refresh }; + queuedThreadRequests.set(threadId, request); + return worker.enqueue(request); + }); + const processEvent = (event: OrchestrationV2DomainEvent) => { switch (event.type) { case "thread.created": case "thread.unarchived": case "thread.metadata-updated": - return worker.enqueue({ threadId: event.threadId, refresh: false }); + return enqueueThread(event.threadId, false); case "thread.unsettled": case "checkpoint.captured": - return worker.enqueue({ threadId: event.threadId, refresh: true }); + return enqueueThread(event.threadId, true); case "run.updated": if ( event.payload.status === "completed" || @@ -375,7 +422,7 @@ export const make = Effect.gen(function* () { event.payload.status === "cancelled" || event.payload.status === "interrupted" ) { - return worker.enqueue({ threadId: event.threadId, refresh: true }); + return enqueueThread(event.threadId, true); } break; } diff --git a/apps/server/src/orchestration-v2/ThreadSearch.test.ts b/apps/server/src/orchestration-v2/ThreadSearch.test.ts index e758280f9388..421e79f5a090 100644 --- a/apps/server/src/orchestration-v2/ThreadSearch.test.ts +++ b/apps/server/src/orchestration-v2/ThreadSearch.test.ts @@ -1,5 +1,6 @@ import { assert, it } from "@effect/vitest"; import { + EnvironmentId, EventId, MessageId, ProjectId, @@ -18,9 +19,11 @@ import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as SqlClient from "effect/sql/SqlClient"; +import { serializeAssistantCitation } from "@t3tools/shared/assistantCitations"; import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import { rebuildMessageSearch } from "./messageSearchIndex.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as ProjectStore from "./ProjectStore.ts"; import * as ThreadSearch from "./ThreadSearch.ts"; @@ -150,6 +153,20 @@ const message = ( }, }); +// The prefix and suffix are hidden selection context, not text the reader sees. +const quote = serializeAssistantCitation({ + version: 1, + environmentId: EnvironmentId.make("environment:local"), + threadId: ThreadId.make("thread:quoted"), + messageId: MessageId.make("quoted-answer"), + text: "A lighthouse guides ships safely through the darkness.", + comment: "what are you talking about", + start: 0, + end: 54, + prefix: "beacon01", + suffix: "harbor", +}); + it.effect.each([true, false])( "ThreadSearch excludes bootstrap text before the hit and candidate limits (indexed: %s)", (indexed) => @@ -349,6 +366,96 @@ it.layer(layerTest)("ThreadSearch", (it) => { }), ); + it.effect("matches the readable text of quotes and their comments", () => + Effect.gen(function* () { + const projections = yield* ProjectionStore.ProjectionStoreV2; + const search = yield* ThreadSearch.ThreadSearch; + const project = ProjectId.make("project:search-citation"); + const threadId = ThreadId.make("thread:citation"); + yield* createProject(project); + yield* Effect.forEach( + [ + thread(threadId, project), + message(threadId, "citation", "user", `${quote} quiet silver lantern`), + ], + projections.apply, + { discard: true }, + ); + const snippets = (query: string) => + Effect.map(search.search({ query }), (result) => + result.matches + .filter((match) => match.threadId === threadId) + .map((match) => match.snippet), + ); + + const readable = + "A lighthouse guides ships safely through the darkness. Comment: what are you talking about quiet silver lantern"; + assert.deepEqual(yield* snippets("what are you talking about"), [ + `…${readable.slice(readable.indexOf("the darkness"))}`, + ]); + assert.deepEqual(yield* snippets("ships safely through"), [readable]); + assert.deepEqual(yield* snippets("beacon01"), []); + assert.deepEqual(yield* snippets("harbor"), []); + + yield* projections.apply(message(threadId, "citation", "user", "quiet silver lantern")); + assert.deepEqual(yield* snippets("what are you talking about"), []); + assert.deepEqual(yield* snippets("lantern"), ["quiet silver lantern"]); + }), + ); + + it.effect("finds accented, CJK, and apostrophe words of a quote through the index", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const search = yield* ThreadSearch.ThreadSearch; + const project = ProjectId.make("project:search-citation-index"); + const threadId = ThreadId.make("thread:citation-index"); + // The stored link percent-encodes these words; the index must hold them as they read. + const encodedQuote = serializeAssistantCitation({ + version: 1, + environmentId: EnvironmentId.make("environment:local"), + threadId: ThreadId.make("thread:quoted"), + messageId: MessageId.make("quoted-answer"), + text: "The kittiwake's résumé mentions 灯台の光 twice.", + comment: "naïve question в логах", + start: 0, + end: 44, + prefix: "", + suffix: "", + }); + yield* createProject(project); + yield* Effect.forEach( + [thread(threadId, project), message(threadId, "citation-index", "user", encodedQuote)], + projections.apply, + { discard: true }, + ); + const index = yield* sql` + SELECT 1 FROM sqlite_master WHERE name = 'orchestration_v2_message_search' + `; + assert.lengthOf(index, 1); + const threadIds = (query: string) => + search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId))); + + for (const query of [ + "kittiwake's", + "kittiwake’s", + "résumé", + "灯台の光", + "naïve", + "логах в", + ]) { + assert.deepEqual(yield* threadIds(query), [threadId], query); + } + // `t3 doctor` rebuilds the index after VACUUM; quotes must still read. + yield* rebuildMessageSearch(sql); + assert.deepEqual(yield* threadIds("résumé"), [threadId]); + + yield* projections.apply(message(threadId, "citation-index", "user", "plain words")); + assert.deepEqual(yield* threadIds("résumé"), []); + assert.deepEqual(yield* threadIds("plain"), [threadId]); + }), + ); + it.effect("indexes a message when it finishes streaming and follows later edits", () => Effect.gen(function* () { const projections = yield* ProjectionStore.ProjectionStoreV2; diff --git a/apps/server/src/orchestration-v2/ThreadSearch.ts b/apps/server/src/orchestration-v2/ThreadSearch.ts index 04f68ef64c34..d3c96a40ce2b 100644 --- a/apps/server/src/orchestration-v2/ThreadSearch.ts +++ b/apps/server/src/orchestration-v2/ThreadSearch.ts @@ -33,7 +33,11 @@ import { type ThreadSearchSubject, } from "@t3tools/shared/paletteSearch"; import { threadSearchPullRequests } from "@t3tools/shared/threadPullRequests"; -import { indexedMessageCondition, nonPlaceholderMessageCondition } from "./messageSearchIndex.ts"; +import { + indexedMessageCondition, + messageSearchText, + nonPlaceholderMessageCondition, +} from "./messageSearchIndex.ts"; import { loadOlderSettledThreadsPage } from "./ShellStream.ts"; // When the user last wrote in the thread aliased `threads`, as the clients @@ -203,7 +207,7 @@ export const messageHitsQuery = ( messages.thread_id AS "threadId", eligible.project_id AS "projectId", messages.role AS source, - json_extract(messages.payload_json, '$.text') AS text, + ${messageSearchText(sql)} AS text, messages.created_at AS "messageCreatedAt" FROM orchestration_v2_projection_messages AS messages INNER JOIN eligible ON eligible.thread_id = messages.thread_id @@ -220,7 +224,7 @@ export const messageHitsQuery = ( messages.thread_id AS "threadId", eligible.project_id AS "projectId", messages.role AS source, - json_extract(messages.payload_json, '$.text') AS text, + ${messageSearchText(sql)} AS text, messages.created_at AS "messageCreatedAt" FROM orchestration_v2_message_search INNER JOIN orchestration_v2_projection_messages AS messages @@ -241,13 +245,13 @@ export const messageHitsQuery = ( AND messages.message_id NOT LIKE 'async-answer:%' AND NOT EXISTS ( SELECT 1 FROM json_each(${substrings}) AS token - WHERE instr(lower(json_extract(messages.payload_json, '$.text')), token.value) = 0 + WHERE instr(lower(${messageSearchText(sql)}), token.value) = 0 ) AND NOT EXISTS ( SELECT 1 FROM json_each(${shortSpellings}) AS token WHERE NOT EXISTS ( SELECT 1 FROM json_each(token.value) AS spelling - WHERE instr(json_extract(messages.payload_json, '$.text'), spelling.value) > 0 + WHERE instr(${messageSearchText(sql)}, spelling.value) > 0 ) ) -- The index's own rowid, so FTS5 returns hits newest first and stops at diff --git a/apps/server/src/orchestration-v2/ThreadSettlementService.test.ts b/apps/server/src/orchestration-v2/ThreadSettlementService.test.ts index 09f1e7b43c9a..8d8445c3406f 100644 --- a/apps/server/src/orchestration-v2/ThreadSettlementService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadSettlementService.test.ts @@ -6,8 +6,10 @@ import { ProjectId, EventId, ProviderInstanceId, + MessageId, RuntimeRequestId, RunId, + type OrchestrationV2Run, ThreadId, TurnItemId, type OrchestrationProjectShell, @@ -35,6 +37,7 @@ import * as Stream from "effect/Stream"; import { TestClock } from "effect/testing"; import * as GitManager from "../git/GitManager.ts"; +import * as SqlitePersistence from "../persistence/Sqlite.ts"; import * as ProjectSetupScriptRunner from "../project/ProjectSetupScriptRunner.ts"; import * as PullRequestService from "../pullRequest/PullRequestService.ts"; import * as ServerActivation from "../serverActivation.ts"; @@ -563,6 +566,7 @@ interface HarnessOptions { ) => Effect.Effect; readonly onCloseIdle?: () => Effect.Effect; readonly currentThreads?: ReadonlyArray; + readonly projections?: ProjectionStore.ProjectionStoreV2Shape; } const makeHarness = Effect.fn("makeThreadSettlementHarness")(function* (options: HarnessOptions) { @@ -589,6 +593,7 @@ const makeHarness = Effect.fn("makeThreadSettlementHarness")(function* (options: const summaryRecovery = yield* Ref.make>([]); const invalidatedCwds = yield* Ref.make>([]); const domainEvents = yield* PubSub.unbounded(); + const observedEvents = yield* Queue.unbounded(); const closedIdle = yield* Queue.unbounded<{ readonly threadId: string }>(); const scriptRuns = yield* Queue.unbounded(); @@ -659,18 +664,24 @@ const makeHarness = Effect.fn("makeThreadSettlementHarness")(function* (options: listShells: () => Ref.get(snapshots).pipe(Effect.map((snapshot) => snapshot.projects)), }), Layer.mock(ProjectionStore.ProjectionStoreV2)({ + ...options.projections, getSettlementCandidates: (threadId) => Ref.update(candidateReads, (reads) => [...reads, threadId]).pipe( Effect.andThen(Ref.updateAndGet(snapshotReadCount, (count) => count + 1)), Effect.tap((count) => Queue.offer(snapshotReads, count)), - Effect.andThen(Ref.get(snapshots)), - Effect.map((snapshot) => - threadId === undefined - ? snapshot.threads - : snapshot.threads.filter((thread) => thread.id === threadId), + Effect.andThen( + options.projections?.getSettlementCandidates(threadId) ?? + Ref.get(snapshots).pipe( + Effect.map((snapshot) => + threadId === undefined + ? snapshot.threads + : snapshot.threads.filter((thread) => thread.id === threadId), + ), + ), ), ), getThread: (threadId) => { + if (options.projections) return options.projections.getThread(threadId); const thread = options.currentThreads?.find((candidate) => candidate.id === threadId); return thread ? Effect.succeed(thread) @@ -678,7 +689,9 @@ const makeHarness = Effect.fn("makeThreadSettlementHarness")(function* (options: }, }), Layer.mock(Orchestrator.OrchestratorV2)({ - streamDomainEvents: Stream.fromPubSub(domainEvents), + streamDomainEvents: Stream.fromPubSub(domainEvents).pipe( + Stream.tap((event) => Queue.offer(observedEvents, event.id)), + ), dispatch, }), Layer.mock(TerminalManager.TerminalManager)({ @@ -723,6 +736,7 @@ const makeHarness = Effect.fn("makeThreadSettlementHarness")(function* (options: return { activation, + settings, snapshots, snapshotReadCount, settingsReadCount, @@ -735,6 +749,7 @@ const makeHarness = Effect.fn("makeThreadSettlementHarness")(function* (options: invalidatedCwds, closedIdle, scriptRuns, + observedEvents, publishEvent: (event: OrchestrationV2DomainEvent) => PubSub.publish(domainEvents, event), updateSettings, publishMerge: PubSub.publish(mergedPullRequests, { @@ -761,6 +776,48 @@ const startHarness = Effect.fn("startThreadSettlementHarness")(function* ( yield* reactor.drain; }); +const projectionStoreSql = ProjectionStore.layer.pipe( + Layer.provideMerge(SqlitePersistence.layerMemory), +); + +const appThread = ( + id: string, + settledOverride: OrchestrationV2AppThread["settledOverride"], + worktreePath: string | null = null, +): OrchestrationV2AppThread => { + const threadId = ThreadId.make(id); + return { + createdBy: "user", + creationSource: "web", + id: threadId, + projectId: PROJECT_ID, + title: id, + providerInstanceId: ProviderInstanceId.make("codex"), + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5" }, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath, + activeProviderThreadId: null, + lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId }, + forkedFrom: null, + createdAt: DateTime.makeUnsafe(NOW), + updatedAt: DateTime.makeUnsafe(NOW), + archivedAt: null, + settledOverride, + settledAt: settledOverride === "settled" ? DateTime.makeUnsafe(NOW) : null, + lastVisitedAt: null, + deletedAt: null, + }; +}; +const settledEvent = (thread: OrchestrationV2AppThread): OrchestrationV2DomainEvent => ({ + type: "thread.settled", + id: EventId.make(`event:settled:${thread.id}`), + threadId: thread.id, + occurredAt: DateTime.makeUnsafe(NOW), + payload: thread, +}); + describe("ThreadSettlementServiceV2 worker", () => { it.effect("skips settings and PR decisions for recent branchless threads", () => Effect.scoped( @@ -912,6 +969,345 @@ describe("ThreadSettlementServiceV2 worker", () => { ), ); + it.effect( + "drains idle subagents of a settled parent and retries busy children without PR lookups", + () => + Effect.scoped( + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse(NOW)); + const store = yield* ProjectionStore.ProjectionStoreV2; + const parent = appThread("subagent-parent", "settled"); + const child = (id: string, overrides: Partial = {}) => ({ + ...appThread(id, null), + branch: "feature", + lineage: { + rootThreadId: parent.id, + parentThreadId: parent.id, + relationshipToParent: "subagent" as const, + }, + ...overrides, + }); + const empty = child("empty"); + const recent = child("recent"); + const busy = child("busy"); + const marker = appThread("marker", "settled"); + for (const thread of [ + parent, + empty, + recent, + busy, + marker, + child("pinned", { pinnedAt: DateTime.makeUnsafe(NOW) }), + child("opted-out", { autoSettleDisabledAt: DateTime.makeUnsafe(NOW) }), + ]) { + yield* store.apply({ + type: "thread.created", + id: EventId.make(`created:${thread.id}`), + threadId: thread.id, + occurredAt: thread.createdAt, + payload: thread, + }); + } + const run = ( + threadId: ThreadId, + status: "running" | "completed", + ): OrchestrationV2Run => ({ + id: RunId.make(`run:${threadId}`), + threadId, + ordinal: 1, + providerInstanceId: parent.providerInstanceId, + modelSelection: parent.modelSelection, + providerThreadId: null, + userMessageId: MessageId.make(`message:${threadId}`), + rootNodeId: null, + activeAttemptId: null, + status, + requestedAt: DateTime.makeUnsafe(NOW), + startedAt: DateTime.makeUnsafe(NOW), + completedAt: status === "completed" ? DateTime.makeUnsafe(NOW) : null, + checkpointId: null, + contextHandoffId: null, + }); + for (const payload of [run(recent.id, "completed"), run(busy.id, "running")]) { + yield* store.apply({ + type: "run.created", + id: EventId.make(`created:${payload.id}`), + threadId: payload.threadId, + occurredAt: DateTime.makeUnsafe(NOW), + payload, + }); + } + const fixture = yield* makeHarness({ + snapshot: makeSnapshot([]), + projections: store, + settings: { ...DEFAULT_SERVER_SETTINGS, sidebarAutoSettleAfterDays: 2 }, + onDispatch: (command) => + store.getThread(command.threadId).pipe( + Effect.flatMap((thread) => + store.apply( + settledEvent({ + ...thread, + settledOverride: "settled", + settledAt: DateTime.makeUnsafe(NOW), + }), + ), + ), + Effect.orDie, + ), + }); + yield* Effect.gen(function* () { + const service = yield* ThreadSettlementService.ThreadSettlementServiceV2; + yield* startHarness(service, fixture.activation, fixture.snapshotReads); + const commands = yield* Ref.get(fixture.commands); + expect(commands.map((command) => command.threadId).toSorted()).toEqual([ + empty.id, + recent.id, + ]); + for (const command of commands) { + expect(command.snapshotAt).toEqual(DateTime.makeUnsafe(NOW)); + expect(command.settledAt).toBeUndefined(); + } + const finished: OrchestrationV2DomainEvent = { + type: "run.updated", + id: EventId.make("busy:completed"), + threadId: busy.id, + occurredAt: DateTime.makeUnsafe(NOW), + payload: run(busy.id, "completed"), + }; + yield* store.apply(finished); + yield* fixture.publishEvent(finished); + // Cleanup of the following event marks delivery of the run event before draining. + yield* fixture.publishEvent(settledEvent(marker)); + expect(yield* Queue.take(fixture.closedIdle)).toEqual({ threadId: marker.id }); + yield* service.drain; + expect((yield* Ref.get(fixture.commands)).map((command) => command.threadId)).toEqual([ + empty.id, + recent.id, + busy.id, + ]); + expect(yield* Ref.get(fixture.branchCalls)).toEqual([]); + expect(yield* Ref.get(fixture.summaryCalls)).toEqual([]); + }).pipe(Effect.provide(fixture.layer)); + }).pipe(Effect.provide(projectionStoreSql)), + ), + ); + + it.effect( + "manual parent settlement drains hidden children with automatic settlement disabled", + () => + Effect.scoped( + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse(NOW)); + const store = yield* ProjectionStore.ProjectionStoreV2; + const parent = appThread("manual-parent", null); + const child = { + ...appThread("manual-child", null), + branch: "feature", + lineage: { + rootThreadId: parent.id, + parentThreadId: parent.id, + relationshipToParent: "subagent" as const, + }, + }; + const marker = appThread("marker", "settled"); + for (const thread of [parent, child, marker]) { + yield* store.apply({ + type: "thread.created", + id: EventId.make(`created:${thread.id}`), + threadId: thread.id, + occurredAt: thread.createdAt, + payload: thread, + }); + } + const fixture = yield* makeHarness({ + snapshot: makeSnapshot([]), + projections: store, + settings: { ...DEFAULT_SERVER_SETTINGS, sidebarAutoSettleAfterDays: 2 }, + onDispatch: (command) => + store.getThread(command.threadId).pipe( + Effect.flatMap((thread) => + store.apply( + settledEvent({ + ...thread, + settledOverride: "settled", + settledAt: DateTime.makeUnsafe(NOW), + }), + ), + ), + Effect.orDie, + ), + }); + yield* Effect.gen(function* () { + const service = yield* ThreadSettlementService.ThreadSettlementServiceV2; + yield* startHarness(service, fixture.activation, fixture.snapshotReads); + expect(yield* Ref.get(fixture.commands)).toEqual([]); + expect(yield* Ref.get(fixture.branchCalls)).toEqual([]); + // Avoid a settings-change sweep: the parent event must trigger the child dispatch. + yield* Ref.set(fixture.settings, { + ...DEFAULT_SERVER_SETTINGS, + sidebarAutoSettleAfterDays: null, + sidebarAutoSettleOnMerge: false, + }); + const event = settledEvent({ + ...parent, + settledOverride: "settled", + settledAt: DateTime.makeUnsafe(NOW), + }); + yield* store.apply(event); + yield* fixture.publishEvent(event); + yield* fixture.publishEvent(settledEvent(marker)); + expect(yield* Queue.take(fixture.closedIdle)).toEqual({ threadId: parent.id }); + expect(yield* Queue.take(fixture.closedIdle)).toEqual({ threadId: marker.id }); + yield* service.drain; + expect(yield* Ref.get(fixture.commands)).toEqual([ + expect.objectContaining({ + type: "thread.auto-settle", + threadId: child.id, + snapshotAt: child.updatedAt, + }), + ]); + expect(yield* Ref.get(fixture.branchCalls)).toEqual([]); + expect(yield* Ref.get(fixture.summaryCalls)).toEqual([]); + }).pipe(Effect.provide(fixture.layer)); + }).pipe(Effect.provide(projectionStoreSql)), + ), + ); + + it.effect( + "coalesces bulk cleanup wakes and still drains children discovered during a sweep", + () => + Effect.scoped( + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse(NOW)); + const store = yield* ProjectionStore.ProjectionStoreV2; + const sweepStarted = yield* Deferred.make(); + const releaseSweep = yield* Deferred.make(); + const childSettled = yield* Deferred.make(); + const parent = appThread("bulk-parent", null); + const child = { + ...appThread("bulk-child", null), + lineage: { + parentThreadId: parent.id, + rootThreadId: parent.id, + relationshipToParent: "subagent" as const, + }, + }; + const marker = appThread("bulk-marker", "settled"); + const parents = [ + parent, + ...Array.from({ length: 233 }, (_, i) => appThread(`bulk-parent-${i}`, null)), + ]; + for (const thread of [...parents, child, marker]) { + yield* store.apply({ + type: "thread.created", + id: EventId.make(`created:${thread.id}`), + threadId: thread.id, + occurredAt: thread.createdAt, + payload: thread, + }); + } + let blockNextSweep = false; + const fixture = yield* makeHarness({ + snapshot: makeSnapshot([]), + settings: { + ...DEFAULT_SERVER_SETTINGS, + sidebarAutoSettleAfterDays: null, + sidebarAutoSettleOnMerge: false, + }, + projections: { + ...store, + getSettlementCandidates: (threadId) => + Effect.gen(function* () { + const candidates = yield* store.getSettlementCandidates(threadId); + if (blockNextSweep) { + blockNextSweep = false; + yield* Deferred.succeed(sweepStarted, undefined); + yield* Deferred.await(releaseSweep); + } + return candidates; + }), + }, + onDispatch: (command): Effect.Effect => + Effect.gen(function* () { + const thread = yield* store.getThread(command.threadId); + const event = settledEvent({ + ...thread, + settledOverride: "settled", + settledAt: DateTime.makeUnsafe(NOW), + }); + yield* store.apply(event); + yield* fixture.publishEvent(event); + yield* Deferred.succeed(childSettled, undefined); + }).pipe(Effect.orDie), + }); + yield* Effect.gen(function* () { + const service = yield* ThreadSettlementService.ThreadSettlementServiceV2; + yield* startHarness(service, fixture.activation, fixture.snapshotReads); + yield* Ref.set(fixture.candidateReads, []); + blockNextSweep = true; + yield* fixture.publishEvent(settledEvent(marker)); + yield* Deferred.await(sweepStarted); + + for (const [i, thread] of parents.entries()) { + const type = (["thread.settled", "thread.archived", "thread.deleted"] as const)[ + i % 3 + ]!; + const event: OrchestrationV2DomainEvent = { + type, + id: EventId.make(`bulk:${thread.id}`), + threadId: thread.id, + occurredAt: DateTime.makeUnsafe(NOW), + payload: { + ...thread, + ...(type === "thread.settled" + ? { settledOverride: "settled" as const, settledAt: DateTime.makeUnsafe(NOW) } + : type === "thread.archived" + ? { archivedAt: DateTime.makeUnsafe(NOW) } + : { deletedAt: DateTime.makeUnsafe(NOW) }), + }, + }; + yield* store.apply(event); + yield* fixture.publishEvent(event); + } + const markDelivery = (id: string) => + fixture + .publishEvent({ + type: "thread.created", + id: EventId.make(id), + threadId: marker.id, + occurredAt: marker.createdAt, + payload: marker, + }) + .pipe( + // The next event is observed only after the previous handler has finished enqueueing. + Effect.andThen( + Queue.take(fixture.observedEvents).pipe( + Effect.repeat({ until: (seen) => seen === id }), + ), + ), + ); + yield* markDelivery("bulk-delivered"); + yield* Deferred.succeed(releaseSweep, undefined); + yield* Deferred.await(childSettled); + yield* markDelivery("child-delivered"); + yield* service.drain; + + expect(yield* Ref.get(fixture.candidateReads)).toEqual([ + undefined, + undefined, + undefined, + ]); + expect((yield* Ref.get(fixture.commands)).map((command) => command.threadId)).toEqual([ + child.id, + ]); + expect((yield* store.getThread(child.id)).settledOverride).toBe("settled"); + expect(yield* Ref.get(fixture.branchCalls)).toEqual([]); + expect(yield* Ref.get(fixture.summaryCalls)).toEqual([]); + }).pipe(Effect.provide(fixture.layer)); + }).pipe(Effect.provide(projectionStoreSql)), + ), + ); + it.effect("looks up a branch in its worktree only while the worktree has .git", () => Effect.scoped( Effect.gen(function* () { @@ -1419,44 +1815,6 @@ describe("ThreadSettlementServiceV2 worker", () => { }); describe("ThreadSettlementServiceV2 terminals", () => { - const appThread = ( - id: string, - settledOverride: OrchestrationV2AppThread["settledOverride"], - worktreePath: string | null = null, - ): OrchestrationV2AppThread => { - const threadId = ThreadId.make(id); - return { - createdBy: "user", - creationSource: "web", - id: threadId, - projectId: PROJECT_ID, - title: id, - providerInstanceId: ProviderInstanceId.make("codex"), - modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5" }, - runtimeMode: "full-access", - interactionMode: "default", - branch: null, - worktreePath, - activeProviderThreadId: null, - lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId }, - forkedFrom: null, - createdAt: DateTime.makeUnsafe(NOW), - updatedAt: DateTime.makeUnsafe(NOW), - archivedAt: null, - settledOverride, - settledAt: settledOverride === "settled" ? DateTime.makeUnsafe(NOW) : null, - lastVisitedAt: null, - deletedAt: null, - }; - }; - const settledEvent = (thread: OrchestrationV2AppThread): OrchestrationV2DomainEvent => ({ - type: "thread.settled", - id: EventId.make(`event:settled:${thread.id}`), - threadId: thread.id, - occurredAt: DateTime.makeUnsafe(NOW), - payload: thread, - }); - it.effect("keeps the terminals of a thread re-engaged before the event ran", () => Effect.scoped( Effect.gen(function* () { @@ -1533,6 +1891,41 @@ describe("ThreadSettlementServiceV2 terminals", () => { ), ); + it.effect("skips the settle script for a subagent in its parent's worktree", () => + Effect.scoped( + Effect.gen(function* () { + const parent = appThread("subagent-parent", "settled", "/worktrees/parent"); + const subagent = (id: string, worktreePath: string) => ({ + ...appThread(id, "settled", worktreePath), + lineage: { + parentThreadId: parent.id, + relationshipToParent: "subagent" as const, + rootThreadId: parent.id, + }, + }); + const shared = subagent("subagent-shared", "/worktrees/parent"); + const own = subagent("subagent-own", "/worktrees/own"); + const fixture = yield* makeHarness({ + snapshot: makeSnapshot([]), + currentThreads: [parent, shared, own], + existingWorktreePaths: ["/worktrees/parent", "/worktrees/own"], + }); + + yield* Effect.gen(function* () { + const service = yield* ThreadSettlementService.ThreadSettlementServiceV2; + yield* startHarness(service, fixture.activation, fixture.snapshotReads); + // Events run in order, so the first run belongs to the second event + // only if the shared-worktree child was skipped. Both still close. + yield* fixture.publishEvent(settledEvent(shared)); + yield* fixture.publishEvent(settledEvent(own)); + assert.deepStrictEqual(yield* Queue.take(fixture.closedIdle), { threadId: shared.id }); + assert.deepStrictEqual(yield* Queue.take(fixture.closedIdle), { threadId: own.id }); + assert.strictEqual((yield* Queue.take(fixture.scriptRuns)).threadId, own.id); + }).pipe(Effect.provide(fixture.layer)); + }), + ), + ); + it.effect("runs the settle script once when a settlement is re-emitted", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/ThreadSettlementService.ts b/apps/server/src/orchestration-v2/ThreadSettlementService.ts index 8c852426da32..05a13cd8a399 100644 --- a/apps/server/src/orchestration-v2/ThreadSettlementService.ts +++ b/apps/server/src/orchestration-v2/ThreadSettlementService.ts @@ -320,20 +320,34 @@ export const make = Effect.gen(function* () { threadId?: ThreadId, ) { const settings = yield* settingsService.getSettings; - if (!autoSettlementConfigured(settings)) { - return; - } + const automaticSettlementEnabled = autoSettlementConfigured(settings); const threads = yield* projections.getSettlementCandidates(threadId); if (threads.length === 0) return; const projectShells = yield* projectStore.listShells(); const nowMs = DateTime.toEpochMillis(yield* DateTime.now); const projects = new Map(projectShells.map((project) => [project.id, project])); // Merge events resweep all candidates because unlinked threads settle through branch lookup. - const candidates = threads.filter((thread) => isAutoSettlementCandidate(thread, nowMs)); + const candidates = threads.filter( + (thread) => + (automaticSettlementEnabled || thread.lineage.relationshipToParent === "subagent") && + isAutoSettlementCandidate(thread, nowMs), + ); // Runs per candidate thread, thousands per sweep on busy servers; the sweep owns the span. const settleThread = Effect.fnUntraced( function* (thread: (typeof candidates)[number], pullRequest: SettlementPullRequest | null) { + if (thread.lineage.relationshipToParent === "subagent") { + // The store returns a subagent only once its parent is done, so it + // settles with the parent and never runs its own lookup. + const uuid = yield* random.uuidV4(); + yield* orchestrator.dispatch({ + type: "thread.auto-settle", + commandId: CommandId.make(`server:auto-settle:${thread.id}:${uuid}`), + threadId: thread.id, + snapshotAt: thread.updatedAt, + }); + return null; + } // thread.auto-settle does not recheck settings, so read them per decision; reads are cached. const currentSettings = resolveProjectSettings( yield* settingsService.getSettings, @@ -375,6 +389,7 @@ export const make = Effect.gen(function* () { const lookupCandidates = (yield* Effect.forEach( candidates, (thread) => { + if (thread.lineage.relationshipToParent === "subagent") return settleThread(thread, null); const projectSettings = resolveProjectSettings(settings, thread.projectId).settings; // Skip no-op decisions; actual settlement still reads fresh settings before dispatch. const settledAt = resolveAutoSettlementAt({ @@ -566,9 +581,20 @@ export const make = Effect.gen(function* () { }), ), ); + let fullSweepPending = false; // A thread that just stopped must not wait behind a full sweep or another thread's host // reads: clients hold it in Working only briefly, so a late settle makes it appear and then vanish. - const fullSweeps = yield* makeDrainableWorker(() => runSweep(null)); + const fullSweeps = yield* makeDrainableWorker(() => { + // Clear at the start so events during this sweep can request one follow-up. + fullSweepPending = false; + return runSweep(null); + }); + const enqueueFullSweep = () => + Effect.suspend(() => { + if (fullSweepPending) return Effect.void; + fullSweepPending = true; + return fullSweeps.enqueue(undefined); + }); const threadSweeps = yield* FiberSet.make(); const threadSweepSlots = yield* Semaphore.make(8); // One sweep per thread at a time; events during a running sweep coalesce into one more pass. @@ -609,6 +635,19 @@ export const make = Effect.gen(function* () { if (thread.settledOverride !== "settled") return; const settledAtMs = toMillis(thread.settledAt); if (settledAtMs === null || settleActionRunAt.get(threadId) === settledAtMs) return; + // The parent's worktree belongs to the parent; shared-worktree children + // skip duplicate cleanup there. + if ( + thread.lineage.relationshipToParent === "subagent" && + thread.lineage.parentThreadId !== null + ) { + const parent = yield* projections.getThread(thread.lineage.parentThreadId).pipe( + Effect.catchTags({ + ProjectionStoreThreadNotFoundError: () => Effect.succeed(null), + }), + ); + if (parent?.worktreePath === worktreePath) return; + } const run = yield* projectScripts.runForThread({ threadId, projectId: thread.projectId, @@ -637,7 +676,10 @@ export const make = Effect.gen(function* () { const processEvent = (event: OrchestrationV2DomainEvent) => { switch (event.type) { case "thread.settled": - return cleanUpSettledThread(event.threadId); + return cleanUpSettledThread(event.threadId).pipe(Effect.andThen(enqueueFullSweep())); + case "thread.archived": + case "thread.deleted": + return enqueueFullSweep(); case "thread.pull-request-link-updated": case "thread.pull-request-synced": case "provider-session.detached": @@ -692,7 +734,7 @@ export const make = Effect.gen(function* () { let lastSettlementSettings = autoSettlementSettingsKey(initialSettings); yield* forkParked( Effect.gen(function* () { - yield* fullSweeps.enqueue(undefined); + yield* enqueueFullSweep(); yield* fullSweeps.drain; }).pipe(Effect.repeat(Schedule.spaced("1 minute")), Effect.asVoid), ); @@ -703,7 +745,7 @@ export const make = Effect.gen(function* () { return Effect.void; } lastSettlementSettings = key; - return fullSweeps.enqueue(undefined); + return enqueueFullSweep(); }), ); yield* forkParked(Stream.runForEach(mergedPullRequests, (event) => runSweep(event))); diff --git a/apps/server/src/orchestration-v2/ThreadStop.test.ts b/apps/server/src/orchestration-v2/ThreadStop.test.ts index 6b3ea08a8644..a0e6d4bae4c8 100644 --- a/apps/server/src/orchestration-v2/ThreadStop.test.ts +++ b/apps/server/src/orchestration-v2/ThreadStop.test.ts @@ -796,6 +796,98 @@ it.effect("a message the user sends after Stop releases the held queue; an agent }).pipe(Effect.provide(layerTest)), ); +it.effect( + "an agent's auto send queues behind a run that is still stopping instead of steering it", + () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const threads = yield* ThreadManagementService.ThreadManagementService; + const threadId = ThreadId.make("thread:stop-mcp-send"); + yield* createWatchingThread(threadId, 6); + yield* send(threadId, "active", "start_immediately"); + const initial = yield* orchestrator.getThreadProjection(threadId); + const run = initial.runs[0]!; + const providerSessionId = ProviderSessionId.make("session:stop-mcp-send"); + const now = yield* DateTime.now; + yield* projections.apply({ + id: EventId.make("event:stop-mcp-send:session"), + type: "provider-session.updated", + threadId, + occurredAt: now, + payload: { + id: providerSessionId, + driver: adapter.driver, + providerInstanceId: instanceId, + status: "ready", + cwd: "/repo", + model: modelSelection.model, + capabilities: CodexProviderCapabilitiesV2, + createdAt: now, + updatedAt: now, + lastError: null, + }, + }); + yield* projections.apply({ + id: EventId.make("event:stop-mcp-send:provider-thread"), + type: "provider-thread.updated", + threadId, + occurredAt: now, + payload: { ...initial.providerThreads[0]!, providerSessionId }, + }); + yield* projections.apply({ + id: EventId.make("event:stop-mcp-send:running"), + type: "run.updated", + threadId, + runId: run.id, + occurredAt: now, + payload: { ...run, status: "running", startedAt: now }, + }); + yield* projections.apply({ + id: EventId.make("event:stop-mcp-send:turn"), + type: "provider-turn.updated", + threadId, + occurredAt: now, + payload: { + id: ProviderTurnId.make("provider-turn:stop-mcp-send"), + providerThreadId: run.providerThreadId!, + nodeId: run.rootNodeId!, + runAttemptId: run.activeAttemptId!, + nativeTurnRef: null, + ordinal: 1, + status: "running", + startedAt: now, + completedAt: null, + }, + }); + yield* send(threadId, "queued before the stop", "queue_after_active"); + yield* orchestrator.dispatch({ + type: "thread.stop", + commandId: CommandId.make("stop-mcp-send"), + threadId, + }); + assert.deepEqual((yield* threadState(threadId)).runs, ["running", "queued:held"]); + + const result = yield* threads.sendToThread({ + projectId: ProjectId.make("project:thread-stop"), + commandId: CommandId.make("command:stop-mcp-send"), + threadId, + messageId: MessageId.make("message:stop-mcp-send"), + text: "Follow up after Stop", + attachments: [], + mode: "auto", + createdBy: "agent", + creationSource: "mcp", + }); + assert.equal(result.delivery, "queued"); + assert.deepEqual((yield* threadState(threadId)).runs, [ + "running", + "queued:held", + "queued:held", + ]); + }).pipe(Effect.provide(layerTest)), +); + it.effect( "thread.stop ends an idle thread's watches and accepts a thread with nothing to stop", () => diff --git a/apps/server/src/orchestration-v2/ThreadTitleRegenerationService.ts b/apps/server/src/orchestration-v2/ThreadTitleRegenerationService.ts index 6c80def39059..686d18292717 100644 --- a/apps/server/src/orchestration-v2/ThreadTitleRegenerationService.ts +++ b/apps/server/src/orchestration-v2/ThreadTitleRegenerationService.ts @@ -170,11 +170,13 @@ const make = Effect.gen(function* () { const result = yield* textGeneration.generateThreadTitle({ cwd: worktreeExists ? worktreePath : project.value.workspaceRoot, message: context.message, + projectTitle: project.value.title, attachments: context.attachments, ...(input.kind.type === "regenerate" ? { previousTitle: projection.thread.title } : {}), ...(inherited === undefined ? {} : { forkSource: inherited }), instructions: settings.threadTitleInstructions, modelSelection: settings.textGenerationModelSelection, + projectId: projection.thread.projectId, }); const generatedTitle = result.title.trim(); return generatedTitle === "New thread" || diff --git a/apps/server/src/orchestration-v2/TurnItemPositionStore.test.ts b/apps/server/src/orchestration-v2/TurnItemPositionStore.test.ts index 579c82f03ee4..ea3a70bdf22a 100644 --- a/apps/server/src/orchestration-v2/TurnItemPositionStore.test.ts +++ b/apps/server/src/orchestration-v2/TurnItemPositionStore.test.ts @@ -56,3 +56,24 @@ it.effect("reuses an item's position without attempting another allocation inser ); }).pipe(Effect.provide(layerTest)), ); + +it.effect("places a run-start item ahead of the run's items, then appends once taken", () => + Effect.gen(function* () { + const positions = yield* TurnItemPositionStore.TurnItemPositionStoreV2; + const threadId = ThreadId.make("thread:run-start"); + const allocate = (id: string, placement?: "run_start") => + positions.allocate({ + threadId, + turnItemId: TurnItemId.make(id), + runId: RunId.make("run:2"), + runOrdinal: 2, + ...(placement === undefined ? {} : { placement }), + }); + const message = yield* allocate("item:message"); + const recovery = yield* allocate("item:recovery", "run_start"); + assert.isBelow(recovery, message); + assert.isAbove(recovery, 1_999_999); + assert.equal(yield* allocate("item:retry-recovery", "run_start"), message + 1); + assert.equal(yield* allocate("item:reply"), message + 2); + }).pipe(Effect.provide(layerTest)), +); diff --git a/apps/server/src/orchestration-v2/TurnItemPositionStore.ts b/apps/server/src/orchestration-v2/TurnItemPositionStore.ts index 03636b34153c..95eb9cdc2415 100644 --- a/apps/server/src/orchestration-v2/TurnItemPositionStore.ts +++ b/apps/server/src/orchestration-v2/TurnItemPositionStore.ts @@ -22,6 +22,8 @@ export interface TurnItemPositionStoreV2Shape { readonly turnItemId: TurnItemId; readonly runId: RunId | null; readonly runOrdinal?: number; + // `run_start` takes the run's unused first slot, ahead of its items, and appends once it is taken. + readonly placement?: "append" | "run_start"; }) => Effect.Effect; readonly normalize: ( item: OrchestrationV2TurnItem, @@ -42,7 +44,13 @@ export const layer: Layer.Layer` SELECT ordinal FROM orchestration_v2_turn_item_positions @@ -68,7 +76,11 @@ export const layer: Layer.Layer ${lowerBound} THEN ${lowerBound} ELSE MAX(ordinal) + 1 END` + : sql`COALESCE(MAX(ordinal), ${lowerBound}) + 1` + } FROM orchestration_v2_turn_item_positions WHERE thread_id = ${threadId} AND ordinal >= ${lowerBound} diff --git a/apps/server/src/orchestration-v2/WireProjection.test.ts b/apps/server/src/orchestration-v2/WireProjection.test.ts index 66e79183da1b..6814bf9e2b19 100644 --- a/apps/server/src/orchestration-v2/WireProjection.test.ts +++ b/apps/server/src/orchestration-v2/WireProjection.test.ts @@ -175,6 +175,15 @@ describe("orchestration V2 wire projection", () => { summary: '"'.repeat(159) + "…", truncated: true, }); + + const boundary = projectTurnItemForWire({ + ...base, + input: { text: "a".repeat(149) + "😀" + "b".repeat(17_000) }, + }); + expect(boundary.type === "dynamic_tool" ? boundary.input : null).toEqual({ + summary: '{"text":"' + "a".repeat(149) + "…", + truncated: true, + }); }); it("truncates detail at a UTF-8 boundary without changing the source", () => { diff --git a/apps/server/src/orchestration-v2/WireProjection.ts b/apps/server/src/orchestration-v2/WireProjection.ts index ccb88bec074a..7c46799a1c6a 100644 --- a/apps/server/src/orchestration-v2/WireProjection.ts +++ b/apps/server/src/orchestration-v2/WireProjection.ts @@ -98,8 +98,10 @@ function summarizeDynamicValue(value: unknown): unknown { pendingSpace = false; if (firstLine.length > 160) break; } + const last = firstLine.charCodeAt(158); + const end = last >= 0xd800 && last <= 0xdbff ? 158 : 159; return { - summary: firstLine.length <= 160 ? firstLine : `${firstLine.slice(0, 159).trimEnd()}…`, + summary: firstLine.length <= 160 ? firstLine : `${firstLine.slice(0, end).trimEnd()}…`, truncated: true, }; } diff --git a/apps/server/src/orchestration-v2/assistantStreaming.test.ts b/apps/server/src/orchestration-v2/assistantStreaming.test.ts index d71042312bfb..0f46d467fa60 100644 --- a/apps/server/src/orchestration-v2/assistantStreaming.test.ts +++ b/apps/server/src/orchestration-v2/assistantStreaming.test.ts @@ -1,5 +1,8 @@ -import { describe, expect, it } from "vite-plus/test"; -import { splitBufferedAssistantText } from "./assistantStreaming.ts"; +import { MessageId, ProviderDriverKind, ThreadId } from "@t3tools/contracts"; +import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import * as DateTime from "effect/DateTime"; +import { assert, describe, expect, it, vi } from "vite-plus/test"; +import { makeAssistantStreamingFilter, splitBufferedAssistantText } from "./assistantStreaming.ts"; describe("V2 assistant streaming", () => { it("keeps code fences intact", () => { @@ -24,3 +27,195 @@ it("holds a streamed section heading until its content has a boundary", () => { rest: "Next", }); }); + +type Step = + // A snapshot at `at` ms, and the text it delivers at once (null when held back). + | { readonly at: number; readonly text: string; readonly shows: string | null } + | { readonly at: number; readonly final: string } + | { readonly id: string; readonly at: number; readonly text: string; readonly shows: string } + // Held snapshots due by `idle` ms, delivered whole. + | { readonly idle: number; readonly shows: ReadonlyArray } + | { readonly dueAt: number | null }; + +const createdAt = DateTime.makeUnsafe(0); +const snapshot = (id: string, text: string, streaming: boolean) => + ({ + type: "message.updated", + driver: ProviderDriverKind.make("codex"), + message: { + createdBy: "agent", + creationSource: "provider", + id: MessageId.make(id), + threadId: ThreadId.make("thread"), + runId: null, + nodeId: null, + role: "assistant", + text, + attachments: [], + streaming, + createdAt, + updatedAt: createdAt, + }, + }) satisfies ProviderAdapter.ProviderAdapterV2Event; +const shown = (event: ProviderAdapter.ProviderAdapterV2Event | null) => + event?.type === "message.updated" ? event.message.text : null; + +it.each<{ readonly name: string; readonly mode?: "turn"; readonly steps: ReadonlyArray }>([ + { + name: "an unfinished code fence", + steps: [ + { at: 0, text: "A\n\n```ts\nx()", shows: "A\n\n" }, + { dueAt: 800 }, + { idle: 799, shows: [] }, + { idle: 800, shows: ["A\n\n```ts\nx()"] }, + { dueAt: null }, + { at: 1_300, text: "A\n\n```ts\nx()\n```\n\nB", shows: "A\n\n```ts\nx()\n```\n\n" }, + ], + }, + { + name: "a list's partial last item", + steps: [ + { at: 0, text: "Steps:\n\n- one\n- tw", shows: "Steps:\n\n- one\n" }, + { idle: 800, shows: ["Steps:\n\n- one\n- tw"] }, + { at: 1_300, text: "Steps:\n\n- one\n- two\n- th", shows: "Steps:\n\n- one\n- two\n" }, + ], + }, + { + name: "an incomplete table row", + steps: [ + { at: 0, text: "Intro.\n\n| a | b |\n|---|---|\n| 1 |", shows: "Intro.\n\n" }, + { idle: 800, shows: ["Intro.\n\n| a | b |\n|---|---|\n| 1 |"] }, + { + at: 1_300, + text: "Intro.\n\n| a | b |\n|---|---|\n| 1 | 2 |\n\nAfter", + shows: "Intro.\n\n| a | b |\n|---|---|\n| 1 | 2 |\n\n", + }, + ], + }, + { + name: "more of the same paragraph after an idle delivery", + steps: [ + { at: 0, text: "A\n\nB", shows: "A\n\n" }, + { idle: 800, shows: ["A\n\nB"] }, + { at: 900, text: "A\n\nB more", shows: null }, + { idle: 1_699, shows: [] }, + { idle: 1_700, shows: ["A\n\nB more"] }, + { idle: 5_000, shows: [] }, + // Its only boundary is behind what the idle delivery showed. + { at: 5_100, text: "A\n\nB more and", shows: null }, + { at: 5_200, text: "A\n\nB more and.\n\nC", shows: "A\n\nB more and.\n\n" }, + { at: 5_300, final: "A\n\nB more and.\n\nC." }, + { idle: 10_000, shows: [] }, + ], + }, + { + name: "completion at the idle deadline", + steps: [ + { at: 0, text: "A\n\nB", shows: "A\n\n" }, + { at: 800, final: "A\n\nB." }, + { idle: 800, shows: [] }, + ], + }, + { + name: "completion right after an idle delivery at the same instant", + steps: [ + { at: 0, text: "A\n\nB", shows: "A\n\n" }, + { idle: 800, shows: ["A\n\nB"] }, + { at: 800, final: "A\n\nB." }, + { idle: 5_000, shows: [] }, + ], + }, + { + name: "several fast paragraphs inside the throttle", + steps: [ + { at: 0, text: "A\n\nB", shows: "A\n\n" }, + { at: 100, text: "A\n\nB\n\nC", shows: null }, + { at: 200, text: "A\n\nB\n\nC\n\nD", shows: null }, + { at: 300, text: "A\n\nB\n\nC\n\nD\n\nE", shows: null }, + { idle: 1_099, shows: [] }, + { idle: 1_100, shows: ["A\n\nB\n\nC\n\nD\n\nE"] }, + ], + }, + { + name: "two messages due at different times", + steps: [ + { id: "a", at: 0, text: "A\n\nx", shows: "A\n\n" }, + { id: "b", at: 300, text: "B\n\ny", shows: "B\n\n" }, + { dueAt: 800 }, + { idle: 800, shows: ["A\n\nx"] }, + { dueAt: 1_100 }, + { idle: 1_100, shows: ["B\n\ny"] }, + ], + }, + { + name: "empty text", + steps: [{ at: 0, text: "", shows: null }, { dueAt: null }], + }, + { + name: "whitespace-only text", + steps: [ + { at: 0, text: " \n", shows: null }, + { idle: 800, shows: [" \n"] }, + { idle: 5_000, shows: [] }, + ], + }, + { + // Rewrites replace delivered text: shorter at once, same length after the throttle. + name: "provider rewrites and truncation", + steps: [ + { at: 0, text: "Original long paragraph.\n\n", shows: "Original long paragraph.\n\n" }, + { at: 1_000, text: "New.\n\n", shows: "New.\n\n" }, + { at: 1_100, text: "Old.\n\n", shows: null }, + { idle: 1_900, shows: ["Old.\n\n"] }, + { at: 2_500, text: "Old", shows: null }, + { idle: 3_300, shows: ["Old"] }, + { at: 4_000, text: "", shows: null }, + { idle: 4_800, shows: [""] }, + ], + }, + { + name: "a rewrite that drops an open code fence", + steps: [ + { at: 0, text: "Intro\n\n```ts\nx()\n", shows: "Intro\n\n" }, + { at: 400, text: "Retry\n\n- a\n- b", shows: "Retry\n\n- a\n" }, + ], + }, + { + name: "turn mode", + mode: "turn", + steps: [{ at: 0, text: "A\n\nB", shows: null }, { dueAt: null }], + }, +])("paragraph streaming holds and idle-delivers $name", ({ mode, steps }) => { + const filter = makeAssistantStreamingFilter(mode ?? "paragraph", 800); + for (const step of steps) { + if ("dueAt" in step) assert.equal(filter.nextIdleFlushAt(), step.dueAt); + else if ("idle" in step) + assert.deepEqual(filter.takeIdleFlushes(step.idle).map(shown), [...step.shows]); + else if ("final" in step) + assert.equal(shown(filter.filter(snapshot("m", step.final, false), step.at)), step.final); + else + assert.equal( + shown(filter.filter(snapshot("id" in step ? step.id : "m", step.text, true), step.at)), + step.shows, + ); + } +}); + +it("scans each streamed line once while a long code fence is still open", () => { + const filter = makeAssistantStreamingFilter("paragraph", 800); + const lineCount = 400; + let text = "Intro\n\n```ts\n"; + const indexOf = vi.spyOn(String.prototype, "indexOf"); + try { + for (let line = 0; line < lineCount; line++) { + text += `const value${line} = ${line};\n`; + filter.filter(snapshot("m", text, true), line * 400); + } + const newlineSearches = indexOf.mock.calls.filter(([search]) => search === "\n").length; + expect(newlineSearches).toBeLessThan(lineCount * 4); + } finally { + indexOf.mockRestore(); + } + text += "```\n"; + assert.equal(shown(filter.filter(snapshot("m", text, true), lineCount * 400)), text); +}); diff --git a/apps/server/src/orchestration-v2/assistantStreaming.ts b/apps/server/src/orchestration-v2/assistantStreaming.ts index 62a173d7318c..6f0cee57b854 100644 --- a/apps/server/src/orchestration-v2/assistantStreaming.ts +++ b/apps/server/src/orchestration-v2/assistantStreaming.ts @@ -11,17 +11,44 @@ const SECTION_TITLE_PATTERN = /^ {0,3}(?:#{1,6}(?:[ \t]|$)|\*\*(?:[^*]|\*(?!\*)) // Unindented ATX headings end preceding paragraphs and lists without blank lines; bold lines do not. const TOP_LEVEL_HEADING_PATTERN = /^#{1,6}(?:[ \t]|$)/; -/** Emit terminated lines only, except partial list-start lookahead for tight lists. Hold section titles until content follows. */ -export function splitBufferedAssistantText(text: string): { ready: string; rest: string } { - let openFence: { marker: string; indent: number } | null = null; - let boundary = -1; - let lineStart = 0; - let titleAwaitingContent = false; +// Matches LIST_ITEM_START_PATTERN on the trimmed partial line without copying it. +const PARTIAL_LIST_ITEM_START_PATTERN = /[ \t]*(?:[-*+]|\d{1,9}[.)])[ \t][ \t\r]*[^ \t\r]/y; + +interface AssistantTextScan { + readonly text: string; + // Start of the partial line; every line before it has been scanned. + readonly lineStart: number; + readonly openFence: { readonly marker: string; readonly indent: number } | null; + readonly boundary: number; + readonly titleAwaitingContent: boolean; +} + +const EMPTY_SCAN: AssistantTextScan = { + text: "", + lineStart: 0, + openFence: null, + boundary: -1, + titleAwaitingContent: false, +}; + +function trimLineEnd(text: string, start: number, end: number): string { + while (end > start) { + const char = text.charCodeAt(end - 1); + if (char !== 32 && char !== 9 && char !== 13) break; + end--; + } + return text.slice(start, end); +} + +/** Scans the terminated lines of `text`, resuming after `previous` when `text` extends it. */ +function advanceScan(previous: AssistantTextScan, text: string): AssistantTextScan { + const resumed = text.startsWith(previous.text); + let { openFence, boundary, lineStart, titleAwaitingContent } = resumed ? previous : EMPTY_SCAN; + let searchFrom = resumed ? previous.text.length : 0; for (;;) { - const newline = text.indexOf("\n", lineStart); - const line = text - .slice(lineStart, newline === -1 ? text.length : newline) - .replace(/[ \t\r]+$/, ""); + const newline = text.indexOf("\n", searchFrom); + if (newline === -1) break; + const line = trimLineEnd(text, lineStart, newline); if ( openFence === null && lineStart > 0 && @@ -30,9 +57,6 @@ export function splitBufferedAssistantText(text: string): { ready: string; rest: ) { boundary = lineStart; } - if (newline === -1) { - break; - } const fenceMatch = MARKDOWN_FENCE_PATTERN.exec(line); if (fenceMatch) { const indent = fenceMatch[1]!.length; @@ -61,51 +85,158 @@ export function splitBufferedAssistantText(text: string): { ready: string; rest: titleAwaitingContent = SECTION_TITLE_PATTERN.test(line); } lineStart = newline + 1; + searchFrom = lineStart; + } + return { text, lineStart, openFence, boundary, titleAwaitingContent }; +} + +// The partial line may still change, so its list item start is checked here, not kept in the scan. +function readyBoundary(scan: AssistantTextScan): number { + if (scan.openFence === null && scan.lineStart > 0 && !scan.titleAwaitingContent) { + PARTIAL_LIST_ITEM_START_PATTERN.lastIndex = scan.lineStart; + if (PARTIAL_LIST_ITEM_START_PATTERN.test(scan.text)) return scan.lineStart; } + return scan.boundary; +} + +/** Emit terminated lines only, except partial list-start lookahead for tight lists. Hold section titles until content follows. */ +export function splitBufferedAssistantText(text: string): { ready: string; rest: string } { + const boundary = readyBoundary(advanceScan(EMPTY_SCAN, text)); if (boundary === -1) { return { ready: "", rest: text }; } return { ready: text.slice(0, boundary), rest: text.slice(boundary) }; } -export function makeAssistantStreamingFilter(mode: ResponseStreamingMode) { - const delivered = new Map(); - return ( - event: ProviderAdapter.ProviderAdapterV2Event, - now: number, - ): ProviderAdapter.ProviderAdapterV2Event | null => { - if ( - event.type === "node.updated" && - event.node.kind === "assistant_message" && - event.node.status === "running" - ) - return null; - const message = - event.type === "message.updated" && event.message.role === "assistant" - ? event.message - : event.type === "turn_item.updated" && - (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning") - ? event.turnItem - : null; - if (!message) return event; - const key = `${event.type}:${message.id}`; - if (!message.streaming) { - delivered.delete(key); - return event; - } - if (mode === "turn") return null; - const previous = delivered.get(key); - if (previous && now - previous.at < 400) return null; - const { ready } = splitBufferedAssistantText(message.text); - if (!ready || ready === previous?.text) return null; - delivered.set(key, { text: ready, at: now }); - if (event.type === "message.updated") - return { ...event, message: { ...event.message, text: ready } }; - if ( - event.type === "turn_item.updated" && - (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning") - ) - return { ...event, turnItem: { ...event.turnItem, text: ready } }; - return event; +// Long enough to ride out normal gaps between chunks; short enough that text written before a +// long tool call or background wait does not look lost. +const ASSISTANT_IDLE_FLUSH_MS = 800; + +type StreamedMessage = { + readonly id: string; + readonly text: string; + readonly streaming: boolean; +}; + +interface StreamTrack { + readonly text: string; + readonly at: number; + // Latest snapshot whose text has not all been delivered yet. + readonly held: ProviderAdapter.ProviderAdapterV2Event | null; + readonly dueAt: number; + readonly scan: AssistantTextScan; +} + +function streamedMessage(event: ProviderAdapter.ProviderAdapterV2Event): StreamedMessage | null { + if (event.type === "message.updated" && event.message.role === "assistant") return event.message; + if ( + event.type === "turn_item.updated" && + (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning") + ) + return event.turnItem; + return null; +} + +function withText( + event: ProviderAdapter.ProviderAdapterV2Event, + text: string, +): ProviderAdapter.ProviderAdapterV2Event { + if (event.type === "message.updated") return { ...event, message: { ...event.message, text } }; + if ( + event.type === "turn_item.updated" && + (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning") + ) + return { ...event, turnItem: { ...event.turnItem, text } }; + return event; +} + +/** + * Paragraph mode delivers streamed text at Markdown boundaries, at most every 400ms. A snapshot + * held back that way is delivered whole once its message has been idle for `idleFlushMs`, so a + * provider that pauses mid-turn still shows everything it wrote. + */ +export function makeAssistantStreamingFilter( + mode: ResponseStreamingMode, + idleFlushMs = ASSISTANT_IDLE_FLUSH_MS, +) { + const tracks = new Map(); + // Earliest deadline of any held snapshot, or earlier: a newer snapshot only pushes its own + // deadline later, so the cache stays O(1) per event and is recomputed when it comes due. + let nextDueAt: number | null = null; + const track = (key: string, value: StreamTrack) => { + tracks.set(key, value); + if (value.held !== null && (nextDueAt === null || value.dueAt < nextDueAt)) + nextDueAt = value.dueAt; + }; + return { + filter( + event: ProviderAdapter.ProviderAdapterV2Event, + now: number, + ): ProviderAdapter.ProviderAdapterV2Event | null { + if ( + event.type === "node.updated" && + event.node.kind === "assistant_message" && + event.node.status === "running" + ) + return null; + const message = streamedMessage(event); + if (!message) return event; + const key = `${event.type}:${message.id}`; + if (!message.streaming) { + tracks.delete(key); + return event; + } + if (mode === "turn") return null; + const previous = tracks.get(key); + const delivered = previous?.text ?? ""; + const hold = (scan = previous?.scan ?? EMPTY_SCAN) => { + track(key, { + text: delivered, + at: previous?.at ?? Number.NEGATIVE_INFINITY, + held: message.text !== delivered ? event : null, + dueAt: now + idleFlushMs, + scan, + }); + return null; + }; + if (previous && now - previous.at < 400) return hold(); + const scan = advanceScan(previous?.scan ?? EMPTY_SCAN, message.text); + const boundary = readyBoundary(scan); + const ready = boundary === -1 ? "" : message.text.slice(0, boundary); + // An idle delivery may already show more than the next boundary; never shrink it. A + // provider rewrite or truncation is not a prefix and replaces it. + if (delivered.startsWith(ready)) return hold(scan); + track(key, { + text: ready, + at: now, + held: ready !== message.text ? event : null, + dueAt: now + idleFlushMs, + scan, + }); + return withText(event, ready); + }, + nextIdleFlushAt: (): number | null => nextDueAt, + /** Returns the held snapshots due by `now` and counts them as delivered. */ + takeIdleFlushes(now: number): ReadonlyArray { + const due: Array = []; + nextDueAt = null; + for (const [key, current] of tracks) { + const event = current.held; + if (event === null) continue; + if (current.dueAt > now) { + if (nextDueAt === null || current.dueAt < nextDueAt) nextDueAt = current.dueAt; + continue; + } + due.push(event); + tracks.set(key, { + text: streamedMessage(event)?.text ?? current.text, + at: now, + held: null, + dueAt: now, + scan: current.scan, + }); + } + return due; + }, }; } diff --git a/apps/server/src/orchestration-v2/legacy/LegacyV1ThreadImporter.test.ts b/apps/server/src/orchestration-v2/legacy/LegacyV1ThreadImporter.test.ts index a50b3ed3c191..4cea500f6e7b 100644 --- a/apps/server/src/orchestration-v2/legacy/LegacyV1ThreadImporter.test.ts +++ b/apps/server/src/orchestration-v2/legacy/LegacyV1ThreadImporter.test.ts @@ -155,12 +155,16 @@ it.effect( const importer = yield* LegacyV1ThreadImporter.LegacyV1ThreadImporter; const projections = yield* ProjectionStore.ProjectionStoreV2; yield* seedPendingTranscripts; - yield* sql` - UPDATE projection_thread_messages SET created_at = '0000-not-a-date' - WHERE message_id = 'thread:first:old' - `; - yield* importer.importPendingTranscripts; + yield* importer.importPendingTranscripts.pipe( + Effect.provideService(Statement.CurrentTransformer, (statement) => { + const [query, params] = statement.compile(); + return query.includes("FROM projection_thread_messages") && + params.includes("thread:first") + ? Effect.die("transcript read failed") + : Effect.succeed(statement); + }), + ); const later = yield* projections.getThreadProjection(ThreadId.make("thread:later")); assert.deepStrictEqual( @@ -219,6 +223,53 @@ it.effect("propagates interruption from transcript reads and retry-marker writes }).pipe(Effect.provide(layerTest)), ); +// regression: #30813 a malformed V1 timestamp stopped the server from starting +it.effect("imports legacy rows with malformed timestamps and keeps the archive marker", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const importer = yield* LegacyV1ThreadImporter.LegacyV1ThreadImporter; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const malformed = "2026-07-13T14:13:00.3NZ"; + yield* sql` + INSERT INTO projection_projects ( + project_id, title, workspace_root, scripts_json, created_at, updated_at + ) VALUES ( + 'project:malformed', 'Legacy project', '/tmp/legacy-malformed', '[]', + '2026-01-01T00:00:00.000Z', '2026-01-01T00:00:00.000Z' + ) + `; + yield* sql` + INSERT INTO projection_threads ( + thread_id, project_id, title, model_selection_json, runtime_mode, + interaction_mode, created_at, updated_at, archived_at + ) VALUES ( + 'thread:malformed', 'project:malformed', 'Legacy thread', + '{"instanceId":"codex","model":"gpt-5.4"}', 'full-access', 'default', + '2026-07-13T14:00:00.000Z', ${malformed}, ${malformed} + ) + `; + yield* sql` + INSERT INTO projection_thread_messages ( + message_id, thread_id, role, text, is_streaming, created_at, updated_at + ) VALUES ( + 'message:malformed', 'thread:malformed', 'user', 'Question', 0, + ${malformed}, '2026-07-13T14:13:00.000Z' + ) + `; + + assert.deepStrictEqual(yield* importer.reconcileShells, { + importedThreadCount: 1, + importedMessageCount: 1, + }); + const projection = yield* projections.getThreadProjection(ThreadId.make("thread:malformed")); + assert.isNotNull(projection.thread.archivedAt); + assert.deepStrictEqual( + projection.messages.map((message) => message.text), + ["Question"], + ); + }).pipe(Effect.provide(layerTest)), +); + it.effect("caches the absent legacy transcript for a native V2 thread", () => Effect.gen(function* () { const importer = yield* LegacyV1ThreadImporter.LegacyV1ThreadImporter; diff --git a/apps/server/src/orchestration-v2/legacy/LegacyV1ThreadImporter.ts b/apps/server/src/orchestration-v2/legacy/LegacyV1ThreadImporter.ts index 8c134d3b57d7..ede38bdd3968 100644 --- a/apps/server/src/orchestration-v2/legacy/LegacyV1ThreadImporter.ts +++ b/apps/server/src/orchestration-v2/legacy/LegacyV1ThreadImporter.ts @@ -186,15 +186,19 @@ function settledOverrideFor(value: string | null): OrchestrationV2AppThread["set return value === "settled" || value === "active" ? value : null; } -function dateTime(value: string): DateTime.Utc { - return DateTime.makeUnsafe(value); +// V1 rows can hold unparseable timestamps; fall back to another time rather than fail the import. +function dateTimeOr(value: string | null, fallback: DateTime.Utc): DateTime.Utc { + return value === null ? fallback : Option.getOrElse(DateTime.make(value), () => fallback); } -function nullableDateTime(value: string | null): DateTime.Utc | null { - return value === null ? null : dateTime(value); +// A malformed marker stays set so archived or deleted threads keep that state. +function nullableDateTime(value: string | null, fallback: DateTime.Utc): DateTime.Utc | null { + return value === null ? null : dateTimeOr(value, fallback); } -function importedThread(row: LegacyThreadRow): OrchestrationV2AppThread { +function importedThread(row: LegacyThreadRow, importedAt: DateTime.Utc): OrchestrationV2AppThread { + const createdAt = dateTimeOr(row.created_at, dateTimeOr(row.updated_at, importedAt)); + const updatedAt = dateTimeOr(row.updated_at, createdAt); const threadId = ThreadId.make(row.thread_id); const modelSelection = modelSelectionFor(row); const branch = row.branch?.trim() || null; @@ -234,27 +238,30 @@ function importedThread(row: LegacyThreadRow): OrchestrationV2AppThread { rootThreadId: threadId, }, forkedFrom: null, - createdAt: dateTime(row.created_at), - updatedAt: dateTime(row.updated_at), - archivedAt: nullableDateTime(row.archived_at), + createdAt, + updatedAt, + archivedAt: nullableDateTime(row.archived_at, updatedAt), settledOverride: settledOverrideFor(row.settled_override), - settledAt: nullableDateTime(row.settled_at), - unsettledAt: nullableDateTime(row.unsettled_at), - snoozedUntil: nullableDateTime(row.snoozed_until), - snoozedAt: nullableDateTime(row.snoozed_at), - pinnedAt: nullableDateTime(row.pinned_at), - autoSettleDisabledAt: nullableDateTime(row.auto_settle_disabled_at), + settledAt: nullableDateTime(row.settled_at, updatedAt), + unsettledAt: nullableDateTime(row.unsettled_at, updatedAt), + snoozedUntil: nullableDateTime(row.snoozed_until, updatedAt), + snoozedAt: nullableDateTime(row.snoozed_at, updatedAt), + pinnedAt: nullableDateTime(row.pinned_at, updatedAt), + autoSettleDisabledAt: nullableDateTime(row.auto_settle_disabled_at, updatedAt), pinOrderKey: row.pin_order_key?.trim() || null, lastVisitedAt: null, - deletedAt: nullableDateTime(row.deleted_at), + deletedAt: nullableDateTime(row.deleted_at, updatedAt), }; } -function messageEvents(row: LegacyMessageRow): ReadonlyArray { +function messageEvents( + row: LegacyMessageRow, + importedAt: DateTime.Utc, +): ReadonlyArray { const threadId = ThreadId.make(row.thread_id); const messageId = MessageId.make(row.message_id); - const createdAt = dateTime(row.created_at); - const updatedAt = dateTime(row.updated_at); + const createdAt = dateTimeOr(row.created_at, dateTimeOr(row.updated_at, importedAt)); + const updatedAt = dateTimeOr(row.updated_at, createdAt); const attachments = attachmentsFor(row); const context = contextFor(row); const message: OrchestrationV2ConversationMessage = { @@ -433,7 +440,7 @@ const make = Effect.gen(function* () { }); const reconcileShellsBase = Effect.gen(function* () { - const now = DateTime.formatIso(yield* DateTime.now); + const now = yield* DateTime.now; const repairRows = yield* sql` SELECT thread.thread_id, @@ -482,7 +489,7 @@ const make = Effect.gen(function* () { const decoded = decodeStoredThread(row.payload_json); if (Option.isNone(decoded)) continue; const current = decoded.value; - const legacy = importedThread(row); + const legacy = importedThread(row, now); const legacyPullRequests = legacy.pullRequests ?? []; const repaired: OrchestrationV2AppThread = { ...current, @@ -531,7 +538,7 @@ const make = Effect.gen(function* () { type: "thread.metadata-updated", threadId: repaired.id, providerInstanceId: repaired.providerInstanceId, - occurredAt: dateTime(now), + occurredAt: now, payload: repaired, }, ], @@ -578,7 +585,7 @@ const make = Effect.gen(function* () { let importedThreadCount = repairedThreadCount; let importedMessageCount = 0; for (const row of rows) { - const thread = importedThread(row); + const thread = importedThread(row, now); const previews = yield* listShellMessages(thread.id); const events: Array = [ { @@ -589,7 +596,7 @@ const make = Effect.gen(function* () { occurredAt: thread.createdAt, payload: thread, }, - ...previews.flatMap(messageEvents), + ...previews.flatMap((message) => messageEvents(message, now)), { id: EventId.make(`${IMPORT_EVENT_PREFIX}:thread:${row.thread_id}:shell`), type: "thread.metadata-updated", @@ -632,7 +639,7 @@ const make = Effect.gen(function* () { VALUES ( ${thread.id}, ${row.updated_at}, - ${now}, + ${DateTime.formatIso(now)}, NULL, ${previews.length}, NULL @@ -695,6 +702,7 @@ const make = Effect.gen(function* () { confirmedTranscriptThreadIds.add(threadId); return { importedThreadCount: 0, importedMessageCount: 0 }; } + const importedAt = yield* DateTime.now; const messages = yield* listMessages(threadId); // Hint the index to skip V1 events without scanning their payloads. const existingRows = yield* sql<{ readonly event_id: string }>` @@ -728,7 +736,9 @@ const make = Effect.gen(function* () { `, { discard: true }, ); - yield* eventSink.write({ events: batch.flatMap(messageEvents) }); + yield* eventSink.write({ + events: batch.flatMap((message) => messageEvents(message, importedAt)), + }); yield* Effect.yieldNow; } const now = DateTime.formatIso(yield* DateTime.now); diff --git a/apps/server/src/orchestration-v2/messageSearchIndex.ts b/apps/server/src/orchestration-v2/messageSearchIndex.ts index dbd3545e5906..87d7256b6809 100644 --- a/apps/server/src/orchestration-v2/messageSearchIndex.ts +++ b/apps/server/src/orchestration-v2/messageSearchIndex.ts @@ -1,3 +1,4 @@ +import { assistantCitationsToPlainText } from "@t3tools/shared/assistantCitations"; import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences"; import * as Effect from "effect/Effect"; import type * as SqlClient from "effect/sql/SqlClient"; @@ -40,46 +41,83 @@ export const indexedMessageCondition = (sql: SqlClient.SqlClient, row = "message AND ${nonPlaceholderMessageCondition(sql, row)} `; +// Curly apostrophes straight, as search folds a query, since iOS keyboards +// type "’" and others type "'". +const foldApostrophes = (text: string) => `replace(replace(${text}, '’', ''''), '‘', '''')`; + +const readableText = (row: string) => + `COALESCE(${row}.search_text, json_extract(${row}.payload_json, '$.text'))`; + +/** + * What search_text holds for a message: its text as it reads, a quote's + * percent-encoded link replaced by its words and comment. NULL when that is + * the stored text, so most messages store nothing extra. + */ +export const readableSearchText = (text: string) => { + const readable = assistantCitationsToPlainText(text); + return readable === text ? null : readable; +}; + +/** A message's text as it reads, for thread search. */ +export const messageSearchText = (sql: SqlClient.SqlClient) => + sql.literal(readableText("messages")); + /** - * What the index stores for a message: its text with curly apostrophes - * straight, as search folds a query, since iOS keyboards type "’" and others - * type "'". Migration 69's triggers use it too. `payload` is the message's + * A message's stored text with apostrophes folded, what the index held before + * migration 81. Migration 69's backfill uses it. `payload` is the message's * JSON column. */ export const indexedMessageText = (sql: SqlClient.SqlClient, payload: string) => - sql.literal(`replace(replace(json_extract(${payload}, '$.text'), '’', ''''), '‘', '''')`); + sql.literal(foldApostrophes(`json_extract(${payload}, '$.text')`)); + +// What the index stores: a message's readable text with apostrophes folded. +// Migrations 69 and 74 create the triggers before migration 81 adds +// search_text, and `t3 doctor` can rebuild a database that has not migrated, +// so those fall back to the stored text. +const indexedText = Effect.fn("MessageSearchIndex.indexedText")(function* ( + sql: SqlClient.SqlClient, +) { + const column = yield* sql` + SELECT 1 FROM pragma_table_info('orchestration_v2_projection_messages') + WHERE name = 'search_text' + `; + return column.length > 0 + ? (row: string) => sql.literal(foldApostrophes(readableText(row))) + : (row: string) => indexedMessageText(sql, `${row}.payload_json`); +}); export const createMessageSearchTriggers = Effect.fn("MessageSearchIndex.createTriggers")( function* (sql: SqlClient.SqlClient) { + const text = yield* indexedText(sql); yield* sql` CREATE TRIGGER orchestration_v2_message_search_insert AFTER INSERT ON orchestration_v2_projection_messages WHEN ${indexedMessageCondition(sql, "new")} BEGIN INSERT INTO orchestration_v2_message_search (rowid, text) - SELECT new.rowid, ${indexedMessageText(sql, "new.payload_json")} + SELECT new.rowid, ${text("new")} WHERE json_extract(new.payload_json, '$.text') IS NOT NULL; END `; // Streaming deltas rewrite the row many times a second; skip them until the - // message settles. Upserts rewrite every column, so a finished message - // re-indexes only when what the index reads changed. Deleting a rowid the - // index never held is a no-op. + // message settles. Upserts rewrite every column, so re-index only when what + // the index stores changed; migration 81's backfill writes only search_text. + // Deleting a rowid the index never held is a no-op. yield* sql` CREATE TRIGGER orchestration_v2_message_search_update - AFTER UPDATE OF payload_json, streaming, role ON orchestration_v2_projection_messages + AFTER UPDATE ON orchestration_v2_projection_messages WHEN (old.streaming = 0 OR new.streaming = 0) AND ( old.streaming IS NOT new.streaming OR old.role IS NOT new.role - OR json_extract(old.payload_json, '$.text') IS NOT json_extract(new.payload_json, '$.text') + OR ${text("old")} IS NOT ${text("new")} OR json_extract(old.payload_json, '$.createdBy') IS NOT json_extract(new.payload_json, '$.createdBy') ) BEGIN DELETE FROM orchestration_v2_message_search WHERE rowid = old.rowid AND old.streaming = 0; INSERT INTO orchestration_v2_message_search (rowid, text) - SELECT new.rowid, ${indexedMessageText(sql, "new.payload_json")} + SELECT new.rowid, ${text("new")} WHERE ${indexedMessageCondition(sql, "new")} AND json_extract(new.payload_json, '$.text') IS NOT NULL; END @@ -95,6 +133,20 @@ export const createMessageSearchTriggers = Effect.fn("MessageSearchIndex.createT }, ); +/** Replaces the search triggers with their current definition, where the index exists. */ +export const recreateMessageSearchTriggers = Effect.fn("MessageSearchIndex.recreateTriggers")( + function* (sql: SqlClient.SqlClient) { + const index = yield* sql` + SELECT 1 FROM sqlite_master WHERE name = 'orchestration_v2_message_search' + `; + if (index.length === 0) return; + yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`; + yield* sql`DROP TRIGGER orchestration_v2_message_search_update`; + yield* sql`DROP TRIGGER orchestration_v2_message_search_delete`; + yield* createMessageSearchTriggers(sql); + }, +); + /** * Refills the search index from the messages table. The index is keyed by * message rowid, so anything that may renumber or copy rows (VACUUM, a dev @@ -108,6 +160,7 @@ export const rebuildMessageSearch = Effect.fn("MessageSearchIndex.rebuild")(func SELECT 1 FROM sqlite_master WHERE name = 'orchestration_v2_message_search' `; if (index.length === 0) return; + const text = yield* indexedText(sql); // One transaction: an interrupt between clear and refill would leave an // empty index that nothing repairs, since search only checks the table exists. yield* sql.withTransaction( @@ -118,7 +171,7 @@ export const rebuildMessageSearch = Effect.fn("MessageSearchIndex.rebuild")(func `; yield* sql` INSERT INTO orchestration_v2_message_search (rowid, text) - SELECT messages.rowid, ${indexedMessageText(sql, "messages.payload_json")} + SELECT messages.rowid, ${text("messages")} FROM orchestration_v2_projection_messages AS messages WHERE ${indexedMessageCondition(sql)} AND json_extract(messages.payload_json, '$.text') IS NOT NULL diff --git a/apps/server/src/orchestration-v2/pullRequestWatch.test.ts b/apps/server/src/orchestration-v2/pullRequestWatch.test.ts index 6ab019164949..dece3e344983 100644 --- a/apps/server/src/orchestration-v2/pullRequestWatch.test.ts +++ b/apps/server/src/orchestration-v2/pullRequestWatch.test.ts @@ -31,6 +31,8 @@ const watch = (overrides: Partial = {}): ThreadPullReque ...overrides, }); +const run = (id: number) => `https://github.com/o/r/actions/runs/${id}`; + const check = (name: string, status: PullRequestCheck["status"]): PullRequestCheck => ({ name, status, @@ -79,6 +81,136 @@ const remark = ( const noRemarks: ReadonlyArray = []; describe("evaluatePullRequestWatch", () => { + it("waits for workflow approval once, through reads that miss it, until the run finishes", () => { + const bot = check("bot", "success"); + const approval = { ...check("CI", "action-required"), url: run(1) }; + const gated = detail({ checks: [bot, approval] }); + const first = evaluatePullRequestWatch(watch(), gated, noRemarks); + assert.deepEqual(first.changes, [{ kind: "checks-awaiting-approval", checks: [approval] }]); + + // GitHub's approval read can answer empty while the rest of the checks read fine. + const missing = evaluatePullRequestWatch(first.next, detail({ checks: [bot] }), noRemarks); + assert.deepEqual(missing.changes, []); + assert.isFalse(missing.next.passed); + assert.deepEqual(evaluatePullRequestWatch(missing.next, gated, noRemarks).changes, []); + const later = { ...check("Preview", "action-required"), url: run(2) }; + assert.deepEqual( + evaluatePullRequestWatch(missing.next, detail({ checks: [bot, approval, later] }), noRemarks) + .changes, + [], + ); + const unnamed = { ...check("Docs", "action-required"), url: null }; + const named = detail({ checks: [bot, unnamed] }); + const alone = evaluatePullRequestWatch(watch(), named, noRemarks); + assert.deepEqual(alone.changes, [{ kind: "checks-awaiting-approval", checks: [unnamed] }]); + const placeholder = { ...unnamed, name: "Workflow approval status", placeholder: true }; + const unlisted = evaluatePullRequestWatch( + alone.next, + detail({ checks: [bot, placeholder] }), + noRemarks, + ); + assert.deepEqual(unlisted.changes, []); + assert.isFalse(unlisted.next.passed); + assert.deepEqual(evaluatePullRequestWatch(unlisted.next, named, noRemarks).changes, []); + const dropped = evaluatePullRequestWatch(alone.next, detail({ checks: [bot] }), noRemarks); + assert.deepEqual(dropped.changes, []); + assert.isFalse(dropped.next.passed); + assert.deepEqual(dropped.next.awaitingApproval, [unnamed.name]); + assert.deepEqual(evaluatePullRequestWatch(dropped.next, named, noRemarks).changes, []); + const namedPending = evaluatePullRequestWatch( + dropped.next, + detail({ + checks: [ + { ...bot, required: true }, + { ...unnamed, status: "pending" }, + ], + }), + noRemarks, + ); + assert.deepEqual(namedPending.changes, []); + assert.isFalse(namedPending.next.passed); + const namedDone = detail({ checks: [bot, { ...unnamed, status: "success" }] }); + const namedFinished = evaluatePullRequestWatch(namedPending.next, namedDone, noRemarks); + assert.deepEqual(namedFinished.changes, [{ kind: "checks-passed", count: 2, required: false }]); + assert.deepEqual(namedFinished.next.awaitingApproval, []); + assert.deepEqual( + evaluatePullRequestWatch(namedFinished.next, namedDone, noRemarks).changes, + [], + ); + const pushed = detail({ headSha: "bbbbbbbbbb", checks: [bot] }); + assert.deepEqual(evaluatePullRequestWatch(missing.next, pushed, noRemarks).changes, [ + { kind: "checks-passed", count: 1, required: false }, + ]); + + // Approved, the run's jobs carry their own names under the run's URL. + const job = { ...check("lint", "success"), url: `${run(1)}/job/7` }; + const pending = { ...check("test", "pending"), url: `${run(1)}/job/8` }; + const required = { ...bot, required: true }; + const running = evaluatePullRequestWatch( + missing.next, + detail({ checks: [required, job, pending] }), + noRemarks, + ); + assert.deepEqual(running.changes, []); + assert.isFalse(running.next.passed); + assert.deepEqual(running.next.awaitingApproval, [run(1)]); + const overlap = evaluatePullRequestWatch( + running.next, + detail({ checks: [required, job, { ...pending, url: `${run(2)}/job/9` }] }), + noRemarks, + ); + assert.deepEqual(overlap.changes, []); + assert.isFalse(overlap.next.passed); + assert.deepEqual(overlap.next.awaitingApproval, [run(1)]); + const unlinked = evaluatePullRequestWatch( + overlap.next, + detail({ checks: [required, job, { ...pending, url: null }] }), + noRemarks, + ); + assert.deepEqual(unlinked.changes, []); + assert.isFalse(unlinked.next.passed); + assert.deepEqual(unlinked.next.awaitingApproval, [run(1)]); + const advisory = evaluatePullRequestWatch( + unlinked.next, + detail({ checks: [required, job, { ...pending, url: "https://ci.example/test" }] }), + noRemarks, + ); + assert.deepEqual(advisory.changes, [{ kind: "checks-passed", count: 1, required: true }]); + assert.deepEqual(advisory.next.awaitingApproval, []); + const completed = detail({ checks: [required, job, { ...pending, status: "success" }] }); + const finished = evaluatePullRequestWatch(unlinked.next, completed, noRemarks); + assert.deepEqual(finished.changes, [{ kind: "checks-passed", count: 1, required: true }]); + assert.deepEqual(finished.next.awaitingApproval, []); + assert.deepEqual(evaluatePullRequestWatch(finished.next, completed, noRemarks).changes, []); + + const done = evaluatePullRequestWatch(missing.next, detail({ checks: [bot, job] }), noRemarks); + assert.deepEqual(done.changes, [{ kind: "checks-passed", count: 2, required: false }]); + }); + + it("does not pass or fail while the host cannot tell whether workflows await approval", () => { + const bot = check("bot", "success"); + const unknown = { + ...check("Workflow approval status", "action-required"), + url: null, + placeholder: true, + }; + const unread = evaluatePullRequestWatch(watch(), detail({ checks: [bot, unknown] }), noRemarks); + assert.deepEqual(unread.changes, []); + assert.isFalse(unread.next.passed); + const read = evaluatePullRequestWatch(unread.next, detail({ checks: [bot] }), noRemarks); + assert.deepEqual(read.changes, [{ kind: "checks-passed", count: 1, required: false }]); + const blocked = evaluatePullRequestWatch( + read.next, + detail({ checks: [{ ...bot, status: "pending" }, unknown] }), + noRemarks, + ); + assert.isFalse(blocked.next.passed); + assert.deepEqual( + evaluatePullRequestWatch(blocked.next, detail({ checks: [bot] }), noRemarks).changes, + [{ kind: "checks-passed", count: 1, required: false }], + ); + }); + it("reports each failure at once, even while another check never finishes", () => { const bot = check("CodeRabbit", "pending"); const first = detail({ checks: [check("lint", "failure"), check("test", "pending"), bot] }); @@ -95,6 +227,13 @@ describe("evaluatePullRequestWatch", () => { const pushed = detail({ ...second, headSha: "bbbbbbbbbb" }); assert.equal(evaluatePullRequestWatch(test.next, pushed, noRemarks).changes.length, 1); + + const approval = { ...check("CI", "action-required"), url: run(1) }; + const gated = detail({ checks: [approval, check("lint", "failure")] }); + assert.deepEqual(evaluatePullRequestWatch(watch(), gated, noRemarks).changes, [ + { kind: "checks-failed", failed: [check("lint", "failure")] }, + { kind: "checks-awaiting-approval", checks: [approval] }, + ]); }); it("reports passed once the required checks pass, whatever the others do", () => { @@ -429,6 +568,15 @@ describe("evaluatePullRequestWatch", () => { assert.isTrue(anHourLater(legacy, detail({ checks: [] })).every((kinds) => kinds.length === 0)); const pushed = anHourLater(legacy, detail({ headSha: "bbbbbbbbbb", checks: [] })); assert.deepEqual(pushed.flat(), ["no-checks"]); + const unknown = detail({ + checks: [{ ...check("Workflow approval status", "action-required"), placeholder: true }], + }); + assert.deepEqual(anHourLater(watch(), unknown).flat(), []); + const unread = evaluate(watch(), unknown, noRemarks, 0); + assert.deepEqual( + evaluate(unread.next, detail({ checks: [] }), noRemarks, 60 * 60_000).changes, + [{ kind: "no-checks", minutes: 60 }], + ); }); it("never reports missing checks on a host where an empty list says nothing", () => { diff --git a/apps/server/src/orchestration-v2/pullRequestWatch.ts b/apps/server/src/orchestration-v2/pullRequestWatch.ts index 62f4daeb7831..4421d62d2958 100644 --- a/apps/server/src/orchestration-v2/pullRequestWatch.ts +++ b/apps/server/src/orchestration-v2/pullRequestWatch.ts @@ -17,6 +17,7 @@ const SNIPPET_LENGTH = 200; export type PullRequestWatchChange = | { readonly kind: "checks-failed"; readonly failed: ReadonlyArray } | { readonly kind: "checks-passed"; readonly count: number; readonly required: boolean } + | { readonly kind: "checks-awaiting-approval"; readonly checks: ReadonlyArray } | { readonly kind: "remarks"; readonly remarks: ReadonlyArray } | { readonly kind: "no-checks"; readonly minutes: number } | { readonly kind: "conflicting" }; @@ -27,9 +28,12 @@ export interface PullRequestWatchReport { readonly exhausted: boolean; } -// action-required is a finished check needing human input. const isFailedCheck = (check: PullRequestCheck) => - check.status === "failure" || check.status === "cancelled" || check.status === "action-required"; + check.status === "failure" || check.status === "cancelled"; + +// A GitHub Actions job's URL extends its workflow run's URL; their names need not match. +const workflowRunOf = (url: string | null) => url?.match(/^.*\/actions\/runs\/\d+/)?.[0] ?? null; +const approvalKey = (check: PullRequestCheck) => workflowRunOf(check.url) ?? check.name; export function evaluatePullRequestWatch( watch: ThreadPullRequestWatch, @@ -50,15 +54,59 @@ export function evaluatePullRequestWatch( let passed = headMoved ? false : watch.passed; let passedChecks = headMoved ? [] : watch.passedChecks; let owedChecks = headMoved ? [] : (watch.owedChecks ?? []); - if (detail.checks.length > 0) { - const failed = detail.checks.filter(isFailedCheck); + let awaitingApproval = headMoved ? [] : (watch.awaitingApproval ?? []); + let approvalAnnounced = headMoved ? false : (watch.approvalAnnounced ?? false); + // An unavailable approval read cannot establish success or that no checks exist. + const unread = detail.checks.some((check) => check.placeholder === true); + const checks = detail.checks.filter((check) => check.placeholder !== true); + const failed = checks.filter(isFailedCheck); + const awaiting = checks.filter((check) => check.status === "action-required"); + if (checks.length > 0) { const newlyFailed = failed.filter((check) => !failedChecks.includes(check.name)); if (newlyFailed.length > 0) changes.push({ kind: "checks-failed", failed: newlyFailed }); + if (!approvalAnnounced && awaiting.length > 0) { + changes.push({ kind: "checks-awaiting-approval", checks: awaiting }); + approvalAnnounced = true; + } failedChecks = failed.map((check) => check.name); - const required = detail.checks.filter((check) => check.required === true); - const gate = required.length > 0 ? required : detail.checks; - const passedNow = gate.every((check) => check.status !== "pending" && !isFailedCheck(check)); + // Approval reads can omit gated runs. Keep each hold until its jobs finish. + const awaitingNow = [...new Set(awaiting.map(approvalKey))]; + const nameUnfinished = (name: string) => { + const matching = checks.filter((check) => check.name === name); + return ( + matching.length === 0 || + matching.some((check) => check.status === "pending" || check.status === "action-required") + ); + }; + // GitHub deduplicates jobs across runs, so a visible pending job may hide one in an awaited run. + // A pending check without a URL could be such a job; one with another host's URL cannot. + const jobsPending = checks.some( + (check) => + check.status === "pending" && (check.url === null || workflowRunOf(check.url) !== null), + ); + const finished = new Set( + jobsPending + ? [] + : checks + .filter((check) => check.status !== "action-required" && check.status !== "pending") + .map((check) => workflowRunOf(check.url)), + ); + awaitingApproval = [ + ...awaitingNow, + ...awaitingApproval.filter( + (key) => + !awaitingNow.includes(key) && + (workflowRunOf(key) === key ? !finished.has(key) : nameUnfinished(key)), + ), + ]; + } + const required = checks.filter((check) => check.required === true); + const gate = required.length > 0 ? required : checks; + const passedNow = + awaitingApproval.length === 0 && + gate.every((check) => check.status !== "pending" && !isFailedCheck(check)); + if (checks.length > 0 && !(unread && passedNow)) { const gateNames = gate.map((check) => check.name); const told = passed && passedChecks.length === 0 ? gateNames : passedChecks; // Required checks can finish between sweeps; detect new names. With no required checks, ignore advisory gate growth. @@ -67,23 +115,25 @@ export function evaluatePullRequestWatch( // Only those are owed, so later advisory churn never wakes. One turned required is gate news. const owed = passedNow && passed - ? detail.checks + ? checks .filter((check) => check.required !== true && owedChecks.includes(check.name)) .map((check) => check.name) : []; const settled = - failed.length === 0 && detail.checks.every((check) => check.status !== "pending"); + failed.length === 0 && + awaitingApproval.length === 0 && + checks.every((check) => check.status !== "pending"); owedChecks = owed; if (passedNow && (!passed || gateGrew)) { changes.push({ kind: "checks-passed", count: gate.length, required: required.length > 0 }); owedChecks = required.length > 0 - ? detail.checks + ? checks .filter((check) => check.required !== true && check.status === "pending") .map((check) => check.name) : []; } else if (settled && owed.length > 0) { - changes.push({ kind: "checks-passed", count: detail.checks.length, required: false }); + changes.push({ kind: "checks-passed", count: checks.length, required: false }); owedChecks = []; } passed = passedNow; @@ -99,9 +149,9 @@ export function evaluatePullRequestWatch( const awaitedBefore = headMoved ? DateTime.formatIso(DateTime.makeUnsafe(now)) : (watch.checksAwaitedSince ?? null); - checksAwaitedSince = detail.checks.length > 0 ? null : awaitedBefore; + checksAwaitedSince = checks.length > 0 ? null : awaitedBefore; const awaitedMs = checksAwaitedSince === null ? 0 : now - Date.parse(checksAwaitedSince); - if (awaitedMs >= PULL_REQUEST_WATCH_NO_CHECKS_MS) { + if (!unread && awaitedMs >= PULL_REQUEST_WATCH_NO_CHECKS_MS) { changes.push({ kind: "no-checks", minutes: Math.round(awaitedMs / 60_000) }); checksAwaitedSince = null; } @@ -148,6 +198,8 @@ export function evaluatePullRequestWatch( passed, passedChecks, owedChecks, + awaitingApproval, + approvalAnnounced, remarksThrough, remarkIds, conflicting, @@ -190,6 +242,11 @@ function changeLines( ` - ${check.name}${check.status === "failure" ? "" : ` (${check.status})`}${check.url ? ` ${check.url}` : ""}`, ), ]; + case "checks-awaiting-approval": + return [ + `- Checks are waiting for approval or action from a maintainer${context.commit}:`, + ...listed(change.checks, (check) => ` - ${check.name}${check.url ? ` ${check.url}` : ""}`), + ]; case "checks-passed": return [ `- All ${change.count} ${change.required ? "required " : ""}${change.count === 1 ? "check" : "checks"} passed${context.commit}.`, @@ -216,6 +273,7 @@ function changeLines( const SUMMARY: Record = { "checks-failed": "checks failed", "checks-passed": "checks passed", + "checks-awaiting-approval": "checks await approval", remarks: "new comments", "no-checks": "no checks reported", conflicting: "merge conflict", diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index cde562e3e46d..c8fcb6140e5a 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -40,6 +40,7 @@ import * as Effect from "effect/Effect"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Fiber from "effect/Fiber"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; @@ -160,17 +161,35 @@ const alternateProviderInstance = { instanceId: alternateInstanceId, }, } satisfies ProviderInstance; +const claudeInstanceId = ProviderInstanceId.make("claude_test"); +const claudeDriver = ProviderDriverKind.make("claudeAgent"); +const claudeProviderInstance = { + ...providerInstance, + instanceId: claudeInstanceId, + driverKind: claudeDriver, + continuationIdentity: { driverKind: claudeDriver, continuationKey: "claude:test" }, + displayName: "Claude test", + orchestrationAdapter: { + ...orchestrationAdapter, + instanceId: claudeInstanceId, + driver: claudeDriver, + }, +} satisfies ProviderInstance; const layerTestProviderInstanceRegistry = Layer.succeed( ProviderInstanceRegistry.ProviderInstanceRegistry, { getInstance: (instanceId) => Effect.succeed( - [providerInstance, alternateProviderInstance].find( + [providerInstance, alternateProviderInstance, claudeProviderInstance].find( (instance) => instanceId === instance.instanceId, ), ), - listInstances: Effect.succeed([providerInstance, alternateProviderInstance]), + listInstances: Effect.succeed([ + providerInstance, + alternateProviderInstance, + claudeProviderInstance, + ]), listUnavailable: Effect.succeed([]), streamChanges: Stream.empty, subscribeChanges: Effect.never, @@ -519,6 +538,9 @@ it.layer(layerProjectDeletionTest)("project deletion during thread commands", (i }), ); + // Coder: upstream's "%s releases the thread's devices" test (#34175) is not carried; there + // are no device hosts. + it.effect("waits for an in-flight thread update before planning deletion", () => Effect.gen(function* () { const projects = yield* ProjectService.ProjectService; @@ -865,6 +887,7 @@ const layerSharedApplicationDataPlaneTest = Layer.mergeAll( repositoryIdentityResolved: false, }), invalidate: () => Effect.void, + refreshRepositoryIdentity: () => Effect.void, subscribeChanges: Effect.never, }), ), @@ -986,6 +1009,228 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { }), ); + it.effect("captures and repeatedly rewinds non-Git conversations without losing run links", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; + const fileSystem = yield* FileSystem.FileSystem; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-non-git-rewind-" }); + const threadId = ThreadId.make("runtime-non-git-rewind"); + const rollbackThread = vi.fn( + ({ providerThread }: ProviderAdapter.ProviderAdapterV2RollbackThreadInput) => + Effect.succeed({ + providerThread: { + ...providerThread, + nativeThreadRef: { + driver, + nativeId: "rewound-native-thread", + strength: "strong" as const, + }, + }, + }), + ); + const sessionSpy = vi.spyOn(sessions, "open").mockReturnValue( + Effect.succeed({ + rollbackThread, + } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime), + ); + yield* Effect.addFinalizer(() => Effect.sync(() => sessionSpy.mockRestore())); + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("non-git-create"), + createdBy: "user", + creationSource: "web", + threadId, + projectId: ProjectId.make("non-git-project"), + title: "Non-Git rewind", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: cwd, + }); + + const completeMessage = (name: string) => + Effect.gen(function* () { + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(name), + createdBy: "user", + creationSource: "web", + threadId, + messageId: MessageId.make(name), + text: name, + attachments: [], + dispatchMode: { type: "start_immediately" }, + }); + const projection = yield* orchestrator.getThreadProjection(threadId); + const run = projection.runs.find((candidate) => candidate.userMessageId === name)!; + const node = projection.nodes.find((candidate) => candidate.id === run.rootNodeId)!; + const providerThread = projection.providerThreads.find( + (candidate) => candidate.id === run.providerThreadId, + )!; + const now = yield* DateTime.now; + const commandId = CommandId.make(`${name}-complete`); + // The provider completion is the fixture boundary; capture and rollback use real effects. + yield* outbox.cancelUnsettled({ + threadId, + effectTypes: ["provider-turn.start"], + reason: "provider completion supplied by fixture", + }); + yield* eventSink.writeWithEffects({ + commandId, + events: [ + { + id: EventId.make(`${name}-run`), + type: "run.updated", + threadId, + runId: run.id, + occurredAt: now, + payload: { ...run, status: "waiting", startedAt: now }, + }, + { + id: EventId.make(`${name}-node`), + type: "node.updated", + threadId, + runId: run.id, + nodeId: node.id, + occurredAt: now, + payload: { ...node, status: "completed", completedAt: now }, + }, + { + id: EventId.make(`${name}-turn`), + type: "provider-turn.updated", + threadId, + runId: run.id, + occurredAt: now, + payload: { + id: ProviderTurnId.make(`${name}-turn`), + providerThreadId: providerThread.id, + nodeId: node.id, + runAttemptId: run.activeAttemptId, + nativeTurnRef: null, + ordinal: run.ordinal, + status: "completed", + startedAt: now, + completedAt: now, + }, + }, + ], + effects: [ + { + id: `${name}-capture`, + commandId, + threadId, + request: { + type: "checkpoint.capture", + runId: run.id, + scopeId: node.checkpointScopeId!, + }, + }, + ], + }); + yield* worker.drain(); + const captured = yield* orchestrator.getThreadProjection(threadId); + assert.equal( + captured.runs.find((candidate) => candidate.id === run.id)?.status, + "completed", + ); + const checkpoint = captured.checkpoints.find((candidate) => candidate.runId === run.id)!; + assert.equal(checkpoint.status, "missing"); + return { run, checkpoint }; + }); + + const first = yield* completeMessage("non-git-first"); + const second = yield* completeMessage("non-git-second"); + const before = yield* orchestrator.getThreadProjection(threadId); + assert.equal( + before.checkpoints.find((checkpoint) => checkpoint.id === first.checkpoint.id)?.runId, + first.run.id, + "capturing a later run must preserve the earlier checkpoint's run link", + ); + const baseline = before.checkpoints.find( + (checkpoint) => checkpoint.ordinalWithinScope === 0, + )!; + assert.equal(baseline.status, "missing"); + for (const checkpoint of [baseline, first.checkpoint]) { + for (const restoreFiles of [undefined, true]) { + const commandId = CommandId.make(`non-git-files-${checkpoint.id}-${restoreFiles}`); + yield* orchestrator + .dispatch({ + type: "checkpoint.rollback", + commandId, + threadId, + checkpointId: checkpoint.id, + scopeId: checkpoint.scopeId, + ...(restoreFiles === undefined ? {} : { restoreFiles }), + }) + .pipe(Effect.flip); + assert.deepEqual(yield* outbox.listByCommandId(commandId), []); + } + } + assert.equal(rollbackThread.mock.calls.length, 0); + + const rewind = (name: string) => + Effect.gen(function* () { + const commandId = CommandId.make(name); + yield* orchestrator.dispatch({ + type: "checkpoint.rollback", + commandId, + threadId, + checkpointId: baseline.id, + scopeId: baseline.scopeId, + restoreFiles: false, + }); + yield* worker.drain(); + assert.deepEqual( + (yield* outbox.listByCommandId(commandId)).map((effect) => effect.status), + ["succeeded"], + ); + const projection = yield* orchestrator.getThreadProjection(threadId); + assert.ok(projection.runs.every((run) => run.status === "rolled_back")); + assert.isNull(projection.providerThreads[0]!.lastRunOrdinal); + assert.equal( + projection.providerThreads[0]!.nativeThreadRef?.nativeId, + "rewound-native-thread", + ); + assert.isNull(projection.thread.rollbackFailure); + }); + + yield* rewind("non-git-rewind-first"); + assert.equal(rollbackThread.mock.calls[0]![0].target.type, "thread_start"); + assert.deepEqual( + rollbackThread.mock.calls[0]![0].providerThreadTurns.map((turn) => turn.id), + [ProviderTurnId.make("non-git-first-turn"), ProviderTurnId.make("non-git-second-turn")], + ); + const discardedCommandId = CommandId.make("non-git-discarded-target"); + yield* orchestrator + .dispatch({ + type: "checkpoint.rollback", + commandId: discardedCommandId, + threadId, + checkpointId: second.checkpoint.id, + scopeId: second.checkpoint.scopeId, + restoreFiles: false, + }) + .pipe(Effect.flip); + assert.deepEqual(yield* outbox.listByCommandId(discardedCommandId), []); + assert.equal(rollbackThread.mock.calls.length, 1); + + const resent = yield* completeMessage("non-git-resent"); + assert.isAbove(resent.run.ordinal, second.run.ordinal); + yield* rewind("non-git-rewind-again"); + assert.equal(rollbackThread.mock.calls.length, 2); + assert.equal(rollbackThread.mock.calls[1]![0].target.type, "thread_start"); + assert.deepEqual( + rollbackThread.mock.calls[1]![0].providerThreadTurns.map((turn) => turn.id), + [ProviderTurnId.make("non-git-resent-turn")], + ); + }).pipe(Effect.provide(Layer.merge(Layer.fresh(layerTest), NodeServices.layer))), + ); + it.effect("rejects non-ready rollback targets before persisting events or effects", () => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; @@ -1272,6 +1517,7 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { providerThreadId: providerThread.id, providerTurnId: providerTurn.id, messageId: steerMessageId, + modelSelection: run.modelSelection, }, ], ); @@ -1566,10 +1812,16 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { assert.equal(native.runtimeRequests[0]?.responseCapability.type, "not_resumable"); yield* recovery.reconcile("shutdown"); yield* recovery.reconcile("startup"); - assert.deepEqual( - (yield* orchestrator.getThreadProjection(threadId)).runtimeRequests, - recovered.runtimeRequests, - ); + const rerecovered = yield* orchestrator.getThreadProjection(threadId); + assert.deepEqual(rerecovered.runtimeRequests, recovered.runtimeRequests); + const runItems = rerecovered.turnItems.filter((item) => item.runId === runId); + const notices = runItems.filter((item) => item.type === "system_notice"); + assert.lengthOf(notices, scope === "run" ? 1 : 0); + if (scope === "run") { + assert.equal(notices[0]?.title, "Server restarted"); + assert.equal(notices[0]?.nodeId, recovered.runs[0]?.rootNodeId); + assert.equal(notices[0]?.ordinal, Math.max(...runItems.map((item) => item.ordinal))); + } yield* orchestrator.dispatch({ type: "runtime-request.respond", commandId: CommandId.make(`recovery-answer-${trigger}-${scope}`), @@ -3391,6 +3643,75 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => { }), ); + it.effect("ends a watch on a closed pull request whose activity cannot be read", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const threadId = ThreadId.make("runtime-pull-request-watch-closed-unreadable"); + const projectId = ProjectId.make("pr-watch-closed-unreadable-project"); + yield* seedProject({ + projectId, + title: "Watch closed unreadable", + workspaceRoot: "/workspace/watch-closed-unreadable", + defaultModelSelection: null, + createdAt: "2026-10-01T00:00:00.000Z", + }); + yield* orchestrator.dispatch({ + type: "thread.create", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make("pr-watch-closed-unreadable-create"), + threadId, + projectId, + title: "Watch closed unreadable", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + }); + yield* orchestrator.dispatch({ + type: "thread.pull-request.watch", + commandId: CommandId.make("pr-watch-closed-unreadable-start"), + threadId, + host: "github.com", + repository: "pingdotgg/t3code", + number: 8, + watching: true, + link: { url: "https://github.com/pingdotgg/t3code/pull/8", source: "agent" }, + }); + const reactor = yield* PullRequestWatchReactor.make.pipe( + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + NodeCrypto.layer, + Layer.mock(PullRequestService.PullRequestService)({ + watchFingerprint: () => Effect.succeed(null), + detail: () => + Effect.succeed({ + ...watchedPullRequestDetail({ + projectId, + number: 8, + at: "2026-10-02T12:00:00.000Z", + }), + state: "closed" as const, + }), + activity: () => Effect.die("host unreachable"), + }), + ), + ), + ); + yield* reactor.sweep; + + const thread = yield* orchestrator.getThreadShell(threadId); + assert.isUndefined(thread?.pullRequests?.[0]?.watch); + const { messages } = yield* orchestrator.getThreadRecords(threadId, ["messages"]); + assert.deepEqual( + messages.flatMap((message) => message.notification?.summary ?? []), + ["!8: closed, stopped watching"], + ); + }), + ); + it.effect("wakes once for a state it reported, across a restart and a new watch", () => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; @@ -3489,12 +3810,12 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => { }), ); - it.effect("finishes owed checks once after pending sweeps, a restart, and a re-watch", () => + it.effect.each(["pending", "action-required"] as const)("recovers %s watches", (initialStatus) => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; - const threadId = ThreadId.make("runtime-pull-request-watch-owed"); - const projectId = ProjectId.make("pr-watch-owed-project"); + const threadId = ThreadId.make(`runtime-pull-request-watch-owed-${initialStatus}`); + const projectId = ProjectId.make(`pr-watch-owed-project-${initialStatus}`); yield* seedProject({ projectId, title: "Watch owed", @@ -3506,7 +3827,7 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => { type: "thread.create", createdBy: "user", creationSource: "web", - commandId: CommandId.make("pr-watch-owed-create"), + commandId: CommandId.make(`pr-watch-owed-create-${initialStatus}`), threadId, projectId, title: "Watch owed", @@ -3519,7 +3840,7 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => { const watch = (on: boolean, id: string) => orchestrator.dispatch({ type: "thread.pull-request.watch", - commandId: CommandId.make(`pr-watch-owed-${id}`), + commandId: CommandId.make(`pr-watch-owed-${initialStatus}-${id}`), threadId, host: "github.com", repository: "pingdotgg/t3code", @@ -3529,7 +3850,10 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => { }); yield* watch(true, "start"); - let remainingStatus: "pending" | "success" = "pending"; + let remainingStatus: "pending" | "action-required" | "success" = initialStatus; + const approval = initialStatus === "action-required"; + const firstSummary = approval ? "!12: checks await approval" : "!12: checks passed"; + const runUrl = "https://github.com/pingdotgg/t3code/actions/runs/123"; const host = Layer.mergeAll( NodeServices.layer, NodeCrypto.layer, @@ -3546,7 +3870,7 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => { name, status: name === "test" ? remainingStatus : ("success" as const), description: null, - url: null, + url: name === "test" ? `${runUrl}/job/7` : null, required: name === "lint", })), })), @@ -3571,38 +3895,42 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => { yield* TestClock.adjust("10 minutes"); yield* reactor.sweep; yield* worker.drain(); - assert.deepEqual(yield* summaries, ["!12: checks passed"]); + assert.deepEqual(yield* summaries, [firstSummary]); assert.deepEqual( - (yield* orchestrator.getThreadShell(threadId))?.pullRequests?.[0]?.watch?.owedChecks, - ["test"], + (yield* orchestrator.getThreadShell(threadId))?.pullRequests?.[0]?.watch?.[ + approval ? "awaitingApproval" : "owedChecks" + ], + approval ? [runUrl] : ["test"], ); // The fresh reactor reloads the nonempty obligation from the persisted watch. const restarted = yield* PullRequestWatchReactor.make.pipe(Effect.provide(host)); yield* restarted.sweep; yield* worker.drain(); - assert.deepEqual(yield* summaries, ["!12: checks passed"]); + assert.deepEqual(yield* summaries, [firstSummary]); // A new watch on the same pull request knows what the last one told. yield* watch(false, "stop"); yield* watch(true, "again"); yield* restarted.sweep; yield* worker.drain(); - assert.deepEqual(yield* summaries, ["!12: checks passed"]); + assert.deepEqual(yield* summaries, [firstSummary]); assert.deepEqual( - (yield* orchestrator.getThreadShell(threadId))?.pullRequests?.[0]?.watch?.owedChecks, - ["test"], + (yield* orchestrator.getThreadShell(threadId))?.pullRequests?.[0]?.watch?.[ + approval ? "awaitingApproval" : "owedChecks" + ], + approval ? [runUrl] : ["test"], ); remainingStatus = "success"; yield* TestClock.adjust("10 minutes"); yield* restarted.sweep; yield* worker.drain(); - assert.deepEqual(yield* summaries, ["!12: checks passed", "!12: checks passed"]); + assert.deepEqual(yield* summaries, [firstSummary, "!12: checks passed"]); yield* TestClock.adjust("10 minutes"); yield* restarted.sweep; yield* worker.drain(); - assert.deepEqual(yield* summaries, ["!12: checks passed", "!12: checks passed"]); + assert.deepEqual(yield* summaries, [firstSummary, "!12: checks passed"]); assert.deepEqual( (yield* orchestrator.getThreadShell(threadId))?.pullRequests?.[0]?.watch?.owedChecks, [], @@ -4325,40 +4653,391 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => { }), ); - it.effect("reads each watched PR as soon as its own fingerprint lands", () => + const createWatchedThread = Effect.fnUntraced(function* (name: string, reopened = false) { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const threadId = ThreadId.make(`pr-wake-${name}`); + yield* orchestrator.dispatch({ + type: "thread.create", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make(`${threadId}-create`), + threadId, + projectId: ProjectId.make(`${threadId}-project`), + title: "Watch wake", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: process.cwd(), + }); + const key = { host: "github.com", repository: "pingdotgg/t3code", number: 7 }; + yield* orchestrator.dispatch({ + type: "thread.pull-request.watch", + commandId: CommandId.make(`${threadId}-watch`), + threadId, + ...key, + watching: true, + link: { url: "https://github.com/pingdotgg/t3code/pull/7", source: "agent" }, + }); + let watch = (yield* orchestrator.getThreadShell(threadId))?.pullRequests?.[0]?.watch; + assert.isDefined(watch); + const snapshotSyncedAt = reopened + ? DateTime.formatIso(yield* DateTime.now) + : "2026-10-02T12:00:00.000Z"; + if (reopened) { + yield* orchestrator.dispatch({ + type: "thread.pull-request-watch.sync", + commandId: CommandId.make(`${threadId}-end-watch`), + threadId, + ...key, + startedAt: watch.startedAt, + watch: null, + }); + yield* TestClock.adjust("1 millis"); + yield* orchestrator.dispatch({ + type: "thread.pull-request.watch", + commandId: CommandId.make(`${threadId}-rewatch`), + threadId, + ...key, + watching: true, + }); + watch = (yield* orchestrator.getThreadShell(threadId))?.pullRequests?.[0]?.watch; + assert.isDefined(watch); + } + return { threadId, key, watch, snapshotSyncedAt }; + }); + + it.effect.each([ + "unwatch", + "unlink", + "new watch", + "merged", + "closed", + "newer head", + "comments then newer head", + "comment limit", + "still watching", + "closed notice", + "closed notice unlinked and relinked", + "reopened", + ] as const)("revalidates a queued pull-request wake before promotion: %s", (mode) => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; - const projectId = ProjectId.make("pr-watch-pipeline-project"); - yield* seedProject({ - projectId, - title: "Watch pipeline", - workspaceRoot: "/workspace/watch-pipeline", - defaultModelSelection: null, - createdAt: "2026-10-01T00:00:00.000Z", + const eventSink = yield* EventSink.EventSinkV2; + const { threadId, key, watch, snapshotSyncedAt } = yield* createWatchedThread( + `queued-${mode}`, + mode === "reopened", + ); + const finalNotice = + mode === "closed notice" || mode === "closed notice unlinked and relinked"; + const deliverable = + mode === "still watching" || + mode === "closed notice" || + mode === "reopened" || + mode === "comments then newer head" || + mode === "comment limit"; + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make(`${threadId}-active`), + threadId, + messageId: MessageId.make(`${threadId}-active`), + text: "Active work", + attachments: [], + modelSelection, + dispatchMode: { type: "start_immediately" }, }); - for (const number of [21, 22]) { - const threadId = ThreadId.make(`runtime-pull-request-watch-pipeline-${number}`); - yield* orchestrator.dispatch({ - type: "thread.create", - createdBy: "user", - creationSource: "web", - commandId: CommandId.make(`pr-watch-pipeline-create-${number}`), - threadId, - projectId, - title: "Watch pipeline", - modelSelection, - runtimeMode: "full-access", - interactionMode: "default", - branch: null, - worktreePath: null, - }); + const wakeMessageId = MessageId.make(`${threadId}-wake`); + const recordedWatch = { ...watch, headSha: "abc123", wakes: 1 }; + yield* orchestrator.dispatch({ + type: "thread.pull-request-watch.sync", + commandId: CommandId.make(`${threadId}-wake`), + threadId, + ...key, + startedAt: watch.startedAt, + watch: finalNotice ? null : recordedWatch, + wake: { + messageId: wakeMessageId, + text: finalNotice ? "Closed, stopped watching" : "Checks failed", + notification: { + source: { kind: "monitor" }, + outcome: "updated", + summary: finalNotice ? "#7: closed, stopped watching" : "#7: checks failed", + }, + ...(mode === "comments then newer head" || mode === "comment limit" + ? { includesRemarks: true } + : {}), + }, + }); + const userMessageId = MessageId.make(`${threadId}-user`); + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make(`${threadId}-user`), + threadId, + messageId: userMessageId, + text: "User follow-up", + attachments: [], + modelSelection, + dispatchMode: { type: "queue_after_active" }, + }); + const before = yield* orchestrator.getThreadProjection(threadId); + const activeRun = before.runs.find((run) => run.status === "starting"); + const wakeRun = before.runs.find((run) => run.userMessageId === wakeMessageId); + const userRun = before.runs.find((run) => run.userMessageId === userMessageId); + assert.isDefined(activeRun); + assert.isDefined(wakeRun); + assert.isDefined(userRun); + assert.equal(wakeRun.status, "queued"); + assert.equal(userRun.status, "queued"); + + if (mode === "unwatch" || mode === "new watch") { yield* orchestrator.dispatch({ type: "thread.pull-request.watch", - commandId: CommandId.make(`pr-watch-pipeline-start-${number}`), + commandId: CommandId.make(`${threadId}-unwatch`), threadId, - host: "github.com", - repository: "pingdotgg/t3code", - number, + ...key, + watching: false, + }); + if (mode === "new watch") { + yield* TestClock.adjust("1 millis"); + yield* orchestrator.dispatch({ + type: "thread.pull-request.watch", + commandId: CommandId.make(`${threadId}-rewatch`), + threadId, + ...key, + watching: true, + }); + const restarted = (yield* orchestrator.getThreadShell(threadId))?.pullRequests?.[0] + ?.watch; + assert.isDefined(restarted); + assert.notEqual(restarted.startedAt, watch.startedAt); + yield* orchestrator.dispatch({ + type: "thread.pull-request-watch.sync", + commandId: CommandId.make(`${threadId}-same-head`), + threadId, + ...key, + startedAt: restarted.startedAt, + watch: { ...restarted, headSha: recordedWatch.headSha }, + }); + } + } else if (mode === "unlink" || mode === "closed notice unlinked and relinked") { + yield* orchestrator.dispatch({ + type: "thread.pull-request.unlink", + commandId: CommandId.make(`${threadId}-unlink`), + threadId, + ...key, + }); + if (mode === "closed notice unlinked and relinked") { + yield* orchestrator.dispatch({ + type: "thread.pull-request.link", + commandId: CommandId.make(`${threadId}-relink`), + threadId, + ...key, + url: "https://github.com/pingdotgg/t3code/pull/7", + source: "manual", + }); + } + } else if ( + mode === "merged" || + mode === "closed" || + mode === "closed notice" || + mode === "reopened" + ) { + yield* orchestrator.dispatch({ + type: "thread.pull-request-link.sync", + commandId: CommandId.make(`${threadId}-snapshot`), + threadId, + ...key, + snapshot: { + state: mode === "merged" ? "merged" : "closed", + title: "Watched pull request", + headBranch: "feature", + baseBranch: "main", + isDraft: false, + updatedAt: null, + // A delayed sync can still report the closure from before this watch restarted. + syncedAt: snapshotSyncedAt, + }, + stack: null, + }); + } else if (mode === "comment limit") { + yield* orchestrator.dispatch({ + type: "thread.pull-request-watch.sync", + commandId: CommandId.make(`${threadId}-exhausted`), + threadId, + ...key, + startedAt: watch.startedAt, + watch: null, + endedWatch: recordedWatch, + wake: { + messageId: MessageId.make(`${threadId}-exhausted`), + text: "New comments, stopped watching", + notification: { + source: { kind: "monitor" }, + outcome: "updated", + summary: "#7: new comments, stopped watching", + }, + includesRemarks: true, + }, + }); + } else if (mode === "newer head" || mode === "comments then newer head") { + yield* orchestrator.dispatch({ + type: "thread.pull-request-watch.sync", + commandId: CommandId.make(`${threadId}-advance`), + threadId, + ...key, + startedAt: watch.startedAt, + watch: { ...recordedWatch, headSha: "def456" }, + }); + } + + const afterSequence = yield* orchestrator.getThreadEventSequence(threadId); + const completedAt = yield* DateTime.now; + yield* eventSink.write({ + events: [ + { + id: EventId.make(`${threadId}-completed`), + type: "run.updated", + threadId, + runId: activeRun.id, + ...(activeRun.rootNodeId === null ? {} : { nodeId: activeRun.rootNodeId }), + providerInstanceId: activeRun.providerInstanceId, + occurredAt: completedAt, + payload: { ...activeRun, status: "completed", completedAt }, + }, + ], + }); + yield* eventSink.stream({ threadId, afterSequence }).pipe( + Stream.filter( + (stored) => + stored.event.type === "run.updated" && stored.event.payload.status === "starting", + ), + Stream.take(1), + Stream.runDrain, + ); + const after = yield* orchestrator.getThreadProjection(threadId); + assert.equal( + after.runs.find((run) => run.id === wakeRun.id)?.status, + deliverable ? "starting" : "cancelled", + ); + assert.equal( + after.runs.find((run) => run.id === userRun.id)?.status, + deliverable ? "queued" : "starting", + ); + if (!deliverable) { + assert.equal( + after.attempts.find((attempt) => attempt.id === wakeRun.activeAttemptId)?.status, + "cancelled", + ); + assert.equal( + after.nodes.find((node) => node.id === wakeRun.rootNodeId)?.status, + "cancelled", + ); + assert.isFalse(after.turnItems.some((item) => item.runId === wakeRun.id)); + } + assert.deepEqual( + after.messages + .filter((message) => message.id === wakeMessageId) + .map((message) => message.notification?.summary), + [finalNotice ? "#7: closed, stopped watching" : "#7: checks failed"], + ); + }), + ); + + it.effect.each(["open", "merged", "closed", "closed notice", "reopened"] as const)( + "revalidates a pull-request wake before idle dispatch: %s", + (mode) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const { threadId, key, watch, snapshotSyncedAt } = yield* createWatchedThread( + `idle-${mode}`, + mode === "reopened", + ); + yield* orchestrator.dispatch({ + type: "thread.pull-request-link.sync", + commandId: CommandId.make(`${threadId}-snapshot`), + threadId, + ...key, + snapshot: { + state: mode === "closed notice" || mode === "reopened" ? "closed" : mode, + title: "Watched pull request", + headBranch: "feature", + baseBranch: "main", + isDraft: false, + updatedAt: null, + syncedAt: snapshotSyncedAt, + }, + stack: null, + }); + const wakeMessageId = MessageId.make(`${threadId}-wake`); + yield* orchestrator.dispatch({ + type: "thread.pull-request-watch.sync", + commandId: CommandId.make(`${threadId}-wake`), + threadId, + ...key, + startedAt: watch.startedAt, + watch: mode === "closed notice" ? null : { ...watch, headSha: "abc123", wakes: 1 }, + wake: { + messageId: wakeMessageId, + text: mode === "closed notice" ? "Closed, stopped watching" : "Checks failed", + notification: { + source: { kind: "monitor" }, + outcome: "updated", + summary: + mode === "closed notice" ? "#7: closed, stopped watching" : "#7: checks failed", + }, + }, + }); + const after = yield* orchestrator.getThreadProjection(threadId); + const deliverable = mode === "open" || mode === "closed notice" || mode === "reopened"; + assert.deepEqual( + after.runs.map((run) => [run.userMessageId, run.status]), + deliverable ? [[wakeMessageId, "starting"]] : [], + ); + assert.deepEqual( + after.messages.map((message) => message.id), + deliverable ? [wakeMessageId] : [], + ); + }), + ); + + it.effect("reads each watched PR as soon as its own fingerprint lands", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const projectId = ProjectId.make("pr-watch-pipeline-project"); + yield* seedProject({ + projectId, + title: "Watch pipeline", + workspaceRoot: "/workspace/watch-pipeline", + defaultModelSelection: null, + createdAt: "2026-10-01T00:00:00.000Z", + }); + for (const number of [21, 22]) { + const threadId = ThreadId.make(`runtime-pull-request-watch-pipeline-${number}`); + yield* orchestrator.dispatch({ + type: "thread.create", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make(`pr-watch-pipeline-create-${number}`), + threadId, + projectId, + title: "Watch pipeline", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + }); + yield* orchestrator.dispatch({ + type: "thread.pull-request.watch", + commandId: CommandId.make(`pr-watch-pipeline-start-${number}`), + threadId, + host: "github.com", + repository: "pingdotgg/t3code", + number, watching: true, link: { url: `https://github.com/pingdotgg/t3code/pull/${number}`, source: "agent" }, }); @@ -5342,6 +6021,218 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => { }), ); + it.effect.each([ + { origin: "agent" as const, delivery: "queued" as const }, + { origin: "agent" as const, delivery: "after-completion" as const }, + { origin: "agent" as const, delivery: "held-resume" as const }, + { origin: "agent" as const, delivery: "resume-before-stop-ends" as const }, + { origin: "child" as const, delivery: "after-completion" as const }, + { origin: "user" as const, delivery: "queued" as const }, + ])( + "preserves queued agent and child updates but resets fresh requests (%j)", + ({ origin, delivery }) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const threadId = ThreadId.make(`work-clock-${origin}-${delivery}`); + const originalMessageId = MessageId.make(`${threadId}-original`); + const followUpMessageId = MessageId.make(`${threadId}-follow-up`); + yield* orchestrator.dispatch({ + type: "thread.create", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make(`${threadId}-create`), + threadId, + projectId: ProjectId.make(`${threadId}-project`), + title: "Agent message work clock", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: process.cwd(), + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make(`${threadId}-original`), + threadId, + messageId: originalMessageId, + text: "Original user work", + attachments: [], + modelSelection, + dispatchMode: { type: "start_immediately" }, + }); + const dispatchFollowUp = orchestrator.dispatch({ + type: "message.dispatch", + createdBy: origin === "user" ? "user" : "agent", + creationSource: origin === "user" ? "web" : "mcp", + ...(origin === "user" ? {} : { senderThreadId: ThreadId.make("sending-agent") }), + commandId: CommandId.make(`${threadId}-follow-up`), + threadId, + messageId: followUpMessageId, + text: "Follow-up work", + attachments: [], + modelSelection, + dispatchMode: { + type: delivery === "after-completion" ? "start_immediately" : "queue_after_active", + }, + }); + const held = delivery === "held-resume" || delivery === "resume-before-stop-ends"; + const drainsOnEnd = delivery === "queued" || delivery === "resume-before-stop-ends"; + if (delivery === "queued" || held) yield* dispatchFollowUp; + const startedRunIds = yield* Queue.unbounded(); + const afterSequence = yield* orchestrator.getThreadEventSequence(threadId); + yield* eventSink.stream({ threadId, afterSequence }).pipe( + Stream.runForEach((stored) => + stored.event.type === "run.updated" && stored.event.payload.status === "starting" + ? Queue.offer(startedRunIds, stored.event.payload.id) + : Effect.void, + ), + Effect.forkScoped, + ); + yield* Effect.yieldNow; + const original = (yield* orchestrator.getThreadProjection(threadId)).runs.find( + (run) => run.userMessageId === originalMessageId, + ); + assert.isDefined(original); + if (origin === "child") { + assert.isNotNull(original.rootNodeId); + yield* eventSink.write({ + events: [ + { + id: EventId.make(`${threadId}-child`), + type: "subagent.updated", + threadId, + runId: original.id, + occurredAt: original.requestedAt, + payload: { + id: NodeId.make(`${threadId}-child`), + threadId, + runId: original.id, + parentNodeId: original.rootNodeId, + origin: "app_owned", + createdBy: "agent", + driver, + providerInstanceId: original.providerInstanceId, + providerThreadId: null, + childThreadId: ThreadId.make("sending-agent"), + nativeTaskRef: null, + prompt: "Delegated work", + title: null, + model: null, + status: "completed", + result: "Done", + startedAt: original.requestedAt, + completedAt: original.requestedAt, + updatedAt: original.requestedAt, + }, + }, + ], + }); + } + if (held) { + const queued = (yield* orchestrator.getThreadProjection(threadId)).runs.find( + (run) => run.userMessageId === followUpMessageId, + ); + assert.isDefined(queued); + yield* eventSink.write({ + events: [ + { + id: EventId.make(`${threadId}-hold`), + type: "run.updated", + threadId, + occurredAt: original.requestedAt, + payload: { ...queued, queueHeld: true }, + }, + { + id: EventId.make(`${threadId}-interrupt-request`), + type: "turn-item.updated", + threadId, + runId: original.id, + occurredAt: original.requestedAt, + payload: { + id: TurnItemId.make(`${threadId}-interrupt-request`), + type: "run_interrupt_request", + threadId, + runId: original.id, + nodeId: original.rootNodeId, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 90, + status: "completed", + title: "Interrupt requested", + startedAt: original.requestedAt, + completedAt: original.requestedAt, + updatedAt: original.requestedAt, + message: "Stopped by user", + }, + }, + ], + }); + } + if (delivery === "resume-before-stop-ends") { + yield* orchestrator.dispatch({ + type: "queue.resume", + commandId: CommandId.make(`${threadId}-resume`), + threadId, + }); + } + const originalStart = DateTime.makeUnsafe("2026-10-21T14:20:17.457Z"); + const completedAt = DateTime.makeUnsafe("2026-10-21T14:23:53.509Z"); + yield* eventSink.write({ + events: [ + { + id: EventId.make(`${threadId}-completed`), + type: "run.updated", + threadId, + runId: original.id, + ...(original.rootNodeId === null ? {} : { nodeId: original.rootNodeId }), + providerInstanceId: original.providerInstanceId, + occurredAt: completedAt, + payload: { + ...original, + status: held ? "interrupted" : "completed", + startedAt: originalStart, + completedAt, + }, + }, + ], + }); + if (delivery === "held-resume") { + yield* TestClock.adjust("3 hours"); + const queued = (yield* orchestrator.getThreadProjection(threadId)).runs.find( + (run) => run.userMessageId === followUpMessageId, + ); + assert.isDefined(queued); + assert.equal(queued.status, "queued"); + yield* orchestrator.dispatch({ + type: "queue.resume", + commandId: CommandId.make(`${threadId}-resume`), + threadId, + }); + } else if (delivery === "after-completion") yield* dispatchFollowUp; + const startedRunId = drainsOnEnd ? yield* Queue.take(startedRunIds) : null; + const followUp = (yield* orchestrator.getThreadProjection(threadId)).runs.find( + (run) => run.userMessageId === followUpMessageId, + ); + assert.isDefined(followUp); + if (drainsOnEnd) assert.equal(startedRunId, followUp.id); + assert.equal(followUp.status, "starting"); + if (origin === "child" || (origin === "agent" && delivery === "queued")) { + assert.isDefined(followUp.workStartedAt); + assert.equal( + DateTime.toEpochMillis(followUp.workStartedAt), + DateTime.toEpochMillis(originalStart), + ); + } else { + assert.isUndefined(followUp.workStartedAt); + } + }), + ); + it.effect.each(["usage_limit", "provider_error"] as const)( "handles a queued message after a %s failure", (failureClass) => @@ -5902,10 +6793,143 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => { }), ); + it.effect.each(["the reset passes", "the user resumes"] as const)( + "holds a limited provider's queue through another provider's turn until %s", + (release) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const threadId = ThreadId.make(`runtime-layer-queue-limit-switch-${release}`); + const switchedSelection = { instanceId: alternateInstanceId, model: "gpt-5.5" }; + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make(`${threadId}:create`), + threadId, + createdBy: "user", + creationSource: "web", + projectId: ProjectId.make(`${threadId}:project`), + title: "Limit switch", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: process.cwd(), + }); + const send = ( + name: string, + selection: typeof modelSelection | typeof switchedSelection, + type: "start_immediately" | "queue_after_active", + ) => + orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`${threadId}:${name}`), + threadId, + messageId: MessageId.make(`${threadId}:${name}`), + text: name, + attachments: [], + createdBy: "user", + creationSource: "web", + modelSelection: selection, + dispatchMode: { type }, + }); + yield* send("limited", modelSelection, "start_immediately"); + yield* send("queued", modelSelection, "queue_after_active"); + const [limited, queued] = (yield* orchestrator.getThreadProjection(threadId)).runs; + assert.isDefined(limited); + assert.isDefined(queued); + const now = yield* DateTime.now; + yield* eventSink.write({ + events: [ + { + id: EventId.make(`${threadId}:error`), + type: "turn-item.updated", + threadId, + runId: limited.id, + nodeId: limited.rootNodeId!, + providerInstanceId: limited.providerInstanceId, + occurredAt: now, + payload: { + id: TurnItemId.make(`${threadId}:error`), + type: "error", + threadId, + runId: limited.id, + nodeId: limited.rootNodeId!, + providerThreadId: limited.providerThreadId, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 2, + status: "failed", + title: "Usage limit", + startedAt: now, + completedAt: now, + updatedAt: now, + failure: { + class: "usage_limit", + message: "Usage limit reached", + code: "usage_limit", + retryable: null, + resetAt: DateTime.formatIso(DateTime.add(now, { hours: 1 })), + }, + }, + }, + { + id: EventId.make(`${threadId}:failed`), + type: "run.updated", + threadId, + runId: limited.id, + providerInstanceId: limited.providerInstanceId, + occurredAt: now, + payload: { ...limited, status: "failed", completedAt: now }, + }, + ], + }); + + // A message on another provider starts ahead of the held queue and + // completes there. + yield* send("switch", switchedSelection, "start_immediately"); + const switched = (yield* orchestrator.getThreadProjection(threadId)).runs.find( + (run) => run.userMessageId === MessageId.make(`${threadId}:switch`), + ); + assert.equal(switched?.status, "starting"); + yield* eventSink.write({ + events: [ + { + id: EventId.make(`${threadId}:switch:completed`), + type: "run.updated", + threadId, + runId: switched!.id, + providerInstanceId: switched!.providerInstanceId, + occurredAt: now, + payload: { ...switched!, status: "completed", completedAt: now }, + }, + ], + }); + + yield* orchestrator.resumeQueuedRuns; + const held = yield* orchestrator.getThreadProjection(threadId); + assert.equal(held.runs.find((run) => run.id === queued.id)?.status, "queued"); + + if (release === "the reset passes") { + yield* TestClock.adjust("61 minutes"); + yield* orchestrator.resumeQueuedRuns; + } else { + yield* orchestrator.dispatch({ + type: "queue.resume", + commandId: CommandId.make(`${threadId}:resume`), + threadId, + }); + } + const reset = yield* orchestrator.getThreadProjection(threadId); + const resumed = reset.runs.find((run) => run.id === queued.id); + assert.equal(resumed?.status, "starting"); + assert.deepEqual(resumed?.modelSelection, modelSelection); + }), + ); + it.effect.each([ ["server", modelSelection], ["provider", modelSelection], - ["provider", { ...modelSelection, instanceId: alternateInstanceId }], ] as const)( "delivers a queued monitor update from %s after conversation selection %j", ([creationSource, currentSelection]) => @@ -6018,7 +7042,6 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => { it.effect.each([ { ...modelSelection, model: "gpt-5.5", options: [{ id: "reasoningEffort", value: "high" }] }, - { instanceId: alternateInstanceId, model: "gpt-5.5" }, ])( "starts a queued delegated completion on a selection set while it waited: %j", (newerSelection) => @@ -6126,6 +7149,220 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => { }), ); + it.effect("keeps a model picked ahead of a queued agent message that names none", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const threadId = ThreadId.make("runtime-layer-queued-agent-message-model-pick"); + const pickedSelection = { ...modelSelection, model: "gpt-5.5" }; + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make(`${threadId}:create`), + threadId, + projectId: ProjectId.make(`${threadId}:project`), + createdBy: "user", + creationSource: "web", + title: "Queued model pick", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: process.cwd(), + }); + for (const [key, selection] of [ + ["active", modelSelection], + ["picked", pickedSelection], + ] as const) { + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`${threadId}:${key}`), + threadId, + messageId: MessageId.make(`${threadId}:${key}`), + text: key, + attachments: [], + modelSelection: selection, + dispatchMode: { type: key === "active" ? "start_immediately" : "queue_after_active" }, + createdBy: "user", + creationSource: "web", + }); + } + // What t3_thread_send dispatches: no model of its own. + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`${threadId}:agent`), + threadId, + messageId: MessageId.make(`${threadId}:agent`), + text: "Message from another thread", + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "agent", + creationSource: "mcp", + }); + const [active, picked, agent] = (yield* orchestrator.getThreadProjection(threadId)).runs; + assert.isDefined(active); + assert.isDefined(picked); + assert.isDefined(agent); + const now = yield* DateTime.now; + for (const runId of [active.id, picked.id]) { + const current = yield* orchestrator.getThreadProjection(threadId); + yield* eventSink.write({ + events: [ + { + id: EventId.make(`${runId}:completed`), + type: "run.updated", + threadId, + runId, + occurredAt: now, + payload: { + ...current.runs.find((run) => run.id === runId)!, + status: "completed", + completedAt: now, + }, + }, + ], + }); + yield* orchestrator.resumeQueuedRuns; + } + const after = yield* orchestrator.getThreadProjection(threadId); + const delivered = after.runs.find((run) => run.id === agent.id)!; + assert.equal(delivered.status, "starting"); + assert.deepEqual(delivered.modelSelection, pickedSelection); + assert.deepEqual(after.thread.modelSelection, pickedSelection); + }), + ); + + it.effect("keeps a buffered provider wake on its own provider behind a queued pick", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const threadId = ThreadId.make("runtime-layer-buffered-wake-behind-queued-pick"); + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make(`${threadId}:create`), + threadId, + projectId: ProjectId.make(`${threadId}:project`), + createdBy: "user", + creationSource: "web", + title: "Buffered wake behind a pick", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: process.cwd(), + }); + for (const [key, selection] of [ + ["active", modelSelection], + ["picked", { ...modelSelection, instanceId: alternateInstanceId }], + ] as const) { + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`${threadId}:${key}`), + threadId, + messageId: MessageId.make(`${threadId}:${key}`), + text: key, + attachments: [], + modelSelection: selection, + dispatchMode: { type: key === "active" ? "start_immediately" : "queue_after_active" }, + createdBy: "user", + creationSource: "web", + }); + } + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`${threadId}:wake`), + threadId, + messageId: MessageId.make(`${threadId}:wake`), + text: "Background activity updated", + attachments: [], + dispatchMode: { type: "queue_after_active" }, + createdBy: "agent", + creationSource: "provider", + notification: { + source: { kind: "background_task" }, + outcome: "updated", + summary: "Background activity updated", + }, + }); + const projection = yield* orchestrator.getThreadProjection(threadId); + const active = projection.runs.find( + (run) => run.userMessageId === MessageId.make(`${threadId}:active`), + ); + const wake = projection.runs.find( + (run) => run.userMessageId === MessageId.make(`${threadId}:wake`), + ); + assert.deepEqual(wake?.modelSelection, modelSelection); + assert.equal(wake?.providerThreadId, active?.providerThreadId); + }), + ); + + it.effect("checks a queued goal pause against the model picked ahead of it", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const threadId = ThreadId.make("runtime-layer-goal-pause-behind-queued-pick"); + const claudeSelection = { ...modelSelection, instanceId: claudeInstanceId }; + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make(`${threadId}:create`), + threadId, + projectId: ProjectId.make(`${threadId}:project`), + createdBy: "user", + creationSource: "web", + title: "Goal pause behind a pick", + modelSelection: claudeSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: process.cwd(), + }); + for (const [key, selection] of [ + ["active", claudeSelection], + ["picked", modelSelection], + ] as const) { + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`${threadId}:${key}`), + threadId, + messageId: MessageId.make(`${threadId}:${key}`), + text: key, + attachments: [], + modelSelection: selection, + dispatchMode: { type: key === "active" ? "start_immediately" : "queue_after_active" }, + createdBy: "user", + creationSource: "web", + }); + } + const sendGoal = (key: string, text: string, deliveryIntent?: "auto") => + orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`${threadId}:${key}`), + threadId, + messageId: MessageId.make(`${threadId}:${key}`), + text, + attachments: [], + dispatchMode: { type: "queue_after_active" }, + ...(deliveryIntent === undefined ? {} : { deliveryIntent }), + createdBy: "agent", + creationSource: "mcp", + }); + // It runs after the pick, on a provider that can pause goals. + yield* sendGoal("pause", "/goal pause"); + // A resume stops the active run and starts before the pick, so the thread's + // Claude instance still refuses it. + const sequence = yield* orchestrator.getThreadEventSequence(threadId); + const refused = yield* Effect.flip(sendGoal("resume", "/goal resume", "auto")); + assert.instanceOf(refused, Orchestrator.OrchestratorDispatchError); + assert.equal( + refused.cause, + "Claude goals cannot be paused or resumed. Send /goal clear to remove the goal.", + ); + assert.equal(yield* orchestrator.getThreadEventSequence(threadId), sequence); + + const pause = (yield* orchestrator.getThreadProjection(threadId)).runs.find( + (run) => run.userMessageId === MessageId.make(`${threadId}:pause`), + ); + assert.deepEqual(pause?.modelSelection, modelSelection); + }), + ); + it.effect("keeps the queue after a user interrupts the active run", () => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; @@ -6917,7 +8154,7 @@ it.layer(layerSharedApplicationDataPlaneTest)("pending provider interruption", ( ["user_message", "run_interrupt_request", "run_interrupt_result"], ); assert.deepEqual(interrupted.providerTurns, []); - assert.isFalse(yield* effectWorker.runOnce); + assert.isFalse(yield* effectWorker.runOnce()); }), ); }); @@ -8688,6 +9925,12 @@ it.layer(layerTest)("usage-limit recovery", (it) => { ), [run.id, resumed.runs.at(-1)!.id], ); + assert.deepEqual( + resumed.turnItems.flatMap((item) => + item.type === "system_notice" && item.title === "Server restarted" ? [item.runId] : [], + ), + scenario === "resume" ? [] : [continuation.id], + ); assert.deepEqual( resumed.runs.map((row) => row.status), scenario === "resume" ? ["failed", "cancelled", "starting"] : ["failed", "cancelled"], diff --git a/apps/server/src/orchestration-v2/runtimeLayer.ts b/apps/server/src/orchestration-v2/runtimeLayer.ts index 682c1574ff97..4149a292ab5f 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.ts @@ -10,6 +10,7 @@ import * as McpAppRequests from "../mcpApps/McpAppRequests.ts"; import * as ProviderSessionRuntime from "../persistence/ProviderSessionRuntime.ts"; import * as TextGeneration from "../textGeneration/TextGeneration.ts"; import * as ProviderAuthService from "../provider/ProviderAuthService.ts"; +import * as ProviderInstallationGate from "../provider/ProviderInstallationGate.ts"; import * as AgentSessionImporter from "../project/AgentSessionImporter.ts"; import * as AgentSessionScanner from "../project/AgentSessionScanner.ts"; import * as ProjectService from "../project/ProjectService.ts"; @@ -138,6 +139,7 @@ const layerProviderSessionManagerProvided = ProviderSessionManager.layer.pipe( IdAllocator.layer, layerProviderEventIngestorProvided, ProjectionStore.layer, + ProviderInstallationGate.layer, ), ), ); @@ -170,6 +172,7 @@ const layerProviderTurnStartServiceProvided = ProviderTurnStartService.layer.pip layerProviderAuthServiceProvided, layerRunExecutionServiceProvided, layerRuntimePolicyProvided, + TurnItemPositionStore.layer, ), ), ); @@ -232,15 +235,21 @@ const layerOrchestratorProvided = Orchestrator.layer.pipe( ), ); +// One scanner serves the wizard's scan and imports, so imports can reuse its candidates. +const layerAgentSessionScannerProvided = AgentSessionScanner.layer.pipe( + Layer.provide(ProviderSessionRuntime.layer), +); const layerAgentSessionImporterProvided = AgentSessionImporter.layer.pipe( Layer.provide( Layer.mergeAll( - AgentSessionScanner.layer, + layerAgentSessionScannerProvided, layerProjectService, layerOrchestratorProvided, layerEventSink, IdAllocator.layer, ProviderSessionRuntime.layer, + ProviderAdapterRegistry.layerFromProviderInstanceRegistry, + ThreadCommandExecutor.layer, ), ), ); @@ -309,7 +318,11 @@ const layerProviderContinuationWorkerProvided = ProviderContinuationService.laye ); const layerThreadTitleRegenerationProvided = ThreadTitleRegenerationService.layer.pipe( Layer.provide( - Layer.mergeAll(layerThreadManagementProvided, ProjectStore.layer, TextGeneration.layer), + Layer.mergeAll( + layerThreadManagementProvided, + ProjectStore.layer, + TextGeneration.layer.pipe(Layer.provide(ProviderInstallationGate.layer)), + ), ), ); const layerEffectExecutorProvided = EffectWorker.layerExecutor.pipe( @@ -387,6 +400,7 @@ export const layerProduction = Layer.mergeAll( ), ), layerProviderContinuationWorkerProvided, + layerAgentSessionScannerProvided, layerAgentSessionImporterProvided, EffectOutbox.layerPruneWorker.pipe(Layer.provide(EffectOutbox.layer)), ).pipe( diff --git a/apps/server/src/orchestration-v2/testkit/AcpRegistryReplayFixtures.integration.test.ts b/apps/server/src/orchestration-v2/testkit/AcpRegistryReplayFixtures.integration.test.ts index e0a9a233bfd9..31ac7d0db5d2 100644 --- a/apps/server/src/orchestration-v2/testkit/AcpRegistryReplayFixtures.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/AcpRegistryReplayFixtures.integration.test.ts @@ -164,7 +164,7 @@ describe("ACP registry replay fixtures", () => { // deadline, draining effects instead of polling the projection. for (let attempt = 0; attempt < 10; attempt++) { yield* worker.drain(); - const next = yield* worker.nextClaimableAt; + const next = yield* worker.nextClaimableAt(); if (Option.isNone(next)) break; const now = yield* DateTime.now; yield* TestClock.adjust( diff --git a/apps/server/src/orchestration-v2/testkit/OrchestratorReplayRecovery.integration.test.ts b/apps/server/src/orchestration-v2/testkit/OrchestratorReplayRecovery.integration.test.ts index acaa3d41baa5..e7d9808f8d37 100644 --- a/apps/server/src/orchestration-v2/testkit/OrchestratorReplayRecovery.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/OrchestratorReplayRecovery.integration.test.ts @@ -335,7 +335,6 @@ describe("orchestrator replay recovery", () => { assert.deepEqual(child?.thread.modelSelection, { instanceId: CLAUDE_MODEL_SELECTION.instanceId, model: "claude-sonnet-5", - options: [{ id: "effort", value: "high" }], }); const conversation = (child?.turnItems ?? []) diff --git a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts index 15ff142f59c2..5ce06f5e00cc 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts @@ -40,6 +40,7 @@ import * as ProviderAuthService from "../../provider/ProviderAuthService.ts"; import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests"; import * as ProviderContinuationService from "../ProviderContinuationService.ts"; import * as ProviderEventIngestor from "../ProviderEventIngestor.ts"; +import * as ProviderInstallationGate from "../../provider/ProviderInstallationGate.ts"; import * as ProviderInstanceRegistry from "../../provider/ProviderInstanceRegistry.ts"; import * as ProviderRuntimeRecoveryService from "../ProviderRuntimeRecoveryService.ts"; import * as ProviderSessionManager from "../ProviderSessionManager.ts"; @@ -203,6 +204,10 @@ export function layerWithRegistry( readonly runContinuationWorker?: boolean; readonly recoverOnStartup?: boolean; readonly continueThreadsAfterServerUpdate?: boolean; + /** Lets a test hold a projection read open to order concurrent dispatches. */ + readonly wrapProjectionStore?: ( + store: ProjectionStore.ProjectionStoreV2Shape, + ) => ProjectionStore.ProjectionStoreV2Shape; } = {}, ): Layer.Layer< | Orchestrator.OrchestratorV2 @@ -235,9 +240,19 @@ export function layerWithRegistry( ? {} : { continueThreadsAfterServerUpdate: options.continueThreadsAfterServerUpdate }), }).pipe(Layer.orDie); + const wrapProjectionStore = options.wrapProjectionStore; + const layerProjectionStore = + wrapProjectionStore === undefined + ? ProjectionStore.layer + : Layer.effect( + ProjectionStore.ProjectionStoreV2, + Effect.gen(function* () { + return wrapProjectionStore(yield* ProjectionStore.ProjectionStoreV2); + }), + ).pipe(Layer.provide(ProjectionStore.layer)); const layerStores = Layer.mergeAll( EventStore.layer, - ProjectionStore.layer, + layerProjectionStore, ProjectStore.layer, CommandReceiptStore.layer, EffectOutbox.layer, @@ -304,6 +319,7 @@ export function layerWithRegistry( McpSessionRegistryTestkit.layer, layerProviderEventIngestorProvided, layerStores, + ProviderInstallationGate.layer, ), ), ); @@ -409,6 +425,7 @@ export function layerWithRegistry( getThreadRecords: orchestrator.getThreadRecords, getThreadProjection: orchestrator.getThreadProjection, recoverDelegatedTask: orchestrator.recoverDelegatedTask, + cancelDelegatedTask: orchestrator.cancelDelegatedTask, delegatedTaskResultPending: orchestrator.delegatedTaskResultPending, }); }), diff --git a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts index 8af560a05134..f02524375a12 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts @@ -114,6 +114,7 @@ function makeTestAdapter(input: { readonly failure?: ReturnType; readonly failedRunOrdinals?: ReadonlySet; readonly interruptedRunOrdinals?: ReadonlySet; + readonly cancelledRunOrdinals?: ReadonlySet; readonly holdRunOrdinal?: number; readonly pendingBackgroundWork?: Ref.Ref; readonly subagentOnlyBackgroundWork?: boolean; @@ -295,7 +296,9 @@ function makeTestAdapter(input: { ? "failed" : input.interruptedRunOrdinals?.has(turnInput.runOrdinal) ? "interrupted" - : "completed"; + : input.cancelledRunOrdinals?.has(turnInput.runOrdinal) + ? "cancelled" + : "completed"; const response = input.responseByThreadId?.[turnInput.threadId]?.[turnInput.runOrdinal] ?? input.responseByRunOrdinal[turnInput.runOrdinal] ?? @@ -1332,6 +1335,20 @@ describe("orchestration v2 provider switching", () => { assert.isFalse(turns[1]?.nativeThreadHasTurns); assert.equal(turns[2]?.nativeThreadId, turns[1]?.nativeThreadId); assert.isTrue(turns[2]?.nativeThreadHasTurns); + + const projection = yield* orchestrator.getThreadProjection(threadId); + const run2 = projection.runs[1]!; + const handoff = projection.contextHandoffs.find( + (candidate) => candidate.targetRunId === run2.id, + ); + assert.deepEqual( + projection.turnItems + .filter((item) => item.runId === run2.id) + .toSorted((left, right) => left.ordinal - right.ordinal) + .slice(0, 2) + .map((item) => (item.type === "handoff" ? item.contextHandoffId : item.type)), + [handoff?.id, "user_message"], + ); }).pipe( Effect.provide( ProviderReplayHarness.layerWithRegistry( @@ -1587,6 +1604,8 @@ describe("orchestration v2 provider switching", () => { ["failed", true, true, true], ["interrupted", false, true, false], ["interrupted", true, false, true], + ["cancelled", false, false, false], + ["cancelled", true, true, false], ] as const ).map( ([status, queued, returning, native]) => @@ -1631,6 +1650,7 @@ describe("orchestration v2 provider switching", () => { interruptedRunOrdinals: new Set( status === "interrupted" ? [sourceOrdinal] : [], ), + cancelledRunOrdinals: new Set(status === "cancelled" ? [sourceOrdinal] : []), holdRunOrdinal: sourceOrdinal, holdFirstTurn: started, releaseFirstTurn: release, @@ -1644,7 +1664,7 @@ describe("orchestration v2 provider switching", () => { const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; const waitForRun = ( ordinal: number, - expectedStatus: "completed" | "failed" | "interrupted", + expectedStatus: "completed" | "failed" | "interrupted" | "cancelled", ) => orchestrator.streamStoredEvents.pipe( Stream.filter( diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/thread_fork_native/claude_transcript.ndjson b/apps/server/src/orchestration-v2/testkit/fixtures/thread_fork_native/claude_transcript.ndjson index 135911941dca..321ca5b5417b 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/thread_fork_native/claude_transcript.ndjson +++ b/apps/server/src/orchestration-v2/testkit/fixtures/thread_fork_native/claude_transcript.ndjson @@ -8,7 +8,7 @@ {"type":"emit_inbound","label":"rate_limit_event","frame":{"type":"rate_limit_event","rate_limit_info":{"status":"allowed"},"uuid":"84451d3f-e7ba-4d80-aea3-4fbc442f4b3b","session_id":"8d0dafab-45d3-45eb-abef-41b9e55d5ff9"}} {"type":"emit_inbound","label":"result","frame":{"type":"result","subtype":"success","is_error":false,"api_error_status":null,"duration_ms":2066,"duration_api_ms":1815,"num_turns":1,"result":"source fork seed ok","stop_reason":"end_turn","session_id":"8d0dafab-45d3-45eb-abef-41b9e55d5ff9","total_cost_usd":0.01172175,"usage":{"input_tokens":3,"cache_creation_input_tokens":2339,"cache_read_input_tokens":9455,"output_tokens":7,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":2339,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[{"input_tokens":3,"output_tokens":7,"cache_read_input_tokens":9455,"cache_creation_input_tokens":2339,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":2339},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-sonnet-4-6":{"inputTokens":3,"outputTokens":7,"cacheReadInputTokens":9455,"cacheCreationInputTokens":2339,"webSearchRequests":0,"costUSD":0.01172175,"contextWindow":200000,"maxOutputTokens":32000}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","uuid":"892cdc1f-e2d5-440d-ae57-ca27d32f63f1"}} {"type":"runtime_exit","status":"success"} -{"type":"expect_outbound","label":"session.fork","frame":{"type":"session.fork","sessionId":"8d0dafab-45d3-45eb-abef-41b9e55d5ff9","options":{"dir":"/tmp/claude-replay-thread_fork_native","upToMessageId":"68baa075-b7d4-4689-866b-96bd51e570c3"}}} +{"type":"expect_outbound","label":"session.fork","frame":{"type":"session.fork","sessionId":"8d0dafab-45d3-45eb-abef-41b9e55d5ff9","options":{"upToMessageId":"68baa075-b7d4-4689-866b-96bd51e570c3"}}} {"type":"emit_inbound","label":"session.forked","frame":{"type":"session.forked","sessionId":"7e7ad6ec-db4a-4510-85ff-59df35cf7fad"}} {"type":"expect_outbound","label":"query.open:fork","frame":{"type":"query.open","options":{"model":"claude-sonnet-4-6","tools":{"type":"preset","preset":"claude_code"},"permissionMode":"bypassPermissions","allowDangerouslySkipPermissions":true,"settings":{"showThinkingSummaries":true},"resumeSessionAt":"68baa075-b7d4-4689-866b-96bd51e570c3","forkSession":true,"resume":"8d0dafab-45d3-45eb-abef-41b9e55d5ff9","sessionId":"7e7ad6ec-db4a-4510-85ff-59df35cf7fad"}}} {"type":"expect_outbound","label":"prompt.offer:2","frame":{"type":"prompt.offer","message":{"type":"user","message":{"role":"user","content":"Respond with the following text: fork native ok"},"parent_tool_use_id":null}}} diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/thread_fork_native_fork_local_rollback/claude_transcript.ndjson b/apps/server/src/orchestration-v2/testkit/fixtures/thread_fork_native_fork_local_rollback/claude_transcript.ndjson index e1b60eaa069d..08242f67d515 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/thread_fork_native_fork_local_rollback/claude_transcript.ndjson +++ b/apps/server/src/orchestration-v2/testkit/fixtures/thread_fork_native_fork_local_rollback/claude_transcript.ndjson @@ -8,7 +8,7 @@ {"type":"emit_inbound","label":"rate_limit_event","frame":{"type":"rate_limit_event","rate_limit_info":{"status":"allowed","resetsAt":1779846000,"rateLimitType":"five_hour","overageStatus":"rejected","overageDisabledReason":"org_level_disabled","isUsingOverage":false},"uuid":"94e56646-7610-4a7a-8245-9301469861ae","session_id":"273fe17b-65fb-451f-ad99-ff19d323852d"}} {"type":"emit_inbound","label":"result","frame":{"type":"result","subtype":"success","is_error":false,"api_error_status":null,"duration_ms":2928,"duration_api_ms":1977,"num_turns":1,"result":"fork local source alpha","stop_reason":"end_turn","session_id":"273fe17b-65fb-451f-ad99-ff19d323852d","total_cost_usd":0.0036539999999999997,"usage":{"input_tokens":3,"cache_creation_input_tokens":0,"cache_read_input_tokens":11800,"output_tokens":7,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[{"input_tokens":3,"output_tokens":7,"cache_read_input_tokens":11800,"cache_creation_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-sonnet-4-6":{"inputTokens":3,"outputTokens":7,"cacheReadInputTokens":11800,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":0.0036539999999999997,"contextWindow":200000,"maxOutputTokens":32000}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","uuid":"684990a5-0d00-4928-a332-fdc2c66769c6"}} {"type":"runtime_exit","status":"success"} -{"type":"expect_outbound","label":"session.fork","frame":{"type":"session.fork","sessionId":"273fe17b-65fb-451f-ad99-ff19d323852d","options":{"dir":"/tmp/claude-replay-thread_fork_native_fork_local_rollback","upToMessageId":"7659a146-b729-47b6-aac2-8fc0c0981e92"}}} +{"type":"expect_outbound","label":"session.fork","frame":{"type":"session.fork","sessionId":"273fe17b-65fb-451f-ad99-ff19d323852d","options":{"upToMessageId":"7659a146-b729-47b6-aac2-8fc0c0981e92"}}} {"type":"emit_inbound","label":"session.forked","frame":{"type":"session.forked","sessionId":"c9784d72-7e4f-4994-8c21-2e7e670ff91c"}} {"type":"expect_outbound","label":"query.open:fork","frame":{"type":"query.open","options":{"model":"claude-sonnet-4-6","tools":{"type":"preset","preset":"claude_code"},"permissionMode":"bypassPermissions","allowDangerouslySkipPermissions":true,"settings":{"showThinkingSummaries":true},"resumeSessionAt":"7659a146-b729-47b6-aac2-8fc0c0981e92","forkSession":true,"resume":"273fe17b-65fb-451f-ad99-ff19d323852d","sessionId":"c9784d72-7e4f-4994-8c21-2e7e670ff91c"}}} {"type":"expect_outbound","label":"prompt.offer:fork-first","frame":{"type":"prompt.offer","message":{"type":"user","message":{"role":"user","content":"For this fork-local rollback fixture, respond with exactly: fork local first"},"parent_tool_use_id":null}}} diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/thread_fork_native_prior_turn/claude_transcript.ndjson b/apps/server/src/orchestration-v2/testkit/fixtures/thread_fork_native_prior_turn/claude_transcript.ndjson index d8a11c34c18c..362fb061f5d7 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/thread_fork_native_prior_turn/claude_transcript.ndjson +++ b/apps/server/src/orchestration-v2/testkit/fixtures/thread_fork_native_prior_turn/claude_transcript.ndjson @@ -12,7 +12,7 @@ {"type":"emit_inbound","label":"assistant","frame":{"type":"assistant","message":{"model":"claude-sonnet-4-6","id":"msg_0181K4CdzyVanM6Px26CQP7L","type":"message","role":"assistant","content":[{"type":"text","text":"fork boundary beta"}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":3,"cache_creation_input_tokens":23,"cache_read_input_tokens":11797,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":23},"output_tokens":1,"service_tier":"standard","inference_geo":"not_available"},"context_management":null},"parent_tool_use_id":null,"session_id":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99","uuid":"c517b74b-9537-4260-afcb-186c4056c0a4"}} {"type":"emit_inbound","label":"result","frame":{"type":"result","subtype":"success","is_error":false,"api_error_status":null,"duration_ms":2545,"duration_api_ms":3001,"num_turns":1,"result":"fork boundary beta","stop_reason":"end_turn","session_id":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99","total_cost_usd":0.01544235,"usage":{"input_tokens":3,"cache_creation_input_tokens":23,"cache_read_input_tokens":11797,"output_tokens":6,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":23,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[{"input_tokens":3,"output_tokens":6,"cache_read_input_tokens":11797,"cache_creation_input_tokens":23,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":23},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-sonnet-4-6":{"inputTokens":6,"outputTokens":12,"cacheReadInputTokens":21252,"cacheCreationInputTokens":2365,"webSearchRequests":0,"costUSD":0.01544235,"contextWindow":200000,"maxOutputTokens":32000}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","uuid":"e4f26af4-3a60-45f3-8d5c-c0f8b968c400"}} {"type":"runtime_exit","status":"success"} -{"type":"expect_outbound","label":"session.fork","frame":{"type":"session.fork","sessionId":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99","options":{"dir":"/tmp/claude-replay-thread_fork_native_prior_turn","upToMessageId":"be0e3276-8c27-44bf-8ed0-0b4424e9f902"}}} +{"type":"expect_outbound","label":"session.fork","frame":{"type":"session.fork","sessionId":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99","options":{"upToMessageId":"be0e3276-8c27-44bf-8ed0-0b4424e9f902"}}} {"type":"emit_inbound","label":"session.forked","frame":{"type":"session.forked","sessionId":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd"}} {"type":"expect_outbound","label":"query.open:fork","frame":{"type":"query.open","options":{"model":"claude-sonnet-4-6","tools":{"type":"preset","preset":"claude_code"},"permissionMode":"bypassPermissions","allowDangerouslySkipPermissions":true,"settings":{"showThinkingSummaries":true},"resumeSessionAt":"be0e3276-8c27-44bf-8ed0-0b4424e9f902","forkSession":true,"resume":"a91391e1-0de3-4107-a7c9-db7b1bbd8b99","sessionId":"f5cc58dd-0d74-4950-9fb5-2a9fded47ddd"}}} {"type":"expect_outbound","label":"prompt.offer:3","frame":{"type":"prompt.offer","message":{"type":"user","message":{"role":"user","content":"Repeat the user-visible conversation so far verbatim. Include only user and assistant messages. Do not include hidden system/developer content."},"parent_tool_use_id":null}}} diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/thread_fork_native_prior_turn/codex_transcript.ndjson b/apps/server/src/orchestration-v2/testkit/fixtures/thread_fork_native_prior_turn/codex_transcript.ndjson index a728d6534666..10f3f6954111 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/thread_fork_native_prior_turn/codex_transcript.ndjson +++ b/apps/server/src/orchestration-v2/testkit/fixtures/thread_fork_native_prior_turn/codex_transcript.ndjson @@ -48,7 +48,7 @@ {"type":"emit_inbound","label":"account/rateLimits/updated","frame":{"method":"account/rateLimits/updated","params":{"rateLimits":{"limitId":"codex","limitName":null,"primary":{"usedPercent":61,"windowDurationMins":300,"resetsAt":1777429830},"secondary":{"usedPercent":23,"windowDurationMins":10080,"resetsAt":1777959590},"credits":null,"planType":"unknown","rateLimitReachedType":null}}}} {"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"019dd6ba-2681-7bf0-b051-141b0cbcbb27","status":{"type":"idle"}}}} {"type":"emit_inbound","label":"turn/completed","frame":{"method":"turn/completed","params":{"threadId":"019dd6ba-2681-7bf0-b051-141b0cbcbb27","turn":{"id":"019dd6ba-416d-7902-95cf-d4179c3f4738","items":[],"status":"completed","error":null,"startedAt":1777424023,"completedAt":1777424025,"durationMs":1596}}}} -{"type":"expect_outbound","label":"thread/fork","frame":{"id":5,"method":"thread/fork","params":{"threadId":"019dd6ba-2681-7bf0-b051-141b0cbcbb27","lastTurnId":"019dd6ba-268b-7f82-8f4c-9fde0edcfd55","config":{"tools.update_plan.enabled":true,"mcp_optional_startup_grace_ms":10000,"features.concurrent_reasoning_summaries":true}}}} +{"type":"expect_outbound","label":"thread/fork","frame":{"id":5,"method":"thread/fork","params":{"threadId":"019dd6ba-2681-7bf0-b051-141b0cbcbb27","excludeTurns":true,"lastTurnId":"019dd6ba-268b-7f82-8f4c-9fde0edcfd55","config":{"tools.update_plan.enabled":true,"mcp_optional_startup_grace_ms":10000,"features.concurrent_reasoning_summaries":true}}}} {"type":"emit_inbound","label":"deprecationNotice","frame":{"method":"deprecationNotice","params":{"summary":"`[features].collab` is deprecated. Use `[features].multi_agent` instead.","details":"Enable it with `--enable multi_agent` or `[features].multi_agent` in config.toml. See https://developers.openai.com/codex/config-basic#feature-flags for details."}}} {"type":"emit_inbound","label":"warning","frame":{"method":"warning","params":{"threadId":"019dd6ba-47b7-7092-8688-9cf7fe5f6498","message":"Under-development features enabled: apply_patch_freeform. Under-development features are incomplete and may behave unpredictably. To suppress this warning, set `suppress_unstable_features_warning = true` in /home/replay-user/.codex/config.toml."}}} {"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"name":"uidotsh","status":"starting","error":null}}} diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/thread_merge_back_siblings/claude_transcript.ndjson b/apps/server/src/orchestration-v2/testkit/fixtures/thread_merge_back_siblings/claude_transcript.ndjson index fc6e8ed264e0..ddd97d4a4ece 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/thread_merge_back_siblings/claude_transcript.ndjson +++ b/apps/server/src/orchestration-v2/testkit/fixtures/thread_merge_back_siblings/claude_transcript.ndjson @@ -8,7 +8,7 @@ {"type":"emit_inbound","label":"rate_limit_event","frame":{"type":"rate_limit_event","rate_limit_info":{"status":"allowed"},"uuid":"04dac113-90a6-48fa-ab30-d02d34e76f76","session_id":"093127f0-d3a7-42f8-938e-b1162927a0ea"}} {"type":"emit_inbound","label":"result","frame":{"type":"result","subtype":"success","is_error":false,"api_error_status":null,"duration_ms":2354,"duration_api_ms":1892,"ttft_ms":1845,"num_turns":1,"result":"merge sibling source stored","stop_reason":"end_turn","session_id":"093127f0-d3a7-42f8-938e-b1162927a0ea","total_cost_usd":0.005823599999999999,"usage":{"input_tokens":3,"cache_creation_input_tokens":0,"cache_read_input_tokens":18982,"output_tokens":8,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":3,"output_tokens":8,"cache_read_input_tokens":18982,"cache_creation_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-sonnet-4-6":{"inputTokens":3,"outputTokens":8,"cacheReadInputTokens":18982,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":0.005823599999999999,"contextWindow":200000,"maxOutputTokens":32000}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","uuid":"148a66d3-6af8-42ec-95d9-51106ba6b80c"}} {"type":"runtime_exit","status":"success"} -{"type":"expect_outbound","label":"session.fork:1","frame":{"type":"session.fork","sessionId":"093127f0-d3a7-42f8-938e-b1162927a0ea","options":{"dir":"/tmp/claude-replay-thread_merge_back_siblings","upToMessageId":"1fc21da3-d746-4ed9-9a01-47a5bced6ad7"}}} +{"type":"expect_outbound","label":"session.fork:1","frame":{"type":"session.fork","sessionId":"093127f0-d3a7-42f8-938e-b1162927a0ea","options":{"upToMessageId":"1fc21da3-d746-4ed9-9a01-47a5bced6ad7"}}} {"type":"emit_inbound","label":"session.forked:1","frame":{"type":"session.forked","sessionId":"aeec447f-87e5-4874-8fe7-7f937f7c3311"}} {"type":"expect_outbound","label":"query.open:fork:1","frame":{"type":"query.open","options":{"model":"claude-sonnet-4-6","tools":{"type":"preset","preset":"claude_code"},"permissionMode":"bypassPermissions","allowDangerouslySkipPermissions":true,"settings":{"showThinkingSummaries":true},"resumeSessionAt":"1fc21da3-d746-4ed9-9a01-47a5bced6ad7","forkSession":true,"resume":"093127f0-d3a7-42f8-938e-b1162927a0ea","sessionId":"aeec447f-87e5-4874-8fe7-7f937f7c3311"}}} {"type":"expect_outbound","label":"prompt.offer:2","frame":{"type":"prompt.offer","message":{"type":"user","message":{"role":"user","content":"Remember the fork-local marker merge-sibling-first-6V2J. Respond with exactly: first merge sibling stored"},"parent_tool_use_id":null}}} @@ -19,7 +19,7 @@ {"type":"emit_inbound","label":"rate_limit_event","frame":{"type":"rate_limit_event","rate_limit_info":{"status":"allowed"},"uuid":"3b7a92ea-d3b5-47b8-94df-b214af9ccd75","session_id":"aeec447f-87e5-4874-8fe7-7f937f7c3311"}} {"type":"emit_inbound","label":"result","frame":{"type":"result","subtype":"success","is_error":false,"api_error_status":null,"duration_ms":2344,"duration_api_ms":1855,"ttft_ms":1992,"num_turns":1,"result":"first merge sibling stored","stop_reason":"end_turn","session_id":"aeec447f-87e5-4874-8fe7-7f937f7c3311","total_cost_usd":0.005865,"usage":{"input_tokens":3,"cache_creation_input_tokens":0,"cache_read_input_tokens":19120,"output_tokens":8,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":3,"output_tokens":8,"cache_read_input_tokens":19120,"cache_creation_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-sonnet-4-6":{"inputTokens":3,"outputTokens":8,"cacheReadInputTokens":19120,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":0.005865,"contextWindow":200000,"maxOutputTokens":32000}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","uuid":"64da0416-b03b-4e9e-9ae9-9c85eb8ac0ef"}} {"type":"runtime_exit","status":"success"} -{"type":"expect_outbound","label":"session.fork:2","frame":{"type":"session.fork","sessionId":"093127f0-d3a7-42f8-938e-b1162927a0ea","options":{"dir":"/tmp/claude-replay-thread_merge_back_siblings","upToMessageId":"1fc21da3-d746-4ed9-9a01-47a5bced6ad7"}}} +{"type":"expect_outbound","label":"session.fork:2","frame":{"type":"session.fork","sessionId":"093127f0-d3a7-42f8-938e-b1162927a0ea","options":{"upToMessageId":"1fc21da3-d746-4ed9-9a01-47a5bced6ad7"}}} {"type":"emit_inbound","label":"session.forked:2","frame":{"type":"session.forked","sessionId":"f85cf945-d406-4573-b2ae-0e799d9a9365"}} {"type":"expect_outbound","label":"query.open:fork:2","frame":{"type":"query.open","options":{"model":"claude-sonnet-4-6","tools":{"type":"preset","preset":"claude_code"},"permissionMode":"bypassPermissions","allowDangerouslySkipPermissions":true,"settings":{"showThinkingSummaries":true},"resumeSessionAt":"1fc21da3-d746-4ed9-9a01-47a5bced6ad7","forkSession":true,"resume":"093127f0-d3a7-42f8-938e-b1162927a0ea","sessionId":"f85cf945-d406-4573-b2ae-0e799d9a9365"}}} {"type":"expect_outbound","label":"prompt.offer:3","frame":{"type":"prompt.offer","message":{"type":"user","message":{"role":"user","content":"Remember the fork-local marker merge-sibling-second-9X5B. Respond with exactly: second merge sibling stored"},"parent_tool_use_id":null}}} diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/thread_merge_back_siblings/codex_transcript.ndjson b/apps/server/src/orchestration-v2/testkit/fixtures/thread_merge_back_siblings/codex_transcript.ndjson index 23768db91787..d5443c56aa12 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/thread_merge_back_siblings/codex_transcript.ndjson +++ b/apps/server/src/orchestration-v2/testkit/fixtures/thread_merge_back_siblings/codex_transcript.ndjson @@ -27,7 +27,7 @@ {"type":"emit_inbound","label":"thread/tokenUsage/updated","frame":{"method":"thread/tokenUsage/updated","params":{"threadId":"01a0d610-155f-7391-843e-120bf1475357","turnId":"01a0d610-1619-7d91-bae2-ec4a7b899193","tokenUsage":{"total":{"totalTokens":17752,"inputTokens":17744,"cachedInputTokens":14848,"cacheWriteInputTokens":0,"outputTokens":8,"reasoningOutputTokens":0},"last":{"totalTokens":17752,"inputTokens":17744,"cachedInputTokens":14848,"cacheWriteInputTokens":0,"outputTokens":8,"reasoningOutputTokens":0},"modelContextWindow":258400}},"emittedAtMs":1790297778003}} {"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"01a0d610-155f-7391-843e-120bf1475357","status":{"type":"idle"}},"emittedAtMs":1790297778011}} {"type":"emit_inbound","label":"turn/completed","frame":{"method":"turn/completed","params":{"threadId":"01a0d610-155f-7391-843e-120bf1475357","turn":{"id":"01a0d610-1619-7d91-bae2-ec4a7b899193","items":[{"type":"agentMessage","id":"msg_0c4d0b070a58ed52016ab5c6b1b0fc87d0a540ea4be4112eb0","text":"merge sibling source stored","phase":"final_answer","memoryCitation":null,"delivery":null,"questions":null}],"itemsView":"summary","status":"completed","error":null,"startedAt":1790297773,"completedAt":1790297778,"durationMs":4400}},"emittedAtMs":1790297778011}} -{"type":"expect_outbound","label":"thread/fork","frame":{"id":4,"method":"thread/fork","params":{"config":{"tools.update_plan.enabled":true,"mcp_optional_startup_grace_ms":10000,"features.concurrent_reasoning_summaries":true},"cwd":"","lastTurnId":"01a0d610-1619-7d91-bae2-ec4a7b899193","model":"gpt-6-sol","threadId":"01a0d610-155f-7391-843e-120bf1475357"}}} +{"type":"expect_outbound","label":"thread/fork","frame":{"id":4,"method":"thread/fork","params":{"config":{"tools.update_plan.enabled":true,"mcp_optional_startup_grace_ms":10000,"features.concurrent_reasoning_summaries":true},"cwd":"","excludeTurns":true,"lastTurnId":"01a0d610-1619-7d91-bae2-ec4a7b899193","model":"gpt-6-sol","threadId":"01a0d610-155f-7391-843e-120bf1475357"}}} {"type":"emit_inbound","label":"deprecationNotice","frame":{"method":"deprecationNotice","params":{"summary":"Full-history hydration is deprecated for paginated threads; use `excludeTurns: true`, then page with `thread/turns/list` and `thread/items/list`.","details":null},"emittedAtMs":1790297778016}} {"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"01a0d610-155f-7391-843e-120bf1475357","name":"xcodebuildmcp","status":"ready","error":null,"failureReason":null},"emittedAtMs":1790297778263}} {"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"01a0d610-2789-7bb3-a492-ad7fa8a7db9c","name":"codex_apps","status":"starting","error":null,"failureReason":null},"emittedAtMs":1790297778313}} @@ -52,7 +52,7 @@ {"type":"emit_inbound","label":"thread/tokenUsage/updated","frame":{"method":"thread/tokenUsage/updated","params":{"threadId":"01a0d610-2789-7bb3-a492-ad7fa8a7db9c","turnId":"01a0d610-28ad-7071-82a1-f55ea8178ae8","tokenUsage":{"total":{"totalTokens":39723,"inputTokens":39707,"cachedInputTokens":26624,"cacheWriteInputTokens":0,"outputTokens":16,"reasoningOutputTokens":0},"last":{"totalTokens":21971,"inputTokens":21963,"cachedInputTokens":11776,"cacheWriteInputTokens":0,"outputTokens":8,"reasoningOutputTokens":0},"modelContextWindow":258400}},"emittedAtMs":1790297782125}} {"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"01a0d610-2789-7bb3-a492-ad7fa8a7db9c","status":{"type":"idle"}},"emittedAtMs":1790297782161}} {"type":"emit_inbound","label":"turn/completed","frame":{"method":"turn/completed","params":{"threadId":"01a0d610-2789-7bb3-a492-ad7fa8a7db9c","turn":{"id":"01a0d610-28ad-7071-82a1-f55ea8178ae8","items":[{"type":"agentMessage","id":"msg_0c4d0b070a58ed52016ab5c6b5b44c87d08e7e5612c5312499","text":"first merge sibling stored","phase":"final_answer","memoryCitation":null,"delivery":null,"questions":null}],"itemsView":"summary","status":"completed","error":null,"startedAt":1790297778,"completedAt":1790297782,"durationMs":3783}},"emittedAtMs":1790297782161}} -{"type":"expect_outbound","label":"thread/fork","frame":{"id":6,"method":"thread/fork","params":{"config":{"tools.update_plan.enabled":true,"mcp_optional_startup_grace_ms":10000,"features.concurrent_reasoning_summaries":true},"cwd":"","lastTurnId":"01a0d610-1619-7d91-bae2-ec4a7b899193","model":"gpt-6-sol","threadId":"01a0d610-155f-7391-843e-120bf1475357"}}} +{"type":"expect_outbound","label":"thread/fork","frame":{"id":6,"method":"thread/fork","params":{"config":{"tools.update_plan.enabled":true,"mcp_optional_startup_grace_ms":10000,"features.concurrent_reasoning_summaries":true},"cwd":"","excludeTurns":true,"lastTurnId":"01a0d610-1619-7d91-bae2-ec4a7b899193","model":"gpt-6-sol","threadId":"01a0d610-155f-7391-843e-120bf1475357"}}} {"type":"emit_inbound","label":"deprecationNotice","frame":{"method":"deprecationNotice","params":{"summary":"Full-history hydration is deprecated for paginated threads; use `excludeTurns: true`, then page with `thread/turns/list` and `thread/items/list`.","details":null},"emittedAtMs":1790297782170}} {"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"01a0d610-37d1-7ba1-918f-136cfdbd9bfd","name":"xcodebuildmcp","status":"starting","error":null,"failureReason":null},"emittedAtMs":1790297782559}} {"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"01a0d610-37d1-7ba1-918f-136cfdbd9bfd","name":"codex_apps","status":"starting","error":null,"failureReason":null},"emittedAtMs":1790297782559}} diff --git a/apps/server/src/orchestration-v2/threadHistoryPaging.ts b/apps/server/src/orchestration-v2/threadHistoryPaging.ts index 8bc794ce36c4..37b653c451fc 100644 --- a/apps/server/src/orchestration-v2/threadHistoryPaging.ts +++ b/apps/server/src/orchestration-v2/threadHistoryPaging.ts @@ -183,7 +183,7 @@ function isThreadHistoryTurnStart(item: HistoryRow["item"]): boolean { ); } -const DEFERRED_WORK_ITEM_TYPES = [ +export const DEFERRED_WORK_ITEM_TYPES = [ "command_execution", "reasoning", "file_change", diff --git a/apps/server/src/os-jank.test.ts b/apps/server/src/os-jank.test.ts index aaac95bc3e88..346c711ba634 100644 --- a/apps/server/src/os-jank.test.ts +++ b/apps/server/src/os-jank.test.ts @@ -64,6 +64,59 @@ effectIt.effect("hydrates PATH when the desktop handoff is absent", () => ), ); +const timedOut = (file: string) => + Object.assign(new Error(`spawnSync ${file} ETIMEDOUT`), { code: "ETIMEDOUT" }); + +effectIt.effect("waits longer for a login shell that times out at startup", () => + Effect.acquireUseRelease( + Effect.sync(() => + vi.spyOn(NodeChildProcess, "execFileSync").mockImplementation((file, _args, options) => { + // This profile takes 8 seconds while the machine is under load. + if ((options?.timeout ?? 0) < 8_000) throw timedOut(String(file)); + return "__T3CODE_ENV_PATH_START__\n/opt/tools/bin\n__T3CODE_ENV_PATH_END__\n"; + }), + ), + () => + Effect.gen(function* () { + const env: NodeJS.ProcessEnv = { SHELL: "/bin/zsh", PATH: "/usr/bin" }; + + yield* fixPath().pipe( + Effect.provideService(HostProcess.Platform, "linux"), + Effect.provideService(HostProcess.Environment, env), + Effect.provide(NodeServices.layer), + ); + + assert.equal(env.PATH, "/opt/tools/bin:/usr/bin"); + }), + (probe) => Effect.sync(() => probe.mockRestore()), + ), +); + +effectIt.effect("waits longer for only one hung login shell", () => + Effect.acquireUseRelease( + Effect.sync(() => + vi.spyOn(NodeChildProcess, "execFileSync").mockImplementation((file) => { + throw timedOut(String(file)); + }), + ), + (probe) => + Effect.gen(function* () { + const env: NodeJS.ProcessEnv = { SHELL: "/bin/zsh", PATH: "/usr/bin" }; + + yield* fixPath().pipe( + Effect.provideService(HostProcess.Platform, "linux"), + Effect.provideService(HostProcess.Environment, env), + Effect.provide(NodeServices.layer), + ); + + const timeouts = probe.mock.calls.map(([, , options]) => options?.timeout); + assert.equal(timeouts.filter((timeout) => (timeout ?? 0) > 5_000).length, 1); + assert.equal(env.PATH, "/usr/bin"); + }), + (probe) => Effect.sync(() => probe.mockRestore()), + ), +); + effectIt.effect("keeps mise's PATH record with the login shell PATH", () => Effect.acquireUseRelease( Effect.sync(() => diff --git a/apps/server/src/os-jank.ts b/apps/server/src/os-jank.ts index dece291aa1ca..e7fcdcd68719 100644 --- a/apps/server/src/os-jank.ts +++ b/apps/server/src/os-jank.ts @@ -10,6 +10,8 @@ import { import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; +// @effect-diagnostics-next-line nodeBuiltinImport:off -- the shared login shell probe runs synchronously through execFileSync. +import * as NodeChildProcess from "node:child_process"; import * as NodeOS from "node:os"; function logPathHydrationWarning(message: string, error?: unknown): void { @@ -18,16 +20,46 @@ function logPathHydrationWarning(message: string, error?: unknown): void { ); } +// mise's record of the PATH entries it added. It must travel with the +// shell's PATH, or `mise x` can rebuild PATH behind a same-named wrapper +// (`exec mise x claude -- claude`) that then re-runs itself forever. +const LOGIN_SHELL_ENVIRONMENT_NAMES = ["PATH", "__MISE_DIFF", "JAVA_HOME"]; +const SLOW_LOGIN_SHELL_TIMEOUT_MS = 20_000; + +function readSlowLoginShellEnvironment(shell: string): Partial> { + return readEnvironmentFromLoginShell( + shell, + LOGIN_SHELL_ENVIRONMENT_NAMES, + (file, args, options) => + NodeChildProcess.execFileSync(file, args, { + ...options, + timeout: SLOW_LOGIN_SHELL_TIMEOUT_MS, + }), + ); +} + function hydratePosixPath(env: NodeJS.ProcessEnv, platform: NodeJS.Platform): void { let shellEnvironment: Partial> = {}; + let waitedForSlowShell = false; for (const shell of listLoginShellCandidates(platform, env.SHELL)) { try { - // mise's record of the PATH entries it added. It must travel with the - // shell's PATH, or `mise x` can rebuild PATH behind a same-named wrapper - // (`exec mise x claude -- claude`) that then re-runs itself forever. - shellEnvironment = readEnvironmentFromLoginShell(shell, ["PATH", "__MISE_DIFF", "JAVA_HOME"]); + shellEnvironment = readEnvironmentFromLoginShell(shell, LOGIN_SHELL_ENVIRONMENT_NAMES); } catch (error) { logPathHydrationWarning(`Failed to read PATH from login shell ${shell}.`, error); + // A profile can be slow only while the machine boots. Wait for it once, + // here: providers copy this env when they are built, so a later read + // would never reach them. + if (!waitedForSlowShell && (error as NodeJS.ErrnoException | null)?.code === "ETIMEDOUT") { + waitedForSlowShell = true; + try { + shellEnvironment = readSlowLoginShellEnvironment(shell); + } catch (retryError) { + logPathHydrationWarning( + `Failed to read PATH from slow login shell ${shell}.`, + retryError, + ); + } + } } if (shellEnvironment.PATH) break; diff --git a/apps/server/src/peer/RemoteDelegation.ts b/apps/server/src/peer/RemoteDelegation.ts index fdd26137534f..858f6b087127 100644 --- a/apps/server/src/peer/RemoteDelegation.ts +++ b/apps/server/src/peer/RemoteDelegation.ts @@ -6,6 +6,8 @@ import { type OrchestratorMcpDelegateTaskInput, type OrchestratorMcpTaskCancelResult, type OrchestratorMcpThreadRun, + type OrchestratorMcpThreadSendInput, + type OrchestratorMcpThreadSendResult, type OrchestratorMcpThreadTimelineItem, type OrchestrationV2RemoteTaskChild, type OrchestrationV2Subagent, @@ -17,6 +19,7 @@ import { } from "@t3tools/contracts"; import * as Context from "effect/Context"; import * as SecureRandom from "effect/crypto/SecureRandom"; +import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; @@ -34,6 +37,7 @@ import { resolveRuntimeMode, taskPrompt, } from "../mcp/delegatedTaskModes.ts"; +import { followUpOwnerRun } from "../orchestration-v2/CommandPolicy.ts"; import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; import * as ThreadManagement from "../orchestration-v2/ThreadManagementService.ts"; // Coder: the identity service lives in `coderEnvironment.ts`; `environment/` is not carried. @@ -80,6 +84,18 @@ export class RemoteDelegation extends Context.Service< }; }, ) => Effect.Effect<{ readonly taskId: NodeId }, OrchestratorMcpFailure>; + /** + * A `t3_thread_send` there that started or queued a run on the thread of + * one of this thread's tasks that has its result: like a follow-up to a + * finished child here, that run is a new task, whose id the send returns. + * Any other send is returned as it came. + */ + readonly followUp: ( + scope: McpInvocationContext.McpInvocationScope, + environmentId: EnvironmentId, + request: Pick, + sent: OrchestratorMcpThreadSendResult, + ) => Effect.Effect; /** * Stops the task's thread there, then completes it here as cancelled. A * task that already has its result keeps it; its thread there is still @@ -360,6 +376,90 @@ const make = Effect.gen(function* () { }, ); + const followUp: RemoteDelegation["Service"]["followUp"] = Effect.fn("RemoteDelegation.followUp")( + function* (scope, environmentId, request, sent) { + // A steer joins a run that is already some task's, or no one's. + if (scope.thread === undefined || (sent.delivery !== "started" && sent.delivery !== "queued")) + return sent; + const parentThreadId = scope.thread.threadId; + const parent = yield* threads + .getThreadRecords(parentThreadId, ["runs", "subagents"]) + .pipe(Effect.orElseSucceed(() => undefined)); + const isThere = (task: OrchestrationV2Subagent) => + task.remoteChild?.environmentId === environmentId && + task.remoteChild.threadId === sent.threadId; + const finished = parent?.subagents.find((task) => isThere(task) && task.result !== null); + // Bound as a follow-up here is: a background caller's foreground run may have ended. + const parentRun = parent === undefined ? null : followUpOwnerRun(parent.runs); + if ( + finished?.remoteChild === undefined || + finished.model === null || + parentRun === null || + parentRun.rootNodeId === null + ) + return sent; + // The message id there is the send's, so a retry that reached the same send records once. + const commandId = CommandId.make( + `command:remote-followup:${encodeURIComponent(parentThreadId)}:${encodeURIComponent(environmentId)}:${encodeURIComponent(sent.messageId)}`, + ); + const recorded = yield* threads + .dispatch({ + type: "delegated_task.remote.request", + commandId, + parentThreadId, + parentRunId: parentRun.id, + parentNodeId: parentRun.rootNodeId, + task: request.message, + ...(finished.title === null ? {} : { title: finished.title }), + driver: finished.driver, + modelSelection: { instanceId: finished.providerInstanceId, model: finished.model }, + remoteChild: { + environmentId, + threadId: sent.threadId, + label: finished.remoteChild.label, + runId: sent.runId, + }, + completionWake: "always", + }) + .pipe( + Effect.tapError((cause) => + Effect.logWarning("remote-delegation.followup-record-failed", { + parentThreadId, + cause, + }), + ), + Effect.mapError( + () => + new OrchestratorMcpFailure({ + code: "orchestration_error", + message: + request.clientRequestId === undefined + ? "The message was sent, but its task was not recorded; read the thread for its answer instead of sending it again." + : "The message was sent, but its task was not recorded; send it again with the same clientRequestId to record it.", + }), + ), + ); + const taskEvent = recorded.storedEvents.find( + (stored) => stored.event.type === "subagent.updated", + ); + const taskId = + taskEvent?.event.type === "subagent.updated" + ? taskEvent.event.payload.id + : // A replayed request recorded the task the first time. + yield* threads.getThreadRecords(parentThreadId, ["subagents"]).pipe( + Effect.map( + ({ subagents }) => + subagents.find((task) => isThere(task) && task.remoteChild?.runId === sent.runId) + ?.id, + ), + Effect.orElseSucceed(() => undefined), + ); + if (taskId === undefined) return sent; + yield* followOne(parentThreadId, taskId).pipe(Effect.forkIn(followers)); + return { ...sent, taskId }; + }, + ); + /** * Completes `task` here with its thread there's final state. Each attempt is its own * command, so a failed one never blocks the next; a report refused because the task already @@ -564,7 +664,9 @@ const make = Effect.gen(function* () { ((task.status === "cancelled" || task.status === "interrupted") && remote.stoppedThere !== true) ) { - const key = remote.stopCount === undefined ? "stop" : `stop-${remote.stopCount}`; + const count = remote.stopCount === undefined ? "stop" : `stop-${remote.stopCount}`; + // Named for its run too: a follow-up task shares the thread there and counts its own stops. + const key = remote.runId === undefined ? count : `${count}:${remote.runId}`; yield* stopThere(scope, remote, key).pipe( // Its thread there now runs above what this side may touch, say // because the parent here was lowered since; no retry changes that. @@ -633,10 +735,31 @@ const make = Effect.gen(function* () { return yield* settled(parentThreadId, taskId); } if (!isTerminal(waited.status)) return false; - const resultRun = waited.resultRun ?? { - runId: remote.runId ?? waited.runId, - status: waited.status, - }; + // A later run's answer there may be another open follow-up's, or its restart, which no run id + // here names; concurrent sends may record here out of their order there. Then this task's + // answer is its own run's latest restart, the one waited on. + const sharedThread = + waited.resultRun !== undefined && + waited.resultRun.runId !== waited.runId && + (yield* threads.getThreadRecords(parentThreadId, ["subagents"]).pipe( + Effect.map(({ subagents }) => { + const recordedAt = subagents.find((task) => task.id === taskId)?.startedAt ?? null; + return subagents.some( + (other) => + other.id !== taskId && + other.remoteChild?.environmentId === remote.environmentId && + other.remoteChild.threadId === remote.threadId && + // One that settled before this was recorded ran before it there. + (recordedAt === null || + other.completedAt === null || + DateTime.isGreaterThan(other.completedAt, recordedAt)), + ); + }), + )); + const resultRun = + waited.resultRun !== undefined && !sharedThread + ? waited.resultRun + : { runId: waited.runId, status: waited.status }; const result = yield* lastReply(scope, remote, resultRun.runId, resultRun.status === "failed"); yield* complete( parentThreadId, @@ -794,7 +917,7 @@ const make = Effect.gen(function* () { }).pipe(Effect.ignoreCause({ log: true })), ); - return RemoteDelegation.of({ delegate, cancel, start }); + return RemoteDelegation.of({ delegate, followUp, cancel, start }); }); const isTerminal = (status: string) => diff --git a/apps/server/src/persistence/DatabaseMaintenance.test.ts b/apps/server/src/persistence/DatabaseMaintenance.test.ts index 864d9f4da1bc..533a31b40446 100644 --- a/apps/server/src/persistence/DatabaseMaintenance.test.ts +++ b/apps/server/src/persistence/DatabaseMaintenance.test.ts @@ -227,18 +227,22 @@ it.layer(DatabaseMaintenance.layer.pipe(Layer.provideMerge(NodeServices.layer))) const clock = yield* Clock.Clock; const pauses: number[] = []; let activeNanos = 0n; - const result = yield* maintenance.compact(dbPath).pipe( - Effect.provideService(Clock.Clock, { - ...clock, - // Exceed the background budget whenever the compactor checks elapsed time. - monotonicTimeNanos: Effect.sync(() => (activeNanos += 30_000_000n)), - sleep: (duration) => - Effect.sync(() => { - pauses.push(Duration.toMillis(duration)); - }), - }), - ); + const steps: string[] = []; + const result = yield* maintenance + .compact(dbPath, (step) => Effect.sync(() => steps.push(step))) + .pipe( + Effect.provideService(Clock.Clock, { + ...clock, + // Exceed the background budget whenever the compactor checks elapsed time. + monotonicTimeNanos: Effect.sync(() => (activeNanos += 30_000_000n)), + sleep: (duration) => + Effect.sync(() => { + pauses.push(Duration.toMillis(duration)); + }), + }), + ); assert.deepEqual(pauses, [], "offline doctor must not throttle compaction"); + assert.deepEqual(steps, ["prune-events", "vacuum", "rebuild-search", "checkpoint"]); assert.equal(result.deletedReceiptCount, 51); assert.isBelow(result.afterBytes, result.beforeBytes - 3 * 1024 * 1024); yield* Effect.gen(function* () { diff --git a/apps/server/src/persistence/DatabaseMaintenance.ts b/apps/server/src/persistence/DatabaseMaintenance.ts index d192f6f44596..79580761090d 100644 --- a/apps/server/src/persistence/DatabaseMaintenance.ts +++ b/apps/server/src/persistence/DatabaseMaintenance.ts @@ -71,6 +71,8 @@ export class DiskSpaceCheckError extends Schema.TaggedError } } +export type DatabaseCompactStep = "prune-events" | "vacuum" | "rebuild-search" | "checkpoint"; + export class DatabaseMaintenance extends Context.Service< DatabaseMaintenance, { @@ -84,8 +86,14 @@ export class DatabaseMaintenance extends Context.Service< }, DatabaseMissingError | DatabaseInspectError >; - /** Compacts history, then VACUUMs into incremental auto_vacuum. Refuses while a server has it open. */ - readonly compact: (databasePath: string) => Effect.Effect< + /** + * Compacts history, then VACUUMs into incremental auto_vacuum. Refuses while a server has it open. + * Calls `onStep` as each step starts; VACUUM time grows with the whole database size. + */ + readonly compact: ( + databasePath: string, + onStep?: (step: DatabaseCompactStep) => Effect.Effect, + ) => Effect.Effect< { readonly beforeBytes: number; readonly afterBytes: number; @@ -171,18 +179,25 @@ const make = Effect.gen(function* () { ); const compact = Effect.fn("DatabaseMaintenance.compact")( - function* (databasePath: string) { + function* ( + databasePath: string, + onStep: (step: DatabaseCompactStep) => Effect.Effect = () => Effect.void, + ) { yield* locate(databasePath); const beforeBytes = Number((yield* fs.stat(databasePath)).size); const result = yield* Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; yield* lockExclusively(databasePath); const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; + yield* onStep("prune-events"); const summary = yield* maintenance.pruneEventStore(); + yield* onStep("vacuum"); // VACUUM is required to enable incremental auto_vacuum on existing databases. yield* sql`PRAGMA auto_vacuum = INCREMENTAL`; yield* sql`VACUUM`; + yield* onStep("rebuild-search"); yield* rebuildMessageSearch(sql); + yield* onStep("checkpoint"); yield* sql`PRAGMA wal_checkpoint(TRUNCATE)`; return summary; }).pipe( diff --git a/apps/server/src/persistence/DatabaseStorage.test.ts b/apps/server/src/persistence/DatabaseStorage.test.ts index ddd5001d3366..c8b43e1281e3 100644 --- a/apps/server/src/persistence/DatabaseStorage.test.ts +++ b/apps/server/src/persistence/DatabaseStorage.test.ts @@ -7,6 +7,7 @@ import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as Option from "effect/Option"; import * as Stream from "effect/Stream"; +import * as Tracer from "effect/Tracer"; import * as TestClock from "effect/testing/TestClock"; import * as SqlClient from "effect/sql/SqlClient"; @@ -44,9 +45,26 @@ it.effect("a new database returns freed pages online without losing data", () => yield* sql`CREATE TABLE retained (value TEXT)`; yield* sql`INSERT INTO retained VALUES ('keep')`; yield* sql`CREATE TABLE discarded (value BLOB)`; - yield* sql`INSERT INTO discarded VALUES (zeroblob(1048576))`; + yield* sql`INSERT INTO discarded VALUES (zeroblob(6291456))`; yield* sql`DROP TABLE discarded`; - assert.isAbove(yield* freePages, 0); + const before = yield* freePages; + assert.isAbove(before, 1024); + const spans: Array = []; + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options); + spans.push(span); + return span; + }, + }); + yield* storage.reclaim.pipe(Effect.withTracer(tracer), Effect.withTracerEnabled(true)); + assert.equal(before - (yield* freePages), 1024); + assert.lengthOf(spans, 1); + assert.deepEqual( + spans.map((span) => span.name), + ["DatabaseStorage.reclaim"], + ); + assert.equal(spans[0]?.attributes.get("db.reclaimed_pages"), 1024); yield* storage.reclaim; assert.equal(yield* freePages, 0); assert.deepEqual(yield* sql`SELECT value FROM retained`, [{ value: "keep" }]); diff --git a/apps/server/src/persistence/DatabaseStorage.ts b/apps/server/src/persistence/DatabaseStorage.ts index d2d400d00e9b..bece6ab8f2e0 100644 --- a/apps/server/src/persistence/DatabaseStorage.ts +++ b/apps/server/src/persistence/DatabaseStorage.ts @@ -114,18 +114,25 @@ const make = Effect.gen(function* () { Effect.catch((cause) => Effect.logWarning("Database storage inspection failed", { cause })), ); - const reclaim = Effect.gen(function* () { - if (!(yield* readLayout).incrementalVacuum) return; - // The driver steps once per statement; incremental_vacuum frees one page per step. Yield between pages. - for (let page = 0; page < RECLAIM_PAGES_PER_PASS; page++) { - const free = yield* sql<{ readonly freelist_count: number }>`PRAGMA freelist_count`; - if ((free[0]?.freelist_count ?? 0) === 0) break; - yield* sql`PRAGMA incremental_vacuum(1)`; - yield* Effect.yieldNow; - } - }).pipe( - Effect.catch((cause) => Effect.logWarning("Database space reclamation failed", { cause })), - ); + const reclaim = Effect.suspend(() => { + let page = 0; + return Effect.gen(function* () { + if (!(yield* readLayout).incrementalVacuum) return; + // The driver steps once per statement; incremental_vacuum frees one page per step. Yield between pages. + for (; page < RECLAIM_PAGES_PER_PASS; page++) { + const free = yield* sql<{ readonly freelist_count: number }>`PRAGMA freelist_count`; + if ((free[0]?.freelist_count ?? 0) === 0) break; + yield* sql`PRAGMA incremental_vacuum(1)`; + yield* Effect.yieldNow; + } + }).pipe( + Effect.ensuring(Effect.suspend(() => Effect.annotateCurrentSpan("db.reclaimed_pages", page))), + Effect.catch((cause) => Effect.logWarning("Database space reclamation failed", { cause })), + // The pass owns the span; tracing each page's two PRAGMAs floods the server trace. + Effect.withTracerEnabled(false), + Effect.withSpan("DatabaseStorage.reclaim"), + ); + }); // Even a pass that deletes nothing scans the event store. Persist completion // so restarting does not trigger another scan before the six-hour interval. diff --git a/apps/server/src/persistence/Migrations.test.ts b/apps/server/src/persistence/Migrations.test.ts index 5867647f5548..dc42559afa3c 100644 --- a/apps/server/src/persistence/Migrations.test.ts +++ b/apps/server/src/persistence/Migrations.test.ts @@ -148,6 +148,20 @@ describe("runMigrations", () => { }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))), ); + it.effect("reapplies peer link migrations whose ledger rows are missing", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* runMigrations({ toMigrationInclusive: 72 }); + yield* sql`DELETE FROM effect_sql_migrations WHERE migration_id BETWEEN 70 AND 72`; + + assert.deepStrictEqual( + yield* runMigrations(), + migrationManifest.filter(([id]) => id >= 70), + ); + assert.deepStrictEqual(yield* history, migrationManifest); + }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))), + ); + it.effect("still starts after a newer build recorded a later V2 migration", () => Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index dce795eb89b0..54bbc83fca78 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -76,6 +76,8 @@ import Migration0077 from "./Migrations/077_RepairBooleanSettledOverride.ts"; import Migration0078 from "./Migrations/078_DropOrchestrationEventCorrelationIndex.ts"; import Migration0079 from "./Migrations/079_DelegatedRecoveryIndexes.ts"; import Migration0080 from "./Migrations/080_ScheduledTaskThreadMode.ts"; +import Migration0081 from "./Migrations/081_ProjectionMessageSearchText.ts"; +import Migration0082 from "./Migrations/082_TurnItemConversationIndex.ts"; // Coder: T3 Coder has no auth sessions or pairing. Upstream's auth migrations keep // their IDs so later upstream migrations stay aligned, but they create nothing. @@ -166,6 +168,8 @@ export const migrationEntries = [ [78, "DropOrchestrationEventCorrelationIndex", Migration0078], [79, "DelegatedRecoveryIndexes", Migration0079], [80, "ScheduledTaskThreadMode", Migration0080], + [81, "ProjectionMessageSearchText", Migration0081], + [82, "TurnItemConversationIndex", Migration0082], ] as const; export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const); @@ -214,6 +218,7 @@ const assertV2MigrationsCanRun = Effect.fn("assertV2MigrationsCanRun")(function* }); }); +// Coder: no `hasPendingMigrations` (#26763); only the cut `t3` CLI asks before migrating. const run = Migrator.make({}); export interface RunMigrationsOptions { diff --git a/apps/server/src/persistence/Migrations/072_WaitingLinkRequestIndex.ts b/apps/server/src/persistence/Migrations/072_WaitingLinkRequestIndex.ts index e025a00d49ba..feaafe620590 100644 --- a/apps/server/src/persistence/Migrations/072_WaitingLinkRequestIndex.ts +++ b/apps/server/src/persistence/Migrations/072_WaitingLinkRequestIndex.ts @@ -6,7 +6,7 @@ export default Effect.gen(function* () { // Keep this predicate aligned with ProjectionStore's waiting link card query. yield* sql` - CREATE INDEX orchestration_v2_projection_turn_items_waiting_link_idx + CREATE INDEX IF NOT EXISTS orchestration_v2_projection_turn_items_waiting_link_idx ON orchestration_v2_projection_turn_items(thread_id, updated_at DESC, turn_item_id DESC) WHERE type = 'link_request' AND status = 'waiting' `; diff --git a/apps/server/src/persistence/Migrations/081_ProjectionMessageSearchText.test.ts b/apps/server/src/persistence/Migrations/081_ProjectionMessageSearchText.test.ts new file mode 100644 index 000000000000..84ca7f9de23a --- /dev/null +++ b/apps/server/src/persistence/Migrations/081_ProjectionMessageSearchText.test.ts @@ -0,0 +1,96 @@ +import { assert, it } from "@effect/vitest"; +import { EnvironmentId, MessageId, ThreadId } from "@t3tools/contracts"; +import { serializeAssistantCitation } from "@t3tools/shared/assistantCitations"; +import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +import { runMigrations } from "../Migrations.ts"; + +const createdAt = "2026-10-01T00:00:00.000Z"; +const quote = serializeAssistantCitation({ + version: 1, + environmentId: EnvironmentId.make("environment:local"), + threadId: ThreadId.make("thread:quoted"), + messageId: MessageId.make("quoted-answer"), + text: "A lighthouse guides ships safely through the darkness.", + comment: "what are you talking about", + start: 0, + end: 54, + prefix: "beacon01", + suffix: "harbor", +}); +const readable = + "A lighthouse guides ships safely through the darkness.\nComment: what are you talking about"; +const indexed = (sql: SqlClient.SqlClient, phrase: string) => + sql<{ readonly message_id: string }>` + SELECT messages.message_id + FROM orchestration_v2_message_search AS search + INNER JOIN orchestration_v2_projection_messages AS messages + ON messages.rowid = search.rowid + WHERE orchestration_v2_message_search MATCH ${phrase} + `.pipe(Effect.map((rows) => rows.map((row) => row.message_id))); + +it.layer(NodeSqliteClient.layer({ filename: ":memory:" }))( + "081_ProjectionMessageSearchText", + (it) => { + it.effect("backfills readable text only for messages whose citations read differently", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* runMigrations({ toMigrationInclusive: 80 }); + const insertMessage = (messageId: string, text: unknown) => sql` + INSERT INTO orchestration_v2_projection_messages ( + message_id, thread_id, role, streaming, created_at, updated_at, payload_json + ) VALUES ( + ${messageId}, 'thread:quote', 'user', 0, ${createdAt}, ${createdAt}, + ${JSON.stringify({ text })} + ) + `; + yield* insertMessage("quoted", `${quote} quiet silver lantern`); + yield* insertMessage("plain", "quiet silver lantern"); + yield* insertMessage("invalid", "[Assistant quote](t3-citation://v1/not-a-citation)"); + yield* insertMessage("unreadable", 20261001); + yield* insertMessage( + "accented", + serializeAssistantCitation({ + version: 1, + environmentId: EnvironmentId.make("environment:local"), + threadId: ThreadId.make("thread:quoted"), + messageId: MessageId.make("quoted-answer"), + text: "The kittiwake's résumé", + start: 0, + end: 22, + prefix: "", + suffix: "", + }), + ); + yield* Effect.forEach( + Array.from({ length: 501 }, (_, index) => `page-${String(index).padStart(3, "0")}`), + (messageId) => insertMessage(messageId, quote), + { discard: true }, + ); + + assert.deepEqual(yield* runMigrations({ toMigrationInclusive: 81 }), [ + [81, "ProjectionMessageSearchText"], + ]); + const rows = yield* sql<{ + readonly message_id: string; + readonly search_text: string | null; + }>`SELECT message_id, search_text FROM orchestration_v2_projection_messages`; + const searchText = new Map(rows.map((row) => [row.message_id, row.search_text])); + assert.equal(searchText.get("quoted"), `${readable} quiet silver lantern`); + assert.isNull(searchText.get("plain")); + assert.isNull(searchText.get("invalid")); + assert.isNull(searchText.get("unreadable")); + assert.equal(searchText.get("page-000"), readable); + assert.equal(searchText.get("page-500"), readable); + + // Messages indexed as stored are indexed again as they read. + assert.deepEqual(yield* indexed(sql, '"résumé"'), ["accented"]); + assert.deepEqual(yield* indexed(sql, `"kittiwake's"`), ["accented"]); + assert.deepEqual(yield* indexed(sql, '"beacon01"'), []); + assert.lengthOf(yield* indexed(sql, '"what are you talking about"'), 502); + }), + ); + }, +); diff --git a/apps/server/src/persistence/Migrations/081_ProjectionMessageSearchText.ts b/apps/server/src/persistence/Migrations/081_ProjectionMessageSearchText.ts new file mode 100644 index 000000000000..4bbcec4d6182 --- /dev/null +++ b/apps/server/src/persistence/Migrations/081_ProjectionMessageSearchText.ts @@ -0,0 +1,44 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +import { + readableSearchText, + recreateMessageSearchTriggers, +} from "../../orchestration-v2/messageSearchIndex.ts"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`ALTER TABLE orchestration_v2_projection_messages ADD COLUMN search_text TEXT`; + + // Migration 74's triggers index the stored text, where a quote's words are + // percent-encoded. Recreated now that search_text exists, they index it, and + // the backfill below re-indexes every message that reads differently. + yield* recreateMessageSearchTriggers(sql); + + // Only messages holding a citation link read differently. The raw check + // first skips parsing the JSON of every other message. + const pageSize = 500; + let afterMessageId = ""; + while (true) { + const rows = yield* sql<{ readonly message_id: string; readonly text: string }>` + SELECT message_id, json_extract(payload_json, '$.text') AS text + FROM orchestration_v2_projection_messages + WHERE message_id > ${afterMessageId} + AND payload_json LIKE '%t3-citation://%' + AND json_type(payload_json, '$.text') = 'text' + ORDER BY message_id + LIMIT ${pageSize} + `; + for (const row of rows) { + const searchText = readableSearchText(row.text); + if (searchText === null) continue; + yield* sql` + UPDATE orchestration_v2_projection_messages + SET search_text = ${searchText} + WHERE message_id = ${row.message_id} + `; + } + if (rows.length < pageSize) break; + afterMessageId = rows.at(-1)!.message_id; + } +}); diff --git a/apps/server/src/persistence/Migrations/082_TurnItemConversationIndex.ts b/apps/server/src/persistence/Migrations/082_TurnItemConversationIndex.ts new file mode 100644 index 000000000000..fb1caac44f3e --- /dev/null +++ b/apps/server/src/persistence/Migrations/082_TurnItemConversationIndex.ts @@ -0,0 +1,16 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + + // The thread window finds its newest conversation rows here instead of walking the tool calls + // between them. The read in ProjectionStore must spell this type list the same way. + yield* sql` + CREATE INDEX IF NOT EXISTS orchestration_v2_projection_turn_items_conversation_idx + ON orchestration_v2_projection_turn_items(thread_id, ordinal, type, run_id) + WHERE type NOT IN ( + 'command_execution', 'reasoning', 'file_change', 'file_search', 'web_search', 'dynamic_tool' + ) + `; +}); diff --git a/apps/server/src/persistence/Sqlite.test.ts b/apps/server/src/persistence/Sqlite.test.ts index a2d29f12f181..fb9335c7fa5f 100644 --- a/apps/server/src/persistence/Sqlite.test.ts +++ b/apps/server/src/persistence/Sqlite.test.ts @@ -183,3 +183,5 @@ it.effect("drops planner statistics a database already has", () => { Effect.ensuring(Effect.sync(() => NodeFS.rmSync(tempDir, { recursive: true, force: true }))), ); }); + +// Coder: upstream's `layerCliConfig` tests (#26763) are not carried; there is no `t3` CLI. diff --git a/apps/server/src/persistence/Sqlite.ts b/apps/server/src/persistence/Sqlite.ts index 38913376cd12..a73f1821d502 100644 --- a/apps/server/src/persistence/Sqlite.ts +++ b/apps/server/src/persistence/Sqlite.ts @@ -11,68 +11,81 @@ import * as ServerConfig from "../config.ts"; const WAL_SIZE_LIMIT_BYTES = 128 * 1024 * 1024; -const layerSetup = Layer.effectDiscard( - Effect.gen(function* () { - const sql = yield* SqlClient.SqlClient; - // CLI and server write from separate processes; wait rather than fail with SQLITE_BUSY. - yield* sql`PRAGMA busy_timeout = 5000;`; - yield* sql`PRAGMA foreign_keys = ON;`; - // Existing databases need doctor's offline VACUUM to add pointer maps for this mode. - const tables = yield* sql<{ - readonly count: number; - }>`SELECT COUNT(*) AS count FROM sqlite_schema WHERE type = 'table'`; - if (tables[0]?.count === 0) yield* sql`PRAGMA auto_vacuum = INCREMENTAL;`; - yield* sql`PRAGMA journal_mode = WAL;`; - // FULL fsyncs every event commit on the main thread; NORMAL syncs only at WAL checkpoints. - // App crashes lose nothing; OS crashes or power loss can roll back post-checkpoint commits without corruption. - yield* sql`PRAGMA synchronous = NORMAL;`; - // macOS fsync misses drive caches; F_FULLFSYNC flushes them at checkpoints and is ignored elsewhere. - yield* sql`PRAGMA checkpoint_fullfsync = ON;`; - // Keep statement journals and sort spills off the main thread's disk path. - // Large statements and open savepoints can increase peak memory use. - yield* sql`PRAGMA temp_store = MEMORY;`; - // PASSIVE checkpoints retain peak WAL size until the last connection closes. - yield* sql.unsafe(`PRAGMA journal_size_limit = ${WAL_SIZE_LIMIT_BYTES};`); - yield* runMigrations(); - // Collecting stats scans every page, and stale small-table stats cause scans after growth; use indexes by default. - // Connections cache stats; reload after deleting them. - const statTables = yield* sql`SELECT 1 FROM sqlite_schema WHERE name = 'sqlite_stat1'`; - if (statTables.length > 0) { - yield* sql`DELETE FROM sqlite_stat1;`; - yield* sql`ANALYZE sqlite_schema;`; - } - }).pipe(Effect.withSpan("Sqlite.setup")), -); +// Coder: no `t3` CLI, so no `ServerOwnsDatabaseError` or owner-held migrations (#26763's +// `runMigrationsAsOwner` and `layerCliConfig`); only the helper opens the database. +const makeLayerSetup = (migrate: Effect.Effect) => + Layer.effectDiscard( + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + // CLI and server write from separate processes; wait rather than fail with SQLITE_BUSY. + yield* sql`PRAGMA busy_timeout = 5000;`; + yield* sql`PRAGMA foreign_keys = ON;`; + // Existing databases need doctor's offline VACUUM to add pointer maps for this mode. + const tables = yield* sql<{ + readonly count: number; + }>`SELECT COUNT(*) AS count FROM sqlite_schema WHERE type = 'table'`; + if (tables[0]?.count === 0) yield* sql`PRAGMA auto_vacuum = INCREMENTAL;`; + yield* sql`PRAGMA journal_mode = WAL;`; + // FULL fsyncs every event commit on the main thread; NORMAL syncs only at WAL checkpoints. + // App crashes lose nothing; OS crashes or power loss can roll back post-checkpoint commits without corruption. + yield* sql`PRAGMA synchronous = NORMAL;`; + // macOS fsync misses drive caches; F_FULLFSYNC flushes them at checkpoints and is ignored elsewhere. + yield* sql`PRAGMA checkpoint_fullfsync = ON;`; + // Keep statement journals and sort spills off the main thread's disk path. + // Large statements and open savepoints can increase peak memory use. + yield* sql`PRAGMA temp_store = MEMORY;`; + // PASSIVE checkpoints retain peak WAL size until the last connection closes. + yield* sql.unsafe(`PRAGMA journal_size_limit = ${WAL_SIZE_LIMIT_BYTES};`); + yield* migrate; + // Collecting stats scans every page, and stale small-table stats cause scans after growth; use indexes by default. + // Connections cache stats; reload after deleting them. + const statTables = yield* sql`SELECT 1 FROM sqlite_schema WHERE name = 'sqlite_stat1'`; + if (statTables.length > 0) { + yield* sql`DELETE FROM sqlite_stat1;`; + yield* sql`ANALYZE sqlite_schema;`; + } + }).pipe(Effect.withSpan("Sqlite.setup")), + ); -export const layerFromPath = Effect.fn("makeSqlitePersistenceLive")(function* (dbPath: string) { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - yield* fs.makeDirectory(path.dirname(dbPath), { recursive: true }); +const layerSetup = makeLayerSetup(runMigrations()); - return Layer.provideMerge( - layerSetup, - NodeSqliteClient.layer({ - filename: dbPath, - readerWorker: true, - // Run checkpoint fsyncs off the event loop so slow disk flushes cannot stall clients. - checkpointWorker: true, - spanAttributes: { - "db.name": path.basename(dbPath), - "service.name": "t3code-server", - }, - }), - ); -}, Layer.unwrap); +const makeLayerFromPath = (setup: Layer.Layer) => + Effect.fn("makeSqlitePersistenceLive")(function* (dbPath: string) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* fs.makeDirectory(path.dirname(dbPath), { recursive: true }); + + return Layer.provideMerge( + setup, + NodeSqliteClient.layer({ + filename: dbPath, + readerWorker: true, + // Run checkpoint fsyncs off the event loop so slow disk flushes cannot stall clients. + checkpointWorker: true, + spanAttributes: { + "db.name": path.basename(dbPath), + "service.name": "t3code-server", + }, + }), + ); + }, Layer.unwrap); + +export const layerFromPath = makeLayerFromPath(layerSetup); export const layerMemory = Layer.provideMerge( layerSetup, NodeSqliteClient.layer({ filename: ":memory:" }), ); -export const layerConfig = Layer.unwrap( - Effect.gen(function* () { - const { dbPath } = yield* ServerConfig.ServerConfig; - yield* initializeV2Database(dbPath); - return layerFromPath(dbPath); - }), -); +const makeLayerConfig = ( + fromPath: (dbPath: string) => Layer.Layer, +) => + Layer.unwrap( + Effect.gen(function* () { + const { dbPath } = yield* ServerConfig.ServerConfig; + yield* initializeV2Database(dbPath); + return fromPath(dbPath); + }), + ); + +export const layerConfig = makeLayerConfig(layerFromPath); diff --git a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts index 84fa4ddb44c7..88348b742b47 100644 --- a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts +++ b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts @@ -60,6 +60,8 @@ describe("V2 preview upgrade", () => { [78, "DropOrchestrationEventCorrelationIndex"], [79, "DelegatedRecoveryIndexes"], [80, "ScheduledTaskThreadMode"], + [81, "ProjectionMessageSearchText"], + [82, "TurnItemConversationIndex"], ]); assert.deepStrictEqual(yield* runMigrations(), []); assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports); @@ -163,6 +165,8 @@ describe("V2 preview upgrade", () => { [78, "DropOrchestrationEventCorrelationIndex"], [79, "DelegatedRecoveryIndexes"], [80, "ScheduledTaskThreadMode"], + [81, "ProjectionMessageSearchText"], + [82, "TurnItemConversationIndex"], ]); }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))), ); diff --git a/apps/server/src/project/AgentSessionImporter.test.ts b/apps/server/src/project/AgentSessionImporter.test.ts index 0e3a3c67a766..97cceaacbaea 100644 --- a/apps/server/src/project/AgentSessionImporter.test.ts +++ b/apps/server/src/project/AgentSessionImporter.test.ts @@ -5,12 +5,18 @@ import { ProjectId, ProviderDriverKind, ProviderInstanceId, + MessageId, + RunId, ThreadId, type ApplicationStoredEvent, type OrchestrationV2AppThread, + type OrchestrationV2DomainEvent, type OrchestrationV2ProviderThread, } from "@t3tools/contracts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; +import * as KeyedLock from "@t3tools/shared/KeyedLock"; +import * as AcpRegistryRuntimeCoordinator from "@t3tools/provider-acp-registry/server/AcpRegistryRuntimeCoordinator"; +import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; @@ -20,11 +26,14 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Stream from "effect/Stream"; import * as SqlClient from "effect/sql/SqlClient"; +import * as TestClock from "effect/testing/TestClock"; import * as EventSink from "../orchestration-v2/EventSink.ts"; import * as EventStore from "../orchestration-v2/EventStore.ts"; import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; +import * as ProviderAdapterRegistry from "../orchestration-v2/ProviderAdapterRegistry.ts"; +import * as ThreadCommandExecutor from "../orchestration-v2/ThreadCommandExecutor.ts"; import { CLAUDE_MODEL_SELECTION, CODEX_MODEL_SELECTION, @@ -170,6 +179,8 @@ const seedOwner = Effect.fn(function* ( const makeImporter = Effect.fn(function* ( candidates: ReadonlyArray>, beforeTransaction: Effect.Effect = Effect.void, + getAdapter?: ProviderAdapterRegistry.ProviderAdapterRegistryV2Shape["get"], + threadCommands?: KeyedLock.KeyedLock, ) { const sql = yield* SqlClient.SqlClient; const events = yield* EventStore.EventStoreV2; @@ -236,12 +247,23 @@ const makeImporter = Effect.fn(function* ( ), ), }), + Layer.mock(ProviderAdapterRegistry.ProviderAdapterRegistryV2)( + getAdapter === undefined ? {} : { get: getAdapter }, + ), + Layer.succeed( + ThreadCommandExecutor.ThreadCommandExecutor, + threadCommands ?? (yield* KeyedLock.make()), + ), ), ), ), ); const importer = Context.get(context, AgentSessionImporter.AgentSessionImporter); - return { run: importer.importRecentAgentThreads({ projectId }), published }; + return { + run: importer.importRecentAgentThreads({ projectId }), + importNativeHistory: importer.importNativeHistory, + published, + }; }); it.effect( @@ -416,3 +438,318 @@ it.effect( }); }).pipe(Effect.scoped, Effect.provide(persistence)), ); + +it.effect( + "copies native session history into the imported thread once, retrying a failed read", + () => + Effect.gen(function* () { + const projections = yield* ProjectionStore.ProjectionStoreV2; + const source = candidate(CODEX_MODEL_SELECTION, "native-history"); + const threadId = yield* seedOwner(source); + const resolved: Array = []; + const opened: Array = []; + let closed = false; + let failRead = true; + const replayed = (role: "user" | "assistant", text: string) => ({ + createdBy: role === "user" ? ("user" as const) : ("agent" as const), + creationSource: "provider" as const, + id: MessageId.make(`replayed:${text}`), + threadId, + runId: null, + nodeId: null, + role, + text, + attachments: [], + streaming: false, + createdAt: now, + updatedAt: now, + }); + const adapter = { + openSession: (input: ProviderAdapter.ProviderAdapterV2OpenSessionInput) => + Effect.acquireRelease( + Effect.sync(() => { + opened.push(input); + return { + readThreadSnapshot: ({ providerThread }) => + failRead + ? Effect.fail( + new ProviderAdapter.ProviderAdapterReadThreadSnapshotError({ + driver: providerThread.driver, + providerThreadId: providerThread.id, + }), + ) + : Effect.succeed({ + providerThread, + providerTurns: [], + runtimeRequests: [], + reasoningMessageIds: [MessageId.make("replayed:Check the build log")], + messages: [ + replayed("user", "Fix the build"), + replayed("assistant", "Check the build log"), + replayed("assistant", "Fixed it"), + replayed("assistant", " "), + ], + }), + } as Pick; + }), + () => + Effect.sync(() => { + closed = true; + }), + ), + } as unknown as ProviderAdapter.ProviderAdapterV2["Service"]; + const importer = yield* makeImporter([], Effect.void, (instanceId) => + Effect.sync(() => { + resolved.push(instanceId); + return adapter; + }), + ); + + const importHistory = importer.importNativeHistory({ + threadId, + workspaceRoot: "/workspace/project", + }); + + expect((yield* Effect.flip(importHistory))._tag).toBe( + "ProviderAdapterReadThreadSnapshotError", + ); + expect(closed).toBe(true); + expect( + (yield* projections.getThreadRecords(threadId, [])).thread.historyOrigin, + ).toBeUndefined(); + + failRead = false; + expect(yield* importHistory).toBe(3); + expect(resolved).toEqual([ + CODEX_MODEL_SELECTION.instanceId, + CODEX_MODEL_SELECTION.instanceId, + ]); + expect(opened[1]).toMatchObject({ + threadId, + initialNativeThreadId: source.source.providerSessionId, + runtimePolicy: { cwd: "/workspace/project" }, + }); + // Forks keep run-less history only for this origin. + expect((yield* projections.getThreadRecords(threadId, [])).thread.historyOrigin).toBe( + "v1_import", + ); + expect(yield* importHistory).toBe(0); + expect(opened).toHaveLength(2); + const page = yield* projections.getTimelinePage(threadId, { view: "activity", limit: 10 }); + expect( + page.items.map(({ item }) => [ + item.type, + item.runId, + "text" in item ? item.text : undefined, + ]), + ).toEqual([ + ["user_message", null, "Fix the build"], + ["reasoning", null, "Check the build log"], + ["assistant_message", null, "Fixed it"], + ]); + }).pipe(Effect.scoped, Effect.provide(persistence)), +); + +it.effect("releases the ACP import lock when a native session never answers", () => + Effect.gen(function* () { + const projections = yield* ProjectionStore.ProjectionStoreV2; + const coordinator = yield* AcpRegistryRuntimeCoordinator.AcpRegistryRuntimeCoordinator; + const threadId = yield* seedOwner(candidate(CODEX_MODEL_SELECTION, "native-history-hang")); + const opened = yield* Deferred.make(); + let closed = false; + // An agent that only resumes sessions and never answers its first session/resume. + const adapter = { + openSession: () => + Effect.suspend(() => { + if (!closed) { + return Effect.addFinalizer(() => + Effect.sync(() => { + closed = true; + }), + ).pipe( + Effect.andThen(Deferred.succeed(opened, undefined)), + Effect.andThen(Effect.never), + ); + } + return Effect.succeed({ + readThreadSnapshot: ({ providerThread }) => + Effect.succeed({ + providerThread, + providerTurns: [], + runtimeRequests: [], + messages: [ + { + createdBy: "user", + creationSource: "provider", + id: MessageId.make("replayed:hang"), + threadId, + runId: null, + nodeId: null, + role: "user", + text: "Fix the build", + attachments: [], + streaming: false, + createdAt: now, + updatedAt: now, + }, + ], + }), + } as Pick); + }), + } as unknown as ProviderAdapter.ProviderAdapterV2["Service"]; + const importer = yield* makeImporter([], Effect.void, () => Effect.succeed(adapter)); + const importHistory = coordinator.withSessionMutation( + importer.importNativeHistory({ threadId, workspaceRoot: "/workspace/project" }), + ); + + const importing = yield* importHistory.pipe(Effect.flip, Effect.forkChild); + yield* Deferred.await(opened); + // Any other ACP import or delete waits for the same lock. + const next = yield* coordinator + .withSessionMutation(Effect.succeed("next")) + .pipe(Effect.forkChild); + yield* TestClock.adjust("2 minutes"); + + expect((yield* Fiber.join(importing))._tag).toBe("TimeoutError"); + expect(closed).toBe(true); + expect(yield* Fiber.join(next)).toBe("next"); + expect( + (yield* projections.getThreadRecords(threadId, [])).thread.historyOrigin, + ).toBeUndefined(); + expect(yield* importHistory).toBe(1); + expect((yield* projections.getThreadRecords(threadId, [])).thread.historyOrigin).toBe( + "v1_import", + ); + }).pipe( + Effect.scoped, + Effect.provide(Layer.merge(persistence, AcpRegistryRuntimeCoordinator.layer)), + ), +); + +/** + * Runs a thread command that read the thread before the history load ended and + * writes once the importer is about to commit, as an unserialized write would. + */ +const importDuringCommand = Effect.fn(function* ( + name: string, + command: (thread: OrchestrationV2AppThread) => ReadonlyArray, +) { + const projections = yield* ProjectionStore.ProjectionStoreV2; + const sink = yield* EventSink.EventSinkV2; + const threadId = yield* seedOwner(candidate(CODEX_MODEL_SELECTION, name)); + const lock = yield* KeyedLock.make(); + const loaded = yield* Deferred.make(); + const importerCommitting = yield* Deferred.make(); + const commandWritten = yield* Deferred.make(); + const committing = Deferred.succeed(importerCommitting, undefined); + const adapter = { + openSession: () => + Effect.succeed({ + readThreadSnapshot: ({ providerThread }) => + Deferred.await(loaded).pipe( + Effect.as({ + providerThread, + providerTurns: [], + runtimeRequests: [], + messages: [ + { + createdBy: "user" as const, + creationSource: "provider" as const, + id: MessageId.make(`replayed:${name}`), + threadId, + runId: null, + nodeId: null, + role: "user" as const, + text: "Fix the build", + attachments: [], + streaming: false, + createdAt: now, + updatedAt: now, + }, + ], + }), + ), + } as Pick), + } as unknown as ProviderAdapter.ProviderAdapterV2["Service"]; + const importer = yield* makeImporter( + [], + committing.pipe(Effect.andThen(Deferred.await(commandWritten))), + () => Effect.succeed(adapter), + { + ...lock, + withLock: (key, effect) => committing.pipe(Effect.andThen(lock.withLock(key, effect))), + }, + ); + const commanding = yield* Effect.forkScoped( + lock.withLock( + threadId, + Effect.gen(function* () { + const { thread } = yield* projections.getThreadRecords(threadId, []); + yield* Deferred.succeed(loaded, undefined); + yield* Deferred.await(importerCommitting); + yield* sink.write({ events: command(thread) }); + yield* Deferred.succeed(commandWritten, undefined); + }), + ), + ); + const imported = yield* importer.importNativeHistory({ + threadId, + workspaceRoot: "/workspace/project", + }); + yield* Fiber.join(commanding); + const records = yield* projections.getThreadRecords(threadId, ["runs"]); + const page = yield* projections.getTimelinePage(threadId, { view: "messages", limit: 10 }); + return { imported, thread: records.thread, runs: records.runs, items: page.items }; +}); + +it.effect("keeps a thread edit made while the native history loads", () => + Effect.gen(function* () { + const result = yield* importDuringCommand("renamed-during-load", (thread) => [ + { + id: EventId.make(`${thread.id}:renamed`), + type: "thread.metadata-updated", + threadId: thread.id, + occurredAt: now, + payload: { ...thread, title: "Renamed" }, + }, + ]); + expect(result.imported).toBe(1); + expect(result.thread.title).toBe("Renamed"); + expect(result.thread.historyOrigin).toBe("v1_import"); + expect(result.items).toHaveLength(1); + }).pipe(Effect.scoped, Effect.provide(persistence)), +); + +it.effect("skips the native history when a run starts while it loads", () => + Effect.gen(function* () { + const result = yield* importDuringCommand("run-during-load", (thread) => [ + { + id: EventId.make(`${thread.id}:run`), + type: "run.updated", + threadId: thread.id, + occurredAt: now, + payload: { + id: RunId.make(`${thread.id}:run`), + threadId: thread.id, + ordinal: 1, + providerInstanceId: thread.providerInstanceId, + modelSelection: thread.modelSelection, + providerThreadId: null, + userMessageId: MessageId.make(`${thread.id}:prompt`), + rootNodeId: null, + activeAttemptId: null, + status: "queued", + requestedAt: now, + startedAt: null, + completedAt: null, + checkpointId: null, + contextHandoffId: null, + }, + }, + ]); + expect(result.imported).toBe(0); + expect(result.runs).toHaveLength(1); + expect(result.thread.historyOrigin).toBeUndefined(); + expect(result.items).toEqual([]); + }).pipe(Effect.scoped, Effect.provide(persistence)), +); diff --git a/apps/server/src/project/AgentSessionImporter.ts b/apps/server/src/project/AgentSessionImporter.ts index a3f587c1c6d4..cc9b3411cce5 100644 --- a/apps/server/src/project/AgentSessionImporter.ts +++ b/apps/server/src/project/AgentSessionImporter.ts @@ -5,7 +5,6 @@ import { DEFAULT_RUNTIME_MODE, AgentSessionImportProjectChangedError, AgentSessionImportProjectNotFoundError, - AgentSessionImportSource, AgentSessionScanError, AgentSessionSource, EventId, @@ -34,6 +33,8 @@ import * as Stream from "effect/Stream"; import * as EventSink from "../orchestration-v2/EventSink.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; +import * as ProviderAdapterRegistry from "../orchestration-v2/ProviderAdapterRegistry.ts"; +import * as ThreadCommandExecutor from "../orchestration-v2/ThreadCommandExecutor.ts"; import * as ProviderSessionRuntime from "../persistence/ProviderSessionRuntime.ts"; import * as AgentSessionScanner from "./AgentSessionScanner.ts"; import * as ProjectService from "./ProjectService.ts"; @@ -41,12 +42,6 @@ import * as ProjectService from "./ProjectService.ts"; const IMPORT_EVENT_PREFIX = "agent-session-import:v2"; const CLAUDE_SESSION_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; -const decodeImportedTranscriptPayload = Schema.decodeUnknownOption( - Schema.Struct({ - cwd: Schema.optional(Schema.String), - importedTranscripts: Schema.optional(Schema.Array(AgentSessionImportSource)), - }), -); class AgentSessionUnresumableSessionError extends Schema.TaggedError()( "AgentSessionUnresumableSessionError", @@ -82,14 +77,34 @@ class AgentSessionThreadModifiedError extends Schema.TaggedError & { + readonly role: AgentSessionScanner.AgentSessionThreadMessage["role"] | "reasoning"; +}; + function messageEvents(input: { readonly threadId: ThreadId; readonly index: number; - readonly message: AgentSessionScanner.AgentSessionThreadMessage; + readonly message: HistoryMessage; }): ReadonlyArray { const ordinal = input.index + 1; const suffix = String(input.index).padStart(6, "0"); @@ -98,20 +113,6 @@ function messageEvents(input: { `${IMPORT_EVENT_PREFIX}:turn-item:${input.threadId}:${suffix}`, ); const at = dateTime(input.message.createdAt); - const message: OrchestrationV2ConversationMessage = { - createdBy: input.message.role === "user" ? "user" : "agent", - creationSource: "server", - id: messageId, - threadId: input.threadId, - runId: null, - nodeId: null, - role: input.message.role, - text: input.message.text, - attachments: [], - streaming: false, - createdAt: at, - updatedAt: at, - }; const common = { id: turnItemId, threadId: input.threadId, @@ -128,6 +129,32 @@ function messageEvents(input: { completedAt: at, updatedAt: at, }; + const turnItemEvent = (payload: OrchestrationV2TurnItem): OrchestrationV2DomainEvent => ({ + id: EventId.make(turnItemId), + type: "turn-item.updated", + threadId: input.threadId, + occurredAt: at, + payload, + }); + if (input.message.role === "reasoning") { + return [ + turnItemEvent({ ...common, type: "reasoning", text: input.message.text, streaming: false }), + ]; + } + const message: OrchestrationV2ConversationMessage = { + createdBy: input.message.role === "user" ? "user" : "agent", + creationSource: "server", + id: messageId, + threadId: input.threadId, + runId: null, + nodeId: null, + role: input.message.role, + text: input.message.text, + attachments: [], + streaming: false, + createdAt: at, + updatedAt: at, + }; const turnItem: OrchestrationV2TurnItem = input.message.role === "user" ? { @@ -155,13 +182,7 @@ function messageEvents(input: { occurredAt: at, payload: message, }, - { - id: EventId.make(`${IMPORT_EVENT_PREFIX}:turn-item:${input.threadId}:${suffix}`), - type: "turn-item.updated", - threadId: input.threadId, - occurredAt: at, - payload: turnItem, - }, + turnItemEvent(turnItem), ]; } @@ -172,6 +193,8 @@ const make = Effect.gen(function* () { const eventSink = yield* EventSink.EventSinkV2; const idAllocator = yield* IdAllocator.IdAllocatorV2; const runtimes = yield* ProviderSessionRuntime.ProviderSessionRuntimeRepository; + const registry = yield* ProviderAdapterRegistry.ProviderAdapterRegistryV2; + const threadCommands = yield* ThreadCommandExecutor.ThreadCommandExecutor; const importRecentAgentThreads = Effect.fn("importRecentAgentThreadsV2")(function* ( input: AgentSessionImportInput, ) { @@ -200,7 +223,7 @@ const make = Effect.gen(function* () { ), ); const completedSources = runtimeRows.flatMap((runtime) => { - const payload = decodeImportedTranscriptPayload(runtime.runtimePayload); + const payload = AgentSessionScanner.decodeImportedTranscriptPayload(runtime.runtimePayload); if ( Option.isNone(payload) || payload.value.cwd === undefined || @@ -432,7 +455,105 @@ const make = Effect.gen(function* () { return { importedCount, skippedCount } satisfies AgentSessionImportResult; }); - return { importRecentAgentThreads }; + /** + * Loading a native session replays its history only into the adapter's own + * snapshot, so copy that snapshot into the thread as run-less history. The + * `v1_import` origin keeps it in forks and marks the import done, so a call + * after a failed read repairs the thread and any later call does nothing. + */ + const importNativeHistory = Effect.fn("importNativeHistoryV2")(function* (input: { + readonly threadId: ThreadId; + readonly workspaceRoot: string; + }) { + const pendingThread = Effect.map( + orchestrator.getThreadRecords(input.threadId, ["providerThreads", "runs"]), + (records) => + isNativeHistoryPending({ + historyOrigin: records.thread.historyOrigin, + hasRun: records.runs.length > 0, + }) + ? records + : null, + ); + // Thread commands write whole thread rows, so hold their lock from the + // re-read through the write to keep both this marker and their changes. + const commit = (messages: ReadonlyArray) => + threadCommands.withLock( + input.threadId, + Effect.gen(function* () { + const current = yield* pendingThread; + if (current === null) return 0; + yield* eventSink.write({ + events: [ + ...messages.flatMap((message, index) => + messageEvents({ threadId: input.threadId, index, message }), + ), + { + id: EventId.make(`${IMPORT_EVENT_PREFIX}:thread:${input.threadId}:native-history`), + type: "thread.metadata-updated", + threadId: input.threadId, + providerInstanceId: current.thread.providerInstanceId, + occurredAt: yield* DateTime.now, + payload: { ...current.thread, historyOrigin: "v1_import" }, + }, + ], + }); + return messages.length; + }), + ); + const records = yield* pendingThread; + if (records === null) return 0; + const { thread, providerThreads } = records; + const providerThread = providerThreads.find( + (candidate) => candidate.appThreadId === input.threadId && candidate.nativeThreadRef !== null, + ); + const nativeThreadId = providerThread?.nativeThreadRef?.nativeId; + if (providerThread === undefined || nativeThreadId == null) return 0; + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: providerThread.providerInstanceId, + threadId: input.threadId, + }); + // The registry admits this open against sign-in changes, like any other open. + const adapter = yield* registry.get(thread.modelSelection.instanceId); + const snapshot = yield* adapter + .openSession({ + threadId: input.threadId, + providerSessionId, + modelSelection: thread.modelSelection, + runtimePolicy: { + runtimeMode: thread.runtimeMode, + interactionMode: thread.interactionMode, + cwd: thread.worktreePath ?? input.workspaceRoot, + }, + initialNativeThreadId: nativeThreadId, + ...(providerThread.nativeMetadata?.itemIdentityVersion === undefined + ? {} + : { + initialProviderItemIdentityVersion: providerThread.nativeMetadata.itemIdentityVersion, + }), + }) + .pipe( + Effect.flatMap((session) => session.readThreadSnapshot({ providerThread })), + Effect.timeout(NATIVE_HISTORY_READ_TIMEOUT), + Effect.scoped, + ); + const reasoningIds = new Set(snapshot.reasoningMessageIds); + const messages = snapshot.messages.flatMap((message) => + (message.role === "user" || message.role === "assistant") && message.text.trim() !== "" + ? [ + { + role: reasoningIds.has(message.id) ? ("reasoning" as const) : message.role, + text: message.text, + createdAt: DateTime.formatIso(message.createdAt), + }, + ] + : [], + ); + // Loading the session can take a while, and the thread is usable meanwhile. + return yield* commit(messages); + }); + + return { importRecentAgentThreads, importNativeHistory }; }); type AgentSessionImporterShape = Effect.Success; diff --git a/apps/server/src/project/AgentSessionScanner.test.ts b/apps/server/src/project/AgentSessionScanner.test.ts index 5012ea5bc1f0..4d5318dff8b9 100644 --- a/apps/server/src/project/AgentSessionScanner.test.ts +++ b/apps/server/src/project/AgentSessionScanner.test.ts @@ -4,6 +4,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import * as NodeChildProcess from "node:child_process"; import * as NodeFS from "node:fs"; import * as NodeOS from "node:os"; +import * as NodeSqlite from "node:sqlite"; import { describe, expect, it } from "@effect/vitest"; import { AgentSessionImportSource, @@ -12,6 +13,7 @@ import { ProviderDriverKind, ProviderInstanceId, type ServerSettings as ContractServerSettings, + ThreadId, } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { symlinksSupported } from "@t3tools/shared/testing/symlinks"; @@ -30,6 +32,7 @@ import * as TestClock from "effect/testing/TestClock"; import * as ServerConfig from "../config.ts"; import * as ProcessRunner from "../processRunner.ts"; import * as ProjectStore from "../orchestration-v2/ProjectStore.ts"; +import * as ProviderSessionRuntime from "../persistence/ProviderSessionRuntime.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as AgentSessionScanner from "./AgentSessionScanner.ts"; @@ -58,9 +61,33 @@ interface ScannerTestInput { readonly importedWorkspaceRoots?: ReadonlyArray; readonly configBaseDir?: string; readonly providerInstances?: ContractServerSettings["providerInstances"]; + readonly importedTranscripts?: { + readonly cwd: string; + readonly sources: ReadonlyArray; + }; readonly processRunner?: Layer.Layer; } +const layerRuntimeRepository = (imported: ScannerTestInput["importedTranscripts"]) => + Layer.mock(ProviderSessionRuntime.ProviderSessionRuntimeRepository)({ + list: () => + Effect.sync(() => + (imported?.sources ?? []).map((source): ProviderSessionRuntime.ProviderSessionRuntime => ({ + threadId: ThreadId.make( + `import:${source.providerInstanceId}:${source.providerSessionId}`, + ), + providerName: source.provider, + providerInstanceId: source.providerInstanceId, + adapterKey: source.provider, + runtimeMode: "full-access", + status: "stopped", + lastSeenAt: "2026-01-01T00:00:00.000Z", + resumeCursor: null, + runtimePayload: { cwd: imported?.cwd, importedTranscripts: [source] }, + })), + ), + }); + const layerScannerTest = (input: ScannerTestInput) => (input.processRunner === undefined ? AgentSessionScanner.layer @@ -102,6 +129,7 @@ const layerScannerTest = (input: ScannerTestInput) => input.configBaseDir ?? { prefix: "t3code-scanner-config-" }, ), layerProjectStore(input.importedWorkspaceRoots ?? []), + layerRuntimeRepository(input.importedTranscripts), ), ), ); @@ -1091,6 +1119,38 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => { ]); }), ); + + it.effect("keeps Codex rollout cwds when the Codex state database is unreadable", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const claudeHomePath = yield* makeTempDir("t3code-claude-home-"); + const codexHomePath = yield* makeTempDir("t3code-codex-home-"); + const workspace = yield* makeTempDir("t3code-workspace-"); + yield* writeTranscript({ + filePath: path.join( + codexHomePath, + "sessions", + "2026", + "05", + "03", + "rollout-2026-05-03T00-00-00-019a5f00-0000-7000-8000-00000000000a.jsonl", + ), + contents: codexRolloutLine(workspace), + mtimeMs: Date.parse("2026-05-03T00:00:00.000Z"), + }); + yield* fileSystem.writeFileString( + path.join(codexHomePath, "state_5.sqlite"), + "not a database", + ); + + const result = yield* runScan({ claudeHomePath, codexHomePath }); + + expect(result.candidates.map(({ path, threadCount }) => ({ path, threadCount }))).toEqual([ + { path: workspace, threadCount: 1 }, + ]); + }), + ); }); describe("recentThreads", () => { @@ -1349,6 +1409,140 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => { }), ); + it.effect("imports Codex chats moved from a scratch folder into a Codex project", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const nowMs = Date.parse("2026-11-08T12:00:00.000Z"); + yield* TestClock.setTime(nowMs); + const claudeHomePath = yield* makeTempDir("t3code-claude-home-"); + const codexHomePath = yield* makeTempDir("t3code-codex-home-"); + const homeDir = yield* makeTempDir("t3code-home-"); + const projectDir = path.join(homeDir, "code", "my-project"); + const scratchDir = path.join(homeDir, "Documents", "Codex", "2026-11-08", "new-chat"); + const otherDir = path.join(homeDir, "code", "other-project"); + yield* fileSystem.makeDirectory(projectDir, { recursive: true }); + yield* fileSystem.makeDirectory(scratchDir, { recursive: true }); + yield* fileSystem.makeDirectory(otherDir, { recursive: true }); + + const projectId = "019a5f00-0000-7000-8000-000000000001"; + const chats = [ + { + id: "019a5f00-0000-7000-8000-00000000000a", + cwd: scratchDir, + message: "Started with no project", + }, + { + id: "019a5f00-0000-7000-8000-00000000000b", + cwd: projectDir, + message: "Started in the project", + }, + { + id: "019a5f00-0000-7000-8000-00000000000c", + cwd: otherDir, + message: "Started in another folder", + }, + { + id: "019a5f00-0000-7000-8000-00000000000d", + rolloutId: "019a5f00-0000-7000-8000-0000000000d1", + cwd: scratchDir, + message: "Rewound with no project", + }, + ]; + const rolloutPaths: Array = []; + for (const [index, chat] of chats.entries()) { + const filePath = path.join( + codexHomePath, + "sessions", + "2026", + "11", + "08", + `rollout-2026-11-08T08-0${index}-00-${"rolloutId" in chat ? `${chat.id}_${chat.rolloutId}` : chat.id}.jsonl`, + ); + rolloutPaths.push(filePath); + yield* writeTranscript({ + filePath, + contents: [ + encodeTranscriptRecord({ + type: "session_meta", + payload: { id: chat.id, cwd: chat.cwd }, + }), + encodeTranscriptRecord({ type: "turn_context", payload: { cwd: chat.cwd } }), + encodeTranscriptRecord({ + type: "event_msg", + payload: { type: "user_message", message: chat.message }, + }), + ].join("\n"), + mtimeMs: nowMs - 60_000 + index, + }); + } + // Codex records a chat's project only in its state DB; the rollout keeps the original cwd. + yield* Effect.sync(() => { + const db = new NodeSqlite.DatabaseSync(path.join(codexHomePath, "state_5.sqlite")); + try { + db.exec(` + CREATE TABLE projects ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + metadata TEXT NOT NULL DEFAULT '{}', + position INTEGER NOT NULL, + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL + ); + CREATE TABLE project_roots ( + project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE, + position INTEGER NOT NULL, + path TEXT NOT NULL, + PRIMARY KEY (project_id, position) + ); + CREATE TABLE threads ( + id TEXT PRIMARY KEY, + rollout_path TEXT NOT NULL, + cwd TEXT NOT NULL, + project_id TEXT REFERENCES projects(id) ON DELETE SET NULL + ); + `); + db.prepare("INSERT INTO projects VALUES (?, ?, '{}', 0, ?, ?)").run( + projectId, + "my-project", + nowMs, + nowMs, + ); + db.prepare("INSERT INTO project_roots VALUES (?, 0, ?)").run(projectId, projectDir); + const insertThread = db.prepare("INSERT INTO threads VALUES (?, ?, ?, ?)"); + for (const [index, chat] of chats.entries()) { + insertThread.run(chat.id, rolloutPaths[index]!, chat.cwd, projectId); + } + } finally { + db.close(); + } + }); + + const threads = yield* runRecentThreads({ + claudeHomePath, + codexHomePath, + workspaceRoot: projectDir, + }).pipe(Effect.provideService(HostProcess.HomeDirectory, homeDir)); + expect(threads.map((thread) => thread.title).toSorted()).toEqual([ + "Rewound with no project", + "Started in the project", + "Started with no project", + ]); + + const result = yield* runScan({ claudeHomePath, codexHomePath }).pipe( + Effect.provideService(HostProcess.HomeDirectory, homeDir), + ); + expect( + result.candidates + .map(({ path, threadCount }) => ({ path, threadCount })) + .toSorted((left, right) => left.path.localeCompare(right.path)), + ).toEqual([ + { path: projectDir, threadCount: 3 }, + { path: otherDir, threadCount: 1 }, + ]); + }), + ); + it.effect("imports a rewound Codex thread with the history its new rollout inherits", () => Effect.gen(function* () { const path = yield* Path.Path; @@ -1456,16 +1650,20 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => { null, { custom: "launcher" }, ]; + const fileSystem = yield* FileSystem.FileSystem; + // Rollouts imported before subagents were excluded still have import records. + const legacyImports: Array = []; for (const [index, source] of sources.entries()) { + const filePath = path.join( + codexHomePath, + "sessions", + "2026", + "08", + "24", + `rollout-${index}.jsonl`, + ); yield* writeTranscript({ - filePath: path.join( - codexHomePath, - "sessions", - "2026", - "08", - "24", - `rollout-${index}.jsonl`, - ), + filePath, contents: [ { type: "session_meta", payload: { id: `session-${index}`, cwd: workspace, source } }, // A later cwd must not make an excluded rollout eligible again. @@ -1479,6 +1677,21 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => { .join("\n"), mtimeMs: nowMs - index * 60_000, }); + if (index >= 100) continue; + const stats = yield* fileSystem.stat(filePath); + legacyImports.push( + decodeImportSource({ + provider: "codex", + providerInstanceId: "codex", + providerSessionId: `session-${index}`, + filePath, + size: Number(stats.size), + mtimeMs: Option.getOrNull(Option.map(stats.mtime, (date) => date.getTime())), + device: Number(stats.dev), + inode: Option.getOrNull(Option.map(stats.ino, String)), + birthtimeMs: Option.getOrNull(Option.map(stats.birthtime, (date) => date.getTime())), + }), + ); } yield* Effect.gen(function* () { @@ -1491,7 +1704,15 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => { outcome._tag === "Importable" ? outcome.thread.providerSessionId : outcome._tag, ), ).toEqual(["session-100", "session-101", "session-102", "session-103"]); - }).pipe(Effect.provide(layerScannerTest({ claudeHomePath, codexHomePath }))); + }).pipe( + Effect.provide( + layerScannerTest({ + claudeHomePath, + codexHomePath, + importedTranscripts: { cwd: workspace, sources: legacyImports }, + }), + ), + ); }), ); @@ -1627,6 +1848,123 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => { }), ); + it.effect("repeated imports reach older transcripts behind already-imported ones", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const nowMs = Date.parse("2026-08-24T12:00:00.000Z"); + yield* TestClock.setTime(nowMs); + const claudeHomePath = yield* makeTempDir("t3code-repeat-import-claude-"); + const codexHomePath = yield* makeTempDir("t3code-repeat-import-codex-"); + const workspace = yield* makeTempDir("t3code-repeat-import-workspace-"); + const sessionIds = Array.from( + { length: 102 }, + (_, index) => `00000000-0000-4000-8000-${String(index).padStart(12, "0")}`, + ); + for (const [index, sessionId] of sessionIds.entries()) { + yield* writeTranscript({ + filePath: path.join(claudeHomePath, "projects", "p", `${sessionId}.jsonl`), + // 999 cwd-less records cost 1,000 metadata records, so 100 newer transcripts + // exhaust MAX_METADATA_RECORDS_PER_SOURCE. + contents: + "{}\n".repeat(999) + + `${encodeTranscriptRecord({ + type: "user", + cwd: workspace, + sessionId, + timestamp: "2026-08-23T12:00:00.000Z", + message: { role: "user", content: `Prompt ${index}` }, + })}\n`, + mtimeMs: nowMs - (index + 1) * 60_000, + }); + } + + const completed: Array = []; + const imported = yield* Effect.gen(function* () { + const scanner = yield* AgentSessionScanner.AgentSessionScanner; + while (true) { + const outcomes = yield* scanner + .recentThreads(workspace, completed) + .pipe(Stream.runCollect); + const added = Array.from(outcomes).flatMap((outcome) => + outcome._tag === "Importable" ? [outcome.source] : [], + ); + if (added.length === 0) return completed; + completed.push(...added); + } + }).pipe( + Effect.provide( + layerScannerTest({ + claudeHomePath, + codexHomePath, + importedTranscripts: { cwd: workspace, sources: completed }, + }), + ), + ); + + expect(imported.map((source) => source.providerSessionId).toSorted()).toEqual(sessionIds); + }), + ); + + it.effect("lists projects whose transcripts sit behind already-imported ones", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const nowMs = Date.parse("2026-08-24T12:00:00.000Z"); + yield* TestClock.setTime(nowMs); + const claudeHomePath = yield* makeTempDir("t3code-hidden-project-claude-"); + const codexHomePath = yield* makeTempDir("t3code-hidden-project-codex-"); + const imported = yield* makeTempDir("t3code-hidden-project-imported-"); + const hidden = yield* makeTempDir("t3code-hidden-project-hidden-"); + const hiddenSessionId = "00000000-0000-4000-8000-000000000100"; + for (const index of Array.from({ length: 101 }, (_, value) => value)) { + const sessionId = `00000000-0000-4000-8000-${String(index).padStart(12, "0")}`; + yield* writeTranscript({ + filePath: path.join(claudeHomePath, "projects", "p", `${sessionId}.jsonl`), + contents: + "{}\n".repeat(999) + + `${encodeTranscriptRecord({ + type: "user", + cwd: index === 100 ? hidden : imported, + sessionId, + message: { role: "user", content: `Prompt ${index}` }, + })}\n`, + mtimeMs: nowMs - (index + 1) * 60_000, + }); + } + + const completed: Array = []; + const scans = yield* Effect.gen(function* () { + const scanner = yield* AgentSessionScanner.AgentSessionScanner; + const before = yield* scanner.scan; + const outcomes = yield* scanner.recentThreads(imported).pipe(Stream.runCollect); + const sources = Array.from(outcomes).flatMap((outcome) => + outcome._tag === "Importable" ? [outcome.source] : [], + ); + // A record whose file identity no longer matches must not lend its cwd. + const newest = sources[0]!; + completed.push(...sources, { + ...newest, + providerSessionId: hiddenSessionId, + filePath: path.join(claudeHomePath, "projects", "p", `${hiddenSessionId}.jsonl`), + }); + return { before, after: yield* scanner.scan }; + }).pipe( + Effect.provide( + layerScannerTest({ + claudeHomePath, + codexHomePath, + importedTranscripts: { cwd: imported, sources: completed }, + }), + ), + ); + + expect(scans.before.candidates.map((candidate) => candidate.path)).toEqual([imported]); + expect(scans.after.candidates.map((candidate) => candidate.path)).toEqual([ + imported, + hidden, + ]); + }), + ); + for (const source of ["claudeAgent", "codex"] as const) { for (const replacement of [ "same root", @@ -1871,6 +2209,95 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => { }), ); + it.effect("imports long Claude history beyond the selected-history and record limits", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const nowMs = Date.parse("2026-08-24T12:00:00.000Z"); + yield* TestClock.setTime(nowMs); + const claudeHomePath = yield* makeTempDir("t3code-long-conversation-claude-"); + const codexHomePath = yield* makeTempDir("t3code-long-conversation-codex-"); + const workspace = yield* makeTempDir("t3code-long-conversation-workspace-"); + const record = (fields: Record) => + encodeTranscriptRecord({ sessionId: "claude-long", ...fields }); + const longText = "x".repeat(64 * 1024); + const middleCount = 600; + yield* writeTranscript({ + filePath: path.join(claudeHomePath, "projects", "-slug", "claude-long.jsonl"), + contents: [ + record({ + type: "user", + cwd: workspace, + message: { role: "user", content: "First prompt" }, + }), + // 600 x 64 KiB exceeds the 32 MiB selected-history budget; the shown window fits. + ...Array.from({ length: middleCount }, (_, index) => { + const role = index % 2 === 0 ? "assistant" : "user"; + return record({ type: role, message: { role, content: longText } }); + }), + // Tool and progress records push the file past the 100,000-record limit. + ...Array.from({ length: 100_000 }, () => "{}"), + record({ type: "user", message: { role: "user", content: "Latest prompt" } }), + ].join("\n"), + mtimeMs: nowMs - 60_000, + }); + + const outcomes = yield* runRecentThreadOutcomes({ + claudeHomePath, + codexHomePath, + workspaceRoot: workspace, + }); + + expect(outcomes.map((outcome) => outcome._tag)).toEqual(["Importable"]); + const thread = outcomes[0]?._tag === "Importable" ? outcomes[0].thread : undefined; + expect(thread?.messages[0]).toMatchObject({ role: "user", text: "First prompt" }); + expect(thread?.messages.at(-1)).toMatchObject({ role: "user", text: "Latest prompt" }); + expect(thread?.messages).toHaveLength(200); + expect(thread?.omittedMessageCount).toBe(middleCount + 2 - 200); + }), + ); + + it.effect("imports a late first Claude prompt once when older replies are dropped", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const nowMs = Date.parse("2026-08-24T12:00:00.000Z"); + yield* TestClock.setTime(nowMs); + const claudeHomePath = yield* makeTempDir("t3code-late-prompt-claude-"); + const codexHomePath = yield* makeTempDir("t3code-late-prompt-codex-"); + const workspace = yield* makeTempDir("t3code-late-prompt-workspace-"); + const contents = [ + encodeTranscriptRecord({ type: "attachment", sessionId: "late-prompt", cwd: workspace }), + // The first prompt is still among the latest 200 messages after older replies drop out. + ...Array.from({ length: 250 }, (_, index) => { + const role = index < 60 || index % 2 === 1 ? "assistant" : "user"; + return encodeTranscriptRecord({ + type: role, + sessionId: "late-prompt", + message: { role, content: `${role} ${index}` }, + }); + }), + ].join("\n"); + const filePath = path.join(claudeHomePath, "projects", "-slug", "late-prompt.jsonl"); + yield* writeTranscript({ filePath, contents, mtimeMs: nowMs - 60_000 }); + + const [thread] = yield* runRecentThreads({ + claudeHomePath, + codexHomePath, + workspaceRoot: workspace, + }); + + expect(thread?.messages[0]).toMatchObject({ role: "assistant", text: "assistant 50" }); + expect(thread).toEqual( + AgentSessionScanner.parseAgentSessionTranscript({ + source: "claudeAgent", + providerInstanceId: ProviderInstanceId.make("claudeAgent"), + fallbackSessionId: "late-prompt", + lastActiveAtMs: nowMs - 60_000, + contents, + }), + ); + }), + ); + it.effect("skips growth during reading without exceeding the reserved bytes", () => Effect.gen(function* () { const path = yield* Path.Path; @@ -2422,58 +2849,6 @@ describe("parseAgentSessionTranscript", () => { expect(thread?.title).toBe(expected); }); - it("drops injected Codex instructions while keeping the visible user event", () => { - const thread = AgentSessionScanner.parseAgentSessionTranscript({ - contents: [ - JSON.stringify({ type: "session_meta", payload: { id: "codex-session" } }), - JSON.stringify({ - type: "response_item", - payload: { - type: "message", - role: "user", - internal_chat_message_metadata_passthrough: { turn_id: "turn-1" }, - content: [ - { - type: "input_text", - text: "\nInternal setup instructions\n", - }, - ], - }, - }), - JSON.stringify({ - type: "event_msg", - payload: { type: "user_message", message: "Fix the actual bug" }, - }), - JSON.stringify({ - type: "response_item", - payload: { - type: "message", - role: "user", - internal_chat_message_metadata_passthrough: { turn_id: "turn-1" }, - content: [{ type: "input_text", text: "Fix the actual bug" }], - }, - }), - JSON.stringify({ - type: "response_item", - payload: { - type: "message", - role: "assistant", - content: [{ type: "output_text", text: "Fixed" }], - }, - }), - ].join("\n"), - source: "codex", - providerInstanceId: ProviderInstanceId.make("codex"), - fallbackSessionId: "fallback", - lastActiveAtMs: Date.parse("2026-08-24T12:00:00.000Z"), - }); - - expect(thread?.messages.map((message) => message.text)).toEqual([ - "Fix the actual bug", - "Fixed", - ]); - }); - it.each([199, 200])("keeps the canonical first prompt with %s later Codex messages", (count) => { const canonicalPrompt = "\n Keep the canonical prompt \n"; const canonicalTimestamp = "2026-08-24T10:01:00.000Z"; diff --git a/apps/server/src/project/AgentSessionScanner.ts b/apps/server/src/project/AgentSessionScanner.ts index 463257831be3..c84d2a2a9da0 100644 --- a/apps/server/src/project/AgentSessionScanner.ts +++ b/apps/server/src/project/AgentSessionScanner.ts @@ -1,13 +1,14 @@ import * as NodeOS from "node:os"; +import * as NodeSqlite from "node:sqlite"; import { + AgentSessionImportSource, AgentSessionScanError, ClaudeSettings, CodexSettings, ProviderDriverKind, ProviderInstanceId, resolveProviderInstanceEnabled, - type AgentSessionImportSource, type AgentSessionProjectCandidate, type AgentSessionProjectGit, type AgentSessionScanResult, @@ -33,6 +34,7 @@ import { repositoryPathFromRemoteUrl } from "@t3tools/source-control-core/server import * as ServerConfig from "../config.ts"; import * as ProcessRunner from "../processRunner.ts"; import * as ProjectStore from "../orchestration-v2/ProjectStore.ts"; +import * as ProviderSessionRuntime from "../persistence/ProviderSessionRuntime.ts"; import { resolveCodexHomeLayout } from "../provider/Drivers/CodexHomeLayout.ts"; import { mergeProviderInstanceEnvironment } from "@t3tools/provider-core/server/instanceEnvironment"; import { expandHomePath } from "@t3tools/provider-core/server/pathExpansion"; @@ -71,9 +73,19 @@ class TranscriptParseError extends Schema.TaggedError()( return `Could not parse the transcript at ${this.filePath}.`; } } + +class CodexStateReadError extends Schema.TaggedError()("CodexStateReadError", { + databasePath: Schema.String, + cause: Schema.Defect(), +}) { + override get message(): string { + return `Could not read Codex projects from ${this.databasePath}.`; + } +} const METADATA_READ_BYTES = 8 * 1024; const MAX_TRANSCRIPT_SCAN_BYTES = 1024 * 1024; const MAX_CODEX_SESSION_INDEX_BYTES = 1024 * 1024; +const MAX_CODEX_PROJECT_ROOT_ROWS = 100_000; const MAX_TRANSCRIPTS_PER_SOURCE = 5000; @@ -170,12 +182,20 @@ const decodePiEntryLinks = Schema.decodeUnknownOption( ); const selectTranscriptPath = createTranscriptJsonSelector(TranscriptRecord); const decodeCodexTurnMetadata = Schema.decodeUnknownOption(CodexTurnMetadata); +export const decodeImportedTranscriptPayload = Schema.decodeUnknownOption( + Schema.Struct({ + cwd: Schema.optional(Schema.String), + importedTranscripts: Schema.optional(Schema.Array(AgentSessionImportSource)), + }), +); const decodeCodexHistoryBase = Schema.decodeUnknownOption( Schema.Struct({ thread_id: Schema.String, end_byte_offset: Schema.Number }), ); // Original rollouts end in their thread ID, rewound ones in `_`. const CODEX_ROLLOUT_ID_PATTERN = - /[-_]([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.jsonl$/i; + /-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:_([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}))?\.jsonl$/i; +// Codex records a chat's project only here; moving a chat leaves its rollout cwd unchanged. +const CODEX_STATE_DATABASE = "state_5.sqlite"; const decodeCodexSessionIndexEntry = Schema.decodeUnknownOption( Schema.fromJsonString(CodexSessionIndexEntry), ); @@ -240,6 +260,7 @@ interface RawCandidate { readonly transcripts: ReadonlyArray<{ readonly filePath: string; readonly mtimeMs: number | null; + readonly projectRoots?: ReadonlyArray | undefined; }>; } @@ -248,7 +269,8 @@ interface TranscriptCandidate { readonly mtimeMs: number; readonly providerInstanceId: ProviderInstanceId; readonly homePath: string; - readonly size: number; + readonly identity: TranscriptIdentity; + readonly projectRoots?: ReadonlyArray | undefined; } interface MetadataReadBudget { @@ -432,13 +454,14 @@ export function parseAgentSessionTranscript( input.source === "claudeAgent" ? records.reduce(foldClaudeFields, emptyClaudeFields) : emptyClaudeFields; - return parseAgentSessionRecords(input, records, claudeFields); + return parseAgentSessionRecords(input, records, claudeFields, 0); } function parseAgentSessionRecords( input: AgentSessionTranscriptMetadata, records: ReadonlyArray, claudeFields: ClaudeFields, + droppedMessageCount: number, ): AgentSessionThread | null { const fallbackTimestamp = DateTime.formatIso(DateTime.makeUnsafe(input.lastActiveAtMs)); // Claude filenames are session IDs. Codex rollout filenames include extra @@ -651,7 +674,7 @@ function parseAgentSessionRecords( createdAt: retainedMessages[0]?.createdAt ?? fallbackTimestamp, updatedAt: fallbackTimestamp, messages: retainedMessages, - omittedMessageCount: conversationMessageCount - retainedMessages.length, + omittedMessageCount: conversationMessageCount + droppedMessageCount - retainedMessages.length, }; } @@ -688,6 +711,31 @@ function shouldRetainDecodedRecord( ); } +// The import shows only the first user message and the latest messages, so a +// long Claude session holds just those and counts the rest as dropped. +function createClaudeHistoryWindow() { + const recent: Array<{ readonly record: DecodedTranscriptRecord; readonly bytes: number }> = []; + let firstUser: DecodedTranscriptRecord | undefined; + let droppedMessageCount = 0; + return { + // Takes a record shouldRetainDecodedRecord kept; returns the bytes it released. + push: (record: DecodedTranscriptRecord, bytes: number): number => { + if (firstUser === undefined && record.type === "user") firstUser = record; + recent.push({ record, bytes }); + if (recent.length <= MAX_IMPORTED_MESSAGES) return 0; + const evicted = recent.shift(); + if (evicted === undefined || evicted.record === firstUser) return 0; + droppedMessageCount += 1; + return evicted.bytes; + }, + records: () => { + const held = recent.map((entry) => entry.record); + return firstUser === undefined || held.includes(firstUser) ? held : [firstUser, ...held]; + }, + droppedMessageCount: () => droppedMessageCount, + }; +} + function normalizeForWorktreeMatch(value: string, caseFold: boolean): string { const normalized = `${value.replaceAll("\\", "/")}/`; return caseFold ? normalized.toLowerCase() : normalized; @@ -789,6 +837,7 @@ export const make = Effect.gen(function* () { const serverConfig = yield* ServerConfig.ServerConfig; const serverSettings = yield* ServerSettings.ServerSettingsService; const projectStore = yield* ProjectStore.ProjectStoreV2; + const runtimes = yield* ProviderSessionRuntime.ProviderSessionRuntimeRepository; const baseDir = path.resolve(serverConfig.baseDir); const worktreesDir = path.resolve(serverConfig.worktreesDir); const isWindows = (yield* HostProcess.Platform) === "win32"; @@ -962,7 +1011,7 @@ export const make = Effect.gen(function* () { transcript: TranscriptCandidate, budget: MetadataReadBudget, ) { - if (transcript.size === 0) return null; + if (transcript.identity.size === 0) return null; if ( budget.bytesRemaining === 0 || budget.operationsRemaining < 2 || @@ -980,7 +1029,7 @@ export const make = Effect.gen(function* () { let remaining = ""; let bytesRead = 0; let recordsRead = 0; - const maxBytes = Math.min(MAX_TRANSCRIPT_SCAN_BYTES, transcript.size); + const maxBytes = Math.min(MAX_TRANSCRIPT_SCAN_BYTES, transcript.identity.size); const reserveRecord = () => { if ( recordsRead === MAX_METADATA_RECORDS_PER_TRANSCRIPT || @@ -1029,7 +1078,7 @@ export const make = Effect.gen(function* () { } } - if (bytesRead < transcript.size) { + if (bytesRead < transcript.identity.size) { budget.truncated = true; return null; } @@ -1057,6 +1106,7 @@ export const make = Effect.gen(function* () { return null; } const records: Array = []; + const claudeHistory = createClaudeHistoryWindow(); let cwd = Option.none(); let claudeFields = emptyClaudeFields; let historyBytes = prefix?.historyBytes ?? 0; @@ -1078,7 +1128,8 @@ export const make = Effect.gen(function* () { const finishRecord = () => { reader.write(decoder.decode()); recordCount += 1; - if (recordCount > MAX_IMPORT_RECORDS) return false; + // Claude keeps a bounded window, so only the byte budgets limit its length. + if (recordCount > MAX_IMPORT_RECORDS && source !== "claudeAgent") return false; const value = reader.finish(); const decoded = decodeTranscriptValue(value); if (Option.isNone(decoded) && source === "pi") { @@ -1090,7 +1141,9 @@ export const make = Effect.gen(function* () { claudeFields = foldClaudeFields(claudeFields, decoded.value); } if (shouldRetainDecodedRecord(source, decoded.value)) { - if ( + if (source === "claudeAgent") { + historyBytes += recordBytes - claudeHistory.push(decoded.value, recordBytes); + } else if ( source === "pi" && decoded.value.type === "message" && !isPiConversationRecord(decoded.value) @@ -1148,7 +1201,13 @@ export const make = Effect.gen(function* () { ) return null; return sameTranscriptIdentity(expected, transcriptIdentity(filePath, yield* file.stat)) - ? { records, cwd, claudeFields, historyBytes } + ? { + records: source === "claudeAgent" ? claudeHistory.records() : records, + droppedMessageCount: claudeHistory.droppedMessageCount(), + cwd, + claudeFields, + historyBytes, + } : null; }), ), @@ -1242,7 +1301,7 @@ export const make = Effect.gen(function* () { mtimeMs: stats.value.mtime.value.getTime(), providerInstanceId, homePath, - size: Number(stats.value.size), + identity: transcriptIdentity(filePath, stats.value), }); } } @@ -1250,6 +1309,47 @@ export const make = Effect.gen(function* () { }, ); + const readCodexProjectRoots = Effect.fn("AgentSessionScanner.readCodexProjectRoots")(function* ( + homePath: string, + ) { + const rootsByThread = new Map>(); + const databasePath = path.join(homePath, CODEX_STATE_DATABASE); + return yield* Effect.try({ + try: () => { + const database = new NodeSqlite.DatabaseSync(databasePath, { readOnly: true }); + try { + database.exec("PRAGMA busy_timeout = 100"); + const rows = database + .prepare( + `SELECT threads.id AS id, project_roots.path AS path + FROM threads JOIN project_roots ON project_roots.project_id = threads.project_id + ORDER BY threads.rowid DESC, project_roots.position LIMIT ?`, + ) + .all(MAX_CODEX_PROJECT_ROOT_ROWS); + for (const { id, path: root } of rows) { + if (typeof id !== "string" || typeof root !== "string" || root.trim().length === 0) { + continue; + } + const roots = rootsByThread.get(id); + if (roots === undefined) rootsByThread.set(id, [root]); + else roots.push(root); + } + return rootsByThread; + } finally { + database.close(); + } + }, + catch: (cause) => new CodexStateReadError({ databasePath, cause }), + }).pipe(Effect.orElseSucceed((): ReadonlyMap> => new Map())); + }); + + // Only chats left in an excluded folder, such as Codex's scratch folder, take their project's root. + const projectCwd = (cwd: string, projectRoots: ReadonlyArray | undefined) => + projectRoots !== undefined && + isExcludedProjectPath(path.resolve(expandHomePath(cwd.trim(), homeDir))) + ? projectRoots[0]! + : cwd; + const discoverCodexTranscripts = Effect.fn("AgentSessionScanner.discoverCodexTranscripts")( function* (homePath: string, providerInstanceId: ProviderInstanceId, operationBudget: number) { const sessionsDir = path.join(homePath, "sessions"); @@ -1304,14 +1404,22 @@ export const make = Effect.gen(function* () { mtimeMs: stats.value.mtime.value.getTime(), providerInstanceId, homePath, - size: Number(stats.value.size), + identity: transcriptIdentity(filePath, stats.value), }); } } } } } - return { transcripts, truncated }; + const projectRoots = yield* readCodexProjectRoots(homePath); + return { + transcripts: transcripts.map((transcript) => { + const threadId = CODEX_ROLLOUT_ID_PATTERN.exec(transcript.filePath)?.[1]; + const roots = threadId === undefined ? undefined : projectRoots.get(threadId); + return roots === undefined ? transcript : { ...transcript, projectRoots: roots }; + }), + truncated, + }; }, ); @@ -1361,7 +1469,7 @@ export const make = Effect.gen(function* () { mtimeMs: stats.value.mtime.value.getTime(), providerInstanceId, homePath: sessionsDir, - size: Number(stats.value.size), + identity: transcriptIdentity(filePath, stats.value), }); } } @@ -1373,6 +1481,10 @@ export const make = Effect.gen(function* () { source: AgentSessionSource, transcripts: ReadonlyArray, budget: MetadataReadBudget, + importedByFile: ReadonlyMap< + string, + ReadonlyArray<{ readonly cwd: string; readonly source: AgentSessionImportSource }> + >, ) { const byOwnerAndCwd = new Map< string, @@ -1381,13 +1493,25 @@ export const make = Effect.gen(function* () { providerInstanceId: ProviderInstanceId; homePath: string; lastActiveAtMs: number; - transcripts: Array<{ filePath: string; mtimeMs: number }>; + transcripts: Array; } >(); for (const transcript of transcripts) { - const cwd = yield* readCwd(source, transcript, budget); - if (cwd === null || cwd === false) continue; + // Unchanged imported transcripts keep their import cwd, so they never spend the read budget. + // Codex is excluded: its first record carries the cwd and the subagent marker, and + // rollouts imported before subagents were skipped must still be excluded by it. + const rolloutCwd = + (source === "codex" + ? undefined + : importedByFile.get(`${transcript.providerInstanceId}\0${transcript.filePath}`) + )?.find( + (imported) => + imported.source.provider === source && + sameTranscriptIdentity(imported.source, transcript.identity), + )?.cwd ?? (yield* readCwd(source, transcript, budget)); + if (rolloutCwd === null || rolloutCwd === false) continue; + const cwd = projectCwd(rolloutCwd, transcript.projectRoots); const key = `${transcript.providerInstanceId}\0${cwd}`; const existing = byOwnerAndCwd.get(key); if (existing) { @@ -1420,8 +1544,25 @@ export const make = Effect.gen(function* () { Effect.mapError((cause) => new AgentSessionScanError({ operation: "read-settings", cause })), ); + const runtimeRows = yield* runtimes + .list() + .pipe( + Effect.mapError( + (cause) => new AgentSessionScanError({ operation: "read-projects", cause }), + ), + ); + const importedByFile = Map.groupBy( + runtimeRows.flatMap((runtime) => { + const payload = decodeImportedTranscriptPayload(runtime.runtimePayload); + if (Option.isNone(payload) || payload.value.cwd === undefined) return []; + const cwd = payload.value.cwd; + return (payload.value.importedTranscripts ?? []).map((source) => ({ cwd, source })); + }), + ({ source }) => `${source.providerInstanceId}\0${source.filePath}`, + ); const raw: Array = []; let truncated = false; + let metadataTruncated = false; for (const source of ["claudeAgent", "codex", "pi"] as const) { const instances: Array<{ @@ -1532,18 +1673,25 @@ export const make = Effect.gen(function* () { recordsRemaining: MAX_METADATA_RECORDS_PER_SOURCE, truncated: false, }; - raw.push(...(yield* groupTranscriptsByCwd(source, selectedTranscripts, metadataBudget))); - truncated ||= metadataBudget.truncated; + raw.push( + ...(yield* groupTranscriptsByCwd( + source, + selectedTranscripts, + metadataBudget, + importedByFile, + )), + ); + metadataTruncated ||= metadataBudget.truncated; } - return { candidates: raw, truncated }; + return { candidates: raw, truncated: truncated || metadataTruncated, metadataTruncated }; }); - let cachedCandidates: ReadonlyArray | null = null; + let cachedCandidates: Effect.Success> | null = null; const scan: AgentSessionScanner["Service"]["scan"] = Effect.gen(function* () { - const { candidates: raw, truncated } = yield* collectCandidates(); - cachedCandidates = raw; + cachedCandidates = yield* collectCandidates(); + const { candidates: raw, truncated } = cachedCandidates; const merged = new Map< string, @@ -1683,8 +1831,11 @@ export const make = Effect.gen(function* () { const nowMs = DateTime.toEpochMillis(yield* DateTime.now); const cutoffMs = nowMs - RECENT_THREAD_WINDOW_MS; - const candidates = cachedCandidates ?? (yield* collectCandidates()).candidates; - cachedCandidates = candidates; + // Imports recorded since a budget-truncated collection can free budget for transcripts it cut off. + if (cachedCandidates === null || cachedCandidates.metadataTruncated) { + cachedCandidates = yield* collectCandidates(); + } + const { candidates } = cachedCandidates; const eligibleTranscripts: Array<{ readonly candidate: RawCandidate; @@ -1734,7 +1885,8 @@ export const make = Effect.gen(function* () { candidates.flatMap((candidate) => candidate.source === "codex" && candidate.providerInstanceId === providerInstanceId ? candidate.transcripts.flatMap((transcript) => { - const id = CODEX_ROLLOUT_ID_PATTERN.exec(transcript.filePath)?.[1]; + const match = CODEX_ROLLOUT_ID_PATTERN.exec(transcript.filePath); + const id = match?.[2] ?? match?.[1]; return id === undefined ? [] : [[id, transcript.filePath] as const]; }) : [], @@ -1820,7 +1972,10 @@ export const make = Effect.gen(function* () { if (Option.isNone(snapshot.cwd)) { return Option.some({ _tag: "Skipped" }); } - const expandedCwd = expandHomePath(snapshot.cwd.value.trim(), homeDir); + const expandedCwd = expandHomePath( + projectCwd(snapshot.cwd.value, transcript.projectRoots).trim(), + homeDir, + ); if ( !path.isAbsolute(expandedCwd) || (yield* directoryIdentity(path.resolve(expandedCwd))) !== rootIdentity @@ -1847,6 +2002,7 @@ export const make = Effect.gen(function* () { ? snapshot.records : [...inheritedRecords, ...snapshot.records], snapshot.claudeFields, + snapshot.droppedMessageCount, ); if (parsedThread === null) { return Option.some({ _tag: "Skipped" }); diff --git a/apps/server/src/project/ManagedProjectFolders.test.ts b/apps/server/src/project/ManagedProjectFolders.test.ts index bf6f43a3fdb3..011b54a6851b 100644 --- a/apps/server/src/project/ManagedProjectFolders.test.ts +++ b/apps/server/src/project/ManagedProjectFolders.test.ts @@ -8,9 +8,11 @@ import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Logger from "effect/Logger"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as PlatformError from "effect/PlatformError"; +import { ChildProcessSpawner } from "effect/process"; import * as Scope from "effect/Scope"; import * as ServerConfig from "../config.ts"; @@ -54,7 +56,7 @@ const layerRealGit = GitVcsDriver.layer.pipe(Layer.provide(VcsProcess.layer)); interface HarnessOptions { readonly vcsProcess?: Layer.Layer; - readonly git?: Layer.Layer; + readonly git?: typeof layerRealGit; readonly projects?: ( real: ProjectService.ProjectService["Service"], ) => ProjectService.ProjectService["Service"]; @@ -111,23 +113,38 @@ const requireRoot = Effect.gen(function* () { return Option.getOrThrow(yield* scratch.scratchRoot); }); -it.effect("offers nothing when the data dir sits inside a Git checkout", () => - Effect.gen(function* () { +it.effect("offers nothing, and says why once, when the data dir sits inside a Git checkout", () => { + const warnings: string[] = []; + const logger = Logger.make(({ logLevel, message }) => { + if (logLevel === "Warn") warnings.push(String(message)); + }); + return Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; const checkout = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-scratch-repo-" }); yield* git(checkout, ["init", "--quiet"]); const baseDir = path.join(checkout, ".t3"); yield* fileSystem.makeDirectory(baseDir); + const repositoryRoot = (yield* git(baseDir, ["rev-parse", "--show-toplevel"])).stdout.trim(); yield* Effect.gen(function* () { const scratch = yield* ManagedProjectFolders.ManagedProjectFolders; assert.isTrue(Option.isNone(yield* scratch.scratchRoot)); assert.isFalse(yield* scratch.isScratchProject(ProjectId.make("project:any"))); const failure = yield* Effect.flip(scratch.ensureScratchProject); assert.equal(failure._tag, "ScratchUnavailableError"); - }).pipe(Effect.provide(layer(baseDir))); - }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), -); + assert.include(failure.message, baseDir); + // The root prefixes the data dir, so it must appear on its own. + assert.include(failure.message.replaceAll(baseDir, ""), repositoryRoot); + yield* Effect.flip(scratch.ensureScratchProject); + }).pipe( + Effect.provide( + Layer.mergeAll(layer(baseDir), Logger.layer([logger], { mergeWithExisting: false })), + ), + ); + assert.equal(warnings.length, 1); + assert.include(warnings[0], baseDir); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))); +}); it.effect("offers a Scratch folder when Git is not installed", () => withScratch( @@ -290,6 +307,31 @@ it.effect("leaves threads in other projects alone", () => ), ); +it.effect("still knows No project when its folder is the target of a symlinked Scratch", () => + withScratch(({ baseDir }) => + Effect.gen(function* () { + const scratch = yield* ManagedProjectFolders.ManagedProjectFolders; + const projects = yield* ProjectService.ProjectService; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* requireRoot; + const target = path.join(baseDir, "visible-chats"); + yield* fileSystem.makeDirectory(target); + yield* fileSystem.symlink(target, root); + const { projectId } = yield* scratch.ensureScratchProject; + yield* projects.update({ + commandId: CommandId.make("command:point-at-target"), + projectId, + workspaceRoot: target, + }); + + assert.isTrue(yield* scratch.isScratchProject(projectId)); + assert.equal((yield* scratch.ensureScratchProject).projectId, projectId); + assert.deepStrictEqual(yield* scratch.scratchRoot, Option.some(target)); + }), + ), +); + const GIT_IDENTITY_KEYS = [ "GIT_AUTHOR_NAME", "GIT_AUTHOR_EMAIL", @@ -434,6 +476,41 @@ it.effect("keeps a named project and reports why when Git cannot commit", () => ), ); +it.effect("says Git is not installed when a named project cannot start Git", () => + withScratch( + () => + Effect.gen(function* () { + const folders = yield* ManagedProjectFolders.ManagedProjectFolders; + + const failure = yield* Effect.flip(folders.createNamedProject({ name: "T3Code Test" })); + + assert.strictEqual( + failure.message, + "Git isn't installed on this machine. Install Git to create a project.", + ); + }), + { + git: GitVcsDriver.layer.pipe( + Layer.provide( + Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.fail( + PlatformError.systemError({ + _tag: "NotFound", + module: "ChildProcess", + method: "spawn", + syscall: "spawn git", + }), + ), + ), + ), + ), + ), + }, + ), +); + it.effect("keeps a folder that another project owns when the create conflicts", () => withScratch(({ baseDir }) => withGitEnv( diff --git a/apps/server/src/project/ManagedProjectFolders.ts b/apps/server/src/project/ManagedProjectFolders.ts index 4b0991e1a64c..27316997077b 100644 --- a/apps/server/src/project/ManagedProjectFolders.ts +++ b/apps/server/src/project/ManagedProjectFolders.ts @@ -1,5 +1,6 @@ import { CommandId, + GitCommandError, type GitManagerServiceError, ProjectId, type ThreadId, @@ -18,6 +19,7 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as PlatformError from "effect/PlatformError"; +import * as Result from "effect/Result"; import * as Schema from "effect/Schema"; import * as ServerConfig from "../config.ts"; @@ -27,10 +29,16 @@ import * as ProjectService from "./ProjectService.ts"; export class ScratchUnavailableError extends Schema.TaggedError()( "ScratchUnavailableError", - {}, + { + baseDir: Schema.String, + repositoryRoot: Schema.optional(Schema.String), + }, ) { override get message(): string { - return "Threads without a project are not available on this environment."; + if (this.repositoryRoot === undefined) { + return "Threads without a project are not available on this environment."; + } + return `Threads without a project are off because the T3 home ${this.baseDir} is inside the Git repository ${this.repositoryRoot}. If that repository was created by mistake, move ${this.repositoryRoot}/.git out of the way; otherwise point T3CODE_HOME (or --base-dir) at a folder outside it. Then restart T3 Code.`; } } @@ -60,6 +68,14 @@ export class ScratchProjectError extends Schema.TaggedError export type ScratchError = ScratchUnavailableError | ScratchFolderError | ScratchProjectError; +// The spawner checks cwd before spawning, so NotFound from spawn itself means a missing executable. +const isMissingExecutable = (cause: unknown) => + PlatformError.isPlatformError(cause) && + cause.reason._tag === "NotFound" && + cause.reason.method === "spawn"; + +const isGitCommandError = Schema.is(GitCommandError); + export class NamedProjectFolderError extends Schema.TaggedError()( "NamedProjectFolderError", { @@ -68,7 +84,9 @@ export class NamedProjectFolderError extends Schema.TaggedError Effect.succeed(false)), - Effect.map((isRepository) => - isRepository ? Option.none() : Option.some(path.resolve(config.baseDir, "scratch")), + gitWorkflow.repositoryRoot(config.baseDir).pipe( + Effect.catchIf(isGitMissing, () => Effect.succeed(null)), + Effect.flatMap( + (repositoryRoot): Effect.Effect> => { + if (repositoryRoot === null) { + return Effect.succeed(Result.succeed(path.resolve(config.baseDir, "scratch"))); + } + const unavailable = new ScratchUnavailableError({ + baseDir: config.baseDir, + repositoryRoot, + }); + return Effect.logWarning(unavailable.message).pipe(Effect.as(Result.fail(unavailable))); + }, ), Effect.catchCause((cause) => - Cause.hasInterrupts(cause) ? Effect.interrupt : Effect.succeed(Option.none()), + Cause.hasInterrupts(cause) + ? Effect.interrupt + : Effect.succeed(Result.fail(new ScratchUnavailableError({ baseDir: config.baseDir }))), ), ), Duration.infinity, ); - const scratchRoot = probe.pipe(Effect.onInterrupt(() => invalidate)); + const scratchProbe = probe.pipe(Effect.onInterrupt(() => invalidate)); + // A symlinked scratch folder lets the No project live at either the link or its target. + const scratchProjectRoot = (root: string) => + fileSystem.realPath(root).pipe( + Effect.flatMap((target) => + target === root + ? Effect.succeed(root) + : projects + .getByWorkspaceRoot(target) + .pipe(Effect.map((project) => (Option.isSome(project) ? target : root))), + ), + Effect.orElseSucceed(() => root), + ); + const scratchRoot = scratchProbe.pipe( + Effect.flatMap((probed) => + Result.isSuccess(probed) + ? scratchProjectRoot(probed.success).pipe(Effect.asSome) + : Effect.succeedNone, + ), + ); const makeScratchFolder = (folder: string) => fileSystem @@ -252,15 +300,9 @@ const make = Effect.gen(function* () { const ensureScratchProject: ManagedProjectFolders["Service"]["ensureScratchProject"] = Effect.gen( function* () { - const workspaceRoot = yield* scratchRoot.pipe( - Effect.flatMap( - Option.match({ - onNone: () => Effect.fail(new ScratchUnavailableError()), - onSome: Effect.succeed, - }), - ), - ); - yield* makeScratchFolder(workspaceRoot); + const probedRoot = yield* scratchProbe.pipe(Effect.flatMap(Effect.fromResult)); + yield* makeScratchFolder(probedRoot); + const workspaceRoot = yield* scratchProjectRoot(probedRoot); const id = yield* random .uuidV4() .pipe(Effect.mapError((cause) => new ScratchProjectError({ workspaceRoot, cause }))); diff --git a/apps/server/src/project/ProjectCloneTracker.test.ts b/apps/server/src/project/ProjectCloneTracker.test.ts index ed775e55a985..0d0a614e467a 100644 --- a/apps/server/src/project/ProjectCloneTracker.test.ts +++ b/apps/server/src/project/ProjectCloneTracker.test.ts @@ -16,6 +16,7 @@ import * as TestClock from "effect/testing/TestClock"; import * as SourceControlRepositoryService from "../sourceControl/SourceControlRepositoryService.ts"; import * as ProjectCloneTracker from "./ProjectCloneTracker.ts"; +import { ProjectConflictError } from "./ProjectService.ts"; import { parseGitCloneProgressLine } from "./gitCloneProgress.ts"; const projectId = ProjectId.make("project-1"); @@ -282,6 +283,35 @@ describe("ProjectCloneTracker", () => { }).pipe(Effect.provide(harness.layer)); }); + it.effect("clones for the project that already owns the empty destination", () => { + const owner = ProjectId.make("project-stranded"); + const harness = makeHarness(); + const hooks: ProjectCloneTracker.ProjectCloneHooks = { + ...harness.hooks, + createProject: (input) => + Effect.fail( + new ProjectConflictError({ + projectId: input.projectId, + workspaceRoot: input.workspaceRoot, + conflictingProjectId: owner, + }), + ), + }; + return Effect.gen(function* () { + const tracker = yield* ProjectCloneTracker.ProjectCloneTracker; + const result = yield* tracker.start(startInput, hooks); + expect(result.projectId).toBe(owner); + yield* Effect.yieldNow; + expect(yield* tracker.get(owner)).toMatchObject({ + projectId: owner, + destinationPath: "/workspace/t3code", + phase: "done", + }); + expect(harness.cloned).toEqual([owner]); + expect(yield* tracker.get(projectId)).toBeNull(); + }).pipe(Effect.provide(harness.layer)); + }); + it.effect("does not create a project when the clone cannot be prepared", () => { const harness = makeHarness(); const layer = ProjectCloneTracker.layer.pipe( diff --git a/apps/server/src/project/ProjectCloneTracker.ts b/apps/server/src/project/ProjectCloneTracker.ts index 0a512bde2aa9..0a24f11ffd89 100644 --- a/apps/server/src/project/ProjectCloneTracker.ts +++ b/apps/server/src/project/ProjectCloneTracker.ts @@ -31,6 +31,7 @@ import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import * as SourceControlRepositoryService from "../sourceControl/SourceControlRepositoryService.ts"; +import type { ProjectConflictError } from "./ProjectService.ts"; export class ProjectCloneTracker extends Context.Service< ProjectCloneTracker, @@ -53,12 +54,13 @@ export class ProjectCloneTracker extends Context.Service< >()("t3/project/ProjectCloneTracker") {} export interface ProjectCloneHooks { + /** Fails with `ProjectConflictError` when another project already owns the destination. */ readonly createProject: (input: { readonly projectId: ProjectId; readonly title: string; readonly workspaceRoot: string; readonly createdAt: string; - }) => Effect.Effect; + }) => Effect.Effect; readonly onCloned: (input: { readonly projectId: ProjectId; readonly workspaceRoot: string; @@ -266,6 +268,16 @@ export const make = Effect.gen(function* () { }); }); + const moveClaim = (from: ProjectId, to: ProjectId) => + Ref.modify(clones, (current) => { + const claim = current.get(from); + if (!claim || current.has(to)) return [false, current] as const; + const next = new Map(current); + next.delete(from); + next.set(to, { ...claim, snapshot: { ...claim.snapshot, projectId: to } }); + return [true, next] as const; + }); + const start: ProjectCloneTracker["Service"]["start"] = Effect.fn("ProjectCloneTracker.start")( function* (input, hooks) { const prepared = yield* repositories.prepareClone(input); @@ -306,30 +318,44 @@ export const make = Effect.gen(function* () { }); return [true, next] as const; }); - if (!claimed) { - return yield* new SourceControlRepositoryError({ - operation: "cloneRepository", - provider: input.provider ?? SourceControlProviderKind.make("unknown"), - detail: "A clone into this destination is already in progress.", - }); - } + const alreadyInProgress = new SourceControlRepositoryError({ + operation: "cloneRepository", + provider: input.provider ?? SourceControlProviderKind.make("unknown"), + detail: "A clone into this destination is already in progress.", + }); + if (!claimed) return yield* alreadyInProgress; // Finish or release the claim despite disconnects; a claim without a fiber cannot be cancelled or retried. - yield* Effect.uninterruptible( + const projectId = yield* Effect.uninterruptible( Effect.gen(function* () { yield* clearRetention(input.projectId); // Register before creation so racing thread.create and project.delete see the clone. - yield* hooks.createProject({ - projectId: input.projectId, - title: input.title, - workspaceRoot: prepared.destinationPath, - createdAt: input.createdAt, - }); + const projectId = yield* hooks + .createProject({ + projectId: input.projectId, + title: input.title, + workspaceRoot: prepared.destinationPath, + createdAt: input.createdAt, + }) + .pipe( + Effect.as(input.projectId), + // A failed clone's project still owns its empty folder after a restart + // forgets the clone; clone for that project instead. + Effect.catchTags({ + ProjectConflictError: (conflict) => + moveClaim(input.projectId, conflict.conflictingProjectId).pipe( + Effect.flatMap((moved) => + moved ? Effect.succeed(conflict.conflictingProjectId) : alreadyInProgress, + ), + ), + }), + ); yield* publish; - yield* launch(input.projectId); + yield* launch(projectId); + return projectId; }).pipe(Effect.tapError(() => remove(input.projectId))), ); return { - projectId: input.projectId, + projectId, cwd: prepared.destinationPath, remoteUrl: prepared.remoteUrl, repository: prepared.repository, diff --git a/apps/server/src/project/ProjectEnrichmentService.test.ts b/apps/server/src/project/ProjectEnrichmentService.test.ts index 6da9ae31203b..1db904c3b4c4 100644 --- a/apps/server/src/project/ProjectEnrichmentService.test.ts +++ b/apps/server/src/project/ProjectEnrichmentService.test.ts @@ -1,10 +1,13 @@ import { assert, it } from "@effect/vitest"; import type { RepositoryIdentity } from "@t3tools/contracts"; import * as Deferred from "effect/Deferred"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as PubSub from "effect/PubSub"; import * as Ref from "effect/Ref"; +import * as TestClock from "effect/testing/TestClock"; import * as ProjectEnrichment from "./ProjectEnrichmentService.ts"; import * as ProjectFaviconResolver from "./ProjectFaviconResolver.ts"; @@ -257,3 +260,131 @@ it.effect("deduplicates requests, bounds pending work, and reloads invalidated r ); }), ); + +it.effect( + "keeps the last resolved repository identity after it expires and while it refreshes", + () => + Effect.gen(function* () { + const refreshStarted = yield* Deferred.make(); + const releaseRefresh = yield* Deferred.make(); + const resolveCalls = yield* Ref.make(0); + const layerMetadata = Layer.merge( + Layer.succeed(RepositoryIdentityResolver.RepositoryIdentityResolver, { + resolve: (workspaceRoot) => + Effect.gen(function* () { + if ((yield* Ref.updateAndGet(resolveCalls, (count) => count + 1)) > 1) { + yield* Deferred.succeed(refreshStarted, undefined); + yield* Deferred.await(releaseRefresh); + } + return identity(workspaceRoot); + }), + }), + Layer.succeed(ProjectFaviconResolver.ProjectFaviconResolver, { + invalidate: () => Effect.void, + resolvePath: () => Effect.succeed(null), + }), + ); + + yield* Effect.gen(function* () { + const service = yield* ProjectEnrichment.ProjectEnrichmentService; + const changes = yield* service.subscribeChanges; + yield* service.request("/widgets"); + const resolved = yield* PubSub.take(changes); + assert.equal( + resolved.enrichment.repositoryIdentity?.canonicalKey, + "example.test/v1/widgets", + ); + + yield* TestClock.adjust(Duration.seconds(61)); + + const expired = yield* service.getAvailable("/widgets"); + assert.equal( + expired.repositoryIdentity?.canonicalKey, + "example.test/v1/widgets", + "an expired repository identity should still be returned until a refresh replaces it", + ); + + yield* Deferred.await(refreshStarted); + const refreshing = yield* service.peek("/widgets"); + assert.equal( + refreshing.repositoryIdentity?.canonicalKey, + "example.test/v1/widgets", + "a repository identity should still be returned while its refresh is pending", + ); + }).pipe(Effect.provide(layer(layerMetadata))); + }), +); + +it.effect("does not let a lookup invalidated mid-flight replace a newer last identity", () => + Effect.gen(function* () { + const firstStarted = yield* Deferred.make(); + const releaseFirst = yield* Deferred.make(); + const resolveCalls = yield* Ref.make(0); + const layerMetadata = Layer.merge( + Layer.succeed(RepositoryIdentityResolver.RepositoryIdentityResolver, { + resolve: (workspaceRoot) => + Effect.gen(function* () { + const call = yield* Ref.updateAndGet(resolveCalls, (count) => count + 1); + if (call === 1) { + yield* Deferred.succeed(firstStarted, undefined); + yield* Deferred.await(releaseFirst); + } + return identity(workspaceRoot, call); + }), + }), + Layer.succeed(ProjectFaviconResolver.ProjectFaviconResolver, { + invalidate: () => Effect.void, + resolvePath: () => Effect.succeed(null), + }), + ); + + yield* Effect.gen(function* () { + const service = yield* ProjectEnrichment.ProjectEnrichmentService; + const firstLookup = yield* Effect.forkChild(service.readRepositoryIdentity("/widgets")); + yield* Deferred.await(firstStarted); + yield* service.invalidate(["/widgets"]); + const newer = yield* service.readRepositoryIdentity("/widgets"); + assert.equal(newer?.canonicalKey, "example.test/v2/widgets"); + + yield* Deferred.succeed(releaseFirst, undefined); + yield* Fiber.join(firstLookup); + yield* TestClock.adjust(Duration.seconds(61)); + + const expired = yield* service.peek("/widgets"); + assert.equal(expired.repositoryIdentity?.canonicalKey, "example.test/v2/widgets"); + }).pipe(Effect.provide(layer(layerMetadata))); + }), +); + +it.effect("keeps a lookup's identity when another root is invalidated mid-flight", () => + Effect.gen(function* () { + const started = yield* Deferred.make(); + const release = yield* Deferred.make(); + const layerMetadata = Layer.merge( + Layer.succeed(RepositoryIdentityResolver.RepositoryIdentityResolver, { + resolve: (workspaceRoot) => + Deferred.succeed(started, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.as(identity(workspaceRoot)), + ), + }), + Layer.succeed(ProjectFaviconResolver.ProjectFaviconResolver, { + invalidate: () => Effect.void, + resolvePath: () => Effect.succeed(null), + }), + ); + + yield* Effect.gen(function* () { + const service = yield* ProjectEnrichment.ProjectEnrichmentService; + const lookup = yield* Effect.forkChild(service.readRepositoryIdentity("/widgets")); + yield* Deferred.await(started); + yield* service.invalidate(["/other"]); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(lookup); + yield* TestClock.adjust(Duration.seconds(61)); + + const expired = yield* service.peek("/widgets"); + assert.equal(expired.repositoryIdentity?.canonicalKey, "example.test/v1/widgets"); + }).pipe(Effect.provide(layer(layerMetadata))); + }), +); diff --git a/apps/server/src/project/ProjectEnrichmentService.ts b/apps/server/src/project/ProjectEnrichmentService.ts index 538beed6f1f0..c08bba24bcce 100644 --- a/apps/server/src/project/ProjectEnrichmentService.ts +++ b/apps/server/src/project/ProjectEnrichmentService.ts @@ -51,6 +51,8 @@ export class ProjectEnrichmentService extends Context.Service< workspaceRoot: string, ) => Effect.Effect; readonly invalidate: (workspaceRoots: Iterable) => Effect.Effect; + /** Re-resolves past both identity caches and publishes a found identity, leaving the favicon alone. */ + readonly refreshRepositoryIdentity: (workspaceRoot: string) => Effect.Effect; readonly subscribeChanges: Effect.Effect< PubSub.Subscription, never, @@ -91,8 +93,44 @@ export const make = Effect.fn("ProjectEnrichmentService.make")(function* ( const maxPending = Math.max(1, options.maxPending ?? DEFAULT_MAX_PENDING); const concurrency = Math.max(1, options.concurrency ?? DEFAULT_CONCURRENCY); + // Outlives the cache entry so an expired or refreshing identity still reads as its last value. + const lastRepositoryIdentities = new Map(); + // `invalidate` drops a root's in-flight lookup so its late result cannot overwrite a newer one. + const lookupsInFlight = new Map(); + const rememberRepositoryIdentity = ( + workspaceRoot: string, + repositoryIdentity: RepositoryIdentity | null, + ) => { + lastRepositoryIdentities.delete(workspaceRoot); + if (repositoryIdentity === null) return; + if (lastRepositoryIdentities.size >= cacheCapacity) { + const oldestRoot = lastRepositoryIdentities.keys().next().value; + if (oldestRoot !== undefined) lastRepositoryIdentities.delete(oldestRoot); + } + lastRepositoryIdentities.set(workspaceRoot, repositoryIdentity); + }; + const repositoryIdentityCache = yield* Cache.makeWith( - (workspaceRoot: string) => Effect.exit(repositoryIdentityResolver.resolve(workspaceRoot)), + (workspaceRoot: string) => + Effect.suspend(() => { + const lookup = {}; + lookupsInFlight.set(workspaceRoot, lookup); + const isCurrent = () => lookupsInFlight.get(workspaceRoot) === lookup; + return Effect.exit(repositoryIdentityResolver.resolve(workspaceRoot)).pipe( + Effect.tap((result) => + Effect.sync(() => { + if (Exit.isSuccess(result) && isCurrent()) { + rememberRepositoryIdentity(workspaceRoot, result.value); + } + }), + ), + Effect.ensuring( + Effect.sync(() => { + if (isCurrent()) lookupsInFlight.delete(workspaceRoot); + }), + ), + ); + }), { capacity: cacheCapacity, timeToLive: Exit.match({ @@ -238,10 +276,13 @@ export const make = Effect.fn("ProjectEnrichmentService.make")(function* ( ] as const, { concurrency: "unbounded" }, ); + const repositoryIdentityResolved = isSuccessfullyResolved(repositoryIdentity); return { - repositoryIdentity: availableValue(repositoryIdentity), + repositoryIdentity: repositoryIdentityResolved + ? availableValue(repositoryIdentity) + : (lastRepositoryIdentities.get(workspaceRoot) ?? null), faviconPath: availableValue(faviconPath), - repositoryIdentityResolved: isSuccessfullyResolved(repositoryIdentity), + repositoryIdentityResolved, }; }, ); @@ -287,6 +328,10 @@ export const make = Effect.fn("ProjectEnrichmentService.make")(function* ( Effect.all( [ Cache.invalidate(repositoryIdentityCache, workspaceRoot), + Effect.sync(() => { + lookupsInFlight.delete(workspaceRoot); + lastRepositoryIdentities.delete(workspaceRoot); + }), Cache.invalidate(faviconCache, workspaceRoot), // The resolver keeps its own misses, which would outlive a clone filling the folder. faviconResolver.invalidate(workspaceRoot), @@ -297,12 +342,24 @@ export const make = Effect.fn("ProjectEnrichmentService.make")(function* ( ); }); + const refreshRepositoryIdentity: ProjectEnrichmentService["Service"]["refreshRepositoryIdentity"] = + Effect.fn("ProjectEnrichmentService.refreshRepositoryIdentity")(function* (workspaceRoot) { + // Remote-less folders hit this after every turn; publishing their unchanged null would + // push a shell frame to every subscriber. + if ((yield* repositoryIdentityResolver.resolve(workspaceRoot, { refresh: true })) === null) { + return; + } + yield* Cache.invalidate(repositoryIdentityCache, workspaceRoot); + yield* resolveRepositoryIdentity(workspaceRoot); + }); + return ProjectEnrichmentService.of({ peek, request, getAvailable, readRepositoryIdentity, invalidate, + refreshRepositoryIdentity, subscribeChanges: PubSub.subscribe(changes), }); }); diff --git a/apps/server/src/project/ProjectService.test.ts b/apps/server/src/project/ProjectService.test.ts index 11290ace3947..ee8020002a2f 100644 --- a/apps/server/src/project/ProjectService.test.ts +++ b/apps/server/src/project/ProjectService.test.ts @@ -200,8 +200,8 @@ const layerProjectServiceDependencies = Layer.mergeAll( Layer.provideMerge(layerMetadata), Layer.provideMerge(SqlitePersistence.layerMemory), Layer.provideMerge(ServerSettings.layerTest()), - Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "project-service-race-" })), - Layer.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), + Layer.provideMerge(ServerConfig.layerTest(process.cwd(), { prefix: "project-service-race-" })), + Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), ); const waitForProject = Effect.fn("ProjectServiceTest.waitForProject")(function* ( diff --git a/apps/server/src/project/ProjectService.ts b/apps/server/src/project/ProjectService.ts index aa2bd364b40a..8b78f4d76675 100644 --- a/apps/server/src/project/ProjectService.ts +++ b/apps/server/src/project/ProjectService.ts @@ -12,6 +12,7 @@ import * as KeyedLock from "@t3tools/shared/KeyedLock"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Result from "effect/Result"; @@ -31,6 +32,7 @@ import * as ProjectStore from "../orchestration-v2/ProjectStore.ts"; import * as ThreadCommandExecutor from "../orchestration-v2/ThreadCommandExecutor.ts"; import { planThreadDeletion } from "../orchestration-v2/ThreadDeletion.ts"; import * as ServerSettings from "../serverSettings.ts"; +import { makeIsManagedWorktreePath } from "../worktreesDirectory.ts"; import * as ProjectEnrichmentService from "./ProjectEnrichmentService.ts"; import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; @@ -141,6 +143,11 @@ export class ProjectService extends Context.Service< workspaceRoot: string, options?: { readonly includeDeleted?: boolean }, ) => Effect.Effect, ProjectOperationError>; + /** + * Where to scan `cwd`'s workspace inventory. A thread's removed managed worktree scans its + * project, which turn start recreates the worktree from; anything else, or a failure, is `cwd`. + */ + readonly workspaceScanDirectory: (cwd: string) => Effect.Effect; readonly snapshot: Effect.Effect; /** * A page of active projects in creation order, each with its repository @@ -172,6 +179,8 @@ export const make = Effect.gen(function* () { const legacyImporter = yield* LegacyV1ThreadImporter.LegacyV1ThreadImporter; const threadCommands = yield* ThreadCommandExecutor.ThreadCommandExecutor; const serverSettings = yield* ServerSettings.ServerSettingsService; + const fileSystem = yield* FileSystem.FileSystem; + const isManagedWorktreePath = yield* makeIsManagedWorktreePath; const projectLocks = yield* KeyedLock.make(); const workspaceLocks = yield* KeyedLock.make(); @@ -331,6 +340,22 @@ export const make = Effect.gen(function* () { }, ); + const workspaceScanDirectory = Effect.fn("ProjectService.workspaceScanDirectory")( + function* (cwd: string) { + if (yield* fileSystem.exists(cwd).pipe(Effect.orElseSucceed(() => true))) return cwd; + if (!(yield* isManagedWorktreePath(cwd))) return cwd; + const snapshot = yield* Effect.flatten( + threadProjections.readShellSnapshot({ worktreePath: cwd }), + ); + const thread = [...snapshot.threads, ...snapshot.archivedThreads].find( + (candidate) => candidate.branch !== null, + ); + if (thread === undefined) return cwd; + return Option.getOrUndefined(yield* readRow(thread.projectId))?.workspaceRoot ?? cwd; + }, + (effect, cwd) => Effect.orElseSucceed(effect, () => cwd), + ); + const getByWorkspaceRoot: ProjectService["Service"]["getByWorkspaceRoot"] = Effect.fn( "ProjectService.getByWorkspaceRoot", )(function* (workspaceRoot, options) { @@ -611,6 +636,7 @@ export const make = Effect.gen(function* () { delete: deleteProject, getById, getByWorkspaceRoot, + workspaceScanDirectory, snapshot, listPage, getShell, diff --git a/apps/server/src/project/ScratchThreadPromotion.ts b/apps/server/src/project/ScratchThreadPromotion.ts index f8d28a9fe007..91ef48adae53 100644 --- a/apps/server/src/project/ScratchThreadPromotion.ts +++ b/apps/server/src/project/ScratchThreadPromotion.ts @@ -7,6 +7,7 @@ import { import * as Context from "effect/Context"; import * as SecureRandom from "effect/crypto/SecureRandom"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; @@ -50,6 +51,9 @@ const make = Effect.gen(function* () { const scheduledTasks = yield* ScheduledTaskService.ScheduledTaskService; const random = yield* SecureRandom.SecureRandom; const path = yield* Path.Path; + const fileSystem = yield* FileSystem.FileSystem; + const realPath = (folder: string) => + fileSystem.realPath(folder).pipe(Effect.orElseSucceed(() => folder)); const fail = (reason: string) => (cause: unknown) => new ScratchThreadPromotionError({ reason, cause }); @@ -76,7 +80,7 @@ const make = Effect.gen(function* () { if ( folder === undefined || scratchRoot === undefined || - path.dirname(folder) !== scratchRoot || + (yield* realPath(path.dirname(folder))) !== (yield* realPath(scratchRoot)) || !(isScratch || alreadyPromoted) ) { return yield* new ScratchThreadPromotionError({ diff --git a/apps/server/src/provider/AcpRegistryProbe.test.ts b/apps/server/src/provider/AcpRegistryProbe.test.ts index d83be92680ae..5f51ccfc8057 100644 --- a/apps/server/src/provider/AcpRegistryProbe.test.ts +++ b/apps/server/src/provider/AcpRegistryProbe.test.ts @@ -12,6 +12,7 @@ import { ProviderInstanceId } from "@t3tools/contracts"; import { AcpRegistrySettings } from "@t3tools/provider-acp-registry/settings"; import { describe, expect, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Schema from "effect/Schema"; import * as EffectAcpErrors from "effect-acp/errors"; @@ -276,7 +277,7 @@ describe("ACP Registry probe", () => { }, ); - it("bounds and de-duplicates advertised commands", () => { + it("keeps every distinct advertised command and de-duplicates the rest", () => { const commands = normalizeAcpRegistryCommands([ { name: "create_plan", @@ -288,9 +289,11 @@ describe("ACP Registry probe", () => { name: `command_${index}`, description: "", })), + { name: "$late-skill", description: "Advertised after 128 commands" }, ]); - expect(commands.slashCommands).toHaveLength(128); + expect(commands.slashCommands).toHaveLength(141); + expect(commands.slashCommands.at(-1)).toEqual({ name: "command_139" }); expect(commands.slashCommands[0]).toEqual({ name: "create_plan", description: "Create a plan", @@ -299,7 +302,7 @@ describe("ACP Registry probe", () => { expect( commands.slashCommands.filter((command) => command.name.toLowerCase() === "create_plan"), ).toHaveLength(1); - expect(commands.skills).toEqual([]); + expect(commands.skills.map((skill) => skill.name)).toEqual(["late-skill"]); }); it("routes dollar-prefixed ACP commands to the provider skill menu", () => { @@ -399,6 +402,51 @@ describe("ACP Registry probe", () => { ); }); + it.live.each(["succeeds", "fails"] as const)( + "deletes its health-probe session when deletion is advertised and %s", + (deleteOutcome) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "acp-probe-delete-" }); + const requestLogPath = NodePath.join(directory, "requests.ndjson"); + // Coder: the catalog resolves only the local command, so the test's agent is local. + const catalog = AcpRegistrySupport.AcpRegistryCatalog.of({ + inspect: () => Effect.die("unused inspect"), + resolve: () => + Effect.succeed({ + distribution: "local" as const, + spawn: { + command: "node", + args: [mockAgentPath], + env: { + ...process.env, + T3_ACP_V2_MANAGEMENT: "1", + T3_ACP_REQUEST_LOG_PATH: requestLogPath, + ...(deleteOutcome === "fails" ? { T3_ACP_FAIL_DELETE_SESSION: "1" } : {}), + }, + }, + }), + }); + + const result = yield* probeAcpRegistryConfiguration({ + instanceId, + settings: decodeSettings({}), + cwd: process.cwd(), + environment: process.env, + }).pipe(Effect.provideService(AcpRegistrySupport.AcpRegistryCatalog, catalog)); + expect(result.probe.ready).toBe(true); + expect(result.probe.sessionManagement.canDelete).toBe(true); + + const requests = (yield* fs.readFileString(requestLogPath)) + .split("\n") + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line)); + expect(requests.filter((request) => request.method === "session/delete")).toEqual([ + expect.objectContaining({ params: { sessionId: "mock-session-1" } }), + ]); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), + ); + it("distinguishes authentication failures from generic probe failures", () => { const advertised = normalizeAcpRegistryAuthMethods([ { id: "grok-login", name: "Log in with Grok" }, diff --git a/apps/server/src/provider/AcpRegistrySessions.ts b/apps/server/src/provider/AcpRegistrySessions.ts new file mode 100644 index 000000000000..339fd21e9685 --- /dev/null +++ b/apps/server/src/provider/AcpRegistrySessions.ts @@ -0,0 +1,227 @@ +import { + AcpRegistryOperationError, + CommandId, + type AcpRegistryImportSessionInput, + type AcpRegistryImportSessionResult, + type Project, + type ProjectId, + type ProviderDriverKind, + type ProviderInstanceId, +} from "@t3tools/contracts"; +import * as AcpRegistryRuntimeCoordinator from "@t3tools/provider-acp-registry/server/AcpRegistryRuntimeCoordinator"; +import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; +import type { ProviderInstance } from "@t3tools/provider-core/server/driver"; +import * as Context from "effect/Context"; +import * as SecureRandom from "effect/crypto/SecureRandom"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Result from "effect/Result"; + +import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts"; +import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts"; +import * as AgentSessionImporter from "../project/AgentSessionImporter.ts"; +import * as ProjectService from "../project/ProjectService.ts"; +import * as ProviderInstanceRegistry from "./ProviderInstanceRegistry.ts"; +import * as ProviderRegistry from "./ProviderRegistry.ts"; + +export const importedThreadId = (input: { + readonly driver: ProviderDriverKind; + readonly instanceId: ProviderInstanceId; + readonly sessionId: string; +}) => + IdAllocator.deriveThreadFromProviderThread({ + driver: input.driver, + providerInstanceId: input.instanceId, + nativeThreadId: input.sessionId, + }); + +export class AcpRegistrySessions extends Context.Service< + AcpRegistrySessions, + { + readonly project: (projectId: ProjectId) => Effect.Effect; + readonly sessionManager: (instanceId: ProviderInstanceId) => Effect.Effect< + { + readonly instance: ProviderInstance; + readonly manager: NonNullable; + }, + AcpRegistryOperationError + >; + /** + * Binds a native ACP session to a T3 thread and copies its history in. Importing a + * session whose thread still lacks its history repairs that thread instead. + */ + readonly importSession: ( + input: AcpRegistryImportSessionInput, + ) => Effect.Effect; + } +>()("t3/provider/AcpRegistrySessions") {} + +const make = Effect.gen(function* () { + const projectService = yield* ProjectService.ProjectService; + const providerInstances = yield* ProviderInstanceRegistry.ProviderInstanceRegistry; + const providerRegistry = yield* ProviderRegistry.ProviderRegistry; + const coordinator = yield* AcpRegistryRuntimeCoordinator.AcpRegistryRuntimeCoordinator; + const threadManagement = yield* ThreadManagementService.ThreadManagementService; + const threadLaunch = yield* ThreadLaunchService.ThreadLaunchService; + const agentSessionImporter = yield* AgentSessionImporter.AgentSessionImporter; + const random = yield* SecureRandom.SecureRandom; + + const project = Effect.fn("AcpRegistrySessions.project")(function* (projectId: ProjectId) { + const found = yield* projectService.getById(projectId).pipe( + Effect.mapError( + (cause) => + new AcpRegistryOperationError({ + reason: "project_not_found", + message: `Project ${projectId} is unavailable.`, + cause, + }), + ), + ); + return yield* Effect.fromOption( + found, + () => + new AcpRegistryOperationError({ + reason: "project_not_found", + message: `Project ${projectId} was not found.`, + }), + ); + }); + + const sessionManager = Effect.fn("AcpRegistrySessions.sessionManager")(function* ( + instanceId: ProviderInstanceId, + ) { + const instance = yield* providerInstances.getInstance(instanceId); + if (instance === undefined) { + return yield* new AcpRegistryOperationError({ + reason: "instance_not_found", + message: `Provider instance ${instanceId} was not found.`, + }); + } + if (instance.acpSessionManagement === undefined) { + return yield* new AcpRegistryOperationError({ + reason: "session_list_unsupported", + message: `Provider instance ${instanceId} does not expose ACP session management.`, + }); + } + return { instance, manager: instance.acpSessionManagement }; + }); + + const importSession = Effect.fn("AcpRegistrySessions.importSession")(function* ( + input: AcpRegistryImportSessionInput, + ) { + const { workspaceRoot } = yield* project(input.projectId); + const { instance } = yield* sessionManager(input.instanceId); + const providerSnapshot = yield* instance.snapshot.getSnapshot; + if ( + providerSnapshot.nativeSessions?.canLoad !== true && + providerSnapshot.nativeSessions?.canResume !== true + ) { + return yield* new AcpRegistryOperationError({ + reason: "session_resume_unsupported", + message: "The ACP agent cannot load or resume native sessions.", + }); + } + const threadId = importedThreadId({ + driver: instance.driverKind, + instanceId: input.instanceId, + sessionId: input.sessionId, + }); + const existing = yield* threadManagement.getThreadShell(threadId).pipe( + Effect.mapError( + (cause) => + new AcpRegistryOperationError({ + reason: "session_import_failed", + message: "Could not inspect the imported ACP session mapping.", + cause, + }), + ), + ); + if (existing !== null) { + const repairing = AgentSessionImporter.isNativeHistoryPending({ + historyOrigin: existing.historyOrigin, + hasRun: existing.latestRunId !== null, + }); + yield* agentSessionImporter.importNativeHistory({ threadId, workspaceRoot }).pipe( + Effect.mapError( + (cause) => + new AcpRegistryOperationError({ + reason: "session_import_failed", + message: "Could not import the ACP session history.", + cause, + }), + ), + ); + return { threadId, imported: repairing } as const; + } + + const provider = (yield* providerRegistry.getProviders).find( + (candidate) => candidate.instanceId === input.instanceId, + ); + const model = + provider?.models.find((candidate) => candidate.isDefault)?.slug ?? + provider?.models[0]?.slug ?? + "default"; + const commandId = CommandId.make(yield* random.uuidV4()); + const launched = yield* Effect.result( + // Coder: no startup command queue; workspace startup completes before this service is + // built. + threadLaunch.launch({ + commandId, + threadId, + projectId: input.projectId, + title: input.title ?? "Imported ACP session", + modelSelection: { instanceId: input.instanceId, model }, + runtimeMode: "approval-required", + interactionMode: "default", + workspaceStrategy: { type: "root" }, + importedNativeThread: { + ref: { + driver: instance.driverKind, + nativeId: input.sessionId, + strength: "strong", + }, + metadata: { + itemIdentityVersion: 2, + ...(input.title === undefined ? {} : { title: input.title }), + ...(input.updatedAt === undefined ? {} : { updatedAt: input.updatedAt }), + }, + }, + createdBy: "user", + creationSource: "web", + }), + ); + if (Result.isFailure(launched)) { + const racedImport = yield* threadManagement.getThreadShell(threadId).pipe( + Effect.mapError( + (cause) => + new AcpRegistryOperationError({ + reason: "session_import_failed", + message: "Could not inspect the imported ACP session after launch failed.", + cause, + }), + ), + ); + if (racedImport !== null) return { threadId, imported: false } as const; + return yield* new AcpRegistryOperationError({ + reason: "session_import_failed", + message: "Could not create a T3 thread for the ACP session.", + cause: launched.failure, + }); + } + // The thread is usable without its history, so a failed read only logs. + yield* agentSessionImporter.importNativeHistory({ threadId, workspaceRoot }).pipe( + Effect.catch((cause) => + Effect.logWarning("Could not import ACP session history", { + instanceId: input.instanceId, + sessionId: input.sessionId, + cause, + }), + ), + ); + return { threadId, imported: true } as const; + }, coordinator.withSessionMutation); + + return AcpRegistrySessions.of({ project, sessionManager, importSession }); +}); + +export const layer = Layer.effect(AcpRegistrySessions, make); diff --git a/apps/server/src/provider/ClaudeModelCatalog.test.ts b/apps/server/src/provider/ClaudeModelCatalog.test.ts index 76436f729fad..c8c8abdfcf2b 100644 --- a/apps/server/src/provider/ClaudeModelCatalog.test.ts +++ b/apps/server/src/provider/ClaudeModelCatalog.test.ts @@ -3,6 +3,8 @@ import { ProviderDriverKind, ProviderInstanceId } from "@t3tools/contracts"; import { resolveProviderCatalog, type ModelManifestData } from "./ModelManifest.ts"; import { + capClaudeCatalogEffort, + type ClaudeModelCatalog, formatClaudeVersionUpgradeMessage, getClaudeCatalogModelCapabilities, normalizeClaudeCatalogEffort, @@ -230,4 +232,242 @@ describe("Claude model catalog", () => { ["claude-synthetic-next", "claude-custom-tuned"], ); }); + + it("marks effort options that would run above the cap without changing ids or defaults", () => { + const effortModel = (slug: string) => ({ + slug, + name: slug, + isCustom: false, + capabilities: { + optionDescriptors: [ + { + id: "effort", + label: "Reasoning", + type: "select" as const, + options: [ + { id: "medium", label: "Medium" }, + { id: "high", label: "High", isDefault: true }, + { id: "xhigh", label: "Extra High" }, + { id: "max", label: "Max" }, + { id: "ultracode", label: "Ultracode" }, + { id: "ultrathink", label: "Ultrathink" }, + ], + }, + ], + }, + }); + const models = [effortModel("claude-plain"), effortModel("claude-remapped")]; + const catalog: ClaudeModelCatalog = { + models: [ + { + model: models[0]!, + runtime: { effortMap: { ultracode: "xhigh", ultrathink: null } }, + compatibility: {}, + }, + { + model: models[1]!, + runtime: { effortMap: { max: "high", xhigh: "max", ultrathink: null } }, + compatibility: {}, + }, + ], + }; + const options = (cap: "high" | "xhigh") => + capClaudeCatalogEffort(catalog, models, [{ maxEffortLevel: cap }]).map((model) => { + const effort = model.capabilities?.optionDescriptors?.[0]; + return effort?.type === "select" ? effort.options : undefined; + }); + + assert.deepStrictEqual(options("high"), [ + [ + { id: "medium", label: "Medium" }, + { id: "high", label: "High", isDefault: true }, + { id: "xhigh", label: "Extra High (capped to High)" }, + { id: "max", label: "Max (capped to High)" }, + { id: "ultracode", label: "Ultracode (capped to High)" }, + { id: "ultrathink", label: "Ultrathink" }, + ], + [ + { id: "medium", label: "Medium" }, + { id: "high", label: "High", isDefault: true }, + { id: "xhigh", label: "Extra High (capped to High)" }, + { id: "max", label: "Max" }, + { id: "ultracode", label: "Ultracode" }, + { id: "ultrathink", label: "Ultrathink" }, + ], + ]); + // An effortMap that sends Extra High as max stays unmarked under an xhigh cap: the CLI runs + // it at xhigh, which is what the label says. + assert.deepStrictEqual( + options("xhigh").map((model) => model?.map((option) => option.label)), + [ + ["Medium", "High", "Extra High", "Max (capped to Extra High)", "Ultracode", "Ultrathink"], + ["Medium", "High", "Extra High", "Max", "Ultracode", "Ultrathink"], + ], + ); + + // A per-model cap applies without a top-level one and leaves other models uncapped. + assert.deepStrictEqual( + capClaudeCatalogEffort(catalog, models, [ + { modelMaxEffortLevels: new Map([["claude-plain", "high"]]) }, + ]).map((model) => { + const effort = model.capabilities?.optionDescriptors?.[0]; + return effort?.type === "select" ? effort.options.map((option) => option.label) : []; + }), + [ + [ + "Medium", + "High", + "Extra High (capped to High)", + "Max (capped to High)", + "Ultracode (capped to High)", + "Ultrathink", + ], + ["Medium", "High", "Extra High", "Max", "Ultracode", "Ultrathink"], + ], + ); + + // Without an Extra High option (Opus 4.6), the CLI clamps an xhigh cap down to High. + const noXhigh = { + slug: "claude-no-xhigh", + name: "claude-no-xhigh", + isCustom: false, + capabilities: { + optionDescriptors: [ + { + id: "effort", + label: "Reasoning", + type: "select" as const, + options: [ + { id: "low", label: "Low" }, + { id: "medium", label: "Medium" }, + { id: "high", label: "High", isDefault: true }, + { id: "max", label: "Max" }, + ], + }, + ], + }, + }; + const [capped] = capClaudeCatalogEffort( + { models: [] }, + [noXhigh], + [{ maxEffortLevel: "xhigh" }], + ); + const cappedEffort = capped?.capabilities?.optionDescriptors?.[0]; + assert.deepStrictEqual( + cappedEffort?.type === "select" ? cappedEffort.options.map((option) => option.label) : [], + ["Low", "Medium", "High", "Max (capped to High)"], + ); + }); + + it("leaves custom model effort labels alone under a cap", () => { + const custom = { + slug: "claude-custom-copy", + name: "Copy", + isCustom: true, + capabilities: { + optionDescriptors: [ + { + id: "effort", + label: "Reasoning", + type: "select" as const, + options: [ + { id: "high", label: "High", isDefault: true }, + { id: "max", label: "Max (capped to High)" }, + ], + }, + ], + }, + }; + + assert.deepStrictEqual( + capClaudeCatalogEffort({ models: [] }, [custom], [{ maxEffortLevel: "high" }]), + [custom], + ); + }); + + it("resolves maxEffortLevel per settings source and per model key like Claude Code", () => { + const opus = { + slug: "claude-opus-5-5", + name: "Opus 5.5", + isCustom: false, + aliases: ["opus", "opus-5.5", "claude-opus-5.5"], + capabilities: { + optionDescriptors: [ + { + id: "effort", + label: "Reasoning", + type: "select" as const, + options: [ + { id: "low", label: "Low" }, + { id: "medium", label: "Medium" }, + { id: "high", label: "High", isDefault: true }, + { id: "max", label: "Max" }, + ], + }, + ], + }, + }; + const catalog: ClaudeModelCatalog = { + models: [ + { model: opus, runtime: {}, compatibility: {} }, + { + model: { + slug: "claude-opus-5", + name: "Opus 5", + isCustom: false, + capabilities: { optionDescriptors: [] }, + }, + runtime: {}, + compatibility: {}, + }, + ], + }; + const maxLabel = (sources: Parameters[2]) => { + const effort = capClaudeCatalogEffort(catalog, [opus], sources)[0]?.capabilities + ?.optionDescriptors?.[0]; + return effort?.type === "select" ? effort.options.at(-1)?.label : undefined; + }; + const perModel = (entries: Record) => ({ + modelMaxEffortLevels: new Map(Object.entries(entries)), + }); + + // Claude Code matches family aliases and [1m] spellings case-insensitively. + assert.strictEqual(maxLabel([perModel({ Opus: "medium" })]), "Max (capped to Medium)"); + assert.strictEqual( + maxLabel([perModel({ "claude-opus-5-5[1M]": "medium" })]), + "Max (capped to Medium)", + ); + assert.strictEqual(maxLabel([perModel({ " OPUS[1m] ": "medium" })]), "Max (capped to Medium)"); + // Other keys name the most specific model id they contain, so dated and provider-prefixed ids + // match while the catalog's dotted aliases and a less specific model id do not. + for (const key of [ + "claude-opus-5-5-20260101", + "CLAUDE-OPUS-5-5-20260101[1m]", + "us.anthropic.claude-opus-5-5-v1:0", + "claude-opus-5-5-latest", + ]) { + assert.strictEqual(maxLabel([perModel({ [key]: "medium" })]), "Max (capped to Medium)", key); + } + for (const key of ["claude-opus-5.5", "opus-5.5", "opus-5-5", "opus [1m]", "claude-opus-5"]) { + assert.strictEqual(maxLabel([perModel({ [key]: "medium" })]), "Max", key); + } + // Keys naming the same model keep the lowest cap, and a `max` key does not exempt it. + assert.strictEqual( + maxLabel([perModel({ opus: "max", "claude-opus-5-5": "low" })]), + "Max (capped to Low)", + ); + // A per-model `max` only exempts the model from its own source's top-level cap. + assert.strictEqual( + maxLabel([{ maxEffortLevel: "medium", ...perModel({ opus: "max" }) }]), + "Max", + ); + assert.strictEqual( + maxLabel([perModel({ opus: "max" }), { maxEffortLevel: "high" }]), + "Max (capped to High)", + ); + assert.strictEqual( + maxLabel([{ maxEffortLevel: "medium" }, perModel({ opus: "high" })]), + "Max (capped to Medium)", + ); + }); }); diff --git a/apps/server/src/provider/ClaudeModelCatalog.ts b/apps/server/src/provider/ClaudeModelCatalog.ts index 4efca48f253b..ceeb94a355e8 100644 --- a/apps/server/src/provider/ClaudeModelCatalog.ts +++ b/apps/server/src/provider/ClaudeModelCatalog.ts @@ -322,6 +322,137 @@ export function formatClaudeVersionUpgradeMessage( return `Claude Code ${versionLabel} is too old for ${unavailable.model.name}. Upgrade to v${unavailable.compatibility.minVersion} or newer to access it.`; } +/** Claude Code's effort levels, lowest first. */ +export const CLAUDE_EFFORT_LEVELS = ["low", "medium", "high", "xhigh", "max"] as const; +export type ClaudeEffortLevel = (typeof CLAUDE_EFFORT_LEVELS)[number]; + +function claudeEffortRank(level: string | undefined): number { + return CLAUDE_EFFORT_LEVELS.indexOf(level as ClaudeEffortLevel); +} + +/** One settings source's effort caps, with `modelSettings` keys as the user wrote them. */ +export interface ClaudeEffortCapSource { + readonly maxEffortLevel?: ClaudeEffortLevel; + readonly modelMaxEffortLevels?: ReadonlyMap; +} + +const CLAUDE_1M_SUFFIX = "[1m]"; +const CLAUDE_CODE_FAMILY_ALIASES: ReadonlySet = new Set([ + "opus", + "sonnet", + "haiku", + "fable", +]); + +/** + * Names the model a `modelSettings` key caps the way Claude Code does, which is narrower than the + * catalog's aliases: only a family alias (optionally `[1m]`) resolves as an alias, and any other key + * names the most specific model id it contains, so `claude-opus-5-5-20260101` and Bedrock ids match + * while `claude-opus-5.5` matches nothing. + */ +function resolveClaudeModelSettingsKey( + catalog: ClaudeModelCatalog, + key: string, +): string | undefined { + const lowered = key.trim().toLowerCase(); + const unsuffixed = lowered.endsWith(CLAUDE_1M_SUFFIX) + ? lowered.slice(0, -CLAUDE_1M_SUFFIX.length) + : lowered; + if (CLAUDE_CODE_FAMILY_ALIASES.has(unsuffixed)) { + return catalog.models.find( + (entry) => !entry.model.isCustom && entry.model.aliases?.includes(unsuffixed), + )?.model.slug; + } + let match: string | undefined; + for (const { model } of catalog.models) { + if (!model.isCustom && lowered.includes(model.slug) && model.slug.length > (match?.length ?? 0)) + match = model.slug; + } + return match; +} + +/** Keys `modelSettings` caps by canonical slug, keeping the lowest cap when several keys name one model. */ +function resolveClaudeModelMaxEffortLevels( + catalog: ClaudeModelCatalog, + levels: ReadonlyMap | undefined, +): ReadonlyMap { + const resolved = new Map(); + for (const [key, level] of levels ?? []) { + const slug = resolveClaudeModelSettingsKey(catalog, key); + if (!slug) continue; + const current = resolved.get(slug); + if (!current || claudeEffortRank(level) < claudeEffortRank(current)) resolved.set(slug, level); + } + return resolved; +} + +/** + * Claude Code clamps efforts above `maxEffortLevel` to the model's highest level under it, so + * such options are relabelled, not removed, keeping stored selections valid. Per-model caps + * replace their own source's top-level cap and `max` means none; across sources the lowest wins. + */ +export function capClaudeCatalogEffort( + catalog: ClaudeModelCatalog, + models: ReadonlyArray, + sources: ReadonlyArray, +): ReadonlyArray { + if (sources.length === 0) return models; + const resolvedSources = sources.map((source) => ({ + maxEffortLevel: source.maxEffortLevel, + modelMaxEffortLevels: resolveClaudeModelMaxEffortLevels(catalog, source.modelMaxEffortLevels), + })); + return models.map((model) => { + // Custom model labels are user-authored, so they are left alone. + if (model.isCustom || !model.capabilities?.optionDescriptors) return model; + const modelMaxEffortLevel = resolvedSources.reduce( + (lowest, source) => { + const level = source.modelMaxEffortLevels.get(model.slug) ?? source.maxEffortLevel; + return level && + level !== "max" && + (!lowest || claudeEffortRank(level) < claudeEffortRank(lowest)) + ? level + : lowest; + }, + undefined, + ); + if (!modelMaxEffortLevel) return model; + const capRank = claudeEffortRank(modelMaxEffortLevel); + return { + ...model, + capabilities: { + ...model.capabilities, + optionDescriptors: model.capabilities.optionDescriptors.map((descriptor) => { + if (descriptor.id !== "effort" || descriptor.type !== "select") return descriptor; + const cap = descriptor.options.reduce<{ rank: number; label: string } | undefined>( + (best, option) => { + const rank = claudeEffortRank(option.id); + return rank >= 0 && rank <= capRank && rank > (best?.rank ?? -1) + ? { rank, label: option.label } + : best; + }, + undefined, + ); + if (!cap) return descriptor; + return { + ...descriptor, + options: descriptor.options.map((option) => { + const idRank = claudeEffortRank(option.id); + const sentRank = claudeEffortRank( + normalizeClaudeCatalogEffort(catalog, option.id, model.slug), + ); + const displayedRank = idRank >= 0 ? idRank : sentRank; + const runsRank = sentRank >= 0 ? sentRank : idRank; + return displayedRank > cap.rank && runsRank > cap.rank + ? { ...option, label: `${option.label} (capped to ${cap.label})` } + : option; + }), + }; + }), + }, + }; + }); +} + export function resolveClaudeCatalogEffort( catalog: ClaudeModelCatalog, model: string | null | undefined, @@ -329,7 +460,7 @@ export function resolveClaudeCatalogEffort( ): string | undefined { if (!resolveClaudeCatalogModel(catalog, model)) { const effort = raw?.trim(); - return effort && ["low", "medium", "high", "xhigh", "max"].includes(effort) + return effort && (CLAUDE_EFFORT_LEVELS as ReadonlyArray).includes(effort) ? effort : undefined; } diff --git a/apps/server/src/provider/ClaudeProvider.ts b/apps/server/src/provider/ClaudeProvider.ts index d957e182b72f..0018e2cafab3 100644 --- a/apps/server/src/provider/ClaudeProvider.ts +++ b/apps/server/src/provider/ClaudeProvider.ts @@ -35,13 +35,18 @@ import { COMPACT_SLASH_COMMAND, DEFAULT_TIMEOUT_MS, isCommandMissingCause, - parseGenericCliVersion, + parseCliVersionResult, providerModelsFromSettings, + providerProbeEnvironment, spawnAndCollect, type ServerProviderDraft, } from "@t3tools/provider-core/server/snapshotProbe"; import { hasClaudeManagedMcpConfig, makeClaudeEnvironment } from "./Drivers/ClaudeHome.ts"; -import { discoverClaudeSkills, readClaudeSettingsEnv } from "./Drivers/ClaudeSkills.ts"; +import { + customClaudeApiEndpointHost, + discoverClaudeSkills, + readClaudeSettingsEnv, +} from "./Drivers/ClaudeSkills.ts"; import type { ProviderWorkspaceSnapshot } from "@t3tools/provider-core/server/driver"; import { makeUnavailableUsageLimits } from "@t3tools/provider-core/server/usageLimits"; import { @@ -51,6 +56,10 @@ import { } from "./claudeUsageLimits.ts"; import { BUNDLED_CLAUDE_MODEL_CATALOG, + CLAUDE_EFFORT_LEVELS, + capClaudeCatalogEffort, + type ClaudeEffortCapSource, + type ClaudeEffortLevel, type ClaudeModelCatalog, claudeReportedModels, formatClaudeVersionUpgradeMessage, @@ -239,7 +248,7 @@ export function buildClaudeCapabilitiesProbeQueryOptions(input: { mcpServers: {}, ...(input.managedMcpConfig ? {} : { strictMcpConfig: true }), env: { - ...input.environment, + ...providerProbeEnvironment(input.environment), // Disable claude.ai MCP separately; it is discovered outside filesystem config. ENABLE_CLAUDEAI_MCP_SERVERS: "false", // Skipping this avoids a Windows tasklist | findstr process tree on every refresh. @@ -273,6 +282,8 @@ export type ClaudeCapabilitiesProbe = { /** Undefined means get_usage failed; absent windows on success mean no subscription windows, as with API keys. */ readonly usage?: Pick; readonly usageCheckedAt?: string; + /** Each settings source's `maxEffortLevel` caps; Claude Code clamps every higher effort. */ + readonly effortCapSources?: ReadonlyArray; }; // Coder: runtime modes follow the workspace's effective Claude permission policy. @@ -293,6 +304,60 @@ function readClaudePermissionModeDisabled( ); } +function parseClaudeMaxEffortLevel(value: unknown): ClaudeEffortLevel | undefined { + return CLAUDE_EFFORT_LEVELS.find((level) => level === value); +} + +function parseClaudeModelMaxEffortLevels( + modelSettings: unknown, +): ReadonlyMap | undefined { + if (!modelSettings || typeof modelSettings !== "object") return undefined; + const levels = new Map(); + for (const [model, settings] of Object.entries(modelSettings)) { + const level = parseClaudeMaxEffortLevel( + settings && typeof settings === "object" + ? (settings as { readonly maxEffortLevel?: unknown }).maxEffortLevel + : undefined, + ); + if (level) levels.set(model, level); + } + return levels.size > 0 ? levels : undefined; +} + +function parseClaudeEffortCapSource(settings: unknown): ClaudeEffortCapSource | undefined { + if (!settings || typeof settings !== "object") return undefined; + const { maxEffortLevel, modelSettings } = settings as { + readonly maxEffortLevel?: unknown; + readonly modelSettings?: unknown; + }; + const level = parseClaudeMaxEffortLevel(maxEffortLevel); + const modelMaxEffortLevels = parseClaudeModelMaxEffortLevels(modelSettings); + return level || modelMaxEffortLevels + ? { + ...(level ? { maxEffortLevel: level } : {}), + ...(modelMaxEffortLevels ? { modelMaxEffortLevels } : {}), + } + : undefined; +} + +// `sources` holds each settings file's raw settings. CLIs that report only the merged `effective` +// settings are read as a single source. +function parseClaudeEffortCapSources( + settings: { readonly effective?: unknown; readonly sources?: unknown } | undefined, +): ReadonlyArray | undefined { + const sources = Array.isArray(settings?.sources) + ? settings.sources.flatMap((source) => { + const parsed = parseClaudeEffortCapSource( + source && typeof source === "object" + ? (source as { readonly settings?: unknown }).settings + : undefined, + ); + return parsed ? [parsed] : []; + }) + : [parseClaudeEffortCapSource(settings?.effective)].filter((source) => source !== undefined); + return sources.length > 0 ? sources : undefined; +} + function parseClaudeInitializationCommands( commands: ReadonlyArray | undefined, ): ReadonlyArray { @@ -394,7 +459,7 @@ const probeClaudeCapabilities = ( claudeSettings: ClaudeSettings, environment?: NodeJS.ProcessEnv, cwd?: string, - includeUsage = true, + includeStatus = true, ) => { const abort = new AbortController(); const abortProbe = Effect.sync(() => { @@ -405,7 +470,7 @@ const probeClaudeCapabilities = ( const executablePath = claudeSettings.binaryPath; const managedMcpConfig = yield* hasClaudeManagedMcpConfig(); // Workspace probes (no usage) discard models; the machine probe refreshes the shared cache. - const graceMs = includeUsage + const graceMs = includeStatus ? claudeGatewayDiscoveryGraceMs({ ...claudeEnvironment, ...(yield* readClaudeSettingsEnv(claudeSettings, cwd, claudeEnvironment)), @@ -444,8 +509,9 @@ const probeClaudeCapabilities = ( // Usage has its own deadline so a slow optional request cannot discard initialization. // Only the rate limits are read, so skip the local transcript scan that fills // `behaviors`: with a few GB of transcripts it outlasts the deadline. + // The effort caps are read in parallel under the same deadline; workspace probes skip both. const usageCheckedAt = DateTime.formatIso(yield* DateTime.now); - const usageResult = includeUsage + const usageResult = includeStatus ? yield* Effect.tryPromise({ // Coder: the workspace CLI transport answers the SDK's usage control request. try: () => @@ -455,6 +521,9 @@ const probeClaudeCapabilities = ( catch: (cause) => new ClaudeCapabilitiesProbeSdkError({ operation: "usage", cause }), }).pipe(Effect.timeout(DEFAULT_TIMEOUT_MS), Effect.result) : undefined; + // Coder: the effort caps come from the permission-policy settings read above. + const settingsResult = + includeStatus && Option.isSome(settings) ? Result.succeed(settings.value) : undefined; const usage = usageResult && Result.isSuccess(usageResult) ? { @@ -462,6 +531,9 @@ const probeClaudeCapabilities = ( rate_limits: usageResult.success.rate_limits, } : undefined; + const effortCapSources = parseClaudeEffortCapSources( + settingsResult && Result.isSuccess(settingsResult) ? settingsResult.success : undefined, + ); const account = init.account as | { readonly email?: string; @@ -486,6 +558,7 @@ const probeClaudeCapabilities = ( slashCommands: parseClaudeInitializationCommands(init.commands), ...(init.models?.length ? { models: init.models } : {}), ...(usage ? { usage, usageCheckedAt } : {}), + ...(effortCapSources ? { effortCapSources } : {}), } satisfies ClaudeCapabilitiesProbe; }).pipe( Effect.tap(() => @@ -591,7 +664,7 @@ export const checkClaudeProviderStatus = Effect.fn("checkClaudeProviderStatus")( const versionProbe = yield* runClaudeCommand( claudeSettings, ["--version"], - resolvedEnvironment, + providerProbeEnvironment(resolvedEnvironment), cwd, ).pipe(Effect.timeoutOption(CLAUDE_VERSION_CHECK_TIMEOUT_MS), Effect.result); @@ -635,7 +708,7 @@ export const checkClaudeProviderStatus = Effect.fn("checkClaudeProviderStatus")( } const version = versionProbe.success.value; - const parsedVersion = parseGenericCliVersion(`${version.stdout}\n${version.stderr}`); + const parsedVersion = parseCliVersionResult(version); if (version.code !== 0) { yield* Effect.logWarning("Claude Agent CLI version probe exited with a non-zero status.", { exitCode: version.code, @@ -658,6 +731,7 @@ export const checkClaudeProviderStatus = Effect.fn("checkClaudeProviderStatus")( } const updateRequiredModels = resolveClaudeUpdateRequiredModels(modelCatalog, parsedVersion); + const versionUpgradeMessage = formatClaudeVersionUpgradeMessage(modelCatalog, parsedVersion); const probeResult = resolveCapabilities ? yield* resolveCapabilities(claudeSettings).pipe(Effect.result) : undefined; @@ -665,12 +739,16 @@ export const checkClaudeProviderStatus = Effect.fn("checkClaudeProviderStatus")( probeResult && Result.isSuccess(probeResult) ? probeResult.success : undefined; const probeFailure = probeResult && Result.isFailure(probeResult) ? probeResult.failure : undefined; - const models = claudeProviderModels( - [ - ...resolveClaudeModelsForVersion(modelCatalog, parsedVersion), - ...claudeReportedModels(modelCatalog, claudeSettings.customModels, capabilities?.models), - ], - claudeSettings.customModels, + const models = capClaudeCatalogEffort( + modelCatalog, + claudeProviderModels( + [ + ...resolveClaudeModelsForVersion(modelCatalog, parsedVersion), + ...claudeReportedModels(modelCatalog, claudeSettings.customModels, capabilities?.models), + ], + claudeSettings.customModels, + ), + capabilities?.effortCapSources ?? [], ); // Coder: runtime modes follow the workspace's Claude permission policy, failing closed when // it could not be read. @@ -681,7 +759,6 @@ export const checkClaudeProviderStatus = Effect.fn("checkClaudeProviderStatus")( fullAccess: capabilities !== undefined && !capabilities.bypassPermissionsDisabled, }), }; - const versionUpgradeMessage = formatClaudeVersionUpgradeMessage(modelCatalog, parsedVersion); const skills = yield* discoverClaudeSkills(claudeSettings, cwd, resolvedEnvironment); const slashCommands = [COMPACT_SLASH_COMMAND, ...(capabilities?.slashCommands ?? [])]; const dedupedSlashCommands = dedupeSlashCommands(slashCommands); @@ -713,12 +790,7 @@ export const checkClaudeProviderStatus = Effect.fn("checkClaudeProviderStatus")( const baseUrl = (yield* readClaudeSettingsEnv(claudeSettings, cwd, resolvedEnvironment)).ANTHROPIC_BASE_URL ?? resolvedEnvironment.ANTHROPIC_BASE_URL; - const endpoint = URL.parse(baseUrl?.trim() ?? ""); - const customApiEndpoint = - endpoint !== null && - (endpoint.protocol === "http:" || endpoint.protocol === "https:") && - endpoint.hostname !== "" && - endpoint.hostname !== "api.anthropic.com"; + const customApiEndpoint = customClaudeApiEndpointHost(baseUrl) !== undefined; if (!customApiEndpoint && claudeAuthStatus(capabilities) === "unauthenticated") { return buildServerProvider({ presentation, @@ -733,7 +805,8 @@ export const checkClaudeProviderStatus = Effect.fn("checkClaudeProviderStatus")( version: parsedVersion, status: "error", auth: { status: "unauthenticated" }, - // Coder: providers authenticate through the workspace's API configuration. + // Coder: providers authenticate through the workspace's API configuration, so the + // message names no config directory to sign in (upstream's `claudeSignedOutMessage`). message: "Claude Code is not authenticated. Check its API credentials in the workspace provider configuration.", }, diff --git a/apps/server/src/provider/CodexDeveloperInstructions.test.ts b/apps/server/src/provider/CodexDeveloperInstructions.test.ts index 2b24d4a2ccd4..67596f0cd7f3 100644 --- a/apps/server/src/provider/CodexDeveloperInstructions.test.ts +++ b/apps/server/src/provider/CodexDeveloperInstructions.test.ts @@ -56,19 +56,6 @@ describe("buildCodexAdditionalContext", () => { }); describe("Codex runtime context", () => { - it("describes Markdown media support in the runtime context in both modes", () => { - { - const instructions = runtimeInstructions({ - model: "gpt-5.3-codex", - reasoningEffort: "high", - }); - NodeAssert.match( - instructions, - /.*embed images, video, and audio.*Markdown image syntax with absolute file paths.*Link any other file.*absolute path.*files viewer.*<\/runtime_info>/, - ); - } - }); - it("flattens multiline metadata into single-line runtime info", () => { const instructions = runtimeInstructions({ model: "gpt\n5.3\ncodex", diff --git a/apps/server/src/provider/CodexProvider.test.ts b/apps/server/src/provider/CodexProvider.test.ts index 30f63b21f318..c9a992bcf516 100644 --- a/apps/server/src/provider/CodexProvider.test.ts +++ b/apps/server/src/provider/CodexProvider.test.ts @@ -1,6 +1,10 @@ import { assert, it } from "@effect/vitest"; -import { applyPreferredCodexDefaultModel, mapCodexModelCapabilities } from "./CodexProvider.ts"; +import { + applyPreferredCodexDefaultModel, + codexRoutesSpendNativeQuota, + mapCodexModelCapabilities, +} from "./CodexProvider.ts"; it("maps current Codex model capability fields", () => { const capabilities = mapCodexModelCapabilities({ @@ -153,3 +157,28 @@ it("ignores custom models that shadow a preferred slug", () => { assert.deepStrictEqual(models.find((model) => model.isDefault)?.slug, "gpt-5.4"); }); + +it("reports native quota only for an OpenAI login on a native route", () => { + const chatgpt = { type: "chatgpt", email: "a@example.com", planType: "pro" } as const; + const hub = { + model_provider: "hub", + model_providers: { hub: { base_url: "https://hub.example.com/v1" } }, + }; + const cases = [ + ["default route", chatgpt, true, {}, true], + ["openai base url", chatgpt, true, { openai_base_url: "https://chatgpt.com/x" }, true], + ["openai proxy url", chatgpt, true, { openai_base_url: "https://proxy.example.com" }, false], + ["custom provider", chatgpt, true, hub, false], + ["unknown provider id", chatgpt, true, { model_provider: "missing" }, false], + ["unreadable config", chatgpt, true, null, false], + ["login not required", chatgpt, false, {}, false], + ["api key", { type: "apiKey" }, true, {}, false], + ] as const; + for (const [name, account, requiresOpenaiAuth, config, expected] of cases) { + assert.strictEqual( + codexRoutesSpendNativeQuota({ account, requiresOpenaiAuth }, config), + expected, + name, + ); + } +}); diff --git a/apps/server/src/provider/CodexProvider.ts b/apps/server/src/provider/CodexProvider.ts index 2adec72f9523..429205f191a7 100644 --- a/apps/server/src/provider/CodexProvider.ts +++ b/apps/server/src/provider/CodexProvider.ts @@ -47,6 +47,7 @@ import { COMPACT_SLASH_COMMAND, DEFAULT_TIMEOUT_MS, providerModelsFromSettings, + providerProbeEnvironment, spawnAndCollect, type ServerProviderDraft, } from "@t3tools/provider-core/server/snapshotProbe"; @@ -75,6 +76,44 @@ type CodexRateLimitsProbe = } | { readonly failure: string }; +const CodexRoutingConfig = Schema.Struct({ + model_provider: Schema.optionalKey(Schema.NullOr(Schema.String)), + openai_base_url: Schema.optionalKey(Schema.NullOr(Schema.String)), + model_providers: Schema.optionalKey( + Schema.Record( + Schema.String, + Schema.Struct({ base_url: Schema.optionalKey(Schema.NullOr(Schema.String)) }), + ), + ), +}); +const decodeCodexRoutingConfig = Schema.decodeUnknownOption(CodexRoutingConfig); + +function routesToNativeOpenai(config: unknown): boolean { + const decoded = decodeCodexRoutingConfig(config); + if (Option.isNone(decoded)) return false; + const routing = decoded.value; + const providerId = routing.model_provider ?? "openai"; + const provider = routing.model_providers?.[providerId]; + if (providerId !== "openai" && !provider) return false; + const baseUrl = + providerId === "openai" ? routing.openai_base_url || undefined : provider?.base_url; + // A provider without an override uses Codex's auth-dependent OpenAI default. + if (baseUrl == null) return true; + if (!URL.canParse(baseUrl)) return false; + const url = new URL(baseUrl); + return ( + url.protocol === "https:" && + (url.hostname === "api.openai.com" || url.hostname === "chatgpt.com") + ); +} + +// A stored login does not mean this provider's traffic spends that login's quota. +export const codexRoutesSpendNativeQuota = ( + account: CodexSchema.V2GetAccountResponse, + config: unknown, +): boolean => + account.requiresOpenaiAuth && account.account?.type !== "apiKey" && routesToNativeOpenai(config); + const CODEX_APP_SERVER_PROBE_FORCE_KILL_AFTER = "2 seconds" as const; const CODEX_CATALOG_REFRESH_TIMEOUT_MS = 3_000; const CODEX_CONFIG_PROBE_TIMEOUT_MS = 3_000; @@ -97,6 +136,7 @@ const CODEX_PRESENTATION = { export interface CodexAppServerProviderSnapshot { readonly account: CodexSchema.V2GetAccountResponse; + readonly nativeUsage?: boolean; readonly rateLimits?: CodexRateLimitsProbe; readonly version: string | undefined; readonly models: ReadonlyArray | undefined; @@ -535,6 +575,34 @@ export const withCodexAppServerClient = Effect.fn("withCodexAppServerClient")(fu return { client, initialize }; }); +// Usage is an enrichment: a failure or a slow answer degrades to "no usage this +// probe" rather than costing the account and models. +const readCodexRateLimits = ( + client: CodexClient.CodexAppServerClient["Service"], + deadlineMs: number, +) => + withCodexDiscoveryDeadline( + client.request("account/rateLimits/read", null), + deadlineMs, + RATE_LIMITS_PROBE_TIMEOUT_MS, + ).pipe( + Effect.map((response): CodexRateLimitsProbe => + response + ? { + snapshot: response.rateLimits, + rateLimitsByLimitId: response.rateLimitsByLimitId, + resetCredits: response.rateLimitResetCredits, + accountId: response.accountId, + } + : { failure: "Codex did not answer the usage request." }, + ), + Effect.catch((error) => + Effect.logDebug("Codex rate-limit read failed.", { cause: error }).pipe( + Effect.as({ failure: codexRateLimitsFailureMessage(error) }), + ), + ), + ); + const probeCodexAppServerProvider = Effect.fn("probeCodexAppServerProvider")(function* (input: { readonly binaryPath: string; readonly homePath?: string; @@ -561,65 +629,63 @@ const probeCodexAppServerProvider = Effect.fn("probeCodexAppServerProvider")(fun // Optional discovery must finish before the outer timeout can discard the account result. const deadlineMs = spawnedAtMs + CODEX_AUTH_PROBE_TIMEOUT_MS - CODEX_DISCOVERY_MARGIN_MS; - const [skillsResponse, models, serviceTier, requirementsResponse, rateLimits] = yield* Effect.all( - [ - withCodexDiscoveryDeadline(client.request("skills/list", { cwds: [input.cwd] }), deadlineMs), - accountResponse.account?.type === "chatgpt" - ? requestCodexChatGptModels({ - client, - codexHome: initialize.codexHome, - spawnedAtMs, - deadlineMs, - }) - : withCodexDiscoveryDeadline(requestAllCodexModels(client), deadlineMs), - // The shared catalog must exclude config from the server's launch project. - withCodexDiscoveryDeadline( - client.request("config/read", { includeLayers: false }), - deadlineMs, - CODEX_CONFIG_PROBE_TIMEOUT_MS, - ).pipe( - Effect.map((response) => response?.config.service_tier ?? undefined), - Effect.catch(() => Effect.logDebug("Codex config read failed.").pipe(Effect.as(undefined))), - ), - // Coder: runtime modes follow the workspace's Codex approval and sandbox requirements. - withCodexDiscoveryDeadline( - client.raw - .request("configRequirements/read") - .pipe(Effect.flatMap(decodeCodexRuntimeRequirementsResponse)), - deadlineMs, - CODEX_CONFIG_PROBE_TIMEOUT_MS, - ).pipe(Effect.orElseSucceed(() => undefined)), - // Usage is an enrichment: a failure or a slow answer degrades to "no - // usage this probe" rather than costing the account and models. - input.skipNativeUsage - ? Effect.undefined - : withCodexDiscoveryDeadline( - client.request("account/rateLimits/read", null), - deadlineMs, - RATE_LIMITS_PROBE_TIMEOUT_MS, - ).pipe( - Effect.map((response): CodexRateLimitsProbe => - response - ? { - snapshot: response.rateLimits, - rateLimitsByLimitId: response.rateLimitsByLimitId, - resetCredits: response.rateLimitResetCredits, - accountId: response.accountId, - } - : { failure: "Codex did not answer the usage request." }, - ), - Effect.catch((error) => - Effect.logDebug("Codex rate-limit read failed.", { cause: error }).pipe( - Effect.as({ failure: codexRateLimitsFailureMessage(error) }), - ), - ), + const [skillsResponse, models, { serviceTier, nativeUsage, rateLimits }, requirementsResponse] = + yield* Effect.all( + [ + withCodexDiscoveryDeadline( + client.request("skills/list", { cwds: [input.cwd] }), + deadlineMs, + ), + accountResponse.account?.type === "chatgpt" + ? requestCodexChatGptModels({ + client, + codexHome: initialize.codexHome, + spawnedAtMs, + deadlineMs, + }) + : withCodexDiscoveryDeadline(requestAllCodexModels(client), deadlineMs), + // The shared catalog must exclude config from the server's launch project. + withCodexDiscoveryDeadline( + client.request("config/read", { includeLayers: false }), + deadlineMs, + CODEX_CONFIG_PROBE_TIMEOUT_MS, + ).pipe( + Effect.catch(() => + Effect.logDebug("Codex config read failed.").pipe(Effect.as(undefined)), ), - ], - { concurrency: "unbounded" }, - ); + Effect.flatMap((config) => { + // Only a confirmed native route may probe or publish the login's quota. + const nativeUsage = + !input.skipNativeUsage && + config !== undefined && + codexRoutesSpendNativeQuota(accountResponse, config.config); + const readUsage: Effect.Effect = nativeUsage + ? readCodexRateLimits(client, deadlineMs) + : Effect.undefined; + return readUsage.pipe( + Effect.map((rateLimits) => ({ + serviceTier: config?.config.service_tier ?? undefined, + nativeUsage, + rateLimits, + })), + ); + }), + ), + // Coder: runtime modes follow the workspace's Codex approval and sandbox requirements. + withCodexDiscoveryDeadline( + client.raw + .request("configRequirements/read") + .pipe(Effect.flatMap(decodeCodexRuntimeRequirementsResponse)), + deadlineMs, + CODEX_CONFIG_PROBE_TIMEOUT_MS, + ).pipe(Effect.orElseSucceed(() => undefined)), + ], + { concurrency: "unbounded" }, + ); return { account: accountResponse, + nativeUsage, ...(rateLimits ? { rateLimits } : {}), version, models, @@ -669,7 +735,10 @@ export const probeCodexSkillsForCwd = Effect.fn("probeCodexSkillsForCwd")(functi readonly cwd: string; readonly environment?: NodeJS.ProcessEnv; }) { - const { client } = yield* withCodexAppServerClient(input); + const { client } = yield* withCodexAppServerClient({ + ...input, + environment: providerProbeEnvironment(input.environment ?? process.env), + }); const skillsResponse = yield* client.request("skills/list", { cwds: [input.cwd] }); return parseCodexSkillsListResponse(skillsResponse, input.cwd); }); @@ -772,7 +841,7 @@ export const checkCodexProviderStatus = Effect.fn("checkCodexProviderStatus")(fu ServerSettingsError, ChildProcessSpawner.ChildProcessSpawner | FileSystem.FileSystem | Path.Path > { - const resolvedEnvironment = environment ?? process.env; + const resolvedEnvironment = providerProbeEnvironment(environment ?? process.env); const checkedAt = DateTime.formatIso(yield* DateTime.now); const emptyModels = emptyCodexModelsFromSettings(codexSettings); @@ -860,8 +929,9 @@ export const checkCodexProviderStatus = Effect.fn("checkCodexProviderStatus")(fu const accountStatus = managedAuth ? { status: "ready" as const, auth: managedAuth, message: undefined } : accountProbeStatus(snapshot); + // Sign-in guidance still applies when Codex itself says it needs an OpenAI login. const usageLimits = - snapshot.account.account?.type === "apiKey" + !snapshot.nativeUsage && !accountStatus.message ? makeUnavailableUsageLimits({ checkedAt, reason: "unsupported" }) : snapshot.rateLimits === undefined || "failure" in snapshot.rateLimits ? makeUnavailableUsageLimits({ diff --git a/apps/server/src/provider/Drivers/ClaudeMaintenance.test.ts b/apps/server/src/provider/Drivers/ClaudeMaintenance.test.ts index 7897fe9b75c5..6fdf4c52e2a9 100644 --- a/apps/server/src/provider/Drivers/ClaudeMaintenance.test.ts +++ b/apps/server/src/provider/Drivers/ClaudeMaintenance.test.ts @@ -42,19 +42,31 @@ const snapshot: ServerProvider = { }; const fixture = Effect.fn("ClaudeMaintenance.test.fixture")(function* ( - npm = false, + install: "native" | "xdgNative" | "npm" | "wrapper" = "native", platform: NodeJS.Platform = "linux", programFiles?: string, ) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const directory = yield* fs.makeTempDirectoryScoped(); - const binaryPath = npm - ? path.join(directory, "lib/node_modules/@anthropic-ai/claude-code/cli.js") - : path.join(directory, ".local/bin", platform === "win32" ? "claude.exe" : "claude"); - yield* fs.makeDirectory(path.dirname(binaryPath), { recursive: true }); - yield* fs.writeFileString(binaryPath, "#!/bin/sh\n"); - yield* fs.chmod(binaryPath, 0o755); + const binaryPath = + install === "npm" + ? path.join(directory, "lib/node_modules/@anthropic-ai/claude-code/cli.js") + : path.join(directory, ".local/bin", platform === "win32" ? "claude.exe" : "claude"); + // Unix native installs link the launcher to a versioned binary; Windows places a real .exe. + const realBinaryPath = + install === "native" && platform !== "win32" + ? path.join(directory, ".local/share/claude/versions/2.1.314") + : install === "xdgNative" + ? path.join(directory, "xdg/data/claude/versions/2.1.314") + : binaryPath; + yield* fs.makeDirectory(path.dirname(realBinaryPath), { recursive: true }); + yield* fs.writeFileString(realBinaryPath, "#!/bin/sh\n"); + yield* fs.chmod(realBinaryPath, 0o755); + if (realBinaryPath !== binaryPath) { + yield* fs.makeDirectory(path.dirname(binaryPath), { recursive: true }); + yield* fs.symlink(realBinaryPath, binaryPath); + } const configDirectory = path.join(directory, "config"); yield* fs.makeDirectory(configDirectory); const settingsPath = path.join(configDirectory, "settings.json"); @@ -194,7 +206,7 @@ it.layer(NodeServices.layer)("ClaudeMaintenance", (it) => { }, ])("advertises the installable version for $name", (row) => Effect.gen(function* () { - const f = yield* fixture(row.npm); + const f = yield* fixture(row.npm ? "npm" : "native"); if (row.settings) yield* f.fs.writeFileString(f.settingsPath, row.settings); const requests: string[] = []; const client = HttpClient.make((request) => { @@ -239,7 +251,7 @@ it.layer(NodeServices.layer)("ClaudeMaintenance", (it) => { "separates channel caches and honors $name managed overrides and feed invalidation", (row) => Effect.gen(function* () { - const f = yield* fixture(false, row.platform, row.programFiles); + const f = yield* fixture("native", row.platform, row.programFiles); const requests: string[] = []; const client = HttpClient.make((request) => { requests.push(request.url); @@ -288,6 +300,26 @@ it.layer(NodeServices.layer)("ClaudeMaintenance", (it) => { }).pipe(Effect.scoped), ); + it.effect.each([ + { name: "a wrapper script", install: "wrapper", lockKey: undefined }, + { + name: "a native install under XDG_DATA_HOME", + install: "xdgNative", + lockKey: "claudeAgent-native", + }, + ] as const)("resolves the native launcher path for $name", (row) => + Effect.gen(function* () { + const f = yield* fixture(row.install); + const resolve = yield* f.makeResolution.pipe( + Effect.provideService( + HttpClient.HttpClient, + HttpClient.make(() => Effect.die("resolution must not look up a version")), + ), + ); + expect((yield* resolve()).update?.lockKey).toBe(row.lockKey); + }).pipe(Effect.scoped), + ); + it.effect("automatic refresh rereads the native channel while preserving both caches", () => Effect.gen(function* () { const f = yield* fixture(); diff --git a/apps/server/src/provider/Drivers/ClaudeMaintenance.ts b/apps/server/src/provider/Drivers/ClaudeMaintenance.ts index e278c25d6d3d..cb225d1fde59 100644 --- a/apps/server/src/provider/Drivers/ClaudeMaintenance.ts +++ b/apps/server/src/provider/Drivers/ClaudeMaintenance.ts @@ -30,14 +30,17 @@ const update = makePackageManagedProviderMaintenanceResolver({ mise: { tool: "claude" }, nativeUpdate: { args: ["update"], + // On Unix the native launcher links into ${XDG_DATA_HOME:-~/.local/share}/claude/versions; a + // plain file at ~/.local/bin/claude is a wrapper whose owner `claude update` cannot see. isCommandPath: (commandPath) => { const normalized = normalizeCommandPath(commandPath); return ( - normalized.endsWith("/.local/bin/claude") || normalized.endsWith("/.local/bin/claude.exe") || - normalized.includes("/.local/share/claude/") + normalized.includes("/.local/share/claude/") || + normalized.includes("/claude/versions/") ); }, + requiresCommandPath: true, }, }); diff --git a/apps/server/src/provider/Drivers/ClaudeSkills.test.ts b/apps/server/src/provider/Drivers/ClaudeSkills.test.ts index b4ce31cac866..71359312854c 100644 --- a/apps/server/src/provider/Drivers/ClaudeSkills.test.ts +++ b/apps/server/src/provider/Drivers/ClaudeSkills.test.ts @@ -7,6 +7,9 @@ import * as Path from "effect/Path"; import { discoverClaudeSkills } from "./ClaudeSkills.ts"; import { planClaudeSkillDispatch } from "./ClaudeSkillDispatch.ts"; +// Default macOS and Windows volumes ignore case. +const caseInsensitiveTmpdir = process.platform !== "linux"; + const writeSkill = Effect.fn(function* ( skillsDir: string, directoryName: string, @@ -354,6 +357,235 @@ it.layer(NodeServices.layer)("discoverClaudeSkills", (it) => { }), ); + it.effect("discovers skills from the current account's claude.ai synced plugins", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const configDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-claude-skills-" }); + const synced = path.join(configDir, "plugins", "synced"); + const organization = "11111111-2222-4333-8444-555555555555"; + const writeBucket = Effect.fn(function* (account: string, skillSuffix = "") { + const bucket = path.join(synced, `${organization}_${account}`); + const plugins = [ + { + pluginId: "plugin_01pstack", + name: "pstack", + directory: "pstack", + skill: "poteto-mode", + }, + { + pluginId: "plugin_01design", + name: "0f6a7c9e-1111-4222-8333-444444444444", + generation: 2, + directory: "0f6a7c9e-1111-4222-8333-444444444444~g2", + prefix: "design", + skill: "design-critique", + }, + ]; + for (const plugin of plugins) { + const pluginDir = path.join(bucket, plugin.directory); + yield* writeSkill(path.join(pluginDir, "skills"), plugin.skill + skillSuffix, "# Synced"); + yield* fs.makeDirectory(path.join(pluginDir, ".claude-plugin")); + yield* fs.writeFileString( + path.join(pluginDir, ".claude-plugin", "plugin.json"), + JSON.stringify({ name: plugin.prefix ?? plugin.name }), + ); + } + yield* fs.writeFileString( + path.join(bucket, "manifest.json"), + JSON.stringify({ + lastUpdated: 1761990000000, + plugins: plugins.map(({ pluginId, name, generation }) => ({ + pluginId, + name, + ...(generation ? { generation } : {}), + })), + }), + ); + yield* fs.writeFileString(path.join(synced, `.bucket-${organization}_${account}`), ""); + return bucket; + }); + const bucket = yield* writeBucket("66666666-7777-4888-9999-aaaaaaaaaaaa"); + yield* writeBucket("77777777-8888-4999-aaaa-bbbbbbbbbbbb", "-other-account"); + yield* fs.writeFileString( + path.join(configDir, ".claude.json"), + JSON.stringify({ + oauthAccount: { + organizationUuid: organization, + accountUuid: "66666666-7777-4888-9999-aaaaaaaaaaaa", + }, + }), + ); + + assert.deepEqual( + (yield* discoverClaudeSkills({ homePath: configDir }, undefined, {})).map((skill) => [ + skill.name, + skill.path, + ]), + [ + [ + "design:design-critique", + path.join( + bucket, + "0f6a7c9e-1111-4222-8333-444444444444~g2", + "skills", + "design-critique", + "SKILL.md", + ), + ], + ["pstack:poteto-mode", path.join(bucket, "pstack", "skills", "poteto-mode", "SKILL.md")], + ], + ); + assert.deepEqual( + (yield* discoverClaudeSkills({ homePath: configDir }, undefined, { + CLAUDE_CODE_PLUGIN_CACHE_DIR: path.join(configDir, "relocated-cache"), + })).map((skill) => skill.name), + ["design:design-critique", "pstack:poteto-mode"], + ); + assert.deepEqual( + (yield* discoverClaudeSkills({ homePath: configDir }, undefined, { + CLAUDE_CODE_OAUTH_TOKEN: "token", + CLAUDE_CODE_ACCOUNT_UUID: "77777777-8888-4999-aaaa-bbbbbbbbbbbb", + })).map((skill) => skill.name), + ["design:design-critique-other-account", "pstack:poteto-mode-other-account"], + ); + yield* fs.writeFileString( + path.join(configDir, "settings.json"), + '{"enabledPlugins":{"pstack@synced":false}}', + ); + assert.deepEqual( + (yield* discoverClaudeSkills({ homePath: configDir }, undefined, {})).map( + (skill) => skill.name, + ), + ["design:design-critique"], + ); + yield* fs.writeFileString( + path.join(configDir, "settings.json"), + '{"syncClaudeAiPlugins":false}', + ); + assert.deepEqual(yield* discoverClaudeSkills({ homePath: configDir }, undefined, {}), []); + }), + ); + + // Coder: the case tells `pstack` and `PSTACK` folders apart, which a case-insensitive volume (the + // default on macOS development machines) cannot hold; Linux workspaces and CI run it. + it.effect.skipIf(caseInsensitiveTmpdir)( + "gives a synced namespace to an installed plugin, then the first synced row", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const configDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-claude-skills-" }); + const bucketName = "11111111-2222-4333-8444-555555555555_unbound"; + const synced = path.join(configDir, "plugins", "synced"); + const bucket = path.join(synced, bucketName); + for (const [directory, prefix, skill] of [ + ["pstack", "pstack", "poteto-mode"], + ["pstack-copy", "pstack", "copied"], + ["tools", "tools", "synced-tool"], + ["a_b", undefined, "renamed"], + ["PSTACK", "PSTACK", "shouted"], + ] as const) { + yield* writeSkill(path.join(bucket, directory, "skills"), skill, "# Synced"); + if (prefix === undefined) continue; + yield* fs.makeDirectory(path.join(bucket, directory, ".claude-plugin")); + yield* fs.writeFileString( + path.join(bucket, directory, ".claude-plugin", "plugin.json"), + JSON.stringify({ name: prefix }), + ); + } + yield* fs.writeFileString( + path.join(bucket, "manifest.json"), + JSON.stringify({ + plugins: ["pstack", "pstack-copy", "tools", "a:b. ", "PSTACK", "no-id"].map( + (name, index) => + name === "no-id" ? { name } : { pluginId: `plugin_0${index}`, name }, + ), + }), + ); + yield* fs.writeFileString(path.join(synced, `.bucket-${bucketName}`), ""); + const installed = path.join(configDir, "plugins", "cache", "shop", "tools"); + yield* writeSkill(path.join(installed, "skills"), "installed-tool", "# Installed"); + yield* fs.writeFileString( + path.join(configDir, "plugins", "installed_plugins.json"), + JSON.stringify({ + plugins: { + "tools@shop": [{ scope: "user", installPath: installed }], + "pstack@shop": [{ scope: "user", installPath: path.join(configDir, "missing") }], + }, + }), + ); + const names = (enabled: boolean) => + Effect.gen(function* () { + yield* fs.writeFileString( + path.join(configDir, "settings.json"), + JSON.stringify({ enabledPlugins: { "tools@shop": enabled, "pstack@shop": true } }), + ); + const skills = yield* discoverClaudeSkills({ homePath: configDir }, undefined, { + CLAUDE_CODE_ORGANIZATION_UUID: "11111111-2222-4333-8444-555555555555", + CLAUDE_CODE_ACCOUNT_UUID: "not-a-uuid", + }); + return skills.map((skill) => skill.name); + }); + + assert.deepEqual(yield* names(true), [ + "a_b:renamed", + "pstack:poteto-mode", + "tools:installed-tool", + ]); + assert.deepEqual(yield* names(false), [ + "a_b:renamed", + "pstack:poteto-mode", + "tools:synced-tool", + ]); + yield* writeSkill(path.join(bucket, "no-id", "skills"), "unlisted", "# Synced"); + assert.notInclude( + (yield* discoverClaudeSkills({ homePath: configDir }, undefined, { + CLAUDE_CODE_ORGANIZATION_UUID: "11111111-2222-4333-8444-555555555555", + })).map((skill) => skill.name), + "no-id:unlisted", + ); + }), + ); + + it.effect("hides synced plugins from local settings but not shared project settings", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const tempDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-claude-skills-" }); + const configDir = path.join(tempDir, "claude-home"); + const workspace = path.join(tempDir, "workspace"); + const bucketName = + "11111111-2222-4333-8444-555555555555_66666666-7777-4888-9999-aaaaaaaaaaaa"; + const synced = path.join(configDir, "plugins", "synced"); + yield* writeSkill(path.join(synced, bucketName, "pstack", "skills"), "poteto-mode", "# S"); + yield* fs.writeFileString( + path.join(synced, bucketName, "manifest.json"), + '{"plugins":[{"pluginId":"plugin_01pstack","name":"pstack"}]}', + ); + yield* fs.writeFileString(path.join(synced, `.bucket-${bucketName}`), ""); + yield* fs.makeDirectory(path.join(workspace, ".claude"), { recursive: true }); + yield* fs.writeFileString( + path.join(workspace, ".claude", "settings.json"), + '{"syncClaudeAiPlugins":false}', + ); + const environment = { + CLAUDE_CODE_ORGANIZATION_UUID: "11111111-2222-4333-8444-555555555555", + CLAUDE_CODE_ACCOUNT_UUID: "66666666-7777-4888-9999-AAAAAAAAAAAA", + }; + const names = Effect.map( + discoverClaudeSkills({ homePath: configDir }, workspace, environment), + (skills) => skills.map((skill) => skill.name), + ); + assert.deepEqual(yield* names, ["pstack:poteto-mode"]); + yield* fs.writeFileString( + path.join(workspace, ".claude", "settings.local.json"), + '{"syncClaudeAiPlugins":false}', + ); + assert.deepEqual(yield* names, []); + }), + ); + it.effect("discovers user and project skills with frontmatter metadata", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/server/src/provider/Drivers/ClaudeSkills.ts b/apps/server/src/provider/Drivers/ClaudeSkills.ts index d8bef60c972a..d5f527cd34c5 100644 --- a/apps/server/src/provider/Drivers/ClaudeSkills.ts +++ b/apps/server/src/provider/Drivers/ClaudeSkills.ts @@ -12,6 +12,8 @@ import { parse as parseYamlDocument } from "yaml"; import { expandHomePath } from "@t3tools/provider-core/server/pathExpansion"; +import { claudeAccountConfigPath, readClaudeOauthAccount } from "../claudeResetCredits.ts"; + type ClaudeSkillScope = "user" | "project"; type ClaudeSkillRoot = { @@ -217,6 +219,7 @@ const SkillSettings = fromLenientJson( Schema.Struct({ skillOverrides: Schema.optional(Schema.Record(Schema.String, SkillOverrideValue)), enabledPlugins: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), + syncClaudeAiPlugins: Schema.optional(Schema.Unknown), }), ); const decodeSkillSettings = Schema.decodeUnknownEffect(SkillSettings); @@ -248,7 +251,10 @@ const readSkillSettings = Effect.fn("readSkillSettings")(function* ( const platform = yield* HostProcess.Platform; const overridesByName = new Map(); const enabledPlugins = new Map(); + let syncedPluginsHidden = false; const repositoryRoot = cwd === undefined ? undefined : yield* findRepositoryRoot(cwd); + const projectSettingsPath = + cwd === undefined ? undefined : path.join(cwd, ".claude", "settings.json"); for (const settingsPath of claudeSettingsPaths( path, @@ -285,9 +291,13 @@ const readSkillSettings = Effect.fn("readSkillSettings")(function* ( enabledPlugins.delete(key); } } + // Claude ignores this switch in shared project settings, and honors only false. + if (parsed?.syncClaudeAiPlugins === false && settingsPath !== projectSettingsPath) { + syncedPluginsHidden = true; + } } - return { skillOverrides: overridesByName, enabledPlugins }; + return { skillOverrides: overridesByName, enabledPlugins, syncedPluginsHidden }; }); const InstalledPlugins = Schema.fromJsonString( @@ -307,15 +317,128 @@ const PluginManifest = Schema.fromJsonString( }), ); +const SyncedPlugins = Schema.fromJsonString( + Schema.Struct({ plugins: Schema.Array(Schema.Unknown) }), +); +const decodeSyncedPlugin = Schema.decodeUnknownOption( + Schema.Struct({ + pluginId: Schema.String, + name: Schema.String, + generation: Schema.optional(Schema.Number), + }), +); +const ACCOUNT_UUID_PATTERN = /^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i; + +/** Claude compares plugin names case-insensitively. */ +function pluginNameKey(name: string): string { + return name.normalize("NFC").toLowerCase(); +} + +/** Claude's folder for a synced row: reserved characters become `_`, trailing dots and spaces go. */ +function syncedPluginDirectoryName(name: string): string { + const replaced = name.replace(/[<>:"|?*\\/]/g, "_"); + let end = replaced.length; + while (end > 0 && (replaced[end - 1] === "." || replaced[end - 1] === " ")) end--; + return replaced.slice(0, end); +} + function isInsideDirectory(path: Path.Path, directory: string, candidate: string): boolean { const relative = path.relative(directory, candidate); return relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative); } +const readPluginManifest = Effect.fn("readPluginManifest")(function* (pluginPath: string) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + return yield* fileSystem + .readFileString(path.join(pluginPath, ".claude-plugin", "plugin.json")) + .pipe( + Effect.flatMap(Schema.decodeUnknownEffect(PluginManifest)), + Effect.orElseSucceed(() => undefined), + ); +}); + +const pluginSkillRoots = Effect.fn("pluginSkillRoots")(function* ( + installPath: string, + scope: ClaudeSkillScope, + prefix: string, + manifest: typeof PluginManifest.Type | undefined, +) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const roots: Array = []; + const defaultDirectory = path.join(installPath, "skills"); + const hasDefaultDirectory = yield* fileSystem + .exists(defaultDirectory) + .pipe(Effect.orElseSucceed(() => false)); + if (hasDefaultDirectory) roots.push({ directory: defaultDirectory, scope, prefix }); + const declared = manifest?.skills; + if (declared === undefined && !hasDefaultDirectory) { + const hasRootSkill = yield* fileSystem + .exists(path.join(installPath, "SKILL.md")) + .pipe(Effect.orElseSucceed(() => false)); + if (hasRootSkill) roots.push({ directory: installPath, scope, prefix }); + } + + const canonicalInstallPath = yield* fileSystem + .realPath(installPath) + .pipe(Effect.orElseSucceed(() => undefined)); + for (const relative of typeof declared === "string" ? [declared] : (declared ?? [])) { + const directory = path.resolve(installPath, relative); + if (!isInsideDirectory(path, installPath, directory) || canonicalInstallPath === undefined) { + continue; + } + const canonicalDirectory = yield* fileSystem + .realPath(directory) + .pipe(Effect.orElseSucceed(() => undefined)); + // The native loader rejects manifest roots that escape through a symlink, too. + if ( + canonicalDirectory !== undefined && + isInsideDirectory(path, canonicalInstallPath, canonicalDirectory) + ) { + roots.push({ directory, scope, prefix }); + } + } + return roots; +}); + +/** The `_` folder Claude syncs the signed-in account's claude.ai plugins into. */ +const resolveSyncedPluginBucket = Effect.fn("resolveSyncedPluginBucket")(function* ( + config: Pick, + configDirPath: string, + environment: NodeJS.ProcessEnv, +) { + const accountConfigPath = yield* claudeAccountConfigPath( + config.homePath.trim() || environment.CLAUDE_CONFIG_DIR?.trim() ? configDirPath : undefined, + ); + const account = yield* readClaudeOauthAccount(accountConfigPath); + const organizationId = environment.CLAUDE_CODE_ORGANIZATION_UUID ?? account?.organizationUuid; + // A stored claude.ai login names its own account; an injected token uses the env account. + const environmentAccountId = environment.CLAUDE_CODE_ACCOUNT_UUID; + const accountId = + account !== undefined && !environment.CLAUDE_CODE_OAUTH_TOKEN + ? typeof account.accountUuid === "string" + ? account.accountUuid + : undefined + : environmentAccountId && ACCOUNT_UUID_PATTERN.test(environmentAccountId) + ? environmentAccountId + : "unbound"; + if ( + !organizationId || + !accountId || + !ACCOUNT_UUID_PATTERN.test(organizationId) || + (accountId !== "unbound" && !ACCOUNT_UUID_PATTERN.test(accountId)) + ) { + return undefined; + } + return `${organizationId.toLowerCase()}_${accountId.toLowerCase()}`; +}); + const discoverPluginSkillRoots = Effect.fn("discoverPluginSkillRoots")(function* ( configDirPath: string, cwd: string | undefined, enabledPlugins: ReadonlyMap, + syncedBucket: string | undefined, environment: NodeJS.ProcessEnv, ) { const fileSystem = yield* FileSystem.FileSystem; @@ -331,8 +454,8 @@ const discoverPluginSkillRoots = Effect.fn("discoverPluginSkillRoots")(function* Effect.flatMap(Schema.decodeUnknownEffect(InstalledPlugins)), Effect.orElseSucceed(() => undefined), ); - if (installed === undefined) return []; const roots: Array = []; + const claimedPluginNames = new Set(); // The CLI's process.cwd() resolves symlinks; registry projectPath values remain lexical. const workspace = cwd === undefined @@ -343,7 +466,7 @@ const discoverPluginSkillRoots = Effect.fn("discoverPluginSkillRoots")(function* const repositoryRoot = workspace === undefined ? undefined : yield* findPluginRepositoryRoot(workspace); - for (const [key, entries] of Object.entries(installed.plugins).sort()) { + for (const [key, entries] of Object.entries(installed?.plugins ?? {}).sort()) { if (enabledPlugins.get(key) !== true) continue; const applicable = yield* Effect.filter( entries.flatMap((entry) => Option.toArray(decodeInstalledPlugin(entry))), @@ -370,47 +493,48 @@ const discoverPluginSkillRoots = Effect.fn("discoverPluginSkillRoots")(function* if (!install) continue; const scope = install.scope === "user" || install.scope === "managed" ? "user" : "project"; const installPath = path.resolve(cwd ?? ".", install.installPath); - const manifest = yield* fileSystem - .readFileString(path.join(installPath, ".claude-plugin", "plugin.json")) - .pipe( - Effect.flatMap(Schema.decodeUnknownEffect(PluginManifest)), - Effect.orElseSucceed(() => undefined), - ); + const manifest = yield* readPluginManifest(installPath); const prefix = manifest?.name ?? key.split("@")[0]; if (!prefix) continue; - - const defaultDirectory = path.join(installPath, "skills"); - const hasDefaultDirectory = yield* fileSystem - .exists(defaultDirectory) - .pipe(Effect.orElseSucceed(() => false)); - if (hasDefaultDirectory) roots.push({ directory: defaultDirectory, scope, prefix }); - const declared = manifest?.skills; - if (declared === undefined && !hasDefaultDirectory) { - const hasRootSkill = yield* fileSystem - .exists(path.join(installPath, "SKILL.md")) - .pipe(Effect.orElseSucceed(() => false)); - if (hasRootSkill) roots.push({ directory: installPath, scope, prefix }); - } - - const canonicalInstallPath = yield* fileSystem - .realPath(installPath) - .pipe(Effect.orElseSucceed(() => undefined)); - for (const relative of typeof declared === "string" ? [declared] : (declared ?? [])) { - const directory = path.resolve(installPath, relative); - if (!isInsideDirectory(path, installPath, directory) || canonicalInstallPath === undefined) { - continue; - } - const canonicalDirectory = yield* fileSystem - .realPath(directory) - .pipe(Effect.orElseSucceed(() => undefined)); - // The native loader rejects manifest roots that escape through a symlink, too. - if ( - canonicalDirectory !== undefined && - isInsideDirectory(path, canonicalInstallPath, canonicalDirectory) - ) { - roots.push({ directory, scope, prefix }); - } - } + // A registry record whose install is gone loads nothing and claims no namespace. + if (Option.isSome(existing)) claimedPluginNames.add(pluginNameKey(prefix)); + roots.push(...(yield* pluginSkillRoots(installPath, scope, prefix, manifest))); + } + if (syncedBucket === undefined) return roots; + + const syncedPath = path.join(configDirPath, "plugins", "synced"); + const bucketPath = path.join(syncedPath, syncedBucket); + const disabledSyncedPlugins = new Set( + [...enabledPlugins].flatMap(([pluginKey, enabled]) => + !enabled && pluginKey.endsWith("@synced") ? [pluginNameKey(pluginKey.slice(0, -7))] : [], + ), + ); + const hasBucketMarker = yield* fileSystem + .exists(path.join(syncedPath, `.bucket-${syncedBucket}`)) + .pipe(Effect.orElseSucceed(() => false)); + if (!hasBucketMarker) return roots; + const synced = yield* fileSystem.readFileString(path.join(bucketPath, "manifest.json")).pipe( + Effect.flatMap(Schema.decodeUnknownEffect(SyncedPlugins)), + Effect.orElseSucceed(() => undefined), + ); + for (const row of synced?.plugins ?? []) { + const plugin = Option.getOrUndefined(decodeSyncedPlugin(row)); + const directoryName = plugin === undefined ? "" : syncedPluginDirectoryName(plugin.name); + if (!plugin || !directoryName || directoryName.startsWith(".")) continue; + const generation = plugin.generation; + const pluginPath = path.join( + bucketPath, + generation !== undefined && Number.isInteger(generation) && generation >= 2 + ? `${directoryName}~g${generation}` + : directoryName, + ); + const manifest = yield* readPluginManifest(pluginPath); + const prefix = manifest?.name ?? directoryName; + // An installed plugin of the same name, then the first synced row, holds the namespace. + const key = pluginNameKey(prefix); + if (claimedPluginNames.has(key) || disabledSyncedPlugins.has(key)) continue; + claimedPluginNames.add(key); + roots.push(...(yield* pluginSkillRoots(pluginPath, "user", prefix, manifest))); } return roots; }); @@ -473,6 +597,17 @@ export const readClaudeSettingsEnv = Effect.fn("readClaudeSettingsEnv")(function return settingsEnv; }); +/** The host of an `ANTHROPIC_BASE_URL` that is not Anthropic's own API. */ +export function customClaudeApiEndpointHost(baseUrl: string | undefined): string | undefined { + const endpoint = URL.parse(baseUrl?.trim() ?? ""); + return endpoint !== null && + (endpoint.protocol === "http:" || endpoint.protocol === "https:") && + endpoint.hostname !== "" && + endpoint.hostname !== "api.anthropic.com" + ? endpoint.hostname + : undefined; +} + // Claude's --add-dir is repeatable and variadic; parseCliArgs keeps only one value. export function claudeLaunchArgDirectories(launchArgs?: string): ReadonlyArray { const directories: string[] = []; @@ -501,12 +636,22 @@ export const discoverClaudeSkills = Effect.fn("discoverClaudeSkills")(function* ): Effect.fn.Return, never, FileSystem.FileSystem | Path.Path> { const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const configDirPath = yield* resolveClaudeConfigDirPath(config, environment ?? process.env, cwd); - const { skillOverrides, enabledPlugins } = yield* readSkillSettings( + const env = environment ?? process.env; + const configDirPath = yield* resolveClaudeConfigDirPath(config, env, cwd); + const { skillOverrides, enabledPlugins, syncedPluginsHidden } = yield* readSkillSettings( configDirPath, cwd, - environment ?? process.env, + env, ); + // Most installs have no synced folder; skip reading `.claude.json` for them. + const hasSyncedPlugins = + !syncedPluginsHidden && + (yield* fileSystem + .exists(path.join(configDirPath, "plugins", "synced")) + .pipe(Effect.orElseSucceed(() => false))); + const syncedBucket = hasSyncedPlugins + ? yield* resolveSyncedPluginBucket(config, configDirPath, env) + : undefined; const roots: ReadonlyArray = [ { directory: path.join(configDirPath, "skills"), scope: "user" }, @@ -517,12 +662,7 @@ export const discoverClaudeSkills = Effect.fn("discoverClaudeSkills")(function* directory: path.join(path.resolve(cwd ?? ".", directory), ".claude", "skills"), scope: "project" as const, })), - ...(yield* discoverPluginSkillRoots( - configDirPath, - cwd, - enabledPlugins, - environment ?? process.env, - )), + ...(yield* discoverPluginSkillRoots(configDirPath, cwd, enabledPlugins, syncedBucket, env)), ]; const skillsByName = new Map(); diff --git a/apps/server/src/provider/Drivers/CodexDriver.test.ts b/apps/server/src/provider/Drivers/CodexDriver.test.ts index c5114253d904..80510386dfa5 100644 --- a/apps/server/src/provider/Drivers/CodexDriver.test.ts +++ b/apps/server/src/provider/Drivers/CodexDriver.test.ts @@ -6,11 +6,15 @@ import * as NodePath from "node:path"; import { expect, it } from "@effect/vitest"; import { ProviderInstanceId } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as PlatformError from "effect/PlatformError"; +import * as Queue from "effect/Queue"; import * as Sink from "effect/Sink"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; import { HttpClient } from "effect/http"; import * as ChildProcess from "effect/process/ChildProcess"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; @@ -463,4 +467,54 @@ it.layer(layerTest)("CodexDriver", (it) => { }).pipe(Effect.scoped), { skip: windowsHost }, ); + + it.effect("probes again when auth.json in the Codex home changes", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const codexHome = yield* fs.makeTempDirectoryScoped({ prefix: "t3-codex-auth-watch-" }); + const watchedPath = yield* Deferred.make(); + const homeEvents = yield* Queue.unbounded(); + const probes = yield* Queue.unbounded(); + yield* CodexDriver.create({ + instanceId: ProviderInstanceId.make("codex-auth-watch"), + displayName: "Codex auth watch", + enabled: true, + environment: [], + config: { ...CodexDriver.defaultConfig(), homePath: codexHome }, + }).pipe( + // OS watchers can miss events written right after they start, so the test supplies them. + Effect.provideService(FileSystem.FileSystem, { + ...fs, + watch: (path) => + Stream.unwrap( + Deferred.succeed(watchedPath, path).pipe(Effect.as(Stream.fromQueue(homeEvents))), + ), + }), + Effect.provideService( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make((command) => + (ChildProcess.isStandardCommand(command) && command.args.includes("app-server") + ? Queue.offer(probes, undefined) + : Effect.void + ).pipe( + Effect.andThen( + PlatformError.badArgument({ + module: "ChildProcessSpawner", + method: "spawn", + description: "The fixture has no Codex executable", + }), + ), + ), + ), + ), + ); + expect(yield* Deferred.await(watchedPath)).toBe(codexHome); + yield* Queue.take(probes); + + yield* Queue.offer(homeEvents, { _tag: "Update", path: "auth.json" }); + yield* TestClock.adjust("250 millis"); + + yield* Queue.take(probes); + }).pipe(Effect.scoped), + ); }); diff --git a/apps/server/src/provider/Drivers/CodexDriver.ts b/apps/server/src/provider/Drivers/CodexDriver.ts index cad0d34af621..df13768003b3 100644 --- a/apps/server/src/provider/Drivers/CodexDriver.ts +++ b/apps/server/src/provider/Drivers/CodexDriver.ts @@ -2,10 +2,12 @@ import { CodexSettings, ProviderDriverKind } from "@t3tools/contracts"; import * as Digest from "effect/crypto/Digest"; import * as SecureRandom from "effect/crypto/SecureRandom"; import type * as Signature from "effect/crypto/Signature"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; import { HttpClient } from "effect/http"; import { ChildProcessSpawner } from "effect/process"; @@ -287,6 +289,20 @@ export const CodexDriver: ProviderDriver pathService.basename(event.path) === "auth.json"), + Stream.debounce(Duration.millis(250)), + Stream.runForEach(() => snapshot.refresh.pipe(Effect.ignoreCause({ log: true }))), + Effect.catchReason("PlatformError", "NotFound", () => Effect.void), + Effect.ignoreCause({ log: true }), + Effect.forkScoped, + ); + } return { instanceId, driverKind: DRIVER_KIND, diff --git a/apps/server/src/provider/ModelManifest.test.ts b/apps/server/src/provider/ModelManifest.test.ts index a27bb5499870..4293cc69aa5e 100644 --- a/apps/server/src/provider/ModelManifest.test.ts +++ b/apps/server/src/provider/ModelManifest.test.ts @@ -81,6 +81,7 @@ describe("resolveProviderCatalog", () => { current: Effect.succeed(manifest), refresh: Effect.die("Catalog reads must not refresh the manifest"), forceRefresh: Effect.die("Catalog reads must not refresh the manifest"), + refreshOnDemand: Effect.die("Catalog reads must not refresh the manifest"), refreshInBackground: Effect.die("Catalog reads must not refresh the manifest"), }), ), diff --git a/apps/server/src/provider/ModelManifest.ts b/apps/server/src/provider/ModelManifest.ts index 5060285ca809..745b91972ced 100644 --- a/apps/server/src/provider/ModelManifest.ts +++ b/apps/server/src/provider/ModelManifest.ts @@ -136,6 +136,8 @@ export class ModelManifest extends Context.Service< readonly current: Effect.Effect; readonly refresh: Effect.Effect; readonly forceRefresh: Effect.Effect; + /** For a user looking at the model list: tolerates far less staleness than `refresh`. */ + readonly refreshOnDemand: Effect.Effect; /** Process-shared fetches must survive the triggering instance's teardown. */ readonly refreshInBackground: Effect.Effect; } @@ -145,6 +147,7 @@ const BundledOnlyModelManifest: ModelManifest["Service"] = { current: Effect.succeed(BUNDLED_MODEL_MANIFEST), refresh: Effect.succeed(BUNDLED_MODEL_MANIFEST), forceRefresh: Effect.succeed(BUNDLED_MODEL_MANIFEST), + refreshOnDemand: Effect.succeed(BUNDLED_MODEL_MANIFEST), refreshInBackground: Effect.void, }; diff --git a/apps/server/src/provider/ProviderInstallationGate.test.ts b/apps/server/src/provider/ProviderInstallationGate.test.ts new file mode 100644 index 000000000000..96d958e789f5 --- /dev/null +++ b/apps/server/src/provider/ProviderInstallationGate.test.ts @@ -0,0 +1,49 @@ +import { ProviderInstanceId } from "@t3tools/contracts"; +import { assert, it } from "@effect/vitest"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; + +import * as ProviderInstallationGate from "./ProviderInstallationGate.ts"; + +const INSTANCE_ID = ProviderInstanceId.make("codex"); + +it.effect("queues work that starts while an update waits for in-flight work", () => + Effect.gen(function* () { + const gate = yield* ProviderInstallationGate.ProviderInstallationGate; + const inFlightStarted = yield* Deferred.make(); + const releaseInFlight = yield* Deferred.make(); + const log: Array = []; + + const inFlight = yield* gate + .execute( + INSTANCE_ID, + Deferred.succeed(inFlightStarted, undefined).pipe( + Effect.andThen(Deferred.await(releaseInFlight)), + Effect.andThen(Effect.sync(() => log.push("in-flight work finished"))), + ), + ) + .pipe(Effect.forkScoped); + yield* Deferred.await(inFlightStarted); + // Starting immediately runs each fiber until it blocks: the update while draining in-flight + // work, by which point it holds the turnstile, and later work at the gate. + const update = yield* gate + .update( + [INSTANCE_ID], + Effect.sync(() => log.push("update ran")), + ) + .pipe(Effect.forkScoped({ startImmediately: true })); + const later = yield* gate + .execute( + INSTANCE_ID, + Effect.sync(() => log.push("later work ran")), + ) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Deferred.succeed(releaseInFlight, undefined); + yield* Fiber.join(inFlight); + yield* Fiber.join(update); + yield* Fiber.join(later); + + assert.deepStrictEqual(log, ["in-flight work finished", "update ran", "later work ran"]); + }).pipe(Effect.provide(ProviderInstallationGate.layer)), +); diff --git a/apps/server/src/provider/ProviderInstallationGate.ts b/apps/server/src/provider/ProviderInstallationGate.ts new file mode 100644 index 000000000000..3c526f9ee825 --- /dev/null +++ b/apps/server/src/provider/ProviderInstallationGate.ts @@ -0,0 +1,73 @@ +import type { ProviderInstanceId } from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Semaphore from "effect/Semaphore"; + +// Executions share `room` and an update drains it. A Semaphore lets a smaller request overtake a +// waiting larger one, so an update first holds `turnstile` and new executions queue behind it. +interface InstanceGate { + readonly turnstile: Semaphore.Semaphore; + readonly room: Semaphore.Semaphore; +} + +const ROOM_PERMITS = Number.MAX_SAFE_INTEGER; + +export class ProviderInstallationGate extends Context.Service< + ProviderInstallationGate, + { + /** Runs provider work once no update of the instance's installation is running or waiting. */ + readonly execute: ( + instanceId: ProviderInstanceId, + effect: Effect.Effect, + ) => Effect.Effect; + /** Runs an installation update after the instances' in-flight work, excluding new work. */ + readonly update: ( + instanceIds: ReadonlyArray, + effect: Effect.Effect, + ) => Effect.Effect; + } +>()("t3/provider/ProviderInstallationGate") {} + +export const layer = Layer.sync(ProviderInstallationGate, () => { + // Instances are few and configured, so their gates are kept for the server's lifetime. + const gates = new Map(); + const gateFor = (instanceId: ProviderInstanceId) => + Effect.sync(() => { + const existing = gates.get(instanceId); + if (existing) return existing; + const gate = { + turnstile: Semaphore.makeUnsafe(1), + room: Semaphore.makeUnsafe(ROOM_PERMITS), + }; + gates.set(instanceId, gate); + return gate; + }); + + const execute = Effect.fn("ProviderInstallationGate.execute")(function* ( + instanceId: ProviderInstanceId, + effect: Effect.Effect, + ) { + const gate = yield* gateFor(instanceId); + yield* gate.turnstile.withPermits(1)(Effect.void); + return yield* gate.room.withPermits(1)(effect); + }); + + const update = Effect.fn("ProviderInstallationGate.update")(function* ( + instanceIds: ReadonlyArray, + effect: Effect.Effect, + ) { + // A stable order keeps two updates that share instances from deadlocking. + const instanceGates = yield* Effect.forEach(Array.from(new Set(instanceIds)).sort(), gateFor); + const drained = instanceGates.reduceRight( + (inner, gate) => gate.room.withPermits(ROOM_PERMITS)(inner), + effect, + ); + return yield* instanceGates.reduceRight( + (inner, gate) => gate.turnstile.withPermits(1)(inner), + drained, + ); + }); + + return ProviderInstallationGate.of({ execute, update }); +}); diff --git a/apps/server/src/provider/ProviderInstanceRegistry.test.ts b/apps/server/src/provider/ProviderInstanceRegistry.test.ts index e8b4de4fb1b4..d91fc33bee15 100644 --- a/apps/server/src/provider/ProviderInstanceRegistry.test.ts +++ b/apps/server/src/provider/ProviderInstanceRegistry.test.ts @@ -39,6 +39,7 @@ import * as ModelManifest from "./ModelManifest.ts"; import * as ProviderEventLoggers from "@t3tools/provider-core/server/ProviderEventLoggers"; import { makeProviderInstanceRegistry } from "./ProviderInstanceRegistry.ts"; import { ProviderDriverError } from "./Errors.ts"; +import * as ProviderInstallationGate from "./ProviderInstallationGate.ts"; import * as ProviderOrchestrationAdapterInfrastructure from "./ProviderOrchestrationAdapterInfrastructure.ts"; import * as PortScanner from "../preview/PortScanner.ts"; import * as ProviderHostLive from "./ProviderHostLive.ts"; @@ -95,8 +96,12 @@ const makeTildeProviderFixtures = Effect.fn( directory: homePath, prefix: ".t3-provider-path-test-", }); + yield* fileSystem.writeFileString( + path.join(fixtureDir, "package.json"), + JSON.stringify({ type: "commonjs" }), + ); const codexPath = path.join(fixtureDir, "codex"); - const claudePath = path.join(fixtureDir, "claude"); + const claudePath = path.join(fixtureDir, "claude.mjs"); const claudeHomePath = path.join(fixtureDir, "claude-home"); const codexScriptPath = path.join(fixtureDir, "codex-script.json"); const codexFixtureDir = path.join(import.meta.dirname, "testFixtures"); @@ -158,6 +163,7 @@ describe("ProviderInstanceRegistry — multi-instance codex slice", () => { ), Layer.provideMerge(ProviderLatestVersions.layer), Layer.provideMerge(McpProviderSessions.layer), + Layer.provideMerge(ProviderInstallationGate.layer), Layer.provideMerge(ModelManifest.layerTest), ); const layerBase = ProviderHostLive.layer.pipe(Layer.provideMerge(layerBaseDeps)); @@ -407,6 +413,93 @@ describe("ProviderInstanceRegistry — multi-instance codex slice", () => { }).pipe(Effect.provide(layerTest)), ); + it.live("applies the lowest maxEffortLevel across Claude's settings sources", () => + Effect.gen(function* () { + if (yield* HostProcess.isWindows) return; + + const fixtures = yield* makeTildeProviderFixtures(); + const claudeId = ProviderInstanceId.make("claude_source_effort_cap"); + const { registry } = yield* makeProviderInstanceRegistry({ + drivers: [ClaudeDriver], + configMap: { + [claudeId]: { + driver: ProviderDriverKind.make("claudeAgent"), + enabled: true, + environment: [ + { + name: "T3_CLAUDE_SETTINGS_SOURCES", + // The merged `effective` settings read as exempting Opus 5; the flag tier's + // top-level cap still applies to it. + value: JSON.stringify([ + { + source: "userSettings", + settings: { modelSettings: { "claude-opus-5": { maxEffortLevel: "max" } } }, + }, + { source: "flagSettings", settings: { maxEffortLevel: "high" } }, + ]), + sensitive: false, + }, + ], + config: makeClaudeConfig({ + enabled: true, + binaryPath: fixtures.claudeBinaryPath, + homePath: fixtures.claudeHomePath, + }), + }, + }, + }); + const claude = yield* registry.getInstance(claudeId); + + const snapshot = yield* claude!.snapshot.refresh; + const maxLabel = snapshot.models + .find((model) => model.slug === "claude-opus-5") + ?.capabilities?.optionDescriptors?.flatMap((descriptor) => + descriptor.id === "effort" && descriptor.type === "select" + ? descriptor.options.filter((option) => option.id === "max") + : [], + )[0]?.label; + expect(maxLabel).toBe("Max (capped to High)"); + }).pipe(Effect.provide(layerTest)), + ); + + it.live("keeps usage and uncapped efforts when the CLI rejects get_settings", () => + Effect.gen(function* () { + if (yield* HostProcess.isWindows) return; + + const fixtures = yield* makeTildeProviderFixtures(); + const claudeId = ProviderInstanceId.make("claude_settings_unsupported"); + const { registry } = yield* makeProviderInstanceRegistry({ + drivers: [ClaudeDriver], + configMap: { + [claudeId]: { + driver: ProviderDriverKind.make("claudeAgent"), + enabled: true, + environment: [{ name: "T3_CLAUDE_SETTINGS_UNSUPPORTED", value: "1", sensitive: false }], + config: makeClaudeConfig({ + enabled: true, + binaryPath: fixtures.claudeBinaryPath, + homePath: fixtures.claudeHomePath, + }), + }, + }, + }); + const claude = yield* registry.getInstance(claudeId); + + const snapshot = yield* claude!.snapshot.refresh; + const effortLabels = snapshot.models.flatMap((model) => + (model.capabilities?.optionDescriptors ?? []).flatMap((descriptor) => + descriptor.id === "effort" && descriptor.type === "select" + ? descriptor.options.map((option) => option.label) + : [], + ), + ); + expect(effortLabels).toContain("Max"); + expect(effortLabels.some((label) => label.includes("capped"))).toBe(false); + expect(snapshot.auth).toMatchObject({ status: "authenticated", email: "test@example.com" }); + expect(snapshot.usageLimits?.windows[0]?.usedPercent).toBe(100); + }).pipe(Effect.provide(layerTest)), + ); + // Coder: upstream's organization-scoped fallback test, without the organization change the // fork does not read, against the CLI transport. it.effect("reuses the last good Claude capabilities once after a failed probe", () => diff --git a/apps/server/src/provider/ProviderRegistry.test.ts b/apps/server/src/provider/ProviderRegistry.test.ts index 7cc567cdaa7c..f81eadcac6d3 100644 --- a/apps/server/src/provider/ProviderRegistry.test.ts +++ b/apps/server/src/provider/ProviderRegistry.test.ts @@ -49,6 +49,7 @@ import { applyProviderCompatibility } from "./providerCompatibility.ts"; import * as ProviderLatestVersions from "@t3tools/provider-core/server/ProviderLatestVersions"; import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions"; import * as ProviderEventLoggers from "@t3tools/provider-core/server/ProviderEventLoggers"; +import * as ProviderInstallationGate from "./ProviderInstallationGate.ts"; import * as ProviderInstanceRegistryHydration from "./ProviderInstanceRegistryHydration.ts"; import * as PortScanner from "../preview/PortScanner.ts"; import * as ServerConfig from "../config.ts"; @@ -740,6 +741,7 @@ it.layer( } as const satisfies ServerProvider; const refreshedProvider = { ...previousProvider, + status: "warning", checkedAt: "2026-04-14T00:01:00.000Z", models: [], slashCommands: [], @@ -931,6 +933,45 @@ it.layer( ); }); + it("drops Cursor models a completed catalog read no longer returns", () => { + const previousProvider = { + instanceId: ProviderInstanceId.make("cursor"), + driver: ProviderDriverKind.make("cursor"), + status: "ready", + enabled: true, + installed: true, + auth: { status: "authenticated" }, + checkedAt: "2026-10-30T00:00:00.000Z", + version: "1.0.0", + models: [ + { slug: "composer-2", name: "Composer 2", isCustom: false, capabilities: null }, + { slug: "retired-model-x", name: "Retired", isCustom: false, capabilities: null }, + ], + slashCommands: [], + skills: [], + } as const satisfies ServerProvider; + const refreshedProvider = { + ...previousProvider, + checkedAt: "2026-10-30T00:01:00.000Z", + models: [previousProvider.models[0]], + } satisfies ServerProvider; + const failedProvider = { + ...refreshedProvider, + status: "error", + auth: { status: "unknown" }, + message: "Cursor SDK model discovery timed out.", + } satisfies ServerProvider; + + assert.deepStrictEqual( + ProviderRegistry.mergeProviderSnapshot(previousProvider, refreshedProvider).models, + [previousProvider.models[0]], + ); + assert.deepStrictEqual( + ProviderRegistry.mergeProviderSnapshot(previousProvider, failedProvider).models, + [...previousProvider.models], + ); + }); + it("retains stale OpenCode models when a refresh fails", () => { const previousProvider = { instanceId: ProviderInstanceId.make("opencode"), @@ -1505,6 +1546,7 @@ it.layer( it.effect("invalidates model caches only for an explicit model refresh", () => Effect.gen(function* () { const forceRefreshCalls = yield* Ref.make(0); + const manifest = yield* Ref.make(ModelManifest.BUNDLED_MODEL_MANIFEST); const makeInstance = (name: string) => Effect.gen(function* () { const driver = ProviderDriverKind.make("codex"); @@ -1523,6 +1565,7 @@ it.layer( skills: [], } as const satisfies ServerProvider; const invalidations = yield* Ref.make(0); + const probes = yield* Ref.make(0); const maintenanceReads = yield* Ref.make>([]); const instance = { instanceId, @@ -1544,7 +1587,7 @@ it.layer( ), ), getSnapshot: Effect.succeed(provider), - refresh: Effect.succeed(provider), + refresh: Ref.update(probes, (count) => count + 1).pipe(Effect.as(provider)), streamChanges: Stream.empty, applyUsageLimits: () => Effect.void, }, @@ -1553,7 +1596,7 @@ it.layer( textGeneration: {} as ProviderInstance["textGeneration"], } satisfies ProviderInstance; const takeMaintenanceReads = Ref.getAndSet(maintenanceReads, []); - return { instance, invalidations, takeMaintenanceReads }; + return { instance, invalidations, probes, takeMaintenanceReads }; }); const first = yield* makeInstance("codex_first"); const second = yield* makeInstance("codex_second"); @@ -1570,11 +1613,12 @@ it.layer( }, ); const layerModelManifest = Layer.succeed(ModelManifest.ModelManifest, { - current: Effect.succeed(ModelManifest.BUNDLED_MODEL_MANIFEST), + current: Ref.get(manifest), refresh: Effect.succeed(ModelManifest.BUNDLED_MODEL_MANIFEST), forceRefresh: Ref.update(forceRefreshCalls, (count) => count + 1).pipe( Effect.as(ModelManifest.BUNDLED_MODEL_MANIFEST), ), + refreshOnDemand: Ref.get(manifest), refreshInBackground: Effect.void, }); const layerLatestVersions = Layer.effect( @@ -1584,9 +1628,9 @@ it.layer( ["codex_second-package", "1.0.0"], ]), ); - const readCachedVersion = (key: string) => + const readCachedVersion = (key: string, latest = "2.0.0") => ProviderLatestVersions.ProviderLatestVersions.use((latestVersions) => - latestVersions.cached(key, Effect.succeed("2.0.0")), + latestVersions.cached(key, Effect.succeed(latest)), ); yield* Effect.gen(function* () { @@ -1620,6 +1664,21 @@ it.layer( assert.deepStrictEqual(yield* first.takeMaintenanceReads, [true]); yield* registry.invalidateModelCaches({ ...workspace, fresh: true }); assert.deepStrictEqual(yield* first.takeMaintenanceReads, [false]); + + // An opened model picker probes only when the manifest differs from what the last pass saw. + const probesBefore = yield* Ref.get(second.probes); + yield* registry.refreshModelCatalog; + assert.strictEqual(yield* Ref.get(second.probes), probesBefore); + assert.strictEqual(yield* readCachedVersion("codex_second-package", "3.0.0"), "2.0.0"); + yield* Ref.set(manifest, { + ...ModelManifest.BUNDLED_MODEL_MANIFEST, + updatedAt: "2999-01-01T00:00:00Z", + }); + yield* registry.refreshModelCatalog; + yield* registry.refreshModelCatalog; + assert.strictEqual(yield* Ref.get(second.probes), probesBefore + 1); + // A changed manifest also rechecks latest CLI versions. + assert.strictEqual(yield* readCachedVersion("codex_second-package", "3.0.0"), "3.0.0"); }).pipe( Effect.provide( ProviderRegistry.layer.pipe( @@ -1732,6 +1791,97 @@ it.layer( }), ); + it.effect("scans scanCwd and keys the workspace snapshot by cwd", () => + Effect.gen(function* () { + const driver = ProviderDriverKind.make("codex"); + const instanceId = ProviderInstanceId.make("codex"); + const provider = { + instanceId, + driver, + status: "ready", + enabled: true, + installed: true, + auth: { status: "authenticated" }, + checkedAt: "2026-06-10T00:00:00.000Z", + version: "1.0.0", + models: [], + slashCommands: [], + skills: [], + } as const satisfies ServerProvider; + const scannedDirectories = yield* Ref.make>([]); + const instance: ProviderInstance = { + instanceId, + driverKind: driver, + continuationIdentity: { driverKind: driver, continuationKey: "codex:instance:codex" }, + displayName: undefined, + enabled: true, + snapshot: { + resolveMaintenance: () => + Effect.succeed( + makeManualOnlyProviderMaintenanceCapabilities({ + provider: driver, + packageName: null, + }), + ), + getSnapshot: Effect.succeed(provider), + refresh: Effect.succeed(provider), + streamChanges: Stream.empty, + applyUsageLimits: () => Effect.void, + }, + snapshotForCwd: (directory) => + Ref.update(scannedDirectories, (directories) => [...directories, directory]).pipe( + Effect.as({ + ...provider, + skills: [{ name: "finalize", path: `${directory}/finalize`, enabled: true }], + }), + ), + orchestrationAdapter: {} as ProviderInstance["orchestrationAdapter"], + textGeneration: {} as ProviderInstance["textGeneration"], + }; + const layerInstanceRegistry = Layer.succeed( + ProviderInstanceRegistry.ProviderInstanceRegistry, + { + getInstance: (requestedId) => + Effect.succeed(requestedId === instanceId ? instance : undefined), + listInstances: Effect.succeed([instance]), + listUnavailable: Effect.succeed([]), + streamChanges: Stream.empty, + subscribeChanges: Effect.flatMap(PubSub.unbounded(), PubSub.subscribe), + }, + ); + const scope = yield* Scope.make(); + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); + const runtimeServices = yield* Layer.build( + ProviderRegistry.layer.pipe( + Layer.provide(layerUnusedMutator), + Layer.provideMerge(layerInstanceRegistry), + Layer.provideMerge( + ServerConfig.layerTest(process.cwd(), { + prefix: "t3-provider-registry-scan-cwd-", + }), + ), + Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), + ), + ).pipe(Scope.provide(scope)); + + yield* Effect.gen(function* () { + const registry = yield* ProviderRegistry.ProviderRegistry; + const providers = yield* registry.refreshWorkspaceSnapshot({ + instanceId, + cwd: "/worktree", + scanCwd: "/project", + }); + assert.deepStrictEqual(yield* Ref.get(scannedDirectories), ["/project"]); + assert.deepStrictEqual( + providers[0]?.workspaceSnapshots + ?.find((snapshot) => snapshot.cwd === "/worktree") + ?.skills?.map((skill) => skill.path), + ["/project/finalize"], + ); + }).pipe(Effect.provide(runtimeServices)); + }), + ); + it.effect("publishes workspace skills while machine health is in error", () => Effect.gen(function* () { const driver = ProviderDriverKind.make("codex"); @@ -2242,6 +2392,7 @@ it.layer( } as const satisfies ServerProvider; const refreshedProvider = { ...initialProvider, + status: "warning", checkedAt: "2026-04-14T00:01:00.000Z", models: [], } satisfies ServerProvider; @@ -2744,7 +2895,9 @@ it.layer( yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); const layerProviderRegistry = ProviderRegistry.layer.pipe( Layer.provideMerge( - ProviderInstanceRegistryHydration.layer.pipe(Layer.provide(PortScanner.layerTest)), + ProviderInstanceRegistryHydration.layer.pipe( + Layer.provide(Layer.mergeAll(PortScanner.layerTest, ProviderInstallationGate.layer)), + ), ), Layer.provideMerge( Layer.succeed(ServerSettingsModule.ServerSettingsService, serverSettings), @@ -2857,7 +3010,9 @@ it.layer( yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); const layerProviderRegistry = ProviderRegistry.layer.pipe( Layer.provideMerge( - ProviderInstanceRegistryHydration.layer.pipe(Layer.provide(PortScanner.layerTest)), + ProviderInstanceRegistryHydration.layer.pipe( + Layer.provide(Layer.mergeAll(PortScanner.layerTest, ProviderInstallationGate.layer)), + ), ), Layer.provideMerge( Layer.succeed(ServerSettingsModule.ServerSettingsService, serverSettings), @@ -2983,7 +3138,9 @@ it.layer( ); readSeed = false; const services = yield* Layer.build( - ProviderInstanceRegistryHydration.layer.pipe(Layer.provide(PortScanner.layerTest)), + ProviderInstanceRegistryHydration.layer.pipe( + Layer.provide(Layer.mergeAll(PortScanner.layerTest, ProviderInstallationGate.layer)), + ), ).pipe(Effect.provide(dependencies)); failCurrentRead = input.failCurrentRead ?? false; yield* PubSub.publish(changes, undefined); @@ -3106,6 +3263,49 @@ it.layer( ), ); + it.effect("does not let a lazy mise shim install Claude during a status check (#33415)", () => + Effect.gen(function* () { + if ((yield* HostProcess.Platform) === "win32") return; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3-fake-mise-claude-" }); + const installMarker = path.join(directory, "installed"); + const binaryPath = path.join(directory, "claude"); + // A `lazy = true` mise shim downloads and installs the tool unless MISE_OFFLINE is set. + yield* fs.writeFileString( + binaryPath, + [ + `#!/bin/sh`, + `case "$MISE_OFFLINE" in`, + ` 1|true) cat >&2 <<'EOF'`, + `mise WARN Failed to resolve tool version list for claude: [/home/u/.config/mise/config.toml] claude@latest: no versions found for claude`, + `mise ERROR Failed to install aqua:anthropics/claude-code@latest: no versions found for claude`, + `mise ERROR Version: 2026.11.5 linux-x64 (2026-11-09)`, + `mise ERROR Run with --verbose or MISE_VERBOSE=1 for more information`, + `EOF`, + ` exit 1 ;;`, + `esac`, + `touch "${installMarker}"`, + `echo "2.1.0 (Claude Code)"`, + ].join("\n"), + ); + yield* fs.chmod(binaryPath, 0o755); + const { MISE_OFFLINE: _, ...environment } = process.env; + + const status = yield* checkClaudeProviderStatus( + { ...defaultClaudeSettings, binaryPath }, + claudeCapabilities(), + environment, + ); + + assert.isFalse( + yield* fs.exists(installMarker), + "the Claude --version status probe let mise install Claude", + ); + assert.isNull(status.version, "mise's own version is not Claude's"); + }), + ); + it.effect("reports a logged-out CLI as unauthenticated", () => Effect.gen(function* () { const status = yield* checkClaudeProviderStatus( @@ -3167,6 +3367,8 @@ it.layer( ), ); + // Coder: the signed-out message names API credentials, not a config directory + // (upstream's "tells a logged-out instance on $name where to sign in"). it.effect("tells a logged-out CLI which models need a Claude Code update", () => Effect.gen(function* () { const status = yield* checkClaudeProviderStatus( diff --git a/apps/server/src/provider/ProviderRegistry.ts b/apps/server/src/provider/ProviderRegistry.ts index fb6aeaac26b6..fe5375136dc2 100644 --- a/apps/server/src/provider/ProviderRegistry.ts +++ b/apps/server/src/provider/ProviderRegistry.ts @@ -78,6 +78,8 @@ export class ProviderRegistry extends Context.Service< readonly refreshWorkspaceSnapshot: (input: { readonly instanceId: ProviderInstanceId; readonly cwd: string; + /** Directory the driver scans; the snapshot stays keyed by `cwd`. Defaults to `cwd`. */ + readonly scanCwd?: string; readonly fresh?: boolean; }) => Effect.Effect>; @@ -92,6 +94,12 @@ export class ProviderRegistry extends Context.Service< readonly fresh?: boolean | undefined; }) => Effect.Effect; + /** + * For an opened model picker: refetch a model manifest older than a few + * minutes. When it changed, recheck latest CLI versions and probe providers. + */ + readonly refreshModelCatalog: Effect.Effect; + readonly getProviderMaintenanceCapabilitiesForInstance: ( instanceId: ProviderInstanceId, provider: ProviderDriverKind, @@ -187,9 +195,12 @@ const shouldRetainMissingProviderModels = (provider: ServerProvider): boolean => ); } - if (provider.driver === ProviderDriverKind.make("grok")) { - // Grok reports ready only after it discovers a model list; the pending, - // ACP-failed, empty-inventory, and logged-out probes are warning/error. + if ( + provider.driver === ProviderDriverKind.make("grok") || + provider.driver === ProviderDriverKind.make("cursor") + ) { + // Grok and Cursor report ready only after they discover a model list; the pending, + // failed, empty-inventory, and logged-out probes are warning/error. return !(provider.installed && provider.status === "ready"); } @@ -624,6 +635,9 @@ export const layer = Layer.effect( Option.none(), ); + // What the last full pass could see; a later manifest fetch does not reach providers by itself. + const probedManifestUpdatedAt = yield* Ref.make(initialManifest.updatedAt); + const refreshAll = Effect.fn("refreshAll")(function* () { const claimed = yield* Ref.modify( refreshAllInFlightRef, @@ -643,15 +657,20 @@ export const layer = Layer.effect( if (claimed.owner) { // Detach shared passes so the first caller's interruption cannot cancel other callers' work. // Syncing first keeps a refresh right after a rebuild from probing the replaced instance. - yield* syncLiveSourcesAndContinue.pipe( - Effect.andThen(Ref.get(liveSubsRef)), - Effect.flatMap((instances) => - Effect.forEach(instances.values(), (instance) => refreshOneSource(instance), { - concurrency: "unbounded", - discard: true, - }), + yield* manifestService.current.pipe( + Effect.flatMap(({ updatedAt }) => + syncLiveSourcesAndContinue.pipe( + Effect.andThen(Ref.get(liveSubsRef)), + Effect.flatMap((instances) => + Effect.forEach(instances.values(), (instance) => refreshOneSource(instance), { + concurrency: "unbounded", + discard: true, + }), + ), + Effect.andThen(Ref.get(providersRef)), + Effect.tap(() => Ref.set(probedManifestUpdatedAt, updatedAt)), + ), ), - Effect.andThen(Ref.get(providersRef)), Effect.onExit((exit) => Ref.set(refreshAllInFlightRef, Option.none()).pipe( Effect.andThen(Deferred.done(claimed.completion, exit)), @@ -937,6 +956,7 @@ export const layer = Layer.effect( const refreshWorkspaceSnapshot = Effect.fn("refreshWorkspaceSnapshot")(function* (input: { readonly instanceId: ProviderInstanceId; readonly cwd: string; + readonly scanCwd?: string; readonly fresh?: boolean; }) { if (input.fresh) { @@ -975,7 +995,7 @@ export const layer = Layer.effect( ) : Effect.void; const runScan = refreshMachineSnapshot.pipe( - Effect.andThen(instance.snapshotForCwd(input.cwd)), + Effect.andThen(instance.snapshotForCwd(input.scanCwd ?? input.cwd)), // Scans fail on error; returned inventory remains valid even when provider health reports an error. Effect.flatMap((scopedSnapshot) => instanceRegistry.getInstance(input.instanceId).pipe( @@ -1069,6 +1089,17 @@ export const layer = Layer.effect( ); }); + const refreshModelCatalog = Effect.fn("ProviderRegistry.refreshModelCatalog")(function* () { + const { updatedAt } = yield* manifestService.refreshOnDemand; + // The first pass can be one that started before the fetch, so allow one more. + for (let pass = 0; pass < 2; pass++) { + if ((yield* Ref.get(probedManifestUpdatedAt)) === updatedAt) return; + // A new model usually ships with the CLI release that supports it. + yield* latestVersions.invalidateAll; + yield* refreshAll(); + } + }); + return { getProviders: Ref.get(providersRef), refresh: (provider?: ProviderDriverKind) => @@ -1081,6 +1112,10 @@ export const layer = Layer.effect( refreshWorkspaceSnapshot(input).pipe(Effect.catchCause(recoverRefreshFailure)), refreshAfterUpdate, invalidateModelCaches, + refreshModelCatalog: refreshModelCatalog().pipe( + Effect.catchCause(recoverRefreshFailure), + Effect.asVoid, + ), getProviderMaintenanceCapabilitiesForInstance, setProviderMaintenanceActionState, get streamChanges() { diff --git a/apps/server/src/provider/claudeResetCredits.ts b/apps/server/src/provider/claudeResetCredits.ts new file mode 100644 index 000000000000..ef6e263a57b4 --- /dev/null +++ b/apps/server/src/provider/claudeResetCredits.ts @@ -0,0 +1,46 @@ +// Coder: reduced to the `.claude.json` account readers that `ClaudeSkills.ts` imports for +// claude.ai-synced plugin skills (#28633). Reset-credit reads and redemption call claude.ai's +// API for subscription account management, which this fork does not carry. +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; + +const Config = Schema.Struct({ + oauthAccount: Schema.optional( + Schema.Struct({ + organizationUuid: Schema.optional(Schema.String), + accountUuid: Schema.optional(Schema.Unknown), + }), + ), +}); + +const readJson = Effect.fn("claudeResetCredits.readJson")(function* ( + schema: S, + file: string, +) { + const fs = yield* FileSystem.FileSystem; + return yield* fs.readFileString(file).pipe( + Effect.catchTags({ + PlatformError: (error) => + error.reason._tag === "NotFound" ? Effect.succeed("{}") : Effect.fail(error), + }), + Effect.flatMap(Schema.decodeEffect(Schema.fromJsonString(schema))), + ); +}); + +export const readClaudeOauthAccount = (accountConfigPath: string) => + readJson(Config, accountConfigPath).pipe( + Effect.map((config) => config.oauthAccount), + Effect.orElseSucceed(() => undefined), + ); + +export const claudeAccountConfigPath = Effect.fn("claudeAccountConfigPath")(function* ( + configDir: string | undefined, +) { + const path = yield* Path.Path; + return configDir + ? path.join(configDir, ".claude.json") + : path.join(yield* HostProcess.HomeDirectory, ".claude.json"); +}); diff --git a/apps/server/src/provider/codexLaunchArgs.test.ts b/apps/server/src/provider/codexLaunchArgs.test.ts index ce18c8f3f1bd..8f4e1b7ca6d6 100644 --- a/apps/server/src/provider/codexLaunchArgs.test.ts +++ b/apps/server/src/provider/codexLaunchArgs.test.ts @@ -2,14 +2,31 @@ import * as NodeAssert from "node:assert/strict"; import { describe, it } from "vite-plus/test"; -import { codexAppServerArgs, codexExecLaunchArgs } from "./codexLaunchArgs.ts"; +import { + codexAppServerArgs, + codexExecLaunchArgs, + codexMcpListLaunchArgs, +} from "./codexLaunchArgs.ts"; describe("codexAppServerArgs", () => { - it("preserves escaped TOML quotes in config values", () => { - NodeAssert.deepStrictEqual( - codexAppServerArgs(String.raw`--config model=\"gpt-6.1-sol\" --strict-config`), - ["app-server", "--config", 'model="gpt-6.1-sol"', "--strict-config"], - ); + it.each([ + { + name: "preserves escaped TOML quotes in config values", + launchArgs: String.raw`--config model=\"gpt-6.1-sol\" --strict-config`, + expected: ["--config", 'model="gpt-6.1-sol"', "--strict-config"], + }, + { + name: "lets launch args override the unload delay", + launchArgs: "-c thread_unload_delay_secs=1800", + expected: ["-c", "thread_unload_delay_secs=1800"], + }, + ])("asks Codex to unload unsubscribed threads immediately and $name", (row) => { + NodeAssert.deepStrictEqual(codexAppServerArgs(row.launchArgs), [ + "app-server", + "-c", + "thread_unload_delay_secs=0", + ...row.expected, + ]); }); }); @@ -27,3 +44,32 @@ describe("codexExecLaunchArgs", () => { ]); }); }); + +describe("codex exec and mcp list launch args", () => { + const launchArgs = + "--strict-config --oss --local-provider ollama --profile local -p=work --listen off --enable foo"; + + it("keeps the local model provider flags for codex exec", () => { + NodeAssert.deepStrictEqual(codexExecLaunchArgs(launchArgs), [ + "--strict-config", + "--oss", + "--local-provider", + "ollama", + "--profile", + "local", + "-p=work", + "--enable", + "foo", + ]); + }); + + it("keeps the selected profile and drops exec-only flags for codex mcp list", () => { + NodeAssert.deepStrictEqual(codexMcpListLaunchArgs(launchArgs), [ + "--profile", + "local", + "-p=work", + "--enable", + "foo", + ]); + }); +}); diff --git a/apps/server/src/provider/codexLaunchArgs.ts b/apps/server/src/provider/codexLaunchArgs.ts index 33def5ce9a61..31ff37e31deb 100644 --- a/apps/server/src/provider/codexLaunchArgs.ts +++ b/apps/server/src/provider/codexLaunchArgs.ts @@ -9,31 +9,61 @@ export const resolveCodexLaunchArgs = ( const codexLaunchArgv = (launchArgs?: string): ReadonlyArray => tokenizeCliArgs(launchArgs); +// T3 unsubscribes only when it unloads a thread; Codex's own unload delay would stack on that. export const codexAppServerArgs = (launchArgs?: string) => [ "app-server", + "-c", + "thread_unload_delay_secs=0", ...codexLaunchArgv(launchArgs), ]; -export const codexExecLaunchArgs = (launchArgs?: string) => { +interface LaunchArgFilter { + readonly switches: ReadonlySet; + readonly valueFlags: ReadonlySet; + readonly inlinePrefixes: ReadonlyArray; +} + +const SHARED_VALUE_FLAGS = ["--config", "-c", "--enable", "--disable"]; +const SHARED_INLINE_PREFIXES = ["--config=", "-c=", "--enable=", "--disable="]; + +const MCP_LIST_FILTER: LaunchArgFilter = { + switches: new Set(), + valueFlags: new Set([...SHARED_VALUE_FLAGS, "--profile", "-p"]), + inlinePrefixes: [...SHARED_INLINE_PREFIXES, "--profile=", "-p="], +}; + +const EXEC_FILTER: LaunchArgFilter = { + switches: new Set(["--strict-config", "--oss"]), + valueFlags: new Set([...SHARED_VALUE_FLAGS, "--local-provider", "--profile", "-p"]), + inlinePrefixes: [...SHARED_INLINE_PREFIXES, "--local-provider=", "--profile=", "-p="], +}; + +const filterLaunchArgs = (launchArgs: string | undefined, allowed: LaunchArgFilter) => { const args = codexLaunchArgv(launchArgs); - const execArgs: Array = []; + const kept: Array = []; for (let index = 0; index < args.length; index++) { const arg = args[index]; if (arg === undefined) continue; - if (arg === "--strict-config" || arg.startsWith("--config=") || arg.startsWith("-c=")) { - execArgs.push(arg); - } else if (arg === "--config" || arg === "-c" || arg === "--enable" || arg === "--disable") { + if (allowed.switches.has(arg)) { + kept.push(arg); + } else if (allowed.valueFlags.has(arg)) { const value = args[index + 1]; if (value !== undefined && !value.startsWith("-")) { - execArgs.push(arg, value); + kept.push(arg, value); index++; } - } else if (arg.startsWith("--enable=") || arg.startsWith("--disable=")) { - execArgs.push(arg); + } else if (allowed.inlinePrefixes.some((prefix) => arg.startsWith(prefix))) { + kept.push(arg); } } - return execArgs; + return kept; }; + +export const codexExecLaunchArgs = (launchArgs?: string) => + filterLaunchArgs(launchArgs, EXEC_FILTER); + +export const codexMcpListLaunchArgs = (launchArgs?: string) => + filterLaunchArgs(launchArgs, MCP_LIST_FILTER); diff --git a/apps/server/src/provider/codexUsageLimits.test.ts b/apps/server/src/provider/codexUsageLimits.test.ts index 9c863f54c086..8249233b8db9 100644 --- a/apps/server/src/provider/codexUsageLimits.test.ts +++ b/apps/server/src/provider/codexUsageLimits.test.ts @@ -39,6 +39,8 @@ describe("codexRateLimitsToLimits", () => { expect(codexRateLimitsToUpdate(snapshot)).toEqual({ windows }); }); + // Coder: upstream's zero-credit-cap case (#34243) checks the cut `shared/usageLimits.ts`. + it("maps primary and secondary onto the session and weekly windows", () => { expect( codexRateLimitsToLimits({ diff --git a/apps/server/src/provider/model-manifest.json b/apps/server/src/provider/model-manifest.json index df88f64a98cf..678ad71161c3 100644 --- a/apps/server/src/provider/model-manifest.json +++ b/apps/server/src/provider/model-manifest.json @@ -1,6 +1,6 @@ { "version": 1, - "updatedAt": "2026-11-05T22:38:29Z", + "updatedAt": "2026-11-10T17:25:47Z", "compatibility": [ { "driver": "codex", @@ -125,8 +125,15 @@ "gpt-daybreak-blue-latest", "gpt-daybreak-red-latest" ], - "claudeAgent": ["claude-fable-5-1", "claude-opus-5-5", "claude-sonnet-5-5", "claude-haiku-5-5"], - "antigravity": ["gemini-3.8-flash-high", "gemini-3.8-flash-medium", "gemini-3.8-flash-low"] + "claudeAgent": ["claude-fable-5-2", "claude-opus-5-5", "claude-sonnet-5-5", "claude-haiku-5-5"], + "antigravity": [ + "gemini-4-carbon", + "gemini-4-argon", + "gemini-3.8-flash", + "gemini-3.8-flash-high", + "gemini-3.8-flash-medium", + "gemini-3.8-flash-low" + ] }, "providers": { "codex": { @@ -757,11 +764,19 @@ "profile": "sonnet-5-5", "adapter": { "claudeCode": { "minVersion": "2.1.284" } } }, + { + "slug": "claude-fable-5-2", + "name": "Claude Fable 5.2", + "aliases": ["fable", "fable-5.2", "claude-fable-5.2"], + "status": "current", + "profile": "fable-5", + "adapter": { "claudeCode": { "minVersion": "2.1.290" } } + }, { "slug": "claude-fable-5-1", "name": "Claude Fable 5.1", - "aliases": ["fable", "fable-5.1", "claude-fable-5.1"], - "status": "current", + "aliases": ["fable-5.1", "claude-fable-5.1"], + "status": "legacy", "profile": "fable-5", "adapter": { "claudeCode": { @@ -867,6 +882,14 @@ }, "profiles": {}, "models": [ + { + "slug": "gemini-4-carbon", + "name": "Gemini 4 Carbon", + "status": "current", + "releasedAt": "2026-11-10" + }, + { "slug": "gemini-4-argon", "name": "Gemini 4 Argon", "status": "current" }, + { "slug": "gemini-3.8-flash", "name": "Gemini 3.8 Flash", "status": "current" }, { "slug": "gemini-3.8-flash-high", "name": "Gemini 3.8 Flash (High)", diff --git a/apps/server/src/provider/providerCompatibility.test.ts b/apps/server/src/provider/providerCompatibility.test.ts index 352c5902462b..883ded6bb4fa 100644 --- a/apps/server/src/provider/providerCompatibility.test.ts +++ b/apps/server/src/provider/providerCompatibility.test.ts @@ -261,6 +261,7 @@ it.effect("a remote policy refresh preserves a newer health result on the regist current: Ref.get(manifest), refresh, forceRefresh: refresh, + refreshOnDemand: refresh, refreshInBackground: Effect.void, }), Layer.succeed(ProviderInstanceRegistry.ProviderInstanceRegistry, { diff --git a/apps/server/src/provider/providerMaintenanceRunner.test.ts b/apps/server/src/provider/providerMaintenanceRunner.test.ts index 77a44c7fa29a..546450632348 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.test.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.test.ts @@ -1,3 +1,4 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; import { describe, it, assert } from "@effect/vitest"; import { ProviderDriverKind, @@ -22,6 +23,13 @@ import { ChildProcessSpawner } from "effect/process"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { SpawnExecutableResolution } from "@t3tools/shared/shell"; +import type { ProviderInstance } from "@t3tools/provider-core/server/driver"; + +import * as ServerSettings from "../serverSettings.ts"; +import * as SourceControlProviderRegistry from "../sourceControl/SourceControlProviderRegistry.ts"; +import * as TextGeneration from "../textGeneration/TextGeneration.ts"; +import * as ProviderInstallationGate from "./ProviderInstallationGate.ts"; +import * as ProviderInstanceRegistry from "./ProviderInstanceRegistry.ts"; import * as ProviderRegistry from "./ProviderRegistry.ts"; import * as ModelManifest from "./ModelManifest.ts"; import * as ProviderMaintenanceRunner from "./providerMaintenanceRunner.ts"; @@ -212,6 +220,7 @@ function makeRegistry( }, refreshWorkspaceSnapshot: () => Ref.get(providersRef), invalidateModelCaches: () => Effect.void, + refreshModelCatalog: Effect.void, getProviderMaintenanceCapabilitiesForInstance: (_instanceId, provider) => Effect.succeed(lifecycleFor(provider)), setProviderMaintenanceActionState, @@ -249,9 +258,12 @@ const makeTestRunner = ( current: Effect.succeed(manifest), refresh: Effect.succeed(manifest), forceRefresh: Effect.succeed(manifest), + refreshOnDemand: Effect.succeed(manifest), refreshInBackground: Effect.void, }), latestVersions, + // Layers memoize by reference, so a test that provides the gate shares it. + ProviderInstallationGate.layer, ), ), ), @@ -270,6 +282,7 @@ function layerFailingLatestLookupAfterFirst() { return yield* lookup; }), invalidate: () => Effect.void, + invalidateAll: Effect.void, }); } @@ -293,6 +306,7 @@ describe("providerMaintenanceRunner", () => { [ ["queued", "Waiting for another provider update to finish."], ["running", "Checking for the latest version"], + ["running", "Waiting for running provider work to finish."], ["running", "Running native-agent update"], ["running", "Verifying the installed version"], ["succeeded", "Provider updated."], @@ -557,6 +571,43 @@ describe("providerMaintenanceRunner", () => { ); }); + it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( + "runs the updater with no stdin, so a command that reads it cannot wait forever", + () => + Effect.gen(function* () { + const { registry } = yield* makeRegistry(baseProvider); + const updater = yield* makeTestRunner({ + ...registry, + getProviderMaintenanceCapabilitiesForInstance: (_instanceId, provider) => + Effect.succeed({ + ...lifecycleFor(provider), + update: { + command: "codex update", + executable: "/bin/sh", + // An open pipe here would block a CLI that reads stdin until the update times out. + args: ["-c", "[ -c /dev/stdin ] || { echo 'stdin is an open pipe' >&2; exit 1; }"], + lockKey: "codex-native", + }, + }), + }); + + const result = yield* updater.updateProvider(CODEX_DRIVER); + // The registry reports no new version, so a command that exits 0 ends as "unchanged". + assert.strictEqual( + result.providers[0]?.updateState?.message, + "Update command completed, but T3 Code still detects an outdated provider version.", + ); + }).pipe( + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + layerNonWindowsPlatform, + layerLatestVersionHttpClient("1.0.0"), + ), + ), + ), + ); + it.effect("re-resolves ownership before running and executes the fresh command", () => { const calls: Array<{ command: string; args: ReadonlyArray }> = []; const fresh: Array = []; @@ -710,6 +761,65 @@ describe("providerMaintenanceRunner", () => { ), ); + it.effect("reports the updater's error rather than a trailing hint after it", () => + Effect.forEach( + [ + { + stdout: "Current version: 2.1.310\nChecking for updates to latest version...\n", + stderr: [ + "Error: Failed to install native update", + "TelemetrySafeError: Failed to fetch version from https://downloads.claude.ai/claude-code-releases/latest after 3 attempt(s): Request failed with status code 403", + 'Try running "claude doctor" for diagnostics', + "", + ].join("\n"), + reason: "Request failed with status code 403", + hint: "claude doctor", + }, + { + stdout: "", + stderr: [ + "Failed to check for updates", + "Unable to fetch latest version from npm registry", + "", + "Possible causes:", + " • Network connectivity issues", + " • npm registry is unreachable", + " • Corporate proxy/firewall blocking npm", + "", + "Try:", + " • Check your internet connection", + " • Run with --debug flag for more details", + " • Manually check: npm view @anthropic-ai/claude-code version", + " • Check if you need to login: npm whoami", + "", + ].join("\n"), + reason: "Failed to check for updates", + hint: "npm whoami", + }, + ], + ({ stdout, stderr, reason, hint }) => + Effect.gen(function* () { + const { registry } = yield* makeRegistry(baseNativeCliProvider); + const updater = yield* makeTestRunner(registry); + + const result = yield* updater.updateProvider(NATIVE_CLI_DRIVER); + const message = result.providers[0]?.updateState?.message ?? ""; + + assert.include(message, reason); + assert.notInclude(message, hint); + }).pipe( + Effect.provide( + Layer.mergeAll( + layerNonWindowsPlatform, + layerLatestVersionHttpClient("0.0.0"), + layerMockSpawner(() => ({ stdout, stderr, code: 1 })), + ), + ), + ), + { discard: true }, + ), + ); + it.effect("keeps the exit code message when a failed command prints nothing", () => Effect.gen(function* () { const { registry } = yield* makeRegistry(); @@ -793,6 +903,66 @@ describe("providerMaintenanceRunner", () => { ); }); + it.effect("an interrupted update ends failed instead of running forever", () => { + const started = Deferred.makeUnsafe(); + return Effect.gen(function* () { + const { registry, updateStatesRef } = yield* makeRegistry(baseNativeCliProvider); + const updater = yield* makeTestRunner(registry); + const update = yield* updater.updateProvider(NATIVE_CLI_DRIVER).pipe(Effect.forkScoped); + yield* Deferred.await(started); + yield* Fiber.interrupt(update); + const last = (yield* Ref.get(updateStatesRef)).at(-1); + assert.deepStrictEqual( + [last?.status, last?.message], + ["failed", "Update stopped before it finished."], + ); + }).pipe( + Effect.provide( + Layer.mergeAll( + layerNonWindowsPlatform, + layerLatestVersionHttpClient("1.0.0"), + layerMockSpawner(() => ({ + exitCode: Deferred.succeed(started, undefined).pipe(Effect.andThen(Effect.never)), + })), + ), + ), + ); + }); + + it.effect("a request dropped while the final state is written still records it", () => { + const writing = Deferred.makeUnsafe(); + const release = Deferred.makeUnsafe(); + return Effect.gen(function* () { + const { registry, updateStatesRef } = yield* makeRegistry(baseNativeCliProvider); + const updater = yield* makeTestRunner({ + ...registry, + setProviderMaintenanceActionState: (input) => + input.state?.status === "succeeded" + ? Deferred.succeed(writing, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.andThen(registry.setProviderMaintenanceActionState(input)), + ) + : registry.setProviderMaintenanceActionState(input), + }); + + const update = yield* updater.updateProvider(NATIVE_CLI_DRIVER).pipe(Effect.forkScoped); + yield* Deferred.await(writing); + update.interruptUnsafe(); + yield* Deferred.succeed(release, undefined); + yield* Fiber.await(update); + + assert.strictEqual((yield* Ref.get(updateStatesRef)).at(-1)?.status, "succeeded"); + }).pipe( + Effect.provide( + Layer.mergeAll( + layerNonWindowsPlatform, + layerLatestVersionHttpClient("1.0.0"), + layerMockSpawner(() => ({ stdout: "updated" })), + ), + ), + ); + }); + it.effect("prevents concurrent updates for the same provider", () => { const startedLatch: { resolve: () => void } = { resolve: () => {} }; const releaseLatch: { resolve: () => void } = { resolve: () => {} }; @@ -840,6 +1010,143 @@ describe("providerMaintenanceRunner", () => { ); }); + it.effect("holds text generation for an instance until its update finishes", () => { + const updateStarted = Deferred.makeUnsafe(); + const log: Array = []; + return Effect.gen(function* () { + const { registry } = yield* makeRegistry(baseProvider, { + installedVersionAfterUpdate: "1.0.0", + }); + const updater = yield* makeTestRunner(registry); + const instance: ProviderInstance = { + instanceId: CODEX_INSTANCE_ID, + driverKind: CODEX_DRIVER, + continuationIdentity: { driverKind: CODEX_DRIVER, continuationKey: CODEX_INSTANCE_ID }, + displayName: undefined, + enabled: true, + snapshot: { getSnapshot: Effect.succeed(baseProvider) } as ProviderInstance["snapshot"], + orchestrationAdapter: {} as ProviderInstance["orchestrationAdapter"], + textGeneration: TextGeneration.TextGeneration.of({ + generateCommitMessage: () => Effect.die("unused"), + generatePrContent: () => Effect.die("unused"), + generateBranchName: () => Effect.die("unused"), + generateReviewGuide: () => Effect.die("unused"), + generateThreadTitle: () => + Effect.sync(() => { + log.push("text generation spawned provider"); + return { title: "Generated title" }; + }), + }), + }; + const textGeneration = yield* TextGeneration.make.pipe( + Effect.provide( + Layer.mergeAll( + Layer.mock(ProviderInstanceRegistry.ProviderInstanceRegistry)({ + listInstances: Effect.succeed([instance]), + getInstance: (id) => Effect.succeed(id === CODEX_INSTANCE_ID ? instance : undefined), + }), + ServerSettings.layerTest(), + // Coder: no text generators (#33147); TextGeneration needs only provider instances. + Layer.mock(SourceControlProviderRegistry.SourceControlProviderRegistry)({ + resolveLink: () => Effect.die("unused"), + }), + ), + ), + ); + + const update = yield* updater.updateProvider(CODEX_DRIVER).pipe(Effect.forkScoped); + yield* Deferred.await(updateStarted); + const generation = yield* textGeneration + .generateThreadTitle({ + cwd: "/tmp", + message: "hello", + linkedContext: "", + modelSelection: { instanceId: CODEX_INSTANCE_ID, model: "gpt-5" }, + }) + .pipe(Effect.forkScoped); + yield* TestClock.adjust("1 minute"); + const result = yield* Fiber.join(update); + yield* Fiber.join(generation); + + assert.strictEqual(result.providers[0]?.updateState?.status, "succeeded"); + assert.deepStrictEqual(log, ["update command exited", "text generation spawned provider"]); + }).pipe( + Effect.provide( + Layer.mergeAll( + ProviderInstallationGate.layer, + layerNonWindowsPlatform, + layerLatestVersionHttpClient("1.0.0"), + layerMockSpawner(() => ({ + exitCode: Deferred.succeed(updateStarted, undefined).pipe( + Effect.andThen(Effect.sleep("1 minute")), + Effect.andThen(Effect.sync(() => log.push("update command exited"))), + Effect.as(ChildProcessSpawner.ExitCode(0)), + ), + })), + ), + ), + ); + }); + + it.effect("holds every instance that shares the updated installation", () => { + const CODEX_WORK_INSTANCE_ID = ProviderInstanceId.make("codex_work"); + const updateStarted = Deferred.makeUnsafe(); + const log: Array = []; + return Effect.gen(function* () { + const { registry } = yield* makeRegistry( + [ + baseProvider, + { ...baseProvider, instanceId: CODEX_WORK_INSTANCE_ID }, + baseOpenCodeProvider, + ], + { installedVersionAfterUpdate: "1.0.0" }, + ); + // OpenCode installs into the same npm prefix, so it shares the lock key but not the package. + const updater = yield* makeTestRunner(registry); + const gate = yield* ProviderInstallationGate.ProviderInstallationGate; + + const update = yield* updater.updateProvider(CODEX_DRIVER).pipe(Effect.forkScoped); + yield* Deferred.await(updateStarted); + const shared = yield* gate + .execute( + CODEX_WORK_INSTANCE_ID, + Effect.sync(() => log.push("shared installation ran")), + ) + .pipe(Effect.forkScoped); + const other = yield* gate + .execute( + OPENCODE_INSTANCE_ID, + Effect.sync(() => log.push("other driver ran")), + ) + .pipe(Effect.forkScoped); + yield* TestClock.adjust("1 minute"); + yield* Fiber.join(update); + yield* Fiber.join(shared); + yield* Fiber.join(other); + + assert.deepStrictEqual(log, [ + "other driver ran", + "update command exited", + "shared installation ran", + ]); + }).pipe( + Effect.provide( + Layer.mergeAll( + ProviderInstallationGate.layer, + layerNonWindowsPlatform, + layerLatestVersionHttpClient("1.0.0"), + layerMockSpawner(() => ({ + exitCode: Deferred.succeed(updateStarted, undefined).pipe( + Effect.andThen(Effect.sleep("1 minute")), + Effect.andThen(Effect.sync(() => log.push("update command exited"))), + Effect.as(ChildProcessSpawner.ExitCode(0)), + ), + })), + ), + ), + ); + }); + it.effect( "releases the running-provider marker when interrupted after queuing but before the lock run starts", () => @@ -975,6 +1282,12 @@ describe("providerMaintenanceRunner", () => { }); it.effect.each([ + { + mode: "default-precurrent", + version: "1.14.19", + installation: { installedVersion: "1.14.20" }, + status: "succeeded", + }, { mode: "default", version: "1.14.19", @@ -1030,7 +1343,11 @@ it.effect.each([ ...registry, refreshInstance: () => Ref.updateAndGet(providersRef, (providers) => - providers.map((provider) => ({ ...provider, checkedAt: "2026-04-10T00:00:01.000Z" })), + providers.map((provider) => ({ + ...provider, + ...(mode === "default-precurrent" ? installation : {}), + checkedAt: "2026-04-10T00:00:01.000Z", + })), ), getProviderMaintenanceCapabilitiesForInstance: () => Effect.succeed(pinnedCapabilities), refreshAfterUpdate: () => @@ -1056,17 +1373,27 @@ it.effect.each([ const result = yield* updater.updateProvider({ provider: OPENCODE_DRIVER, - ...(mode === "default" ? {} : { targetVersion: "1.14.20" }), + ...(mode.startsWith("default") ? {} : { targetVersion: "1.14.20" }), }); - assert.deepStrictEqual(calls, [ - `npm install -g opencode-ai@${mode === "default" ? "latest" : "1.14.20"}`, - ]); + assert.deepStrictEqual( + calls, + mode === "default-precurrent" + ? [] + : [`npm install -g opencode-ai@${mode.startsWith("default") ? "latest" : "1.14.20"}`], + ); const provider = result.providers[0]; assert.ok(provider); assert.strictEqual(provider.updateState?.status, status); + if (mode === "default" || mode === "default-precurrent" || mode === "targeted") { + assert.strictEqual( + provider.updateState?.message, + "OpenCode CLI updated. Restart the configured local OpenCode server to use the new version.", + ); + } assert.strictEqual(provider.version, version); assert.strictEqual("installedVersion" in provider, mode !== "legacy"); + if (mode === "default-precurrent") return; assert.strictEqual( provider.versionAdvisory?.status, mode === "unknown" ? "unknown" : "current", diff --git a/apps/server/src/provider/providerMaintenanceRunner.ts b/apps/server/src/provider/providerMaintenanceRunner.ts index d3fe7fdf8673..80248acea577 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.ts @@ -25,6 +25,7 @@ import * as NodeUtil from "node:util"; import * as ModelManifest from "./ModelManifest.ts"; import { resolveProviderCompatibility } from "./providerCompatibility.ts"; +import * as ProviderInstallationGate from "./ProviderInstallationGate.ts"; import * as ProviderRegistry from "./ProviderRegistry.ts"; import { makeProviderMaintenanceCommandCoordinator } from "./providerMaintenanceCommandCoordinator.ts"; import { @@ -94,6 +95,18 @@ interface VerifiedProviderRefresh { const nowIso = Effect.map(DateTime.now, DateTime.formatIso); +const OPENCODE_SERVER_RESTART_MESSAGE = + "OpenCode CLI updated. Restart the configured local OpenCode server to use the new version."; + +const needsOpenCodeServerRestart = (providers: ReadonlyArray) => + providers.some( + (provider) => + provider.driver === ProviderDriverKind.make("opencode") && + provider.installedVersion != null && + provider.version != null && + provider.installedVersion !== provider.version, + ); + const runProviderMaintenanceCommandWithSpawner = Effect.fn("ProviderMaintenanceRunner.runCommand")( function* (input: { readonly spawner: ChildProcessSpawner.ChildProcessSpawner["Service"]; @@ -111,6 +124,8 @@ const runProviderMaintenanceCommandWithSpawner = Effect.fn("ProviderMaintenanceR .spawn( ChildProcess.make(resolved.command, resolved.args, { shell: resolved.shell, + // Nothing answers an update command, so one that reads stdin must see EOF, not wait. + stdin: "ignore", ...env, }), ) @@ -250,18 +265,20 @@ function commandOutput(result: ProviderMaintenanceCommandResult): string | null return truncateText(output, UPDATE_OUTPUT_MAX_BYTES); } -// Updaters print their reason last, usually on stderr. -function lastOutputLine(result: ProviderMaintenanceCommandResult): string | null { - for (const output of [result.stderr, result.stdout]) { - const line = output +// Updaters report on stderr, but may follow their error with hints (`claude update` +// ends with "Try running claude doctor"), so prefer the last line naming an error. +function failureReason(result: ProviderMaintenanceCommandResult): string | null { + const streams = [result.stderr, result.stdout].map((output) => + output .split(/\r\n|\r|\n/) .map(toProgressLine) - .findLast(Boolean); - if (line) { - return line; - } - } - return null; + .filter((line) => line !== null), + ); + return ( + streams.map((lines) => lines.findLast((line) => /error|fail/i.test(line))).find(Boolean) ?? + streams.map((lines) => lines.at(-1)).find(Boolean) ?? + null + ); } function failureMessage(result: ProviderMaintenanceCommandResult): string { @@ -272,7 +289,7 @@ function failureMessage(result: ProviderMaintenanceCommandResult): string { result.exitCode !== null && result.exitCode !== 0 ? `Update command exited with code ${result.exitCode}` : "Update command failed"; - const reason = lastOutputLine(result); + const reason = failureReason(result); return reason ? `${summary}: ${reason}` : `${summary}.`; } @@ -307,6 +324,7 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const httpClient = yield* HttpClient.HttpClient; const latestVersions = yield* ProviderLatestVersions.ProviderLatestVersions; + const installationGate = yield* ProviderInstallationGate.ProviderInstallationGate; const runMaintenanceCommand = ( update: ProviderMaintenanceCommandAction, onProgress: (line: string) => Effect.Effect, @@ -318,6 +336,28 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { onProgress, ...(update.env ? { env: update.env } : {}), }); + // Instances with other homes or accounts can still run the binary this update replaces. The + // lock key names the package manager location, so other drivers there keep their own package. + const instancesSharingInstallation = Effect.fn( + "ProviderMaintenanceRunner.instancesSharingInstallation", + )(function* (instanceId: ProviderInstanceId, provider: ProviderDriverKind, lockKey: string) { + const providers = yield* providerRegistry.getProviders; + const shared = yield* Effect.forEach( + providers.filter( + (candidate) => candidate.instanceId !== instanceId && candidate.driver === provider, + ), + (candidate) => + providerRegistry + .getProviderMaintenanceCapabilitiesForInstance(candidate.instanceId, candidate.driver) + .pipe( + Effect.map((capabilities) => + capabilities.update?.lockKey === lockKey ? [candidate.instanceId] : [], + ), + ), + { concurrency: "unbounded" }, + ); + return [instanceId, ...shared.flat()]; + }); const commandCoordinator = yield* makeProviderMaintenanceCommandCoordinator({ makeAlreadyRunningError: () => new ServerProviderUpdateError({ @@ -403,6 +443,8 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { }); } + const startedAtRef = yield* Ref.make(null); + const settled = yield* Ref.make(false); const setUpdateState = (state: ServerProviderUpdateState | null) => providerRegistry.setProviderMaintenanceActionState({ instanceId, @@ -420,9 +462,13 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { const runProviderUpdate = Effect.fn("ProviderMaintenanceRunner.runProviderUpdate")( function* () { + // Uninterruptible, so a dropped request cannot leave settled set without the state. const finish = (state: ServerProviderUpdateState) => - setUpdateState(state).pipe(Effect.map((providers) => ({ providers }))); - const startedAtRef = yield* Ref.make(null); + Ref.set(settled, true).pipe( + Effect.andThen(setUpdateState(state)), + Effect.map((providers) => ({ providers })), + Effect.uninterruptible, + ); const runCommandAndVerify = Effect.fn("ProviderMaintenanceRunner.runCommandAndVerify")( function* () { @@ -527,14 +573,26 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { status: "succeeded", startedAt, finishedAt: yield* nowIso, - message: "Already up to date.", + message: needsOpenCodeServerRestart(precheck.verifiedProviders) + ? OPENCODE_SERVER_RESTART_MESSAGE + : "Already up to date.", }), ); } } - yield* setRunningMessage(`Running ${describeCommand(command)}`); - const result = yield* runMaintenanceCommand(command, setRunningMessage); + const sharingInstances = yield* instancesSharingInstallation( + instanceId, + provider, + update.lockKey, + ); + yield* setRunningMessage("Waiting for running provider work to finish."); + const result = yield* installationGate.update( + sharingInstances, + setRunningMessage(`Running ${describeCommand(command)}`).pipe( + Effect.andThen(runMaintenanceCommand(command, setRunningMessage)), + ), + ); const finishedAt = yield* nowIso; if (result.timedOut || result.exitCode !== 0) { yield* Effect.logWarning("Provider update command failed", { @@ -600,7 +658,9 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { ? "Update command completed, but T3 Code could not verify the provider version." : stillOutdated ? "Update command completed, but T3 Code still detects an outdated provider version." - : "Provider updated.", + : needsOpenCodeServerRestart(verifiedProviders) + ? OPENCODE_SERVER_RESTART_MESSAGE + : "Provider updated.", output: commandOutput(result), }), ); @@ -635,6 +695,21 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { run: runProviderUpdate(), }) .pipe( + // A dropped request interrupts the update; a "running" state would spin until restart. + Effect.onInterrupt(() => + Effect.gen(function* () { + // A finished state stays, even if the request drops while it is being written. + if (yield* Ref.get(settled)) return; + yield* setUpdateState( + makeUpdateState({ + status: "failed", + startedAt: yield* Ref.get(startedAtRef), + finishedAt: yield* nowIso, + message: "Update stopped before it finished.", + }), + ); + }).pipe(Effect.ignore), + ), Effect.mapError((error) => isServerProviderUpdateError(error) ? new ServerProviderUpdateError({ diff --git a/apps/server/src/provider/testUtils/providerRegistryMock.ts b/apps/server/src/provider/testUtils/providerRegistryMock.ts index f75ecd141e71..3ae8079414a9 100644 --- a/apps/server/src/provider/testUtils/providerRegistryMock.ts +++ b/apps/server/src/provider/testUtils/providerRegistryMock.ts @@ -15,6 +15,7 @@ const makeProviderRegistryMock = ( refreshAfterUpdate: () => Effect.succeed(providers), refreshWorkspaceSnapshot: () => Effect.succeed(providers), invalidateModelCaches: () => Effect.void, + refreshModelCatalog: Effect.void, getProviderMaintenanceCapabilitiesForInstance: (_instanceId, provider) => Effect.succeed(makeManualOnlyProviderMaintenanceCapabilities({ provider, packageName: null })), setProviderMaintenanceActionState: () => Effect.succeed(providers), diff --git a/apps/server/src/provider/testing/ProviderInstanceRegistryLive.fixture.mjs b/apps/server/src/provider/testing/ProviderInstanceRegistryLive.fixture.mjs index 5c4c84f2aa3e..b996e420f85f 100755 --- a/apps/server/src/provider/testing/ProviderInstanceRegistryLive.fixture.mjs +++ b/apps/server/src/provider/testing/ProviderInstanceRegistryLive.fixture.mjs @@ -68,6 +68,40 @@ lines.on("line", (line) => { ); return; } + if (message.request?.subtype === "get_settings" && process.env.T3_CLAUDE_SETTINGS_UNSUPPORTED) { + process.stdout.write( + JSON.stringify({ + type: "control_response", + response: { + subtype: "error", + request_id: message.request_id, + error: "Unsupported control request subtype: get_settings", + }, + }) + "\n", + ); + return; + } + if (message.request?.subtype === "get_settings") { + const sources = JSON.parse(process.env.T3_CLAUDE_SETTINGS_SOURCES ?? "[]"); + // Like Claude Code, `effective` is the last-writer merge, which hides per-source caps. + const effective = { modelSettings: {} }; + for (const { settings } of sources) { + Object.assign(effective, settings, { + modelSettings: Object.assign(effective.modelSettings, settings.modelSettings), + }); + } + process.stdout.write( + JSON.stringify({ + type: "control_response", + response: { + subtype: "success", + request_id: message.request_id, + response: { effective, sources }, + }, + }) + "\n", + ); + return; + } if (message.request?.subtype !== "initialize") return; process.stdout.write( JSON.stringify({ diff --git a/apps/server/src/pullRequest/PullRequestReviewGuideService.ts b/apps/server/src/pullRequest/PullRequestReviewGuideService.ts index fe994e7ab3ee..fe81e29a8426 100644 --- a/apps/server/src/pullRequest/PullRequestReviewGuideService.ts +++ b/apps/server/src/pullRequest/PullRequestReviewGuideService.ts @@ -404,6 +404,7 @@ const make = Effect.gen(function* () { ...reviewGuideInputFromPullRequest(detail, activity, checkout.baseCommit), cwd: checkout.path, modelSelection, + projectId: input.projectId, instructions: projectSettings.reviewGuideInstructions, stack: checkout.stack, }), diff --git a/apps/server/src/pullRequest/PullRequestService.test.ts b/apps/server/src/pullRequest/PullRequestService.test.ts index 7ef531309c3b..e067c68ec4f8 100644 --- a/apps/server/src/pullRequest/PullRequestService.test.ts +++ b/apps/server/src/pullRequest/PullRequestService.test.ts @@ -752,6 +752,106 @@ it.effect("tells two hosts of one kind apart in the switcher and the filter", () }), ); +it.effect("retains an unavailable host's summary when it is the only queried host", () => + Effect.gen(function* () { + for (const reason of ["missing-tool", "unauthenticated"] as const) { + const service = yield* makeService({ + projects: [ + project({ + id: "p1", + title: "t3code", + workspaceRoot: "/a", + repository: "pingdotgg/t3code", + }), + ], + providers: [ + fakePullRequestProvider(SourceControlProviderKind.make("github"), { + getViewer: () => + Effect.fail(unusable(SourceControlProviderKind.make("github"), reason)), + }), + ], + }); + + for (const filter of [{}, { host: "github.com" }]) { + const error = yield* service.list({ state: "open", ...filter }).pipe(Effect.flip); + assert.strictEqual(error._tag, "PullRequestUnavailableError"); + assert.strictEqual( + error._tag === "PullRequestUnavailableError" ? error.reason : null, + reason === "missing-tool" ? "cli-missing" : "cli-unauthenticated", + ); + assert.isDefined(error.listResult); + const result = error.listResult!; + assert.deepStrictEqual(result.entries, []); + assert.deepStrictEqual(result.viewers, {}); + assert.deepStrictEqual( + result.providers.map(({ host, configured, projectIds }) => ({ + host, + configured, + projectIds: projectIds?.map(String), + })), + [{ host: "github.com", configured: false, projectIds: undefined }], + ); + assert.strictEqual(result.providers[0]?.detail?.includes("github"), true); + assert.deepStrictEqual( + result.errors.map(({ projectId }) => projectId), + ["p1"], + ); + } + } + }), +); + +it.effect("retains a Bitbucket host summary when the provider rejects saved credentials", () => + Effect.gen(function* () { + const provider = fakePullRequestProvider(SourceControlProviderKind.make("bitbucket"), { + getViewer: () => + Effect.fail( + new PullRequestProviderError({ + provider: SourceControlProviderKind.make("bitbucket"), + operation: "getViewer", + reason: "failed", + detail: "HTTP 403", + }), + ), + }); + const service = yield* makeService({ + projects: [ + project({ + id: "bitbucket", + title: "Widget", + workspaceRoot: "/widget", + repository: "acme/widget", + provider: SourceControlProviderKind.make("bitbucket"), + host: "bitbucket.org", + }), + ], + providers: [provider], + }); + for (const filter of [{}, { host: "bitbucket.org" }]) { + const error = yield* service.list({ state: "open", ...filter }).pipe(Effect.flip); + assert.strictEqual(error._tag, "PullRequestOperationError"); + assert.isDefined(error.listResult); + const result = error.listResult!; + assert.deepStrictEqual( + result.providers.map(({ host, configured, projectIds }) => ({ + host, + configured, + projectIds: projectIds?.map(String), + })), + [{ host: "bitbucket.org", configured: false, projectIds: undefined }], + ); + assert.deepStrictEqual(result.entries, []); + assert.deepStrictEqual( + result.errors.map(({ projectId }) => projectId), + ["bitbucket"], + ); + } + }), +); + +// Coder: upstream's Forgejo authority test (#21530) needs the Forgejo driver, which the +// GitLab-only registry does not carry. + it.effect("reads a repository once when several worktrees share it", () => Effect.gen(function* () { let calls = 0; @@ -783,6 +883,7 @@ it.effect("reads a repository once when several worktrees share it", () => assert.strictEqual(calls, 1); assert.strictEqual(result.entries.length, 1); + assert.isUndefined(result.providers[0]?.projectIds); }), ); @@ -1480,6 +1581,71 @@ it.effect("stops new reads after a rate limit while leaving manual actions avail }), ); +it.effect( + "opening a pull request during a rate-limit pause still reads its detail and activity", + () => + Effect.gen(function* () { + let detailReads = 0; + let activityReads = 0; + const service = yield* makeService({ + projects: [ + project({ id: "p1", title: "cloud", workspaceRoot: "/cloud", repository: "acme/web" }), + ], + providers: [ + fakePullRequestProvider(SourceControlProviderKind.make("github"), { + getChangeRequest: ({ number }) => + Effect.suspend(() => { + detailReads += 1; + return number === 1 + ? Effect.fail( + new PullRequestProviderError({ + provider: SourceControlProviderKind.make("github"), + operation: "getChangeRequest", + reason: "rate-limited", + detail: "GitHub API rate limit exceeded.", + }), + ) + : Effect.succeed({ ...hostedChangeRequest("Body"), number }); + }), + getChangeRequestActivity: () => + Effect.sync(() => { + activityReads += 1; + return { + comments: [], + commentCount: 0, + commentsTruncated: false, + reviewThreads: [], + commits: [], + }; + }), + }), + ], + }); + const reference = { projectId: "p1" as ProjectId, repository: "acme/web", number: 2 }; + + yield* Effect.result(service.detail({ ...reference, number: 1 })); + const background = yield* Effect.result(service.detail(reference)); + const opened = yield* Effect.result( + service + .detail(reference) + .pipe(Effect.provideService(SourceControlRateLimit.Interactive, true)), + ); + const timeline = yield* Effect.result( + service + .activity(reference) + .pipe(Effect.provideService(SourceControlRateLimit.Interactive, true)), + ); + + expect(background).toMatchObject({ _tag: "Failure" }); + expect({ opened, timeline }).toMatchObject({ + opened: { _tag: "Success", success: { number: 2 } }, + timeline: { _tag: "Success", success: { commentCount: 0 } }, + }); + assert.strictEqual(detailReads, 2); + assert.strictEqual(activityReads, 1); + }), +); + it.effect("refuses a repository that does not belong to the requested project", () => Effect.gen(function* () { const service = yield* makeService({ @@ -3067,35 +3233,6 @@ it.effect("keeps viewed files itself for a host that keeps none of its own", () }), ); -it.effect("reports a file pushed to since it was cleared as changed", () => - Effect.gen(function* () { - const revisions = new Map([ - ["src/a.ts", "blob-a"], - ["src/b.ts", "blob-b"], - ]); - const service = yield* environmentViewedService(revisions, []); - - yield* service.setFilesViewed({ - ...GITLAB_REFERENCE, - files: [ - { path: "src/a.ts", viewed: true }, - { path: "src/b.ts", viewed: true }, - ], - }); - revisions.set("src/a.ts", "blob-a-again"); - yield* service.invalidate({ reference: GITLAB_REFERENCE }); - const marked = yield* service.filesViewed(GITLAB_REFERENCE); - - assert.deepStrictEqual( - [...marked.files].toSorted((left, right) => left.path.localeCompare(right.path)), - [ - { path: "src/a.ts", state: "dismissed" }, - { path: "src/b.ts", state: "viewed" }, - ], - ); - }), -); - it.effect("keeps a deleted file cleared, which the head has no version of at all", () => Effect.gen(function* () { const service = yield* environmentViewedService(new Map(), []); diff --git a/apps/server/src/pullRequest/PullRequestService.ts b/apps/server/src/pullRequest/PullRequestService.ts index f171896a0eaf..fcf8afa08557 100644 --- a/apps/server/src/pullRequest/PullRequestService.ts +++ b/apps/server/src/pullRequest/PullRequestService.ts @@ -325,6 +325,7 @@ interface WorkspaceProjects { { readonly kind: SourceControlProviderKind; readonly projectCount: number } >; readonly viewerRoots: ReadonlyMap>; + readonly refinedProjectIdsByHost: ReadonlyMap>; } // Inherited by fibers forked inside the scope, so it is closed, not just dropped, when the scope ends. @@ -451,32 +452,37 @@ function withRateLimitBackoff( const protect = ( operation: string, effect: Effect.Effect, - allowPaused: boolean, + alwaysInteractive: boolean, ) => - limits.check(key, allowPaused ? { allowPaused: true } : undefined).pipe( - Effect.mapError( - (error) => - new PullRequestProviderError({ - provider: api.kind, - operation, - reason: "rate-limited", - detail: error.detail, - retryAt: error.retryAt, - cause: error, - }), - ), - Effect.flatMap((lease) => - effect.pipe( - Effect.provideService(SourceControlRateLimit.Interactive, allowPaused), - Effect.tap(() => limits.recordSuccess({ ...key, lease })), - Effect.tapError((error) => - error.reason === "rate-limited" - ? limits.recordRateLimit({ - ...key, - lease, - ...(error.retryAt === undefined ? {} : { retryAt: error.retryAt }), - }) - : Effect.void, + SourceControlRateLimit.Interactive.pipe( + Effect.map((callerInteractive) => alwaysInteractive || callerInteractive), + Effect.flatMap((allowPaused) => + limits.check(key, allowPaused ? { allowPaused: true } : undefined).pipe( + Effect.mapError( + (error) => + new PullRequestProviderError({ + provider: api.kind, + operation, + reason: "rate-limited", + detail: error.detail, + retryAt: error.retryAt, + cause: error, + }), + ), + Effect.flatMap((lease) => + effect.pipe( + Effect.provideService(SourceControlRateLimit.Interactive, allowPaused), + Effect.tap(() => limits.recordSuccess({ ...key, lease })), + Effect.tapError((error) => + error.reason === "rate-limited" + ? limits.recordRateLimit({ + ...key, + lease, + ...(error.retryAt === undefined ? {} : { retryAt: error.retryAt }), + }) + : Effect.void, + ), + ), ), ), ), @@ -485,10 +491,10 @@ function withRateLimitBackoff( , A>( operation: string, call: (...args: Args) => Effect.Effect, - allowPaused = false, + alwaysInteractive = false, ) => (...args: Args) => - protect(operation, call(...args), allowPaused); + protect(operation, call(...args), alwaysInteractive); const interactive = , A>( operation: string, call: (...args: Args) => Effect.Effect, @@ -903,6 +909,7 @@ export const make = Effect.gen(function* () { { kind: SourceControlProviderKind; projectCount: number } >(); const viewerRoots = new Map(); + const refinedProjectIdsByHost = new Map(); const seen = new Set(); for (const project of projects) { if (filter.projectId !== undefined && project.id !== filter.projectId) continue; @@ -923,6 +930,11 @@ export const make = Effect.gen(function* () { if (filter.host !== undefined && host !== filter.host.toLowerCase()) { continue; } + if (host !== sourceControlDefinitions.changeRequestHost(identity, identity.provider)) { + const projectIds = refinedProjectIdsByHost.get(host); + if (projectIds === undefined) refinedProjectIdsByHost.set(host, [project.id]); + else projectIds.push(project.id); + } const registered = registry.get(kind); const api = registered?.servesHost?.(host) === false ? null : registered; if (api !== null) { @@ -956,7 +968,7 @@ export const make = Effect.gen(function* () { : sourceControlDefinitions.canonicalRepositoryKey(repositoryKey.toLowerCase()), }); } - return { supported, unimplemented, viewerRoots }; + return { supported, unimplemented, viewerRoots, refinedProjectIdsByHost }; }), ); @@ -1263,7 +1275,12 @@ export const make = Effect.gen(function* () { const involvement = input.involvement ?? "all"; const epochAtStart = epochCounter; const continuation = yield* decodeCursors(input.cursors); - const { supported: projects, unimplemented, viewerRoots } = yield* listWorkspaceProjects(input); + const { + supported: projects, + unimplemented, + viewerRoots, + refinedProjectIdsByHost, + } = yield* listWorkspaceProjects(input); const projectCounts = new Map(); for (const { host } of projects) { projectCounts.set(host, (projectCounts.get(host) ?? 0) + 1); @@ -1285,6 +1302,7 @@ export const make = Effect.gen(function* () { searchesOnHost: capabilities?.search ?? false, actions: capabilities?.actions ?? [], projectCount: projectCounts.get(result.host) ?? 1, + projectIds: refinedProjectIdsByHost.get(result.host), configured: result.viewer !== null, detail: result.error === null ? null : providerDetail(requirementOf, result.error), }; @@ -1295,6 +1313,7 @@ export const make = Effect.gen(function* () { searchesOnHost: false, actions: [], projectCount, + projectIds: refinedProjectIdsByHost.get(host), configured: false, detail: "This host cannot be browsed here yet.", })), @@ -1313,6 +1332,14 @@ export const make = Effect.gen(function* () { message: `${repository} could not be read.`, })); if (readable.length === 0) { + const listResult: PullRequestListResult = { + viewers: viewers as PullRequestListResult["viewers"], + providers, + entries: [], + errors: unreadable, + truncated: false, + nextCursors: {}, + }; const errors = viewerResults.flatMap((result) => result.error === null || !selected.some(({ host }) => host === result.host) ? [] @@ -1320,17 +1347,13 @@ export const make = Effect.gen(function* () { ); const blocking = errors.find(isProviderUnusable) ?? errors[0]; if (blocking) { - return yield* toPullRequestError("list")(blocking); + // Preserve failure/retry semantics while letting clients try another server for these hosts. + const error = toPullRequestError("list")(blocking); + return yield* error._tag === "PullRequestUnavailableError" + ? new PullRequestUnavailableError({ ...error, listResult }) + : new PullRequestOperationError({ ...error, listResult }); } - - return { - viewers: viewers as PullRequestListResult["viewers"], - providers, - entries: [], - errors: [], - truncated: false, - nextCursors: {}, - }; + return listResult; } const limit = input.limit ?? DEFAULT_REPOSITORY_LIST_LIMIT; @@ -1810,6 +1833,9 @@ export const make = Effect.gen(function* () { ? {} : { reviewThreadsTruncated: activity.reviewThreadsTruncated }), reviewThreads: activity.reviewThreads, + ...(activity.overviewThreads === undefined + ? {} + : { overviewThreads: activity.overviewThreads }), commits: activity.commits, ...(activity.reactions === undefined ? {} : { reactions: activity.reactions }), })), diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.schedule.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.schedule.test.ts index 1070b49a0dcf..933e79d10c0b 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.schedule.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.schedule.test.ts @@ -1,11 +1,18 @@ import * as Scheduler from "../scheduling/Scheduler.ts"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import { expect, it } from "@effect/vitest"; -import { ScheduledTaskUpsertInput } from "@t3tools/contracts"; +import { + type OrchestrationV2ConversationMessage, + type OrchestrationV2Run, + MessageId, + RunId, + ScheduledTaskUpsertInput, +} from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import * as TestClock from "effect/testing/TestClock"; @@ -169,6 +176,156 @@ it.effect("keeps the task list order when a toggle or edit rewrites updated_at", }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); +it.effect("keeps one queued occurrence of a recurring task while its bound thread is busy", () => + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse("2026-07-01T09:00:00.000Z")); + // The bound thread's own run never finishes, so every scheduled send lands + // in its queue and stays there. + const activeRun = { + id: RunId.make("run:busy"), + status: "running", + userMessageId: MessageId.make("message:busy"), + } as OrchestrationV2Run; + const thread = yield* Ref.make<{ + readonly runs: ReadonlyArray; + readonly messages: ReadonlyArray; + }>({ runs: [activeRun], messages: [] }); + const runDueWork = yield* Ref.make>(Effect.void); + const layerDependencies = Layer.mergeAll( + NodeCrypto.layer, + // Capture the poll pass so the test drives each due interval directly. + Layer.mock(Scheduler.Scheduler)({ + register: (name, work) => + name !== "scheduled-tasks" + ? Effect.void + : Effect.context>().pipe( + Effect.flatMap((context) => + Ref.set(runDueWork, work.pipe(Effect.provideContext(context), Effect.ignore)), + ), + ), + }), + Layer.mock(ThreadLaunchService.ThreadLaunchService)({}), + Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ + namedProjectsRoot: "/projects", + isScratchProject: () => Effect.succeed(false), + }), + Layer.mock(ThreadManagementService.ThreadManagementService)({ + sendToThread: (input) => + Ref.modify(thread, (current) => { + const run = { + id: RunId.make(`run:${input.messageId}`), + status: "queued", + userMessageId: input.messageId, + } as OrchestrationV2Run; + const message = { + id: input.messageId, + threadId: input.threadId, + runId: run.id, + role: "user", + text: input.text, + ...(input.scheduledTaskId === undefined + ? {} + : { scheduledTaskId: input.scheduledTaskId }), + } as OrchestrationV2ConversationMessage; + return [ + { + delivery: "queued", + run, + message, + } as ThreadManagementService.ThreadManagementSendResult, + { runs: [...current.runs, run], messages: [...current.messages, message] }, + ]; + }), + getThreadRecords: () => Ref.get(thread).pipe(Effect.map((records) => records as never)), + getThreadShell: () => + Effect.succeed({ + projectId: "project-busy-thread", + deletedAt: null, + archivedAt: null, + } as never), + }), + Layer.mock(SecretRequests.SecretRequests)({}), + ); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* service.upsert( + yield* decodeUpsertInput({ + id: "scheduled-task:busy-thread", + title: "Check progress", + prompt: "Check progress", + enabled: true, + schedule: { type: "interval", everyMs: 60_000 }, + projectId: "project-busy-thread", + threadId: "thread:busy", + workspaceStrategy: { type: "root" }, + modelSelection: { instanceId: "codex", model: "gpt-5.4" }, + runtimeMode: "full-access", + interactionMode: "default", + }), + ); + + for (let interval = 0; interval < 3; interval++) { + yield* TestClock.adjust("61 seconds"); + yield* Effect.flatten(Ref.get(runDueWork)); + } + + const queuedOccurrencesOf = (taskId: typeof task.id) => + Ref.get(thread).pipe( + Effect.map( + ({ runs, messages }) => + runs + .filter((run) => run.status === "queued") + .map((run) => messages.find((message) => message.id === run.userMessageId)) + .filter((message) => message?.scheduledTaskId === taskId).length, + ), + ); + const queuedOccurrences = queuedOccurrencesOf(task.id); + expect(yield* queuedOccurrences).toBe(1); + // Skipped intervals still move the schedule forward. + const [listed] = (yield* service.list()).tasks; + expect(Date.parse(listed?.nextRunAt ?? "")).toBeGreaterThan( + Date.parse("2026-07-01T09:03:03.000Z"), + ); + + // Run now is an explicit request, so it still queues behind the held one. + yield* service.runNow({ id: task.id }); + expect(yield* queuedOccurrences).toBe(2); + + // Once the queued copies start, the next interval sends again. + yield* Ref.update(thread, (current) => ({ + ...current, + runs: current.runs.map((run) => + run.status === "queued" ? ({ ...run, status: "running" } as OrchestrationV2Run) : run, + ), + })); + yield* TestClock.adjust("61 seconds"); + yield* Effect.flatten(Ref.get(runDueWork)); + expect(yield* queuedOccurrences).toBe(1); + + // A one-time task's only scheduled occurrence still sends behind a queued Run now. + const { task: once } = yield* service.upsert( + yield* decodeUpsertInput({ + id: "scheduled-task:busy-thread-once", + title: "Check once", + prompt: "Check once", + enabled: true, + schedule: { type: "once", runAt: "2026-07-01T09:05:00.000Z" }, + projectId: "project-busy-thread", + threadId: "thread:busy", + workspaceStrategy: { type: "root" }, + modelSelection: { instanceId: "codex", model: "gpt-5.4" }, + runtimeMode: "full-access", + interactionMode: "default", + }), + ); + yield* service.runNow({ id: once.id }); + yield* TestClock.adjust("61 seconds"); + yield* Effect.flatten(Ref.get(runDueWork)); + expect(yield* queuedOccurrencesOf(once.id)).toBe(2); + }).pipe(Effect.provide(ScheduledTaskService.layer.pipe(Layer.provide(layerDependencies)))); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + it.effect( "idle polls still wake for tasks saved later, recorded due times, and beside far-future tasks", () => diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index b9f6f02981a1..f1b49d2422ea 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -803,6 +803,24 @@ export const layer = Layer.effect( ), ); + // Reads every run row but only the queued runs' messages, so the cost per + // due interval tracks the thread's run count, not its transcript. + const hasQueuedOccurrence = Effect.fn("ScheduledTaskService.hasQueuedOccurrence")(function* ( + taskId: ScheduledTaskId, + threadId: ThreadId, + ) { + const { runs } = yield* threadManagement.getThreadRecords(threadId, ["runs"]); + const queued = runs.filter((run) => run.status === "queued"); + if (queued.length === 0) return false; + const queuedMessageIds = new Set(queued.map((run) => run.userMessageId)); + const { messages } = yield* threadManagement.getThreadRecords(threadId, ["messages"], { + messageIds: [...queuedMessageIds], + }); + return messages.some( + (message) => queuedMessageIds.has(message.id) && message.scheduledTaskId === taskId, + ); + }); + // The run already started the thread, so a failed write only costs the binding. // A thread deleted before the binding landed already ran its task cleanup, so apply that // cleanup here; a later deletion finds the binding. An unconfirmed lookup keeps the binding. @@ -869,6 +887,19 @@ export const layer = Layer.effect( ) { return { task: active, threadId: null, dispatchFailed: false }; } + // A thread that stays busy would otherwise collect one copy of the + // prompt per interval. A failed read falls through to the normal send. + if ( + trigger === "scheduled" && + active.schedule.type !== "once" && + active.threadId !== null && + (yield* hasQueuedOccurrence(active.id, ThreadId.make(active.threadId)).pipe( + Effect.orElseSucceed(() => false), + )) + ) { + yield* rescheduleMissedRun(active, startedAt, "previous occurrence still queued"); + return { task: active, threadId: null, dispatchFailed: false }; + } // Reject queued deliveries after pause, same-id recreation or trigger changes. if (webhook !== undefined) { const reason = @@ -1089,10 +1120,12 @@ export const layer = Layer.effect( const rescheduleMissedRun = Effect.fn("ScheduledTaskService.rescheduleMissedRun")(function* ( task: ScheduledTask, now: DateTime.DateTime, + reason = "missed", ) { const next = nextRunAt(task, now); - yield* Effect.logInfo("Skipping missed schedule task run", { + yield* Effect.logInfo("Skipping schedule task run", { taskId: task.id, + reason, missedRunAt: task.nextRunAt, rescheduledTo: next, }); @@ -1212,7 +1245,10 @@ export const layer = Layer.effect( const list: ScheduledTaskService["Service"]["list"] = () => listRows().pipe( - Effect.map((tasks) => ({ tasks })), + Effect.map((tasks) => ({ + tasks, + timeZone: DateTime.zoneToString(DateTime.zoneMakeLocal()), + })), Effect.mapError((cause) => taskError("Could not list schedule tasks.", { cause })), ); diff --git a/apps/server/src/sourceControl/SourceControlDiscovery.test.ts b/apps/server/src/sourceControl/SourceControlDiscovery.test.ts index 2b76ccb09ba9..45db826c8a9b 100644 --- a/apps/server/src/sourceControl/SourceControlDiscovery.test.ts +++ b/apps/server/src/sourceControl/SourceControlDiscovery.test.ts @@ -1,10 +1,20 @@ -import { SourceControlProviderKind, VcsProcessSpawnError } from "@t3tools/contracts"; +import { + SourceControlProviderKind, + type VcsError, + VcsProcessExitError, + VcsProcessSpawnError, + VcsProcessTimeoutError, +} from "@t3tools/contracts"; import { assert, it } from "@effect/vitest"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; -import { providerAuth } from "@t3tools/source-control-core/server/discovery"; +import * as PlatformError from "effect/PlatformError"; +import { + providerAuth, + type SourceControlProviderDiscoverySpec, +} from "@t3tools/source-control-core/server/discovery"; import type * as SourceControlHost from "@t3tools/source-control-core/server/SourceControlHost"; import { fakeDiscovery, @@ -29,7 +39,29 @@ const run = ( SourceControlHost.SourceControlProcessInput, "operation" | "command" | "args" | "cwd" >, -): Effect.Effect => { +): Effect.Effect => { + const context = { operation: input.operation, command: input.command, cwd: input.cwd }; + if (input.command === "slow-cli") { + return Effect.fail(new VcsProcessTimeoutError({ ...context, timeoutMs: 5_000 })); + } + if (input.command === "broken-cli") { + return Effect.fail( + new VcsProcessExitError({ ...context, exitCode: 1, detail: "broken-cli: bad install" }), + ); + } + if (input.command === "absent-cli") { + return Effect.fail( + new VcsProcessSpawnError({ + ...context, + cause: PlatformError.systemError({ + _tag: "NotFound", + module: "ChildProcess", + method: "spawn", + pathOrDescriptor: input.command, + }), + }), + ); + } if (input.args[0] === "--version") { return input.command === "missing-cli" ? Effect.fail( @@ -47,29 +79,37 @@ const run = ( ); }; -const cliDiscovery = (name: string) => ({ +const cliDiscovery = ( + name: string, + executable = name === "beta" ? "missing-cli" : `${name}-cli`, +) => ({ type: "cli" as const, kind: kind(name), label: name, installHint: `Install ${name}.`, - executable: name === "beta" ? "missing-cli" : `${name}-cli`, + executable, versionArgs: ["--version"], authArgs: ["auth", "status"], parseAuth: ({ stdout }: { readonly stdout: string }) => providerAuth({ status: "authenticated", account: /as (\S+)/.exec(stdout)?.[1] }), }); -const layerDiscovery = (runProcess = run) => +const defaultDiscoveries = [ + cliDiscovery("alpha"), + cliDiscovery("beta"), + fakeDiscovery("gamma", { + probeAuth: Effect.succeed(providerAuth({ status: "authenticated", account: "api-user" })), + }), +]; + +const layerDiscovery = ( + runProcess = run, + discoveries: ReadonlyArray = defaultDiscoveries, +) => Layer.effect( SourceControlProviderRegistry.SourceControlProviderRegistry, SourceControlProviderRegistry.makeWithProviders( - [ - cliDiscovery("alpha"), - cliDiscovery("beta"), - fakeDiscovery("gamma", { - probeAuth: Effect.succeed(providerAuth({ status: "authenticated", account: "api-user" })), - }), - ].map((discovery) => ({ + discoveries.map((discovery) => ({ kind: discovery.kind, provider: fakeSourceControlProvider(discovery.kind), discovery, @@ -133,6 +173,41 @@ it.effect("reports every hosting integration in registry order with its sign-in" }).pipe(Effect.provide(SourceControlDiscovery.layer.pipe(Layer.provide(layerDiscovery())))), ); +it.effect("explains a hosting CLI whose version probe failed instead of advising an install", () => + Effect.gen(function* () { + const discovery = yield* SourceControlDiscovery.SourceControlDiscovery; + const result = yield* discovery.discover; + const item = (name: string) => { + const found = result.sourceControlProviders.find((entry) => entry.kind === kind(name)); + assert.isDefined(found); + return found; + }; + const authDetail = (name: string) => Option.getOrElse(item(name).auth.detail, () => ""); + + assert.notInclude(item("slow").installHint, "Install slow."); + assert.include(item("slow").installHint, "timed out after 5000ms"); + assert.strictEqual(authDetail("slow"), item("slow").installHint); + + assert.notInclude(item("broken").installHint, "Install broken."); + assert.include(item("broken").installHint, "exited with status 1"); + assert.strictEqual(authDetail("broken"), item("broken").installHint); + + assert.strictEqual(item("absent").installHint, "Install absent."); + }).pipe( + Effect.provide( + SourceControlDiscovery.layer.pipe( + Layer.provide( + layerDiscovery(run, [ + cliDiscovery("slow", "slow-cli"), + cliDiscovery("broken", "broken-cli"), + cliDiscovery("absent", "absent-cli"), + ]), + ), + ), + ), + ), +); + it.effect.each(["darwin", "linux"] as const)( "names incompatible executables only on macOS (%s)", (platform) => { diff --git a/apps/server/src/sourceControl/SourceControlRepositoryService.test.ts b/apps/server/src/sourceControl/SourceControlRepositoryService.test.ts index c785b4cf3361..e4a09101b7a5 100644 --- a/apps/server/src/sourceControl/SourceControlRepositoryService.test.ts +++ b/apps/server/src/sourceControl/SourceControlRepositoryService.test.ts @@ -198,6 +198,9 @@ function makePublishService(reason?: GitCommandError["reason"]) { ); } +// Coder: upstream's "says a clone that ran out of time timed out" test (#27588) builds +// upstream's GitHub clone harness, which this file replaces with GitLab cases. + it.effect("explains a rejected SSH push after creating the repository", () => Effect.gen(function* () { const service = yield* makePublishService("authentication_failed"); diff --git a/apps/server/src/sourceControl/SourceControlRepositoryService.ts b/apps/server/src/sourceControl/SourceControlRepositoryService.ts index be3ca71d54c4..18d5a0d19150 100644 --- a/apps/server/src/sourceControl/SourceControlRepositoryService.ts +++ b/apps/server/src/sourceControl/SourceControlRepositoryService.ts @@ -1,4 +1,5 @@ import * as Context from "effect/Context"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; @@ -362,12 +363,15 @@ export const make = Effect.gen(function* () { }; // Git misreads @ in a pathless URL's query as userinfo and echoes its suffix as the host. const normalizedCloneUrl = cloneUrl.replace(/^([a-z][a-z0-9+.-]*:\/\/[^/?#]+)\?/i, "$1/?"); - yield* git + const provider = input.provider ?? SourceControlProviderKind.make("unknown"); + const timeoutMs = options?.timeoutMs === undefined ? CLONE_TIMEOUT_MS : options.timeoutMs; + const clone = git .execute({ operation: "SourceControlRepositoryService.cloneRepository", cwd: path.dirname(destinationPath), args: ["clone", "--progress", "--", normalizedCloneUrl, path.basename(destinationPath)], - timeoutMs: options?.timeoutMs === undefined ? CLONE_TIMEOUT_MS : options.timeoutMs, + // The deadline is applied below so a timeout is reported as one, not as git's last line. + timeoutMs: null, // Buffered output is unused and progress can be huge; cap it while retaining line callbacks. maxOutputBytes: 256 * 1024, appendTruncationMarker: true, @@ -380,7 +384,7 @@ export const make = Effect.gen(function* () { (cause) => new SourceControlRepositoryError({ operation: "cloneRepository", - provider: input.provider ?? SourceControlProviderKind.make("unknown"), + provider, detail: stderrTail.length > 0 ? stderrTail.join(" ") @@ -389,6 +393,21 @@ export const make = Effect.gen(function* () { }), ), ); + if (timeoutMs === null) return yield* clone; + // Interrupting the clone closes the scope that owns git, which kills it. + return yield* clone.pipe( + Effect.timeoutOrElse({ + duration: timeoutMs, + orElse: () => + Effect.fail( + new SourceControlRepositoryError({ + operation: "cloneRepository", + provider, + detail: `The clone timed out after ${Duration.format(Duration.millis(timeoutMs))}.`, + }), + ), + }), + ); }); const cloneRepository = Effect.fn("SourceControlRepositoryService.cloneRepository")(function* ( diff --git a/apps/server/src/storageCleanup.test.ts b/apps/server/src/storageCleanup.test.ts index 95bea38cab09..643c0e201773 100644 --- a/apps/server/src/storageCleanup.test.ts +++ b/apps/server/src/storageCleanup.test.ts @@ -2117,6 +2117,21 @@ describe("storage cleanup reports and local file policies", () => { }), ), ); + it.live("skips a stored worktree path whose parent is a regular file", () => + runCleanupTest( + Effect.gen(function* () { + const fixture = yield* cleanupFixture(); + const legacyRoot = `${fixture.config.worktreesDir}/legacy-root`; + yield* fixture.fs.writeFileString(legacyRoot, ""); + fixture.setThreads([ + shell({ branch: "feature", worktreePath: `${legacyRoot}/stored-workspace` }), + ]); + const report = yield* fixture.service.runNow; + expect(report.entries).toEqual([]); + expect(report.counts).toEqual({ removed: 0, kept: 0, failed: 0 }); + }), + ), + ); it.live.each(["before-check", "while-measuring"] as const)( "keeps a worktree protected by a missing project root (%s)", (timing) => @@ -2673,6 +2688,120 @@ describe("storage cleanup reports and local file policies", () => { }), ), ); + it.live.each(["--assume-unchanged", "--skip-worktree"] as const)( + "keeps tracked edits that git status hides (%s)", + (flag) => + runCleanupTest( + Effect.gen(function* () { + const { service, fs, worktree, command } = yield* cleanupFixture(); + yield* command(worktree, ["update-index", flag, "tracked.txt"]); + yield* fs.writeFileString(`${worktree}/tracked.txt`, "hidden edit\n"); + expect((yield* service.runNow).entries[0]).toMatchObject({ outcome: "kept" }); + expect(yield* fs.readFileString(`${worktree}/tracked.txt`)).toBe("hidden edit\n"); + }), + ), + ); + // macOS and Windows file systems reject names that are not valid UTF-8. + it.live.skipIf(HostProcess.Platform.defaultValue() !== "linux")( + "keeps hidden edits to files whose names are not valid UTF-8", + () => + runCleanupTest( + Effect.gen(function* () { + const { service, fs, worktree, command } = yield* cleanupFixture(); + const git = yield* GitVcsDriver.GitVcsDriver; + // Git unquotes C-quoted paths on stdin, which reaches a name a JS string cannot. + const quoted = '"bad-\\377.txt"'; + const stdin = (args: string[], input: string) => + git.execute({ operation: "cleanup-test", cwd: worktree, args, stdin: input }); + const blob = (content: string) => + stdin(["hash-object", "-w", "--stdin"], content).pipe( + Effect.map((result) => result.stdout.trim()), + ); + const original = yield* blob("original\n"); + yield* stdin(["update-index", "--index-info"], `100644 ${original}\t${quoted}\n`); + yield* stdin(["checkout-index", "--stdin"], `${quoted}\n`); + yield* command(worktree, ["commit", "-m", "bad name"]); + // The committed content moves on while the file on disk keeps the other version. + const edited = yield* blob("edited\n"); + yield* stdin(["update-index", "--index-info"], `100644 ${edited}\t${quoted}\n`); + yield* command(worktree, ["commit", "-m", "edit"]); + yield* stdin(["update-index", "--assume-unchanged", "--stdin"], `${quoted}\n`); + expect((yield* command(worktree, ["status", "--porcelain"])).stdout).toBe(""); + expect((yield* service.runNow).entries[0]).toMatchObject({ outcome: "kept" }); + expect((yield* fs.readDirectory(worktree)).some((name) => name.startsWith("bad-"))).toBe( + true, + ); + }), + ), + ); + it.live("removes worktrees whose flagged files are unchanged or outside a sparse checkout", () => + runCleanupTest( + Effect.gen(function* () { + const { service, fs, worktree, command } = yield* cleanupFixture(); + yield* command(worktree, ["update-index", "--skip-worktree", "tracked.txt"]); + yield* fs.remove(`${worktree}/tracked.txt`); + yield* command(worktree, ["update-index", "--assume-unchanged", ".gitignore"]); + expect((yield* service.runNow).entries[0]).toMatchObject({ outcome: "removed" }); + }), + ), + ); + it.live.each([ + ["missing", "symlink"], + ["tracked.txt", "symlink"], + ["missing", "file"], + ["tracked.txt", "file"], + ] as const)("compares flagged tracked symlinks by target (%s as a %s)", ([target, kind]) => + runCleanupTest( + Effect.gen(function* () { + const { service, fs, worktree, command } = yield* cleanupFixture(); + yield* fs.symlink("tracked.txt", `${worktree}/link`); + yield* command(worktree, ["add", "link"]); + yield* command(worktree, ["commit", "-m", "link"]); + yield* command(worktree, ["update-index", "--assume-unchanged", "link"]); + yield* fs.remove(`${worktree}/link`); + // Without core.symlinks, Git checks a link out as a regular file holding its target. + if (kind === "file") { + yield* command(worktree, ["config", "core.symlinks", "false"]); + yield* fs.writeFileString(`${worktree}/link`, target); + } else yield* fs.symlink(target, `${worktree}/link`); + expect((yield* service.runNow).entries[0]).toMatchObject( + target === "missing" + ? { outcome: "kept", reason: expect.stringContaining("(link)") } + : { outcome: "removed" }, + ); + }), + ), + ); + it.live.each([ + ["uncommitted-changes", "untracked", "Local changes exceed the inspection limit"], + ["any-local-files", "ignored", "Ignored files exceed the inspection limit"], + ] as const)( + "keeps a worktree under %s when its %s file listing exceeds the inspection limit", + ([policy, kind, reason]) => + runCleanupTest( + Effect.gen(function* () { + const { service, fs, worktree, setPolicy } = yield* cleanupFixture(); + setPolicy(policy); + const files = Array.from({ length: 3000 }, (_, i) => + kind === "untracked" + ? `${worktree}/many/untracked-file-with-a-long-name-${i}.txt` + : `${worktree}/many/ignored-directory-with-a-long-name-${i}/.env`, + ); + yield* fs.makeDirectory(`${worktree}/many`); + yield* Effect.forEach( + files, + (file) => + Effect.gen(function* () { + if (kind === "ignored") yield* fs.makeDirectory(file.slice(0, -"/.env".length)); + yield* fs.writeFileString(file, "local\n"); + }), + { concurrency: 64, discard: true }, + ); + expect((yield* service.runNow).entries[0]).toMatchObject({ outcome: "kept", reason }); + expect(yield* fs.exists(files[0]!)).toBe(true); + }), + ), + ); it.live("serializes concurrent requests and returns each run's report", () => runCleanupTest( Effect.gen(function* () { diff --git a/apps/server/src/storageCleanup.ts b/apps/server/src/storageCleanup.ts index 0ab5fc708192..fae43706f812 100644 --- a/apps/server/src/storageCleanup.ts +++ b/apps/server/src/storageCleanup.ts @@ -47,6 +47,7 @@ import * as Settings from "./serverSettings.ts"; import * as TerminalManager from "./terminal/Manager.ts"; import { isFilesystemRoot, managedWorktreesDirectories } from "./worktreesDirectory.ts"; import * as GitVcsDriver from "./vcs/GitVcsDriver.ts"; +import { isNonRepositoryGitStderr } from "./vcs/GitVcsDriverCore.ts"; import { removeLinkedWorktree } from "./vcs/worktreeRemoval.ts"; import { withWorkspaceLease } from "./workspace/workspaceLease.ts"; import * as HostProcess from "@t3tools/shared/HostProcess"; @@ -85,6 +86,8 @@ const ORPHAN_WORKTREE_GRACE_MS = 60 * 60 * 1_000; const sentence = (text: string) => text.charAt(0).toUpperCase() + text.slice(1); const REPORT_ENTRY_LIMIT = 200; const isGitCommandError = Schema.is(GitCommandError); +const errorCode = (error: PlatformError) => + error.cause instanceof Error && "code" in error.cause ? error.cause.code : undefined; function cleanupFailureReason(error: { readonly message: string }) { return isGitCommandError(error) @@ -379,36 +382,34 @@ const make = Effect.gen(function* () { let entries = 1; let bytes = 0; while (pending.length > 0) { - const target = pending.pop()!; - if (target === skipped) continue; - // Effect's stat follows links. Probe with readLink first to get lstat - // semantics, including skipping dangling links and directory links. - const isLink = yield* fs.readLink(target).pipe( - Effect.as(true), - Effect.catchIf( - (error) => - error.cause instanceof Error && - "code" in error.cause && - error.cause.code === "EINVAL", - () => Effect.succeed(false), - ), + const inspected = yield* Effect.forEach( + pending.splice(-8), + // Untraced: this runs once per entry, and a worktree can hold millions. + Effect.fnUntraced(function* (target) { + if (target === skipped) return { target, bytes: 0, names: [] }; + const stat = yield* fs.lstat(target); + if (stat.type === "SymbolicLink") return { target, bytes: 0, names: [] }; + if ((yield* fs.realPath(target)) !== target) return null; + if (stat.type === "Directory") + return { target, bytes: 0, names: yield* fs.readDirectory(target) }; + // Removing a shared hardlink leaves its blocks allocated. + if (stat.type !== "File" || Option.getOrElse(stat.nlink, () => 1n) !== 1n) + return { target, bytes: 0, names: [] }; + const bytes = Option.match(stat.blocks, { + onNone: () => Number(stat.size), + onSome: (blocks) => Number(blocks * 512n), + }); + return { target, bytes, names: [] }; + }), + { concurrency: "unbounded" }, ); - if (isLink) continue; - if ((yield* fs.realPath(target)) !== target) return null; - const stat = yield* fs.stat(target); - if (stat.type === "File") { - // Removing a shared hardlink leaves its blocks allocated. - if (Option.getOrElse(stat.nlink, () => 1n) !== 1n) continue; - bytes += Option.match(stat.blocks, { - onNone: () => Number(stat.size), - onSome: (blocks) => Number(blocks * 512n), - }); - } else if (stat.type === "Directory") { - const names = yield* fs.readDirectory(target); - entries += names.length; + for (const result of inspected) { + if (result === null) return null; + bytes += result.bytes; + entries += result.names.length; if (entries > 2_000_000) return null; - for (const name of names) { - const child = path.join(target, name); + for (const name of result.names) { + const child = path.join(result.target, name); if (!inside(root, child)) return null; pending.push(child); } @@ -618,7 +619,7 @@ const make = Effect.gen(function* () { ); if (result === null) return null; if (result.exitCode === 0 || result.exitCode === allowedExitCode) return result; - if (result.stderr.toLowerCase().includes("not a git repository")) return null; + if (isNonRepositoryGitStderr(result.stderr)) return null; return yield* failure( result.stderr.trim().split(/\r?\n/)[0]?.slice(0, 1000) || "command failed", ); @@ -637,6 +638,99 @@ const make = Effect.gen(function* () { return (name?.startsWith("(") ? null : name) !== branch; }); + // Git status and non-forced worktree removal both skip assume-unchanged and skip-worktree files. + const hiddenTrackedChange = Effect.fn("StorageCleanup.hiddenTrackedChange")(function* ( + cwd: string, + ) { + const inspect = (args: string[], stdin?: string) => + git.execute({ + operation: "StorageCleanup.hiddenTrackedChanges", + cwd, + args, + ...(stdin === undefined ? {} : { stdin }), + maxOutputBytes: 1024 * 1024, + }); + const sparse = yield* inspect([ + "config", + "--type=bool", + "--default=false", + "core.sparseCheckout", + ]); + // Sparse checkouts flag every file outside the cone; Git 2.37+ unflags those it finds on disk. + const hiddenTag = sparse.stdout.trim() === "true" ? /^[a-z]$/ : /^(?:S|[a-z])$/; + const flagged: Array<{ readonly mode: string; readonly id: string; readonly file: string }> = + []; + let overflow = false; + yield* git.execute({ + operation: "StorageCleanup.hiddenTrackedChanges", + cwd, + args: ["-c", "core.quotePath=false", "ls-files", "-s", "-v"], + maxOutputBytes: 64 * 1024, + appendTruncationMarker: true, + keepLineCallbacksAfterTruncation: true, + progress: { + onStdoutLine: (line) => + Effect.sync(() => { + const match = /^(\S) (\d+) ([0-9a-f]+) \d\t(.*)$/s.exec(line); + if (match === null || !hiddenTag.test(match[1]!) || match[2] === "160000") return; + if (flagged.length < 10_000) + flagged.push({ mode: match[2]!, id: match[3]!, file: match[4]! }); + else overflow = true; + }), + }, + }); + if (overflow) return "hidden tracked files exceed the inspection limit"; + const reason = (file: string) => `has uncommitted changes hidden from git status (${file})`; + // Quoted names (control characters, quotes, backslashes) and names that are not valid UTF-8 + // decode to text that is not the path on disk. + const unreadable = flagged.find(({ file }) => file.startsWith('"') || file.includes("\uFFFD")); + if (unreadable !== undefined) return reason(unreadable.file); + const onDisk = yield* Effect.forEach( + flagged, + (entry) => { + const file = path.join(cwd, entry.file); + return fs.readLink(file).pipe( + Effect.map((target) => ({ ...entry, type: "SymbolicLink", target })), + Effect.catchIf( + (error) => errorCode(error) === "EINVAL", + () => fs.stat(file).pipe(Effect.map((info) => ({ ...entry, type: info.type }))), + ), + Effect.catchIf( + (error) => errorCode(error) === "ENOENT" || errorCode(error) === "ENOTDIR", + () => Effect.succeed(null), + ), + ); + }, + { concurrency: 16 }, + ); + const files: Array<{ readonly id: string; readonly file: string }> = []; + // Without core.symlinks a tracked link is a regular file holding its target, compared unfiltered. + const linkFiles: typeof files = []; + for (const entry of onDisk) { + if (entry === null) continue; + if (entry.type === "File") (entry.mode === "120000" ? linkFiles : files).push(entry); + else if (!("target" in entry) || entry.mode !== "120000") return reason(entry.file); + // Hashing a link's path would hash the file it points to. + else if ( + (yield* inspect(["hash-object", "--stdin"], entry.target)).stdout.trim() !== entry.id + ) + return reason(entry.file); + } + const hashPaths = (entries: ReadonlyArray<{ readonly file: string }>, flags: string[]) => + entries.length === 0 + ? Effect.succeed([]) + : inspect( + ["hash-object", ...flags, "--stdin-paths"], + entries.map(({ file }) => `${file}\n`).join(""), + ).pipe(Effect.map((result) => result.stdout.split("\n"))); + const worktreeIds = yield* hashPaths(files, []); + const linkIds = yield* hashPaths(linkFiles, ["--no-filters"]); + const changed = + files.find(({ id }, i) => worktreeIds[i] !== id) ?? + linkFiles.find(({ id }, i) => linkIds[i] !== id); + return changed === undefined ? null : reason(changed.file); + }); + const localChangesReason = Effect.fn("StorageCleanup.localChangesReason")(function* ( cwd: string, rules: WorktreeCleanupRules, @@ -654,6 +748,7 @@ const make = Effect.gen(function* () { : "--untracked-files=all", ], maxOutputBytes: 64 * 1024, + appendTruncationMarker: true, }); if (status.stdoutTruncated) return "local changes exceed the inspection limit"; const records = status.stdout.split("\0").filter(Boolean); @@ -663,12 +758,15 @@ const make = Effect.gen(function* () { if (/^[RC]|^.[RC]/.test(records[i]!)) i++; } if (count > 0) return `has uncommitted changes (${count} ${count === 1 ? "file" : "files"})`; + const hidden = yield* hiddenTrackedChange(cwd); + if (hidden !== null) return hidden; if (rules.worktreeKeepWhen !== "any-local-files") return null; const ignored = yield* git.execute({ operation: "StorageCleanup.ignoredFiles", cwd, args: ["ls-files", "--others", "--ignored", "--exclude-standard", "--directory", "-z"], maxOutputBytes: 64 * 1024, + appendTruncationMarker: true, }); if (ignored.stdoutTruncated) return "ignored files exceed the inspection limit"; const file = ignored.stdout @@ -925,7 +1023,17 @@ const make = Effect.gen(function* () { return reason; }; yield* Effect.gen(function* () { - if (!(yield* fs.exists(worktreePath))) return; + const exists = yield* fs.exists(worktreePath).pipe( + // A stored path under a regular file cannot exist, but access() fails with ENOTDIR. + Effect.catchIf( + (error) => + error.cause instanceof Error && + "code" in error.cause && + error.cause.code === "ENOTDIR", + () => Effect.succeed(false), + ), + ); + if (!exists) return; if (deleted && !settings.worktreeOnDelete) return keep("no rules apply"); if (members.length > 1 && worktreeOwner(members) === undefined) return keep(`shared by ${members.length} threads`); diff --git a/apps/server/src/terminal/Manager.test.ts b/apps/server/src/terminal/Manager.test.ts index 4b3435585970..6f1c3cc8b30f 100644 --- a/apps/server/src/terminal/Manager.test.ts +++ b/apps/server/src/terminal/Manager.test.ts @@ -40,6 +40,7 @@ import * as ProjectSetupScriptRunner from "../project/ProjectSetupScriptRunner.t import * as TerminalManager from "./Manager.ts"; import { terminalOutputStream } from "./OutputProtocol.ts"; import * as PtyAdapter from "@t3tools/shared/PtyAdapter"; +import packageJson from "../../package.json" with { type: "json" }; vi.mock("effect/SynchronizedRef", async (importOriginal) => { const original = await importOriginal(); @@ -1927,6 +1928,51 @@ it.layer( }), ); + it.effect.each([ + { + platform: "linux", + parentEnv: { TERM_PROGRAM: "Apple_Terminal", TERM_PROGRAM_VERSION: "455" }, + runtimeEnv: { TERM_PROGRAM: "", TERM_PROGRAM_VERSION: "custom-version" }, + }, + { + platform: "win32", + parentEnv: { term_program: "Apple_Terminal", Term_Program_Version: "455" }, + runtimeEnv: { Term_Program: "", term_program_version: "custom-version" }, + }, + ] as const)( + "identifies as T3 Code unless the terminal env overrides it on $platform", + ({ platform, parentEnv, runtimeEnv }) => + Effect.gen(function* () { + const originalEnv = { ...parentEnv }; + const env = Object.freeze(parentEnv); + const { manager, ptyAdapter } = yield* createManager(5, { env }).pipe( + Effect.provide(layerWithHostPlatform(platform)), + ); + + yield* manager.open(openInput()); + yield* manager.open(openInput({ terminalId: "override", env: runtimeEnv })); + + expect(ptyAdapter.spawnInputs).toHaveLength(2); + for (const [index, expected] of [ + { TERM_PROGRAM: "t3code", TERM_PROGRAM_VERSION: packageJson.version }, + { TERM_PROGRAM: "", TERM_PROGRAM_VERSION: "custom-version" }, + ].entries()) { + const spawnEnv = ptyAdapter.spawnInputs[index]!.env; + for (const [key, value] of Object.entries(expected)) { + expect + .soft( + Object.entries(spawnEnv) + .filter(([candidate]) => candidate.toUpperCase() === key) + .map(([, entryValue]) => entryValue), + `${index === 0 ? "default" : "override"} ${key}`, + ) + .toEqual([value]); + } + } + expect(env).toEqual(originalEnv); + }), + ); + it.effect("filters app runtime env variables from terminal sessions", () => Effect.gen(function* () { const { manager, ptyAdapter } = yield* createManager(5, { @@ -2018,6 +2064,20 @@ it.layer( }), ); + it.effect("expands a provider home named in any case on Windows", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const home = yield* HostProcess.HomeDirectory; + const { manager, ptyAdapter } = yield* createManager(5, { + env: { GROK_HOME: "C:\\inherited-grok" }, + }); + + yield* manager.open(openInput({ env: { grok_home: "~/.grok-work" } })); + + expect(ptyAdapter.spawnInputs[0]?.env.GROK_HOME).toBe(path.join(home, ".grok-work")); + }).pipe(Effect.provide(layerWithHostPlatform("win32"))), + ); + it.effect("fails closed when a provider instance is missing", () => Effect.gen(function* () { const providerInstanceId = ProviderInstanceId.make("deleted_instance"); diff --git a/apps/server/src/terminal/Manager.ts b/apps/server/src/terminal/Manager.ts index 4383bda4b55c..8f35a7d5c818 100644 --- a/apps/server/src/terminal/Manager.ts +++ b/apps/server/src/terminal/Manager.ts @@ -60,13 +60,14 @@ import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import * as SynchronizedRef from "effect/SynchronizedRef"; +import packageJson from "../../package.json" with { type: "json" }; import * as ServerConfig from "../config.ts"; import { mergeProviderInstanceEnvironment } from "@t3tools/provider-core/server/instanceEnvironment"; import { resolveCodexHomeLayout } from "../provider/Drivers/CodexHomeLayout.ts"; import { makeClaudeEnvironment } from "../provider/Drivers/ClaudeHome.ts"; import { deriveProviderInstanceConfigMap } from "../provider/ProviderInstanceRegistryHydration.ts"; import * as ServerSettings from "../serverSettings.ts"; -import { expandHomePath } from "@t3tools/provider-core/server/pathExpansion"; +import { expandHomeVariable } from "@t3tools/provider-core/server/pathExpansion"; import * as ProcessRunner from "../processRunner.ts"; import * as PortScanner from "../preview/PortScanner.ts"; import { terminalOutputStream } from "./OutputProtocol.ts"; @@ -1375,16 +1376,27 @@ function createTerminalSpawnEnv( for (const [key, value] of Object.entries(baseEnv)) { if (value === undefined) continue; if (shouldExcludeTerminalEnvKey(key)) continue; + if (platform === "win32") { + const normalizedKey = key.toUpperCase(); + if (normalizedKey === "TERM_PROGRAM" || normalizedKey === "TERM_PROGRAM_VERSION") continue; + } spawnEnv[key] = value; } + // The server inherits its launcher's identity (e.g. Apple_Terminal); shells + // and tools inside the integrated terminal should see T3 Code instead. + spawnEnv.TERM_PROGRAM = "t3code"; + spawnEnv.TERM_PROGRAM_VERSION = packageJson.version; if (runtimeEnv) { for (const [key, value] of Object.entries(runtimeEnv)) { const existingKey = platform === "win32" ? Object.keys(spawnEnv).find((candidate) => candidate.toLowerCase() === key.toLowerCase()) : undefined; - spawnEnv[existingKey ?? key] = - key === "CODEX_HOME" || key === "CLAUDE_CONFIG_DIR" ? expandHomePath(value, home) : value; + spawnEnv[existingKey ?? key] = expandHomeVariable( + platform === "win32" ? key.toUpperCase() : key, + value, + home, + ); } } if (!spawnEnv.COLORTERM && runtimeEnv?.COLORTERM === undefined) { diff --git a/apps/server/src/textGeneration/CodexTextGeneration.test.ts b/apps/server/src/textGeneration/CodexTextGeneration.test.ts index aa4c0fb30261..fd6f23da6209 100644 --- a/apps/server/src/textGeneration/CodexTextGeneration.test.ts +++ b/apps/server/src/textGeneration/CodexTextGeneration.test.ts @@ -1,10 +1,12 @@ import { DEFAULT_TEXT_GENERATION_MODEL_BY_PROVIDER, + ProjectId, ProviderDriverKind, type ModelSelection, type ServerProvider, } from "@t3tools/contracts"; import type { ProviderInstance } from "@t3tools/provider-core/server/driver"; +import * as ProviderInstallationGate from "../provider/ProviderInstallationGate.ts"; import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as SourceControlProviderRegistry from "../sourceControl/SourceControlProviderRegistry.ts"; @@ -431,8 +433,8 @@ it.layer(layerCodexTextGenerationTest)("CodexTextGeneration", (it) => { subject: "Add important change", body: "", }), - launchArgs: "--strict-config --listen off", - requireArg: "--strict-config", + launchArgs: "--strict-config --oss --local-provider ollama --listen off", + requireArg: "--strict-config --oss --local-provider ollama", forbidArg: "--listen", }, (textGeneration) => @@ -543,6 +545,7 @@ const PREFLIGHT_PRIMARY_STATES = [ "pending", // Coder: no "exhausted" case; the out-of-usage fallback (#26164) is not carried. "not installed", + "not installed, fallback disabled for project", "signed out", "signed out, fallback failed", ] as const; @@ -551,6 +554,7 @@ it.effect.each(PREFLIGHT_PRIMARY_STATES)("text generation preflight: %s", (prima Effect.gen(function* () { const primary = ProviderInstanceId.make("codex"); const fallback = ProviderInstanceId.make("claude_work"); + const projectId = ProjectId.make("project-a"); const selection = { instanceId: primary, model: "chat-model", @@ -574,7 +578,7 @@ it.effect.each(PREFLIGHT_PRIMARY_STATES)("text generation preflight: %s", (prima ...(primaryState === "pending" && instanceId === primary ? { installed: false, status: "warning", auth: { status: "unknown" } } : {}), - ...(primaryState === "not installed" && instanceId === primary + ...(primaryState.startsWith("not installed") && instanceId === primary ? { installed: false, status: "error", auth: { status: "unknown" } } : {}), ...(primaryState.startsWith("signed out") && instanceId === primary @@ -617,10 +621,18 @@ it.effect.each(PREFLIGHT_PRIMARY_STATES)("text generation preflight: %s", (prima providerInstances: { [fallback]: { driver: ProviderDriverKind.make("claudeAgent"), config: {} }, }, + ...(primaryState === "not installed, fallback disabled for project" + ? { + projectSettingsOverrides: { + [projectId]: { providerInstances: { [fallback]: { enabled: false } } }, + }, + } + : {}), }), Layer.mock(SourceControlProviderRegistry.SourceControlProviderRegistry)({ resolveLink: () => Effect.die("unused"), }), + ProviderInstallationGate.layer, ), ), ); @@ -630,11 +642,13 @@ it.effect.each(PREFLIGHT_PRIMARY_STATES)("text generation preflight: %s", (prima message: "hello", linkedContext: "", modelSelection: selection, + projectId, }), ).toEqual({ title: "Generated title" }); expect(calls).toEqual([ primaryState === "ready" || primaryState === "pending" || + primaryState === "not installed, fallback disabled for project" || primaryState === "signed out, fallback failed" ? selection : { diff --git a/apps/server/src/textGeneration/CodexTextGeneration.ts b/apps/server/src/textGeneration/CodexTextGeneration.ts index 7b9191a7f81e..87027a8b9cf4 100644 --- a/apps/server/src/textGeneration/CodexTextGeneration.ts +++ b/apps/server/src/textGeneration/CodexTextGeneration.ts @@ -19,7 +19,11 @@ import * as ServerConfig from "../config.ts"; import { expandHomePath } from "@t3tools/provider-core/server/pathExpansion"; // Coder: provider input reads images only through PastedImageAttachments. import { resolvePastedImageAttachment } from "../provider/PastedImageAttachments.ts"; -import { codexExecLaunchArgs, resolveCodexLaunchArgs } from "../provider/codexLaunchArgs.ts"; +import { + codexExecLaunchArgs, + codexMcpListLaunchArgs, + resolveCodexLaunchArgs, +} from "../provider/codexLaunchArgs.ts"; import * as TextGenerationOperations from "@t3tools/provider-core/server/textGenerationOperations"; import { normalizeCliError, @@ -285,19 +289,17 @@ export const makeCodexTextGeneration = Effect.fn("makeCodexTextGeneration")(func } : {}), }; - const execLaunchArgs = codexExecLaunchArgs(launchArgs); const mcpServersOff = yield* listEnabledMcpServers({ binary, cwd, env, - // `codex mcp` rejects --strict-config; the overrides still apply. - launchArgs: execLaunchArgs.filter((arg) => arg !== "--strict-config"), + launchArgs: codexMcpListLaunchArgs(launchArgs), }).pipe(Effect.map(codexMcpServersOff)); const spawnCommand = yield* resolveSpawnCommand( binary, [ "exec", - ...execLaunchArgs, + ...codexExecLaunchArgs(launchArgs), // A user's MCP server could act on the text-generation prompt. ...CODEX_NO_PLUGIN_ARGS, ...(mcpServersOff ? ["--config", mcpServersOff] : []), diff --git a/apps/server/src/textGeneration/TextGeneration.ts b/apps/server/src/textGeneration/TextGeneration.ts index c17e22d50f00..f7e539a1db1b 100644 --- a/apps/server/src/textGeneration/TextGeneration.ts +++ b/apps/server/src/textGeneration/TextGeneration.ts @@ -1,7 +1,7 @@ import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; -import type { ModelSelection, ProviderInstanceId } from "@t3tools/contracts"; +import type { ModelSelection, ProjectId, ProviderInstanceId } from "@t3tools/contracts"; import { isProviderAvailable, isProviderTextGenerationCapable, @@ -12,6 +12,7 @@ import * as ServerSettings from "../serverSettings.ts"; import type { ProviderTextGeneration } from "@t3tools/provider-core/server/textGeneration"; import { TextGenerationError } from "@t3tools/contracts"; +import * as ProviderInstallationGate from "../provider/ProviderInstallationGate.ts"; import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts"; import type { ProviderInstance } from "@t3tools/provider-core/server/driver"; import * as SourceControlProviderRegistry from "../sourceControl/SourceControlProviderRegistry.ts"; @@ -67,9 +68,11 @@ export const make = Effect.gen(function* () { const registry = yield* ProviderInstanceRegistry.ProviderInstanceRegistry; const sourceControl = yield* SourceControlProviderRegistry.SourceControlProviderRegistry; const serverSettings = yield* ServerSettings.ServerSettingsService; + const installationGate = yield* ProviderInstallationGate.ProviderInstallationGate; const generate = Effect.fn("TextGeneration.generate")(function* ( operation: TextGenerationOp, selection: ModelSelection, + projectId: ProjectId | null | undefined, invoke: ( service: ProviderTextGeneration, modelSelection: ModelSelection, @@ -99,54 +102,66 @@ export const make = Effect.gen(function* () { ), ); selection = - fallbackTextGenerationModelSelection( - settings.providerInstances, - (instanceId, driver) => { - const provider = snapshots.find((candidate) => candidate.instanceId === instanceId); - return ( - instanceId !== selection.instanceId && - provider?.enabled === true && - resolveProjectProviderInstanceEnabled(settings, null, instanceId) && - provider.installed && - provider.auth.status !== "unauthenticated" && - provider.status !== "error" && - isProviderAvailable(provider) && - isProviderTextGenerationCapable(provider) && - (operation !== "generateReviewGuide" || INVESTIGATING_DRIVERS.has(driver)) - ); - }, - { includeConfiguredInstances: true }, - ) ?? selection; + fallbackTextGenerationModelSelection(settings.providerInstances, (instanceId, driver) => { + const provider = snapshots.find((candidate) => candidate.instanceId === instanceId); + return ( + instanceId !== selection.instanceId && + provider?.enabled === true && + resolveProjectProviderInstanceEnabled(settings, projectId ?? null, instanceId) && + provider.installed && + provider.auth.status !== "unauthenticated" && + provider.status !== "error" && + isProviderAvailable(provider) && + isProviderTextGenerationCapable(provider) && + (operation !== "generateReviewGuide" || INVESTIGATING_DRIVERS.has(driver)) + ); + }) ?? selection; } - return yield* resolveInstance(registry, operation, selection.instanceId).pipe( - Effect.flatMap((service) => invoke(service, selection)), + return yield* installationGate.execute( + selection.instanceId, + resolveInstance(registry, operation, selection.instanceId).pipe( + Effect.flatMap((service) => invoke(service, selection)), + ), ); }); return TextGeneration.of({ generateCommitMessage: Effect.fn("TextGeneration.generateCommitMessage")(function (input) { - return generate("generateCommitMessage", input.modelSelection, (service, modelSelection) => - service.generateCommitMessage({ ...input, modelSelection }), + return generate( + "generateCommitMessage", + input.modelSelection, + input.projectId, + (service, modelSelection) => service.generateCommitMessage({ ...input, modelSelection }), ); }), generatePrContent: Effect.fn("TextGeneration.generatePrContent")(function (input) { - return generate("generatePrContent", input.modelSelection, (service, modelSelection) => - service.generatePrContent({ ...input, modelSelection }), + return generate( + "generatePrContent", + input.modelSelection, + input.projectId, + (service, modelSelection) => service.generatePrContent({ ...input, modelSelection }), ); }), generateBranchName: Effect.fn("TextGeneration.generateBranchName")(function (input) { - return generate("generateBranchName", input.modelSelection, (service, modelSelection) => - service.generateBranchName({ ...input, modelSelection }), + return generate( + "generateBranchName", + input.modelSelection, + input.projectId, + (service, modelSelection) => service.generateBranchName({ ...input, modelSelection }), ); }), generateReviewGuide: Effect.fn("TextGeneration.generateReviewGuide")(function (input) { - return generate("generateReviewGuide", input.modelSelection, (service, modelSelection) => - service.generateReviewGuide({ ...input, modelSelection }), + return generate( + "generateReviewGuide", + input.modelSelection, + input.projectId, + (service, modelSelection) => service.generateReviewGuide({ ...input, modelSelection }), ); }), generateThreadTitle: Effect.fn("TextGeneration.generateThreadTitle")(function (input) { return generate( "generateThreadTitle", input.modelSelection, + input.projectId, (textGeneration, modelSelection) => Effect.gen(function* () { const linkedContext = diff --git a/apps/server/src/vcs/GitVcsDriver.test.ts b/apps/server/src/vcs/GitVcsDriver.test.ts index 7be13d395d26..b492c4738761 100644 --- a/apps/server/src/vcs/GitVcsDriver.test.ts +++ b/apps/server/src/vcs/GitVcsDriver.test.ts @@ -13,6 +13,7 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as PlatformError from "effect/PlatformError"; +import { ChildProcessSpawner } from "effect/process"; import { assert, it } from "@effect/vitest"; import { CheckpointRef, GitCommandError, VcsProcessExitError } from "@t3tools/contracts"; @@ -99,35 +100,54 @@ runVcsDriverContractSuite({ }, }); -it.effect("lists the fetch and push URLs of a partial clone remote", () => - Effect.gen(function* () { - const fileSystem = yield* FileSystem.FileSystem; - const cwd = yield* fileSystem.makeTempDirectoryScoped({ - prefix: "t3-partial-clone-remotes-", - }); - const driver = yield* GitVcsDriver.makeVcsDriverShape(); - yield* runGit(cwd, ["init"]); - yield* runGit(cwd, ["remote", "add", "origin", "https://github.com/pingdotgg/t3code.git"]); - yield* runGit(cwd, [ - "remote", - "set-url", - "--push", - "origin", - "git@github.com:octocat/t3code.git", - ]); - yield* runGit(cwd, ["config", "remote.origin.promisor", "true"]); - yield* runGit(cwd, ["config", "remote.origin.partialclonefilter", "blob:none"]); - - const result = yield* driver.listRemotes(cwd); - assert.deepEqual(result.remotes, [ - { - name: "origin", - url: "https://github.com/pingdotgg/t3code.git", - pushUrl: Option.some("git@github.com:octocat/t3code.git"), - isPrimary: true, - }, - ]); - }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +it.effect.each([ + { + fetchUrl: "https://github.com/pingdotgg/t3code.git", + pushUrl: "git@github.com:octocat/t3code.git", + lineEnding: "\n", + }, + { fetchUrl: "../team repos/shared.git", pushUrl: "../team repos/push.git", lineEnding: "\n" }, + { + fetchUrl: " ../team repos/shared (fetch) [blob:none].git ", + pushUrl: "../team repos/push (push).git\t", + lineEnding: "\r\n", + }, +])( + "lists the fetch and push URLs of a partial clone remote (%j)", + ({ fetchUrl, pushUrl, lineEnding }) => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-partial-clone-remotes-", + }); + const process = yield* VcsProcess.VcsProcess; + const driver = yield* GitVcsDriver.makeVcsDriverShape().pipe( + Effect.provideService(VcsProcess.VcsProcess, { + run: (input) => + process.run(input).pipe( + Effect.map((result) => ({ + ...result, + stdout: result.stdout.replace(/\r?\n/g, lineEnding), + })), + ), + }), + ); + yield* runGit(cwd, ["init"]); + yield* runGit(cwd, ["remote", "add", "origin", fetchUrl]); + yield* runGit(cwd, ["remote", "set-url", "--push", "origin", pushUrl]); + yield* runGit(cwd, ["config", "remote.origin.promisor", "true"]); + yield* runGit(cwd, ["config", "remote.origin.partialclonefilter", "blob:none"]); + + const result = yield* driver.listRemotes(cwd); + assert.deepEqual(result.remotes, [ + { + name: "origin", + url: fetchUrl, + pushUrl: Option.some(pushUrl), + isPrimary: true, + }, + ]); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), ); const makeCheckpointFixture = Effect.fn("makeCheckpointFixture")(function* ( @@ -403,6 +423,33 @@ it.effect("checkpoint capture skips an ignored project while keeping its allowed }).pipe(Effect.scoped, Effect.provide(layerGitContract)), ); +it.effect.each([ + { workspace: "the repository root", subdirectory: "" }, + { workspace: "a subdirectory", subdirectory: "sub" }, +])( + "checkpoint capture keeps new files in $workspace when the root .gitignore has a blank CRLF line", + ({ subdirectory }) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const parent = yield* fs.makeTempDirectoryScoped({ prefix: "t3-checkpoint-crlf-ignore-" }); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, parent); + yield* fs.writeFileString(path.join(parent, ".gitignore"), "a.txt\r\n\r\nb.txt\r\n"); + const cwd = path.join(parent, subdirectory); + yield* fs.makeDirectory(cwd, { recursive: true }); + yield* fs.writeFileString(path.join(cwd, "notes.md"), "notes\n"); + const notesPath = subdirectory === "" ? "notes.md" : `${subdirectory}/notes.md`; + + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + + assert.include( + (yield* git(["ls-tree", "-r", "--name-only", checkpointRef])).stdout.split("\n"), + notesPath, + ); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + it.effect.each([ { tracking: "committed-only", committed: ["committed.txt"], staged: [] }, { tracking: "staged-only", committed: [], staged: ["staged.txt"] }, @@ -1004,7 +1051,6 @@ it.effect.each(["committed", "unborn"] as const)( "rev-parse", "config", ...(head === "committed" ? ["read-tree", "ls-files"] : []), - "check-ignore", "add", "write-tree", "commit-tree", @@ -1057,13 +1103,13 @@ it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( const commands = yield* captureRecordingGitCommands(cwd, checkpointRef); assert.deepStrictEqual(commands, [ + "rev-parse", "rev-parse", "rev-parse", "config", "rev-parse", "read-tree", "ls-files", - "check-ignore", "add", "write-tree", "commit-tree", @@ -1287,4 +1333,126 @@ it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), ); +it.effect("GitVcsDriver fails repository probes when git cannot read the repository config", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-bad-config-" }); + // The included file's path appears in Git's message and must not read as "no repository". + const includeDir = path.join(root, "not a git repository"); + yield* fileSystem.makeDirectory(includeDir); + yield* fileSystem.writeFileString(path.join(includeDir, "included"), "invalid config\n"); + const configs = { + direct: "invalid config\n", + included: `[include]\n\tpath = ${path.join(includeDir, "included")}\n`, + }; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const checkpointRef = CheckpointRef.make("refs/t3/checkpoints/bad-config/0"); + + for (const [name, config] of Object.entries(configs)) { + const cwd = path.join(root, name); + yield* fileSystem.makeDirectory(cwd); + yield* runGit(cwd, ["init"]); + yield* fileSystem.writeFileString(path.join(cwd, ".git", "config"), config); + const outcomes = { + isInsideWorkTree: Exit.isFailure(yield* Effect.exit(driver.isInsideWorkTree(cwd))), + detectRepository: Exit.isFailure(yield* Effect.exit(driver.detectRepository(cwd))), + hasCheckpointRef: Exit.isFailure( + yield* Effect.exit(driver.checkpoints.hasCheckpointRef({ cwd, checkpointRef })), + ), + }; + assert.deepStrictEqual( + outcomes, + { isInsideWorkTree: true, detectRepository: true, hasCheckpointRef: true }, + `a bad ${name} config is an operational failure, not an absent repository or ref`, + ); + } + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect("GitVcsDriver reports no repository for a deleted cwd or a bare repository", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-no-work-tree-" }); + const bare = path.join(root, "bare.git"); + yield* fileSystem.makeDirectory(bare); + yield* runGit(bare, ["init", "--bare"]); + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + + for (const cwd of [path.join(root, "deleted"), bare]) { + assert.equal(yield* driver.isInsideWorkTree(cwd), false, cwd); + assert.equal(yield* driver.detectRepository(cwd), null, cwd); + } + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +// Git outputs CI's Git version cannot produce with a real repository. +const makeStubbedDriver = Effect.fn("test.makeStubbedDriver")(function* (output: { + readonly code: number; + readonly stdout: string; + readonly stderr: string; +}) { + const process = yield* VcsProcess.make.pipe( + Effect.provideService(ProcessRunner.ProcessRunner, { + run: () => + Effect.succeed({ + ...output, + code: ChildProcessSpawner.ExitCode(output.code), + timedOut: false, + stdoutTruncated: false, + stderrTruncated: false, + stdoutInvalidUtf8: false, + stderrInvalidUtf8: false, + }), + }), + ); + return yield* GitVcsDriver.makeVcsDriverShape().pipe( + Effect.provideService(VcsProcess.VcsProcess, process), + ); +}); + +it.effect("GitVcsDriver reports no repository when old Git answers false before failing", () => + Effect.gen(function* () { + // Git before 2.38 ignores safe.bareRepository, so a bare repo prints "false" and exits 128. + const driver = yield* makeStubbedDriver({ + code: 128, + stdout: "false\n", + stderr: "fatal: this operation must be run in a work tree\n", + }); + + assert.equal(yield* driver.detectRepository(globalThis.process.cwd()), null); + }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), +); + +it.effect("GitVcsDriver fails a checkpoint ref lookup when git cannot read its object", () => + Effect.gen(function* () { + const driver = yield* makeStubbedDriver({ + code: 1, + stdout: "", + stderr: "error: unable to open loose object 0733271782d6: Permission denied\n", + }); + const checkpointRef = CheckpointRef.make("refs/t3/checkpoints/unreadable/0"); + + const exit = yield* Effect.exit( + driver.checkpoints.hasCheckpointRef({ cwd: globalThis.process.cwd(), checkpointRef }), + ); + assert.isTrue(Exit.isFailure(exit), "an unreadable checkpoint object is not an absent ref"); + }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), +); + +it.effect("GitVcsDriver reads an absent checkpoint ref as absent despite a Git warning", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-git-warning-" }); + yield* runGit(cwd, ["init"]); + // Valid legacy config that makes every Git command print a deprecation warning. + yield* runGit(cwd, ["config", "core.fsyncObjectFiles", "true"]); + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const checkpointRef = CheckpointRef.make("refs/t3/checkpoints/absent/0"); + + assert.isFalse(yield* driver.checkpoints.hasCheckpointRef({ cwd, checkpointRef })); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + // Coder: upstream's Git command metric tests are not carried; the fork exports no metrics. diff --git a/apps/server/src/vcs/GitVcsDriver.ts b/apps/server/src/vcs/GitVcsDriver.ts index ab70b561d394..f066826a1874 100644 --- a/apps/server/src/vcs/GitVcsDriver.ts +++ b/apps/server/src/vcs/GitVcsDriver.ts @@ -56,7 +56,6 @@ export interface ExecuteGitInput { readonly env?: NodeJS.ProcessEnv; readonly allowNonZeroExit?: boolean; readonly timeoutMs?: number | null; - readonly forceKillAfterMs?: number; readonly maxOutputBytes?: number; readonly appendTruncationMarker?: boolean; /** With appendTruncationMarker, line callbacks continue beyond the buffer cap for progress-only consumers. */ @@ -183,6 +182,11 @@ export interface GitPushResult { setUpstream?: boolean | undefined; } +export interface GitPushTarget { + remoteName: string; + branchName: string; +} + export interface GitRangeContext { commitSummary: string; diffSummary: string; @@ -348,6 +352,11 @@ export class GitVcsDriver extends Context.Service< fallbackBranch: string | null, options?: { readonly remoteName?: string | null }, ) => Effect.Effect; + /** Where `pushCurrentBranch` would push the checked-out branch; null with no remote or when Git picks. */ + readonly resolvePushTarget: ( + cwd: string, + current: { readonly branch: string; readonly hasUpstream: boolean }, + ) => Effect.Effect; readonly readRangeContext: ( cwd: string, baseRef: string, @@ -519,14 +528,10 @@ function parseGitRemoteVerboseOutput( output: string, ): Map { const remotes = new Map(); - for (const line of output.split("\n")) { - const trimmed = line.trim(); - if (trimmed.length === 0) { - continue; - } - + for (const line of output.split(/\r?\n/)) { + // Git uses a tab before the URL and one space before the direction; both can occur in URLs. // Partial clones append the promisor filter, e.g. `(fetch) [blob:none]`. - const match = /^(\S+)\s+(\S+)\s+\((fetch|push)\)(?:\s+\[[^\]]+\])?$/.exec(trimmed); + const match = /^(\S+)\t(.+) \((fetch|push)\)(?:\s+\[[^\]]+\])?$/.exec(line); if (!match) { continue; } @@ -548,6 +553,13 @@ function parseGitRemoteVerboseOutput( return remotes; } +// Trace output and deprecation warnings can precede Git's own message, so check every line. +const hasStderrLine = (stderr: string, prefixes: ReadonlyArray) => + stderr.split("\n").some((line) => prefixes.some((prefix) => line.startsWith(prefix))); + +/** Operation of the error raised when Git cannot say whether a cwd is a work tree. */ +export const WORK_TREE_PROBE_OPERATION = "GitVcsDriver.probeWorkTree"; + const gitCommand = Effect.fn("GitVcsDriver.gitCommand")( ( process: VcsProcess.VcsProcess["Service"], @@ -590,21 +602,50 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( const vcsProcess = yield* VcsProcess.VcsProcess; const random = yield* SecureRandom.SecureRandom; const hostPlatform = yield* HostProcess.Platform; + // Returns stdout, or null when there is no work tree. A non-repository exits 128 like a broken + // one, so only Git's message (pinned to the C locale) or a missing cwd tells them apart. + const probeWorkTree = Effect.fn("GitVcsDriver.probeWorkTree")(function* ( + operation: string, + cwd: string, + args: ReadonlyArray, + maxOutputBytes: number, + ) { + const result = yield* gitCommand(vcsProcess, operation, cwd, args, { + env: { LC_ALL: "C" }, + allowNonZeroExit: true, + timeoutMs: 5_000, + maxOutputBytes, + }).pipe(Effect.catchIf(isMissingExecutable, () => Effect.succeed(null))); + if (result === null) return null; + if (result.exitCode === 0) return result.stdout; + // Git before 2.38 answers "false" for a bare repo, then fails on --show-toplevel. + // Match line prefixes only: a path inside a diagnostic may contain the same words. + if ( + result.stdout.startsWith("false") || + hasStderrLine(result.stderr, [ + "fatal: not a git repository", + "fatal: cannot use bare repository", + ]) || + !(yield* fileSystem.exists(cwd).pipe(Effect.orElseSucceed(() => true))) + ) { + return null; + } + return yield* new VcsProcessExitError({ + operation: WORK_TREE_PROBE_OPERATION, + command: "git rev-parse", + cwd, + exitCode: result.exitCode, + detail: result.stderr.trim() || "git rev-parse failed", + }); + }); + const isInsideWorkTree: VcsDriver.VcsDriver["Service"]["isInsideWorkTree"] = (cwd) => - gitCommand( - vcsProcess, + probeWorkTree( "GitVcsDriver.isInsideWorkTree", cwd, ["rev-parse", "--is-inside-work-tree"], - { - allowNonZeroExit: true, - timeoutMs: 5_000, - maxOutputBytes: 4_096, - }, - ).pipe( - Effect.map((result) => result.exitCode === 0 && result.stdout.trim() === "true"), - Effect.catchIf(isMissingExecutable, () => Effect.succeed(false)), - ); + 4_096, + ).pipe(Effect.map((stdout) => stdout?.trim() === "true")); const execute: VcsDriver.VcsDriver["Service"]["execute"] = (input) => gitCommand(vcsProcess, input.operation, input.cwd, input.args, { @@ -623,19 +664,14 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( "detectRepository", )(function* (cwd) { // One spawn answers all three; outside a work tree it exits non-zero or prints "false". - const result = yield* gitCommand( - vcsProcess, + const stdout = yield* probeWorkTree( "GitVcsDriver.detectRepository", cwd, ["rev-parse", "--is-inside-work-tree", "--show-toplevel", "--git-common-dir"], - { - allowNonZeroExit: true, - timeoutMs: 5_000, - maxOutputBytes: 16_384, - }, - ).pipe(Effect.catchIf(isMissingExecutable, () => Effect.succeed(null))); - if (result === null || result.exitCode !== 0) return null; - const lines = result.stdout.replace(/\r?\n$/u, "").split(/\r?\n/u); + 16_384, + ); + if (stdout === null) return null; + const lines = stdout.replace(/\r?\n$/u, "").split(/\r?\n/u); if (lines[0]?.trim() !== "true") return null; // A newline inside a path breaks the line framing; ask for each path on its own. if (lines.length !== 3) { @@ -834,14 +870,28 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( operation: "GitVcsDriver.checkpoints.resolveCheckpointCommit", cwd, args: ["rev-parse", "--verify", "--quiet", `${checkpointRef}^{commit}`], + env: { LC_ALL: "C" }, allowNonZeroExit: true, }).pipe( - Effect.map((result) => { + Effect.flatMap((result) => { + // --quiet reports an absent ref as exit 1 with no error; an unreadable object also exits 1 + // but reports an error line. + if (result.exitCode === 1 && !hasStderrLine(result.stderr, ["error:", "fatal:"])) { + return Effect.succeed(null); + } if (result.exitCode !== 0) { - return null; + return Effect.fail( + new VcsProcessExitError({ + operation: "GitVcsDriver.checkpoints.resolveCheckpointCommit", + command: "git rev-parse", + cwd, + exitCode: result.exitCode, + detail: result.stderr.trim() || "git rev-parse failed", + }), + ); } const commit = result.stdout.trim(); - return commit.length > 0 ? commit : null; + return Effect.succeed(commit.length > 0 ? commit : null); }), ); @@ -873,6 +923,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( "--git-common-dir", "--git-path", "index", + "--show-prefix", "--verify", "--quiet", "HEAD", @@ -882,19 +933,27 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( const headExists = result.exitCode === 0; const headChecked = headExists || result.exitCode === 1; const lines = result.stdout.split("\n"); - const [gitCommonDir = "", indexPath = ""] = lines; - if (headChecked && lines.length === (headExists ? 4 : 3) && lines.at(-1) === "") { + const [gitCommonDir = "", indexPath = "", prefix = ""] = lines; + if (headChecked && lines.length === (headExists ? 5 : 4) && lines.at(-1) === "") { return { gitCommonDir: gitCommonDir.trim(), indexPath: indexPath.trim(), + prefix, headExists, }; } // Git before 2.31 echoes --path-format, and a path containing a newline spans lines. // Separate commands read the paths as before and report a failure the same way. + const fallbackCommonDir = yield* resolveGitCommonDir(cwd); + const prefixResult = yield* execute({ + operation: VcsProcess.CHECKPOINT_CAPTURE_OPERATION, + cwd, + args: ["rev-parse", "--show-prefix"], + }); return { - gitCommonDir: yield* resolveGitCommonDir(cwd), + gitCommonDir: fallbackCommonDir, indexPath: undefined, + prefix: prefixResult.stdout.replace(/\n$/, ""), headExists: headChecked ? headExists : yield* hasHeadCommit(cwd), }; }); @@ -931,7 +990,9 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( "-c", "splitIndex.sharedIndexExpire=never", ]; - const { gitCommonDir, indexPath, headExists } = yield* resolveCaptureRepository(input.cwd); + const { gitCommonDir, indexPath, prefix, headExists } = yield* resolveCaptureRepository( + input.cwd, + ); const indexId = yield* random.uuidV4(); const tempIndexPath = path.join(gitCommonDir, `t3-checkpoint-index-${indexId}`); const hostEnv = yield* HostProcess.Environment; @@ -1125,7 +1186,12 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( } } - const workspaceIgnored = (yield* filterIgnoredPaths(input.cwd, ["."])).length === 0; + // The working-tree root is never ignored. Probe a subdirectory without the trailing slash + // Git adds to ".", which a blank or whitespace-only ignore line matches as an empty pattern. + const workspaceIgnored = + prefix !== "" && + (yield* filterIgnoredPaths(input.cwd, [`../${prefix.slice(0, -1).split("/").at(-1)}`])) + .length === 0; if (workspaceIgnored) { // HEAD omits staged-only paths. Seed just this workspace's additions, keeping // unrelated staged changes and the scratch index's sparse flags untouched. diff --git a/apps/server/src/vcs/GitVcsDriverCore.test.ts b/apps/server/src/vcs/GitVcsDriverCore.test.ts index cf01e452351e..83c3f67abfb0 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.test.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.test.ts @@ -13,6 +13,7 @@ import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as Queue from "effect/Queue"; import * as PlatformError from "effect/PlatformError"; @@ -23,6 +24,7 @@ import * as Schema from "effect/Schema"; import * as Sink from "effect/Sink"; import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; +import * as Tracer from "effect/Tracer"; import { ChildProcess, ChildProcessSpawner, CommandLine } from "effect/process"; import { @@ -37,6 +39,7 @@ import { import { symlinksSupported } from "@t3tools/shared/testing/symlinks"; import * as ServerConfig from "../config.ts"; import { + isNonRepositoryGitStderr, makeGitVcsDriverCore, parseGitCheckoutProgressLine, splitNullSeparatedGitStdoutPaths, @@ -130,6 +133,27 @@ const makeDriverReleasingLock = ( return { driver, failures }; }); +describe("isNonRepositoryGitStderr", () => { + it.each([ + "fatal: not a git repository (or any of the parent directories): .git", + "fatal: not a git repository: /missing-git-directory", + // Git 2.56+ wording for a `.git` gitfile that points at a missing directory. + "fatal: gitfile does not point to a valid repository: /tmp/worktrees/feature/.git", + "FATAL: GITFILE DOES NOT POINT TO A VALID REPOSITORY: /tmp/x/.git", + ])("classifies %j as a non-repository", (stderr) => { + assert.isTrue(isNonRepositoryGitStderr(stderr)); + }); + + it.each([ + "", + "fatal: ambiguous argument 'HEAD': unknown revision or path not in the working tree.", + "fatal: Unable to create '/repo/.git/index.lock': File exists.", + "fatal: invalid gitfile format: /repo/.git", + ])("does not classify %j as a non-repository", (stderr) => { + assert.isFalse(isNonRepositoryGitStderr(stderr)); + }); +}); + it.effect("runs background Git in its own lane so user commands never queue behind it", () => Effect.gen(function* () { const gate = yield* Deferred.make(); @@ -610,7 +634,7 @@ it.effect.each(["bare", "unborn"] as const)( assert.equal(Result.isSuccess(result), state === "bare"); if (Result.isFailure(result)) { assert.equal(result.failure.operation, "GitVcsDriver.createWorktree.createBranch"); - assert.equal(result.failure.detail, "git worktree add failed"); + assert.equal(result.failure.detail, "git branch failed"); } assert.equal(spawnedArgs.filter((args) => args.includes("--git-common-dir")).length, 1); assert.isFalse(spawnedArgs.some((args) => args[0] === "symbolic-ref")); @@ -1805,6 +1829,45 @@ describe("rejected git refs", () => { }); it.layer(layerTest)("GitVcsDriver core integration", (it) => { + // regression: #26577 a timed-out git that outlives its timeout holds a process permit + it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( + "kills a timed-out git command that ignores SIGTERM", + () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const pathService = yield* Path.Path; + const fileSystem = yield* FileSystem.FileSystem; + const markerPath = pathService.join(yield* makeTmpDir(), "outlived-timeout"); + const driver = yield* GitVcsDriver.GitVcsDriver; + const trapped = yield* Deferred.make(); + + const command = yield* driver + .execute({ + operation: "GitVcsDriver.test.ignoresSigterm", + cwd, + args: [ + "-c", + `alias.hang=!trap '' TERM; echo trapped; sleep 3; touch '${markerPath}'`, + "hang", + ], + progress: { + onStdoutLine: (line) => + line === "trapped" ? Deferred.succeed(trapped, undefined) : Effect.void, + }, + }) + .pipe(Effect.forkChild); + yield* Deferred.await(trapped); + // The driver's timeout ends a command the same way: by interrupting it. + yield* Fiber.interrupt(command); + + assert.isFalse( + yield* fileSystem.exists(markerPath), + "the timed-out git command kept running until it finished on its own", + ); + }), + ); + it.effect("keeps large change sets available for status and commit preparation", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -1935,7 +1998,7 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { it.effect.each([ { action: "worktree add", detail: "git worktree add failed" }, - { action: "worktree add -b", detail: "git worktree add failed" }, + { action: "worktree add -b", detail: "git branch failed" }, { action: "switch", detail: "git checkout failed" }, ])( "reports a missing ref during $action with stable, redacted diagnostics", @@ -1984,6 +2047,54 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("lets a slow remote branch checkout finish without blocking config writes", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const remote = yield* makeTmpDir("git-remote-"); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["push", "origin", "HEAD:refs/heads/feature"]); + yield* git(cwd, ["fetch", "origin"]); + const checkoutStarted = yield* Deferred.make(); + const releaseCheckout = yield* Deferred.make(); + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + // Holds the real checkout open like a slow LFS smudge filter would. + const spawner = ChildProcessSpawner.make((command) => + Effect.gen(function* () { + const handle = yield* delegate.spawn(command); + if (!ChildProcess.isStandardCommand(command) || command.args[0] !== "checkout") { + return handle; + } + yield* Deferred.succeed(checkoutStarted, undefined); + return ChildProcessSpawner.makeHandle({ + ...handle, + exitCode: Deferred.await(releaseCheckout).pipe(Effect.andThen(handle.exitCode)), + }); + }), + ); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provide(layerServerConfig), + ); + + const switching = yield* driver + .switchRef({ cwd, refName: "origin/feature" }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(checkoutStarted); + yield* TestClock.adjust("60 seconds"); + yield* driver.createRef({ cwd, refName: "other" }); + yield* Deferred.succeed(releaseCheckout, undefined); + + const result = yield* Fiber.join(switching); + assert.equal(result.refName, "feature"); + assert.equal( + yield* git(cwd, ["rev-parse", "--abbrev-ref", "feature@{upstream}"]), + "origin/feature", + ); + }), + ); + it.effect.each([ // Git cannot tell a missing revision from a missing file in these. { label: "an argument that may be a path", args: ["log", "-1", "missing"] }, @@ -4006,6 +4117,44 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("records no failed span reading status of a repository without a remote", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + yield* git(cwd, ["checkout", "-b", "feature"]); + const driver = yield* GitVcsDriver.GitVcsDriver; + const spans: Array = []; + const tracer = Tracer.make({ + span: (options) => { + const span = new Tracer.NativeSpan(options); + spans.push(span); + return span; + }, + }); + const traced = (effect: Effect.Effect) => + Effect.gen(function* () { + spans.length = 0; + const exit = yield* effect.pipe(Effect.withTracer(tracer), Effect.exit); + const failedSpans = spans.flatMap((span) => + span.status._tag === "Ended" && Exit.isFailure(span.status.exit) ? [span.name] : [], + ); + return { exit, failedSpans }; + }); + + const status = yield* traced(driver.statusDetailsRemote(cwd)); + assert.isTrue(Exit.isSuccess(status.exit)); + assert.deepStrictEqual(status.failedSpans, []); + + // A caller that needs a remote still fails, in its own span. + const fetch = yield* traced(driver.fetchRefCommit({ cwd, ref: "refs/pull/1/head" })); + assert.include( + Exit.isFailure(fetch.exit) ? Cause.pretty(fetch.exit.cause) : "", + "No git remote is configured for this repository.", + ); + assert.deepStrictEqual(fetch.failedSpans, ["GitVcsDriver.fetchRefCommit"]); + }), + ); + it.effect("can read cached remote divergence without fetching upstream", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -4658,6 +4807,79 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.isTrue(Result.isSuccess(result), "worktree creation failed in the hook"); }), ); + it.effect("refreshes a new worktree's index once its checkout second is over", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.GitVcsDriver; + for (let file = 0; file < 50; file++) { + yield* writeTextFile(cwd, `src/file-${file}.txt`, `contents ${file}\n`); + } + yield* git(cwd, ["add", "."]); + yield* git(cwd, ["commit", "-m", "more files"]); + const worktreePath = path.join(yield* makeTmpDir("git-worktrees-"), "fresh"); + + // The refresh waits on the clock that stamps the files. + yield* TestClock.withLive( + driver.createWorktree({ + cwd, + path: worktreePath, + refName: initialBranch, + newRefName: "feature/fresh", + }), + ); + + // Git re-reads every entry stamped in the index's own second or later. + const indexPath = path.resolve( + worktreePath, + yield* git(worktreePath, ["rev-parse", "--git-path", "index"]), + ); + const indexMtime = Option.getOrThrow((yield* fs.stat(indexPath)).mtime); + const indexSecond = Math.floor(indexMtime.getTime() / 1000); + const entrySeconds = Array.from( + (yield* git(worktreePath, ["ls-files", "--debug"])).matchAll(/^ {2}mtime: (\d+):/gm), + ([, seconds]) => Number(seconds), + ); + assert.equal(entrySeconds.length, 51); + assert.deepEqual( + entrySeconds.filter((seconds) => seconds >= indexSecond), + [], + ); + }), + ); + it.effect("returns the worktree when its index refresh fails", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.GitVcsDriver; + // Something else takes the new worktree's index lock as soon as its checkout ends. + yield* writeTextFile( + cwd, + ".git/hooks/post-checkout", + '#!/bin/sh\ntouch "$(git rev-parse --git-path index).lock"\n', + ); + yield* fs.chmod(path.join(cwd, ".git/hooks/post-checkout"), 0o755); + const worktreePath = path.join(yield* makeTmpDir("git-worktrees-"), "locked"); + + const created = yield* driver.createWorktree({ + cwd, + path: worktreePath, + refName: initialBranch, + newRefName: "feature/locked", + }); + + assert.equal(created.worktree.path, worktreePath); + const indexPath = path.resolve( + worktreePath, + yield* git(worktreePath, ["rev-parse", "--git-path", "index"]), + ); + assert.isTrue(yield* fs.exists(`${indexPath}.lock`)); + }), + ); it.effect.each(["commit", "short-commit", "tag", "revision"] as const)( "creates a worktree from a %s without recording it as a PR base", (kind) => @@ -4693,6 +4915,28 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("explains a new branch that collides with an existing branch namespace", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + yield* git(cwd, ["branch", "werkstatt"]); + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.GitVcsDriver; + const error = yield* driver + .createWorktree({ + cwd, + path: path.join(yield* makeTmpDir("git-worktrees-"), "conflict"), + refName: initialBranch, + newRefName: "werkstatt/baseline", + }) + .pipe(Effect.flip); + + assert.equal(error._tag, "GitCommandError"); + assert.equal(error.operation, "GitVcsDriver.createWorktree.createBranch"); + assert.equal(error.reason, "ref_name_conflict"); + }), + ); + // regression: #28354 a failed worktree add left its new branch behind and blocked retries it.effect("removes the new branch when the worktree cannot be added", () => Effect.gen(function* () { @@ -5499,6 +5743,61 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect.each(["\n", "\r\n"])( + "ensureRemote preserves spaces in fetch URLs and reserves their names (%j)", + (lineEnding) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const fetchUrl = " ../team repos/shared (fetch) [blob:none].git "; + const pushUrl = "../team repos/push (push).git"; + yield* git(cwd, ["init"]); + yield* git(cwd, ["remote", "add", "origin", fetchUrl]); + yield* git(cwd, ["remote", "set-url", "--push", "origin", pushUrl]); + yield* git(cwd, ["config", "remote.origin.promisor", "true"]); + yield* git(cwd, ["config", "remote.origin.partialclonefilter", "blob:none"]); + const liveDriver = yield* GitVcsDriver.GitVcsDriver; + const listing = yield* liveDriver.execute({ + cwd, + args: ["remote", "-v"], + operation: "test", + }); + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const spawner = ChildProcessSpawner.make((command) => + ChildProcess.isStandardCommand(command) && + command.args[0] === "remote" && + command.args[1] === "-v" + ? Effect.succeed(makeSuccessfulHandle(listing.stdout.replace(/\r?\n/g, lineEnding))) + : delegate.spawn(command), + ); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provide(layerServerConfig), + ); + + assert.equal( + yield* driver.ensureRemote({ cwd, preferredName: "origin", url: fetchUrl }), + "origin", + ); + assert.equal( + yield* driver.ensureRemote({ cwd, preferredName: "origin", url: pushUrl }), + "origin-1", + ); + const fetch = yield* liveDriver.execute({ + cwd, + args: ["config", "--get", "remote.origin.url"], + operation: "test", + }); + const push = yield* liveDriver.execute({ + cwd, + args: ["config", "--get", "remote.origin.pushurl"], + operation: "test", + }); + assert.equal(fetch.stdout, `${fetchUrl}\n`); + assert.equal(push.stdout, `${pushUrl}\n`); + assert.equal(yield* git(cwd, ["config", "--get", "remote.origin-1.url"]), pushUrl); + }), + ); + it.effect("ensureRemote returns the remote a parallel call added for the same URL", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -5726,6 +6025,60 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }); describe("commit context", () => { + it.effect("reads history through a chain of shared clones", () => + Effect.gen(function* () { + const root = yield* makeTmpDir(); + const fileSystem = yield* FileSystem.FileSystem; + const driver = yield* GitVcsDriver.GitVcsDriver; + const seed = `${root}/clone-0`; + yield* fileSystem.makeDirectory(seed); + yield* initRepoWithCommit(seed); + let previous = seed; + for (let index = 1; index <= 6; index += 1) { + const next = `${root}/clone-${index}`; + yield* git(root, ["clone", "--quiet", "--shared", previous, next]); + previous = next; + } + // Six shared clones deep is within Git's alternate limit, not one level deeper. + yield* writeTextFile(previous, "README.md", "changed\n"); + const chained = yield* driver.prepareCommitContext(previous, ["README.md"]); + assert.include(chained?.stagedSummary ?? "", "M\tREADME.md"); + assert.include(chained?.stagedPatch ?? "", "-# test"); + }), + ); + + it.effect("leaves no objects behind when no commit follows, in a linked worktree too", () => + Effect.gen(function* () { + const repo = yield* makeTmpDir(); + yield* initRepoWithCommit(repo); + const linked = `${repo}/linked`; + yield* git(repo, ["worktree", "add", "--quiet", linked]); + const driver = yield* GitVcsDriver.GitVcsDriver; + + for (const cwd of [repo, linked]) { + yield* writeTextFile(cwd, "README.md", `changed in ${cwd}\n`); + yield* writeTextFile(cwd, "draft.txt", `drafted in ${cwd}\n`); + const blobs = [ + yield* git(cwd, ["hash-object", "draft.txt"]), + yield* git(cwd, ["hash-object", "README.md"]), + ]; + const context = yield* driver.prepareCommitContext(cwd); + assert.include(context?.stagedPatch ?? "", `+drafted in ${cwd}`); + assert.include(context?.stagedPatch ?? "", "-# test"); + for (const blob of blobs) { + const probe = yield* driver.execute({ + operation: "GitVcsDriver.test.git", + cwd, + args: ["cat-file", "-e", blob], + allowNonZeroExit: true, + timeoutMs: 10_000, + }); + assert.notEqual(probe.exitCode, 0, `${blob} stayed in the repository`); + } + } + }), + ); + it.effect("prepares source patches without running textconv", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -6375,42 +6728,67 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); - it.effect("pushes with upstream setup and skips when already up to date", () => - Effect.gen(function* () { - const cwd = yield* makeTmpDir(); - const remote = yield* makeTmpDir("git-remote-"); - yield* initRepoWithCommit(cwd); - yield* git(remote, ["init", "--bare"]); - yield* git(cwd, ["remote", "add", "origin", remote]); - yield* (yield* GitVcsDriver.GitVcsDriver).createRef({ - cwd, - refName: "feature/push", - }); - yield* (yield* GitVcsDriver.GitVcsDriver).switchRef({ - cwd, - refName: "feature/push", - }); - yield* writeTextFile(cwd, "feature.txt", "feature\n"); - yield* git(cwd, ["add", "-A"]); - yield* (yield* GitVcsDriver.GitVcsDriver).commit(cwd, "Add feature", ""); + it.effect.each([false, true])( + "pushes with upstream setup and skips when already up to date (autoSetupMerge=%s)", + (autoSetupMerge) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const remote = yield* makeTmpDir("git-remote-"); + const { initialBranch } = yield* initRepoWithCommit(cwd); + if (autoSetupMerge) { + yield* git(cwd, ["config", "branch.autoSetupMerge", "always"]); + } + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* (yield* GitVcsDriver.GitVcsDriver).createRef({ + cwd, + refName: "feature/push", + }); + yield* (yield* GitVcsDriver.GitVcsDriver).switchRef({ + cwd, + refName: "feature/push", + }); + yield* writeTextFile(cwd, "feature.txt", "feature\n"); + yield* git(cwd, ["add", "-A"]); + yield* (yield* GitVcsDriver.GitVcsDriver).commit(cwd, "Add feature", ""); - const pushed = yield* (yield* GitVcsDriver.GitVcsDriver).pushCurrentBranch(cwd, null); - assert.deepInclude(pushed, { - status: "pushed", - branch: "feature/push", - setUpstream: true, - }); - assert.equal( - yield* git(cwd, ["rev-parse", "--abbrev-ref", "@{upstream}"]), - "origin/feature/push", - ); + const driver = yield* GitVcsDriver.GitVcsDriver; + if (autoSetupMerge) { + assert.equal(yield* driver.readConfigValue(cwd, "branch.feature/push.remote"), "."); + assert.equal( + yield* driver.readConfigValue(cwd, "branch.feature/push.merge"), + `refs/heads/${initialBranch}`, + ); + } - const skipped = yield* (yield* GitVcsDriver.GitVcsDriver).pushCurrentBranch(cwd, null); - assert.deepInclude(skipped, { - status: "skipped_up_to_date", - branch: "feature/push", - }); - }), + const pushed = yield* (yield* GitVcsDriver.GitVcsDriver).pushCurrentBranch(cwd, null); + assert.deepInclude(pushed, { + status: "pushed", + branch: "feature/push", + setUpstream: true, + }); + assert.equal( + yield* git(cwd, ["rev-parse", "--abbrev-ref", "@{upstream}"]), + "origin/feature/push", + ); + + if (autoSetupMerge) { + assert.equal( + yield* driver.readConfigValue(cwd, "branch.feature/push.t3-merge-base"), + initialBranch, + ); + assert.equal( + yield* driver.readConfigValue(cwd, "branch.feature/push.t3-merge-base-ref"), + `refs/heads/${initialBranch}`, + ); + } + + const skipped = yield* (yield* GitVcsDriver.GitVcsDriver).pushCurrentBranch(cwd, null); + assert.deepInclude(skipped, { + status: "skipped_up_to_date", + branch: "feature/push", + }); + }), ); it.effect("rebases a pushed branch's pulls when branch.autoSetupRebase asks for it", () => @@ -6558,62 +6936,170 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); - it.effect("publishes a branch tracking its base under its own name, not the base", () => - Effect.gen(function* () { - const cwd = yield* makeTmpDir(); - const remote = yield* makeTmpDir("git-remote-"); - const upstream = yield* makeTmpDir("git-upstream-"); - yield* initRepoWithCommit(cwd); - const driver = yield* GitVcsDriver.GitVcsDriver; - yield* git(cwd, ["branch", "-M", "main"]); - yield* git(remote, ["init", "--bare"]); - yield* git(cwd, ["remote", "add", "origin", remote]); - yield* git(cwd, ["push", "-u", "origin", "main"]); - yield* git(cwd, ["checkout", "-b", "dev"]); - yield* git(cwd, ["push", "-u", "origin", "dev"]); - const devSha = yield* git(cwd, ["rev-parse", "HEAD"]); - yield* git(upstream, ["init", "--bare"]); - yield* git(cwd, ["remote", "add", "upstream", upstream]); - yield* writeTextFile(cwd, "upstream.txt", "upstream work\n"); - yield* git(cwd, ["add", "-A"]); - yield* driver.commit(cwd, "Add upstream update", ""); - yield* git(cwd, ["push", "upstream", "dev"]); - const upstreamDevSha = yield* git(cwd, ["rev-parse", "HEAD"]); - yield* git(cwd, ["checkout", "-b", "feature/x", "upstream/dev"]); - yield* writeTextFile(cwd, "feature.txt", "feature\n"); - yield* git(cwd, ["add", "-A"]); - yield* driver.commit(cwd, "Add feature", ""); + it.effect.each([ + { baseBranch: "dev", local: false }, + { baseBranch: "dev", local: true }, + { baseBranch: "feature/base", local: true }, + { baseBranch: "upstream", local: true }, + { baseBranch: "upstream/feature/x", local: true }, + ])( + "publishes a branch tracking $baseBranch (local=$local) under its own name, not the base", + ({ baseBranch, local }) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const remote = yield* makeTmpDir("git-remote-"); + const upstream = yield* makeTmpDir("git-upstream-"); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* git(cwd, ["branch", "-M", "main"]); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["push", "-u", "origin", "main"]); + yield* git(cwd, ["checkout", "--no-track", "-b", baseBranch]); + yield* git(cwd, ["push", "-u", "origin", baseBranch]); + const devSha = yield* git(cwd, ["rev-parse", "HEAD"]); + yield* git(upstream, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "upstream", upstream]); + yield* writeTextFile(cwd, "upstream.txt", "upstream work\n"); + yield* git(cwd, ["add", "-A"]); + yield* driver.commit(cwd, "Add upstream update", ""); + yield* git(cwd, ["push", "upstream", baseBranch]); + const upstreamDevSha = yield* git(cwd, ["rev-parse", "HEAD"]); + yield* git(cwd, [ + "checkout", + "--track", + "-b", + "feature/x", + local ? `refs/heads/${baseBranch}` : `refs/remotes/upstream/${baseBranch}`, + ]); + assert.equal( + yield* driver.readConfigValue(cwd, "branch.feature/x.remote"), + local ? "." : "upstream", + ); + yield* writeTextFile(cwd, "feature.txt", "feature\n"); + yield* git(cwd, ["add", "-A"]); + yield* driver.commit(cwd, "Add feature", ""); - const pushed = yield* driver.pushCurrentBranch(cwd, null); + const pushed = yield* driver.pushCurrentBranch(cwd, null); - assert.deepInclude(pushed, { - status: "pushed", - branch: "feature/x", - upstreamBranch: "origin/feature/x", - setUpstream: true, - }); - assert.equal( - yield* git(remote, ["--git-dir=.", "log", "-1", "--pretty=%s", "feature/x"]), - "Add feature", - ); - assert.equal(yield* git(remote, ["--git-dir=.", "rev-parse", "dev"]), devSha); - assert.equal(yield* git(upstream, ["--git-dir=.", "rev-parse", "dev"]), upstreamDevSha); - assert.equal( - yield* git(cwd, ["rev-parse", "--abbrev-ref", "@{upstream}"]), - "origin/feature/x", - ); - assert.equal(yield* driver.readConfigValue(cwd, "branch.feature/x.t3-merge-base"), "dev"); + assert.deepInclude(pushed, { + status: "pushed", + branch: "feature/x", + upstreamBranch: "origin/feature/x", + setUpstream: true, + }); + assert.equal( + yield* git(remote, ["--git-dir=.", "log", "-1", "--pretty=%s", "feature/x"]), + "Add feature", + ); + assert.equal( + yield* git(remote, ["--git-dir=.", "rev-parse", `refs/heads/${baseBranch}`]), + devSha, + ); + assert.equal( + yield* git(upstream, ["--git-dir=.", "rev-parse", `refs/heads/${baseBranch}`]), + upstreamDevSha, + ); + assert.equal( + yield* git(cwd, ["rev-parse", "--abbrev-ref", "@{upstream}"]), + "origin/feature/x", + ); + assert.equal( + yield* driver.readConfigValue(cwd, "branch.feature/x.t3-merge-base"), + baseBranch, + ); + assert.equal( + yield* driver.readConfigValue(cwd, "branch.feature/x.t3-merge-base-ref"), + local ? `refs/heads/${baseBranch}` : `refs/remotes/upstream/${baseBranch}`, + ); + assert.equal(yield* git(cwd, ["rev-parse", `refs/heads/${baseBranch}`]), upstreamDevSha); - const preview = yield* driver.getReviewDiffPreview({ cwd }); - const changes = preview.sources.find((source) => source.kind === "branch-range")!; - assert.deepStrictEqual(changes.files, [ - { path: "feature.txt", previousPath: null, additions: 1, deletions: 0 }, - ]); - assert.equal(changes.baseRef, "upstream/dev"); - }), + const preview = yield* driver.getReviewDiffPreview({ cwd }); + const changes = preview.sources.find((source) => source.kind === "branch-range")!; + assert.deepStrictEqual(changes.files, [ + { path: "feature.txt", previousPath: null, additions: 1, deletions: 0 }, + ]); + assert.equal( + changes.baseRef, + local ? `refs/heads/${baseBranch}` : `upstream/${baseBranch}`, + ); + }), ); - it.effect("keeps a recorded merge base when publishing a tracked branch", () => + it.effect.each([ + { upstreamRef: "refs/remotes/origin/main", baseKey: "gh-merge-base" }, + { upstreamRef: "refs/heads/main", baseKey: "gh-merge-base" }, + { upstreamRef: "refs/heads/main", baseKey: "t3-merge-base" }, + ])( + "keeps a recorded $baseKey when publishing a branch tracking $upstreamRef", + ({ upstreamRef, baseKey }) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const remote = yield* makeTmpDir("git-remote-"); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* git(cwd, ["branch", "-M", "main"]); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["push", "-u", "origin", "main"]); + yield* git(cwd, ["checkout", "--track", "-b", "feature/y", upstreamRef]); + yield* git(cwd, ["config", `branch.feature/y.${baseKey}`, "release/v2"]); + yield* git(cwd, [ + "config", + "branch.feature/y.t3-merge-base-ref", + "refs/heads/release/v2", + ]); + yield* writeTextFile(cwd, "feature.txt", "feature\n"); + yield* git(cwd, ["add", "-A"]); + yield* driver.commit(cwd, "Add feature", ""); + + const pushed = yield* driver.pushCurrentBranch(cwd, null); + + assert.deepInclude(pushed, { + status: "pushed", + branch: "feature/y", + upstreamBranch: "origin/feature/y", + setUpstream: true, + }); + assert.equal( + yield* driver.readConfigValue(cwd, `branch.feature/y.${baseKey}`), + "release/v2", + ); + assert.equal(yield* driver.readBranchBase(cwd, "feature/y"), "release/v2"); + assert.equal( + yield* driver.readConfigValue(cwd, "branch.feature/y.t3-merge-base-ref"), + "refs/heads/release/v2", + ); + }), + ); + + it.effect.each(["main", "feature/base"])( + "rejects publishing a branch tracking local %s without a git remote", + (parent) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* git(cwd, ["branch", "-M", parent]); + yield* git(cwd, ["checkout", "--track", "-b", "child", `refs/heads/${parent}`]); + yield* writeTextFile(cwd, "feature.txt", "feature\n"); + yield* git(cwd, ["add", "-A"]); + yield* driver.commit(cwd, "Add feature", ""); + const parentSha = yield* git(cwd, ["rev-parse", `refs/heads/${parent}`]); + + const error = yield* driver.pushCurrentBranch(cwd, null).pipe(Effect.flip); + + assert.equal(error._tag, "GitCommandError"); + assert.equal(error.operation, "GitVcsDriver.pushCurrentBranch"); + assert.equal( + error.detail, + "Cannot push because no git remote is configured for this repository.", + ); + assert.equal(yield* git(cwd, ["rev-parse", `refs/heads/${parent}`]), parentSha); + }), + ); + + it.effect("publishes a local-tracking branch named like a remote ref under its full name", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); const remote = yield* makeTmpDir("git-remote-"); @@ -6623,8 +7109,8 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { yield* git(remote, ["init", "--bare"]); yield* git(cwd, ["remote", "add", "origin", remote]); yield* git(cwd, ["push", "-u", "origin", "main"]); - yield* git(cwd, ["checkout", "-b", "feature/y", "origin/main"]); - yield* git(cwd, ["config", "branch.feature/y.gh-merge-base", "release/v2"]); + const mainSha = yield* git(cwd, ["rev-parse", "refs/heads/main"]); + yield* git(cwd, ["checkout", "--track", "-b", "origin/main", "refs/heads/main"]); yield* writeTextFile(cwd, "feature.txt", "feature\n"); yield* git(cwd, ["add", "-A"]); yield* driver.commit(cwd, "Add feature", ""); @@ -6633,13 +7119,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.deepInclude(pushed, { status: "pushed", - branch: "feature/y", - upstreamBranch: "origin/feature/y", - setUpstream: true, + branch: "origin/main", + upstreamBranch: "origin/origin/main", }); + assert.equal(yield* git(remote, ["--git-dir=.", "rev-parse", "refs/heads/main"]), mainSha); assert.equal( - yield* driver.readConfigValue(cwd, "branch.feature/y.gh-merge-base"), - "release/v2", + yield* git(remote, ["--git-dir=.", "log", "-1", "--pretty=%s", "refs/heads/origin/main"]), + "Add feature", ); }), ); @@ -6812,3 +7298,60 @@ describe("Windows long path configuration", () => { } }); }); + +it.layer(layerTest)("branch change totals after a failed untracked listing", (it) => { + it.effect("do not list untracked files again on every status read", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = yield* makeTmpDir("git-untracked-totals-"); + yield* initRepoWithCommit(cwd); + const many = path.join(cwd, "many"); + yield* fs.makeDirectory(many); + yield* Effect.forEach( + Array.from({ length: 5_001 }, (_, index) => index), + (index) => fs.writeFileString(path.join(many, `file-${index}`), ""), + { concurrency: 64, discard: true }, + ); + + let listings = 0; + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const spawner = ChildProcessSpawner.make((command) => + Effect.suspend(() => { + if ( + ChildProcess.isStandardCommand(command) && + command.args.includes("ls-files") && + command.args.includes("--others") + ) + listings++; + return delegate.spawn(command); + }), + ); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provide(layerServerConfig), + ); + const read = driver.statusDetailsLocal(cwd, { includeBranchChanges: true }); + + assert.strictEqual((yield* read).branchChanges, undefined, "too many files to count"); + assert.strictEqual(listings, 1); + yield* read; + yield* read; + assert.strictEqual(listings, 1, "the failed listing is not repeated within the backoff"); + + yield* TestClock.adjust("1 minute"); + yield* read; + assert.strictEqual(listings, 2, "it is retried once the backoff passes"); + yield* TestClock.adjust("1 minute"); + yield* read; + assert.strictEqual(listings, 2, "and the next backoff is longer"); + + // A change at the untracked-roots level scans again at once and counts again. + yield* fs.remove(many, { recursive: true }); + yield* writeTextFile(cwd, "notes.txt", "one\ntwo\n"); + const status = yield* read; + assert.strictEqual(listings, 3); + assert.deepStrictEqual(status.branchChanges, { baseRef: null, insertions: 2, deletions: 0 }); + }), + ); +}); diff --git a/apps/server/src/vcs/GitVcsDriverCore.ts b/apps/server/src/vcs/GitVcsDriverCore.ts index ef817f92e292..d913b2752bc2 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.ts @@ -64,6 +64,7 @@ const branchBaseConfigKey = (branch: string) => `branch.${branch}.t3-merge-base` const legacyBranchBaseConfigKey = (branch: string) => `branch.${branch}.gh-merge-base`; const gitProcesses = Semaphore.makeUnsafe(8); const backgroundGitProcesses = Semaphore.makeUnsafe(2); +const WORKTREE_INDEX_REFRESH_TIMEOUT_MS = Duration.toMillis(Duration.minutes(5)); const WORKTREE_REMOVE_TIMEOUT_MS = Duration.toMillis(Duration.minutes(5)); const DEFAULT_MAX_OUTPUT_BYTES = 1_000_000; const OUTPUT_TRUNCATED_MARKER = "\n\n[truncated]"; @@ -75,6 +76,10 @@ const REVIEW_DIFF_PATCH_MAX_OUTPUT_BYTES = 120_000; const REVIEW_METADATA_MAX_OUTPUT_BYTES = 16 * 1024 * 1024; // Temporary-index git add scales poorly (50k files took ~30 s); report incomplete totals above this cap. const REVIEW_UNTRACKED_MAX_FILES = 5_000; +// The listing walks the whole untracked tree before the cap applies, so a folder whose listing +// failed (too many files, output limit, timeout) is not listed again on every status poll. +const UNTRACKED_TOTALS_RETRY_MIN_MS = 60_000; +const UNTRACKED_TOTALS_RETRY_MAX_MS = 30 * 60_000; const REVIEW_DIFF_FILE_MAX_OUTPUT_BYTES = 1024 * 1024; const REVIEW_BRANCH_COMMIT_LIMIT = 100; const REVIEW_BRANCH_SUBJECT_LIMIT = 300; @@ -187,7 +192,6 @@ interface GitRefsSnapshot { interface ExecuteGitOptions { stdin?: string | undefined; timeoutMs?: number | null | undefined; - forceKillAfterMs?: number | undefined; allowNonZeroExit?: boolean | undefined; fallbackErrorDetail?: string | undefined; env?: NodeJS.ProcessEnv | undefined; @@ -348,11 +352,10 @@ function sanitizeRemoteName(value: string): string { function parseRemoteFetchUrls(stdout: string): Map { const remotes = new Map(); - for (const line of stdout.split("\n")) { - const trimmed = line.trim(); - if (trimmed.length === 0) continue; + for (const line of stdout.split(/\r?\n/)) { + // Git uses a tab before the URL and one space before the direction; both can occur in URLs. // Partial clones append the promisor filter, e.g. `(fetch) [blob:none]`. - const match = /^(\S+)\s+(\S+)\s+\((fetch|push)\)(?:\s+\[[^\]]+\])?$/.exec(trimmed); + const match = /^(\S+)\t(.+) \((fetch|push)\)(?:\s+\[[^\]]+\])?$/.exec(line); if (!match) continue; const [, remoteName = "", remoteUrl = "", direction = ""] = match; if (direction !== "fetch" || remoteName.length === 0 || remoteUrl.length === 0) { @@ -445,6 +448,8 @@ const GIT_FAILURE_REASON_PATTERNS: ReadonlyArray normalized.includes(phrase)); } function isUnbornHeadStderr(stderr: string): boolean { const normalized = stderr.toLowerCase(); @@ -984,7 +997,8 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ChildProcess.make(executable, commandInput.args, { cwd: commandInput.cwd, env: spawnEnv, - forceKillAfter: input.forceKillAfterMs, + // SIGTERM first lets git remove its lock files; SIGKILL ends one that ignores it. + forceKillAfter: "1 second", ...(runsGitWithoutConsole(hostPlatform, executable, commandInput.args) ? { detached: true, windowsHide: true } : {}), @@ -1144,9 +1158,6 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ...(options.env !== undefined ? { env: options.env } : {}), allowNonZeroExit: true, ...(options.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}), - ...(options.forceKillAfterMs !== undefined - ? { forceKillAfterMs: options.forceKillAfterMs } - : {}), ...(options.maxOutputBytes !== undefined ? { maxOutputBytes: options.maxOutputBytes } : {}), ...(options.appendTruncationMarker !== undefined ? { appendTruncationMarker: options.appendTruncationMarker } @@ -1361,7 +1372,6 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* env: STATUS_UPSTREAM_REFRESH_ENV, fallbackErrorDetail: "Background Git fetch exited with a non-zero status.", timeoutMs: Duration.toMillis(STATUS_UPSTREAM_FETCH_TIMEOUT), - forceKillAfterMs: 1_000, }).pipe( Effect.asVoid, Effect.tap(() => @@ -1607,6 +1617,15 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* }); const statusRemoteRefreshFailures = new Map(); + const untrackedTotalsFailures = new Map< + string, + { + readonly roots: string; + readonly count: number; + readonly retryAt: number; + readonly detail: string; + } + >(); const statusRemoteRefreshFailureKey = (cacheKey: StatusRemoteRefreshCacheKey) => `${cacheKey.gitCommonDir}\0${cacheKey.remoteName}`; const recordStatusRemoteRefreshFailure = Effect.fn( @@ -1725,25 +1744,32 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* return parsedRemoteRef?.branchName ?? branchName; }); - const resolvePrimaryRemoteName = Effect.fn("resolvePrimaryRemoteName")(function* (cwd: string) { + // Local-only repositories are normal and status polls them, so the span succeeds with null. + // Callers that need a remote fail in their own span. + const findPrimaryRemoteName = Effect.fn("resolvePrimaryRemoteName")(function* (cwd: string) { if (yield* originRemoteExists(cwd)) { return "origin"; } - const remotes = yield* listRemoteNames(cwd); - const [firstRemote] = remotes; - if (firstRemote) { - return firstRemote; - } - return yield* new GitCommandError({ - ...gitCommandContext({ - operation: "GitVcsDriver.resolvePrimaryRemoteName", - cwd, - args: ["remote"], - }), - detail: "No git remote is configured for this repository.", - }); + const [firstRemote] = yield* listRemoteNames(cwd); + return firstRemote ?? null; }); + const resolvePrimaryRemoteName = (cwd: string) => + findPrimaryRemoteName(cwd).pipe( + Effect.filterOrFail( + (remoteName): remoteName is string => remoteName !== null, + () => + new GitCommandError({ + ...gitCommandContext({ + operation: "GitVcsDriver.resolvePrimaryRemoteName", + cwd, + args: ["remote"], + }), + detail: "No git remote is configured for this repository.", + }), + ), + ); + const resolvePushRemoteName = Effect.fn("resolvePushRemoteName")(function* ( cwd: string, refName: string, @@ -2488,17 +2514,45 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* Effect.fn("prepareCommitContext")(function* (cwd, filePaths, options) { return yield* Effect.scoped( Effect.gen(function* () { - const indexValue = yield* runGitStdout( - "GitVcsDriver.prepareCommitContext.indexPath", + const [indexValue = "", objectsValue = ""] = (yield* runGitStdout( + "GitVcsDriver.prepareCommitContext.gitPaths", cwd, - ["rev-parse", "--git-path", "index"], - ); + ["rev-parse", "--git-path", "index", "--git-path", "objects"], + )).split("\n"); const indexPath = path.resolve(cwd, indexValue.trim()); const directory = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-commit-index-", }); const tempIndexPath = path.join(directory, "index"); - const env = { GIT_INDEX_FILE: tempIndexPath } satisfies NodeJS.ProcessEnv; + // Staging writes blobs; put them in the temp directory so a draft leaves nothing behind. + // List every object directory Git resolved (nested and environment alternates) flat, + // since nesting the repository's objects one level deeper can pass Git's depth limit. + const objectDirectories = yield* runGitStdout( + "GitVcsDriver.prepareCommitContext.alternates", + cwd, + ["count-objects", "-v"], + ); + // Git prints an entry C-quoted when it needs quoting; quote the rest the same way, so a + // path holding the list separator stays one entry (Git 2.11.1+). + const quoted = (entry: string) => + entry.startsWith('"') ? entry : `"${entry.replace(/[\\"]/g, "\\$&")}"`; + const alternates = [ + path.resolve(cwd, objectsValue.trim()), + ...objectDirectories + .split("\n") + .flatMap((line) => + line.startsWith("alternate: ") ? [line.slice("alternate: ".length)] : [], + ), + ]; + const tempObjectsPath = path.join(directory, "objects"); + yield* fileSystem.makeDirectory(tempObjectsPath); + const env = { + GIT_INDEX_FILE: tempIndexPath, + GIT_OBJECT_DIRECTORY: tempObjectsPath, + GIT_ALTERNATE_OBJECT_DIRECTORIES: alternates + .map(quoted) + .join(hostPlatform === "win32" ? ";" : ":"), + } satisfies NodeJS.ProcessEnv; const config = [ "-c", "core.splitIndex=false", @@ -2644,6 +2698,72 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ), ); + // `publishes` is false only when HEAD goes to the head of its current `upstream`. + const resolvePushDestination = Effect.fn("resolvePushDestination")(function* ( + cwd: string, + branch: string, + hasUpstream: boolean, + ) { + if (!hasUpstream) { + const remoteName = yield* resolvePushRemoteName(cwd, branch); + if (!remoteName) return null; + const branchName = yield* resolvePublishBranchName(cwd, branch); + return { remoteName, branchName, publishes: true, upstream: null }; + } + + if ((yield* readConfigValue(cwd, `branch.${branch}.remote`)) === ".") { + const localUpstream = yield* readCurrentUpstreamRef(cwd).pipe( + Effect.orElseSucceed(() => null), + ); + if (localUpstream === null) return null; + // A local upstream is a base; publish under the full local name so `origin/main` never lands on remote `main`. + const remoteName = yield* resolvePushRemoteName(cwd, branch).pipe( + Effect.orElseSucceed(() => null), + ); + if (!remoteName) return null; + return { + remoteName, + branchName: branch, + publishes: true, + upstream: { remoteName: ".", branchName: localUpstream, upstreamRef: localUpstream }, + }; + } + + const upstream = yield* resolveCurrentUpstream(cwd).pipe(Effect.orElseSucceed(() => null)); + if (!upstream) return null; + // Differently named upstreams are bases, not publish targets; pushing HEAD there would write features to shared branches. + const isAliasOfUpstreamHead = + branch === upstream.branchName || + (branch.endsWith(`/${upstream.branchName}`) && upstream.upstreamRef.endsWith(`/${branch}`)); + if (isAliasOfUpstreamHead) { + return { + remoteName: upstream.remoteName, + branchName: upstream.branchName, + publishes: false, + upstream, + }; + } + const publishRemoteName = yield* resolvePushRemoteName(cwd, branch).pipe( + Effect.orElseSucceed(() => null), + ); + const branchName = yield* resolvePublishBranchName(cwd, branch); + return { + remoteName: publishRemoteName ?? upstream.remoteName, + branchName, + publishes: true, + upstream, + }; + }); + + const resolvePushTarget: GitVcsDriver.GitVcsDriver["Service"]["resolvePushTarget"] = Effect.fn( + "resolvePushTarget", + )(function* (cwd, current) { + const destination = yield* resolvePushDestination(cwd, current.branch, current.hasUpstream); + return destination + ? { remoteName: destination.remoteName, branchName: destination.branchName } + : null; + }); + const pushCurrentBranch: GitVcsDriver.GitVcsDriver["Service"]["pushCurrentBranch"] = Effect.fn( "pushCurrentBranch", )(function* (cwd, fallbackBranch, options) { @@ -2678,30 +2798,27 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* } const hasNoLocalDelta = details.aheadCount === 0 && details.behindCount === 0; - if (hasNoLocalDelta) { - if (details.hasUpstream) { - return { - status: "skipped_up_to_date" as const, - branch, - ...(details.upstreamRef ? { upstreamBranch: details.upstreamRef } : {}), - }; - } + if (hasNoLocalDelta && details.hasUpstream) { + return { + status: "skipped_up_to_date" as const, + branch, + ...(details.upstreamRef ? { upstreamBranch: details.upstreamRef } : {}), + }; + } + const destination = yield* resolvePushDestination(cwd, branch, details.hasUpstream); + if (hasNoLocalDelta) { const comparableBaseBranch = yield* resolveBaseBranchForNoUpstream(cwd, branch).pipe( Effect.orElseSucceed(() => null), ); if (comparableBaseBranch) { - const publishRemoteName = yield* resolvePushRemoteName(cwd, branch).pipe( - Effect.orElseSucceed(() => null), - ); - if (!publishRemoteName) { + if (!destination) { return { status: "skipped_up_to_date" as const, branch, }; } - const publishBranch = yield* resolvePublishBranchName(cwd, branch); // The configured diff base can contain HEAD while the publish branch is still behind. const publishContainsHead = yield* executeGit( "GitVcsDriver.pushCurrentBranch.publishContainsHead", @@ -2710,7 +2827,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* "merge-base", "--is-ancestor", "HEAD", - `refs/remotes/${publishRemoteName}/${publishBranch}`, + `refs/remotes/${destination.remoteName}/${destination.branchName}`, ], { allowNonZeroExit: true }, ).pipe( @@ -2726,87 +2843,67 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* } } - if (!details.hasUpstream) { - const publishRemoteName = yield* resolvePushRemoteName(cwd, branch); - if (!publishRemoteName) { - return yield* new GitCommandError({ - ...gitCommandContext({ - operation: "GitVcsDriver.pushCurrentBranch", - cwd, - args: ["push"], - }), - detail: "Cannot push because no git remote is configured for this repository.", + if ( + !destination && + (!details.hasUpstream || (yield* readConfigValue(cwd, `branch.${branch}.remote`)) === ".") + ) { + return yield* new GitCommandError({ + ...gitCommandContext({ + operation: "GitVcsDriver.pushCurrentBranch", + cwd, + args: ["push"], + }), + detail: "Cannot push because no git remote is configured for this repository.", + }); + } + + if (destination?.publishes) { + const { remoteName, branchName: publishBranch, upstream } = destination; + // Publishing retargets the upstream, so preserve its comparison base first. + if (upstream && (yield* readBranchBase(cwd, branch)) === null) { + yield* writeConfigValue({ + cwd, + key: branchBaseConfigKey(branch), + value: upstream.branchName, }); + if (!(yield* readConfigValue(cwd, `branch.${branch}.t3-merge-base-ref`))) { + yield* writeConfigValue({ + cwd, + key: `branch.${branch}.t3-merge-base-ref`, + value: + upstream.remoteName === "." + ? `refs/heads/${upstream.branchName}` + : `refs/remotes/${upstream.upstreamRef}`, + }); + } } - const publishBranch = yield* resolvePublishBranchName(cwd, branch); yield* runGit( - "GitVcsDriver.pushCurrentBranch.pushWithUpstream", + upstream + ? "GitVcsDriver.pushCurrentBranch.pushOwnBranch" + : "GitVcsDriver.pushCurrentBranch.pushWithUpstream", cwd, - ["push", "--", publishRemoteName, `HEAD:refs/heads/${publishBranch}`], + ["push", "--", remoteName, `HEAD:refs/heads/${publishBranch}`], { timeoutMs: null }, ); return { status: "pushed" as const, branch, - upstreamBranch: `${publishRemoteName}/${publishBranch}`, - setUpstream: yield* recordPushedUpstream(cwd, branch, publishRemoteName, publishBranch), + upstreamBranch: `${remoteName}/${publishBranch}`, + setUpstream: yield* recordPushedUpstream(cwd, branch, remoteName, publishBranch), }; } - const currentUpstream = yield* resolveCurrentUpstream(cwd).pipe( - Effect.orElseSucceed(() => null), - ); - if (currentUpstream) { - // Differently named upstreams are bases, not publish targets; pushing HEAD there would write features to shared branches. - const isAliasOfUpstreamHead = - branch === currentUpstream.branchName || - (branch.endsWith(`/${currentUpstream.branchName}`) && - currentUpstream.upstreamRef.endsWith(`/${branch}`)); - if (!isAliasOfUpstreamHead) { - const publishRemoteName = yield* resolvePushRemoteName(cwd, branch).pipe( - Effect.orElseSucceed(() => null), - ); - const remoteName = publishRemoteName ?? currentUpstream.remoteName; - const publishBranch = yield* resolvePublishBranchName(cwd, branch); - // Publishing retargets the upstream, so preserve its comparison base first. - if ((yield* readBranchBase(cwd, branch)) === null) { - yield* writeConfigValue({ - cwd, - key: branchBaseConfigKey(branch), - value: currentUpstream.branchName, - }); - if (!(yield* readConfigValue(cwd, `branch.${branch}.t3-merge-base-ref`))) { - yield* writeConfigValue({ - cwd, - key: `branch.${branch}.t3-merge-base-ref`, - value: `refs/remotes/${currentUpstream.upstreamRef}`, - }); - } - } - yield* runGit( - "GitVcsDriver.pushCurrentBranch.pushOwnBranch", - cwd, - ["push", "--", remoteName, `HEAD:refs/heads/${publishBranch}`], - { timeoutMs: null }, - ); - return { - status: "pushed" as const, - branch, - upstreamBranch: `${remoteName}/${publishBranch}`, - setUpstream: yield* recordPushedUpstream(cwd, branch, remoteName, publishBranch), - }; - } - + if (destination?.upstream) { yield* runGit( "GitVcsDriver.pushCurrentBranch.pushUpstream", cwd, - ["push", "--", currentUpstream.remoteName, `HEAD:refs/heads/${currentUpstream.branchName}`], + ["push", "--", destination.remoteName, `HEAD:refs/heads/${destination.branchName}`], { timeoutMs: null }, ); return { status: "pushed" as const, branch, - upstreamBranch: currentUpstream.upstreamRef, + upstreamBranch: destination.upstream.upstreamRef, setUpstream: false, }; } @@ -3107,7 +3204,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* return stdout.trim(); }); - const prepareUntrackedReviewIndex = Effect.fn("prepareUntrackedReviewIndex")(function* ( + const prepareUntrackedReviewIndexOrReason = Effect.fn("prepareUntrackedReviewIndex")(function* ( cwd: string, pathArgs: ReadonlyArray, onlyPath?: string, @@ -3117,15 +3214,21 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* cwd, ["ls-files", "--others", "--exclude-standard", "-z", "--", ...pathArgs], { maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES }, - ).pipe(Effect.orElseSucceed(() => null)); - if (untracked === null) return null; + ).pipe( + Effect.catchTags({ + GitCommandError: (error) => + Effect.succeed({ unavailable: `Could not list untracked files: ${error.detail}` }), + }), + ); + if ("unavailable" in untracked) return untracked; // ls-files lists a nested repository with a trailing slash; its diff path has none. const paths = splitNullSeparatedGitStdoutPaths(untracked).filter( (candidate) => onlyPath === undefined || candidate === onlyPath || candidate === `${onlyPath}/`, ); if (paths.length === 0) return { env: undefined }; - if (paths.length > REVIEW_UNTRACKED_MAX_FILES) return null; + if (paths.length > REVIEW_UNTRACKED_MAX_FILES) + return { unavailable: "Too many untracked files to count." }; const env = yield* prepareReviewIndex(cwd, paths).pipe( Effect.catchTags({ PlatformError: (cause) => @@ -3142,6 +3245,14 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ); return { env }; }); + const prepareUntrackedReviewIndex = ( + cwd: string, + pathArgs: ReadonlyArray, + onlyPath?: string, + ) => + prepareUntrackedReviewIndexOrReason(cwd, pathArgs, onlyPath).pipe( + Effect.map((result) => ("unavailable" in result ? null : result)), + ); const resolveReviewMergeBase = Effect.fn("resolveReviewMergeBase")(function* ( cwd: string, @@ -3215,16 +3326,46 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* roots.length <= 128 && roots.reduce((length, root) => length + root.length + 16, 0) < 8_000 ? roots.map((root) => `:(top,literal)${root}`) : []; + // Changed untracked roots skip the backoff and scan again. + const rootsKey = roots.join("\0"); + const remembered = untrackedTotalsFailures.get(cwd); + if ( + remembered !== undefined && + remembered.roots === rootsKey && + (yield* Clock.currentTimeMillis) < remembered.retryAt + ) { + return yield* new GitCommandError({ + operation: "GitVcsDriver.readBranchChangeTotals", + command: "git ls-files", + cwd, + detail: remembered.detail, + }); + } // Empty status can mean untracked files are hidden; retain the full scan. - const untracked = yield* prepareUntrackedReviewIndex(cwd, pathspecs); - if (untracked === null) { + const untracked = yield* prepareUntrackedReviewIndexOrReason(cwd, pathspecs); + if ("unavailable" in untracked) { + const count = remembered?.roots === rootsKey ? remembered.count + 1 : 1; + untrackedTotalsFailures.delete(cwd); + untrackedTotalsFailures.set(cwd, { + roots: rootsKey, + count, + retryAt: + (yield* Clock.currentTimeMillis) + + Math.min(UNTRACKED_TOTALS_RETRY_MIN_MS * 2 ** (count - 1), UNTRACKED_TOTALS_RETRY_MAX_MS), + detail: untracked.unavailable, + }); + if (untrackedTotalsFailures.size > STATUS_UPSTREAM_REFRESH_CACHE_CAPACITY) { + const oldestKey = untrackedTotalsFailures.keys().next().value; + if (oldestKey !== undefined) untrackedTotalsFailures.delete(oldestKey); + } return yield* new GitCommandError({ operation: "GitVcsDriver.readBranchChangeTotals", command: "git ls-files", cwd, - detail: "Too many untracked files to count.", + detail: untracked.unavailable, }); } + untrackedTotalsFailures.delete(cwd); const readNumstat = (ref: string) => executeGit( "GitVcsDriver.readBranchChangeTotals", @@ -4260,6 +4401,52 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* { timeoutMs: WORKTREE_REMOVE_TIMEOUT_MS }, ).pipe(Effect.asVoid); + // A checkout stamps most files in the second it writes the index, and Git re-reads every file + // stamped in the index's own second or later ("racily clean") on each status, diff and add. + // T3's own git calls never rewrite a worktree's index, so refresh it once. + const refreshNewWorktreeIndex = Effect.fn("refreshNewWorktreeIndex")(function* ( + worktreePath: string, + ) { + const indexPath = path.resolve( + worktreePath, + (yield* runGitStdout("GitVcsDriver.createWorktree.indexPath", worktreePath, [ + "rev-parse", + "--git-path", + "index", + ])).trim(), + ); + const indexSecond = fileSystem.stat(indexPath).pipe( + Effect.map(({ mtime }) => Option.map(mtime, (date) => Math.floor(date.getTime() / 1000))), + Effect.orElseSucceed(() => Option.none()), + ); + // Older Git (such as 2.30) skips the write when racy entries merely check out clean. + // A refresh re-reads every checked-out file, so allow five minutes outside shared permits. + const refresh = runGit( + "GitVcsDriver.createWorktree.refreshIndex", + worktreePath, + ["update-index", "-q", "--refresh", "--force-write-index"], + { timeoutMs: WORKTREE_INDEX_REFRESH_TIMEOUT_MS }, + ); + const checkoutSecond = yield* indexSecond; + yield* refresh; + const refreshedSecond = yield* indexSecond; + if ( + Option.isNone(checkoutSecond) || + Option.isNone(refreshedSecond) || + refreshedSecond.value > checkoutSecond.value + ) { + return; + } + // The refresh finished within the checkout's second, so its index is just as racy. Wait that + // second out (file timestamps can trail the clock by a kernel tick) and refresh again. Two + // seconds or more means this clock does not match the file timestamps, as on a share stamped + // by another machine, so it cannot say when that second ends. + const delay = (checkoutSecond.value + 1) * 1000 + 50 - (yield* Clock.currentTimeMillis); + if (delay >= 2_000) return; + if (delay > 0) yield* Effect.sleep(Duration.millis(delay)); + yield* refresh; + }); + const createWorktree: GitVcsDriver.GitVcsDriver["Service"]["createWorktree"] = Effect.fn( "createWorktree", )(function* (input, options) { @@ -4301,7 +4488,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* yield* createBranchAt( "GitVcsDriver.createWorktree.createBranch", { cwd: input.cwd, branch: newRefName, startPoint: input.refName }, - "git worktree add failed", + "git branch failed", ); } const args = ["worktree", "add", "--", worktreePath, newRefName ?? input.refName]; @@ -4428,6 +4615,17 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ); } + // Before the worktree is returned: a refresh holds index.lock, so running it in the + // background would race the setup script's and the agent's own git commands. + yield* refreshNewWorktreeIndex(worktreePath).pipe( + Effect.catch((cause) => + Effect.logWarning( + "worktree index refresh failed; Git keeps re-reading its files until the index is rewritten", + { worktreePath, cause }, + ), + ), + ); + if (newRefName && base) { yield* Effect.gen(function* () { yield* writeConfigValue({ @@ -4966,28 +5164,51 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ).pipe(Effect.map((result) => result.exitCode === 0)) : false; + const newTrackingBranch = + !localInputExists && remoteExists && !localTrackingBranch && !localTrackedBranchTargetExists + ? localTrackedBranchCandidate + : null; + const checkoutArgs = localInputExists ? ["checkout", input.refName] : remoteExists && !localTrackingBranch && localTrackedBranchTargetExists ? ["checkout", input.refName] - : remoteExists && !localTrackingBranch - ? ["checkout", "--track", input.refName] - : remoteExists && localTrackingBranch - ? ["checkout", localTrackingBranch] - : ["checkout", input.refName]; + : newTrackingBranch + ? ["checkout", newTrackingBranch] + : remoteExists && !localTrackingBranch + ? ["checkout", "--track", input.refName] + : remoteExists && localTrackingBranch + ? ["checkout", localTrackingBranch] + : ["checkout", input.refName]; yield* refuseOptionLikeRefName("GitVcsDriver.switchRef", input.cwd, checkoutArgs.at(-1)); + // Created before the checkout so the shared-config lock is not held while files are written. + // Not rolled back on failure: Git may already have switched HEAD to it (post-checkout hooks). + if (newTrackingBranch) { + yield* executeGit( + "GitVcsDriver.switchRef.createTrackingBranch", + input.cwd, + ["branch", "--track", "--", newTrackingBranch, input.refName], + { + timeoutMs: 10_000, + fallbackErrorDetail: "git branch create failed", + writesSharedConfig: true, + }, + ); + } + // A stale ref must not turn into a path checkout that discards local edits. yield* executeGitWithStableDiagnostics( "GitVcsDriver.switchRef.checkout", input.cwd, [...checkoutArgs, "--"], { - timeoutMs: 10_000, + // Killing a checkout midway (large or LFS repos) leaves a half-updated working tree. + timeoutMs: null, fallbackErrorDetail: "git checkout failed", - // Checking out a remote branch creates a local one that tracks it. - writesSharedConfig: !localInputExists, + // Creating a branch writes its upstream; Git also guesses a remote branch for a missing name. + writesSharedConfig: checkoutArgs[1] === "--track" || (!localInputExists && !remoteExists), }, ); @@ -5114,6 +5335,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* commit: Effect.fnUntraced(function* (cwd, subject, body, options) { return yield* withListRefsInvalidation(cwd, commit(cwd, subject, body, options)); }), + resolvePushTarget, pushCurrentBranch: Effect.fnUntraced(function* (cwd, fallbackBranch, options) { return yield* withListRefsInvalidation(cwd, pushCurrentBranch(cwd, fallbackBranch, options)); }), diff --git a/apps/server/src/vcs/VcsDriverRegistry.test.ts b/apps/server/src/vcs/VcsDriverRegistry.test.ts index c13d44f2d813..6fee790bc4bd 100644 --- a/apps/server/src/vcs/VcsDriverRegistry.test.ts +++ b/apps/server/src/vcs/VcsDriverRegistry.test.ts @@ -6,6 +6,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import { ChildProcessSpawner } from "effect/process"; +import { VcsProcessExitError } from "@t3tools/contracts"; import * as VcsProcess from "./VcsProcess.ts"; import * as VcsProjectConfig from "./VcsProjectConfig.ts"; @@ -128,4 +129,53 @@ describe("VcsDriverRegistry", () => { }).pipe(Effect.provide(makeDiskBackedLayer(repoDir, calls))); }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), ); + const makeFailingLayer = (detectOutput: VcsProcess.VcsProcessOutput) => + Layer.effect(VcsDriverRegistry.VcsDriverRegistry, VcsDriverRegistry.make).pipe( + Layer.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), + Layer.provide( + Layer.mock(VcsProjectConfig.VcsProjectConfig)({ + resolveKind: (input) => Effect.succeed(input.requestedKind ?? "auto"), + }), + ), + Layer.provide( + Layer.mock(VcsProcess.VcsProcess)({ + run: (input) => + normalizeGitArgs(input.args).join(" ") === DETECT_COMMAND + ? Effect.succeed(detectOutput) + : Effect.fail( + new VcsProcessExitError({ + operation: input.operation, + command: "git", + cwd: input.cwd, + exitCode: 128, + detail: "fatal: bad config line 1 in file .git/config", + }), + ), + }), + ), + ); + + it.effect("reads only a failed work-tree probe as no repository", () => + Effect.gen(function* () { + const probeFailure = yield* VcsDriverRegistry.VcsDriverRegistry.pipe( + Effect.flatMap((registry) => registry.detect({ cwd: "/repo" })), + Effect.provide( + makeFailingLayer({ + ...processOutput(""), + exitCode: ChildProcessSpawner.ExitCode(128), + stderr: "fatal: bad config line 1 in file .git/config", + }), + ), + ); + assert.equal(probeFailure, null); + + // A newline in the root path makes detection ask for it again, and main surfaced that failure. + const fallbackFailure = yield* VcsDriverRegistry.VcsDriverRegistry.pipe( + Effect.flatMap((registry) => registry.detect({ cwd: "/repo" })), + Effect.provide(makeFailingLayer(processOutput("true\n/re\npo\n.git\n"))), + Effect.flip, + ); + assert.equal(fallbackFailure._tag, "VcsProcessExitError"); + }), + ); }); diff --git a/apps/server/src/vcs/VcsDriverRegistry.ts b/apps/server/src/vcs/VcsDriverRegistry.ts index d42cae5cc347..5bf7263ebe83 100644 --- a/apps/server/src/vcs/VcsDriverRegistry.ts +++ b/apps/server/src/vcs/VcsDriverRegistry.ts @@ -85,7 +85,16 @@ export const make = Effect.gen(function* () { driver: VcsDriver.VcsDriver["Service"], cwd: string, ) { - const repository = yield* driver.detectRepository(cwd); + // Status, review and source control show a repository Git cannot read as none at all. + // Checkpoints need the failure, so CheckpointStore asks the driver directly. + const repository = yield* driver.detectRepository(cwd).pipe( + Effect.catchTags({ + VcsProcessExitError: (error) => + error.operation === GitVcsDriver.WORK_TREE_PROBE_OPERATION + ? Effect.succeed(null) + : Effect.fail(error), + }), + ); if (!repository) { return null; } diff --git a/apps/server/src/vcs/VcsStatusBroadcaster.test.ts b/apps/server/src/vcs/VcsStatusBroadcaster.test.ts index f2af7c0ab18a..ea2c253e6d26 100644 --- a/apps/server/src/vcs/VcsStatusBroadcaster.test.ts +++ b/apps/server/src/vcs/VcsStatusBroadcaster.test.ts @@ -19,6 +19,7 @@ import * as Tracer from "effect/Tracer"; import type { VcsStatusLocalResult, VcsStatusRemoteResult, + VcsStatusResult, VcsStatusStreamEvent, } from "@t3tools/contracts"; // Coder: exercise the fork settings policy and lightweight ref-stream contract. @@ -30,6 +31,7 @@ import { type ServerSettings as ServerSettingsValue, type VcsRefStatusStreamEvent, } from "@t3tools/contracts"; +import { applyGitStatusStreamEvent } from "@t3tools/shared/git"; import * as ProjectStore from "../orchestration-v2/ProjectStore.ts"; import * as ServerSettings from "../serverSettings.ts"; @@ -93,11 +95,15 @@ function layerTestFor(state: { branchRead?: () => Effect.Effect; fileSystem?: FileSystem.FileSystem; pullCurrentBranch?: GitWorkflowService.GitWorkflowService["Service"]["pullCurrentBranch"]; + headPath?: string; + headPathCalls?: number; + failLocalStatus?: boolean; + onLocalStatusFailure?: Effect.Effect; }) { return VcsStatusBroadcaster.layer.pipe( Layer.provideMerge( state.fileSystem - ? Layer.succeed(FileSystem.FileSystem, state.fileSystem) + ? Layer.merge(Layer.succeed(FileSystem.FileSystem, state.fileSystem), Path.layer) : NodeServices.layer, ), Layer.provide(layerBackgroundPolicy(() => state.backgroundWorkEnabled !== false)), @@ -112,6 +118,19 @@ function layerTestFor(state: { localStatus: () => Effect.suspend(() => { state.localStatusCalls += 1; + if (state.failLocalStatus) { + return (state.onLocalStatusFailure ?? Effect.void).pipe( + Effect.andThen( + Effect.fail( + new GitManagerError({ + operation: "VcsStatusBroadcaster.test", + cwd: "/repo", + detail: "index.lock is held", + }), + ), + ), + ); + } return (state.beforeLocalStatus ?? Effect.void).pipe( Effect.as(state.currentLocalStatus), ); @@ -151,6 +170,11 @@ function layerTestFor(state: { state.remoteInvalidationCalls += 1; }), ...(state.pullCurrentBranch ? { pullCurrentBranch: state.pullCurrentBranch } : {}), + headPath: () => + Effect.sync(() => { + state.headPathCalls = (state.headPathCalls ?? 0) + 1; + return state.headPath ?? null; + }), }), ), ); @@ -364,17 +388,19 @@ describe("VcsStatusBroadcaster", () => { yield* Deferred.await(subscribed); state.currentLocalStatus = { ...baseLocalStatus, refName: "main" }; + state.currentRemoteStatus = baseRemoteStatus; const refreshed = yield* broadcaster.refreshStatus("/repo", { localOnly: true }); assert.equal(refreshed.refName, "main"); assert.isNull(refreshed.pr); assert.equal(state.localInvalidationCalls, 1); assert.equal(state.remoteStatusCalls, 1); - assert.equal(state.remoteInvalidationCalls, 2); + assert.equal(state.remoteInvalidationCalls, 1); assert.deepStrictEqual((yield* Fiber.join(events)).slice(1), [ { _tag: "remoteUpdated", remote: null }, { _tag: "localUpdated", local: state.currentLocalStatus }, ]); assert.isNull((yield* broadcaster.getStatus({ cwd: "/repo" })).pr); + assert.equal(state.remoteStatusCalls, 2); state.currentLocalStatus = { ...state.currentLocalStatus, hasWorkingTreeChanges: true }; yield* broadcaster.refreshLocalStatus("/repo"); state.currentLocalStatus = { ...baseLocalStatus, refName: "union" }; @@ -422,6 +448,98 @@ describe("VcsStatusBroadcaster", () => { }, ); + it.effect("a local-only refresh that overlaps the first remote poll keeps its PR", () => { + const state: Parameters[0] = { + currentLocalStatus: baseLocalStatus, + currentRemoteStatus: remoteStatusWithPr, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + }; + + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const pollStarted = yield* Deferred.make(); + const releasePoll = yield* Deferred.make(); + state.beforeRemoteStatus = Deferred.succeed(pollStarted, undefined).pipe( + Effect.andThen(Deferred.await(releasePoll)), + ); + const localSeen = yield* Deferred.make(); + yield* broadcaster.streamStatus({ cwd: "/repo" }).pipe( + Stream.runForEach((event) => + event._tag === "localUpdated" ? Deferred.succeed(localSeen, undefined) : Effect.void, + ), + Effect.forkScoped, + ); + yield* Deferred.await(pollStarted); + state.beforeRemoteStatus = Effect.void; + + state.currentLocalStatus = { ...baseLocalStatus, hasWorkingTreeChanges: true }; + const refresh = yield* broadcaster + .refreshStatus("/repo", { localOnly: true }) + .pipe(Effect.forkScoped); + yield* Deferred.await(localSeen); + yield* Deferred.succeed(releasePoll, undefined); + yield* Fiber.join(refresh); + + const status = yield* broadcaster.getStatus({ cwd: "/repo" }); + assert.deepStrictEqual(status.pr, remoteStatusWithPr.pr); + assert.equal(state.remoteStatusCalls, 1); + }).pipe(Effect.provide(layerTestFor(state))); + }); + + it.effect( + "a local-only refresh before any remote read leaves the PR refresh able to read", + () => { + const state: Parameters[0] = { + currentLocalStatus: baseLocalStatus, + currentRemoteStatus: remoteStatusWithPr, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + }; + + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + yield* broadcaster.refreshStatus("/repo", { localOnly: true }); + + const remote = yield* broadcaster.refreshPullRequestStatus("/repo"); + assert.deepStrictEqual(remote?.pr, remoteStatusWithPr.pr); + assert.equal(state.remoteStatusCalls, 1); + }).pipe(Effect.provide(layerTestFor(state))); + }, + ); + + it.effect("a local-only refresh after a branch switch leaves the PR refresh able to read", () => { + const state: Parameters[0] = { + currentLocalStatus: baseLocalStatus, + currentRemoteStatus: remoteStatusWithPr, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + }; + + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + yield* broadcaster.getStatus({ cwd: "/repo" }); + + state.currentLocalStatus = { ...baseLocalStatus, refName: "feature/next" }; + const nextBranchRemote: VcsStatusRemoteResult = { + ...baseRemoteStatus, + pr: { ...remoteStatusWithPr.pr!, number: 3001, headRef: "feature/next" }, + }; + state.currentRemoteStatus = nextBranchRemote; + yield* broadcaster.refreshStatus("/repo", { localOnly: true }); + + const remote = yield* broadcaster.refreshPullRequestStatus("/repo"); + assert.deepStrictEqual(remote?.pr, nextBranchRemote.pr); + assert.equal(state.remoteStatusCalls, 2); + }).pipe(Effect.provide(layerTestFor(state))); + }); + it.effect("clears the previous branch's remote status after an in-flight PR refresh", () => { const remoteReadStarted = Deferred.makeUnsafe(); const releaseRemoteRead = Deferred.makeUnsafe(); @@ -452,13 +570,14 @@ describe("VcsStatusBroadcaster", () => { yield* Deferred.succeed(releaseRemoteRead, undefined); yield* Fiber.join(remoteRefresh); yield* Fiber.join(localRefresh); + state.currentRemoteStatus = baseRemoteStatus; const status = yield* broadcaster.getStatus({ cwd: "/repo" }); assert.equal(status.refName, "main"); assert.isNull(status.pr); assert.equal(status.aheadCount, 0); assert.equal(status.behindCount, 0); - assert.equal(state.remoteStatusCalls, 2); + assert.equal(state.remoteStatusCalls, 3); }).pipe(Effect.provide(layerTestFor(state))); }); @@ -1183,6 +1302,259 @@ describe("VcsStatusBroadcaster", () => { }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); }); + it.effect("a terminal checkout publishes the new branch without waiting for a poll", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const gitDir = yield* fs.makeTempDirectoryScoped({ prefix: "vcs-head-watch-" }); + const headPath = path.join(gitDir, "HEAD"); + yield* fs.writeFileString(headPath, "ref: refs/heads/main\n"); + const state = { + currentLocalStatus: { ...baseLocalStatus, isDefaultRef: true, refName: "main" }, + currentRemoteStatus: remoteStatusWithPr as VcsStatusRemoteResult | null, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + remoteStatusRefreshUpstreamValues: [] as Array, + headPath, + }; + + yield* Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + yield* broadcaster.getStatus({ cwd: "/repo" }); + const clientStates: Array = []; + const subscribed = yield* Deferred.make(); + const checkedOut = yield* Deferred.make(); + yield* Stream.runForEach(broadcaster.streamStatus({ cwd: "/repo" }), (event) => + Effect.gen(function* () { + const next = applyGitStatusStreamEvent(clientStates.at(-1) ?? null, event); + clientStates.push(next); + yield* Deferred.succeed(subscribed, undefined); + if (next.refName === "repro") yield* Deferred.succeed(checkedOut, undefined); + }), + ).pipe(Effect.forkScoped); + yield* Deferred.await(subscribed); + + // What `git switch -c repro` does to HEAD, with the test clock never advanced. + state.currentLocalStatus = { ...baseLocalStatus, isDefaultRef: false, refName: "repro" }; + state.currentRemoteStatus = { ...baseRemoteStatus, hasUpstream: false }; + const lockPath = path.join(gitDir, "HEAD.lock"); + yield* fs.writeFileString(lockPath, "ref: refs/heads/repro\n"); + yield* fs.rename(lockPath, headPath); + yield* Deferred.await(checkedOut); + + assert.deepStrictEqual( + clientStates.filter((status) => status.refName === "repro").map((status) => status.pr), + [null], + ); + // A checkout must not trigger a network fetch. + assert.deepStrictEqual(state.remoteStatusRefreshUpstreamValues.slice(1), [false]); + }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + ); + + it.effect("a terminal checkout updates branch-only subscribers without full status", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const gitDir = yield* fs.makeTempDirectoryScoped({ prefix: "vcs-head-watch-" }); + const headPath = path.join(gitDir, "HEAD"); + yield* fs.writeFileString(headPath, "ref: refs/heads/main\n"); + const state = { + currentLocalStatus: { ...baseLocalStatus, refName: "main" }, + currentRemoteStatus: null, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + headPath, + }; + + yield* Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const branches = yield* Stream.toPull(broadcaster.streamBranchStatus({ cwd: "/repo" })); + assert.deepEqual(yield* branches, [{ refName: "main" }]); + + state.currentLocalStatus = { ...baseLocalStatus, refName: "repro" }; + const lockPath = path.join(gitDir, "HEAD.lock"); + yield* fs.writeFileString(lockPath, "ref: refs/heads/repro\n"); + yield* fs.rename(lockPath, headPath); + + assert.deepEqual(yield* branches, [{ refName: "repro" }]); + assert.equal(state.localStatusCalls, 0); + }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + ); + + // Coder: macOS reports one rename into a watched folder as two `HEAD` events, so the sliding + // buffer keeps a second refresh; Linux, where upstream CI and the workspace run, reports one. + it.effect.skipIf(process.platform === "darwin")( + "a checkout read that fails while index.lock is held retries until it succeeds", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const gitDir = yield* fs.makeTempDirectoryScoped({ prefix: "vcs-head-watch-" }); + const headPath = path.join(gitDir, "HEAD"); + yield* fs.writeFileString(headPath, "ref: refs/heads/main\n"); + const failed = yield* Deferred.make(); + const state = { + currentLocalStatus: { ...baseLocalStatus, isDefaultRef: true, refName: "main" }, + currentRemoteStatus: null, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + headPath, + failLocalStatus: false, + onLocalStatusFailure: Deferred.succeed(failed, undefined).pipe(Effect.asVoid), + }; + + yield* Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const subscribed = yield* Deferred.make(); + const checkedOut = yield* Deferred.make(); + let clientState: VcsStatusResult | null = null; + yield* Stream.runForEach( + broadcaster.streamStatus({ cwd: "/repo", includeRemote: false }), + (event) => + Effect.gen(function* () { + clientState = applyGitStatusStreamEvent(clientState, event); + yield* Deferred.succeed(subscribed, undefined); + if (clientState.refName === "repro") yield* Deferred.succeed(checkedOut, undefined); + }), + ).pipe(Effect.forkScoped); + yield* Deferred.await(subscribed); + + // A terminal commit holds index.lock across the checkout, so the HEAD event's read fails. + state.failLocalStatus = true; + state.currentLocalStatus = { ...baseLocalStatus, isDefaultRef: false, refName: "repro" }; + const lockPath = path.join(gitDir, "HEAD.lock"); + yield* fs.writeFileString(lockPath, "ref: refs/heads/repro\n"); + yield* fs.rename(lockPath, headPath); + yield* Deferred.await(failed); + const failedCalls = state.localStatusCalls; + + // Removing index.lock moves no HEAD, and no remote poll runs for this subscriber. + state.failLocalStatus = false; + yield* TestClock.adjust(Duration.seconds(30)); + yield* Deferred.await(checkedOut); + assert.strictEqual(state.localStatusCalls, failedCalls + 1); + }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + ); + + it.effect("subscribers share one HEAD watch that ends with the last of them", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const gitDir = yield* fs.makeTempDirectoryScoped({ prefix: "vcs-head-watch-" }); + const headPath = path.join(gitDir, "HEAD"); + yield* fs.writeFileString(headPath, "ref: refs/heads/main\n"); + const state = { + currentLocalStatus: baseLocalStatus, + currentRemoteStatus: baseRemoteStatus as VcsStatusRemoteResult | null, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + headPath, + headPathCalls: 0, + }; + + yield* Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const subscribe = Effect.gen(function* () { + const received = yield* Deferred.make(); + const fiber = yield* Stream.runForEach( + broadcaster.streamStatus({ cwd: "/repo", includeRemote: false }), + () => Deferred.succeed(received, undefined), + ).pipe(Effect.forkScoped); + yield* Deferred.await(received); + return fiber; + }); + + const first = yield* subscribe; + const second = yield* subscribe; + assert.strictEqual(state.headPathCalls, 1); + + yield* Fiber.interrupt(first); + yield* Fiber.interrupt(second); + yield* subscribe; + assert.strictEqual(state.headPathCalls, 2); + }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + ); + + it.effect("a HEAD watch that ended restarts for the next subscriber and keeps earlier ones", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const gitDir = yield* fs.makeTempDirectoryScoped({ prefix: "vcs-head-watch-" }); + const headPath = path.join(gitDir, "HEAD"); + yield* fs.writeFileString(headPath, "ref: refs/heads/main\n"); + const state: Parameters[0] = { + currentLocalStatus: { ...baseLocalStatus, isDefaultRef: true, refName: "main" }, + currentRemoteStatus: null, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + headPathCalls: 0, + }; + + yield* Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const subscribe = Effect.gen(function* () { + const subscribed = yield* Deferred.make(); + const refs = new Map>(); + const sawRef = (refName: string) => { + const existing = refs.get(refName); + if (existing) return existing; + const created = Deferred.makeUnsafe(); + refs.set(refName, created); + return created; + }; + let clientState: VcsStatusResult | null = null; + const fiber = yield* Stream.runForEach( + broadcaster.streamStatus({ cwd: "/repo", includeRemote: false }), + (event) => + Effect.gen(function* () { + clientState = applyGitStatusStreamEvent(clientState, event); + yield* Deferred.succeed(subscribed, undefined); + yield* Deferred.succeed(sawRef(clientState.refName ?? ""), undefined); + }), + ).pipe(Effect.forkScoped); + yield* Deferred.await(subscribed); + return { fiber, sawRef }; + }); + const checkout = (refName: string) => + Effect.gen(function* () { + state.currentLocalStatus = { ...baseLocalStatus, isDefaultRef: false, refName }; + const lockPath = path.join(gitDir, "HEAD.lock"); + yield* fs.writeFileString(lockPath, `ref: refs/heads/${refName}\n`); + yield* fs.rename(lockPath, headPath); + }); + + // The repository has no HEAD yet, so the first watch ends at once. + const first = yield* subscribe; + state.headPath = headPath; + const second = yield* subscribe; + assert.strictEqual(state.headPathCalls, 2); + + yield* checkout("one"); + yield* Deferred.await(first.sawRef("one")); + yield* Deferred.await(second.sawRef("one")); + + // The replacement belongs to both subscribers, so the second leaving keeps it. + yield* Fiber.interrupt(second.fiber); + yield* checkout("two"); + yield* Deferred.await(first.sawRef("two")); + }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + ); + it.effect("releases remote demand while passive observers remain", () => { const state = { currentLocalStatus: baseLocalStatus, diff --git a/apps/server/src/vcs/VcsStatusBroadcaster.ts b/apps/server/src/vcs/VcsStatusBroadcaster.ts index 83ba243e9d34..4e468dd1d89b 100644 --- a/apps/server/src/vcs/VcsStatusBroadcaster.ts +++ b/apps/server/src/vcs/VcsStatusBroadcaster.ts @@ -1,11 +1,13 @@ import * as Cause from "effect/Cause"; import * as Context from "effect/Context"; +import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; import * as PubSub from "effect/PubSub"; import * as Ref from "effect/Ref"; // Coder: map local status changes onto the lightweight ref stream. @@ -41,6 +43,12 @@ import { resolveProjectAutoPull } from "./projectAutoPull.ts"; const VCS_STATUS_REFRESH_FAILURE_BASE_DELAY = Duration.seconds(30); const VCS_STATUS_REFRESH_FAILURE_MAX_DELAY = Duration.minutes(15); const MAX_FAILURE_DIAGNOSTIC_VALUES = 8; +// A held index.lock fails the read, and its removal moves no HEAD to trigger another. +const CHECKOUT_REFRESH_RETRY = Schedule.exponential(Duration.seconds(1)).pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(30))), + ), +); const MAX_FAILURE_DIAGNOSTIC_VALUE_LENGTH = 128; function boundedDiagnosticValue(value: string): string { @@ -145,6 +153,12 @@ interface CachedVcsStatus { readonly remote: CachedRemoteStatus | null; } +interface ActiveHeadWatcher { + readonly fiber: Fiber.Fiber; + readonly ready: Deferred.Deferred; + readonly subscriberCount: number; +} + interface ActiveRemotePoller { readonly fiber: Fiber.Fiber; readonly subscriberCount: number; @@ -262,6 +276,7 @@ export const make = Effect.gen(function* () { const workflow = yield* GitWorkflowService.GitWorkflowService; const backgroundPolicy = yield* BackgroundPolicy.BackgroundPolicy; const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; const changesPubSub = yield* Effect.acquireRelease( PubSub.unbounded(), (pubsub) => PubSub.shutdown(pubsub), @@ -301,6 +316,7 @@ export const make = Effect.gen(function* () { const withRemoteWriteLock = (cwd: string, effect: Effect.Effect) => remoteWriteLocks.withLock(cwd, effect); const pollersRef = yield* SynchronizedRef.make(new Map()); + const headWatchersRef = yield* SynchronizedRef.make(new Map()); const getCachedStatus = Effect.fn("VcsStatusBroadcaster.getCachedStatus")(function* ( cwd: string, @@ -547,24 +563,11 @@ export const make = Effect.gen(function* () { "VcsStatusBroadcaster.refreshStatus", )(function* (rawCwd, options) { const cwd = yield* withFileSystem(normalizeCwd(rawCwd)); - const previous = options?.localOnly ? yield* getCachedStatus(cwd) : null; - const local = yield* refreshLocalStatusCore(cwd); + yield* refreshLocalStatusCore(cwd); if (options?.localOnly) { - return yield* withRemoteWriteLock( - cwd, - Effect.gen(function* () { - if (previous?.local?.value.refName !== local.refName || !previous?.remote) { - yield* workflow.invalidateRemoteStatus(cwd); - yield* updateCachedRemoteStatus( - cwd, - { branch: local.refName, fingerprint: fingerprintStatusPart(null), value: null }, - { publish: true }, - ); - } - const parts = yield* getStatusParts(cwd); - return mergeGitStatusParts(parts.local, parts.remote); - }), - ); + // Never cache a remote here: readers take `value: null` as "read and found no PR". + const parts = yield* getStatusParts(cwd); + return mergeGitStatusParts(parts.local, parts.remote); } return yield* withRemoteWriteLock( cwd, @@ -770,52 +773,6 @@ export const make = Effect.gen(function* () { } }); - const streamStatus: VcsStatusBroadcaster["Service"]["streamStatus"] = (input) => - Stream.unwrap( - Effect.gen(function* () { - const cwd = yield* withFileSystem(normalizeCwd(input.cwd)); - yield* Effect.acquireRelease( - Effect.sync(() => { - const previous = detailedSubscribers.get(cwd) ?? 0; - detailedSubscribers.set(cwd, previous + 1); - }), - () => - Effect.sync(() => { - const remaining = (detailedSubscribers.get(cwd) ?? 1) - 1; - if (remaining === 0) detailedSubscribers.delete(cwd); - else detailedSubscribers.set(cwd, remaining); - }), - ); - const subscription = yield* PubSub.subscribe(changesPubSub); - const initialLocal = yield* getOrLoadLocalStatus(cwd); - const cachedStatus = yield* getCachedStatus(cwd); - const initialRemote = cachedStatus?.remote?.value ?? null; - if (input.includeRemote !== false) { - yield* retainRemotePoller( - cwd, - input.cwd, - cachedStatus?.remote === null || cachedStatus?.remote === undefined, - ); - } - const release = - input.includeRemote === false - ? Effect.void - : releaseRemotePoller(cwd, input.cwd).pipe(Effect.ignore, Effect.asVoid); - - return Stream.concat( - Stream.make({ - _tag: "snapshot" as const, - local: initialLocal, - remote: initialRemote, - }), - Stream.fromSubscription(subscription).pipe( - Stream.filter((event) => event.cwd === cwd), - Stream.map((event) => event.event), - ), - ).pipe(Stream.ensuring(release)); - }), - ); - // Coder: subscribe before the non-mutating ref snapshot so concurrent updates are retained. const streamRefStatus: VcsStatusBroadcaster["Service"]["streamRefStatus"] = (rawCwd) => Stream.unwrap( @@ -877,6 +834,151 @@ export const make = Effect.gen(function* () { ), ); + // A checkout outside T3 Code, such as `git switch` in the terminal, calls no refresh RPC. + // Rereading remote too, without a fetch, sends the new branch with its own PR and counts. + const refreshCheckedOutRef = Effect.fn("VcsStatusBroadcaster.refreshCheckedOutRef")(function* ( + cwd: string, + ) { + if (!detailedSubscribers.has(cwd)) { + yield* refreshBranchStatusCore(cwd); + } else if ((yield* getCachedStatus(cwd))?.remote) { + yield* refreshRemoteStatus(cwd, { refreshUpstream: false }); + } else { + yield* refreshLocalStatusCore(cwd); + } + }); + + // Runs for the subscription lifetime, so tracing would leave a span open indefinitely. + const watchHead = Effect.fnUntraced(function* (cwd: string, ready: Deferred.Deferred) { + const headPath = yield* workflow.headPath(cwd); + if (headPath === null) return; + // Git replaces HEAD by renaming HEAD.lock, which a watch on the file itself would lose. + yield* fs + .watch(path.dirname(headPath), { + onReady: () => Deferred.doneUnsafe(ready, Exit.void), + }) + .pipe( + Stream.filter((event) => path.basename(event.path) === "HEAD"), + // A rebase moves HEAD many times; refresh once for whatever arrived during a refresh. + Stream.buffer({ capacity: 1, strategy: "sliding" }), + Stream.runForEach(() => + refreshCheckedOutRef(cwd).pipe( + Effect.retry(CHECKOUT_REFRESH_RETRY), + Effect.catchCause((cause) => + Effect.logWarning("VCS checked-out ref refresh failed", { + cwdLength: cwd.length, + ...remoteRefreshFailureDiagnostics(cause), + }), + ), + ), + ), + ); + }); + + const retainHeadWatcher = Effect.fn("VcsStatusBroadcaster.retainHeadWatcher")(function* ( + cwd: string, + ) { + const ready = yield* SynchronizedRef.modifyEffect(headWatchersRef, (activeWatchers) => { + const existing = activeWatchers.get(cwd); + if (existing && existing.fiber.pollUnsafe() === undefined) { + const nextWatchers = new Map(activeWatchers).set(cwd, { + ...existing, + subscriberCount: existing.subscriberCount + 1, + }); + return Effect.succeed([existing.ready, nextWatchers] as const); + } + // A watch that ended, say before the repository existed, restarts for everyone holding it. + return Effect.gen(function* () { + const ready = yield* Deferred.make(); + const fiber = yield* watchHead(cwd, ready).pipe( + Effect.catchCause((cause) => + Effect.logWarning("VCS HEAD watch failed", { + cwdLength: cwd.length, + ...remoteRefreshFailureDiagnostics(cause), + }), + ), + // Subscribers wait for the watch, so one that cannot start must not hold them. + Effect.ensuring(Deferred.succeed(ready, undefined)), + Effect.forkIn(broadcasterScope), + ); + const nextWatchers = new Map(activeWatchers).set(cwd, { + fiber, + ready, + subscriberCount: (existing?.subscriberCount ?? 0) + 1, + }); + return [ready, nextWatchers] as const; + }); + }); + yield* Deferred.await(ready); + }); + + const releaseHeadWatcher = Effect.fn("VcsStatusBroadcaster.releaseHeadWatcher")(function* ( + cwd: string, + ) { + const watcherToInterrupt = yield* SynchronizedRef.modify(headWatchersRef, (activeWatchers) => { + const existing = activeWatchers.get(cwd); + if (!existing) return [null, activeWatchers] as const; + const nextWatchers = new Map(activeWatchers); + if (existing.subscriberCount > 1) { + nextWatchers.set(cwd, { ...existing, subscriberCount: existing.subscriberCount - 1 }); + return [null, nextWatchers] as const; + } + nextWatchers.delete(cwd); + return [existing.fiber, nextWatchers] as const; + }); + if (watcherToInterrupt) { + yield* Fiber.interrupt(watcherToInterrupt).pipe(Effect.ignore); + } + }); + + const streamStatus: VcsStatusBroadcaster["Service"]["streamStatus"] = (input) => + Stream.unwrap( + Effect.gen(function* () { + const cwd = yield* withFileSystem(normalizeCwd(input.cwd)); + yield* Effect.acquireRelease( + Effect.sync(() => { + const previous = detailedSubscribers.get(cwd) ?? 0; + detailedSubscribers.set(cwd, previous + 1); + }), + () => + Effect.sync(() => { + const remaining = (detailedSubscribers.get(cwd) ?? 1) - 1; + if (remaining === 0) detailedSubscribers.delete(cwd); + else detailedSubscribers.set(cwd, remaining); + }), + ); + const subscription = yield* PubSub.subscribe(changesPubSub); + // Watch before reading so a checkout in between still publishes. + yield* Effect.acquireRelease(retainHeadWatcher(cwd), () => releaseHeadWatcher(cwd)); + const initialLocal = yield* getOrLoadLocalStatus(cwd); + const cachedStatus = yield* getCachedStatus(cwd); + const initialRemote = cachedStatus?.remote?.value ?? null; + if (input.includeRemote !== false) { + yield* retainRemotePoller( + cwd, + input.cwd, + cachedStatus?.remote === null || cachedStatus?.remote === undefined, + ); + } + const release = + input.includeRemote === false + ? Effect.void + : releaseRemotePoller(cwd, input.cwd).pipe(Effect.ignore, Effect.asVoid); + + return Stream.concat( + Stream.make({ + _tag: "snapshot" as const, + local: initialLocal, + remote: initialRemote, + }), + Stream.fromSubscription(subscription).pipe( + Stream.filter((event) => event.cwd === cwd), + Stream.map((event) => event.event), + ), + ).pipe(Stream.ensuring(release)); + }), + ); + const refreshBranchStatus: VcsStatusBroadcaster["Service"]["refreshBranchStatus"] = Effect.fn( "VcsStatusBroadcaster.refreshBranchStatus", )(function* (input) { @@ -891,6 +993,7 @@ export const make = Effect.gen(function* () { Effect.gen(function* () { const cwd = yield* withFileSystem(normalizeCwd(input.cwd)); const subscription = yield* PubSub.subscribe(branchChangesPubSub); + yield* Effect.acquireRelease(retainHeadWatcher(cwd), () => releaseHeadWatcher(cwd)); const branch = yield* refreshBranchStatusCore(cwd); return Stream.concat( Stream.make({ refName: branch.refName }), diff --git a/apps/server/src/workspace/WorkspaceEntries.test.ts b/apps/server/src/workspace/WorkspaceEntries.test.ts index 404897631e52..4db37e53002c 100644 --- a/apps/server/src/workspace/WorkspaceEntries.test.ts +++ b/apps/server/src/workspace/WorkspaceEntries.test.ts @@ -434,6 +434,20 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { }), ); + it.effect("matches square brackets in the query literally", () => + Effect.gen(function* () { + const cwd = yield* makeTempDir({ prefix: "t3code-workspace-brackets-" }); + yield* writeTextFile(cwd, "src/app/game/[id]/index.tsx"); + + for (const query of ["game/[i", "game/[id]"]) { + const result = yield* searchWorkspaceEntries({ cwd, query, limit: 10 }); + expect(result.entries.map((entry) => entry.path)).toContain( + "src/app/game/[id]/index.tsx", + ); + } + }), + ); + it.effect("excludes gitignored paths for git repositories", () => Effect.gen(function* () { const cwd = yield* makeTempDir({ prefix: "t3code-workspace-gitignore-", git: true }); diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts index 2c41e4ed0b3d..2f9ac05bf9e3 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts @@ -5,8 +5,12 @@ import { type GrepOptions, type GrepResult, } from "@ff-labs/fff-node"; +import * as NodeServices from "@effect/platform-node/NodeServices"; import { afterEach, expect, it } from "@effect/vitest"; +import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; import { vi } from "vite-plus/test"; import * as WorkspaceSearchIndex from "./WorkspaceSearchIndex.ts"; @@ -139,3 +143,34 @@ it.effect("continues whole-word searches after a filtered grep page", () => }), ), ); + +// fff reads HOME on Linux and macOS but the user profile on Windows, so a temp HOME cannot stand in there. +it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( + "refuses to index a workspace rooted at the home directory", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const home = yield* fs.realPath( + yield* fs.makeTempDirectoryScoped({ prefix: "t3-search-home-" }), + ); + yield* fs.makeDirectory(path.join(home, "OneDrive", "Remix"), { recursive: true }); + yield* fs.writeFileString(path.join(home, "OneDrive", "Remix", "part.stl"), "solid part\n"); + yield* Effect.acquireRelease( + Effect.sync(() => vi.stubEnv("HOME", home)), + () => Effect.sync(() => vi.unstubAllEnvs()), + ); + + const result = yield* WorkspaceSearchIndex.make(home).pipe( + Effect.flatMap((searchIndex) => searchIndex.search("stl", 50, undefined, false)), + Effect.map(({ entries }) => entries), + Effect.catchTags({ + WorkspaceSearchIndexCreateFailed: (error) => Effect.succeed(error._tag), + }), + ); + + expect(result).toBe("WorkspaceSearchIndexCreateFailed"); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.ts b/apps/server/src/workspace/WorkspaceSearchIndex.ts index ecb8c6ba87a7..11a19e183986 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.ts @@ -296,8 +296,6 @@ const createFinder = Effect.fn("WorkspaceSearchIndex.createFinder")(function* ( // Content indexing costs CPU/memory; path-only consumers use the lightweight index. disableContentIndexing: variant !== "content", aiMode: false, - enableFsRootScanning: true, - enableHomeDirScanning: true, }), catch: (cause) => new WorkspaceSearchIndexCreateFailed({ @@ -451,7 +449,10 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( const search: WorkspaceSearchIndex["Service"]["search"] = Effect.fn( "WorkspaceSearchIndex.search", - )(function* (query, limit, kind, imageOnly) { + )(function* (rawQuery, limit, kind, imageOnly) { + // fff parses any word containing "[" as a glob and has no escape for it; fuzzy + // matching is subsequence-based, so dropping "[" still matches "[id]" folders. + const query = rawQuery.replaceAll("[", ""); const pageSize = imageOnly ? WORKSPACE_INDEX_PAGE_SIZE : Math.max(1, limit + 1); const incompleteBeforeQuery = hasIncompleteInitialScan(); if (kind === "file" || imageOnly) { diff --git a/apps/server/src/ws.projects.test.ts b/apps/server/src/ws.projects.test.ts index e11831037275..4c3678c74508 100644 --- a/apps/server/src/ws.projects.test.ts +++ b/apps/server/src/ws.projects.test.ts @@ -57,6 +57,7 @@ import * as ProviderInstanceRegistry from "./provider/ProviderInstanceRegistry.t import * as ProviderLatestVersions from "@t3tools/provider-core/server/ProviderLatestVersions"; import * as ProviderRegistry from "./provider/ProviderRegistry.ts"; import * as AcpRegistryRuntimeCoordinator from "@t3tools/provider-acp-registry/server/AcpRegistryRuntimeCoordinator"; +import * as AcpRegistrySessions from "./provider/AcpRegistrySessions.ts"; import * as PullRequestService from "./pullRequest/PullRequestService.ts"; import * as PullRequestReviewGuideService from "./pullRequest/PullRequestReviewGuideService.ts"; import * as ReviewService from "./review/ReviewService.ts"; @@ -239,6 +240,7 @@ const harness = ( stub(DocumentText.DocumentText), stub(ProcessRunner.ProcessRunner), stub(AcpRegistryRuntimeCoordinator.AcpRegistryRuntimeCoordinator), + stub(AcpRegistrySessions.AcpRegistrySessions), Layer.succeed(BuiltInDrivers.SourceControlDrivers, []), ServerConfig.layer(config), WorkspacePaths.layer, diff --git a/apps/server/src/ws.test.ts b/apps/server/src/ws.test.ts index b1ab7affe2b5..c4ebef17c6b5 100644 --- a/apps/server/src/ws.test.ts +++ b/apps/server/src/ws.test.ts @@ -552,7 +552,8 @@ it.effect("shell snapshots omit idle children and older settled threads only on // session-scoped `layerWsRpc` with mocks of the auth, preview, device, and usage services the // helper does not serve; the helper's `subscribeServerConfig` keeps upstream's // subscribe-before-snapshot order and strips workspace snapshots the same way. Upstream's -// failed-publish remote refresh test (#29688) builds the same layer, so it is not carried either. +// failed-publish remote refresh test (#29688) and missing-worktree skills test (#32471) build +// the same layer, so they are not carried either. // Pruned history cannot be replayed: a cursor below the boundary must resync from a snapshot, // or, for a shell client that loads it over HTTP, be told to without the server building one. @@ -742,3 +743,7 @@ it.effect("thread resume at its own high-water below the pruned boundary skips t assert.isTrue(Exit.isSuccess(exit)); }).pipe(Effect.scoped, Effect.provide(ReplayTestLayer)), ); + +// Coder: upstream's "ACP session RPCs list a thread missing its history and repair it on import" +// test (#22155) serves upstream's session-scoped `layerWsRpc`; `AcpRegistrySessions.ts` carries +// the repair, and the helper serves the two RPCs through the same service. diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 4cb421c386d5..1b3a7f22b936 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -11,14 +11,12 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as Queue from "effect/Queue"; -import * as Result from "effect/Result"; import * as Stream from "effect/Stream"; import { rpcInitialItems } from "./rpcInitialItems.ts"; import { AcpRegistryOperationError, CommandId, type GitActionProgressEvent, - type AcpRegistryImportSessionInput, type AcpRegistryListSessionsInput, type GitManagerServiceError, OrchestrationGetFullThreadDiffError, @@ -47,9 +45,6 @@ import { ProjectWriteFileError, ProjectMutationError, type ServerLifecycleStreamEvent, - type ProjectId, - type ProviderDriverKind, - type ProviderInstanceId, type ServerProvider, type FilesystemBrowseFailure, FilesystemBrowseError, @@ -126,8 +121,8 @@ import * as ThreadSearch from "./orchestration-v2/ThreadSearch.ts"; import * as OrchestrationEventStore from "./persistence/OrchestrationEventStore.ts"; import { userFacingDispatchErrorMessage } from "./orchestration-v2/UserFacingErrors.ts"; import * as ProviderRegistry from "./provider/ProviderRegistry.ts"; +import * as AcpRegistrySessions from "./provider/AcpRegistrySessions.ts"; import * as ProviderInstanceRegistry from "./provider/ProviderInstanceRegistry.ts"; -import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProviderMaintenanceRunner from "./provider/providerMaintenanceRunner.ts"; import * as ServerLifecycleEvents from "./serverLifecycleEvents.ts"; import * as ServerSettings from "./serverSettings.ts"; @@ -162,9 +157,9 @@ import * as SqlClient from "effect/sql/SqlClient"; import * as PullRequestSyncReactor from "./orchestration-v2/PullRequestSyncReactor.ts"; import * as SourceControlDiscovery from "./sourceControl/SourceControlDiscovery.ts"; import * as SourceControlRepositoryService from "./sourceControl/SourceControlRepositoryService.ts"; +import * as SourceControlRateLimit from "@t3tools/source-control-core/server/SourceControlRateLimit"; import * as AgentSessionScanner from "./project/AgentSessionScanner.ts"; import * as AgentSessionImporter from "./project/AgentSessionImporter.ts"; -import * as AcpRegistryRuntimeCoordinator from "@t3tools/provider-acp-registry/server/AcpRegistryRuntimeCoordinator"; // Coder: the helper's group, its Coder services, and the fork-only methods. import { CoderWsRpcGroup } from "@t3tools/contracts"; import { DEFAULT_RESOLVED_KEYBINDINGS } from "@t3tools/shared/keybindings"; @@ -1040,9 +1035,8 @@ const layerWsRpc = () => const portDiscovery = yield* PortScanner.PortDiscovery; const providerRegistry = yield* ProviderRegistry.ProviderRegistry; const providerInstances = yield* ProviderInstanceRegistry.ProviderInstanceRegistry; - // Coder: of upstream's ACP Registry services, only the runtime coordinator is used here. - const acpRegistryRuntimeCoordinator = - yield* AcpRegistryRuntimeCoordinator.AcpRegistryRuntimeCoordinator; + // Coder: of upstream's ACP Registry services, only the session service is used here. + const acpRegistrySessions = yield* AcpRegistrySessions.AcpRegistrySessions; const providerMaintenanceRunner = yield* ProviderMaintenanceRunner.ProviderMaintenanceRunner; const config = yield* ServerConfig.ServerConfig; const lifecycleEvents = yield* ServerLifecycleEvents.ServerLifecycleEvents; @@ -1061,67 +1055,16 @@ const layerWsRpc = () => yield* SourceControlRepositoryService.SourceControlRepositoryService; // Coder: GitLab uses the workspace glab login; no viewer routing credential exists. const withPullRequestViewer = (_reference: unknown, effect: T) => effect; + // The panel's reads for a pull request the user opened go out during a rate-limit pause. + const openedPullRequestRead = Effect.provideService(SourceControlRateLimit.Interactive, true); // Coder: of upstream's ACP Registry handlers, only Kilo Code's native session list and // import are carried. - const acpRegistryProject = Effect.fn("ws.acpRegistry.project")(function* ( - projectId: ProjectId, - ) { - const project = yield* projectService.getById(projectId).pipe( - Effect.mapError( - (cause) => - new AcpRegistryOperationError({ - reason: "project_not_found", - message: `Project ${projectId} is unavailable.`, - cause, - }), - ), - ); - return yield* Effect.fromOption( - project, - () => - new AcpRegistryOperationError({ - reason: "project_not_found", - message: `Project ${projectId} was not found.`, - }), - ); - }); - - const acpSessionManager = Effect.fn("ws.acpRegistry.sessionManager")(function* ( - instanceId: ProviderInstanceId, - ) { - const instance = yield* providerInstances.getInstance(instanceId); - if (instance === undefined) { - return yield* new AcpRegistryOperationError({ - reason: "instance_not_found", - message: `Provider instance ${instanceId} was not found.`, - }); - } - if (instance.acpSessionManagement === undefined) { - return yield* new AcpRegistryOperationError({ - reason: "session_list_unsupported", - message: `Provider instance ${instanceId} does not expose ACP session management.`, - }); - } - return { instance, manager: instance.acpSessionManagement }; - }); - - const importedAcpThreadId = (input: { - readonly driver: ProviderDriverKind; - readonly instanceId: ProviderInstanceId; - readonly sessionId: string; - }) => - IdAllocator.deriveThreadFromProviderThread({ - driver: input.driver, - providerInstanceId: input.instanceId, - nativeThreadId: input.sessionId, - }); - const listAcpRegistrySessions = Effect.fn("ws.acpRegistry.listSessions")(function* ( input: AcpRegistryListSessionsInput, ) { - const project = yield* acpRegistryProject(input.projectId); - const { instance, manager } = yield* acpSessionManager(input.instanceId); + const project = yield* acpRegistrySessions.project(input.projectId); + const { instance, manager } = yield* acpRegistrySessions.sessionManager(input.instanceId); const listed = yield* manager.listSessions({ cwd: project.workspaceRoot, ...(input.cursor === undefined ? {} : { cursor: input.cursor }), @@ -1129,7 +1072,7 @@ const layerWsRpc = () => const sessions = yield* Effect.forEach( listed.sessions, (session) => { - const threadId = importedAcpThreadId({ + const threadId = AcpRegistrySessions.importedThreadId({ driver: instance.driverKind, instanceId: input.instanceId, sessionId: session.sessionId, @@ -1138,6 +1081,13 @@ const layerWsRpc = () => Effect.map((thread) => ({ ...session, importedThreadId: thread === null ? null : threadId, + ...(thread !== null && + AgentSessionImporter.isNativeHistoryPending({ + historyOrigin: thread.historyOrigin, + hasRun: thread.latestRunId !== null, + }) + ? { nativeHistoryPending: true } + : {}), })), Effect.mapError( (cause) => @@ -1154,99 +1104,6 @@ const layerWsRpc = () => return { ...listed, sessions }; }); - const importAcpRegistrySession = Effect.fn("ws.acpRegistry.importSession")(function* ( - input: AcpRegistryImportSessionInput, - ) { - return yield* acpRegistryRuntimeCoordinator.withSessionMutation( - Effect.gen(function* () { - yield* acpRegistryProject(input.projectId); - const { instance } = yield* acpSessionManager(input.instanceId); - const providerSnapshot = yield* instance.snapshot.getSnapshot; - if ( - providerSnapshot.nativeSessions?.canLoad !== true && - providerSnapshot.nativeSessions?.canResume !== true - ) { - return yield* new AcpRegistryOperationError({ - reason: "session_resume_unsupported", - message: "The ACP agent cannot load or resume native sessions.", - }); - } - const threadId = importedAcpThreadId({ - driver: instance.driverKind, - instanceId: input.instanceId, - sessionId: input.sessionId, - }); - const existing = yield* threadManagement.getThreadShell(threadId).pipe( - Effect.mapError( - (cause) => - new AcpRegistryOperationError({ - reason: "session_import_failed", - message: "Could not inspect the imported ACP session mapping.", - cause, - }), - ), - ); - if (existing !== null) return { threadId, imported: false } as const; - - const provider = (yield* providerRegistry.getProviders).find( - (candidate) => candidate.instanceId === input.instanceId, - ); - const model = - provider?.models.find((candidate) => candidate.isDefault)?.slug ?? - provider?.models[0]?.slug ?? - "default"; - const commandId = CommandId.make(yield* random.uuidV4()); - // Coder: no startup command queue; workspace startup completes before this layer is - // built. - const launched = yield* Effect.result( - threadLaunch.launch({ - commandId, - threadId, - projectId: input.projectId, - title: input.title ?? "Imported ACP session", - modelSelection: { instanceId: input.instanceId, model }, - runtimeMode: "approval-required", - interactionMode: "default", - workspaceStrategy: { type: "root" }, - importedNativeThread: { - ref: { - driver: instance.driverKind, - nativeId: input.sessionId, - strength: "strong", - }, - metadata: { - itemIdentityVersion: 2, - ...(input.title === undefined ? {} : { title: input.title }), - ...(input.updatedAt === undefined ? {} : { updatedAt: input.updatedAt }), - }, - }, - createdBy: "user", - creationSource: "web", - }), - ); - if (Result.isFailure(launched)) { - const racedImport = yield* threadManagement.getThreadShell(threadId).pipe( - Effect.mapError( - (cause) => - new AcpRegistryOperationError({ - reason: "session_import_failed", - message: "Could not inspect the imported ACP session after launch failed.", - cause, - }), - ), - ); - if (racedImport !== null) return { threadId, imported: false } as const; - return yield* new AcpRegistryOperationError({ - reason: "session_import_failed", - message: "Could not create a T3 thread for the ACP session.", - cause: launched.failure, - }); - } - return { threadId, imported: true } as const; - }), - ); - }); - // Coder: publish the trimmed workspace config without auth, editors, telemetry, usage // limits, or event-loop stall notices. const loadServerConfig = Effect.fn("ws.loadServerConfig")(function* (options: { @@ -1538,67 +1395,97 @@ const layerWsRpc = () => "orchestration_v2.project_id": input.projectId, }).pipe( Effect.andThen( - ThreadMessageIntake.launchThread({ - commandId: input.commandId, - ...(input.threadId === undefined ? {} : { threadId: input.threadId }), - ...(input.reuseExistingThread === undefined - ? {} - : { reuseExistingThread: input.reuseExistingThread }), - projectId: input.projectId, - title: input.title, - ...(input.generateTitle === undefined - ? {} - : { generateTitle: input.generateTitle }), - modelSelection: input.modelSelection, - runtimeMode: input.runtimeMode, - interactionMode: input.interactionMode, - workspaceStrategy: input.workspaceStrategy, - ...(input.initialMessage === undefined - ? {} - : { - initialMessage: { - ...(input.initialMessage.messageId === undefined - ? {} - : { messageId: input.initialMessage.messageId }), - text: input.initialMessage.text, - attachments: input.initialMessage.attachments, - ...(input.initialMessage.context === undefined - ? {} - : { context: input.initialMessage.context }), - }, - }), - createdBy: "user", - creationSource: input.creationSource ?? "web", - }).pipe( - Effect.provide(intakeContext), - Effect.map((result) => ({ - ...result, - projection: projectThreadProjectionForWire(result.projection), - })), - Effect.catchTags({ - AttachmentClaimError: (cause) => - new OrchestrationV2ThreadLaunchError({ - commandId: input.commandId, - projectId: input.projectId, - message: cause.message, - cause, - }), - ThreadLaunchError: (cause) => - new OrchestrationV2ThreadLaunchError({ - commandId: input.commandId, - projectId: input.projectId, - message: "Failed to launch thread", - cause, - }), - ThreadManagementProviderInstanceDisabledError: (cause) => - new OrchestrationV2ThreadLaunchError({ + threadLaunch + .checkWorktreeBase({ + commandId: input.commandId, + projectId: input.projectId, + workspaceStrategy: input.workspaceStrategy, + }) + .pipe( + Effect.andThen( + ThreadMessageIntake.launchThread({ commandId: input.commandId, + ...(input.threadId === undefined ? {} : { threadId: input.threadId }), + ...(input.reuseExistingThread === undefined + ? {} + : { reuseExistingThread: input.reuseExistingThread }), projectId: input.projectId, - message: cause.message, - cause, + title: input.title, + ...(input.generateTitle === undefined + ? {} + : { generateTitle: input.generateTitle }), + modelSelection: input.modelSelection, + runtimeMode: input.runtimeMode, + interactionMode: input.interactionMode, + workspaceStrategy: input.workspaceStrategy, + ...(input.initialMessage === undefined + ? {} + : { + initialMessage: { + ...(input.initialMessage.messageId === undefined + ? {} + : { messageId: input.initialMessage.messageId }), + text: input.initialMessage.text, + attachments: input.initialMessage.attachments, + ...(input.initialMessage.context === undefined + ? {} + : { context: input.initialMessage.context }), + }, + }), + createdBy: "user", + creationSource: input.creationSource ?? "web", }), - }), - ), + ), + Effect.provide(intakeContext), + Effect.map((result) => ({ + ...result, + projection: projectThreadProjectionForWire(result.projection), + })), + Effect.catchTags({ + AttachmentClaimError: (cause) => + new OrchestrationV2ThreadLaunchError({ + commandId: input.commandId, + projectId: input.projectId, + message: cause.message, + cause, + }), + ThreadLaunchError: (cause) => + new OrchestrationV2ThreadLaunchError({ + commandId: input.commandId, + projectId: input.projectId, + message: "Failed to launch thread", + cause, + }), + ThreadManagementProviderInstanceDisabledError: (cause) => + new OrchestrationV2ThreadLaunchError({ + commandId: input.commandId, + projectId: input.projectId, + message: cause.message, + cause, + }), + ThreadLaunchBaseRefError: (cause) => + new OrchestrationV2ThreadLaunchError({ + commandId: input.commandId, + projectId: input.projectId, + message: cause.message, + cause, + }), + ThreadLaunchBranchConflictError: (cause) => + new OrchestrationV2ThreadLaunchError({ + commandId: input.commandId, + projectId: input.projectId, + message: cause.message, + cause, + }), + ThreadLaunchNotRepositoryError: (cause) => + new OrchestrationV2ThreadLaunchError({ + commandId: input.commandId, + projectId: input.projectId, + message: cause.message, + cause, + }), + }), + ), ), ), // Coder: the stream takes its services from `streamContext`. @@ -1648,6 +1535,10 @@ const layerWsRpc = () => [WS_METHODS.serverRefreshProviders]: /* Request instrumentation already owns the span; keep this helper in that context. */ Effect.fnUntraced(function* (input) { + if (input.modelCatalogOnly === true) { + yield* providerRegistry.refreshModelCatalog; + return { providers: [] }; + } if (input.refreshModels) { yield* providerRegistry.invalidateModelCaches({ instanceId: input.instanceId, @@ -1661,6 +1552,7 @@ const layerWsRpc = () => ? providerRegistry.refreshWorkspaceSnapshot({ instanceId: input.instanceId, cwd: input.cwd, + scanCwd: yield* projectService.workspaceScanDirectory(input.cwd), fresh: input.fresh === true, }) : input.reloadSecrets === true @@ -1730,7 +1622,7 @@ const layerWsRpc = () => Effect.annotateCurrentSpan({ "provider.instance_id": input.instanceId, "project.id": input.projectId, - }).pipe(Effect.andThen(importAcpRegistrySession(input))), + }).pipe(Effect.andThen(acpRegistrySessions.importSession(input))), [WS_METHODS.pullRequestsList]: (input) => pullRequests.list(input), [WS_METHODS.pullRequestsListStats]: (input) => pullRequests.listStats(input), [WS_METHODS.pullRequestsFilterOptions]: (input) => pullRequests.filterOptions(input), @@ -1749,13 +1641,13 @@ const layerWsRpc = () => ), ), [WS_METHODS.pullRequestsDetail]: (input) => - withPullRequestViewer(input, pullRequests.detail(input)), + withPullRequestViewer(input, openedPullRequestRead(pullRequests.detail(input))), [WS_METHODS.pullRequestsPreview]: (input) => withPullRequestViewer(input, pullRequests.preview(input)), [WS_METHODS.pullRequestsChecks]: (input) => withPullRequestViewer(input, pullRequests.checks(input)), [WS_METHODS.pullRequestsActivity]: (input) => - withPullRequestViewer(input, pullRequests.activity(input)), + withPullRequestViewer(input, openedPullRequestRead(pullRequests.activity(input))), [WS_METHODS.pullRequestsReviewGuide]: (input) => withPullRequestViewer(input, reviewGuides.get(input)), [WS_METHODS.pullRequestsGenerateReviewGuide]: (input) => @@ -1836,12 +1728,13 @@ const layerWsRpc = () => }) .pipe( Effect.asVoid, - Effect.mapError( - (cause) => - new OrchestrationDispatchCommandError({ - message: "Failed to create clone project.", - cause, - }), + Effect.mapError((cause) => + cause._tag === "ProjectConflictError" + ? cause + : new OrchestrationDispatchCommandError({ + message: "Failed to create clone project.", + cause, + }), ), ), onCloned: (project) => @@ -2100,7 +1993,9 @@ const layerWsRpc = () => }) .pipe( Effect.matchCauseEffect({ - onFailure: (cause) => Queue.failCause(queue, cause), + // Earlier steps (branch, commit) may have landed before the failing one. + onFailure: (cause) => + refreshGitStatus(input.cwd).pipe(Effect.andThen(Queue.failCause(queue, cause))), onSuccess: (result) => (input.threadId === undefined ? Effect.void diff --git a/apps/web/index.html b/apps/web/index.html index a7a5e7450b3d..ef9517a884a5 100644 --- a/apps/web/index.html +++ b/apps/web/index.html @@ -668,7 +668,10 @@ white-space: nowrap; } - T3 Code (Beta) + + T3 Code

diff --git a/apps/web/package.json b/apps/web/package.json index 6185d8d66aee..20876054b3c3 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -1,6 +1,6 @@ { "name": "@t3tools/web", - "version": "0.1.7", + "version": "0.1.9", "private": true, "type": "module", "scripts": { @@ -14,8 +14,9 @@ "@base-ui/react": "^1.9.0", "@betteroffice/docx": "0.4.3", "@betteroffice/fonts": "0.4.0", - "@betteroffice/pptx": "0.2.0", - "@betteroffice/xlsx": "0.3.0", + "@betteroffice/fonts-cjk": "0.4.0", + "@betteroffice/pptx": "0.5.1", + "@betteroffice/xlsx": "0.5.1", "@daypicker/react": "^10.0.1", "@dnd-kit/core": "^6.3.1", "@dnd-kit/modifiers": "^9.0.0", @@ -24,6 +25,7 @@ "@effect/atom-react": "catalog:", "@fontsource-variable/inter": "^5.3.0", "@legendapp/list": "catalog:", + "@noble/hashes": "catalog:", "@pierre/diffs": "catalog:", "@pierre/trees": "1.0.0-beta.6", "@t3tools/client-runtime": "workspace:*", @@ -60,11 +62,9 @@ "jsonc-parser": "3.3.1", "jszip": "3.10.1", "katex": "^0.18.2", - "lucide": "^0.564.0", "lucide-react": "^0.564.0", "mdast-util-from-markdown": "^2.0.3", "micromark-util-combine-extensions": "^2.0.1", - "morphicons": "^1.7.1", "pdfjs-dist": "6.4.299", "pele": "catalog:", "react": "catalog:", @@ -78,7 +78,6 @@ "sql.js": "1.15.0", "tailwind-merge": "^3.4.0", "tiff": "7.1.3", - "unbash": "^5.0.0", "unified": "catalog:", "use-sync-external-store": "^1.6.0", "vfile": "^6.0.3", @@ -102,7 +101,7 @@ "jsdom": "^30.0.1", "micromark-util-types": "catalog:", "oxc-transform-react": "~0.152.0", - "playwright": "1.60.0", + "playwright": "1.64.1", "react-test-renderer": "19.3.0", "tailwindcss": "^4.0.0", "vite": "catalog:", diff --git a/apps/web/src/appInit.tsx b/apps/web/src/appInit.tsx index f5fa3787faef..9b57d5884dd6 100644 --- a/apps/web/src/appInit.tsx +++ b/apps/web/src/appInit.tsx @@ -1,4 +1,9 @@ -import { scopedThreadKey, scopeThreadRef } from "@t3tools/client-runtime/environment"; +import { + scopedThreadKey, + scopeProjectRef, + scopeThreadRef, +} from "@t3tools/client-runtime/environment"; +import type { ScopedThreadRef } from "@t3tools/contracts"; import type { ClientSettings } from "@t3tools/contracts/settings"; import { createBrowserHistory } from "@tanstack/react-router"; import * as Schema from "effect/Schema"; @@ -14,7 +19,8 @@ import { startSlowRpcRequestToasts } from "./components/SlowRpcRequestToastCoord import { startThreadNotifications } from "./components/ThreadNotificationCoordinator"; import { startGuideRunNotifications } from "./components/pullRequest/guide/PullRequestGuideRunCoordinator"; import { APP_BASE_NAME, APP_DISPLAY_NAME, APP_STAGE_LABEL } from "./branding"; -import { resolveServerBackedAppDisplayName } from "./branding.logic"; +import { resolveServerBackedAppDisplayName, resolveWindowTitle } from "./branding.logic"; +import { isElectron } from "./env"; import { subscribeComposerDraftModelReconciliation, subscribeDraftThreadPromotion, @@ -31,13 +37,14 @@ import { syncBrowserChromeTheme } from "./hooks/useTheme"; import { startThreadVisitedMigration } from "./hooks/useThreadVisitedMigration"; import { getRouter, type AppRouter } from "./router"; import { appAtomRegistry } from "./rpc/atomRegistry"; +import { environmentProjects } from "./state/projects"; import { primaryServerConfigAtom } from "./state/server"; import { environmentShell } from "./state/shell"; import { installChatRouteCommands } from "./shortcuts/chatRouteCommands"; import { startWorkspaceFocus } from "./lib/workspaceFocus"; import { installCommandPaletteCommands } from "./shortcuts/commandPaletteCommands"; import { environmentThreadShells, threadEnvironment } from "./state/threads"; -import { resolveThreadRouteTarget } from "./threadRoutes"; +import { resolveActiveThreadRouteRef, resolveThreadRouteTarget } from "./threadRoutes"; /** Writes appearance settings onto the document, touching only the groups that changed. */ function startClientAppearance(): void { @@ -99,19 +106,74 @@ function startClientAppearance(): void { apply(); } -function startDocumentTitle(): void { - appAtomRegistry.subscribe( - primaryServerConfigAtom, - (config) => { - document.title = resolveServerBackedAppDisplayName({ +function readOpenThreadRef(router: AppRouter): ScopedThreadRef | null { + const target = resolveThreadRouteTarget(router.state.matches.at(-1)?.params ?? {}); + const draft = + target?.kind === "draft" + ? useComposerDraftStore.getState().getDraftSession(target.draftId) + : null; + return resolveActiveThreadRouteRef(target, draft); +} + +/** Names the open thread and its project in the window title, or the app when none is open. */ +function startDocumentTitle(router: AppRouter): void { + // Coder: no guest page, which titles its own tab upstream. + let openThreadRef = readOpenThreadRef(router); + const write = () => { + const thread = + openThreadRef === null + ? null + : appAtomRegistry.get(environmentThreadShells.quietThreadShellAtom(openThreadRef)); + const project = + thread === null + ? null + : appAtomRegistry.get( + environmentProjects.projectAtom( + scopeProjectRef(thread.environmentId, thread.projectId), + ), + ); + const title = resolveWindowTitle({ + appDisplayName: resolveServerBackedAppDisplayName({ baseName: APP_BASE_NAME, fallbackDisplayName: APP_DISPLAY_NAME, fallbackStageLabel: APP_STAGE_LABEL, - primaryServerVersion: config?.environment.serverVersion ?? null, - }); - }, - { immediate: true }, - ); + primaryServerVersion: + appAtomRegistry.get(primaryServerConfigAtom)?.environment.serverVersion ?? null, + }), + projectTitle: project?.title ?? null, + threadTitle: thread?.title ?? null, + desktop: isElectron, + }); + // Projects change whenever any project does; most changes leave the title alone. + if (document.title !== title) document.title = title; + }; + // The open thread's own shell, since the shell lists skip threads hidden by a tag filter. + let unsubscribeOpenThread = () => {}; + const watchOpenThread = () => { + unsubscribeOpenThread(); + unsubscribeOpenThread = + openThreadRef === null + ? () => {} + : appAtomRegistry.subscribe( + environmentThreadShells.quietThreadShellAtom(openThreadRef), + write, + ); + }; + // The draft store changes on every keystroke; only a new open thread matters here. + const updateOpenThread = () => { + const next = readOpenThreadRef(router); + if ((next && scopedThreadKey(next)) === (openThreadRef && scopedThreadKey(openThreadRef))) { + return; + } + openThreadRef = next; + watchOpenThread(); + write(); + }; + watchOpenThread(); + appAtomRegistry.subscribe(primaryServerConfigAtom, write, { immediate: true }); + appAtomRegistry.subscribe(environmentProjects.projectsAtom, write); + router.subscribe("onResolved", updateOpenThread); + useComposerDraftStore.subscribe(updateOpenThread); } /** @@ -155,7 +217,7 @@ export function startAppServices(): AppRouter { const router = getRouter(createBrowserHistory()); startClientAppearance(); - startDocumentTitle(); + startDocumentTitle(router); startEnvironmentThemeRepaint(); startBrowserChromeThemeSync(router); startModelPreferencesMigration(); diff --git a/apps/web/src/appearanceFonts.browser.test.ts b/apps/web/src/appearanceFonts.browser.test.ts index 9bdc5c2ef986..289e48ecad05 100644 --- a/apps/web/src/appearanceFonts.browser.test.ts +++ b/apps/web/src/appearanceFonts.browser.test.ts @@ -52,4 +52,40 @@ describe("applyAppearanceFontVariables", () => { else root.setAttribute("style", style); } }); + + it("widens the numbered-list gutter only when the interface font's marker would clip", () => { + const root = document.documentElement; + const style = root.getAttribute("style"); + const gutter = (sans: string) => { + applyAppearanceFontVariables(root, { + sans, + code: "", + composer: "", + sizeInterface: 16, + sizePrompt: 14, + sizeCode: 13, + smoothing: true, + separate: false, + }); + const markdown = document.createElement("div"); + markdown.className = "chat-markdown"; + markdown.style.fontSize = "14px"; + const list = document.createElement("ol"); + markdown.append(list); + document.body.append(markdown); + try { + return Number.parseFloat(getComputedStyle(list).paddingInlineStart); + } finally { + markdown.remove(); + } + }; + try { + // `0. ` in a 14px monospace font is about 25px, past the 20px default. + expect(gutter("monospace")).toBeGreaterThan(24); + expect(gutter("sans-serif")).toBe(20); + } finally { + if (style === null) root.removeAttribute("style"); + else root.setAttribute("style", style); + } + }); }); diff --git a/apps/web/src/appearanceFonts.ts b/apps/web/src/appearanceFonts.ts index e7c2867cb279..19f2d88d3c29 100644 --- a/apps/web/src/appearanceFonts.ts +++ b/apps/web/src/appearanceFonts.ts @@ -88,6 +88,16 @@ export function applyAppearanceFontVariables( } } + const markerWidth = probeWidth( + appearanceFontStack(preferences.sans, DEFAULT_SANS_FONT_STACK), + LIST_MARKER_PROBE_TEXT, + ); + if (markerWidth === null) { + root.style.removeProperty("--list-marker-sans"); + } else { + root.style.setProperty("--list-marker-sans", `${(markerWidth / PROBE_FONT_SIZE).toFixed(3)}em`); + } + root.style.fontSize = `${clampInterfaceFontSize(preferences.sizeInterface)}px`; root.style.setProperty( "--font-size-prompt", @@ -127,15 +137,18 @@ function clampCodeFontSize(value: number): number { } const FONT_PROBE_TEXT = "mmmmmmmmMMWli1O0@# fjord"; +// The widest single-digit `ol` marker; CSS can't measure it to size the list gutter. +const LIST_MARKER_PROBE_TEXT = "0. "; +const PROBE_FONT_SIZE = 16; let fontProbeContext: CanvasRenderingContext2D | null | undefined; -function probeWidth(fontList: string): number | null { +function probeWidth(fontList: string, text = FONT_PROBE_TEXT): number | null { if (fontProbeContext === undefined) { fontProbeContext = document.createElement("canvas").getContext("2d"); } if (fontProbeContext === null) return null; - fontProbeContext.font = `16px ${fontList}`; - return fontProbeContext.measureText(FONT_PROBE_TEXT).width; + fontProbeContext.font = `${PROBE_FONT_SIZE}px ${fontList}`; + return fontProbeContext.measureText(text).width; } /** Probes canvas metrics because document.fonts.check() accepts uninstalled families. */ diff --git a/apps/web/src/branding.logic.ts b/apps/web/src/branding.logic.ts index 126fb7706248..395c2cd7ac6b 100644 --- a/apps/web/src/branding.logic.ts +++ b/apps/web/src/branding.logic.ts @@ -11,6 +11,9 @@ export function formatAppDisplayName(input: { return `${input.baseName} (${input.stageLabel})`; } +// Coder: no resolveHostedAppChannelLabel or resolveWebAppStageLabel; branding.ts fixes the name +// and vite.config.ts stamps no title, so nothing resolves a hosted channel or web stage. + export function resolveServerBackedAppStageLabel(input: { readonly primaryServerVersion: string | null | undefined; readonly fallbackStageLabel: string; @@ -36,3 +39,23 @@ export function resolveServerBackedAppDisplayName(input: { ? input.fallbackDisplayName : formatAppDisplayName({ baseName: input.baseName, stageLabel }); } + +/** + * Names the open thread: `project / thread` on desktop, where the OS already + * shows the app, with the app name appended in browser tabs. + */ +export function resolveWindowTitle(input: { + readonly appDisplayName: string; + readonly projectTitle: string | null; + readonly threadTitle: string | null; + readonly desktop: boolean; +}): string { + const threadTitle = input.threadTitle?.trim() ?? ""; + if (threadTitle === "") { + return input.appDisplayName; + } + + const projectTitle = input.projectTitle?.trim() ?? ""; + const context = projectTitle === "" ? threadTitle : `${projectTitle} / ${threadTitle}`; + return input.desktop ? context : `${context} - ${input.appDisplayName}`; +} diff --git a/apps/web/src/components/AppSidebarLayout.tsx b/apps/web/src/components/AppSidebarLayout.tsx index 47bc82a04eba..0c0199ea9325 100644 --- a/apps/web/src/components/AppSidebarLayout.tsx +++ b/apps/web/src/components/AppSidebarLayout.tsx @@ -53,6 +53,7 @@ import { SidebarProvider, SidebarRail, SidebarTrigger, + useMobileSidebarSheetMounted, useSidebar, useSidebarVisibility, } from "./ui/sidebar"; @@ -268,7 +269,8 @@ function SidebarControl({ showSidebar }: { readonly showSidebar: boolean }) { // Coder: there is no usage page. const usagePageOpen = false; const keybindings = useAtomValue(primaryServerKeybindingsAtom); - const { toggleSidebar, isMobile, mobileSheetMounted } = useSidebar(); + const { toggleSidebar } = useSidebar(); + const mobileSidebarSheetMounted = useMobileSidebarSheetMounted(); const isSidebarVisible = useSidebarVisibility(); const environmentIdentificationMode = useEnvironmentIdentificationMode(); const stageBackdropVariant = useSidebarStageBackdropVariant(environmentIdentificationMode); @@ -327,7 +329,7 @@ function SidebarControl({ showSidebar }: { readonly showSidebar: boolean }) { ref={appShellCommandsRef} className={cn( "pointer-events-none fixed left-[var(--workspace-controls-left)] top-[var(--workspace-controls-top)] z-50 ml-px flex h-[var(--workspace-topbar-height)] items-center", - isMobile && mobileSheetMounted && "hidden", + mobileSidebarSheetMounted && "hidden", )} data-sidebar-control="" > diff --git a/apps/web/src/components/BranchPicker.tsx b/apps/web/src/components/BranchPicker.tsx index 963755a4e773..a6598de5c85e 100644 --- a/apps/web/src/components/BranchPicker.tsx +++ b/apps/web/src/components/BranchPicker.tsx @@ -32,6 +32,7 @@ export function BranchPicker({ isFetchingNextPage, onLoadNext, statusText, + notice, originControl, footerAction, popupProps, @@ -51,6 +52,7 @@ export function BranchPicker({ isFetchingNextPage: boolean; onLoadNext: () => void; statusText: string | null; + notice?: string | null | undefined; originControl?: { checked: boolean; onCheckedChange: (checked: boolean) => void } | undefined; /** Coder: an action row under the list, such as renaming the checked-out branch. */ footerAction?: ReactNode; @@ -137,6 +139,7 @@ export function BranchPicker({ keyExtractor={(item) => item} {...(getItemType ? { getItemType } : {})} renderItem={({ item, index }) => renderItem(item, index)} + extraData={renderItem} estimatedItemSize={28} drawDistance={336} onScroll={updateBranchListScrollFades} @@ -184,6 +187,7 @@ export function BranchPicker({ ) : null} {footerAction} + {notice ? {notice} : null} {statusText ? {statusText} : null}
diff --git a/apps/web/src/components/BranchToolbar.logic.test.ts b/apps/web/src/components/BranchToolbar.logic.test.ts index 903a20066b39..d5b74f1cb116 100644 --- a/apps/web/src/components/BranchToolbar.logic.test.ts +++ b/apps/web/src/components/BranchToolbar.logic.test.ts @@ -1,13 +1,15 @@ -import { type VcsRef } from "@t3tools/contracts"; +import { ProviderInstanceId, RunId, type VcsRef } from "@t3tools/contracts"; import { describe, expect, it } from "vite-plus/test"; import { dedupeRemoteBranchesWithLocalMatches } from "@t3tools/shared/git"; import { deriveLocalBranchNameFromRemoteRef, + isWorktreeChangeBlocked, resolveBranchSelectionTarget, resolveDraftEnvModeAfterBranchChange, resolveEffectiveEnvMode, resolveBranchToolbarPrBranch, resolveLocalCheckoutBranchMismatch, + resolveLockedWorkspaceLabel, sanitizeNewRefName, shouldIncludeBranchPickerItem, } from "./BranchToolbar.logic"; @@ -92,6 +94,16 @@ describe("resolveEffectiveEnvMode", () => { }); }); +describe("resolveLockedWorkspaceLabel", () => { + it.each([ + ["/home/me/.t3/worktrees/app/t3-ff466c42", "t3-ff466c42"], + ["/home/me/.t3/worktrees/app/t3-ff466c42/", "t3-ff466c42"], + ["C:\\Users\\me\\worktrees\\feature-x", "feature-x"], + ])("names the worktree at %s", (path, label) => { + expect(resolveLockedWorkspaceLabel(path, "worktree")).toBe(label); + }); +}); + describe("deriveLocalBranchNameFromRemoteRef", () => { it("supports remote names that contain slashes", () => { expect(deriveLocalBranchNameFromRemoteRef("my-org/upstream/feature/demo")).toBe( @@ -101,16 +113,23 @@ describe("deriveLocalBranchNameFromRemoteRef", () => { }); describe("dedupeRemoteBranchesWithLocalMatches", () => { - it("hides remote refs when the matching local ref exists", () => { + it.each([ + ["feature/demo", "origin/feature/demo", true], + ["feature/demo", "feature/demo", false], + ["demo", "demo", true], + ["origin/", "origin/", true], + ["/demo", "/demo", true], + ["", "", false], + ])("filters origin ref %s / %s (hidden: %s)", (localName, remoteName, hidden) => { const input: VcsRef[] = [ { - name: "feature/demo", + name: localName, current: false, isDefault: false, worktreePath: null, }, { - name: "origin/feature/demo", + name: remoteName, isRemote: true, remoteName: "origin", current: false, @@ -127,10 +146,12 @@ describe("dedupeRemoteBranchesWithLocalMatches", () => { }, ]; - expect(dedupeRemoteBranchesWithLocalMatches(input).map((ref) => ref.name)).toEqual([ - "feature/demo", - "origin/feature/remote-only", - ]); + const result = dedupeRemoteBranchesWithLocalMatches(input); + const expected = hidden ? [input[0], input[2]] : input; + expect(result).toEqual(expected); + for (const [index, ref] of result.entries()) { + expect(ref).toBe(expected[index]); + } }); it("keeps non-origin remote refs visible when git tracks with first-slash local naming", () => { @@ -234,6 +255,35 @@ describe("resolveBranchSelectionTarget", () => { }); }); +describe("isWorktreeChangeBlocked", () => { + const runtime = (status: "preparing" | "running" | "completed") => ({ + status, + activeRunId: status === "completed" ? null : RunId.make("run"), + providerInstanceId: ProviderInstanceId.make("codex"), + providerName: null, + lastError: null, + }); + + it.each([ + ["preparing", "/repo/wt-a", null, true], + ["running", "/repo/wt-a", "/repo/wt-b", true], + ["running", null, "/repo/wt-b", true], + ["running", "/repo/wt-a", "/repo/wt-a", false], + ["completed", "/repo/wt-a", null, false], + ] as const)( + "a %s turn moving %s -> %s is blocked: %s", + (status, currentWorktreePath, nextWorktreePath, blocked) => { + expect( + isWorktreeChangeBlocked({ + runtime: runtime(status), + currentWorktreePath, + nextWorktreePath, + }), + ).toBe(blocked); + }, + ); +}); + describe("shouldIncludeBranchPickerItem", () => { it("keeps the synthetic checkout PR item visible for gh pr checkout input", () => { expect( diff --git a/apps/web/src/components/BranchToolbar.logic.ts b/apps/web/src/components/BranchToolbar.logic.ts index 332103466456..dd1add3bb864 100644 --- a/apps/web/src/components/BranchToolbar.logic.ts +++ b/apps/web/src/components/BranchToolbar.logic.ts @@ -5,10 +5,12 @@ import type { ProjectId, WorktreeSubmodules, } from "@t3tools/contracts"; +import type { ThreadStructureRuntime } from "@t3tools/client-runtime/state/shell"; import * as Schema from "effect/Schema"; import { sanitizeNewRefName } from "@t3tools/shared/git"; import { workspaceRelativeFilePath } from "@t3tools/shared/path"; import { toSortableTimestamp } from "@t3tools/client-runtime/state/thread-sort"; +import { deriveCanInterruptRunningThread } from "../session-logic"; export { deriveLocalBranchNameFromRemoteRef, sanitizeNewRefName } from "@t3tools/shared/git"; export interface EnvironmentOption { @@ -104,7 +106,7 @@ export function resolveLockedWorkspaceLabel( activeWorktreePath: string | null, effectiveEnvMode: EnvMode, ): string { - if (activeWorktreePath) return "Worktree"; + if (activeWorktreePath) return activeWorktreePath.split(/[\\/]/).findLast(Boolean) ?? "Worktree"; return effectiveEnvMode === "worktree" ? resolveEnvModeLabel("worktree") : "Local checkout"; } @@ -278,6 +280,17 @@ export function resolveBranchSelectionTarget(input: { }; } +export function isWorktreeChangeBlocked(input: { + runtime: ThreadStructureRuntime | null; + currentWorktreePath: string | null; + nextWorktreePath: string | null; +}): boolean { + return ( + input.nextWorktreePath !== input.currentWorktreePath && + deriveCanInterruptRunningThread(true, input.runtime) + ); +} + export function shouldIncludeBranchPickerItem(input: { itemValue: string; normalizedQuery: string; diff --git a/apps/web/src/components/BranchToolbarBranchSelector.tsx b/apps/web/src/components/BranchToolbarBranchSelector.tsx index fb18047913ac..980d0ac273db 100644 --- a/apps/web/src/components/BranchToolbarBranchSelector.tsx +++ b/apps/web/src/components/BranchToolbarBranchSelector.tsx @@ -26,7 +26,7 @@ import { writeTextToClipboard } from "../hooks/useCopyToClipboard"; import { readLocalApi } from "../localApi"; import { useOpenPrLink } from "../lib/openPullRequestLink"; import { usePaginatedBranches } from "../state/queries"; -import { useProject, useQuietThreadShell } from "../state/entities"; +import { readThreadShell, useProject, useQuietThreadShell } from "../state/entities"; import { useEnvironmentQuery } from "../state/query"; import { readEnvironmentScope, useEnvironmentScope } from "~/state/session"; import { threadEnvironment } from "../state/threads"; @@ -40,6 +40,7 @@ import { getSourceControlPresentation } from "../sourceControlPresentation"; import { useComposerMenuProps } from "./chat/composerEventScope"; import { deriveLocalBranchNameFromRemoteRef, + isWorktreeChangeBlocked, resolveBranchTriggerLabel, resolveBranchToolbarPrBranch, resolveBranchSelectionTarget, @@ -52,6 +53,7 @@ import { import { ThreadPullRequestBadgeControl, prStatusIndicator, + resolveDisplayedThreadPr, resolveThreadPullRequestBadge, useLinkedThreadPullRequest, } from "./ThreadStatusIndicators"; @@ -99,6 +101,8 @@ interface BranchToolbarBranchSelectorProps { onComposerFocusRequest?: () => void; } +const WORKTREE_CHANGE_BLOCKED_MESSAGE = "Stop the current turn to switch worktrees."; + function toBranchActionErrorMessage(error: unknown): string { return error instanceof Error ? error.message : "An error occurred."; } @@ -166,7 +170,9 @@ export function BranchToolbarBranchSelector({ : (serverThread?.branch ?? draftThread?.branch ?? null); const activeWorktreePath = forceNewWorktree ? null - : (serverThread?.worktreePath ?? draftThread?.worktreePath ?? null); + : serverThread + ? serverThread.worktreePath + : (draftThread?.worktreePath ?? null); const activeProjectCwd = activeProject?.workspaceRoot ?? null; const branchCwd = activeWorktreePath ?? activeProjectCwd; const hasServerThread = serverThread !== null; @@ -180,6 +186,10 @@ export function BranchToolbarBranchSelector({ draftThreadEnvMode: draftThread?.envMode, }); + const readLatestThreadState = () => + readThreadShell(threadRef) ?? + serverThread ?? { runtime: null, worktreePath: activeWorktreePath }; + const setThreadBranch = ( branch: string | null, worktreePath: string | null, @@ -190,8 +200,22 @@ export function BranchToolbarBranchSelector({ !activeProject || (hasServerThread && !readEnvironmentScope(environmentId, AuthOrchestrationOperateScope)) ) - return; - if (serverSession && worktreePath !== activeWorktreePath) { + return false; + const latest = readLatestThreadState(); + if (automatic && (latest.runtime || latest.worktreePath)) return false; + if ( + isWorktreeChangeBlocked({ + runtime: latest.runtime, + currentWorktreePath: latest.worktreePath, + nextWorktreePath: worktreePath, + }) + ) { + toastManager.add( + stackedThreadToast({ type: "warning", title: WORKTREE_CHANGE_BLOCKED_MESSAGE }), + ); + return false; + } + if (latest.runtime && worktreePath !== latest.worktreePath) { void stopThreadSession({ environmentId, input: { threadId: activeThreadId }, @@ -210,7 +234,7 @@ export function BranchToolbarBranchSelector({ } if (hasServerThread) { onActiveThreadBranchOverrideChange?.(branch); - return; + return true; } const nextDraftEnvMode = resolveDraftEnvModeAfterBranchChange({ nextWorktreePath: worktreePath, @@ -224,6 +248,7 @@ export function BranchToolbarBranchSelector({ environmentSelection: automatic ? (draftThread?.environmentSelection ?? "auto") : "manual", projectRef: scopeProjectRef(environmentId, activeProject.id), }); + return true; }; const [isBranchMenuOpen, setIsBranchMenuOpen] = useState(false); @@ -257,6 +282,20 @@ export function BranchToolbarBranchSelector({ const prReference = parsePullRequestReference(trimmedBranchQuery); const isSelectingWorktreeBase = effectiveEnvMode === "worktree" && !envLocked && !activeWorktreePath; + const isWorktreeChangeBlockedForRef = (refName: VcsRef) => + activeProjectCwd !== null && + isWorktreeChangeBlocked({ + runtime: serverSession, + currentWorktreePath: serverThread ? serverThread.worktreePath : activeWorktreePath, + nextWorktreePath: isSelectingWorktreeBase + ? null + : resolveBranchSelectionTarget({ + activeProjectCwd, + activeProjectCheckoutRoot, + activeWorktreePath, + refName, + }).nextWorktreePath, + }); const checkoutPullRequestItemValue = canChangeThreadBranch && prReference && onCheckoutPullRequestRequest ? `__checkout_pull_request__:${prReference}` @@ -464,20 +503,30 @@ export function BranchToolbarBranchSelector({ const selectBranch = (refName: VcsRef) => { if (!canUpdateThreadBranch || !branchCwd || !activeProjectCwd || isBranchActionPending) return; + const latest = readLatestThreadState(); if (isSelectingWorktreeBase) { + if (latest.worktreePath) return; setThreadBranch(refName.name, null); setIsBranchMenuOpen(false); onComposerFocusRequest?.(); return; } - const selectionTarget = resolveBranchSelectionTarget({ activeProjectCwd, activeProjectCheckoutRoot, - activeWorktreePath, + activeWorktreePath: latest.worktreePath, refName, }); + if ( + isWorktreeChangeBlocked({ + runtime: latest.runtime, + currentWorktreePath: latest.worktreePath, + nextWorktreePath: selectionTarget.nextWorktreePath, + }) + ) + return; + if (selectionTarget.reuseExistingWorktree) { setThreadBranch(refName.name, selectionTarget.nextWorktreePath); setIsBranchMenuOpen(false); @@ -506,8 +555,11 @@ export function BranchToolbarBranchSelector({ const nextBranchName = refName.isRemote ? (checkoutResult.value.refName ?? selectedBranchName) : selectedBranchName; - setOptimisticBranch(nextBranchName); - setThreadBranch(nextBranchName, selectionTarget.nextWorktreePath); + setOptimisticBranch( + setThreadBranch(nextBranchName, selectionTarget.nextWorktreePath) + ? nextBranchName + : previousBranch, + ); return; } setOptimisticBranch(previousBranch); @@ -543,8 +595,10 @@ export function BranchToolbarBranchSelector({ }, }); if (createBranchResult._tag === "Success") { - setOptimisticBranch(createBranchResult.value.refName); - setThreadBranch(createBranchResult.value.refName, activeWorktreePath); + const nextBranchName = createBranchResult.value.refName; + setOptimisticBranch( + setThreadBranch(nextBranchName, activeWorktreePath) ? nextBranchName : previousBranch, + ); return; } setOptimisticBranch(previousBranch); @@ -620,10 +674,10 @@ export function BranchToolbarBranchSelector({ activeThreadBranch, resolvedActiveBranch, }); - const branchPr = - branchPrBranch !== null && branchStatusQuery.data?.refName === branchPrBranch - ? (branchStatusQuery.data.pr ?? null) - : null; + const branchPr = resolveDisplayedThreadPr({ + threadBranch: branchPrBranch, + gitStatus: branchStatusQuery.data, + }); const supportsMultiplePullRequests = useSupportsMultiplePullRequests(environmentId); const linkedStatus = useLinkedThreadPullRequest( environmentId, @@ -719,7 +773,8 @@ export function BranchToolbarBranchSelector({ activeProjectCheckoutRoot, activeWorktreePath, refName, - }).reuseExistingWorktree)) + }).reuseExistingWorktree)) || + isWorktreeChangeBlockedForRef(refName) } onClick={() => selectPickerItem(itemValue)} onContextMenu={(event) => handleBranchContextMenu(event, itemValue)} @@ -742,6 +797,11 @@ export function BranchToolbarBranchSelector({ isFetchingNextPage={isFetchingNextPage} onLoadNext={branchRefState.loadNext} statusText={branchStatusText} + notice={ + refs.some((refName) => isWorktreeChangeBlockedForRef(refName)) + ? WORKTREE_CHANGE_BLOCKED_MESSAGE + : null + } renderItem={renderPickerItem} getItemType={(item) => item === checkoutPullRequestItemValue diff --git a/apps/web/src/components/BranchToolbarEnvModeSelector.tsx b/apps/web/src/components/BranchToolbarEnvModeSelector.tsx index ca511de43642..e9ca7d3f7c94 100644 --- a/apps/web/src/components/BranchToolbarEnvModeSelector.tsx +++ b/apps/web/src/components/BranchToolbarEnvModeSelector.tsx @@ -97,7 +97,8 @@ export const BranchToolbarEnvModeSelector = memo(function BranchToolbarEnvModeSe {forceNewWorktree ? "Each model starts in its own worktree." - : resolveLockedWorkspaceLabel(activeWorktreePath, effectiveEnvMode)} + : (activeWorktreePath ?? + resolveLockedWorkspaceLabel(activeWorktreePath, effectiveEnvMode))} ); diff --git a/apps/web/src/components/ChatMarkdown.test.tsx b/apps/web/src/components/ChatMarkdown.test.tsx index d55786bd9306..d0a7e03f6739 100644 --- a/apps/web/src/components/ChatMarkdown.test.tsx +++ b/apps/web/src/components/ChatMarkdown.test.tsx @@ -65,6 +65,9 @@ vi.mock("./chat/AssistantCitationChip", () => ({ AssistantCitationChip: ({ citation }: { citation: AssistantCitation }) => ( {citation.text} ), + AssistantCitationEmbed: ({ citation }: { citation: AssistantCitation }) => ( + {citation.text} + ), })); vi.mock("~/lib/openPullRequestLink", () => ({ findProjectOnChangeRequestHost: () => undefined, @@ -74,6 +77,9 @@ vi.mock("~/lib/openPullRequestLink", () => ({ })); import ChatMarkdown from "./ChatMarkdown"; + +// Coder: markup without tags, with the quotes the renderer escapes. +const renderedText = (html: string) => html.replace(/<[^>]+>/g, "").replaceAll(""", '"'); import { hasMarkdownFilePrimaryAction, shouldUseMarkdownFileBrowserPrimaryAction, @@ -669,8 +675,9 @@ describe("ChatMarkdown file option chips", () => { />, ); - expect(html).toContain(":codex-file-citation"); - expect(html).not.toContain("chat-markdown-file-link"); + // Upstream now links the bare path inside the unfinished directive, labelled with its file + // name; the directive itself stays literal text. + expect(renderedText(html)).toContain('Created :codex-file-citation{path="report.xlsx"'); }); it("leaves malformed and similarly named file directives literal", () => { @@ -680,8 +687,9 @@ describe("ChatMarkdown file option chips", () => { ]) { const html = renderToStaticMarkup(); - expect(html).toContain(text.replaceAll('"', """)); - expect(html).not.toContain("chat-markdown-file-link"); + expect(renderedText(html)).toContain( + text.replace("/tmp/project/outputs/report.xlsx", "report.xlsx"), + ); } }); @@ -706,8 +714,7 @@ describe("ChatMarkdown file option chips", () => { />, ); - expect(html).toContain(":codex-file-citation"); - expect(html).not.toContain("chat-markdown-file-link"); + expect(renderedText(html)).toContain(':codex-file-citation{path="report.xlsx"}'); }); it("does not create a nested link for citations inside link text", () => { diff --git a/apps/web/src/components/ChatMarkdown.tsx b/apps/web/src/components/ChatMarkdown.tsx index 03758d31e524..4fa8b23b1f5a 100644 --- a/apps/web/src/components/ChatMarkdown.tsx +++ b/apps/web/src/components/ChatMarkdown.tsx @@ -20,6 +20,7 @@ import { CHAT_MARKDOWN_REMARK_PLUGINS, CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS, CHAT_MARKDOWN_REHYPE_PLUGINS, + remarkAssistantInsights, } from "@t3tools/shared/markdownPipeline"; import { useAtomValue } from "@effect/atom-react"; import { @@ -38,7 +39,7 @@ import { MessageSquareWarningIcon, OctagonAlertIcon, PresentationIcon, - SparklesIcon, + StarIcon, TriangleAlertIcon, type LucideIcon, } from "lucide-react"; @@ -86,7 +87,7 @@ import { defaultUrlTransform } from "react-markdown"; import { parseAssistantCitationHref } from "@t3tools/shared/assistantCitations"; import { parseComposerContextHref } from "@t3tools/shared/composerContextReferences"; import { parseThreadLinkHref } from "@t3tools/shared/threadLinks"; -import { AssistantCitationChip } from "./chat/AssistantCitationChip"; +import { AssistantCitationEmbed } from "./chat/AssistantCitationChip"; import { MarkdownThreadLink } from "./chat/MarkdownThreadLink"; import { mathKind } from "@t3tools/shared/markdownMath"; import remarkMath from "remark-math"; @@ -137,10 +138,9 @@ import { filesystemEnvironment } from "../state/filesystem"; import { claimWorkspaceBasenameLookup, needsWorkspaceBasenameLookup, - pickStrippedWorkspaceMatch, - pickWorkspaceBasenameMatch, - stripRepeatedWorkspacePrefix, + pickWorkspaceLinkMatch, WORKSPACE_BASENAME_LOOKUP_LIMIT, + workspaceLinkLookupQuery, } from "@t3tools/client-runtime/workspace-basename-lookup"; import { resolvePullRequestPreviewTarget } from "@t3tools/client-runtime/change-request-links"; import { sourceControlClients } from "@t3tools/client-runtime/source-control-clients"; @@ -182,6 +182,7 @@ import { reportMarkdownActionFailure } from "./chat/markdownActionFailure"; const isAssetNotFoundError = Schema.is(AssetWorkspaceAssetNotFoundError); interface ChatMarkdownProps { + renderInsights?: boolean; /** Compact markdown transforms remove blocks before rendering inside a label. */ inline?: boolean; text: string; @@ -251,11 +252,8 @@ function CodexArtifactTemplateCard(props: { data-skill-name={props.template.skillName} >
- + - - -
@@ -283,13 +281,18 @@ function CodexArtifactTemplateCard(props: { const WINDOWS_DRIVE_PATH_REGEX = /^[A-Za-z]:[\\/]/; function orderedListGutterStyle( - itemCount: number, + itemValues: ReadonlyArray, start: unknown, ): { "--list-gutter": string } | undefined { const parsedStart = Number.parseInt(String(start ?? 1), 10); - const firstNumber = Number.isNaN(parsedStart) ? 1 : parsedStart; - const lastNumber = firstNumber + Math.max(itemCount - 1, 0); - const markerWidth = Math.max(String(firstNumber).length, String(lastNumber).length); + let number = Number.isNaN(parsedStart) ? 1 : parsedStart; + let markerWidth = String(number).length; + for (const value of itemValues) { + const written = Number.parseInt(String(value ?? ""), 10); + if (!Number.isNaN(written)) number = written; + markerWidth = Math.max(markerWidth, String(number).length); + number += 1; + } if (markerWidth <= 1) return undefined; return { "--list-gutter": `${markerWidth + 2}ch` }; } @@ -298,6 +301,12 @@ const GITHUB_ALERT_PRESENTATIONS: Record< string, { label: string; Icon: typeof InfoIcon; borderClassName: string; titleClassName: string } > = { + insight: { + label: "Insight", + Icon: StarIcon, + borderClassName: "border-info", + titleClassName: "text-info", + }, note: { label: "Note", Icon: InfoIcon, @@ -611,33 +620,24 @@ function useChatMarkdownState({ ) { return null; } - const isBasename = needsWorkspaceBasenameLookup(workspaceRelativePath); - const strippedPath = isBasename - ? null - : stripRepeatedWorkspacePrefix(workspaceRelativePath, cwd); - if (!isBasename && strippedPath === null) return null; - const result = await searchProjectEntries({ - environmentId, - input: { - cwd, - query: strippedPath ?? workspaceRelativePath, - limit: WORKSPACE_BASENAME_LOOKUP_LIMIT, - kind: "file", - }, - }); - if (result._tag !== "Success") return null; - if (strippedPath === null) { - return pickWorkspaceBasenameMatch(workspaceRelativePath, result.value.entries); + const query = workspaceLinkLookupQuery(workspaceRelativePath, cwd); + if (query === null) return null; + // Search can be truncated or still scanning, so it cannot rule out the literal + // path. A path with a folder opens as written whenever it exists. + if (!needsWorkspaceBasenameLookup(workspaceRelativePath)) { + const metadata = await getFilesystemMetadata({ + environmentId, + input: { paths: [resolveWorkspaceFilePath(workspaceRelativePath, cwd)] }, + }); + if (metadata._tag !== "Success" || metadata.value.entries[0] !== null) return null; } - const match = pickStrippedWorkspaceMatch(strippedPath, cwd, result.value.entries); - if (match === null) return null; - // Search can be truncated or still scanning, so it cannot rule out the - // literal path. Check the host before redirecting to the stripped match. - const metadata = await getFilesystemMetadata({ + const result = await searchProjectEntries({ environmentId, - input: { paths: [resolveWorkspaceFilePath(workspaceRelativePath, cwd)] }, + input: { cwd, query, limit: WORKSPACE_BASENAME_LOOKUP_LIMIT, kind: "file" }, }); - return metadata._tag === "Success" && metadata.value.entries[0] === null ? match : null; + return result._tag === "Success" + ? pickWorkspaceLinkMatch(workspaceRelativePath, cwd, result.value.entries) + : null; }; const openFileInPanel = (fileLinkMeta: MarkdownFileLinkMeta, panelPath: string) => { if (!threadRef) return; @@ -648,8 +648,7 @@ function useChatMarkdownState({ if ( fileLinkMeta.workspaceLookupPath === null || !cwd || - (!needsWorkspaceBasenameLookup(panelPath) && - stripRepeatedWorkspacePrefix(panelPath, cwd) === null) + workspaceLinkLookupQuery(panelPath, cwd) === null ) { openAt(panelPath); return; @@ -694,6 +693,7 @@ function useChatMarkdownState({ path: fileLinkMeta.filePath, ...(fileLinkMeta.line !== undefined ? { line: fileLinkMeta.line } : {}), ...(fileLinkMeta.column !== undefined ? { column: fileLinkMeta.column } : {}), + ...(fileLinkMeta.endLine !== undefined ? { endLine: fileLinkMeta.endLine } : {}), }, fileLinkParentSuffixByPath, )} @@ -763,6 +763,15 @@ const ChatMarkdownRendererContext = React.createContext< // Isolate streamed source text from the renderer context; only offset-based renderers need it. const ChatMarkdownTextContext = React.createContext(""); +// A quote card is a block. These say nothing else shares its paragraph or list item on that side. +const LIST_MARKER = String.raw`(?:[-*+]|\d+[.)])[ \t]+`; +const CITATION_LEADS_BLOCK = new RegExp( + String.raw`(?:(?:^|\n[ \t]*\n)[ \t>]*(?:${LIST_MARKER})?|\n[ \t>]*${LIST_MARKER})$`, +); +const CITATION_ENDS_BLOCK = new RegExp( + String.raw`^[ \t]*(?:$|\n[ \t]*(?:\n|$)|\n[ \t>]*${LIST_MARKER})`, +); + function markdownHeadingRenderer(level: 1 | 2 | 3 | 4 | 5 | 6) { const Tag = `h${level}` as const; return function MarkdownHeading({ @@ -833,10 +842,11 @@ const CHAT_MARKDOWN_COMPONENTS = { ); }, ol: function MarkdownOrderedList({ node, start, style, ...props }) { - const itemCount = - node?.children?.filter((child) => child.type === "element" && child.tagName === "li") - .length ?? 0; - const gutterStyle = orderedListGutterStyle(itemCount, start); + const itemValues = + node?.children?.flatMap((child) => + child.type === "element" && child.tagName === "li" ? [child.properties.value] : [], + ) ?? []; + const gutterStyle = orderedListGutterStyle(itemValues, start); return (
    ); @@ -902,7 +912,17 @@ const CHAT_MARKDOWN_COMPONENTS = { } = use(ChatMarkdownRendererContext); const text = use(ChatMarkdownTextContext); const citation = href ? parseAssistantCitationHref(href) : null; - if (citation) return ; + if (citation) { + const before = node?.position?.start.offset; + const after = node?.position?.end.offset; + return ( + + ); + } const threadLink = href ? parseThreadLinkHref(href) : null; if (threadLink) { const written = hastPlainTextDeep(node); @@ -1438,6 +1458,8 @@ const CHAT_MARKDOWN_REMARK_PLUGIN_SETS = { const combinedPlugins = new WeakMap>(); // One combined list per pair, so a stable extra list shares a renderer and its parse cache. +const INSIGHT_REMARK_PLUGINS: PluggableList = [remarkAssistantInsights]; + function withExtraPlugins(shared: PluggableList, extra: PluggableList): PluggableList { if (extra.length === 0) return shared; let byShared = combinedPlugins.get(extra); @@ -1504,10 +1526,12 @@ function ChatMarkdown({ lineBreaks = false, parseRawHtml = true, extraRemarkPlugins = EMPTY_REMARK_PLUGINS, + renderInsights = false, ...props }: ChatMarkdownProps) { const { componentState, handleCopy, markdownRef, localMediaPreview, closeLocalMediaPreview } = useChatMarkdownState({ text, inline, ...props }); + const hasInsights = renderInsights && text.includes("★ Insight"); // Streaming re-renders only the last block; offsets for task edits need the whole text. const streamingBlocks = props.isStreaming === true && @@ -1518,6 +1542,7 @@ function ChatMarkdown({ : null; const incrementalParsing = props.isStreaming === true && + !hasInsights && streamingBlocks === null && extraRemarkPlugins.length === 0 && /(?:^|\n) {0,3}(?:`{3}|~{3})/.test(text); @@ -1539,7 +1564,12 @@ function ChatMarkdown({ const uncachedPlugins = incrementalParsing; const remarkPlugins = incrementalParsing ? [...sharedRemarkPlugins, ...extraRemarkPlugins, createIncrementalMarkdownPlugin()] - : withExtraPlugins(sharedRemarkPlugins, extraRemarkPlugins); + : withExtraPlugins( + hasInsights + ? withExtraPlugins(INSIGHT_REMARK_PLUGINS, sharedRemarkPlugins) + : sharedRemarkPlugins, + extraRemarkPlugins, + ); const blocks = streamingBlocks ?? [text]; return ( diff --git a/apps/web/src/components/ChatView.logic.browser.test.ts b/apps/web/src/components/ChatView.logic.browser.test.ts index 233af7115eeb..2d887cb5acc2 100644 --- a/apps/web/src/components/ChatView.logic.browser.test.ts +++ b/apps/web/src/components/ChatView.logic.browser.test.ts @@ -1,5 +1,6 @@ import { expect, it } from "vite-plus/test"; import { userEvent } from "vite-plus/test/browser"; +import { createWorkspaceFocusController } from "../lib/workspaceFocus"; import { shouldRefocusComposerOnWindowFocus } from "./ChatView.logic"; it("keeps keyboard focus on a sidebar row on window return, but not a clicked row", async () => { @@ -26,3 +27,35 @@ it("keeps keyboard focus on a sidebar row on window return, but not a clicked ro expect(document.activeElement).toBe(tabbed); expect(shouldRefocusComposerOnWindowFocus(document.activeElement)).toBe(false); }); + +it("refocuses the composer on window return from a clicked area after a bare key press", async () => { + const controller = createWorkspaceFocusController(document); + const root = document.createElement("div"); + root.innerHTML = + '' + + '
    '; + document.body.append(root); + using _root = { + [Symbol.dispose]: () => { + controller.dispose(); + root.remove(); + }, + }; + const conversation = root.querySelector("section")!; + + await userEvent.click(conversation); + expect(document.activeElement).toBe(conversation); + // A window switch can deliver a lone modifier key without moving focus. + await userEvent.keyboard("{Shift}"); + expect( + shouldRefocusComposerOnWindowFocus(document.activeElement), + "a clicked area counts as reached from the keyboard after a bare key press", + ).toBe(true); + + await userEvent.keyboard("{F6}"); + controller.move(-1); + await userEvent.keyboard("{F6}"); + controller.move(1); + expect(document.activeElement).toBe(conversation); + expect(shouldRefocusComposerOnWindowFocus(document.activeElement)).toBe(false); +}); diff --git a/apps/web/src/components/ChatView.logic.test.ts b/apps/web/src/components/ChatView.logic.test.ts index 54ccad72fa7a..36c94a3ce664 100644 --- a/apps/web/src/components/ChatView.logic.test.ts +++ b/apps/web/src/components/ChatView.logic.test.ts @@ -1042,6 +1042,12 @@ describe("planComposerTurn", () => { shouldQueueBehindActiveRun: false, baseBranchForWorktree: null, }); + expect( + planComposerTurn({ ...running, sendQueuesBehindSetup: true, dispatchMode: "steer" }), + ).toMatchObject({ + turnDispatchMode: "steer", + shouldQueueBehindActiveRun: true, + }); // A fresh session cannot join the running turn, and a stale one is not compacted first. expect( planComposerTurn({ diff --git a/apps/web/src/components/ChatView.logic.ts b/apps/web/src/components/ChatView.logic.ts index 39a4c6c2417b..3100c1d78c40 100644 --- a/apps/web/src/components/ChatView.logic.ts +++ b/apps/web/src/components/ChatView.logic.ts @@ -180,16 +180,6 @@ export function resolveProactiveTurnDiffAction(input: { return totals.count >= 3 || totals.additions + totals.deletions >= 50 ? "open" : "ignore"; } -/** Mark reactivated Diff tabs to preserve review scope. Already-active tabs must not leave a marker for - * the next open. */ -export function resolveExplicitDiffOpenOnActivate(input: { - readonly surfaceKind: RightPanelSurface["kind"]; - readonly diffOpen: boolean; - readonly activeThreadRef: ScopedThreadRef; -}): ScopedThreadRef | null { - return input.surfaceKind === "diff" && !input.diffOpen ? input.activeThreadRef : null; -} - export function codexArtifactTemplatePromptToAppend( currentDraft: string, template: CodexArtifactTemplate, @@ -1021,7 +1011,7 @@ export function shouldRefocusComposerOnWindowFocus( } return ( activeElement.closest( - '[data-workspace-area]:focus-visible, [data-sidebar-row]:focus-visible, [role="dialog"], [role="alertdialog"], [role="application"], [role="menu"], [role="listbox"], [data-slot$="-popup"], [data-terminal-owner]', + '[data-workspace-keyboard-focus], [data-sidebar-row]:focus-visible, [role="dialog"], [role="alertdialog"], [role="application"], [role="menu"], [role="listbox"], [data-slot$="-popup"], [data-terminal-owner]', ) === null ); } @@ -1208,7 +1198,9 @@ export function planComposerTurn(input: { baseBranchForWorktree: shouldCreateWorktree ? input.activeThreadBranch : null, compactBeforeSend, turnDispatchMode: - compactBeforeSend || input.sendQueuesBehindSetup ? ("queue" as const) : dispatchMode, + compactBeforeSend || (input.sendQueuesBehindSetup && dispatchMode !== "steer") + ? ("queue" as const) + : dispatchMode, shouldQueueBehindActiveRun: compactBeforeSend || input.sendQueuesBehindSetup || diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 1773f7fff0e0..667e7f2c20f7 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -4,6 +4,7 @@ import { sourceControlClients } from "@t3tools/client-runtime/source-control-cli import { queuedRunEditBlockedReason } from "@t3tools/client-runtime/queued-run-edit"; import { ScrollToEndButton, WorkspaceFileDropOverlay } from "./chat/TimelineOverlays"; import { usePasteToFocusComposer } from "./chat/usePasteToFocusComposer"; +import { useFocusComposerOnThreadOpen } from "./chat/useFocusComposerOnThreadOpen"; import { announcedRun, isDetailBehindShell } from "./chat/threadActivityAnnouncement"; import { deriveAvailableRevertTurnCounts, @@ -11,6 +12,7 @@ import { presentThreadShellFromProjection, } from "./ChatView.logic"; import { usePullRequestDraftThread } from "./chat/usePullRequestDraftThread"; +import { usePullRequestLinking } from "../hooks/usePullRequestLinking"; import { useComposerSelection } from "./chat/useComposerSelection"; import { useProjectCloneBanner } from "./chat/useProjectCloneBanner"; import { useAttachmentPreviewHandoff } from "./chat/useAttachmentPreviewHandoff"; @@ -83,8 +85,10 @@ import { } from "@t3tools/client-runtime/state/subagentRuntime"; import { getUserQueuedThreadRuns, + threadSteersDuringCompaction, threadSupportsProviderHandoff, } from "@t3tools/client-runtime/state/thread-workflows"; +import { compactionBlocksSteering } from "@t3tools/client-runtime/state/composer-dispatch"; import { resolvePendingHandoff } from "@t3tools/client-runtime/handoff"; import { HandoffRow } from "./chat/V2LifecycleRow"; import { @@ -108,7 +112,10 @@ import { type SetStateAction, use, useEffect, useLayoutEffect, useRef, useState import { BackgroundThreadContext } from "./chat/backgroundThread"; import { WorkspaceMutationIdContext } from "./chat/ChatMarkdownMedia"; import { useLocation, useNavigate } from "@tanstack/react-router"; -import { assistantCitationFromLocation } from "../lib/assistantCitationNavigation"; +import { + assistantCitationFromLocation, + useSideChatCitationReveal, +} from "../lib/assistantCitationNavigation"; import type { AssistantCitationSourceAnchor } from "~/lib/assistantTextSelection"; import { useStore } from "zustand"; import { shallow } from "zustand/vanilla/shallow"; @@ -120,7 +127,6 @@ import { } from "@t3tools/client-runtime/state/runtime"; import { AsyncResult } from "effect/reactivity"; import { isElectron } from "../env"; -import { useDiffPanelStore } from "../diffPanelStore"; import { useActiveThreadRef } from "../hooks/useActiveThreadRef"; import { composerRequiresModifier } from "../composer-logic"; import { @@ -193,12 +199,12 @@ import { import { type TerminalContextDraft } from "../lib/terminalContext"; import { selectThreadTerminalUiState, useTerminalUiStateStore } from "../terminalUiStateStore"; import { useKnownTerminalSessions, useThreadRunningTerminalIds } from "../state/terminalSessions"; -import { useEnvironmentQuery } from "../state/query"; +import { useEnvironmentQueryRefresh } from "../state/query"; import { useEnvironmentScope } from "~/state/session"; import { primaryServerKeybindingsAtom, serverEnvironment } from "../state/server"; import { threadEnvironment } from "../state/threads"; import { resolveProviderSkillsForCwd } from "@t3tools/client-runtime/providerSkills"; -import { vcsEnvironment } from "../state/vcs"; +import { chatGitStatusAtom, NO_CHAT_GIT_STATUS_ATOM, vcsEnvironment } from "../state/vcs"; import { useEnvironment, useEnvironmentProviders, useEnvironments } from "../state/environments"; import { resolveThreadDetailRef, @@ -328,6 +334,7 @@ import type { CodexArtifactTemplate } from "@t3tools/shared/codexArtifactTemplat const EMPTY_ANCHORED_TIMELINE_MESSAGES: ReadonlyArray = []; const EMPTY_PROVIDER_SKILLS: ServerProvider["skills"] = []; const EMPTY_TURN_ITEMS: ThreadStructure["turnItems"] = []; +const EMPTY_RUNS: ThreadStructure["runs"] = []; const EMPTY_COMPOSER_BANNER_ITEMS: ReadonlyArray = []; const EMPTY_PENDING_FILE_SURFACE_IDS: ReadonlySet = new Set(); @@ -510,12 +517,16 @@ export default function ChatView(props: ChatViewProps) { select: (location) => location.state.assistantCitationActivation ?? location.state.__TSR_key, }); const citationLocation = { href: citationHref, key: citationKey }; - const citationRequest = (() => { - const citation = assistantCitationFromLocation(citationLocation.href); - return citation && citation.environmentId === environmentId && citation.threadId === threadId - ? { citation, key: citationLocation.key ?? citationLocation.href } - : null; - })() as AssistantCitationRequest | null; + const locationCitation = assistantCitationFromLocation(citationLocation.href); + const citationActivation = citationLocation.key ?? citationLocation.href; + const citationRequest = ( + locationCitation && + locationCitation.environmentId === environmentId && + locationCitation.threadId === threadId + ? { citation: locationCitation, key: citationActivation } + : null + ) as AssistantCitationRequest | null; + useSideChatCitationReveal(embedded ? null : routeThreadRef, locationCitation, citationActivation); const citationHistory = useCitedHistory(citationRequest); const { resolvedTheme } = useTheme(); const composerRuntimeMode = useComposerDraftStore( @@ -913,20 +924,6 @@ export default function ChatView(props: ChatViewProps) { selectActiveRightPanel(state.byThreadKey, activeThreadRef), ); const diffOpen = activeRightPanelKind === "diff"; - const explicitDiffOpenRef = useRef(null); - // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- resets the diff panel store to branch scope when the diff opens without an explicit request - useLayoutEffect(() => { - const explicitThreadRef = explicitDiffOpenRef.current; - explicitDiffOpenRef.current = null; - if ( - diffOpen && - activeThreadRef && - explicitThreadRef !== activeThreadRef && - !useDiffPanelStore.getState().consumeKeepSelectionOnOpen(activeThreadRef) - ) { - useDiffPanelStore.getState().selectGitScope(activeThreadRef, "branch"); - } - }, [activeThreadRef, diffOpen]); const rightPanelState = useRightPanelStore((state) => selectThreadRightPanelState(state.byThreadKey, activeThreadRef), ); @@ -1075,6 +1072,13 @@ export default function ChatView(props: ChatViewProps) { canPickEnvironment: hasMultipleEnvironments, }); + const pullRequestLinking = usePullRequestLinking(environmentId); + // Best effort: the thread already exists, so a failed link is only logged. + const linkCheckoutPullRequest = useStableCallback((threadRef: ScopedThreadRef, url: string) => { + pullRequestLinking.changeLink(threadRef, url, true).catch((error: unknown) => { + console.warn("Failed to link the checked-out pull request to its thread.", error); + }); + }); const { handlePreparedPullRequestThreadStable, openPullRequestDialogStable } = usePullRequestDraftThread({ routeKind, @@ -1253,9 +1257,17 @@ export default function ChatView(props: ChatViewProps) { item.runId === activeRuntime?.activeRunId && item.status === "running", ); + const compactCommandIsActive = compactRequestIsActive && !compactionSettled; const isCompacting = (isSendBusy || phase === "connecting" || phase === "running") && - (compactionRunning || (compactRequestIsActive && !compactionSettled)); + (compactionRunning || compactCommandIsActive); + const compactionBlocksSteer = compactionBlocksSteering({ + compactCommandIsActive, + compactionRunning, + steersDuringCompaction: threadSteersDuringCompaction(serverProjection), + run: serverProjection?.runs.find((run) => run.id === activeRuntime?.activeRunId), + messages: serverProjection?.messages, + }); const isWorking = phase === "running" || isSendBusy || isCompacting || runlessWorkStartedAt !== null; const activeContextWindow = deriveLatestContextWindowSnapshot( @@ -1397,7 +1409,7 @@ export default function ChatView(props: ChatViewProps) { const turnDiffSummaries = useThreadCheckpointSummaries(routeThreadDetailRef); const availableRevertTurnCounts = deriveAvailableRevertTurnCounts(serverProjection); // A paged history window cannot identify the first loaded run as the thread start. - const threadStartCheckpointReady = + const threadStartCheckpointAvailable = !serverThreadHistory.hasMoreHistory && findRevertCheckpoint(serverProjection?.checkpoints ?? [], 0) !== undefined; @@ -1413,14 +1425,17 @@ export default function ChatView(props: ChatViewProps) { }) : null; const gitStatusCwd = activeThread?.worktreePath ?? gitCwd; - const gitStatusQuery = useEnvironmentQuery( + const gitStatusAtom = gitStatusCwd === null ? null : vcsEnvironment.status({ environmentId, input: { cwd: gitStatusCwd }, - }), - ); + }); + const gitStatusQuery = { + data: useAtomValue(gitStatusAtom ? chatGitStatusAtom(gitStatusAtom) : NO_CHAT_GIT_STATUS_ATOM), + refresh: useEnvironmentQueryRefresh(gitStatusAtom).refresh, + }; useWorkspaceMutationRefresh({ enabled: gitStatusCwd !== null, mutationId: workspaceMutationId, @@ -1863,16 +1878,11 @@ export default function ChatView(props: ChatViewProps) { setTimelineAnchor, }); - // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- schedules a frame to focus the composer DOM element when a thread opens with the terminal closed - useEffect(() => { - if (embedded || !activeThread?.id || terminalUiState.terminalOpen) return; - const frame = window.requestAnimationFrame(() => { - focusComposerStable(); - }); - return () => { - window.cancelAnimationFrame(frame); - }; - }, [activeThread?.id, embedded, focusComposerStable, terminalUiState.terminalOpen]); + useFocusComposerOnThreadOpen( + activeThread?.id, + !embedded && !terminalUiState.terminalOpen, + focusComposerStable, + ); // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- window focus listener that refocuses the composer after the window regains focus useEffect(() => { @@ -2115,7 +2125,6 @@ export default function ChatView(props: ChatViewProps) { pullRequestsSurfaceAvailable, gitCwd, activeProjectCwd, - explicitDiffOpenRef, onDiffPanelOpen, closeRightPanelSurface, }); @@ -2352,7 +2361,7 @@ export default function ChatView(props: ChatViewProps) { document.activeElement?.getAttribute("data-testid") === "composer-editor" && isInSideChatPane(document.activeElement) === embedded, draftThreadRoute: routeKind === "draft", - turnRunning: phase === "running", + turnRunning: phase === "running" || phase === "connecting", composerRequiresModifier: composerRequiresModifier(settings.sendShortcut, promptRef.current), isWeb: !isElectron, isDesktop: isElectron, @@ -2604,6 +2613,7 @@ export default function ChatView(props: ChatViewProps) { sideChatDisabledReason, handleNewThreadInActiveProject, persistThreadSettingsForNextTurn, + linkCheckoutPullRequest, }); const onWorktreeSetupWorkLocally = () => { @@ -3027,7 +3037,7 @@ export default function ChatView(props: ChatViewProps) { parentThreadLink={parentThreadLink} workflowOpening={workflowOpening} supportsConversationRollback={supportsConversationRollback} - threadStartCheckpointReady={threadStartCheckpointReady} + threadStartCheckpointAvailable={threadStartCheckpointAvailable} availableRevertTurnCounts={availableRevertTurnCounts} onRevertToTurnCount={onRevertTimelineTurnStable} onUseArtifactTemplate={insertArtifactTemplateStable} @@ -3160,6 +3170,7 @@ export default function ChatView(props: ChatViewProps) { draftId={draftId} activeThreadId={activeThreadId} activeThread={activeThread} + activeThreadRuns={serverProjection?.runs ?? EMPTY_RUNS} promptHistoryEntries={timelineEntries} forceExpandedOnMobile={ forceExpandedMobileComposer && isDraftHeroState @@ -3168,7 +3179,7 @@ export default function ChatView(props: ChatViewProps) { isLocalDraftThread && activeProject === null } phase={phase} - isCompacting={isCompacting} + canSteer={!compactionBlocksSteer} canInterrupt={canInterruptRunningThread} isSendBusy={ (isSendBusy && !sendQueuesBehindSetup) || @@ -3392,6 +3403,7 @@ export default function ChatView(props: ChatViewProps) { } keybindings={keybindings} gitCwd={gitCwd} + workspaceMutationId={workspaceMutationId} isGitRepo={isGitRepo} envLocked={envLocked} availableEnvironments={logicalProjectEnvironments} @@ -3504,7 +3516,11 @@ export default function ChatView(props: ChatViewProps) { ) : null} - + {embedded ? null : } diff --git a/apps/web/src/components/CommandPalette.browser.test.tsx b/apps/web/src/components/CommandPalette.browser.test.tsx index 3f1ab04da99a..59d5271db97d 100644 --- a/apps/web/src/components/CommandPalette.browser.test.tsx +++ b/apps/web/src/components/CommandPalette.browser.test.tsx @@ -2,7 +2,7 @@ import { EnvironmentId, ProjectId, ThreadId } from "@t3tools/contracts"; import { describe, expect, it, vi } from "vite-plus/test"; import { page, userEvent } from "vite-plus/test/browser"; -import { render } from "../../test/render"; +import { act, render } from "../../test/render"; import { openCommandPalette } from "../commandPaletteBus"; import { useCommandPaletteStore } from "../commandPaletteStore"; import { makeThreadFixture } from "../test-fixtures"; @@ -35,6 +35,56 @@ function setup(state: { projects?: ReadonlyArray; threads?: ReadonlyArr } describe("CommandPalette open intents", () => { + it("drags a searched thread into the current composer without navigating", async () => { + const environmentId = EnvironmentId.make("remote"); + const threadId = ThreadId.make("dragged-thread"); + setup({ + threads: [makeThreadFixture({ environmentId, id: threadId, title: "Drag this thread" })], + }); + const onDrop = vi.fn(); + await using screen = await render( + +
    { + node?.addEventListener("t3-thread-context-drop", onDrop); + }} + style={{ position: "fixed", bottom: 0, width: 300, height: 100 }} + > + Composer +
    +
    , + ); + const pathname = screen.router?.state.location.pathname; + openCommandPalette({ query: "Drag this thread" }); + const row = page.getByRole("option", { name: /Drag this thread/ }); + await expect.element(row).toBeVisible(); + const bounds = row.element().getBoundingClientRect(); + const pointer = (type: string, x: number, y: number) => + new PointerEvent(type, { + bubbles: true, + cancelable: true, + pointerId: 1, + isPrimary: true, + button: 0, + buttons: type === "pointerup" ? 0 : 1, + clientX: x, + clientY: y, + }); + await act(() => { + row.element().dispatchEvent(pointer("pointerdown", bounds.x + 20, bounds.y + 10)); + document.dispatchEvent(pointer("pointermove", bounds.x + 30, bounds.y + 20)); + }); + expect(useCommandPaletteStore.getState().open).toBe(false); + await act(() => { + document.dispatchEvent(pointer("pointermove", 50, window.innerHeight - 30)); + document.dispatchEvent(pointer("pointerup", 50, window.innerHeight - 30)); + }); + expect(onDrop).toHaveBeenCalledOnce(); + expect((onDrop.mock.calls[0]![0] as CustomEvent).detail).toEqual([{ environmentId, threadId }]); + expect(screen.router?.state.location.pathname).toBe(pathname); + }); + it("opens New thread in... with focus in the search input, not the Back button", async () => { setup(); await using _screen = await render({null}); diff --git a/apps/web/src/components/CommandPalette.composition.browser.test.tsx b/apps/web/src/components/CommandPalette.composition.browser.test.tsx index 0382e082a1e9..83083d70802f 100644 --- a/apps/web/src/components/CommandPalette.composition.browser.test.tsx +++ b/apps/web/src/components/CommandPalette.composition.browser.test.tsx @@ -2,11 +2,17 @@ import { describe, expect, it, vi } from "vite-plus/test"; import { page, userEvent } from "vite-plus/test/browser"; import { DEFAULT_RESOLVED_KEYBINDINGS } from "@t3tools/shared/keybindings"; import { DEFAULT_CLIENT_SETTINGS } from "@t3tools/contracts/settings"; +import { AuthOrchestrationOperateScope, AuthSourceControlWriteScope } from "@t3tools/contracts"; import { act, render } from "../../test/render"; import { CommandPalette } from "./CommandPalette"; import { openCommandPalette } from "../commandPaletteBus"; import { primaryServerKeybindingsAtom } from "../state/server"; +const scopes = vi.hoisted(() => ({ denied: undefined as string | undefined })); +const setup = (deniedScope?: string) => { + scopes.denied = deniedScope; +}; + vi.mock("@effect/atom-react", async (original) => ({ ...(await original()), useAtomValue: (atom: unknown) => @@ -47,7 +53,8 @@ vi.mock("../hooks/useHandleNewThread", () => ({ vi.mock("../hooks/useOpenPanelPullRequestUrl", () => ({ useOpenPanelPullRequestUrl: () => null })); vi.mock("../connection/useDesktopLocalBootstraps", () => ({ useDesktopLocalBootstraps: () => [] })); vi.mock("../state/use-atom-command", () => ({ - useAtomCommand: () => async () => ({ _tag: "Success", value: undefined }), + // The add-project destination step starts a clone and follows its project id. + useAtomCommand: () => async () => ({ _tag: "Success", value: { projectId: "project-cloned" } }), })); vi.mock("../state/use-atom-query-runner", () => ({ useAtomQueryRunner: () => async () => ({ _tag: "Success", value: undefined }), @@ -64,6 +71,7 @@ vi.mock("../state/environments", async (original) => ({ serverConfig: { environment: { platform: { os: "linux" }, capabilities: { projectCloneTracking: true } }, settings: { addProjectBaseDirectory: "/workspace/" }, + newProjectsRoot: "/workspace/", }, }, ], @@ -92,7 +100,7 @@ vi.mock("../state/queries", async (original) => ({ })); vi.mock("../state/session", async (original) => ({ ...(await original()), - useEnvironmentScope: () => true, + useEnvironmentScope: (_environmentId: unknown, scope: string) => scope !== scopes.denied, readEnvironmentScope: () => true, })); vi.mock("../state/filesystem", async (original) => ({ @@ -113,6 +121,7 @@ describe.each(["repository", "folder", "destination"] as const)("add-project %s" { isComposing: true, keyCode: 13 }, { isComposing: false, keyCode: 229 }, ])("keeps Enter in the input during composition: %j", async (composition) => { + setup(); await using _screen = await render({null}); await act(() => openCommandPalette({ open: "add-project" })); // Coder: Add project opens the workspace picker first. @@ -144,7 +153,46 @@ describe.each(["repository", "folder", "destination"] as const)("add-project %s" }); }); +it.each([ + { step: "folder", source: "Local folder", keys: "", description: /^(?!.*(Ctrl|⌘)).*Add.*Enter/ }, + { + step: "highlighted folder", + source: "Local folder", + keys: "{ArrowDown}", + description: /Add.*(Ctrl|⌘) Enter/, + }, + { step: "repository", source: "Git URL", keys: "", description: /Continue.*Enter/ }, + { step: "new project", source: "New project", keys: "", description: /Create.*Enter/ }, + { + step: "permission-blocked folder", + source: "Local folder", + keys: "", + deniedScope: AuthOrchestrationOperateScope, + description: "This connection cannot add projects.", + }, + { + step: "permission-blocked destination", + source: "Git URL", + keys: "https://example.com/team/project.git{Enter}", + deniedScope: AuthSourceControlWriteScope, + description: "This connection needs permission to write source control and add projects.", + }, +])( + "tells screen readers how to submit the $step step", + async ({ source, keys, deniedScope, description }) => { + setup(deniedScope); + await using _screen = await render({null}); + await act(() => openCommandPalette({ open: "add-project" })); + // Coder: Add project opens the workspace picker first. + await page.getByText("Local", { exact: true }).click(); + await page.getByText(source, { exact: true }).click(); + if (keys) await userEvent.keyboard(keys); + await expect.element(page.getByRole("combobox")).toHaveAccessibleDescription(description); + }, +); + it("keeps the caret in place when an edit briefly makes the folder path not a path", async () => { + setup(); await using _screen = await render({null}); await act(() => openCommandPalette({ open: "add-project" })); // Coder: Add project opens the workspace picker first. @@ -159,3 +207,20 @@ it("keeps the caret in place when an edit briefly makes the folder path not a pa await userEvent.keyboard("w"); await expect.element(input()).toHaveValue("/wworkspace/project"); }); + +it.each([ + { flow: "Git URL", value: "https://github.com/pingdotgg/t3code.git", submit: "Continue (Enter)" }, + { flow: "New project", value: "demo-tab-order", submit: /^Create \(/ }, +])("moves focus from the $flow input to its enabled submit button on Tab", async (step) => { + await using _screen = await render({null}); + await act(() => openCommandPalette({ open: "add-project" })); + // Coder: Add project opens the workspace picker first. + await page.getByText("Local", { exact: true }).click(); + await page.getByText(step.flow, { exact: true }).click(); + const input = page.getByRole("combobox"); + await input.fill(step.value); + const submit = page.getByRole("button", { name: step.submit }); + await expect.element(submit).toBeEnabled(); + await userEvent.keyboard("{Tab}"); + await expect.element(submit).toHaveFocus(); +}); diff --git a/apps/web/src/components/CommandPalette.drag.ts b/apps/web/src/components/CommandPalette.drag.ts new file mode 100644 index 000000000000..3dec40e02b0f --- /dev/null +++ b/apps/web/src/components/CommandPalette.drag.ts @@ -0,0 +1,36 @@ +import type { ScopedThreadRef } from "@t3tools/contracts"; +import { flushSync } from "react-dom"; +import { dispatchCommandPalette } from "../commandPaletteStore"; +import { SidebarPointerSensor } from "./Sidebar.pointer"; +import { + dropThreadContext, + endThreadContextDrag, + moveThreadContextDrag, +} from "./chat/threadContextDrag"; + +export function startPaletteThreadDrag( + event: PointerEvent, + thread: ScopedThreadRef, + title: string, +) { + if (!event.isPrimary || event.button !== 0 || event.pointerType === "touch") return; + // The document-owned sensor must survive closing and unmounting the palette on activation. + return new SidebarPointerSensor({ + event, + active: `${thread.environmentId}:${thread.threadId}`, + options: { + distance: 6, + onAttach: () => {}, + onFinish: endThreadContextDrag, + onDrop: (point) => dropThreadContext(point, [thread]), + }, + onStart: () => { + flushSync(() => dispatchCommandPalette({ _tag: "SetOpen", open: false })); + }, + onMove: (point) => moveThreadContextDrag(point, { title, count: 1 }), + onPending: () => {}, + onAbort: () => {}, + onCancel: () => {}, + onEnd: () => {}, + }); +} diff --git a/apps/web/src/components/CommandPalette.logic.ts b/apps/web/src/components/CommandPalette.logic.ts index cc8fd9091ded..db7ea4895b7c 100644 --- a/apps/web/src/components/CommandPalette.logic.ts +++ b/apps/web/src/components/CommandPalette.logic.ts @@ -19,6 +19,7 @@ import { type AddProjectRemoteSourceReadiness, } from "@t3tools/client-runtime/operations/projects"; import { + type ScopedThreadRef, type EnvironmentId, type ProjectId, type EnvironmentMachineKind, @@ -48,6 +49,7 @@ export function buildLinkedThreadActionItems( return input.threads.map((thread) => ({ kind: "action", value: `thread:${input.environmentId}:${thread.id}`, + threadRef: { environmentId: input.environmentId, threadId: thread.id }, title: thread.title || "Untitled thread", description: thread.archivedAt === null ? "Linked thread" : "Archived thread", searchTerms: [input.query, thread.title], @@ -194,6 +196,7 @@ export interface CommandPaletteItem { readonly timestamp?: string; /** Thread rows rank by this, with use history, instead of by `searchTerms`. */ readonly threadSearch?: ThreadSearchSubject; + readonly threadRef?: ScopedThreadRef; /** Shown in place of the description when the match is not in the title. */ readonly matchHint?: SearchMatchHint; readonly icon: ReactNode; @@ -466,6 +469,7 @@ export function buildThreadActionItems + { @@ -1919,7 +1922,7 @@ function OpenCommandPaletteDialog(props: { function getDefaultCloneDestination( environmentId: EnvironmentId, remoteUrl: string, - repository: string, + repository?: string, ): string { const environment = environments.find((candidate) => candidate.environmentId === environmentId); return getCloneDestinationPath( @@ -1974,7 +1977,6 @@ function OpenCommandPaletteDialog(props: { const destinationPath = getDefaultCloneDestination( addProjectCloneFlow.environmentId, remoteUrl, - rawRepository, ); setAddProjectCloneFlow({ step: "confirm", @@ -2080,12 +2082,11 @@ function OpenCommandPaletteDialog(props: { return; } - const projectId = newProjectId(); setIsRemoteProjectCloning(true); const startResult = await startProjectClone({ environmentId: addProjectCloneFlow.environmentId, input: { - projectId, + projectId: newProjectId(), title: inferProjectTitleFromPath(destinationPath), // oxlint-disable-next-line react/purity -- runs in an event handler (oxc#27309) createdAt: new Date().toISOString(), @@ -2107,7 +2108,10 @@ function OpenCommandPaletteDialog(props: { return; } setOpen(false); - const projectRef = scopeProjectRef(addProjectCloneFlow.environmentId, projectId); + const projectRef = scopeProjectRef( + addProjectCloneFlow.environmentId, + startResult.value.projectId, + ); await waitForProject(projectRef, 3_000).catch(() => null); const navigationResult = await settlePromise(() => handleNewThread(projectRef)); if (navigationResult._tag === "Failure") { @@ -2536,6 +2540,7 @@ function OpenCommandPaletteDialog(props: { scrollCommandPaletteRowIntoView( resultListRef.current, resultRows.rowIndexByItemIndex[0] ?? 0, + true, ); return; } @@ -2580,9 +2585,17 @@ function OpenCommandPaletteDialog(props: { }); } + const submitButtonId = useId(); + const browseSubmitLabel = + isCloneDestinationStep && !canCloneProject + ? "This connection needs permission to write source control and add projects." + : canCreateProject + ? `${submitActionLabel} (${addShortcutLabel})` + : "This connection cannot add projects."; const inputAccessory = newProjectFlow !== null ? ( ) : addProjectCloneFlow?.step === "repository" ? ( ) : isBrowsing ? ( handleQueryChange(nextQuery, event.target)} diff --git a/apps/web/src/components/CommandPaletteAddProject.tsx b/apps/web/src/components/CommandPaletteAddProject.tsx index 808536403a4b..ce3797848e31 100644 --- a/apps/web/src/components/CommandPaletteAddProject.tsx +++ b/apps/web/src/components/CommandPaletteAddProject.tsx @@ -34,6 +34,7 @@ export function sourceControlHostIcon( /** The input's trailing submit button for the add-project, clone, and new-project flows. */ export function CommandPaletteSubmitButton(props: { + readonly id: string; readonly label: ReactNode; readonly shortcutLabel: string; readonly ariaLabel: string; @@ -46,9 +47,9 @@ export function CommandPaletteSubmitButton(props: { { + if (props.item.threadRef) { + startPaletteThreadDrag( + event.nativeEvent, + props.item.threadRef, + props.item.searchTitle ?? + (typeof props.item.title === "string" ? props.item.title : "Thread"), + ); + } + }} onMouseDown={(event) => { event.preventDefault(); }} diff --git a/apps/web/src/components/ComposerPromptEditorTiptap.citationDraft.browser.test.tsx b/apps/web/src/components/ComposerPromptEditorTiptap.citationDraft.browser.test.tsx index a1a69a7968ee..8da54f5f7214 100644 --- a/apps/web/src/components/ComposerPromptEditorTiptap.citationDraft.browser.test.tsx +++ b/apps/web/src/components/ComposerPromptEditorTiptap.citationDraft.browser.test.tsx @@ -31,9 +31,11 @@ const promptBefore = `See ${serializeAssistantCitation(citation)} `; function Composer({ question, editorRef, + send = "clears", }: { question: boolean; editorRef: React.RefObject; + send?: "clears" | "blocked" | "refusedLater"; }) { const [prompt, setPrompt] = useState(promptBefore); const [answer, setAnswer] = useState(""); @@ -61,6 +63,11 @@ function Composer({ onCitationDraftSave={(replacedValue, savedValue) => { if (replacedValue === prompt) setPrompt(savedValue); }} + onCitationSubmitAndSend={() => { + if (send === "blocked") return false; + if (send === "clears") setPrompt(""); + return true; + }} onPaste={() => {}} /> @@ -97,3 +104,58 @@ it("keeps an unsaved citation comment when a question takes the composer", async ); await expect.element(page.getByLabelText("Edit citation comment")).toBeInTheDocument(); }); + +it("clears the sent prompt when an empty citation comment sends it", async () => { + const editorRef = createRef(); + await using screen = await render(); + + const sourceAnchor = findAssistantCitationSourceAnchor(document, citation); + if (!sourceAnchor) throw new Error("Citation source did not resolve"); + await act(async () => + editorRef.current?.requestCitationComment({ + previousValue: "See ", + value: promptBefore, + citationStart: "See ".length, + sourceAnchor, + insertedSpaces: { before: false, after: true }, + }), + ); + const comment = page.getByLabelText("Comment on selected text"); + await expect.element(comment).toHaveFocus(); + await userEvent.keyboard("{Control>}{Enter}{/Control}"); + + await expect.element(comment).not.toBeInTheDocument(); + await expect.element(page.getByTestId("composer-editor")).toHaveFocus(); + await expect.element(screen.getByRole("status")).toHaveAttribute("data-prompt", ""); +}); + +it.each(["blocked", "refusedLater"] as const)( + "keeps the caret after the citation when save-and-send is %s", + async (send) => { + const editorRef = createRef(); + await using screen = await render( + , + ); + + const sourceAnchor = findAssistantCitationSourceAnchor(document, citation); + if (!sourceAnchor) throw new Error("Citation source did not resolve"); + await act(async () => + editorRef.current?.requestCitationComment({ + previousValue: "See ", + value: promptBefore, + citationStart: "See ".length, + sourceAnchor, + insertedSpaces: { before: false, after: true }, + }), + ); + const comment = page.getByLabelText("Comment on selected text"); + await expect.element(comment).toHaveFocus(); + await userEvent.keyboard("{Control>}{Enter}{/Control}"); + await expect.element(comment).not.toBeInTheDocument(); + await userEvent.keyboard("x"); + + await expect + .element(screen.getByRole("status")) + .toHaveAttribute("data-prompt", `See ${serializeAssistantCitation(citation)}x `); + }, +); diff --git a/apps/web/src/components/ComposerPromptEditorTiptap.tsx b/apps/web/src/components/ComposerPromptEditorTiptap.tsx index bb8f5a5a2211..c443825dffd5 100644 --- a/apps/web/src/components/ComposerPromptEditorTiptap.tsx +++ b/apps/web/src/components/ComposerPromptEditorTiptap.tsx @@ -21,6 +21,7 @@ import { Plugin, PluginKey, TextSelection, + type Transaction, } from "@tiptap/pm/state"; import { Decoration, DecorationSet, type EditorView } from "@tiptap/pm/view"; import type { NodeType, ResolvedPos } from "@tiptap/pm/model"; @@ -91,6 +92,7 @@ import { serializeSelection, skillChipReplacements, splitOrLiftListItem, + surroundSelectionWithMark, stepCaretAcrossStyledEdge, type IsKnownModel, type SkillMeta, @@ -186,7 +188,7 @@ export interface ComposerPromptEditorProps { onPageScrollKeyDown?: (key: "PageUp" | "PageDown") => void; onPageScrollKeyUp?: (key: string) => void; onPageScrollRelease?: () => void; - onCitationSubmitAndSend?: () => void; + onCitationSubmitAndSend?: () => boolean; /** Saves unsaved citation text into replacedValue when a controlled rewrite replaces the document. */ onCitationDraftSave?: (replacedValue: string, savedValue: string) => void; onPaste: React.ClipboardEventHandler; @@ -213,12 +215,12 @@ const ComposerCitationCommentContext = createContext<{ openComment: OpenCitationComment | null; onOpenChange: (citeKey: string, open: boolean) => void; onDraftChange: (citeKey: string, comment: string) => void; - onSubmitAndSend: () => void; + onSubmitAndSend: () => boolean; }>({ openComment: null, onOpenChange: () => {}, onDraftChange: () => {}, - onSubmitAndSend: () => {}, + onSubmitAndSend: () => false, }); const RichComposerSkillsContext = createContext>([]); @@ -477,13 +479,23 @@ function ComposerCitationNodeView({ node, editor, getPos }: NodeViewProps) { editor.chain().focus().deleteRange({ from, to }).run(); }; + const sentRef = useRef(false); const onRestoreFocus = () => { + const sent = sentRef.current; + sentRef.current = false; if (!editor.isEditable) return; const pos = nodePos(); - if (pos === null) return; - const current = editor.state.doc.nodeAt(pos); - if (!current || current.type.name !== "composer-citation") return; - editor.commands.focus(pos + current.nodeSize); + const current = pos === null ? null : editor.state.doc.nodeAt(pos); + if (pos === null || current?.type.name !== "composer-citation") { + if (sent) editor.view.focus(); + return; + } + // The send's controlled rewrite may still be pending; emitting this caret move would restore the old prompt. + editor + .chain() + .setMeta(SILENT_SELECTION_META, sent) + .focus(pos + current.nodeSize) + .run(); }; return ( @@ -512,7 +524,6 @@ function ComposerCitationNodeView({ node, editor, getPos }: NodeViewProps) { > commentContext.onDraftChange(citeKey, comment), onSaveAndSend: (comment) => { if (!onSaveComment(comment)) return false; - commentContext.onSubmitAndSend(); + sentRef.current = commentContext.onSubmitAndSend(); return true; }, onRestoreFocus, @@ -729,6 +740,8 @@ function chipSelectionDecorations({ doc, selection }: EditorState): DecorationSe type TiptapEditor = NonNullable>; +const SILENT_SELECTION_META = "composerSilentSelection"; + function replaceSkillChips( editor: TiptapEditor, replacements: ReturnType, @@ -995,10 +1008,10 @@ function ComposerPromptEditorTiptapInner( onDraftChange: (nodeKey: string, comment: string) => { citationDraftRef.current = { key: nodeKey, comment }; }, - onSubmitAndSend: onCitationSubmitAndSend ?? (() => {}), + onSubmitAndSend: onCitationSubmitAndSend ?? (() => false), }; - const handleEditorChange = (updated: TiptapEditor) => { + const handleEditorChange = (updated: TiptapEditor, transaction: Transaction) => { const map = serializeEditorDoc(updated.state.doc); const contextIds = literalText ? [] : [...map.contextIds]; const { from, to } = updated.state.selection; @@ -1025,7 +1038,7 @@ function ComposerPromptEditorTiptapInner( previousSnapshot.contextIds.length === contextIds.length && previousSnapshot.contextIds.every((id, index) => id === contextIds[index]) ); - if (isApplyingControlledUpdateRef.current) return; + if (isApplyingControlledUpdateRef.current || transaction.getMeta(SILENT_SELECTION_META)) return; if (!snapshotChanged) { if (didComposerSelectionChangeVisibly(previousSelectionRange, nextSelectionRange)) { readCallbacks().onVisibleSelectionChange?.(); @@ -1343,6 +1356,11 @@ function ComposerPromptEditorTiptapInner( const startMd = flatToMarkdown(map, pmToFlat(map, from)); const endMd = flatToMarkdown(map, pmToFlat(map, to)); if (selectionTouchesMentionBoundary(map.value, startMd, endMd)) return false; + const markTr = surroundSelectionWithMark(view.state, from, to, text); + if (markTr) { + view.dispatch(markTr); + return true; + } const tr = view.state.tr.insertText(closer, to).insertText(text, from); tr.setSelection(TextSelection.create(tr.doc, from + text.length, to + text.length)); view.dispatch(tr); @@ -1459,10 +1477,10 @@ function ComposerPromptEditorTiptapInner( return; } } - handleEditorChange(updated); + handleEditorChange(updated, transaction); }, - onSelectionUpdate: ({ editor: updated }) => { - handleEditorChange(updated); + onSelectionUpdate: ({ editor: updated, transaction }) => { + handleEditorChange(updated, transaction); }, }, [], diff --git a/apps/web/src/components/ConfirmDialogHost.tsx b/apps/web/src/components/ConfirmDialogHost.tsx index 8992dc43d7c1..514c719dba73 100644 --- a/apps/web/src/components/ConfirmDialogHost.tsx +++ b/apps/web/src/components/ConfirmDialogHost.tsx @@ -9,6 +9,7 @@ import { import { AlertDialog, AlertDialogClose, + AlertDialogAction, AlertDialogDescription, AlertDialogFooter, AlertDialogHeader, @@ -59,6 +60,7 @@ export function ConfirmDialogHost() { const copy = resolveConfirmDialogCopy(state.status === "idle" ? "" : state.message); const confirmVariant = state.status === "idle" ? "default" : state.variant; + const focusConfirm = state.status !== "idle" && state.initialFocus === "confirm"; const onCancel = () => respondToConfirmDialog(false); const onConfirm = () => respondToConfirmDialog(true); @@ -83,9 +85,13 @@ export function ConfirmDialogHost() { }>Cancel - + {focusConfirm && confirmVariant === "default" ? ( + Confirm + ) : ( + + )} diff --git a/apps/web/src/components/CustomSnoozeDialog.tsx b/apps/web/src/components/CustomSnoozeDialog.tsx index 91d4a54fd9e6..6781cf1b4a09 100644 --- a/apps/web/src/components/CustomSnoozeDialog.tsx +++ b/apps/web/src/components/CustomSnoozeDialog.tsx @@ -85,7 +85,7 @@ function CustomSnoozeDialog() { if (!open) finish(null); }} > - +
    { diff --git a/apps/web/src/components/DiffFilePathCopyButton.tsx b/apps/web/src/components/DiffFilePathCopyButton.tsx index ab42f126189e..5e78949336cf 100644 --- a/apps/web/src/components/DiffFilePathCopyButton.tsx +++ b/apps/web/src/components/DiffFilePathCopyButton.tsx @@ -1,14 +1,12 @@ -import { Check, Copy } from "lucide"; +import { CheckIcon, CopyIcon } from "lucide-react"; import { useRef } from "react"; import { useCopyToClipboard } from "../hooks/useCopyToClipboard"; -import { cn } from "../lib/utils"; import { ANCHORED_COPY_TOAST_TIMEOUT_MS, showAnchoredCopyErrorToast, showAnchoredCopySuccessToast, } from "./ui/anchoredCopyToast"; import { Button } from "./ui/button"; -import { MorphIcon } from "~/components/MorphIcon"; import { Tooltip, TooltipPopup, TooltipTrigger } from "./ui/tooltip"; export function DiffFilePathCopyButton({ filePath }: { filePath: string }) { @@ -32,10 +30,7 @@ export function DiffFilePathCopyButton({ filePath }: { filePath: string }) { /> } > - + {isCopied ? : }

    {isCopied ? "Copied" : "Copy path"}

    diff --git a/apps/web/src/components/DiffPanel.browser.test.tsx b/apps/web/src/components/DiffPanel.browser.test.tsx index 929fd6f18147..832e78570143 100644 --- a/apps/web/src/components/DiffPanel.browser.test.tsx +++ b/apps/web/src/components/DiffPanel.browser.test.tsx @@ -120,7 +120,6 @@ function setup(failedBaseRef: string | null) { useDiffPanelStore.setState({ byThreadKey: {}, branchBaseRefByThreadKey: {}, - keepSelectionOnOpenThreadKey: null, }); useDiffPanelSessionStore.setState({ collapsedFileKeysByScopeKey: {} }); }; diff --git a/apps/web/src/components/DiffPanel.tsx b/apps/web/src/components/DiffPanel.tsx index 9fd56fd5431a..08d2d096567b 100644 --- a/apps/web/src/components/DiffPanel.tsx +++ b/apps/web/src/components/DiffPanel.tsx @@ -11,8 +11,10 @@ import { PilcrowIcon, Rows3Icon, TextWrapIcon, + ChevronRightIcon, + ChevronsDownUpIcon, + ChevronsUpDownIcon, } from "lucide-react"; -import { ChevronDown, ChevronRight, ChevronsDownUp, ChevronsUpDown } from "lucide"; import * as Schema from "effect/Schema"; import { type MouseEvent, useEffect, useRef, useState } from "react"; import { useCodeViewFileReveal } from "./diffs/useCodeViewFileReveal"; @@ -67,7 +69,6 @@ import { type DiffFileExpansionError, } from "./diffs/DiffFileExpansionErrorNotice"; import { Button } from "./ui/button"; -import { MorphIcon } from "~/components/MorphIcon"; import { ToggleGroup, Toggle } from "./ui/toggle-group"; import { Switch } from "./ui/switch"; import { @@ -174,10 +175,11 @@ function DiffFileCollapseToggle({ /> } > - + {collapsed ? ( + + ) : ( + + )} {collapsed ? "Expand diff" : "Collapse diff"} @@ -232,7 +234,8 @@ export default function DiffPanel({ })); const diffLayout = settings.diffLayout; const [wordWrap, setWordWrap] = useState(settings.wordWrap); - const [diffIgnoreWhitespace, setDiffIgnoreWhitespace] = useState(settings.diffIgnoreWhitespace); + const diffIgnoreWhitespace = + useDiffPanelSessionStore((state) => state.ignoreWhitespace) ?? settings.diffIgnoreWhitespace; const [fileTreeOpen, setFileTreeOpen] = useLocalStorage( DIFF_FILE_TREE_STORAGE_KEY, false, @@ -1181,10 +1184,11 @@ export default function DiffPanel({ /> } > - + {allDiffFilesCollapsed ? ( + + ) : ( + + )} {allDiffFilesCollapsed ? "Expand all files" : "Collapse all files"} @@ -1241,7 +1245,7 @@ export default function DiffPanel({ size="sm" pressed={diffIgnoreWhitespace} onPressedChange={(pressed) => { - setDiffIgnoreWhitespace(Boolean(pressed)); + useDiffPanelSessionStore.getState().setIgnoreWhitespace(Boolean(pressed)); }} /> } diff --git a/apps/web/src/components/GitActionsControl.logic.test.ts b/apps/web/src/components/GitActionsControl.logic.test.ts index a2cc2124f468..773136257ca0 100644 --- a/apps/web/src/components/GitActionsControl.logic.test.ts +++ b/apps/web/src/components/GitActionsControl.logic.test.ts @@ -4,6 +4,7 @@ import { resolveAutoFeatureBranchName } from "@t3tools/shared/git"; import { assert, describe, it } from "vite-plus/test"; import { buildMenuItems, + hasUsableGitMenuContent, requiresDefaultBranchConfirmation, resolveChangesTotals, resolveDefaultBranchActionDialogCopy, @@ -77,6 +78,29 @@ describe("when: ref is clean, up to date, and has no open PR", () => { false, ); }); + + it("offers the actions menu only when something in it is usable", () => { + const hasContent = ( + gitStatus: VcsStatusResult, + overrides: Partial[0]> = {}, + ) => + hasUsableGitMenuContent({ + menuItems: buildMenuItems(gitStatus, false, gitStatus.hasPrimaryRemote), + canWriteSourceControl: true, + canPublishRepository: false, + hasNotice: false, + ...overrides, + }); + const clean = status(); + const dirty = status({ hasWorkingTreeChanges: true }); + + assert.isFalse(hasContent(clean)); + assert.isTrue(hasContent(dirty)); + assert.isTrue(hasContent(status({ aheadCount: 1 }))); + assert.isFalse(hasContent(dirty, { canWriteSourceControl: false })); + assert.isTrue(hasContent(status({ hasPrimaryRemote: false }), { canPublishRepository: true })); + assert.isTrue(hasContent(clean, { hasNotice: true })); + }); }); describe("when: on default ref without open PR", () => { @@ -283,6 +307,37 @@ describe("resolveChangesTotals", () => { ); }); + it("uses the panel's branch comparison when it has file stats", () => { + const dirty = status({ + branchChanges: { baseRef: "main", insertions: 20, deletions: 3 }, + }); + const source = { + id: "branch-range", + kind: "branch-range", + title: "Changes vs release", + baseRef: "release", + headRef: "feature/test", + diff: "", + diffHash: "hash", + truncated: false, + } as const; + assert.deepEqual( + resolveChangesTotals(dirty, { + ...source, + files: [ + { path: "a.ts", previousPath: null, additions: 4, deletions: 1 }, + { path: "b.ts", previousPath: null, additions: 2, deletions: 0 }, + ], + }), + { insertions: 6, deletions: 1, baseRef: "release" }, + ); + assert.deepEqual(resolveChangesTotals(dirty, source), { + insertions: 20, + deletions: 3, + baseRef: "main", + }); + }); + it("falls back to the working tree when branchChanges is omitted", () => { assert.deepInclude( resolveChangesTotals( diff --git a/apps/web/src/components/GitActionsControl.logic.ts b/apps/web/src/components/GitActionsControl.logic.ts index 5effb44c79a8..399e01306836 100644 --- a/apps/web/src/components/GitActionsControl.logic.ts +++ b/apps/web/src/components/GitActionsControl.logic.ts @@ -1,6 +1,7 @@ import type { GitRunStackedActionResult, GitStackedAction, + ReviewDiffPreviewSource, VcsStatusResult, } from "@t3tools/contracts"; import { isTemporaryWorktreeBranch } from "@t3tools/shared/git"; @@ -174,6 +175,19 @@ export function buildMenuItems( ]; } +export function hasUsableGitMenuContent(input: { + readonly menuItems: readonly GitActionMenuItem[]; + readonly canWriteSourceControl: boolean; + readonly canPublishRepository: boolean; + readonly hasNotice: boolean; +}): boolean { + return ( + input.canPublishRepository || + input.hasNotice || + (input.canWriteSourceControl && input.menuItems.some((item) => !item.disabled)) + ); +} + export function resolveQuickAction( gitStatus: VcsStatusResult | null, isBusy: boolean, @@ -435,9 +449,21 @@ export type ChangesLineTotals = { // `branchChanges ?? workingTree` treats `{0,0}` as present, so a zero branch total hid a dirty // working tree and the Changes row read +0 while Diff still showed the work. +// The Diff panel's branch comparison wins when it has file stats, so a remembered base or +// hidden whitespace reads the same in the row as in the panel. export function resolveChangesTotals( status: VcsStatusResult | null | undefined, + branchPreview?: ReviewDiffPreviewSource | null, ): ChangesLineTotals | null { + if (branchPreview?.files) { + let insertions = 0; + let deletions = 0; + for (const file of branchPreview.files) { + insertions += file.additions; + deletions += file.deletions; + } + return { insertions, deletions, baseRef: branchPreview.baseRef }; + } if (!status) return null; const branch = status.branchChanges; const tree = { ...status.workingTree, baseRef: null }; diff --git a/apps/web/src/components/GitActionsControl.tsx b/apps/web/src/components/GitActionsControl.tsx index 16c2d81117af..4b4b29a4e1ea 100644 --- a/apps/web/src/components/GitActionsControl.tsx +++ b/apps/web/src/components/GitActionsControl.tsx @@ -46,6 +46,7 @@ import { buildMenuItems, formatGitActionElapsed, GIT_ACTION_SUCCESS_VISIBLE_MS, + hasUsableGitMenuContent, type GitActionProgressPresentation, type GitActionIconName, type GitActionMenuItem, @@ -92,6 +93,15 @@ import { import { useQuietThreadProjection, useQuietThreadShell } from "~/state/entities"; import { readEnvironmentScope, useEnvironmentScope } from "~/state/session"; import { useEnvironmentQuery } from "~/state/query"; +import { useFilesystemReadAccess } from "~/state/filesystem"; +import { reviewEnvironment } from "~/state/review"; +import { revisionedQuery } from "~/state/revisionedQuery"; +import { + selectThreadBranchBaseRef, + useDiffPanelSessionStore, + useDiffPanelStore, +} from "~/diffPanelStore"; +import { useClientSettings } from "~/hooks/useSettings"; import { sourceControlEnvironment } from "~/state/sourceControl"; import { threadEnvironment } from "~/state/threads"; import { useAtomCommand } from "~/state/use-atom-command"; @@ -121,6 +131,7 @@ interface GitActionsControlProps { showActions?: boolean; showChanges?: boolean; onOpenChanges?: () => void; + workspaceMutationId?: string | null; } interface PendingDefaultBranchAction { @@ -932,6 +943,7 @@ export default function GitActionsControl({ showActions = true, showChanges = true, onOpenChanges, + workspaceMutationId = null, }: GitActionsControlProps) { const panelAnchorRef = useRef(null); const updateThreadMetadata = useAtomCommand( @@ -1045,7 +1057,53 @@ export default function GitActionsControl({ const isRepo = gitStatus?.isRepo ?? true; const hasPrimaryRemote = gitStatus?.hasPrimaryRemote ?? false; const gitStatusForActions = gitStatus; - const resolvedChangesTotals = resolveChangesTotals(gitStatusForActions); + const changesBaseRef = useDiffPanelStore((state) => + selectThreadBranchBaseRef(state, activeThreadRef), + ); + const ignoreWhitespaceSetting = useClientSettings((settings) => settings.diffIgnoreWhitespace); + const ignoreWhitespace = + useDiffPanelSessionStore((state) => state.ignoreWhitespace) ?? ignoreWhitespaceSetting; + const { canReadFiles } = useFilesystemReadAccess(activeEnvironmentId); + // Same input as the Diff panel's branch query, so both read one cached result. Status + // totals join the revision because edits, pulls and resets outside a turn leave the mutation + // id as is. + const changesRevision = gitStatusForActions + ? JSON.stringify([ + workspaceMutationId, + gitStatusForActions.refName, + gitStatusForActions.workingTree.insertions, + gitStatusForActions.workingTree.deletions, + gitStatusForActions.branchChanges?.baseRef, + gitStatusForActions.branchChanges?.insertions, + gitStatusForActions.branchChanges?.deletions, + ]) + : null; + const changesPreview = useEnvironmentQuery( + showChanges && + isRepo && + canReadFiles && + changesRevision !== null && + activeEnvironmentId !== null && + gitCwd !== null + ? revisionedQuery( + reviewEnvironment.diffPreview({ + environmentId: activeEnvironmentId, + input: { + cwd: gitCwd, + ...(changesBaseRef ? { baseRef: changesBaseRef } : {}), + ignoreWhitespace, + metadataOnly: true, + sourceKind: "branch-range", + }, + }), + changesRevision, + ) + : null, + ); + const resolvedChangesTotals = resolveChangesTotals( + gitStatusForActions, + changesPreview.data?.sources.find((source) => source.kind === "branch-range"), + ); const changesTotals = resolvedChangesTotals ?? { insertions: 0, deletions: 0 }; const changesBaseLabel = resolvedChangesTotals ? resolvedChangesTotals.baseRef === null @@ -1375,6 +1433,23 @@ export default function GitActionsControl({ const canPublishRepository = canWriteSourceControl && isRepo && gitStatusForActions !== null && !hasPrimaryRemote; + const isDetachedHead = gitStatusForActions?.refName === null; + const isCleanAndBehind = + gitStatusForActions !== null && + gitStatusForActions.refName !== null && + !gitStatusForActions.hasWorkingTreeChanges && + gitStatusForActions.behindCount > 0 && + gitStatusForActions.aheadCount === 0; + const hasGitMenuContent = hasUsableGitMenuContent({ + menuItems: gitActionMenuItems, + canWriteSourceControl, + canPublishRepository, + hasNotice: isDetachedHead || isCleanAndBehind || Boolean(gitStatusError), + }); + + // A disabled menu cannot refresh on open, and terminal edits or commits publish no status. + const refreshStatusForEmptyMenu = () => + requestVcsStatusRefresh(refreshVcsStatus, activeEnvironmentId, gitCwd); const initializeGit = () => { void (async () => { @@ -1446,19 +1521,15 @@ export default function GitActionsControl({ Publish repository... ) : null} - {gitStatusForActions?.refName === null && ( + {isDetachedHead && (

    Detached HEAD: create and check out a branch to enable push and{" "} {changeRequestTerminology.singular} actions.

    )} - {gitStatusForActions && - gitStatusForActions.refName !== null && - !gitStatusForActions.hasWorkingTreeChanges && - gitStatusForActions.behindCount > 0 && - gitStatusForActions.aheadCount === 0 && ( -

    Behind upstream. Pull/rebase first.

    - )} + {isCleanAndBehind && ( +

    Behind upstream. Pull/rebase first.

    + )} {gitStatusError &&

    {gitStatusError}

    } ); @@ -1486,6 +1557,8 @@ export default function GitActionsControl({ role="group" aria-label="Git actions" ref={panelAnchorRef} + onPointerEnter={hasGitMenuContent ? undefined : refreshStatusForEmptyMenu} + onFocus={hasGitMenuContent ? undefined : refreshStatusForEmptyMenu} className={cn( "shrink-0", THREAD_DETAILS_PANEL_SPLIT_GROUP_CLASS, @@ -1593,7 +1666,7 @@ export default function GitActionsControl({ panel /> } - disabled={isGitActionRunning} + disabled={isGitActionRunning || !hasGitMenuContent} >