diff --git a/apps/server/src/orchestration-v2/CheckpointService.test.ts b/apps/server/src/orchestration-v2/CheckpointService.test.ts index 62fa68c4c53af..fec015b5a1b63 100644 --- a/apps/server/src/orchestration-v2/CheckpointService.test.ts +++ b/apps/server/src/orchestration-v2/CheckpointService.test.ts @@ -11,6 +11,7 @@ import { VcsProcessTimeoutError, VcsUnsupportedOperationError, } from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; @@ -164,12 +165,23 @@ it.effect( assert.strictEqual((yield* Fiber.join(postTurn)).status, "ready"); assert.strictEqual((yield* Fiber.join(waitingBaseline))._tag, "Success"); - const slowBaseline = yield* checkpoints - .captureBaseline({ scope, ordinalWithinScope: 2 }) + for (const ordinalWithinScope of [2, 3]) { + const slowBaseline = yield* checkpoints + .captureBaseline({ scope, ordinalWithinScope }) + .pipe(Effect.result, Effect.forkChild); + yield* TestClock.adjust("5 seconds"); + const result = yield* Fiber.join(slowBaseline); + assert.strictEqual(result._tag, "Failure"); + if (result._tag === "Failure") { + assert.isTrue(Cause.isTimeoutError(result.failure.cause)); + } + } + const backedOffBaseline = yield* checkpoints + .captureBaseline({ scope, ordinalWithinScope: 4 }) .pipe(Effect.result, Effect.forkChild); yield* TestClock.adjust("5 seconds"); - assert.strictEqual((yield* Fiber.join(slowBaseline))._tag, "Failure"); - assert.strictEqual(yield* Ref.get(snapshots), 2); + assert.strictEqual((yield* Fiber.join(backedOffBaseline))._tag, "Success"); + assert.strictEqual(yield* Ref.get(snapshots), 3); assert.strictEqual((yield* Ref.get(refs)).size, 1); }).pipe(Effect.provide(layerTest)); }), @@ -233,3 +245,113 @@ it.effect("records a repository probe failure as an error checkpoint, not a miss assert.deepEqual(statuses, { baseline: "error", turn: "error" }); }), ); + +it.effect("backs off only baseline retries after repeated workspace capture timeouts", () => + Effect.gen(function* () { + const scope: OrchestrationV2CheckpointScope = { + id: CheckpointScopeId.make("checkpoint-scope:slow-capture"), + threadId: ThreadId.make("thread:slow-capture"), + runId: RunId.make("run:slow-capture:1"), + nodeId: NodeId.make("node:slow-capture:1"), + parentScopeId: null, + providerThreadId: ProviderThreadId.make("provider-thread:slow-capture"), + kind: "root_run", + ordinalWithinParent: 0, + advancesAppRunCount: true, + cwd: "/repo/slow", + createdAt: DateTime.makeUnsafe("2026-07-28T00:00:00.000Z"), + }; + const timeout = new VcsProcessTimeoutError({ + operation: "GitVcsDriver.checkpoints.captureCheckpoint", + command: "git", + cwd: scope.cwd, + timeoutMs: 30_000, + }); + let attempts = 0; + let healthy = false; + let nonTimeoutFailure = false; + const failure = new VcsProcessExitError({ + operation: "test.capture", + command: "git", + cwd: scope.cwd, + exitCode: 128, + detail: "capture failed", + }); + const layerTest = CheckpointService.layer.pipe( + Layer.provide( + Layer.mergeAll( + IdAllocator.layer, + Layer.mock(CheckpointStore.CheckpointStore)({ + isGitRepository: () => Effect.succeed(true), + hasCheckpointRef: () => Effect.succeed(false), + captureCheckpoint: () => + Effect.suspend(() => { + attempts++; + return healthy ? Effect.void : Effect.fail(nonTimeoutFailure ? failure : timeout); + }), + }), + ), + ), + Layer.provideMerge(NodeCrypto.layer), + ); + yield* Effect.gen(function* () { + const checkpoints = yield* CheckpointService.CheckpointServiceV2; + const baseline = { scope, ordinalWithinScope: 0 }; + const turn = { + scope, + ordinalWithinScope: 1, + runId: scope.runId, + nodeId: scope.nodeId, + appRunOrdinal: 1, + capturedAt: scope.createdAt, + }; + yield* Effect.result(checkpoints.captureBaseline(baseline)); + assert.strictEqual((yield* checkpoints.capture(turn)).status, "error"); + assert.strictEqual(attempts, 2); + + const otherThread = { ...scope, id: CheckpointScopeId.make("checkpoint-scope:other") }; + const skipped = yield* Effect.result( + checkpoints.captureBaseline({ scope: otherThread, ordinalWithinScope: 0 }), + ); + assert.strictEqual(attempts, 2, "baseline retried a repeatedly timing-out workspace"); + assert.strictEqual(skipped._tag, "Success"); + + yield* Effect.result( + checkpoints.captureBaseline({ + scope: { ...otherThread, cwd: "/repo/other" }, + ordinalWithinScope: 0, + }), + ); + assert.strictEqual(attempts, 3, "another workspace must still attempt capture"); + + const failedTurn = yield* checkpoints.capture(turn); + assert.strictEqual(failedTurn.status, "error"); + assert.strictEqual(attempts, 4, "post-turn capture must not back off"); + + yield* checkpoints.captureBaseline(baseline); + assert.strictEqual(attempts, 4, "baseline retries must wait for post-turn recovery"); + healthy = true; + assert.strictEqual((yield* checkpoints.capture(turn)).status, "ready"); + assert.strictEqual(attempts, 5); + yield* checkpoints.captureBaseline(baseline); + assert.strictEqual(attempts, 6, "a successful post-turn capture must clear backoff"); + + healthy = false; + yield* Effect.result(checkpoints.captureBaseline(baseline)); + yield* Effect.result(checkpoints.captureBaseline(baseline)); + yield* checkpoints.captureBaseline(baseline); + assert.strictEqual(attempts, 8); + nonTimeoutFailure = true; + assert.strictEqual((yield* checkpoints.capture(turn)).status, "error"); + nonTimeoutFailure = false; + yield* Effect.result(checkpoints.captureBaseline(baseline)); + assert.strictEqual(attempts, 10, "a non-timeout failure must clear backoff"); + healthy = true; + yield* checkpoints.captureBaseline(baseline); + healthy = false; + yield* Effect.result(checkpoints.captureBaseline(baseline)); + yield* Effect.result(checkpoints.captureBaseline(baseline)); + assert.strictEqual(attempts, 13, "a successful baseline must clear the timeout streak"); + }).pipe(Effect.provide(layerTest)); + }), +); diff --git a/apps/server/src/orchestration-v2/CheckpointService.ts b/apps/server/src/orchestration-v2/CheckpointService.ts index 3e0fbe95e8aa4..fb6490834c7f6 100644 --- a/apps/server/src/orchestration-v2/CheckpointService.ts +++ b/apps/server/src/orchestration-v2/CheckpointService.ts @@ -9,6 +9,7 @@ import { RunId, ThreadId, } from "@t3tools/contracts"; +import * as Cache from "effect/Cache"; import * as Context from "effect/Context"; import * as Digest from "effect/crypto/Digest"; import * as DateTime from "effect/DateTime"; @@ -17,6 +18,7 @@ import * as Base64Url from "effect/encoding/Base64Url"; import * as Hex from "effect/encoding/Hex"; import * as KeyedLock from "@t3tools/shared/KeyedLock"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import { isGitImport, parseTurnDiffFilesFromNumstat } from "../checkpointing/Diffs.ts"; @@ -250,6 +252,28 @@ export const layer: Layer.Layer< const withWorkspaceLock = (cwd: string, effect: Effect.Effect) => workspaceLocks.withLock(cwd, effect); + const captureTimeouts = yield* Cache.make({ + capacity: 128, + lookup: () => Effect.succeed(0), + }); + // Both callers hold the workspace lock; post-turn captures always probe for recovery. + const captureWorkspaceCheckpoint = Effect.fn("CheckpointService.captureWorkspaceCheckpoint")( + function* (input: CheckpointStore.CaptureCheckpointInput, baseline = false) { + const capture = checkpointStore.captureCheckpoint(input); + yield* (baseline ? capture.pipe(Effect.timeout(BASELINE_CAPTURE_BUDGET)) : capture).pipe( + Effect.tap(() => Cache.invalidate(captureTimeouts, input.cwd)), + Effect.tapError((error) => + error._tag === "VcsProcessTimeoutError" || error._tag === "TimeoutError" + ? Effect.gen(function* () { + const previous = yield* Cache.get(captureTimeouts, input.cwd); + yield* Cache.set(captureTimeouts, input.cwd, previous + 1); + }) + : Cache.invalidate(captureTimeouts, input.cwd), + ), + ); + }, + ); + // A failed probe is an operational error, not proof that the workspace has no repository. const probeRepository = (scope: OrchestrationV2CheckpointScope) => checkpointStore.isGitRepository(scope.cwd).pipe( @@ -284,17 +308,19 @@ export const layer: Layer.Layer< return; } - yield* checkpointStore - .captureCheckpoint({ - cwd: input.scope.cwd, - checkpointRef, - }) - .pipe( - Effect.timeout(BASELINE_CAPTURE_BUDGET), - Effect.catchTags({ - VcsUnsupportedOperationError: () => Effect.void, - }), + const timeouts = yield* Cache.getOption(captureTimeouts, input.scope.cwd); + if (Option.isSome(timeouts) && timeouts.value >= 2) { + return yield* Effect.logWarning( + "orchestration V2 checkpoint baseline skipped after repeated capture timeouts", + { scopeId: input.scope.id, ordinalWithinScope: input.ordinalWithinScope }, ); + } + + yield* captureWorkspaceCheckpoint({ cwd: input.scope.cwd, checkpointRef }, true).pipe( + Effect.catchTags({ + VcsUnsupportedOperationError: () => Effect.void, + }), + ); }), ).pipe( Effect.mapError( @@ -406,25 +432,23 @@ export const layer: Layer.Layer< }); } - const captureStatus = yield* checkpointStore - .captureCheckpoint({ - cwd: input.scope.cwd, - checkpointRef, - ...(input.ordinalWithinScope > 0 ? { previousCheckpointRef } : {}), - }) - .pipe( - Effect.as("ready" as const), - Effect.catchTags({ - VcsUnsupportedOperationError: () => Effect.succeed("missing" as const), - }), - Effect.catch((cause) => - Effect.logWarning("orchestration V2 checkpoint capture failed", { - scopeId: input.scope.id, - checkpointRef, - cause: String(cause), - }).pipe(Effect.as("error" as const)), - ), - ); + const captureStatus = yield* captureWorkspaceCheckpoint({ + cwd: input.scope.cwd, + checkpointRef, + ...(input.ordinalWithinScope > 0 ? { previousCheckpointRef } : {}), + }).pipe( + Effect.as("ready" as const), + Effect.catchTags({ + VcsUnsupportedOperationError: () => Effect.succeed("missing" as const), + }), + Effect.catch((cause) => + Effect.logWarning("orchestration V2 checkpoint capture failed", { + scopeId: input.scope.id, + checkpointRef, + cause: String(cause), + }).pipe(Effect.as("error" as const)), + ), + ); if (captureStatus !== "ready") { return makeCheckpoint({