diff --git a/apps/server/src/orchestration-v2/CheckpointService.test.ts b/apps/server/src/orchestration-v2/CheckpointService.test.ts
index 62fa68c4c53af..fec015b5a1b63 100644
--- a/apps/server/src/orchestration-v2/CheckpointService.test.ts
+++ b/apps/server/src/orchestration-v2/CheckpointService.test.ts
@@ -11,6 +11,7 @@ import {
VcsProcessTimeoutError,
VcsUnsupportedOperationError,
} from "@t3tools/contracts";
+import * as Cause from "effect/Cause";
import * as DateTime from "effect/DateTime";
import * as Deferred from "effect/Deferred";
import * as Effect from "effect/Effect";
@@ -164,12 +165,23 @@ it.effect(
assert.strictEqual((yield* Fiber.join(postTurn)).status, "ready");
assert.strictEqual((yield* Fiber.join(waitingBaseline))._tag, "Success");
- const slowBaseline = yield* checkpoints
- .captureBaseline({ scope, ordinalWithinScope: 2 })
+ for (const ordinalWithinScope of [2, 3]) {
+ const slowBaseline = yield* checkpoints
+ .captureBaseline({ scope, ordinalWithinScope })
+ .pipe(Effect.result, Effect.forkChild);
+ yield* TestClock.adjust("5 seconds");
+ const result = yield* Fiber.join(slowBaseline);
+ assert.strictEqual(result._tag, "Failure");
+ if (result._tag === "Failure") {
+ assert.isTrue(Cause.isTimeoutError(result.failure.cause));
+ }
+ }
+ const backedOffBaseline = yield* checkpoints
+ .captureBaseline({ scope, ordinalWithinScope: 4 })
.pipe(Effect.result, Effect.forkChild);
yield* TestClock.adjust("5 seconds");
- assert.strictEqual((yield* Fiber.join(slowBaseline))._tag, "Failure");
- assert.strictEqual(yield* Ref.get(snapshots), 2);
+ assert.strictEqual((yield* Fiber.join(backedOffBaseline))._tag, "Success");
+ assert.strictEqual(yield* Ref.get(snapshots), 3);
assert.strictEqual((yield* Ref.get(refs)).size, 1);
}).pipe(Effect.provide(layerTest));
}),
@@ -233,3 +245,113 @@ it.effect("records a repository probe failure as an error checkpoint, not a miss
assert.deepEqual(statuses, { baseline: "error", turn: "error" });
}),
);
+
+it.effect("backs off only baseline retries after repeated workspace capture timeouts", () =>
+ Effect.gen(function* () {
+ const scope: OrchestrationV2CheckpointScope = {
+ id: CheckpointScopeId.make("checkpoint-scope:slow-capture"),
+ threadId: ThreadId.make("thread:slow-capture"),
+ runId: RunId.make("run:slow-capture:1"),
+ nodeId: NodeId.make("node:slow-capture:1"),
+ parentScopeId: null,
+ providerThreadId: ProviderThreadId.make("provider-thread:slow-capture"),
+ kind: "root_run",
+ ordinalWithinParent: 0,
+ advancesAppRunCount: true,
+ cwd: "/repo/slow",
+ createdAt: DateTime.makeUnsafe("2026-07-28T00:00:00.000Z"),
+ };
+ const timeout = new VcsProcessTimeoutError({
+ operation: "GitVcsDriver.checkpoints.captureCheckpoint",
+ command: "git",
+ cwd: scope.cwd,
+ timeoutMs: 30_000,
+ });
+ let attempts = 0;
+ let healthy = false;
+ let nonTimeoutFailure = false;
+ const failure = new VcsProcessExitError({
+ operation: "test.capture",
+ command: "git",
+ cwd: scope.cwd,
+ exitCode: 128,
+ detail: "capture failed",
+ });
+ const layerTest = CheckpointService.layer.pipe(
+ Layer.provide(
+ Layer.mergeAll(
+ IdAllocator.layer,
+ Layer.mock(CheckpointStore.CheckpointStore)({
+ isGitRepository: () => Effect.succeed(true),
+ hasCheckpointRef: () => Effect.succeed(false),
+ captureCheckpoint: () =>
+ Effect.suspend(() => {
+ attempts++;
+ return healthy ? Effect.void : Effect.fail(nonTimeoutFailure ? failure : timeout);
+ }),
+ }),
+ ),
+ ),
+ Layer.provideMerge(NodeCrypto.layer),
+ );
+ yield* Effect.gen(function* () {
+ const checkpoints = yield* CheckpointService.CheckpointServiceV2;
+ const baseline = { scope, ordinalWithinScope: 0 };
+ const turn = {
+ scope,
+ ordinalWithinScope: 1,
+ runId: scope.runId,
+ nodeId: scope.nodeId,
+ appRunOrdinal: 1,
+ capturedAt: scope.createdAt,
+ };
+ yield* Effect.result(checkpoints.captureBaseline(baseline));
+ assert.strictEqual((yield* checkpoints.capture(turn)).status, "error");
+ assert.strictEqual(attempts, 2);
+
+ const otherThread = { ...scope, id: CheckpointScopeId.make("checkpoint-scope:other") };
+ const skipped = yield* Effect.result(
+ checkpoints.captureBaseline({ scope: otherThread, ordinalWithinScope: 0 }),
+ );
+ assert.strictEqual(attempts, 2, "baseline retried a repeatedly timing-out workspace");
+ assert.strictEqual(skipped._tag, "Success");
+
+ yield* Effect.result(
+ checkpoints.captureBaseline({
+ scope: { ...otherThread, cwd: "/repo/other" },
+ ordinalWithinScope: 0,
+ }),
+ );
+ assert.strictEqual(attempts, 3, "another workspace must still attempt capture");
+
+ const failedTurn = yield* checkpoints.capture(turn);
+ assert.strictEqual(failedTurn.status, "error");
+ assert.strictEqual(attempts, 4, "post-turn capture must not back off");
+
+ yield* checkpoints.captureBaseline(baseline);
+ assert.strictEqual(attempts, 4, "baseline retries must wait for post-turn recovery");
+ healthy = true;
+ assert.strictEqual((yield* checkpoints.capture(turn)).status, "ready");
+ assert.strictEqual(attempts, 5);
+ yield* checkpoints.captureBaseline(baseline);
+ assert.strictEqual(attempts, 6, "a successful post-turn capture must clear backoff");
+
+ healthy = false;
+ yield* Effect.result(checkpoints.captureBaseline(baseline));
+ yield* Effect.result(checkpoints.captureBaseline(baseline));
+ yield* checkpoints.captureBaseline(baseline);
+ assert.strictEqual(attempts, 8);
+ nonTimeoutFailure = true;
+ assert.strictEqual((yield* checkpoints.capture(turn)).status, "error");
+ nonTimeoutFailure = false;
+ yield* Effect.result(checkpoints.captureBaseline(baseline));
+ assert.strictEqual(attempts, 10, "a non-timeout failure must clear backoff");
+ healthy = true;
+ yield* checkpoints.captureBaseline(baseline);
+ healthy = false;
+ yield* Effect.result(checkpoints.captureBaseline(baseline));
+ yield* Effect.result(checkpoints.captureBaseline(baseline));
+ assert.strictEqual(attempts, 13, "a successful baseline must clear the timeout streak");
+ }).pipe(Effect.provide(layerTest));
+ }),
+);
diff --git a/apps/server/src/orchestration-v2/CheckpointService.ts b/apps/server/src/orchestration-v2/CheckpointService.ts
index 3e0fbe95e8aa4..fb6490834c7f6 100644
--- a/apps/server/src/orchestration-v2/CheckpointService.ts
+++ b/apps/server/src/orchestration-v2/CheckpointService.ts
@@ -9,6 +9,7 @@ import {
RunId,
ThreadId,
} from "@t3tools/contracts";
+import * as Cache from "effect/Cache";
import * as Context from "effect/Context";
import * as Digest from "effect/crypto/Digest";
import * as DateTime from "effect/DateTime";
@@ -17,6 +18,7 @@ import * as Base64Url from "effect/encoding/Base64Url";
import * as Hex from "effect/encoding/Hex";
import * as KeyedLock from "@t3tools/shared/KeyedLock";
import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
import * as Schema from "effect/Schema";
import { isGitImport, parseTurnDiffFilesFromNumstat } from "../checkpointing/Diffs.ts";
@@ -250,6 +252,28 @@ export const layer: Layer.Layer<
const withWorkspaceLock = (cwd: string, effect: Effect.Effect) =>
workspaceLocks.withLock(cwd, effect);
+ const captureTimeouts = yield* Cache.make({
+ capacity: 128,
+ lookup: () => Effect.succeed(0),
+ });
+ // Both callers hold the workspace lock; post-turn captures always probe for recovery.
+ const captureWorkspaceCheckpoint = Effect.fn("CheckpointService.captureWorkspaceCheckpoint")(
+ function* (input: CheckpointStore.CaptureCheckpointInput, baseline = false) {
+ const capture = checkpointStore.captureCheckpoint(input);
+ yield* (baseline ? capture.pipe(Effect.timeout(BASELINE_CAPTURE_BUDGET)) : capture).pipe(
+ Effect.tap(() => Cache.invalidate(captureTimeouts, input.cwd)),
+ Effect.tapError((error) =>
+ error._tag === "VcsProcessTimeoutError" || error._tag === "TimeoutError"
+ ? Effect.gen(function* () {
+ const previous = yield* Cache.get(captureTimeouts, input.cwd);
+ yield* Cache.set(captureTimeouts, input.cwd, previous + 1);
+ })
+ : Cache.invalidate(captureTimeouts, input.cwd),
+ ),
+ );
+ },
+ );
+
// A failed probe is an operational error, not proof that the workspace has no repository.
const probeRepository = (scope: OrchestrationV2CheckpointScope) =>
checkpointStore.isGitRepository(scope.cwd).pipe(
@@ -284,17 +308,19 @@ export const layer: Layer.Layer<
return;
}
- yield* checkpointStore
- .captureCheckpoint({
- cwd: input.scope.cwd,
- checkpointRef,
- })
- .pipe(
- Effect.timeout(BASELINE_CAPTURE_BUDGET),
- Effect.catchTags({
- VcsUnsupportedOperationError: () => Effect.void,
- }),
+ const timeouts = yield* Cache.getOption(captureTimeouts, input.scope.cwd);
+ if (Option.isSome(timeouts) && timeouts.value >= 2) {
+ return yield* Effect.logWarning(
+ "orchestration V2 checkpoint baseline skipped after repeated capture timeouts",
+ { scopeId: input.scope.id, ordinalWithinScope: input.ordinalWithinScope },
);
+ }
+
+ yield* captureWorkspaceCheckpoint({ cwd: input.scope.cwd, checkpointRef }, true).pipe(
+ Effect.catchTags({
+ VcsUnsupportedOperationError: () => Effect.void,
+ }),
+ );
}),
).pipe(
Effect.mapError(
@@ -406,25 +432,23 @@ export const layer: Layer.Layer<
});
}
- const captureStatus = yield* checkpointStore
- .captureCheckpoint({
- cwd: input.scope.cwd,
- checkpointRef,
- ...(input.ordinalWithinScope > 0 ? { previousCheckpointRef } : {}),
- })
- .pipe(
- Effect.as("ready" as const),
- Effect.catchTags({
- VcsUnsupportedOperationError: () => Effect.succeed("missing" as const),
- }),
- Effect.catch((cause) =>
- Effect.logWarning("orchestration V2 checkpoint capture failed", {
- scopeId: input.scope.id,
- checkpointRef,
- cause: String(cause),
- }).pipe(Effect.as("error" as const)),
- ),
- );
+ const captureStatus = yield* captureWorkspaceCheckpoint({
+ cwd: input.scope.cwd,
+ checkpointRef,
+ ...(input.ordinalWithinScope > 0 ? { previousCheckpointRef } : {}),
+ }).pipe(
+ Effect.as("ready" as const),
+ Effect.catchTags({
+ VcsUnsupportedOperationError: () => Effect.succeed("missing" as const),
+ }),
+ Effect.catch((cause) =>
+ Effect.logWarning("orchestration V2 checkpoint capture failed", {
+ scopeId: input.scope.id,
+ checkpointRef,
+ cause: String(cause),
+ }).pipe(Effect.as("error" as const)),
+ ),
+ );
if (captureStatus !== "ready") {
return makeCheckpoint({