From f188c79a5f9d92edf108fc379501b6035c2fb45a Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 6 Nov 2026 10:52:48 +0000 Subject: [PATCH 1/6] test(server): reproduce #31954, failing pull request reads are retried every sweep A link whose summary read keeps failing is read on every one-minute sweep: a never-synced link has no lastSyncedAt, and a link that arrived through requestSync stays requested after the failure. These tests reproduce #31954 and fail on main. Co-Authored-By: Claude Opus 5.5 --- .../PullRequestSyncReactor.test.ts | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts b/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts index 0161054781567..c6133a54317c9 100644 --- a/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts +++ b/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts @@ -302,6 +302,24 @@ const sweepAgain = Effect.fn("sweepPullRequestSyncHarness")(function* ( yield* reactor.drain; }); +const sweepFor = Effect.fn("sweepPullRequestSyncHarnessFor")(function* ( + fixture: Harness, + reactor: PullRequestSyncReactor.PullRequestSyncReactor["Service"], + minutes: number, +) { + for (let minute = 0; minute < minutes; minute++) yield* sweepAgain(fixture, reactor); +}); + +const failingGitLabLink = makeLink(13, null, { + host: "gitlab.com", + url: "https://gitlab.com/owner/repository/-/merge_requests/13", +}); + +const gitLabCliFailure = new PullRequestOperationError({ + operation: "summary", + detail: "GitLab CLI command failed", +}); + function runUpdated( threadId: ThreadId, status: OrchestrationV2Run["status"], @@ -1519,6 +1537,77 @@ describe("PullRequestSyncReactor", () => { ), ); + it.effect("backs off reading a never-synced link whose host read keeps failing", () => + Effect.scoped( + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse(NOW)); + const fixture = yield* makeHarness({ + snapshot: makeSnapshot([makeThread("one", { pullRequests: [failingGitLabLink] })]), + summary: () => Effect.fail(gitLabCliFailure), + }); + yield* Effect.gen(function* () { + const reactor = yield* startAndSweep(fixture); + yield* sweepFor(fixture, reactor, 10); + const readsInTenMinutes = (yield* Ref.get(fixture.summaryCalls)).length; + assert.isAtMost( + readsInTenMinutes, + 5, + "a link whose read keeps failing is read on every one-minute sweep", + ); + yield* sweepFor(fixture, reactor, 6); + assert.isAbove( + (yield* Ref.get(fixture.summaryCalls)).length, + readsInTenMinutes, + "a link whose read keeps failing is never retried", + ); + }).pipe(Effect.provide(fixture.layer)); + }), + ), + ); + + it.effect("backs off reading a newly linked pull request whose host read keeps failing", () => + Effect.scoped( + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse(NOW)); + const fixture = yield* makeHarness({ + snapshot: makeSnapshot([makeThread("one")]), + summary: () => Effect.fail(gitLabCliFailure), + }); + yield* Effect.gen(function* () { + const reactor = yield* startAndSweep(fixture); + yield* Ref.set( + fixture.snapshots, + makeSnapshot([makeThread("one", { pullRequests: [failingGitLabLink] })], 2), + ); + yield* Queue.offer(fixture.domainEvents, { + id: EventId.make("event:one:link-updated"), + type: "thread.pull-request-link-updated", + threadId: ThreadId.make("one"), + occurredAt: DateTime.makeUnsafe(NOW), + payload: failingGitLabLink, + }); + yield* Queue.take(fixture.snapshotReads); + yield* reactor.drain; + assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, 1); + + yield* sweepFor(fixture, reactor, 10); + const readsInTenMinutes = (yield* Ref.get(fixture.summaryCalls)).length; + assert.isAtMost( + readsInTenMinutes, + 5, + "a requested link whose read keeps failing is read on every one-minute sweep", + ); + yield* sweepFor(fixture, reactor, 6); + assert.isAbove( + (yield* Ref.get(fixture.summaryCalls)).length, + readsInTenMinutes, + "a requested link whose read keeps failing is never retried", + ); + }).pipe(Effect.provide(fixture.layer)); + }), + ), + ); + it.effect.each([ "gh pr merge 7 --squash 2>&1 | tail -3", "cafe pr merge 7 --squash", From 8edf3a2ff2462ef687c20b32f6a8ba39ff70bd14 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 6 Nov 2026 10:58:03 +0000 Subject: [PATCH 2/6] fix(server): pull request links whose reads keep failing back off instead of re-reading every minute A link with no snapshot, or one requested through requestSync, was due on every one-minute sweep, and only a successful read moved its cadence clock or answered its request. A link whose host read kept failing (a wrong project path, revoked access, a generic GitLab CLI failure) was read every minute forever. A failed read that is not a rate limit now backs its key off: one sweep after the first failure, doubling after each further one, capped at the slow sync interval. A successful read clears it. A pending request still waits for the backoff, so a failed refresh is retried rather than dropped, while a new request reads at once. Rate limits keep their per-host pause. Fixes pingdotgg/t3code#31954 Co-Authored-By: Claude Opus 5.5 --- .../PullRequestSyncReactor.test.ts | 25 +++++++++++++++++++ .../PullRequestSyncReactor.ts | 19 +++++++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts b/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts index c6133a54317c9..920197ca6f112 100644 --- a/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts +++ b/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts @@ -1565,6 +1565,31 @@ describe("PullRequestSyncReactor", () => { ), ); + it.effect("reads a backed-off link at once when a refresh asks, then keeps backing off", () => + Effect.scoped( + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse(NOW)); + const fixture = yield* makeHarness({ + snapshot: makeSnapshot([makeThread("one", { pullRequests: [failingGitLabLink] })]), + summary: () => Effect.fail(gitLabCliFailure), + }); + yield* Effect.gen(function* () { + const reactor = yield* startAndSweep(fixture); + yield* sweepFor(fixture, reactor, 4); + const backedOffReads = (yield* Ref.get(fixture.summaryCalls)).length; + + yield* reactor.requestSync(failingGitLabLink); + yield* Queue.take(fixture.snapshotReads); + yield* reactor.drain; + assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, backedOffReads + 1); + + yield* sweepAgain(fixture, reactor); + assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, backedOffReads + 1); + }).pipe(Effect.provide(fixture.layer)); + }), + ), + ); + it.effect("backs off reading a newly linked pull request whose host read keeps failing", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/PullRequestSyncReactor.ts b/apps/server/src/orchestration-v2/PullRequestSyncReactor.ts index 9dffe4babb424..19ce7de1f8ddb 100644 --- a/apps/server/src/orchestration-v2/PullRequestSyncReactor.ts +++ b/apps/server/src/orchestration-v2/PullRequestSyncReactor.ts @@ -47,6 +47,8 @@ const QUIET_SYNC_INTERVAL_MS = 5 * 60 * 1_000; // A pull request closed this long ago is rarely reopened, so the sweep stops reading it, as it // does merged ones. A reader that sees it open, or a requested refresh, still reads it at once. const CLOSED_FINAL_AFTER_MS = 24 * 60 * 60 * 1_000; +// A link whose host read failed waits a sweep, then twice as long after each further failure. +const FAILED_READ_BACKOFF_MS = 60 * 1_000; const isChangeRequestCloseCommand = (command: string) => sourceControlDefinitions.definitions.some( @@ -346,10 +348,12 @@ export const make = Effect.gen(function* () { // linking dozens of pull requests) is read together and shares the summary batches. let requestedSweepQueued = false; const retryStacks = new Set(); + const failedReads = new Map(); // Pause per project and host because credentials are project-specific; leave paused PRs due until reset. const pausedUntil = new Map(); const isDue = (key: string, entries: ReadonlyArray, nowMs: number): boolean => { + if ((failedReads.get(key)?.retryAtMs ?? 0) > nowMs) return false; if (requested.has(key) || retryStacks.has(key)) return true; if (entries.some((entry) => entry.link.snapshot === null)) return true; const active = entries.filter((entry) => isUnsettled(entry.thread)); @@ -397,12 +401,17 @@ export const make = Effect.gen(function* () { // A host-level refresh reaches every project, each retaining its own pending retry. for (const [key, entries] of groups) { - if (pendingRequests.has(threadPullRequestKeyOf(entries[0]!.link))) requested.add(key); + if (!pendingRequests.has(threadPullRequestKeyOf(entries[0]!.link))) continue; + requested.add(key); + // A new request reads at once; a failure after it backs off further, not from the start. + const failed = failedReads.get(key); + if (failed !== undefined) failedReads.set(key, { ...failed, retryAtMs: nowMs }); } pendingRequests.clear(); for (const key of lastSyncedAt.keys()) if (!groups.has(key)) lastSyncedAt.delete(key); for (const key of retryStacks) if (!groups.has(key)) retryStacks.delete(key); + for (const key of failedReads.keys()) if (!groups.has(key)) failedReads.delete(key); for (const key of requested.keys()) if (!groups.has(key)) requested.delete(key); const linkedThisSweep = new Set(); @@ -562,6 +571,7 @@ export const make = Effect.gen(function* () { } // The host answered, so the cadence clock ticks even if a dispatch below is rejected. lastSyncedAt.set(key, nowMs); + failedReads.delete(key); requested.delete(key); yield* Effect.forEach( entries, @@ -587,6 +597,13 @@ export const make = Effect.gen(function* () { const retryAt = rateLimitRetryAt(cause); if (retryAt !== undefined) { pausedUntil.set(pauseKey, Math.max(retryAt, pausedUntil.get(pauseKey) ?? 0)); + } else { + const previous = failedReads.get(key); + const backoffMs = + previous === undefined + ? FAILED_READ_BACKOFF_MS + : Math.min(previous.backoffMs * 2, SLOW_SYNC_INTERVAL_MS); + failedReads.set(key, { retryAtMs: nowMs + backoffMs, backoffMs }); } const reason = skipReason(cause); const skip = skips.get(reason); From 4397f86c8208f2e94f4416879f7395ad320b8e42 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 6 Nov 2026 11:27:38 +0000 Subject: [PATCH 3/6] test(server): pin the failed pull request read backoff cap and its reset after a success Co-Authored-By: Claude Opus 5.5 --- .../PullRequestSyncReactor.test.ts | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts b/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts index 920197ca6f112..c58e6cc940828 100644 --- a/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts +++ b/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts @@ -1565,6 +1565,48 @@ describe("PullRequestSyncReactor", () => { ), ); + it.effect( + "caps the failed-read backoff at the slow cadence and restarts it after a success", + () => + Effect.scoped( + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse(NOW)); + const failing = yield* Ref.make(true); + const fixture = yield* makeHarness({ + snapshot: makeSnapshot([makeThread("one", { pullRequests: [failingGitLabLink] })]), + summary: (input) => + Ref.get(failing).pipe( + Effect.flatMap((fails) => + fails ? Effect.fail(gitLabCliFailure) : Effect.succeed(makeSummary(input)), + ), + ), + }); + yield* Effect.gen(function* () { + const reactor = yield* startAndSweep(fixture); + const readMinutes = [0]; + const sweepMinutes = Effect.fnUntraced(function* (from: number, to: number) { + for (let minute = from; minute <= to; minute++) { + const before = (yield* Ref.get(fixture.summaryCalls)).length; + yield* sweepAgain(fixture, reactor); + if ((yield* Ref.get(fixture.summaryCalls)).length > before) + readMinutes.push(minute); + } + }); + + yield* sweepMinutes(1, 60); + assert.deepStrictEqual(readMinutes, [0, 1, 3, 7, 15, 30, 45, 60]); + + yield* Ref.set(failing, false); + yield* sweepMinutes(61, 75); + yield* Ref.set(failing, true); + readMinutes.length = 0; + yield* sweepMinutes(76, 80); + assert.deepStrictEqual(readMinutes, [76, 77, 79]); + }).pipe(Effect.provide(fixture.layer)); + }), + ), + ); + it.effect("reads a backed-off link at once when a refresh asks, then keeps backing off", () => Effect.scoped( Effect.gen(function* () { From 9eab4baebfea79fa80e64dae04c4e805c48d836b Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 6 Nov 2026 11:33:45 +0000 Subject: [PATCH 4/6] test(server): drop the never-synced backoff test the exact schedule test covers Co-Authored-By: Claude Opus 5.5 --- .../PullRequestSyncReactor.test.ts | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts b/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts index c58e6cc940828..ab55dc83cdb12 100644 --- a/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts +++ b/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts @@ -1537,34 +1537,6 @@ describe("PullRequestSyncReactor", () => { ), ); - it.effect("backs off reading a never-synced link whose host read keeps failing", () => - Effect.scoped( - Effect.gen(function* () { - yield* TestClock.setTime(Date.parse(NOW)); - const fixture = yield* makeHarness({ - snapshot: makeSnapshot([makeThread("one", { pullRequests: [failingGitLabLink] })]), - summary: () => Effect.fail(gitLabCliFailure), - }); - yield* Effect.gen(function* () { - const reactor = yield* startAndSweep(fixture); - yield* sweepFor(fixture, reactor, 10); - const readsInTenMinutes = (yield* Ref.get(fixture.summaryCalls)).length; - assert.isAtMost( - readsInTenMinutes, - 5, - "a link whose read keeps failing is read on every one-minute sweep", - ); - yield* sweepFor(fixture, reactor, 6); - assert.isAbove( - (yield* Ref.get(fixture.summaryCalls)).length, - readsInTenMinutes, - "a link whose read keeps failing is never retried", - ); - }).pipe(Effect.provide(fixture.layer)); - }), - ), - ); - it.effect( "caps the failed-read backoff at the slow cadence and restarts it after a success", () => From cc131a6e98b25f431fd26839ad19a726505fa0d9 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 6 Nov 2026 11:44:36 +0000 Subject: [PATCH 5/6] test(server): assert a failed refresh doubles the pull request read backoff Co-Authored-By: Claude Opus 5.5 --- .../src/orchestration-v2/PullRequestSyncReactor.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts b/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts index ab55dc83cdb12..6ff53f6548722 100644 --- a/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts +++ b/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts @@ -1597,8 +1597,11 @@ describe("PullRequestSyncReactor", () => { yield* reactor.drain; assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, backedOffReads + 1); - yield* sweepAgain(fixture, reactor); + // The refresh failed at minute 4 with the backoff doubled to 8 minutes. + yield* sweepFor(fixture, reactor, 7); assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, backedOffReads + 1); + yield* sweepAgain(fixture, reactor); + assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, backedOffReads + 2); }).pipe(Effect.provide(fixture.layer)); }), ), From 42add8817aabdf3447f6b160320fde6cb39315bb Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 10 Nov 2026 21:11:05 +0000 Subject: [PATCH 6/6] test(server): drop the newly linked backoff test the schedule test already covers AI-assisted; verified with the tests and tools listed above. --- .../PullRequestSyncReactor.test.ts | 43 ------------------- 1 file changed, 43 deletions(-) diff --git a/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts b/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts index 6ff53f6548722..9b5174fdc6fdb 100644 --- a/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts +++ b/apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts @@ -1607,49 +1607,6 @@ describe("PullRequestSyncReactor", () => { ), ); - it.effect("backs off reading a newly linked pull request whose host read keeps failing", () => - Effect.scoped( - Effect.gen(function* () { - yield* TestClock.setTime(Date.parse(NOW)); - const fixture = yield* makeHarness({ - snapshot: makeSnapshot([makeThread("one")]), - summary: () => Effect.fail(gitLabCliFailure), - }); - yield* Effect.gen(function* () { - const reactor = yield* startAndSweep(fixture); - yield* Ref.set( - fixture.snapshots, - makeSnapshot([makeThread("one", { pullRequests: [failingGitLabLink] })], 2), - ); - yield* Queue.offer(fixture.domainEvents, { - id: EventId.make("event:one:link-updated"), - type: "thread.pull-request-link-updated", - threadId: ThreadId.make("one"), - occurredAt: DateTime.makeUnsafe(NOW), - payload: failingGitLabLink, - }); - yield* Queue.take(fixture.snapshotReads); - yield* reactor.drain; - assert.strictEqual((yield* Ref.get(fixture.summaryCalls)).length, 1); - - yield* sweepFor(fixture, reactor, 10); - const readsInTenMinutes = (yield* Ref.get(fixture.summaryCalls)).length; - assert.isAtMost( - readsInTenMinutes, - 5, - "a requested link whose read keeps failing is read on every one-minute sweep", - ); - yield* sweepFor(fixture, reactor, 6); - assert.isAbove( - (yield* Ref.get(fixture.summaryCalls)).length, - readsInTenMinutes, - "a requested link whose read keeps failing is never retried", - ); - }).pipe(Effect.provide(fixture.layer)); - }), - ), - ); - it.effect.each([ "gh pr merge 7 --squash 2>&1 | tail -3", "cafe pr merge 7 --squash",