diff --git a/.github/scripts/thread-transfer-report.cjs b/.github/scripts/thread-transfer-report.cjs index 09b25131e4c48..304b56d57e400 100644 --- a/.github/scripts/thread-transfer-report.cjs +++ b/.github/scripts/thread-transfer-report.cjs @@ -3,6 +3,7 @@ const path = require("node:path"); const ARTIFACT_NAME = "thread-transfer-results"; const RESULT_FILE = "thread-transfer-result.json"; +const SCENARIOS_FILE = "thread-transfer-scenarios.json"; const COMMENT_MARKER = ""; const PROVIDERS = ["codex", "claudeAgent"]; const OBSERVED_KEYS = [ @@ -29,6 +30,12 @@ const SCENARIO_KEYS = [ "measuredMcpResultBytes", ]; +const SCENARIO_ROW_KEYS = ["phase", "metric", "unit", "observed", "ceiling"]; +// These labels come from the PR and reach a comment the bot posts: no markup, links or mentions. +// GitHub still links plain text such as `GH-26`, so they are rendered as code spans. +const SCENARIO_LABEL = /^[A-Za-z0-9][A-Za-z0-9 ,-]{0,79}$/; +const MAX_SCENARIO_ROWS = 100; + function resultShaMarker(sha) { return ``; } @@ -85,15 +92,63 @@ function validateResult(value) { return value; } -function readResult(directory) { +function validateScenarios(value) { + assertExactKeys(value, ["schemaVersion", "providers"], "scenarios"); + if (value.schemaVersion !== 1) { + throw new Error("scenarios.schemaVersion must be 1"); + } + assertExactKeys(value.providers, PROVIDERS, "scenarios.providers"); + for (const provider of PROVIDERS) { + const rows = value.providers[provider]; + if (!Array.isArray(rows) || rows.length > MAX_SCENARIO_ROWS) { + throw new Error( + `scenarios.providers.${provider} must list at most ${MAX_SCENARIO_ROWS} rows`, + ); + } + const seen = new Set(); + for (const [index, row] of rows.entries()) { + const label = `scenarios.providers.${provider}[${index}]`; + assertExactKeys(row, SCENARIO_ROW_KEYS, label); + for (const key of ["phase", "metric"]) { + if (typeof row[key] !== "string" || !SCENARIO_LABEL.test(row[key])) { + throw new Error(`${label}.${key} must be a short plain label`); + } + } + if (row.unit !== "bytes" && row.unit !== "count") { + throw new Error(`${label}.unit must be bytes or count`); + } + assertMetric(row.observed, `${label}.observed`); + assertMetric(row.ceiling, `${label}.ceiling`); + const key = scenarioKey(row); + if (seen.has(key)) throw new Error(`${label} repeats an earlier row`); + seen.add(key); + } + } + return value; +} + +function scenarioKey(row) { + return `${row.phase}|${row.metric}`; +} + +function readArtifactFile(directory, name, validate) { if (!directory) return undefined; - const file = path.join(directory, RESULT_FILE); + const file = path.join(directory, name); if (!fs.existsSync(file)) return undefined; const stat = fs.lstatSync(file); if (!stat.isFile() || stat.size > 64 * 1_024) { - throw new Error("thread transfer result must be a regular file smaller than 64 KiB"); + throw new Error(`${name} must be a regular file smaller than 64 KiB`); } - return validateResult(JSON.parse(fs.readFileSync(file, "utf8"))); + return validate(JSON.parse(fs.readFileSync(file, "utf8"))); +} + +function readResult(directory) { + return readArtifactFile(directory, RESULT_FILE, validateResult); +} + +// Older runs upload only the result file, so a run without scenarios is still a valid run. +function readScenarios(directory) { + return readArtifactFile(directory, SCENARIOS_FILE, validateScenarios); } function formatBytes(bytes) { @@ -103,7 +158,7 @@ function formatBytes(bytes) { } function formatValue(value, kind) { - return kind === "messages" ? value.toLocaleString("en-US") : formatBytes(value); + return kind === "bytes" ? formatBytes(value) : value.toLocaleString("en-US"); } function formatImpact(current, baseline, kind) { @@ -136,6 +191,77 @@ const METRICS = [ { key: "measuredTurnWebSocketMessages", label: "Live turn messages", kind: "messages" }, ]; +function providerLabel(provider) { + return provider === "codex" ? "Codex" : "Claude"; +} + +/** + * The scenario rows of a run against the same rows on `main`. Rows that fail or differ from + * `main` are listed in the open; the full table is collapsed below them. + */ +function renderScenarios(current, baseline, ceilingChanges) { + if (!current) return { lines: [], failed: false }; + const header = [ + "| Provider | Scenario | Metric | Main baseline | This PR | Impact | PR ceiling | |", + "| --- | --- | --- | ---: | ---: | ---: | ---: | --- |", + ]; + const all = []; + const notable = []; + let failed = false; + let withoutBaseline = 0; + + for (const provider of PROVIDERS) { + const baselineRows = new Map( + (baseline?.providers[provider] ?? []).map((row) => [scenarioKey(row), row]), + ); + for (const row of current.providers[provider]) { + const before = baselineRows.get(scenarioKey(row)); + const beforeObserved = before?.unit === row.unit ? before.observed : undefined; + const pass = row.observed <= row.ceiling; + failed ||= !pass; + if (beforeObserved === undefined) withoutBaseline += 1; + const line = `| ${providerLabel(provider)} | \`${row.phase}\` | \`${row.metric}\` | ${beforeObserved === undefined ? "—" : formatValue(beforeObserved, row.unit)} | ${formatValue(row.observed, row.unit)} | ${formatImpact(row.observed, beforeObserved, row.unit)} | ${formatValue(row.ceiling, row.unit)} | ${pass ? "✅" : "❌"} |`; + all.push(line); + if (!pass || (beforeObserved !== undefined && beforeObserved !== row.observed)) { + notable.push(line); + } + if (beforeObserved !== undefined && before.ceiling !== row.ceiling) { + ceilingChanges.push( + `- ${providerLabel(provider)} \`${row.phase}\` \`${row.metric}\`: ${formatValue(before.ceiling, row.unit)} → ${formatValue(row.ceiling, row.unit)}`, + ); + } + } + } + + const compared = all.length - withoutBaseline; + const summary = !baseline + ? `No \`main\` baseline exists yet for these ${all.length} scenario measurements.` + : [ + notable.length > 0 + ? `${notable.length} of ${all.length} scenario measurements changed or failed.` + : `${compared === all.length ? "All " : ""}${compared} scenario measurements match \`main\`.`, + ...(withoutBaseline > 0 ? [`${withoutBaseline} have no \`main\` baseline.`] : []), + ].join(" "); + return { + failed, + lines: [ + "### Other scenarios", + "", + summary, + ...(notable.length > 0 ? ["", ...header, ...notable] : []), + "", + "
", + `All ${all.length} scenario measurements`, + "", + ...header, + ...all, + "", + "
", + "", + ], + }; +} + function renderComment(input) { const current = input.current; const baseline = input.baseline; @@ -154,16 +280,22 @@ function renderComment(input) { const pass = observed <= ceiling; failed ||= !pass; rows.push( - `| ${provider === "codex" ? "Codex" : "Claude"} | ${metric.label} | ${baselineObserved === undefined ? "—" : formatValue(baselineObserved, metric.kind)} | ${formatValue(observed, metric.kind)} | ${formatImpact(observed, baselineObserved, metric.kind)} | ${formatValue(ceiling, metric.kind)} | ${pass ? "✅" : "❌"} |`, + `| ${providerLabel(provider)} | ${metric.label} | ${baselineObserved === undefined ? "—" : formatValue(baselineObserved, metric.kind)} | ${formatValue(observed, metric.kind)} | ${formatImpact(observed, baselineObserved, metric.kind)} | ${formatValue(ceiling, metric.kind)} | ${pass ? "✅" : "❌"} |`, ); if (comparable && baseline.providers[provider].ceiling[metric.key] !== ceiling) { ceilingChanges.push( - `- ${provider === "codex" ? "Codex" : "Claude"} ${metric.label}: ${formatValue(baseline.providers[provider].ceiling[metric.key], metric.kind)} → ${formatValue(ceiling, metric.kind)}`, + `- ${providerLabel(provider)} ${metric.label}: ${formatValue(baseline.providers[provider].ceiling[metric.key], metric.kind)} → ${formatValue(ceiling, metric.kind)}`, ); } } } + const scenarios = renderScenarios( + input.currentScenarios, + input.baselineScenarios, + ceilingChanges, + ); + failed ||= scenarios.failed; const baselineLink = input.baselineRun ? `[\`${input.baselineRun.sha.slice(0, 7)}\`](${input.baselineRun.url})` @@ -203,6 +335,7 @@ function renderComment(input) { "| --- | --- | ---: | ---: | ---: | ---: | --- |", ...rows, "", + ...scenarios.lines, `Baseline: ${baselineLink} · PR result: ${currentLink} · Source CI: ${input.currentRun.conclusion}`, "", "
", @@ -212,7 +345,7 @@ function renderComment(input) { "", ...PROVIDERS.map( (provider) => - `- ${provider === "codex" ? "Codex" : "Claude"} decoded thread snapshot: ${formatBytes(current.providers[provider].observed.threadSnapshotDecodedBytes)}`, + `- ${providerLabel(provider)} decoded thread snapshot: ${formatBytes(current.providers[provider].observed.threadSnapshotDecodedBytes)}`, ), "", "
", @@ -437,7 +570,14 @@ async function publish({ github, context, core }) { url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.BASELINE_RUN_ID}`, } : undefined; - const body = renderComment({ current, baseline, currentRun, baselineRun }); + const body = renderComment({ + current, + baseline, + currentScenarios: readScenarios(process.env.PR_RESULT_DIR), + baselineScenarios: readScenarios(process.env.BASELINE_RESULT_DIR), + currentRun, + baselineRun, + }); const published = await upsertCommentForCurrentHead( github, context, @@ -458,4 +598,5 @@ module.exports = { resolve, upsertCommentForCurrentHead, validateResult, + validateScenarios, }; diff --git a/.github/scripts/thread-transfer-report.test.cjs b/.github/scripts/thread-transfer-report.test.cjs index 8c4595ccf0c6a..3a953a8bfc16b 100644 --- a/.github/scripts/thread-transfer-report.test.cjs +++ b/.github/scripts/thread-transfer-report.test.cjs @@ -2,9 +2,11 @@ const assert = require("node:assert/strict"); const test = require("node:test"); const { + renderComment, resolve, upsertCommentForCurrentHead, validateResult, + validateScenarios, } = require("./thread-transfer-report.cjs"); function result(overrides = {}) { @@ -40,6 +42,29 @@ function result(overrides = {}) { }; } +function scenarios(overrides = {}) { + const rows = [ + { + phase: "long turn finished, cold open", + metric: "HTTP decoded", + unit: "bytes", + observed: 390_000, + ceiling: 420_000, + }, + { + phase: "idle thread resume", + metric: "WebSocket messages", + unit: "count", + observed: 1, + ceiling: 1, + }, + ]; + return { + schemaVersion: 1, + providers: { codex: [{ ...rows[0], ...overrides }, rows[1]], claudeAgent: rows }, + }; +} + test("validates the fixed artifact schema", () => { assert.equal(validateResult(result()).schemaVersion, 1); assert.throws( @@ -50,6 +75,60 @@ test("validates the fixed artifact schema", () => { () => validateResult(result({ totalWireBytes: "lots" })), /non-negative safe integer/, ); + + assert.equal(validateScenarios(scenarios()).schemaVersion, 1); + for (const phase of ["@everyone", "[link](https://example.com)", "a | b", "", ""]) { + assert.throws(() => validateScenarios(scenarios({ phase })), /short plain label/); + } + assert.throws(() => validateScenarios(scenarios({ note: "extra" })), /unexpected fields/); + assert.throws(() => validateScenarios(scenarios({ ceiling: null })), /non-negative safe integer/); + assert.throws( + () => + validateScenarios(scenarios({ phase: "idle thread resume", metric: "WebSocket messages" })), + /repeats an earlier row/, + ); +}); + +test("lists scenario rows that changed or failed above the collapsed table", () => { + const run = { sha: "abcdef1234", url: "https://example.com/run", conclusion: "success" }; + const render = (currentScenarios, baselineScenarios) => + renderComment({ + current: result(), + baseline: result(), + currentScenarios, + baselineScenarios, + currentRun: run, + baselineRun: { ...run, matchesBase: true }, + }); + const open = (body) => body.slice(0, body.indexOf("
")); + + const unchanged = render(scenarios(), scenarios()); + assert.match(unchanged, /✅ Thread transfer remains within every enforced ceiling/); + assert.match(open(unchanged), /All 4 scenario measurements match `main`/); + assert.doesNotMatch(open(unchanged), /long turn finished, cold open/); + + const grown = render(scenarios({ observed: 400_000 }), scenarios()); + assert.match(open(grown), /1 of 4 scenario measurements changed or failed/); + assert.match( + open(grown), + /\| Codex \| `long turn finished, cold open` \| `HTTP decoded` \| 380\.9 KiB \| 390\.6 KiB \| \+9\.8 KiB \(\+2\.6%\) \| 410\.2 KiB \| ✅ \|/, + ); + + const over = render(scenarios({ observed: 430_000, ceiling: 425_000 }), scenarios()); + assert.match(over, /❌ One or more thread transfer ceilings were exceeded/); + assert.match( + over, + /Codex `long turn finished, cold open` `HTTP decoded`: 410\.2 KiB → 415\.0 KiB/, + ); + + // A row `main` does not have is not reported as matching it. + const added = render(scenarios({ phase: "GH-26" }), scenarios()); + assert.match(open(added), /3 scenario measurements match `main`\. 1 have no `main` baseline\./); + assert.match(added, /\| Codex \| `GH-26` \|/); + + // A main run from before the scenario file has nothing to compare; a PR run from before it has no section. + assert.match(open(render(scenarios(), undefined)), /No `main` baseline exists yet/); + assert.doesNotMatch(render(undefined, scenarios()), /Other scenarios/); }); test("resolves a fallback PR with a redacted head repo and exact main baseline", async () => { diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f75c92562d03..d27b2c6a85729 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -503,6 +503,7 @@ jobs: env: T3CODE_TRANSFER_BUDGET_REPORT_PATH: ${{ runner.temp }}/t3code-transfer-budget.md T3CODE_TRANSFER_BUDGET_RESULT_PATH: ${{ runner.temp }}/thread-transfer-result.json + T3CODE_TRANSFER_BUDGET_SCENARIOS_PATH: ${{ runner.temp }}/thread-transfer-scenarios.json run: >- vp run test:cached -- --shard ${{ matrix.shard }}/${{ strategy.job-total }} --uncached apps/server/integration/transferBudgetV2.integration.test.ts @@ -543,7 +544,9 @@ jobs: uses: actions/upload-artifact@v7 with: name: thread-transfer-results - path: ${{ runner.temp }}/thread-transfer-result.json + path: | + ${{ runner.temp }}/thread-transfer-result.json + ${{ runner.temp }}/thread-transfer-scenarios.json if-no-files-found: ignore retention-days: 30 diff --git a/apps/server/integration/TransferBudgetReport.integration.ts b/apps/server/integration/TransferBudgetReport.integration.ts index 0b3cdb78589a0..005c15f8e1669 100644 --- a/apps/server/integration/TransferBudgetReport.integration.ts +++ b/apps/server/integration/TransferBudgetReport.integration.ts @@ -38,6 +38,11 @@ const STALE_TODO_RECORD_KINDS = [ ] as const; type StaleTodoRecordCounts = Readonly>; +/** One new socket, read from the subscribe call until the subscription caught up. */ +export interface SocketOpenMeasurement { + readonly totals: WebSocketTransferTotals; +} + export interface TransferBudgetRun { readonly provider: ProviderDriverKind; readonly threadSnapshot: HttpTransferMeasurement; @@ -49,15 +54,28 @@ export interface TransferBudgetRun { readonly reconnectShell: WebSocketCatchUpMeasurement; readonly measuredTurnSqlStatements: number; readonly reconnectSqlStatements: number; - /** The compact bounded snapshot web and mobile load when the long-turn thread opens. */ - readonly longTurnColdOpen: TimelineHttpMeasurement; - readonly longTurnHistoryPage: TimelineHttpMeasurement; + /** A thread resumed from the cursor it held before one ordinary turn. */ + readonly shortTurnResume: SocketOpenMeasurement; + /** The long-turn thread while its last turn runs: the bounded snapshot, then one older page. */ + readonly longTurnRunningColdOpen: TimelineHttpMeasurement; + readonly longTurnRunningHistoryPage: TimelineHttpMeasurement; /** The same reads as an older client that only sends `compactTurnItems`, to show the savings. */ - readonly longTurnColdOpenLegacy: TimelineHttpMeasurement; - readonly longTurnHistoryPageLegacy: TimelineHttpMeasurement; + readonly longTurnRunningColdOpenLegacy: TimelineHttpMeasurement; + readonly longTurnRunningHistoryPageLegacy: TimelineHttpMeasurement; + /** The same thread once the turn finished, over HTTP (web) and over the socket alone (mobile). */ + readonly longTurnColdOpen: TimelineHttpMeasurement; + readonly longTurnSocketColdOpen: SocketOpenMeasurement & { readonly rows: number }; + /** The first page of the finished turn's deferred work, loaded when its group is expanded. */ + readonly longTurnWorkPage: TimelineHttpMeasurement; + /** A thread resumed from the cursor of its own last event. */ + readonly idleThreadResume: SocketOpenMeasurement; /** The cold-connect shell snapshot after the settled-thread fleet is written. */ readonly fleetShell: TimelineHttpMeasurement; readonly fleetShellLegacy: TimelineHttpMeasurement; + /** The shell subscription a cold connect opens from that snapshot's cursor. */ + readonly fleetShellSubscribe: SocketOpenMeasurement; + /** A shell resumed from a cursor too old to replay, by a client that reloads over HTTP. */ + readonly staleShellResume: SocketOpenMeasurement; /** * The compact bounded snapshot of the long thread whose finished runs left todo lists open, * with the count of each record kind that rides along with its rows. @@ -89,14 +107,44 @@ const TRANSFER_BUDGETS: Readonly> = { claudeAgent: TRANSFER_BUDGET, }; -// Decoded bytes, not wire bytes: phones parse and keep the decoded JSON. About 7% above the -// observed 103 KB, so a cold open without body dedupe (117 KB), the ID table, or row budgets -// (all 920 rows) fails. Kept apart from TRANSFER_BUDGET because the PR comment validates its keys. -const LONG_TURN_BUDGET = { +// Decoded bytes, not wire bytes: phones parse and keep the decoded JSON. About 15% above the +// observed 95 KB history page, so a page without the ID table (130 KB) or row budgets fails. +// Kept apart from TRANSFER_BUDGET because the PR comment validates its keys. +const LONG_TURN_PAGE_BUDGET = { decodedBytes: 110_000, rows: 75, }; +// About 7% above the observed 290 KB. The window holds 75 rows, but every message of a running +// turn rides along with it: 196 KB here. Without body dedupe or the ID table it is 354 KB. +const LONG_TURN_RUNNING_BUDGET = { + decodedBytes: 310_000, + rows: 75, +}; + +// About 7% above the observed 393 KB. A finished turn defers its 770 work rows and stays whole: +// 151 rows of messages, checkpoints and subagents, on both transports. +const LONG_TURN_FINISHED_BUDGET = { + decodedBytes: 420_000, + rows: 151, +}; + +// About 6% above the observed 193 KB: one older page of work, twice the cold-open row budget. +const LONG_TURN_WORK_PAGE_BUDGET = { + decodedBytes: 205_000, + rows: 150, +}; + +const SOCKET_BUDGETS = { + // One ordinary turn replays in 11 KB. With its tool bodies it is over 900 KB. + shortTurnResume: { decodedBytes: 13_000, messages: 2 }, + longTurnColdOpen: { decodedBytes: LONG_TURN_FINISHED_BUDGET.decodedBytes, messages: 2 }, + // These three send no rows. A snapshot in their place is 393 KB for the thread, 42 KB for the shell. + idleThreadResume: { decodedBytes: 200, messages: 1 }, + fleetShellSubscribe: { decodedBytes: 200, messages: 1 }, + staleShellResume: { decodedBytes: 400, messages: 2 }, +}; + // About 30% above the observed 42 KB. Without the shell opt-ins it carries all 142 rows (217 KB). const FLEET_SHELL_BUDGET = { decodedBytes: 55_000, @@ -211,31 +259,139 @@ function webSocketRows( ]; } -function timelineRows( - provider: ProviderDriverKind, +interface ScenarioRow { + readonly phase: string; + readonly metric: string; + readonly unit: "bytes" | "count"; + readonly observed: number; + /** Absent on figures that only inform: wire bytes, and what an older client receives. */ + readonly maximum: number | undefined; +} + +function httpRows( phase: string, measured: TimelineHttpMeasurement, - budget: { readonly decodedBytes: number; readonly rows: number }, -): string[] { + budget?: { readonly decodedBytes: number; readonly rows: number }, +): ScenarioRow[] { return [ - infoRow(provider, phase, "HTTP wire", measured.http.wireBytes), - row(provider, phase, "HTTP decoded", measured.http.decodedBodyBytes, budget.decodedBytes), - row(provider, phase, "rows", measured.rows, budget.rows, String), + { + phase, + metric: "HTTP wire", + unit: "bytes", + observed: measured.http.wireBytes, + maximum: undefined, + }, + { + phase, + metric: "HTTP decoded", + unit: "bytes", + observed: measured.http.decodedBodyBytes, + maximum: budget?.decodedBytes, + }, + { phase, metric: "rows", unit: "count", observed: measured.rows, maximum: budget?.rows }, ]; } -function legacyRows( - provider: ProviderDriverKind, +function socketRows( phase: string, - measured: TimelineHttpMeasurement, -): string[] { + { totals }: SocketOpenMeasurement, + budget: { readonly decodedBytes: number; readonly messages: number }, +): ScenarioRow[] { + return [ + { + phase, + metric: "WebSocket wire", + unit: "bytes", + observed: totals.wireBytes, + maximum: undefined, + }, + { + phase, + metric: "WebSocket decoded", + unit: "bytes", + observed: totals.decodedBytes, + maximum: budget.decodedBytes, + }, + { + phase, + metric: "WebSocket messages", + unit: "count", + observed: totals.messages, + maximum: budget.messages, + }, + ]; +} + +/** + * Every scenario figure beyond the thread totals. The violations, the report table and the PR + * comment's scenario file all read this list, so a row added here reaches all three. The PR + * comment only accepts letters, digits, spaces, commas and hyphens in `phase` and `metric`. + */ +function scenarioRows(run: TransferBudgetRun): ScenarioRow[] { return [ - infoRow(provider, `${phase}, older client`, "HTTP wire", measured.http.wireBytes), - infoRow(provider, `${phase}, older client`, "HTTP decoded", measured.http.decodedBodyBytes), - infoRow(provider, `${phase}, older client`, "rows", measured.rows, String), + ...socketRows("short turn resume", run.shortTurnResume, SOCKET_BUDGETS.shortTurnResume), + ...httpRows( + "long turn running, cold open", + run.longTurnRunningColdOpen, + LONG_TURN_RUNNING_BUDGET, + ), + ...httpRows("long turn running, cold open, older client", run.longTurnRunningColdOpenLegacy), + ...httpRows( + "long turn running, history page", + run.longTurnRunningHistoryPage, + LONG_TURN_PAGE_BUDGET, + ), + ...httpRows( + "long turn running, history page, older client", + run.longTurnRunningHistoryPageLegacy, + ), + ...httpRows("long turn finished, cold open", run.longTurnColdOpen, LONG_TURN_FINISHED_BUDGET), + ...socketRows( + "long turn finished, socket cold open", + run.longTurnSocketColdOpen, + SOCKET_BUDGETS.longTurnColdOpen, + ), + { + phase: "long turn finished, socket cold open", + metric: "rows", + unit: "count", + observed: run.longTurnSocketColdOpen.rows, + maximum: LONG_TURN_FINISHED_BUDGET.rows, + }, + ...httpRows("long turn finished, work page", run.longTurnWorkPage, LONG_TURN_WORK_PAGE_BUDGET), + ...socketRows("idle thread resume", run.idleThreadResume, SOCKET_BUDGETS.idleThreadResume), + ...httpRows("fleet shell snapshot", run.fleetShell, FLEET_SHELL_BUDGET), + ...httpRows("fleet shell snapshot, older client", run.fleetShellLegacy), + ...socketRows( + "fleet shell subscribe", + run.fleetShellSubscribe, + SOCKET_BUDGETS.fleetShellSubscribe, + ), + ...socketRows("stale shell resume", run.staleShellResume, SOCKET_BUDGETS.staleShellResume), + ...httpRows("stale todo list cold open", run.staleTodoColdOpen, STALE_TODO_BUDGET), + ...STALE_TODO_RECORD_KINDS.map((kind): ScenarioRow => ({ + phase: "stale todo list cold open", + metric: kind, + unit: "count", + observed: run.staleTodoColdOpen.records[kind], + maximum: STALE_TODO_BUDGET.records[kind], + })), ]; } +/** The budgeted scenario figures, for the PR comment to compare with `main`. */ +export function formatTransferBudgetScenarios(runs: ReadonlyArray): string { + const providers = Object.fromEntries( + runs.map((run) => [ + run.provider, + scenarioRows(run).flatMap(({ maximum, ...row }) => + maximum === undefined ? [] : [{ ...row, ceiling: maximum }], + ), + ]), + ); + return `${JSON.stringify({ schemaVersion: 1, providers }, null, 2)}\n`; +} + export function transferBudgetViolations(runs: ReadonlyArray): string[] { const violations: string[] = []; for (const run of runs) { @@ -262,37 +418,8 @@ export function transferBudgetViolations(runs: ReadonlyArray) run.measuredTurnWebSocket.messages, budget.measuredTurnWebSocketMessages, ], - [ - "long-turn cold open decoded bytes", - run.longTurnColdOpen.http.decodedBodyBytes, - LONG_TURN_BUDGET.decodedBytes, - ], - ["long-turn cold open rows", run.longTurnColdOpen.rows, LONG_TURN_BUDGET.rows], - [ - "long-turn history page decoded bytes", - run.longTurnHistoryPage.http.decodedBodyBytes, - LONG_TURN_BUDGET.decodedBytes, - ], - ["long-turn history page rows", run.longTurnHistoryPage.rows, LONG_TURN_BUDGET.rows], - [ - "fleet shell snapshot decoded bytes", - run.fleetShell.http.decodedBodyBytes, - FLEET_SHELL_BUDGET.decodedBytes, - ], - ["fleet shell snapshot rows", run.fleetShell.rows, FLEET_SHELL_BUDGET.rows], - [ - "stale todo list cold open decoded bytes", - run.staleTodoColdOpen.http.decodedBodyBytes, - STALE_TODO_BUDGET.decodedBytes, - ], - ["stale todo list cold open rows", run.staleTodoColdOpen.rows, STALE_TODO_BUDGET.rows], - ...STALE_TODO_RECORD_KINDS.map( - (kind) => - [ - `stale todo list cold open ${kind}`, - run.staleTodoColdOpen.records[kind], - STALE_TODO_BUDGET.records[kind], - ] as const, + ...scenarioRows(run).flatMap(({ phase, metric, observed, maximum }) => + maximum === undefined ? [] : [[`${phase} ${metric}`, observed, maximum] as const], ), ] as const; for (const [metric, observed, maximum] of checks) { @@ -311,8 +438,9 @@ export function formatTransferBudgetReport(runs: ReadonlyArray - row( - run.provider, - "stale todo list cold open", - kind, - run.staleTodoColdOpen.records[kind], - STALE_TODO_BUDGET.records[kind], - String, - ), - ), + ...scenarioRows(run).map(({ phase, metric, unit, observed, maximum }) => { + const format = unit === "bytes" ? formatBytes : String; + return maximum === undefined + ? infoRow(run.provider, phase, metric, observed, format) + : row(run.provider, phase, metric, observed, maximum, format); + }), ); } diff --git a/apps/server/integration/TransferBudgetV2Fixture.integration.ts b/apps/server/integration/TransferBudgetV2Fixture.integration.ts index 9a857120c512f..a87788c7fdc75 100644 --- a/apps/server/integration/TransferBudgetV2Fixture.integration.ts +++ b/apps/server/integration/TransferBudgetV2Fixture.integration.ts @@ -217,10 +217,10 @@ export const LONG_TURN_THREAD_ID = ThreadId.make("transfer-budget-long-turn"); export const LONG_TURN_WORK_ROWS = 900; /** - * Three short user turns, then one agent turn far longer than a cold-open - * window, with the item kinds a long agent run produces. One batch per turn. - * Run, node, provider and item IDs follow the composite shapes the Claude - * adapter stores, and every assistant message has its record, as real threads do. + * Three short user turns, then one agent turn far longer than a cold-open window, with the item + * kinds a long agent run produces. One batch per turn; the last batch only finishes the long turn. + * Run, node, provider and item IDs follow the composite shapes the Claude adapter stores, and + * every assistant message has its record, as real threads do. */ export function longTurnThreadBatches( provider: ProviderDriverKind, @@ -434,12 +434,14 @@ export function longTurnThreadBatches( return events; }; + const longTurn = turn(3, LONG_TURN_WORK_ROWS); return [ [threadCreated(provider, threadId)], turn(0, 4), turn(1, 4), turn(2, 4), - turn(3, LONG_TURN_WORK_ROWS), + longTurn.slice(0, -1), + longTurn.slice(-1), ]; } diff --git a/apps/server/integration/transferBudgetV2.integration.test.ts b/apps/server/integration/transferBudgetV2.integration.test.ts index 313c583eecf7a..16ee48d6de0e1 100644 --- a/apps/server/integration/transferBudgetV2.integration.test.ts +++ b/apps/server/integration/transferBudgetV2.integration.test.ts @@ -23,8 +23,14 @@ import { OrchestrationV2GetThreadProjectionError, OrchestrationV2GetShellSnapshotError, ProviderDriverKind, + type EnvironmentOrchestrationShellSnapshotQuery, + type EnvironmentOrchestrationThreadBoundedSnapshotQuery, + type EnvironmentOrchestrationThreadHistoryQuery, + type OrchestrationV2SubscribeShellInput, + type OrchestrationV2SubscribeThreadInput, type OrchestrationV2ThreadStreamItem, type OrchestrationV2ShellStreamItem, + type ThreadId, } from "@t3tools/contracts"; import * as Context from "effect/Context"; import * as Config from "effect/Config"; @@ -63,6 +69,7 @@ import { makeSqlStatementCounter } from "./SqlStatementCounter.integration.ts"; import { formatTransferBudgetReport, formatTransferBudgetResult, + formatTransferBudgetScenarios, transferBudgetViolations, type TransferBudgetRun, } from "./TransferBudgetReport.integration.ts"; @@ -83,11 +90,41 @@ import { turnEvents, } from "./TransferBudgetV2Fixture.integration.ts"; -// The opt-ins current web and mobile clients send; keep in sync with client-runtime's -// boundedThreadSnapshotHttp.ts, threadHistoryHttp.ts, shellSnapshotHttp.ts, threads.ts and shell.ts. -const CLIENT_BOUNDED_QUERY = "compactTurnItems=1&dedupeBodies=1&compactIds=1"; -const CLIENT_HISTORY_QUERY = "compactIds=1"; -const CLIENT_SHELL_QUERY = "omitIdleSubagents=1&omitOlderSettled=1"; +// The opt-ins current web and mobile clients send, as client-runtime's boundedThreadSnapshotHttp.ts, +// threadHistoryHttp.ts, shellSnapshotHttp.ts, threads.ts and shell.ts send them. Each is `Required`, +// so an opt-in added to a contract fails typecheck here until this test sends it. +const CLIENT_BOUNDED_OPT_INS = { + compactTurnItems: "1", + dedupeBodies: "1", + compactIds: "1", + deferWork: "1", +} satisfies Required; +const CLIENT_HISTORY_OPT_INS = { compactIds: "1", deferWork: "1" } satisfies Required< + Omit +>; +const CLIENT_SHELL_OPT_INS = { + omitIdleSubagents: "1", + omitOlderSettled: "1", +} satisfies Required; +const CLIENT_THREAD_SUBSCRIPTION = { + requestCompletionMarker: true, + acceptBoundedSnapshot: true, + acceptDeferredWork: true, + acceptCompactTurnItems: true, + acceptCompactIds: true, + acceptDedupedBodies: true, +} satisfies Required>; +const CLIENT_SHELL_SUBSCRIPTION = { + requestCompletionMarker: true, + omitIdleSubagents: true, + omitOlderSettled: true, +} satisfies Required< + Omit +>; +const query = (optIns: Readonly>) => new URLSearchParams(optIns).toString(); +const CLIENT_BOUNDED_QUERY = query(CLIENT_BOUNDED_OPT_INS); +const CLIENT_HISTORY_QUERY = query(CLIENT_HISTORY_OPT_INS); +const CLIENT_SHELL_QUERY = query(CLIENT_SHELL_OPT_INS); const decodeThreadSnapshot = Schema.decodeUnknownEffect( Schema.fromJsonString(Schema.toCodecJson(OrchestrationV2ThreadDetailSnapshot)), @@ -224,30 +261,36 @@ const collectUntil = Effect.fn("transfer.collectUntil")(function* ) + | ({ readonly shell: true } & Pick< + OrchestrationV2SubscribeShellInput, + "afterSequence" | "httpSnapshotFallback" + >); +function subscriptionStream( + client: MeasuredWsClient, + subscription: Subscription, +): Stream.Stream { + if ("thread" in subscription) { + const { thread, ...resume } = subscription; + return client.client[ORCHESTRATION_V2_WS_METHODS.subscribeThread]({ + ...CLIENT_THREAD_SUBSCRIPTION, + ...resume, + threadId: thread, + }); + } + const { shell: _, ...resume } = subscription; + return client.client[ORCHESTRATION_V2_WS_METHODS.subscribeShell]({ + ...CLIENT_SHELL_SUBSCRIPTION, + ...resume, + }); +} const subscribe = Effect.fn("transfer.subscribe")(function* ( client: MeasuredWsClient, - kind: "thread" | "shell", - afterSequence: number, + subscription: Subscription, ) { const queue = yield* Queue.unbounded(); - const stream: Stream.Stream = - kind === "thread" - ? client.client[ORCHESTRATION_V2_WS_METHODS.subscribeThread]({ - threadId: THREAD_ID, - afterSequence, - requestCompletionMarker: true, - acceptBoundedSnapshot: true, - acceptCompactTurnItems: true, - acceptCompactIds: true, - acceptDedupedBodies: true, - }) - : client.client[ORCHESTRATION_V2_WS_METHODS.subscribeShell]({ - afterSequence, - requestCompletionMarker: true, - omitIdleSubagents: true, - omitOlderSettled: true, - }); - yield* stream.pipe( + yield* subscriptionStream(client, subscription).pipe( Stream.runForEach((item) => Queue.offer(queue, item)), Scope.provide(client.scope), Effect.forkIn(client.scope), @@ -324,10 +367,11 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () => const shell = yield* openMeasuredWsClient({ url, cookie: "" }); const second = yield* openMeasuredWsClient({ url, cookie: "" }); assert.include(thread.recorder.negotiatedExtensions(), "permessage-deflate"); - const tq = yield* subscribe(thread, "thread", sequence); - const sq = yield* subscribe(shell, "shell", sequence); - const ttq = yield* subscribe(second, "thread", sequence); - const ssq = yield* subscribe(second, "shell", sequence); + const fromSnapshot = { afterSequence: sequence }; + const tq = yield* subscribe(thread, { thread: THREAD_ID, ...fromSnapshot }); + const sq = yield* subscribe(shell, { shell: true, ...fromSnapshot }); + const ttq = yield* subscribe(second, { thread: THREAD_ID, ...fromSnapshot }); + const ssq = yield* subscribe(second, { shell: true, ...fromSnapshot }); for (const queue of [tq, sq, ttq, ssq]) assert.equal(yield* synchronized(queue), "replay"); const threadBefore = thread.recorder.totals(); @@ -376,8 +420,8 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () => const reconnectSqlStart = counter.count(); const resumedThread = yield* openMeasuredWsClient({ url, cookie: "" }); const resumedShell = yield* openMeasuredWsClient({ url, cookie: "" }); - const rq = yield* subscribe(resumedThread, "thread", sequence); - const rsq = yield* subscribe(resumedShell, "shell", sequence); + const rq = yield* subscribe(resumedThread, { thread: THREAD_ID, ...fromSnapshot }); + const rsq = yield* subscribe(resumedShell, { shell: true, ...fromSnapshot }); const threadMode = yield* synchronized(rq); const shellMode = yield* synchronized(rsq); const reconnectThread = { ...resumedThread.recorder.totals(), mode: threadMode }; @@ -385,7 +429,6 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () => const reconnectSqlStatements = counter.count() - reconnectSqlStart; assert.equal(threadMode, "snapshot"); - for (const events of longTurnThreadBatches(provider)) yield* sink.write({ events }); const get = (path: string) => measureHttpGet({ url: `${base}${path}`, @@ -398,8 +441,9 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () => Effect.gen(function* () { const http = yield* get(`${longTurnPath}/bounded?${query}`); const snapshot = yield* decodeBoundedSnapshot(decoded(http)); - const rows = snapshot.projection.visibleTurnItems.length; - return { http, rows, cursor: snapshot.historyCursor }; + const rows = snapshot.projection.visibleTurnItems; + const workAnchor = rows.find((row) => row.item.deferredWork !== undefined); + return { http, rows: rows.length, cursor: snapshot.historyCursor, workAnchor }; }); const readHistoryPage = (cursor: string, query: string) => Effect.gen(function* () { @@ -413,15 +457,75 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () => const readShell = (query: string) => Effect.gen(function* () { const http = yield* get(`/api/orchestration/shell?${query}`); - return { http, rows: (yield* decodeShellSnapshot(decoded(http))).threads.length }; + const { threads, snapshotSequence } = yield* decodeShellSnapshot(decoded(http)); + return { http, rows: threads.length, snapshotSequence }; + }); + // A new socket, read until the subscription caught up or the server ended it. + const open = (subscription: Subscription) => + Effect.gen(function* () { + const socket = yield* openMeasuredWsClient({ url, cookie: "" }); + const items = yield* subscriptionStream(socket, subscription).pipe( + Stream.takeUntil((item) => item.kind === "synchronized"), + Stream.runCollect, + Scope.provide(socket.scope), + Effect.orDie, + ); + const totals = socket.recorder.totals(); + yield* socket.close; + return { kinds: items.map((item) => item.kind), items, totals }; }); - const { cursor: _, ...longTurnColdOpenLegacy } = - yield* readColdOpen("compactTurnItems=1"); - const { cursor, ...longTurnColdOpen } = yield* readColdOpen(CLIENT_BOUNDED_QUERY); + // A phone that comes back after one ordinary turn replays it, without tool bodies. + const shortTurn = yield* sink.write({ + events: turnEvents(provider, TRANSFER_HISTORY_TURN_COUNT + 1, false), + }); + const shortTurnResume = yield* open({ thread: THREAD_ID, afterSequence: last.sequence }); + assert.notInclude(shortTurnResume.kinds, "snapshot"); + // The byte cap below also passes when the resume skips the turn. + assert.isTrue( + shortTurnResume.items.some( + (item) => "sequence" in item && item.sequence === shortTurn.at(-1)!.sequence, + ), + ); + + // A turn that is still running cannot defer its work, so it loads in row-budget windows. + const [finishLongTurn, ...longTurnHistory] = longTurnThreadBatches(provider).toReversed(); + for (const events of longTurnHistory.toReversed()) yield* sink.write({ events }); + const longTurnRunningColdOpenLegacy = yield* readColdOpen("compactTurnItems=1"); + const longTurnRunningColdOpen = yield* readColdOpen(CLIENT_BOUNDED_QUERY); + const { cursor } = longTurnRunningColdOpen; if (cursor === null) return yield* Effect.die("Expected older long-turn history"); - const longTurnHistoryPageLegacy = yield* readHistoryPage(cursor, ""); - const longTurnHistoryPage = yield* readHistoryPage(cursor, CLIENT_HISTORY_QUERY); + const longTurnRunningHistoryPageLegacy = yield* readHistoryPage(cursor, ""); + const longTurnRunningHistoryPage = yield* readHistoryPage(cursor, CLIENT_HISTORY_QUERY); + + // Once it finishes, its work stays on the server until a group is expanded. + yield* sink.write({ events: finishLongTurn! }); + const longTurnColdOpen = yield* readColdOpen(CLIENT_BOUNDED_QUERY); + const { workAnchor } = longTurnColdOpen; + if (workAnchor?.item.deferredWork === undefined) + return yield* Effect.die("Expected the finished long turn to defer its work"); + const longTurnWorkPage = yield* readHistoryPage( + workAnchor.item.deferredWork.cursor, + `view=work&throughEntryId=${encodeURIComponent(workAnchor.sourceItemId)}&compactIds=1`, + ); + // Mobile opens a thread over the socket alone. + const socketOpen = yield* open({ thread: LONG_TURN_THREAD_ID }); + const socketSnapshot = socketOpen.items.find( + (item) => item.kind === "snapshot" && "projection" in item, + ); + if (socketSnapshot?.kind !== "snapshot" || !("projection" in socketSnapshot)) + return yield* Effect.die("Expected a bounded socket snapshot"); + const longTurnSocketColdOpen = { + ...socketOpen, + rows: socketSnapshot.projection.visibleTurnItems.length, + }; + // The row cap below also passes when the socket ignores the deferral and sends a window. + assert.equal(longTurnSocketColdOpen.rows, longTurnColdOpen.rows); + const idleThreadResume = yield* open({ + thread: LONG_TURN_THREAD_ID, + afterSequence: socketSnapshot.snapshotSequence, + }); + assert.deepEqual(idleThreadResume.kinds, ["synchronized"]); // The transfer thread plus the long-turn thread. const earlierThreads = decodedShell.threads.length + 1; @@ -432,6 +536,19 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () => fleetShellLegacy.rows, earlierThreads + SHELL_SETTLED_THREADS + SHELL_IDLE_SUBAGENTS, ); + // A cold connect subscribes from the snapshot it just loaded. + const fleetShellSubscribe = yield* open({ + shell: true, + afterSequence: fleetShell.snapshotSequence, + }); + assert.notInclude(fleetShellSubscribe.kinds, "snapshot"); + // A cursor too old to replay is told to reload over HTTP, not sent a socket snapshot. + const staleShellResume = yield* open({ + shell: true, + afterSequence: sequence, + httpSnapshotFallback: true, + }); + assert.deepEqual(staleShellResume.kinds, ["snapshot-required"]); for (const events of staleTodoListThreadBatches(provider)) yield* sink.write({ events }); const staleTodoHttp = yield* get( @@ -472,12 +589,19 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () => reconnectThread, reconnectShell, reconnectSqlStatements, - longTurnColdOpenLegacy, + shortTurnResume, + longTurnRunningColdOpenLegacy, + longTurnRunningColdOpen, + longTurnRunningHistoryPageLegacy, + longTurnRunningHistoryPage, longTurnColdOpen, - longTurnHistoryPageLegacy, - longTurnHistoryPage, + longTurnWorkPage, + longTurnSocketColdOpen, + idleThreadResume, fleetShellLegacy, fleetShell, + fleetShellSubscribe, + staleShellResume, staleTodoColdOpen, } satisfies TransferBudgetRun; }).pipe( @@ -492,10 +616,14 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () => const outputPaths = yield* Config.all({ report: Config.String("T3CODE_TRANSFER_BUDGET_REPORT_PATH").pipe(Config.withDefault("")), result: Config.String("T3CODE_TRANSFER_BUDGET_RESULT_PATH").pipe(Config.withDefault("")), + scenarios: Config.String("T3CODE_TRANSFER_BUDGET_SCENARIOS_PATH").pipe( + Config.withDefault(""), + ), }); for (const [path, contents] of [ [outputPaths.report, report], [outputPaths.result, formatTransferBudgetResult(runs)], + [outputPaths.scenarios, formatTransferBudgetScenarios(runs)], ]) if (path && contents) yield* fs.writeFileString(path, contents); yield* Effect.log(report); diff --git a/docs/internals/performance-regressions.md b/docs/internals/performance-regressions.md index a7d5be74fd175..95719e4901087 100644 --- a/docs/internals/performance-regressions.md +++ b/docs/internals/performance-regressions.md @@ -19,7 +19,8 @@ The v2 checks pin these invariants: large work groups until expansion. Messages and items outside those groups remain in normal history paging. Required control records and the at-least-one-row rule can exceed the byte target; it is not a hard cap. -- The bounded snapshot does not retain the full duplicate conversation-message table. +- The bounded snapshot does not retain the full duplicate conversation-message table. Messages of + a run that is still in progress are the exception: all of them accompany the window. - Older activity is fetched in bounded HTTP pages and merged without disturbing the live scroll window. - Clients with confirmed history paging opt into socket snapshot fallbacks that use the same @@ -68,13 +69,22 @@ These are neither compressed WebSocket captures nor measurements of server RSS, fixtures exclude unrelated settings calls and subscription events. Payload budgets belong in these tests so regressions fail locally. The transfer budget integration test (`apps/server/integration/transferBudgetV2.integration.test.ts`) sends the opt-ins current clients -send and caps the decoded bytes and rows of the bounded snapshot and one older page for a 900-row -agent turn, and of the cold-connect shell for a sidebar of settled threads and idle subagents. It -also caps the bytes and the node, subagent, plan, provider turn and checkpoint counts of the -bounded snapshot for a thread of 40 finished runs behind one working run, so finished runs cannot -ride along with a window. The working run has a ready context handoff, so its history cannot ride -along either. When clients gain an opt-in, add it to the test's client queries and give a fixture -the data it saves. +send. Its opt-in objects are typed `Required` against the contracts, so an opt-in added to a +contract fails typecheck until the test sends it; give a fixture the data the opt-in saves. The +test caps decoded bytes and rows for a 900-row agent turn in both states: while the turn runs (the +bounded snapshot and one older page) and after it finishes (the deferred cold open over HTTP and +over the socket alone, and the first page of its deferred work). It caps the socket bytes of three +resumes that must not send a snapshot: a thread after one ordinary turn, an idle thread, and a +shell cursor too old to replay. It caps the cold-connect shell for a sidebar of settled threads and +idle subagents, and the subscription a client opens from that snapshot. It also caps the bytes and +the node, subagent, plan, provider turn and checkpoint counts of the bounded snapshot for a thread +of 40 finished runs behind one working run, so finished runs cannot ride along with a window. The +working run has a ready context handoff, so its history cannot ride along either. + +CI prints every row in the job summary. The pull request comment compares the budgeted rows with +`main`, lists the rows that changed or failed, and names each ceiling a pull request raises. A row +added to `scenarioRows` in `TransferBudgetReport.integration.ts` reaches the pass/fail check, the +summary and the comment. The server config has its own budget in `apps/server/src/ws.test.ts`. A client that asks for catalog digests and status deltas gets the provider list of its first config in under 1 KiB, for a diff --git a/packages/contracts/src/environmentHttp.ts b/packages/contracts/src/environmentHttp.ts index 4bbae77fb922d..b9a2cacf6e7ce 100644 --- a/packages/contracts/src/environmentHttp.ts +++ b/packages/contracts/src/environmentHttp.ts @@ -682,19 +682,19 @@ const EnvironmentOrchestrationThreadSnapshotParams = Schema.Struct({ threadId: ThreadId, }); -const EnvironmentOrchestrationShellSnapshotQuery = Schema.Struct({ +export const EnvironmentOrchestrationShellSnapshotQuery = Schema.Struct({ omitIdleSubagents: Schema.optionalKey(Schema.String), omitOlderSettled: Schema.optionalKey(Schema.String), }); -const EnvironmentOrchestrationThreadBoundedSnapshotQuery = Schema.Struct({ +export const EnvironmentOrchestrationThreadBoundedSnapshotQuery = Schema.Struct({ deferWork: Schema.optionalKey(Schema.String), compactTurnItems: Schema.optionalKey(Schema.String), dedupeBodies: Schema.optionalKey(Schema.String), compactIds: Schema.optionalKey(Schema.String), }); -const EnvironmentOrchestrationThreadHistoryQuery = Schema.Struct({ +export const EnvironmentOrchestrationThreadHistoryQuery = Schema.Struct({ cursor: TrimmedNonEmptyString, throughEntryId: Schema.optional(TrimmedNonEmptyString), view: Schema.optional(Schema.Literals(["conversation", "activity", "work"])), diff --git a/packages/shared/src/testing/providerEnv.ts b/packages/shared/src/testing/providerEnv.ts index 21bf59e52b500..4961b70141b84 100644 --- a/packages/shared/src/testing/providerEnv.ts +++ b/packages/shared/src/testing/providerEnv.ts @@ -8,6 +8,7 @@ const KEPT_ENV = new Set([ // CI sets these so the transfer budget integration test writes its report for the job summary. "T3CODE_TRANSFER_BUDGET_REPORT_PATH", "T3CODE_TRANSFER_BUDGET_RESULT_PATH", + "T3CODE_TRANSFER_BUDGET_SCENARIOS_PATH", // Opt-in: points the HtmlRender tests at a headless browser, which they skip without. "T3CODE_TEST_HEADLESS_SHELL", ]);