diff --git a/.agents/skills/test-t3-mobile/SKILL.md b/.agents/skills/test-t3-mobile/SKILL.md index 306038007e9a..a11eee4e3f4e 100644 --- a/.agents/skills/test-t3-mobile/SKILL.md +++ b/.agents/skills/test-t3-mobile/SKILL.md @@ -43,10 +43,15 @@ prints one JSON line; on failure it names the next command. Mobile verification includes that step unless the user prohibits it. For a device you opened in an agent-device session elsewhere (a device pool, -cloud, or remote host), run `vp run mobile:install --session ` -from the directory you opened the session in. It installs the cached build into -that session (`--abi arm64-v8a` for an ARM Android device; `--metro ` also -opens it against Metro) and never builds. +cloud, or remote host), run `vp run mobile:install --session +--config ` from the directory you opened the session in, passing the +`--config` file from `device_open`'s `targetArgs`. Without it, a device pool's +gateway refuses the install, wrongly saying agent-device is too old. It installs +the cached build into that session (`--abi arm64-v8a` for an ARM Android device; +pool Android emulators are x86_64, the default; `--metro ` also opens it +against Metro). Without a cached build for this checkout it builds first, which can +outlast the lease, so run `vp run mobile:install --prepare` (with +the same `--abi`) before you open the device. Start `vp run dev:client` from `apps/mobile`, or reuse a healthy Metro belonging to this checkout. Open its printed development-client URL with AgentDevice diff --git a/.github/VOUCHED.td b/.github/VOUCHED.td index 69c1ea6575a4..315ff5742fd5 100644 --- a/.github/VOUCHED.td +++ b/.github/VOUCHED.td @@ -22,6 +22,7 @@ github:btsouth github:cestercian github:chrisdeeming github:chuks-qua +-github:coygeek Filed over 270 automated issues, about 50 a day at peak, far past what maintainers can triage github:cursoragent github:D3OXY github:dbalders diff --git a/.github/actions/setup-ci-deps/action.yml b/.github/actions/setup-ci-deps/action.yml index e27a8c90fdff..cc6dc80511ac 100644 --- a/.github/actions/setup-ci-deps/action.yml +++ b/.github/actions/setup-ci-deps/action.yml @@ -35,7 +35,7 @@ runs: - name: Cache Vite+ id: vp_cache if: steps.vp.outputs.version != '' - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.vite-plus key: vite-plus-home-v2-${{ runner.os }}-${{ runner.arch }}-${{ steps.vp.outputs.version }}-node${{ steps.vp.outputs.node }}-${{ steps.vp.outputs.pnpm }} @@ -56,7 +56,7 @@ runs: # partial bytes, so a shorter limit costs a slow download only a retry. - name: Setup Vite+ if: steps.vp_cache.outputs.cache-hit != 'true' - uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1 + uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1.15.0 env: VP_DOWNLOAD_TIMEOUT: "120" with: @@ -92,7 +92,7 @@ runs: - name: Restore pnpm store id: pnpm_cache if: inputs.install == 'true' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.pnpm_store.outputs.path }} key: pnpm-${{ steps.pnpm_store.outputs.namespace }}-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('pnpm-lock.yaml', 'pnpm-workspace.yaml', 'package.json') }} @@ -106,7 +106,7 @@ runs: - name: Restore lockfile verification id: pnpm_verified if: inputs.install == 'true' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.pnpm_store.outputs.verified }} key: pnpm-verified-v1-${{ hashFiles('pnpm-lock.yaml', 'pnpm-workspace.yaml', 'package.json') }} @@ -121,7 +121,7 @@ runs: - name: Save lockfile verification if: inputs.install == 'true' && steps.pnpm_verified.outputs.cache-hit != 'true' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.pnpm_store.outputs.verified }} key: ${{ steps.pnpm_verified.outputs.cache-primary-key }} @@ -138,7 +138,7 @@ runs: steps.pnpm_cache.outputs.cache-hit != 'true' && (steps.pnpm_store.outputs.namespace == 'full' || steps.pnpm_cache.outputs.cache-matched-key != format('pnpm-full-v1-{0}-{1}-{2}', runner.os, runner.arch, hashFiles('pnpm-lock.yaml', 'pnpm-workspace.yaml', 'package.json'))) - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.pnpm_store.outputs.path }} key: ${{ steps.pnpm_cache.outputs.cache-primary-key }} diff --git a/.github/readme/hero.webp b/.github/readme/hero.webp new file mode 100644 index 000000000000..8cc713bf541d Binary files /dev/null and b/.github/readme/hero.webp differ diff --git a/.github/scripts/check-nightly-release.cjs b/.github/scripts/check-nightly-release.cjs index 95581e2b2278..a9502b26c8c8 100644 --- a/.github/scripts/check-nightly-release.cjs +++ b/.github/scripts/check-nightly-release.cjs @@ -121,9 +121,54 @@ async function resolveLatestNightlyCommit({ github, context, core }) { return { tag, sha: commit.sha, version }; } +// The two places a nightly install looks for an update: the `update-feed` pointer, and +// releases.atom, which holds the 10 newest git tags of any kind. Older installs take the first +// nightly in the feed. Reports what each names beside the newest published nightly. +async function readNightlyUpdateFeed({ github, context, fetch = globalThis.fetch }) { + const latest = await findLatestNightly({ github, context }); + if (!latest) return {}; + + const base = `https://github.com/${context.repo.owner}/${context.repo.repo}`; + // Only a missing or malformed pointer is stale. Rewriting deletes the old file first, so a + // GitHub outage must not look like one. + // Node waits five minutes for a stalled response, which is this job's whole time limit. + const signal = () => AbortSignal.timeout(30_000); + const pointer = await fetch(`${base}/releases/download/update-feed/nightly.json`, { + signal: signal(), + }); + if (!pointer.ok && pointer.status !== 404) { + throw new Error(`The update-feed pointer answered HTTP ${pointer.status}.`); + } + let pointerTag; + if (pointer.ok) { + // A body that fails to arrive throws here; only one that is not JSON counts as malformed. + const body = await pointer.text(); + try { + pointerTag = JSON.parse(body)?.tag_name; + } catch {} + } + + // An unreadable feed leaves `feedTags` unset, so the pointer is still repaired. An error page + // can answer HTTP 200, so only an Atom document counts as read. + let feedTags; + try { + const feed = await fetch(`${base}/releases.atom`, { signal: signal() }); + const body = feed.ok ? await feed.text() : ""; + if (/]/.test(body)) { + feedTags = Array.from(body.matchAll(/href="[^"]*\/releases\/tag\/([^"]+)"/g), (match) => + decodeURIComponent(match[1]), + ); + } + } catch {} + const feedNightly = feedTags?.find((tag) => /^v?\d+\.\d+\.\d+-nightly\./.test(tag)); + + return { tag: latest.tagName, pointerTag, feedNightly, feedTags }; +} + module.exports = { assertCommitOnDefaultBranch, assertReleaseSource, shouldReleaseNightly, resolveLatestNightlyCommit, + readNightlyUpdateFeed, }; diff --git a/.github/scripts/check-nightly-release.test.cjs b/.github/scripts/check-nightly-release.test.cjs index 3030080d946c..2f3d385755ca 100644 --- a/.github/scripts/check-nightly-release.test.cjs +++ b/.github/scripts/check-nightly-release.test.cjs @@ -237,3 +237,81 @@ test("stable releases fail when no nightly is published", async () => { const { options } = nightlyCommitFixture({ releases: [nightly(1, { tagName: "v1.0.0" })] }); await assert.rejects(resolveLatestNightlyCommit(options), /No published nightly found/); }); + +const { readNightlyUpdateFeed } = require("./check-nightly-release.cjs"); + +// GitHub percent-encodes tag names in the feed, and any tag in the repository takes a slot. +const atomEntry = (tag) => + ``; +const atomFeed = (tags) => new Response(`${tags.map(atomEntry).join("")}`); +const NEWEST = "v1.0.1-nightly.20260905.123"; +const OLDER = "v1.0.1-nightly.20260905.100"; + +function updateFeedFixture({ feed, pointer }) { + const { options } = fixture({ + releases: [nightly(1), nightly(0, { tagName: "v1.0.2-preview.1" })], + }); + options.fetch = async (url) => + url === "https://github.com/example/app/releases.atom" + ? feed + : url === "https://github.com/example/app/releases/download/update-feed/nightly.json" + ? pointer + : assert.fail(`unexpected fetch ${url}`); + return options; +} + +const missing = () => new Response("Not Found", { status: 404 }); +// HTTP 200, then the connection drops before the body arrives. +const brokenBody = () => + new Response(new ReadableStream({ start: (controller) => controller.error(new Error("reset")) })); +for (const [name, { feed, pointer }, expected] of [ + [ + "a nightly crowded out of the feed and a missing pointer", + { feed: atomFeed(["v1.0.2-preview.1", "mobile-android-build/2.3.0/abc"]), pointer: missing() }, + { pointerTag: undefined, feedNightly: undefined }, + ], + [ + "the first nightly in the feed, which is the one older installs take", + { feed: atomFeed(["update-feed", OLDER, NEWEST]), pointer: Response.json({ tag_name: OLDER }) }, + { pointerTag: OLDER, feedNightly: OLDER }, + ], + [ + "a malformed pointer as stale and an unreadable feed as naming nothing", + { feed: new Response("", { status: 503 }), pointer: Response.json(null) }, + { pointerTag: undefined, feedNightly: undefined, unreadable: true }, + ], + [ + "an HTML page with release links as an unreadable feed", + { + feed: new Response( + `x`, + ), + pointer: Response.json({ tag_name: NEWEST }), + }, + { pointerTag: NEWEST, feedNightly: undefined, unreadable: true }, + ], + [ + "the pointer when the feed's body fails to arrive", + { feed: brokenBody(), pointer: Response.json({ tag_name: NEWEST }) }, + { pointerTag: NEWEST, feedNightly: undefined, unreadable: true }, + ], +]) { + test(`update feed check reports ${name}`, async () => { + const { tag, pointerTag, feedNightly, feedTags } = await readNightlyUpdateFeed( + updateFeedFixture({ feed, pointer }), + ); + const { unreadable = false, ...named } = expected; + assert.deepEqual({ tag, pointerTag, feedNightly }, { tag: NEWEST, ...named }); + assert.equal(feedTags === undefined, unreadable); + }); +} + +for (const [name, pointer, error] of [ + ["answers with an error", () => new Response("", { status: 503 }), /pointer answered HTTP 503/], + ["drops the connection", brokenBody, /reset/], +]) { + test(`update feed check does not call a pointer stale when GitHub ${name}`, async () => { + const options = updateFeedFixture({ feed: atomFeed([NEWEST]), pointer: pointer() }); + await assert.rejects(readNightlyUpdateFeed(options), error); + }); +} diff --git a/.github/scripts/pr-limits.cjs b/.github/scripts/pr-limits.cjs index 77c09785e410..f3eb528a0804 100644 --- a/.github/scripts/pr-limits.cjs +++ b/.github/scripts/pr-limits.cjs @@ -100,14 +100,22 @@ function parseExemptions(text) { } // VOUCHED.td: one `github:login` per line, `#` comments, and `-github:login reason` to denounce. -function vouchedLogins(text) { - const logins = new Set(); +// A login listed both ways is an error: pr-vouch.yml's action reads its first line and +// issue-vouch.yml its last, so no answer here would agree with both labels. +function parseVouchList(text) { + const vouched = new Set(); + const denounced = new Set(); for (const raw of text.split("\n")) { - const entry = raw.trim().split(/\s+/)[0]; - if (entry.startsWith("github:")) logins.add(entry.slice("github:".length).toLowerCase()); + const match = /^(-?)github:(\S+)/.exec(raw.trim()); + if (!match) continue; + const login = match[2].toLowerCase(); + const [listed, other] = match[1] ? [denounced, vouched] : [vouched, denounced]; + if (other.has(login)) + throw new Error(`${VOUCHED_PATH}: ${login} is both vouched and denounced`); + listed.add(login); } - if (logins.size === 0) throw new Error(`${VOUCHED_PATH}: no entries`); - return logins; + if (vouched.size === 0) throw new Error(`${VOUCHED_PATH}: no entries`); + return { vouched, denounced }; } function labeledAt(pr, name) { @@ -278,7 +286,8 @@ async function enforceOpenLimit({ github, context, core, dryRun }) { // lookups as "not vouched". Every read below throws on failure, and a throw closes nothing. if (pr.user.type === "Bot") return; if (parseExemptions(fs.readFileSync(EXEMPTIONS_PATH, "utf8")).has(author.toLowerCase())) return; - if (vouchedLogins(fs.readFileSync(VOUCHED_PATH, "utf8")).has(author.toLowerCase())) return; + const { vouched, denounced } = parseVouchList(fs.readFileSync(VOUCHED_PATH, "utf8")); + if (vouched.has(author.toLowerCase())) return; const { data: access } = await github.rest.repos.getCollaboratorPermissionLevel({ ...context.repo, username: author, @@ -289,6 +298,21 @@ async function enforceOpenLimit({ github, context, core, dryRun }) { const reopened = context.payload.action === "reopened"; if (reopened && context.payload.sender.login.toLowerCase() !== author.toLowerCase()) return; + if (denounced.has(author.toLowerCase())) { + if (dryRun) { + core.notice(`Dry run. PR #${pr.number} would close: ${author} is denounced.`); + return; + } + await closeWithComment( + github, + context, + pr.number, + `Closing: ${author} is banned from contributing here, so this PR will not be reviewed. See [bans](${guideLink(context, "bans")}).`, + ); + core.info(`PR #${pr.number}: closed, ${author} is denounced.`); + return; + } + const q = `repo:${context.repo.owner}/${context.repo.repo} is:pr is:open author:${author} sort:created-asc`; const { search } = await read(github, AUTHOR_OPEN_PRS_QUERY, { q }); // A new PR counts only the older ones, so two opened together cannot close each other. diff --git a/.github/scripts/thread-transfer-report.cjs b/.github/scripts/thread-transfer-report.cjs index 09b25131e4c4..304b56d57e40 100644 --- a/.github/scripts/thread-transfer-report.cjs +++ b/.github/scripts/thread-transfer-report.cjs @@ -3,6 +3,7 @@ const path = require("node:path"); const ARTIFACT_NAME = "thread-transfer-results"; const RESULT_FILE = "thread-transfer-result.json"; +const SCENARIOS_FILE = "thread-transfer-scenarios.json"; const COMMENT_MARKER = ""; const PROVIDERS = ["codex", "claudeAgent"]; const OBSERVED_KEYS = [ @@ -29,6 +30,12 @@ const SCENARIO_KEYS = [ "measuredMcpResultBytes", ]; +const SCENARIO_ROW_KEYS = ["phase", "metric", "unit", "observed", "ceiling"]; +// These labels come from the PR and reach a comment the bot posts: no markup, links or mentions. +// GitHub still links plain text such as `GH-26`, so they are rendered as code spans. +const SCENARIO_LABEL = /^[A-Za-z0-9][A-Za-z0-9 ,-]{0,79}$/; +const MAX_SCENARIO_ROWS = 100; + function resultShaMarker(sha) { return ``; } @@ -85,15 +92,63 @@ function validateResult(value) { return value; } -function readResult(directory) { +function validateScenarios(value) { + assertExactKeys(value, ["schemaVersion", "providers"], "scenarios"); + if (value.schemaVersion !== 1) { + throw new Error("scenarios.schemaVersion must be 1"); + } + assertExactKeys(value.providers, PROVIDERS, "scenarios.providers"); + for (const provider of PROVIDERS) { + const rows = value.providers[provider]; + if (!Array.isArray(rows) || rows.length > MAX_SCENARIO_ROWS) { + throw new Error( + `scenarios.providers.${provider} must list at most ${MAX_SCENARIO_ROWS} rows`, + ); + } + const seen = new Set(); + for (const [index, row] of rows.entries()) { + const label = `scenarios.providers.${provider}[${index}]`; + assertExactKeys(row, SCENARIO_ROW_KEYS, label); + for (const key of ["phase", "metric"]) { + if (typeof row[key] !== "string" || !SCENARIO_LABEL.test(row[key])) { + throw new Error(`${label}.${key} must be a short plain label`); + } + } + if (row.unit !== "bytes" && row.unit !== "count") { + throw new Error(`${label}.unit must be bytes or count`); + } + assertMetric(row.observed, `${label}.observed`); + assertMetric(row.ceiling, `${label}.ceiling`); + const key = scenarioKey(row); + if (seen.has(key)) throw new Error(`${label} repeats an earlier row`); + seen.add(key); + } + } + return value; +} + +function scenarioKey(row) { + return `${row.phase}|${row.metric}`; +} + +function readArtifactFile(directory, name, validate) { if (!directory) return undefined; - const file = path.join(directory, RESULT_FILE); + const file = path.join(directory, name); if (!fs.existsSync(file)) return undefined; const stat = fs.lstatSync(file); if (!stat.isFile() || stat.size > 64 * 1_024) { - throw new Error("thread transfer result must be a regular file smaller than 64 KiB"); + throw new Error(`${name} must be a regular file smaller than 64 KiB`); } - return validateResult(JSON.parse(fs.readFileSync(file, "utf8"))); + return validate(JSON.parse(fs.readFileSync(file, "utf8"))); +} + +function readResult(directory) { + return readArtifactFile(directory, RESULT_FILE, validateResult); +} + +// Older runs upload only the result file, so a run without scenarios is still a valid run. +function readScenarios(directory) { + return readArtifactFile(directory, SCENARIOS_FILE, validateScenarios); } function formatBytes(bytes) { @@ -103,7 +158,7 @@ function formatBytes(bytes) { } function formatValue(value, kind) { - return kind === "messages" ? value.toLocaleString("en-US") : formatBytes(value); + return kind === "bytes" ? formatBytes(value) : value.toLocaleString("en-US"); } function formatImpact(current, baseline, kind) { @@ -136,6 +191,77 @@ const METRICS = [ { key: "measuredTurnWebSocketMessages", label: "Live turn messages", kind: "messages" }, ]; +function providerLabel(provider) { + return provider === "codex" ? "Codex" : "Claude"; +} + +/** + * The scenario rows of a run against the same rows on `main`. Rows that fail or differ from + * `main` are listed in the open; the full table is collapsed below them. + */ +function renderScenarios(current, baseline, ceilingChanges) { + if (!current) return { lines: [], failed: false }; + const header = [ + "| Provider | Scenario | Metric | Main baseline | This PR | Impact | PR ceiling | |", + "| --- | --- | --- | ---: | ---: | ---: | ---: | --- |", + ]; + const all = []; + const notable = []; + let failed = false; + let withoutBaseline = 0; + + for (const provider of PROVIDERS) { + const baselineRows = new Map( + (baseline?.providers[provider] ?? []).map((row) => [scenarioKey(row), row]), + ); + for (const row of current.providers[provider]) { + const before = baselineRows.get(scenarioKey(row)); + const beforeObserved = before?.unit === row.unit ? before.observed : undefined; + const pass = row.observed <= row.ceiling; + failed ||= !pass; + if (beforeObserved === undefined) withoutBaseline += 1; + const line = `| ${providerLabel(provider)} | \`${row.phase}\` | \`${row.metric}\` | ${beforeObserved === undefined ? "—" : formatValue(beforeObserved, row.unit)} | ${formatValue(row.observed, row.unit)} | ${formatImpact(row.observed, beforeObserved, row.unit)} | ${formatValue(row.ceiling, row.unit)} | ${pass ? "✅" : "❌"} |`; + all.push(line); + if (!pass || (beforeObserved !== undefined && beforeObserved !== row.observed)) { + notable.push(line); + } + if (beforeObserved !== undefined && before.ceiling !== row.ceiling) { + ceilingChanges.push( + `- ${providerLabel(provider)} \`${row.phase}\` \`${row.metric}\`: ${formatValue(before.ceiling, row.unit)} → ${formatValue(row.ceiling, row.unit)}`, + ); + } + } + } + + const compared = all.length - withoutBaseline; + const summary = !baseline + ? `No \`main\` baseline exists yet for these ${all.length} scenario measurements.` + : [ + notable.length > 0 + ? `${notable.length} of ${all.length} scenario measurements changed or failed.` + : `${compared === all.length ? "All " : ""}${compared} scenario measurements match \`main\`.`, + ...(withoutBaseline > 0 ? [`${withoutBaseline} have no \`main\` baseline.`] : []), + ].join(" "); + return { + failed, + lines: [ + "### Other scenarios", + "", + summary, + ...(notable.length > 0 ? ["", ...header, ...notable] : []), + "", + "
", + `All ${all.length} scenario measurements`, + "", + ...header, + ...all, + "", + "
", + "", + ], + }; +} + function renderComment(input) { const current = input.current; const baseline = input.baseline; @@ -154,16 +280,22 @@ function renderComment(input) { const pass = observed <= ceiling; failed ||= !pass; rows.push( - `| ${provider === "codex" ? "Codex" : "Claude"} | ${metric.label} | ${baselineObserved === undefined ? "—" : formatValue(baselineObserved, metric.kind)} | ${formatValue(observed, metric.kind)} | ${formatImpact(observed, baselineObserved, metric.kind)} | ${formatValue(ceiling, metric.kind)} | ${pass ? "✅" : "❌"} |`, + `| ${providerLabel(provider)} | ${metric.label} | ${baselineObserved === undefined ? "—" : formatValue(baselineObserved, metric.kind)} | ${formatValue(observed, metric.kind)} | ${formatImpact(observed, baselineObserved, metric.kind)} | ${formatValue(ceiling, metric.kind)} | ${pass ? "✅" : "❌"} |`, ); if (comparable && baseline.providers[provider].ceiling[metric.key] !== ceiling) { ceilingChanges.push( - `- ${provider === "codex" ? "Codex" : "Claude"} ${metric.label}: ${formatValue(baseline.providers[provider].ceiling[metric.key], metric.kind)} → ${formatValue(ceiling, metric.kind)}`, + `- ${providerLabel(provider)} ${metric.label}: ${formatValue(baseline.providers[provider].ceiling[metric.key], metric.kind)} → ${formatValue(ceiling, metric.kind)}`, ); } } } + const scenarios = renderScenarios( + input.currentScenarios, + input.baselineScenarios, + ceilingChanges, + ); + failed ||= scenarios.failed; const baselineLink = input.baselineRun ? `[\`${input.baselineRun.sha.slice(0, 7)}\`](${input.baselineRun.url})` @@ -203,6 +335,7 @@ function renderComment(input) { "| --- | --- | ---: | ---: | ---: | ---: | --- |", ...rows, "", + ...scenarios.lines, `Baseline: ${baselineLink} · PR result: ${currentLink} · Source CI: ${input.currentRun.conclusion}`, "", "
", @@ -212,7 +345,7 @@ function renderComment(input) { "", ...PROVIDERS.map( (provider) => - `- ${provider === "codex" ? "Codex" : "Claude"} decoded thread snapshot: ${formatBytes(current.providers[provider].observed.threadSnapshotDecodedBytes)}`, + `- ${providerLabel(provider)} decoded thread snapshot: ${formatBytes(current.providers[provider].observed.threadSnapshotDecodedBytes)}`, ), "", "
", @@ -437,7 +570,14 @@ async function publish({ github, context, core }) { url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.BASELINE_RUN_ID}`, } : undefined; - const body = renderComment({ current, baseline, currentRun, baselineRun }); + const body = renderComment({ + current, + baseline, + currentScenarios: readScenarios(process.env.PR_RESULT_DIR), + baselineScenarios: readScenarios(process.env.BASELINE_RESULT_DIR), + currentRun, + baselineRun, + }); const published = await upsertCommentForCurrentHead( github, context, @@ -458,4 +598,5 @@ module.exports = { resolve, upsertCommentForCurrentHead, validateResult, + validateScenarios, }; diff --git a/.github/scripts/thread-transfer-report.test.cjs b/.github/scripts/thread-transfer-report.test.cjs index 8c4595ccf0c6..3a953a8bfc16 100644 --- a/.github/scripts/thread-transfer-report.test.cjs +++ b/.github/scripts/thread-transfer-report.test.cjs @@ -2,9 +2,11 @@ const assert = require("node:assert/strict"); const test = require("node:test"); const { + renderComment, resolve, upsertCommentForCurrentHead, validateResult, + validateScenarios, } = require("./thread-transfer-report.cjs"); function result(overrides = {}) { @@ -40,6 +42,29 @@ function result(overrides = {}) { }; } +function scenarios(overrides = {}) { + const rows = [ + { + phase: "long turn finished, cold open", + metric: "HTTP decoded", + unit: "bytes", + observed: 390_000, + ceiling: 420_000, + }, + { + phase: "idle thread resume", + metric: "WebSocket messages", + unit: "count", + observed: 1, + ceiling: 1, + }, + ]; + return { + schemaVersion: 1, + providers: { codex: [{ ...rows[0], ...overrides }, rows[1]], claudeAgent: rows }, + }; +} + test("validates the fixed artifact schema", () => { assert.equal(validateResult(result()).schemaVersion, 1); assert.throws( @@ -50,6 +75,60 @@ test("validates the fixed artifact schema", () => { () => validateResult(result({ totalWireBytes: "lots" })), /non-negative safe integer/, ); + + assert.equal(validateScenarios(scenarios()).schemaVersion, 1); + for (const phase of ["@everyone", "[link](https://example.com)", "a | b", "", ""]) { + assert.throws(() => validateScenarios(scenarios({ phase })), /short plain label/); + } + assert.throws(() => validateScenarios(scenarios({ note: "extra" })), /unexpected fields/); + assert.throws(() => validateScenarios(scenarios({ ceiling: null })), /non-negative safe integer/); + assert.throws( + () => + validateScenarios(scenarios({ phase: "idle thread resume", metric: "WebSocket messages" })), + /repeats an earlier row/, + ); +}); + +test("lists scenario rows that changed or failed above the collapsed table", () => { + const run = { sha: "abcdef1234", url: "https://example.com/run", conclusion: "success" }; + const render = (currentScenarios, baselineScenarios) => + renderComment({ + current: result(), + baseline: result(), + currentScenarios, + baselineScenarios, + currentRun: run, + baselineRun: { ...run, matchesBase: true }, + }); + const open = (body) => body.slice(0, body.indexOf("
")); + + const unchanged = render(scenarios(), scenarios()); + assert.match(unchanged, /✅ Thread transfer remains within every enforced ceiling/); + assert.match(open(unchanged), /All 4 scenario measurements match `main`/); + assert.doesNotMatch(open(unchanged), /long turn finished, cold open/); + + const grown = render(scenarios({ observed: 400_000 }), scenarios()); + assert.match(open(grown), /1 of 4 scenario measurements changed or failed/); + assert.match( + open(grown), + /\| Codex \| `long turn finished, cold open` \| `HTTP decoded` \| 380\.9 KiB \| 390\.6 KiB \| \+9\.8 KiB \(\+2\.6%\) \| 410\.2 KiB \| ✅ \|/, + ); + + const over = render(scenarios({ observed: 430_000, ceiling: 425_000 }), scenarios()); + assert.match(over, /❌ One or more thread transfer ceilings were exceeded/); + assert.match( + over, + /Codex `long turn finished, cold open` `HTTP decoded`: 410\.2 KiB → 415\.0 KiB/, + ); + + // A row `main` does not have is not reported as matching it. + const added = render(scenarios({ phase: "GH-26" }), scenarios()); + assert.match(open(added), /3 scenario measurements match `main`\. 1 have no `main` baseline\./); + assert.match(added, /\| Codex \| `GH-26` \|/); + + // A main run from before the scenario file has nothing to compare; a PR run from before it has no section. + assert.match(open(render(scenarios(), undefined)), /No `main` baseline exists yet/); + assert.doesNotMatch(render(undefined, scenarios()), /Other scenarios/); }); test("resolves a fallback PR with a redacted head repo and exact main baseline", async () => { diff --git a/.github/workflows/agent-context-budget.yml b/.github/workflows/agent-context-budget.yml index 41efb7b69a5a..5935fb4653c3 100644 --- a/.github/workflows/agent-context-budget.yml +++ b/.github/workflows/agent-context-budget.yml @@ -10,10 +10,7 @@ on: permissions: contents: read -concurrency: - group: agent-context-budget-${{ github.event.pull_request.number }} - cancel-in-progress: true - +# No concurrency group: label events arrive in bursts, and a cancelled run reports as a failed check. jobs: budget: name: Agent context budget @@ -24,7 +21,7 @@ jobs: ALLOW_INCREASE: ${{ contains(github.event.pull_request.labels.*.name, 'agent-context-increase') }} steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: sparse-checkout: apps/server/src/provider/agentContextBudget.json sparse-checkout-cone-mode: false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f9db201d88f..f6950bccc00a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: # Only the package manifests the script reads to build the dependency graph. # Depth 2 brings the base commit the change is diffed against. - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: fetch-depth: 2 sparse-checkout: | @@ -69,7 +69,7 @@ jobs: timeout-minutes: 30 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: sparse-checkout: | /* @@ -93,7 +93,7 @@ jobs: # electron's install.js (run by ensure:electron) reads this cache before it # downloads from GitHub. The key changes only with the Electron version. - name: Cache Electron download - uses: actions/cache@v6 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cache/electron key: electron-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('node_modules/.pnpm/electron@*/node_modules/electron/package.json') }} @@ -125,7 +125,7 @@ jobs: timeout-minutes: 30 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: sparse-checkout: | /* @@ -169,7 +169,7 @@ jobs: # electron's install.js (run by ensure:electron) reads this cache before it # downloads from GitHub. The key changes only with the Electron version. - name: Cache Electron download - uses: actions/cache@v6 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cache/electron key: electron-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('node_modules/.pnpm/electron@*/node_modules/electron/package.json') }} @@ -221,7 +221,7 @@ jobs: - name: Upload desktop launch check if: ${{ failure() && steps.desktop_launch.outcome == 'failure' }} - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: desktop-launch-check path: ${{ runner.temp }}/desktop-launch-check @@ -239,7 +239,7 @@ jobs: - name: Cache Chromium id: chromium_cache if: ${{ !cancelled() && steps.build.outcome == 'success' && needs.changes.outputs.test_web != 'false' }} - uses: actions/cache@v6 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cache/ms-playwright key: playwright-chromium-shell-${{ runner.os }}-${{ runner.arch }}-${{ steps.playwright_version.outputs.version }} @@ -287,7 +287,7 @@ jobs: timeout-minutes: 30 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: sparse-checkout: | /* @@ -322,7 +322,7 @@ jobs: # electron's install.js (run by ensure:electron) reads this cache before it # downloads from GitHub. The key changes only with the Electron version. - name: Cache Electron download - uses: actions/cache@v6 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cache/electron key: electron-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('node_modules/.pnpm/electron@*/node_modules/electron/package.json') }} @@ -385,7 +385,7 @@ jobs: shard: [1, 2] steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: sparse-checkout: | /* @@ -406,7 +406,7 @@ jobs: - name: Cache Chromium id: chromium_cache - uses: actions/cache@v6 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cache/ms-playwright key: playwright-chromium-shell-${{ runner.os }}-${{ runner.arch }}-${{ steps.playwright_version.outputs.version }} @@ -451,7 +451,7 @@ jobs: steps: # Depth 2 brings the base commit the change is diffed against. - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: fetch-depth: 2 sparse-checkout: | @@ -482,7 +482,7 @@ jobs: - name: Cache Chromium id: chromium_cache if: steps.changes.outputs.test_server != 'false' - uses: actions/cache@v6 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cache/ms-playwright key: playwright-chromium-shell-${{ runner.os }}-${{ runner.arch }}-${{ steps.playwright_version.outputs.version }} @@ -505,7 +505,7 @@ jobs: # Only pushes to main save it, so pull requests never evict each other's entries. - name: Restore Vitest module cache if: steps.changes.outputs.test_server != 'false' - uses: actions/cache/restore@v6 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: node_modules/.vitest-cache key: vitest-modules-server-${{ matrix.shard }}-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}-${{ github.sha }} @@ -516,14 +516,16 @@ jobs: env: T3CODE_TRANSFER_BUDGET_REPORT_PATH: ${{ runner.temp }}/t3code-transfer-budget.md T3CODE_TRANSFER_BUDGET_RESULT_PATH: ${{ runner.temp }}/thread-transfer-result.json + T3CODE_TRANSFER_BUDGET_SCENARIOS_PATH: ${{ runner.temp }}/thread-transfer-scenarios.json + # One process per file spends ~2s starting before its tests run; one file per vCPU overlaps that. run: >- - vp run test:cached -- --shard ${{ matrix.shard }}/${{ strategy.job-total }} + vp run test:cached -- --shard ${{ matrix.shard }}/${{ strategy.job-total }} --concurrency 4 --uncached apps/server/integration/transferBudgetV2.integration.test.ts --verify-hits 1 apps/server - name: Save Vitest module cache if: github.event_name == 'push' && steps.changes.outputs.test_server != 'false' - uses: actions/cache/save@v6 + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: node_modules/.vitest-cache key: vitest-modules-server-${{ matrix.shard }}-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}-${{ github.sha }} @@ -553,10 +555,12 @@ jobs: - name: Upload thread transfer result if: always() && steps.transfer_budget.outputs.present == 'true' - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: thread-transfer-results - path: ${{ runner.temp }}/thread-transfer-result.json + path: | + ${{ runner.temp }}/thread-transfer-result.json + ${{ runner.temp }}/thread-transfer-scenarios.json if-no-files-found: ignore retention-days: 30 @@ -570,7 +574,7 @@ jobs: timeout-minutes: 30 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: sparse-checkout: | /* @@ -579,8 +583,9 @@ jobs: sparse-checkout-cone-mode: false - name: Setup Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: + toolchain: stable components: rustfmt - name: Check Rust formatting @@ -603,7 +608,7 @@ jobs: timeout-minutes: 30 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: sparse-checkout: | /* @@ -611,7 +616,7 @@ jobs: sparse-checkout-cone-mode: false - name: Setup Vite+ - uses: voidzero-dev/setup-vp@v1 + uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1.15.0 with: node-version-file: package.json cache: true @@ -636,7 +641,7 @@ jobs: timeout-minutes: 30 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: sparse-checkout: | /* @@ -694,7 +699,7 @@ jobs: # server tests ran must have produced it. - name: Require transfer measurements if: needs.test_server.result == 'success' && needs.changes.outputs.test_server != 'false' - uses: actions/download-artifact@v8 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: thread-transfer-results path: ${{ runner.temp }}/thread-transfer diff --git a/.github/workflows/deploy-relay.yml b/.github/workflows/deploy-relay.yml index 5db7a86eb98c..da779c2783d4 100644 --- a/.github/workflows/deploy-relay.yml +++ b/.github/workflows/deploy-relay.yml @@ -51,7 +51,7 @@ jobs: ALCHEMY_TELEMETRY_DISABLED: "1" steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: sparse-checkout: | /* @@ -59,7 +59,7 @@ jobs: sparse-checkout-cone-mode: false - name: Setup Vite+ - uses: voidzero-dev/setup-vp@v1 + uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1.15.0 with: node-version-file: package.json cache: true @@ -83,7 +83,7 @@ jobs: T3CHAT_LINK_SECRET: ${{ secrets.T3CHAT_LINK_SECRET }} - name: Publish relay deploy commit status - uses: actions/github-script@v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: script: | await github.rest.repos.createCommitStatus({ diff --git a/.github/workflows/desktop-macos-preview-publish.yml b/.github/workflows/desktop-macos-preview-publish.yml index 6e8805b7f790..88f96079b897 100644 --- a/.github/workflows/desktop-macos-preview-publish.yml +++ b/.github/workflows/desktop-macos-preview-publish.yml @@ -55,7 +55,7 @@ jobs: steps: - id: pr name: Resolve the pull request behind the build - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: script: | const run = context.payload.workflow_run; @@ -134,7 +134,7 @@ jobs: - id: vouch name: Check PR author trust if: steps.pr.outputs.eligible == 'true' - uses: mitchellh/vouch/action/check-user@d66fa29a64600490892131ad87597c30c91fcac4 # v1 + uses: mitchellh/vouch/action/check-user@d66fa29a64600490892131ad87597c30c91fcac4 # v1.5.0 with: user: ${{ steps.pr.outputs.author }} allow-fail: true @@ -149,7 +149,7 @@ jobs: - id: consume name: Consume the preview label if: steps.pr.outputs.eligible == 'true' - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: PR_NUMBER: ${{ steps.pr.outputs.pr_number }} with: @@ -271,7 +271,7 @@ jobs: # Only validated bundle files cross into the signing job's artifact. - name: Stage JS bundle for packaging if: steps.gate.outputs.eligible == 'true' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: js-bundle path: ${{ runner.temp }}/js-bundle @@ -316,6 +316,7 @@ jobs: rust_target: aarch64-apple-darwin resource_key: darwin-arm64 cli_archive: false + untrusted_bundle: true # Release assets download without a GitHub account, unlike workflow # artifacts. All preview DMGs live on one rolling prerelease tagged @@ -338,7 +339,7 @@ jobs: pull-requests: write steps: - name: Download macOS artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: desktop-mac-arm64 path: release @@ -431,7 +432,7 @@ jobs: - name: Comment download link if: steps.upload.outputs.download_url != '' - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: PR_NUMBER: ${{ needs.resolve.outputs.pr_number }} DOWNLOAD_URL: ${{ steps.upload.outputs.download_url }} @@ -552,7 +553,7 @@ jobs: - name: Mark the preview comment as removed if: steps.delete.outputs.removed == 'true' - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: script: | const marker = ""; diff --git a/.github/workflows/desktop-macos-preview.yml b/.github/workflows/desktop-macos-preview.yml index 0213f3fce6a9..2c54052ea66d 100644 --- a/.github/workflows/desktop-macos-preview.yml +++ b/.github/workflows/desktop-macos-preview.yml @@ -50,7 +50,7 @@ jobs: git checkout --detach FETCH_HEAD - name: Setup Vite+ - uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1 + uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1.15.0 with: node-version-file: package.json cache: true @@ -98,7 +98,7 @@ jobs: # Same layout as release.yml's js-bundle so release-desktop.yml can # package it unchanged. - name: Upload JS bundle - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: js-bundle path: | diff --git a/.github/workflows/issue-labels.yml b/.github/workflows/issue-labels.yml index d6571d65d453..fd9ba6b60302 100644 --- a/.github/workflows/issue-labels.yml +++ b/.github/workflows/issue-labels.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Ensure managed issue labels exist - uses: actions/github-script@v7 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 with: script: | const managedLabels = [ diff --git a/.github/workflows/issue-vouch.yml b/.github/workflows/issue-vouch.yml new file mode 100644 index 000000000000..d65882593c64 --- /dev/null +++ b/.github/workflows/issue-vouch.yml @@ -0,0 +1,110 @@ +name: Issue Vouch + +on: + issues: + types: [opened, reopened] + push: + branches: + - main + paths: + - .github/VOUCHED.td + - .github/workflows/issue-vouch.yml + workflow_dispatch: + +permissions: + contents: read + issues: write + +concurrency: + group: issue-vouch-${{ github.event.issue.number || 'all' }} + cancel-in-progress: true + +jobs: + label: + name: Label issues by author trust + runs-on: ubuntu-24.04 + steps: + - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + with: + script: | + const { owner, repo } = context.repo; + // Only listed authors get a label; open issues number in the thousands, + // so unvouched is the default and stays unlabelled. + const labelByStatus = { vouched: "vouch:trusted", denounced: "vouch:denounced" }; + const managedLabelNames = new Set(["vouch:trusted", "vouch:unvouched", "vouch:denounced"]); + + const { data: file } = await github.rest.repos.getContent({ + owner, + repo, + path: ".github/VOUCHED.td", + ref: context.payload.repository.default_branch, + }); + const statusByUser = new Map(); + const listedBothWays = new Set(); + for (const raw of Buffer.from(file.content, "base64").toString("utf8").split("\n")) { + const match = /^(-?)github:(\S+)/.exec(raw.trim()); + if (!match) continue; + const login = match[2].toLowerCase(); + const status = match[1] ? "denounced" : "vouched"; + if (statusByUser.has(login) && statusByUser.get(login) !== status) listedBothWays.add(login); + statusByUser.set(login, status); + } + + const issues = + context.eventName === "issues" + ? [context.payload.issue] + : ( + await github.paginate(github.rest.issues.listForRepo, { + owner, + repo, + state: "open", + per_page: 100, + }) + ).filter((issue) => !issue.pull_request); + + for (const issue of issues) { + const status = statusByUser.get(issue.user.login.toLowerCase()); + const next = labelByStatus[status]; + const current = issue.labels.map((label) => label.name).filter((name) => managedLabelNames.has(name)); + + for (const name of current) { + if (name === next) continue; + try { + await github.rest.issues.removeLabel({ owner, repo, issue_number: issue.number, name }); + } catch (error) { + if (error.status !== 404) throw error; + } + } + if (next && !current.includes(next)) { + await github.rest.issues.addLabels({ owner, repo, issue_number: issue.number, labels: [next] }); + core.info(`Issue #${issue.number} (${issue.user.login}): ${status} -> ${next}`); + } + + // Only a newly opened issue is closed. One filed before its author was denounced + // may already have a fix on the way, so the sweep leaves those to maintainers. + // Someone else reopening it is a maintainer keeping it open. + const reopenedByOther = + context.payload.action === "reopened" && + context.payload.sender.login.toLowerCase() !== issue.user.login.toLowerCase(); + // The PR rules refuse a list that names a login both ways, so nothing closes for one here. + const trusted = + issue.user.type === "Bot" || + ["OWNER", "MEMBER", "COLLABORATOR"].includes(issue.author_association) || + listedBothWays.has(issue.user.login.toLowerCase()); + if (status === "denounced" && context.eventName === "issues" && !reopenedByOther && !trusted) { + await github.rest.issues.createComment({ + owner, + repo, + issue_number: issue.number, + body: `Closing: ${issue.user.login} is banned from contributing here, so this issue will not be triaged. See [bans](https://github.com/${owner}/${repo}/blob/main/CONTRIBUTING.md#bans).`, + }); + await github.rest.issues.update({ + owner, + repo, + issue_number: issue.number, + state: "closed", + state_reason: "not_planned", + }); + core.info(`Issue #${issue.number}: closed, ${issue.user.login} is denounced.`); + } + } diff --git a/.github/workflows/mobile-android-apk.yml b/.github/workflows/mobile-android-apk.yml index 18d977c50f4a..8650b5a6d527 100644 --- a/.github/workflows/mobile-android-apk.yml +++ b/.github/workflows/mobile-android-apk.yml @@ -37,7 +37,7 @@ jobs: - name: Checkout if: steps.credentials.outputs.present == 'true' - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: sparse-checkout: .github/scripts diff --git a/.github/workflows/mobile-dev-client.yml b/.github/workflows/mobile-dev-client.yml index 5f8cb86befb5..b3e11c1ed6b6 100644 --- a/.github/workflows/mobile-dev-client.yml +++ b/.github/workflows/mobile-dev-client.yml @@ -83,7 +83,7 @@ jobs: T3_NATIVE_CLIENT_VERBOSE: "1" steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ inputs.ref || github.sha }} @@ -144,7 +144,7 @@ jobs: # the runs holding the cache login restore. - name: Mount build cache if: github.event_name != 'pull_request' - uses: useblacksmith/stickydisk@v1 + uses: useblacksmith/stickydisk@3f6be1451e3cad893b778bc93495c80452a5da26 # v1.7.2 with: key: ${{ github.repository }}-android-dev-client-${{ matrix.abi }} path: ~/.cache/t3-android-dev-client @@ -165,7 +165,7 @@ jobs: install-args: --filter=@t3tools/mobile... --filter=@t3tools/scripts... - name: Setup Java - uses: actions/setup-java@v5 + uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0 with: distribution: temurin java-version: 17 @@ -219,7 +219,7 @@ jobs: ios: name: iOS simulator if: github.repository == 'pingdotgg/t3code' && (github.event_name != 'workflow_dispatch' || inputs.platform == 'ios') - runs-on: blacksmith-12vcpu-macos-26 + runs-on: blacksmith-12vcpu-macos-27 timeout-minutes: 60 concurrency: group: mobile-dev-client-ios-${{ github.event_name == 'pull_request' && github.event.pull_request.number || inputs.fingerprint || github.sha }} @@ -232,7 +232,7 @@ jobs: T3_NATIVE_CLIENT_VERBOSE: "1" steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ inputs.ref || github.sha }} diff --git a/.github/workflows/mobile-eas-preview-fork.yml b/.github/workflows/mobile-eas-preview-fork.yml index a8bf5725af70..13ca814defd7 100644 --- a/.github/workflows/mobile-eas-preview-fork.yml +++ b/.github/workflows/mobile-eas-preview-fork.yml @@ -40,7 +40,7 @@ jobs: # If a maintainer removed the label first, that removal wins. - id: consume name: Consume the deployment label - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: script: | try { @@ -75,7 +75,7 @@ jobs: # the PR's scripts to read. - name: Checkout reviewed commit if: steps.expo-token.outputs.present == 'true' - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 @@ -84,7 +84,7 @@ jobs: # The comment script comes from main, not from the PR. - name: Checkout trusted scripts if: steps.expo-token.outputs.present == 'true' - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: path: .trusted sparse-checkout: .github/scripts @@ -120,7 +120,7 @@ jobs: # trusted workflows with other secrets would restore what PR code wrote. - name: Setup Vite+ if: steps.expo-token.outputs.present == 'true' - uses: voidzero-dev/setup-vp@v1 + uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1.15.0 with: node-version-file: package.json cache: false @@ -138,7 +138,7 @@ jobs: - name: Setup EAS if: steps.expo-token.outputs.present == 'true' - uses: expo/expo-github-action@v8 + uses: expo/expo-github-action@c7b66a9c327a43a8fa7c0158e7f30d6040d2481e # v8.2.1 with: # Pinned and installed with npm for the same reasons as mobile-eas-preview.yml. eas-version: 21.8.0 @@ -166,7 +166,7 @@ jobs: - name: Deploy with fingerprint check if: steps.expo-token.outputs.present == 'true' - uses: expo/expo-github-action/continuous-deploy-fingerprint@main + uses: expo/expo-github-action/continuous-deploy-fingerprint@5273c2136bd09c2a341fe3340d3e767ce5a13015 # v9.0.2 env: EXPO_TOKEN: ${{ secrets.EXPO_PREVIEW_TOKEN }} RUNNER_DEBUG: "1" @@ -249,14 +249,14 @@ jobs: ORG_GRADLE_PROJECT_reactNativeArchitectures: armeabi-v7a,arm64-v8a steps: - name: Checkout reviewed commit - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 # The comment script comes from main, not from the PR. - name: Checkout trusted scripts - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: path: .trusted sparse-checkout: .github/scripts @@ -288,7 +288,7 @@ jobs: fi - name: Setup Node - uses: actions/setup-node@v6 + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 with: node-version-file: package.json @@ -302,13 +302,13 @@ jobs: run: pnpm install --frozen-lockfile --filter=@t3tools/mobile... - name: Setup Java - uses: actions/setup-java@v5 + uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0 with: distribution: temurin java-version: 17 - name: Setup EAS - uses: expo/expo-github-action@v8 + uses: expo/expo-github-action@c7b66a9c327a43a8fa7c0158e7f30d6040d2481e # v8.2.1 with: eas-version: latest eas-cache: false diff --git a/.github/workflows/mobile-eas-preview.yml b/.github/workflows/mobile-eas-preview.yml index 5989d7a0c0cc..b46e090434f5 100644 --- a/.github/workflows/mobile-eas-preview.yml +++ b/.github/workflows/mobile-eas-preview.yml @@ -38,7 +38,7 @@ jobs: - name: Checkout if: steps.expo-token.outputs.present == 'true' - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: fetch-depth: 0 # No sparse-checkout here: it makes actions/checkout fetch with @@ -48,7 +48,7 @@ jobs: - name: Setup Vite+ if: steps.expo-token.outputs.present == 'true' - uses: voidzero-dev/setup-vp@v1 + uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1.15.0 with: node-version-file: package.json cache: true @@ -66,7 +66,7 @@ jobs: - name: Setup EAS if: steps.expo-token.outputs.present == 'true' - uses: expo/expo-github-action@v8 + uses: expo/expo-github-action@c7b66a9c327a43a8fa7c0158e7f30d6040d2481e # v8.2.1 with: # eas-cli 22 renamed `project` to `app` in build JSON, and # continuous-deploy-fingerprint still reads `build.project.ownerAccount`. @@ -99,7 +99,7 @@ jobs: - name: Deploy with fingerprint check if: steps.expo-token.outputs.present == 'true' - uses: expo/expo-github-action/continuous-deploy-fingerprint@main + uses: expo/expo-github-action/continuous-deploy-fingerprint@5273c2136bd09c2a341fe3340d3e767ce5a13015 # v9.0.2 env: EXPO_TOKEN: ${{ secrets.EXPO_PREVIEW_TOKEN }} # The Expo action otherwise suppresses the EAS command's failure output. @@ -154,7 +154,7 @@ jobs: - name: Checkout if: steps.expo-token.outputs.present == 'true' - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: fetch-depth: 0 @@ -187,7 +187,7 @@ jobs: - name: Setup Vite+ if: steps.expo-token.outputs.present == 'true' - uses: voidzero-dev/setup-vp@v1 + uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1.15.0 with: node-version-file: package.json cache: true @@ -205,7 +205,7 @@ jobs: - name: Setup EAS if: steps.expo-token.outputs.present == 'true' - uses: expo/expo-github-action@v8 + uses: expo/expo-github-action@c7b66a9c327a43a8fa7c0158e7f30d6040d2481e # v8.2.1 with: eas-version: latest token: ${{ secrets.EXPO_PREVIEW_TOKEN }} @@ -290,7 +290,7 @@ jobs: ORG_GRADLE_PROJECT_reactNativeArchitectures: armeabi-v7a,arm64-v8a steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: fetch-depth: 0 @@ -349,7 +349,7 @@ jobs: - name: Mount build cache if: steps.record.outputs.built != 'true' - uses: useblacksmith/stickydisk@v1 + uses: useblacksmith/stickydisk@3f6be1451e3cad893b778bc93495c80452a5da26 # v1.7.2 with: key: ${{ github.repository }}-android-preview-build path: ~/.cache/t3-android-preview @@ -367,7 +367,7 @@ jobs: - name: Setup Node if: steps.record.outputs.built != 'true' - uses: actions/setup-node@v6 + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 with: node-version-file: package.json @@ -384,14 +384,14 @@ jobs: - name: Setup Java if: steps.record.outputs.built != 'true' - uses: actions/setup-java@v5 + uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0 with: distribution: temurin java-version: 17 - name: Setup EAS if: steps.record.outputs.built != 'true' - uses: expo/expo-github-action@v8 + uses: expo/expo-github-action@c7b66a9c327a43a8fa7c0158e7f30d6040d2481e # v8.2.1 with: eas-version: latest token: ${{ secrets.EXPO_PREVIEW_TOKEN }} diff --git a/.github/workflows/mobile-eas-production.yml b/.github/workflows/mobile-eas-production.yml index 1bf16d8a8f65..cdabb1e4b7ca 100644 --- a/.github/workflows/mobile-eas-production.yml +++ b/.github/workflows/mobile-eas-production.yml @@ -124,7 +124,7 @@ jobs: - id: version_app_token name: Mint release app token for version override if: steps.expo-token.outputs.present == 'true' && github.event_name == 'workflow_dispatch' && inputs.mode == 'build' && inputs.version != '' && inputs.profile != 'testflight' - uses: actions/create-github-app-token@v2 + uses: actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349 # v2.2.2 with: app-id: ${{ secrets.RELEASE_APP_ID }} private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} @@ -132,7 +132,7 @@ jobs: - name: Checkout if: steps.expo-token.outputs.present == 'true' - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: fetch-depth: 0 token: ${{ steps.version_app_token.outputs.token || github.token }} @@ -143,7 +143,7 @@ jobs: - name: Setup Vite+ if: steps.expo-token.outputs.present == 'true' - uses: voidzero-dev/setup-vp@v1 + uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1.15.0 with: node-version-file: package.json cache: true @@ -161,7 +161,7 @@ jobs: - name: Setup EAS if: steps.expo-token.outputs.present == 'true' - uses: expo/expo-github-action@v8 + uses: expo/expo-github-action@c7b66a9c327a43a8fa7c0158e7f30d6040d2481e # v8.2.1 with: eas-version: latest token: ${{ secrets.EXPO_PRODUCTION_TOKEN }} @@ -498,7 +498,7 @@ jobs: BUILD_TAG: mobile-android-build/${{ needs.production.outputs.android_version }}/${{ needs.production.outputs.android_fingerprint }} steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ needs.production.outputs.android_sha }} fetch-depth: 0 @@ -551,7 +551,7 @@ jobs: - name: Mount build cache if: steps.record.outputs.built != 'true' - uses: useblacksmith/stickydisk@v1 + uses: useblacksmith/stickydisk@3f6be1451e3cad893b778bc93495c80452a5da26 # v1.7.2 with: key: ${{ github.repository }}-android-store-build path: ~/.cache/t3-android-store @@ -569,7 +569,7 @@ jobs: - name: Setup Node if: steps.record.outputs.built != 'true' - uses: actions/setup-node@v6 + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 with: node-version-file: package.json @@ -588,14 +588,14 @@ jobs: - name: Setup Java if: steps.record.outputs.built != 'true' - uses: actions/setup-java@v5 + uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0 with: distribution: temurin java-version: 17 - name: Setup EAS if: steps.record.outputs.built != 'true' - uses: expo/expo-github-action@v8 + uses: expo/expo-github-action@c7b66a9c327a43a8fa7c0158e7f30d6040d2481e # v8.2.1 with: eas-version: latest token: ${{ secrets.EXPO_PRODUCTION_TOKEN }} @@ -642,7 +642,7 @@ jobs: # A --path submission doesn't keep its archive on EAS; this is the only downloadable copy. - name: Keep the AAB if: steps.record.outputs.built != 'true' - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: android-store-aab-${{ github.run_id }}-${{ github.run_attempt }} path: ${{ runner.temp }}/t3code.aab @@ -680,8 +680,8 @@ jobs: echo ":rocket: android: submitted and recorded \`$BUILD_TAG\`" >> "$GITHUB_STEP_SUMMARY" # Off unless the MOBILE_IOS_LOCAL_BUILDS repository variable is `true`; the - # production job only sets ios_build then. Store binaries need the final - # Xcode 27.1; MOBILE_IOS_ALLOW_XCODE_RC=true accepts the release candidate. + # production job only sets ios_build then. Xcode 27.1 matches the EAS build + # image in eas.json. # No sticky disk: it mounts ext4 and only works on Linux runners. ios: name: iOS store build @@ -701,10 +701,9 @@ jobs: EXPO_TOKEN: ${{ secrets.EXPO_PRODUCTION_TOKEN }} PROFILE: ${{ needs.production.outputs.ios_profile }} BUILD_TAG: mobile-ios-build/${{ needs.production.outputs.ios_profile }}/${{ needs.production.outputs.ios_version }}/${{ needs.production.outputs.ios_fingerprint }} - ALLOW_XCODE_RC: ${{ vars.MOBILE_IOS_ALLOW_XCODE_RC == 'true' }} steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ needs.production.outputs.ios_sha }} fetch-depth: 0 @@ -739,12 +738,9 @@ jobs: if: steps.record.outputs.built != 'true' run: | xcode=/Applications/Xcode_27.1.app - if [ ! -d "$xcode" ] && [ "$ALLOW_XCODE_RC" = "true" ]; then - xcode=/Applications/Xcode_27.1_Release_Candidate.app - fi if [ ! -d "$xcode" ]; then ls -d /Applications/Xcode*.app - echo "::error::Xcode 27.1 (final) is not on this runner. Set the MOBILE_IOS_ALLOW_XCODE_RC repository variable to true to build store binaries with the release candidate." + echo "::error::Xcode 27.1 is not on this runner." exit 1 fi sudo xcode-select -s "$xcode/Contents/Developer" @@ -761,7 +757,7 @@ jobs: - name: Setup Node if: steps.record.outputs.built != 'true' - uses: actions/setup-node@v6 + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 with: node-version-file: package.json @@ -780,7 +776,7 @@ jobs: - name: Setup EAS if: steps.record.outputs.built != 'true' - uses: expo/expo-github-action@v8 + uses: expo/expo-github-action@c7b66a9c327a43a8fa7c0158e7f30d6040d2481e # v8.2.1 with: eas-version: latest token: ${{ secrets.EXPO_PRODUCTION_TOKEN }} diff --git a/.github/workflows/mobile-fingerprint-check.yml b/.github/workflows/mobile-fingerprint-check.yml index 6ef032450e04..6bc4639fcda2 100644 --- a/.github/workflows/mobile-fingerprint-check.yml +++ b/.github/workflows/mobile-fingerprint-check.yml @@ -33,7 +33,7 @@ jobs: NODE_OPTIONS: --max-old-space-size=8192 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: # The merge ref can advance after the event is queued. Pin its commit # and compare against its first parent, rather than the event's base. @@ -58,7 +58,7 @@ jobs: - name: Setup Vite+ if: steps.changes.outputs.relevant == 'true' - uses: voidzero-dev/setup-vp@v1 + uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1.15.0 with: node-version-file: package.json cache: true @@ -136,7 +136,7 @@ jobs: printf '%s\n' "$CHANGED_PLATFORMS" > "$RUNNER_TEMP/fingerprint-result/changed-platforms" - name: Upload changed platforms - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: fingerprint-result path: ${{ runner.temp }}/fingerprint-result diff --git a/.github/workflows/mobile-fingerprint-label.yml b/.github/workflows/mobile-fingerprint-label.yml index 45e4c9f2ad42..a48462eec8c5 100644 --- a/.github/workflows/mobile-fingerprint-label.yml +++ b/.github/workflows/mobile-fingerprint-label.yml @@ -33,7 +33,7 @@ jobs: steps: - id: pr name: Resolve the pull request behind the check - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: script: | const run = context.payload.workflow_run; @@ -100,7 +100,7 @@ jobs: - name: Sync native change label if: steps.pr.outputs.pr_number != '' - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: PR_NUMBER: ${{ steps.pr.outputs.pr_number }} CHANGED_PLATFORMS: ${{ steps.result.outputs.changed_platforms }} diff --git a/.github/workflows/mobile-showcase-screenshots.yml b/.github/workflows/mobile-showcase-screenshots.yml index 53b0a74a1f5a..402ff8644537 100644 --- a/.github/workflows/mobile-showcase-screenshots.yml +++ b/.github/workflows/mobile-showcase-screenshots.yml @@ -45,13 +45,13 @@ jobs: ios: name: iPhone, iPhone Duo, and iPad 13 if: inputs.platform == 'all' || inputs.platform == 'ios' - runs-on: blacksmith-12vcpu-macos-26 + runs-on: blacksmith-12vcpu-macos-27 # Capturing every palette multiplies the device matrix by six, and only the # one native build is shared between them. timeout-minutes: ${{ inputs.theme == 'all' && 300 || 60 }} steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: sparse-checkout: | /* @@ -59,7 +59,7 @@ jobs: sparse-checkout-cone-mode: false - name: Setup Vite+ - uses: voidzero-dev/setup-vp@v1 + uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1.15.0 with: node-version-file: package.json cache: true @@ -98,7 +98,7 @@ jobs: - name: Upload iOS screenshots if: always() - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: app-store-connect-screenshots path: artifacts/app-store/screenshots/apple/ @@ -116,7 +116,7 @@ jobs: T3_SHOWCASE_ANDROID_ABI: x86_64 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: sparse-checkout: | /* @@ -124,7 +124,7 @@ jobs: sparse-checkout-cone-mode: false - name: Setup Vite+ - uses: voidzero-dev/setup-vp@v1 + uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1.15.0 with: node-version-file: package.json cache: true @@ -142,13 +142,13 @@ jobs: "$vp_pnpm_bin/pnpm" --version - name: Setup Java - uses: actions/setup-java@v5 + uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0 with: distribution: temurin java-version: 17 - name: Setup Gradle cache - uses: gradle/actions/setup-gradle@v5 + uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 - name: Enable KVM run: | @@ -158,7 +158,7 @@ jobs: sudo udevadm trigger --name-match=kvm - name: Capture Android showcase - uses: reactivecircus/android-emulator-runner@v2 + uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # v2.38.0 with: api-level: 36 target: google_apis @@ -175,7 +175,7 @@ jobs: - name: Upload Android screenshots if: always() - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: google-play-screenshots path: artifacts/app-store/screenshots/google-play/ diff --git a/.github/workflows/pr-limits.yml b/.github/workflows/pr-limits.yml index 75943062735c..d6a2864f94a9 100644 --- a/.github/workflows/pr-limits.yml +++ b/.github/workflows/pr-limits.yml @@ -34,13 +34,13 @@ jobs: # branch and never run PR code. Cone mode also checks out the files beside # .github/scripts, which is where TRIAGE_EXEMPTIONS.td and VOUCHED.td live. - name: Checkout trusted scripts - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ github.event.repository.default_branch }} sparse-checkout: .github/scripts - name: Close PRs over the limit - uses: actions/github-script@v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: DRY_RUN: ${{ vars.PR_LIMITS_ENFORCE != 'true' }} with: @@ -57,13 +57,13 @@ jobs: cancel-in-progress: false steps: - name: Checkout trusted scripts - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ github.event.repository.default_branch }} sparse-checkout: .github/scripts - name: Request rebases and close what went unanswered - uses: actions/github-script@v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: DRY_RUN: ${{ vars.PR_LIMITS_ENFORCE != 'true' || (github.event_name == 'workflow_dispatch' && inputs.dry_run) }} with: diff --git a/.github/workflows/pr-size.yml b/.github/workflows/pr-size.yml index cb0c9637fcef..45f5c9b9f776 100644 --- a/.github/workflows/pr-size.yml +++ b/.github/workflows/pr-size.yml @@ -16,7 +16,7 @@ jobs: steps: - id: config name: Build PR size label config - uses: actions/github-script@v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: result-encoding: string script: | @@ -64,7 +64,7 @@ jobs: issues: write steps: - name: Ensure PR size labels exist - uses: actions/github-script@v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: PR_SIZE_LABELS_JSON: ${{ needs.prepare-config.outputs.labels_json }} with: @@ -128,11 +128,11 @@ jobs: # git data. Do not add dependency installs, build/test scripts, or cache # actions here; use pull_request plus workflow_run for that pattern instead. - name: Checkout base repository - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 - name: Sync PR size label - uses: actions/github-script@v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: PR_SIZE_LABELS_JSON: ${{ needs.prepare-config.outputs.labels_json }} with: @@ -295,7 +295,7 @@ jobs: ); # Reads the script from the base checkout above; the PR's own commits are only diffed. - name: Report test suite growth - uses: actions/github-script@v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: script: | const { execFileSync } = require("node:child_process"); diff --git a/.github/workflows/pr-vouch.yml b/.github/workflows/pr-vouch.yml index 75e2ee8a5654..eb53c0789402 100644 --- a/.github/workflows/pr-vouch.yml +++ b/.github/workflows/pr-vouch.yml @@ -29,7 +29,7 @@ jobs: targets: ${{ steps.collect.outputs.targets }} steps: - id: collect - uses: actions/github-script@v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: script: | if (context.eventName === "pull_request_target") { @@ -81,7 +81,7 @@ jobs: steps: - id: vouch name: Check PR author trust - uses: mitchellh/vouch/action/check-user@v1 + uses: mitchellh/vouch/action/check-user@d66fa29a64600490892131ad87597c30c91fcac4 # v1.5.0 with: user: ${{ matrix.target.user }} allow-fail: true @@ -89,7 +89,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Sync PR labels - uses: actions/github-script@v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: PR_NUMBER: ${{ matrix.target.number }} VOUCH_STATUS: ${{ steps.vouch.outputs.status }} @@ -101,17 +101,17 @@ jobs: { name: "vouch:trusted", color: "1f883d", - description: "PR author is trusted by repo permissions or the VOUCHED list.", + description: "Author is trusted by repo permissions or the VOUCHED list.", }, { name: "vouch:unvouched", color: "fbca04", - description: "PR author is not yet trusted in the VOUCHED list.", + description: "Author is not yet trusted in the VOUCHED list.", }, { name: "vouch:denounced", color: "d1242f", - description: "PR author is explicitly blocked by the VOUCHED list.", + description: "Author is explicitly blocked by the VOUCHED list.", }, ]; diff --git a/.github/workflows/publish-aur.yml b/.github/workflows/publish-aur.yml index 4ab6db89ab70..dc88f0150945 100644 --- a/.github/workflows/publish-aur.yml +++ b/.github/workflows/publish-aur.yml @@ -48,7 +48,7 @@ jobs: run: pacman -Syu --noconfirm --needed git github-cli jq namcap openssh sudo - name: Checkout packaging sources - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - name: Create unprivileged build user run: | diff --git a/.github/workflows/relay-preview-cleanup.yml b/.github/workflows/relay-preview-cleanup.yml index eed067adfd77..483e3e0b01b7 100644 --- a/.github/workflows/relay-preview-cleanup.yml +++ b/.github/workflows/relay-preview-cleanup.yml @@ -58,7 +58,7 @@ jobs: ALCHEMY_TELEMETRY_DISABLED: "1" steps: - name: Checkout default branch - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ github.event.repository.default_branch }} sparse-checkout: | @@ -67,7 +67,7 @@ jobs: sparse-checkout-cone-mode: false - name: Setup Vite+ - uses: voidzero-dev/setup-vp@v1 + uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1.15.0 with: node-version-file: package.json cache: true @@ -170,7 +170,7 @@ jobs: - name: Checkout default branch if: steps.eligible.outputs.eligible == 'yes' - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ github.event.repository.default_branch }} sparse-checkout: | @@ -180,7 +180,7 @@ jobs: - name: Setup Vite+ if: steps.eligible.outputs.eligible == 'yes' - uses: voidzero-dev/setup-vp@v1 + uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1.15.0 with: node-version-file: package.json cache: true diff --git a/.github/workflows/relay-preview.yml b/.github/workflows/relay-preview.yml index fdce22c68a59..daec31908775 100644 --- a/.github/workflows/relay-preview.yml +++ b/.github/workflows/relay-preview.yml @@ -53,7 +53,7 @@ jobs: fi - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ github.event.pull_request.head.sha }} sparse-checkout: | @@ -62,7 +62,7 @@ jobs: sparse-checkout-cone-mode: false - name: Setup Vite+ - uses: voidzero-dev/setup-vp@v1 + uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1.15.0 with: node-version-file: package.json cache: true diff --git a/.github/workflows/release-desktop.yml b/.github/workflows/release-desktop.yml index b227a2b8b12f..4252cb46e7a0 100644 --- a/.github/workflows/release-desktop.yml +++ b/.github/workflows/release-desktop.yml @@ -70,6 +70,12 @@ on: required: false default: false type: boolean + # Whether the JS bundle came from a pull request. This job holds the + # signing secrets, so packaging then skips every step that runs the bundle. + untrusted_bundle: + required: false + default: false + type: boolean # A runner of the target's own platform and architecture, for a `runner` # that cannot execute the target. The build job skips everything that # runs the packaged binaries, and a second job runs it there. @@ -134,6 +140,14 @@ jobs: T3CODE_CLERK_CLI_OAUTH_CLIENT_ID: ${{ inputs.clerk_cli_oauth_client_id }} T3CODE_RELAY_URL: ${{ inputs.relay_url }} steps: + # The launch check and the CLI archive smoke test both run the bundle. + - name: Reject running an untrusted bundle + if: inputs.untrusted_bundle && (inputs.launch_check || inputs.cli_archive) + shell: bash + run: | + echo "::error::untrusted_bundle cannot be combined with launch_check or cli_archive, which run the bundle." + exit 1 + # This repository is public, so Git needs no credentials. checkout's # credential cleanup runs submodule foreach even with submodules disabled, # which fails on the orphaned gitlinks in our vendored .repos tree. @@ -157,7 +171,7 @@ jobs: # artifact leaves the cache empty, so installation runs the checks again. - name: Download dependency verification continue-on-error: true - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: release-dependency-verification path: ${{ runner.temp }}/pnpm-metadata @@ -183,7 +197,7 @@ jobs: - name: Restore staged install cache id: stage_pnpm_cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.stage_pnpm.outputs.dir }} key: ${{ steps.stage_pnpm.outputs.prefix }}${{ github.run_id }}-${{ github.run_attempt }} @@ -195,7 +209,7 @@ jobs: - name: Cache resource monitor id: resource_monitor_cache - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: native/resource-monitor/target/${{ inputs.rust_target }}/release/t3-resource-monitor${{ inputs.platform == 'win' && '.exe' || '' }} key: resource-monitor-${{ inputs.rust_target }}-${{ hashFiles('native/resource-monitor/Cargo.lock', 'native/resource-monitor/Cargo.toml', 'native/resource-monitor/src/**') }} @@ -203,7 +217,7 @@ jobs: - name: Cache Linux capture helpers if: inputs.platform == 'linux' id: capture_helper_cache - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: | native/kde-snap-shot/target/${{ inputs.rust_target }}/release/t3-kde-snap-shot @@ -216,7 +230,7 @@ jobs: # files and downloads again on a mismatch, so only raw archives are cached. - name: Cache Linux packaging tool archives if: inputs.platform == 'linux' - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: | ~/.cache/electron-builder/appimage@*/appimage-tools-runtime-*.tar.gz @@ -234,14 +248,14 @@ jobs: - name: Setup Rust if: steps.resource_monitor_cache.outputs.cache-hit != 'true' || (inputs.platform == 'linux' && steps.capture_helper_cache.outputs.cache-hit != 'true') - uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable + uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: toolchain: stable targets: ${{ inputs.rust_target }} - name: Download relay client tracing config if: inputs.relay_client_tracing - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: relay-client-tracing-config path: ${{ runner.temp }}/relay-client-tracing @@ -262,14 +276,14 @@ jobs: # ancestor of its paths), so extracting into `apps` restores # apps/server/dist and apps/desktop/dist-electron at their build paths. - name: Download JS bundle - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: js-bundle path: apps - name: Install cargo-xwin if: inputs.platform == 'win' && steps.resource_monitor_cache.outputs.cache-hit != 'true' - uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2 + uses: taiki-e/install-action@cfdb7ed4ccc7c7dce37759eae89dc5674b469b39 # v2.90.3 with: tool: cargo-xwin@0.23.1 @@ -317,7 +331,7 @@ jobs: - name: Restore target Node.js for the CLI executable id: target_node_cache if: inputs.cli_archive && inputs.platform == 'win' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ runner.temp }}/node-download key: node-download-v${{ steps.target_node.outputs.version }}-win-${{ inputs.arch }} @@ -344,7 +358,7 @@ jobs: # Saved right away so a later failure in this job still keeps the download. - name: Save target Node.js for the CLI executable if: inputs.cli_archive && inputs.platform == 'win' && steps.target_node_cache.outputs.cache-hit != 'true' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ runner.temp }}/node-download key: ${{ steps.target_node_cache.outputs.cache-primary-key }} @@ -479,7 +493,7 @@ jobs: # macOS uploads its archive after "Await CLI notarization" instead. - name: Upload CLI archive if: inputs.cli_archive && inputs.platform != 'mac' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: cli-${{ inputs.platform }}-${{ inputs.arch }} path: release-cli/* @@ -559,7 +573,7 @@ jobs: - name: Download Linux CLI archive for WSL if: inputs.platform == 'win' - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: cli-linux-${{ inputs.arch }} path: wsl-runtime @@ -573,6 +587,7 @@ jobs: pnpm_config_prefer_offline: "true" T3CODE_DESKTOP_REUSE_RESOURCE_MONITOR: true T3CODE_DESKTOP_REUSE_LINUX_CAPTURE_HELPERS: ${{ steps.capture_helper_cache.outputs.cache-hit == 'true' }} + T3CODE_DESKTOP_UNTRUSTED_BUNDLE: ${{ inputs.untrusted_bundle }} CSC_LINK: ${{ secrets.CSC_LINK }} CSC_KEY_PASSWORD: ${{ secrets.CSC_KEY_PASSWORD }} APPLE_API_KEY: ${{ secrets.APPLE_API_KEY }} @@ -680,7 +695,7 @@ jobs: - name: Upload notarized CLI archive if: inputs.cli_archive && inputs.platform == 'mac' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: cli-${{ inputs.platform }}-${{ inputs.arch }} path: release-cli/* @@ -700,7 +715,7 @@ jobs: - name: Save staged install cache if: steps.stage_pnpm_changed.outputs.changed == 'true' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.stage_pnpm.outputs.dir }} key: ${{ steps.stage_pnpm_cache.outputs.cache-primary-key }} @@ -731,7 +746,7 @@ jobs: - name: Upload launch check output if: failure() && inputs.launch_check - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: desktop-launch-check-${{ inputs.platform }}-${{ inputs.arch }} path: ${{ runner.temp }}/desktop-launch-check @@ -759,7 +774,7 @@ jobs: - name: Upload verification inputs if: inputs.verify_runner != '' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: verify-${{ inputs.platform }}-${{ inputs.arch }} path: verify/* @@ -826,7 +841,7 @@ jobs: cp "$source_path" "$target_dir/$binary_name" - name: Upload build artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: desktop-${{ inputs.platform }}-${{ inputs.arch }} path: release-publish/* @@ -835,7 +850,7 @@ jobs: compression-level: 0 - name: Upload resource monitor - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: resource-monitor-${{ inputs.resource_key }} path: resource-monitor-publish/${{ inputs.resource_key }}/* @@ -849,14 +864,14 @@ jobs: timeout-minutes: 15 steps: - name: Download verification inputs - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: verify-${{ inputs.platform }}-${{ inputs.arch }} path: verify - name: Download CLI archive if: inputs.cli_archive - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: cli-${{ inputs.platform }}-${{ inputs.arch }} path: release-cli diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8827876b3829..c1bd673dab60 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -48,9 +48,12 @@ jobs: # triggering commit. Manual stable releases build the commit of the latest # published nightly, so stable only ever ships a build that nightly users # have already run. Scheduled runs also decide here whether a nightly is due. + # The PAUSE_NIGHTLY repository variable skips scheduled runs only; every later + # job needs this one, so a skip here skips the whole run. `update_feed` runs + # without it and checks the variable itself. resolve_commit: name: Resolve release commit - if: github.repository == 'pingdotgg/t3code' + if: github.repository == 'pingdotgg/t3code' && (github.event_name != 'schedule' || vars.PAUSE_NIGHTLY != 'true') runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 5 outputs: @@ -59,13 +62,13 @@ jobs: has_changes: ${{ steps.resolve.outputs.has_changes }} steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: sparse-checkout: .github/scripts - id: resolve name: Resolve release commit - uses: actions/github-script@v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: DISPATCH_CHANNEL: ${{ inputs.channel }} with: @@ -122,7 +125,7 @@ jobs: ref: ${{ needs.resolve_commit.outputs.ref }} steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ needs.resolve_commit.outputs.ref }} fetch-depth: 0 @@ -163,14 +166,12 @@ jobs: echo "is_prerelease=true" >> "$GITHUB_OUTPUT" echo "make_latest=false" >> "$GITHUB_OUTPUT" elif [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" && "${DISPATCH_CHANNEL:-preview}" == "preview" ]]; then - # Manual-only test train: exercises the whole release flow for a - # commit end users must never receive. Never scheduled. + # Manual-only test train: exercises the release flow up to a draft + # release for a commit end users must never receive. Never scheduled. # Same versioning as nightly under its own prerelease identifier. - # A preview release is reachable only by asking for it: npm gets it - # under the `preview` dist-tag, which nothing resolves by default, - # its desktop builds carry no update feed, and no updater manifest - # is attached to the release, so neither stable nor nightly - # installs can ever be offered one. + # A preview release stays a draft (see "Publish prerelease"), its + # desktop builds carry no update feed, no updater manifest is + # attached, and npm gets nothing, so no install can be offered one. nightly_date="$(date -u -d "$NIGHTLY_DATE" +%Y%m%d)" node scripts/resolve-nightly-release.ts \ @@ -181,7 +182,6 @@ jobs: --github-output echo "release_channel=preview" >> "$GITHUB_OUTPUT" - echo "cli_dist_tag=preview" >> "$GITHUB_OUTPUT" echo "is_prerelease=true" >> "$GITHUB_OUTPUT" echo "make_latest=false" >> "$GITHUB_OUTPUT" else @@ -226,7 +226,7 @@ jobs: # Share only the verification results, not the large registry metadata cache. - name: Upload dependency verification continue-on-error: true - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: release-dependency-verification path: ${{ runner.temp }}/pnpm-metadata/lockfile-verified.jsonl @@ -239,7 +239,7 @@ jobs: timeout-minutes: 10 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ needs.preflight.outputs.ref }} sparse-checkout: | @@ -271,7 +271,7 @@ jobs: timeout-minutes: 10 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ needs.preflight.outputs.ref }} sparse-checkout: | @@ -311,7 +311,7 @@ jobs: shard: [1, 2] steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ needs.preflight.outputs.ref }} sparse-checkout: | @@ -330,7 +330,7 @@ jobs: - name: Cache Chromium id: chromium_cache - uses: actions/cache@v6 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cache/ms-playwright key: playwright-chromium-shell-${{ runner.os }}-${{ runner.arch }}-${{ steps.playwright_version.outputs.version }} @@ -370,7 +370,7 @@ jobs: shard: [1, 2, 3, 4, 5, 6, 7, 8] steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ needs.preflight.outputs.ref }} sparse-checkout: | @@ -423,7 +423,7 @@ jobs: CLERK_CLI_OAUTH_CLIENT_ID: ${{ vars.CLERK_CLI_OAUTH_CLIENT_ID }} steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ needs.resolve_commit.outputs.ref }} sparse-checkout: | @@ -463,7 +463,7 @@ jobs: } > "$RUNNER_TEMP/relay-client-tracing.env" - name: Upload relay client tracing config - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: relay-client-tracing-config path: ${{ runner.temp }}/relay-client-tracing.env @@ -533,7 +533,7 @@ jobs: T3CODE_RELAY_URL: ${{ needs.relay_public_config.outputs.relay_url }} steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ needs.preflight.outputs.ref }} sparse-checkout: | @@ -545,7 +545,7 @@ jobs: # artifact leaves the cache empty, so installation runs the checks again. - name: Download dependency verification continue-on-error: true - uses: actions/download-artifact@v8 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: release-dependency-verification path: ${{ runner.temp }}/pnpm-metadata @@ -558,7 +558,7 @@ jobs: pnpm_config_cache_dir: ${{ runner.temp }}/pnpm-metadata - name: Download relay client tracing config - uses: actions/download-artifact@v8 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: relay-client-tracing-config path: ${{ runner.temp }}/relay-client-tracing @@ -604,7 +604,7 @@ jobs: # Two paths under apps/ so the artifact root is apps/; consumers download # into `apps` to restore both at their original locations. - name: Upload JS bundle - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: js-bundle path: | @@ -622,7 +622,7 @@ jobs: # this upload by its ID, which a publish_cli-only rerun keeps. - name: Upload the npm launcher id: npm_launcher - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: npm-launcher path: npm-packages/t3.tgz @@ -817,7 +817,7 @@ jobs: pull-requests: read steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ needs.preflight.outputs.ref }} # Release notes need history back to the previous tag, not file contents. @@ -852,14 +852,14 @@ jobs: fi - name: Download all desktop artifacts - uses: actions/download-artifact@v8 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: desktop-* merge-multiple: true path: release-assets - name: Download all CLI archives - uses: actions/download-artifact@v8 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: cli-* merge-multiple: true @@ -903,34 +903,12 @@ jobs: fi # electron-updater reads one manifest per platform and channel and picks - # the file entry whose name carries the running arch, so the per-arch - # manifests the build jobs wrote are merged back into that one file. - - name: Merge macOS updater manifests + # the file entry whose name carries the running arch. Require every + # platform's two build jobs to have produced matching metadata before + # merging it, so a successful release can never strand one architecture. + - name: Merge desktop updater manifests if: needs.preflight.outputs.release_channel != 'preview' - run: | - shopt -s nullglob - for x64_manifest in release-assets/*-mac-x64.yml; do - arm64_manifest="${x64_manifest%-x64.yml}.yml" - if [[ -f "$arm64_manifest" ]]; then - node scripts/merge-update-manifests.ts --platform mac "$arm64_manifest" "$x64_manifest" - rm -f "$x64_manifest" - fi - done - - - name: Merge Windows updater manifests - if: needs.preflight.outputs.release_channel != 'preview' - run: | - shopt -s nullglob - for x64_manifest in release-assets/*-win-x64.yml; do - arm64_manifest="${x64_manifest%-x64.yml}-arm64.yml" - merged_manifest="${x64_manifest%-win-x64.yml}.yml" - if [[ -f "$arm64_manifest" ]]; then - node scripts/merge-update-manifests.ts --platform win "$x64_manifest" "$arm64_manifest" "$merged_manifest" - rm -f "$x64_manifest" "$arm64_manifest" - else - mv "$x64_manifest" "$merged_manifest" - fi - done + run: node scripts/merge-release-update-manifests.ts release-assets # Updater manifests and blockmaps are what electron-updater consumes. # They are only listed for channels an updater is meant to follow. @@ -1002,7 +980,7 @@ jobs: - name: Publish release if: needs.preflight.outputs.previous_tag != '' - uses: softprops/action-gh-release@v3 + uses: softprops/action-gh-release@8a0cd651e5b2365bd640854a3fa5dfc70e4875f3 # v3.0.4 with: tag_name: ${{ needs.preflight.outputs.tag }} target_commitish: ${{ needs.preflight.outputs.ref }} @@ -1019,7 +997,7 @@ jobs: - name: Publish first release if: needs.preflight.outputs.previous_tag == '' - uses: softprops/action-gh-release@v3 + uses: softprops/action-gh-release@8a0cd651e5b2365bd640854a3fa5dfc70e4875f3 # v3.0.4 with: tag_name: ${{ needs.preflight.outputs.tag }} target_commitish: ${{ needs.preflight.outputs.ref }} @@ -1036,22 +1014,93 @@ jobs: # The action keeps a stable release a draft until its assets finish uploading, but publishes a # prerelease first. Desktop clients update connected servers from the CLI archives as soon as # they see the release, so a prerelease also goes public only once every asset is attached. + # Previews stay drafts, because a draft has no git tag. releases.atom lists the 10 newest tags, + # a day's `preview` tags can sort above its `nightly` ones, and older updaters find nightlies only there. - name: Publish prerelease - if: needs.preflight.outputs.is_prerelease == 'true' + if: needs.preflight.outputs.is_prerelease == 'true' && needs.preflight.outputs.release_channel != 'preview' env: GH_TOKEN: ${{ github.token }} GH_REPO: ${{ github.repository }} RELEASE_TAG: ${{ needs.preflight.outputs.tag }} run: gh release edit "$RELEASE_TAG" --draft=false + # A nightly install finds updates in releases.atom, which holds the 10 newest git tags of any + # kind, so a burst of other tags hides every nightly. The `update-feed` release is the fixed URL + # newer installs also read. Every unpaused scheduled tick and every nightly repairs it, then fails when + # the feed's first nightly is not the newest one. See docs/operations/release.md#update-feed. + update_feed: + name: Check the update feed + needs: [preflight, release] + if: ${{ !cancelled() && github.repository == 'pingdotgg/t3code' && ((github.event_name == 'schedule' && vars.PAUSE_NIGHTLY != 'true') || needs.preflight.outputs.release_channel == 'nightly') }} + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 5 + permissions: + contents: write + steps: + - name: Checkout + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + sparse-checkout: .github/scripts + + - id: feed + name: Read the update feed + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + with: + retries: 3 + script: | + const { readNightlyUpdateFeed } = require('./.github/scripts/check-nightly-release.cjs'); + const { tag, pointerTag, feedNightly, feedTags } = await readNightlyUpdateFeed({ github, context }); + if (!tag) return; + core.info(`Newest nightly: ${tag}. Pointer: ${pointerTag}. Feed: ${feedTags?.join(', ')}.`); + core.setOutput('tag', tag); + core.setOutput('pointer_stale', pointerTag !== tag); + core.setOutput('feed', feedTags === undefined ? 'unreadable' : feedNightly === tag ? 'ok' : 'hidden'); + + - name: Point the update feed at the newest nightly + if: steps.feed.outputs.pointer_stale == 'true' + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + RELEASE_TAG: ${{ steps.feed.outputs.tag }} + run: | + gh api "repos/$GH_REPO/releases/tags/$RELEASE_TAG" --jq '{tag_name, name, body}' > nightly.json + gh release view update-feed > /dev/null 2>&1 || + gh release create update-feed --prerelease --latest=false --title "Update feed" \ + --notes "The desktop updater reads this release to find the newest nightly. Do not delete it. See docs/operations/release.md." + gh release upload update-feed nightly.json --clobber + + # Every run, because the pointer can be current while the release's flags are wrong. A draft's + # files are not public, and a release marked "latest" is the one stable installs update to. + - name: Keep the update feed release public and never latest + if: steps.feed.outputs.tag != '' + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: gh release edit update-feed --draft=false --prerelease --latest=false + + # An unreadable feed says nothing about its tags and the next tick reads it again, so only a + # feed that was read and names another nightly first fails the run. + - name: Fail when the feed hides the newest nightly + if: steps.feed.outputs.tag != '' && steps.feed.outputs.feed != 'ok' + env: + RELEASE_TAG: ${{ steps.feed.outputs.tag }} + FEED: ${{ steps.feed.outputs.feed }} + run: | + if [[ "$FEED" == "unreadable" ]]; then + echo "::warning::releases.atom could not be read, so this run did not check it." + exit 0 + fi + echo "::error::The first nightly in releases.atom is not $RELEASE_TAG. Nightly installs older than 0.1.10-nightly.20261112.4603 cannot reach it, and neither can later builds that do not read the pointer when the feed names an older nightly. A preview from a branch not rebased onto main is the usual cause. See docs/operations/release.md#update-feed." + exit 1 + # The `t3` npm package is a launcher that downloads this version's CLI # archive from the GitHub Release, so it publishes only once that release - # exists. Preview publishes too, under the `preview` dist-tag, which - # nothing resolves unless asked for by name. + # exists. A preview release stays a draft, whose files are not public, so + # a preview never publishes the launcher. publish_cli: name: Publish CLI to npm needs: [preflight, build_bundle, release] - if: ${{ !failure() && !cancelled() && needs.preflight.result == 'success' && needs.release.result == 'success' }} + if: ${{ !failure() && !cancelled() && needs.preflight.result == 'success' && needs.release.result == 'success' && needs.preflight.outputs.release_channel != 'preview' }} runs-on: ubuntu-24.04 # blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 15 permissions: @@ -1060,7 +1109,7 @@ jobs: steps: # Only what setup-ci-deps reads to install the project's Node and npm. - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ needs.preflight.outputs.ref }} sparse-checkout: | @@ -1087,7 +1136,7 @@ jobs: fi - name: Download the npm launcher - uses: actions/download-artifact@v8 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: artifact-ids: ${{ needs.build_bundle.outputs.npm_launcher_artifact_id }} path: npm-packages @@ -1149,7 +1198,7 @@ jobs: VERCEL_TEAM_SLUG: ${{ vars.VERCEL_TEAM_SLUG }} steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ needs.preflight.outputs.ref }} sparse-checkout: | @@ -1163,7 +1212,7 @@ jobs: install-args: --filter=@t3tools/scripts... --filter=@t3tools/web... - name: Download relay client tracing config - uses: actions/download-artifact@v8 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: relay-client-tracing-config path: ${{ runner.temp }}/relay-client-tracing @@ -1249,7 +1298,7 @@ jobs: # Only what setup-ci-deps reads to install vp and Node. The alias call # names its deployment and scope, so it reads nothing else from the repo. - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ needs.preflight.outputs.ref }} sparse-checkout: | @@ -1320,7 +1369,7 @@ jobs: VERCEL_TEAM_SLUG: ${{ vars.VERCEL_TEAM_SLUG }} steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ needs.preflight.outputs.ref }} sparse-checkout: | @@ -1386,7 +1435,7 @@ jobs: VERCEL_TEAM_SLUG: ${{ vars.VERCEL_TEAM_SLUG }} steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ needs.preflight.outputs.ref }} sparse-checkout: | @@ -1424,14 +1473,14 @@ jobs: steps: - id: app_token name: Mint release app token - uses: actions/create-github-app-token@v2 + uses: actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349 # v2.2.2 with: app-id: ${{ secrets.RELEASE_APP_ID }} private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} owner: ${{ github.repository_owner }} - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: main fetch-depth: 0 @@ -1515,7 +1564,7 @@ jobs: timeout-minutes: 10 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ needs.preflight.outputs.ref }} sparse-checkout: | diff --git a/.github/workflows/stale-needs-more-info.yml b/.github/workflows/stale-needs-more-info.yml index b7b475a6117d..9d994115705f 100644 --- a/.github/workflows/stale-needs-more-info.yml +++ b/.github/workflows/stale-needs-more-info.yml @@ -13,7 +13,7 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v9 + - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9.1.0 with: only-issue-labels: "needs more info" # The label is applied by the triage flow, so never auto-mark; close 7 days after it was added. diff --git a/.github/workflows/thread-transfer-report.yml b/.github/workflows/thread-transfer-report.yml index 23eec72923bd..94987b447745 100644 --- a/.github/workflows/thread-transfer-report.yml +++ b/.github/workflows/thread-transfer-report.yml @@ -22,7 +22,7 @@ jobs: # workflow_run has a write-capable token even for fork PRs. Only load the # publisher from the trusted default branch and never execute PR code. - name: Checkout trusted publisher - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ github.event.repository.default_branch }} sparse-checkout: .github/scripts @@ -32,7 +32,7 @@ jobs: - id: resolve name: Resolve PR and baseline artifacts - uses: actions/github-script@v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: script: | const reporter = require("./.github/scripts/thread-transfer-report.cjs"); @@ -40,7 +40,7 @@ jobs: - name: Download PR result if: steps.resolve.outputs.publish == 'true' && steps.resolve.outputs.pr_artifact == 'true' - uses: actions/download-artifact@v8 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: thread-transfer-results path: ${{ runner.temp }}/thread-transfer/pr @@ -49,7 +49,7 @@ jobs: - name: Download main baseline if: steps.resolve.outputs.publish == 'true' && steps.resolve.outputs.baseline_artifact == 'true' - uses: actions/download-artifact@v8 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: thread-transfer-results path: ${{ runner.temp }}/thread-transfer/main @@ -58,7 +58,7 @@ jobs: - name: Update thread transfer comment if: steps.resolve.outputs.publish == 'true' - uses: actions/github-script@v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: PR_NUMBER: ${{ steps.resolve.outputs.pull_number }} PR_SHA: ${{ steps.resolve.outputs.pr_sha }} diff --git a/.github/workflows/web-preview.yml b/.github/workflows/web-preview.yml index 4b3f8952d503..69db6b6025e8 100644 --- a/.github/workflows/web-preview.yml +++ b/.github/workflows/web-preview.yml @@ -46,7 +46,7 @@ jobs: VERCEL_TEAM_SLUG: ${{ vars.VERCEL_TEAM_SLUG }} steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ github.event.pull_request.head.sha }} sparse-checkout: | @@ -55,7 +55,7 @@ jobs: sparse-checkout-cone-mode: false - name: Setup Vite+ - uses: voidzero-dev/setup-vp@v1 + uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1.15.0 with: node-version-file: package.json cache: true @@ -97,7 +97,7 @@ jobs: echo "deployment_url=$deployment_url" >> "$GITHUB_OUTPUT" - name: Comment deployment URL - uses: actions/github-script@v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: DEPLOYMENT_URL: ${{ steps.deploy.outputs.deployment_url }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/windows-tests.yml b/.github/workflows/windows-tests.yml index 21a61e91cc8e..cf0f7c030c23 100644 --- a/.github/workflows/windows-tests.yml +++ b/.github/workflows/windows-tests.yml @@ -30,7 +30,7 @@ jobs: timeout-minutes: 45 steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: sparse-checkout: | /* @@ -40,7 +40,7 @@ jobs: # setup-vp's own cache restores a Linux-shaped store on Windows, which is # slower than no cache (see #7975). Cache pnpm's Windows store directly. - name: Setup Vite+ - uses: voidzero-dev/setup-vp@v1 + uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1.15.0 with: node-version-file: package.json cache: false @@ -52,7 +52,7 @@ jobs: run: '"path=$(vp pm cache dir)" >> $env:GITHUB_OUTPUT' - name: Cache packages - uses: actions/cache@v6 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.package_cache_path.outputs.path }} key: windows-tests-packages-v1-${{ hashFiles('pnpm-lock.yaml') }} diff --git a/.gitignore b/.gitignore index 85b7c5b4bf02..1637f475fbbd 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,5 @@ node_modules/ # pnpm content-addressable store; never belongs in the repo. .pnpm-store/ + +/packages/ssh/.ssh-launcher-test-*/ diff --git a/AGENTS.md b/AGENTS.md index 9073e293deba..e5941a175264 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,6 @@ # T3 Code -T3 Code is a minimal GUI for coding agents. A Node WebSocket server wraps provider CLIs and agents (Codex, Claude Code, Cursor, Grok, OpenCode, Antigravity, Kiro) and serves web, desktop, and mobile clients. +T3 Code is a minimal GUI for coding agents. A Node WebSocket server wraps provider CLIs and agents (Codex, Claude Code, Cursor, Grok, OpenCode, Antigravity, Kiro, Pi, Muse Code, fx) and serves web, desktop, and mobile clients. You can think of T3 Code as an open source "bring-your-own-subscription" alternative to apps like Claude Desktop, Codex App, Cursor Glass and Conductor. @@ -68,7 +68,7 @@ The most common defect in this repo is a change that works on the path you teste - **Entry points.** A behavior reachable from the chat view is usually also reachable from Settings, the command palette, and a keybinding. Fixing one is not fixing the feature. - **Clients.** Web, desktop (wraps web, adds Electron shell/IPC), and mobile (React Native, separate navigation). Shared logic lives in `packages/client-runtime` -- **Providers.** Codex, Claude, Cursor, Grok, OpenCode, Antigravity, and Kiro each have an adapter. Provider-shaped features need a decision per adapter, even if the decision is "not supported here". +- **Providers.** Codex, Claude, Cursor, Grok, OpenCode, Antigravity, Kiro, Pi, Muse Code, fx, and ACP Registry each have an adapter. Provider-shaped features need a decision per adapter, even if the decision is "not supported here". - **Agents.** A capability a user can trigger is usually one an agent should reach through MCP tools, and scheduled tasks run the same paths. That only works when it is a service method, not handler code. - **Contracts.** Anything crossing the wire is typed in `packages/contracts`. Change the schema and the server, web, mobile, and desktop all follow. - **Reverse states.** If you added a way in, add the way out and the way to see it. Snooze needs unsnooze. Close needs reopen. A one-way door is a bug. @@ -83,7 +83,8 @@ The most common defect in this repo is a change that works on the path you teste - Ports derive from the worktree path and are stable across restarts, but read the real ones from the `[dev-runner]` line since occupied ports shift. - Sharing over the tailnet is three steps: run `vp run dev --share` in the background, wait for the `pairingUrl:` line in its output, then give that full URL to an unpaired browser. Do not wire up `tailscale serve` by hand, open the URL yourself, or consume the user's pairing link. A browser with the reusable dev cookie can use the bare origin. If a normal one-time token was consumed, mint a fresh one with `node apps/server/src/bin.ts pair`. It carries standard scopes, while the startup URL carries admin scopes needed for Connections settings. - To reuse web dev auth across worktrees, configure one fixed `T3CODE_DEV_AUTH_TOKEN` in the main checkout's gitignored `.env`. The `t3.json` setup links that file into worktrees. Never commit or publish the token or a startup URL. See [Reusable dev credential](docs/operations/development.md#reusable-dev-credential). -- Stop what you started, by the PID you tracked. See rule 1. +- Stop what you started, by the PID you tracked. See rule 1. `vp` does not pass SIGTERM on, so signalling the `vp run dev` PID alone leaves Vite and the server running. Stop the background task with your harness's own stop, which ends the whole tree. +- Run dev servers, Metro, and builds from your own shell so they end with your session. Do not detach them with `systemd-run`, `nohup`, `setsid`, or `tmux` unless the user asks for one that outlives the session. Detached servers ran for weeks and filled RAM and swap. The dev runner stops itself above `--memory-limit-gb` (half of RAM, at most 16 GiB) on Linux. - Need a second checkout? `git worktree add` it next to this worktree, not in `/tmp`. Remove it when done. ## Test data @@ -149,7 +150,7 @@ For mobile verification, a missing or outdated native client is an install step, ## Pull requests - Never make a PR unless the developer explicitly asks you to do so. -- If `origin` is git.cafe, follow [GitCafe mirror](docs/operations/development.md#gitcafe-mirror-optional): push to `origin`, open and track PRs with `cafe` and their git.cafe URLs, and keep `gh` for merging. +- `origin` is not always GitHub. Check `git remote get-url origin` before your first forge command and use that forge's CLI and skill. See [Forge mirror](docs/operations/development.md#forge-mirror-optional). - Conventional commit titles, plain language: `fix(web): new threads no longer spike CPU`. - Body: the problem in a sentence or two, then how you fixed it. End with the model and harness that did the work. - UI changes need before/after images. Motion or timing needs a short video. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 63e71dc1e250..0c886bd49338 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -115,6 +115,9 @@ the open PR limit. - If you make our lives harder as reviewers and maintainers, you will be banned. Examples are repeated pings, a fight over a closure, and a second account to get around a limit. +A banned account is denounced in [.github/VOUCHED.td](.github/VOUCHED.td). Any issue or PR it opens +is closed when it is opened. + ## When your PR is closed diff --git a/README.md b/README.md index d50e32b606e6..fce93b721fdf 100644 --- a/README.md +++ b/README.md @@ -1,147 +1,94 @@ -# T3 Code +

+ + T3 Code logo + +

-T3 Code is an "agent harness control surface". It enables control of the agents on your machine with a best-in-class mobile app ([iOS](https://apps.apple.com/us/app/t3-code-remote-claude-more/id6787819824), [Android](https://play.google.com/store/apps/details?id=com.t3tools.t3code)), [web app](https://app.t3.codes) and [Electron-based desktop app](https://t3.codes). +

T3 Code

-Works with your subscriptions on Claude Code, Codex, Cursor, Grok Build, OpenCode, Google Antigravity, and fx. If they're set up on your computer, T3 Code can control them. +

The open-source control plane for coding agents.

-## "Wait, what are you selling me?" - -Nothing. We built T3 Code because we wanted the best possible development experience with agents. We were inspired by existing solutions like the Codex desktop app, Conductor, Claude Desktop and Cursor Glass, but none met our bar. +

+ Download · + iOS · + Android · + Web · + Docs · + Discord +

-We wanted something performant, remote-ready, and truly open. If we ever go the wrong direction, we want you to have everything you need to fork and build the editor that you want. +

+ + T3 Code on desktop with a finished Claude thread, next to the phone app asking to approve a command from Codex + +

-## Installation +T3 Code runs the coding agents on your machine and gives them one fast interface. Start a thread at your desk, then approve the next step from your phone. -> [!WARNING] -> T3 Code currently supports Codex, Claude, Cursor, Grok Build, OpenCode, Antigravity, and fx. Install and authenticate at least one provider before use: -> -> - Codex: install [Codex CLI](https://developers.openai.com/codex/cli) and run `codex login` -> - Claude: install [Claude Code](https://claude.com/product/claude-code) and run `claude auth login` -> - Cursor: install [Cursor CLI](https://cursor.com/cli) and run `agent login` -> - Grok Build: install [Grok Build CLI](https://x.ai/cli) and run `grok login` -> - OpenCode: install [OpenCode](https://opencode.ai) and run `opencode auth login` -> - Antigravity: enable it in Settings, then use **Install Antigravity** and **Sign in with Google**. No CLI is required. -> - fx: install [fx](https://fx.sh) and run `fx login` on the machine running the T3 Code server ([setup guide](./docs/user/providers-fx.md)) +It works with Claude Code, Codex, Cursor, OpenCode, Grok Build, Antigravity, Kiro, Pi, Muse Code, fx, and any [ACP](./docs/user/providers-acp.md) agent. If an agent is set up on your computer, T3 Code can control it. -### Command line +## What you get -```bash -npx --registry=https://npm-registry.rtvision.com/ @rtvision/t3@latest -``` +- **Your subscriptions.** T3 Code does not resell tokens. It drives the agents you are already signed in to. +- **Every agent in one place.** Change models in the middle of a thread. Let Claude hand a task to Codex. +- **Desktop, web, and phone.** One server, and every client shows the same threads. There are apps for iOS and Android. +- **Remote ready.** Run agents on any machine. Connect over your local network, Tailscale, or [T3 Connect](./docs/user/remote-access.md). +- **Git built in.** Each thread can work in its own worktree. Review the diff, then commit, push, and open a pull request with one button. +- **Fast.** Performance is a feature here. We measure it, and we treat a dropped frame as a bug. +- **Open.** MIT licensed. If you don't like something, fork it. -Then run `t3` to start the server and open the local web app. `t3 service install` keeps it running in the background, `t3 update` moves to a newer release, and `t3 --help` has the full reference. - -Tip: Use `npx --registry=https://npm-registry.rtvision.com/ @rtvision/t3@latest --help` for the full CLI reference. +## Install ### Desktop app -Install the latest version of the desktop app from [GitHub Releases](https://github.com/RTVision/t3code/releases), or from your favorite package registry: - -#### Windows (`winget`) - -```bash -winget install T3Tools.T3Code -``` - -#### Windows (`scoop`) +Download the Windows x64 or Linux x64 installer from [RTVision GitHub Releases](https://github.com/RTVision/t3code/releases). Desktop installers are unsigned. -```bash -scoop install extras/t3code -``` - -#### macOS (Homebrew) - -```bash -brew install --cask t3-code -``` - -#### Debian, Ubuntu (`.deb`) - -Download the `.deb` from [GitHub Releases](https://github.com/RTVision/t3code/releases), then: - -```bash -sudo apt install ./T3-Code-*.deb -``` - -#### Arch Linux (AUR) +### Command line -Stable: +This installs the `t3` command. Run `t3` to start the server and open the web app. ```bash -yay -S t3code-bin +npx --registry=https://npm-registry.rtvision.com/ @rtvision/t3@latest ``` -Nightly: +Use `npx --registry=https://npm-registry.rtvision.com/ @rtvision/t3@latest --help` for the full CLI reference. -```bash -yay -S t3code-nightly-bin -``` +### Phone -The AUR packaging is maintained in this repository under [`packaging/aur`](./packaging/aur). +Get the app from the [App Store](https://apps.apple.com/us/app/t3-code-remote-claude-more/id6787819824) or [Google Play](https://play.google.com/store/apps/details?id=com.t3tools.t3code). Then follow [remote access](./docs/user/remote-access.md) to connect it to your computer. -#### Nix (nixpkgs) +### Sign in to an agent -```bash -nix profile add github:NixOS/nixpkgs/nixos-unstable#t3code -``` +T3 Code needs one agent that is installed and signed in. For example, install [Claude Code](https://claude.com/product/claude-code) and run `claude auth login`, or install [Codex CLI](https://developers.openai.com/codex/cli) and run `codex login`. -The package is maintained in [nixpkgs](https://github.com/NixOS/nixpkgs/tree/nixos-unstable/pkgs/by-name/t3/t3code) -and includes the desktop app (`t3code-desktop`) and the `t3` command. -See [Nix installation](./docs/user/install.md#nix-nixpkgs) for NixOS configuration and updates. +[Provider setup](./docs/user/install.md#providers) has the steps for each agent. [Install T3 Code](./docs/user/install.md) covers nightly builds, AppImage, WSL, and NixOS. -## Some notes +## "Wait, what are you selling me?" -We are very very early in this project. Expect bugs. +Nothing. We built T3 Code because we wanted the best possible development experience with agents. We were inspired by existing solutions like the Codex desktop app, Conductor, Claude Desktop and Cursor Glass, but none met our bar. -We are (mostly) not accepting contributions yet. Small fixes may be considered. Big features will not be. +We wanted something performant, remote-ready, and truly open. If we ever go the wrong direction, we want you to have everything you need to fork and build the tool that you want. ## Documentation -Full docs live in [docs/](./docs). There's no docs site yet. - -- [FAQ](./docs/user/faq.md) - [Install and first run](./docs/user/install.md) -- [Permission modes](./docs/user/permission-modes.md) -- [Keyboard shortcuts](./docs/user/keybindings.md) -- [Project settings](./docs/user/project-settings.md) -- [Appearance preferences](./docs/user/appearance.md) - [Remote access from a phone or another machine](./docs/user/remote-access.md) -- [Connect Claude Code, Codex, ChatGPT and other agents over MCP](./docs/user/outside-agents.md) -- [Keeping app and server in sync](./docs/user/updating.md) -- [Source control integrations](./docs/user/source-control.md) -- Multiple accounts: [Codex](./docs/user/providers-codex.md) · [Claude](./docs/user/providers-claude.md) -- [Run T3 Code as a background service](./docs/user/background-service.md) - -Building from source? Start at [docs/internals/overview.md](./docs/internals/overview.md). - -## If you REALLY want to contribute still.... read this first - -### Install `vp` - -T3 Code uses Vite+ so you'll need to install the global `vp` command-line tool. - -#### macOS / Linux - -```bash -curl -fsSL https://vite.plus | bash -``` - -#### Windows - -```bash -irm https://vite.plus/ps1 | iex -``` +- [Permission modes](./docs/user/permission-modes.md) +- [Source control](./docs/user/source-control.md) +- [Connect outside agents over MCP](./docs/user/outside-agents.md) +- [FAQ](./docs/user/faq.md) -Checkout their getting started guide for more information: https://viteplus.dev/guide/ +[All docs](./docs/README.md) has the full list. -### Install dependencies +## Contributing -```bash -vp i -``` +We have limited review capacity. Small, focused fixes are the most likely to be merged. Features need an approved discussion first. -Read [CONTRIBUTING.md](./CONTRIBUTING.md) before reporting a bug or opening a PR. +- Read [CONTRIBUTING.md](./CONTRIBUTING.md) before you report a bug or open a pull request. +- Put feature requests in [Ideas](https://github.com/pingdotgg/t3code/discussions/categories/ideas). +- Build from source with the [development runbook](./docs/operations/development.md#first-checkout). The [architecture overview](./docs/internals/overview.md) explains how it works. +- Get help in [Discord](https://discord.gg/jn4EGJjrvv). -Have a feature request? Start an [Ideas discussion](https://github.com/pingdotgg/t3code/discussions/categories/ideas). +## License -Need support? Join the [Discord](https://discord.gg/jn4EGJjrvv). +[MIT](./LICENSE) diff --git a/apps/desktop/package.json b/apps/desktop/package.json index f9ab77d92ffd..48fc70397086 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -1,6 +1,6 @@ { "name": "@t3tools/desktop", - "version": "0.0.110", + "version": "0.0.111", "private": true, "type": "module", "main": "dist-electron/boot.cjs", diff --git a/apps/desktop/scripts/ensure-electron-runtime.mjs b/apps/desktop/scripts/ensure-electron-runtime.mjs index 933a763dfa35..741a2d9ee85d 100644 --- a/apps/desktop/scripts/ensure-electron-runtime.mjs +++ b/apps/desktop/scripts/ensure-electron-runtime.mjs @@ -209,6 +209,10 @@ export function ensureElectronRuntime() { return electronPath; } +export function setup() { + ensureElectronRuntime(); +} + // `file://${argv[1]}` never matches on Windows (drive letters need `file:///C:/`). if (process.argv[1] && NodeURL.pathToFileURL(process.argv[1]).href === import.meta.url) { const electronPath = ensureElectronRuntime(); diff --git a/apps/desktop/src/app/DesktopApp.test.ts b/apps/desktop/src/app/DesktopApp.test.ts index 20047278aeb1..ce2c53cbe1bf 100644 --- a/apps/desktop/src/app/DesktopApp.test.ts +++ b/apps/desktop/src/app/DesktopApp.test.ts @@ -86,6 +86,7 @@ it.effect.each([false, true])( const historyStarted = yield* Deferred.make(); const releaseHistory = yield* Deferred.make(); const quitting = yield* Ref.make(false); + const tokenRequested = yield* Deferred.make(); const events: string[] = []; const record = (event: string) => Effect.sync(() => { @@ -144,7 +145,7 @@ it.effect.each([false, true])( configure: Effect.void, } as PreviewPasskeys.PreviewPasskeys["Service"]), Effect.provideService(DesktopApplicationMenu.DesktopApplicationMenu, { - configure: Effect.void, + configure: record("menu"), }), Effect.provideService(DesktopUpdates.DesktopUpdates, { configure: Effect.void, @@ -222,7 +223,12 @@ it.effect.each([false, true])( Layer.mock(DesktopClientSettings.DesktopClientSettings, {}), Layer.mock(DesktopConnectionCatalogStore.DesktopConnectionCatalogStore, {}), Layer.mock(DesktopIpc.DesktopIpc, {}), - Layer.mock(DesktopLocalEnvironmentAuth.DesktopLocalEnvironmentAuth, {}), + Layer.mock(DesktopLocalEnvironmentAuth.DesktopLocalEnvironmentAuth, { + getBearerToken: record("token").pipe( + Effect.andThen(Deferred.succeed(tokenRequested, undefined)), + Effect.as("token"), + ), + }), Layer.mock(DesktopManagedPolicy.DesktopManagedPolicy, {}), Layer.succeed(DesktopPrimaryPaste.DesktopPrimaryPaste, { enabled: true }), Layer.mock(DesktopSshEnvironment.DesktopSshEnvironment, {}), @@ -254,7 +260,13 @@ it.effect.each([false, true])( ); yield* Deferred.await(registrationStarted); yield* Deferred.await(windowStarted); - assert.deepEqual(events, ["identity", "ready", "identity", "backend", "window"]); + yield* Deferred.await(tokenRequested); + // The backend spawns before menus, and its token is minted without waiting for the renderer. + assert.deepEqual( + events.filter((event) => event !== "token"), + ["identity", "ready", "identity", "backend", "menu", "window"], + ); + assert.isAbove(events.indexOf("token"), events.indexOf("backend")); assert.isFalse(yield* Deferred.isDone(registrationStopped)); const shutdown = yield* DesktopShutdown.DesktopShutdown; yield* shutdown.request; diff --git a/apps/desktop/src/app/DesktopApp.ts b/apps/desktop/src/app/DesktopApp.ts index 7ebeaa74a021..f69bf965d005 100644 --- a/apps/desktop/src/app/DesktopApp.ts +++ b/apps/desktop/src/app/DesktopApp.ts @@ -22,6 +22,7 @@ import * as PreviewPasskeys from "../preview/Passkeys.ts"; import * as DesktopBackendManager from "../backend/DesktopBackendManager.ts"; import * as DesktopBackendPool from "../backend/DesktopBackendPool.ts"; import * as DesktopHomeOwner from "../backend/DesktopHomeOwner.ts"; +import * as DesktopLocalEnvironmentAuth from "../backend/DesktopLocalEnvironmentAuth.ts"; import * as DesktopEnvironment from "./DesktopEnvironment.ts"; import * as DesktopLegacyLocalStorage from "./DesktopLegacyLocalStorage.ts"; import * as DesktopLifecycle from "./DesktopLifecycle.ts"; @@ -168,13 +169,12 @@ const stopAllPoolInstances = Effect.fn("desktop.app.stopAllPoolInstances")( }, ); -const bootstrap = Effect.gen(function* () { +// Spawns the local backend as soon as the protocol, IPC, and legacy storage that every window needs exist. +const startLocalBackend = Effect.gen(function* () { const state = yield* DesktopState.DesktopState; const environment = yield* DesktopEnvironment.DesktopEnvironment; const desktopSettings = yield* DesktopAppSettings.DesktopAppSettings; const desktopWindow = yield* DesktopWindow.DesktopWindow; - const snapShot = yield* DesktopSnapShot.DesktopSnapShot; - const appActivation = yield* DesktopAppActivation.DesktopAppActivation; yield* logBootstrapInfo("bootstrap start"); const settings = yield* desktopSettings.get; @@ -193,32 +193,13 @@ const bootstrap = Effect.gen(function* () { yield* (yield* DesktopAppIdentity.DesktopAppIdentity).resolveUserDataPath, ); - yield* snapShot.initialize; - // Only the user's own login shell PATH shows every t3 a new terminal runs; a fresh install has - // no cached one. Without it, the command is left to Settings. - const installCli = (yield* DesktopCliCommand.DesktopCliCommand).installOnFirstLaunch; - yield* Effect.forkScoped( - (yield* DesktopShellEnvironment.DesktopShellEnvironment).awaitUserShellDiscovery.pipe( - Effect.flatMap((found) => - found || environment.platform === "win32" - ? installCli - : logBootstrapInfo("skipped installing the t3 command without the shell PATH"), - ), - ), - ); - if (!settings.localEnvironmentEnabled) { yield* logBootstrapInfo("bootstrap skipping local environment (disabled in settings)"); - if (!(yield* Ref.get(state.quitting))) { - yield* desktopWindow.createMainIfBackendReady; - } - return; + return false; } - const pool = yield* DesktopBackendPool.DesktopBackendPool; - const primaryBackend = yield* pool.primary; + const primaryBackend = yield* (yield* DesktopBackendPool.DesktopBackendPool).primary; const serverExposure = yield* DesktopServerExposure.DesktopServerExposure; - const wslBackend = yield* DesktopWslBackend.DesktopWslBackend; if (environment.isDevelopment && Option.isNone(environment.configuredBackendPort)) { return yield* new DesktopDevelopmentBackendPortRequiredError(); @@ -237,7 +218,8 @@ const bootstrap = Effect.gen(function* () { "This T3 home is unavailable", DesktopHomeOwner.homeOwnedMessage(owner), ); - return yield* (yield* ElectronApp.ElectronApp).quit; + yield* (yield* ElectronApp.ElectronApp).quit; + return false; } const backendPortSelection = yield* resolveDesktopBackendPort(environment.configuredBackendPort); @@ -275,29 +257,71 @@ const bootstrap = Effect.gen(function* () { ); } - if (!(yield* Ref.get(state.quitting))) { - if (settings.wslOnly === true && settings.wslBackendEnabled === true) { - yield* desktopWindow.showConnectingSplash; - } - yield* primaryBackend.start; - yield* logBootstrapInfo("bootstrap backend start requested"); - // WSL-only fallback can change the primary endpoint before readiness. - const config = yield* primaryBackend.currentConfig; - if ( - !environment.isDevelopment && - !settings.wslOnly && - Option.isSome(config) && - Option.isNone(config.value.preflightFailure) - ) { - yield* desktopWindow.ensureMain; + if (yield* Ref.get(state.quitting)) { + return false; + } + if (settings.wslOnly === true && settings.wslBackendEnabled === true) { + yield* desktopWindow.showConnectingSplash; + } + yield* primaryBackend.start; + yield* logBootstrapInfo("bootstrap backend start requested"); + // The renderer asks for this token first; minting it at readiness saves a round trip. + yield* Effect.forkScoped( + (yield* DesktopLocalEnvironmentAuth.DesktopLocalEnvironmentAuth).getBearerToken.pipe( + Effect.ignore, + ), + ); + return true; +}).pipe(Effect.withSpan("desktop.startLocalBackend")); + +const bootstrap = Effect.fn("desktop.bootstrap")(function* (backendStarted: boolean) { + const state = yield* DesktopState.DesktopState; + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const desktopSettings = yield* DesktopAppSettings.DesktopAppSettings; + const desktopWindow = yield* DesktopWindow.DesktopWindow; + const snapShot = yield* DesktopSnapShot.DesktopSnapShot; + const appActivation = yield* DesktopAppActivation.DesktopAppActivation; + const settings = yield* desktopSettings.get; + if (settings.localEnvironmentEnabled && (!backendStarted || (yield* Ref.get(state.quitting)))) { + return; + } + + yield* snapShot.initialize; + // Only the user's own login shell PATH shows every t3 a new terminal runs; a fresh install has + // no cached one. Without it, the command is left to Settings. + const installCli = (yield* DesktopCliCommand.DesktopCliCommand).installOnFirstLaunch; + yield* Effect.forkScoped( + (yield* DesktopShellEnvironment.DesktopShellEnvironment).awaitUserShellDiscovery.pipe( + Effect.flatMap((found) => + found || environment.platform === "win32" + ? installCli + : logBootstrapInfo("skipped installing the t3 command without the shell PATH"), + ), + ), + ); + + if (!settings.localEnvironmentEnabled) { + if (!(yield* Ref.get(state.quitting))) { + yield* desktopWindow.createMainIfBackendReady; } - yield* appActivation.start.pipe( - Effect.tap(() => logBootstrapInfo("desktop app control socket ready")), - Effect.catch((error) => logStartupError("desktop app control socket unavailable", { error })), - ); - yield* Effect.forkScoped(wslBackend.reconcile); + return; } -}).pipe(Effect.withSpan("desktop.bootstrap")); + + const primaryBackend = yield* (yield* DesktopBackendPool.DesktopBackendPool).primary; + const config = yield* primaryBackend.currentConfig; + if ( + !environment.isDevelopment && + Option.isSome(config) && + Option.isNone(config.value.preflightFailure) + ) { + yield* desktopWindow.ensureMain; + } + yield* appActivation.start.pipe( + Effect.tap(() => logBootstrapInfo("desktop app control socket ready")), + Effect.catch((error) => logStartupError("desktop app control socket unavailable", { error })), + ); + yield* Effect.forkScoped((yield* DesktopWslBackend.DesktopWslBackend).reconcile); +}); const startup = Effect.gen(function* () { const appIdentity = yield* DesktopAppIdentity.DesktopAppIdentity; @@ -369,14 +393,20 @@ const startup = Effect.gen(function* () { }); } yield* appIdentity.configure; - yield* previewPasskeys.configure; - yield* applicationMenu.configure; + // Sets canInstallUpdates, which the backend bootstrap payload reads. yield* updates.configure; yield* DesktopRemoteUpdates.listen; yield* DesktopStartupRecovery.listen; + const backendStarted = yield* startLocalBackend.pipe( + Effect.catchCause((cause) => fatalStartupCause("bootstrap", cause)), + ); + yield* previewPasskeys.configure; + yield* applicationMenu.configure; // Portal identity is prepared pre-ready; MIME helpers must not gate the backend or window. yield* linuxUrlHandler.register.pipe(Effect.forkScoped); - yield* bootstrap.pipe(Effect.catchCause((cause) => fatalStartupCause("bootstrap", cause))); + yield* bootstrap(backendStarted).pipe( + Effect.catchCause((cause) => fatalStartupCause("bootstrap", cause)), + ); }).pipe(Effect.withSpan("desktop.startup")); const scopedProgram = Effect.scoped( diff --git a/apps/desktop/src/app/DesktopClerk.test.ts b/apps/desktop/src/app/DesktopClerk.test.ts index 8a823cde8db7..de687d41a4e6 100644 --- a/apps/desktop/src/app/DesktopClerk.test.ts +++ b/apps/desktop/src/app/DesktopClerk.test.ts @@ -424,7 +424,7 @@ it.effect.each(["startup", "open-url", "early-second-instance"] as const)( ), Effect.provideService(DesktopWebLinks.DesktopWebLinks, ignoreWebLinks), ); - }).pipe(Effect.scoped), + }), ); it.effect("hands web and app links to the renderer and leaves other links alone", () => @@ -481,5 +481,5 @@ it.effect("hands web and app links to the renderer and leaves other links alone" }), ), ); - }).pipe(Effect.scoped), + }), ); diff --git a/apps/desktop/src/app/DesktopCliCommand.test.ts b/apps/desktop/src/app/DesktopCliCommand.test.ts index 1655a8f0a8f1..5662378d754f 100644 --- a/apps/desktop/src/app/DesktopCliCommand.test.ts +++ b/apps/desktop/src/app/DesktopCliCommand.test.ts @@ -160,7 +160,7 @@ it.layer(NodeServices.layer)("DesktopCliCommand", (it) => { expect(yield* fs.exists(upgradedLink)).toBe(false); expect((yield* upgraded.install).installedPath).toBe(upgradedLink); } - }).pipe(Effect.scoped), + }), ); it.effect("never replaces or removes a t3 it did not create", () => @@ -202,7 +202,7 @@ it.layer(NodeServices.layer)("DesktopCliCommand", (it) => { expect((yield* Effect.flip(command.install)).message).toContain("Could not set up"); yield* command.uninstall; expect(yield* fs.readFileString(launcher)).toBe("npm's t3\n"); - }).pipe(Effect.scoped), + }), ); it.effect("finds and removes a link left by a previous T3 home", () => @@ -259,7 +259,7 @@ it.layer(NodeServices.layer)("DesktopCliCommand", (it) => { ); expect(yield* fs.exists(path.join(home, ".local", "bin", "t3"))).toBe(false); expect(yield* fs.readFileString(path.join(home, "bin", "t3.staging"))).toBe("mine"); - }).pipe(Effect.scoped), + }), ); it.effect("refuses to install behind another t3 that runs first", () => @@ -309,7 +309,7 @@ it.layer(NodeServices.layer)("DesktopCliCommand", (it) => { installedPath: path.join(home, ".local", "bin", "t3"), }); expect(installed.shadowedBy).toBeUndefined(); - }).pipe(Effect.scoped), + }), ); it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( @@ -343,7 +343,7 @@ it.layer(NodeServices.layer)("DesktopCliCommand", (it) => { onPath: true, installedPath: path.join(home, ".local", "bin", "t3"), }); - }).pipe(Effect.scoped), + }), ); it.effect("does not treat its own launcher as another t3 when its folder is first on PATH", () => @@ -372,7 +372,7 @@ it.layer(NodeServices.layer)("DesktopCliCommand", (it) => { expect(yield* command.state).toEqual(removed); expect(yield* fs.exists(installed.installedPath!)).toBe(false); expect(yield* fs.exists(launcher)).toBe(true); - }).pipe(Effect.scoped), + }), ); it.effect("leaves the Windows PATH alone when it cannot be read", () => @@ -414,7 +414,7 @@ it.layer(NodeServices.layer)("DesktopCliCommand", (it) => { const reader = yield* commandIn({ home, platform: "win32" }, inherited.spawner); yield* reader.install; expect(inherited.registry).toMatchObject({ path: `${userPath};${launcherDir}`, writes: 1 }); - }).pipe(Effect.scoped), + }), ); it.effect("does not claim or remove a Windows PATH entry the user added", () => @@ -444,6 +444,6 @@ it.layer(NodeServices.layer)("DesktopCliCommand", (it) => { yield* (yield* commandIn({ home: besideHome, platform: "win32" }, beside.spawner)) .installOnFirstLaunch; expect(beside.registry).toMatchObject({ path: "C:\\Tools", writes: 0 }); - }).pipe(Effect.scoped), + }), ); }); diff --git a/apps/desktop/src/app/DesktopCliShim.test.ts b/apps/desktop/src/app/DesktopCliShim.test.ts index e2f97c7d2ff7..3e8b4967dfde 100644 --- a/apps/desktop/src/app/DesktopCliShim.test.ts +++ b/apps/desktop/src/app/DesktopCliShim.test.ts @@ -39,7 +39,7 @@ const run = (shim: string, args: ReadonlyArray, env: NodeJS.ProcessEnv = }); const REPORTER = - '#!/bin/sh\necho "node=$ELECTRON_RUN_AS_NODE cli=$T3CODE_CLI_PATH home=$T3CODE_HOME"\nprintf "%s\\n" "$@"\nexit 3\n'; + '#!/bin/sh\necho "node=$ELECTRON_RUN_AS_NODE cli=$T3CODE_CLI_PATH home=$T3CODE_HOME appimage=$APPIMAGE"\nprintf "%s\\n" "$@"\nexit 3\n'; const fakeAppImage = (canMount = true, app = REPORTER) => { const root = tempRoot(); @@ -56,7 +56,7 @@ const fakeAppImage = (canMount = true, app = REPORTER) => { `echo $$ >'${root}/mounter.pid'`, canMount ? `echo '${image}'` : 'echo "Cannot mount AppImage"; exit 1', // Like the FUSE runtime, any of these unmounts. fd 3 reports to the test when it is open. - `trap 'echo unmounted >&3 2>/dev/null; touch "${root}/unmounted"; exit 0' TERM HUP INT`, + "trap 'echo unmounted >&3 2>/dev/null; exit 0' TERM HUP INT", "trap 'echo mounted >&3 2>/dev/null' WINCH", "while :; do sleep 0.05; done", "", @@ -150,6 +150,14 @@ describe("renderCliShim", () => { ); }); + // The server skips its compile cache only under APPIMAGE; each mount path is new. + it("tells the CLI it runs from the mounted AppImage", () => { + const { appImage, shim } = fakeAppImage(); + const result = run(shim, ["--version"]); + expect(result.status).toBe(3); + expect(result.stdout).toContain(`appimage=${appImage}`); + }); + it("keeps the install's T3 home under sudo, unless one is set", () => { const root = tempRoot(); writeExecutable(NodePath.join(root, "app"), REPORTER); diff --git a/apps/desktop/src/app/DesktopCliShim.ts b/apps/desktop/src/app/DesktopCliShim.ts index eba61f561f52..217f3b327dc9 100644 --- a/apps/desktop/src/app/DesktopCliShim.ts +++ b/apps/desktop/src/app/DesktopCliShim.ts @@ -109,6 +109,8 @@ export const renderCliShim = (input: { ' (cd "$work" && "$app" --appimage-extract >/dev/null) || { echo "Could not open $app." >&2; exit 1; }', ' mount="$work/squashfs-root"', "fi", + // The runtime sets this when it launches the app; the CLI skips its path-keyed compile cache on it. + 'export APPIMAGE="$app"', run("$mount"), "", ].join("\n"); diff --git a/apps/desktop/src/app/DesktopConnectionCatalogStore.test.ts b/apps/desktop/src/app/DesktopConnectionCatalogStore.test.ts index 98c75385afc5..b58b583fd6cd 100644 --- a/apps/desktop/src/app/DesktopConnectionCatalogStore.test.ts +++ b/apps/desktop/src/app/DesktopConnectionCatalogStore.test.ts @@ -99,7 +99,7 @@ const withStore = ( prefix: "t3-desktop-connection-catalog-test-", }); return yield* effect.pipe(Effect.provide(layerFor(baseDir, encryptionAvailable))); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped); + }).pipe(Effect.provide(NodeServices.layer)); describe("DesktopConnectionCatalogStore", () => { it.effect("persists and reads an encrypted connection catalog", () => @@ -298,6 +298,68 @@ describe("DesktopConnectionCatalogStore", () => { assert.notEqual(error.message, decryptError.message); yield* Ref.set(failDecrypt, false); assert.deepStrictEqual(yield* store.get, Option.some('{"schemaVersion":1,"targets":[]}')); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + const emptyCatalog = + '{"schemaVersion":1,"targets":[],"profiles":[],"credentials":[],"remoteDpopTokens":[],"disabledEnvironmentIds":[]}'; + const savedFilesHolding = Effect.fn(function* (stateDir: string, contents: string) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const kept: string[] = []; + for (const name of yield* fileSystem.readDirectory(stateDir)) { + const file = path.join(stateDir, name); + if ((yield* fileSystem.stat(file)).type !== "File") continue; + if ((yield* fileSystem.readFileString(file)) === contents) kept.push(name); + } + return kept; + }); + + it.effect("keeps a catalog this build cannot decode when the next save replaces it", () => + withStore( + Effect.gen(function* () { + const path = yield* Path.Path; + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const fileSystem = yield* FileSystem.FileSystem; + const store = yield* DesktopConnectionCatalogStore.DesktopConnectionCatalogStore; + const catalogPath = path.join(environment.stateDir, "connection-catalog.json"); + assert.isTrue( + yield* store.set( + '{"schemaVersion":1,"targets":[{"_tag":"FutureConnectionTarget","environmentId":"remote"}],"profiles":[],"credentials":[],"remoteDpopTokens":[],"disabledEnvironmentIds":[]}', + ), + ); + const original = yield* fileSystem.readFileString(catalogPath); + + yield* store.get; + assert.isTrue(yield* store.set(emptyCatalog)); + + assert.isNotEmpty(yield* savedFilesHolding(environment.stateDir, original)); + }), + ), + ); + + it.effect("keeps a catalog that no longer decrypts when the next save replaces it", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const fileSystem = yield* FileSystem.FileSystem; + const baseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-desktop-connection-catalog-test-", + }); + const failDecrypt = yield* Ref.make(false); + const store = yield* DesktopConnectionCatalogStore.DesktopConnectionCatalogStore.pipe( + Effect.provide(layerFor(baseDir, true, failDecrypt)), + ); + const stateDir = path.join(baseDir, "userdata"); + assert.isTrue(yield* store.set('{"schemaVersion":1,"targets":[]}')); + const original = yield* fileSystem.readFileString( + path.join(stateDir, "connection-catalog.json"), + ); + + yield* Ref.set(failDecrypt, true); + yield* store.get.pipe(Effect.flip); + assert.isTrue(yield* store.set(emptyCatalog)); + + assert.isNotEmpty(yield* savedFilesHolding(stateDir, original)); }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), ); }); diff --git a/apps/desktop/src/app/DesktopConnectionCatalogStore.ts b/apps/desktop/src/app/DesktopConnectionCatalogStore.ts index 3240df3636e3..5db49e5f9504 100644 --- a/apps/desktop/src/app/DesktopConnectionCatalogStore.ts +++ b/apps/desktop/src/app/DesktopConnectionCatalogStore.ts @@ -15,6 +15,7 @@ import { fromLenientJson } from "@t3tools/shared/schemaJson"; import * as Context from "effect/Context"; import * as SecureRandom from "effect/crypto/SecureRandom"; import * as Effect from "effect/Effect"; +import * as Clock from "effect/Clock"; import * as Base64 from "effect/encoding/Base64"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; @@ -45,6 +46,9 @@ const RuntimeConnectionCatalogDocumentJson = Schema.fromJsonString( const encodeRuntimeConnectionCatalogDocumentJson = Schema.encodeEffect( RuntimeConnectionCatalogDocumentJson, ); +const decodeRuntimeConnectionCatalogDocumentJson = Schema.decodeEffect( + RuntimeConnectionCatalogDocumentJson, +); const DesktopConnectionCatalogStoreWriteOperation = Schema.Literals([ "create-temporary-file-name", @@ -52,6 +56,7 @@ const DesktopConnectionCatalogStoreWriteOperation = Schema.Literals([ "create-directory", "write-temporary-file", "replace-catalog-file", + "keep-unreadable-catalog", ]); const DesktopConnectionCatalogStoreMigrationOperation = Schema.Literals([ @@ -395,6 +400,57 @@ export const make = Effect.gen(function* () { ), ); + const decryptDocument = (document: EncryptedConnectionCatalogDocument) => + decodeSecretBytes(catalogPath, document.encryptedCatalog).pipe( + Effect.flatMap((encryptedCatalog) => + safeStorage.decryptString(encryptedCatalog).pipe( + Effect.mapError( + (cause) => + new DesktopConnectionCatalogStoreProtectionError({ + operation: "decrypt-catalog", + catalogPath, + cause, + }), + ), + ), + ), + ); + + // A catalog this build cannot decrypt or decode may still hold another build's saved + // environments and credentials, so a save copies it aside instead of destroying it. + const keepUnreadableCatalog = Effect.gen(function* () { + const readable = yield* readDocument(fileSystem, catalogPath).pipe( + Effect.flatMap( + Option.match({ + onNone: () => Effect.succeed(true), + onSome: (document) => + decryptDocument(document).pipe( + Effect.flatMap(decodeRuntimeConnectionCatalogDocumentJson), + Effect.as(true), + ), + }), + ), + Effect.orElseSucceed(() => false), + ); + if (readable) { + return; + } + const keptPath = `${catalogPath}.unreadable-${yield* Clock.currentTimeMillis}`; + yield* fileSystem.copyFile(catalogPath, keptPath).pipe( + Effect.mapError( + (cause) => + new DesktopConnectionCatalogStoreWriteError({ + operation: "keep-unreadable-catalog", + path: keptPath, + cause, + }), + ), + ); + yield* Effect.logWarning("Kept an unreadable connection catalog before replacing it.", { + keptPath, + }); + }); + const writeCatalog = Effect.fn("desktop.connectionCatalogStore.writeCatalog")(function* ( catalog: string, ) { @@ -470,26 +526,13 @@ export const make = Effect.gen(function* () { if (!(yield* encryptionAvailable)) { return Option.none(); } - const decrypted = yield* decodeSecretBytes(catalogPath, document.value.encryptedCatalog).pipe( - Effect.flatMap((encryptedCatalog) => - safeStorage.decryptString(encryptedCatalog).pipe( - Effect.mapError( - (cause) => - new DesktopConnectionCatalogStoreProtectionError({ - operation: "decrypt-catalog", - catalogPath, - cause, - }), - ), - ), - ), - ); - return Option.some(decrypted); + return Option.some(yield* decryptDocument(document.value)); }).pipe(Effect.withSpan("desktop.connectionCatalogStore.get")), set: Effect.fn("desktop.connectionCatalogStore.set")(function* (catalog) { if (!(yield* encryptionAvailable)) { return false; } + yield* keepUnreadableCatalog; yield* writeCatalog(catalog); return true; }), diff --git a/apps/desktop/src/app/DesktopExitChildren.test.ts b/apps/desktop/src/app/DesktopExitChildren.test.ts new file mode 100644 index 000000000000..800a0f5bd4c0 --- /dev/null +++ b/apps/desktop/src/app/DesktopExitChildren.test.ts @@ -0,0 +1,62 @@ +// @effect-diagnostics nodeBuiltinImport:off - Spawns a real process tree for /proc. +import * as NodeChildProcess from "node:child_process"; +import * as NodeFS from "node:fs"; +import { assert, describe, it } from "@effect/vitest"; + +import { stopChildProcessesBeforeExit, stopZygotesBeforeExit } from "./DesktopExitChildren.ts"; + +const isRunning = (pid: number) => { + try { + const stat = NodeFS.readFileSync(`/proc/${pid}/stat`, "utf8"); + return !"ZX".includes(stat.charAt(stat.lastIndexOf(")") + 2)); + } catch { + return false; + } +}; + +describe.runIf(NodeFS.existsSync("/proc/self/task"))("stopChildProcessesBeforeExit", () => { + it("terminates every child of the given process before returning", async () => { + const parent = NodeChildProcess.spawn( + "sh", + ["-c", "sleep 60 & echo $!; sleep 60 & echo $!; wait"], + { stdio: ["ignore", "pipe", "ignore"] }, + ); + try { + const childPids = await new Promise>((resolve) => { + let output = ""; + parent.stdout.on("data", (chunk: Buffer) => { + output += chunk.toString(); + const pids = output.split("\n").filter(Boolean).map(Number); + if (pids.length === 2) resolve(pids); + }); + }); + assert.isTrue(childPids.every(isRunning)); + + assert.deepStrictEqual(stopChildProcessesBeforeExit(parent.pid!), []); + + assert.isFalse(childPids.some(isRunning)); + } finally { + parent.kill("SIGKILL"); + } + }); + + it("kills zygote children before returning and leaves other helpers running", async () => { + const spawnTitled = async (title: string) => { + const child = NodeChildProcess.spawn("sleep", ["60"], { argv0: title, stdio: "ignore" }); + await new Promise((resolve) => child.once("spawn", resolve)); + return child; + }; + // Chromium rewrites a zygote's argv into one space-separated title. + const zygote = await spawnTitled("/opt/T3 Code/t3code --type=zygote --no-sandbox"); + const utility = await spawnTitled("/opt/T3 Code/t3code --type=utility"); + try { + stopZygotesBeforeExit(process.pid); + + assert.isFalse(isRunning(zygote.pid!)); + assert.isTrue(isRunning(utility.pid!)); + } finally { + zygote.kill("SIGKILL"); + utility.kill("SIGKILL"); + } + }); +}); diff --git a/apps/desktop/src/app/DesktopExitChildren.ts b/apps/desktop/src/app/DesktopExitChildren.ts new file mode 100644 index 000000000000..9426c1b4ff88 --- /dev/null +++ b/apps/desktop/src/app/DesktopExitChildren.ts @@ -0,0 +1,87 @@ +// @effect-diagnostics nodeBuiltinImport:off - Runs synchronously in the process "exit" event, where nothing async completes. +import * as NodeFS from "node:fs"; + +// Matches the backend's quit budget: SIGTERM, 5 s grace, SIGKILL, 1 s wait. +const TERMINATE_GRACE_MS = 5_000; +const KILL_WAIT_MS = 1_000; + +const readProc = (path: string): string | undefined => { + try { + return NodeFS.readFileSync(path, "utf8"); + } catch { + return undefined; + } +}; + +// Chromium launches helpers from its own threads, so each thread's children file is read. +const childPids = (pid: number): ReadonlyArray => { + let tasks: ReadonlyArray; + try { + tasks = NodeFS.readdirSync(`/proc/${pid}/task`); + } catch { + return []; + } + return tasks.flatMap((task) => + (readProc(`/proc/${pid}/task/${task}/children`) ?? "") + .split(" ") + .filter((value) => value.trim() !== "") + .map(Number), + ); +}; + +// A zygote rewrites its argv into one space-separated title. +const isZygote = (pid: number): boolean => + (readProc(`/proc/${pid}/cmdline`) ?? "") + .replaceAll("\0", " ") + .split(" ") + .includes("--type=zygote"); + +// A zombie has already released its memory mappings. +const isRunning = (pid: number): boolean => { + const stat = readProc(`/proc/${pid}/stat`); + if (stat === undefined) return false; + const state = stat.charAt(stat.lastIndexOf(")") + 2); + return state !== "Z" && state !== "X"; +}; + +const signal = (pids: ReadonlyArray, name: NodeJS.Signals) => { + for (const pid of pids) { + try { + process.kill(pid, name); + } catch {} + } +}; + +const waitUntilStopped = (pids: ReadonlyArray, maxMs: number): ReadonlyArray => { + const tick = new Int32Array(new SharedArrayBuffer(4)); + let running = pids.filter(isRunning); + for (let waited = 0; waited < maxMs && running.length > 0; waited += 10) { + Atomics.wait(tick, 0, 0, 10); + running = running.filter(isRunning); + } + return running; +}; + +/** + * Stops the main process's children while it is still alive. On an AppImage the image unmounts + * as soon as main exits, so children left running then die with SIGBUS. Linux only (/proc). + * Returns the pids that needed SIGKILL. + */ +export const stopChildProcessesBeforeExit = (pid: number): ReadonlyArray => { + const children = childPids(pid); + signal(children, "SIGTERM"); + const survivors = waitUntilStopped(children, TERMINATE_GRACE_MS); + signal(survivors, "SIGKILL"); + waitUntilStopped(survivors, KILL_WAIT_MS); + return survivors; +}; + +/** + * Chromium's zygotes exit only after main does, so on an AppImage they outlive the mount even on + * an orderly quit. Kills them while main still holds the image mounted. Linux only (/proc). + */ +export const stopZygotesBeforeExit = (pid: number): void => { + const zygotes = childPids(pid).filter(isZygote); + signal(zygotes, "SIGKILL"); + waitUntilStopped(zygotes, KILL_WAIT_MS); +}; diff --git a/apps/desktop/src/app/DesktopLifecycle.test.ts b/apps/desktop/src/app/DesktopLifecycle.test.ts index 30df96c58ad7..e11e946dd41b 100644 --- a/apps/desktop/src/app/DesktopLifecycle.test.ts +++ b/apps/desktop/src/app/DesktopLifecycle.test.ts @@ -2,6 +2,7 @@ import { assert, describe, it } from "@effect/vitest"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Ref from "effect/Ref"; import * as TestClock from "effect/testing/TestClock"; @@ -117,6 +118,7 @@ describe("DesktopLifecycle", () => { const layerEnvironment = Layer.succeed(DesktopEnvironment.DesktopEnvironment, { platform, isDevelopment: false, + appImagePath: Option.none(), } as DesktopEnvironment.DesktopEnvironment["Service"]); const layer = DesktopLifecycle.layer.pipe( diff --git a/apps/desktop/src/app/DesktopLifecycle.ts b/apps/desktop/src/app/DesktopLifecycle.ts index 0b5039bd623d..9568e72dde31 100644 --- a/apps/desktop/src/app/DesktopLifecycle.ts +++ b/apps/desktop/src/app/DesktopLifecycle.ts @@ -3,6 +3,7 @@ import * as Context from "effect/Context"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; @@ -12,6 +13,7 @@ import type * as Electron from "electron"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as DesktopEnvironment from "./DesktopEnvironment.ts"; +import { stopChildProcessesBeforeExit, stopZygotesBeforeExit } from "./DesktopExitChildren.ts"; import { makeComponentLogger } from "./DesktopObservability.ts"; import * as DesktopShutdown from "./DesktopShutdown.ts"; import * as ElectronApp from "../electron/ElectronApp.ts"; @@ -69,15 +71,18 @@ function addScopedListener>( target: unknown, eventName: string, listener: (...args: Args) => void, + options: { readonly prepend?: boolean } = {}, ): Effect.Effect { const eventTarget = target as { on: (eventName: string, listener: (...args: Array) => void) => unknown; + prependListener: (eventName: string, listener: (...args: Array) => void) => unknown; removeListener: (eventName: string, listener: (...args: Array) => void) => unknown; }; const untypedListener = listener as unknown as (...args: Array) => void; return Effect.acquireRelease( Effect.sync(() => { - eventTarget.on(eventName, untypedListener); + if (options.prepend) eventTarget.prependListener(eventName, untypedListener); + else eventTarget.on(eventName, untypedListener); }), () => Effect.sync(() => { @@ -300,6 +305,34 @@ export const make = DesktopLifecycle.of({ ); }); + // Electron can exit without before-quit, e.g. when the X11/Wayland connection drops. + // Children (the backend too) get SIGTERM while main still holds an AppImage mount open. + // Prepended so the record is buffered before the trace sink's own exit flush runs. + yield* addScopedListener( + process, + "exit", + (exitCode: number) => { + Effect.runSyncWith(context)( + Effect.gen(function* () { + const shutdown = yield* DesktopShutdown.DesktopShutdown; + // "allowed" without a finished shutdown is the updater's quit; its relaunched app is a child. + if (quit.phase === "allowed" || (yield* shutdown.isComplete)) return; + const attributes = { exitCode, quitPhase: quit.phase }; + yield* Effect.gen(function* () { + yield* logLifecycleError( + "desktop main process exiting before shutdown completed", + attributes, + ); + if (environment.platform !== "linux") return; + const killed = stopChildProcessesBeforeExit(process.pid); + yield* Effect.annotateCurrentSpan({ killedChildPids: killed }); + }).pipe(Effect.withSpan("desktop.lifecycle.exit", { attributes })); + }), + ); + }, + { prepend: true }, + ); + if (environment.platform !== "win32") { yield* addScopedListener(process, "SIGINT", () => { quitFromSignal("SIGINT", runEffect, quit); @@ -308,6 +341,13 @@ export const make = DesktopLifecycle.of({ quitFromSignal("SIGTERM", runEffect, quit); }); } + + // Unscoped: the app scope closes before Electron exits, and every exit path must run this. + if (environment.platform === "linux" && Option.isSome(environment.appImagePath)) { + yield* Effect.sync(() => { + process.once("exit", () => stopZygotesBeforeExit(process.pid)); + }); + } }).pipe(Effect.withSpan("desktop.lifecycle.register")), }); diff --git a/apps/desktop/src/app/DesktopLinuxUrlHandler.test.ts b/apps/desktop/src/app/DesktopLinuxUrlHandler.test.ts index 7e685358865c..8e8a2812ab24 100644 --- a/apps/desktop/src/app/DesktopLinuxUrlHandler.test.ts +++ b/apps/desktop/src/app/DesktopLinuxUrlHandler.test.ts @@ -204,6 +204,35 @@ describe("DesktopLinuxUrlHandler", () => { assert.include(entry, "Icon=/home/al ice/icons/T3\\\\x.png"); }); + it.each([ + { from: "T3-Code-0.1.8-x86_64.AppImage", to: "T3-Code-x86_64.AppImage", restored: true }, + { + from: "T3-Code-0.1.9-nightly.20261110.4449-arm64.AppImage", + to: "T3-Code-Nightly-arm64.AppImage", + restored: true, + }, + { from: "T3-Code-0.1.8-x86_64.AppImage", to: "T3-Code-0.1.9-x86_64.AppImage", restored: false }, + { from: "t3code.AppImage", to: "T3-Code-x86_64.AppImage", restored: false }, + { from: "T3-Code-x86_64.AppImage", to: "T3-Code-x86_64.AppImage", restored: false }, + { from: "../T3-Code-0.1.8-x86_64.AppImage", to: "T3-Code-x86_64.AppImage", restored: false }, + ])("finds the launch path an update from $from to $to removed", ({ from, to, restored }) => { + for (const dir of ["/home/alice/Applications", '/home/al ice/My "100%" $Apps\\x']) { + const previous = dir + "/" + from; + const previousEntry = DesktopLinuxUrlHandler.renderUrlHandlerDesktopEntry({ + displayName: "T3 Code (Beta)", + execTarget: previous, + scheme: "t3code", + }); + assert.equal( + DesktopLinuxUrlHandler.replacedVersionedAppImagePath({ + previousEntry, + appImagePath: dir + "/" + to, + }), + restored ? previous : null, + ); + } + }); + it.effect( "writes the handler entry, refreshes the MIME cache, and claims the scheme default", () => { @@ -602,7 +631,7 @@ it.layer(NodeServices.layer)("DesktopLinuxUrlHandler theme icons", (it) => { } assert.isFalse(yield* fileSystem.exists(fixture.cachePath)); assert.equal(iconCacheRuns(run), 0); - }).pipe(Effect.scoped), + }), ); it.effect("replaces same-length stale icons and refreshes an existing cache once", () => @@ -634,7 +663,7 @@ it.layer(NodeServices.layer)("DesktopLinuxUrlHandler theme icons", (it) => { run.commands.filter(({ command }) => command === "gtk-update-icon-cache"), [{ command: "gtk-update-icon-cache", args: ["-f", "-t", fixture.hicolorDir] }], ); - }).pipe(Effect.scoped), + }), ); it.effect("refreshes when this launch changed an icon even if the cache looks newer", () => @@ -655,7 +684,7 @@ it.layer(NodeServices.layer)("DesktopLinuxUrlHandler theme icons", (it) => { assert.isTrue(yield* fileSystem.exists(path.join(fixture.appsDir, "com.t3tools.T3Code.png"))); assert.equal(iconCacheRuns(run), 1); - }).pipe(Effect.scoped), + }), ); it.effect("retries a failed icon cache refresh on the next launch", () => @@ -678,7 +707,7 @@ it.layer(NodeServices.layer)("DesktopLinuxUrlHandler theme icons", (it) => { yield* registerWithRealFileSystem(recovering); yield* registerWithRealFileSystem(recovering); assert.equal(iconCacheRuns(recovering), 1); - }).pipe(Effect.scoped), + }), ); it.effect("still registers the handler when the theme icon install fails", () => @@ -702,6 +731,6 @@ it.layer(NodeServices.layer)("DesktopLinuxUrlHandler theme icons", (it) => { missingTool.commands.map(({ command }) => command), ["gtk-update-icon-cache", "update-desktop-database", "xdg-mime"], ); - }).pipe(Effect.scoped), + }), ); }); diff --git a/apps/desktop/src/app/DesktopLinuxUrlHandler.ts b/apps/desktop/src/app/DesktopLinuxUrlHandler.ts index 64535c5b3ac1..35fd46dd7ae5 100644 --- a/apps/desktop/src/app/DesktopLinuxUrlHandler.ts +++ b/apps/desktop/src/app/DesktopLinuxUrlHandler.ts @@ -96,6 +96,38 @@ export function renderUrlHandlerDesktopEntry(input: { ].join("\n"); } +function parseUrlHandlerExecTarget(entry: string): string | null { + const exec = /^Exec=(.*) %U$/mu.exec(entry)?.[1]; + if (exec === undefined) return null; + const unescaped = exec.replace(/\\([\\nrt])/gu, (_, char: string) => + char === "n" ? "\n" : char === "r" ? "\r" : char === "t" ? "\t" : char, + ); + const quoted = /^"(.*)"$/su.exec(unescaped)?.[1]; + return ( + quoted === undefined ? unescaped : quoted.replace(/\\([\\`$"])/gu, (_, char: string) => char) + ).replaceAll("%%", "%"); +} + +const APPIMAGE_NAME_VERSION = /\d+\.\d+\.\d+/u; + +// An updater from before version-free AppImage names deletes its own file and installs the update +// beside it under the new name. The previous launch's entry still names the path launchers use. +export function replacedVersionedAppImagePath(input: { + readonly previousEntry: string; + readonly appImagePath: string; +}): string | null { + const previous = parseUrlHandlerExecTarget(input.previousEntry); + if (previous === null) return null; + const previousName = previous.slice(previous.lastIndexOf("/") + 1); + const currentName = input.appImagePath.slice(input.appImagePath.lastIndexOf("/") + 1); + return previous.startsWith("/") && + previous.slice(0, -previousName.length) === input.appImagePath.slice(0, -currentName.length) && + APPIMAGE_NAME_VERSION.test(previousName) && + !APPIMAGE_NAME_VERSION.test(currentName) + ? previous + : null; +} + // Docks may look up the Wayland app id without a desktop entry; icon theme lookups are case-sensitive. const LINUX_THEME_ICON_SIZE = "256x256"; diff --git a/apps/desktop/src/app/DesktopObservability.test.ts b/apps/desktop/src/app/DesktopObservability.test.ts index ab8ea0b8e9d6..ceb29b149e3d 100644 --- a/apps/desktop/src/app/DesktopObservability.test.ts +++ b/apps/desktop/src/app/DesktopObservability.test.ts @@ -281,6 +281,27 @@ describe("DesktopObservability", () => { ), ); + it.effect("startup prunes only canonical rotation suffixes", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const directory = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-desktop-rotating-prune-test-", + }); + const filePath = `${directory}/server-child.log`; + const retained = ["1", "3", "04", "4.0", "4e0", "0x4", "+4", "notes"]; + for (const suffix of [...retained, "4", "50"]) { + yield* fileSystem.writeFileString(`${filePath}.${suffix}`, suffix); + } + + yield* DesktopObservability.makeRotatingLogFileWriter({ filePath, maxFiles: 3 }); + + assert.deepEqual( + (yield* fileSystem.readDirectory(directory)).sort(), + retained.map((suffix) => `server-child.log.${suffix}`).sort(), + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + it.effect("stops every export when the OpenTelemetry SDK is disabled", () => { const requests: Array = []; return Effect.gen(function* () { diff --git a/apps/desktop/src/app/DesktopObservability.ts b/apps/desktop/src/app/DesktopObservability.ts index 3cd3936a59bf..16192fff46de 100644 --- a/apps/desktop/src/app/DesktopObservability.ts +++ b/apps/desktop/src/app/DesktopObservability.ts @@ -263,8 +263,10 @@ export const makeRotatingLogFileWriter = Effect.fn("makeRotatingLogFileWriter")( const entries = yield* fileSystem.readDirectory(directory).pipe(Effect.orElseSucceed(() => [])); for (const entry of entries) { if (!entry.startsWith(`${baseName}.`)) continue; - const suffix = Number(entry.slice(baseName.length + 1)); - if (!Number.isInteger(suffix) || suffix <= maxFiles) continue; + const suffixText = entry.slice(baseName.length + 1); + const suffix = Number(suffixText); + if (!Number.isInteger(suffix) || String(suffix) !== suffixText || suffix <= maxFiles) + continue; yield* fileSystem.remove(path.join(directory, entry), { force: true }).pipe(Effect.ignore); } }); @@ -654,6 +656,8 @@ const layerTelemetry = Layer.unwrap( maxFiles: DESKTOP_LOG_FILE_MAX_FILES, batchWindowMs: DESKTOP_TRACE_BATCH_WINDOW_MS, }); + // Unscoped: Electron can exit without closing the app scope, e.g. when the display drops. + process.once("exit", sink.flushSync); const delegate = endpoints.traces === undefined ? undefined diff --git a/apps/desktop/src/app/DesktopPreReadyPlatform.test.ts b/apps/desktop/src/app/DesktopPreReadyPlatform.test.ts index 34607e08b844..45887a928abb 100644 --- a/apps/desktop/src/app/DesktopPreReadyPlatform.test.ts +++ b/apps/desktop/src/app/DesktopPreReadyPlatform.test.ts @@ -16,6 +16,7 @@ const { copyFileSyncMock, execFileSyncMock, readFileSyncMock, + symlinkSyncMock, } = vi.hoisted(() => ({ appendSwitchMock: vi.fn(), getSwitchValueMock: vi.fn(), @@ -27,6 +28,7 @@ const { copyFileSyncMock: vi.fn(), execFileSyncMock: vi.fn(), readFileSyncMock: vi.fn(), + symlinkSyncMock: vi.fn(), })); vi.mock("electron", () => ({ @@ -51,6 +53,7 @@ vi.mock("node:fs", () => ({ mkdirSync: mkdirSyncMock, writeFileSync: writeFileSyncMock, copyFileSync: copyFileSyncMock, + symlinkSync: symlinkSyncMock, })); vi.mock("node:child_process", () => ({ @@ -72,6 +75,7 @@ describe("DesktopPreReadyPlatform", () => { copyFileSyncMock.mockReset(); execFileSyncMock.mockReset(); readFileSyncMock.mockReset().mockReturnValue("{}"); + symlinkSyncMock.mockReset(); }); it.effect("persists Linux scaling synchronously across relaunches and settings rewrites", () => { @@ -181,6 +185,47 @@ describe("DesktopPreReadyPlatform", () => { ).pipe(Effect.ensuring(Effect.sync(() => vi.unstubAllEnvs()))); }); + it.effect.each([ + { relaunchedByUpdater: "true", restored: true }, + { relaunchedByUpdater: "", restored: false }, + ])( + "restores the versioned AppImage path an update removed (updater relaunch: $relaunchedByUpdater)", + ({ relaunchedByUpdater, restored }) => { + vi.stubEnv("VITE_DEV_SERVER_URL", ""); + getSwitchValueMock.mockReturnValue(""); + const entryPath = "/xdg/applications/com.t3tools.T3Code.desktop"; + const files = new Map([[entryPath, "Exec=/Applications/T3-Code-0.1.8-x86_64.AppImage %U\n"]]); + readFileSyncMock.mockImplementation((path: string) => files.get(path) ?? "{}"); + writeFileSyncMock.mockImplementation((path: string, contents: string | Buffer) => { + files.set(path, String(contents)); + }); + + return Effect.scoped( + Effect.gen(function* () { + yield* Layer.build( + DesktopPreReadyPlatform.layer.pipe( + Layer.provide(Layer.succeed(HostProcess.Platform, "linux")), + Layer.provide( + DesktopConfig.layerTest({ + XDG_DATA_HOME: "/xdg", + APPIMAGE: "/Applications/T3-Code-x86_64.AppImage", + APPIMAGE_SILENT_INSTALL: relaunchedByUpdater, + }), + ), + ), + ); + assert.deepEqual( + symlinkSyncMock.mock.calls, + restored + ? [["T3-Code-x86_64.AppImage", "/Applications/T3-Code-0.1.8-x86_64.AppImage"]] + : [], + ); + assert.include(files.get(entryPath), "Exec=/Applications/T3-Code-x86_64.AppImage %U"); + }), + ).pipe(Effect.ensuring(Effect.sync(() => vi.unstubAllEnvs()))); + }, + ); + it.effect("points the early portal entry at a configured distribution launcher", () => { vi.stubEnv("VITE_DEV_SERVER_URL", ""); getSwitchValueMock.mockReturnValue(""); diff --git a/apps/desktop/src/app/DesktopPreReadyPlatform.ts b/apps/desktop/src/app/DesktopPreReadyPlatform.ts index df8059133fc2..8ac4654636ab 100644 --- a/apps/desktop/src/app/DesktopPreReadyPlatform.ts +++ b/apps/desktop/src/app/DesktopPreReadyPlatform.ts @@ -14,7 +14,10 @@ import * as HostProcess from "@t3tools/shared/HostProcess"; import * as DesktopEarlyElectronStartup from "./DesktopEarlyElectronStartup.ts"; import { trimmedString } from "./DesktopConfig.ts"; import { resolveDesktopAppBranding } from "./DesktopEnvironment.ts"; -import { renderUrlHandlerDesktopEntry } from "./DesktopLinuxUrlHandler.ts"; +import { + renderUrlHandlerDesktopEntry, + replacedVersionedAppImagePath, +} from "./DesktopLinuxUrlHandler.ts"; import * as ElectronProtocol from "../electron/ElectronProtocol.ts"; export interface DesktopPreReadyCommandLineReader { @@ -56,6 +59,8 @@ const LinuxDesktopEntryConfig = Config.all({ xdgDataHome: trimmedString("XDG_DATA_HOME"), launcherPath: trimmedString("T3CODE_DESKTOP_LAUNCHER_PATH"), appImagePath: trimmedString("APPIMAGE"), + // electron-updater sets this on the AppImage it relaunches after installing an update. + relaunchedByUpdater: trimmedString("APPIMAGE_SILENT_INSTALL"), }); /** @public */ @@ -116,8 +121,26 @@ export const make = Effect.gen(function* () { ); } catch {} } + const desktopEntryPath = NodePath.posix.join(applicationsDir, linux.linuxDesktopEntryName); + const { appImagePath } = linuxDesktopEntryConfig; + if ( + Option.isSome(appImagePath) && + Option.isNone(linuxDesktopEntryConfig.launcherPath) && + Option.contains(linuxDesktopEntryConfig.relaunchedByUpdater, "true") + ) { + try { + const replacedPath = replacedVersionedAppImagePath({ + previousEntry: NodeFS.readFileSync(desktopEntryPath, "utf8"), + appImagePath: appImagePath.value, + }); + // Relative, so the pair survives a moved folder; symlink never overwrites a file. + if (replacedPath !== null) { + NodeFS.symlinkSync(NodePath.posix.basename(appImagePath.value), replacedPath); + } + } catch {} + } NodeFS.writeFileSync( - NodePath.posix.join(applicationsDir, linux.linuxDesktopEntryName), + desktopEntryPath, renderUrlHandlerDesktopEntry({ displayName: resolveDesktopAppBranding({ isDevelopment: linux.isDevelopment, diff --git a/apps/desktop/src/app/DesktopStartupRecovery.ts b/apps/desktop/src/app/DesktopStartupRecovery.ts index 625c5d9b7470..1af40ea15a6e 100644 --- a/apps/desktop/src/app/DesktopStartupRecovery.ts +++ b/apps/desktop/src/app/DesktopStartupRecovery.ts @@ -217,7 +217,6 @@ export const listen: Effect.Effect< const download = yield* updates.download; if (!download.completed) return; } - // Waits for a running background check; plain install would refuse. yield* updates.installPrepared(current.version); return; } diff --git a/apps/desktop/src/app/stdioErrors.test.ts b/apps/desktop/src/app/stdioErrors.test.ts new file mode 100644 index 000000000000..ccfa3ed95117 --- /dev/null +++ b/apps/desktop/src/app/stdioErrors.test.ts @@ -0,0 +1,22 @@ +import * as NodeOS from "node:os"; +import { assert, it } from "vite-plus/test"; + +import { handleStdioError } from "./stdioErrors.ts"; + +const errorWithCode = (code: string, errno?: number) => + Object.assign(new Error(code), { code, errno }); +const EDQUOT_ERRNO = NodeOS.constants.errno.EDQUOT; + +it.each(["EPIPE", "ENOSPC"])("ignores %s on console output", (code) => { + assert.doesNotThrow(() => handleStdioError(errorWithCode(code))); +}); + +// Node reports EDQUOT as UNKNOWN because libuv has no name for it. +it.skipIf(EDQUOT_ERRNO === undefined)("ignores EDQUOT on console output", () => { + assert.doesNotThrow(() => handleStdioError(errorWithCode("UNKNOWN", -(EDQUOT_ERRNO ?? 0)))); +}); + +it("rethrows other console output errors", () => { + assert.throws(() => handleStdioError(errorWithCode("EIO")), "EIO"); + assert.throws(() => handleStdioError(errorWithCode("UNKNOWN", -5)), "UNKNOWN"); +}); diff --git a/apps/desktop/src/app/stdioErrors.ts b/apps/desktop/src/app/stdioErrors.ts new file mode 100644 index 000000000000..b23c0e89e79a --- /dev/null +++ b/apps/desktop/src/app/stdioErrors.ts @@ -0,0 +1,13 @@ +import * as NodeOS from "node:os"; + +// libuv has no name for EDQUOT, so Node reports it as code "UNKNOWN" with the raw errno. +const EDQUOT_ERRNO = NodeOS.constants.errno.EDQUOT; + +// Redirected console output can fail when its log volume is full. +export function handleStdioError(error: unknown): void { + if (typeof error === "object" && error !== null) { + if ("code" in error && (error.code === "EPIPE" || error.code === "ENOSPC")) return; + if (EDQUOT_ERRNO !== undefined && "errno" in error && error.errno === -EDQUOT_ERRNO) return; + } + throw error; +} diff --git a/apps/desktop/src/backend/DesktopBackendConfiguration.test.ts b/apps/desktop/src/backend/DesktopBackendConfiguration.test.ts index b047b6cabf17..ba1f463f21c4 100644 --- a/apps/desktop/src/backend/DesktopBackendConfiguration.test.ts +++ b/apps/desktop/src/backend/DesktopBackendConfiguration.test.ts @@ -373,7 +373,7 @@ describe("DesktopBackendConfiguration", () => { ); assert.equal(config.runningDistro, "Ubuntu"); - assert.deepEqual(config.args.slice(0, 2), ["-d", "Ubuntu"]); + assert.deepEqual(config.args.slice(0, 5), ["-d", "Ubuntu", "--cd", "~", "--exec"]); assert.deepEqual(observedDistros, ["Ubuntu", "Ubuntu", "Ubuntu"]); assert.isTrue(Option.isNone(config.preflightFailure)); }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), @@ -385,7 +385,7 @@ describe("DesktopBackendConfiguration", () => { runtimeId: string; sha256: string; }> = []; - const observedProbeRoots: string[] = []; + let probeCount = 0; let legacyCleanupCount = 0; const linuxAppRoot = "/home/test/.t3/wsl-runtime/1.2.3-x64"; const resolvedPath = "/home/test/.local/bin:/usr/bin:/bin"; @@ -406,8 +406,8 @@ describe("DesktopBackendConfiguration", () => { }); return { ok: true, linuxAppRoot }; }, - probeRuntime: (_distro, root) => { - observedProbeRoots.push(root); + probeRuntime: () => { + probeCount += 1; return { ok: true, resolvedPath }; }, ensureNodePty: () => { @@ -428,7 +428,7 @@ describe("DesktopBackendConfiguration", () => { sha256: archiveHash, }, ]); - assert.deepEqual(observedProbeRoots, [linuxAppRoot]); + assert.equal(probeCount, 1); assert.equal( config.entryPath, path.join(baseDir, "server.asar/apps/server/dist/bin.mjs"), @@ -436,6 +436,8 @@ describe("DesktopBackendConfiguration", () => { assert.deepEqual(config.args, [ "-d", "Ubuntu", + "--cd", + "~", "--exec", "env", `PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${resolvedPath}`, @@ -521,10 +523,10 @@ describe("DesktopBackendConfiguration", () => { ); }); - it.effect("resolveWsl retires a staged runtime whose executable does not start", () => { + it.effect("resolveWsl retires a staged runtime whose launch probe fails", () => { const archiveHash = "c".repeat(64); const stagedAppRoot = `/home/test/.t3/wsl-runtime/sha256-${archiveHash}`; - const observedProbeRoots: string[] = []; + let probeCount = 0; const observedNodePtyRoots: string[] = []; const invalidatedRuntimeIds: string[] = []; return withPackagedWslHarness( @@ -536,9 +538,9 @@ describe("DesktopBackendConfiguration", () => { Effect.sync(() => { invalidatedRuntimeIds.push(runtimeId); }), - probeRuntime: (_distro, root) => { - observedProbeRoots.push(root); - return { ok: false, reason: `${root}/t3 --version failed (exit 127)` }; + probeRuntime: () => { + probeCount += 1; + return { ok: false, reason: "WSL login-shell PATH could not be resolved" }; }, ensureNodePty: (_distro, root) => { observedNodePtyRoots.push(root); @@ -551,7 +553,7 @@ describe("DesktopBackendConfiguration", () => { const configuration = yield* DesktopBackendConfiguration.DesktopBackendConfiguration; const config = yield* configuration.resolveWsl({ port: 5000, distro: "Ubuntu" }); - assert.deepEqual(observedProbeRoots, [stagedAppRoot]); + assert.equal(probeCount, 1); assert.deepEqual(observedNodePtyRoots, [mountedAppRoot]); assert.include(config.args, "/usr/bin/node"); assert.include(config.args, `${mountedAppRoot}/apps/server/dist/bin.mjs`); @@ -576,7 +578,7 @@ describe("DesktopBackendConfiguration", () => { Effect.sync(() => { invalidatedRuntimeIds.push(runtimeId); }), - probeRuntime: () => ({ ok: false, reason: "t3 --version failed (exit 1)" }), + probeRuntime: () => ({ ok: false, reason: "WSL login-shell PATH could not be resolved" }), ensureNodePty: () => ({ ok: false, reason: "WSL backend preflight timed out while probing for Node.js.", @@ -655,6 +657,8 @@ describe("DesktopBackendConfiguration", () => { "Ubuntu", "--user", "alice", + "--cd", + "~", "--exec", "env", "PATH=/home/test user's/.nvm/versions/node/v22.0.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/test user/bin:/opt/test's tools/bin:/usr/bin:/bin", @@ -687,6 +691,43 @@ describe("DesktopBackendConfiguration", () => { ), ); + it.effect("gives the Windows and WSL backends one host scope that survives a restart", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const baseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-desktop-backend-config-test-", + }); + const launch = Effect.gen(function* () { + const configuration = yield* DesktopBackendConfiguration.DesktopBackendConfiguration; + const primary = yield* configuration.resolvePrimary; + const wsl = yield* configuration.resolveWsl({ port: 5050, distro: null }); + assert.equal(wsl.env.T3CODE_DESKTOP_HOST_SCOPE, primary.env.T3CODE_DESKTOP_HOST_SCOPE); + assert.include((wsl.env.WSLENV ?? "").split(":"), "T3CODE_DESKTOP_HOST_SCOPE"); + return primary.env.T3CODE_DESKTOP_HOST_SCOPE; + }).pipe( + Effect.provide( + layerBackendConfiguration().pipe( + Layer.provideMerge(layerServerExposure), + Layer.provideMerge(DesktopAppSettings.layerTest()), + Layer.provideMerge(DesktopWslServerTree.layerTest()), + Layer.provideMerge( + DesktopWslEnvironment.layerTest({ + isAvailable: true, + windowsToWslPath: () => Option.some("/mnt/c/repo/apps/server/src/index.ts"), + getDistroIp: () => Option.some("172.27.0.99"), + }), + ), + Layer.provideMerge(layerEnvironment(baseDir, { platform: "win32" })), + ), + ), + ); + + const first = yield* launch; + assert.match(first ?? "", /^[0-9a-f]{32}$/); + assert.equal(yield* launch, first); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), + ); + it.effect("resolveWsl carries the kill switch and resource attributes into the distro", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; @@ -752,7 +793,7 @@ describe("DesktopBackendConfiguration", () => { assert.equal(config.env.T3CODE_OTLP_PROTOCOL, "http/protobuf"); assert.equal( config.env.WSLENV, - "GOPATH/p:OPENAI_API_KEY/u:EMPTY::AZURE_DEVOPS_EXT_PAT/u:ANTHROPIC_API_KEY:T3CODE_OTLP_HEADERS:T3CODE_OTLP_PROTOCOL", + "GOPATH/p:OPENAI_API_KEY/u:EMPTY::AZURE_DEVOPS_EXT_PAT/u:T3CODE_DESKTOP_HOST_SCOPE:ANTHROPIC_API_KEY:T3CODE_OTLP_HEADERS:T3CODE_OTLP_PROTOCOL", ); }).pipe( Effect.provide( diff --git a/apps/desktop/src/backend/DesktopBackendConfiguration.ts b/apps/desktop/src/backend/DesktopBackendConfiguration.ts index 040e765446a1..dbd5bfae11b3 100644 --- a/apps/desktop/src/backend/DesktopBackendConfiguration.ts +++ b/apps/desktop/src/backend/DesktopBackendConfiguration.ts @@ -243,8 +243,38 @@ interface SharedBootstrapInput { readonly bootstrapToken: string; readonly bootstrapSecret: string; readonly observabilitySettings: BackendObservabilitySettings; + readonly hostScope: string; } +// Matches the server's DESKTOP_HOST_SCOPE_ENV. +const DESKTOP_HOST_SCOPE_ENV = "T3CODE_DESKTOP_HOST_SCOPE"; +const HOST_SCOPE_FILE_NAME = "desktop-host-scope"; +const HOST_SCOPE_PATTERN = /^[0-9a-f]{32}$/; + +/** + * Not a credential: it only tells this install's Windows and WSL servers that + * they run on one machine, so usage both read from one Windows directory counts + * once. Persisted so summaries from earlier launches still match. + */ +const readOrCreateHostScope = Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const random = yield* SecureRandom.SecureRandom; + const scopePath = environment.path.join(environment.stateDir, HOST_SCOPE_FILE_NAME); + const existing = (yield* fileSystem + .readFileString(scopePath) + .pipe(Effect.orElseSucceed(() => ""))).trim(); + if (HOST_SCOPE_PATTERN.test(existing)) return existing; + const scope = Hex.encode(yield* random.randomBytes(16).pipe(Effect.orDie)); + yield* fileSystem.makeDirectory(environment.stateDir, { recursive: true }).pipe( + Effect.andThen(fileSystem.writeFileString(scopePath, scope)), + Effect.catch((cause) => + Effect.logWarning("Could not persist the desktop host scope.", { cause }), + ), + ); + return scope; +}); + type WslPreflightRuntime = | { readonly kind: "executable"; @@ -395,7 +425,7 @@ const runWslPreflight = Effect.fn("desktop.backendConfiguration.wslPreflight")(f const runtime = yield* wslEnv.prepareRuntime(runningDistro, input.runtimeArchive); if (runtime.ok) { // Staged runtimes bundle Node and node-pty; provider PATH discovery must not require them. - const stagedProbe = yield* wslEnv.probeRuntime(runningDistro, runtime.linuxAppRoot); + const stagedProbe = yield* wslEnv.probeRuntime(runningDistro); if (stagedProbe.ok) { yield* wslServerTree.cleanupLegacy; return { @@ -409,7 +439,7 @@ const runWslPreflight = Effect.fn("desktop.backendConfiguration.wslPreflight")(f } as const; } yield* Effect.logWarning( - "The staged WSL runtime did not start; retrying from the mounted server tree.", + "The staged WSL runtime probe failed; retrying from the mounted server tree.", { reason: stagedProbe.reason }, ); stagedFailure = { runtimeId: input.runtimeArchive.runtimeId, reason: stagedProbe.reason }; @@ -543,6 +573,7 @@ const resolvePrimaryStartConfig = Effect.fn("desktop.backendConfiguration.resolv ELECTRON_RUN_AS_NODE: "1", T3CODE_DESKTOP_SHELL_DISCOVERY_PENDING: "1", T3CODE_CLI_PATH: Option.getOrUndefined(input.cliPath), + [DESKTOP_HOST_SCOPE_ENV]: input.hostScope, }, extendEnv: true, bootstrap, @@ -643,8 +674,8 @@ const resolveWslStartConfig = Effect.fn("desktop.backendConfiguration.resolveWsl ...(distroForConfig ? ["-d", distroForConfig] : []), ...(runningUser ? ["--user", runningUser] : []), ]; - const forwardedEnv: Record = {}; - const forwardedEnvNames: string[] = []; + const forwardedEnv: Record = { [DESKTOP_HOST_SCOPE_ENV]: input.hostScope }; + const forwardedEnvNames: string[] = [DESKTOP_HOST_SCOPE_ENV]; for (const name of WSL_FORWARDED_ENV_NAMES) { const value = env[name]; if (value !== undefined && value.length > 0) { @@ -719,6 +750,9 @@ const resolveWslStartConfig = Effect.fn("desktop.backendConfiguration.resolveWsl : {}), args: [ ...distroArgs, + // WSL otherwise inherits the Windows-mounted desktop working directory. + "--cd", + "~", "--exec", "env", `PATH=${launchPath}`, @@ -746,6 +780,13 @@ export const make = Effect.gen(function* () { const random = yield* SecureRandom.SecureRandom; // Random generation yields; serialize it so concurrent backends cannot capture distinct bootstrap secrets. const secretRef = yield* SynchronizedRef.make(Option.none()); + const hostScope = yield* Effect.cached( + readOrCreateHostScope.pipe( + Effect.provideService(FileSystem.FileSystem, fileSystem), + Effect.provideService(DesktopEnvironment.DesktopEnvironment, environment), + Effect.provideService(SecureRandom.SecureRandom, random), + ), + ); const getOrCreateBootstrapSecret = SynchronizedRef.modifyEffect(secretRef, (current) => Option.match(current, { onSome: (secret) => Effect.succeed([secret, current] as const), @@ -773,6 +814,7 @@ export const make = Effect.gen(function* () { bootstrapToken, bootstrapSecret, observabilitySettings, + hostScope: yield* hostScope, } satisfies SharedBootstrapInput; }); diff --git a/apps/desktop/src/backend/DesktopBackendManager.test.ts b/apps/desktop/src/backend/DesktopBackendManager.test.ts index 3126d12a2e93..ff5b7b8c08e4 100644 --- a/apps/desktop/src/backend/DesktopBackendManager.test.ts +++ b/apps/desktop/src/backend/DesktopBackendManager.test.ts @@ -5,7 +5,13 @@ import { type DesktopBackendBootstrap as DesktopBackendBootstrapValue, DesktopTelemetryControlMessage, } from "@t3tools/contracts"; +import { + DATABASE_MIGRATION_PROGRESS_MESSAGE, + DATABASE_MIGRATIONS_FINISHED_MESSAGE, + DATABASE_MIGRATIONS_STARTED_MESSAGE, +} from "@t3tools/shared/databaseMigrationLog"; import { assert, describe, it } from "@effect/vitest"; +import * as Clock from "effect/Clock"; import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; @@ -194,129 +200,162 @@ function makeTestInstance(input: MakeInstanceInput) { describe("DesktopBackendManager", () => { it.effect("notifies readiness waiters on HTTP readiness and stop without clock advancement", () => - Effect.scoped( - Effect.gen(function* () { - const probe = yield* Deferred.make(); - const ready = yield* Deferred.make(); - const exit = yield* Deferred.make(); - const instance = yield* makeTestInstance({ - spawnerLayer: Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => - Effect.acquireRelease( - Effect.succeed( - makeProcess({ - exitCode: Deferred.await(exit), - kill: () => - Deferred.succeed(exit, ChildProcessSpawner.ExitCode(0)).pipe(Effect.asVoid), - }), - ), - () => Deferred.succeed(exit, ChildProcessSpawner.ExitCode(0)), + Effect.gen(function* () { + const probe = yield* Deferred.make(); + const ready = yield* Deferred.make(); + const exit = yield* Deferred.make(); + const instance = yield* makeTestInstance({ + spawnerLayer: Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.acquireRelease( + Effect.succeed( + makeProcess({ + exitCode: Deferred.await(exit), + kill: () => + Deferred.succeed(exit, ChildProcessSpawner.ExitCode(0)).pipe(Effect.asVoid), + }), ), + () => Deferred.succeed(exit, ChildProcessSpawner.ExitCode(0)), ), ), - httpClientLayer: Layer.succeed( - HttpClient.HttpClient, - HttpClient.make((request) => - Deferred.succeed(probe, undefined).pipe( - Effect.andThen(Deferred.await(ready)), - Effect.as(HttpClientResponse.fromWeb(request, new Response("{}"))), - ), + ), + httpClientLayer: Layer.succeed( + HttpClient.HttpClient, + HttpClient.make((request) => + Deferred.succeed(probe, undefined).pipe( + Effect.andThen(Deferred.await(ready)), + Effect.as(HttpClientResponse.fromWeb(request, new Response("{}"))), ), ), - }); - yield* instance.start; - yield* Deferred.await(probe); - const waiter = yield* instance.waitForReady(Duration.seconds(15)).pipe(Effect.forkChild); - yield* Deferred.succeed(ready, undefined); - assert.isTrue(yield* Fiber.join(waiter)); - yield* instance.stop(); - assert.isFalse(yield* instance.waitForReady(Duration.seconds(15))); - }), - ), + ), + }); + yield* instance.start; + yield* Deferred.await(probe); + const waiter = yield* instance.waitForReady(Duration.seconds(15)).pipe(Effect.forkChild); + yield* Deferred.succeed(ready, undefined); + assert.isTrue(yield* Fiber.join(waiter)); + yield* instance.stop(); + assert.isFalse(yield* instance.waitForReady(Duration.seconds(15))); + }), ); - it.effect("spawns the backend with fd3 bootstrap and fd4 telemetry", () => - Effect.scoped( - Effect.gen(function* () { - let spawnedCommand: ChildProcess.Command | undefined; - let bootstrapJson = ""; - let telemetryJson = ""; - let readyCount = 0; - const ready = yield* Deferred.make(); - const exited = yield* Queue.unbounded(); - - const layerSpawner = Layer.succeed( + it.effect("notices a WSL backend soon after it listens despite slow refused connects", () => + Effect.gen(function* () { + const ready = yield* Deferred.make(); + const exit = yield* Deferred.make(); + const instance = yield* makeTestInstance({ + config: { ...baseConfig, httpBaseUrl: new URL("http://172.20.84.98:3773") }, + onReady: Deferred.succeed(ready, undefined).pipe(Effect.asVoid), + spawnerLayer: Layer.succeed( ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make((command) => - Effect.gen(function* () { - spawnedCommand = command; - if (command._tag === "StandardCommand") { - const fd3 = command.options.additionalFds?.fd3; - if (fd3?.type === "input" && fd3.stream) { - bootstrapJson = yield* fd3.stream.pipe(Stream.decodeText(), Stream.mkString); - } - const fd4 = command.options.additionalFds?.fd4; - if (fd4?.type === "input" && fd4.stream) { - telemetryJson = yield* fd4.stream.pipe(Stream.decodeText(), Stream.mkString); - } + ChildProcessSpawner.make(() => + Effect.acquireRelease( + Effect.succeed(makeProcess({ exitCode: Deferred.await(exit) })), + () => Deferred.succeed(exit, ChildProcessSpawner.ExitCode(0)), + ), + ), + ), + // Before the server listens at 300 ms, each refused connect takes a second to fail. + httpClientLayer: layerHttpClient((request) => + Clock.currentTimeMillis.pipe( + Effect.flatMap((now) => + now < 300 + ? Effect.sleep(Duration.seconds(1)).pipe( + Effect.as(responseForRequest(request, 503)), + ) + : Effect.succeed(responseForRequest(request, 200)), + ), + ), + ), + }); + yield* instance.start; + yield* TestClock.adjust(Duration.millis(600)); + + assert.isTrue(yield* Deferred.isDone(ready)); + yield* instance.stop(); + }), + ); + + it.effect("spawns the backend with fd3 bootstrap and fd4 telemetry", () => + Effect.gen(function* () { + let spawnedCommand: ChildProcess.Command | undefined; + let bootstrapJson = ""; + let telemetryJson = ""; + let readyCount = 0; + const ready = yield* Deferred.make(); + const exited = yield* Queue.unbounded(); + + const layerSpawner = Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make((command) => + Effect.gen(function* () { + spawnedCommand = command; + if (command._tag === "StandardCommand") { + const fd3 = command.options.additionalFds?.fd3; + if (fd3?.type === "input" && fd3.stream) { + bootstrapJson = yield* fd3.stream.pipe(Stream.decodeText(), Stream.mkString); + } + const fd4 = command.options.additionalFds?.fd4; + if (fd4?.type === "input" && fd4.stream) { + telemetryJson = yield* fd4.stream.pipe(Stream.decodeText(), Stream.mkString); } + } - return makeProcess({ - exitCode: Deferred.await(ready).pipe(Effect.as(ChildProcessSpawner.ExitCode(0))), - }); - }), - ), - ); + return makeProcess({ + exitCode: Deferred.await(ready).pipe(Effect.as(ChildProcessSpawner.ExitCode(0))), + }); + }), + ), + ); - const instance = yield* makeTestInstance({ - config: { - ...baseConfig, - bootstrap: configWithObservability, - }, - spawnerLayer: layerSpawner, - desktopTelemetryStream: Stream.encodeText( - Stream.make('{"version":1,"type":"desktopTelemetryHello","electronPid":123}\n'), - ), - onReady: Effect.sync(() => { - readyCount += 1; - }).pipe(Effect.andThen(Deferred.succeed(ready, void 0)), Effect.asVoid), - backendOutputLog: { - persistFailure: () => Queue.offer(exited, void 0).pipe(Effect.asVoid), - }, - }); + const instance = yield* makeTestInstance({ + config: { + ...baseConfig, + bootstrap: configWithObservability, + }, + spawnerLayer: layerSpawner, + desktopTelemetryStream: Stream.encodeText( + Stream.make('{"version":1,"type":"desktopTelemetryHello","electronPid":123}\n'), + ), + onReady: Effect.sync(() => { + readyCount += 1; + }).pipe(Effect.andThen(Deferred.succeed(ready, void 0)), Effect.asVoid), + backendOutputLog: { + persistFailure: () => Queue.offer(exited, void 0).pipe(Effect.asVoid), + }, + }); - yield* instance.start; - yield* Queue.take(exited); - - assert.equal(readyCount, 1); - assert.isDefined(spawnedCommand); - if (spawnedCommand._tag !== "StandardCommand") { - throw new Error("Expected backend to spawn a standard command."); - } - - assert.equal(spawnedCommand.command, "/electron"); - assert.deepEqual(spawnedCommand.args, ["/server/bin.mjs", "--bootstrap-fd", "3"]); - assert.equal(spawnedCommand.options.cwd, "/server"); - assert.equal(spawnedCommand.options.extendEnv, true); - assert.equal(spawnedCommand.options.stdout, "pipe"); - assert.equal(spawnedCommand.options.stderr, "pipe"); - assert.equal(spawnedCommand.options.killSignal, "SIGTERM"); - assert.isDefined(spawnedCommand.options.forceKillAfter); - assert.equal(spawnedCommand.options.additionalFds?.fd4?.type, "input"); - assert.equal(spawnedCommand.options.additionalFds?.fd5?.type, "output"); - assert.equal( - Duration.toMillis(Duration.fromInputUnsafe(spawnedCommand.options.forceKillAfter)), - 5_000, - ); - - assert.deepEqual(yield* decodeBootstrap(bootstrapJson), configWithObservability); - assert.equal( - telemetryJson, - '{"version":1,"type":"desktopTelemetryHello","electronPid":123}\n', - ); - }), - ), + yield* instance.start; + yield* Queue.take(exited); + + assert.equal(readyCount, 1); + assert.isDefined(spawnedCommand); + if (spawnedCommand._tag !== "StandardCommand") { + throw new Error("Expected backend to spawn a standard command."); + } + + assert.equal(spawnedCommand.command, "/electron"); + assert.deepEqual(spawnedCommand.args, ["/server/bin.mjs", "--bootstrap-fd", "3"]); + assert.equal(spawnedCommand.options.cwd, "/server"); + assert.equal(spawnedCommand.options.extendEnv, true); + assert.equal(spawnedCommand.options.stdout, "pipe"); + assert.equal(spawnedCommand.options.stderr, "pipe"); + assert.equal(spawnedCommand.options.killSignal, "SIGTERM"); + assert.isDefined(spawnedCommand.options.forceKillAfter); + assert.equal(spawnedCommand.options.additionalFds?.fd4?.type, "input"); + assert.equal(spawnedCommand.options.additionalFds?.fd5?.type, "output"); + assert.equal( + Duration.toMillis(Duration.fromInputUnsafe(spawnedCommand.options.forceKillAfter)), + 5_000, + ); + + assert.deepEqual(yield* decodeBootstrap(bootstrapJson), configWithObservability); + assert.equal( + telemetryJson, + '{"version":1,"type":"desktopTelemetryHello","electronPid":123}\n', + ); + }), ); it.effect("reports a SIGKILL exit without an exit code", () => @@ -344,285 +383,308 @@ describe("DesktopBackendManager", () => { ); it.effect("reports child exit before waiting for trailing output to drain", () => - Effect.scoped( - Effect.gen(function* () { - const exitObserved = yield* Deferred.make(); - const finishOutputDrain = yield* Deferred.make(); - const layerSpawner = Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => - Effect.succeed( - makeProcess({ - stdout: Stream.fromEffect( - Deferred.await(finishOutputDrain).pipe( - Effect.as(new TextEncoder().encode("trailing output\n")), - ), + Effect.gen(function* () { + const exitObserved = yield* Deferred.make(); + const finishOutputDrain = yield* Deferred.make(); + const layerSpawner = Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.succeed( + makeProcess({ + stdout: Stream.fromEffect( + Deferred.await(finishOutputDrain).pipe( + Effect.as(new TextEncoder().encode("trailing output\n")), ), - exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(1)), - }), - ), + ), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(1)), + }), ), - ); + ), + ); - const runFiber = yield* DesktopBackendManager.runBackendProcess({ - ...baseConfig, - desktopTelemetryStream: Stream.empty, - onExitObserved: () => Deferred.succeed(exitObserved, void 0).pipe(Effect.asVoid), - }).pipe( - Effect.provide(Layer.merge(layerSpawner, layerHealthyHttpClient)), - Effect.forkChild, - ); - - yield* Deferred.await(exitObserved); - assert.isUndefined(runFiber.pollUnsafe()); - - yield* Deferred.succeed(finishOutputDrain, void 0); - assert.equal((yield* Fiber.join(runFiber)).code.pipe(Option.getOrUndefined), 1); - }), - ), + const runFiber = yield* DesktopBackendManager.runBackendProcess({ + ...baseConfig, + desktopTelemetryStream: Stream.empty, + onExitObserved: () => Deferred.succeed(exitObserved, void 0).pipe(Effect.asVoid), + }).pipe(Effect.provide(Layer.merge(layerSpawner, layerHealthyHttpClient)), Effect.forkChild); + + yield* Deferred.await(exitObserved); + assert.isUndefined(runFiber.pollUnsafe()); + + yield* Deferred.succeed(finishOutputDrain, void 0); + assert.equal((yield* Fiber.join(runFiber)).code.pipe(Option.getOrUndefined), 1); + }), ); - it.effect("continues routing desktop telemetry control messages after an invalid line", () => + it.effect("becomes ready from the server ready line while HTTP probes stall", () => Effect.scoped( Effect.gen(function* () { - const handled = yield* Deferred.make(); - const controlMessage = encodeDesktopTelemetryControl({ - version: 1, - type: "setDiagnosticsDemand", - enabled: true, - }); - const layerSpawner = Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => - Effect.succeed( - makeProcess({ - getOutputFd: (fd) => - fd === 5 - ? Stream.encodeText(Stream.make(`not-json\n${controlMessage}\n`)) - : Stream.empty, - exitCode: Deferred.await(handled).pipe(Effect.as(ChildProcessSpawner.ExitCode(0))), - }), + const probeStarted = yield* Deferred.make(); + const exit = yield* Deferred.make(); + const readyLine = encodeDesktopTelemetryControl({ version: 1, type: "serverReady" }); + const instance = yield* makeTestInstance({ + spawnerLayer: Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.succeed( + makeProcess({ + getOutputFd: (fd) => + fd === 5 + ? Stream.fromEffect(Deferred.await(probeStarted)).pipe( + Stream.flatMap(() => Stream.encodeText(Stream.make(`${readyLine}\n`))), + ) + : Stream.empty, + exitCode: Deferred.await(exit), + kill: () => + Deferred.succeed(exit, ChildProcessSpawner.ExitCode(0)).pipe(Effect.asVoid), + }), + ), ), ), - ); - const instance = yield* makeTestInstance({ - spawnerLayer: layerSpawner, - desktopTelemetryPublisher: { - handleControlForSource: (_sourceId, message) => - message.type === "setDiagnosticsDemand" - ? Deferred.succeed(handled, message.enabled).pipe(Effect.asVoid) - : Effect.void, - }, + httpClientLayer: layerHttpClient(() => + Deferred.succeed(probeStarted, undefined).pipe(Effect.andThen(Effect.never)), + ), }); yield* instance.start; - assert.isTrue(yield* Deferred.await(handled)); + assert.isTrue(yield* instance.waitForReady(Duration.seconds(15))); }), ), ); + it.effect("continues routing desktop telemetry control messages after an invalid line", () => + Effect.gen(function* () { + const handled = yield* Deferred.make(); + const controlMessage = encodeDesktopTelemetryControl({ + version: 1, + type: "setDiagnosticsDemand", + enabled: true, + }); + const layerSpawner = Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.succeed( + makeProcess({ + getOutputFd: (fd) => + fd === 5 + ? Stream.encodeText(Stream.make(`not-json\n${controlMessage}\n`)) + : Stream.empty, + exitCode: Deferred.await(handled).pipe(Effect.as(ChildProcessSpawner.ExitCode(0))), + }), + ), + ), + ); + const instance = yield* makeTestInstance({ + spawnerLayer: layerSpawner, + desktopTelemetryPublisher: { + handleControlForSource: (_sourceId, message) => + message.type === "setDiagnosticsDemand" + ? Deferred.succeed(handled, message.enabled).pipe(Effect.asVoid) + : Effect.void, + }, + }); + + yield* instance.start; + assert.isTrue(yield* Deferred.await(handled)); + }), + ); + it.effect("stops the scoped process with output still pending", () => - Effect.scoped( - Effect.gen(function* () { - let startCount = 0; - let closedCount = 0; - const closed = yield* Deferred.make(); - const teardownStarted = yield* Deferred.make(); - const finishTeardown = yield* Deferred.make(); - const startedPids = yield* Queue.unbounded(); - const ready = yield* Deferred.make(); - const backendReadyFlag = yield* Ref.make(false); - let shutdownCount = 0; - let persistedFailureCount = 0; - let discardedSessionCount = 0; - let removedTelemetrySources = 0; - - const layerSpawner = Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => - Effect.gen(function* () { - const scope = yield* Scope.Scope; - startCount += 1; - yield* Queue.offer(startedPids, 123); - const close = Deferred.succeed(teardownStarted, undefined).pipe( - Effect.andThen(Deferred.await(finishTeardown)), - Effect.andThen( - Effect.sync(() => { - closedCount += 1; - }), - ), - Effect.andThen(Deferred.succeed(closed, void 0)), - Effect.asVoid, - ); + Effect.gen(function* () { + let startCount = 0; + let closedCount = 0; + const closed = yield* Deferred.make(); + const teardownStarted = yield* Deferred.make(); + const finishTeardown = yield* Deferred.make(); + const startedPids = yield* Queue.unbounded(); + const ready = yield* Deferred.make(); + const backendReadyFlag = yield* Ref.make(false); + let shutdownCount = 0; + let persistedFailureCount = 0; + let discardedSessionCount = 0; + let removedTelemetrySources = 0; - yield* Scope.addFinalizer(scope, close); + const layerSpawner = Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.gen(function* () { + const scope = yield* Scope.Scope; + startCount += 1; + yield* Queue.offer(startedPids, 123); + const close = Deferred.succeed(teardownStarted, undefined).pipe( + Effect.andThen(Deferred.await(finishTeardown)), + Effect.andThen( + Effect.sync(() => { + closedCount += 1; + }), + ), + Effect.andThen(Deferred.succeed(closed, void 0)), + Effect.asVoid, + ); + + yield* Scope.addFinalizer(scope, close); + + return makeProcess({ + stdout: Stream.never, + exitCode: Deferred.await(closed).pipe(Effect.as(ChildProcessSpawner.ExitCode(0))), + kill: () => close, + }); + }), + ), + ); - return makeProcess({ - stdout: Stream.never, - exitCode: Deferred.await(closed).pipe(Effect.as(ChildProcessSpawner.ExitCode(0))), - kill: () => close, - }); + const instance = yield* makeTestInstance({ + spawnerLayer: layerSpawner, + onReady: Ref.set(backendReadyFlag, true).pipe( + Effect.andThen(Deferred.succeed(ready, void 0)), + Effect.asVoid, + ), + onShutdown: Ref.set(backendReadyFlag, false).pipe( + Effect.andThen( + Effect.sync(() => { + shutdownCount += 1; }), ), - ); - - const instance = yield* makeTestInstance({ - spawnerLayer: layerSpawner, - onReady: Ref.set(backendReadyFlag, true).pipe( - Effect.andThen(Deferred.succeed(ready, void 0)), - Effect.asVoid, - ), - onShutdown: Ref.set(backendReadyFlag, false).pipe( - Effect.andThen( - Effect.sync(() => { - shutdownCount += 1; - }), - ), - ), - backendOutputLog: { - persistFailure: () => - Effect.sync(() => { - persistedFailureCount += 1; - }), - discardSession: Effect.sync(() => { - discardedSessionCount += 1; + ), + backendOutputLog: { + persistFailure: () => + Effect.sync(() => { + persistedFailureCount += 1; }), - }, - desktopTelemetryPublisher: { - removeControlSource: () => - Effect.sync(() => { - removedTelemetrySources += 1; - }), - }, - }); - assert.isTrue(Option.isNone(yield* instance.currentConfig)); - - yield* instance.start; - assert.equal(yield* Queue.take(startedPids), 123); - yield* Deferred.await(ready); - assert.isTrue(yield* Ref.get(backendReadyFlag)); - assert.deepEqual(yield* instance.currentConfig, Option.some(baseConfig)); - - const runningSnapshot = yield* instance.snapshot; - assert.equal(runningSnapshot.ready, true); - assert.deepEqual(runningSnapshot.activePid, Option.some(123)); - - const stopFiber = yield* instance.stop().pipe(Effect.forkChild); - yield* Deferred.await(teardownStarted).pipe(Effect.timeout("1 second")); - assert.isFalse(yield* Ref.get(backendReadyFlag)); - assert.equal(shutdownCount, 1); - yield* Deferred.succeed(finishTeardown, undefined); - yield* Fiber.join(stopFiber).pipe(Effect.timeout("1 second")); - assert.equal(startCount, 1); - assert.equal(closedCount, 1); - assert.equal(persistedFailureCount, 0); - assert.equal(discardedSessionCount, 1); - assert.equal(removedTelemetrySources, 1); - - const stoppedSnapshot = yield* instance.snapshot; - assert.isFalse(yield* Ref.get(backendReadyFlag)); - assert.equal(shutdownCount, 1); - assert.equal(stoppedSnapshot.desiredRunning, false); - assert.equal(stoppedSnapshot.ready, false); - assert.equal(Option.isNone(stoppedSnapshot.activePid), true); - }), - ), + discardSession: Effect.sync(() => { + discardedSessionCount += 1; + }), + }, + desktopTelemetryPublisher: { + removeControlSource: () => + Effect.sync(() => { + removedTelemetrySources += 1; + }), + }, + }); + assert.isTrue(Option.isNone(yield* instance.currentConfig)); + + yield* instance.start; + assert.equal(yield* Queue.take(startedPids), 123); + yield* Deferred.await(ready); + assert.isTrue(yield* Ref.get(backendReadyFlag)); + assert.deepEqual(yield* instance.currentConfig, Option.some(baseConfig)); + + const runningSnapshot = yield* instance.snapshot; + assert.equal(runningSnapshot.ready, true); + assert.deepEqual(runningSnapshot.activePid, Option.some(123)); + + const stopFiber = yield* instance.stop().pipe(Effect.forkChild); + yield* Deferred.await(teardownStarted).pipe(Effect.timeout("1 second")); + assert.isFalse(yield* Ref.get(backendReadyFlag)); + assert.equal(shutdownCount, 1); + yield* Deferred.succeed(finishTeardown, undefined); + yield* Fiber.join(stopFiber).pipe(Effect.timeout("1 second")); + assert.equal(startCount, 1); + assert.equal(closedCount, 1); + assert.equal(persistedFailureCount, 0); + assert.equal(discardedSessionCount, 1); + assert.equal(removedTelemetrySources, 1); + + const stoppedSnapshot = yield* instance.snapshot; + assert.isFalse(yield* Ref.get(backendReadyFlag)); + assert.equal(shutdownCount, 1); + assert.equal(stoppedSnapshot.desiredRunning, false); + assert.equal(stoppedSnapshot.ready, false); + assert.equal(Option.isNone(stoppedSnapshot.activePid), true); + }), ); it.effect("restarts when start is requested during stop teardown", () => - Effect.scoped( - Effect.gen(function* () { - const starts = yield* Queue.unbounded(); - const teardownStarted = yield* Deferred.make(); - const finishTeardown = yield* Deferred.make(); - let startCount = 0; + Effect.gen(function* () { + const starts = yield* Queue.unbounded(); + const teardownStarted = yield* Deferred.make(); + const finishTeardown = yield* Deferred.make(); + let startCount = 0; - const layerSpawner = Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => - Effect.gen(function* () { - const scope = yield* Scope.Scope; - const closed = yield* Deferred.make(); - startCount += 1; - yield* Queue.offer(starts, startCount); - if (startCount === 1) { - yield* Scope.addFinalizer( - scope, - Deferred.succeed(teardownStarted, undefined).pipe( - Effect.andThen(Deferred.await(finishTeardown)), - Effect.andThen(Deferred.succeed(closed, undefined)), - Effect.asVoid, - ), - ); - } else { - yield* Scope.addFinalizer( - scope, - Deferred.succeed(closed, undefined).pipe(Effect.asVoid), - ); - } - return makeProcess({ - exitCode: Deferred.await(closed).pipe(Effect.as(ChildProcessSpawner.ExitCode(0))), - }); - }), - ), - ); + const layerSpawner = Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.gen(function* () { + const scope = yield* Scope.Scope; + const closed = yield* Deferred.make(); + startCount += 1; + yield* Queue.offer(starts, startCount); + if (startCount === 1) { + yield* Scope.addFinalizer( + scope, + Deferred.succeed(teardownStarted, undefined).pipe( + Effect.andThen(Deferred.await(finishTeardown)), + Effect.andThen(Deferred.succeed(closed, undefined)), + Effect.asVoid, + ), + ); + } else { + yield* Scope.addFinalizer( + scope, + Deferred.succeed(closed, undefined).pipe(Effect.asVoid), + ); + } + return makeProcess({ + exitCode: Deferred.await(closed).pipe(Effect.as(ChildProcessSpawner.ExitCode(0))), + }); + }), + ), + ); - const instance = yield* makeTestInstance({ - spawnerLayer: layerSpawner, - httpClientLayer: layerHttpClient(() => Effect.never), - }); + const instance = yield* makeTestInstance({ + spawnerLayer: layerSpawner, + httpClientLayer: layerHttpClient(() => Effect.never), + }); - yield* instance.start; - assert.equal(yield* Queue.take(starts), 1); + yield* instance.start; + assert.equal(yield* Queue.take(starts), 1); - const stopFiber = yield* instance.stop().pipe(Effect.forkChild); - yield* Deferred.await(teardownStarted).pipe(Effect.timeout("1 second")); - yield* instance.start; - assert.equal((yield* instance.snapshot).desiredRunning, true); + const stopFiber = yield* instance.stop().pipe(Effect.forkChild); + yield* Deferred.await(teardownStarted).pipe(Effect.timeout("1 second")); + yield* instance.start; + assert.equal((yield* instance.snapshot).desiredRunning, true); - yield* Deferred.succeed(finishTeardown, undefined); - yield* Fiber.join(stopFiber).pipe(Effect.timeout("1 second")); - yield* TestClock.adjust(Duration.millis(500)); + yield* Deferred.succeed(finishTeardown, undefined); + yield* Fiber.join(stopFiber).pipe(Effect.timeout("1 second")); + yield* TestClock.adjust(Duration.millis(500)); - assert.equal(yield* Queue.take(starts).pipe(Effect.timeout("1 second")), 2); - const restarted = yield* instance.snapshot; - assert.equal(restarted.desiredRunning, true); - assert.deepEqual(restarted.activePid, Option.some(123)); - }).pipe(Effect.provide(TestClock.layer())), - ), + assert.equal(yield* Queue.take(starts).pipe(Effect.timeout("1 second")), 2); + const restarted = yield* instance.snapshot; + assert.equal(restarted.desiredRunning, true); + assert.deepEqual(restarted.activePid, Option.some(123)); + }).pipe(Effect.provide(TestClock.layer())), ); it.effect("reports state-directory ownership once without scheduling another backend", () => - Effect.scoped( - Effect.gen(function* () { - let startCount = 0; - const refused = yield* Deferred.make(); - const instance = yield* makeTestInstance({ - spawnerLayer: Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => - Effect.sync(() => { - startCount += 1; - return makeProcess({ - exitCode: Effect.succeed( - ChildProcessSpawner.ExitCode(SERVER_EXIT_CODE_STATE_DIR_OWNED), - ), - }); - }), - ), + Effect.gen(function* () { + let startCount = 0; + const refused = yield* Deferred.make(); + const instance = yield* makeTestInstance({ + spawnerLayer: Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.sync(() => { + startCount += 1; + return makeProcess({ + exitCode: Effect.succeed( + ChildProcessSpawner.ExitCode(SERVER_EXIT_CODE_STATE_DIR_OWNED), + ), + }); + }), ), - httpClientLayer: layerHttpClient(() => Effect.never), - onStateDirOwned: Deferred.succeed(refused, undefined).pipe(Effect.asVoid), - }); - yield* instance.start; - yield* Deferred.await(refused); - const snapshot = yield* instance.snapshot; - assert.isFalse(snapshot.desiredRunning); - assert.isFalse(snapshot.restartScheduled); - yield* TestClock.adjust(Duration.minutes(1)); - assert.equal(startCount, 1); - }).pipe(Effect.provide(TestClock.layer())), - ), + ), + httpClientLayer: layerHttpClient(() => Effect.never), + onStateDirOwned: Deferred.succeed(refused, undefined).pipe(Effect.asVoid), + }); + yield* instance.start; + yield* Deferred.await(refused); + const snapshot = yield* instance.snapshot; + assert.isFalse(snapshot.desiredRunning); + assert.isFalse(snapshot.restartScheduled); + yield* TestClock.adjust(Duration.minutes(1)); + assert.equal(startCount, 1); + }).pipe(Effect.provide(TestClock.layer())), ); // A policy refusal is reported on the first exit, since retrying cannot fix it. @@ -632,145 +694,167 @@ describe("DesktopBackendManager", () => { ])( "reports exit $code with the server output after $exits exits", ({ code, exits, managedPolicy }) => - Effect.scoped( - Effect.gen(function* () { - const message = "Error: Cannot find module 'zod/v4' in /Users/René"; - let exitCount = 0; - const firstExit = yield* Deferred.make(); - const reports = yield* Queue.unbounded<{ - readonly exitCount: number; - readonly failure: DesktopBackendManager.BackendStartupFailure; - }>(); - const instance = yield* makeTestInstance({ - spawnerLayer: Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => - Effect.succeed( - makeProcess({ - // "é" split across chunks must not decode as U+FFFD, and the 4,000-char cut - // lands inside the emoji's surrogate pair. - stderr: Stream.make( - new TextEncoder().encode( - `😀${"x".repeat(3_999 - message.length)}${message.slice(0, -1)}`, - ), - new Uint8Array([0xc3]), - new Uint8Array([0xa9]), + Effect.gen(function* () { + const message = "Error: Cannot find module 'zod/v4' in /Users/René"; + let exitCount = 0; + const firstExit = yield* Deferred.make(); + const reports = yield* Queue.unbounded<{ + readonly exitCount: number; + readonly failure: DesktopBackendManager.BackendStartupFailure; + }>(); + const instance = yield* makeTestInstance({ + spawnerLayer: Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.succeed( + makeProcess({ + // "é" split across chunks must not decode as U+FFFD, and the 4,000-char cut + // lands inside the emoji's surrogate pair. + stderr: Stream.make( + new TextEncoder().encode( + `😀${"x".repeat(3_999 - message.length)}${message.slice(0, -1)}`, ), - exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(code)), - }), - ), + new Uint8Array([0xc3]), + new Uint8Array([0xa9]), + ), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(code)), + }), ), ), - httpClientLayer: layerHttpClient(() => Effect.never), - backendOutputLog: { - persistFailure: () => { - exitCount += 1; - return Deferred.succeed(firstExit, undefined).pipe(Effect.asVoid); - }, + ), + httpClientLayer: layerHttpClient(() => Effect.never), + backendOutputLog: { + persistFailure: () => { + exitCount += 1; + return Deferred.succeed(firstExit, undefined).pipe(Effect.asVoid); }, - onStartupFailure: (failure) => - Queue.offer(reports, { exitCount, failure }).pipe(Effect.asVoid), - }); - yield* instance.start; - yield* Deferred.await(firstExit); - yield* TestClock.adjust(Duration.minutes(1)); - const { exitCount: exitsWhenReported, failure } = yield* Queue.take(reports); - assert.equal(exitsWhenReported, exits); - assert.equal(failure.reason, `The server could not start (code=${code}).`); - assert.equal(failure.managedPolicy, managedPolicy); - assert.isTrue(failure.output.endsWith(message)); - assert.isTrue(failure.output.isWellFormed()); - }).pipe(Effect.provide(TestClock.layer())), - ), + }, + onStartupFailure: (failure) => + Queue.offer(reports, { exitCount, failure }).pipe(Effect.asVoid), + }); + yield* instance.start; + yield* Deferred.await(firstExit); + yield* TestClock.adjust(Duration.minutes(1)); + const { exitCount: exitsWhenReported, failure } = yield* Queue.take(reports); + assert.equal(exitsWhenReported, exits); + assert.equal(failure.reason, `The server could not start (code=${code}).`); + assert.equal(failure.managedPolicy, managedPolicy); + assert.isTrue(failure.output.endsWith(message)); + assert.isTrue(failure.output.isWellFormed()); + }).pipe(Effect.provide(TestClock.layer())), ); it.effect("waits out a slow startup and reports one that never becomes ready", () => - Effect.scoped( - Effect.gen(function* () { - const reports = yield* Queue.unbounded(); - const startInstance = (status: Ref.Ref, onReady?: Effect.Effect) => - makeTestInstance({ - spawnerLayer: Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => - Effect.succeed(makeProcess({ exitCode: Effect.never })), + Effect.gen(function* () { + const reports = yield* Queue.unbounded(); + const startInstance = (status: Ref.Ref, onReady?: Effect.Effect, stdout = "") => + makeTestInstance({ + spawnerLayer: Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.succeed( + makeProcess({ + exitCode: Effect.never, + stdout: Stream.make(new TextEncoder().encode(stdout)), + }), ), ), - httpClientLayer: layerHttpClient((request) => - Ref.get(status).pipe(Effect.map((code) => responseForRequest(request, code))), - ), - onStartupFailure: (failure) => Queue.offer(reports, failure).pipe(Effect.asVoid), - ...(onReady ? { onReady } : {}), - }).pipe(Effect.tap((instance) => instance.start)); - - const slowStatus = yield* Ref.make(503); - const ready = yield* Deferred.make(); - yield* startInstance(slowStatus, Deferred.succeed(ready, undefined).pipe(Effect.asVoid)); - yield* TestClock.adjust(Duration.seconds(66)); - yield* Ref.set(slowStatus, 200); - yield* TestClock.adjust(Duration.millis(100)); - yield* Deferred.await(ready); - assert.isTrue(Option.isNone(yield* Queue.poll(reports))); - - yield* startInstance(yield* Ref.make(503)); - yield* TestClock.adjust(Duration.seconds(179)); - assert.isTrue(Option.isNone(yield* Queue.poll(reports))); - yield* TestClock.adjust(Duration.seconds(1)); - const failure = yield* Queue.take(reports); - assert.equal(failure.reason, "The server started but did not become ready in time."); - }).pipe(Effect.provide(TestClock.layer())), - ), + ), + httpClientLayer: layerHttpClient((request) => + Ref.get(status).pipe(Effect.map((code) => responseForRequest(request, code))), + ), + onStartupFailure: (failure) => Queue.offer(reports, failure).pipe(Effect.asVoid), + ...(onReady ? { onReady } : {}), + }).pipe(Effect.tap((instance) => instance.start)); + + const slowStatus = yield* Ref.make(503); + const ready = yield* Deferred.make(); + yield* startInstance(slowStatus, Deferred.succeed(ready, undefined).pipe(Effect.asVoid)); + yield* TestClock.adjust(Duration.seconds(66)); + yield* Ref.set(slowStatus, 200); + yield* TestClock.adjust(Duration.millis(100)); + yield* Deferred.await(ready); + assert.isTrue(Option.isNone(yield* Queue.poll(reports))); + + const migratingStatus = yield* Ref.make(503); + const migrated = yield* Deferred.make(); + yield* startInstance( + migratingStatus, + Deferred.succeed(migrated, undefined).pipe(Effect.asVoid), + // Enough progress lines that the start line scrolls out of the output tail. + `INFO ${DATABASE_MIGRATIONS_STARTED_MESSAGE}\n` + + Array.from( + { length: 80 }, + (_, index) => + `INFO ${DATABASE_MIGRATION_PROGRESS_MESSAGE} migration=${index}_Migration progress="${index + 1} of 80"\n`, + ).join(""), + ); + yield* TestClock.adjust(Duration.minutes(10)); + assert.isTrue(Option.isNone(yield* Queue.poll(reports))); + yield* Ref.set(migratingStatus, 200); + yield* TestClock.adjust(Duration.millis(100)); + yield* Deferred.await(migrated); + + yield* startInstance( + yield* Ref.make(503), + undefined, + `${DATABASE_MIGRATIONS_STARTED_MESSAGE}\n${DATABASE_MIGRATIONS_FINISHED_MESSAGE}\n`, + ); + yield* TestClock.adjust(Duration.seconds(179)); + assert.isTrue(Option.isNone(yield* Queue.poll(reports))); + yield* TestClock.adjust(Duration.seconds(1)); + const failure = yield* Queue.take(reports); + assert.equal(failure.reason, "The server started but did not become ready in time."); + }).pipe(Effect.provide(TestClock.layer())), ); it.effect("the quit stop keeps waiting for a backend that outlasts its deadline", () => - Effect.scoped( - Effect.gen(function* () { - const spawned = yield* Deferred.make(); - const teardownStarted = yield* Deferred.make(); - const teardownFinished = yield* Deferred.make(); - const instance = yield* makeTestInstance({ - spawnerLayer: Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => - Effect.gen(function* () { - const scope = yield* Scope.Scope; - yield* Deferred.succeed(spawned, undefined); - yield* Scope.addFinalizer( - scope, - Deferred.succeed(teardownStarted, undefined).pipe( - Effect.andThen(Effect.sleep(Duration.seconds(10))), - Effect.andThen(Deferred.succeed(teardownFinished, undefined)), - Effect.asVoid, - ), - ); - return makeProcess({ - exitCode: Deferred.await(teardownFinished).pipe( - Effect.as(ChildProcessSpawner.ExitCode(0)), - ), - }); - }), - ), + Effect.gen(function* () { + const spawned = yield* Deferred.make(); + const teardownStarted = yield* Deferred.make(); + const teardownFinished = yield* Deferred.make(); + const instance = yield* makeTestInstance({ + spawnerLayer: Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.gen(function* () { + const scope = yield* Scope.Scope; + yield* Deferred.succeed(spawned, undefined); + yield* Scope.addFinalizer( + scope, + Deferred.succeed(teardownStarted, undefined).pipe( + Effect.andThen(Effect.sleep(Duration.seconds(10))), + Effect.andThen(Deferred.succeed(teardownFinished, undefined)), + Effect.asVoid, + ), + ); + return makeProcess({ + exitCode: Deferred.await(teardownFinished).pipe( + Effect.as(ChildProcessSpawner.ExitCode(0)), + ), + }); + }), ), - httpClientLayer: layerHttpClient(() => Effect.never), - }); - yield* instance.start; - yield* Deferred.await(spawned); - - // App quit and the updater stop every backend with this deadline. - const quitDone = yield* Deferred.make(); - const quitFiber = yield* instance - .stop({ timeout: DesktopBackendManager.BACKEND_QUIT_STOP_TIMEOUT }) - .pipe(Effect.andThen(Deferred.succeed(quitDone, undefined)), Effect.forkChild); - - yield* Deferred.await(teardownStarted); - // Past the quit deadline, quit must still wait for the backend to exit. - yield* TestClock.adjust(Duration.seconds(9)); - assert.isFalse(yield* Deferred.isDone(quitDone)); - - yield* TestClock.adjust(Duration.seconds(1)); - yield* Fiber.join(quitFiber); - assert.isTrue(yield* Deferred.isDone(teardownFinished)); - }).pipe(Effect.provide(TestClock.layer())), - ), + ), + httpClientLayer: layerHttpClient(() => Effect.never), + }); + yield* instance.start; + yield* Deferred.await(spawned); + + // App quit and the updater stop every backend with this deadline. + const quitDone = yield* Deferred.make(); + const quitFiber = yield* instance + .stop({ timeout: DesktopBackendManager.BACKEND_QUIT_STOP_TIMEOUT }) + .pipe(Effect.andThen(Deferred.succeed(quitDone, undefined)), Effect.forkChild); + + yield* Deferred.await(teardownStarted); + // Past the quit deadline, quit must still wait for the backend to exit. + yield* TestClock.adjust(Duration.seconds(9)); + assert.isFalse(yield* Deferred.isDone(quitDone)); + + yield* TestClock.adjust(Duration.seconds(1)); + yield* Fiber.join(quitFiber); + assert.isTrue(yield* Deferred.isDone(teardownFinished)); + }).pipe(Effect.provide(TestClock.layer())), ); }); diff --git a/apps/desktop/src/backend/DesktopBackendManager.ts b/apps/desktop/src/backend/DesktopBackendManager.ts index 63b641e8e42e..76366419a0f3 100644 --- a/apps/desktop/src/backend/DesktopBackendManager.ts +++ b/apps/desktop/src/backend/DesktopBackendManager.ts @@ -1,5 +1,6 @@ import * as Brand from "effect/Brand"; import * as Cause from "effect/Cause"; +import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; @@ -25,6 +26,7 @@ import { DesktopTelemetryControlMessage, type DesktopTelemetryControlMessage as DesktopTelemetryControlMessageValue, } from "@t3tools/contracts"; +import { isRunningDatabaseMigrations } from "@t3tools/shared/databaseMigrationLog"; import { waitForHttpReady as waitForHttpReadyShared } from "@t3tools/shared/httpReadiness"; import * as DesktopObservability from "../app/DesktopObservability.ts"; @@ -209,7 +211,7 @@ interface RunBackendProcessOptions extends DesktopBackendStartConfig { readonly outputDrainTimeout?: Duration.Duration; readonly onStarted?: (pid: number) => Effect.Effect; readonly onExitObserved?: () => Effect.Effect; - readonly onReady?: () => Effect.Effect; + readonly onReady?: (pid: number) => Effect.Effect; readonly onReadinessFailure?: (error: BackendReadinessTimeoutError) => Effect.Effect; readonly onOutput?: ( streamName: BackendProcessOutputStream, @@ -256,8 +258,8 @@ export interface BackendInstanceSpec { readonly id: BackendInstanceId; readonly label: Effect.Effect; readonly configResolve: Effect.Effect; - /** Receives the ready run's resolved URL; reading current config could race a subsequent restart. */ - readonly onReady?: (httpBaseUrl: URL) => Effect.Effect; + /** Receives the ready run's URL and pid; reading current config could race a subsequent restart. */ + readonly onReady?: (httpBaseUrl: URL, pid: number) => Effect.Effect; readonly onShutdown?: () => Effect.Effect; readonly onPreflightFailed?: (failure: PreflightFailure) => Effect.Effect; readonly onStateDirOwned?: () => Effect.Effect; @@ -347,8 +349,26 @@ const closeRun = ( ); }; +const LOOPBACK_HOSTNAMES = new Set(["127.0.0.1", "localhost", "[::1]"]); +// Windows takes about a second to fail a refused connection to a non-loopback address such as a +// WSL distro IP, so one sequential poller notices readiness up to a second late. +const STAGGERED_READINESS_POLLERS = 4; +const READINESS_POLLER_STAGGER = Duration.millis(250); + const waitForHttpReady = ( options: BackendProcessContext & { readonly timeout: Duration.Duration }, +): Effect.Effect => { + const poll = pollHttpReady(options); + if (LOOPBACK_HOSTNAMES.has(options.httpBaseUrl.hostname)) return poll; + return Effect.raceAll( + Array.from({ length: STAGGERED_READINESS_POLLERS }, (_, index) => + Effect.sleep(Duration.times(READINESS_POLLER_STAGGER, index)).pipe(Effect.andThen(poll)), + ), + ); +}; + +const pollHttpReady = ( + options: BackendProcessContext & { readonly timeout: Duration.Duration }, ): Effect.Effect => { const readinessUrl = new URL(BACKEND_READINESS_PATH, options.httpBaseUrl); return waitForHttpReadyShared({ @@ -488,21 +508,23 @@ export const runBackendProcess = Effect.fn("runBackendProcess")(function* ( ), ); const outputFibers: Array> = []; + const serverReady = yield* Deferred.make(); yield* options.onStarted?.(handle.pid) ?? Effect.void; - if ( - options.bootstrap.desktopTelemetryControlFd !== undefined && - options.onDesktopTelemetryControl !== undefined - ) { + if (options.bootstrap.desktopTelemetryControlFd !== undefined) { const controlFd = options.bootstrap.desktopTelemetryControlFd; - const handleControl = options.onDesktopTelemetryControl; + const handleControl = options.onDesktopTelemetryControl ?? (() => Effect.void); yield* handle.getOutputFd(controlFd).pipe( Stream.decodeText(), Stream.splitLines, Stream.filter((line) => line.trim().length > 0), Stream.runForEach((line) => decodeDesktopTelemetryControlLine(line).pipe( - Effect.flatMap(handleControl), + Effect.flatMap((message) => + message.type === "serverReady" + ? Deferred.succeed(serverReady, undefined) + : handleControl(message), + ), Effect.catchCause((cause) => logBackendProcessWarning("ignored invalid desktop telemetry control message", { fd: controlFd, @@ -574,15 +596,19 @@ export const runBackendProcess = Effect.fn("runBackendProcess")(function* ( ); } // Cold WSL boots across /mnt/c can exceed one probe budget; retry while the backend is alive. + // HTTP polling stays as the fallback for backends without the control fd, such as WSL. const probeReadiness = Effect.fn("desktop.backendProcess.probeReadiness")(() => - waitForHttpReady({ - executablePath: options.executablePath, - entryPath: options.entryPath, - cwd: options.cwd, - httpBaseUrl: options.httpBaseUrl, - timeout: options.readinessTimeout ?? DEFAULT_BACKEND_READINESS_TIMEOUT, - }).pipe( - Effect.flatMap(() => options.onReady?.() ?? Effect.void), + Effect.raceFirst( + waitForHttpReady({ + executablePath: options.executablePath, + entryPath: options.entryPath, + cwd: options.cwd, + httpBaseUrl: options.httpBaseUrl, + timeout: options.readinessTimeout ?? DEFAULT_BACKEND_READINESS_TIMEOUT, + }), + Deferred.await(serverReady), + ).pipe( + Effect.flatMap(() => options.onReady?.(handle.pid) ?? Effect.void), Effect.as(true), Effect.catchTags({ BackendReadinessTimeoutError: (error) => @@ -943,7 +969,7 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( ...run, exitObserved: true, })), - onReady: Effect.fn("desktop.backendInstance.onReady")(function* () { + onReady: Effect.fn("desktop.backendInstance.onReady")(function* (pid) { const isCurrentRun = yield* SubscriptionRef.modify(state, (latest) => { const activeRun = Option.getOrUndefined(latest.active); if (activeRun?.id !== runId) { @@ -963,7 +989,7 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( return; } - yield* spec.onReady?.(config.value.httpBaseUrl) ?? Effect.void; + yield* spec.onReady?.(config.value.httpBaseUrl, pid) ?? Effect.void; if ( config.value.runningDistro !== undefined && config.value.wslRuntimeId !== undefined @@ -976,6 +1002,12 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( }), onReadinessFailure: Effect.fn("desktop.backendInstance.onReadinessFailure")( function* (error) { + if (isRunningDatabaseMigrations(yield* Ref.get(outputTail))) { + yield* logInstanceWarning("backend is still migrating its database; waiting", { + error: error.message, + }); + return; + } yield* logInstanceWarning("backend readiness check failed during bootstrap", { error: error.message, }); diff --git a/apps/desktop/src/backend/DesktopBackendPool.test.ts b/apps/desktop/src/backend/DesktopBackendPool.test.ts index c46e0b3a2607..005a1cde1224 100644 --- a/apps/desktop/src/backend/DesktopBackendPool.test.ts +++ b/apps/desktop/src/backend/DesktopBackendPool.test.ts @@ -1,5 +1,5 @@ -import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, describe, it } from "@effect/vitest"; +import { DESKTOP_UPDATE_IN_PROGRESS_TTL_MS } from "@t3tools/contracts"; import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; @@ -110,27 +110,56 @@ const runningConfig = { function layerPool( labelRef: Ref.Ref, - backend?: Effect.Success>, + options: { + readonly config?: DesktopBackendStartConfig; + readonly resolvePrimary?: Effect.Effect; + readonly onMainContentsCommand?: ( + command: DesktopWindow.MainWindowContentsCommand, + ) => Effect.Effect; + readonly fileSystem?: Partial; + readonly spawner?: ChildProcessSpawner.ChildProcessSpawner["Service"]; + readonly httpClient?: HttpClient.HttpClient; + readonly onReady?: Effect.Effect; + readonly backend?: Effect.Success>; + } = {}, ): Layer.Layer { return DesktopBackendPool.layer.pipe( Layer.provideMerge( Layer.mergeAll( - FileSystem.layerNoop({ exists: () => Effect.succeed(true) }), + DesktopEnvironment.layer({ + dirname: "/repo/apps/desktop/src", + homeDirectory: "/test-home", + platform: "linux", + processArch: "x64", + appVersion: "1.2.3", + appPath: "/repo", + isPackaged: true, + resourcesPath: "/missing/resources", + runningUnderArm64Translation: false, + }).pipe( + Layer.provide( + Layer.mergeAll(Path.layer, DesktopConfig.layerTest({ T3CODE_HOME: "/test-home" })), + ), + Layer.orDie, + ), + FileSystem.layerNoop(options.fileSystem ?? { exists: () => Effect.succeed(true) }), Path.layer, - backend?.layer ?? + options.backend?.layer ?? Layer.succeed( ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => Effect.die("unexpected child process spawn")), + options.spawner ?? + ChildProcessSpawner.make(() => Effect.die("unexpected child process spawn")), ), Layer.succeed( HttpClient.HttpClient, - HttpClient.make((request) => - backend === undefined - ? Effect.die("unexpected HTTP request") - : Effect.succeed( - HttpClientResponse.fromWeb(request, new Response(null, { status: 200 })), - ), - ), + options.httpClient ?? + HttpClient.make((request) => + options.backend === undefined + ? Effect.die("unexpected HTTP request") + : Effect.succeed( + HttpClientResponse.fromWeb(request, new Response(null, { status: 200 })), + ), + ), ), Layer.succeed(DesktopObservability.DesktopBackendOutputLogFactory, { forInstance: () => @@ -155,29 +184,24 @@ function layerPool( }), DesktopBrowserHost.layer, Layer.succeed(DesktopBackendConfiguration.DesktopBackendConfiguration, { - resolvePrimary: Effect.die("unexpected primary config resolve"), + resolvePrimary: + options.resolvePrimary ?? + (options.config + ? Effect.succeed(options.config) + : Effect.die("unexpected primary config resolve")), resolvePrimaryLabel: Ref.get(labelRef), resolveWsl: () => Effect.die("unexpected WSL config resolve"), currentBootstrapToken: Effect.die("unexpected bootstrap token read"), } satisfies DesktopBackendConfiguration.DesktopBackendConfiguration["Service"]), DesktopAppSettings.layerTest(), - DesktopEnvironment.layer({ - dirname: "/repo/apps/desktop/src", - homeDirectory: "/home/test", - platform: "linux", - processArch: "x64", - appVersion: "0.0.0", - appPath: "/repo", - isPackaged: true, - resourcesPath: "/missing/resources", - runningUnderArm64Translation: false, - }).pipe( - Layer.provide(Layer.mergeAll(NodeServices.layer, DesktopConfig.layerTest({}))), - Layer.orDie, - ), DesktopState.layer, DesktopWslEnvironment.layerTest(), - ElectronDialog.layer, + Layer.succeed(ElectronDialog.ElectronDialog, { + pickFolder: () => Effect.die("unexpected folder picker"), + pickFiles: () => Effect.die("unexpected file picker"), + showMessageBox: () => Effect.die("unexpected message box"), + showErrorBox: () => Effect.void, + } satisfies ElectronDialog.ElectronDialog["Service"]), ElectronApp.layer, Layer.succeed(DesktopWindow.DesktopWindow, { createMain: Effect.die("unexpected window create"), @@ -186,14 +210,15 @@ function layerPool( activate: Effect.die("unexpected window activate"), createMainIfBackendReady: Effect.die("unexpected window create"), showConnectingSplash: Effect.void, - handleBackendReady: () => Effect.void, + handleBackendReady: () => options.onReady ?? Effect.void, handleBackendNotReady: Effect.void, flushMainWindowBounds: Effect.void, prepareCaptureReveal: Effect.void, dispatchMenuAction: () => Effect.die("unexpected menu action"), dispatchSnapShotEvent: () => Effect.void, zoomMain: () => Effect.die("unexpected zoom"), - runMainContentsCommand: () => Effect.die("unexpected main contents command"), + runMainContentsCommand: + options.onMainContentsCommand ?? (() => Effect.die("unexpected main contents command")), syncAppearance: Effect.void, } satisfies DesktopWindow.DesktopWindow["Service"]), ), @@ -201,86 +226,334 @@ function layerPool( ); } +const markerPath = "/test-home/runtime/desktop-update-in-progress"; +const runtimePath = "/test-home/userdata/server-runtime.json"; + +const makeUpdateMarkerFixture = () => + Effect.gen(function* () { + const allowReady = yield* Deferred.make(); + const runtimeChecked = yield* Deferred.make(); + const markerRemoved = yield* Deferred.make(); + const readyRuns = yield* Queue.unbounded(); + let nextPid = 123; + const files = new Map([[markerPath, JSON.stringify({ startedAt: 0, expiresAt: 60_000 })]]); + const removed: string[] = []; + const config: DesktopBackendStartConfig = { + executablePath: "/electron", + args: [], + entryPath: "/server/bin.mjs", + cwd: "/server", + env: {}, + bootstrap: { + mode: "desktop", + noBrowser: true, + port: 3774, + t3Home: "/test-home", + host: "127.0.0.1", + desktopBootstrapToken: "token", + tailscaleServeEnabled: false, + tailscaleServePort: 443, + }, + bootstrapDelivery: "fd3", + extendEnv: true, + httpBaseUrl: new URL("http://127.0.0.1:3774"), + captureOutput: false, + preflightFailure: Option.none(), + }; + const layer = layerPool(Ref.makeUnsafe("Local"), { + config, + fileSystem: { + exists: (path) => Effect.succeed(files.has(path) || path === config.entryPath), + readFileString: (path) => + (path === runtimePath ? Deferred.succeed(runtimeChecked, undefined) : Effect.void).pipe( + Effect.andThen(Effect.sync(() => files.get(path) ?? "")), + ), + remove: (path) => + Effect.sync(() => { + removed.push(path); + files.delete(path); + }).pipe( + Effect.andThen( + path === markerPath ? Deferred.succeed(markerRemoved, undefined) : Effect.void, + ), + Effect.asVoid, + ), + }, + spawner: ChildProcessSpawner.make(() => + Effect.gen(function* () { + const exited = yield* Deferred.make(); + const exit = Deferred.succeed(exited, ChildProcessSpawner.ExitCode(0)).pipe( + Effect.asVoid, + ); + // Like the real spawner, closing the run's scope ends the process. + yield* Effect.addFinalizer(() => exit); + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(nextPid++), + stdout: Stream.empty, + stderr: Stream.empty, + all: Stream.empty, + exitCode: Deferred.await(exited), + isRunning: Effect.succeed(true), + kill: () => exit, + stdin: Sink.drain, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }); + }), + ), + httpClient: HttpClient.make((request) => + Deferred.await(allowReady).pipe( + Effect.as(HttpClientResponse.fromWeb(request, new Response(null, { status: 200 }))), + ), + ), + onReady: Queue.offer(readyRuns, undefined).pipe(Effect.asVoid), + }); + const publishRuntime = (pid: number) => + files.set( + runtimePath, + JSON.stringify({ + version: 1, + pid, + port: 3774, + origin: "http://127.0.0.1:3774", + startedAt: "2026-10-27T00:00:00.000Z", + }), + ); + return { + layer, + files, + removed, + allowReady, + readyRuns, + runtimeChecked, + markerRemoved, + publishRuntime, + }; + }); + describe("DesktopBackendPool", () => { - it.effect("unregister during pool quit does not interrupt sibling stops", () => - Effect.scoped( + it.effect( + "keeps the update marker after HTTP readiness until the backend publishes its runtime", + () => Effect.gen(function* () { - const backend = yield* makeFakeBackendProcess(); + const fixture = yield* makeUpdateMarkerFixture(); yield* Effect.gen(function* () { const pool = yield* DesktopBackendPool.DesktopBackendPool; - const id = DesktopBackendPool.BackendInstanceId("wsl:Ubuntu"); - const ready = yield* Deferred.make(); - const siblingStarted = yield* Deferred.make(); - const finishSibling = yield* Deferred.make(); - let siblingCompleted = false; - const instance = yield* pool.register({ - id, - label: Effect.succeed("WSL (Ubuntu)"), - configResolve: Effect.succeed(runningConfig), - onReady: () => Deferred.succeed(ready, undefined).pipe(Effect.asVoid), - }); - yield* instance.start; - yield* backend.started; - yield* Deferred.await(ready); - const quitFiber = yield* Effect.all( - [ - instance.stop({ timeout: Duration.seconds(5) }), - Deferred.succeed(siblingStarted, undefined).pipe( - Effect.andThen(Deferred.await(finishSibling)), - Effect.andThen( - Effect.sync(() => { - siblingCompleted = true; - }), - ), - ), - ], - { concurrency: "unbounded" }, - ).pipe(Effect.forkChild); - yield* Deferred.await(siblingStarted); - assert.equal(yield* Queue.take(backend.signals), "SIGTERM"); - const unregisterFiber = yield* pool.unregister(id).pipe(Effect.forkChild); + const primary = yield* pool.primary; + yield* primary.start; + yield* Deferred.succeed(fixture.allowReady, undefined); + yield* Queue.take(fixture.readyRuns); + assert.isTrue( + fixture.files.has(markerPath), + "a launcher could neither wait nor adopt between readiness and publication", + ); + yield* Deferred.await(fixture.runtimeChecked); yield* TestClock.adjust(Duration.seconds(5)); - assert.equal(yield* Queue.take(backend.signals), "SIGKILL"); - assert.isUndefined(quitFiber.pollUnsafe()); - assert.isUndefined(unregisterFiber.pollUnsafe()); - assert.isTrue(yield* backend.isRunning); - yield* TestClock.adjust(backend.killWait); - yield* Fiber.join(unregisterFiber); - yield* Deferred.succeed(finishSibling, undefined); - yield* Fiber.join(quitFiber); - - assert.isTrue(siblingCompleted); - assert.isFalse(yield* backend.isRunning); - assert.isTrue(Option.isNone((yield* instance.snapshot).activePid)); - assert.isTrue(Option.isNone(yield* pool.get(id))); - }).pipe(Effect.provide(layerPool(Ref.makeUnsafe("Windows"), backend))); + assert.isTrue(fixture.files.has(markerPath)); + fixture.publishRuntime(456); + yield* TestClock.adjust(Duration.seconds(1)); + assert.isTrue( + fixture.files.has(markerPath), + "another process's runtime is not the replacement", + ); + fixture.publishRuntime(123); + yield* TestClock.adjust(Duration.seconds(1)); + yield* Deferred.await(fixture.markerRemoved); + assert.deepEqual(fixture.removed, [markerPath]); + }).pipe(Effect.provide(fixture.layer)); }), - ), ); - it.effect("unregister stops the registered instance", () => - Effect.scoped( - Effect.gen(function* () { - const id = DesktopBackendPool.BackendInstanceId("wsl:Ubuntu"); + it.effect("leaves a replacement update marker written while an earlier wait is pending", () => + Effect.gen(function* () { + const fixture = yield* makeUpdateMarkerFixture(); + yield* Effect.gen(function* () { + const pool = yield* DesktopBackendPool.DesktopBackendPool; + const primary = yield* pool.primary; + yield* primary.start; + yield* Deferred.succeed(fixture.allowReady, undefined); + yield* Queue.take(fixture.readyRuns); + yield* Deferred.await(fixture.runtimeChecked); + yield* TestClock.adjust(Duration.millis(DESKTOP_UPDATE_IN_PROGRESS_TTL_MS - 1_000)); + const replacement = JSON.stringify({ startedAt: 59_000, expiresAt: 119_000 }); + fixture.files.set(markerPath, replacement); + yield* TestClock.adjust(Duration.millis(DESKTOP_UPDATE_IN_PROGRESS_TTL_MS)); + assert.equal(fixture.files.get(markerPath), replacement); + assert.deepEqual(fixture.removed, []); + }).pipe(Effect.provide(fixture.layer)); + }), + ); + + it.effect("a later backend run's readiness supersedes the earlier run's marker wait", () => + Effect.gen(function* () { + const fixture = yield* makeUpdateMarkerFixture(); + yield* Effect.gen(function* () { + const pool = yield* DesktopBackendPool.DesktopBackendPool; + const primary = yield* pool.primary; + yield* primary.start; + yield* Deferred.succeed(fixture.allowReady, undefined); + yield* Queue.take(fixture.readyRuns); + yield* Deferred.await(fixture.runtimeChecked); + yield* TestClock.adjust(Duration.seconds(30)); + yield* primary.stop(); + yield* primary.start; + yield* Queue.take(fixture.readyRuns); + yield* TestClock.adjust(Duration.seconds(31)); + assert.isTrue( + fixture.files.has(markerPath), + "the earlier run's wait must not end the later run's handoff", + ); + assert.deepEqual(fixture.removed, []); + fixture.publishRuntime(124); + yield* TestClock.adjust(Duration.millis(100)); + yield* Deferred.await(fixture.markerRemoved); + assert.deepEqual(fixture.removed, [markerPath]); + }).pipe(Effect.provide(fixture.layer)); + }), + ); + + it.effect("unregister during pool quit does not interrupt sibling stops", () => + Effect.gen(function* () { + const backend = yield* makeFakeBackendProcess(); + yield* Effect.gen(function* () { const pool = yield* DesktopBackendPool.DesktopBackendPool; + const id = DesktopBackendPool.BackendInstanceId("wsl:Ubuntu"); + const ready = yield* Deferred.make(); + const siblingStarted = yield* Deferred.make(); + const finishSibling = yield* Deferred.make(); + let siblingCompleted = false; const instance = yield* pool.register({ id, label: Effect.succeed("WSL (Ubuntu)"), - configResolve: Effect.succeed({ - ...runningConfig, - preflightFailure: Option.some({ reason: "WSL starting", fatal: false }), - }), + configResolve: Effect.succeed(runningConfig), + onReady: () => Deferred.succeed(ready, undefined).pipe(Effect.asVoid), }); yield* instance.start; - assert.equal((yield* instance.snapshot).restartScheduled, true); + yield* backend.started; + yield* Deferred.await(ready); + const quitFiber = yield* Effect.all( + [ + instance.stop({ timeout: Duration.seconds(5) }), + Deferred.succeed(siblingStarted, undefined).pipe( + Effect.andThen(Deferred.await(finishSibling)), + Effect.andThen( + Effect.sync(() => { + siblingCompleted = true; + }), + ), + ), + ], + { concurrency: "unbounded" }, + ).pipe(Effect.forkChild); + yield* Deferred.await(siblingStarted); + assert.equal(yield* Queue.take(backend.signals), "SIGTERM"); + const unregisterFiber = yield* pool.unregister(id).pipe(Effect.forkChild); + yield* TestClock.adjust(Duration.seconds(5)); + assert.equal(yield* Queue.take(backend.signals), "SIGKILL"); + assert.isUndefined(quitFiber.pollUnsafe()); + assert.isUndefined(unregisterFiber.pollUnsafe()); + assert.isTrue(yield* backend.isRunning); + yield* TestClock.adjust(backend.killWait); + yield* Fiber.join(unregisterFiber); + yield* Deferred.succeed(finishSibling, undefined); + yield* Fiber.join(quitFiber); - yield* pool.unregister(id); + assert.isTrue(siblingCompleted); + assert.isFalse(yield* backend.isRunning); + assert.isTrue(Option.isNone((yield* instance.snapshot).activePid)); + assert.isTrue(Option.isNone(yield* pool.get(id))); + }).pipe(Effect.provide(layerPool(Ref.makeUnsafe("Windows"), { backend }))); + }), + ); - const stopped = yield* instance.snapshot; - assert.equal(stopped.desiredRunning, false); - assert.equal(stopped.restartScheduled, false); - assert.equal(Option.isNone(yield* pool.get(id)), true); - }).pipe(Effect.provide(layerPool(Ref.makeUnsafe("Windows")))), - ), + it.effect("unregister stops the registered instance", () => + Effect.gen(function* () { + const id = DesktopBackendPool.BackendInstanceId("wsl:Ubuntu"); + const pool = yield* DesktopBackendPool.DesktopBackendPool; + const instance = yield* pool.register({ + id, + label: Effect.succeed("WSL (Ubuntu)"), + configResolve: Effect.succeed({ + ...runningConfig, + preflightFailure: Option.some({ reason: "WSL starting", fatal: false }), + }), + }); + yield* instance.start; + assert.equal((yield* instance.snapshot).restartScheduled, true); + + yield* pool.unregister(id); + + const stopped = yield* instance.snapshot; + assert.equal(stopped.desiredRunning, false); + assert.equal(stopped.restartScheduled, false); + assert.equal(Option.isNone(yield* pool.get(id)), true); + }).pipe(Effect.provide(layerPool(Ref.makeUnsafe("Windows")))), + ); + + it.effect("reloads a main window that read the WSL endpoint before the Windows fallback", () => + Effect.gen(function* () { + const events: string[] = []; + let resolves = 0; + const resolvePrimary = Effect.sync(() => + resolves++ === 0 + ? { + ...runningConfig, + preflightFailure: Option.some({ + reason: "WSL still unavailable", + fatal: false, + retryLimit: 1, + }), + } + : runningConfig, + ); + const ready = yield* Deferred.make(); + yield* Effect.gen(function* () { + const primary = yield* (yield* DesktopBackendPool.DesktopBackendPool).primary; + yield* primary.start; + yield* TestClock.adjust(Duration.seconds(1)); + yield* Deferred.await(ready); + + assert.deepEqual(events, ["reload", "ready"]); + }).pipe( + Effect.provide( + layerPool(Ref.makeUnsafe("WSL"), { + resolvePrimary, + spawner: ChildProcessSpawner.make(() => + Effect.gen(function* () { + const exited = yield* Deferred.make(); + const exit = Deferred.succeed(exited, ChildProcessSpawner.ExitCode(0)).pipe( + Effect.asVoid, + ); + yield* Effect.addFinalizer(() => exit); + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(123), + stdout: Stream.empty, + stderr: Stream.empty, + all: Stream.empty, + exitCode: Deferred.await(exited), + isRunning: Effect.succeed(true), + kill: () => exit, + stdin: Sink.drain, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }); + }), + ), + httpClient: HttpClient.make((request) => + Effect.succeed( + HttpClientResponse.fromWeb(request, new Response(null, { status: 200 })), + ), + ), + onMainContentsCommand: (command) => Effect.sync(() => void events.push(command)), + onReady: Effect.sync(() => void events.push("ready")).pipe( + Effect.andThen(Deferred.succeed(ready, undefined)), + Effect.asVoid, + ), + }), + ), + ); + }), ); }); diff --git a/apps/desktop/src/backend/DesktopBackendPool.ts b/apps/desktop/src/backend/DesktopBackendPool.ts index 82d542b322c8..62c811996b1a 100644 --- a/apps/desktop/src/backend/DesktopBackendPool.ts +++ b/apps/desktop/src/backend/DesktopBackendPool.ts @@ -1,10 +1,17 @@ +import { + DESKTOP_UPDATE_IN_PROGRESS_MARKER_FILE, + DESKTOP_UPDATE_IN_PROGRESS_TTL_MS, +} from "@t3tools/contracts"; import * as Context from "effect/Context"; import * as Deferred from "effect/Deferred"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; +import * as FiberHandle from "effect/FiberHandle"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import * as Schedule from "effect/Schedule"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as SubscriptionRef from "effect/SubscriptionRef"; @@ -31,6 +38,10 @@ import * as ElectronApp from "../electron/ElectronApp.ts"; const { logWarning: logBackendPoolWarning } = DesktopObservability.makeComponentLogger("desktop-backend-pool"); +const decodeServerRuntime = Schema.decodeUnknownEffect( + Schema.fromJsonString(Schema.Struct({ pid: Schema.Number, port: Schema.Number })), +); + export type BackendInstanceId = DesktopBackendManager.BackendInstanceId; export const BackendInstanceId = DesktopBackendManager.BackendInstanceId; export const PRIMARY_INSTANCE_ID = DesktopBackendManager.PRIMARY_INSTANCE_ID; @@ -106,6 +117,8 @@ type UnregisterAction = export const layer = Layer.effect( DesktopBackendPool, Effect.gen(function* () { + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const fileSystem = yield* FileSystem.FileSystem; const configuration = yield* DesktopBackendConfiguration.DesktopBackendConfiguration; const desktopWindow = yield* DesktopWindow.DesktopWindow; const electronDialog = yield* ElectronDialog.ElectronDialog; @@ -119,10 +132,57 @@ export const layer = Layer.effect( FileSystem.FileSystem | Path.Path | DesktopEnvironment.DesktopEnvironment >(); + const updateMarkerPath = environment.path.join( + environment.baseDir, + "runtime", + DESKTOP_UPDATE_IN_PROGRESS_MARKER_FILE, + ); + const serverRuntimePath = environment.path.join(environment.stateDir, "server-runtime.json"); + // The server publishes its runtime after HTTP readiness. Until then an SSH launcher + // finds nothing to adopt, so dropping the marker earlier lets it start a competing server. + const releaseUpdateMarker = Effect.fn("desktop.backendPool.releaseUpdateMarker")(function* ( + httpBaseUrl: URL, + pid: number, + ) { + const marker = yield* fileSystem.readFileString(updateMarkerPath).pipe(Effect.option); + if (Option.isNone(marker)) { + return; + } + yield* fileSystem.readFileString(serverRuntimePath).pipe( + Effect.flatMap(decodeServerRuntime), + Effect.map((runtime) => runtime.pid === pid && runtime.port === Number(httpBaseUrl.port)), + Effect.orElseSucceed(() => false), + Effect.repeat({ + until: (published) => published, + schedule: Schedule.spaced(Duration.millis(100)), + }), + Effect.timeoutOption(Duration.millis(DESKTOP_UPDATE_IN_PROGRESS_TTL_MS)), + ); + // A later update may have replaced the marker; that handoff is not ours to end. + const current = yield* fileSystem.readFileString(updateMarkerPath).pipe(Effect.option); + if (Option.isSome(current) && current.value === marker.value) { + yield* fileSystem.remove(updateMarkerPath, { force: true }).pipe(Effect.ignore); + } + }); + // One wait at a time: each run's shutdown or readiness ends the previous run's wait. + const updateMarkerWait = yield* FiberHandle.make(); + + // The main window loads during startup and reads the primary endpoint once; a Windows fallback + // before the first readiness replaces the WSL endpoint it read. + let primaryWasReady = false; + let reloadMainOnReady = false; + const reloadStaleMain = Effect.suspend(() => { + primaryWasReady = true; + if (!reloadMainOnReady) return Effect.void; + reloadMainOnReady = false; + return desktopWindow.runMainContentsCommand("reload"); + }); + // Persist fatal Windows fallbacks; transient transport failures must preserve the user's WSL preference. const handlePrimaryPreflightFailure = Effect.fn("desktop.backendPool.primaryPreflightFailed")( function* (failure: DesktopBackendManager.PreflightFailure) { const { reason, fatal } = failure; + if (!primaryWasReady) reloadMainOnReady = true; if (!fatal) { yield* logBackendPoolWarning( "primary WSL preflight retry window exhausted; using Windows for this launch", @@ -165,8 +225,10 @@ export const layer = Layer.effect( label: configuration.resolvePrimaryLabel, configResolve: configuration.resolvePrimary, // Absorb window errors so readiness can reset restartAttempt. - onReady: (httpBaseUrl) => - desktopWindow.handleBackendReady(httpBaseUrl).pipe( + onReady: (httpBaseUrl, pid) => + FiberHandle.run(updateMarkerWait, releaseUpdateMarker(httpBaseUrl, pid)).pipe( + Effect.andThen(reloadStaleMain), + Effect.andThen(desktopWindow.handleBackendReady(httpBaseUrl)), // Only once the main window exists, so the recovery window never closes into nothing. Effect.andThen(SubscriptionRef.set(primaryBackendStartup, { _tag: "Ready" })), Effect.catch((error) => @@ -177,7 +239,10 @@ export const layer = Layer.effect( ), onStartupFailure: (failure) => SubscriptionRef.set(primaryBackendStartup, { _tag: "Failing", ...failure }), - onShutdown: () => desktopWindow.handleBackendNotReady, + onShutdown: () => + FiberHandle.clear(updateMarkerWait).pipe( + Effect.andThen(desktopWindow.handleBackendNotReady), + ), onPreflightFailed: handlePrimaryPreflightFailure, onStateDirOwned: () => DesktopHomeOwner.readLiveHomeOwner().pipe( diff --git a/apps/desktop/src/backend/DesktopLocalEnvironmentAuth.test.ts b/apps/desktop/src/backend/DesktopLocalEnvironmentAuth.test.ts index e82e3afea730..c15ce3932bb8 100644 --- a/apps/desktop/src/backend/DesktopLocalEnvironmentAuth.test.ts +++ b/apps/desktop/src/backend/DesktopLocalEnvironmentAuth.test.ts @@ -1,11 +1,13 @@ import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import { assert, describe, it } from "@effect/vitest"; +import * as Clock from "effect/Clock"; import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Ref from "effect/Ref"; +import * as Result from "effect/Result"; import * as HttpClient from "effect/http/HttpClient"; import * as HttpClientRequest from "effect/http/HttpClientRequest"; import * as HttpClientResponse from "effect/http/HttpClientResponse"; @@ -238,4 +240,34 @@ describe("DesktopLocalEnvironmentAuth", () => { assert.strictEqual(yield* Ref.get(requestCount), 1); }).pipe(Effect.provide(TestClock.layer())), ); + + it.effect("waits through a backend that stalls for 30 seconds after it is ready", () => + Effect.gen(function* () { + const requestCount = yield* Ref.make(0); + const layerHttpClient = Layer.succeed( + HttpClient.HttpClient, + HttpClient.make((request) => + Effect.gen(function* () { + yield* Ref.update(requestCount, (count) => count + 1); + const now = yield* Clock.currentTimeMillis; + return now < 30_000 ? yield* Effect.never : tokenResponse(request); + }), + ), + ); + const auth = yield* DesktopLocalEnvironmentAuth.DesktopLocalEnvironmentAuth.pipe( + Effect.provide( + DesktopLocalEnvironmentAuth.layer.pipe( + Layer.provide(Layer.mergeAll(layerPool, layerHttpClient, NodeCrypto.layerMac)), + ), + ), + ); + + const fiber = yield* auth.getBearerToken.pipe(Effect.result, Effect.forkChild); + yield* TestClock.adjust(Duration.seconds(45)); + const result = yield* Fiber.join(fiber); + + assert.isTrue(Result.isSuccess(result)); + assert.strictEqual(yield* Ref.get(requestCount), 4); + }).pipe(Effect.provide(TestClock.layer())), + ); }); diff --git a/apps/desktop/src/backend/DesktopLocalEnvironmentAuth.ts b/apps/desktop/src/backend/DesktopLocalEnvironmentAuth.ts index 6c7d24623d40..89aa3cd2a678 100644 --- a/apps/desktop/src/backend/DesktopLocalEnvironmentAuth.ts +++ b/apps/desktop/src/backend/DesktopLocalEnvironmentAuth.ts @@ -17,7 +17,8 @@ import * as HttpClient from "effect/http/HttpClient"; import * as DesktopBackendPool from "./DesktopBackendPool.ts"; -const BOOTSTRAP_TRANSIENT_RETRY_TIMEOUT = Duration.seconds(15); +// Each /oauth/token attempt may take 10 s, so a backend busy after ready needs several attempts. +const BOOTSTRAP_TRANSIENT_RETRY_TIMEOUT = Duration.seconds(60); const BOOTSTRAP_TRANSIENT_RETRY_INTERVAL = Duration.millis(500); const TRANSIENT_BOOTSTRAP_STATUS_CODES = new Set([502, 503, 504]); diff --git a/apps/desktop/src/backend/tailscaleEndpointProvider.test.ts b/apps/desktop/src/backend/tailscaleEndpointProvider.test.ts index 054655917cec..397a75770483 100644 --- a/apps/desktop/src/backend/tailscaleEndpointProvider.test.ts +++ b/apps/desktop/src/backend/tailscaleEndpointProvider.test.ts @@ -1,7 +1,7 @@ import { assert, describe, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; -import { HttpClient } from "effect/http"; +import { HttpClient, HttpClientResponse } from "effect/http"; import { ChildProcessSpawner } from "effect/process"; import { resolveTailscaleAdvertisedEndpoints } from "./tailscaleEndpointProvider.ts"; @@ -79,4 +79,41 @@ describe("tailscale endpoint provider", () => { ); }).pipe(Effect.provide(layerUnusedTailscaleExternalServices)), ); + + it.effect("offers Tailscale HTTPS only when Serve reaches this desktop's backend", () => + Effect.gen(function* () { + const magicDnsStatus = (servedEnvironmentId: string) => + resolveTailscaleAdvertisedEndpoints({ + port: 3773, + serveEnabled: true, + networkInterfaces: {}, + statusJson: `{"Self":{"DNSName":"desktop.example.ts.net.","TailscaleIPs":[]}}`, + }).pipe( + Effect.map((endpoints) => endpoints.map((endpoint) => endpoint.status)), + Effect.provideService( + HttpClient.HttpClient, + HttpClient.make((request) => + Effect.succeed( + HttpClientResponse.fromWeb( + request, + Response.json({ + environmentId: + new URL(request.url).hostname === "127.0.0.1" + ? "desktop-backend" + : servedEnvironmentId, + label: "T3 server", + platform: { os: "linux", arch: "x64" }, + serverVersion: "0.1.9", + capabilities: { repositoryIdentity: true }, + }), + ), + ), + ), + ), + ); + + assert.deepEqual(yield* magicDnsStatus("desktop-backend"), ["available"]); + assert.deepEqual(yield* magicDnsStatus("another-server"), ["unavailable"]); + }).pipe(Effect.provide(layerUnusedTailscaleExternalServices)), + ); }); diff --git a/apps/desktop/src/backend/tailscaleEndpointProvider.ts b/apps/desktop/src/backend/tailscaleEndpointProvider.ts index 4b75afccdb5b..1b8d9dff4ed4 100644 --- a/apps/desktop/src/backend/tailscaleEndpointProvider.ts +++ b/apps/desktop/src/backend/tailscaleEndpointProvider.ts @@ -1,5 +1,9 @@ import { createAdvertisedEndpoint } from "@t3tools/shared/advertisedEndpoint"; -import type { AdvertisedEndpoint, AdvertisedEndpointProvider } from "@t3tools/contracts"; +import { + ExecutionEnvironmentDescriptor, + type AdvertisedEndpoint, + type AdvertisedEndpointProvider, +} from "@t3tools/contracts"; import { buildTailscaleHttpsBaseUrl, isTailscaleIpv4Address, @@ -11,6 +15,7 @@ import { import * as Effect from "effect/Effect"; import * as Option from "effect/Option"; import * as HttpClient from "effect/http/HttpClient"; +import * as HttpClientResponse from "effect/http/HttpClientResponse"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import type { NetworkInterfaces } from "./DesktopNetworkInterfaces.ts"; @@ -71,10 +76,20 @@ function resolveTailscaleIpAdvertisedEndpoints(input: { return endpoints; } +// The backend may run elsewhere (WSL), so ask it for its id instead of reading the state dir. +const probeServesThisBackend = (localHttpUrl: string) => (baseUrl: string) => + HttpClient.get(new URL("/.well-known/t3/environment", localHttpUrl)).pipe( + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap(HttpClientResponse.schemaBodyJson(ExecutionEnvironmentDescriptor)), + Effect.flatMap(({ environmentId }) => probeTailscaleHttpsEndpoint({ baseUrl, environmentId })), + Effect.orElseSucceed(() => false), + ); + const resolveTailscaleMagicDnsAdvertisedEndpoint = Effect.fn( "resolveTailscaleMagicDnsAdvertisedEndpoint", )(function* (input: { readonly dnsName: string | null; + readonly localHttpUrl: string; readonly serveEnabled: boolean; readonly servePort?: number; readonly probe?: (baseUrl: string) => Effect.Effect; @@ -87,11 +102,7 @@ const resolveTailscaleMagicDnsAdvertisedEndpoint = Effect.fn( magicDnsName: input.dnsName, ...(input.servePort === undefined ? {} : { servePort: input.servePort }), }); - const probe = - input.probe?.(httpBaseUrl) ?? - probeTailscaleHttpsEndpoint({ - baseUrl: httpBaseUrl, - }); + const probe = (input.probe ?? probeServesThisBackend(input.localHttpUrl))(httpBaseUrl); const isReachable = input.serveEnabled ? yield* probe : false; return Option.some( @@ -140,6 +151,7 @@ export const resolveTailscaleAdvertisedEndpoints = Effect.fn("resolveTailscaleAd const ipEndpoints = resolveTailscaleIpAdvertisedEndpoints({ ...input, status }); const magicDnsEndpoint = yield* resolveTailscaleMagicDnsAdvertisedEndpoint({ dnsName: status?.magicDnsName ?? null, + localHttpUrl: `http://127.0.0.1:${input.port}`, serveEnabled: input.serveEnabled === true, ...(input.servePort === undefined ? {} : { servePort: input.servePort }), ...(input.probe === undefined ? {} : { probe: input.probe }), diff --git a/apps/desktop/src/electron/ElectronAudio.test.ts b/apps/desktop/src/electron/ElectronAudio.test.ts new file mode 100644 index 000000000000..b38d7bbb3ae4 --- /dev/null +++ b/apps/desktop/src/electron/ElectronAudio.test.ts @@ -0,0 +1,60 @@ +import * as Effect from "effect/Effect"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import { app } from "electron"; +import { expect, vi } from "vite-plus/test"; +import { it } from "@effect/vitest"; +import * as Fiber from "effect/Fiber"; + +const { getAppMetrics, kill } = vi.hoisted(() => ({ getAppMetrics: vi.fn(), kill: vi.fn() })); +vi.mock("node:process", () => ({ kill })); +vi.mock("electron", async () => { + const { EventEmitter } = await import("node:events"); + return { app: Object.assign(new EventEmitter(), { getAppMetrics }) }; +}); + +import { restart } from "./ElectronAudio.ts"; + +const audio = { pid: 42, type: "Utility", serviceName: "audio.mojom.AudioService" }; + +it.effect("coalesces recovery and waits for its own audio utility to exit", () => + Effect.gen(function* () { + getAppMetrics.mockReturnValue([ + { pid: 1, type: "Utility", serviceName: "network.mojom.NetworkService" }, + { pid: 2, type: "Browser", serviceName: audio.serviceName }, + audio, + ]); + const started = Promise.withResolvers(); + kill.mockReset().mockImplementation(() => { + started.resolve(); + }); + const pending = yield* Effect.forkChild( + Effect.all([restart(), restart()], { concurrency: "unbounded" }), + ); + yield* Effect.promise(() => started.promise); + app.emit( + "child-process-gone", + {}, + { type: "Utility", serviceName: "network.mojom.NetworkService" }, + ); + expect(app.listenerCount("child-process-gone")).toBe(1); + app.emit("child-process-gone", {}, audio); + expect(yield* Fiber.join(pending)).toEqual([true, true]); + expect(kill.mock.calls).toEqual([[42, "SIGTERM"]]); + expect(app.listenerCount("child-process-gone")).toBe(0); + }).pipe(Effect.provideService(HostProcess.Platform, "linux")), +); + +it.effect.each(["linux", "darwin", "win32"] as const)( + "leaves other processes alone on %s", + (platform) => + Effect.gen(function* () { + getAppMetrics.mockReturnValue([ + { pid: 1, type: "Utility", serviceName: "network.mojom.NetworkService" }, + ]); + kill.mockReset(); + expect(yield* restart().pipe(Effect.provideService(HostProcess.Platform, platform))).toBe( + false, + ); + expect(kill).not.toHaveBeenCalled(); + }), +); diff --git a/apps/desktop/src/electron/ElectronAudio.ts b/apps/desktop/src/electron/ElectronAudio.ts new file mode 100644 index 000000000000..6a7f290d6fd8 --- /dev/null +++ b/apps/desktop/src/electron/ElectronAudio.ts @@ -0,0 +1,62 @@ +// @effect-diagnostics globalTimers:off -- The shared native restart deadline must outlive individual IPC callers. +// @effect-diagnostics nodeBuiltinImport:off -- Only Electron's main process can restart its Chromium audio utility. +import * as NodeProcess from "node:process"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import { app, type Details, type Event } from "electron"; + +class AudioServiceRestartError extends Schema.TaggedError()( + "AudioServiceRestartError", + { cause: Schema.Defect() }, +) {} + +let pendingRestart: Promise | undefined; + +export const restart = Effect.fn("ElectronAudio.restart")(function* () { + if ((yield* HostProcess.Platform) !== "linux") return false; + return yield* Effect.tryPromise({ + try: () => { + if (pendingRestart) return pendingRestart; + const audio = app + .getAppMetrics() + .find( + (metric) => + metric.type === "Utility" && metric.serviceName === "audio.mojom.AudioService", + ); + if (!audio) return Promise.resolve(false); + // A failed PulseAudio context survives new getUserMedia calls; Chromium reconnects when its utility exits. + pendingRestart = new Promise((resolve, reject) => { + const cleanup = () => { + clearTimeout(timer); + app.removeListener("child-process-gone", onGone); + }; + const onGone = (_event: Event, details: Details) => { + if (details.type === "Utility" && details.serviceName === audio.serviceName) { + cleanup(); + resolve(true); + } + }; + const timer = setTimeout(() => { + cleanup(); + resolve(false); + }, 5_000); + app.on("child-process-gone", onGone); + try { + NodeProcess.kill(audio.pid, "SIGTERM"); + } catch (cause) { + cleanup(); + reject(cause); + } + }).finally(() => { + pendingRestart = undefined; + }); + return pendingRestart; + }, + catch: (cause) => new AudioServiceRestartError({ cause }), + }).pipe( + Effect.catch((cause) => + Effect.logWarning("Could not restart the audio service", cause).pipe(Effect.as(false)), + ), + ); +}); diff --git a/apps/desktop/src/electron/ElectronMenu.ts b/apps/desktop/src/electron/ElectronMenu.ts index 4ff3df6bf10f..540326346edf 100644 --- a/apps/desktop/src/electron/ElectronMenu.ts +++ b/apps/desktop/src/electron/ElectronMenu.ts @@ -1,4 +1,4 @@ -import type { ContextMenuItem } from "@t3tools/contracts"; +import type { ContextMenuItem, ContextMenuSource } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; @@ -17,6 +17,7 @@ export interface ElectronMenuContextInput { readonly window: Electron.BrowserWindow; readonly items: readonly ContextMenuItem[]; readonly position: Option.Option; + readonly source: Option.Option; } export interface ElectronMenuTemplateInput { @@ -259,7 +260,11 @@ export const make = Effect.gen(function* () { callback: () => complete(Option.none()), }), }); - menu.popup(popupOptions); + menu.popup( + Option.isSome(input.source) + ? { ...popupOptions, sourceType: input.source.value } + : popupOptions, + ); } catch (cause) { if (completed) { return; diff --git a/apps/desktop/src/electron/ElectronProtocol.test.ts b/apps/desktop/src/electron/ElectronProtocol.test.ts index aec3a2d5b45c..b0cf4823126e 100644 --- a/apps/desktop/src/electron/ElectronProtocol.test.ts +++ b/apps/desktop/src/electron/ElectronProtocol.test.ts @@ -63,7 +63,7 @@ describe("ElectronProtocol", () => { assert.equal((yield* request("/%invalid")).status, 400); assert.equal((yield* request("/", { method: "POST" })).status, 405); assert.equal(netFetchMock.mock.calls.length, 0); - }).pipe(Effect.provide(Layer.merge(layerProtocol, NodeServices.layer)), Effect.scoped), + }).pipe(Effect.provide(Layer.merge(layerProtocol, NodeServices.layer))), ); it.effect("proxies the stable renderer origin to the current app server", () => diff --git a/apps/desktop/src/electron/ElectronShell.test.ts b/apps/desktop/src/electron/ElectronShell.test.ts index 9864c19b0d0a..e8618b4f5cc4 100644 --- a/apps/desktop/src/electron/ElectronShell.test.ts +++ b/apps/desktop/src/electron/ElectronShell.test.ts @@ -54,18 +54,30 @@ describe("ElectronShell", () => { }).pipe(Effect.provide(ElectronShell.layer)), ); - it.effect("opens only the bare Grok Bot launch link", () => + it.effect("opens only the Grok Bot prompt link", () => Effect.gen(function* () { openExternalMock.mockResolvedValue(undefined); + const prompt = "grokbot://app/v1/prompt?text=Add%20the%20MCP%20server"; const electronShell = yield* ElectronShell.ElectronShell; const results = yield* Effect.forEach( - ["grokbot://app", "grokbot://app/v1/prompt?id=bot&text=hi", "grokbot://"], + [ + prompt, + "grokbot://app", + "grokbot://app/v1/prompt?id=bot&text=hi", + "grokbot://app/v1/prompt?text=a&text=b", + "grokbot://app/v1/other?text=hi", + "grokbot://app/v1/prompt?text=hi#x", + "grokbot://:secret@app/v1/prompt?text=hi", + "grokbot://user@app/v1/prompt?text=hi", + "grokbot://other/v1/prompt?text=hi", + "other://app/v1/prompt?text=hi", + ], (url) => electronShell.openExternal(url), ); - assert.deepEqual(results, [true, false, false]); - assert.deepEqual(openExternalMock.mock.calls, [["grokbot://app"]]); + assert.deepEqual(results, [true, ...Array(9).fill(false)]); + assert.deepEqual(openExternalMock.mock.calls, [[prompt]]); }).pipe(Effect.provide(ElectronShell.layer)), ); diff --git a/apps/desktop/src/electron/ElectronShell.ts b/apps/desktop/src/electron/ElectronShell.ts index e342361750f4..d27c51d57c6e 100644 --- a/apps/desktop/src/electron/ElectronShell.ts +++ b/apps/desktop/src/electron/ElectronShell.ts @@ -21,9 +21,16 @@ const REMOTE_EDITOR_PROTOCOLS = new Set( }), ); -// "Use T3 Code in another app" opens these apps (apps/web/src/components/mcp/mcpApps.ts). List +// "Use T3 Code in another app" opens these apps (apps/web/src/components/mcp/mcpApps.ts). Match // each exact link shape the wizard sends, never a whole scheme. -const APP_LAUNCH_URLS = new Set(["grokbot://app"]); +const isAppLaunchUrl = (url: URL) => + url.protocol === "grokbot:" && + url.host === "app" && + url.pathname === "/v1/prompt" && + url.username.length === 0 && + url.password.length === 0 && + url.hash.length === 0 && + [...url.searchParams.keys()].join() === "text"; // Zed embeds [user@]host in the path; reject passwords and ports there. const ZED_SSH_PATHNAME = /^\/(?:[^/@:]+@)?[^/@:]+\/.*$/; @@ -50,7 +57,7 @@ export function parseSafeExternalUrl(rawUrl: unknown): Option.Option { try { const url = new URL(rawUrl); return SAFE_WEB_PROTOCOLS.has(url.protocol) || - APP_LAUNCH_URLS.has(url.href) || + isAppLaunchUrl(url) || isRemoteEditorUrl(url) || isLocalEditorFileUrl(url) ? Option.some(url.href) diff --git a/apps/desktop/src/electron/ElectronTheme.test.ts b/apps/desktop/src/electron/ElectronTheme.test.ts index 4239bfe4c5bc..47aa0ce1f8df 100644 --- a/apps/desktop/src/electron/ElectronTheme.test.ts +++ b/apps/desktop/src/electron/ElectronTheme.test.ts @@ -42,7 +42,7 @@ describe("ElectronTheme", () => { assert.equal(nativeTheme.themeSource, "light"); portal.report(null); assert.equal(nativeTheme.themeSource, "system"); - }).pipe(Effect.scoped), + }), ); it.effect("keeps an explicit appearance over the desktop color scheme", () => @@ -57,7 +57,7 @@ describe("ElectronTheme", () => { yield* theme.setSource("system"); assert.equal(nativeTheme.themeSource, "dark"); - }).pipe(Effect.scoped), + }), ); it.effect("stops following the desktop when the layer is released", () => @@ -76,6 +76,6 @@ describe("ElectronTheme", () => { }); yield* theme.setSource("system"); assert.equal(nativeTheme.themeSource, "system"); - }).pipe(Effect.scoped), + }), ); }); diff --git a/apps/desktop/src/electron/ElectronWindow.test.ts b/apps/desktop/src/electron/ElectronWindow.test.ts index 9eb41ed4e240..eac944722e73 100644 --- a/apps/desktop/src/electron/ElectronWindow.test.ts +++ b/apps/desktop/src/electron/ElectronWindow.test.ts @@ -148,6 +148,34 @@ describe("ElectronWindow", () => { }).pipe(Effect.provide(layerTestFor("win32"))), ); + it.effect("reports which windows were created with a title bar overlay", () => + Effect.gen(function* () { + const framed = { isDestroyed: () => false }; + const overlay = { isDestroyed: () => false }; + browserWindowMock + .mockImplementationOnce(function FramedWindowMock() { + return framed; + }) + .mockImplementationOnce(function OverlayWindowMock() { + return overlay; + }); + const electronWindow = yield* ElectronWindow.ElectronWindow; + yield* electronWindow.create({}); + yield* electronWindow.create({ + titleBarStyle: "hidden", + titleBarOverlay: { color: "#01000000", height: 40 }, + }); + getAllWindowsMock.mockReturnValue([framed, overlay, { isDestroyed: () => false }]); + + const seen: Array = []; + yield* electronWindow.syncAllAppearance((_window, { titleBarOverlay }) => + Effect.sync(() => seen.push(titleBarOverlay)), + ); + + assert.deepEqual(seen, [false, true, false]); + }).pipe(Effect.provide(layerTestFor("win32"))), + ); + it.effect("preserves destroy failures and continues with later windows", () => Effect.gen(function* () { const cause = new Error("window destroy failed"); diff --git a/apps/desktop/src/electron/ElectronWindow.ts b/apps/desktop/src/electron/ElectronWindow.ts index 15a117a52b05..b5f89969e66c 100644 --- a/apps/desktop/src/electron/ElectronWindow.ts +++ b/apps/desktop/src/electron/ElectronWindow.ts @@ -125,7 +125,10 @@ export class ElectronWindow extends Context.Service< readonly sendAll: (channel: string, ...args: readonly unknown[]) => Effect.Effect; readonly destroyAll: Effect.Effect; readonly syncAllAppearance: ( - sync: (window: Electron.BrowserWindow) => Effect.Effect, + sync: ( + window: Electron.BrowserWindow, + appearance: { readonly titleBarOverlay: boolean }, + ) => Effect.Effect, ) => Effect.Effect; } >()("@t3tools/desktop/electron/ElectronWindow") {} @@ -145,6 +148,7 @@ export const make = Effect.gen(function* () { yield* Effect.addFinalizer(() => Effect.sync(() => windowsForegroundFocus?.close())); const mainWindowRef = yield* Ref.make>(Option.none()); const auxiliaryWindows = new WeakSet(); + const titleBarOverlayWindows = new WeakSet(); const listWindows = Effect.try({ try: () => Electron.BrowserWindow.getAllWindows(), @@ -241,6 +245,9 @@ export const make = Effect.gen(function* () { try: () => { const window = new Electron.BrowserWindow(options); if (behavior?.auxiliary === true) auxiliaryWindows.add(window); + if (options.titleBarOverlay !== undefined && options.titleBarOverlay !== false) { + titleBarOverlayWindows.add(window); + } return window; }, catch: (cause) => new ElectronWindowCreateError({ options: diagnosticOptions, cause }), @@ -383,14 +390,17 @@ export const make = Effect.gen(function* () { } }), syncAllAppearance: Effect.fn("desktop.electron.window.syncAllAppearance")(function* ( - sync: (window: Electron.BrowserWindow) => Effect.Effect, + sync: ( + window: Electron.BrowserWindow, + appearance: { readonly titleBarOverlay: boolean }, + ) => Effect.Effect, ) { const windows = yield* listWindows; for (const window of windows) { if (yield* isWindowDestroyed(window)) { continue; } - yield* sync(window); + yield* sync(window, { titleBarOverlay: titleBarOverlayWindows.has(window) }); } }), }); diff --git a/apps/desktop/src/ipc/DesktopIpc.test.ts b/apps/desktop/src/ipc/DesktopIpc.test.ts index 45f63b951f36..1973ab49cefe 100644 --- a/apps/desktop/src/ipc/DesktopIpc.test.ts +++ b/apps/desktop/src/ipc/DesktopIpc.test.ts @@ -63,5 +63,5 @@ it.effect("answers only the main window's top frame", () => appSubframe: ["rejected", undefined], previewGuest: ["rejected", undefined], }); - }).pipe(Effect.scoped), + }), ); diff --git a/apps/desktop/src/ipc/DesktopIpcHandlers.ts b/apps/desktop/src/ipc/DesktopIpcHandlers.ts index 842ecf431b98..21d2a16da814 100644 --- a/apps/desktop/src/ipc/DesktopIpcHandlers.ts +++ b/apps/desktop/src/ipc/DesktopIpcHandlers.ts @@ -49,12 +49,14 @@ import { getWindowFullscreenState, openExternal, readMedia, + restartAudioService, openSystemSettings, checkSystemPermission, pasteAsText, probeRemoteEditors, revealWindow, pickFolder, + toLocalBackendPath, pickProjectFavicon, pickThemeFiles, setTheme, @@ -119,6 +121,7 @@ export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers" yield* ipc.handle(getHostEnvironmentId); yield* ipc.handle(getLocalEnvironmentBearerToken); + yield* ipc.handle(restartAudioService); yield* ipc.handle(getClientSettings); yield* ipc.handle(setClientSettings); yield* ipc.handle(getConnectionCatalog); @@ -157,6 +160,7 @@ export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers" yield* ipc.handle(setSshRunner); yield* ipc.handle(pickFolder); + yield* ipc.handle(toLocalBackendPath); yield* ipc.handle(pickProjectFavicon); yield* ipc.handle(pickThemeFiles); yield* ipc.handle(setTheme); diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts index 7df5ffe7f879..a7e099eb32fc 100644 --- a/apps/desktop/src/ipc/channels.ts +++ b/apps/desktop/src/ipc/channels.ts @@ -1,4 +1,5 @@ export const PICK_FOLDER_CHANNEL = "desktop:pick-folder"; +export const TO_LOCAL_BACKEND_PATH_CHANNEL = "desktop:to-local-backend-path"; export const SET_NOTIFICATION_BADGE_CHANNEL = "desktop:set-notification-badge"; export const PICK_PROJECT_FAVICON_CHANNEL = "desktop:pick-project-favicon"; export const PICK_THEME_FILES_CHANNEL = "desktop:pick-theme-files"; @@ -89,6 +90,7 @@ export const PREVIEW_REGISTER_WEBVIEW_CHANNEL = "desktop:preview-register-webvie export const PREVIEW_NAVIGATE_CHANNEL = "desktop:preview-navigate"; export const PREVIEW_GO_BACK_CHANNEL = "desktop:preview-go-back"; export const PREVIEW_GO_FORWARD_CHANNEL = "desktop:preview-go-forward"; +export const PREVIEW_STOP_LOADING_CHANNEL = "desktop:preview-stop-loading"; export const PREVIEW_REFRESH_CHANNEL = "desktop:preview-refresh"; export const PREVIEW_ZOOM_IN_CHANNEL = "desktop:preview-zoom-in"; export const PREVIEW_ZOOM_OUT_CHANNEL = "desktop:preview-zoom-out"; @@ -100,6 +102,7 @@ export const PREVIEW_SET_AUDIO_MUTED_CHANNEL = "desktop:preview-set-audio-muted" export const PREVIEW_FIND_IN_PAGE_CHANNEL = "desktop:preview-find-in-page"; export const PREVIEW_STOP_FIND_IN_PAGE_CHANNEL = "desktop:preview-stop-find-in-page"; export const PREVIEW_SET_SHOWN_CHANNEL = "desktop:preview-set-shown"; +export const PREVIEW_SET_CAPTURE_DRAWN_CHANNEL = "desktop:preview-set-capture-drawn"; export const PREVIEW_OPEN_DEVTOOLS_CHANNEL = "desktop:preview-open-devtools"; export const PREVIEW_CLEAR_COOKIES_CHANNEL = "desktop:preview-clear-cookies"; export const PREVIEW_CLEAR_CACHE_CHANNEL = "desktop:preview-clear-cache"; @@ -141,3 +144,5 @@ export const RECEIVE_PROVIDER_AUTH_CALLBACK_CHANNEL = "desktop:receive-provider- export const CANCEL_PROVIDER_AUTH_CALLBACK_CHANNEL = "desktop:cancel-provider-auth-callback"; export const TAKE_LEGACY_LOCAL_STORAGE_CHANNEL = "desktop:take-legacy-local-storage"; export const COMPLETE_LEGACY_LOCAL_STORAGE_CHANNEL = "desktop:complete-legacy-local-storage"; + +export const RESTART_AUDIO_SERVICE_CHANNEL = "desktop:restart-audio-service"; diff --git a/apps/desktop/src/ipc/methods/preview.ts b/apps/desktop/src/ipc/methods/preview.ts index ea291b66b314..757e1e463f53 100644 --- a/apps/desktop/src/ipc/methods/preview.ts +++ b/apps/desktop/src/ipc/methods/preview.ts @@ -13,6 +13,7 @@ import { DesktopPreviewScreenshotArtifactSchema, DesktopPreviewSetAudioMutedInputSchema, DesktopPreviewSetShownInputSchema, + DesktopPreviewSetCaptureDrawnInputSchema, DesktopPreviewSetColorSchemeInputSchema, DesktopPreviewSetZoomFactorInputSchema, BrowserImportResult, @@ -168,6 +169,11 @@ const refresh = tabMethod( "desktop.ipc.preview.refresh", (manager, tabId) => manager.refresh(tabId), ); +const stopLoading = tabMethod( + IpcChannels.PREVIEW_STOP_LOADING_CHANNEL, + "desktop.ipc.preview.stopLoading", + (manager, tabId) => manager.stopLoading(tabId), +); const zoomIn = tabMethod( IpcChannels.PREVIEW_ZOOM_IN_CHANNEL, "desktop.ipc.preview.zoomIn", @@ -229,6 +235,15 @@ const setShown = DesktopIpc.makeIpcMethod({ yield* manager.setShown(tabId, shown); }), }); +const setCaptureDrawn = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.PREVIEW_SET_CAPTURE_DRAWN_CHANNEL, + payload: DesktopPreviewSetCaptureDrawnInputSchema, + result: Schema.Void, + handler: Effect.fn("desktop.ipc.preview.setCaptureDrawn")(function* ({ tabId, drawn }) { + const manager = yield* PreviewManager.PreviewManager; + yield* manager.setCaptureDrawn(tabId, drawn); + }), +}); const openDevTools = tabMethod( IpcChannels.PREVIEW_OPEN_DEVTOOLS_CHANNEL, "desktop.ipc.preview.openDevTools", @@ -472,6 +487,7 @@ export const methods = [ goBack, goForward, refresh, + stopLoading, zoomIn, zoomOut, resetZoom, @@ -482,6 +498,7 @@ export const methods = [ setZoomFactor, setAudioMuted, setShown, + setCaptureDrawn, openDevTools, clearCookies, clearCache, diff --git a/apps/desktop/src/ipc/methods/window.test.ts b/apps/desktop/src/ipc/methods/window.test.ts index 376f85d2baf9..de202529c411 100644 --- a/apps/desktop/src/ipc/methods/window.test.ts +++ b/apps/desktop/src/ipc/methods/window.test.ts @@ -1,6 +1,8 @@ import { assert, describe, it } from "@effect/vitest"; import * as NodeServices from "@effect/platform-node/NodeServices"; import type { ManagedPolicyLimits } from "@t3tools/contracts"; +import { EDITORS, REMOTE_CAPABLE_EDITOR_IDS } from "@t3tools/contracts"; +import { resolveEditorCommand } from "@t3tools/shared/editor"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { ManagedPolicyFileError } from "@t3tools/shared/managedPolicyFile"; import * as FileSystem from "effect/FileSystem"; @@ -25,6 +27,7 @@ import * as DesktopBackendManager from "../../backend/DesktopBackendManager.ts"; import * as DesktopBackendPool from "../../backend/DesktopBackendPool.ts"; import * as DesktopAppSettings from "../../settings/DesktopAppSettings.ts"; import * as DesktopManagedPolicy from "../../app/DesktopManagedPolicy.ts"; +import * as DesktopWslEnvironment from "../../wsl/DesktopWslEnvironment.ts"; import * as ElectronDialog from "../../electron/ElectronDialog.ts"; import * as ElectronWindow from "../../electron/ElectronWindow.ts"; import * as IpcChannels from "../channels.ts"; @@ -36,6 +39,7 @@ import { probeRemoteEditors, readMedia, syncHtmlFullscreen, + toLocalBackendPath, } from "./window.ts"; const readyWslConfig: DesktopBackendManager.DesktopBackendStartConfig = { @@ -185,6 +189,47 @@ describe("getLocalEnvironmentBootstraps", () => { }); }); +describe("toLocalBackendPath", () => { + const { runningDistro: _, ...windowsConfig } = readyWslConfig; + it.effect("converts paths through wslpath only for backends running in WSL", () => + Effect.gen(function* () { + const convert = (backendId: string, path = "C:\\Users\\me\\project") => + toLocalBackendPath.handler({ backendId, path }); + assert.strictEqual(yield* convert("primary"), "C:\\Users\\me\\project"); + assert.strictEqual(yield* convert("wsl:default"), "/drives/c/Users/me/project"); + assert.strictEqual(yield* convert("wsl:missing"), null); + assert.strictEqual( + yield* convert("wsl:default", "\\\\wsl.localhost\\ubuntu\\home\\me\\repo"), + "/home/me/repo", + ); + assert.strictEqual(yield* convert("wsl:default", "\\\\wsl$\\Debian\\home\\me\\repo"), null); + assert.strictEqual( + yield* convert("primary", "\\\\wsl$\\Debian\\home\\me\\repo"), + "\\\\wsl$\\Debian\\home\\me\\repo", + ); + }).pipe( + Effect.provide( + Layer.mergeAll( + DesktopBackendPool.layerTest([ + { + ...defaultWslInstance, + id: DesktopBackendManager.BackendInstanceId("primary"), + currentConfig: Effect.succeedSome(windowsConfig), + }, + defaultWslInstance, + ]), + DesktopWslEnvironment.layerTest({ + windowsToWslPath: (distro, windowsPath) => + distro === "Ubuntu" && windowsPath === "C:\\Users\\me\\project" + ? Option.some("/drives/c/Users/me/project") + : Option.none(), + }), + ), + ), + ), + ); +}); + describe("installRelayTelemetryEnabled", () => { // The renderer reads this over sync IPC, so the answer must not wait on the filesystem. const relayTelemetryEnabled = ( @@ -474,3 +519,108 @@ it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( assert.include(editors, "webstorm"); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); + +describe("probeRemoteEditors PATH scan", () => { + const platform = HostProcess.Platform.defaultValue(); + const launcherName = (command: string) => (platform === "win32" ? `${command}.cmd` : command); + + const makeFakePath = Effect.fnUntraced(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-probe-path-" }); + const dirs = ["a", "b", "c", "d"].map((name) => path.join(root, name)); + for (const dir of dirs) yield* fs.makeDirectory(dir, { recursive: true }); + // Launchers in different PATH directories, deliberately not in REMOTE_CAPABLE_EDITOR_IDS order. + for (const [dir, command] of [ + [dirs[2]!, "webstorm"], + [dirs[0]!, "cursor"], + [dirs[3]!, "code"], + [dirs[1]!, "zeditor"], + ] as const) { + const file = path.join(dir, launcherName(command)); + yield* fs.writeFileString(file, "#!/bin/sh\n"); + if (platform !== "win32") yield* fs.chmod(file, 0o755); + } + return { + root, + env: { + HOME: root, + USERPROFILE: root, + PATH: dirs.join(platform === "win32" ? ";" : ":"), + PATHEXT: ".COM;.EXE;.BAT;.CMD", + }, + }; + }); + + const countingFileSystem = (counter: { stat: number; reads: Map }) => + Layer.effect( + FileSystem.FileSystem, + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + return FileSystem.FileSystem.of({ + ...fs, + stat: (target) => { + counter.stat += 1; + return fs.stat(target); + }, + readDirectory: (target, options) => { + counter.reads.set(target, (counter.reads.get(target) ?? 0) + 1); + return fs.readDirectory(target, options); + }, + }); + }), + ).pipe(Layer.provideMerge(NodeServices.layer)); + + const serialReference = (env: Record) => + Effect.gen(function* () { + const available: Array = []; + for (const editorId of REMOTE_CAPABLE_EDITOR_IDS) { + const editor = EDITORS.find((candidate) => candidate.id === editorId); + if (editor && Option.isSome(yield* resolveEditorCommand(editor, env))) { + available.push(editorId); + } + } + return available; + }); + + it.effect("reads each PATH directory once instead of stat-ing every candidate per editor", () => + Effect.gen(function* () { + const { env: baseEnv } = yield* makeFakePath(); + const delimiter = platform === "win32" ? ";" : ":"; + const uncachedCounter = { stat: 0, reads: new Map() }; + const cachedCounter = { stat: 0, reads: new Map() }; + const uncachedEnv = { ...baseEnv, PATH: `${baseEnv.PATH}${delimiter}${baseEnv.HOME}x1` }; + const cachedEnv = { ...baseEnv, PATH: `${baseEnv.PATH}${delimiter}${baseEnv.HOME}x2` }; + const uncached = yield* serialReference(uncachedEnv).pipe( + Effect.provide(countingFileSystem(uncachedCounter)), + ); + const cached = yield* probeRemoteEditors + .handler(undefined) + .pipe( + Effect.provideService(HostProcess.Environment, cachedEnv), + Effect.provideService(HostProcess.Platform, platform), + Effect.provide(countingFileSystem(cachedCounter)), + ); + // The host can add editors from app folders the fixture can't redirect (e.g. /Applications). + const fixtureEditors = ["cursor", "vscode", "zed", "webstorm"]; + assert.deepStrictEqual( + uncached.filter((id) => fixtureEditors.includes(id)), + fixtureEditors, + ); + assert.deepStrictEqual(cached, uncached); + assert.isAbove(uncachedCounter.stat, 0); + // On every platform the probe lists each PATH directory at most once. Only PATH entries count: + // resolveEditorCommand also reads app folders such as /Applications on macOS. + const pathReads = cachedEnv.PATH.split(delimiter) + .filter((entry) => entry.length > 0) + .map((entry) => cachedCounter.reads.get(entry) ?? 0); + assert.isAbove(Math.max(...pathReads), 0); + assert.isAtMost(Math.max(...pathReads), 1); + // Without the listings, Windows stats every PATH entry x PATHEXT candidate for each editor; + // elsewhere there is no PATHEXT, so the stat count is about the same either way. + if (platform === "win32") { + assert.isBelow(cachedCounter.stat, uncachedCounter.stat / 2); + } + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +}); diff --git a/apps/desktop/src/ipc/methods/window.ts b/apps/desktop/src/ipc/methods/window.ts index 53a1138d71a9..38887ec71d90 100644 --- a/apps/desktop/src/ipc/methods/window.ts +++ b/apps/desktop/src/ipc/methods/window.ts @@ -1,5 +1,6 @@ import { ContextMenuItemSchema, + ContextMenuSource, DesktopAppBrandingSchema, DesktopEnvironmentBootstrapSchema, DesktopMediaReadSchema, @@ -16,6 +17,7 @@ import { } from "@t3tools/contracts"; import { PROJECT_FAVICON_EXTENSIONS } from "@t3tools/shared/projectFavicon"; import { resolveEditorCommand } from "@t3tools/shared/editor"; +import { withPathDirectoryListings } from "@t3tools/shared/shell"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { isRelayTracingEnabled } from "@t3tools/shared/otelEnvironment"; import * as FileSystem from "effect/FileSystem"; @@ -25,6 +27,7 @@ import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import * as DesktopBackendConfiguration from "../../backend/DesktopBackendConfiguration.ts"; +import * as DesktopBackendManager from "../../backend/DesktopBackendManager.ts"; import * as DesktopBackendPool from "../../backend/DesktopBackendPool.ts"; import * as DesktopLocalEnvironmentAuth from "../../backend/DesktopLocalEnvironmentAuth.ts"; import * as DesktopEnvironment from "../../app/DesktopEnvironment.ts"; @@ -34,6 +37,7 @@ import * as DesktopAppSettings from "../../settings/DesktopAppSettings.ts"; import * as DesktopWslBackend from "../../wsl/DesktopWslBackend.ts"; import * as DesktopWslEnvironment from "../../wsl/DesktopWslEnvironment.ts"; import * as ElectronApp from "../../electron/ElectronApp.ts"; +import * as ElectronAudio from "../../electron/ElectronAudio.ts"; import * as ElectronDialog from "../../electron/ElectronDialog.ts"; import * as ElectronMenu from "../../electron/ElectronMenu.ts"; import * as ElectronShell from "../../electron/ElectronShell.ts"; @@ -58,6 +62,7 @@ const ContextMenuPosition = Schema.Struct({ const ContextMenuInput = Schema.Struct({ items: Schema.Array(ContextMenuItemSchema), position: Schema.optionalKey(ContextMenuPosition), + source: Schema.optionalKey(ContextMenuSource), }); function toWebSocketBaseUrl(httpBaseUrl: URL): string { @@ -204,6 +209,20 @@ export const getLocalEnvironmentBearerToken = DesktopIpc.makeIpcMethod({ }), }); +// Ask the distro: its automount root is configurable, so /mnt/ is only the default. +const windowsPathToWsl = Effect.fn("desktop.ipc.window.windowsPathToWsl")(function* ( + distro: string | null, + windowsPath: string, +) { + const linuxUncPath = wslUncPathToLinuxPath(windowsPath); + if (linuxUncPath !== null) return Option.some(linuxUncPath); + const wslEnvironment = yield* DesktopWslEnvironment.DesktopWslEnvironment; + return yield* wslEnvironment.windowsToWslPath( + extractDistroFromUncPath(windowsPath) ?? distro, + windowsPath, + ); +}); + export const pickFolder = DesktopIpc.makeIpcMethod({ channel: IpcChannels.PICK_FOLDER_CHANNEL, payload: Schema.UndefinedOr(PickFolderOptionsSchema), @@ -250,20 +269,29 @@ export const pickFolder = DesktopIpc.makeIpcMethod({ if (!useWsl) { return selectedPath.value; } - - const linuxUncPath = wslUncPathToLinuxPath(selectedPath.value); - if (linuxUncPath !== null) { - return linuxUncPath; - } - - const converted = yield* wslEnvironment.windowsToWslPath( - extractDistroFromUncPath(selectedPath.value) ?? wslDistro, - selectedPath.value, - ); + const converted = yield* windowsPathToWsl(wslDistro, selectedPath.value); return Option.getOrElse(converted, () => selectedPath.value); }), }); +export const toLocalBackendPath = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.TO_LOCAL_BACKEND_PATH_CHANNEL, + payload: Schema.Struct({ backendId: Schema.String, path: Schema.String }), + result: Schema.NullOr(Schema.String), + handler: Effect.fn("desktop.ipc.window.toLocalBackendPath")(function* ({ backendId, path }) { + const pool = yield* DesktopBackendPool.DesktopBackendPool; + const instance = yield* pool.get(DesktopBackendManager.BackendInstanceId(backendId)); + if (Option.isNone(instance)) return null; + const config = yield* instance.value.currentConfig; + if (Option.isNone(config) || Option.isSome(config.value.preflightFailure)) return null; + const { runningDistro } = config.value; + if (runningDistro === undefined) return path; + const uncDistro = extractDistroFromUncPath(path); + if (uncDistro !== null && uncDistro.toLowerCase() !== runningDistro.toLowerCase()) return null; + return Option.getOrNull(yield* windowsPathToWsl(runningDistro, path)); + }), +}); + export const pickProjectFavicon = DesktopIpc.makeIpcMethod({ channel: IpcChannels.PICK_PROJECT_FAVICON_CHANNEL, payload: Schema.UndefinedOr(Schema.String), @@ -316,6 +344,7 @@ export const showContextMenu = DesktopIpc.makeIpcMethod({ window: window.value, items: input.items, position: Option.fromNullishOr(input.position), + source: Option.fromNullishOr(input.source), }); return Option.getOrNull(selectedItemId); }), @@ -400,20 +429,26 @@ export const openSystemSettings = DesktopIpc.makeIpcMethod({ }), }); +const REMOTE_EDITOR_PROBE_CONCURRENCY = 8; + export const probeRemoteEditors = DesktopIpc.makeIpcMethod({ channel: IpcChannels.PROBE_REMOTE_EDITORS_CHANNEL, payload: Schema.Undefined, result: Schema.Array(EditorId), handler: Effect.fn("desktop.ipc.window.probeRemoteEditors")(function* () { - const available: Array = []; const env = yield* HostProcess.Environment; - for (const editorId of REMOTE_CAPABLE_EDITOR_IDS) { - const editor = EDITORS.find((editor) => editor.id === editorId); - if (editor && Option.isSome(yield* resolveEditorCommand(editor, env))) { - available.push(editorId); - } - } - return available; + // Shared listings stop each editor re-stating every PATH entry x PATHEXT candidate on Windows. + const resolved = yield* Effect.forEach( + REMOTE_CAPABLE_EDITOR_IDS, + (editorId) => { + const editor = EDITORS.find((editor) => editor.id === editorId); + return editor + ? resolveEditorCommand(editor, env).pipe(Effect.map(Option.isSome)) + : Effect.succeed(false); + }, + { concurrency: REMOTE_EDITOR_PROBE_CONCURRENCY }, + ).pipe(withPathDirectoryListings); + return REMOTE_CAPABLE_EDITOR_IDS.filter((_, index) => resolved[index]); }), }); @@ -561,3 +596,10 @@ export const checkSystemPermission = DesktopIpc.makeIpcMethod({ return yield* Effect.promise(check); }), }); + +export const restartAudioService = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.RESTART_AUDIO_SERVICE_CHANNEL, + payload: Schema.Void, + result: Schema.Boolean, + handler: ElectronAudio.restart, +}); diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index ae8f1502cf99..5508c059b22b 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -1,10 +1,10 @@ import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import { resolveDesktopSshRunner } from "./ssh/DesktopSshRunner.ts"; import * as MacPermissions from "./permissions/MacPermissions.ts"; +import { handleStdioError } from "./app/stdioErrors.ts"; + for (const stream of [process.stdout, process.stderr]) { - stream.on("error", (err: NodeJS.ErrnoException) => { - if (err.code !== "EPIPE") throw err; - }); + stream.on("error", handleStdioError); } import * as NodeHttpClient from "@effect/platform-node/NodeHttpClient"; @@ -14,7 +14,9 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; // @effect-diagnostics-next-line nodeBuiltinImport:off - Version output must flush before Electron exits, without acquiring the runtime. import * as NodeFS from "node:fs"; import * as NodeOS from "node:os"; +import * as SecureRandom from "effect/crypto/SecureRandom"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; @@ -139,6 +141,15 @@ const layerDesktopSshEnvironment = Layer.unwrap( return DesktopSshEnvironment.layer({ resolveSshRunner, resolveCliRunner: Effect.succeed(resolveDesktopSshCliRunner(environment)), + resolveLauncherId: yield* Effect.cached( + DesktopSshEnvironment.readOrCreateLauncherId( + environment.path.join(environment.stateDir, "ssh-launcher-id"), + ).pipe( + Effect.provide( + yield* Effect.context(), + ), + ), + ), }); }), ); @@ -228,7 +239,6 @@ const layerDesktopApplication = Layer.mergeAll( DesktopApplicationMenu.layer, DesktopLinuxUrlHandler.layer, DesktopCliCommand.layer, - layerDesktopSsh, ).pipe( Layer.provideMerge(layerDesktopSnapShot), Layer.provideMerge( @@ -238,6 +248,8 @@ const layerDesktopApplication = Layer.mergeAll( Layer.provide(DesktopManagedPolicy.layer), ), ), + // Updates closes the same tunnels the SSH IPC opens before quitAndInstall. + Layer.provideMerge(layerDesktopSsh), Layer.provideMerge(layerDesktopWslBackend), Layer.provideMerge(layerDesktopLocalEnvironmentAuth), ); diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index cf3c669f9696..aa71b138134c 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -115,6 +115,7 @@ contextBridge.exposeInMainWorld("desktopBridge", { ipcRenderer.sendSync(IpcChannels.GET_RELAY_TELEMETRY_ENABLED_CHANNEL) === true, getPathForFile: (file: File) => webUtils.getPathForFile(file), getClientPlatform: () => clientPlatform, + restartAudioService: () => ipcRenderer.invoke(IpcChannels.RESTART_AUDIO_SERVICE_CHANNEL), setNotificationBadge: (badge) => ipcRenderer.invoke(IpcChannels.SET_NOTIFICATION_BADGE_CHANNEL, badge), onNotificationBadgeClear: (listener) => { @@ -239,14 +240,17 @@ contextBridge.exposeInMainWorld("desktopBridge", { setSshRunner: (runner) => ipcRenderer.invoke(IpcChannels.SET_SSH_RUNNER_CHANNEL, runner), setWslOnly: (enabled) => ipcRenderer.invoke(IpcChannels.SET_WSL_ONLY_CHANNEL, enabled), pickFolder: (options) => ipcRenderer.invoke(IpcChannels.PICK_FOLDER_CHANNEL, options), + toLocalBackendPath: (backendId, path) => + ipcRenderer.invoke(IpcChannels.TO_LOCAL_BACKEND_PATH_CHANNEL, { backendId, path }), pickProjectFavicon: (initialPath) => ipcRenderer.invoke(IpcChannels.PICK_PROJECT_FAVICON_CHANNEL, initialPath), pickThemeFiles: () => ipcRenderer.invoke(IpcChannels.PICK_THEME_FILES_CHANNEL, undefined), setTheme: (theme) => ipcRenderer.invoke(IpcChannels.SET_THEME_CHANNEL, theme), - showContextMenu: (items, position) => + showContextMenu: (items, position, source) => ipcRenderer.invoke(IpcChannels.CONTEXT_MENU_CHANNEL, { items, ...(position === undefined ? {} : { position }), + ...(source === undefined ? {} : { source }), }), receiveProviderAuthCallback: (url: string) => ipcRenderer.invoke(IpcChannels.RECEIVE_PROVIDER_AUTH_CALLBACK_CHANNEL, url), @@ -396,6 +400,7 @@ contextBridge.exposeInMainWorld("desktopBridge", { goBack: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_GO_BACK_CHANNEL, { tabId }), goForward: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_GO_FORWARD_CHANNEL, { tabId }), refresh: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_REFRESH_CHANNEL, { tabId }), + stopLoading: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_STOP_LOADING_CHANNEL, { tabId }), zoomIn: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_ZOOM_IN_CHANNEL, { tabId }), zoomOut: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_ZOOM_OUT_CHANNEL, { tabId }), resetZoom: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_RESET_ZOOM_CHANNEL, { tabId }), @@ -417,6 +422,8 @@ contextBridge.exposeInMainWorld("desktopBridge", { ipcRenderer.invoke(IpcChannels.PREVIEW_STOP_FIND_IN_PAGE_CHANNEL, { tabId }), setShown: (tabId, shown) => ipcRenderer.invoke(IpcChannels.PREVIEW_SET_SHOWN_CHANNEL, { tabId, shown }), + setCaptureDrawn: (tabId, drawn) => + ipcRenderer.invoke(IpcChannels.PREVIEW_SET_CAPTURE_DRAWN_CHANNEL, { tabId, drawn }), openDevTools: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_OPEN_DEVTOOLS_CHANNEL, { tabId }), listBrowserImportSources: () => ipcRenderer.invoke(IpcChannels.PREVIEW_IMPORT_SOURCES_CHANNEL), diff --git a/apps/desktop/src/preview/AnnotationStroke.test.ts b/apps/desktop/src/preview/AnnotationStroke.test.ts new file mode 100644 index 000000000000..ad8a21c140b4 --- /dev/null +++ b/apps/desktop/src/preview/AnnotationStroke.test.ts @@ -0,0 +1,73 @@ +import type { PreviewAnnotationPoint } from "@t3tools/contracts"; +import { describe, expect, it } from "vite-plus/test"; + +import { + addStrokePoint, + openStrokePath, + startStroke, + strokeBounds, + STROKE_SEGMENTS_PER_PATH, +} from "./AnnotationStroke.ts"; + +// The Draw tool used to rebuild the whole path and rescan every point on each move. +const fullPath = (points: ReadonlyArray): string => { + let path = `M ${points[0]!.x} ${points[0]!.y}`; + for (let index = 1; index < points.length - 1; index += 1) { + const current = points[index]!; + const next = points[index + 1]!; + path += ` Q ${current.x} ${current.y} ${(current.x + next.x) / 2} ${(current.y + next.y) / 2}`; + } + const last = points[points.length - 1]!; + return `${path} L ${last.x} ${last.y}`; +}; + +const fullBounds = (points: ReadonlyArray, width: number) => { + const xs = points.map((point) => point.x); + const ys = points.map((point) => point.y); + const padding = width + 3; + const left = Math.min(...xs) - padding; + const top = Math.min(...ys) - padding; + const right = Math.max(...xs) + padding; + const bottom = Math.max(...ys) + padding; + return { x: left, y: top, width: right - left, height: bottom - top }; +}; + +// Each later path starts with a move to where the previous one ended. +const joinPaths = (paths: ReadonlyArray): string => + paths + .map((path, index) => { + if (index === 0) return path; + const resume = path.match(/^M (\S+ \S+)/)![1]!; + expect(paths[index - 1]!.endsWith(` ${resume}`)).toBe(true); + return path.slice(`M ${resume}`.length); + }) + .join(""); + +describe("stroke geometry", () => { + it("draws the same curve and bounds as a full rescan at every length", () => { + const points = Array.from({ length: STROKE_SEGMENTS_PER_PATH * 2 + 30 }, (_, index) => ({ + x: index * 3 + ((index * 7) % 11), + y: 200 + ((index * 13) % 29) - index, + })); + const geometry = startStroke(points[0]!); + const closedPaths: string[] = []; + for (let index = 1; index < points.length; index += 1) { + const closedPath = addStrokePoint(geometry, points[index]!); + if (closedPath !== null) closedPaths.push(closedPath); + const drawn = points.slice(0, index + 1); + expect(joinPaths([...closedPaths, openStrokePath(geometry)])).toBe(fullPath(drawn)); + expect(strokeBounds(geometry, 4)).toEqual(fullBounds(drawn, 4)); + } + expect(closedPaths).toHaveLength(2); + expect(geometry.points).toEqual(points); + }); + + it("keeps the rewritten path short for a stroke past the engine's argument limit", () => { + const geometry = startStroke({ x: 0, y: 0 }); + for (let index = 1; index < 250_000; index += 1) { + addStrokePoint(geometry, { x: index % 800, y: index % 600 }); + } + expect(strokeBounds(geometry, 4)).toEqual({ x: -7, y: -7, width: 813, height: 613 }); + expect(openStrokePath(geometry).length).toBeLessThan(STROKE_SEGMENTS_PER_PATH * 40); + }); +}); diff --git a/apps/desktop/src/preview/AnnotationStroke.ts b/apps/desktop/src/preview/AnnotationStroke.ts new file mode 100644 index 000000000000..66510fc0b7cf --- /dev/null +++ b/apps/desktop/src/preview/AnnotationStroke.ts @@ -0,0 +1,69 @@ +import type { PreviewAnnotationPoint, PreviewAnnotationRect } from "@t3tools/contracts"; + +// Setting a path's `d` makes Chromium parse the whole string again, so a long +// stroke is drawn as a run of paths and only the last one is rewritten per move. +export const STROKE_SEGMENTS_PER_PATH = 100; + +export interface StrokeGeometry { + readonly points: PreviewAnnotationPoint[]; + minX: number; + minY: number; + maxX: number; + maxY: number; + // The curve through the newest point is still open: where it ends depends on + // the point after it. This holds only the settled segments of the last path. + openPath: string; + openSegments: number; +} + +export function startStroke(point: PreviewAnnotationPoint): StrokeGeometry { + return { + points: [point], + minX: point.x, + minY: point.y, + maxX: point.x, + maxY: point.y, + openPath: `M ${point.x} ${point.y}`, + openSegments: 0, + }; +} + +// Returns the final `d` of the path this point filled, if it filled one. +export function addStrokePoint( + geometry: StrokeGeometry, + point: PreviewAnnotationPoint, +): string | null { + const previous = geometry.points[geometry.points.length - 1]!; + let closedPath: string | null = null; + if (geometry.points.length > 1) { + const midX = (previous.x + point.x) / 2; + const midY = (previous.y + point.y) / 2; + geometry.openPath += ` Q ${previous.x} ${previous.y} ${midX} ${midY}`; + geometry.openSegments += 1; + if (geometry.openSegments === STROKE_SEGMENTS_PER_PATH) { + closedPath = geometry.openPath; + geometry.openPath = `M ${midX} ${midY}`; + geometry.openSegments = 0; + } + } + geometry.points.push(point); + geometry.minX = Math.min(geometry.minX, point.x); + geometry.minY = Math.min(geometry.minY, point.y); + geometry.maxX = Math.max(geometry.maxX, point.x); + geometry.maxY = Math.max(geometry.maxY, point.y); + return closedPath; +} + +export function openStrokePath(geometry: StrokeGeometry): string { + const last = geometry.points[geometry.points.length - 1]!; + return `${geometry.openPath} L ${last.x} ${last.y}`; +} + +export function strokeBounds(geometry: StrokeGeometry, width: number): PreviewAnnotationRect { + const padding = width + 3; + const left = geometry.minX - padding; + const top = geometry.minY - padding; + const right = geometry.maxX + padding; + const bottom = geometry.maxY + padding; + return { x: left, y: top, width: right - left, height: bottom - top }; +} diff --git a/apps/desktop/src/preview/BrowserImport/BrowserImport.test.ts b/apps/desktop/src/preview/BrowserImport/BrowserImport.test.ts index 9528cd5d2e84..484ba6f9358f 100644 --- a/apps/desktop/src/preview/BrowserImport/BrowserImport.test.ts +++ b/apps/desktop/src/preview/BrowserImport/BrowserImport.test.ts @@ -33,6 +33,7 @@ const layerRejectedBeforeSession = Layer.succeed( isPartition: () => false, getSession: () => Effect.die("getSession must not be reached"), clearCookies: () => Effect.die("clearCookies must not be reached"), + cookiesCleared: () => Effect.die("cookiesCleared must not be reached"), clearCache: () => Effect.die("clearCache must not be reached"), }), ); @@ -88,7 +89,7 @@ describe("BrowserImport.importCookies", () => { assert.instanceOf(error, BrowserImport.BrowserImportFailedError); assert.equal(error.reason, "unknownSourceProfile"); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); it.effect.skipIf(!symlinksSupported)( @@ -113,7 +114,7 @@ describe("BrowserImport.importCookies", () => { assert.instanceOf(error, BrowserImport.BrowserImportFailedError); assert.equal(error.reason, "browserRunning"); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); }); diff --git a/apps/desktop/src/preview/BrowserImport/ChromiumCookies.test.ts b/apps/desktop/src/preview/BrowserImport/ChromiumCookies.test.ts index e5aa05f268e4..eb4dd3143eef 100644 --- a/apps/desktop/src/preview/BrowserImport/ChromiumCookies.test.ts +++ b/apps/desktop/src/preview/BrowserImport/ChromiumCookies.test.ts @@ -134,7 +134,7 @@ describe("readChromiumCookieDatabase", () => { const partitioned = yield* readChromiumCookieDatabase(filename, keys, "linux"); expect(partitioned.cookies).toEqual([]); expect(partitioned.undecryptable).toBe(1); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); it.effect("reads plaintext, encrypted, and genuinely empty cookie values", () => @@ -186,7 +186,7 @@ describe("readChromiumCookieDatabase", () => { { name: "encrypted", value: "stored encrypted" }, { name: "empty", value: "" }, ]); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); it.effect("enforces domain binding only for schema 24 and newer", () => @@ -227,7 +227,7 @@ describe("readChromiumCookieDatabase", () => { ]); expect(result.undecryptable).toBe(2); expect(result.undecryptableHosts).toEqual(["wrong.example", "short.example"]); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); it.effect("decrypts mixed v10 and v11 cookies with their respective keys", () => @@ -273,7 +273,7 @@ describe("readChromiumCookieDatabase", () => { ]); expect(v10Only.undecryptable).toBe(1); expect(v10Only.undecryptableHosts).toEqual(["v11.example"]); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); it.effect("recovers records written with the empty-passphrase key", () => @@ -320,7 +320,7 @@ describe("readChromiumCookieDatabase", () => { const noV11 = yield* readChromiumCookieDatabase(filename, { cbcV10, cbcEmpty }, "linux"); expect(noV11.cookies.map(({ name }) => name)).toEqual(["empty-v10"]); expect(noV11.undecryptableHosts).toEqual(["ev11.example"]); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); it.effect("treats unversioned encrypted values as legacy plaintext on macOS and Linux", () => @@ -355,6 +355,29 @@ describe("readChromiumCookieDatabase", () => { expect(mac.undecryptable).toBe(0); expect(linux.cookies[0]?.value).toBe("legacy cleartext"); expect(linux.undecryptable).toBe(0); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("reads a cookie database the browser has emptied as having no cookies", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const directory = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3code-chromium-cookies-", + }); + const filename = `${directory}/Cookies`; + yield* Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`create table scratch (id integer)`; + yield* sql`drop table scratch`; + }).pipe(Effect.provide(NodeSqliteClient.layer({ filename }))); + + const result = yield* readChromiumCookieDatabase( + filename, + { cbcV10: Buffer.from("0123456789abcdef") }, + "darwin", + ); + + expect(result).toEqual({ cookies: [], undecryptable: 0, undecryptableHosts: [] }); }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), ); @@ -409,6 +432,6 @@ describe("readChromiumCookieDatabase", () => { expect(chips.cookies.map(({ name }) => name)).toEqual(["plain"]); expect(chips.undecryptable).toBe(1); expect(chips.undecryptableHosts).toEqual(["partitioned.example"]); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); }); diff --git a/apps/desktop/src/preview/BrowserImport/ChromiumCookies.ts b/apps/desktop/src/preview/BrowserImport/ChromiumCookies.ts index 63fef9fb3c80..91cbfa03aac7 100644 --- a/apps/desktop/src/preview/BrowserImport/ChromiumCookies.ts +++ b/apps/desktop/src/preview/BrowserImport/ChromiumCookies.ts @@ -181,6 +181,10 @@ export const readChromiumCookieDatabase = Effect.fn("ChromiumCookies.readChromiu function* (snapshotPath: string, keys: ChromiumKeyMaterial, platform: NodeJS.Platform) { const result = yield* Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; + // Chromium razes a damaged jar to a file with no tables and rebuilds it on its next write. + const tables = + yield* sql`select 1 from sqlite_master where type = 'table' and name = 'cookies'`; + if (tables.length === 0) return { rows: [], schemaVersion: 0 }; const schemaVersion = yield* sql`select value from meta where key = 'version' limit 1`.pipe( Effect.flatMap(decodeSchemaVersion), Effect.map(([row]) => row.value), diff --git a/apps/desktop/src/preview/BrowserImport/CookieDatabase.test.ts b/apps/desktop/src/preview/BrowserImport/CookieDatabase.test.ts index 402397d3fde6..ca0a86cedacd 100644 --- a/apps/desktop/src/preview/BrowserImport/CookieDatabase.test.ts +++ b/apps/desktop/src/preview/BrowserImport/CookieDatabase.test.ts @@ -11,7 +11,7 @@ import { snapshotCookieDatabase } from "./CookieDatabase.ts"; const runNode = ( effect: Effect.Effect, -) => effect.pipe(Effect.provide(NodeServices.layer), Effect.scoped); +) => effect.pipe(Effect.provide(NodeServices.layer)); describe("snapshotCookieDatabase", () => { it.effect("includes committed WAL data in one consistent database", () => diff --git a/apps/desktop/src/preview/BrowserImport/FirefoxCookies.test.ts b/apps/desktop/src/preview/BrowserImport/FirefoxCookies.test.ts index be41318fa01e..33e32ec030be 100644 --- a/apps/desktop/src/preview/BrowserImport/FirefoxCookies.test.ts +++ b/apps/desktop/src/preview/BrowserImport/FirefoxCookies.test.ts @@ -78,7 +78,7 @@ const writeFirefoxCookieDatabase = Effect.fnUntraced(function* ( }); const run = (effect: Effect.Effect) => - effect.pipe(Effect.provide(NodeServices.layer), Effect.scoped); + effect.pipe(Effect.provide(NodeServices.layer)); describe("readFirefoxCookies", () => { it.effect("converts millisecond expiries from schema 16 and newer", () => diff --git a/apps/desktop/src/preview/BrowserImport/LinuxBrowserSecret.test.ts b/apps/desktop/src/preview/BrowserImport/LinuxBrowserSecret.test.ts index 8358088bc134..8ef376e4887b 100644 --- a/apps/desktop/src/preview/BrowserImport/LinuxBrowserSecret.test.ts +++ b/apps/desktop/src/preview/BrowserImport/LinuxBrowserSecret.test.ts @@ -64,6 +64,6 @@ it.layer(NodeServices.layer)("Linux browser secret path", (it) => { assert.isUndefined(yield* resolve(false, "win32")); yield* fileSystem.remove(staged); assert.isUndefined(yield* resolve(false)); - }).pipe(Effect.scoped), + }), ); }); diff --git a/apps/desktop/src/preview/BrowserImport/SafariCookies.test.ts b/apps/desktop/src/preview/BrowserImport/SafariCookies.test.ts index 7e92334c957d..f82f2319baeb 100644 --- a/apps/desktop/src/preview/BrowserImport/SafariCookies.test.ts +++ b/apps/desktop/src/preview/BrowserImport/SafariCookies.test.ts @@ -240,7 +240,7 @@ describe("safariAccessDenied", () => { yield* fileSystem.writeFile(jar, new Uint8Array([0x63, 0x6f, 0x6f, 0x6b])); assert.isFalse(yield* safariAccessDenied(jar)); assert.isFalse(yield* safariAccessDenied(`${directory}/absent.binarycookies`)); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); }); diff --git a/apps/desktop/src/preview/BrowserImport/SafariPermission.test.ts b/apps/desktop/src/preview/BrowserImport/SafariPermission.test.ts index 3b33f872d124..dc225b04bac7 100644 --- a/apps/desktop/src/preview/BrowserImport/SafariPermission.test.ts +++ b/apps/desktop/src/preview/BrowserImport/SafariPermission.test.ts @@ -39,7 +39,7 @@ it.effect("detects Safari access becoming available without reading or importing assert.isTrue(yield* Effect.promise(check)); yield* fs.remove(jar); assert.isFalse(yield* Effect.promise(check)); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); it.effect("recognizes access when cookies exist only in a named Safari profile", () => @@ -55,5 +55,5 @@ it.effect("recognizes access when cookies exist only in a named Safari profile", yield* fs.makeDirectory(directory, { recursive: true }); yield* fs.writeFileString(`${directory}/Cookies.binarycookies`, "not parsed"); assert.isTrue(yield* Effect.promise(check)); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); diff --git a/apps/desktop/src/preview/BrowserImport/Sources.test.ts b/apps/desktop/src/preview/BrowserImport/Sources.test.ts index 720a57e64a9d..9b2c3cb26c8f 100644 --- a/apps/desktop/src/preview/BrowserImport/Sources.test.ts +++ b/apps/desktop/src/preview/BrowserImport/Sources.test.ts @@ -51,7 +51,7 @@ const run = ( E, FileSystem.FileSystem | Path.Path | Scope.Scope | ChildProcessSpawner.ChildProcessSpawner >, -) => effect.pipe(Effect.provide(NodeServices.layer), Effect.scoped); +) => effect.pipe(Effect.provide(NodeServices.layer)); const writeCookieDatabase = (file: string, count: number) => Effect.sync(() => { diff --git a/apps/desktop/src/preview/BrowserImport/Sources.ts b/apps/desktop/src/preview/BrowserImport/Sources.ts index 20a35752ed97..6ecd546a1ad5 100644 --- a/apps/desktop/src/preview/BrowserImport/Sources.ts +++ b/apps/desktop/src/preview/BrowserImport/Sources.ts @@ -345,6 +345,12 @@ const countProfileCookies = Effect.fnUntraced(function* ( if (database === undefined) return undefined; return yield* Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; + // readChromiumCookieDatabase imports a razed jar as empty; Firefox has no such reader guard. + if (definition.engine !== "firefox") { + const tables = + yield* sql`select 1 from sqlite_master where type = 'table' and name = 'cookies'`; + if (tables.length === 0) return 0; + } const rows = definition.engine === "firefox" ? yield* sql`select count(*) as count from moz_cookies where originAttributes = ''` diff --git a/apps/desktop/src/preview/BrowserSession.test.ts b/apps/desktop/src/preview/BrowserSession.test.ts index e68e52961fa6..9a9be6213c39 100644 --- a/apps/desktop/src/preview/BrowserSession.test.ts +++ b/apps/desktop/src/preview/BrowserSession.test.ts @@ -1,8 +1,11 @@ import { assert, describe, it } from "@effect/vitest"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as NodeEvents from "node:events"; import { beforeEach, vi } from "vite-plus/test"; const { fromPartition, sessions } = vi.hoisted(() => ({ @@ -12,6 +15,8 @@ const { fromPartition, sessions } = vi.hoisted(() => ({ { readonly clearCache: ReturnType; readonly clearStorageData: ReturnType; + readonly closeAllConnections: ReturnType; + readonly storagePath: string | null; readonly getUserAgent: ReturnType string>>; readonly setPermissionRequestHandler: ReturnType; readonly setPermissionCheckHandler: ReturnType; @@ -21,44 +26,125 @@ const { fromPartition, sessions } = vi.hoisted(() => ({ >(), })); +const storage = vi.hoisted(() => ({ root: "" })); + const { fromWebContents } = vi.hoisted(() => ({ fromWebContents: vi.fn<(webContents: unknown) => unknown>(() => null), })); +const { appListeners } = vi.hoisted(() => ({ + appListeners: new Map void>(), +})); + vi.mock("electron", () => ({ + app: { + on: (eventName: string, listener: (...args: unknown[]) => void) => { + appListeners.set(eventName, listener); + }, + removeListener: (eventName: string) => { + appListeners.delete(eventName); + }, + }, BrowserWindow: { fromWebContents }, session: { fromPartition, }, })); +import * as ElectronApp from "../electron/ElectronApp.ts"; import * as ElectronDialog from "../electron/ElectronDialog.ts"; import * as BrowserSession from "./BrowserSession.ts"; -const { showMessageBox } = vi.hoisted(() => ({ +const { showMessageBox, showErrorBox } = vi.hoisted(() => ({ showMessageBox: vi.fn< - (options: Electron.MessageBoxOptions, owner?: Electron.BrowserWindow) => number + ( + options: Electron.MessageBoxOptions, + owner?: Electron.BrowserWindow, + ) => number | Promise >(() => 1), + showErrorBox: vi.fn<(title: string, content: string) => void>(), })); const layerDialog = Layer.succeed(ElectronDialog.ElectronDialog, { pickFolder: () => Effect.die("unused"), pickFiles: () => Effect.die("unused"), showMessageBox: (options, owner) => - Effect.sync(() => ({ response: showMessageBox(options, owner), checkboxChecked: false })), - showErrorBox: () => Effect.void, + Effect.promise(async () => ({ + response: await showMessageBox(options, owner), + checkboxChecked: false, + })), + showErrorBox: (title, content) => Effect.sync(() => showErrorBox(title, content)), }); const layer = BrowserSession.layer.pipe( - Layer.provide(Layer.merge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer), layerDialog)), + Layer.provide( + Layer.mergeAll(NodeServices.layer, NodeCrypto.layer, layerDialog, ElectronApp.layer), + ), ); +const selfSignedCertificate = { + fingerprint: "sha256/self-signed", + subjectName: "localhost", + issuerName: "localhost", + validStart: 1_735_776_000, + validExpiry: 1_807_056_000, +} as Electron.Certificate; + +/** Raises Electron's `certificate-error` for a preview guest in `browserSession`. */ +const certificateError = ( + browserSession: unknown, + input: { + readonly url: string; + readonly certificate?: Electron.Certificate; + readonly isMainFrame?: boolean; + }, +) => { + const listener = appListeners.get("certificate-error"); + assert.isDefined(listener, "preview sessions should handle certificate errors"); + const guest = Object.assign(new NodeEvents.EventEmitter(), { + session: browserSession, + hostWebContents: null, + isDestroyed: () => false, + }); + const event = { preventDefault: vi.fn() }; + const decision = Deferred.makeUnsafe(); + const callback = vi.fn((trusted: boolean) => { + Deferred.doneUnsafe(decision, Effect.succeed(trusted)); + }); + listener( + event, + guest, + input.url, + "net::ERR_CERT_AUTHORITY_INVALID", + input.certificate ?? selfSignedCertificate, + callback, + input.isMainFrame ?? true, + ); + return { guest, event, callback, trusted: Deferred.await(decision) }; +}; + +/** Answers the certificate prompt with its "Trust…" button. */ +const trustCertificateButton = (options: Electron.MessageBoxOptions): number => { + const index = (options.buttons ?? []).findIndex((label) => label.startsWith("Trust")); + assert.notStrictEqual(index, -1, "the certificate prompt should offer to trust"); + return index; +}; + describe("BrowserSession", () => { beforeEach(() => { sessions.clear(); + appListeners.clear(); + storage.root = ""; + showErrorBox.mockReset(); fromPartition.mockReset(); fromPartition.mockImplementation((partition: string) => { const browserSession = { clearCache: vi.fn(() => Promise.resolve()), clearStorageData: vi.fn(() => Promise.resolve()), + closeAllConnections: vi.fn(() => Promise.resolve()), + // Like Electron, only a `persist:` partition has storage on disk. + storagePath: + storage.root !== "" && partition.startsWith("persist:") + ? `${storage.root}/Partitions/${partition}` + : null, getUserAgent: vi.fn(() => "Mozilla/5.0 Electron/41.5.0 t3code/0.0.27"), setPermissionRequestHandler: vi.fn(), setPermissionCheckHandler: vi.fn(), @@ -129,6 +215,8 @@ describe("BrowserSession", () => { const browserSession = { clearCache: vi.fn(() => Promise.resolve()), clearStorageData: vi.fn(() => Promise.resolve()), + closeAllConnections: vi.fn(() => Promise.resolve()), + storagePath: null, getUserAgent: vi.fn(() => userAgent), setPermissionRequestHandler: vi.fn(), setPermissionCheckHandler: vi.fn(), @@ -333,4 +421,236 @@ describe("BrowserSession", () => { assert.strictEqual(vi.mocked(other.clearStorageData).mock.calls.length, 0); }).pipe(Effect.provide(layer)), ); + + it.effect("asks before trusting an untrusted certificate and remembers the approval", () => + Effect.gen(function* () { + const browserSessions = yield* BrowserSession.BrowserSession; + const partition = yield* browserSessions.getPartition("scope-a"); + yield* browserSessions.getSession("scope-a"); + const browserSession = sessions.get(partition); + + showMessageBox.mockReset(); + showMessageBox.mockImplementation(trustCertificateButton); + const first = certificateError(browserSession, { url: "https://localhost:18443/" }); + assert.strictEqual(first.event.preventDefault.mock.calls.length, 1); + assert.isTrue(yield* first.trusted); + assert.strictEqual(showMessageBox.mock.calls.length, 1); + + // The approval covers this origin, port, and certificate in this profile. + const again = certificateError(browserSession, { url: "https://localhost:18443/other" }); + assert.isTrue(yield* again.trusted); + assert.strictEqual(showMessageBox.mock.calls.length, 1); + // Its sockets reach the same TLS endpoint, and cannot ask on their own. + const socket = certificateError(browserSession, { + url: "wss://localhost:18443/hmr", + isMainFrame: false, + }); + assert.isTrue(yield* socket.trusted); + const otherSocket = certificateError(browserSession, { + url: "wss://localhost:18444/hmr", + isMainFrame: false, + }); + assert.isFalse(yield* otherSocket.trusted); + assert.strictEqual(showMessageBox.mock.calls.length, 1); + + showMessageBox.mockReset(); + showMessageBox.mockReturnValue(1); + const otherPort = certificateError(browserSession, { url: "https://localhost:18444/" }); + assert.isFalse(yield* otherPort.trusted); + const renewed = certificateError(browserSession, { + url: "https://localhost:18443/", + certificate: { ...selfSignedCertificate, fingerprint: "sha256/renewed" }, + }); + assert.isFalse(yield* renewed.trusted); + assert.strictEqual(showMessageBox.mock.calls.length, 2); + + // A Session that is not a preview profile keeps Electron's denial. + const appSession = certificateError({}, { url: "https://localhost:18443/" }); + assert.strictEqual(appSession.event.preventDefault.mock.calls.length, 0); + assert.strictEqual(appSession.callback.mock.calls.length, 0); + }).pipe(Effect.provide(layer)), + ); + + it.effect("revokes certificate approvals when cookies are cleared, not the cache", () => + Effect.gen(function* () { + const browserSessions = yield* BrowserSession.BrowserSession; + const partition = yield* browserSessions.getPartition("scope-a"); + yield* browserSessions.getSession("scope-a"); + const browserSession = sessions.get(partition); + const url = "https://localhost:18443/"; + + showMessageBox.mockReset(); + showMessageBox.mockImplementation(trustCertificateButton); + assert.isTrue(yield* certificateError(browserSession, { url }).trusted); + + yield* browserSessions.clearCache([partition]); + assert.isTrue(yield* certificateError(browserSession, { url }).trusted); + assert.strictEqual(showMessageBox.mock.calls.length, 1); + + // A failed storage clear still forgets the approvals the user asked to clear. + browserSession!.clearStorageData.mockRejectedValueOnce(new Error("disk busy")); + const failure = yield* browserSessions.clearCookies([partition]).pipe(Effect.flip); + assert.strictEqual(failure._tag, "BrowserSessionStorageClearError"); + showMessageBox.mockReset(); + showMessageBox.mockReturnValue(1); + assert.isFalse(yield* certificateError(browserSession, { url }).trusted); + assert.strictEqual(showMessageBox.mock.calls.length, 1); + }).pipe(Effect.provide(layer)), + ); + + it.effect("remembers approvals across restarts, but not incognito ones", () => + Effect.gen(function* () { + storage.root = yield* (yield* FileSystem.FileSystem).makeTempDirectoryScoped(); + const url = "https://localhost:18443/"; + const approve = Effect.gen(function* () { + const browserSessions = yield* BrowserSession.BrowserSession; + const persistent = yield* browserSessions.getPartition("scope-a"); + const incognito = yield* browserSessions.getPartition("scope-a", false); + yield* browserSessions.getSession("scope-a"); + yield* browserSessions.getSession("scope-a", false); + showMessageBox.mockReset(); + showMessageBox.mockImplementation(trustCertificateButton); + assert.isTrue(yield* certificateError(sessions.get(persistent), { url }).trusted); + assert.isTrue(yield* certificateError(sessions.get(incognito), { url }).trusted); + return { persistent, incognito }; + }); + const { persistent, incognito } = yield* approve.pipe(Effect.provide(layer)); + + // A restarted app gets fresh Session objects and reads approvals from disk. + sessions.clear(); + appListeners.clear(); + yield* Effect.gen(function* () { + const browserSessions = yield* BrowserSession.BrowserSession; + yield* browserSessions.getSession("scope-a"); + yield* browserSessions.getSession("scope-a", false); + showMessageBox.mockReset(); + showMessageBox.mockReturnValue(1); + assert.isTrue(yield* certificateError(sessions.get(persistent), { url }).trusted); + assert.strictEqual(showMessageBox.mock.calls.length, 0); + assert.isFalse(yield* certificateError(sessions.get(incognito), { url }).trusted); + assert.strictEqual(showMessageBox.mock.calls.length, 1); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("an answer to a prompt opened before cookies were cleared does not trust", () => + Effect.gen(function* () { + storage.root = yield* (yield* FileSystem.FileSystem).makeTempDirectoryScoped(); + const browserSessions = yield* BrowserSession.BrowserSession; + const partition = yield* browserSessions.getPartition("scope-a"); + yield* browserSessions.getSession("scope-a"); + const browserSession = sessions.get(partition); + const url = "https://localhost:18443/"; + + const asked = yield* Deferred.make(); + let answer = () => {}; + const answered = new Promise((resolve) => { + answer = resolve; + }); + showMessageBox.mockReset(); + showMessageBox.mockImplementation((options) => { + Deferred.doneUnsafe(asked, Effect.void); + return answered.then(() => trustCertificateButton(options)); + }); + const pending = certificateError(browserSession, { url }); + yield* Deferred.await(asked); + yield* browserSessions.clearCookies([partition]); + answer(); + assert.isFalse(yield* pending.trusted); + + showMessageBox.mockReset(); + showMessageBox.mockReturnValue(1); + assert.isFalse(yield* certificateError(browserSession, { url }).trusted); + assert.strictEqual(showMessageBox.mock.calls.length, 1); + assert.isFalse( + yield* (yield* FileSystem.FileSystem).exists( + `${browserSession!.storagePath}/t3code-certificate-approvals.json`, + ), + ); + }).pipe(Effect.scoped, Effect.provide(Layer.merge(layer, NodeServices.layer))), + ); + + it.effect("a tab that closes or navigates away dismisses its certificate prompt", () => + Effect.gen(function* () { + const browserSessions = yield* BrowserSession.BrowserSession; + const partition = yield* browserSessions.getPartition("scope-a"); + yield* browserSessions.getSession("scope-a"); + const browserSession = sessions.get(partition); + const url = "https://localhost:18443/"; + + for (const leave of [ + (guest: NodeEvents.EventEmitter) => + guest.emit("did-start-navigation", { isMainFrame: true, isSameDocument: false }), + (guest: NodeEvents.EventEmitter) => guest.emit("destroyed"), + ]) { + showMessageBox.mockReset(); + // Answering "Trust" as the tab leaves must not approve the certificate. + showMessageBox.mockImplementation( + (options) => + new Promise((resolve) => + options.signal?.addEventListener("abort", () => + resolve(trustCertificateButton(options)), + ), + ), + ); + const pending = certificateError(browserSession, { url }); + pending.guest.emit("did-start-navigation", { isMainFrame: true, isSameDocument: true }); + leave(pending.guest); + assert.isFalse(yield* pending.trusted); + assert.strictEqual(showMessageBox.mock.calls.length, 1); + } + }).pipe(Effect.provide(layer)), + ); + + it.effect("does not trust a certificate whose approval cannot be saved", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + // A file where the profile directory should be makes every write fail. + const root = yield* fileSystem.makeTempDirectoryScoped(); + yield* fileSystem.writeFileString(`${root}/Partitions`, ""); + storage.root = root; + const browserSessions = yield* BrowserSession.BrowserSession; + const partition = yield* browserSessions.getPartition("scope-a"); + yield* browserSessions.getSession("scope-a"); + const browserSession = sessions.get(partition); + const url = "https://localhost:18443/"; + + showMessageBox.mockReset(); + showMessageBox.mockImplementation(trustCertificateButton); + assert.isFalse(yield* certificateError(browserSession, { url }).trusted); + assert.strictEqual(showErrorBox.mock.calls.length, 1); + // Nothing was trusted for this run either, so the next visit asks again. + showMessageBox.mockReset(); + showMessageBox.mockReturnValue(1); + assert.isFalse(yield* certificateError(browserSession, { url }).trusted); + assert.strictEqual(showMessageBox.mock.calls.length, 1); + }).pipe(Effect.scoped, Effect.provide(Layer.merge(layer, NodeServices.layer))), + ); + + it.effect("clearing cookies deletes a saved approval even when none are loaded", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + storage.root = yield* fileSystem.makeTempDirectoryScoped(); + const browserSessions = yield* BrowserSession.BrowserSession; + const partition = yield* browserSessions.getPartition("scope-a"); + yield* browserSessions.getSession("scope-a"); + const browserSession = sessions.get(partition); + // What a clear whose removal failed leaves behind: memory forgot, the file did not. + const file = `${browserSession!.storagePath}/t3code-certificate-approvals.json`; + yield* fileSystem.makeDirectory(browserSession!.storagePath!, { recursive: true }); + yield* fileSystem.writeFileString( + file, + JSON.stringify([ + { + origin: "https://localhost:18443", + fingerprint: selfSignedCertificate.fingerprint, + error: "net::ERR_CERT_AUTHORITY_INVALID", + }, + ]), + ); + + yield* browserSessions.clearCookies([partition]); + assert.isFalse(yield* fileSystem.exists(file)); + }).pipe(Effect.scoped, Effect.provide(Layer.merge(layer, NodeServices.layer))), + ); }); diff --git a/apps/desktop/src/preview/BrowserSession.ts b/apps/desktop/src/preview/BrowserSession.ts index b2523ba62b97..1400f2c9cbe0 100644 --- a/apps/desktop/src/preview/BrowserSession.ts +++ b/apps/desktop/src/preview/BrowserSession.ts @@ -1,14 +1,21 @@ -import type { Session, WebContents } from "electron"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import type { Certificate, Session, WebContents } from "electron"; import { BrowserWindow, session } from "electron"; import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; import * as Digest from "effect/crypto/Digest"; import * as Effect from "effect/Effect"; import * as Hex from "effect/encoding/Hex"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; import * as PlatformError from "effect/PlatformError"; +import * as Result from "effect/Result"; import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; import * as SynchronizedRef from "effect/SynchronizedRef"; +import * as ElectronApp from "../electron/ElectronApp.ts"; import * as ElectronDialog from "../electron/ElectronDialog.ts"; const PREVIEW_PARTITION_PREFIX = "persist:t3code-preview-"; @@ -73,6 +80,64 @@ const permissionOrigin = (rawUrl: string | undefined): string | null => { } }; +/** + * Certificate errors a user may accept for a preview page, with the reason the + * prompt shows. Every other error keeps Electron's denial. + */ +const OVERRIDABLE_CERTIFICATE_ERRORS: ReadonlyMap = new Map([ + ["net::ERR_CERT_AUTHORITY_INVALID", "No trusted authority issued this certificate."], + ["net::ERR_CERT_COMMON_NAME_INVALID", "This certificate is not for this address."], + ["net::ERR_CERT_DATE_INVALID", "This certificate has expired or is not valid yet."], +]); + +/** + * A `wss:` socket reaches the same TLS endpoint as the page, so it shares the + * `https:` origin. It must: a socket is never a main frame, so it could not be + * approved on its own, and a page's dev-server socket would stay blocked. + */ +const certificateOrigin = (rawUrl: string): string | null => { + try { + const url = new URL(rawUrl); + return url.protocol === "https:" || url.protocol === "wss:" ? `https://${url.host}` : null; + } catch { + return null; + } +}; + +const CertificateApproval = Schema.Struct({ + origin: Schema.String, + fingerprint: Schema.String, + error: Schema.String, +}); +type CertificateApproval = typeof CertificateApproval.Type; +const CertificateApprovalsJson = Schema.fromJsonString(Schema.Array(CertificateApproval)); +const decodeCertificateApprovals = Schema.decodeEffect(CertificateApprovalsJson); +const encodeCertificateApprovals = Schema.encodeEffect(CertificateApprovalsJson); + +const certificateApprovalKey = ({ origin, fingerprint, error }: CertificateApproval): string => + JSON.stringify([origin, fingerprint, error]); + +/** Kept beside the profile's cookies, so it lives and dies with them. */ +const CERTIFICATE_APPROVALS_FILE = "t3code-certificate-approvals.json"; + +/** Server-chosen certificate names must not add lines or flood the prompt. */ +const promptCertificateName = (name: string): string => { + const flat = name.replace(/\s+/g, " ").trim(); + return flat.length > 120 ? `${flat.slice(0, 120)}…` : flat; +}; + +const promptCertificateDate = (seconds: number): string => + DateTime.formatIsoDateUtc(DateTime.makeUnsafe(seconds * 1000)); + +/** The certificates a user accepted in one preview Session. */ +interface CertificateTrust { + /** `null` for an incognito Session, whose approvals end with the process. */ + readonly file: string | null; + readonly approvals: Map; + /** Bumped by each revocation, so an answer to an older prompt cannot restore trust. */ + revocations: number; +} + export class BrowserSessionPartitionDerivationError extends Schema.TaggedError()( "BrowserSessionPartitionDerivationError", { @@ -154,6 +219,10 @@ export class BrowserSession extends Context.Service< readonly clearCookies: ( partitions?: ReadonlyArray, ) => Effect.Effect; + /** For cookies cleared outside `clearCookies`, such as over a tab's CDP session. */ + readonly cookiesCleared: ( + browserSession: Session, + ) => Effect.Effect; readonly clearCache: ( partitions?: ReadonlyArray, ) => Effect.Effect; @@ -190,7 +259,11 @@ const previewHostWindow = (guest: WebContents | null): BrowserWindow | undefined /** @public */ export const make = Effect.gen(function* BrowserSessionMake() { const crypto = yield* Digest.Digest; + const electronApp = yield* ElectronApp.ElectronApp; const electronDialog = yield* ElectronDialog.ElectronDialog; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const platform = yield* HostProcess.Platform; const runFork = Effect.runForkWith(yield* Effect.context()); // One prompt at a time, so a page cannot stack dialogs by looping a deep link. let permissionPromptOpen = false; @@ -257,6 +330,229 @@ export const make = Effect.gen(function* BrowserSessionMake() { (answer) => callback(answer === true), ); }; + + const certificateTrust = new WeakMap(); + const writeCertificateApprovals = yield* Semaphore.make(1); + const loadCertificateTrust = Effect.fn("BrowserSession.loadCertificateTrust")(function* ( + browserSession: Session, + ) { + const file = + browserSession.storagePath == null + ? null + : path.join(browserSession.storagePath, CERTIFICATE_APPROVALS_FILE); + const approvals: ReadonlyArray = + file === null + ? [] + : yield* fileSystem.readFileString(file).pipe( + Effect.flatMap(decodeCertificateApprovals), + Effect.catchTags({ + PlatformError: (error) => + error.reason._tag === "NotFound" ? Effect.succeed([]) : Effect.fail(error), + }), + Effect.catch((cause) => + Effect.logWarning("Ignored saved certificate approvals.", { file, cause }).pipe( + Effect.as([]), + ), + ), + ); + const trust: CertificateTrust = { + file, + approvals: new Map(approvals.map((approval) => [certificateApprovalKey(approval), approval])), + revocations: 0, + }; + return trust; + }); + const writeApprovalsFile = Effect.fn("BrowserSession.writeApprovalsFile")(function* ( + file: string, + approvals: ReadonlyMap, + ) { + if (approvals.size === 0) return yield* fileSystem.remove(file, { force: true }); + yield* fileSystem.makeDirectory(path.dirname(file), { recursive: true }); + yield* fileSystem.writeFileString( + file, + yield* encodeCertificateApprovals([...approvals.values()]), + ); + }); + /** + * Saved before it is trusted, so a failed write cannot leave an approval that + * silently ends at restart. Writes share one lock, so a slower write cannot + * restore stale data. + */ + const rememberCertificate = Effect.fn("BrowserSession.rememberCertificate")(function* ( + trust: CertificateTrust, + key: string, + approval: CertificateApproval, + revocations: number, + ) { + if (trust.revocations !== revocations) return false; + if (trust.file !== null) { + yield* writeApprovalsFile(trust.file, new Map(trust.approvals).set(key, approval)); + // A revocation during the write queues its own rewrite behind this one. + if (trust.revocations !== revocations) return false; + } + trust.approvals.set(key, approval); + return true; + }, writeCertificateApprovals.withPermits(1)); + /** + * Rewrites the file from memory. Clearing cookies runs this even when memory + * is already empty, so retrying after a failed removal still deletes the file. + */ + const saveCertificateApprovals = Effect.fn("BrowserSession.saveCertificateApprovals")(function* ( + trust: CertificateTrust, + ) { + if (trust.file !== null) yield* writeApprovalsFile(trust.file, trust.approvals); + }, writeCertificateApprovals.withPermits(1)); + + // One prompt at a time; tabs that hit the same certificate wait on it. + let certificatePrompt: { + readonly trust: CertificateTrust; + readonly key: string; + readonly callbacks: Array<(trusted: boolean) => void>; + } | null = null; + const confirmCertificate = ( + contents: WebContents, + trust: CertificateTrust, + approval: CertificateApproval, + certificate: Certificate, + reason: string, + callback: (trusted: boolean) => void, + ): void => { + const key = certificateApprovalKey(approval); + if (certificatePrompt !== null) { + if (certificatePrompt.trust === trust && certificatePrompt.key === key) { + certificatePrompt.callbacks.push(callback); + } else { + callback(false); + } + return; + } + const prompt = { trust, key, callbacks: [callback] }; + certificatePrompt = prompt; + // A prompt for a tab that closed or moved on would approve a page nobody is waiting for. + const dismiss = new AbortController(); + const abandon = () => dismiss.abort(); + const navigated = ( + event: Electron.Event, + ) => { + if (event.isMainFrame && !event.isSameDocument) abandon(); + }; + contents.once("destroyed", abandon); + contents.on("did-start-navigation", navigated); + const settle = (trusted: boolean) => + Effect.sync(() => { + certificatePrompt = null; + if (!contents.isDestroyed()) { + contents.off("destroyed", abandon); + contents.off("did-start-navigation", navigated); + } + for (const waiter of prompt.callbacks) waiter(trusted); + }); + const host = new URL(approval.origin).host; + const revocations = trust.revocations; + runFork( + electronDialog + .showMessageBox( + { + type: "warning", + buttons: ["Trust and Remember", "Cancel"], + defaultId: 1, + cancelId: 1, + signal: dismiss.signal, + message: `${host} has an untrusted certificate`, + detail: [ + reason, + "", + `Issued to: ${promptCertificateName(certificate.subjectName)}`, + `Issued by: ${promptCertificateName(certificate.issuerName)}`, + `Valid: ${promptCertificateDate(certificate.validStart)} – ${promptCertificateDate(certificate.validExpiry)}`, + `Fingerprint: ${approval.fingerprint}`, + "", + "Trust it only if you know this certificate belongs to the site. Clearing this browser profile's cookies forgets it.", + ].join("\n"), + }, + // In the reporter's Windows test, a box owned by the host window + // blocked all mouse input, its own buttons included, until it closed. + platform === "win32" ? undefined : previewHostWindow(contents), + ) + .pipe( + Effect.flatMap(({ response }) => + response === 0 && !dismiss.signal.aborted + ? rememberCertificate(trust, key, approval, revocations).pipe( + Effect.catch((cause) => + Effect.logWarning("Failed to save a certificate approval.", { cause }).pipe( + Effect.andThen( + electronDialog.showErrorBox( + `Could not trust the certificate for ${host}`, + "T3 Code could not save this approval, so the page stays blocked. Check that the disk is not full and its data folder is writable, then try again.", + ), + ), + Effect.as(false), + ), + ), + ) + : Effect.succeed(false), + ), + Effect.orElseSucceed(() => false), + Effect.flatMap(settle), + Effect.onInterrupt(() => settle(false)), + ), + ); + }; + yield* electronApp.on( + "certificate-error", + ( + event: Electron.Event, + contents: WebContents, + url: string, + error: string, + certificate: Certificate, + callback: (trusted: boolean) => void, + isMainFrame: boolean, + ) => { + if (contents.isDestroyed()) return; + // Sessions this service did not create keep Electron's denial. + const trust = certificateTrust.get(contents.session); + const origin = certificateOrigin(url); + const reason = OVERRIDABLE_CERTIFICATE_ERRORS.get(error); + if (trust === undefined || origin === null || reason === undefined) return; + const approval = { origin, fingerprint: certificate.fingerprint, error }; + event.preventDefault(); + if (trust.approvals.has(certificateApprovalKey(approval))) { + callback(true); + return; + } + // Only a page navigation asks; a subresource the user has not accepted stays blocked. + if (!isMainFrame) { + callback(false); + return; + } + confirmCertificate(contents, trust, approval, certificate, reason, callback); + }, + ); + + /** Clearing a profile's cookies also forgets the certificates its user trusted. */ + const revokeCertificates = Effect.fn("BrowserSession.revokeCertificates")(function* ( + partition: string, + browserSession: Session, + ) { + const trust = certificateTrust.get(browserSession); + if (trust === undefined) return; + const hadApprovals = trust.approvals.size > 0; + trust.revocations += 1; + trust.approvals.clear(); + const saved = yield* saveCertificateApprovals(trust).pipe(Effect.result); + // An open connection already passed its certificate check. + if (hadApprovals) { + yield* Effect.tryPromise({ + try: () => browserSession.closeAllConnections(), + catch: (cause) => new BrowserSessionStorageClearError({ partition, cause }), + }); + } + if (Result.isFailure(saved)) { + return yield* new BrowserSessionStorageClearError({ partition, cause: saved.failure }); + } + }); + const sessionsRef = yield* SynchronizedRef.make>(new Map()); const getPartition = Effect.fn("BrowserSession.getPartition")(function* ( @@ -361,9 +657,7 @@ export const make = Effect.gen(function* BrowserSessionMake() { }); }, ); - const next = new Map(sessions); - next.set(partition, browserSession); - return [browserSession, next] as const; + return browserSession; }, catch: (cause) => new BrowserSessionCreationError({ @@ -371,7 +665,18 @@ export const make = Effect.gen(function* BrowserSessionMake() { partition, cause, }), - }); + }).pipe( + Effect.flatMap((browserSession) => + loadCertificateTrust(browserSession).pipe( + Effect.map((trust) => { + certificateTrust.set(browserSession, trust); + const next = new Map(sessions); + next.set(partition, browserSession); + return [browserSession, next] as const; + }), + ), + ), + ); }); }); @@ -386,20 +691,37 @@ export const make = Effect.gen(function* BrowserSessionMake() { yield* Effect.forEach( selectSessions(sessions, partitions), ([partition, browserSession]) => - Effect.tryPromise({ - try: () => - browserSession.clearStorageData({ - storages: ["cookies", "localstorage", "indexdb", "serviceworkers", "cachestorage"], - }), - catch: (cause) => - new BrowserSessionStorageClearError({ - partition, - cause, - }), + Effect.gen(function* () { + const cleared = yield* Effect.tryPromise({ + try: () => + browserSession.clearStorageData({ + storages: [ + "cookies", + "localstorage", + "indexdb", + "serviceworkers", + "cachestorage", + ], + }), + catch: (cause) => + new BrowserSessionStorageClearError({ + partition, + cause, + }), + }).pipe(Effect.result); + // Approvals go even when the storage clear fails, since the user asked to forget them. + yield* revokeCertificates(partition, browserSession); + if (Result.isFailure(cleared)) return yield* cleared.failure; }), { concurrency: "unbounded", discard: true }, ); }), + cookiesCleared: Effect.fn("BrowserSession.cookiesCleared")(function* (browserSession) { + const sessions = yield* SynchronizedRef.get(sessionsRef); + for (const [partition, candidate] of sessions) { + if (candidate === browserSession) yield* revokeCertificates(partition, browserSession); + } + }), clearCache: Effect.fn("BrowserSession.clearCache")(function* (partitions?) { const sessions = yield* SynchronizedRef.get(sessionsRef); yield* Effect.forEach( diff --git a/apps/desktop/src/preview/CdpRelay.test.ts b/apps/desktop/src/preview/CdpRelay.test.ts index 6e657ba5fc4d..564b4dc57d6f 100644 --- a/apps/desktop/src/preview/CdpRelay.test.ts +++ b/apps/desktop/src/preview/CdpRelay.test.ts @@ -9,6 +9,7 @@ import { const makeTarget = ( send: CdpRelayTarget["send"], setDownloadDirectory: CdpRelayTarget["setDownloadDirectory"] = () => {}, + ackScreencastFrame: CdpRelayTarget["ackScreencastFrame"] = () => {}, ): CdpRelayTarget => ({ send, targetId: async () => "GUEST-TARGET", @@ -16,6 +17,8 @@ const makeTarget = ( title: () => "Fixture", userAgent: () => "Electron", setDownloadDirectory, + receivesScreencast: () => true, + ackScreencastFrame, }); const makeRelay = (target: CdpRelayTarget, write: (raw: string) => void) => @@ -137,9 +140,13 @@ describe("CDP relay", () => { describe("CDP relay extra sessions", () => { it("gives a second page session the page's commands and events", async () => { - const send = vi.fn(async () => ({})); + const send = vi.fn(async () => ({})); const written: Array> = []; - const relay = makeRelay(makeTarget(send), (raw) => written.push(JSON.parse(raw))); + const nativeAcks: Array = []; + const relay = makeRelay( + makeTarget(send, undefined, (frameId) => nativeAcks.push(frameId)), + (raw) => written.push(JSON.parse(raw)), + ); relay.receive(JSON.stringify({ id: 1, method: "Target.setAutoAttach", params: {} })); relay.receive(JSON.stringify({ id: 2, method: "Target.attachToBrowserTarget" })); await settle(); @@ -169,8 +176,63 @@ describe("CDP relay extra sessions", () => { await settle(); expect(send).toHaveBeenLastCalledWith("Page.startScreencast", {}, undefined, extra); written.length = 0; - relay.event("Page.screencastFrame", { data: "x" }, undefined); - expect(written.map((m) => m["sessionId"])).toEqual(["t3-preview-page", extra]); + const frame = (frameId: number) => + relay.event("Page.screencastFrame", { data: "x", sessionId: frameId }, undefined); + const ack = (sessionId: string, frameId: number) => + relay.receive( + JSON.stringify({ + id: 6, + method: "Page.screencastFrameAck", + params: { sessionId: frameId }, + sessionId, + }), + ); + const delivered = () => { + const frames = written + .filter((m) => m["method"]) + .map((m) => [m["sessionId"], (m["params"] as { sessionId?: number }).sessionId]); + written.length = 0; + return frames; + }; + frame(1); + expect(delivered()).toEqual([ + ["t3-preview-page", 1], + [extra, 1], + ]); + // A session that has not acknowledged its frame gets only the newest one, on its ack. + for (let i = 2; i < 100; i++) frame(i); + expect(delivered()).toEqual([]); + // Frames every session is busy for stay unacknowledged, so Chromium pauses at its in-flight limit. + expect(nativeAcks).toEqual([1]); + ack(extra, 1); + await settle(); + expect(delivered()).toEqual([[extra, 99]]); + // Each consumer ack lets Chromium send one more frame. + expect(nativeAcks).toEqual([1, 2]); + ack(extra, 99); + await settle(); + expect(delivered()).toEqual([]); + expect(nativeAcks).toEqual([1, 2, 3]); + frame(100); + expect(delivered()).toEqual([[extra, 100]]); + ack(extra, 99); + await settle(); + frame(101); + expect(delivered()).toEqual([]); + expect(send.mock.calls.some(([method]) => method === "Page.screencastFrameAck")).toBe(false); + // A restart keeps the unacknowledged frame's slot until its own acknowledgement. + relay.receive(JSON.stringify({ id: 7, method: "Page.stopScreencast", sessionId: extra })); + relay.receive(JSON.stringify({ id: 8, method: "Page.startScreencast", sessionId: extra })); + await settle(); + delivered(); + frame(102); + expect(delivered()).toEqual([]); + ack(extra, 100); + await settle(); + expect(delivered()).toEqual([[extra, 102]]); + // Other events still reach a session while its frame is unacknowledged. + relay.event("Page.loadEventFired", { timestamp: 1 }, undefined); + expect(delivered().map(([session]) => session)).toEqual(["t3-preview-page", extra]); }); }); diff --git a/apps/desktop/src/preview/CdpRelay.ts b/apps/desktop/src/preview/CdpRelay.ts index 94a78f47db68..e9f3cf509b44 100644 --- a/apps/desktop/src/preview/CdpRelay.ts +++ b/apps/desktop/src/preview/CdpRelay.ts @@ -9,6 +9,10 @@ export interface CdpRelayTarget { owner?: string, ) => Promise; readonly sessionDetached?: (owner: string) => void; + /** Whether a page session, by owner as `send` names it, gets the page's screencast frames. */ + readonly receivesScreencast: (owner: string | undefined) => boolean; + /** Acknowledges a native screencast frame, letting Chromium send the next one. */ + readonly ackScreencastFrame: (frameId: number) => void; /** The page's own CDP target id; Playwright keys the main frame by it. */ readonly targetId: () => Promise; readonly url: () => string; @@ -101,6 +105,20 @@ export function createCdpRelayConnection( let downloadEventsEnabled = false; /** Relay-created page sessions share one real debugger; child frame sessions retain their own ids. */ const sessions = new Map([[PAGE_SESSION_ID, "page"]]); + // Each page session holds one unacknowledged frame and keeps the newest one skipped meanwhile. + const unackedFrames = new Map(); + const skippedFrames = new Map>(); + // Chromium keeps a few frames in flight and reuses one frame id per run, so pacing it means + // leaving frames every session is busy for unacknowledged and releasing one per consumer ack. + const heldNativeFrames: Array = []; + const releaseHeldFramesWithoutHolders = () => { + if (skippedFrames.size === 0) + for (const frameId of heldNativeFrames.splice(0)) target.ackScreencastFrame(frameId); + }; + const sendFrame = (sessionId: string, params: Record) => { + unackedFrames.set(sessionId, params["sessionId"] as number); + send({ method: "Page.screencastFrame", params, sessionId }); + }; let sessionSequence = 0; const newSession = (kind: "browser" | "page") => { const sessionId = `t3-preview-${kind}-${++sessionSequence}`; @@ -168,6 +186,9 @@ export function createCdpRelayConnection( const sessionId = command.params?.["sessionId"]; if (typeof sessionId === "string" && sessionId !== PAGE_SESSION_ID) { sessions.delete(sessionId); + unackedFrames.delete(sessionId); + skippedFrames.delete(sessionId); + releaseHeldFramesWithoutHolders(); target.sessionDetached?.(sessionId); } return {}; @@ -200,6 +221,27 @@ export function createCdpRelayConnection( } // Playwright asks for this on every page; the host emulates focus only while it is in use. if (command.method === "Emulation.setFocusEmulationEnabled") return Promise.resolve({}); + if (command.method === "Page.screencastFrameAck" && child === undefined) { + // A restart keeps the slot too, or restarting would let frames past an unacknowledged one. + const sessionId = command.sessionId!; + if (unackedFrames.get(sessionId) === command.params?.["sessionId"]) { + unackedFrames.delete(sessionId); + const skipped = skippedFrames.get(sessionId); + skippedFrames.delete(sessionId); + if ( + skipped !== undefined && + target.receivesScreencast(sessionId === PAGE_SESSION_ID ? undefined : sessionId) + ) + sendFrame(sessionId, skipped); + const released = heldNativeFrames.shift(); + if (released !== undefined) target.ackScreencastFrame(released); + } + return Promise.resolve({}); + } + if (command.method === "Page.stopScreencast" && child === undefined) { + skippedFrames.delete(command.sessionId!); + releaseHeldFramesWithoutHolders(); + } // The shared debugger keeps Runtime enabled after an earlier connection, and a repeated // enable reports no existing contexts; Playwright waits for them, so reset once per connection. if (command.method === "Runtime.enable" && child === undefined) { @@ -260,6 +302,27 @@ export function createCdpRelayConnection( send({ method, params, sessionId }); return; } + if (method === "Page.screencastFrame") { + const frame = (params ?? {}) as Record; + const frameId = frame["sessionId"]; + if (typeof frameId !== "number") return; + let held = false; + let delivered = false; + for (const [id, kind] of sessions) { + if (kind !== "page") continue; + if (!target.receivesScreencast(id === PAGE_SESSION_ID ? undefined : id)) continue; + if (unackedFrames.has(id)) { + skippedFrames.set(id, frame); + held = true; + } else { + sendFrame(id, frame); + delivered = true; + } + } + if (held && !delivered) heldNativeFrames.push(frameId); + else target.ackScreencastFrame(frameId); + return; + } for (const [id, kind] of sessions) if (kind === "page") send({ method, params, sessionId: id }); }, diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index 94809f0d10cf..8e96129293fc 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -16,7 +16,21 @@ const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); const decodeCdpReply = Schema.decodeUnknownSync( Schema.fromJsonString(Schema.Struct({ id: Schema.Number })), ); +const decodeCdpMessage = Schema.decodeUnknownSync( + Schema.fromJsonString( + Schema.Struct({ + id: Schema.optional(Schema.Number), + method: Schema.optional(Schema.String), + sessionId: Schema.optional(Schema.String), + }), + ), +); const key = { threadId: "thread-1", tabId: "tab-1" }; + +/** A host whose client already draws `key` at full detail, so its screenshots go out at once. */ +const makeDrawnHost = Effect.tap(DesktopBrowserHost.make, (host) => + Effect.sync(() => host.setCaptureDrawn(key, true)), +); /** Sends a page command whose id the fake debugger records as it receives it. */ const command = ( host: DesktopBrowserHost.DesktopBrowserHost["Service"], @@ -43,6 +57,8 @@ const makeDebuggee = ( const pending: Array<() => void> = []; const focusChanges: Array<() => void> = []; const sent: Array = []; + const methods: Array = []; + const rejects: Array<() => void> = []; const sendWaiters: Array<() => void> = []; const focusEmulation: Array = []; const debuggee = Object.assign(emitter, { @@ -67,6 +83,8 @@ const makeDebuggee = ( return; } sent.push(params?.id); + methods.push(method); + rejects.push(() => reject(new Error("Refused"))); pending.push(() => resolve({ method })); sendWaiters.shift()?.(); }), @@ -94,6 +112,10 @@ const makeDebuggee = ( webContents.emit("did-fail-load", {}, code, "Failed", "http://localhost/", true), /** Ids of the commands the debugger received, in order. */ sent, + /** Methods of the same commands, in order. */ + methods, + /** Refuses the pending command the debugger received with this id. */ + refuse: (id: number) => rejects[sent.indexOf(id)]?.(), focusEmulation, nextSend: () => new Promise((resolve) => sendWaiters.push(resolve)), release: () => pending.splice(0).forEach((resolve) => resolve()), @@ -116,16 +138,24 @@ const takeEvents = (host: DesktopBrowserHost.DesktopBrowserHost["Service"], coun /** * Ids of the replies the host sends, in order. A screenshot's reply leaves after its turn * has passed on, so by then the next waiting screenshot, if any, has been sent. Subscribe - * before sending commands: a new subscriber releases the tab's relay. + * before sending commands: a new subscriber releases the tab's relay. Screencast frames + * go to `frames` by the session they were sent to. */ -const watchReplies = (host: DesktopBrowserHost.DesktopBrowserHost["Service"]) => +const watchReplies = ( + host: DesktopBrowserHost.DesktopBrowserHost["Service"], + frames: Array = [], +) => Effect.gen(function* () { const replies = yield* Queue.unbounded(); yield* host.events.pipe( Stream.map((line) => decodeEvent(new TextDecoder().decode(line))), - Stream.runForEach((event) => - event.type === "cdp" ? Queue.offer(replies, decodeCdpReply(event.message).id) : Effect.void, - ), + Stream.runForEach((event) => { + if (event.type !== "cdp") return Effect.void; + const { id, method, sessionId } = decodeCdpMessage(event.message); + if (id !== undefined) return Queue.offer(replies, id); + if (method === "Page.screencastFrame") frames.push(sessionId); + return Effect.void; + }), Effect.forkScoped, ); yield* Effect.yieldNow; @@ -147,7 +177,7 @@ const watchPainting = (host: DesktopBrowserHost.DesktopBrowserHost["Service"]) = describe("DesktopBrowserHost", () => { it.effect("announces tabs already attached to a backend that starts later", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; host.attach(key, makeDebuggee().tab); expect(yield* takeEvents(host, 1)).toEqual([{ type: "attached", ...key }]); @@ -157,7 +187,7 @@ describe("DesktopBrowserHost", () => { it.effect("announces a guest still loading its first page only once that page is up", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const loading = makeDebuggee({ loading: true }); const other = { ...key, tabId: "tab-2" }; const reader = yield* takeEvents(host, 3).pipe(Effect.forkScoped); @@ -180,7 +210,7 @@ describe("DesktopBrowserHost", () => { "keeps a loading guest unannounced across a replaced navigation and a new backend", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const loading = makeDebuggee({ loading: true }); const other = { ...key, tabId: "tab-2" }; host.attach(key, loading.tab); @@ -202,7 +232,7 @@ describe("DesktopBrowserHost", () => { "announces a first page that never comes up, but never a tab closed or destroyed first", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const closed = { ...key, tabId: "tab-2" }; const destroyed = { ...key, tabId: "tab-3" }; const destroyedGuest = makeDebuggee({ loading: true }); @@ -222,7 +252,7 @@ describe("DesktopBrowserHost", () => { it.effect("re-sends shown to a backend that starts later and when a tab attaches again", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; host.attach(key, makeDebuggee().tab); host.setShown(key, true); const reader = yield* takeEvents(host, 5).pipe(Effect.forkScoped); @@ -243,7 +273,7 @@ describe("DesktopBrowserHost", () => { it.effect("replays an explicit hidden report before announcing another tab", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const other = { ...key, tabId: "tab-2" }; host.setShown(key, false); host.attach(key, makeDebuggee().tab); @@ -260,7 +290,7 @@ describe("DesktopBrowserHost", () => { it.effect("drops replies from a relay the server released", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const reader = yield* takeEvents(host, 2).pipe(Effect.forkScoped); @@ -283,12 +313,12 @@ describe("DesktopBrowserHost", () => { expect(reply).toMatchObject({ type: "cdp" }); expect(decodeCdpReply((reply as { message: string }).message).id).toBe(2); - }).pipe(Effect.scoped), + }), ); it.effect("sends a page one screenshot at a time", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -302,12 +332,41 @@ describe("DesktopBrowserHost", () => { debuggee.releaseFirst(); expect(yield* Queue.take(replies)).toBe(1); expect(debuggee.sent).toEqual([1, 3, 2]); - }).pipe(Effect.scoped), + }), + ); + + it.effect( + "sends a screenshot once the client draws the guest at full detail, or soon after", + () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + + yield* command(host, 1, "Page.captureScreenshot"); + yield* settle; + expect(debuggee.sent).toEqual([]); + host.setCaptureDrawn(key, true); + yield* settle; + expect(debuggee.sent).toEqual([1]); + debuggee.releaseFirst(); + expect(yield* Queue.take(replies)).toBe(1); + + // A client that never draws, such as a hidden window, only delays the capture. + host.setCaptureDrawn(key, false); + yield* command(host, 2, "Page.captureScreenshot"); + yield* settle; + expect(debuggee.sent).toEqual([1]); + yield* TestClock.adjust("1 second"); + yield* settle; + expect(debuggee.sent).toEqual([1, 2]); + }).pipe(Effect.scoped), ); it.effect("stops waiting on a screenshot the page navigated away from", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -327,12 +386,12 @@ describe("DesktopBrowserHost", () => { debuggee.releaseFirst(); expect(yield* Queue.take(replies)).toBe(2); expect(debuggee.sent).toEqual([1, 2, 3]); - }).pipe(Effect.scoped), + }), ); it.effect("drops screenshots a released relay left waiting", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); @@ -352,7 +411,7 @@ describe("DesktopBrowserHost", () => { it.effect("emulates focus while the server drives a guest, then lets a hidden one throttle", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); @@ -383,7 +442,7 @@ describe("DesktopBrowserHost", () => { it.effect("keeps a guest focused while a command it is running is still pending", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -399,12 +458,12 @@ describe("DesktopBrowserHost", () => { expect(yield* Queue.take(replies)).toBe(1); yield* TestClock.adjust("1 minute"); expect(debuggee.focusEmulation).toEqual([true, false]); - }).pipe(Effect.scoped), + }), ); it.effect("lets a guest throttle once Chromium can no longer answer its pending command", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); @@ -426,7 +485,7 @@ describe("DesktopBrowserHost", () => { it.effect("keeps a guest focused for the screenshot a navigation lets start", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); @@ -442,7 +501,7 @@ describe("DesktopBrowserHost", () => { it.effect("never sends a screenshot a navigation superseded while focus was changing", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee({ deferFocus: true }); host.attach(key, debuggee.tab); @@ -459,7 +518,7 @@ describe("DesktopBrowserHost", () => { it.effect("keeps a guest focused when its turn starts after the agent's first command", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const turn = (active: boolean) => @@ -481,7 +540,7 @@ describe("DesktopBrowserHost", () => { it.effect("lets a guest throttle through a turn that only sent it housekeeping", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); yield* host.handleCommandLine( @@ -510,7 +569,7 @@ describe("DesktopBrowserHost", () => { it.effect("keeps a guest the agent used focused through its turn's pauses", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const turn = (active: boolean) => @@ -538,7 +597,7 @@ describe("DesktopBrowserHost", () => { "ends a command's hold with the turn that sent it, even if Chromium never answers", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); yield* host.handleCommandLine( @@ -559,7 +618,7 @@ describe("DesktopBrowserHost", () => { it.effect("ends the agent's own session's holds with its turn, but not a viewer's stream", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -587,12 +646,12 @@ describe("DesktopBrowserHost", () => { yield* turn(false); yield* TestClock.adjust("1 minute"); expect(debuggee.focusEmulation).toEqual([true, false, true]); - }).pipe(Effect.scoped), + }), ); it.effect("paints for a screenshot queued behind one its viewer abandoned mid-capture", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -625,12 +684,12 @@ describe("DesktopBrowserHost", () => { expect(yield* Queue.take(replies)).toBe(4); yield* settle; expect(painting).toEqual([true, false]); - }).pipe(Effect.scoped), + }), ); it.effect("stops painting for a queued screenshot once its session detaches", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -664,12 +723,12 @@ describe("DesktopBrowserHost", () => { expect(painting).toEqual([true, false]); yield* TestClock.adjust("1 minute"); expect(debuggee.focusEmulation).toEqual([true, false]); - }).pipe(Effect.scoped), + }), ); it.effect("never sends a screenshot whose session detached while focus was changing", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee({ deferFocus: true }); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -688,12 +747,12 @@ describe("DesktopBrowserHost", () => { debuggee.settleFocus(); yield* settle; expect(debuggee.sent).toEqual([]); - }).pipe(Effect.scoped), + }), ); it.effect("holds no focus for a screenshot still queued when the agent's turn ends", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const painting = yield* watchPainting(host); @@ -718,12 +777,12 @@ describe("DesktopBrowserHost", () => { debuggee.release(); yield* settle; expect(painting).toEqual([true, false]); - }).pipe(Effect.scoped), + }), ); it.effect("lets a guest throttle when Chromium refuses a viewer's screencast", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee({ refuse: (method) => method === "Page.startScreencast" }); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -739,12 +798,12 @@ describe("DesktopBrowserHost", () => { expect(painting).toEqual([true, false]); yield* TestClock.adjust("1 minute"); expect(debuggee.focusEmulation).toEqual([true, false]); - }).pipe(Effect.scoped), + }), ); it.effect("lets a guest throttle after a person's input through a viewer that then closed", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -768,12 +827,12 @@ describe("DesktopBrowserHost", () => { yield* settle; yield* TestClock.adjust("1 minute"); expect(debuggee.focusEmulation).toEqual([true, false]); - }).pipe(Effect.scoped), + }), ); it.effect("keeps a page's screencast when a child session's screencast is refused", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee({ refuse: (method, sessionId) => method === "Page.startScreencast" && sessionId !== undefined, }); @@ -790,12 +849,12 @@ describe("DesktopBrowserHost", () => { expect(yield* Queue.take(replies)).toBe(2); yield* TestClock.adjust("1 minute"); expect(debuggee.focusEmulation).toEqual([true]); - }).pipe(Effect.scoped), + }), ); it.effect("ignores a screencast that arrives on a session after it detached", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -816,12 +875,12 @@ describe("DesktopBrowserHost", () => { yield* settle; yield* TestClock.adjust("1 minute"); expect(debuggee.focusEmulation).toEqual([true, false]); - }).pipe(Effect.scoped), + }), ); it.effect("lets a guest throttle once the session waiting on its command detaches", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -843,12 +902,12 @@ describe("DesktopBrowserHost", () => { }); yield* TestClock.adjust("1 minute"); expect(debuggee.focusEmulation).toEqual([true, false]); - }).pipe(Effect.scoped), + }), ); it.effect("never sends a screenshot queued by a session that has since detached", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -874,12 +933,12 @@ describe("DesktopBrowserHost", () => { yield* TestClock.adjust("1 minute"); expect(debuggee.focusEmulation).toEqual([true, false]); - }).pipe(Effect.scoped), + }), ); it.effect("keeps a streamed guest focused until its screencast ends", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); @@ -908,7 +967,7 @@ describe("DesktopBrowserHost", () => { it.effect("keeps a guest focused while any of its screencasts runs", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -935,12 +994,12 @@ describe("DesktopBrowserHost", () => { }); yield* TestClock.adjust("1 minute"); expect(debuggee.focusEmulation).toEqual([true, false]); - }).pipe(Effect.scoped), + }), ); it.effect("paints a hidden guest only while the screenshot it is taking is pending", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const painting = yield* watchPainting(host); @@ -961,12 +1020,12 @@ describe("DesktopBrowserHost", () => { debuggee.release(); yield* settle; expect(painting).toEqual([true, false]); - }).pipe(Effect.scoped), + }), ); it.effect("stops painting for a screenshot Chromium never answers once its document goes", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const painting = yield* watchPainting(host); @@ -1006,12 +1065,12 @@ describe("DesktopBrowserHost", () => { host.detach(key); yield* settle; expect(painting).toEqual([true, false, true, false, true, false, true, false]); - }).pipe(Effect.scoped), + }), ); it.effect("paints a streamed guest until its last screencast ends", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -1037,14 +1096,190 @@ describe("DesktopBrowserHost", () => { }); yield* settle; expect(painting).toEqual([true, false]); - }).pipe(Effect.scoped), + }), ); - it.effect("saves a server tab's download under its CDP guid where the server asked", () => + it.effect("keeps a guest's one screencast running until its last streaming session stops", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); + const frames: Array = []; + const replies = yield* watchReplies(host, frames); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + const nativeStops = () => + debuggee.methods.filter((method) => method === "Page.stopScreencast").length; + + // A viewer and a recording share Chromium's single screencast for the guest. + yield* raw({ id: 0, method: "Target.setAutoAttach", params: { autoAttach: true } }); + expect(yield* Queue.take(replies)).toBe(0); + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + yield* raw({ id: 2, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect([yield* Queue.take(replies), yield* Queue.take(replies)]).toEqual([1, 2]); + yield* raw({ id: 3, method: "Page.startScreencast", sessionId: "t3-preview-page-1" }); + yield* raw({ id: 4, method: "Page.startScreencast", sessionId: "t3-preview-page-2" }); + yield* settle; + debuggee.release(); + yield* raw({ id: 5, method: "Page.stopScreencast", sessionId: "t3-preview-page-1" }); + yield* settle; + expect(nativeStops(), "stopped the screencast another session still streams").toBe(0); + debuggee.emit("Page.screencastFrame", { data: "frame", sessionId: 1 }); + yield* settle; + expect(frames, "sent a stopped or idle session the screencast's frames").toEqual([ + "t3-preview-page-2", + ]); + + yield* raw({ id: 6, method: "Page.stopScreencast", sessionId: "t3-preview-page-2" }); + yield* settle; + expect(nativeStops()).toBe(1); + }), + ); + + it.effect("keeps a session's screencast when another of its starts is refused", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const frames: Array = []; + const replies = yield* watchReplies(host, frames); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + + yield* raw({ id: 0, method: "Target.setAutoAttach", params: { autoAttach: true } }); + expect(yield* Queue.take(replies)).toBe(0); + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(1); + yield* raw({ + id: 2, + method: "Page.startScreencast", + params: { id: 2 }, + sessionId: "t3-preview-page-1", + }); + yield* raw({ + id: 3, + method: "Page.startScreencast", + params: { id: 3 }, + sessionId: "t3-preview-page-1", + }); + yield* settle; + debuggee.refuse(2); + debuggee.release(); + expect([yield* Queue.take(replies), yield* Queue.take(replies)].toSorted()).toEqual([2, 3]); + yield* settle; + expect(debuggee.methods).not.toContain("Page.stopScreencast"); + debuggee.emit("Page.screencastFrame", { data: "frame", sessionId: 1 }); + yield* settle; + expect(frames).toContain("t3-preview-page-1"); + + // A refused replacement leaves the session's running stream, shared with another session. + yield* raw({ id: 4, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(4); + yield* raw({ + id: 5, + method: "Page.startScreencast", + params: { id: 5 }, + sessionId: "t3-preview-page-2", + }); + yield* raw({ + id: 6, + method: "Page.startScreencast", + params: { id: 6 }, + sessionId: "t3-preview-page-1", + }); + yield* settle; + debuggee.refuse(6); + debuggee.release(); + expect([yield* Queue.take(replies), yield* Queue.take(replies)].toSorted()).toEqual([5, 6]); + yield* raw({ + id: 7, + method: "Page.screencastFrameAck", + params: { sessionId: 1 }, + sessionId: "t3-preview-page-1", + }); + expect(yield* Queue.take(replies)).toBe(7); + yield* settle; + frames.length = 0; + debuggee.emit("Page.screencastFrame", { data: "frame", sessionId: 2 }); + yield* settle; + expect(frames).toEqual(["t3-preview-page-1", "t3-preview-page-2"]); + }), + ); + + it.effect("stops a guest's screencast once its last streaming session detaches", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + const nativeStops = () => + debuggee.methods.filter((method) => method === "Page.stopScreencast").length; + + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + yield* raw({ id: 2, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect([yield* Queue.take(replies), yield* Queue.take(replies)]).toEqual([1, 2]); + yield* raw({ id: 3, method: "Page.startScreencast", sessionId: "t3-preview-page-1" }); + yield* raw({ id: 4, method: "Page.startScreencast", sessionId: "t3-preview-page-2" }); + yield* settle; + debuggee.release(); + // Streamers go without stopping, such as viewers that closed. + yield* raw({ + id: 5, + method: "Target.detachFromTarget", + params: { sessionId: "t3-preview-page-1" }, + }); + yield* settle; + expect(nativeStops(), "stopped the screencast another session still streams").toBe(0); + yield* raw({ + id: 6, + method: "Target.detachFromTarget", + params: { sessionId: "t3-preview-page-2" }, + }); + const answered = []; + for (let reply = 0; reply < 4; reply++) answered.push(yield* Queue.take(replies)); + expect(answered.toSorted()).toEqual([3, 4, 5, 6]); + yield* settle; + expect(nativeStops()).toBe(1); + }), + ); + + it.effect("stops a screencast whose start lands after its session detached", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee({ deferFocus: true }); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(1); + // The start waits on focus emulation while the viewer closes. + yield* raw({ id: 2, method: "Page.startScreencast", sessionId: "t3-preview-page-1" }); + yield* raw({ + id: 3, + method: "Target.detachFromTarget", + params: { sessionId: "t3-preview-page-1" }, + }); + expect(yield* Queue.take(replies)).toBe(3); + debuggee.settleFocus(); + yield* settle; + debuggee.release(); + expect(yield* Queue.take(replies)).toBe(2); + yield* settle; + expect(debuggee.methods.at(-1), "left the late screencast running").toBe( + "Page.stopScreencast", + ); + }), + ); + + it.effect("saves a server tab's download under its CDP guid where the server asked", () => + Effect.gen(function* () { + const host = yield* makeDrawnHost; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); const paths: Array = []; const item = { @@ -1072,12 +1307,12 @@ describe("DesktopBrowserHost", () => { expect(host.placeDownload(debuggee.tab.webContents, item)).toBe(true); expect(paths).toEqual(["/srv/downloads/guid-1"]); expect(host.placeDownload(debuggee.tab.webContents, item)).toBe(false); - }).pipe(Effect.scoped), + }), ); it.effect("tells a download the person clicked from one the agent's input started", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); @@ -1104,9 +1339,44 @@ describe("DesktopBrowserHost", () => { }), ); - it.effect("detaches only the tab whose CDP event is too large to relay", () => + it.effect("stops crediting the agent once a person acts through a viewer", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + const viewer = (id: number, method: string, type: string) => + raw({ id, method, params: { type }, sessionId: "t3-preview-page-1" }); + const led = () => host.agentLedPage(debuggee.tab.webContents); + + yield* command(host, 1, "DOM.getDocument"); + expect(led()).toBe(false); + yield* raw({ id: 2, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(2); + yield* raw({ id: 3, method: "Page.startScreencast", sessionId: "t3-preview-page-1" }); + + for (const [method, type] of [ + ["Input.dispatchKeyEvent", "rawKeyDown"], + ["Input.dispatchMouseEvent", "mousePressed"], + ["Input.dispatchTouchEvent", "touchStart"], + ] as const) { + yield* command(host, 4, "Runtime.evaluate"); + expect(led()).toBe(true); + // The viewer's pointer only passes over the page the agent drives. + yield* viewer(5, "Input.dispatchMouseEvent", "mouseMoved"); + yield* viewer(6, "Input.dispatchMouseEvent", "mouseWheel"); + expect(led()).toBe(true); + yield* viewer(7, method, type); + expect(led()).toBe(false); + } + }).pipe(Effect.scoped), + ); + + it.effect("detaches only the tab whose CDP event is too large to relay", () => + Effect.gen(function* () { + const host = yield* makeDrawnHost; const tabs = [makeDebuggee({ deferFocus: true }), makeDebuggee()]; const send = vi.spyOn(tabs[0]!.tab.debugger, "sendCommand"); const keys = [key, { threadId: "thread-2", tabId: "tab-2" }]; @@ -1170,6 +1440,6 @@ describe("DesktopBrowserHost", () => { expect(JSON.parse((next as { message: string }).message)).toMatchObject({ method: "Page.loadEventFired", }); - }).pipe(Effect.scoped), + }), ); }); diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index 2cda717a891b..ce5d7074a786 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -26,8 +26,12 @@ const AGENT_INPUT_COMMAND = /"method":"(?:Input\.|Page\.navigate"|Runtime\.(?:evaluate|callFunctionOn)")/; /** The same commands as above, by method, for the relay's sends. */ const AGENT_INPUT_METHOD = /^(?:Input\.|Page\.navigate$|Runtime\.(?:evaluate|callFunctionOn)$)/; -const AGENT_DOWNLOAD_WINDOW_MS = 5_000; +const AGENT_INPUT_WINDOW_MS = 5_000; +/** A viewer's pointer travelling over the stream activates nothing on the page. */ +const POINTER_TRAVEL = new Set(["mouseMoved", "mouseWheel"]); const FOCUS_IDLE = "5 seconds"; +/** How long a screenshot waits for the client to draw its guest at full detail. */ +const CAPTURE_DRAW_WAIT = "1 second"; /** Well inside the server's 10 s DESKTOP_ATTACH_TIMEOUT, for a first page that never reports it is up. */ const FIRST_PAGE_TIMEOUT = "5 seconds"; const ERR_ABORTED = -3; @@ -40,6 +44,8 @@ export interface DesktopBrowserTabKey { export interface DesktopBrowserTabDebugger { readonly webContents: Electron.WebContents; readonly debugger: Electron.Debugger; + /** Runs after the server clears the tab's cookies, so state kept beside them goes too. */ + readonly cookiesCleared?: () => Promise; } const keyOf = ({ threadId, tabId }: DesktopBrowserTabKey) => `${threadId}\u0000${tabId}`; @@ -58,6 +64,8 @@ interface AttachedTab { downloadDirectory: string | null; pendingDownloadGuid: string | null; agentInputAt: number; + /** When the agent last drove the page with no person's input through a viewer since. */ + agentLedAt: number; focusEmulated: boolean; focusChange: Promise | null; focusUses: number; @@ -73,8 +81,8 @@ interface AttachedTab { */ agentUsed: boolean; focusReleasing: boolean; - /** The sessions streaming the guest; one stopping leaves the others' screencasts running. */ - readonly screencasts: Set; + /** The sessions sharing the guest's one Chromium screencast; it stops with the last of them. */ + readonly screencasts: Map; readonly onMessage: ( event: Electron.Event, method: string, @@ -95,7 +103,11 @@ export class DesktopBrowserHost extends Context.Service< readonly detach: (key: DesktopBrowserTabKey) => void; readonly loadFailed: (key: DesktopBrowserTabKey, failure: PreviewAutomationLoadError) => void; readonly setShown: (key: DesktopBrowserTabKey, shown: boolean | undefined) => void; + /** Whether the client draws the guest at full detail while it is captured. */ + readonly setCaptureDrawn: (key: DesktopBrowserTabKey, drawn: boolean | undefined) => void; readonly humanStartedDownload: (source: Electron.WebContents) => boolean; + /** Whether what the page does now follows the agent's input, not a viewer's. */ + readonly agentLedPage: (source: Electron.WebContents) => boolean; readonly placeDownload: (source: Electron.WebContents, item: Electron.DownloadItem) => boolean; readonly pointers: Stream.Stream<{ readonly key: DesktopBrowserTabKey; @@ -130,6 +142,8 @@ export const make = Effect.gen(function* () { // Kept apart from attachment: the renderer can show a webview before its debugger attaches, // and the server forgets `shown` on detach, for example while DevTools holds the debugger. const shownTabs = new Map(); + const drawnTabs = new Set(); + const drawWaiters = new Map void>>(); const emit = (event: DesktopBrowserEventType) => runFork(PubSub.publish(outbox, event)); const nextScreenshot = (tab: AttachedTab) => { @@ -258,12 +272,32 @@ export const make = Effect.gen(function* () { }); }; + /** + * A scaled guest rasters at its on-screen size until the client draws it at full detail, + * and a screenshot taken before that copies the blurred frame. A client that cannot draw + * (a hidden window, say) only delays the capture. + */ + const whenDrawn = (tab: AttachedTab) => + new Promise((resolve) => { + const id = keyOf(tab.key); + const waiters = drawWaiters.get(id) ?? new Set(); + drawWaiters.set(id, waiters); + const done = () => { + waiters.delete(done); + if (waiters.size === 0 && drawWaiters.get(id) === waiters) drawWaiters.delete(id); + resolve(); + }; + waiters.add(done); + runFork(Effect.sleep(CAPTURE_DRAW_WAIT).pipe(Effect.andThen(Effect.sync(done)))); + }); + /** Chromium answers a capture, and frees its wake lock, only once the guest paints a frame. */ const paintUntilAnswered = (tab: AttachedTab, send: () => Promise) => { const sent = {}; tab.capturing.add(sent); syncPainting(tab); - return send().finally(() => { + // Sent at once when already drawn, so it keeps its place among the commands after it. + return (drawnTabs.has(keyOf(tab.key)) ? send() : whenDrawn(tab).then(send)).finally(() => { tab.capturing.delete(sent); syncPainting(tab); }); @@ -287,9 +321,15 @@ export const make = Effect.gen(function* () { dropFocusHolds(tab); }; + /** An owner that leaves without stopping would otherwise keep the screencast running unseen. */ + const leaveScreencast = (tab: AttachedTab, owner: string | undefined) => { + if (!tab.screencasts.delete(owner) || tab.screencasts.size > 0) return; + void tab.debuggee.debugger.sendCommand("Page.stopScreencast").catch(() => {}); + }; + /** A detached session, such as a closed viewer, waits on nothing it sent. */ const dropSessionHolds = (tab: AttachedTab, owner: string) => { - tab.screencasts.delete(owner); + leaveScreencast(tab, owner); for (const [pending, holder] of tab.focusPending) { if (holder === owner) tab.focusPending.delete(pending); } @@ -315,9 +355,12 @@ export const make = Effect.gen(function* () { const relayFor = (tab: AttachedTab) => { if (tab.relay) return tab.relay; - const { webContents, debugger: debuggee } = tab.debuggee; - const send = (method: string, params: Record, sessionId?: string) => - sendPageCommand(webContents, debuggee, method, params, sessionId); + const { webContents, debugger: debuggee, cookiesCleared } = tab.debuggee; + const send = async (method: string, params: Record, sessionId?: string) => { + const result = await sendPageCommand(webContents, debuggee, method, params, sessionId); + if (method === "Network.clearBrowserCookies") await cookiesCleared?.(); + return result; + }; const detachedOwners = new Set(); /** Viewers and recordings stream; the agent's own sessions never do. */ const streamingOwners = new Set(); @@ -325,30 +368,41 @@ export const make = Effect.gen(function* () { { send: (method, params, sessionId, owner) => { // A child session's screencast is not the page's, such as a late one on a detached session. - if (sessionId === undefined && method === "Page.startScreencast") { - tab.screencasts.add(owner); + const stream = + sessionId === undefined && method === "Page.startScreencast" + ? (tab.screencasts.get(owner) ?? { started: false, pending: 0 }) + : undefined; + if (stream) { + stream.pending += 1; + tab.screencasts.set(owner, stream); if (owner !== undefined) streamingOwners.add(owner); syncPainting(tab); } else if (sessionId === undefined && method === "Page.stopScreencast") { tab.screencasts.delete(owner); syncPainting(tab); + if (tab.screencasts.size > 0) return Promise.resolve({}); } // A person's input through a viewer, or Playwright's own housekeeping, is not the agent's. - if ( - AGENT_INPUT_METHOD.test(method) && - (owner === undefined || !streamingOwners.has(owner)) - ) { - tab.agentUsed = true; + if (AGENT_INPUT_METHOD.test(method)) { + if (owner === undefined || !streamingOwners.has(owner)) { + tab.agentUsed = true; + tab.agentLedAt = performance.now(); + } else if (method.startsWith("Input.") && !POINTER_TRAVEL.has(params["type"])) { + tab.agentLedAt = Number.NEGATIVE_INFINITY; + } } // Waits for focus emulation; insertText and other non-CDP input would not queue behind it. const forward = (current?: () => boolean) => { const change = tab.focusChange; - const run = () => - tab.relay !== relay || (current !== undefined && !current()) + const stale = () => tab.relay !== relay || (current !== undefined && !current()); + const sendUnlessStale = () => + stale() ? Promise.reject(new Error("The page this screenshot was for has moved on.")) - : current === undefined - ? send(method, params, sessionId) - : paintUntilAnswered(tab, () => send(method, params, sessionId)); + : send(method, params, sessionId); + const run = () => + current === undefined || stale() + ? sendUnlessStale() + : paintUntilAnswered(tab, sendUnlessStale); return change === null ? run() : change.then(run); }; return method === "Page.captureScreenshot" @@ -360,19 +414,46 @@ export const make = Effect.gen(function* () { held ? holdFocus(tab, owner, () => forward(current)) : forward(current), ) : method === "Page.startScreencast" - ? holdFocus(tab, owner, forward).catch((cause: unknown) => { - // A viewer can outlive a refused stream; nothing is streaming for it. - if (sessionId === undefined && tab.relay === relay) tab.screencasts.delete(owner); - releaseFocusWhenIdle(tab); - syncPainting(tab); - throw cause; - }) + ? holdFocus(tab, owner, forward).then( + (result) => { + if (stream) { + stream.pending -= 1; + stream.started = true; + // A detach's stop can overtake a start still waiting on focus; nobody owns it now. + if (tab.screencasts.get(owner) !== stream && tab.screencasts.size === 0) { + void debuggee.sendCommand("Page.stopScreencast").catch(() => {}); + } + } + return result; + }, + (cause: unknown) => { + if (stream) stream.pending -= 1; + // A viewer can outlive a refused start; its running or pending stream still holds it. + if ( + stream && + !stream.started && + stream.pending === 0 && + tab.relay === relay && + tab.screencasts.get(owner) === stream + ) { + leaveScreencast(tab, owner); + } + releaseFocusWhenIdle(tab); + syncPainting(tab); + throw cause; + }, + ) : holdFocus(tab, owner, forward); }, sessionDetached: (owner) => { detachedOwners.add(owner); dropSessionHolds(tab, owner); }, + receivesScreencast: (owner) => tab.screencasts.has(owner), + ackScreencastFrame: (frameId) => + void debuggee + .sendCommand("Page.screencastFrameAck", { sessionId: frameId }) + .catch(() => {}), targetId: () => debuggee .sendCommand("Target.getTargetInfo") @@ -450,6 +531,7 @@ export const make = Effect.gen(function* () { downloadDirectory: null, pendingDownloadGuid: null, agentInputAt: Number.NEGATIVE_INFINITY, + agentLedAt: Number.NEGATIVE_INFINITY, focusEmulated: false, focusChange: null, focusUses: 0, @@ -458,7 +540,7 @@ export const make = Effect.gen(function* () { painting: false, agentUsed: false, focusReleasing: false, - screencasts: new Set(), + screencasts: new Map(), onMessage: (_event, method, params, sessionId) => { if (method === "Page.downloadWillBegin") { const guid = (params as { guid?: unknown } | undefined)?.guid; @@ -534,6 +616,17 @@ export const make = Effect.gen(function* () { if (tabs.get(id)?.stopWaiting === null) emit({ type: "shown", ...key, shown }); }; + const setCaptureDrawn = (key: DesktopBrowserTabKey, drawn: boolean | undefined) => { + const id = keyOf(key); + if (!drawn) { + drawnTabs.delete(id); + return; + } + drawnTabs.add(id); + for (const done of drawWaiters.get(id) ?? []) done(); + drawWaiters.delete(id); + }; + const loadFailed: DesktopBrowserHost["Service"]["loadFailed"] = (key, failure) => { if (tabs.has(keyOf(key))) emit({ type: "loadFailed", ...key, ...failure }); }; @@ -564,7 +657,12 @@ export const make = Effect.gen(function* () { const humanStartedDownload = (source: Electron.WebContents) => { const tab = [...tabs.values()].find((candidate) => candidate.debuggee.webContents === source); - return tab !== undefined && performance.now() - tab.agentInputAt > AGENT_DOWNLOAD_WINDOW_MS; + return tab !== undefined && performance.now() - tab.agentInputAt > AGENT_INPUT_WINDOW_MS; + }; + + const agentLedPage = (source: Electron.WebContents) => { + const tab = [...tabs.values()].find((candidate) => candidate.debuggee.webContents === source); + return tab !== undefined && performance.now() - tab.agentLedAt <= AGENT_INPUT_WINDOW_MS; }; const announceAll = Effect.suspend(() => { @@ -605,8 +703,10 @@ export const make = Effect.gen(function* () { detach, loadFailed, setShown, + setCaptureDrawn, placeDownload, humanStartedDownload, + agentLedPage, }); }); diff --git a/apps/desktop/src/preview/GuestInput.test.ts b/apps/desktop/src/preview/GuestInput.test.ts index a0889940a668..48191ce2c4b9 100644 --- a/apps/desktop/src/preview/GuestInput.test.ts +++ b/apps/desktop/src/preview/GuestInput.test.ts @@ -44,6 +44,30 @@ describe("preview guest input", () => { expect(guest.sendCommand).not.toHaveBeenCalledWith("Input.insertText", expect.anything()); }); + it("stops the guest navigation without stopping its embedder", async () => { + let guestLoading = true; + let hostLoading = true; + const guest = { + stop: () => { + guestLoading = false; + }, + }; + const debuggee = { + sendCommand: async () => { + hostLoading = false; + return {}; + }, + }; + await sendPageCommand( + guest as unknown as Electron.WebContents, + debuggee as unknown as Electron.Debugger, + "Page.stopLoading", + {}, + ); + expect(guestLoading).toBe(false); + expect(hostLoading).toBe(true); + }); + it.each([ { sessionId: undefined, diff --git a/apps/desktop/src/preview/GuestInput.ts b/apps/desktop/src/preview/GuestInput.ts index 2af13af8d7da..4d794beea62b 100644 --- a/apps/desktop/src/preview/GuestInput.ts +++ b/apps/desktop/src/preview/GuestInput.ts @@ -185,6 +185,10 @@ export const sendPageCommand = async ( else webContents.reload(); return {}; } + if (method === "Page.stopLoading") { + webContents.stop(); + return {}; + } if (!FOCUS_ROUTED_INPUT.has(method) || webContents.focusedFrame) { return debuggee.sendCommand(method, params); } diff --git a/apps/desktop/src/preview/GuestProtocol.ts b/apps/desktop/src/preview/GuestProtocol.ts index 8e7eee53ba68..ef3323c6d614 100644 --- a/apps/desktop/src/preview/GuestProtocol.ts +++ b/apps/desktop/src/preview/GuestProtocol.ts @@ -6,13 +6,16 @@ export const ANNOTATION_DRAFT_CHANNEL = "preview:annotation-draft"; export const ANNOTATION_THEME_CHANNEL = "preview:annotation-theme"; export const ANNOTATION_SEND_ENABLED_CHANNEL = "preview:annotation-send-enabled"; export const HUMAN_INPUT_CHANNEL = "preview:human-input"; -export const MOUSE_NAVIGATE_CHANNEL = "preview:mouse-navigate"; +// Sent to the hosting element (ipc-message). The web app mirrors this +// literal in HostedBrowserWebview because it cannot import from the desktop app. +export const POINTER_DOWN_HOST_CHANNEL = "preview:pointer-down"; export const PAGE_UNHANDLED_SHORTCUT_CHANNEL = "preview:page-unhandled-shortcut"; export const RECORDING_CURSOR_CHANNEL = "preview:recording-cursor"; export const RECORDING_POINTER_CHANNEL = "preview:recording-pointer"; export const RECORDING_KEY_CHANNEL = "preview:recording-key"; export const RECORDING_INPUT_CHANNEL = "preview:recording-input"; export const RECORDING_CONTROLLER_CHANNEL = "preview:recording-controller"; +export const THUMB_PRESS_CHANNEL = "preview:thumb-press"; export const PASSKEY_CREATE_CHANNEL = "preview:passkey-create"; export const PASSKEY_GET_CHANNEL = "preview:passkey-get"; export const PASSKEY_BRIDGE_ARGUMENT = "--t3code-preview-passkey-bridge"; diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 1760a3cea5a7..36bdb143afdf 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -205,6 +205,7 @@ const layerBrowserSession = Layer.succeed( isPartition: (partition) => partition.startsWith("persist:t3code-preview-"), getSession: () => Effect.succeed(previewSession as unknown as Electron.Session), clearCookies: () => Effect.void, + cookiesCleared: () => Effect.void, clearCache: () => Effect.void, }), ); @@ -348,6 +349,47 @@ const makeTestPreviewWebContents = ( } as unknown as TestPreviewWebContents; }; +const makeStillImage = (pixels = "fresh-png") => ({ + toPNG: () => Buffer.from(pixels), + toJPEG: () => Buffer.from(pixels), + getSize: () => ({ width: 100, height: 80 }), +}); + +const makeTestStillImageWebContents = (...args: Parameters) => { + const wc = makeTestPreviewWebContents(...args); + Object.assign(wc, { isDevToolsOpened: () => false }); + holdStillImagePaint(wc, Promise.resolve()); + return wc; +}; + +const holdStillImagePaint = ( + wc: TestPreviewWebContents, + paint: Promise | Array>, +) => { + vi.mocked(wc.debugger.sendCommand).mockImplementation(async (method) => { + if (method === "Page.getFrameTree") return { frameTree: { frame: { id: "main" } } }; + if (method === "Page.createIsolatedWorld") return { executionContextId: 7 }; + if (method === "Runtime.evaluate") { + await (Array.isArray(paint) ? paint.shift() : paint); + return { result: { value: true } }; + } + return undefined; + }); +}; + +const setupStill = (manager: PreviewManager.PreviewManager["Service"], paint = Promise.resolve()) => + Effect.gen(function* () { + const image = makeStillImage(); + const capturePage = vi.fn<() => Promise>().mockResolvedValue(image); + const wc = makeTestStillImageWebContents(capturePage); + holdStillImagePaint(wc, paint); + fromId.mockReturnValue(wc); + yield* manager.createTab("tab_still"); + yield* manager.registerWebview("tab_still", wc.id); + yield* manager.setColorScheme("tab_still", "system"); + return { image, capturePage, wc, capture: manager.captureScreenshot("tab_still") }; + }); + const setupRecordingRaceTabs = (manager: PreviewManager.PreviewManager["Service"]) => Effect.gen(function* () { const capturePage = vi.fn(async () => ({ @@ -575,6 +617,32 @@ describe("PreviewManager", () => { webviewSend.mockClear(); }); + effectIt.effect("preserves recording artifacts saved in the same millisecond", () => + withManager((manager) => + Effect.gen(function* () { + const stored = new Map(); + const save = (path: string, bytes: Uint8Array) => { + stored.set(path, Uint8Array.from(bytes)); + return undefined; + }; + writeFile.mockImplementationOnce(save).mockImplementationOnce(save); + const firstBytes = new Uint8Array([1, 2, 3]); + const secondBytes = new Uint8Array([4, 5, 6]); + const [first, second] = yield* Effect.all( + [ + manager.saveRecording("tab_first", "video/webm", firstBytes), + manager.saveRecording("tab_second", "video/webm", secondBytes), + ], + { concurrency: "unbounded" }, + ); + + expect(first.path).not.toBe(second.path); + expect(stored.get(first.path)).toEqual(firstBytes); + expect(stored.get(second.path)).toEqual(secondBytes); + }), + ), + ); + effectIt.effect.each([false, true])( "preserves simultaneous same-name downloads with an existing destination: %s", (existing) => @@ -616,8 +684,10 @@ describe("PreviewManager", () => { attach: () => {}, detach: () => {}, setShown: () => {}, + setCaptureDrawn: () => {}, loadFailed: () => {}, humanStartedDownload: () => true, + agentLedPage: () => false, placeDownload: () => true, }), Layer.mock(PreviewPasskeys.PreviewPasskeys)({ @@ -670,7 +740,7 @@ describe("PreviewManager", () => { if (existing) { expect(yield* fs.readFileString(`${downloads}/report.txt`)).toBe("existing download"); } - }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), Effect.scoped), + }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), ); effectIt.effect.each([ @@ -744,6 +814,137 @@ describe("PreviewManager", () => { ), ); + effectIt.effect.each([ + "navigate", + "reload", + "goBack", + "goForward", + "refresh", + "hardReload", + ] as const)("%s stops a page script that never yields before it navigates", (operation) => + withManager((manager) => + Effect.gen(function* () { + const preview = makeFaviconWebContents(); + // As in Chromium: the hung renderer answers nothing, and a terminate reaches it only + // while no navigation is pending. + let stopped = false; + const probed = yield* Deferred.make(); + const navigatedAfterStop: Array = []; + const navigated = () => { + navigatedAfterStop.push(stopped); + }; + const sendCommand = async (method: string) => { + if (method === "Runtime.evaluate") { + Deferred.doneUnsafe(probed, Effect.void); + await new Promise(() => {}); + } + if (method === "Runtime.terminateExecution") stopped = true; + }; + Object.assign(preview.webContents, { + getURL: () => "http://localhost:5173/busy", + reload: navigated, + reloadIgnoringCache: navigated, + loadURL: async () => navigated(), + navigationHistory: { + canGoBack: () => true, + canGoForward: () => true, + goBack: navigated, + goForward: navigated, + }, + debugger: { + isAttached: () => false, + attach: vi.fn(), + sendCommand, + on: vi.fn(), + off: vi.fn(), + }, + }); + fromId.mockReturnValue(preview.webContents); + yield* manager.createTab("tab_busy"); + yield* manager.registerWebview("tab_busy", 42); + + const leaving = yield* ( + operation === "navigate" + ? manager.navigate("tab_busy", "http://localhost:5173/next") + : operation === "reload" + ? manager.navigate("tab_busy", "http://localhost:5173/busy") + : manager[operation]("tab_busy") + ).pipe(Effect.forkChild); + const probing = yield* Effect.raceFirst( + Deferred.await(probed).pipe(Effect.as(true)), + Fiber.await(leaving).pipe(Effect.as(false)), + ); + if (probing) yield* TestClock.adjust(1_000); + yield* Fiber.join(leaving); + + expect(navigatedAfterStop).toEqual([true]); + }), + ), + ); + + effectIt.effect("keeps mouse back and forward buttons inside the preview tab", () => + withManager((manager) => + Effect.gen(function* () { + const preview = makeFaviconWebContents(); + const goBack = vi.fn(); + const goForward = vi.fn(); + const send = vi.fn(); + let canGoBack = false; + Object.assign(preview.webContents, { + navigationHistory: { + canGoBack: () => canGoBack, + canGoForward: () => true, + goBack, + goForward, + }, + getZoomFactor: () => 2, + send, + }); + fromId.mockReturnValue(preview.webContents); + yield* manager.createTab("tab_mouse_navigation"); + yield* manager.registerWebview("tab_mouse_navigation", 42); + + // Electron reports thumb buttons as "back"/"forward" for every frame of the guest, iframes included. + const beforeMouse = preview.listeners.get("before-mouse-event"); + expect(beforeMouse, "guest has no before-mouse-event listener").toBeDefined(); + const press = (type: "mouseDown" | "mouseUp", button: string) => { + const preventDefault = vi.fn(); + beforeMouse!({ preventDefault } as never, { type, button, x: 10, y: 10 } as never); + return preventDefault; + }; + + // At the start of tab history the press is still swallowed, or the host app goes back instead. + expect(press("mouseDown", "back")).toHaveBeenCalledOnce(); + expect(press("mouseUp", "back")).toHaveBeenCalledOnce(); + expect(goBack).not.toHaveBeenCalled(); + + canGoBack = true; + expect(press("mouseDown", "back")).toHaveBeenCalledOnce(); + expect(goBack).not.toHaveBeenCalled(); + expect(press("mouseUp", "back")).toHaveBeenCalledOnce(); + expect(goBack).toHaveBeenCalledOnce(); + expect(press("mouseDown", "forward")).toHaveBeenCalledOnce(); + expect(press("mouseUp", "forward")).toHaveBeenCalledOnce(); + expect(goForward).toHaveBeenCalledOnce(); + for (const button of ["left", "right"]) { + expect(press("mouseDown", button)).not.toHaveBeenCalled(); + expect(press("mouseUp", button)).not.toHaveBeenCalled(); + } + expect(goBack).toHaveBeenCalledOnce(); + expect(goForward).toHaveBeenCalledOnce(); + // The guest replays dropped presses for human input and recordings, in CSS px. + expect(send.mock.calls.filter(([channel]) => channel === "preview:thumb-press")).toEqual([ + ["preview:thumb-press", "down", 5, 5, 3], + ["preview:thumb-press", "up", 5, 5, 3], + ["preview:thumb-press", "down", 5, 5, 3], + ["preview:thumb-press", "up", 5, 5, 3], + ["preview:thumb-press", "down", 5, 5, 4], + ["preview:thumb-press", "up", 5, 5, 4], + ]); + }), + ), + ); + effectIt.effect("keeps editing and forwarded shortcuts in whichever tab holds focus", () => withManager((manager) => Effect.gen(function* () { @@ -819,7 +1020,7 @@ describe("PreviewManager", () => { ["linux", false], ["win32", false], ] as const)( - "lets the page and its iframes claim the find shortcut before the host opens find on %s", + "lets the page and its iframes claim the find and zoom shortcuts before the host acts on %s", ([platform, usesMeta]) => withManager( (manager) => @@ -837,8 +1038,23 @@ describe("PreviewManager", () => { yield* manager.registerWebview("tab_page_find", 42); yield* manager.setForwardedShortcuts([ { command: "preview.find", shortcut: parseKeybindingShortcut("mod+f")! }, + { command: "preview.zoomIn", shortcut: parseKeybindingShortcut("mod+=")! }, ]); const beforeInput = preview.listeners.get("before-input-event")!; + const keepsZoom = vi.fn(); + beforeInput( + { preventDefault: keepsZoom } as never, + { + type: "keyDown", + key: "=", + code: "Equal", + meta: usesMeta, + control: !usesMeta, + shift: false, + alt: false, + } as never, + ); + expect(keepsZoom).not.toHaveBeenCalled(); const findKey = { type: "keyDown", key: "f", @@ -1676,6 +1892,30 @@ describe("PreviewManager", () => { ), ); + effectIt.effect("captures final pixels after warmup fails and paint readiness settles", () => + withManager((manager) => + Effect.gen(function* () { + const paint = Promise.withResolvers(); + const { capturePage, wc, capture } = yield* setupStill(manager, paint.promise); + capturePage + .mockRejectedValueOnce(new Error("UnknownVizError")) + .mockRejectedValueOnce(new Error("UnknownVizError")) + .mockRejectedValueOnce(new Error("UnknownVizError")) + .mockResolvedValue(makeStillImage("final-png")); + const fiber = yield* capture.pipe(Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust(300); + expect(capturePage).toHaveBeenCalledTimes(3); + expect(writeFile).not.toHaveBeenCalled(); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false]]); + paint.resolve(); + const artifact = yield* Fiber.join(fiber); + expect(capturePage).toHaveBeenCalledTimes(4); + expect(writeFile.mock.calls).toEqual([[artifact.path, Buffer.from("final-png")]]); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false], [true]]); + }), + ), + ); + effectIt.effect("preserves both screenshots of the same site in the same millisecond", () => withManager((manager) => Effect.gen(function* () { @@ -1689,6 +1929,8 @@ describe("PreviewManager", () => { const capturePage = vi .fn<() => Promise>>() .mockResolvedValueOnce(pngImage(firstPng)) + .mockResolvedValueOnce(pngImage(firstPng)) + .mockResolvedValueOnce(pngImage(secondPng)) .mockResolvedValueOnce(pngImage(secondPng)); fromId.mockReturnValue(makeTestPreviewWebContents(capturePage)); yield* manager.createTab("tab_1"); @@ -1709,6 +1951,115 @@ describe("PreviewManager", () => { ), ); + effectIt.effect( + "waits for paint before capturing and restores rendering after overlapping requests", + () => + withManager((manager) => + Effect.gen(function* () { + const paints = [Promise.withResolvers(), Promise.withResolvers()]; + const pendingPaints = paints.map((paint) => paint.promise); + const { image, capturePage, wc, capture } = yield* setupStill(manager); + holdStillImagePaint(wc, pendingPaints); + const first = Promise.withResolvers(); + const second = Promise.withResolvers(); + capturePage + .mockResolvedValueOnce(makeStillImage("discarded-warmup")) + .mockResolvedValueOnce(makeStillImage("discarded-warmup")) + .mockReturnValueOnce(first.promise) + .mockReturnValueOnce(second.promise); + const hostThrottling = vi.fn(); + yield* manager.setMainWindow({ + isDestroyed: () => false, + once: vi.fn(), + webContents: { setBackgroundThrottling: hostThrottling }, + } as never); + const one = yield* capture.pipe(Effect.forkChild({ startImmediately: true })); + const two = yield* capture.pipe(Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust(0); + expect(capturePage).toHaveBeenCalledTimes(2); + expect(pendingPaints).toHaveLength(0); + expect(writeFile).not.toHaveBeenCalled(); + expect(hostThrottling.mock.calls).toEqual([[false]]); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false]]); + paints.forEach((paint) => paint.resolve()); + yield* TestClock.adjust(0); + expect(capturePage).toHaveBeenCalledTimes(4); + first.resolve(image); + yield* Fiber.join(one); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false]]); + second.resolve(image); + yield* Fiber.join(two); + const saved = writeFile.mock.calls.map(([, bytes]) => bytes); + expect(saved).toEqual([image.toPNG(), image.toPNG()]); + expect(hostThrottling.mock.calls).toEqual([[false], [true]]); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false], [true]]); + }), + ), + ); + + effectIt.effect("keeps picture-in-picture rendering after a still image completes", () => + withManager((manager) => + Effect.gen(function* () { + const { image, capturePage, wc, capture } = yield* setupStill(manager); + const pending = Promise.withResolvers(); + capturePage + .mockResolvedValueOnce(image) + .mockResolvedValueOnce(makeStillImage("discarded-warmup")) + .mockReturnValueOnce(pending.promise); + const { pictureInPictureWindow } = makeTestPictureInPictureWindow(); + browserWindowConstructor.mockImplementation(function () { + return pictureInPictureWindow; + }); + yield* manager.openPictureInPicture("tab_still"); + const still = yield* capture.pipe(Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust(0); + expect(capturePage).toHaveBeenCalledTimes(3); + expect(writeFile).not.toHaveBeenCalled(); + pending.resolve(image); + yield* Fiber.join(still); + expect(writeFile.mock.calls.map((call) => call[1])).toEqual([Buffer.from("fresh-png")]); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false]]); + yield* manager.closePictureInPicture("tab_still"); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false], [true]]); + }), + ), + ); + + effectIt.effect("keeps a pending still image rendering when recording ends", () => + withManager((manager) => + Effect.gen(function* () { + const { image, capturePage, wc, capture } = yield* setupStill(manager); + const pending = Promise.withResolvers(); + capturePage.mockResolvedValueOnce(image).mockReturnValue(pending.promise); + yield* manager.startRecording("tab_still"); + const fiber = yield* capture.pipe(Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust(0); + yield* manager.stopRecording("tab_still"); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false]]); + pending.resolve(image); + yield* Fiber.join(fiber); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false], [true]]); + }), + ), + ); + + effectIt.effect("saves available pixels when paint readiness never settles", () => + withManager((manager) => + Effect.gen(function* () { + const paint = Promise.withResolvers(); + const { capturePage, wc, capture } = yield* setupStill(manager, paint.promise); + const first = yield* Effect.exit(capture).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* TestClock.adjust(1_001); + expect(Exit.isSuccess(yield* Fiber.join(first))).toBe(true); + expect(capturePage).toHaveBeenCalledTimes(2); + expect(writeFile.mock.calls.map((call) => call[1])).toEqual([Buffer.from("fresh-png")]); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false], [true]]); + }), + ), + ); + effectIt.effect("keeps every recorded guest unthrottled until its frame capture stops", () => withManager((manager) => Effect.gen(function* () { @@ -2055,7 +2406,7 @@ describe("PreviewManager", () => { expect(yield* nextType).toBe("attached"); yield* manager.closeTab("tab_1"); expect(yield* nextType).toBe("detached"); - }).pipe(Effect.provide(managerLayer()), Effect.scoped), + }).pipe(Effect.provide(managerLayer())), ); effectIt.effect( @@ -2112,8 +2463,74 @@ describe("PreviewManager", () => { }), ); expect(yield* Queue.take(capturing)).toBe(true); + yield* manager.setCaptureDrawn("tab_1", true); screenshot.resolve({ data: "" }); expect(yield* Queue.take(capturing)).toBe(false); + }).pipe(Effect.provide(managerLayer())), + ); + + effectIt.effect( + "opens a new-tab link in the background only when the agent's input led to it", + () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.DesktopBrowserHost; + const serverTab = { threadId: "thread-1", tabId: "server-tab-1" }; + const contents = Object.assign( + makeTestPreviewWebContents(async () => { + throw new Error("unused"); + }, 42), + { + isDevToolsOpened: () => false, + getUserAgent: () => "Electron", + focusedFrame: null, + }, + ); + fromId.mockReturnValue(contents); + const manager = yield* PreviewManager.PreviewManager; + const openLinks = yield* Queue.unbounded(); + yield* manager.subscribeOpenLinks(({ background }) => + Queue.offer(openLinks, background).pipe(Effect.asVoid), + ); + const attached = yield* Deferred.make(); + yield* host.events.pipe( + Stream.runForEach((line) => + new TextDecoder().decode(line).includes('"type":"attached"') + ? Deferred.succeed(attached, undefined) + : Effect.void, + ), + Effect.forkScoped, + ); + yield* manager.createTab("tab_1", { serverTab }); + yield* manager.registerWebview("tab_1", 42); + yield* Deferred.await(attached); + const openWindow = vi.mocked(contents.setWindowOpenHandler).mock.calls[0]![0]; + const openBlankLink = Effect.suspend(() => { + openWindow({ + url: "https://example.com/next", + disposition: "foreground-tab", + } as Electron.HandlerDetails); + return Queue.take(openLinks); + }); + + expect(yield* openBlankLink).toBe(false); + + yield* host.handleCommandLine( + JSON.stringify({ + type: "cdp", + ...serverTab, + message: JSON.stringify({ + id: 1, + method: "Runtime.evaluate", + params: { expression: "document.querySelector('a').click()" }, + sessionId: "t3-preview-page", + }), + }), + ); + expect(yield* openBlankLink).toBe(true); + + // A person clicking in the guest itself gives it a focused frame. + Object.assign(contents, { focusedFrame: {} }); + expect(yield* openBlankLink).toBe(false); }).pipe(Effect.provide(managerLayer()), Effect.scoped), ); @@ -2205,7 +2622,7 @@ describe("PreviewManager", () => { expect(JSON.parse(yield* Queue.take(lines))).toEqual({ type: "detached", ...serverTab }); expect(JSON.parse(yield* Queue.take(lines))).toEqual({ type: "attached", ...serverTab }); expect(statuses.at(-1)?.kind).toBe("Success"); - }).pipe(Effect.provide(managerLayer()), Effect.scoped), + }).pipe(Effect.provide(managerLayer())), ); effectIt.effect("discards a screenshot that resolves after its guest is replaced", () => @@ -2367,6 +2784,116 @@ describe("PreviewManager", () => { ), ); + effectIt.effect( + "publishes human ownership only on transitions while retaining every recording press", + () => + withManager((manager) => + Effect.gen(function* () { + let humanInput: ((input: unknown) => void) | undefined; + const contents = makeTestPreviewWebContents(async () => ({ + toJPEG: () => Buffer.from("frame"), + getSize: () => ({ width: 800, height: 600 }), + })); + contents.ipc.on = (channel, callback) => { + if (channel === "preview:human-input") + humanInput = (input) => callback({} as Electron.IpcMainEvent, input); + return contents.ipc; + }; + fromId.mockReturnValue(contents); + yield* manager.createTab("tab_human"); + yield* manager.registerWebview("tab_human", 42); + yield* manager.startRecording("tab_human", { + showKeyPresses: true, + showMousePresses: true, + }); + const controllers: string[] = []; + const feedback = yield* Queue.unbounded(); + webviewSend.mockImplementation((channel) => { + if (channel === "preview:recording-controller") Queue.offerUnsafe(feedback, undefined); + }); + yield* manager.subscribeStateChanges((_tabId, state) => + Effect.sync(() => { + controllers.push(state.controller); + }), + ); + for (const press of [ + { kind: "pointer", x: 10, y: 20, button: 0 }, + { kind: "pointer", x: 30, y: 40, button: 0 }, + { kind: "key", key: "a", code: "KeyA" }, + ]) { + humanInput?.(press); + yield* Queue.take(feedback); + } + expect(controllers).toEqual(["human"]); + expect( + webviewSend.mock.calls.filter( + ([channel]) => channel === "preview:recording-controller", + ), + ).toEqual([ + ["preview:recording-controller", "human", { x: 10, y: 20 }], + ["preview:recording-controller", "human", { x: 30, y: 40 }], + ["preview:recording-controller", "human", undefined], + ]); + yield* TestClock.adjust(750); + expect(controllers).toEqual(["human", "none"]); + }), + ), + ); + + effectIt.effect("keeps human ownership until the last input's idle deadline", () => + withManager((manager) => + Effect.gen(function* () { + let humanInput: ((input: unknown) => void) | undefined; + const contents = makeTestPreviewWebContents(async () => ({ + toJPEG: () => Buffer.from("frame"), + getSize: () => ({ width: 800, height: 600 }), + })); + contents.ipc.on = (channel, callback) => { + if (channel === "preview:human-input") + humanInput = (input) => callback({} as Electron.IpcMainEvent, input); + return contents.ipc; + }; + fromId.mockReturnValue(contents); + yield* manager.createTab("tab_idle"); + yield* manager.registerWebview("tab_idle", 42); + const controllers: string[] = []; + yield* manager.subscribeStateChanges((_tabId, state) => + Effect.sync(() => { + controllers.push(state.controller); + }), + ); + const press = () => humanInput?.({ kind: "key", key: "a", code: "KeyA" }); + press(); + yield* TestClock.adjust(500); + press(); + yield* TestClock.adjust(749); + expect(controllers).toEqual(["human"]); + yield* TestClock.adjust(1); + expect(controllers).toEqual(["human", "none"]); + + press(); + yield* TestClock.adjust(500); + const staleInput = humanInput; + const replacement = makeTestPreviewWebContents( + async () => ({ + toJPEG: () => Buffer.from("frame"), + getSize: () => ({ width: 800, height: 600 }), + }), + 43, + ); + replacement.ipc.on = contents.ipc.on; + contents.isDestroyed = () => true; + fromId.mockImplementation((id) => (id === 43 ? replacement : null)); + yield* manager.registerWebview("tab_idle", 43); + expect(controllers.at(-1)).toBe("none"); + const count = controllers.length; + staleInput?.({ kind: "key", key: "b", code: "KeyB" }); + yield* TestClock.adjust(750); + expect(controllers).toHaveLength(count); + }), + ), + ); + effectIt.effect("gates recording decorations and isolates failed subscribers", () => withManager((manager) => Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 41288e308944..90104ebe9ef6 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -82,13 +82,13 @@ import { CANCEL_PICK_CHANNEL, ELEMENT_PICKED_CHANNEL, HUMAN_INPUT_CHANNEL, - MOUSE_NAVIGATE_CHANNEL, PAGE_UNHANDLED_SHORTCUT_CHANNEL, RECORDING_CURSOR_CHANNEL, RECORDING_INPUT_CHANNEL, RECORDING_CONTROLLER_CHANNEL, RECORDING_POINTER_CHANNEL, START_PICK_CHANNEL, + THUMB_PRESS_CHANNEL, } from "./GuestProtocol.ts"; import { isPreviewAnnotationDraft, @@ -141,6 +141,7 @@ const PICTURE_IN_PICTURE_JPEG_QUALITY = 80; const CAPTURE_PAGE_RETRY_ATTEMPTS = 3; const CAPTURE_PAGE_RETRY_DELAY_MS = 120; const CAPTURE_PAGE_ATTEMPT_TIMEOUT_MS = 1_000; +const BUSY_PAGE_PROBE_MS = 1_000; const PICTURE_IN_PICTURE_INITIAL_WIDTH = 480; const PICTURE_IN_PICTURE_INITIAL_HEIGHT = 320; const PICTURE_IN_PICTURE_MIN_WIDTH = 240; @@ -399,7 +400,8 @@ interface ManagedListeners { readonly webContents: Electron.WebContents; } -type FrameCaptureConsumer = "picture-in-picture" | "recording"; +// Each still-image request owns a separate, short-lived rendering lease. +type FrameCaptureConsumer = "picture-in-picture" | "recording" | symbol; interface FrameCaptureSession { readonly recordingInputOptions?: RecordingInputOptions; @@ -457,6 +459,9 @@ type FindResultListener = (event: DesktopPreviewFindResultEvent) => Effect.Effec const PAGE_FIRST_FORWARDED_COMMANDS: ReadonlySet = new Set([ "preview.find", + "preview.zoomIn", + "preview.zoomOut", + "preview.resetZoom", ]); const decodePageShortcutKeys = Schema.decodeUnknownOption( @@ -576,9 +581,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const random = yield* SecureRandom.SecureRandom; const parentScope = yield* Scope.Scope; const browserHost = yield* DesktopBrowserHost.DesktopBrowserHost; + const browserSession = yield* BrowserSession.BrowserSession; const passkeys = yield* PreviewPasskeys.PreviewPasskeys; const context = yield* Effect.context(); const runFork = Effect.runForkWith(context); + const runPromise = Effect.runPromiseWith(context); const resolvedArtifactDirectory = path.resolve(artifactDirectory); let forwardedShortcuts: ReadonlyArray = []; @@ -895,7 +902,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const update = Effect.fn("PreviewManager.update")(function* ( tabId: string, patch: Partial, - humanPoint?: { readonly x: number; readonly y: number }, ) { const updatedAt = yield* currentIso; const next = yield* SynchronizedRef.modify(tabsRef, (tabs) => { @@ -911,17 +917,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }); // syncTabAudible can commit after this update; emitIfCurrent prevents publishing a stale snapshot. if (Option.isSome(next)) { - if (patch.controller !== undefined && next.value.webContentsId != null) { - const capture = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId); - const webContentsId = next.value.webContentsId; - if (capture?.consumers.has("recording")) { - yield* attempt({ operation: "recording.controller", tabId }, () => { - const contents = webContents.fromId(webContentsId); - if (contents && !contents.isDestroyed()) - contents.send(RECORDING_CONTROLLER_CHANNEL, patch.controller, humanPoint); - }).pipe(Effect.ignore); - } - } yield* emitIfCurrent(tabId, next.value); } }); @@ -1114,28 +1109,13 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ), scope, ); - const handleDebuggerMessage = Effect.fnUntraced(function* ( - method: string, - params: Record, - ) { - if (method === "Page.screencastFrame") { - const sessionId = params["sessionId"]; - if (typeof sessionId === "number") { - yield* attemptPromise( - { - operation: "ackScreencastFrame", - webContentsId: wc.id, - }, - () => wcDebugger.sendCommand("Page.screencastFrameAck", { sessionId }), - ).pipe(Effect.ignore); - } - } + const handleDebuggerMessage = Effect.fnUntraced(function* (method: string) { if (method === "Runtime.consoleAPICalled" || method === "Runtime.exceptionThrown") { yield* Queue.offer(consoleReleases, undefined); } }); - const onMessage: BrowserControlSession["onMessage"] = (_event, method, params) => { - runFork(handleDebuggerMessage(method, params)); + const onMessage: BrowserControlSession["onMessage"] = (_event, method) => { + runFork(handleDebuggerMessage(method)); }; const control: BrowserControlSession = { webContentsId: wc.id, @@ -1450,21 +1430,75 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }), ); }; + let humanInputWorker = false; + let humanInputDeadline = 0; + let listening = true; + const publishHumanController = Effect.fn("PreviewManager.publishHumanController")(function* ( + controller: "human" | "none", + humanPoint?: { readonly x: number; readonly y: number }, + ) { + const now = yield* currentMillis; + const updatedAt = yield* currentIso; + const result = yield* SynchronizedRef.modify(tabsRef, (tabs) => { + const current = tabs.get(tabId); + if ( + !current || + current.webContentsId !== wc.id || + (controller === "human" + ? !listening || webContents.fromId(wc.id) !== wc + : current.controller !== "human" || humanInputDeadline > now) + ) + return [undefined, tabs] as const; + const changed = current.controller !== controller; + const state = changed ? { ...current, controller, updatedAt } : current; + return [ + { state, changed }, + changed + ? replaceMap(tabs, (copy) => { + copy.set(tabId, state); + }) + : tabs, + ] as const; + }); + if (!result) return; + const capture = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId); + if (capture?.consumers.has("recording")) { + yield* attempt({ operation: "recording.controller", tabId }, () => { + if (!wc.isDestroyed()) wc.send(RECORDING_CONTROLLER_CHANNEL, controller, humanPoint); + }).pipe(Effect.ignore); + } + if (result.changed) yield* emitIfCurrent(tabId, result.state); + }); + yield* Scope.addFinalizer( + scope, + Effect.gen(function* () { + listening = false; + humanInputDeadline = 0; + yield* publishHumanController("none"); + }), + ); const handleHumanInput = Effect.fn("PreviewManager.handleHumanInput")(function* ( rawSignal?: unknown, ) { - yield* update( - tabId, - { controller: "human" }, + if (!listening) return; + humanInputDeadline = (yield* currentMillis) + 750; + if (!humanInputWorker) { + humanInputWorker = true; + yield* Effect.gen(function* () { + let remaining: number; + while ((remaining = humanInputDeadline - (yield* currentMillis)) > 0) { + yield* Effect.sleep(remaining); + } + humanInputWorker = false; + yield* publishHumanController("none"); + }).pipe(Effect.forkIn(scope)); + } + yield* publishHumanController( + "human", isPreviewInputSignal(rawSignal) && rawSignal.kind === "pointer" ? { x: rawSignal.x, y: rawSignal.y } : undefined, ); - yield* Effect.sleep(750); - const tabs = yield* SynchronizedRef.get(tabsRef); - if (tabs.get(tabId)?.controller === "human") { - yield* update(tabId, { controller: "none" }); - } }); const recordingInput = (_event: unknown, input: unknown) => { if (!isRecordingInput(input)) return; @@ -1498,15 +1532,27 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const humanInput = (_event: unknown, rawSignal?: unknown): void => { runFork(handleHumanInput(rawSignal)); }; - const mouseNavigate = (_event: unknown, payload?: unknown): void => { - const direction = - typeof payload === "object" && payload !== null && "direction" in payload - ? (payload as { direction?: unknown }).direction - : undefined; - if (direction !== "back" && direction !== "forward") return; + // Chromium navigates the outermost WebContents (the host app) on thumb buttons, even from guest iframes. + const beforeMouse = (event: Electron.Event, mouse: Electron.MouseInputEvent): void => { + // Electron reports thumb buttons as "back" and "forward", which its typings omit. + const button: string | undefined = mouse.button; + if (button !== "back" && button !== "forward") return; + if (mouse.type !== "mouseDown" && mouse.type !== "mouseUp") return; + event.preventDefault(); + // The page never sees the press, so the guest replays it; these coordinates are DIPs, not CSS px. + const zoom = wc.getZoomFactor(); + const phase = mouse.type === "mouseDown" ? "down" : "up"; + wc.send( + THUMB_PRESS_CHANNEL, + phase, + mouse.x / zoom, + mouse.y / zoom, + button === "back" ? 3 : 4, + ); + if (phase !== "up") return; runFork( attempt({ operation: "mouseNavigate", tabId, webContentsId: wc.id }, () => { - if (direction === "back") { + if (button === "back") { if (wc.navigationHistory.canGoBack()) wc.navigationHistory.goBack(); } else if (wc.navigationHistory.canGoForward()) { wc.navigationHistory.goForward(); @@ -1564,7 +1610,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }); if (forwarded && host) { // The page keeps these keys. Only the main-frame preload can report one it - // left unhandled, so a focused iframe (e.g. Google Docs' input) keeps its own find. + // left unhandled, so a focused iframe (e.g. Google Docs' input) keeps its own find or zoom. if (PAGE_FIRST_FORWARDED_COMMANDS.has(forwarded.command)) return; event.preventDefault(); if (!input.isAutoRepeat) { @@ -1616,9 +1662,9 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function wc.off("found-in-page", foundInPage as never); wc.off("did-create-window", windowCreated); wc.off("before-input-event", beforeInput); + wc.off("before-mouse-event", beforeMouse); wc.ipc.off(HUMAN_INPUT_CHANNEL, humanInput); wc.ipc.off(RECORDING_INPUT_CHANNEL, recordingInput); - wc.ipc.off(MOUSE_NAVIGATE_CHANNEL, mouseNavigate); wc.ipc.off(PAGE_UNHANDLED_SHORTCUT_CHANNEL, pageUnhandledShortcut); detachPasskeys(); }).pipe(Effect.ignore), @@ -1639,7 +1685,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function wc.on("found-in-page", foundInPage as never); wc.ipc.on(HUMAN_INPUT_CHANNEL, humanInput); wc.ipc.on(RECORDING_INPUT_CHANNEL, recordingInput); - wc.ipc.on(MOUSE_NAVIGATE_CHANNEL, mouseNavigate); wc.ipc.on(PAGE_UNHANDLED_SHORTCUT_CHANNEL, pageUnhandledShortcut); detachPasskeys = passkeys.attachGuest(wc); wc.setWindowOpenHandler((details) => { @@ -1653,7 +1698,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function emitOpenLink({ tabId, url: details.url, - background: details.disposition === "background-tab", + // A person's own click or key leaves the guest a focused frame; GuestInput hands + // focus back after the agent's press, so a foreground tab would take it again. + background: + details.disposition === "background-tab" || + (wc.focusedFrame === null && browserHost.agentLedPage(wc)), }), ); return { action: "deny" }; @@ -1667,6 +1716,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }); wc.on("did-create-window", windowCreated); wc.on("before-input-event", beforeInput); + wc.on("before-mouse-event", beforeMouse); }); yield* Ref.update(attachedRef, (attached) => replaceMap(attached, (copy) => { @@ -1790,7 +1840,10 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }); if (Option.isNone(tab)) return; const closedTab = tab.value; - if (closedTab.serverTab) browserHost.setShown(closedTab.serverTab, undefined); + if (closedTab.serverTab) { + browserHost.setShown(closedTab.serverTab, undefined); + browserHost.setCaptureDrawn(closedTab.serverTab, undefined); + } if (closedTab.webContentsId != null) { yield* Effect.all( [ @@ -2030,6 +2083,28 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ); }); + /** + * A same-site navigation reuses the renderer, so it never commits while a page script holds it. + * Runs before the navigation starts: Chromium does not deliver a terminate once one is pending. + */ + const leaveBusyPage = Effect.fn("PreviewManager.leaveBusyPage")(function* ( + wc: Electron.WebContents, + ) { + const control = (yield* SynchronizedRef.get(controlSessionsRef)).get(wc.id); + if (!control) return; + const answered = yield* Effect.promise(() => + control.debugger.sendCommand("Runtime.evaluate", { expression: "0" }).then( + () => undefined, + () => undefined, + ), + ).pipe(Effect.timeoutOption(BUSY_PAGE_PROBE_MS)); + if (Option.isSome(answered)) return; + // Not awaited: a renderer blocked on something other than script may never acknowledge it. + yield* Effect.sync(() => { + void control.debugger.sendCommand("Runtime.terminateExecution").catch(() => undefined); + }); + }); + const navigate = Effect.fn("PreviewManager.navigate")(function* (tabId: string, rawUrl: string) { const url = yield* attempt({ operation: "navigate.normalizeUrl", tabId }, () => normalizePreviewUrl(rawUrl), @@ -2112,6 +2187,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ); return; } + yield* leaveBusyPage(wc); if (wc.getURL() === url) { yield* attempt({ operation: "navigate.reload", tabId, webContentsId: wc.id }, () => wc.reload(), @@ -2147,17 +2223,28 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function yield* attempt({ operation, tabId, webContentsId: wc.id }, () => use(wc)); }); + const leavePage = Effect.fn("PreviewManager.leavePage")(function* ( + operation: string, + tabId: string, + use: (wc: Electron.WebContents) => void, + ) { + const wc = yield* requireWebContents(tabId); + yield* leaveBusyPage(wc); + yield* attempt({ operation, tabId, webContentsId: wc.id }, () => use(wc)); + }); + const goBack = (tabId: string) => - withWebContents("goBack", tabId, (wc) => { + leavePage("goBack", tabId, (wc) => { if (wc.navigationHistory.canGoBack()) wc.navigationHistory.goBack(); }); const goForward = (tabId: string) => - withWebContents("goForward", tabId, (wc) => { + leavePage("goForward", tabId, (wc) => { if (wc.navigationHistory.canGoForward()) wc.navigationHistory.goForward(); }); - const refresh = (tabId: string) => withWebContents("refresh", tabId, (wc) => wc.reload()); + const refresh = (tabId: string) => leavePage("refresh", tabId, (wc) => wc.reload()); + const stopLoading = (tabId: string) => withWebContents("stopLoading", tabId, (wc) => wc.stop()); const hardReload = (tabId: string) => - withWebContents("hardReload", tabId, (wc) => wc.reloadIgnoringCache()); + leavePage("hardReload", tabId, (wc) => wc.reloadIgnoringCache()); const findInPage = ( tabId: string, text: string, @@ -2532,7 +2619,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function return; } if (afterAttach.serverTab) { - browserHost.attach(afterAttach.serverTab, { webContents: wc, debugger: control.debugger }); + browserHost.attach(afterAttach.serverTab, { + webContents: wc, + debugger: control.debugger, + cookiesCleared: () => runPromise(browserSession.cookiesCleared(wc.session)), + }); // CDP cannot recover a failure that happened before it attached. const { navStatus } = afterAttach; if (navStatus.kind === "LoadFailed") { @@ -2607,12 +2698,21 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function if (serverTab) browserHost.setShown(serverTab, shown); }); + const setCaptureDrawn = Effect.fn("PreviewManager.setCaptureDrawn")(function* ( + tabId: string, + drawn: boolean, + ) { + const serverTab = (yield* SynchronizedRef.get(tabsRef)).get(tabId)?.serverTab; + if (serverTab) browserHost.setCaptureDrawn(serverTab, drawn); + }); + // A window reload or guest crash destroys the guest without the renderer reporting it hidden. // The renderer reports again once a new guest registers. const hideDestroyedGuest = Effect.fnUntraced(function* (webContentsId: number) { for (const tab of (yield* SynchronizedRef.get(tabsRef)).values()) { if (tab.webContentsId === webContentsId && tab.serverTab) { browserHost.setShown(tab.serverTab, false); + browserHost.setCaptureDrawn(tab.serverTab, false); } } }); @@ -2651,7 +2751,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const [createdAt, millis, image] = yield* Effect.all([ currentIso, currentMillis, - capturePageWithRetry( + captureStillImage( { operation: "captureScreenshot.capturePage", tabId, @@ -2699,6 +2799,100 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }; }); + const captureStillImage = Effect.fn("PreviewManager.captureStillImage")(function* ( + errorContext: PreviewOperationContext, + tabId: string, + wc: Electron.WebContents, + ) { + const consumer = Symbol("still-image"); + return yield* Effect.acquireUseRelease( + startFrameCapture(tabId, consumer), + () => + Effect.gen(function* () { + // A DOM mutation can precede its compositor frame even in a shown + // window. Keep the original guest rendering through the next paint; + // a hidden guest otherwise returns old pixels or never completes. + const probe = Effect.fn("PreviewManager.probeStillImagePaint")(function* ( + send: ( + method: string, + params?: Record, + ) => Effect.Effect, + ) { + // CDP executes against the current document without Electron's + // did-stop-loading gate. Isolate scheduling globals from page code. + const tree = (yield* send("Page.getFrameTree")) as + | { frameTree?: { frame?: { id?: string } } } + | undefined; + const frameId = tree?.frameTree?.frame?.id; + if (typeof frameId !== "string") + return yield* new PreviewOperationError({ + ...errorContext, + cause: new Error("The preview paint frame is unavailable."), + }); + const world = (yield* send("Page.createIsolatedWorld", { + frameId, + worldName: "t3-preview-still-image", + })) as { executionContextId?: number } | undefined; + if (typeof world?.executionContextId !== "number") + return yield* new PreviewOperationError({ + ...errorContext, + cause: new Error("The preview paint context is unavailable."), + }); + const response = (yield* send("Runtime.evaluate", { + // Settle and release scheduled work when only timers keep running. + // The outer deadline still bounds a frozen renderer. + expression: `new Promise(resolve => { let frame; const timer = setTimeout(() => { cancelAnimationFrame(frame); resolve(false); }, ${CAPTURE_PAGE_ATTEMPT_TIMEOUT_MS}); frame = requestAnimationFrame(() => { frame = requestAnimationFrame(() => { clearTimeout(timer); resolve(true); }); }); })`, + contextId: world.executionContextId, + awaitPromise: true, + returnByValue: true, + })) as { result?: { value?: unknown }; exceptionDetails?: unknown } | undefined; + if (response?.exceptionDetails || response?.result?.value !== true) + return yield* new PreviewOperationError({ + ...errorContext, + cause: new Error("The preview paint boundary was not confirmed."), + }); + }); + const paintReady = Effect.gen(function* () { + // Only use an existing owned session; never attach or displace + // another debugger to obtain a screenshot. + const control = (yield* SynchronizedRef.get(controlSessionsRef)).get(wc.id); + if (!control) + return yield* new PreviewOperationError({ + ...errorContext, + cause: new Error("The preview paint session is unavailable."), + }); + return yield* probe((method, params) => + attemptPromise(errorContext, () => control.debugger.sendCommand(method, params)), + ); + }).pipe( + Effect.timeout(CAPTURE_PAGE_ATTEMPT_TIMEOUT_MS), + Effect.catch((error) => + Effect.logWarning( + "Preview paint readiness was not confirmed; capturing available pixels with mutation freshness unconfirmed.", + { tabId, webContentsId: wc.id, error }, + ), + ), + ); + // Minimized guests can suspend frames even with throttling disabled. + // A native capture can resume them; discard its pixels and read again + // after the bounded paint probe. + const warmup = capturePageWithRetry(errorContext, tabId, wc).pipe( + Effect.catchTags({ + PreviewOperationError: (error) => + Effect.logWarning("Preview warmup capture failed; continuing with final capture.", { + tabId, + webContentsId: wc.id, + error, + }), + }), + ); + yield* Effect.all([paintReady, warmup], { concurrency: 2, discard: true }); + return yield* capturePageWithRetry(errorContext, tabId, wc); + }), + () => stopFrameCapture(tabId, consumer), + ); + }); + const capturePreviewFrame = Effect.fn("PreviewManager.capturePreviewFrame")(function* ( tabId: string, ) { @@ -3381,7 +3575,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function data: Uint8Array, ) { const [createdAt, millis] = yield* Effect.all([currentIso, currentMillis]); - const id = `browser-recording-${millis.toString(36)}`; + const uuid = yield* random.uuidV4(); + const id = `browser-recording-${millis.toString(36)}-${uuid.slice(0, 8)}`; const extension = recordingFileExtension(mimeType); const artifactPath = path.join(resolvedArtifactDirectory, `${id}.${extension}`); yield* fileSystem.makeDirectory(resolvedArtifactDirectory, { recursive: true }).pipe( @@ -3395,7 +3590,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }), ), ); - yield* fileSystem.writeFile(artifactPath, data).pipe( + yield* fileSystem.writeFile(artifactPath, data, { flag: "wx" }).pipe( Effect.mapError( (cause) => new PreviewOperationError({ @@ -3582,6 +3777,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function prepareWebview, reapplyZoom, refresh, + stopLoading, registerWebview, resetZoom: (tabId: string) => applyZoom(tabId, () => DEFAULT_ZOOM_FACTOR), setZoomFactor: (tabId: string, zoomFactor: number) => @@ -3592,6 +3788,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function setAnnotationSendEnabled, setAudioMuted, setShown, + setCaptureDrawn, setColorScheme, setMainWindow, setForwardedShortcuts: (shortcuts: ReadonlyArray) => @@ -3783,6 +3980,7 @@ export class PreviewManager extends Context.Service< readonly goBack: (tabId: string) => Effect.Effect; readonly goForward: (tabId: string) => Effect.Effect; readonly refresh: (tabId: string) => Effect.Effect; + readonly stopLoading: (tabId: string) => Effect.Effect; readonly zoomIn: (tabId: string) => Effect.Effect; readonly zoomOut: (tabId: string) => Effect.Effect; readonly resetZoom: (tabId: string) => Effect.Effect; @@ -3807,6 +4005,7 @@ export class PreviewManager extends Context.Service< audioMuted: boolean, ) => Effect.Effect; readonly setShown: (tabId: string, shown: boolean) => Effect.Effect; + readonly setCaptureDrawn: (tabId: string, drawn: boolean) => Effect.Effect; readonly openDevTools: (tabId: string) => Effect.Effect; readonly clearCookies: ( partitions?: ReadonlyArray, @@ -3957,6 +4156,7 @@ export const make = Effect.gen(function* PreviewManagerMake() { goBack: operations.goBack, goForward: operations.goForward, refresh: operations.refresh, + stopLoading: operations.stopLoading, zoomIn: operations.zoomIn, zoomOut: operations.zoomOut, resetZoom: operations.resetZoom, @@ -3968,6 +4168,7 @@ export const make = Effect.gen(function* PreviewManagerMake() { setZoomFactor: operations.setZoomFactor, setAudioMuted: operations.setAudioMuted, setShown: operations.setShown, + setCaptureDrawn: operations.setCaptureDrawn, openDevTools: operations.openDevTools, clearCookies: Effect.fn("PreviewManager.clearCookies")(function* (partitions) { yield* browserSession diff --git a/apps/desktop/src/preview/PickPreload.test.ts b/apps/desktop/src/preview/PickPreload.test.ts index 8a080183b383..61d3941ba41a 100644 --- a/apps/desktop/src/preview/PickPreload.test.ts +++ b/apps/desktop/src/preview/PickPreload.test.ts @@ -62,3 +62,46 @@ it("keeps the tool while typing a tool key in the comment", () => { expect(document.documentElement.getAttribute("data-t3code-annotation-tool")).toBe("select"); }); + +it("keeps comment keystrokes away from the page's capture-phase shortcuts", () => { + const pageShortcut = vi.fn((event: KeyboardEvent) => event.preventDefault()); + window.addEventListener("keydown", pageShortcut, { capture: true }); + const shadowRoot = startAnnotation(); + const comment = shadowRoot.querySelector("textarea"); + if (!comment) throw new Error("comment textarea not found"); + + comment.focus(); + const space = new KeyboardEvent("keydown", { + key: " ", + code: "Space", + bubbles: true, + cancelable: true, + composed: true, + }); + comment.dispatchEvent(space); + window.removeEventListener("keydown", pageShortcut, { capture: true }); + + expect(pageShortcut).not.toHaveBeenCalled(); + expect(space.defaultPrevented).toBe(false); +}); + +it("keeps the annotation when Escape dismisses an IME candidate in the comment", () => { + const shadowRoot = startAnnotation(); + const comment = shadowRoot.querySelector("textarea"); + if (!comment) throw new Error("comment textarea not found"); + + comment.value = "draft"; + comment.focus(); + comment.dispatchEvent( + new KeyboardEvent("keydown", { + key: "Escape", + isComposing: true, + bubbles: true, + cancelable: true, + composed: true, + }), + ); + + expect(shadowRoot.host.isConnected).toBe(true); + expect(comment.value).toBe("draft"); +}); diff --git a/apps/desktop/src/preview/PickPreload.ts b/apps/desktop/src/preview/PickPreload.ts index f831565a7def..42867b9d9cb4 100644 --- a/apps/desktop/src/preview/PickPreload.ts +++ b/apps/desktop/src/preview/PickPreload.ts @@ -20,17 +20,18 @@ import type { import { createAnnotationIdSource } from "./AnnotationIds.ts"; import { resolveAnnotationSubmission } from "./AnnotationKeyboard.ts"; -import { previewAnnotationStyles } from "./AnnotationStyles.generated.ts"; import { - isPreviewAnnotationDraft, - type PreviewAnnotationDraft, - type PreviewAnnotationTool, -} from "./PickedElementPayload.ts"; + addStrokePoint, + openStrokePath, + startStroke, + strokeBounds, + type StrokeGeometry, +} from "./AnnotationStroke.ts"; +import { previewAnnotationStyles } from "./AnnotationStyles.generated.ts"; import { installRecordingCursor } from "./RecordingCursor.ts"; import { DEFAULT_RECORDING_INPUT_OPTIONS } from "./RecordingInput.ts"; import { ANNOTATION_CAPTURED_CHANNEL, - ANNOTATION_DRAFT_CHANNEL, ANNOTATION_SEND_ENABLED_CHANNEL, ANNOTATION_THEME_CHANNEL, ANNOTATION_VOICE_EVENT_CHANNEL, @@ -38,7 +39,7 @@ import { CANCEL_PICK_CHANNEL, ELEMENT_PICKED_CHANNEL, HUMAN_INPUT_CHANNEL, - MOUSE_NAVIGATE_CHANNEL, + POINTER_DOWN_HOST_CHANNEL, PAGE_UNHANDLED_SHORTCUT_CHANNEL, RECORDING_CURSOR_CHANNEL, RECORDING_POINTER_CHANNEL, @@ -46,6 +47,7 @@ import { RECORDING_INPUT_CHANNEL, RECORDING_CONTROLLER_CHANNEL, START_PICK_CHANNEL, + THUMB_PRESS_CHANNEL, } from "./GuestProtocol.ts"; const OVERLAY_ATTRIBUTE = "data-t3code-annotation-ui"; const INPUT_LAYER_ATTRIBUTE = "data-t3code-annotation-input"; @@ -56,9 +58,6 @@ const MAX_MARQUEE_ELEMENTS = 20; const ELEMENT_CONTEXT_TIMEOUT_MS = 5_000; const CONTENT_LAYER_Z_INDEX = 1; const CHROME_LAYER_Z_INDEX = 10; -/** How long a restored draft keeps looking for elements the reloaded page renders late. */ -const DRAFT_RESTORE_TIMEOUT_MS = 5_000; -const DRAFT_RESTORE_INTERVAL_MS = 200; // Page dialogs (Radix, react-remove-scroll) dismiss, refocus or block scrolling on document events // that shadow retargeting attributes to the overlay host. const PAGE_ISOLATED_EVENTS = [ @@ -144,6 +143,8 @@ ipcRenderer.on(RECORDING_POINTER_CHANNEL, (_event, point: unknown) => { ); }); +type AnnotationTool = "select" | "marquee" | "draw" | "erase"; + interface SelectedElement { id: string; element: Element; @@ -155,8 +156,8 @@ interface SelectedElement { interface AnnotationSession { teardown: (notifyMain: boolean) => void; applyTheme: (theme: DesktopPreviewAnnotationTheme) => void; - syncDraft: () => void; setSendEnabled: (enabled: boolean) => void; + handleOverlayKey: (event: KeyboardEvent) => void; } let activeSession: AnnotationSession | null = null; @@ -193,6 +194,9 @@ const applyAnnotationTheme = ( const reportHumanPointerInput = (event: PointerEvent): void => { if (!event.isTrusted) return; + // The guest does not always take focus on a click, so the host's webview + // `focus` event can't be relied on to dismiss open menus and popovers. + ipcRenderer.sendToHost(POINTER_DOWN_HOST_CHANNEL); ipcRenderer.send(HUMAN_INPUT_CHANNEL, { kind: "pointer", x: event.clientX, @@ -233,34 +237,40 @@ const reportUnhandledShortcut = (event: KeyboardEvent): void => { window.addEventListener("keydown", reportUnhandledShortcut, true); -const MOUSE_BUTTON_BACK = 3; -const MOUSE_BUTTON_FORWARD = 4; - -const navigationDirectionForButton = (button: number): "back" | "forward" | null => { - if (button === MOUSE_BUTTON_BACK) return "back"; - if (button === MOUSE_BUTTON_FORWARD) return "forward"; - return null; -}; - -// Chromium routes thumb-button navigation to the focused WebContents; navigate this hovered guest explicitly. -const suppressNavigationButton = (event: MouseEvent): void => { - if (!event.isTrusted || navigationDirectionForButton(event.button) === null) return; - event.preventDefault(); +// Registered before page scripts so their capture listeners never see keys typed into the +// overlay; the closed shadow root retargets those to the host, hiding that they edit text. +const isolateOverlayKey = (event: KeyboardEvent): void => { + if (!activeSession || !(event.target instanceof Element) || !isAnnotationNode(event.target)) + return; + activeSession.handleOverlayKey(event); event.stopImmediatePropagation(); }; -const requestNavigationForButton = (event: MouseEvent): void => { - if (!event.isTrusted) return; - const direction = navigationDirectionForButton(event.button); - if (direction === null) return; - event.preventDefault(); - event.stopImmediatePropagation(); - ipcRenderer.send(MOUSE_NAVIGATE_CHANNEL, { direction }); -}; +for (const type of ["keydown", "keypress", "keyup"] as const) + window.addEventListener(type, isolateOverlayKey, true); -window.addEventListener("mousedown", suppressNavigationButton, true); -window.addEventListener("mouseup", requestNavigationForButton, true); -window.addEventListener("auxclick", suppressNavigationButton, true); +let replayedThumbPress: { readonly x: number; readonly y: number } | null = null; +ipcRenderer.on( + THUMB_PRESS_CHANNEL, + (_event, phase: unknown, x: unknown, y: unknown, button: unknown) => { + if (phase !== "down" && phase !== "up") return; + if (typeof x !== "number" || typeof y !== "number" || typeof button !== "number") return; + // Main reports presses inside a frame in that frame's coordinates. + const inFrame = document.querySelector("iframe:hover, frame:hover, object:hover, embed:hover"); + if (phase === "up") { + const pressed = replayedThumbPress; + replayedThumbPress = null; + // A replayed press is always released, at its own point if the release lands in a frame. + if (pressed) recordingCursor?.press(inFrame ? pressed : { x, y }, "up"); + return; + } + if (inFrame) return; + replayedThumbPress = { x, y }; + ipcRenderer.sendToHost(POINTER_DOWN_HOST_CHANNEL); + ipcRenderer.send(HUMAN_INPUT_CHANNEL, { kind: "pointer", x, y, button }); + recordingCursor?.press({ x, y }, "down"); + }, +); const nextId = createAnnotationIdSource(); @@ -317,36 +327,6 @@ function pickFromPoint(clientX: number, clientY: number): Element | null { return null; } -/** A CSS path that finds `element` again in a reloaded copy of the page. */ -function cssPathFor(element: Element): string { - const parts: string[] = []; - let current: Element | null = element; - while (current && current !== document.documentElement) { - if (current.id) { - const idSelector = `#${CSS.escape(current.id)}`; - if (document.querySelectorAll(idSelector).length === 1) { - parts.unshift(idSelector); - return parts.join(" > "); - } - } - const node: Element = current; - const siblings = node.parentElement ? Array.from(node.parentElement.children) : [node]; - const index = siblings.filter((sibling) => sibling.localName === node.localName).indexOf(node); - parts.unshift(`${CSS.escape(node.localName)}:nth-of-type(${index + 1})`); - current = node.parentElement; - } - return [":root", ...parts].join(" > "); -} - -function findBySelector(selector: string): Element | null { - try { - const element = document.querySelector(selector); - return element && !isAnnotationNode(element) ? element : null; - } catch { - return null; - } -} - function describeRawElement(element: Element): string { const tag = element.tagName.toLowerCase(); const id = element.id ? `#${element.id}` : ""; @@ -571,37 +551,8 @@ function createUnitInput(unit: string, placeholder = "0"): HTMLInputElement { return input; } -function pathFromPoints(points: ReadonlyArray): string { - if (points.length === 0) return ""; - if (points.length === 1) return `M ${points[0]!.x} ${points[0]!.y} l 0.01 0.01`; - let path = `M ${points[0]!.x} ${points[0]!.y}`; - for (let index = 1; index < points.length - 1; index += 1) { - const current = points[index]!; - const next = points[index + 1]!; - path += ` Q ${current.x} ${current.y} ${(current.x + next.x) / 2} ${(current.y + next.y) / 2}`; - } - const last = points[points.length - 1]!; - path += ` L ${last.x} ${last.y}`; - return path; -} - -function strokeBounds( - points: ReadonlyArray, - width: number, -): PreviewAnnotationRect { - const xs = points.map((point) => point.x); - const ys = points.map((point) => point.y); - const padding = width + 3; - const left = Math.min(...xs) - padding; - const top = Math.min(...ys) - padding; - const right = Math.max(...xs) + padding; - const bottom = Math.max(...ys) + padding; - return { x: left, y: top, width: right - left, height: bottom - top }; -} - function startAnnotation( sendEnabled: boolean, - draft: PreviewAnnotationDraft | null, voice: DesktopPreviewAnnotationVoiceConfig | undefined, ): void { activeSession?.teardown(false); @@ -922,7 +873,7 @@ function startAnnotation( const regions: PreviewAnnotationRegionTarget[] = []; const strokes: PreviewAnnotationStrokeTarget[] = []; const styleChanges = new Map(); - const toolButtons = new Map(); + const toolButtons = new Map(); // Chromium draws a PDF in its own viewer frame, so pointer and key input over it // never reaches this document. A transparent layer above the viewer catches the // pointer and takes focus for Escape and the tool keys. A PDF has no elements to @@ -934,49 +885,20 @@ function startAnnotation( inputLayer.tabIndex = -1; inputLayer.style.cssText = `position:fixed;inset:0;z-index:${Z_INDEX_OVERLAY - 1};cursor:crosshair;outline:none`; } - let tool: PreviewAnnotationTool = isPdf ? "marquee" : "select"; + let tool: AnnotationTool = isPdf ? "marquee" : "select"; let dragStart: PreviewAnnotationPoint | null = null; - let activeStroke: { target: PreviewAnnotationStrokeTarget; path: SVGPathElement } | null = null; + let activeStroke: { + target: PreviewAnnotationStrokeTarget; + geometry: StrokeGeometry; + group: SVGGElement; + path: SVGPathElement; + } | null = null; let pendingCapture = false; let editorExpanded = false; let editorWasShown = false; let editorPosition: { left: number; top: number } | null = null; let editorDrag: { pointerId: number; offsetX: number; offsetY: number } | null = null; let editorLayoutFrame: number | null = null; - let restoreTimer: number | null = null; - // Restored elements the reloaded page has not rendered yet. Each one holds - // its old spot as a placeholder region until it turns up. - const unresolved: Array< - PreviewAnnotationDraft["elements"][number] & { - placeholderId: string; - styleChanges: ReadonlyArray; - } - > = []; - - // Main keeps the latest copy so a reload of the page can hand it back. - const syncDraft = (): void => { - if (pendingCapture) return; - const placeholderIds = new Set(unresolved.map((element) => element.placeholderId)); - const snapshot: PreviewAnnotationDraft = { - comment: comment.value, - tool, - elements: [ - ...Array.from(selected.values(), (target) => ({ - id: target.id, - selector: cssPathFor(target.element), - rect: rectFromDomRect(target.element.getBoundingClientRect()), - })), - ...unresolved.map(({ id, selector, rect }) => ({ id, selector, rect })), - ], - regions: regions.filter((region) => !placeholderIds.has(region.id)), - strokes, - styleChanges: [ - ...styleChanges.values(), - ...unresolved.flatMap((element) => element.styleChanges), - ], - }; - ipcRenderer.send(ANNOTATION_DRAFT_CHANNEL, snapshot); - }; const resizeComment = (): void => { const maxHeight = 96; @@ -988,7 +910,6 @@ function startAnnotation( }; comment.addEventListener("input", () => { resizeComment(); - syncDraft(); sendVoice("sync"); }); @@ -1008,7 +929,6 @@ function startAnnotation( editorWasShown = true; window.setTimeout(() => comment.focus({ preventScroll: true }), 0); } - syncDraft(); }; const refreshToolButtons = (): void => { @@ -1021,7 +941,6 @@ function startAnnotation( if (tool !== "select") hoverOutline.style.display = "none"; if (tool !== "marquee") marqueeBox.style.display = "none"; document.documentElement.setAttribute("data-t3code-annotation-tool", tool); - syncDraft(); }; const removeSelected = (target: SelectedElement): void => { @@ -1039,10 +958,10 @@ function startAnnotation( } }; - const addSelected = (element: Element, id = nextId("element")): SelectedElement | null => { - if (selected.has(element)) return null; + const addSelected = (element: Element): void => { + if (selected.has(element)) return; const target: SelectedElement = { - id, + id: nextId("element"), element, outline: createBox(PRIMARY, PRIMARY_FILL), label: createLabel(), @@ -1055,7 +974,6 @@ function startAnnotation( stylePanel.style.display = "grid"; syncStyleControls(); } - return target; }; const toggleSelected = (element: Element, additive: boolean): void => { @@ -1071,38 +989,27 @@ function startAnnotation( updateStatus(); }; - const applyStyleChange = ( - target: SelectedElement, - change: PreviewAnnotationStyleChange, - ): void => { - if (!(target.element instanceof HTMLElement || target.element instanceof SVGElement)) return; - if (!target.baselineStyles.has(change.property)) { - target.baselineStyles.set( - change.property, - target.element.style.getPropertyValue(change.property), - ); - } - target.element.style.setProperty(change.property, change.value, "important"); - styleChanges.set(`${target.id}:${change.property}`, change); - updateSelectedVisual(target); - }; - const setStyleForSelected = (property: string, value: string): void => { for (const target of selected.values()) { if (!(target.element instanceof HTMLElement || target.element instanceof SVGElement)) continue; + if (!target.baselineStyles.has(property)) { + target.baselineStyles.set(property, target.element.style.getPropertyValue(property)); + } + const key = `${target.id}:${property}`; const previousValue = - styleChanges.get(`${target.id}:${property}`)?.previousValue ?? + styleChanges.get(key)?.previousValue ?? getComputedStyle(target.element).getPropertyValue(property).trim(); - applyStyleChange(target, { + target.element.style.setProperty(property, value, "important"); + styleChanges.set(key, { targetId: target.id, selector: null, property, previousValue, value, }); + updateSelectedVisual(target); } - syncDraft(); }; const textSection = createStyleSection(); @@ -1328,7 +1235,7 @@ function startAnnotation( gap.value = computed.gap === "normal" ? "0px" : computed.gap; }; - const tools: ReadonlyArray<[PreviewAnnotationTool, string, string]> = [ + const tools: ReadonlyArray<[AnnotationTool, string, string]> = [ ["select", "Select", "Select elements (V)"], ["marquee", "Region", "Draw a region or marquee-select elements (R)"], ["draw", "Draw", "Draw freehand (D)"], @@ -1505,7 +1412,8 @@ function startAnnotation( y <= region.rect.y + region.rect.height, ); if (regionIndex >= 0) { - removeRegion(regions[regionIndex]!.id); + const [removed] = regions.splice(regionIndex, 1); + root.querySelector(`[data-region-id="${removed?.id}"]`)?.remove(); updateStatus(); return true; } @@ -1560,36 +1468,6 @@ function startAnnotation( return candidates.length; }; - const addRegion = (region: PreviewAnnotationRegionTarget): void => { - regions.push(region); - const regionBox = createBox(PRIMARY, "color-mix(in srgb, var(--t3-primary) 6%, transparent)"); - regionBox.setAttribute("data-region-id", region.id); - positionBox(regionBox, region.rect); - root.appendChild(regionBox); - }; - - const removeRegion = (regionId: string): void => { - const index = regions.findIndex((region) => region.id === regionId); - if (index >= 0) regions.splice(index, 1); - root.querySelector(`[data-region-id="${regionId}"]`)?.remove(); - const unresolvedIndex = unresolved.findIndex((element) => element.placeholderId === regionId); - if (unresolvedIndex >= 0) unresolved.splice(unresolvedIndex, 1); - }; - - const createStrokePath = (stroke: PreviewAnnotationStrokeTarget): SVGPathElement => { - const path = document.createElementNS("http://www.w3.org/2000/svg", "path"); - path.setAttribute(OVERLAY_ATTRIBUTE, ""); - path.setAttribute("data-stroke-id", stroke.id); - path.setAttribute("fill", "none"); - path.setAttribute("stroke", stroke.color); - path.setAttribute("stroke-width", String(stroke.width)); - path.setAttribute("stroke-linecap", "round"); - path.setAttribute("stroke-linejoin", "round"); - path.setAttribute("d", pathFromPoints(stroke.points)); - svg.appendChild(path); - return path; - }; - const clearHoverOutline = (): void => { hoverOutline.style.display = "none"; }; @@ -1614,15 +1492,18 @@ function startAnnotation( return; } if (tool === "draw" && activeStroke) { - activeStroke.target.points = [ - ...activeStroke.target.points, - { x: event.clientX, y: event.clientY }, - ]; - activeStroke.target.bounds = strokeBounds( - activeStroke.target.points, - activeStroke.target.width, - ); - activeStroke.path.setAttribute("d", pathFromPoints(activeStroke.target.points)); + const closedPath = addStrokePoint(activeStroke.geometry, { + x: event.clientX, + y: event.clientY, + }); + activeStroke.target.bounds = strokeBounds(activeStroke.geometry, activeStroke.target.width); + if (closedPath !== null) { + activeStroke.path.setAttribute("d", closedPath); + activeStroke.path = activeStroke.group.appendChild( + document.createElementNS("http://www.w3.org/2000/svg", "path"), + ); + } + activeStroke.path.setAttribute("d", openStrokePath(activeStroke.geometry)); } }; @@ -1641,14 +1522,27 @@ function startAnnotation( } dragStart = { x: event.clientX, y: event.clientY }; if (tool === "draw") { + const geometry = startStroke(dragStart); const stroke: PreviewAnnotationStrokeTarget = { id: nextId("stroke"), color: annotationTheme?.primary ?? "#2563eb", width: 4, - points: [dragStart], + points: geometry.points, bounds: { x: dragStart.x, y: dragStart.y, width: 1, height: 1 }, }; - activeStroke = { target: stroke, path: createStrokePath(stroke) }; + const group = document.createElementNS("http://www.w3.org/2000/svg", "g"); + group.setAttribute(OVERLAY_ATTRIBUTE, ""); + group.setAttribute("data-stroke-id", stroke.id); + group.setAttribute("fill", "none"); + group.setAttribute("stroke", stroke.color); + group.setAttribute("stroke-width", String(stroke.width)); + group.setAttribute("stroke-linecap", "round"); + group.setAttribute("stroke-linejoin", "round"); + const path = group.appendChild( + document.createElementNS("http://www.w3.org/2000/svg", "path"), + ); + svg.appendChild(group); + activeStroke = { target: stroke, geometry, group, path }; } }; @@ -1661,11 +1555,21 @@ function startAnnotation( marqueeBox.style.display = "none"; if (isUsableRect(rect)) { const found = selectElementsInRect(rect); - if (found === 0) addRegion({ id: nextId("region"), rect }); + if (found === 0) { + const region: PreviewAnnotationRegionTarget = { id: nextId("region"), rect }; + regions.push(region); + const regionBox = createBox( + PRIMARY, + "color-mix(in srgb, var(--t3-primary) 6%, transparent)", + ); + regionBox.setAttribute("data-region-id", region.id); + positionBox(regionBox, rect); + root.appendChild(regionBox); + } } } else if (tool === "draw" && activeStroke) { if (activeStroke.target.points.length > 1) strokes.push(activeStroke.target); - else activeStroke.path.remove(); + else activeStroke.group.remove(); activeStroke = null; } dragStart = null; @@ -1731,7 +1635,6 @@ function startAnnotation( for (const type of PAGE_ISOLATED_EVENTS) host.removeEventListener(type, stopAtOverlay); document.documentElement.removeEventListener("focusout", onFocusOutToOverlay); if (editorLayoutFrame !== null) window.cancelAnimationFrame(editorLayoutFrame); - if (restoreTimer !== null) window.clearTimeout(restoreTimer); ipcRenderer.off(CANCEL_PICK_CHANNEL, onCancel); ipcRenderer.off(ANNOTATION_CAPTURED_CHANNEL, onCaptured); ipcRenderer.off(ANNOTATION_VOICE_STATE_CHANNEL, onVoiceState); @@ -1772,7 +1675,7 @@ function startAnnotation( return; } } - if (event.key === "Escape") { + if (event.key === "Escape" && !event.isComposing) { event.preventDefault(); event.stopPropagation(); teardown(true); @@ -1801,7 +1704,6 @@ function startAnnotation( } if (pendingCapture || (selected.size === 0 && regions.length === 0 && strokes.length === 0)) return; - syncDraft(); pendingCapture = true; submit.disabled = true; submit.textContent = "Capturing…"; @@ -1862,15 +1764,17 @@ function startAnnotation( }); }; submit.addEventListener("click", () => submitAnnotation("attach")); - root.addEventListener("keydown", (event) => { + const handleOverlayKey = (event: KeyboardEvent): void => { + if (event.type === "keyup") onVoiceKeyUp(event); + if (event.type !== "keydown") return; + onKeyDown(event); + if (event.cancelBubble) return; const submission = - event.target === comment ? resolveAnnotationSubmission(event, sendEnabled) : null; - // Bubble phase lets editor inputs handle events before hiding them from the page. - event.stopImmediatePropagation(); + shadowRoot.activeElement === comment ? resolveAnnotationSubmission(event, sendEnabled) : null; if (!submission) return; event.preventDefault(); submitAnnotation(submission); - }); + }; window.addEventListener("pointermove", onPointerMove, { capture: true, passive: false }); window.addEventListener("pointerdown", onPointerDown, { capture: true, passive: false }); @@ -1895,7 +1799,6 @@ function startAnnotation( inputLayer.focus({ preventScroll: true }); } document.documentElement.appendChild(host); - if (draft) restoreDraft(draft); rootResizeObserver.observe(document.documentElement); refreshToolButtons(); updateStatus(); @@ -1903,61 +1806,12 @@ function startAnnotation( activeSession = { teardown, applyTheme: (theme) => applyAnnotationTheme(host, theme), - syncDraft, setSendEnabled: (enabled) => { sendEnabled = enabled; updateSendHint(); }, + handleOverlayKey, }; - - /** - * Rebuilds the markup a reload interrupted. Apps often render after - * DOMContentLoaded, so an element that is not there yet holds its old spot - * as a region and is swapped back in if it shows up before the timeout. - */ - function restoreDraft(restored: PreviewAnnotationDraft): void { - comment.value = restored.comment; - resizeComment(); - tool = restored.tool; - for (const region of restored.regions) addRegion(region); - for (const stroke of restored.strokes) { - strokes.push(stroke); - createStrokePath(stroke); - } - for (const element of restored.elements) { - const placeholderId = nextId("region"); - unresolved.push({ - ...element, - placeholderId, - styleChanges: restored.styleChanges.filter((change) => change.targetId === element.id), - }); - addRegion({ id: placeholderId, rect: element.rect }); - } - const deadline = Date.now() + DRAFT_RESTORE_TIMEOUT_MS; - const resolveElements = (): void => { - restoreTimer = null; - if (finished || pendingCapture) return; - // removeRegion drops each match from `unresolved`, so collect first. - const matches = unresolved.flatMap((element) => { - const found = findBySelector(element.selector); - return found ? [{ element, found }] : []; - }); - for (const { element, found } of matches) { - removeRegion(element.placeholderId); - // The user may have selected it again by hand while it was missing. - const target = selected.get(found) ?? addSelected(found, element.id); - if (!target) continue; - for (const change of element.styleChanges) { - applyStyleChange(target, { ...change, targetId: target.id }); - } - } - updateStatus(); - if (unresolved.length > 0 && Date.now() < deadline) { - restoreTimer = window.setTimeout(resolveElements, DRAFT_RESTORE_INTERVAL_MS); - } - }; - resolveElements(); - } } ipcRenderer.on( @@ -1966,14 +1820,12 @@ ipcRenderer.on( _event, theme: DesktopPreviewAnnotationTheme | undefined, sendEnabled?: boolean, - draft?: unknown, voice?: DesktopPreviewAnnotationVoiceConfig, ) => { if (theme) annotationTheme = theme; - startAnnotation(sendEnabled === true, isPreviewAnnotationDraft(draft) ? draft : null, voice); + startAnnotation(sendEnabled === true, voice); }, ); -window.addEventListener("pagehide", () => activeSession?.syncDraft()); ipcRenderer.on(ANNOTATION_SEND_ENABLED_CHANNEL, (_event, enabled: boolean) => { activeSession?.setSendEnabled(enabled === true); }); diff --git a/apps/desktop/src/preview/RecordingCursor.ts b/apps/desktop/src/preview/RecordingCursor.ts index 29f1865de291..ab74c361b68e 100644 --- a/apps/desktop/src/preview/RecordingCursor.ts +++ b/apps/desktop/src/preview/RecordingCursor.ts @@ -156,13 +156,15 @@ export function installRecordingCursor( moveHuman(point); pointer(point, "move"); }; + const press = (point: { readonly x: number; readonly y: number }, phase: "down" | "up") => { + if (phase === "down") moveHuman(point); + pointer(point, phase); + }; const pointerDown = (event: PointerEvent) => { - if (event.pointerType === "touch") return; - moveHuman({ x: event.clientX, y: event.clientY }); - pointer({ x: event.clientX, y: event.clientY }, "down"); + if (event.pointerType !== "touch") press({ x: event.clientX, y: event.clientY }, "down"); }; const pointerUp = (event: PointerEvent) => { - if (event.pointerType !== "touch") pointer({ x: event.clientX, y: event.clientY }, "up"); + if (event.pointerType !== "touch") press({ x: event.clientX, y: event.clientY }, "up"); }; const keyDown = (event: KeyboardEvent) => { if (event.isComposing || event.repeat) return; @@ -196,6 +198,7 @@ export function installRecordingCursor( window.addEventListener("blur", hide); return { move, + press, keyPress, setController, setTheme, diff --git a/apps/desktop/src/settings/DesktopAppSettings.test.ts b/apps/desktop/src/settings/DesktopAppSettings.test.ts index c31209454fa3..e1d28cc3e41e 100644 --- a/apps/desktop/src/settings/DesktopAppSettings.test.ts +++ b/apps/desktop/src/settings/DesktopAppSettings.test.ts @@ -4,6 +4,7 @@ import { assert, describe, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as PlatformError from "effect/PlatformError"; import * as Schema from "effect/Schema"; import * as Tracer from "effect/Tracer"; @@ -160,6 +161,49 @@ describe("DesktopSettings", () => { ), ); + it.effect("keeps saved preferences when a read fails before a bounds save", () => + withSettings( + Effect.gen(function* () { + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const fileSystem = yield* FileSystem.FileSystem; + yield* writeSettingsPatch({ serverExposureMode: "network-accessible" }); + const cause = PlatformError.systemError({ + _tag: "PermissionDenied", + module: "FileSystem", + method: "readFileString", + pathOrDescriptor: environment.desktopSettingsPath, + }); + let failRead = true; + const settings = yield* DesktopAppSettings.make.pipe( + Effect.provideService( + FileSystem.FileSystem, + FileSystem.FileSystem.of({ + ...fileSystem, + readFileString: (path) => + Effect.suspend(() => { + if (failRead && path === environment.desktopSettingsPath) { + failRead = false; + return Effect.fail(cause); + } + return fileSystem.readFileString(path); + }), + }), + ), + ); + + yield* Effect.exit(settings.load); + yield* Effect.exit( + settings.setMainWindowBounds({ x: 10, y: 20, width: 900, height: 600 }, false), + ); + + const persisted = yield* decodeDesktopSettingsPatch( + yield* fileSystem.readFileString(environment.desktopSettingsPath), + ); + assert.equal(persisted.serverExposureMode, "network-accessible"); + }), + ), + ); + it.effect("loads lenient persisted desktop settings JSON", () => withSettings( Effect.gen(function* () { @@ -197,6 +241,28 @@ describe("DesktopSettings", () => { ), ); + it.effect("keeps Local environment off when another value in the file is unreadable", () => + withSettings( + Effect.gen(function* () { + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const fileSystem = yield* FileSystem.FileSystem; + const settings = yield* DesktopAppSettings.DesktopAppSettings; + yield* fileSystem.makeDirectory(environment.stateDir, { recursive: true }); + yield* fileSystem.writeFileString( + environment.desktopSettingsPath, + `{ "localEnvironmentEnabled": false, "serverExposureMode": "tailnet" }\n`, + ); + + assert.isFalse((yield* settings.load).localEnvironmentEnabled); + yield* settings.setMainWindowBounds({ x: 10, y: 20, width: 800, height: 600 }, false); + assert.include( + yield* fileSystem.readFileString(environment.desktopSettingsPath), + `"localEnvironmentEnabled":false`, + ); + }), + ), + ); + it.effect("rejects window bounds that do not satisfy the domain schema", () => withSettings( Effect.gen(function* () { diff --git a/apps/desktop/src/settings/DesktopAppSettings.ts b/apps/desktop/src/settings/DesktopAppSettings.ts index d343a545d8b1..73e43b7c33a3 100644 --- a/apps/desktop/src/settings/DesktopAppSettings.ts +++ b/apps/desktop/src/settings/DesktopAppSettings.ts @@ -13,6 +13,8 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import * as PlatformError from "effect/PlatformError"; +import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import * as SynchronizedRef from "effect/SynchronizedRef"; @@ -90,21 +92,26 @@ const DesktopWindowBoundsDocument = Schema.Struct({ height: Schema.Number, }); +// A value this build cannot read drops only its own key, so the rest of the file +// still loads and is written back instead of every setting resetting to defaults. +const optionalReadableKey = (schema: S) => + Schema.optionalKey(schema.pipe(Schema.catchDecoding(() => Effect.succeedNone))); + const DesktopSettingsDocument = Schema.Struct({ - localEnvironmentEnabled: Schema.optionalKey(Schema.Boolean), - linuxPasswordStore: Schema.optionalKey(Schema.Unknown), - mainWindowBounds: Schema.optionalKey(Schema.NullOr(DesktopWindowBoundsDocument)), - mainWindowMaximized: Schema.optionalKey(Schema.Boolean), - serverExposureMode: Schema.optionalKey(DesktopServerExposureModeSchema), - tailscaleServeEnabled: Schema.optionalKey(Schema.Boolean), - tailscaleServePort: Schema.optionalKey(Schema.Number), - updateChannel: Schema.optionalKey(DesktopUpdateChannelSchema), - updateChannelConfiguredByUser: Schema.optionalKey(Schema.Boolean), - wslBackendEnabled: Schema.optionalKey(Schema.Boolean), - wslMode: Schema.optionalKey(Schema.Literals(["local", "wsl"])), - wslDistro: Schema.optionalKey(Schema.NullOr(Schema.String)), - sshRunner: Schema.optionalKey(Schema.Literals(["windows", "wsl"])), - wslOnly: Schema.optionalKey(Schema.Boolean), + localEnvironmentEnabled: optionalReadableKey(Schema.Boolean), + linuxPasswordStore: optionalReadableKey(Schema.Unknown), + mainWindowBounds: optionalReadableKey(Schema.NullOr(DesktopWindowBoundsDocument)), + mainWindowMaximized: optionalReadableKey(Schema.Boolean), + serverExposureMode: optionalReadableKey(DesktopServerExposureModeSchema), + tailscaleServeEnabled: optionalReadableKey(Schema.Boolean), + tailscaleServePort: optionalReadableKey(Schema.Number), + updateChannel: optionalReadableKey(DesktopUpdateChannelSchema), + updateChannelConfiguredByUser: optionalReadableKey(Schema.Boolean), + wslBackendEnabled: optionalReadableKey(Schema.Boolean), + wslMode: optionalReadableKey(Schema.Literals(["local", "wsl"])), + wslDistro: optionalReadableKey(Schema.NullOr(Schema.String)), + sshRunner: optionalReadableKey(Schema.Literals(["windows", "wsl"])), + wslOnly: optionalReadableKey(Schema.Boolean), }); type DesktopSettingsDocument = typeof DesktopSettingsDocument.Type; @@ -393,15 +400,20 @@ function applyWslWindowsFallback(settings: DesktopSettings): DesktopSettings { return setWslOnly(setWslBackendEnabled(settings, false), false); } +// Fails only when the file may exist but could not be read; a missing file means defaults. function readSettings( fileSystem: FileSystem.FileSystem, settingsPath: string, appVersion: string, -): Effect.Effect { +): Effect.Effect { const defaultSettings = resolveDefaultDesktopSettings(appVersion); return fileSystem.readFileString(settingsPath).pipe( - Effect.option, + Effect.asSome, + Effect.catchIf( + (cause) => cause.reason._tag === "NotFound", + () => Effect.succeed(Option.none()), + ), Effect.flatMap( Option.match({ onNone: () => Effect.succeed(defaultSettings), @@ -490,6 +502,8 @@ export const make = Effect.gen(function* () { const path = yield* Path.Path; const random = yield* SecureRandom.SecureRandom; const settingsRef = yield* SynchronizedRef.make(environment.defaultDesktopSettings); + // Writing over a settings file we could not read would erase the preferences it holds. + const persistenceEnabled = yield* Ref.make(true); const updateInMemory = (update: (settings: DesktopSettings) => DesktopSettings) => SynchronizedRef.modify(settingsRef, (settings) => { @@ -506,17 +520,23 @@ export const make = Effect.gen(function* () { return Effect.succeed([settingsChange(settings, false), settings] as const); } - return random.uuidV4().pipe( - Effect.map((uuid) => uuid.replace(/-/g, "")), - Effect.flatMap((suffix) => - writeSettings({ - fileSystem, - path, - settingsPath: environment.desktopSettingsPath, - settings: nextSettings, - defaultSettings: environment.defaultDesktopSettings, - suffix, - }), + return Ref.get(persistenceEnabled).pipe( + Effect.flatMap((enabled) => + enabled + ? random.uuidV4().pipe( + Effect.map((uuid) => uuid.replace(/-/g, "")), + Effect.flatMap((suffix) => + writeSettings({ + fileSystem, + path, + settingsPath: environment.desktopSettingsPath, + settings: nextSettings, + defaultSettings: environment.defaultDesktopSettings, + suffix, + }), + ), + ) + : Effect.void, ), Effect.as([settingsChange(nextSettings, true), nextSettings] as const), ); @@ -529,6 +549,17 @@ export const make = Effect.gen(function* () { fileSystem, environment.desktopSettingsPath, environment.appVersion, + ).pipe( + Effect.catch((cause) => + Effect.logWarning( + "Could not read desktop settings; using defaults without saving changes.", + cause, + ).pipe( + Effect.annotateLogs({ settingsPath: environment.desktopSettingsPath }), + Effect.andThen(Ref.set(persistenceEnabled, false)), + Effect.as(resolveDefaultDesktopSettings(environment.appVersion)), + ), + ), ); return yield* SynchronizedRef.setAndGet(settingsRef, settings); }).pipe(Effect.withSpan("desktop.settings.load")), diff --git a/apps/desktop/src/settings/DesktopClientSettings.test.ts b/apps/desktop/src/settings/DesktopClientSettings.test.ts index 1ab9b3159e1f..90c8740f8372 100644 --- a/apps/desktop/src/settings/DesktopClientSettings.test.ts +++ b/apps/desktop/src/settings/DesktopClientSettings.test.ts @@ -110,7 +110,7 @@ const withClientSettings = ( prefix: "t3-desktop-client-settings-test-", }); return yield* effect.pipe(Effect.provide(layer(baseDir))); - }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), Effect.scoped); + }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))); describe("DesktopClientSettings", () => { it.effect("saves through a symlinked client settings file without replacing the link", () => diff --git a/apps/desktop/src/settings/DesktopSavedEnvironments.test.ts b/apps/desktop/src/settings/DesktopSavedEnvironments.test.ts index 45224317989d..6e706aec6bb6 100644 --- a/apps/desktop/src/settings/DesktopSavedEnvironments.test.ts +++ b/apps/desktop/src/settings/DesktopSavedEnvironments.test.ts @@ -151,7 +151,7 @@ const withSavedEnvironments = ( prefix: "t3-desktop-saved-environments-test-", }); return yield* effect.pipe(Effect.provide(layer(baseDir, options))); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped); + }).pipe(Effect.provide(NodeServices.layer)); describe("DesktopSavedEnvironments", () => { it.effect("loads lenient saved environment registry documents", () => diff --git a/apps/desktop/src/shell/DesktopShellEnvironment.test.ts b/apps/desktop/src/shell/DesktopShellEnvironment.test.ts index 0c7869194323..7808682dfd82 100644 --- a/apps/desktop/src/shell/DesktopShellEnvironment.test.ts +++ b/apps/desktop/src/shell/DesktopShellEnvironment.test.ts @@ -2,6 +2,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, describe, it } from "@effect/vitest"; import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Deferred from "effect/Deferred"; @@ -62,6 +63,26 @@ function makeProcess(output: string, exitCode = 0): ChildProcessSpawner.ChildPro }); } +function makeSlowProcess( + output: string, + delay: Duration.Input, +): ChildProcessSpawner.ChildProcessHandle { + const stdout = Stream.fromEffect(Effect.as(Effect.sleep(delay), textEncoder.encode(output))); + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(125), + stdout, + stderr: Stream.empty, + all: stdout, + exitCode: Effect.as(Effect.sleep(delay), ChildProcessSpawner.ExitCode(0)), + isRunning: Effect.succeed(true), + kill: () => Effect.void, + stdin: Sink.drain, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }); +} + function makeHungProcess(): ChildProcessSpawner.ChildProcessHandle { return ChildProcessSpawner.makeHandle({ pid: ChildProcessSpawner.ProcessId(124), @@ -486,7 +507,7 @@ describe("DesktopShellEnvironment", () => { Effect.gen(function* () { const env: NodeJS.ProcessEnv = { SHELL: "/bin/zsh", - PATH: "/Users/test/.local/bin:/usr/bin", + PATH: " /Users/test/.local/bin:/usr/bin", }; const commands: ChildProcess.Command[] = []; @@ -496,7 +517,7 @@ describe("DesktopShellEnvironment", () => { handler: (command) => { commands.push(command); return envOutput({ - PATH: "/opt/homebrew/bin:/usr/bin", + PATH: "/opt/homebrew/bin:/usr/bin:/opt/bin ", SSH_AUTH_SOCK: "/tmp/secretive.sock", HOMEBREW_PREFIX: "/opt/homebrew", }); @@ -505,7 +526,7 @@ describe("DesktopShellEnvironment", () => { assert.equal(commands.length, 1); assert.equal(commands[0]?._tag === "StandardCommand" ? commands[0].command : "", "/bin/zsh"); - assert.equal(env.PATH, "/opt/homebrew/bin:/usr/bin:/Users/test/.local/bin"); + assert.equal(env.PATH, "/opt/homebrew/bin:/usr/bin:/opt/bin : /Users/test/.local/bin"); assert.equal(env.SSH_AUTH_SOCK, "/tmp/secretive.sock"); assert.equal(env.HOMEBREW_PREFIX, "/opt/homebrew"); assert.isTrue(prepared); @@ -580,6 +601,39 @@ describe("DesktopShellEnvironment", () => { }), ); + it.effect.each([ + { + name: "hydrates both Android SDK names when neither is inherited", + inherited: {}, + expected: { ANDROID_HOME: "/shell/android-home", ANDROID_SDK_ROOT: "/shell/android-root" }, + }, + { + // The device host reads ANDROID_HOME before ANDROID_SDK_ROOT, so a shell + // ANDROID_HOME next to an inherited ANDROID_SDK_ROOT would change the SDK. + name: "keeps an inherited ANDROID_SDK_ROOT selected over a login-shell ANDROID_HOME", + inherited: { ANDROID_SDK_ROOT: "/inherited/android-sdk" }, + expected: { ANDROID_HOME: undefined, ANDROID_SDK_ROOT: "/inherited/android-sdk" }, + }, + ])("$name", ({ inherited, expected }) => + Effect.gen(function* () { + const env: NodeJS.ProcessEnv = { SHELL: "/bin/zsh", PATH: "/usr/bin", ...inherited }; + + yield* runShellEnvironment({ + env, + platform: "darwin", + handler: () => + envOutput({ + PATH: "/usr/bin", + ANDROID_HOME: "/shell/android-home", + ANDROID_SDK_ROOT: "/shell/android-root", + }), + }); + + assert.equal(env.ANDROID_HOME, expected.ANDROID_HOME); + assert.equal(env.ANDROID_SDK_ROOT, expected.ANDROID_SDK_ROOT); + }), + ); + it.effect("keeps mise's PATH record with the login shell PATH", () => Effect.gen(function* () { const env: NodeJS.ProcessEnv = { @@ -637,6 +691,175 @@ describe("DesktopShellEnvironment", () => { }), ); + it.effect("retries the user's shell when only the fallback shell answered", () => + Effect.gen(function* () { + const env: NodeJS.ProcessEnv = { SHELL: "/usr/bin/fish", PATH: "/usr/bin" }; + const commands: string[] = []; + let fishReady = false; + yield* runShellEnvironment({ + env, + platform: "linux", + handler: (command) => { + if (command._tag !== "StandardCommand") return ""; + commands.push(command.command); + if (command.command !== "/usr/bin/fish") return envOutput({ PATH: "/usr/bin" }); + return fishReady + ? envOutput({ PATH: "/fish/bin:/usr/bin" }) + : { output: "", exitCode: 7 }; + }, + use: (service) => + Effect.gen(function* () { + assert.isTrue(yield* service.awaitDiscovery); + assert.equal(env.PATH, "/usr/bin"); + // The Linux probe checks the D-Bus socket on the real filesystem, which the clock does not await. + const published = yield* service.updates.pipe( + Stream.filter((snapshot) => snapshot.environment.PATH === "/fish/bin:/usr/bin"), + Stream.runHead, + Effect.forkChild({ startImmediately: true }), + ); + + fishReady = true; + yield* TestClock.adjust("30 seconds"); + assert.isTrue(Option.isSome(yield* Fiber.join(published))); + assert.equal(env.PATH, "/fish/bin:/usr/bin"); + return true; + }), + }).pipe(Effect.provide(Logger.layer([]))); + assert.deepEqual(commands, ["/usr/bin/fish", "/bin/bash", "/usr/bin/fish"]); + + // With no user shell to wait for, the fallback's answer is final. + commands.length = 0; + yield* runShellEnvironment({ + env: { PATH: "/usr/bin" }, + platform: "linux", + handler: (command) => { + if (command._tag === "StandardCommand") commands.push(command.command); + return envOutput({ PATH: "/usr/bin" }); + }, + use: (service) => + Effect.gen(function* () { + assert.isTrue(yield* service.awaitDiscovery); + yield* TestClock.adjust("5 minutes"); + return true; + }), + }).pipe(Effect.provide(Logger.layer([]))); + assert.deepEqual(commands, ["/bin/bash"]); + }), + ); + + it.effect("gives background retries time for a locked or slow shell, then publishes", () => + Effect.gen(function* () { + const env: NodeJS.ProcessEnv = { SHELL: "/bin/zsh", PATH: "/usr/bin" }; + // Startup fails fast, the first retry meets a shell blocked on a lock, the second a slow one. + const spawned = yield* Effect.forEach([0, 1, 2], () => Deferred.make()); + const stopped = yield* Deferred.make(); + let attempts = 0; + yield* runShellEnvironment({ + env, + platform: "darwin", + handler: () => "", + processSpawner: ChildProcessSpawner.make((command) => { + if (command._tag !== "StandardCommand" || command.command !== "/bin/zsh") { + return Effect.succeed(makeProcess("")); + } + const attempt = attempts++; + const spawn = Deferred.succeed(spawned[attempt]!, undefined); + if (attempt === 0) return spawn.pipe(Effect.as(makeProcess(""))); + if (attempt === 1) { + return spawn.pipe( + Effect.andThen( + Effect.acquireRelease(Effect.succeed(makeHungProcess()), () => + Deferred.succeed(stopped, command.options), + ), + ), + ); + } + return spawn.pipe( + Effect.as(makeSlowProcess(envOutput({ PATH: "/fresh/bin" }), "20 seconds")), + ); + }), + use: (service) => + Effect.gen(function* () { + assert.isFalse(yield* service.awaitDiscovery); + const published = yield* service.updates.pipe( + Stream.drop(1), + Stream.runHead, + Effect.forkChild({ startImmediately: true }), + ); + + yield* TestClock.adjust("30 seconds"); + yield* Deferred.await(spawned[1]!); + // pyenv waits up to 60 s on its rehash lock before the shell continues. + yield* TestClock.adjust("1 minute"); + assert.isFalse(yield* Deferred.isDone(stopped)); + yield* TestClock.adjust("1 minute"); + const stop = yield* Deferred.await(stopped); + // pyenv releases that lock in an EXIT trap, which SIGTERM runs and SIGKILL skips. + assert.equal(stop.killSignal, "SIGTERM"); + assert.isAtLeast(Duration.toMillis(stop.forceKillAfter ?? 0), 5_000); + + yield* TestClock.adjust("30 seconds"); + yield* Deferred.await(spawned[2]!); + yield* TestClock.adjust("20 seconds"); + const snapshot = yield* Fiber.join(published); + assert.equal(env.PATH, "/fresh/bin:/usr/bin"); + assert.isTrue(Option.isSome(snapshot) && snapshot.value.discovered); + assert.equal( + Option.isSome(snapshot) ? snapshot.value.environment.PATH : undefined, + "/fresh/bin:/usr/bin", + ); + return true; + }), + }); + }), + ); + + it.effect("recovers the login-shell PATH when the first probe outlasts the timeout", () => + Effect.gen(function* () { + const env: NodeJS.ProcessEnv = { SHELL: "/bin/zsh", PATH: "/usr/bin:/bin:/usr/sbin:/sbin" }; + const output = envOutput({ PATH: "/opt/homebrew/bin:/usr/bin" }); + let shellProbes = 0; + yield* runShellEnvironment({ + env, + platform: "darwin", + handler: () => "", + processSpawner: ChildProcessSpawner.make((command) => { + if (command._tag !== "StandardCommand" || command.command !== "/bin/zsh") { + return Effect.succeed(makeProcess("")); + } + shellProbes += 1; + if (shellProbes > 1) return Effect.succeed(makeProcess(output)); + // A cold start: the first login shell answers, but only after 7s. + const slow = Effect.sleep("7 seconds"); + const stdout = Stream.fromEffect(slow.pipe(Effect.as(textEncoder.encode(output)))); + return Effect.succeed( + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(125), + stdout, + stderr: Stream.empty, + all: stdout, + exitCode: slow.pipe(Effect.as(ChildProcessSpawner.ExitCode(0))), + isRunning: Effect.succeed(true), + kill: () => Effect.void, + stdin: Sink.drain, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }), + ); + }), + use: (service) => + Effect.gen(function* () { + yield* service.installIntoProcess; + yield* TestClock.adjust("10 minutes"); + return true; + }), + }).pipe(Effect.provide(Logger.layer([]))); + + assert.equal(env.PATH, "/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin"); + }), + ); + it.effect("loads PowerShell profile environment on Windows", () => Effect.gen(function* () { const env: NodeJS.ProcessEnv = { diff --git a/apps/desktop/src/shell/DesktopShellEnvironment.ts b/apps/desktop/src/shell/DesktopShellEnvironment.ts index 9d9e390c9549..351150aa4b98 100644 --- a/apps/desktop/src/shell/DesktopShellEnvironment.ts +++ b/apps/desktop/src/shell/DesktopShellEnvironment.ts @@ -3,6 +3,7 @@ import { applyShellEnvironment, buildEnvironmentCaptureCommand, listLoginShellCandidates, + mergePathValues, readUserLoginShell, setWindowsPath, } from "@t3tools/shared/shell"; @@ -14,6 +15,7 @@ import * as Deferred from "effect/Deferred"; import * as Path from "effect/Path"; import * as PubSub from "effect/PubSub"; import * as Ref from "effect/Ref"; +import * as Schedule from "effect/Schedule"; import * as Scope from "effect/Scope"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; @@ -27,9 +29,17 @@ import * as DesktopEnvironment from "../app/DesktopEnvironment.ts"; type EnvironmentPatch = Record; +interface ProbeBudget { + readonly timeout: Duration.Duration; + // One longer wait for the first login shell that times out. + readonly slowTimeout?: Duration.Duration; + readonly terminateGrace: Duration.Duration; +} + interface ShellEnvironmentConfig { readonly env: NodeJS.ProcessEnv; readonly platform: NodeJS.Platform; + readonly budget: ProbeBudget; } interface WindowsProbeOptions { @@ -88,6 +98,8 @@ export class DesktopShellEnvironment extends Context.Service< const LOGIN_SHELL_ENV_NAMES = [ "PATH", + "ANDROID_HOME", + "ANDROID_SDK_ROOT", "DBUS_SESSION_BUS_ADDRESS", "DISPLAY", "LANG", @@ -114,11 +126,28 @@ const LOGIN_SHELL_ENV_NAMES = [ ] as const; const WINDOWS_PROFILE_ENV_NAMES = ["PATH", "FNM_DIR", "FNM_MULTISHELL_PATH"] as const; const LOCALE_ENV_NAMES = ["LANG", "LC_ALL", "LC_CTYPE"] as const; +const ANDROID_SDK_ENV_NAMES = ["ANDROID_HOME", "ANDROID_SDK_ROOT"] as const; const FALLBACK_LC_CTYPE = "en_US.UTF-8"; const WINDOWS_SHELL_CANDIDATES = ["pwsh.exe", "powershell.exe"] as const; const LOGIN_SHELL_TIMEOUT = Duration.seconds(5); +// A profile can be slow only while the machine boots; one longer wait covers it. +const SLOW_LOGIN_SHELL_TIMEOUT = Duration.seconds(20); const LAUNCHCTL_TIMEOUT = Duration.seconds(2); const PROCESS_TERMINATE_GRACE = Duration.seconds(1); +const STARTUP_PROBE: ProbeBudget = { + timeout: LOGIN_SHELL_TIMEOUT, + slowTimeout: SLOW_LOGIN_SHELL_TIMEOUT, + terminateGrace: PROCESS_TERMINATE_GRACE, +}; +// Retries block nothing, so they outlast pyenv's 60 s wait on its rehash lock, and SIGTERM gets +// time to run the exit trap that releases it. A SIGKILL leaves the lock for two minutes. +const RETRY_PROBE: ProbeBudget = { + timeout: Duration.minutes(2), + terminateGrace: Duration.seconds(10), +}; +// A Mac asleep at launch fails the first probe on wake, and nothing else would probe again. +const DISCOVERY_RETRY_DELAY = Duration.seconds(30); +const DISCOVERY_RETRIES = 10; const trimNonEmpty = (value: string | null | undefined): Option.Option => Option.fromNullishOr(value).pipe( @@ -126,10 +155,12 @@ const trimNonEmpty = (value: string | null | undefined): Option.Option = Option.filter((entry) => entry.length > 0), ); -const pathDelimiter = (platform: NodeJS.Platform) => (platform === "win32" ? ";" : ":"); +// POSIX directory names may begin or end with whitespace; mergePathValues trims only on Windows. +const nonEmptyPath = (value: string | null | undefined): Option.Option => + Option.fromNullishOr(value).pipe(Option.filter((entry) => entry.length > 0)); const readEnvPath = (env: NodeJS.ProcessEnv): Option.Option => - trimNonEmpty(env.PATH ?? env.Path ?? env.path); + nonEmptyPath(env.PATH ?? env.Path ?? env.path); const normalizeRuntimeDir = (value: string): string => value.replace(/\/+$/u, ""); @@ -148,39 +179,16 @@ const linuxRuntimeDirCandidates = ( return candidates.filter((candidate) => candidate.length > 0); }; -const pathComparisonKey = (entry: string, platform: NodeJS.Platform) => { - const normalized = entry.trim().replace(/^"+|"+$/g, ""); - return platform === "win32" ? normalized.toLowerCase() : normalized; -}; - -const sanitizePathEntry = (entry: string, platform: NodeJS.Platform) => - platform === "win32" ? entry.replaceAll('"', "") : entry; - const mergePaths = ( platform: NodeJS.Platform, values: ReadonlyArray>, -): Option.Option => { - const delimiter = pathDelimiter(platform); - const entries: string[] = []; - const seen = new Set(); - - for (const value of values) { - if (Option.isNone(value)) continue; - - for (const entry of value.value.split(delimiter)) { - const sanitized = sanitizePathEntry(entry.trim(), platform); - if (sanitized.length === 0) continue; - - const key = pathComparisonKey(sanitized, platform); - if (key.length === 0 || seen.has(key)) continue; - - seen.add(key); - entries.push(sanitized); - } - } - - return entries.length > 0 ? Option.some(entries.join(delimiter)) : Option.none(); -}; +): Option.Option => + Option.fromNullishOr( + values.reduce( + (merged, value) => mergePathValues(merged, Option.getOrUndefined(value), platform), + undefined, + ), + ); const knownWindowsCliDirs = (env: NodeJS.ProcessEnv): ReadonlyArray => [ ...trimNonEmpty(env.APPDATA).pipe( @@ -260,8 +268,13 @@ const runCommandOutput = Effect.fn("desktop.shellEnvironment.runCommandOutput")( readonly args: ReadonlyArray; readonly env: NodeJS.ProcessEnv; readonly timeout: Duration.Duration; + readonly terminateGrace?: Duration.Duration; readonly shell?: boolean; -}): Effect.fn.Return { +}): Effect.fn.Return< + string, + DesktopShellEnvironmentCommandTimeoutError, + ChildProcessSpawner.ChildProcessSpawner +> { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const fiber = yield* Effect.scoped( Effect.gen(function* () { @@ -274,7 +287,7 @@ const runCommandOutput = Effect.fn("desktop.shellEnvironment.runCommandOutput")( stdout: "pipe", stderr: "ignore", killSignal: "SIGTERM", - forceKillAfter: PROCESS_TERMINATE_GRACE, + forceKillAfter: input.terminateGrace ?? PROCESS_TERMINATE_GRACE, }), ); return yield* Effect.all([Stream.mkString(Stream.decodeText(child.stdout)), child.exitCode], { @@ -317,21 +330,35 @@ const runCommandOutput = Effect.fn("desktop.shellEnvironment.runCommandOutput")( return output.value; } - const error = new DesktopShellEnvironmentCommandTimeoutError({ + return yield* new DesktopShellEnvironmentCommandTimeoutError({ probe: input.probe, executable: executableName(input.command), argumentCount: input.args.length, timeoutMs: Duration.toMillis(input.timeout), }); - yield* logShellEnvironmentCommandError(error); - return ""; }); +const orEmptyOnTimeout = ( + effect: Effect.Effect, +): Effect.Effect => + effect.pipe( + Effect.catchTags({ + DesktopShellEnvironmentCommandTimeoutError: (error) => + logShellEnvironmentCommandError(error).pipe(Effect.as("")), + }), + ); + const readLoginShellEnvironment = ( shell: string, names: ReadonlyArray, env: NodeJS.ProcessEnv, -): Effect.Effect => + timeout: Duration.Duration, + terminateGrace: Duration.Duration, +): Effect.Effect< + EnvironmentPatch, + DesktopShellEnvironmentCommandTimeoutError, + ChildProcessSpawner.ChildProcessSpawner +> => names.length === 0 ? Effect.succeed({}) : runCommandOutput({ @@ -339,7 +366,8 @@ const readLoginShellEnvironment = ( command: shell, args: ["-ilc", buildEnvironmentCaptureCommand(shell, names)], env, - timeout: LOGIN_SHELL_TIMEOUT, + timeout, + terminateGrace, }).pipe(Effect.map((output) => extractEnvironment(output, names))); const readLaunchctlPath = (env: NodeJS.ProcessEnv) => @@ -349,13 +377,14 @@ const readLaunchctlPath = (env: NodeJS.ProcessEnv) => command: "/bin/launchctl", args: ["getenv", "PATH"], timeout: LAUNCHCTL_TIMEOUT, - }).pipe(Effect.map(trimNonEmpty)); + }).pipe(orEmptyOnTimeout, Effect.map(trimNonEmpty)); const readWindowsEnvironment = Effect.fn("desktop.shellEnvironment.readWindowsEnvironment")( function* ( names: ReadonlyArray, options: WindowsProbeOptions, env: NodeJS.ProcessEnv, + budget: ProbeBudget, ): Effect.fn.Return { if (names.length === 0) return {}; @@ -373,8 +402,9 @@ const readWindowsEnvironment = Effect.fn("desktop.shellEnvironment.readWindowsEn command, args, env, - timeout: LOGIN_SHELL_TIMEOUT, - }); + timeout: budget.timeout, + terminateGrace: budget.terminateGrace, + }).pipe(orEmptyOnTimeout); const environment = extractEnvironment(output, names); if (Object.keys(environment).length > 0) { return environment; @@ -391,8 +421,13 @@ const installWindowsEnvironment = Effect.fn("desktop.shellEnvironment.installWin ): Effect.fn.Return { const [noProfile, profile] = yield* Effect.all( [ - readWindowsEnvironment(["PATH"], { loadProfile: false }, config.env), - readWindowsEnvironment(WINDOWS_PROFILE_ENV_NAMES, { loadProfile: true }, config.env), + readWindowsEnvironment(["PATH"], { loadProfile: false }, config.env, config.budget), + readWindowsEnvironment( + WINDOWS_PROFILE_ENV_NAMES, + { loadProfile: true }, + config.env, + config.budget, + ), ], { concurrency: 2 }, ); @@ -416,6 +451,9 @@ const installWindowsEnvironment = Effect.fn("desktop.shellEnvironment.installWin }, ); +const listUserShells = (env: NodeJS.ProcessEnv): ReadonlyArray => + [env.SHELL?.trim(), readUserLoginShell()].filter((shell): shell is string => Boolean(shell)); + const installPosixEnvironment = Effect.fn("desktop.shellEnvironment.installPosixEnvironment")( function* ( config: ShellEnvironmentConfig, @@ -426,13 +464,36 @@ const installPosixEnvironment = Effect.fn("desktop.shellEnvironment.installPosix > { const fileSystem = yield* FileSystem.FileSystem; const shellEnvironment: EnvironmentPatch = {}; - const userShells = [config.env.SHELL?.trim(), readUserLoginShell()]; + const userShells = listUserShells(config.env); let answeredBy: string | undefined; + let waitedForSlowShell = false; for (const shell of listLoginShellCandidates(config.platform, config.env.SHELL)) { + const read = (timeout: Duration.Duration) => + readLoginShellEnvironment( + shell, + LOGIN_SHELL_ENV_NAMES, + config.env, + timeout, + config.budget.terminateGrace, + ); Object.assign( shellEnvironment, - yield* readLoginShellEnvironment(shell, LOGIN_SHELL_ENV_NAMES, config.env), + yield* read(config.budget.timeout).pipe( + Effect.catchTags({ + DesktopShellEnvironmentCommandTimeoutError: (error) => { + const slowTimeout = waitedForSlowShell ? undefined : config.budget.slowTimeout; + waitedForSlowShell = true; + return logShellEnvironmentCommandError(error).pipe( + Effect.andThen(slowTimeout ? read(slowTimeout) : Effect.succeed({})), + ); + }, + }), + Effect.catchTags({ + DesktopShellEnvironmentCommandTimeoutError: (error) => + logShellEnvironmentCommandError(error).pipe(Effect.as({})), + }), + ), ); if (shellEnvironment.PATH) { answeredBy = shell; @@ -445,7 +506,7 @@ const installPosixEnvironment = Effect.fn("desktop.shellEnvironment.installPosix ? yield* readLaunchctlPath(config.env) : Option.none(); const mergedPath = mergePaths(config.platform, [ - trimNonEmpty(shellEnvironment.PATH).pipe(Option.orElse(() => launchctlPath)), + nonEmptyPath(shellEnvironment.PATH).pipe(Option.orElse(() => launchctlPath)), readEnvPath(config.env), ]); @@ -492,6 +553,17 @@ const installPosixEnvironment = Effect.fn("desktop.shellEnvironment.installPosix } } + // The device host picks the Android SDK from ANDROID_HOME before ANDROID_SDK_ROOT, + // so a profile-only ANDROID_HOME must not shadow an inherited ANDROID_SDK_ROOT. + if (ANDROID_SDK_ENV_NAMES.every((name) => Option.isNone(trimNonEmpty(config.env[name])))) { + for (const name of ANDROID_SDK_ENV_NAMES) { + const value = trimNonEmpty(shellEnvironment[name]); + if (Option.isSome(value)) { + config.env[name] = value.value; + } + } + } + // Hydrate locale variables together; inherited LC_ALL can override LANG or LC_CTYPE. if ( config.platform === "darwin" && @@ -527,7 +599,10 @@ const installPosixEnvironment = Effect.fn("desktop.shellEnvironment.installPosix } const prepared = Option.isSome(trimNonEmpty(shellEnvironment.PATH)); - return { prepared, fromUserShell: prepared && userShells.includes(answeredBy) }; + return { + prepared, + fromUserShell: prepared && answeredBy !== undefined && userShells.includes(answeredBy), + }; }, ); @@ -611,38 +686,60 @@ export const make = Effect.gen(function* () { yield* Ref.set(latest, Option.some(message)); yield* PubSub.publish(changes, message); }); - const refresh = Effect.gen(function* () { - // Refresh from the inherited baseline so removed cached paths and values do not survive. - const fresh = { ...inherited }; - const discovery = yield* installShellEnvironment({ - env: fresh, - platform: environment.platform, - }); - const { prepared } = discovery; - const patch = Object.fromEntries( - Object.keys(ShellEnvironmentPatch.fields).map((name) => [name, fresh[name] ?? null]), - ); - if (prepared || Option.isNone(yield* Ref.get(latest))) yield* publish(patch, prepared); - if (prepared) { - const cached = Object.fromEntries( - CACHED_ENV_NAMES.flatMap((name) => (fresh[name] ? [[name, fresh[name]]] : [])), - ); - const persist = Effect.gen(function* () { - const encoded = yield* encodeCache({ version: 1, key, environment: cached }); - yield* fileSystem.makeDirectory(environment.stateDir, { recursive: true }); - yield* fileSystem.writeFileString(cachePath + ".tmp", encoded, { mode: 0o600 }); - yield* fileSystem.rename(cachePath + ".tmp", cachePath); + const probe = Effect.fn("desktop.shellEnvironment.refresh")( + function* (budget: ProbeBudget) { + // Refresh from the inherited baseline so removed cached paths and values do not survive. + const fresh = { ...inherited }; + const discovery = yield* installShellEnvironment({ + env: fresh, + platform: environment.platform, + budget, }); - yield* persist.pipe( - Effect.catch(() => Effect.logWarning("Could not cache shell environment")), + const { prepared } = discovery; + const patch = Object.fromEntries( + Object.keys(ShellEnvironmentPatch.fields).map((name) => [name, fresh[name] ?? null]), ); - } - return discovery; - }).pipe( + if (prepared || Option.isNone(yield* Ref.get(latest))) yield* publish(patch, prepared); + if (prepared) { + const cached = Object.fromEntries( + CACHED_ENV_NAMES.flatMap((name) => (fresh[name] ? [[name, fresh[name]]] : [])), + ); + const persist = Effect.gen(function* () { + const encoded = yield* encodeCache({ version: 1, key, environment: cached }); + yield* fileSystem.makeDirectory(environment.stateDir, { recursive: true }); + yield* fileSystem.writeFileString(cachePath + ".tmp", encoded, { mode: 0o600 }); + yield* fileSystem.rename(cachePath + ".tmp", cachePath); + }); + yield* persist.pipe( + Effect.catch(() => Effect.logWarning("Could not cache shell environment")), + ); + } + return discovery; + }, Effect.provideService(FileSystem.FileSystem, fileSystem), Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), - Effect.withSpan("desktop.shellEnvironment.refresh"), + ); + // A fallback shell's PATH can miss folders the user's own shell adds, so keep asking that shell. + const awaitsUserShell = listUserShells(inherited).length > 0; + const settled = ({ prepared, fromUserShell }: ShellDiscovery) => + fromUserShell || (prepared && !awaitsUserShell); + const refresh = probe(STARTUP_PROBE).pipe( Effect.onExit((exit) => Deferred.done(discovered, exit)), + Effect.flatMap((discovery) => + settled(discovery) + ? Effect.void + : Effect.sleep(DISCOVERY_RETRY_DELAY).pipe( + Effect.andThen( + probe(RETRY_PROBE).pipe( + Effect.repeat({ + until: settled, + schedule: Schedule.spaced(DISCOVERY_RETRY_DELAY), + times: DISCOVERY_RETRIES - 1, + }), + ), + ), + ), + ), ); const installIntoProcess = yield* Effect.cached( Effect.gen(function* () { diff --git a/apps/desktop/src/snapShot/PortalCaptureShortcut.test.ts b/apps/desktop/src/snapShot/PortalCaptureShortcut.test.ts index 35c007e35dc0..1a85c9a4cd89 100644 --- a/apps/desktop/src/snapShot/PortalCaptureShortcut.test.ts +++ b/apps/desktop/src/snapShot/PortalCaptureShortcut.test.ts @@ -30,6 +30,7 @@ class FakeBus extends NodeEvents.EventEmitter { requestPath = ""; boundId = ""; foreignHandle = false; + owner = ":1.2"; send(message: Message) { this.sends.push(message); @@ -39,7 +40,7 @@ class FakeBus extends NodeEvents.EventEmitter { member: string, path: string, body: unknown[], - sender = ":1.2", + sender = this.owner, signature = "", ) { this.emit( @@ -71,7 +72,7 @@ class FakeBus extends NodeEvents.EventEmitter { ]), }); } - activate(id = this.boundId, session = this.session, sender = ":1.2") { + activate(id = this.boundId, session = this.session, sender = this.owner) { this.signal(portal, "Activated", root, [session, id, 1, {}], sender, "osta{sv}"); } async call(message: Message) { @@ -87,7 +88,7 @@ class FakeBus extends NodeEvents.EventEmitter { if (this.registryError) throw this.registryError; return reply(); } - if (message.member === "GetNameOwner") return reply([":1.2"]); + if (message.member === "GetNameOwner") return reply([this.owner]); if (message.member === "Get") return reply([new Variant("u", this.version)]); if (message.member === "AddMatch" || message.member === "ConfigureShortcuts") return reply(); const options = message.body.at(-1) as Record>; @@ -338,7 +339,30 @@ it("handles a disappearing portal without keeping a false registered state", asy ); expect(client.state.shortcutRegistered).toBe(false); expect(client.state.shortcutMessage).toContain("restarted"); - expect(bus.disconnect).toHaveBeenCalledOnce(); + expect(client.hasSession).toBe(false); + expect(bus.disconnect).not.toHaveBeenCalled(); + await vi.waitFor(() => expect(client.state.shortcutRegistered).toBe(true)); + expect(bus.calls.filter((call) => call.member === "CreateSession")).toHaveLength(2); +}); + +it("binds a new session when the portal restarts", async () => { + const { bus, client, capture } = start(new FakeBus(), chord, true); + await client.ready; + const ownerChanged = (from: string, to: string) => + bus.signal( + "org.freedesktop.DBus", + "NameOwnerChanged", + "/org/freedesktop/DBus", + ["org.freedesktop.portal.Desktop", from, to], + "org.freedesktop.DBus", + ); + ownerChanged(":1.2", ""); + bus.owner = ":1.3"; + ownerChanged("", ":1.3"); + await vi.waitFor(() => expect(client.state.shortcutActionRegistered).toBe(true)); + expect(bus.calls.filter((call) => call.member === "CreateSession")).toHaveLength(2); + bus.activate(); + expect(capture).toHaveBeenCalledOnce(); }); it("accepts old portals without Registry but does not ignore permission denial", async () => { diff --git a/apps/desktop/src/snapShot/PortalCaptureShortcut.ts b/apps/desktop/src/snapShot/PortalCaptureShortcut.ts index ec42af46a388..8d44ec86d4fb 100644 --- a/apps/desktop/src/snapShot/PortalCaptureShortcut.ts +++ b/apps/desktop/src/snapShot/PortalCaptureShortcut.ts @@ -67,6 +67,7 @@ export class PortalCaptureShortcut { }; readonly ready: Promise; private closed = false; + private connection = 0; private owner = ""; private namespace = ""; private session = ""; @@ -74,8 +75,10 @@ export class PortalCaptureShortcut { private version = 0; private pending: { path: string; resolve: (body: unknown) => void } | undefined; private responses = new Map(); - private readonly stopped: Promise; + private stopped!: Promise; private rejectStopped!: (reason: Error) => void; + private readonly appId: string; + private readonly trigger: string; private readonly onCapture: () => void; private readonly onStateChanged: () => void; private readonly bus: MessageBus; @@ -89,6 +92,8 @@ export class PortalCaptureShortcut { bus: MessageBus = sessionBus(), managedByHyprland = false, ) { + this.appId = appId; + this.trigger = portalShortcutTrigger(shortcut); this.onCapture = onCapture; this.onStateChanged = onStateChanged; this.bus = bus; @@ -99,13 +104,41 @@ export class PortalCaptureShortcut { shortcutPending: true, shortcutMessage: "Connecting to Hyprland shortcuts…", }; + this.resetStopped(); + bus.on("error", this.failed); + bus.on("message", this.message); + this.ready = this.connect(true); + } + + private resetStopped() { this.stopped = new Promise((_, reject) => { this.rejectStopped = reject; }); void this.stopped.catch(() => undefined); - bus.on("error", this.failed); - bus.on("message", this.message); - this.ready = this.initialize(appId, shortcut).catch(this.failed); + } + + private connect(first = false) { + const connection = ++this.connection; + return this.initialize(first).catch((error: unknown) => { + if (connection === this.connection) this.failed(error); + }); + } + + private portalGone() { + this.rejectStopped(new Error("The desktop shortcut service restarted.")); + this.resetStopped(); + this.owner = ""; + this.session = ""; + this.pending = undefined; + this.responses.clear(); + this.update({ + shortcutRegistered: false, + shortcutActionRegistered: false, + shortcutPending: true, + shortcutMessage: "The desktop shortcut service restarted. Reconnecting…", + }); + // Register is sent to the well-known name, so it D-Bus-activates a portal that is not running. + void this.connect(); } close = () => { @@ -209,10 +242,9 @@ export class PortalCaptureShortcut { message.sender === "org.freedesktop.DBus" && message.interface === DBUS && message.member === "NameOwnerChanged" && - message.body[0] === PORTAL && - message.body[1] === this.owner + message.body[0] === PORTAL ) { - this.failed(new Error("The desktop shortcut service restarted. Retry the shortcut request.")); + if (this.owner && message.body[1] === this.owner) this.portalGone(); return; } if (!this.owner || message.sender !== this.owner) return; @@ -270,7 +302,8 @@ export class PortalCaptureShortcut { const response = new Promise((done) => { resolve = done; }); - this.pending = { path: expectedPath, resolve }; + const pending = { path: expectedPath, resolve }; + this.pending = pending; this.responses.clear(); let completed = false; try { @@ -284,7 +317,7 @@ export class PortalCaptureShortcut { }); const handle = string(reply.body[0]); if (!handle.startsWith(this.namespace)) throw new Error("Invalid shortcut request handle."); - this.pending.path = handle; + pending.path = handle; if (this.responses.has(handle)) resolve(this.responses.get(handle)); const [status, results] = decodeResponse( await this.wait(response, member === "BindShortcuts" ? 120_000 : 5_000), @@ -306,9 +339,11 @@ export class PortalCaptureShortcut { } return results; } finally { - if (!completed && !this.closed && this.pending) this.closeObject(this.pending.path, REQUEST); - this.pending = undefined; - this.responses.clear(); + if (this.pending === pending) { + if (!completed && !this.closed) this.closeObject(pending.path, REQUEST); + this.pending = undefined; + this.responses.clear(); + } } } @@ -338,8 +373,8 @@ export class PortalCaptureShortcut { }); } - private async initialize(appId: string, shortcut: SnapShotKeyChord) { - const trigger = portalShortcutTrigger(shortcut); + private async initialize(first: boolean) { + const trigger = this.trigger; if (!process.env.FLATPAK_ID && !process.env.SNAP) { await this.call({ destination: PORTAL, @@ -347,7 +382,7 @@ export class PortalCaptureShortcut { interface: "org.freedesktop.host.portal.Registry", member: "Register", signature: "sa{sv}", - body: [appId, {}], + body: [this.appId, {}], }).catch((error: unknown) => { if ( !( @@ -381,7 +416,11 @@ export class PortalCaptureShortcut { this.version = decodeVersion(version.body[0]).value; for (const rule of [ `type='signal',sender='${this.owner}',path_namespace='${PATH}'`, - `type='signal',sender='org.freedesktop.DBus',interface='${DBUS}',member='NameOwnerChanged',arg0='${PORTAL}'`, + ...(first + ? [ + `type='signal',sender='org.freedesktop.DBus',interface='${DBUS}',member='NameOwnerChanged',arg0='${PORTAL}'`, + ] + : []), ]) await this.call({ destination: DBUS, diff --git a/apps/desktop/src/ssh/DesktopSshEnvironment.test.ts b/apps/desktop/src/ssh/DesktopSshEnvironment.test.ts index 6dbcdde913b6..1363b4e3a2b0 100644 --- a/apps/desktop/src/ssh/DesktopSshEnvironment.test.ts +++ b/apps/desktop/src/ssh/DesktopSshEnvironment.test.ts @@ -137,7 +137,7 @@ describe("sshEnvironment", () => { alias: "bastion.example.com", hostname: "bastion.example.com", username: null, - port: null, + port: 2222, source: "known-hosts", }, { diff --git a/apps/desktop/src/ssh/DesktopSshEnvironment.ts b/apps/desktop/src/ssh/DesktopSshEnvironment.ts index 9aeb5c6275c8..67ad2cf098e0 100644 --- a/apps/desktop/src/ssh/DesktopSshEnvironment.ts +++ b/apps/desktop/src/ssh/DesktopSshEnvironment.ts @@ -23,7 +23,7 @@ import { import * as SshTunnel from "@t3tools/ssh/tunnel"; import * as Context from "effect/Context"; import type * as Digest from "effect/crypto/Digest"; -import type * as SecureRandom from "effect/crypto/SecureRandom"; +import * as SecureRandom from "effect/crypto/SecureRandom"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; @@ -70,14 +70,37 @@ export class DesktopSshEnvironment extends Context.Service< readonly disconnectEnvironment: ( target: DesktopSshEnvironmentTarget, ) => Effect.Effect; + readonly closeAllTunnels: Effect.Effect; } >()("@t3tools/desktop/ssh/DesktopSshEnvironment") {} export interface DesktopSshEnvironmentLayerOptions { readonly resolveCliRunner?: Effect.Effect; readonly resolveSshRunner?: Effect.Effect; + readonly resolveLauncherId?: Effect.Effect; } +// Remote servers record this ID as their owner, so it has to outlive restarts, +// updates and edits to a saved connection. Without a stored one there is none. +export const readOrCreateLauncherId = Effect.fn("desktop.ssh.readOrCreateLauncherId")( + function* (filePath: string) { + const fileSystem = yield* FileSystem.FileSystem; + const random = yield* SecureRandom.SecureRandom; + // Any other read failure may hide a stored ID, which a new one would overwrite. + const stored = yield* fileSystem.readFileString(filePath).pipe( + Effect.catchIf( + (cause) => cause.reason._tag === "NotFound", + () => Effect.succeed(""), + ), + ); + if (/^[0-9a-f]{32}$/u.test(stored.trim())) return stored.trim(); + const launcherId = (yield* random.uuidV4()).replace(/-/g, ""); + yield* fileSystem.writeFileString(filePath, `${launcherId}\n`); + return launcherId; + }, + Effect.orElseSucceed(() => null), +); + function discoverDesktopSshHostsEffect(input?: { readonly homeDir?: string }) { return discoverSshHosts(input ?? {}); } @@ -216,16 +239,20 @@ export const make = Effect.fn("DesktopSshEnvironment.make")(function* ( Effect.withSpan("desktop.ssh.disconnectEnvironment"), ); }), + closeAllTunnels: manager.closeAllTunnels, }); }); export const layer = (options: DesktopSshEnvironmentLayerOptions = {}) => Layer.effect(DesktopSshEnvironment, make(options)).pipe( Layer.provide( - SshTunnel.SshEnvironmentManager.layer( - options.resolveCliRunner === undefined + SshTunnel.SshEnvironmentManager.layer({ + ...(options.resolveCliRunner === undefined ? {} - : { resolveCliRunner: options.resolveCliRunner }, - ), + : { resolveCliRunner: options.resolveCliRunner }), + ...(options.resolveLauncherId === undefined + ? {} + : { resolveLauncherId: options.resolveLauncherId }), + }), ), ); diff --git a/apps/desktop/src/ssh/DesktopSshPasswordPrompts.test.ts b/apps/desktop/src/ssh/DesktopSshPasswordPrompts.test.ts index 1e5b6dafc6ca..15977b34f398 100644 --- a/apps/desktop/src/ssh/DesktopSshPasswordPrompts.test.ts +++ b/apps/desktop/src/ssh/DesktopSshPasswordPrompts.test.ts @@ -144,7 +144,7 @@ describe("DesktopSshPasswordPrompts", () => { yield* prompts.resolve({ requestId: request.requestId, password: "secret" }); assert.equal(yield* Fiber.join(fiber), "secret"); - }).pipe(Effect.provide(layer(testWindow.window)), Effect.scoped); + }).pipe(Effect.provide(layer(testWindow.window))); }); it.effect.each(["answered", "cancelled", "timed out", "interrupted"] as const)( @@ -189,7 +189,7 @@ describe("DesktopSshPasswordPrompts", () => { .map((message) => message.args), [[request.requestId]], ); - }).pipe(Effect.provide(layer(testWindow.window)), Effect.scoped); + }).pipe(Effect.provide(layer(testWindow.window))); }, ); @@ -219,6 +219,6 @@ describe("DesktopSshPasswordPrompts", () => { assert.equal(password, "secret"); assert.equal(testWindow.isFocused(), false); assert.equal(testWindow.closedListenerCount(), 0); - }).pipe(Effect.provide(layer(testWindow.window)), Effect.scoped); + }).pipe(Effect.provide(layer(testWindow.window))); }); }); diff --git a/apps/desktop/src/telemetry/DesktopTelemetryPublisher.ts b/apps/desktop/src/telemetry/DesktopTelemetryPublisher.ts index 0a2b10e08c5a..415901878caf 100644 --- a/apps/desktop/src/telemetry/DesktopTelemetryPublisher.ts +++ b/apps/desktop/src/telemetry/DesktopTelemetryPublisher.ts @@ -352,6 +352,8 @@ export const make = Effect.fn("desktop.telemetryPublisher.make")(function* () { return Queue.offer(updateCommitQueue, message).pipe(Effect.asVoid); case "cancelDesktopUpdate": return Queue.offer(updateCancellationQueue, message).pipe(Effect.asVoid); + case "serverReady": + return Effect.void; } }; const removeControlSource: DesktopTelemetryPublisher["Service"]["removeControlSource"] = ( diff --git a/apps/desktop/src/updates/DesktopRemoteUpdates.test.ts b/apps/desktop/src/updates/DesktopRemoteUpdates.test.ts index 1f5321148b77..fdac65f86a59 100644 --- a/apps/desktop/src/updates/DesktopRemoteUpdates.test.ts +++ b/apps/desktop/src/updates/DesktopRemoteUpdates.test.ts @@ -102,6 +102,29 @@ describe("DesktopRemoteUpdates", () => { assert.equal(terminals[0]?.requestId, "req-1"); assert.equal(harness.quitAndInstalls(), 0); + yield* Queue.offer(requests, request("req-retry")); + yield* settle; + assert.deepEqual( + terminalReports(reports).map((report) => report.outcome), + ["ready-to-install", "ready-to-install"], + ); + assert.equal(terminalReports(reports)[1]?.requestId, "req-retry"); + assert.equal(terminalReports(reports)[1]?.preparedRequestId, "req-1"); + assert.equal(harness.quitAndInstalls(), 0); + assert.equal(harness.downloadCount(), 1); + + yield* Queue.offer(requests, { + ...request("req-other-track"), + channel: "nightly", + targetVersion: "1.2.4-nightly.20261027.1", + }); + yield* settle; + assert.equal(terminalReports(reports)[2]?.outcome, "failed"); + assert.equal( + terminalReports(reports)[2]?.reason, + "A prepared desktop update is already in progress.", + ); + yield* Queue.offer(commits, { version: 1, type: "commitDesktopUpdate", @@ -119,6 +142,49 @@ describe("DesktopRemoteUpdates", () => { ); }); + it.effect("does not renew the preparation lease when a retry recovers its token", () => { + const harness = makeHarness(); + + return runRemoteUpdatesTest(harness, ({ reports, requests, commits }) => + Effect.gen(function* () { + harness.emit("update-downloaded", { version: "1.2.4" }); + yield* settle; + yield* Queue.offer(requests, request("req-original")); + yield* settle; + + yield* TestClock.adjust(Duration.minutes(4)); + harness.emit("update-available", { version: "1.2.4" }); + yield* settle; + yield* Queue.offer(requests, request("req-retry")); + yield* settle; + assert.equal(terminalReports(reports)[1]?.preparedRequestId, "req-original"); + + yield* TestClock.adjust(Duration.minutes(1)); + yield* Queue.offer(commits, { + version: 1, + type: "commitDesktopUpdate", + requestId: "req-original", + }); + yield* settle; + assert.equal( + terminalReports(reports)[2]?.reason, + "This desktop update is no longer prepared.", + ); + assert.equal(harness.quitAndInstalls(), 0); + + yield* Queue.offer(requests, request("req-new")); + yield* settle; + yield* Queue.offer(commits, { + version: 1, + type: "commitDesktopUpdate", + requestId: "req-new", + }); + yield* settle; + assert.equal(harness.quitAndInstalls(), 1); + }), + ); + }); + it.effect("switches to the requested track before preparing its release", () => { // In-memory settings keep the channel write off real disk I/O. const harness = makeHarness({ beforeSetUpdateChannel: Effect.void }); @@ -691,7 +757,7 @@ describe("DesktopRemoteUpdates", () => { const backgroundCheck = yield* updates.check("poll").pipe(Effect.forkChild); yield* Deferred.await(backgroundCheckStarted); - assert.equal((yield* updates.getState).status, "checking"); + assert.isTrue(yield* updates.isActionActive); yield* Queue.offer(commits, { version: 1, @@ -701,7 +767,6 @@ describe("DesktopRemoteUpdates", () => { yield* settle; assert.equal(harness.quitAndInstalls(), 0); - harness.emit("update-downloaded", { version: "1.2.4" }); yield* Deferred.succeed(releaseBackgroundCheck, undefined); yield* Fiber.join(backgroundCheck); yield* settle; diff --git a/apps/desktop/src/updates/DesktopRemoteUpdates.ts b/apps/desktop/src/updates/DesktopRemoteUpdates.ts index 560e2e78d4bb..ce2f23ab3e78 100644 --- a/apps/desktop/src/updates/DesktopRemoteUpdates.ts +++ b/apps/desktop/src/updates/DesktopRemoteUpdates.ts @@ -32,6 +32,8 @@ const PREPARED_UPDATE_TTL = Duration.minutes(5); interface PreparedUpdate { readonly requestId: string; + readonly channel: DesktopTelemetryRequestDesktopUpdate["channel"]; + readonly targetVersion: DesktopTelemetryRequestDesktopUpdate["targetVersion"]; readonly downloadedVersion: string; readonly status: "prepared" | "committing" | "failed"; readonly failureReason?: string; @@ -67,7 +69,11 @@ export const listen: Effect.Effect< const publishReport = ( state: DesktopUpdateState, - terminal?: { readonly outcome: DesktopUpdateRemoteOutcome; readonly reason?: string }, + terminal?: { + readonly outcome: DesktopUpdateRemoteOutcome; + readonly reason?: string; + readonly preparedRequestId?: string; + }, explicitRequestId?: string, ): Effect.Effect => { const reason = normalizeRemoteUpdateReason(terminal?.reason); @@ -82,6 +88,9 @@ export const listen: Effect.Effect< ? { requestId: requestId.value } : {}), ...(terminal === undefined ? {} : { outcome: terminal.outcome }), + ...(terminal?.preparedRequestId === undefined + ? {} + : { preparedRequestId: terminal.preparedRequestId }), ...(reason ? { reason } : {}), state, }), @@ -167,8 +176,22 @@ export const listen: Effect.Effect< : ([current, current] as const), ); if (Option.isSome(prepared) && prepared.value.status !== "failed") { + const state = yield* updates.getState; + if ( + prepared.value.status === "prepared" && + prepared.value.channel === request.channel && + prepared.value.targetVersion === request.targetVersion && + state.downloadedVersion === prepared.value.downloadedVersion + ) { + yield* publishReport( + state, + { outcome: "ready-to-install", preparedRequestId: prepared.value.requestId }, + request.requestId, + ); + return; + } yield* publishReport( - yield* updates.getState, + state, { outcome: "failed", reason: "A prepared desktop update is already in progress.", @@ -301,6 +324,8 @@ export const listen: Effect.Effect< preparedUpdateRef, Option.some({ requestId: request.requestId, + channel: request.channel, + targetVersion: request.targetVersion, downloadedVersion: state.downloadedVersion, status: "prepared", expiresAt: diff --git a/apps/desktop/src/updates/DesktopUpdates.test.ts b/apps/desktop/src/updates/DesktopUpdates.test.ts index 68002f826d42..53bd956ca6da 100644 --- a/apps/desktop/src/updates/DesktopUpdates.test.ts +++ b/apps/desktop/src/updates/DesktopUpdates.test.ts @@ -1,7 +1,7 @@ import * as NodeModule from "node:module"; import { assert, describe, it } from "@effect/vitest"; -import { DESKTOP_UPDATE_RESTART_MARKER_FILE } from "@t3tools/contracts"; +import { DESKTOP_UPDATE_RESTART_MARKER_FILE, DesktopUpdateStateSchema } from "@t3tools/contracts"; import { computeReleaseNotes } from "electron-updater/out/providers/GitHubProvider.js"; import * as Cause from "effect/Cause"; import * as Deferred from "effect/Deferred"; @@ -13,6 +13,7 @@ import * as Logger from "effect/Logger"; import * as Option from "effect/Option"; import * as References from "effect/References"; import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; @@ -25,6 +26,8 @@ import * as DesktopUpdatePolicy from "./DesktopUpdatePolicy.ts"; import * as DesktopUpdates from "./DesktopUpdates.ts"; import { flushCallbacks, makeHarness } from "./updatesTestHarness.ts"; +const encodeUpdateState = Schema.encodeEffect(DesktopUpdateStateSchema); + type ReleaseFeed = Parameters[2]; type RunningVersion = Parameters[0]; @@ -125,7 +128,7 @@ describe("DesktopUpdates", () => { ).pipe(Effect.provide(harness.layer)); }); - it.effect("checks for newer releases after an update has been downloaded", () => { + it.effect("re-checks an offered update without announcing it again", () => { const harness = makeHarness(); return Effect.scoped( @@ -138,14 +141,23 @@ describe("DesktopUpdates", () => { releaseNotes: "## What's changed\n- fix: queued update", }); yield* flushCallbacks; + const offeredStates = harness.sentStates.length; + + yield* updates.check("poll"); + harness.emit("update-available", { version: "1.2.4" }); + yield* flushCallbacks; + assert.equal(harness.sentStates.length, offeredStates); + harness.emit("update-downloaded", { version: "1.2.4" }); yield* flushCallbacks; + const downloadedStates = harness.sentStates.length; const result = yield* updates.check("poll"); assert.isTrue(result.checked); harness.emit("update-available", { version: "1.2.4" }); yield* flushCallbacks; + assert.equal(harness.sentStates.length, downloadedStates); const unchangedState = yield* updates.getState; assert.equal(unchangedState.status, "downloaded"); @@ -247,15 +259,46 @@ describe("DesktopUpdates", () => { ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); }); - it.effect( - "rejects install while a refresh check is in progress and releases the reservation", - () => + it.effect("checks shortly after startup and then every four hours", () => { + const harness = makeHarness(); + + return Effect.scoped( + Effect.gen(function* () { + const updates = yield* DesktopUpdates.DesktopUpdates; + yield* updates.configure; + + yield* TestClock.adjust("15 seconds"); + assert.equal(harness.checkCount(), 1); + + yield* TestClock.adjust("3 hours"); + assert.equal(harness.checkCount(), 1); + + yield* TestClock.adjust("1 hour"); + assert.equal(harness.checkCount(), 2); + }), + ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); + }); + + it.effect.each([ + { action: "download", offer: "update-available", found: null, accepted: true }, + { action: "install", offer: "update-downloaded", found: null, accepted: true }, + { action: "download", offer: "update-available", found: "1.2.5", accepted: true }, + { action: "install", offer: "update-downloaded", found: "1.2.5", accepted: false }, + ] as const)( + "a $action waiting out a refresh check acts on what it found ($found)", + ({ action, offer, found, accepted }) => Effect.gen(function* () { const checkStarted = yield* Deferred.make(); const releaseCheck = yield* Deferred.make(); const harness = makeHarness({ checkForUpdates: Deferred.succeed(checkStarted, undefined).pipe( Effect.andThen(Deferred.await(releaseCheck)), + // electron-updater emits the check's outcome before its promise resolves. + Effect.andThen( + Effect.sync(() => { + if (found !== null) harness.emit("update-available", { version: found }); + }), + ), ), }); @@ -263,22 +306,20 @@ describe("DesktopUpdates", () => { Effect.gen(function* () { const updates = yield* DesktopUpdates.DesktopUpdates; yield* updates.configure; - harness.emit("update-downloaded", { version: "1.2.4" }); + harness.emit(offer, { version: "1.2.4" }); yield* flushCallbacks; - const checkFiber = yield* updates.check("manual").pipe(Effect.forkScoped); + const checkFiber = yield* updates.check("poll").pipe(Effect.forkScoped); yield* Deferred.await(checkStarted); - - const installResult = yield* updates.install; - assert.isFalse(installResult.accepted); + const actionFiber = yield* updates[action].pipe(Effect.forkScoped); + yield* flushCallbacks; yield* Deferred.succeed(releaseCheck, undefined); - const checkResult = yield* Fiber.join(checkFiber); - assert.isTrue(checkResult.checked); - - const followUpCheck = yield* updates.check("manual"); - assert.isTrue(followUpCheck.checked); - assert.equal(harness.checkCount(), 2); + assert.isTrue((yield* Fiber.join(checkFiber)).checked); + assert.equal((yield* Fiber.join(actionFiber)).accepted, accepted); + const state = yield* updates.getState; + assert.equal(state.availableVersion, found ?? "1.2.4"); + if (action === "download") assert.equal(state.status, "downloading"); }), ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); }), @@ -524,14 +565,17 @@ describe("DesktopUpdates", () => { it.effect("marks the backend stop for an install as an update restart", () => { let markersAtStop: ReadonlyArray = []; + let inProgressAtStop: string | undefined; const harness = makeHarness({ stopBackend: Effect.sync(() => { markersAtStop = [...harness.updateRestartMarkers]; + inProgressAtStop = harness.updateInProgressMarker(); }), }); return Effect.scoped( Effect.gen(function* () { + yield* TestClock.setTime(1_000); const environment = yield* DesktopEnvironment.DesktopEnvironment; const updates = yield* DesktopUpdates.DesktopUpdates; yield* updates.configure; @@ -542,6 +586,12 @@ describe("DesktopUpdates", () => { assert.deepEqual(markersAtStop, [ environment.path.join(environment.baseDir, "runtime", DESKTOP_UPDATE_RESTART_MARKER_FILE), ]); + // SSH launchers wait on this until the replacement backend publishes its runtime. + assert.deepEqual(JSON.parse(inProgressAtStop ?? "null"), { + startedAt: 1_000, + expiresAt: 61_000, + }); + assert.isDefined(harness.updateInProgressMarker()); }), ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); }); @@ -624,9 +674,11 @@ describe("DesktopUpdates", () => { const installFiber = yield* updates.install.pipe(Effect.forkScoped); yield* Deferred.await(stopping); assert.equal(harness.updateRestartMarkers.size, 1); + assert.isDefined(harness.updateInProgressMarker()); yield* Fiber.interrupt(installFiber); assert.equal(harness.updateRestartMarkers.size, 0); + assert.isUndefined(harness.updateInProgressMarker()); }), ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); }), @@ -655,8 +707,13 @@ describe("DesktopUpdates", () => { const result = yield* updates.install; assert.isTrue(result.accepted); assert.isFalse(yield* Ref.get(desktopState.quitting)); - assert.deepEqual(harness.installSteps, ["quitAndInstall", "startBackend"]); + assert.deepEqual(harness.installSteps, [ + "closeSshTunnels", + "quitAndInstall", + "startBackend", + ]); assert.equal(harness.updateRestartMarkers.size, 0); + assert.isUndefined(harness.updateInProgressMarker()); assert.equal(harness.installRecords.size, 0); }), ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); @@ -685,7 +742,7 @@ describe("DesktopUpdates", () => { const result = yield* updates.install; assert.equal(result.state.errorContext, "install"); - assert.deepEqual(harness.installSteps, ["quitAndInstall"]); + assert.deepEqual(harness.installSteps, ["closeSshTunnels", "quitAndInstall"]); assert.isFalse((yield* (yield* pool.primary).snapshot).desiredRunning); }), ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); @@ -779,7 +836,11 @@ describe("DesktopUpdates", () => { assert.equal(harness.quitAndInstalls(), 1); harness.emit("error", new Error("duplicate native installer error")); yield* flushCallbacks; - assert.deepEqual(harness.installSteps, ["quitAndInstall", "startBackend"]); + assert.deepEqual(harness.installSteps, [ + "closeSshTunnels", + "quitAndInstall", + "startBackend", + ]); yield* Deferred.succeed(releaseRecovery, undefined); const failedResult = yield* Fiber.join(failedInstall); @@ -1222,6 +1283,7 @@ describe("DesktopUpdates under a managed policy", () => { assert.equal(held.status, "disabled"); assert.include(held.message ?? "", message); assert.isTrue(held.channelBlockedByPolicy); + assert.isTrue((yield* encodeUpdateState(held)).channelBlockedByPolicy); assert.isFalse((yield* updates.check("menu")).checked); assert.equal(harness.checkCount(), 0); diff --git a/apps/desktop/src/updates/DesktopUpdates.ts b/apps/desktop/src/updates/DesktopUpdates.ts index 0f52ec249c2c..055c6574391a 100644 --- a/apps/desktop/src/updates/DesktopUpdates.ts +++ b/apps/desktop/src/updates/DesktopUpdates.ts @@ -1,5 +1,7 @@ import { DESKTOP_UPDATE_RESTART_MARKER_FILE, + DESKTOP_UPDATE_IN_PROGRESS_MARKER_FILE, + DESKTOP_UPDATE_IN_PROGRESS_TTL_MS, DesktopUpdateChannelSchema, type DesktopRuntimeInfo, type DesktopUpdateActionResult, @@ -35,6 +37,7 @@ import * as ElectronUpdater from "../electron/ElectronUpdater.ts"; import * as ElectronWindow from "../electron/ElectronWindow.ts"; import * as IpcChannels from "../ipc/channels.ts"; import * as DesktopAppSettings from "../settings/DesktopAppSettings.ts"; +import * as DesktopSshEnvironment from "../ssh/DesktopSshEnvironment.ts"; import * as DesktopUpdatePolicy from "./DesktopUpdatePolicy.ts"; import { normalizeDesktopUpdateReleaseNotes, @@ -59,8 +62,8 @@ import { const CHANNEL_CHANGE_RETRY_DELAY = Duration.seconds(1); const CHANNEL_CHANGE_RETRIES = 30; const AUTO_UPDATE_STARTUP_DELAY = "15 seconds"; -const AUTO_UPDATE_POLL_INTERVAL = "4 minutes"; -const PREPARED_INSTALL_CHECK_WAIT = Duration.seconds(90); +const AUTO_UPDATE_POLL_INTERVAL = "4 hours"; +const UPDATE_CHECK_WAIT = Duration.seconds(90); type UpdateAction = "check" | "download" | "install" | "install-recovery" | "channel"; @@ -315,6 +318,10 @@ function getCanRetryFromState(state: DesktopUpdateState): boolean { return state.availableVersion !== null || state.downloadedVersion !== null; } +function isOffered(state: DesktopUpdateState): boolean { + return state.status === "available" || state.status === "downloaded"; +} + function shouldBroadcastDownloadProgress( currentState: DesktopUpdateState, nextPercent: number, @@ -369,6 +376,7 @@ function isArm64HostRunningIntelBuild(runtimeInfo: DesktopRuntimeInfo): boolean export const make = Effect.gen(function* () { const config = yield* DesktopConfig.DesktopConfig; const pool = yield* DesktopBackendPool.DesktopBackendPool; + const sshEnvironment = yield* DesktopSshEnvironment.DesktopSshEnvironment; const desktopState = yield* DesktopState.DesktopState; const electronUpdater = yield* ElectronUpdater.ElectronUpdater; const electronWindow = yield* ElectronWindow.ElectronWindow; @@ -514,6 +522,23 @@ export const make = Effect.gen(function* () { ), ); + // Re-runs admit after each finished action, so a caller waiting on a check decides on its result. + const admitAfterCheck = Effect.fn("desktop.updates.admitAfterCheck")(function* ( + admit: Effect.Effect<"admitted" | "refused" | "wait-for-check">, + ) { + const actionCompletions = yield* PubSub.subscribe(finishedUpdateActions); + let admission = yield* stateMutex.withPermits(1)(admit); + while (admission === "wait-for-check") { + const finishedAction = yield* PubSub.take(actionCompletions).pipe( + Effect.timeoutOption(UPDATE_CHECK_WAIT), + ); + admission = Option.isNone(finishedAction) + ? "refused" + : yield* stateMutex.withPermits(1)(admit); + } + return admission; + }); + const applyAutoUpdaterChannel = Effect.fn("desktop.updates.applyAutoUpdaterChannel")(function* ( channel: DesktopUpdateChannel, ) { @@ -555,8 +580,10 @@ export const make = Effect.gen(function* () { if (actionReservation === "acquire" && !(yield* tryStartUpdateAction("check"))) return false; const check = Effect.gen(function* () { - const checkedAt = yield* currentIsoTimestamp; - yield* setState(reduceDesktopUpdateStateOnCheckStart(state, checkedAt)); + // An offer stays on screen while re-checking, so finding it again changes nothing. + if (!isOffered(state)) { + yield* setState(reduceDesktopUpdateStateOnCheckStart(state, yield* currentIsoTimestamp)); + } yield* logUpdaterInfo("checking for updates", { reason }); // The flag is shared updater state, so it flips only while this check holds the reservation. @@ -593,15 +620,30 @@ export const make = Effect.gen(function* () { }); const downloadAvailableUpdate = Effect.gen(function* () { - const state = yield* Ref.get(updateStateRef); - if (!(yield* Ref.get(updaterConfiguredRef)) || state.status !== "available") { - return { accepted: false, completed: false }; - } - - if (!(yield* tryStartUpdateAction("download"))) { + const admission = yield* Effect.scoped( + admitAfterCheck( + Effect.gen(function* () { + const state = yield* Ref.get(updateStateRef); + if (!(yield* Ref.get(updaterConfiguredRef)) || state.status !== "available") { + return "refused" as const; + } + const activeAction = yield* Ref.get(activeUpdateActionRef); + if (Option.isSome(activeAction)) { + return activeAction.value === "check" + ? ("wait-for-check" as const) + : ("refused" as const); + } + return (yield* tryStartUpdateAction("download")) + ? ("admitted" as const) + : ("refused" as const); + }), + ), + ); + if (admission === "refused") { return { accepted: false, completed: false }; } + const state = yield* Ref.get(updateStateRef); return yield* Effect.gen(function* () { yield* setState(reduceDesktopUpdateStateOnDownloadStart(state)); yield* electronUpdater.setDisableDifferentialDownload( @@ -667,6 +709,34 @@ export const make = Effect.gen(function* () { ), ); + const updateInProgressMarkerPath = environment.path.join( + updateRestartMarkerDir, + DESKTOP_UPDATE_IN_PROGRESS_MARKER_FILE, + ); + const encodeUpdateMarker = Schema.encodeEffect( + Schema.fromJsonString(Schema.Struct({ startedAt: Schema.Number, expiresAt: Schema.Number })), + ); + const updateInProgressTempPath = `${updateInProgressMarkerPath}.tmp`; + const writeUpdateInProgressMarker = Effect.gen(function* () { + const startedAt = DateTime.toEpochMillis(yield* DateTime.now); + const contents = yield* encodeUpdateMarker({ + startedAt, + expiresAt: startedAt + DESKTOP_UPDATE_IN_PROGRESS_TTL_MS, + }); + yield* fileSystem.writeFileString(updateInProgressTempPath, contents); + yield* fileSystem.rename(updateInProgressTempPath, updateInProgressMarkerPath); + }).pipe( + Effect.ensuring( + fileSystem.remove(updateInProgressTempPath, { force: true }).pipe(Effect.ignore), + ), + Effect.catch((error) => + logUpdaterWarning("Could not write the update-in-progress marker.", { errorTag: error._tag }), + ), + ); + const removeUpdateInProgressMarker = fileSystem + .remove(updateInProgressMarkerPath, { force: true }) + .pipe(Effect.ignore); + // Failed or interrupted installs leave no replacement backend; later quits must release the tunnel. const removeUpdateRestartMarker = fileSystem .remove(updateRestartMarkerPath, { force: true }) @@ -722,6 +792,7 @@ export const make = Effect.gen(function* () { Ref.set(desktopState.quitting, false), Ref.set(installRunningBackendsRef, []), removeUpdateRestartMarker, + removeUpdateInProgressMarker, removeInstallRecord, ], { discard: true }, @@ -739,6 +810,7 @@ export const make = Effect.gen(function* () { yield* Ref.set(desktopState.quitting, false); yield* removeUpdateRestartMarker; + yield* removeUpdateInProgressMarker; yield* removeInstallRecord; yield* Effect.gen(function* () { const runningInstances = yield* Ref.getAndSet(installRunningBackendsRef, []); @@ -764,47 +836,35 @@ export const make = Effect.gen(function* () { const installDownloadedUpdate = (expectedVersion?: string) => Effect.scoped( Effect.gen(function* () { - const actionCompletions = yield* PubSub.subscribe(finishedUpdateActions); - let admission: "admitted" | "refused" | "wait-for-check" = "wait-for-check"; - while (admission === "wait-for-check") { - admission = yield* stateMutex.withPermits(1)( - Effect.gen(function* () { - const state = yield* Ref.get(updateStateRef); - const activeAction = yield* Ref.get(activeUpdateActionRef); - const hasExpectedDownload = - state.downloadedVersion !== null && - (expectedVersion === undefined || state.downloadedVersion === expectedVersion); - if ( - (yield* Ref.get(desktopState.quitting)) || - !(yield* Ref.get(updaterConfiguredRef)) || - !hasExpectedDownload - ) { - return "refused" as const; - } - if (Option.isSome(activeAction)) { - return activeAction.value === "check" && expectedVersion !== undefined - ? ("wait-for-check" as const) - : ("refused" as const); - } - const hasInstallableDownload = - state.status === "downloaded" || - (state.status === "error" && - (state.errorContext === null || state.errorContext === "install")); - if (!hasInstallableDownload) return "refused" as const; - return (yield* tryStartUpdateAction("install")) - ? ("admitted" as const) + const admission = yield* admitAfterCheck( + Effect.gen(function* () { + const state = yield* Ref.get(updateStateRef); + const activeAction = yield* Ref.get(activeUpdateActionRef); + const hasExpectedDownload = + state.downloadedVersion !== null && + (expectedVersion === undefined || state.downloadedVersion === expectedVersion); + if ( + (yield* Ref.get(desktopState.quitting)) || + !(yield* Ref.get(updaterConfiguredRef)) || + !hasExpectedDownload + ) { + return "refused" as const; + } + if (Option.isSome(activeAction)) { + return activeAction.value === "check" + ? ("wait-for-check" as const) : ("refused" as const); - }), - ); - if (admission === "wait-for-check") { - const finishedAction = yield* PubSub.take(actionCompletions).pipe( - Effect.timeoutOption(PREPARED_INSTALL_CHECK_WAIT), - ); - if (Option.isNone(finishedAction)) { - admission = "refused"; } - } - } + const hasInstallableDownload = + state.status === "downloaded" || + (state.status === "error" && + (state.errorContext === null || state.errorContext === "install")); + if (!hasInstallableDownload) return "refused" as const; + return (yield* tryStartUpdateAction("install")) + ? ("admitted" as const) + : ("refused" as const); + }), + ); if (admission === "refused") { return { accepted: false, completed: false, failed: false }; } @@ -814,19 +874,26 @@ export const make = Effect.gen(function* () { return yield* Effect.gen(function* () { yield* writeUpdateRestartMarker; + yield* writeUpdateInProgressMarker; const { downloadedVersion } = yield* Ref.get(updateStateRef); if (downloadedVersion !== null) yield* writeInstallRecord(downloadedVersion); - // Stop pooled backends before quitAndInstall can outrun their scope finalizers. + // Stop pooled backends and SSH tunnels before quitAndInstall can outrun their scope finalizers. const instances = yield* pool.list; // stop clears desiredRunning, so retain the recovery set beforehand. const runningInstances = yield* Effect.filter(instances, (instance) => instance.snapshot.pipe(Effect.map((snapshot) => snapshot.desiredRunning)), ); yield* Ref.set(installRunningBackendsRef, runningInstances); - yield* Effect.forEach( - instances, - (instance) => - instance.stop({ timeout: DesktopBackendManager.BACKEND_QUIT_STOP_TIMEOUT }), + yield* Effect.all( + [ + Effect.forEach( + instances, + (instance) => + instance.stop({ timeout: DesktopBackendManager.BACKEND_QUIT_STOP_TIMEOUT }), + { concurrency: "unbounded" }, + ), + sshEnvironment.closeAllTunnels, + ], { concurrency: "unbounded" }, ); yield* electronUpdater.quitAndInstall({ @@ -941,6 +1008,7 @@ export const make = Effect.gen(function* () { yield* Ref.set(lastLoggedDownloadMilestoneRef, -1); return; } + if (isOffered(state) && state.availableVersion === info.version) return; const checkedAt = yield* currentIsoTimestamp; const { releaseNotes, omittedReleaseCount } = normalizeDesktopUpdateReleaseNotes( @@ -1120,6 +1188,8 @@ export const make = Effect.gen(function* () { ), configure: Effect.gen(function* () { const context = yield* Effect.context(); + // Runs synchronously up to the first await, so a check's outcome is applied before the + // check's promise resolves and wakes the actions waiting on it. const runEffect = (effect: Effect.Effect) => { void Effect.runPromiseWith(context)(effect); }; diff --git a/apps/desktop/src/updates/githubUpdateFeed.test.ts b/apps/desktop/src/updates/githubUpdateFeed.test.ts new file mode 100644 index 000000000000..d9e928fce186 --- /dev/null +++ b/apps/desktop/src/updates/githubUpdateFeed.test.ts @@ -0,0 +1,107 @@ +import * as NodeModule from "node:module"; + +import { GitHubProvider } from "electron-updater/out/providers/GitHubProvider.js"; +import { describe, expect, it } from "vite-plus/test"; + +type ProviderArgs = ConstructorParameters; + +// electron-updater's own dependency, which apps/desktop does not declare. +const semver = NodeModule.createRequire( + NodeModule.createRequire(import.meta.url).resolve("electron-updater"), +)("semver") as { parse: (version: string) => ProviderArgs[1]["currentVersion"] }; + +const REPO = "/pingdotgg/t3code"; +const nightly = (build: number) => `v0.1.10-nightly.20261112.${build}`; + +const atomFeed = (tags: ReadonlyArray) => + `${tags + .map( + (tag) => + `${tag}${tag} feed notes`, + ) + .join("")}`; + +/** + * The release a nightly install on 4580 is offered when GitHub serves this feed, this `update-feed` + * pointer, and this newest nightly from the REST API scan. + */ +const offeredRelease = async ( + feedTags: ReadonlyArray, + pointerTag: string | undefined, + listedTag?: string | Error, +) => { + let apiScans = 0; + const request = async ({ path = "" }: { readonly path?: string | null }) => { + if (path === `${REPO}/releases.atom`) return atomFeed(feedTags); + if (path === `${REPO}/releases/download/update-feed/nightly.json`) { + if (pointerTag === undefined) throw new Error("404"); + return JSON.stringify({ tag_name: pointerTag, name: pointerTag, body: "pointer notes" }); + } + const manifest = /\/releases\/download\/v([^/]+)\/nightly\.yml$/.exec(path ?? ""); + if (manifest) return `version: ${manifest[1]}\npath: T3-Code.exe\nsha512: abc\n`; + apiScans += 1; + if (listedTag instanceof Error) throw listedTag; + return JSON.stringify(listedTag ? [{ tag_name: listedTag, body: "API notes" }] : []); + }; + const provider = new GitHubProvider( + { provider: "github", owner: "pingdotgg", repo: "t3code" }, + // Only the four fields the provider reads; a real AppUpdater needs Electron. + { + allowPrerelease: true, + channel: "nightly", + currentVersion: semver.parse("0.1.10-nightly.20261112.4580"), + fullChangelog: true, + } as ProviderArgs[1], + { + isUseMultipleRangeRequest: false, + platform: "win32", + executor: { request }, + } as ProviderArgs[2], + ); + const { tag, releaseNotes } = await provider.getLatestVersion(); + return { tag, releaseNotes, apiScans }; +}; + +// What filled releases.atom on 2026-11-12: ten tags, none of them a nightly. +const crowdedFeed = Array.from({ length: 10 }, (_, i) => `v0.1.10-preview.20261112.${4550 + i}`); + +describe("GitHubProvider nightly lookup", () => { + it.each([ + ["other tags fill the feed", crowdedFeed], + ["the feed's first nightly is an older one", [nightly(4603)]], + ])("finds the newest nightly at the fixed pointer when %s", async (_case, feedTags) => { + expect(await offeredRelease(feedTags, nightly(4606))).toEqual({ + tag: nightly(4606), + releaseNotes: "pointer notes", + apiScans: 0, + }); + }); + + it("still scans the API when the feed has no nightly and the pointer offers nothing new", async () => { + expect(await offeredRelease(crowdedFeed, nightly(4580), nightly(4606))).toEqual({ + tag: nightly(4606), + releaseNotes: "API notes", + apiScans: 1, + }); + }); + + it("keeps the pointer's answer when the API scan fails", async () => { + const rateLimited = new Error("403"); + expect(await offeredRelease(crowdedFeed, nightly(4580), rateLimited)).toMatchObject({ + tag: nightly(4580), + apiScans: 1, + }); + }); + + it.each([ + ["the feed is newer than the pointer", nightly(4603)], + ["there is no pointer", undefined], + ["the pointer names another channel", "v0.1.10-preview.20261112.4700"], + ])("keeps the feed's nightly when %s", async (_case, pointerTag) => { + expect(await offeredRelease([nightly(4606)], pointerTag)).toEqual({ + tag: nightly(4606), + releaseNotes: [{ version: nightly(4606).slice(1), note: `${nightly(4606)} feed notes` }], + apiScans: 0, + }); + }); +}); diff --git a/apps/desktop/src/updates/releaseNotes.test.ts b/apps/desktop/src/updates/releaseNotes.test.ts index 6f05b1e9e94a..a8fcc70ea355 100644 --- a/apps/desktop/src/updates/releaseNotes.test.ts +++ b/apps/desktop/src/updates/releaseNotes.test.ts @@ -241,14 +241,20 @@ describe("normalizeDesktopUpdateReleaseNotes", () => { }); }); - it("does not throw on out-of-range numeric entities and keeps the literal", () => { + it("keeps unsupported entities literal, including out-of-range and Object.prototype names", () => { const result = normalizeDesktopUpdateReleaseNotes( - "- Broken entity �", + "- Broken entity �\n- Mentions &constructor; and &toString;", "1.0.0", "latest", ); expect(result).toEqual({ - releaseNotes: [{ version: "1.0.0", items: ["Broken entity �"], totalItems: 1 }], + releaseNotes: [ + { + version: "1.0.0", + items: ["Mentions &constructor; and &toString;", "Broken entity �"], + totalItems: 2, + }, + ], omittedReleaseCount: 0, }); }); diff --git a/apps/desktop/src/updates/releaseNotes.ts b/apps/desktop/src/updates/releaseNotes.ts index 715c1767e6a0..30bfb9ace64b 100644 --- a/apps/desktop/src/updates/releaseNotes.ts +++ b/apps/desktop/src/updates/releaseNotes.ts @@ -20,14 +20,14 @@ const MAX_RELEASE_NOTE_GROUPS = 6; const MAX_RELEASE_NOTE_ITEMS_PER_GROUP = 8; const MAX_RELEASE_NOTE_ITEM_LENGTH = 220; -const HTML_ENTITY_REPLACEMENTS: Readonly> = { - amp: "&", - apos: "'", - gt: ">", - lt: "<", - nbsp: " ", - quot: '"', -}; +const HTML_ENTITY_REPLACEMENTS: ReadonlyMap = new Map([ + ["amp", "&"], + ["apos", "'"], + ["gt", ">"], + ["lt", "<"], + ["nbsp", " "], + ["quot", '"'], +]); function decodeCodePoint(codePoint: number, entity: string): string { if (!Number.isInteger(codePoint) || codePoint < 0 || codePoint > 0x10ffff) { @@ -37,7 +37,7 @@ function decodeCodePoint(codePoint: number, entity: string): string { } function decodeHtmlEntity(entity: string): string { - const named = HTML_ENTITY_REPLACEMENTS[entity]; + const named = HTML_ENTITY_REPLACEMENTS.get(entity); if (named) return named; if (entity.startsWith("#x")) { return decodeCodePoint(Number.parseInt(entity.slice(2), 16), entity); diff --git a/apps/desktop/src/updates/updatesTestHarness.ts b/apps/desktop/src/updates/updatesTestHarness.ts index 86035c4c8539..74cb14529b4b 100644 --- a/apps/desktop/src/updates/updatesTestHarness.ts +++ b/apps/desktop/src/updates/updatesTestHarness.ts @@ -13,6 +13,7 @@ import * as DesktopEnvironment from "../app/DesktopEnvironment.ts"; import * as ElectronUpdater from "../electron/ElectronUpdater.ts"; import * as ElectronWindow from "../electron/ElectronWindow.ts"; import * as DesktopAppSettings from "../settings/DesktopAppSettings.ts"; +import * as DesktopSshEnvironment from "../ssh/DesktopSshEnvironment.ts"; import * as DesktopState from "../app/DesktopState.ts"; import * as DesktopUpdatePolicy from "./DesktopUpdatePolicy.ts"; import * as DesktopUpdates from "./DesktopUpdates.ts"; @@ -136,6 +137,12 @@ export function makeHarness(options: UpdatesHarnessOptions = {}) { syncAllAppearance: () => Effect.void, } satisfies ElectronWindow.ElectronWindow["Service"]); + const layerSshEnvironment = Layer.mock(DesktopSshEnvironment.DesktopSshEnvironment, { + closeAllTunnels: Effect.sync(() => { + installSteps.push("closeSshTunnels"); + }), + }); + let backendDesiredRunning = options.backendDesiredRunning ?? true; const stubBackendInstance: DesktopBackendPool.DesktopBackendInstance = { id: DesktopBackendPool.PRIMARY_INSTANCE_ID, @@ -238,6 +245,7 @@ export function makeHarness(options: UpdatesHarnessOptions = {}) { : DesktopAppSettings.layer; const updateRestartMarkers = new Set(); + const updateMarkerContents = new Map(); const installRecords = new Map(); let previousInstallRecord = options.installRecord; const isInstallRecord = (path: string) => @@ -263,11 +271,21 @@ export function makeHarness(options: UpdatesHarnessOptions = {}) { writeFileString: (path, data) => Effect.sync(() => { if (isInstallRecord(path)) installRecords.set(path, data); - else updateRestartMarkers.add(path); + else { + if (path.endsWith("/desktop-update-restart")) updateRestartMarkers.add(path); + updateMarkerContents.set(path, data); + } + }), + rename: (from, to) => + Effect.sync(() => { + const contents = updateMarkerContents.get(from); + if (contents !== undefined) updateMarkerContents.set(to, contents); + updateMarkerContents.delete(from); }), remove: (path) => Effect.sync(() => { updateRestartMarkers.delete(path); + updateMarkerContents.delete(path); installRecords.delete(path); if (isInstallRecord(path)) previousInstallRecord = undefined; }), @@ -279,6 +297,7 @@ export function makeHarness(options: UpdatesHarnessOptions = {}) { Layer.provideMerge(layerUpdater), Layer.provideMerge(layerWindow), Layer.provideMerge(layerBackend), + Layer.provideMerge(layerSshEnvironment), Layer.provideMerge(DesktopState.layer), Layer.provideMerge(layerSettings), Layer.provideMerge( @@ -299,6 +318,8 @@ export function makeHarness(options: UpdatesHarnessOptions = {}) { quitAndInstalls: () => quitAndInstallCount, installSteps, updateRestartMarkers, + updateInProgressMarker: () => + [...updateMarkerContents].find(([path]) => path.endsWith("/desktop-update-in-progress"))?.[1], installRecords, previousInstallRecord: () => previousInstallRecord, downloadCount: () => downloadCount, diff --git a/apps/desktop/src/window/DesktopApplicationMenu.test.ts b/apps/desktop/src/window/DesktopApplicationMenu.test.ts index 612ec441ba99..46004ca30229 100644 --- a/apps/desktop/src/window/DesktopApplicationMenu.test.ts +++ b/apps/desktop/src/window/DesktopApplicationMenu.test.ts @@ -3,7 +3,9 @@ import { assert, describe, it } from "@effect/vitest"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import type { DesktopUpdateCheckResult, DesktopUpdateState } from "@t3tools/contracts"; import * as Electron from "electron"; import { vi } from "vite-plus/test"; @@ -18,6 +20,7 @@ import * as DesktopApplicationMenu from "./DesktopApplicationMenu.ts"; import * as DesktopConfig from "../app/DesktopConfig.ts"; import * as DesktopEnvironment from "../app/DesktopEnvironment.ts"; import * as DesktopUpdates from "../updates/DesktopUpdates.ts"; +import { createInitialDesktopUpdateState } from "../updates/updateMachine.ts"; import * as DesktopWindow from "./DesktopWindow.ts"; const environmentInput = { @@ -56,27 +59,37 @@ const layerElectronApp = Layer.succeed(ElectronApp.ElectronApp, { on: () => Effect.void, } satisfies ElectronApp.ElectronApp["Service"]); -const layerElectronDialog = Layer.succeed(ElectronDialog.ElectronDialog, { - pickFolder: () => Effect.succeedNone, - pickFiles: () => Effect.succeed([]), - showMessageBox: () => Effect.succeed({ response: 0, checkboxChecked: false }), - showErrorBox: () => Effect.void, -} satisfies ElectronDialog.ElectronDialog["Service"]); - -const layerDesktopUpdates = Layer.succeed(DesktopUpdates.DesktopUpdates, { - getState: Effect.die("unexpected getState"), - isActionActive: Effect.succeed(false), - isInstallActive: Effect.succeed(false), - subscribe: Effect.die("unexpected subscribe"), - emitState: Effect.void, - disabledReason: Effect.succeedNone, - configure: Effect.void, - setChannel: () => Effect.die("unexpected setChannel"), - check: () => Effect.die("unexpected check"), - download: Effect.die("unexpected download"), - install: Effect.die("unexpected install"), - installPrepared: () => Effect.die("unexpected installPrepared"), -} satisfies DesktopUpdates.DesktopUpdates["Service"]); +interface MenuServices { + readonly checkResult?: DesktopUpdateCheckResult; + readonly shownMessageBox?: Deferred.Deferred; +} + +const layerElectronDialog = ({ shownMessageBox }: MenuServices) => + Layer.succeed(ElectronDialog.ElectronDialog, { + pickFolder: () => Effect.succeedNone, + pickFiles: () => Effect.succeed([]), + showMessageBox: (options) => + (shownMessageBox ? Deferred.succeed(shownMessageBox, options) : Effect.void).pipe( + Effect.as({ response: options.cancelId ?? 0, checkboxChecked: false }), + ), + showErrorBox: () => Effect.void, + } satisfies ElectronDialog.ElectronDialog["Service"]); + +const layerDesktopUpdates = ({ checkResult }: MenuServices) => + Layer.succeed(DesktopUpdates.DesktopUpdates, { + getState: Effect.die("unexpected getState"), + isActionActive: Effect.succeed(false), + isInstallActive: Effect.succeed(false), + subscribe: Effect.die("unexpected subscribe"), + emitState: Effect.void, + disabledReason: Effect.succeedNone, + configure: Effect.void, + setChannel: () => Effect.die("unexpected setChannel"), + check: () => (checkResult ? Effect.succeed(checkResult) : Effect.die("unexpected check")), + download: Effect.die("unexpected download"), + install: Effect.die("unexpected install"), + installPrepared: () => Effect.die("unexpected installPrepared"), + } satisfies DesktopUpdates.DesktopUpdates["Service"]); const layerDesktopWindow = (selectedAction: Deferred.Deferred) => Layer.succeed(DesktopWindow.DesktopWindow, { @@ -113,6 +126,7 @@ const configureMenu = ( selectedAction: Deferred.Deferred, applicationMenuTemplate: Deferred.Deferred, environment: Partial = {}, + services: MenuServices = {}, ) => Effect.gen(function* () { const menu = yield* DesktopApplicationMenu.DesktopApplicationMenu; @@ -122,8 +136,8 @@ const configureMenu = ( DesktopApplicationMenu.layer.pipe( Layer.provideMerge(layerElectronMenu(applicationMenuTemplate)), Layer.provideMerge(layerDesktopWindow(selectedAction)), - Layer.provideMerge(layerDesktopUpdates), - Layer.provideMerge(layerElectronDialog), + Layer.provideMerge(layerDesktopUpdates(services)), + Layer.provideMerge(layerElectronDialog(services)), Layer.provideMerge(layerElectronApp), Layer.provideMerge( DesktopEnvironment.layer({ ...environmentInput, ...environment }).pipe( @@ -134,6 +148,54 @@ const configureMenu = ( ), ); +const checkForUpdatesFromMacMenu = (state: Partial) => + Effect.gen(function* () { + const applicationMenuTemplate = + yield* Deferred.make(); + const shownMessageBox = yield* Deferred.make(); + yield* configureMenu( + yield* Deferred.make(), + applicationMenuTemplate, + { platform: "darwin" }, + { + checkResult: { + checked: true, + state: { + ...createInitialDesktopUpdateState( + "0.1.5", + { hostArch: "arm64", appArch: "arm64", runningUnderArm64Translation: false }, + "latest", + ), + enabled: true, + ...state, + }, + }, + shownMessageBox, + }, + ); + + const appMenu = (yield* Deferred.await(applicationMenuTemplate))[0]; + if (!Array.isArray(appMenu?.submenu)) { + throw new Error("Expected the macOS app menu submenu to be an array."); + } + const checkForUpdates = appMenu.submenu.find((item) => item.label === "Check for Updates..."); + if (typeof checkForUpdates?.click !== "function") { + throw new Error("Expected Check for Updates... menu item to have a click handler."); + } + + // Every stub completes synchronously, so the menu action has finished when click returns. + checkForUpdates.click( + {} as Electron.MenuItem, + {} as Electron.BrowserWindow, + {} as KeyboardEvent, + ); + const dialog = yield* Deferred.poll(shownMessageBox); + if (Option.isNone(dialog)) { + throw new Error("Check for Updates... finished without showing any dialog."); + } + return yield* dialog.value; + }); + describe("DesktopApplicationMenu", () => { it.effect("leaves the accelerator to Chromium instead of injecting a paste", () => Effect.gen(function* () { @@ -285,4 +347,25 @@ describe("DesktopApplicationMenu", () => { assert.equal(yield* Deferred.await(selectedAction), "main-reload"); }), ); + it.effect("tells the user which update the menu check found", () => + Effect.gen(function* () { + const dialog = yield* checkForUpdatesFromMacMenu({ + status: "available", + availableVersion: "0.1.7", + }); + assert.include(`${dialog.title ?? ""} ${dialog.message} ${dialog.detail ?? ""}`, "0.1.7"); + }), + ); + + it.effect("tells the user a downloaded update is ready when the menu check runs", () => + Effect.gen(function* () { + const dialog = yield* checkForUpdatesFromMacMenu({ + status: "downloaded", + availableVersion: "0.1.7", + downloadedVersion: "0.1.7", + downloadPercent: 100, + }); + assert.include(`${dialog.title ?? ""} ${dialog.message} ${dialog.detail ?? ""}`, "0.1.7"); + }), + ); }); diff --git a/apps/desktop/src/window/DesktopApplicationMenu.ts b/apps/desktop/src/window/DesktopApplicationMenu.ts index 44d2c234f835..b4025ebfde8e 100644 --- a/apps/desktop/src/window/DesktopApplicationMenu.ts +++ b/apps/desktop/src/window/DesktopApplicationMenu.ts @@ -64,6 +64,9 @@ const runMainContentsCommand = Effect.fn("desktop.menu.runMainContentsCommand")( yield* desktopWindow.runMainContentsCommand(command); }); +const describeUpdate = (version: string | null) => + version === null ? "A new version of T3 Code" : `T3 Code ${version}`; + const checkForUpdatesFromMenu = Effect.gen(function* () { const updates = yield* DesktopUpdates.DesktopUpdates; const electronDialog = yield* ElectronDialog.ElectronDialog; @@ -77,6 +80,35 @@ const checkForUpdatesFromMenu = Effect.gen(function* () { message: `T3 Code ${updateState.currentVersion} is currently the newest version available.`, buttons: ["OK"], }); + } else if (updateState.status === "available") { + const { response } = yield* electronDialog.showMessageBox({ + type: "info", + title: "Update available", + message: `${describeUpdate(updateState.availableVersion)} is available.`, + detail: `You're running T3 Code ${updateState.currentVersion}.`, + buttons: ["Download", "Later"], + defaultId: 0, + cancelId: 1, + }); + if (response === 0) yield* updates.download; + } else if (updateState.status === "downloading") { + yield* electronDialog.showMessageBox({ + type: "info", + title: "Update downloading", + message: `${describeUpdate(updateState.availableVersion)} is already downloading.`, + buttons: ["OK"], + }); + } else if (updateState.status === "downloaded") { + const { response } = yield* electronDialog.showMessageBox({ + type: "info", + title: "Update ready", + message: `${describeUpdate(updateState.downloadedVersion ?? updateState.availableVersion)} is ready to install.`, + detail: "T3 Code will restart to install it. Any running tasks will be interrupted.", + buttons: ["Restart", "Later"], + defaultId: 0, + cancelId: 1, + }); + if (response === 0) yield* updates.install; } else if (updateState.status === "error") { yield* electronDialog.showMessageBox({ type: "warning", diff --git a/apps/desktop/src/window/DesktopWindow.test.ts b/apps/desktop/src/window/DesktopWindow.test.ts index 509e51cd0507..16dbe19f47f1 100644 --- a/apps/desktop/src/window/DesktopWindow.test.ts +++ b/apps/desktop/src/window/DesktopWindow.test.ts @@ -207,17 +207,20 @@ const layerElectronTheme = Layer.succeed(ElectronTheme.ElectronTheme, { onUpdated: () => Effect.void, } satisfies ElectronTheme.ElectronTheme["Service"]); -const layerDesktopEnvironment = DesktopEnvironment.layer(environmentInput).pipe( - Layer.provide( - Layer.mergeAll( - NodeServices.layer, - DesktopConfig.layerTest({ - T3CODE_PORT: "3773", - VITE_DEV_SERVER_URL: "http://127.0.0.1:5733", - }), +const layerDesktopEnvironmentFor = (platform: NodeJS.Platform) => + DesktopEnvironment.layer({ ...environmentInput, platform }).pipe( + Layer.provide( + Layer.mergeAll( + NodeServices.layer, + DesktopConfig.layerTest({ + T3CODE_PORT: "3773", + VITE_DEV_SERVER_URL: "http://127.0.0.1:5733", + }), + ), ), - ), -); + ); + +const layerDesktopEnvironment = layerDesktopEnvironmentFor(environmentInput.platform); const desktopWindowBoundsEquivalence = Schema.toEquivalence( DesktopAppSettings.DesktopWindowBoundsSchema, @@ -243,6 +246,12 @@ function layerTest(input: { readonly previewZoomReapplies?: number[]; readonly onReveal?: (window: Electron.BrowserWindow) => void; readonly focusedWindow?: Electron.BrowserWindow; + readonly fallbackWindow?: Electron.BrowserWindow; + readonly platform?: NodeJS.Platform; + readonly appearanceWindows?: ReadonlyArray<{ + readonly window: Electron.BrowserWindow; + readonly titleBarOverlay: boolean; + }>; readonly primaryPasteEnabled?: boolean; readonly previewPasskeyBridgeEnabled?: boolean; }) { @@ -292,7 +301,9 @@ function layerTest(input: { Effect.as(input.window), ), main: Ref.get(input.mainWindow), - currentMainOrFirst: Ref.get(input.mainWindow), + currentMainOrFirst: Ref.get(input.mainWindow).pipe( + Effect.map(Option.orElse(() => Option.fromNullishOr(input.fallbackWindow ?? null))), + ), focusedMainOrFirst: input.focusedWindow === undefined ? Ref.get(input.mainWindow) @@ -303,7 +314,14 @@ function layerTest(input: { reveal: (window) => Effect.sync(() => input.onReveal?.(window)), sendAll: () => Effect.void, destroyAll: input.destroyAll ?? Effect.void, - syncAllAppearance: (sync) => sync(input.window), + syncAllAppearance: (sync) => + input.appearanceWindows === undefined + ? sync(input.window, { titleBarOverlay: true }) + : Effect.forEach( + input.appearanceWindows, + ({ window, titleBarOverlay }) => sync(window, { titleBarOverlay }), + { discard: true }, + ), } satisfies ElectronWindow.ElectronWindow["Service"]); return DesktopWindow.layer.pipe( @@ -315,7 +333,9 @@ function layerTest(input: { recordMetrics: () => Effect.void, shutdown: Effect.void, }), - layerDesktopEnvironment, + input.platform === undefined + ? layerDesktopEnvironment + : layerDesktopEnvironmentFor(input.platform), layerDesktopAppSettings, layerDesktopClientSettings, layerDesktopServerExposure, @@ -424,7 +444,8 @@ const makeSplashScenario = (createOutcomes: readonly (Electron.BrowserWindow | n reveal: (window) => Ref.update(revealedWindows, (windows) => [...windows, window]), sendAll: () => Effect.void, destroyAll: Effect.void, - syncAllAppearance: (sync) => (fallbackWindow ? sync(fallbackWindow) : Effect.void), + syncAllAppearance: (sync) => + fallbackWindow ? sync(fallbackWindow, { titleBarOverlay: true }) : Effect.void, } satisfies ElectronWindow.ElectronWindow["Service"]; const layer = DesktopWindow.layer.pipe( @@ -827,7 +848,7 @@ describe("DesktopWindow", () => { }), ); - it.effect("reloads and inspects the main window even when a popup has focus", () => + it.effect("routes menu commands to the main window even when a popup has focus", () => Effect.gen(function* () { const fakeWindow = makeFakeBrowserWindow(); const popup = makeFakeBrowserWindow(); @@ -847,7 +868,10 @@ describe("DesktopWindow", () => { yield* desktopWindow.runMainContentsCommand("reload"); yield* desktopWindow.runMainContentsCommand("forceReload"); yield* desktopWindow.runMainContentsCommand("toggleDevTools"); + yield* desktopWindow.dispatchMenuAction("open-settings"); + assert.deepEqual(fakeWindow.send.mock.calls, [["desktop:menu-action", "open-settings"]]); + assert.equal(popup.send.mock.calls.length, 0); assert.equal(fakeWindow.reload.mock.calls.length, 1); assert.equal(fakeWindow.reloadIgnoringCache.mock.calls.length, 1); assert.equal(fakeWindow.toggleDevTools.mock.calls.length, 1); @@ -858,6 +882,38 @@ describe("DesktopWindow", () => { }), ); + it.effect("recreates main for Settings when only an unregistered popup survives", () => + Effect.gen(function* () { + const main = makeFakeBrowserWindow(); + const auxiliary = makeFakeBrowserWindow(); + const createCount = yield* Ref.make(0); + const mainWindow = yield* Ref.make>(Option.none()); + + yield* Effect.gen(function* () { + const windows = yield* DesktopWindow.DesktopWindow; + yield* windows.dispatchMenuAction("open-settings"); + assert.deepEqual(main.send.mock.calls, [["desktop:menu-action", "open-settings"]]); + assert.equal(auxiliary.send.mock.calls.length, 0); + assert.equal(yield* Ref.get(createCount), 1); + assert.strictEqual(yield* windows.ensureMain, main.window); + assert.equal(yield* Ref.get(createCount), 1); + }).pipe( + Effect.provide( + layerTest({ + window: main.window, + createCount, + mainWindow, + fallbackWindow: auxiliary.window, + desktopSettings: { + ...DesktopAppSettings.DEFAULT_DESKTOP_SETTINGS, + localEnvironmentEnabled: false, + }, + }), + ), + ); + }), + ); + it.effect("does not open main or splash windows once quitting has started", () => Effect.gen(function* () { const fake = makeFakeBrowserWindow(); @@ -1171,6 +1227,36 @@ describe("DesktopWindow", () => { }), ); + it.effect("updates title bar overlays only on windows created with one", () => + Effect.gen(function* () { + const framed = makeFakeBrowserWindow(); + vi.mocked(framed.window.setTitleBarOverlay).mockImplementation(() => { + throw new TypeError("Titlebar overlay is not enabled"); + }); + const overlay = makeFakeBrowserWindow(); + const layer = layerTest({ + window: overlay.window, + createCount: yield* Ref.make(0), + mainWindow: yield* Ref.make>(Option.none()), + platform: "win32", + appearanceWindows: [ + { window: framed.window, titleBarOverlay: false }, + { window: overlay.window, titleBarOverlay: true }, + ], + }); + + yield* Effect.gen(function* () { + const desktopWindow = yield* DesktopWindow.DesktopWindow; + yield* desktopWindow.syncAppearance; + }).pipe(Effect.provide(layer)); + + assert.deepEqual(vi.mocked(framed.window.setBackgroundColor).mock.calls, [["#ffffff"]]); + assert.lengthOf(vi.mocked(framed.window.setTitleBarOverlay).mock.calls, 0); + assert.deepEqual(vi.mocked(overlay.window.setBackgroundColor).mock.calls, [["#ffffff"]]); + assert.lengthOf(vi.mocked(overlay.window.setTitleBarOverlay).mock.calls, 1); + }), + ); + it.effect.each([ { window: "normal", write: "flush" }, { window: "normal", write: "debounce" }, diff --git a/apps/desktop/src/window/DesktopWindow.ts b/apps/desktop/src/window/DesktopWindow.ts index 43404e2d6cfb..7be14bc72083 100644 --- a/apps/desktop/src/window/DesktopWindow.ts +++ b/apps/desktop/src/window/DesktopWindow.ts @@ -343,6 +343,7 @@ function getWindowTitleBarOptions( function syncWindowAppearance( window: Electron.BrowserWindow, + hasTitleBarOverlay: boolean, shouldUseDarkColors: boolean, platform: NodeJS.Platform, ): Effect.Effect { @@ -353,7 +354,7 @@ function syncWindowAppearance( window.setBackgroundColor(getInitialWindowBackgroundColor(shouldUseDarkColors)); const { titleBarOverlay } = getWindowTitleBarOptions(shouldUseDarkColors, platform); - if (typeof titleBarOverlay === "object") { + if (hasTitleBarOverlay && typeof titleBarOverlay === "object") { window.setTitleBarOverlay(titleBarOverlay); } }); @@ -424,7 +425,6 @@ export const make = Effect.gen(function* () { ); const currentMainWindow = electronWindow.currentMainOrFirst.pipe(Effect.flatMap(withoutSplash)); - const focusedMainWindow = electronWindow.focusedMainOrFirst.pipe(Effect.flatMap(withoutSplash)); const zoomWindow = Effect.fn("desktop.window.zoomWindow")(function* ( window: Electron.BrowserWindow, @@ -1018,7 +1018,7 @@ export const make = Effect.gen(function* () { const ensureMain = Effect.gen(function* () { if (yield* Ref.get(desktopState.quitting)) return Option.none(); - const existingWindow = yield* currentMainWindow; + const existingWindow = yield* electronWindow.main; if (Option.isSome(existingWindow)) { return existingWindow; } @@ -1105,7 +1105,8 @@ export const make = Effect.gen(function* () { { reveal = true }: { readonly reveal?: boolean } = {}, ) { if (yield* Ref.get(desktopState.quitting)) return; - const existingWindow = yield* reveal ? focusedMainWindow : electronWindow.main; + // Only the main renderer loads the app preload that listens on these channels. + const existingWindow = yield* electronWindow.main; if (Option.isNone(existingWindow) && (!reveal || (yield* waitingForBackend))) return; const window = Option.isSome(existingWindow) ? existingWindow : yield* ensureMain; if (Option.isNone(window) || window.value.isDestroyed()) return; @@ -1202,8 +1203,8 @@ export const make = Effect.gen(function* () { }), syncAppearance: Effect.gen(function* () { const shouldUseDarkColors = yield* electronTheme.shouldUseDarkColors; - yield* electronWindow.syncAllAppearance((window) => - syncWindowAppearance(window, shouldUseDarkColors, environment.platform), + yield* electronWindow.syncAllAppearance((window, { titleBarOverlay }) => + syncWindowAppearance(window, titleBarOverlay, shouldUseDarkColors, environment.platform), ); }).pipe(Effect.withSpan("desktop.window.syncAppearance")), }); diff --git a/apps/desktop/src/wsl/DesktopWslBackend.test.ts b/apps/desktop/src/wsl/DesktopWslBackend.test.ts index 8468a9538630..d9ae6e76c29c 100644 --- a/apps/desktop/src/wsl/DesktopWslBackend.test.ts +++ b/apps/desktop/src/wsl/DesktopWslBackend.test.ts @@ -154,7 +154,7 @@ describe("DesktopWslBackend", () => { assert.equal(startCount, 1); assert.deepEqual(yield* backend.lastPreflightError, refusal); - yield* clearFailure(new URL("http://127.0.0.1:41773")); + yield* clearFailure(new URL("http://127.0.0.1:41773"), 123); assert.deepEqual(yield* backend.lastPreflightError, Option.none()); yield* recordOwnershipFailure(); const settings = yield* DesktopAppSettings.DesktopAppSettings; diff --git a/apps/desktop/src/wsl/DesktopWslEnvironment.test.ts b/apps/desktop/src/wsl/DesktopWslEnvironment.test.ts index 6111360158ca..bab641240119 100644 --- a/apps/desktop/src/wsl/DesktopWslEnvironment.test.ts +++ b/apps/desktop/src/wsl/DesktopWslEnvironment.test.ts @@ -1,19 +1,21 @@ // @effect-diagnostics nodeBuiltinImport:off - the executed suite runs the generated install script through a real POSIX shell. +import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, describe, it } from "@effect/vitest"; -import { afterAll, expect } from "vite-plus/test"; -import * as NodeChildProcess from "node:child_process"; -import * as Deferred from "effect/Deferred"; -import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; -import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import * as Redacted from "effect/Redacted"; import * as Sink from "effect/Sink"; import * as Stream from "effect/Stream"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; +import { afterAll, expect } from "vite-plus/test"; +import * as NodeChildProcess from "node:child_process"; +import * as Deferred from "effect/Deferred"; +import * as Duration from "effect/Duration"; +import * as Fiber from "effect/Fiber"; import * as TestClock from "effect/testing/TestClock"; -import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import * as DesktopConfig from "../app/DesktopConfig.ts"; import * as DesktopEnvironment from "../app/DesktopEnvironment.ts"; @@ -22,6 +24,7 @@ import { ManagedPolicyFileError } from "@t3tools/shared/managedPolicyFile"; import * as DesktopManagedPolicy from "../app/DesktopManagedPolicy.ts"; import { + layer as layerDesktopWslEnvironment, buildWslRuntimeInstallScript, makeIsAvailable, buildWslRuntimeInvalidateScript, @@ -33,6 +36,7 @@ import { DesktopWslEnvironment, layer as wslEnvironmentLayer, resolveWslRunningUser, + windowsToWslPathImpl, } from "./DesktopWslEnvironment.ts"; // The install script only fails the way this file cares about when a real shell @@ -558,7 +562,7 @@ describe.skipIf(posixShellRunner === null)("WSL runtime install script (executed " }", ' eval "$2"', "); }", - buildWslRuntimeProbeScript(fixture.runtimeRoot), + buildWslRuntimeProbeScript(), ].join("\n"), ); @@ -593,7 +597,7 @@ describe.skipIf(posixShellRunner === null)("WSL runtime install script (executed [ "bash() { return 127; }", 'export PATH="/fixture/bin:/usr/bin:/bin"', - buildWslRuntimeProbeScript(fixture.runtimeRoot), + buildWslRuntimeProbeScript(), ].join("\n"), ); @@ -777,6 +781,93 @@ describe.skipIf(posixShellRunner === null)("WSL runtime install script (executed expect(selected.status, `${selected.stdout}\n${selected.stderr}`).toBe(0); }); + // Runs the distro side on the local bash, so a Windows dev box using the wsl.exe runner skips it. + describe.skipIf(posixShellRunner?.file !== "bash")("prepareRuntime through wsl.exe", () => { + // Emulates `wsl.exe [-d ] [--cd ] [--|--exec] ` in a distro with automount off. + const layerFakeWslExe = (home: string) => + Layer.effect( + ChildProcessSpawner.ChildProcessSpawner, + Effect.map(ChildProcessSpawner.ChildProcessSpawner, (local) => + ChildProcessSpawner.make((command) => { + if (!ChildProcess.isStandardCommand(command) || command.command !== "wsl.exe") { + return local.spawn(command); + } + const args = command.args.map((word) => + Redacted.isRedacted(word) ? Redacted.value(word) : word, + ); + while (args.length > 0) { + const flag = args[0]; + if (flag === "-d" || flag === "--distribution" || flag === "--cd") { + args.splice(0, 2); + } else if (flag === "--" || flag === "--exec" || flag === "-e") { + args.shift(); + break; + } else { + break; + } + } + const [file = "sh", ...rest] = + args[0] === "wslpath" + ? [ + "sh", + "-c", + 'printf "wslpath: %s\\n" "$1" >&2; exit 1', + "wslpath", + args.at(-1) ?? "", + ] + : args; + return local.spawn( + ChildProcess.make(file, rest, { + ...command.options, + env: { ...command.options.env, HOME: home }, + extendEnv: true, + }), + ); + }), + ), + ); + + const layerWslEnvironment = (home: string) => + layerDesktopWslEnvironment.pipe( + Layer.provide( + Layer.mergeAll( + layerFakeWslExe(home), + DesktopManagedPolicy.layerTest(), + Layer.effect( + DesktopEnvironment.DesktopEnvironment, + Effect.map( + Path.Path, + (path) => + ({ platform: "win32", path }) as DesktopEnvironment.DesktopEnvironment["Service"], + ), + ), + ), + ), + Layer.provide(NodeServices.layer), + ); + + it.live("installs the runtime when wslpath cannot translate the archive's Windows path", () => { + const fixture = createFixture(); + return Effect.gen(function* () { + const wsl = yield* DesktopWslEnvironment; + // The archive's Windows path; with no drives mounted, wslpath cannot map it into the distro. + const translated = yield* wsl.windowsToWslPath("Ubuntu", fixture.archivePath); + expect(translated._tag).toBe("None"); + + const prepared = yield* wsl.prepareRuntime("Ubuntu", { + windowsPath: fixture.archivePath, + runtimeId: fixture.runtimeId, + sha256: fixture.archiveSha, + }); + + expect(prepared).toEqual({ ok: true, linuxAppRoot: fixture.runtimeRoot }); + const version = runShell(`${sh(fixture.serverEntry)} --version`); + expect(version.status, version.stderr).toBe(0); + expect(version.stdout).toBe("t3code wsl runtime test server 0.0.0\n"); + }).pipe(Effect.provide(layerWslEnvironment(`${fixture.work}/home`))); + }); + }); + it("prunes old and markerless caches without touching retained, active, locked, or unrelated roots", () => { const result = runShell( [ @@ -856,3 +947,48 @@ describe("makeIsAvailable", () => { }), ); }); + +describe("windowsToWslPathImpl", () => { + it.effect("runs wslpath without a shell so folder names stay one literal argument", () => + Effect.gen(function* () { + const spawned: Array = []; + const stdout = Stream.make(new TextEncoder().encode("/mnt/c/a$b & c;(d)\n")); + const spawner = ChildProcessSpawner.make((command) => { + spawned.push(command); + return Effect.succeed( + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + stdout, + stderr: Stream.empty, + all: stdout, + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(0)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + stdin: Sink.drain, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }), + ); + }); + + const converted = yield* windowsToWslPathImpl("Ubuntu", "C:\\a$b & c;(d)").pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + + assert.deepStrictEqual(converted, Option.some("/mnt/c/a$b & c;(d)")); + assert.lengthOf(spawned, 1); + const [command] = spawned; + assert(command?._tag === "StandardCommand"); + assert.strictEqual(command.command, "wsl.exe"); + assert.deepStrictEqual(command.args, [ + "-d", + "Ubuntu", + "--exec", + "wslpath", + "-u", + "C:/a$b & c;(d)", + ]); + }), + ); +}); diff --git a/apps/desktop/src/wsl/DesktopWslEnvironment.ts b/apps/desktop/src/wsl/DesktopWslEnvironment.ts index c5f67d1e2435..b2da57a0e30f 100644 --- a/apps/desktop/src/wsl/DesktopWslEnvironment.ts +++ b/apps/desktop/src/wsl/DesktopWslEnvironment.ts @@ -5,6 +5,8 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import type * as PlatformError from "effect/PlatformError"; +import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; @@ -110,10 +112,7 @@ export class DesktopWslEnvironment extends Context.Service< ) => Effect.Effect; readonly pruneRuntimes: (distro: string | null, runtimeId: string) => Effect.Effect; readonly invalidateRuntime: (distro: string | null, runtimeId: string) => Effect.Effect; - readonly probeRuntime: ( - distro: string | null, - linuxAppRoot: string, - ) => Effect.Effect; + readonly probeRuntime: (distro: string | null) => Effect.Effect; readonly ensureNodePty: ( distro: string | null, linuxAppRoot: string, @@ -175,37 +174,19 @@ const buildWslNodeEnvPreamble = ( ensure_remote_node_path || true `; -// wsl.exe re-escaping mangles quotes inside bash -lc scripts. -const runWslShell = ( - distro: string | null, - bashScript: string, +const runWslProcess = ( + args: ReadonlyArray, + stdin: Stream.Stream, timeout: Duration.Duration, - options: { - readonly nodeEngineRange?: string | null; - readonly resolveNode?: boolean; - } = {}, ): Effect.Effect => { const spawner = ChildProcessSpawner.ChildProcessSpawner; - const resolveNode = options.resolveNode !== false; - const command = ChildProcess.make( - "wsl.exe", - resolveNode - ? [...buildDistroArgs(distro), "--", "bash", "-l", "-s"] - : [...buildDistroArgs(distro), "--exec", "sh", "-s"], - { - stdin: Stream.encodeText( - Stream.make( - resolveNode - ? `${buildWslNodeEnvPreamble(options.nodeEngineRange)}${bashScript}` - : bashScript, - ), - ), - stdout: "pipe", - stderr: "pipe", - killSignal: "SIGTERM", - forceKillAfter: PROCESS_TERMINATE_GRACE, - }, - ); + const command = ChildProcess.make("wsl.exe", args, { + stdin, + stdout: "pipe", + stderr: "pipe", + killSignal: "SIGTERM", + forceKillAfter: PROCESS_TERMINATE_GRACE, + }); return Effect.scoped( Effect.gen(function* () { @@ -251,6 +232,32 @@ const runWslShell = ( ); }; +// wsl.exe re-escaping mangles quotes inside bash -lc scripts. +const runWslShell = ( + distro: string | null, + bashScript: string, + timeout: Duration.Duration, + options: { + readonly nodeEngineRange?: string | null; + readonly resolveNode?: boolean; + } = {}, +): Effect.Effect => { + const resolveNode = options.resolveNode !== false; + return runWslProcess( + resolveNode + ? [...buildDistroArgs(distro), "--", "bash", "-l", "-s"] + : [...buildDistroArgs(distro), "--exec", "sh", "-s"], + Stream.encodeText( + Stream.make( + resolveNode + ? `${buildWslNodeEnvPreamble(options.nodeEngineRange)}${bashScript}` + : bashScript, + ), + ), + timeout, + ); +}; + const shellQuote = (value: string): string => `'${value.replaceAll("'", "'\\''")}'`; const WSL_RUNTIME_READY_MARKER = ".t3code-wsl-runtime-ready"; @@ -259,13 +266,14 @@ const WSL_RUNTIME_SELECTION_GRACE_MINUTES = 5; const sanitizeWslRuntimeId = (value: string): string => value.replace(/[^A-Za-z0-9._-]/g, "_"); +// A null archive path asks for a staging directory to copy the archive into, unless the cache is ready. export const buildWslRuntimeInstallScript = ( - linuxArchivePath: string, + linuxArchivePath: string | null, runtimeId: string, archiveSha256: string, ): string => { const safeRuntimeId = sanitizeWslRuntimeId(runtimeId); - return [ + const readyCheck = [ "set -eu", 'runtime_parent="$HOME/.t3/wsl-runtime"', `runtime_root="$runtime_parent/${safeRuntimeId}"`, @@ -294,6 +302,17 @@ export const buildWslRuntimeInstallScript = ( ` printf 'runtimeRoot:%s\\n' "$runtime_root"`, " exit 0", "fi", + ]; + if (linuxArchivePath === null) { + return [ + ...readyCheck, + // The .tmp. name lets pruning collect a stage orphaned by a crash. + `archive_stage=$(mktemp -d "$runtime_parent/.${safeRuntimeId}.tmp.XXXXXX")`, + `printf 'archiveStage:%s\\n' "$archive_stage"`, + ].join("\n"); + } + return [ + ...readyCheck, `archive_sha=$(sha256sum ${shellQuote(linuxArchivePath)} | cut -d ' ' -f 1)`, `if [ "$archive_sha" != ${shellQuote(archiveSha256)} ]; then`, ` printf 'WSL runtime archive does not match its recorded SHA-256 (expected %s, got %s)\\n' ${shellQuote(archiveSha256)} "$archive_sha" >&2`, @@ -411,14 +430,17 @@ export const buildWslRuntimeInvalidateScript = (runtimeId: string): string => { ].join("\n"); }; -export const parseWslRuntimeRoot = (stdout: string): string | null => { - const prefix = "runtimeRoot:"; +const parseAbsolutePathField = (stdout: string, field: string): string | null => { + const prefix = `${field}:`; const line = stdout.split("\n").find((candidate) => candidate.startsWith(prefix)); if (line === undefined) return null; - const runtimeRoot = line.slice(prefix.length).replace(/\r$/, ""); - return runtimeRoot.startsWith("/") ? runtimeRoot : null; + const value = line.slice(prefix.length).replace(/\r$/, ""); + return value.startsWith("/") ? value : null; }; +export const parseWslRuntimeRoot = (stdout: string): string | null => + parseAbsolutePathField(stdout, "runtimeRoot"); + const NODE_PTY_BINARY_MISSING_EXIT_CODE = 4; const formatNodePtyProbeFailureReason = (exitCode: number): string | null => @@ -460,11 +482,9 @@ require("node-pty"); NODE`; // Resolve version-managed Node for providers without making runtime readiness depend on it. -export const buildWslRuntimeProbeScript = (linuxAppRoot: string) => - [ - `bash -lc ${shellQuote(`${buildWslNodeEnvPreamble()}${RESOLVED_PATH_LINE}`)} 2>/dev/null || ${RESOLVED_PATH_LINE}`, - `${shellQuote(`${linuxAppRoot}/t3`)} --version >/dev/null 2>&1`, - ].join("\n"); +// The install script already ran the staged t3, so running it again here only adds a second startup. +export const buildWslRuntimeProbeScript = () => + `bash -lc ${shellQuote(`${buildWslNodeEnvPreamble()}${RESOLVED_PATH_LINE}`)} 2>/dev/null || ${RESOLVED_PATH_LINE}`; const TOOLCHAIN_CHECK_SCRIPT = [ "for tool in node make g++ python3; do", @@ -586,30 +606,17 @@ export const resolveWslRunningUser = Effect.fn("desktop.wsl.resolveRunningUser") }); const probeWslRuntimeImpl = Effect.fn("DesktopWslEnvironment.probeWslRuntimeImpl")(function* ( distro: string | null, - linuxAppRoot: string, ): Effect.fn.Return { - const probe = yield* runWslShell( - distro, - buildWslRuntimeProbeScript(linuxAppRoot), - PROBE_TIMEOUT, - { - resolveNode: false, - }, - ); + const probe = yield* runWslShell(distro, buildWslRuntimeProbeScript(), PROBE_TIMEOUT, { + resolveNode: false, + }); const transportFailureReason = formatWslShellTransportFailureReason( probe.transportFailure, - "the staged runtime", + "the login-shell PATH", ); if (transportFailureReason !== null) { return { ok: false, reason: transportFailureReason } as const; } - if (probe.exitCode !== 0) { - const trimmedTail = probe.stderr.trim().slice(-500); - return { - ok: false, - reason: `${linuxAppRoot}/t3 --version failed (exit ${probe.exitCode})${trimmedTail ? `: ${trimmedTail}` : ""}`, - } as const; - } const resolvedPath = parseResolvedPath(probe.stdout); if (resolvedPath === null) { return { @@ -801,25 +808,18 @@ const ensureNodePtyImpl = Effect.fn("DesktopWslEnvironment.ensureNodePtyImpl")(f } as const; }); -const prepareWslRuntimeImpl = Effect.fn("desktop.wsl.prepareRuntimeImpl")(function* ( +type WslRuntimeInstallOutcome = + | { readonly ok: true; readonly stdout: string } + | { readonly ok: false; readonly reason: string }; + +const runWslRuntimeInstall = Effect.fn("desktop.wsl.runRuntimeInstall")(function* ( distro: string | null, archive: WslRuntimeArchive, - windowsToWslPath: ( - distro: string | null, - windowsPath: string, - ) => Effect.Effect>, -): Effect.fn.Return { - const linuxArchivePath = yield* windowsToWslPath(distro, archive.windowsPath); - if (Option.isNone(linuxArchivePath)) { - return { - ok: false, - reason: `wslpath conversion failed for ${archive.windowsPath}`, - } as const; - } - + linuxArchivePath: string | null, +): Effect.fn.Return { const install = yield* runWslShell( distro, - buildWslRuntimeInstallScript(linuxArchivePath.value, archive.runtimeId, archive.sha256), + buildWslRuntimeInstallScript(linuxArchivePath, archive.runtimeId, archive.sha256), RUNTIME_INSTALL_TIMEOUT, { resolveNode: false }, ); @@ -839,14 +839,106 @@ const prepareWslRuntimeImpl = Effect.fn("desktop.wsl.prepareRuntimeImpl")(functi reason: `WSL runtime installation failed (exit ${install.exitCode}): ${trimmedTail || "no stderr captured"}`, } as const; } + return { ok: true, stdout: install.stdout } as const; +}); - const linuxAppRoot = parseWslRuntimeRoot(install.stdout); +const toPrepareWslRuntimeResult = (outcome: WslRuntimeInstallOutcome): PrepareWslRuntimeResult => { + if (!outcome.ok) return outcome; + const linuxAppRoot = parseWslRuntimeRoot(outcome.stdout); return linuxAppRoot === null ? { ok: false, reason: "WSL runtime installation completed without reporting its cache path.", } : { ok: true, linuxAppRoot }; +}; + +// Without drive mounts (`[automount] enabled=false`) wslpath cannot reach the archive, so copy it in over stdin. +const installWslRuntimeFromHostCopy = Effect.fn("desktop.wsl.installRuntimeFromHostCopy")( + function* ( + distro: string | null, + archive: WslRuntimeArchive, + archiveBytes: Stream.Stream, + ): Effect.fn.Return { + const staged = yield* runWslRuntimeInstall(distro, archive, null); + if (!staged.ok || parseWslRuntimeRoot(staged.stdout) !== null) { + return toPrepareWslRuntimeResult(staged); + } + const stage = parseAbsolutePathField(staged.stdout, "archiveStage"); + if (stage === null) { + return { + ok: false, + reason: `wslpath conversion failed for ${archive.windowsPath}, and the distro did not provide a directory to copy the runtime archive into.`, + } as const; + } + const linuxArchivePath = `${stage}/runtime.tar.gz`; + + const copyAndInstall = Effect.gen(function* () { + const readError = yield* Ref.make(Option.none()); + // dd reads fd 0 itself; reopening /dev/stdin fails when WSL relays stdin as a socket. + const copy = yield* runWslProcess( + [...buildDistroArgs(distro), "--exec", "dd", `of=${linuxArchivePath}`, "bs=65536"], + archiveBytes.pipe( + // Ending stdin on a read error lets dd exit instead of waiting out the timeout. + Stream.catch((error) => + Stream.drain(Stream.fromEffect(Ref.set(readError, Option.some(error)))), + ), + ), + RUNTIME_INSTALL_TIMEOUT, + ); + const failedRead = yield* Ref.get(readError); + if (Option.isSome(failedRead)) { + return { + ok: false, + reason: `Could not read the WSL runtime archive at ${archive.windowsPath}: ${failedRead.value.message}`, + } as const; + } + if (copy.transportFailure !== null || copy.exitCode !== 0) { + const trimmedTail = copy.stderr.trim().slice(-500); + return { + ok: false, + reason: `Could not copy the WSL runtime archive into the distro (exit ${copy.exitCode})${trimmedTail ? `: ${trimmedTail}` : ""}`, + } as const; + } + return toPrepareWslRuntimeResult( + yield* runWslRuntimeInstall(distro, archive, linuxArchivePath), + ); + }); + + return yield* copyAndInstall.pipe( + Effect.ensuring( + runWslShell(distro, `rm -rf ${shellQuote(stage)}`, RUNTIME_INVALIDATE_TIMEOUT, { + resolveNode: false, + }), + ), + ); + }, +); + +const prepareWslRuntimeImpl = Effect.fn("desktop.wsl.prepareRuntimeImpl")(function* ( + distro: string | null, + archive: WslRuntimeArchive, + windowsToWslPath: ( + distro: string | null, + windowsPath: string, + ) => Effect.Effect>, +): Effect.fn.Return< + PrepareWslRuntimeResult, + never, + ChildProcessSpawner.ChildProcessSpawner | FileSystem.FileSystem +> { + const linuxArchivePath = yield* windowsToWslPath(distro, archive.windowsPath); + if (Option.isNone(linuxArchivePath)) { + const fileSystem = yield* FileSystem.FileSystem; + return yield* installWslRuntimeFromHostCopy( + distro, + archive, + fileSystem.stream(archive.windowsPath), + ); + } + return toPrepareWslRuntimeResult( + yield* runWslRuntimeInstall(distro, archive, linuxArchivePath.value), + ); }); const pruneWslRuntimesImpl = Effect.fn("desktop.wsl.pruneRuntimesImpl")(function* ( @@ -951,7 +1043,7 @@ const preWarmImpl = ( }), ).pipe(Effect.timeoutOption(PRE_WARM_TIMEOUT), Effect.ignore); -const windowsToWslPathImpl = ( +export const windowsToWslPathImpl = ( distro: string | null, windowsPath: string, ): Effect.Effect, never, ChildProcessSpawner.ChildProcessSpawner> => { @@ -962,7 +1054,8 @@ const windowsToWslPathImpl = ( const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const command = ChildProcess.make( "wsl.exe", - [...buildDistroArgs(distro), "--", "wslpath", "-u", normalized], + // --exec skips the distro shell, which would expand `$`, `&` or `;` in folder names. + [...buildDistroArgs(distro), "--exec", "wslpath", "-u", normalized], { stdin: "ignore", stdout: "pipe", @@ -1090,7 +1183,7 @@ export interface DesktopWslEnvironmentTestStub { ) => PrepareWslRuntimeResult; readonly pruneRuntimes?: (distro: string | null, runtimeId: string) => Effect.Effect; readonly invalidateRuntime?: (distro: string | null, runtimeId: string) => Effect.Effect; - readonly probeRuntime?: (distro: string | null, linuxAppRoot: string) => ProbeWslRuntimeResult; + readonly probeRuntime?: (distro: string | null) => ProbeWslRuntimeResult; readonly ensureNodePty?: ( distro: string | null, linuxAppRoot: string, @@ -1137,16 +1230,14 @@ export const layerTest = (stub: DesktopWslEnvironmentTestStub = {}) => { return stub.invalidateRuntime?.(distro, runtimeId) ?? Effect.void; }, ), - probeRuntime: Effect.fn("DesktopWslEnvironment.probeRuntime")( - function (distro, linuxAppRoot) { - return Effect.succeed( - stub.probeRuntime?.(distro, linuxAppRoot) ?? { - ok: true, - resolvedPath: "/usr/bin:/bin", - }, - ); - }, - ), + probeRuntime: Effect.fn("DesktopWslEnvironment.probeRuntime")(function (distro) { + return Effect.succeed( + stub.probeRuntime?.(distro) ?? { + ok: true, + resolvedPath: "/usr/bin:/bin", + }, + ); + }), ensureNodePty: Effect.fn("DesktopWslEnvironment.ensureNodePty")( function (distro, linuxAppRoot, options) { return Effect.succeed( @@ -1224,7 +1315,11 @@ export const layer = Layer.effect( getUserHome, getDistroIp, prepareRuntime: Effect.fn("desktop.wsl.prepareRuntime")(function (distro, archive) { - return provideSpawner(prepareWslRuntimeImpl(distro, archive, windowsToWslPath)); + return provideSpawner( + prepareWslRuntimeImpl(distro, archive, windowsToWslPath).pipe( + Effect.provideService(FileSystem.FileSystem, fileSystem), + ), + ); }), pruneRuntimes: Effect.fn("desktop.wsl.pruneRuntimes")(function (distro, runtimeId) { return provideSpawner(pruneWslRuntimesImpl(distro, runtimeId)); @@ -1232,8 +1327,8 @@ export const layer = Layer.effect( invalidateRuntime: Effect.fn("desktop.wsl.invalidateRuntime")(function (distro, runtimeId) { return provideSpawner(invalidateWslRuntimeImpl(distro, runtimeId)); }), - probeRuntime: Effect.fn("desktop.wsl.probeRuntime")(function (distro, linuxAppRoot) { - return provideSpawner(probeWslRuntimeImpl(distro, linuxAppRoot)); + probeRuntime: Effect.fn("desktop.wsl.probeRuntime")(function (distro) { + return provideSpawner(probeWslRuntimeImpl(distro)); }), ensureNodePty: Effect.fn("desktop.wsl.ensureNodePty")( function (distro, linuxAppRoot, options) { diff --git a/apps/desktop/src/wsl/DesktopWslServerTree.test.ts b/apps/desktop/src/wsl/DesktopWslServerTree.test.ts index 7eaf362a326a..34dcbea08a0e 100644 --- a/apps/desktop/src/wsl/DesktopWslServerTree.test.ts +++ b/apps/desktop/src/wsl/DesktopWslServerTree.test.ts @@ -6,7 +6,6 @@ import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as PlatformError from "effect/PlatformError"; import * as Ref from "effect/Ref"; -import * as Scope from "effect/Scope"; import * as DesktopConfig from "../app/DesktopConfig.ts"; import * as DesktopEnvironment from "../app/DesktopEnvironment.ts"; @@ -42,20 +41,14 @@ const layerEnvironment = (input: { ), ); -const withTempDir = ( - run: (tempDir: string) => Effect.Effect, -): Effect.Effect< - A, - E | PlatformError.PlatformError, - FileSystem.FileSystem | Exclude -> => +const withTempDir = (run: (tempDir: string) => Effect.Effect) => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; const tempDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-wsl-server-tree-test-", }); return yield* run(tempDir); - }).pipe(Effect.scoped); + }); const ensureWith = (input: { readonly baseDir: string; diff --git a/apps/desktop/src/wsl/wslPathParsing.test.ts b/apps/desktop/src/wsl/wslPathParsing.test.ts index 0a814229665d..ed491f58381a 100644 --- a/apps/desktop/src/wsl/wslPathParsing.test.ts +++ b/apps/desktop/src/wsl/wslPathParsing.test.ts @@ -77,8 +77,15 @@ describe("wslUncPathToLinuxPath", () => { expect(wslUncPathToLinuxPath("\\\\wsl.localhost\\Debian\\")).toBe("/"); }); + it("maps UNC paths of distros whose names start or end with a dot or hyphen", () => { + for (const name of ["Ubuntu-", "Ubuntu.", ".Ubuntu", "-Ubuntu"]) { + expect(wslUncPathToLinuxPath(`\\\\wsl.localhost\\${name}\\home\\josh`)).toBe("/home/josh"); + } + }); + it("rejects invalid distro names and non-WSL paths", () => { expect(wslUncPathToLinuxPath("\\\\wsl.localhost\\bad!name\\home")).toBeNull(); + expect(wslUncPathToLinuxPath("\\\\wsl.localhost\\..\\home")).toBeNull(); expect(wslUncPathToLinuxPath("C:\\Users\\Josh\\repo")).toBeNull(); }); }); @@ -107,9 +114,13 @@ describe("resolveWslPickFolderDefaultPath", () => { }); describe("isValidDistroName", () => { - it("rejects names with trailing whitespace, hyphen, or dot", () => { - for (const name of ["Ubuntu ", "Ubuntu-", "Ubuntu."]) { - expect(isValidDistroName(name)).toBe(false); + it("rejects names with trailing whitespace", () => { + expect(isValidDistroName("Ubuntu ")).toBe(false); + }); + + it("accepts names WSL registers with a leading or trailing dot or hyphen", () => { + for (const name of ["Ubuntu-", "Ubuntu.", ".Ubuntu", "-Ubuntu"]) { + expect(isValidDistroName(name)).toBe(true); } }); diff --git a/apps/desktop/src/wsl/wslPathParsing.ts b/apps/desktop/src/wsl/wslPathParsing.ts index a1ce99c129d1..1df64f91dac0 100644 --- a/apps/desktop/src/wsl/wslPathParsing.ts +++ b/apps/desktop/src/wsl/wslPathParsing.ts @@ -9,7 +9,8 @@ export interface WslConfig { } // Literal spaces only; \s would accept controls and corrupt UNC distro paths. -const DISTRO_NAME_PATTERN = /^\w(?:[\w \-.]*\w)?$/; +// "." and ".." would be path segments in a UNC path, not distro names. +const DISTRO_NAME_PATTERN = /^(?!\.{1,2}$)[\w.-](?:[\w .-]*[\w.-])?$/; export function parseWslDistroList(stdout: Buffer): readonly WslDistro[] { const hasUtf16Bom = stdout.length >= 2 && stdout[0] === 0xff && stdout[1] === 0xfe; diff --git a/apps/desktop/vite.config.ts b/apps/desktop/vite.config.ts index c0430d1e36fa..630345983daa 100644 --- a/apps/desktop/vite.config.ts +++ b/apps/desktop/vite.config.ts @@ -164,6 +164,9 @@ export default defineConfig({ // The Windows lane runs workspace suites concurrently; filesystem-heavy // desktop integration tests can exceed Vitest's 5 second default there. testTimeout: 15_000, + // `require("electron")` downloads a missing binary, so concurrent workers race to + // extract it into the same dist/. Install it once before any worker starts. + globalSetup: ["./scripts/ensure-electron-runtime.mjs"], projects: [ { extends: true, diff --git a/apps/marketing/package.json b/apps/marketing/package.json index 04f6830066a1..8e447d16b2c0 100644 --- a/apps/marketing/package.json +++ b/apps/marketing/package.json @@ -7,7 +7,8 @@ "stage:install-scripts": "node scripts/stage-install-scripts.mjs", "dev": "node scripts/stage-install-scripts.mjs && astro dev", "build": "node scripts/stage-install-scripts.mjs && astro build", - "preview": "astro preview" + "preview": "astro preview", + "test": "vp test run" }, "dependencies": { "@fontsource-variable/inter": "^5.3.0", diff --git a/apps/marketing/src/i18n/de.ts b/apps/marketing/src/i18n/de.ts index 648b35beebe0..96925adb2706 100644 --- a/apps/marketing/src/i18n/de.ts +++ b/apps/marketing/src/i18n/de.ts @@ -27,6 +27,8 @@ export const de: Messages = { downloadForMac: "Für macOS herunterladen", downloadForWindows: "Für Windows herunterladen", downloadForLinux: "Für Linux herunterladen", + downloadForIos: "Für iOS herunterladen", + downloadForAndroid: "Für Android herunterladen", sourceLink: "Klau unseren Code (legal)", alsoOnPhone: "Auch auf deinem Handy:", screenshotAlt: "T3 Code führt Subagents von Claude, Codex, Muse und Gemini in einem Thread aus", diff --git a/apps/marketing/src/i18n/en.ts b/apps/marketing/src/i18n/en.ts index 1769d81184c5..1a36424a1bfd 100644 --- a/apps/marketing/src/i18n/en.ts +++ b/apps/marketing/src/i18n/en.ts @@ -27,6 +27,8 @@ export const en = { downloadForMac: "Download for macOS", downloadForWindows: "Download for Windows", downloadForLinux: "Download for Linux", + downloadForIos: "Download for iOS", + downloadForAndroid: "Download for Android", sourceLink: "Steal our code (legally)", alsoOnPhone: "Also on your phone:", screenshotAlt: "T3 Code running Claude, Codex, Muse, and Gemini subagents in one thread", diff --git a/apps/marketing/src/i18n/es.ts b/apps/marketing/src/i18n/es.ts index cb4c28e9871f..90fcf79bd942 100644 --- a/apps/marketing/src/i18n/es.ts +++ b/apps/marketing/src/i18n/es.ts @@ -28,6 +28,8 @@ export const es: Messages = { downloadForMac: "Descargar para macOS", downloadForWindows: "Descargar para Windows", downloadForLinux: "Descargar para Linux", + downloadForIos: "Descargar para iOS", + downloadForAndroid: "Descargar para Android", sourceLink: "Roba nuestro código (legalmente)", alsoOnPhone: "También en tu teléfono:", screenshotAlt: "T3 Code ejecutando subagentes de Claude, Codex, Muse y Gemini en un mismo hilo", diff --git a/apps/marketing/src/i18n/fr.ts b/apps/marketing/src/i18n/fr.ts index 8fd918efc815..3d76747d6c83 100644 --- a/apps/marketing/src/i18n/fr.ts +++ b/apps/marketing/src/i18n/fr.ts @@ -27,6 +27,8 @@ export const fr: Messages = { downloadForMac: "Télécharger pour macOS", downloadForWindows: "Télécharger pour Windows", downloadForLinux: "Télécharger pour Linux", + downloadForIos: "Télécharger pour iOS", + downloadForAndroid: "Télécharger pour Android", sourceLink: "Volez notre code (légalement)", alsoOnPhone: "Aussi sur votre téléphone :", screenshotAlt: diff --git a/apps/marketing/src/i18n/ja.ts b/apps/marketing/src/i18n/ja.ts index 55117b609ac0..9f9fdba51129 100644 --- a/apps/marketing/src/i18n/ja.ts +++ b/apps/marketing/src/i18n/ja.ts @@ -28,6 +28,8 @@ export const ja: Messages = { downloadForMac: "macOS 版をダウンロード", downloadForWindows: "Windows 版をダウンロード", downloadForLinux: "Linux 版をダウンロード", + downloadForIos: "iOS 版をダウンロード", + downloadForAndroid: "Android 版をダウンロード", sourceLink: "コードを盗もう(合法的に)", alsoOnPhone: "スマホでも使えます:", screenshotAlt: diff --git a/apps/marketing/src/i18n/ko.ts b/apps/marketing/src/i18n/ko.ts index f2c09c0af06a..e2f4d4e6f7c7 100644 --- a/apps/marketing/src/i18n/ko.ts +++ b/apps/marketing/src/i18n/ko.ts @@ -27,6 +27,8 @@ export const ko: Messages = { downloadForMac: "macOS용 다운로드", downloadForWindows: "Windows용 다운로드", downloadForLinux: "Linux용 다운로드", + downloadForIos: "iOS용 다운로드", + downloadForAndroid: "Android용 다운로드", sourceLink: "코드 훔쳐 가기(합법적으로)", alsoOnPhone: "휴대폰에서도:", screenshotAlt: "하나의 스레드에서 Claude, Codex, Muse, Gemini 서브에이전트를 실행 중인 T3 Code", diff --git a/apps/marketing/src/i18n/pt-BR.ts b/apps/marketing/src/i18n/pt-BR.ts index 33841e839c52..1947884d3351 100644 --- a/apps/marketing/src/i18n/pt-BR.ts +++ b/apps/marketing/src/i18n/pt-BR.ts @@ -27,6 +27,8 @@ export const ptBR: Messages = { downloadForMac: "Baixar para macOS", downloadForWindows: "Baixar para Windows", downloadForLinux: "Baixar para Linux", + downloadForIos: "Baixar para iOS", + downloadForAndroid: "Baixar para Android", sourceLink: "Roube nosso código (legalmente)", alsoOnPhone: "Também no seu celular:", screenshotAlt: diff --git a/apps/marketing/src/i18n/ru.ts b/apps/marketing/src/i18n/ru.ts index 8fdeefdb9fad..e0d357813bc6 100644 --- a/apps/marketing/src/i18n/ru.ts +++ b/apps/marketing/src/i18n/ru.ts @@ -27,6 +27,8 @@ export const ru: Messages = { downloadForMac: "Скачать для macOS", downloadForWindows: "Скачать для Windows", downloadForLinux: "Скачать для Linux", + downloadForIos: "Скачать для iOS", + downloadForAndroid: "Скачать для Android", sourceLink: "Украдите наш код (легально)", alsoOnPhone: "Есть и для телефона:", screenshotAlt: "T3 Code запускает субагентов Claude, Codex, Muse и Gemini в одном треде", diff --git a/apps/marketing/src/i18n/zh.ts b/apps/marketing/src/i18n/zh.ts index bae711530051..1c554b193c40 100644 --- a/apps/marketing/src/i18n/zh.ts +++ b/apps/marketing/src/i18n/zh.ts @@ -27,6 +27,8 @@ export const zh: Messages = { downloadForMac: "下载 macOS 版", downloadForWindows: "下载 Windows 版", downloadForLinux: "下载 Linux 版", + downloadForIos: "下载 iOS 版", + downloadForAndroid: "下载 Android 版", sourceLink: "来偷我们的代码(合法的)", alsoOnPhone: "手机上也能用:", screenshotAlt: "T3 Code 在同一个线程中运行 Claude、Codex、Muse 和 Gemini 子智能体", diff --git a/apps/marketing/src/layouts/Layout.astro b/apps/marketing/src/layouts/Layout.astro index 0f0a86410174..fa7fa30bc0b1 100644 --- a/apps/marketing/src/layouts/Layout.astro +++ b/apps/marketing/src/layouts/Layout.astro @@ -7,8 +7,8 @@ import { ANDROID_PLAY_STORE_URL, GITHUB_REPOSITORY_URL, IOS_APP_STORE_URL, - MARKETING_STATS, } from "../lib/site"; +import { MARKETING_STATS } from "../lib/stats"; import { type JsonLd, jsonLdGraph, organizationJsonLd, websiteJsonLd } from "../lib/seo"; import { messagesFor } from "../i18n"; import { diff --git a/apps/marketing/src/lib/platform.test.ts b/apps/marketing/src/lib/platform.test.ts new file mode 100644 index 000000000000..d35abb2ac63a --- /dev/null +++ b/apps/marketing/src/lib/platform.test.ts @@ -0,0 +1,45 @@ +import { expect, it } from "vite-plus/test"; + +import { detectPlatform, getMobileStoreUrl } from "./platform"; +import { ANDROID_PLAY_STORE_URL, IOS_APP_STORE_URL } from "./site"; + +it.each([ + { + // Android user agents also contain "Linux". + name: "Android", + userAgent: + "Mozilla/5.0 (Linux; Android 15; Pixel 9) AppleWebKit/537.36 Chrome/140.0.0.0 Mobile Safari/537.36", + maxTouchPoints: 5, + os: "android", + storeUrl: ANDROID_PLAY_STORE_URL, + }, + { + name: "iPhone", + userAgent: + "Mozilla/5.0 (iPhone; CPU iPhone OS 18_6 like Mac OS X) AppleWebKit/605.1.15 Version/18.6 Mobile/15E148 Safari/604.1", + maxTouchPoints: 5, + os: "ios", + storeUrl: IOS_APP_STORE_URL, + }, + { + // iPad Safari sends a desktop Mac user agent by default. + name: "iPad with a desktop user agent", + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) AppleWebKit/605.1.15 Version/18.6 Safari/605.1.15", + maxTouchPoints: 5, + os: "ios", + storeUrl: IOS_APP_STORE_URL, + }, + { + name: "macOS", + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 Version/18.6 Safari/605.1.15", + maxTouchPoints: 0, + os: "mac", + storeUrl: null, + }, +])("$name visitors get the $os download", ({ userAgent, maxTouchPoints, os, storeUrl }) => { + const platform = detectPlatform(userAgent, maxTouchPoints); + expect(platform?.os).toBe(os); + expect(platform && getMobileStoreUrl(platform)).toBe(storeUrl); +}); diff --git a/apps/marketing/src/lib/platform.ts b/apps/marketing/src/lib/platform.ts new file mode 100644 index 000000000000..be17e789a6bb --- /dev/null +++ b/apps/marketing/src/lib/platform.ts @@ -0,0 +1,22 @@ +import { ANDROID_PLAY_STORE_URL, IOS_APP_STORE_URL } from "./site"; + +export type Platform = { os: "mac" | "win" | "linux" | "ios" | "android" }; + +export function detectPlatform(userAgent: string, maxTouchPoints: number): Platform | null { + // Mobile checks come first: Android user agents contain "Linux" and iOS + // user agents contain "Mac OS X". + if (/Android/i.test(userAgent)) return { os: "android" }; + if (/iPhone|iPad|iPod/i.test(userAgent)) return { os: "ios" }; + if (/Win/i.test(userAgent)) return { os: "win" }; + // iPadOS Safari sends a Mac user agent by default; only the touch screen + // tells it apart from a Mac. + if (/Mac/i.test(userAgent)) return { os: maxTouchPoints > 1 ? "ios" : "mac" }; + if (/Linux/i.test(userAgent)) return { os: "linux" }; + return null; +} + +export function getMobileStoreUrl(platform: Platform): string | null { + if (platform.os === "ios") return IOS_APP_STORE_URL; + if (platform.os === "android") return ANDROID_PLAY_STORE_URL; + return null; +} diff --git a/apps/marketing/src/lib/releases.test.ts b/apps/marketing/src/lib/releases.test.ts new file mode 100644 index 000000000000..6527342d674e --- /dev/null +++ b/apps/marketing/src/lib/releases.test.ts @@ -0,0 +1,18 @@ +import { afterEach, describe, expect, it, vi } from "vite-plus/test"; + +import { fetchLatestRelease } from "./releases"; + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("fetchLatestRelease", () => { + it("rejects a GitHub API error response instead of returning it as a release", async () => { + vi.stubGlobal("sessionStorage", { getItem: () => null }); + vi.stubGlobal("fetch", async () => + Response.json({ message: "API rate limit exceeded" }, { status: 403 }), + ); + + await expect(fetchLatestRelease("stable")).rejects.toThrow("403"); + }); +}); diff --git a/apps/marketing/src/lib/releases.ts b/apps/marketing/src/lib/releases.ts index 3a683d2350ea..02fe0e464680 100644 --- a/apps/marketing/src/lib/releases.ts +++ b/apps/marketing/src/lib/releases.ts @@ -25,12 +25,18 @@ function cacheKey(channel: ReleaseChannel) { return `t3code-${channel}-release`; } +async function fetchJson(url: string): Promise { + const response = await fetch(url); + if (!response.ok) throw new Error(`GitHub API responded ${response.status}`); + return response.json(); +} + async function fetchStable(): Promise { - return fetch(LATEST_API_URL).then((r) => r.json()); + return fetchJson(LATEST_API_URL); } async function fetchNightly(): Promise { - const list: Release[] = await fetch(LIST_API_URL).then((r) => r.json()); + const list = await fetchJson(LIST_API_URL); const nightly = Array.isArray(list) ? list.find((release) => release.tag_name?.includes("-nightly.")) : undefined; diff --git a/apps/marketing/src/lib/site.ts b/apps/marketing/src/lib/site.ts index c4cc8aca59f2..319f6dd800bc 100644 --- a/apps/marketing/src/lib/site.ts +++ b/apps/marketing/src/lib/site.ts @@ -5,26 +5,3 @@ export const IOS_APP_STORE_URL = export const ANDROID_PLAY_STORE_URL = "https://play.google.com/store/apps/details?id=com.t3tools.t3code"; - -// Runs at build time, so each deploy picks up the current count. -async function fetchGithubStars(): Promise { - const token = process.env.GITHUB_TOKEN; - try { - const response = await fetch("https://api.github.com/repos/pingdotgg/t3code", { - headers: token ? { Authorization: `Bearer ${token}` } : {}, - }); - if (!response.ok) throw new Error(`GitHub API responded ${response.status}`); - const { stargazers_count: stars } = (await response.json()) as { stargazers_count: number }; - return stars < 1000 ? String(stars) : `${Math.floor(stars / 1000)}k+`; - } catch (error) { - // A deploy must not ship a made-up count; offline dev should still render. - if (import.meta.env.PROD) throw new Error("Could not fetch GitHub stars", { cause: error }); - console.warn("Could not fetch GitHub stars", error); - return "?"; - } -} - -export const MARKETING_STATS = { - githubStars: await fetchGithubStars(), - users: "500,000", -} as const; diff --git a/apps/marketing/src/lib/stats.ts b/apps/marketing/src/lib/stats.ts new file mode 100644 index 000000000000..d291f420b06d --- /dev/null +++ b/apps/marketing/src/lib/stats.ts @@ -0,0 +1,22 @@ +// Runs at build time, so each deploy picks up the current count. +async function fetchGithubStars(): Promise { + const token = process.env.GITHUB_TOKEN; + try { + const response = await fetch("https://api.github.com/repos/pingdotgg/t3code", { + headers: token ? { Authorization: `Bearer ${token}` } : {}, + }); + if (!response.ok) throw new Error(`GitHub API responded ${response.status}`); + const { stargazers_count: stars } = (await response.json()) as { stargazers_count: number }; + return stars < 1000 ? String(stars) : `${Math.floor(stars / 1000)}k+`; + } catch (error) { + // A deploy must not ship a made-up count; offline dev should still render. + if (import.meta.env.PROD) throw new Error("Could not fetch GitHub stars", { cause: error }); + console.warn("Could not fetch GitHub stars", error); + return "?"; + } +} + +export const MARKETING_STATS = { + githubStars: await fetchGithubStars(), + users: "500,000", +} as const; diff --git a/apps/marketing/src/pages/95.astro b/apps/marketing/src/pages/95.astro index 4ddbb9eeca7a..24c66f72be92 100644 --- a/apps/marketing/src/pages/95.astro +++ b/apps/marketing/src/pages/95.astro @@ -1,7 +1,8 @@ --- import RetroIcon from "../components/RetroIcon.astro"; import RetroBox from "../components/RetroBox.astro"; -import { GITHUB_REPOSITORY_URL, MARKETING_STATS } from "../lib/site"; +import { GITHUB_REPOSITORY_URL } from "../lib/site"; +import { MARKETING_STATS } from "../lib/stats"; import "../styles/retro.css"; const agents = [ diff --git a/apps/marketing/src/pages/[...locale]/index.astro b/apps/marketing/src/pages/[...locale]/index.astro index bd652d1d9864..7ede25b8ac98 100644 --- a/apps/marketing/src/pages/[...locale]/index.astro +++ b/apps/marketing/src/pages/[...locale]/index.astro @@ -6,9 +6,9 @@ import { ANDROID_PLAY_STORE_URL, GITHUB_REPOSITORY_URL, IOS_APP_STORE_URL, - MARKETING_STATS, } from "../../lib/site"; import { RELEASES_URL } from "../../lib/releases"; +import { MARKETING_STATS } from "../../lib/stats"; import { tweets } from "../../lib/tweets"; import { softwareJsonLd } from "../../lib/seo"; import { messagesFor } from "../../i18n"; @@ -84,6 +84,8 @@ const mobileEndorsementRows = [ data-label-mac={t.downloadForMac} data-label-win={t.downloadForWindows} data-label-linux={t.downloadForLinux} + data-label-ios={t.downloadForIos} + data-label-android={t.downloadForAndroid} > @@ -414,16 +416,7 @@ const mobileEndorsementRows = [ `); + await expect( + ServerBrowserPage.click(page, { selector: "#save", timeoutMs: 2_000 }), + ).rejects.toThrow(); + expect(await page.locator("body").getAttribute("data-clicked")).toBeNull(); + }); + it("does not retarget a removed ref to a replacement node", async () => { await page.setContent(""); const locator = buttonLocator((await takeSnapshot()).accessibilityTree, "original"); @@ -157,6 +375,25 @@ describe("server browser element refs", () => { expect(await page.getByRole("textbox", { name: "parent" }).inputValue()).toBe(""); }); + it("types after a click by coordinates into a shadow-root field, and refuses a field inside an editable frame", async () => { + await page.setContent( + `
`, + ); + const field = page.locator("#host #field"); + const box = (await field.boundingBox())!; + await page.mouse.click(box.x + box.width / 2, box.y + box.height / 2); + await ServerBrowserPage.type(page, { text: "shadow" }); + expect(await field.inputValue()).toBe("shadow"); + + // Refused for now: focus inside a frame cannot be checked from the top document. + const inner = page.frameLocator("iframe").locator("#inner"); + await inner.click(); + await expect(ServerBrowserPage.type(page, { text: "frame" })).rejects.toMatchObject({ + tag: "PreviewAutomationTargetNotEditableError", + }); + expect(await inner.inputValue()).toBe(""); + }); + it("shows dotted fields as dots wherever their value appears, keeping text values and refs", async () => { await page.setContent(`
password:
@@ -391,7 +628,7 @@ describe("server browser snapshot screenshot", () => { const screenshotOf = async (html: string) => { const page = await context.newPage(); await page.setContent(html); - const { screenshot, fullScreenshot } = await ServerBrowserPage.snapshot({ + const input = { page, cdp: await context.newCDPSession(page), renderScale: 2, @@ -399,7 +636,8 @@ describe("server browser snapshot screenshot", () => { consoleEntries: [], networkEntries: [], actionTimeline: [], - }); + }; + const { screenshot, fullScreenshot } = await ServerBrowserPage.snapshot(input); // The server has no DOM types, so the decoder runs as page source. const points = [ [0.15, 0.75], @@ -418,7 +656,7 @@ describe("server browser snapshot screenshot", () => { Array.from(context.getImageData(x * bitmap.width, y * bitmap.height, 1, 1).data), ); })()`)) as Array<[number, number, number, number]>; - return { screenshot, fullScreenshot, pixels }; + return { screenshot, fullScreenshot, pixels, input }; }; const colorOf = ([red, green, blue]: readonly [number, number, number, number]) => blue - red > 100 ? "blue" : red - blue > 100 ? "red" : `rgb(${red}, ${green}, ${blue})`; @@ -433,13 +671,20 @@ describe("server browser snapshot screenshot", () => { }); it("captures a plain page at the shrunk size with its content in place", async () => { - const { screenshot, fullScreenshot, pixels } = await screenshotOf(halves("")); + const { screenshot, fullScreenshot, pixels, input } = await screenshotOf(halves("")); + const metadata = await ServerBrowserPage.snapshotMetadata(input); expect({ size: [screenshot.width, screenshot.height], fullSize: [fullScreenshot?.width, fullScreenshot?.height], + metadataSize: metadata.screenshot, colors: pixels.map(colorOf), - }).toEqual({ size: [1280, 800], fullSize: [2560, 1600], colors: ["blue", "red", "red"] }); + }).toEqual({ + size: [1280, 800], + fullSize: [2560, 1600], + metadataSize: { width: 1280, height: 800 }, + colors: ["blue", "red", "red"], + }); }); it("records the destination of a finite entrance animation without waiting on infinite ones", async () => { @@ -532,6 +777,144 @@ describe("server browser drag", () => { process.off("unhandledRejection", onUnhandled); } }); + + it("moves the mouse after dragstart, so pages that track a drag with mousemove see it", async () => { + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); + await page.setContent(` +
+
source
+
target
+
+ `); + + await ServerBrowserPage.drag(page, { source: "#source", target: "#target" }); + + const events = (await page.evaluate("window.events")) as Array; + expect(events.slice(0, 2)).toEqual(["dragstart", "mousemove"]); + expect(events.at(-1)).toBe("mouseup"); + } finally { + await browser.close(); + } + }); +}); + +describe("server browser action timeout", () => { + type StepOptions = { readonly timeout?: number }; + it.each([ + { + action: "hover", + act: (page: Page) => ServerBrowserPage.hover(page, { selector: "#a", timeoutMs: 9_000 }), + last: ["#a.hover 1"], + }, + { + action: "drag", + act: (page: Page) => + ServerBrowserPage.drag(page, { source: "#a", target: "#b", timeoutMs: 9_000 }), + last: ["#a.dragTo 1", "#b.boundingBox 1"], + }, + ])("gives the $action's steps one shared timeout", async ({ act, last }) => { + let now = 0; + using _now = vi.spyOn(Date, "now").mockImplementation(() => now); + const steps: Array = []; + const box = { x: 0, y: 0, width: 20, height: 20 }; + const page = { + locator: (selector: string) => { + const record = (step: string, options: StepOptions) => + steps.push(`${selector}.${step} ${options.timeout}`); + return { + // Each step that finds the target takes 5 of the 9 seconds. + scrollIntoViewIfNeeded: async (options: StepOptions) => { + record("scrollIntoViewIfNeeded", options); + now += 5_000; + }, + boundingBox: async (options: StepOptions) => { + record("boundingBox", options); + now += 5_000; + return box; + }, + hover: async (options: StepOptions) => record("hover", options), + dragTo: async (_target: unknown, options: StepOptions) => record("dragTo", options), + }; + }, + } as unknown as Page; + + await act(page); + + expect(steps).toEqual(["#a.scrollIntoViewIfNeeded 9000", "#a.boundingBox 4000", ...last]); + }); +}); + +describe("server browser click", () => { + it("clicks a target the page replaced while it was being scrolled into view", async () => { + let scrolls = 0; + const clicks: Array = []; + const page = { + locator: () => ({ + // Playwright resolves the node once; a page re-rendering it detaches that node. + scrollIntoViewIfNeeded: async () => { + if (scrolls++ === 0) { + throw new Error("locator.scrollIntoViewIfNeeded: Element is not attached to the DOM"); + } + }, + boundingBox: async () => ({ x: 0, y: 0, width: 20, height: 20 }), + click: async (options: unknown) => { + clicks.push(options); + }, + count: async () => 1, + }), + once: () => {}, + off: () => {}, + } as unknown as Page; + + await expect(ServerBrowserPage.click(page, { selector: "#save" })).resolves.toEqual({ + x: 10, + y: 10, + }); + expect(clicks).toHaveLength(1); + }); + + it("retries a replaced target only within the action's timeout", async () => { + vi.useFakeTimers(); + try { + const timeouts: Array = []; + const clicks: Array = []; + const detached = new Error( + "locator.scrollIntoViewIfNeeded: Element is not attached to the DOM", + ); + const page = { + locator: () => ({ + scrollIntoViewIfNeeded: async (options?: { timeout?: number }) => { + timeouts.push(options?.timeout); + vi.advanceTimersByTime(60); + throw detached; + }, + click: async (options: unknown) => { + clicks.push(options); + }, + }), + } as unknown as Page; + + await expect( + ServerBrowserPage.click(page, { selector: "#save", timeoutMs: 100 }), + ).rejects.toBe(detached); + expect(timeouts).toEqual([100, 40]); + expect(clicks).toHaveLength(0); + } finally { + vi.useRealTimers(); + } + }); }); describe("server browser evaluate", () => { @@ -568,6 +951,63 @@ describe("server browser evaluate", () => { }); }); +describe("server browser type", () => { + // As on a desktop tab whose Playwright utility world never answers. + const stalledPage = (focusAnswer: () => Promise, typed: string[]) => + ({ + locator: () => ({ last: () => ({ count: () => new Promise(() => {}) }) }), + evaluate: focusAnswer, + keyboard: { insertText: async (text: string) => void typed.push(text) }, + }) as unknown as Page; + + it("types into the focused field when Playwright's utility world never answers", async () => { + vi.useFakeTimers(); + try { + const typed: string[] = []; + let outcome: unknown = "pending"; + const page = stalledPage(async () => true, typed); + void ServerBrowserPage.type(page, { text: "hello", timeoutMs: 300 }).then( + () => (outcome = "typed"), + (error: unknown) => (outcome = error), + ); + await vi.advanceTimersByTimeAsync(300); + expect(outcome).toBe("typed"); + expect(typed).toEqual(["hello"]); + } finally { + vi.useRealTimers(); + } + }); + + it("stops at its timeout without typing when the page never answers", async () => { + vi.useFakeTimers(); + try { + const typed: string[] = []; + let outcome: unknown = "pending"; + const page = stalledPage(() => new Promise(() => {}), typed); + void ServerBrowserPage.type(page, { text: "hello", timeoutMs: 300 }).then( + () => (outcome = "typed"), + (error: unknown) => (outcome = error), + ); + await vi.advanceTimersByTimeAsync(300); + expect(outcome).toMatchObject({ + tag: "PreviewAutomationTimeoutError", + detail: { timeoutMs: 300 }, + }); + expect(typed).toEqual([]); + } finally { + vi.useRealTimers(); + } + }); +}); + +describe("server browser unresponsive script", () => { + it("leaves a page that answers alone", async () => { + const send = vi.fn(async (_method: string) => ({})); + await ServerBrowserPage.stopUnresponsiveScript({ send } as unknown as CDPSession); + expect(send.mock.calls.map(([method]) => method)).toEqual(["Runtime.evaluate"]); + }); +}); + describe("server browser wait", () => { it("ends at its timeout when a check never answers", async () => { // As on a desktop tab whose Playwright utility world never answers. diff --git a/apps/server/src/preview/ServerBrowserPage.ts b/apps/server/src/preview/ServerBrowserPage.ts index 29ba05c21b7e..ac37b9914ef7 100644 --- a/apps/server/src/preview/ServerBrowserPage.ts +++ b/apps/server/src/preview/ServerBrowserPage.ts @@ -14,6 +14,8 @@ import { type PreviewAutomationSelectInput, type PreviewAutomationSelectResult, type PreviewAutomationSnapshot, + type PreviewAutomationSnapshotMetadata, + type PreviewAutomationSnapshotStage, type PreviewAutomationTypeInput, type PreviewAutomationUploadInput, type PreviewAutomationWaitForInput, @@ -22,6 +24,7 @@ import { constVoid } from "effect/Function"; import * as Schema from "effect/Schema"; import type { CDPSession, ConsoleMessage, Frame, Locator, Page } from "playwright-core"; import * as NodeCrypto from "node:crypto"; +import * as NodeUtil from "node:util"; import { BrowserControlInterrupted } from "./SessionControl.ts"; const isBrowserControlInterrupted = Schema.is(BrowserControlInterrupted); @@ -29,6 +32,8 @@ const isBrowserControlInterrupted = Schema.is(BrowserControlInterrupted); const MAX_EVALUATION_BYTES = 64_000; const MAX_VISIBLE_TEXT_LENGTH = 20_000; const MAX_SCREENSHOT_WIDTH = 1280; +// Two attempts must fit in a snapshot's 10-second deadline. +const SCREENSHOT_TIMEOUT_MS = 3_000; const WAIT_POLL_MS = 100; export const DIAGNOSTIC_BUFFER_LIMIT = 200; @@ -60,7 +65,7 @@ export const toOperationError = ( if (cause instanceof Error && cause.name === "TimeoutError") { return new ServerBrowserOperationError({ tag: "PreviewAutomationTimeoutError", - message: firstLine, + message: withActionabilityReason(firstLine, cause), detail: timeoutMs === undefined ? undefined : { timeoutMs }, }); } @@ -82,6 +87,17 @@ export const toOperationError = ( }); }; +/** Playwright puts why an element was not actionable in its call log, below the first line. */ +const withActionabilityReason = (message: string, cause: Error) => { + // Playwright dims call-log lines with ANSI escapes when colour is on. + const reason = NodeUtil.stripVTControlCharacters(cause.message) + .split("\n") + .map((line) => line.trim()) + .findLast((line) => line.startsWith("- element is ")) + ?.slice(2); + return reason === undefined ? message : `${message} Playwright last saw: ${reason}.`; +}; + const DEFAULT_TIMEOUT_MS = 15_000; const TARGET_COUNT_MS = 250; const MAX_TARGET_CHARS = 200; @@ -145,15 +161,32 @@ export type PointerReporter = ( ) => Promise; const noPointer: PointerReporter = async () => {}; +// Locator.scrollIntoViewIfNeeded resolves its node once and fails when the page replaces it; the +// input actions after it re-resolve, so the scroll does too, within the same timeout. +const DETACHED = /^locator\.scrollIntoViewIfNeeded: Element is not attached to the DOM(?:\n|$)/; + +const scrollIntoView = async (locator: Locator, timeout: number) => { + const deadline = Date.now() + timeout; + for (;;) { + try { + return await locator.scrollIntoViewIfNeeded({ timeout: Math.max(1, deadline - Date.now()) }); + } catch (cause) { + if (!(cause instanceof Error && DETACHED.test(cause.message)) || Date.now() >= deadline) { + throw cause; + } + } + } +}; + const targetPoint = async ( page: Page, locator: Locator | null, input: { readonly x?: number | undefined; readonly y?: number | undefined }, - timeout: number, + timeout: () => number, ) => { if (locator === null) return { x: input.x ?? 0, y: input.y ?? 0 }; - await locator.scrollIntoViewIfNeeded({ timeout }); - const box = await locator.boundingBox({ timeout }); + await scrollIntoView(locator, timeout()); + const box = await locator.boundingBox({ timeout: timeout() }); if (box) return { x: box.x + box.width / 2, y: box.y + box.height / 2 }; const viewport = page.viewportSize(); return { x: (viewport?.width ?? 0) / 2, y: (viewport?.height ?? 0) / 2 }; @@ -246,13 +279,36 @@ const pageChanged = () => message: "The page changed while capturing its snapshot. Take another snapshot.", }); -const tooBusy = () => +type SnapshotStage = typeof PreviewAutomationSnapshotStage.Type; + +/** + * Running stages are counted, since a retry can overlap the previous attempt's unanswered + * screenshot. A stage whose own timeout fired has stopped running but is still the cause. + */ +interface SnapshotStages { + readonly running: Map; + readonly timedOut: Set; +} + +const snapshotTimedOut = (stages?: SnapshotStages) => new ServerBrowserOperationError({ tag: "PreviewAutomationTimeoutError", - message: - "The page was too busy to capture a snapshot in time. Wait for it to settle and take another snapshot.", + message: `Snapshot capture timed out after ${SNAPSHOT_TIMEOUT_MS}ms. Try the snapshot again.`, + detail: { + timeoutMs: SNAPSHOT_TIMEOUT_MS, + pendingStages: stages ? [...new Set([...stages.timedOut, ...stages.running.keys()])] : [], + }, }); +const frameUnanswered = () => + new ServerBrowserOperationError({ + tag: "PreviewAutomationTimeoutError", + message: `A frame on the page didn't answer within ${FRAME_ANSWER_MS}ms, so its content couldn't be checked for hidden values. Try the snapshot again.`, + detail: { timeoutMs: FRAME_ANSWER_MS, pendingStages: ["frames"] }, + }); + +const isTimeout = (cause: unknown) => cause instanceof Error && cause.name === "TimeoutError"; + const VALUE_TOO_LONG = "A password field holds a value too long to hide, so no snapshot was taken."; /** Reuse one CDP session per out-of-process frame across snapshots. */ @@ -428,6 +484,37 @@ const pngSize = (base64: string) => { return { width: header.readUInt32BE(16), height: header.readUInt32BE(20) }; }; +export const jpegSize = (base64: string) => { + const bytes = Buffer.from(base64, "base64"); + if (bytes.length < 4 || bytes.readUInt16BE(0) !== 0xffd8) + throw new Error("Invalid JPEG screenshot."); + let offset = 2; + while (offset < bytes.length) { + if (bytes[offset++] !== 0xff) throw new Error("Invalid JPEG screenshot."); + while (bytes[offset] === 0xff) offset++; + const marker = bytes[offset++]; + if (marker === undefined || marker === 0xd9 || marker === 0xda) break; + if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) continue; + if (offset + 2 > bytes.length) break; + const length = bytes.readUInt16BE(offset); + if (length < 2 || offset + length > bytes.length) break; + if (marker >= 0xc0 && marker <= 0xcf && ![0xc4, 0xc8, 0xcc].includes(marker)) { + if (length < 7) break; + const height = bytes.readUInt16BE(offset + 3); + const width = bytes.readUInt16BE(offset + 5); + if (width && height) return { width, height }; + break; + } + offset += length; + } + throw new Error("Invalid JPEG screenshot."); +}; + +const scaledSize = (size: { readonly width: number; readonly height: number }, scale: number) => + scale < 1 + ? { width: Math.round(size.width * scale), height: Math.round(size.height * scale) } + : size; + /** * A page's display ratio and CSS viewport, read in an isolated world since page script can * reassign `devicePixelRatio`, `innerWidth`, and `innerHeight` in its own, and its page zoom. @@ -468,10 +555,27 @@ export const captureViewport = async ( cdp: CDPSession, options: { readonly format: "png" | "jpeg"; readonly quality?: number }, ) => { - const { data } = await cdp.send("Page.captureScreenshot", { + const capture = cdp.send("Page.captureScreenshot", { format: options.format, ...(options.quality === undefined ? {} : { quality: options.quality }), }); + let timer: ReturnType | undefined; + const expired = new Promise((_, reject) => { + timer = setTimeout( + () => + reject( + new ServerBrowserOperationError({ + tag: "PreviewAutomationExecutionError", + message: + "The browser is not producing screenshot frames. For a desktop tab, show or restore the T3 Code window and try again.", + }), + ), + SCREENSHOT_TIMEOUT_MS, + ); + }); + // Hidden native guests can leave CDP captures pending forever. This deadline also + // releases a viewer's still-capture lock, which an agent snapshot may be waiting for. + const { data } = await Promise.race([capture, expired]).finally(() => clearTimeout(timer)); return data; }; @@ -561,41 +665,73 @@ const renderingOf = async (input: SnapshotInput) => { const capture = async ( input: SnapshotInput, deadline: number, -): Promise => { - const { renderScale, viewport } = await renderingOf(input); + stages: SnapshotStages, + metadataOnly: boolean, +) => { + const { running } = stages; + const track = async
(stage: SnapshotStage, operation: () => Promise) => { + running.set(stage, (running.get(stage) ?? 0) + 1); + try { + const result = await operation(); + stages.timedOut.delete(stage); + return result; + } catch (cause) { + if (isTimeout(cause)) stages.timedOut.add(stage); + throw cause; + } finally { + const count = (running.get(stage) ?? 1) - 1; + if (count === 0) running.delete(stage); + else running.set(stage, count); + } + }; + const { renderScale, viewport } = await track("metrics", () => renderingOf(input)); // A late metrics reply must not begin a capture after viewers have resumed. - if (Date.now() >= deadline) throw tooBusy(); + if (Date.now() >= deadline) throw snapshotTimedOut(); const scale = Math.min(1, MAX_SCREENSHOT_WIDTH / (viewport.width * renderScale)); const state = refsFor(input.page); invalidateRefs(input.page); const generation = state.generation; - const { readers, read } = await openReaders(input.page, input.cdp); - const before = await read(); + const { readers, read } = await track("frames", () => openReaders(input.page, input.cdp)); + const before = await track("frames", () => read()); // Bracket the tree with value reads to detect changes; await tree completion before retrying failed captures. const pageAndScreenshot = Promise.all([ - input.page.evaluate(SNAPSHOT_SCRIPT) as Promise< - Pick< - PreviewAutomationSnapshot, - "url" | "title" | "loading" | "visibleText" | "interactiveElements" - > - >, - captureViewport(input.cdp, { format: "png" }).then(async (full) => - scale < 1 - ? { - data: await shrinkPng(input.cdp, full, scale), - full: input.fullScreenshot ? full : undefined, - } - : { data: full, full: undefined }, + track( + "page state", + () => + input.page.evaluate(SNAPSHOT_SCRIPT) as Promise< + Pick< + PreviewAutomationSnapshot, + "url" | "title" | "loading" | "visibleText" | "interactiveElements" + > + >, + ), + // A cheap real frame still sizes the screenshot; computing it from metrics drifted from the PNG. + track("screenshot", () => + metadataOnly + ? captureViewport(input.cdp, { format: "jpeg", quality: 10 }).then((data) => ({ + data, + full: undefined, + })) + : captureViewport(input.cdp, { format: "png" }).then(async (full) => + scale < 1 + ? { + data: await shrinkPng(input.cdp, full, scale), + full: input.fullScreenshot ? full : undefined, + } + : { data: full, full: undefined }, + ), ), ]); void pageAndScreenshot.catch(constVoid); // Leaves time for the second read; a frame that never loads holds the tree until then. - const tree = await input.page.ariaSnapshot({ - mode: "ai", - boxes: true, - timeout: Math.max(1, deadline - Date.now() - 2_000), - }); - const after = await read(before); + const tree = await track("accessibility tree", () => + input.page.ariaSnapshot({ + mode: "ai", + boxes: true, + timeout: Math.max(1, deadline - Date.now() - 2_000), + }), + ); + const after = await track("frames", () => read(before)); // A value that changed meanwhile may be in the tree under neither reading. if (after.some((values, i) => values && JSON.stringify(values) !== JSON.stringify(before[i]))) throw pageChanged(); @@ -604,11 +740,13 @@ const capture = async ( const unread = readers.flatMap((reader, i) => (after[i] ? [] : reader.frames)); const filled = /^\s*- iframe (?:\[active\] )?\[ref=(\S+?)\].*:$/gm; for (const [, ref] of unread.length ? tree.matchAll(filled) : []) { - const iframe = await input.page.locator(`aria-ref=${ref}`).elementHandle({ timeout: 2_000 }); + const iframe = await track("frames", () => + input.page.locator(`aria-ref=${ref}`).elementHandle({ timeout: 2_000 }), + ); if (!iframe) throw pageChanged(); - const frame = await iframe.contentFrame(); + const frame = await track("frames", () => iframe.contentFrame()); void iframe.dispose().catch(constVoid); - if (frame && unread.includes(frame)) throw tooBusy(); + if (frame && unread.includes(frame)) throw frameUnanswered(); } const [page, { full, data }] = await pageAndScreenshot; if (state.generation !== generation) throw pageChanged(); @@ -632,11 +770,10 @@ const capture = async ( consoleEntries: [...input.consoleEntries], networkEntries: [...input.networkEntries], actionTimeline: [...input.actionTimeline], - screenshot: { - mimeType: "image/png", - data, - ...pngSize(data), - }, + // Reports the size the PNG path would have shrunk to. + screenshot: metadataOnly + ? scaledSize(jpegSize(data), scale) + : { mimeType: "image/png" as const, data, ...pngSize(data) }, ...(full !== undefined ? { fullScreenshot: { data: full, ...pngSize(full) } } : {}), }; }; @@ -644,41 +781,50 @@ const capture = async ( /** Snapshots begun per page; observers' snapshots aren't queued, so two can overlap. */ const snapshotTurns = new WeakMap(); -export const snapshot = async (input: SnapshotInput): Promise => { +const runSnapshot = async (input: SnapshotInput, metadataOnly: boolean) => { const deadline = Date.now() + SNAPSHOT_TIMEOUT_MS; + const stages: SnapshotStages = { running: new Map(), timedOut: new Set() }; let timer: ReturnType | undefined; const expired = new Promise((_, reject) => { - timer = setTimeout(() => reject(tooBusy()), SNAPSHOT_TIMEOUT_MS); + timer = setTimeout(() => reject(snapshotTimedOut(stages)), SNAPSHOT_TIMEOUT_MS); }); const turn = (snapshotTurns.get(input.page) ?? 0) + 1; snapshotTurns.set(input.page, turn); // Retry only before the deadline and the next snapshot, whose refs a later retry would revoke. - const captured = capture(input, deadline).catch((cause: unknown) => + const captured = capture(input, deadline, stages, metadataOnly).catch((cause: unknown) => Date.now() < deadline && snapshotTurns.get(input.page) === turn && !(cause instanceof Error && cause.message === VALUE_TOO_LONG) - ? capture(input, deadline) + ? capture(input, deadline, stages, metadataOnly) : Promise.reject(cause), ); try { return await Promise.race([captured, expired]); } catch (cause) { - throw cause instanceof Error && cause.name === "TimeoutError" ? tooBusy() : cause; + throw isTimeout(cause) ? snapshotTimedOut(stages) : cause; } finally { clearTimeout(timer); } }; +export const snapshot = (input: SnapshotInput) => + runSnapshot(input, false) as Promise; + +export const snapshotMetadata = (input: SnapshotInput) => + runSnapshot(input, true) as Promise; + /** Return when a dialog opens; waiting for the action to finish would block until dialog resolution. */ export const click = async ( page: Page, input: PreviewAutomationClickInput, pointer: PointerReporter = noPointer, ): Promise<{ readonly x: number; readonly y: number }> => { - const timeout = input.timeoutMs ?? DEFAULT_TIMEOUT_MS; + // One deadline across steps keeps the reply inside the broker's grace; Playwright reads 0 as none. + const deadline = Date.now() + (input.timeoutMs ?? DEFAULT_TIMEOUT_MS); + const remaining = () => Math.max(1, deadline - Date.now()); const locator = targetLocator(page, input); try { - return await clickTarget(page, locator, input, timeout, pointer); + return await clickTarget(page, locator, input, remaining, pointer); } catch (cause) { throw locator === null ? cause : await namedTargetTimeout(locator, input, cause); } @@ -714,10 +860,11 @@ const namedTargetTimeout = async ( : matches === 0 ? `The click may have happened and removed its target or navigated; check the page before retrying. No element matches ${target} after the timeout; take a fresh snapshot and use its locator.` : `The click may have happened; check the page before retrying. ${target} still matches an element after the timeout.`; + const message = withActionabilityReason(reason, cause); return new ServerBrowserOperationError({ tag: "PreviewAutomationTimeoutError", - message: reason, - detail: { reason }, + message, + detail: { reason: message }, cause, }); }; @@ -726,7 +873,7 @@ const clickTarget = async ( page: Page, locator: Locator | null, input: PreviewAutomationClickInput, - timeout: number, + timeout: () => number, pointer: PointerReporter, ) => { const point = await targetPoint(page, locator, input, timeout); @@ -735,7 +882,7 @@ const clickTarget = async ( const clicked = locator === null ? page.mouse.click(point.x, point.y, options) - : locator.click({ ...options, timeout }); + : locator.click({ ...options, timeout: timeout() }); let onDialog = constVoid; const dialogOpened = new Promise<"dialog">((resolve) => { onDialog = () => resolve("dialog"); @@ -749,6 +896,76 @@ const clickTarget = async ( return point; }; +type FocusCandidate = { + readonly tagName: string; + readonly type?: string; + readonly disabled?: boolean; + readonly readOnly?: boolean; + readonly isContentEditable?: boolean; + readonly shadowRoot?: { readonly activeElement: FocusCandidate | null } | null; + readonly focus?: () => void; + getRootNode(): { readonly activeElement?: FocusCandidate | null }; + contains(other: FocusCandidate): boolean; +}; + +// Serialized into the page's main world, so it must not reference module scope. +const focusTextTarget = (target: FocusCandidate | null) => { + let element = + target ?? + (globalThis as unknown as { document: { activeElement: FocusCandidate | null } }).document + .activeElement; + if (target === null) { + while (element?.shadowRoot?.activeElement) element = element.shadowRoot.activeElement; + // Focus inside a frame cannot be checked from here, and an editable frame would pass as contentEditable. + if (element && ["IFRAME", "FRAME", "FENCEDFRAME", "OBJECT", "EMBED"].includes(element.tagName)) + return false; + } + if (element == null) return false; + // Require an enabled, focused text target to avoid losing input or sending it to the previous field. + const nonText = [ + "button", + "checkbox", + "color", + "file", + "hidden", + "image", + "radio", + "range", + "reset", + "submit", + ]; + const textControl = + element.tagName === "TEXTAREA" || + (element.tagName === "INPUT" && !nonText.includes(element.type ?? "text")); + if (!(textControl || element.isContentEditable) || element.disabled || element.readOnly) { + return false; + } + element.focus?.(); + const active = element.getRootNode().activeElement; + return active != null && (active === element || element.contains(active)); +}; + +// A `*:focus` locator needs Playwright's utility world, which some desktop tabs never get. +const focusedTextField = async (page: Page, timeout: number) => { + let timer: ReturnType | undefined; + const expired = new Promise((_, reject) => { + timer = setTimeout( + () => + reject( + new ServerBrowserOperationError({ + tag: "PreviewAutomationTimeoutError", + message: `The page did not report its focused element within ${timeout}ms, so no text was typed.`, + detail: { timeoutMs: timeout }, + }), + ), + timeout, + ); + }); + return await Promise.race([page.evaluate(focusTextTarget, null), expired]).finally(() => + clearTimeout(timer), + ); +}; + export const type = async (page: Page, input: PreviewAutomationTypeInput) => { const timeout = input.timeoutMs ?? DEFAULT_TIMEOUT_MS; const locator = targetLocator(page, input); @@ -756,46 +973,10 @@ export const type = async (page: Page, input: PreviewAutomationTypeInput) => { await locator.fill(input.text, { timeout }); return; } - // A shadow host also matches :focus; use its innermost focused descendant. - const target = locator ?? page.locator("*:focus").last(); const focused = - (locator !== null || (await target.count()) > 0) && - (await target.evaluate( - (element) => { - // Require an enabled, focused text target to avoid losing input or sending it to the previous field. - const control = element as unknown as { - readonly type?: string; - readonly disabled?: boolean; - readonly readOnly?: boolean; - readonly isContentEditable?: boolean; - readonly focus?: () => void; - }; - const nonText = [ - "button", - "checkbox", - "color", - "file", - "hidden", - "image", - "radio", - "range", - "reset", - "submit", - ]; - const textControl = - element.tagName === "TEXTAREA" || - (element.tagName === "INPUT" && !nonText.includes(control.type ?? "text")); - if (!(textControl || control.isContentEditable) || control.disabled || control.readOnly) { - return false; - } - control.focus?.(); - const root = element.getRootNode() as { readonly activeElement?: typeof element | null }; - const active = root.activeElement; - return active != null && (active === element || element.contains(active)); - }, - undefined, - { timeout }, - )); + locator === null + ? await focusedTextField(page, timeout) + : await locator.evaluate(focusTextTarget, undefined, { timeout }); if (focused !== true) { throw new ServerBrowserOperationError({ tag: "PreviewAutomationTargetNotEditableError", @@ -814,14 +995,15 @@ export const hover = async ( input: PreviewAutomationHoverInput, pointer: PointerReporter = noPointer, ) => { - const timeout = input.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const deadline = Date.now() + (input.timeoutMs ?? DEFAULT_TIMEOUT_MS); + const remaining = () => Math.max(1, deadline - Date.now()); const locator = targetLocator(page, input); - await pointer(await targetPoint(page, locator, input, timeout), "move"); + await pointer(await targetPoint(page, locator, input, remaining), "move"); if (locator === null) { await page.mouse.move(input.x ?? 0, input.y ?? 0); return; } - await locator.hover({ timeout }); + await locator.hover({ timeout: remaining() }); }; export const select = async ( @@ -864,16 +1046,18 @@ export const drag = async ( input: PreviewAutomationDragInput, pointer: PointerReporter = noPointer, ) => { - const timeout = input.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const deadline = Date.now() + (input.timeoutMs ?? DEFAULT_TIMEOUT_MS); + const remaining = () => Math.max(1, deadline - Date.now()); const source = targetLocator(page, { locator: input.source })!; const target = targetLocator(page, { locator: input.target })!; - await pointer(await targetPoint(page, source, {}, timeout), "move"); + await pointer(await targetPoint(page, source, {}, remaining), "move"); // Observe drag rejection immediately; an unhandled rejection during cursor movement exits the server. - const dragFailure = source.dragTo(target, { timeout }).then( + // Chromium delivers a single move before dragstart; steps give the page mousemoves during the drag. + const dragFailure = source.dragTo(target, { timeout: remaining(), steps: 5 }).then( () => null, (error: unknown) => ({ error }), ); - const end = await target.boundingBox({ timeout }).catch(() => null); + const end = await target.boundingBox({ timeout: remaining() }).catch(() => null); if (end) await pointer({ x: end.x + end.width / 2, y: end.y + end.height / 2 }, "move"); const failure = await dragFailure; if (failure) throw failure.error; @@ -901,6 +1085,26 @@ export const scroll = async (page: Page, input: PreviewAutomationScrollInput) => await locator.evaluate((element, [x, y]) => element.scrollBy(x, y), delta); }; +const UNRESPONSIVE_SCRIPT_PROBE_MS = 1_000; + +/** + * A same-site navigation reuses the renderer, so it never commits while a page script holds it. + * Call before starting the navigation: Chrome does not deliver a terminate once one is pending. + */ +export const stopUnresponsiveScript = async (cdp: CDPSession, budgetMs = Infinity) => { + let timer: ReturnType | undefined; + const answered = cdp.send("Runtime.evaluate", { expression: "0" }).then( + () => true, + () => true, + ); + const expired = new Promise((resolve) => { + timer = setTimeout(() => resolve(false), Math.min(budgetMs, UNRESPONSIVE_SCRIPT_PROBE_MS)); + }); + const responsive = await Promise.race([answered, expired]).finally(() => clearTimeout(timer)); + // Not awaited: a renderer blocked on something other than script may never acknowledge it. + if (!responsive) void cdp.send("Runtime.terminateExecution").catch(constVoid); +}; + /** Terminate expired page scripts so a hung expression cannot block the tab's control queue. */ export const evaluate = async ( cdp: CDPSession, @@ -1083,7 +1287,8 @@ export const recordingEncoderScript = (frameRate: BrowserRecordingFrameRate) => window.__t3Recorder = { async frame(base64, cssWidth) { if (tooLarge) return false; - const bytes = Uint8Array.from(atob(base64), (char) => char.charCodeAt(0)); + // A per-character atob copy costs ~30 ms on a 3200x1800 frame and throttles recordings. + const bytes = Uint8Array.fromBase64(base64); const bitmap = await createImageBitmap(new Blob([bytes], { type: "image/jpeg" })); if (tooLarge) { bitmap.close(); return false; } if (!recorder) { @@ -1165,11 +1370,18 @@ export const recordingEncoderScript = (frameRate: BrowserRecordingFrameRate) => }, async chunk(index) { const bytes = new Uint8Array(await chunks[index].arrayBuffer()); + if (typeof bytes.toBase64 === "function") return bytes.toBase64(); let binary = ""; for (let offset = 0; offset < bytes.length; offset += 0x8000) { binary += String.fromCharCode(...bytes.subarray(offset, offset + 0x8000)); } return btoa(binary); }, + download() { + const link = document.createElement("a"); + link.href = URL.createObjectURL(new Blob(chunks, { type: recorder.mimeType })); + link.download = "recording"; + link.click(); + }, }; })()`; diff --git a/apps/server/src/preview/ServerBrowserStream.test.ts b/apps/server/src/preview/ServerBrowserStream.test.ts index c4a709ed8acc..c0a9fbc923ff 100644 --- a/apps/server/src/preview/ServerBrowserStream.test.ts +++ b/apps/server/src/preview/ServerBrowserStream.test.ts @@ -57,6 +57,7 @@ const mutations = [ { type: "navigate", url: "https://example.com" }, { type: "history", delta: -1 }, { type: "reload" }, + { type: "stopLoading" }, { type: "probe", x: 10, y: 10 }, { type: "takeControl" }, { type: "releaseControl" }, @@ -149,7 +150,7 @@ it.effect.each([ }, ]); expect(auth.requests).toEqual([resource]); - }).pipe(Effect.scoped), + }), ); it.effect("sends replies ahead of stale frames and only the newest frame after an ack", () => @@ -221,7 +222,7 @@ it.effect("sends replies ahead of stale frames and only the newest frame after a yield* Effect.promise(() => next.promise); expect(received).toEqual([1, "probe", 4]); expect(released).toEqual([2, 3, 1]); - }).pipe(Effect.scoped), + }), ); it.effect.each([ @@ -264,7 +265,7 @@ it.effect.each([ ); expect(response.status).toBe(testCase.status); expect(attachments).toBe(0); - }).pipe(Effect.scoped), + }), ); it.effect("serves a tab's download only to an authorized session", () => @@ -369,7 +370,7 @@ it.effect("passes uploaded files to the page's open picker and needs operate sco { chooserId: "chooser-1", files: [{ name: "notes.txt", text: "hello" }] }, ]); expect((yield* upload([AuthPreviewOperateScope], "stale")).status).toBe(409); - }).pipe(Effect.scoped), + }), ); const decodeHostSetup = Schema.decodeUnknownEffect(Schema.fromJsonString(PreviewStreamHostSetup)); @@ -430,5 +431,5 @@ it.effect.each([ need, command: "sudo t3 browser setup", }); - }).pipe(Effect.scoped), + }), ); diff --git a/apps/server/src/process/NonBlockingSpawner.test.ts b/apps/server/src/process/NonBlockingSpawner.test.ts new file mode 100644 index 000000000000..81563cbab70d --- /dev/null +++ b/apps/server/src/process/NonBlockingSpawner.test.ts @@ -0,0 +1,200 @@ +// @effect-diagnostics nodeBuiltinImport:off - the spawner's checks are node:fs/promises calls, which the test spies on. +import * as NodeFSP from "node:fs/promises"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { describe, expect, it } from "@effect/vitest"; +import { vi } from "vite-plus/test"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as ChildProcess from "effect/process/ChildProcess"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; + +import * as NonBlockingSpawner from "./NonBlockingSpawner.ts"; + +vi.mock("node:fs/promises", { spy: true }); + +const hostPath = process.env.PATH; + +const wrapped = NonBlockingSpawner.layer.pipe(Layer.provideMerge(NodeServices.layer)); + +/** What a caller sees of a command: its output, or the error of a spawn that failed. */ +const outcome = (command: ChildProcess.Command) => + ChildProcessSpawner.ChildProcessSpawner.use((spawner) => spawner.string(command)).pipe( + Effect.match({ + onSuccess: (output) => ({ output }), + onFailure: (error) => ({ reason: error.reason._tag, message: error.message }), + }), + ); + +describe.skipIf(HostProcess.Platform.defaultValue() !== "linux")("NonBlockingSpawner", () => { + it.live("gives callers what the native spawner gives them", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const root = yield* fileSystem.realPath(yield* fileSystem.makeTempDirectoryScoped()); + for (const [file, mode] of [ + ["stale/tool", 0o644], + ["fresh/tool", 0o755], + ["target/bin/tool", 0o755], + // GNU env reads a lone `-` as -i and `NAME=value` as an assignment, even after `--`. + ["fresh/-", 0o755], + ["fresh/launch=stable", 0o755], + ] as const) { + yield* fileSystem.makeDirectory(`${root}/${file.slice(0, file.lastIndexOf("/"))}`, { + recursive: true, + }); + yield* fileSystem.writeFileString(`${root}/${file}`, `#!/bin/sh\necho ${file} "$@"\n`); + yield* fileSystem.chmod(`${root}/${file}`, mode); + } + yield* fileSystem.makeDirectory(`${root}/target/subdir`); + yield* fileSystem.symlink(`${root}/target/subdir`, `${root}/link`); + const PATH = `${root}/stale:${root}/fresh`; + + const commands = { + "enters the cwd": ChildProcess.make("pwd", ["-P"], { cwd: root }), + "keeps argv": ChildProcess.make( + "printf", + ["%s\\n", "-n", "--", "a b", `it's "quoted"`, "$HOME", "; exit 3", "", "A=b"], + { cwd: root }, + ), + // dash would drop the exported function and add PWD and OLDPWD. + "keeps the environment": ChildProcess.make("env", [], { + cwd: root, + env: { PATH: hostPath, A: "1", "BASH_FUNC_x%%": "() { echo exported\n}" }, + }), + "skips a non-executable PATH entry": ChildProcess.make("tool", [], { env: { PATH } }), + "resolves `..` in PATH after a symlink": ChildProcess.make("tool", [], { + env: { PATH: `${root}/link/../bin` }, + }), + "searches PATH relative to the cwd": ChildProcess.make("tool", [], { + cwd: root, + env: { PATH: "fresh" }, + }), + "a command named -": ChildProcess.make("-", [], { cwd: root, env: { PATH } }), + "a command containing =": ChildProcess.make("launch=stable", [], { env: { PATH } }), + "shell mode": ChildProcess.make('pwd -P; echo "$0" "a b"', [], { cwd: root, shell: true }), + "a pipeline": ChildProcess.make("printf", ["a\\nb\\n"], { cwd: root }).pipe( + ChildProcess.pipeTo(ChildProcess.make("tail", ["-n", "1"])), + ), + "a missing command": ChildProcess.make("t3code-missing-command", ["--version"]), + "a missing command in a cwd": ChildProcess.make("t3code-missing-command", [], { + cwd: root, + }), + "a non-executable command": ChildProcess.make("tool", [], { + env: { PATH: `${root}/stale` }, + }), + "a missing explicit path": ChildProcess.make("./missing", [], { cwd: root }), + "a non-executable explicit path": ChildProcess.make(`${root}/stale/tool`, []), + "a missing cwd": ChildProcess.make("true", [], { cwd: `${root}/missing` }), + // The native check sees `missing`, although the path resolves to an existing directory. + "a cwd through a missing directory": ChildProcess.make("true", [], { + cwd: `${root}/missing/..`, + }), + }; + for (const [name, command] of Object.entries(commands)) { + const native = yield* outcome(command).pipe(Effect.provide(NodeServices.layer)); + expect(yield* outcome(command).pipe(Effect.provide(wrapped)), name).toEqual(native); + } + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.live("hands the native spawner only `env`, never a cwd or a command to look up", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const directory = yield* fileSystem.makeTempDirectoryScoped(); + const received: Array = []; + const record = (command: ChildProcess.Command): void => { + if (command._tag === "StandardCommand") received.push(command); + else [command.left, command.right].forEach(record); + }; + const recording = Layer.effect( + ChildProcessSpawner.ChildProcessSpawner, + Effect.map(ChildProcessSpawner.ChildProcessSpawner, (native) => + ChildProcessSpawner.make((command) => { + record(command); + return native.spawn(command); + }), + ), + ).pipe(Layer.provideMerge(NodeServices.layer)); + const commands = [ + ChildProcess.make("true", [], { cwd: directory }), + ChildProcess.make("true", []), + ChildProcess.make("/bin/true", []), + ChildProcess.make("t3code-missing-command", []), + ChildProcess.make("true", [], { cwd: directory, shell: true }), + ChildProcess.make("printf", ["a"], { cwd: directory }).pipe( + ChildProcess.pipeTo(ChildProcess.make("cat", [], { cwd: directory })), + ), + ]; + yield* Effect.forEach(commands, (command) => + ChildProcessSpawner.ChildProcessSpawner.use((spawner) => spawner.exitCode(command)).pipe( + Effect.exit, + ), + ).pipe(Effect.provide(NonBlockingSpawner.layer.pipe(Layer.provide(recording)))); + + expect(received.map((command) => [command.command, command.options.cwd])).toEqual( + Array.from({ length: 6 }, () => ["/usr/bin/env", undefined]), + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.live("checks at most two paths at once, however many directories hang", () => + Effect.gen(function* () { + const allArrived = yield* Deferred.make(); + let arrivals = 0; + // Each spawn resolves its cwd, then starts its check within microtasks. + const arrivalPath = Layer.effect( + Path.Path, + Effect.map(Path.Path, (path) => ({ + ...path, + resolve: (...segments: ReadonlyArray) => { + if (++arrivals === 3) setImmediate(() => Deferred.doneUnsafe(allArrived, Effect.void)); + return path.resolve(...segments); + }, + })), + ); + const hung: Array<() => void> = []; + let releasing = false; + let running = 0; + let mostRunning = 0; + using _access = vi.spyOn(NodeFSP, "access").mockImplementation( + () => + new Promise((_resolve, reject) => { + mostRunning = Math.max(mostRunning, ++running); + const fail = () => { + running--; + reject(Object.assign(new Error("ENOENT"), { code: "ENOENT" })); + }; + if (releasing) fail(); + else hung.push(fail); + }), + ); + const spawns = yield* Effect.forEach( + ["a", "b", "c"], + (name) => + ChildProcessSpawner.ChildProcessSpawner.use((spawner) => + spawner.exitCode(ChildProcess.make("true", [], { cwd: `/t3code-hung/${name}` })), + ).pipe(Effect.flip), + { concurrency: "unbounded" }, + ).pipe( + Effect.provide( + NonBlockingSpawner.layer.pipe( + Layer.provide(arrivalPath), + Layer.provide(NodeServices.layer), + ), + ), + Effect.forkChild, + ); + yield* Deferred.await(allArrived); + expect(mostRunning).toBe(2); + releasing = true; + for (const fail of hung.splice(0)) fail(); + yield* Fiber.join(spawns); + expect(mostRunning).toBe(2); + }), + ); +}); diff --git a/apps/server/src/process/NonBlockingSpawner.ts b/apps/server/src/process/NonBlockingSpawner.ts new file mode 100644 index 000000000000..0807b287cb9c --- /dev/null +++ b/apps/server/src/process/NonBlockingSpawner.ts @@ -0,0 +1,178 @@ +// @effect-diagnostics nodeBuiltinImport:off - execvp's X_OK test has no Effect FileSystem equivalent. +import * as NodeFS from "node:fs"; +import * as NodeFSP from "node:fs/promises"; + +import { envTrampoline, supportsEnvChdir } from "@t3tools/shared/envTrampoline"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as PlatformError from "effect/PlatformError"; +import * as Redacted from "effect/Redacted"; +import * as ChildProcess from "effect/process/ChildProcess"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; +import * as CommandLine from "effect/process/CommandLine"; + +// libuv's _PATH_DEFPATH, searched when the child environment has no PATH. +const DEFAULT_PATH = "/usr/bin:/bin"; + +// A check on a hung mount holds a libuv threadpool thread until the mount answers, and the +// pool's four threads also compress every WebSocket frame. Checks therefore run one at a time +// per directory and two at a time overall; the rest wait without holding a thread. +const CHECK_LIMIT = 2; +const directoryTails = new Map>(); +const waiting: Array<() => void> = []; +let running = 0; + +const access = async (path: string, mode: number) => { + if (running < CHECK_LIMIT) running++; + else await new Promise((resume) => waiting.push(resume)); + try { + await NodeFSP.access(path, mode); + return undefined; + } catch (error) { + return error as NodeJS.ErrnoException; + } finally { + // A waiter takes over the slot. + const next = waiting.shift(); + if (next === undefined) running--; + else next(); + } +}; + +/** The errno error of `access(path, mode)`, or undefined when it passes. */ +const check = (directory: string, path: string, mode: number) => { + const result = (directoryTails.get(directory) ?? Promise.resolve()).then(() => + access(path, mode), + ); + directoryTails.set(directory, result); + void result.then(() => { + if (directoryTails.get(directory) === result) directoryTails.delete(directory); + }); + return Effect.promise(() => result); +}; + +// The PATH the child searches, from the environment Effect's Node spawner gives it. +const childPath = ({ env, extendEnv }: ChildProcess.CommandOptions) => { + if (env === undefined) return process.env.PATH; + if (!Object.hasOwn(env, "PATH")) return extendEnv === true ? process.env.PATH : undefined; + const value = env.PATH; + return value === undefined || !Redacted.isRedacted(value) ? value : Redacted.value(value); +}; + +// The spawn error Effect's Node spawner reports when execvp fails with `code`. +const spawnFailure = (command: ChildProcess.StandardCommand, code: "ENOENT" | "EACCES") => { + const syscall = `spawn ${command.command}`; + return PlatformError.systemError({ + _tag: code === "ENOENT" ? "NotFound" : "PermissionDenied", + module: "ChildProcess", + method: "spawn", + pathOrDescriptor: CommandLine.display([command.command, ...command.args], { format: "posix" }), + syscall, + cause: Object.assign(new Error(`${syscall} ${code}`), { + errno: code === "ENOENT" ? -2 : -13, + code, + syscall, + path: command.command, + spawnargs: command.args.map((arg) => (Redacted.isRedacted(arg) ? "" : arg)), + }), + }); +}; + +// The error of the `FileSystem.access(cwd)` Effect's Node spawner runs before it spawns. +const cwdFailure = (cwd: string, cause: NodeJS.ErrnoException) => + PlatformError.systemError({ + _tag: + cause.code === "ENOENT" + ? "NotFound" + : cause.code === "EACCES" || cause.code === "EPERM" + ? "PermissionDenied" + : cause.code === "ENOTDIR" || cause.code === "ELOOP" + ? "BadResource" + : "Unknown", + module: "FileSystem", + method: "access", + pathOrDescriptor: cwd, + syscall: cause.syscall, + cause, + }); + +/** + * Fails as a native spawn does when the command is missing or not executable: callers read + * that error to tell a CLI that is not installed from one that failed. execvp reports EACCES + * when a candidate exists but cannot be run, and ENOENT otherwise. + */ +const findExecutable = Effect.fnUntraced(function* ( + command: ChildProcess.StandardCommand, + cwd: string | undefined, +) { + // Joined as text: the kernel resolves `..` after a symlink differently from a normalized path. + const fromCwd = (path: string) => + cwd === undefined || path.startsWith("/") ? path : `${cwd}/${path}`; + const candidates = command.command.includes("/") + ? [fromCwd(command.command)] + : (childPath(command.options) ?? DEFAULT_PATH) + .split(":") + .map((entry) => `${fromCwd(entry || ".")}/${command.command}`); + let denied = false; + for (const candidate of candidates) { + const error = yield* check( + candidate.slice(0, candidate.lastIndexOf("/")) || "/", + candidate, + NodeFS.constants.X_OK, + ); + // Any other errno is left for `env` to report through its exit code. + if (error === undefined || (error.code !== "ENOENT" && error.code !== "EACCES")) return; + denied ||= error.code === "EACCES"; + } + return yield* spawnFailure(command, denied ? "EACCES" : "ENOENT"); +}); + +const make = Effect.gen(function* () { + const native = yield* ChildProcessSpawner.ChildProcessSpawner; + if ((yield* HostProcess.Platform) !== "linux") return native; + if (!(yield* Effect.promise(supportsEnvChdir))) return native; + const path = yield* Path.Path; + + // Runs inside the NonBlockingSpawner.spawn span. + const prepareStandard = Effect.fnUntraced(function* (input: ChildProcess.StandardCommand) { + const { cwd: givenCwd, shell, ...options } = input.options; + // Effect rejects a shell command with arguments, and a NUL, itself. + if ((shell && input.args.length > 0) || input.command.includes("\0")) return input; + // Node's shell mode on Unix, spelled out so the trampoline can run it. + const command = shell + ? ChildProcess.make(typeof shell === "string" ? shell : "/bin/sh", ["-c", input.command]) + : input; + // Checked as given and entered resolved, as Effect's Node spawner does. + const cwd = givenCwd === undefined ? undefined : path.resolve(givenCwd); + const trampoline = envTrampoline(command.command, command.args, cwd); + if (trampoline === undefined) return input; + if (givenCwd !== undefined && cwd !== undefined) { + const error = yield* check(cwd, givenCwd, NodeFS.constants.F_OK); + if (error !== undefined) return yield* cwdFailure(givenCwd, error); + } + yield* findExecutable(ChildProcess.make(command.command, command.args, options), cwd); + return ChildProcess.make(trampoline.command, trampoline.args, options); + }); + + const prepare = ( + command: ChildProcess.Command, + ): Effect.Effect => + command._tag === "StandardCommand" + ? prepareStandard(command) + : Effect.map(Effect.all([prepare(command.left), prepare(command.right)]), ([left, right]) => + ChildProcess.pipeTo(left, right, command.options), + ); + + return ChildProcessSpawner.make( + Effect.fn("NonBlockingSpawner.spawn")(function* (command) { + return yield* native.spawn(yield* prepare(command)); + }), + ); +}); + +/** + * On Linux with GNU `env`, replaces the provided `ChildProcessSpawner` with one that never + * hands libuv a cwd or a PATH search; elsewhere it passes the provided one through. + */ +export const layer = Layer.effect(ChildProcessSpawner.ChildProcessSpawner, make); diff --git a/apps/server/src/process/SpawnWorker.test.ts b/apps/server/src/process/SpawnWorker.test.ts new file mode 100644 index 000000000000..27d31aa69c17 --- /dev/null +++ b/apps/server/src/process/SpawnWorker.test.ts @@ -0,0 +1,172 @@ +// @effect-diagnostics nodeBuiltinImport:off processEnvInEffect:off - fixtures need real paths, the process resource list and the live environment. +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { describe, expect, it } from "@effect/vitest"; +import { vi } from "vite-plus/test"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Result from "effect/Result"; +import * as Stream from "effect/Stream"; +import * as ChildProcess from "effect/process/ChildProcess"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; +import * as HostProcess from "@t3tools/shared/HostProcess"; + +import * as SpawnWorker from "./SpawnWorker.ts"; + +const envChdir = vi.hoisted(() => ({ supported: true })); +vi.mock("@t3tools/shared/envTrampoline", async (importOriginal) => ({ + ...(await importOriginal()), + supportsEnvChdir: () => Promise.resolve(envChdir.supported), +})); + +const layer = SpawnWorker.layer.pipe(Layer.provide(NodeServices.layer)); +const text = (stream: Stream.Stream) => + stream.pipe(Stream.decodeText(), Stream.mkString); +const node = (script: string, args: ReadonlyArray = [], options = {}) => + ChildProcess.make(process.execPath, ["-e", script, ...args], options); + +describe("SpawnWorker", () => { + it.live("runs children on a launcher thread where the server thread would wait on them", () => + Effect.gen(function* () { + for (const [platform, hasEnvChdir, onServerThread] of [ + ["win32", true, false], + ["linux", true, true], + ["linux", false, false], + ] as const) { + envChdir.supported = hasEnvChdir; + const launched = Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* spawner.spawn( + node("process.stdout.write('ready');process.stdin.resume()"), + ); + expect(yield* child.stdout.pipe(Stream.take(1), text)).toBe("ready"); + // The thread that called spawn owns the native process handle. + expect(process.getActiveResourcesInfo().includes("ProcessWrap")).toBe(onServerThread); + expect(yield* child.isRunning).toBe(true); + }); + yield* launched.pipe( + Effect.scoped, + Effect.provide(SpawnWorker.layerServer.pipe(Layer.provide(NodeServices.layer))), + Effect.provideService(HostProcess.Platform, platform), + ); + } + }), + ); + + it.live("preserves stdin, large output, env, cwd, literal args and additional descriptors", () => + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const literal = "quote' \" $HOME; * & %PATH%"; + const cwd = NodeOS.tmpdir(); + const script = ` + const fs = require('node:fs'); + const input = fs.readFileSync(0, 'utf8'); + fs.writeSync(1, JSON.stringify([input, process.cwd(), process.env.SPAWN_WORKER_TEST, process.argv[1]]) + '\\n'); + fs.writeSync(1, 'a'.repeat(1024 * 1024)); + fs.writeSync(2, 'b'.repeat(1024 * 1024)); + `; + for (let i = 0; i < 3; i++) { + const child = yield* spawner.spawn( + node(script, [literal], { + cwd, + env: { SPAWN_WORKER_TEST: "inherited value" }, + extendEnv: true, + stdin: Stream.encodeText(Stream.make("stdin value")), + }), + ); + const [stdout, stderr, code] = yield* Effect.all( + [text(child.stdout), text(child.stderr), child.exitCode], + { concurrency: "unbounded" }, + ); + expect(stdout).toBe( + JSON.stringify(["stdin value", NodePath.resolve(cwd), "inherited value", literal]) + + "\n" + + "a".repeat(1024 * 1024), + ); + expect(stderr).toBe("b".repeat(1024 * 1024)); + expect(code).toBe(0); + } + // The launcher threads already exist, so this needs the server's environment at launch time. + process.env.SPAWN_WORKER_LATE = "set after start"; + const late = yield* spawner.spawn( + node("process.stdout.write(process.env.SPAWN_WORKER_LATE)"), + ); + expect(yield* text(late.stdout)).toBe("set after start"); + delete process.env.SPAWN_WORKER_LATE; + const extra = yield* spawner.spawn( + node("require('node:fs').writeSync(3, 'fd3 value')", [], { + additionalFds: { fd3: { type: "output" } }, + }), + ); + const [fd3, extraCode] = yield* Effect.all([text(extra.getOutputFd(3)), extra.exitCode], { + concurrency: "unbounded", + }); + expect(fd3).toBe("fd3 value"); + expect(extraCode).toBe(0); + const reader = yield* spawner.spawn( + node("process.stdin.on('end', () => process.stdout.write('eof')).resume()"), + ); + yield* Stream.fail("abandoned input").pipe(Stream.run(reader.stdin), Effect.ignore); + expect(yield* text(reader.stdout)).toBe("eof"); + const pipeline = node("process.stdout.write('piped')").pipe( + ChildProcess.pipeTo( + node("process.stdin.on('data', (d) => process.stdout.write(String(d).toUpperCase()))"), + ), + ); + const piped = yield* spawner.spawn(pipeline); + expect(yield* text(piped.stdout)).toBe("PIPED"); + expect(yield* piped.exitCode).toBe(0); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("reports launch errors and exit codes, and keeps launching after either", () => + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const missing = yield* spawner + .spawn(ChildProcess.make("t3-missing-command-25245")) + .pipe(Effect.result); + expect(Result.isFailure(missing)).toBe(true); + if (Result.isFailure(missing)) expect(missing.failure.reason._tag).toBe("NotFound"); + const failed = yield* spawner.spawn(node("process.exit(23)")); + expect(yield* failed.exitCode).toBe(23); + const healthy = yield* spawner.spawn(node("process.stdout.write('ok')")); + expect(yield* text(healthy.stdout)).toBe("ok"); + expect(yield* healthy.exitCode).toBe(0); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("kills running commands, detached or not, and on scope interruption", () => + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + for (const detached of [true, false]) { + const child = yield* spawner.spawn( + node("process.stdout.write(String(process.pid));process.stdin.resume()", [], { + detached, + }), + ); + const pid = yield* child.stdout.pipe(Stream.take(1), text); + expect(Number(pid)).toBe(child.pid); + yield* child.kill(); + expect(yield* child.isRunning).toBe(false); + } + const started = yield* Deferred.make(); + const running = yield* Effect.gen(function* () { + const scoped = yield* spawner.spawn(node("process.stdin.resume()")); + yield* Deferred.succeed(started, scoped); + return yield* Effect.never; + }).pipe(Effect.scoped, Effect.forkChild); + const scoped = yield* Deferred.await(started); + yield* Fiber.interrupt(running); + expect(yield* scoped.isRunning).toBe(false); + // Direct callers such as the Claude adapter pass an AbortSignal, which cannot cross to a worker. + const direct = SpawnWorker.spawnPiped(process.execPath, ["-e", "process.stdin.resume()"], { + signal: AbortSignal.abort(), + }); + const exited = new Promise((resolve) => direct.once("exit", () => resolve(direct.killed))); + expect(yield* Effect.promise(() => exited)).toBe(true); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); +}); diff --git a/apps/server/src/process/SpawnWorker.ts b/apps/server/src/process/SpawnWorker.ts new file mode 100644 index 000000000000..a43868b6d7b4 --- /dev/null +++ b/apps/server/src/process/SpawnWorker.ts @@ -0,0 +1,541 @@ +// @effect-diagnostics nodeBuiltinImport:off - worker threads and Node streams implement the platform spawner boundary. +import * as NodeEvents from "node:events"; +import type * as NodeChildProcess from "node:child_process"; +import * as NodeStream from "node:stream"; +import * as NodeWorkerThreads from "node:worker_threads"; +import * as NodeChildProcessSpawner from "@effect/platform-node-shared/NodeChildProcessSpawner"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import { supportsEnvChdir } from "@t3tools/shared/envTrampoline"; +import * as HostProcess from "@t3tools/shared/HostProcess"; + +// Windows runs CreateProcess synchronously on the calling thread, and antivirus or a busy disk can +// hold it for seconds; Linux's spawn waits for chdir and the PATH search, which a hung mount +// stalls. Workers own the children, so only the worker waits. +const MAX_WORKERS = 4; +const MAX_IDLE_WORKERS = 1; + +type ToWorker = + | { + readonly type: "spawn"; + readonly id: number; + readonly command: string; + readonly args: ReadonlyArray; + readonly options: NodeChildProcess.SpawnOptions; + } + | { readonly type: "write"; readonly id: number; readonly fd: number; readonly data: Uint8Array } + | { readonly type: "end"; readonly id: number; readonly fd: number } + | { readonly type: "flow"; readonly id: number; readonly fd: number; readonly paused: boolean } + | { readonly type: "destroy"; readonly id: number; readonly fd: number } + | { readonly type: "kill"; readonly id: number; readonly signal: NodeJS.Signals | number }; + +interface SerializedError { + readonly message: string; + readonly [key: string]: unknown; +} + +type FromWorker = + | { readonly type: "spawn"; readonly id: number; readonly pid: number | undefined } + | { readonly type: "error"; readonly id: number; readonly error: SerializedError } + | { readonly type: "data"; readonly id: number; readonly fd: number; readonly data: Uint8Array } + | { + readonly type: "written"; + readonly id: number; + readonly fd: number; + readonly error?: SerializedError; + } + | { readonly type: "eof"; readonly id: number; readonly fd: number } + | { readonly type: "finished"; readonly id: number; readonly fd: number } + | { + readonly type: "stream-error"; + readonly id: number; + readonly fd: number; + readonly error: SerializedError; + } + | { + readonly type: "exit"; + readonly id: number; + readonly code: number | null; + readonly signal: NodeJS.Signals | null; + } + | { readonly type: "close"; readonly id: number }; + +// Inline source runs from bundles, asar archives and single-executable builds without module files. +const workerSource = ` +const { parentPort } = require("node:worker_threads"); +const { spawn } = require("node:child_process"); +const children = new Map(); +const post = (message, transfer) => parentPort.postMessage(message, transfer); +const serialize = (error) => { + const out = { message: String(error && error.message) }; + for (const key of ["code", "errno", "syscall", "path", "spawnargs"]) { + if (error && error[key] !== undefined) out[key] = error[key]; + } + return out; +}; +parentPort.on("message", (request) => { + const { id } = request; + if (request.type === "spawn") { + let child; + try { + child = spawn(request.command, request.args, request.options); + } catch (error) { + post({ type: "error", id, error: serialize(error) }); + return; + } + children.set(id, child); + child.on("spawn", () => post({ type: "spawn", id, pid: child.pid })); + child.on("error", (error) => post({ type: "error", id, error: serialize(error) })); + child.on("exit", (code, signal) => post({ type: "exit", id, code, signal })); + // Output can still arrive after exit; close follows the last of it. + child.on("close", () => { + children.delete(id); + post({ type: "close", id }); + }); + child.stdio?.forEach((stream, fd) => { + if (!stream) return; + stream.on("error", (error) => post({ type: "stream-error", id, fd, error: serialize(error) })); + if (fd === 0) return; + // Cloning a Buffer copies its whole backing store; copy the bytes once and transfer them. + stream.on("data", (chunk) => { + const data = new Uint8Array(chunk); + post({ type: "data", id, fd, data }, [data.buffer]); + }); + stream.on("end", () => post({ type: "eof", id, fd })); + }); + return; + } + const child = children.get(id); + if (child === undefined) { + if (request.type === "write") post({ type: "written", id, fd: request.fd, error: { message: "write after exit", code: "EPIPE" } }); + if (request.type === "end") post({ type: "finished", id, fd: request.fd }); + return; + } + const stream = child.stdio?.[request.fd]; + switch (request.type) { + case "write": + if (!stream || stream.destroyed) { + post({ type: "written", id, fd: request.fd, error: { message: "write after end", code: "EPIPE" } }); + return; + } + try { + stream.write(request.data, (error) => post({ type: "written", id, fd: request.fd, error: error ? serialize(error) : undefined })); + } catch (error) { + post({ type: "written", id, fd: request.fd, error: serialize(error) }); + } + return; + case "end": + if (!stream || stream.destroyed || stream.writableFinished) post({ type: "finished", id, fd: request.fd }); + else stream.end(() => post({ type: "finished", id, fd: request.fd })); + return; + case "flow": + if (request.paused) stream?.pause(); + else stream?.resume(); + return; + case "destroy": + stream?.destroy(); + return; + case "kill": + try { + child.kill(request.signal); + } catch {} + return; + } +}); +`; + +const toError = (serialized: SerializedError) => + Object.assign(new Error(serialized.message), serialized); + +const toBuffer = (data: Uint8Array) => Buffer.from(data.buffer, data.byteOffset, data.byteLength); + +class WorkerChild + extends NodeEvents.EventEmitter + implements NodeChildProcessSpawner.SpawnedChildProcess +{ + pid: number | undefined; + exitCode: number | null = null; + signalCode: NodeJS.Signals | null = null; + stdin: NodeStream.Writable | null = null; + stdout: NodeStream.Readable | null = null; + stderr: NodeStream.Readable | null = null; + readonly stdio: Array = []; + referenced = true; + killed = false; + spawned = false; + exited = false; + readonly #writes = new Map void>>(); + readonly #finals = new Map void>(); + readonly #paused = new Set(); + + readonly worker: SpawnWorker; + readonly id: number; + + constructor(worker: SpawnWorker, id: number, stdio: ReadonlyArray) { + super(); + this.worker = worker; + this.id = id; + stdio.forEach((option, fd) => { + if (option !== "pipe" && option !== "overlapped") return; + const stream = this.#relay(fd); + this.stdio[fd] = stream; + if (fd === 0) this.stdin = stream as NodeStream.Writable; + else if (fd === 1) this.stdout = stream as NodeStream.Readable; + else if (fd === 2) this.stderr = stream as NodeStream.Readable; + }); + } + + #relay(fd: number): NodeStream.Readable | NodeStream.Writable { + const read = () => { + if (this.#paused.delete(fd)) + this.worker.post({ type: "flow", id: this.id, fd, paused: false }); + }; + const write = ( + chunk: Buffer, + _encoding: BufferEncoding, + callback: (error?: Error | null) => void, + ) => { + if (this.worker.closed) { + callback(Object.assign(new Error("write EPIPE"), { code: "EPIPE" })); + return; + } + const pending = this.#writes.get(fd) ?? []; + pending.push(callback); + this.#writes.set(fd, pending); + const data = new Uint8Array(chunk); + this.worker.post({ type: "write", id: this.id, fd, data }, [data.buffer]); + }; + const final = (callback: (error?: Error | null) => void) => { + this.#finals.set(fd, callback); + this.worker.post({ type: "end", id: this.id, fd }); + }; + // A caller that destroys its end closes the child pipe, as it would with a real pipe. + const destroy = (error: Error | null, callback: (error?: Error | null) => void) => { + if (!this.exited) this.worker.post({ type: "destroy", id: this.id, fd }); + callback(error); + }; + if (fd === 0) return new NodeStream.Writable({ write, final, destroy }); + if (fd <= 2) return new NodeStream.Readable({ read, destroy }); + return new NodeStream.Duplex({ read, write, final, destroy, allowHalfOpen: true }); + } + + onData(fd: number, data: Uint8Array): void { + const stream = this.stdio[fd]; + if (!stream || !("push" in stream)) return; + if (!stream.push(toBuffer(data)) && !this.#paused.has(fd)) { + this.#paused.add(fd); + this.worker.post({ type: "flow", id: this.id, fd, paused: true }); + } + } + + onEof(fd: number): void { + const stream = this.stdio[fd]; + if (stream && "push" in stream) stream.push(null); + } + + onFinished(fd: number): void { + const callback = this.#finals.get(fd); + this.#finals.delete(fd); + callback?.(); + } + + onWritten(fd: number, error: SerializedError | undefined): void { + const callback = this.#writes.get(fd)?.shift(); + callback?.(error === undefined ? null : toError(error)); + } + + onStreamError(fd: number, error: SerializedError): void { + this.stdio[fd]?.destroy(toError(error)); + } + + kill(signal: NodeJS.Signals | number = "SIGTERM"): boolean { + if (this.exited) return false; + this.killed = true; + this.worker.post({ type: "kill", id: this.id, signal }); + return true; + } + + ref(): void { + this.referenced = true; + this.worker.syncRef(); + } + + unref(): void { + this.referenced = false; + this.worker.syncRef(); + } + + exit(code: number | null, signal: NodeJS.Signals | null): void { + if (this.exited) return; + this.exited = true; + this.exitCode = code; + this.signalCode = signal; + // Node destroys stdin on exit; the Effect sink may already have removed its error listener. + this.stdin?.on("error", () => {}); + this.stdin?.destroy(); + this.emit("exit", code, signal); + } + + /** The worker is gone: end output so readers finish, and let the exit report the kill. */ + abandon(): void { + // Not destroy(error): Effect leaves an unread stderr's pipe without an error listener. + for (const stream of this.stdio) { + if (stream && "push" in stream && !stream.readableEnded) stream.push(null); + else if (stream && !("push" in stream)) stream.destroy(); + } + } +} + +class SpawnWorker { + readonly worker: NodeWorkerThreads.Worker; + readonly children = new Map(); + readonly pool: SpawnWorkerPool; + /** Launches sent but not yet answered; the worker is inside CreateProcess for one of them. */ + launching = 0; + closed = false; + + constructor(pool: SpawnWorkerPool) { + this.pool = pool; + this.worker = new NodeWorkerThreads.Worker(workerSource, { eval: true }); + this.worker.on("message", (message: FromWorker) => this.onMessage(message)); + this.worker.on("error", (error) => this.fail(error)); + this.worker.on("exit", () => this.fail(new Error("The process launcher thread exited."))); + this.worker.unref(); + } + + post(message: ToWorker, transfer?: ReadonlyArray): void { + if (!this.closed) this.worker.postMessage(message, transfer); + } + + spawn( + id: number, + command: string, + args: ReadonlyArray, + options: NodeChildProcess.SpawnOptions, + ): WorkerChild { + const io = options.stdio; + const stdio = + typeof io === "string" || io === undefined ? [io ?? "pipe", io ?? "pipe", io ?? "pipe"] : io; + const child = new WorkerChild(this, id, stdio); + // An AbortSignal cannot cross to the worker, so this thread kills on abort as Node would. + const { signal, ...rest } = options; + if (signal !== undefined) { + const abort = () => child.kill(); + if (signal.aborted) queueMicrotask(abort); + else { + signal.addEventListener("abort", abort, { once: true }); + child.once("exit", () => signal.removeEventListener("abort", abort)); + } + } + this.children.set(id, child); + this.launching++; + this.syncRef(); + this.post({ + type: "spawn", + id, + command, + args: [...args], + // A worker's process.env is a copy from when it started; children get the server's current one. + options: { ...rest, stdio, env: rest.env ?? { ...process.env } }, + }); + return child; + } + + syncRef(): void { + if (this.closed) return; + const referenced = + this.launching > 0 || [...this.children.values()].some((child) => child.referenced); + if (referenced) this.worker.ref(); + else this.worker.unref(); + } + + onMessage(message: FromWorker): void { + const child = this.children.get(message.id); + if (child === undefined) return; + switch (message.type) { + case "spawn": + this.launching--; + child.spawned = true; + child.pid = message.pid; + child.emit("spawn"); + break; + case "error": + if (!child.spawned) { + this.launching--; + child.exited = true; + this.release(child); + } + child.emit("error", toError(message.error)); + break; + case "data": + child.onData(message.fd, message.data); + break; + case "written": + child.onWritten(message.fd, message.error); + break; + case "eof": + child.onEof(message.fd); + break; + case "finished": + child.onFinished(message.fd); + break; + case "stream-error": + child.onStreamError(message.fd, message.error); + break; + case "exit": + child.exit(message.code, message.signal); + break; + case "close": + child.emit("close", child.exitCode, child.signalCode); + this.release(child); + break; + } + } + + release(child: WorkerChild): void { + this.children.delete(child.id); + this.syncRef(); + this.pool.release(this); + } + + fail(error: Error): void { + if (this.closed) return; + this.close(); + for (const child of this.children.values()) { + if (child.exited) { + child.abandon(); + continue; + } + if (!child.spawned) { + child.exited = true; + child.emit("error", error); + continue; + } + // The worker owned the only handle, so try to end the whole tree before reporting the exit. + const finish = () => { + child.exit(null, "SIGKILL"); + child.abandon(); + }; + if (child.pid === undefined) finish(); + else this.pool.killTree(child.pid, finish); + } + this.children.clear(); + } + + close(): void { + if (this.closed) return; + this.closed = true; + this.pool.workers.delete(this); + void this.worker.terminate(); + } +} + +class SpawnWorkerPool { + readonly workers = new Set(); + readonly platform: NodeJS.Platform; + #nextId = 0; + + constructor(platform: NodeJS.Platform) { + this.platform = platform; + } + + readonly spawn = ( + command: string, + args: ReadonlyArray, + options: NodeChildProcess.SpawnOptions, + ) => { + const workers = [...this.workers]; + // A worker inside CreateProcess cannot start another launch until it returns. + const worker = + workers.find((candidate) => candidate.launching === 0) ?? + (workers.length < MAX_WORKERS + ? this.#add() + : workers.reduce((best, candidate) => + candidate.launching < best.launching ? candidate : best, + )); + return worker.spawn(++this.#nextId, command, args, options); + }; + + killTree(pid: number, done: () => void): void { + const killPid = () => { + try { + process.kill(pid, "SIGKILL"); + } catch {} + }; + if (this.platform !== "win32") { + killPid(); + done(); + return; + } + let settled = false; + const finish = (killed: boolean) => { + if (settled) return; + settled = true; + // As Effect's own kill does when taskkill fails, end at least the direct child. + if (!killed) killPid(); + done(); + }; + this.spawn("taskkill", ["/pid", String(pid), "/T", "/F"], { + stdio: "ignore", + windowsHide: true, + }) + .on("error", () => finish(false)) + .on("exit", (code) => finish(code === 0)); + } + + #add(): SpawnWorker { + const worker = new SpawnWorker(this); + this.workers.add(worker); + return worker; + } + + release(worker: SpawnWorker): void { + if (worker.children.size > 0 || worker.launching > 0) return; + const idle = [...this.workers].filter( + (candidate) => candidate.children.size === 0 && candidate.launching === 0, + ); + if (idle.length > MAX_IDLE_WORKERS) worker.close(); + } + + close(): void { + for (const worker of this.workers) worker.close(); + } +} + +let sharedPool: SpawnWorkerPool | undefined; + +/** A piped `child_process.spawn` for Windows launches outside Effect's spawner. */ +export const spawnPiped = ( + command: string, + args: ReadonlyArray, + options: Omit, +) => + (sharedPool ??= new SpawnWorkerPool("win32")).spawn(command, args, { + ...options, + stdio: ["pipe", "pipe", "pipe"], + }) as WorkerChild & { + readonly stdin: NodeStream.Writable; + readonly stdout: NodeStream.Readable; + readonly stderr: NodeStream.Readable; + }; + +export const layer = Layer.unwrap( + Effect.gen(function* () { + const platform = yield* HostProcess.Platform; + const pool = yield* Effect.acquireRelease( + Effect.sync(() => new SpawnWorkerPool(platform)), + (pool) => Effect.sync(() => pool.close()), + ); + return NodeChildProcessSpawner.layerWithSpawn(pool.spawn); + }), +); + +/** Workers on Windows, and on Linux where `NonBlockingSpawner` has no GNU `env -C` to use. */ +export const layerServer = Layer.unwrap( + Effect.gen(function* () { + const platform = yield* HostProcess.Platform; + if (platform === "win32") return layer; + if (platform === "linux" && !(yield* Effect.promise(supportsEnvChdir))) return layer; + return NodeChildProcessSpawner.layer; + }), +); diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index 0095146a3021..49674354b253 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -12,12 +12,21 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Logger from "effect/Logger"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; +import { vi } from "vite-plus/test"; import * as ServerConfig from "../config.ts"; import { makeCodexAppServerSpawnCommand } from "../orchestration-v2/Adapters/CodexAdapterV2.ts"; import * as ProcessRunner from "../processRunner.ts"; +import * as ProviderProcesses from "./providerProcesses.ts"; import { classifyScope, make, ownedAgentSlice, type ScopeState } from "./agentScope.ts"; +const GiB = 1024 ** 3; +const memory = vi.hoisted(() => ({ total: 16 * 1024 ** 3 })); +vi.mock("node:os", async (importOriginal) => ({ + ...(await importOriginal()), + totalmem: () => memory.total, +})); + const running: ScopeState = { loadState: "loaded", activeState: "active", @@ -48,6 +57,7 @@ const withFakeSystemd = ( readonly scope: Effect.Success; readonly units: Map; readonly calls: ReadonlyArray>; + readonly failingVerbs: Set; readonly stateDir: string; readonly cgroupRoot: string; }) => Effect.Effect, @@ -55,7 +65,14 @@ const withFakeSystemd = ( { oomPolicy = true, stopFails = false, - }: { readonly oomPolicy?: boolean; readonly stopFails?: boolean } = {}, + leftovers = false, + liveOwner = false, + }: { + readonly oomPolicy?: boolean; + readonly stopFails?: boolean; + readonly leftovers?: boolean; + readonly liveOwner?: boolean; + } = {}, ) => Effect.gen(function* () { const bin = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-agent-scope-")); @@ -65,6 +82,7 @@ const withFakeSystemd = ( const cgroupRoot = NodePath.join(bin, "cgroup"); const units = new Map(); const calls: Array> = []; + const failingVerbs = new Set(); const output = (stdout: string, code = 0): ProcessRunner.ProcessRunOutput => ({ stdout, stderr: "", @@ -83,6 +101,7 @@ const withFakeSystemd = ( return output("", 1); } const [, verb, unit = ""] = args; + if (failingVerbs.has(verb!)) return output("", 1); if (stopFails && verb === "stop") { return { ...output(""), @@ -102,7 +121,12 @@ const withFakeSystemd = ( }), }); const fileSystem = yield* FileSystem.FileSystem; + const { stateDir } = yield* ServerConfig.ServerConfig; + if (leftovers) { + units.set(ownedAgentSlice(stateDir), { activeState: "active", result: "success" }); + } const scope = yield* make.pipe( + Effect.provideService(ProviderProcesses.ProviderProcesses, { hasOtherLiveOwner: liveOwner }), Effect.provideService(ProcessRunner.ProcessRunner, runner), Effect.provideService(HostProcess.Platform, "linux"), Effect.provideService(HostProcess.Environment, { PATH: bin, XDG_RUNTIME_DIR: "/run/user/1" }), @@ -114,8 +138,7 @@ const withFakeSystemd = ( fileSystem.writeFileString(path.replace("/sys/fs/cgroup", cgroupRoot), data, options), }), ); - const { stateDir } = yield* ServerConfig.ServerConfig; - return yield* body({ scope, units, calls, stateDir, cgroupRoot }).pipe( + return yield* body({ scope, units, calls, failingVerbs, stateDir, cgroupRoot }).pipe( Effect.ensuring(Effect.sync(() => NodeFS.rmSync(bin, { recursive: true, force: true }))), ); }).pipe( @@ -308,27 +331,68 @@ describe("AgentScope service", () => { ), ); - it.effect("stops agents a killed server left before the first launch", () => - withFakeSystemd(({ scope, units, calls, stateDir }) => + it.effect("follows a live memory change at the next launch", () => { + memory.total = 16 * GiB; + return withFakeSystemd(({ scope, calls, failingVerbs }) => Effect.gen(function* () { - units.set(ownedAgentSlice(stateDir), { activeState: "active", result: "success" }); - const agent = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" }); - const stops = calls.filter(([, verb]) => verb === "stop"); - expect(stops).toEqual([["--user", "stop", ownedAgentSlice(stateDir)]]); - expect(sliceOf(agent)).toBe(ownedAgentSlice(stateDir)); - expect(agent.args).toContain("--property=TimeoutStopSec=10s"); - const terminal = yield* scope.wrap({ - command: "agent", - args: [], - name: "terminal", - survivesRestart: true, - }); - expect(sliceOf(terminal)).toBe("app-t3code-agents.slice"); - expect(calls.filter(([, verb]) => verb === "stop")).toHaveLength(1); + const launch = scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" }); + const memoryLimits = () => + calls + .filter( + ([, verb, , slice]) => verb === "set-property" && slice === "app-t3code-agents.slice", + ) + .map((args) => args.slice(4)); + yield* launch; + yield* launch; + expect(memoryLimits()).toEqual([[`MemoryHigh=${10 * GiB}`, `MemoryMax=${12 * GiB}`]]); + + // A VM balloon or `pct set --memory` grows RAM while the server keeps running. + memory.total = 22 * GiB; + failingVerbs.add("set-property"); + yield* launch; + yield* launch; + failingVerbs.delete("set-property"); + yield* launch; + yield* launch; + const grown = [`MemoryHigh=${16 * GiB}`, `MemoryMax=${18 * GiB}`]; + expect(memoryLimits()).toEqual([ + [`MemoryHigh=${10 * GiB}`, `MemoryMax=${12 * GiB}`], + grown, + grown, + grown, + ]); }), + ); + }); + + it.effect("stops agents a killed server left before startup completes", () => + withFakeSystemd( + ({ scope, calls, stateDir }) => + Effect.gen(function* () { + expect(stopsOf(calls)).toEqual([["--user", "stop", ownedAgentSlice(stateDir)]]); + const agent = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" }); + expect(sliceOf(agent)).toBe(ownedAgentSlice(stateDir)); + expect(agent.args).toContain("--property=TimeoutStopSec=10s"); + const terminal = yield* scope.wrap({ + command: "agent", + args: [], + name: "terminal", + survivesRestart: true, + }); + expect(sliceOf(terminal)).toBe("app-t3code-agents.slice"); + expect(stopsOf(calls)).toHaveLength(1); + }), + { leftovers: true }, ), ); + it.effect("leaves the slice to another live server sharing this home", () => + withFakeSystemd(({ calls }) => Effect.sync(() => expect(stopsOf(calls)).toEqual([])), { + leftovers: true, + liveOwner: true, + }), + ); + it.effect("stops leftover processes only in an active agent scope", () => withFakeSystemd(({ scope, units, calls, cgroupRoot }) => Effect.gen(function* () { diff --git a/apps/server/src/process/agentScope.ts b/apps/server/src/process/agentScope.ts index b9b6b56acb14..1e52e680b6f8 100644 --- a/apps/server/src/process/agentScope.ts +++ b/apps/server/src/process/agentScope.ts @@ -14,9 +14,11 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; +import * as Semaphore from "effect/Semaphore"; import * as ServerConfig from "../config.ts"; import * as ProcessRunner from "../processRunner.ts"; +import * as ProviderProcesses from "./providerProcesses.ts"; const AGENT_SLICE = "app-t3code-agents.slice"; // Owns every agent slice and competes with the server's unit or scope under app.slice. @@ -106,6 +108,7 @@ export const make = Effect.gen(function* () { const path = yield* Path.Path; const hostEnvironment = yield* HostProcess.Environment; const { stateDir } = yield* ServerConfig.ServerConfig; + const { hasOtherLiveOwner } = yield* ProviderProcesses.ProviderProcesses; const ownedSlice = ownedAgentSlice(stateDir); const layerScope = yield* Effect.scope; @@ -130,6 +133,7 @@ export const make = Effect.gen(function* () { const stopLeftoverAgents = Effect.fn("agentScope.stopLeftoverAgents")(function* ( systemctl: string, ) { + if (hasOtherLiveOwner) return; const shown = yield* run(systemctl, [ "--user", "show", @@ -190,20 +194,44 @@ export const make = Effect.gen(function* () { slice, ...properties, ]); - if (result._tag === "None" || result.value.code !== 0) { - yield* Effect.logWarning("Could not set resource limits on an agents slice", { - slice, - properties, - stderr: result._tag === "Some" ? result.value.stderr.trim() : undefined, - }); - } + if (result._tag === "Some" && result.value.code === 0) return true; + yield* Effect.logWarning("Could not set resource limits on an agents slice", { + slice, + properties, + stderr: result._tag === "Some" ? result.value.stderr.trim() : undefined, + }); + return false; }); - const limits = agentSliceMemoryLimits(NodeOS.totalmem()); - yield* setProperties(AGENT_SLICE, [`MemoryHigh=${limits.high}`, `MemoryMax=${limits.max}`]); + // VMs and containers can resize RAM live, so each launch re-checks the total + // the limits came from. A failed update is retried by the next launch. + let appliedTotal: number | undefined; + const memoryLimitsLock = yield* Semaphore.make(1); + const applyMemoryLimits = memoryLimitsLock.withPermits(1)( + Effect.gen(function* () { + const total = NodeOS.totalmem(); + if (total === appliedTotal) return; + const limits = agentSliceMemoryLimits(total); + const applied = yield* setProperties(AGENT_SLICE, [ + `MemoryHigh=${limits.high}`, + `MemoryMax=${limits.max}`, + ]); + if (!applied) return; + if (appliedTotal !== undefined) { + yield* Effect.logInfo("Agent memory limits follow a memory change", { + slice: AGENT_SLICE, + previousTotal: appliedTotal, + total, + ...limits, + }); + } + appliedTotal = total; + }), + ); + yield* applyMemoryLimits; yield* setProperties(AGENTS_PARENT_SLICE, AGENTS_PARENT_PROPERTIES); yield* Effect.logInfo("Agent scopes are on", { slice: AGENT_SLICE, - ...limits, + ...agentSliceMemoryLimits(NodeOS.totalmem()), oomPolicy: supportsOomPolicy, }); return { @@ -211,10 +239,15 @@ export const make = Effect.gen(function* () { systemctl, runtimeDir, supportsOomPolicy, + applyMemoryLimits, }; }), ); + // Layer acquisition is the startup barrier: recovery must not terminalize or + // continue old runs while a previous server's providers can still execute. + yield* systemd; + // `shared` scopes serve other threads too: tracked only for OOM diagnosis on old systemd. const scopeByThread = new Map(); @@ -396,6 +429,7 @@ export const make = Effect.gen(function* () { if (resolved === null) return { command, args }; const scope = yield* systemd; if (scope === undefined) return agentScopeCommand({ command: resolved, args }); + yield* scope.applyMemoryLimits; const unit = `t3code-${name}-${NodeCrypto.randomUUID().slice(0, 8)}.scope`; // A continuing shared scope cannot attribute a tool's OOM to one thread. // Clear the launch thread's previous scope instead. @@ -427,4 +461,7 @@ export const make = Effect.gen(function* () { } satisfies AgentScopeShape; }); -export const layer = Layer.effect(AgentScope, make).pipe(Layer.provide(ProcessRunner.layer)); +export const layer = Layer.effect(AgentScope, make).pipe( + Layer.provide(ProcessRunner.layer), + Layer.provideMerge(ProviderProcesses.layer), +); diff --git a/apps/server/src/process/externalLauncher.test.ts b/apps/server/src/process/externalLauncher.test.ts index ed3dc623af3d..f215fe4ebc6a 100644 --- a/apps/server/src/process/externalLauncher.test.ts +++ b/apps/server/src/process/externalLauncher.test.ts @@ -21,6 +21,7 @@ import { resolveEditorCommand } from "@t3tools/shared/editor"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { SpawnExecutableResolution } from "@t3tools/shared/shell"; import * as ExternalLauncher from "./externalLauncher.ts"; +import { WindowsVerbatimSpawn } from "./windowsVerbatimSpawn.ts"; // Windows drive letters break mocked POSIX PATH splitting; these tests also require /bin/sh stubs. const windowsHost = HostProcess.Platform.defaultValue() === "win32"; @@ -104,36 +105,47 @@ const layerTest = (input: { }; // macOS command lookup cannot search a Windows drive-letter PATH. -it.effect.skipIf(windowsHost)( - "opens home-relative app bundles using the server home directory", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const bin = yield* fs.makeTempDirectoryScoped({ prefix: "t3-app-open-" }); - yield* fs.writeFileString(path.join(bin, "open"), "#!/bin/sh\n"); - yield* fs.chmod(path.join(bin, "open"), 0o755); - const spawned: ChildProcess.StandardCommand[] = []; - yield* Effect.gen(function* () { - const launcher = yield* ExternalLauncher.ExternalLauncher; - for (const cwd of ["~/Applications/My App.app", "/Applications/My App.app"]) { - yield* launcher.launchEditor({ editor: "file-manager", cwd }); - } - }).pipe( - Effect.provide( - layerTest({ - platform: "darwin", - env: { PATH: bin }, - onSpawn: (command) => spawned.push(command), - }), - ), - Effect.provideService(HostProcess.HomeDirectory, "/Users/server"), - ); - assert.deepEqual( - spawned.map((command) => command.args), - [["/Users/server/Applications/My App.app"], ["/Applications/My App.app"]], - ); - }).pipe(Effect.provide(NodeServices.layer)), +it.effect.skipIf(windowsHost)("opens home-relative targets using the server home directory", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const bin = yield* fs.makeTempDirectoryScoped({ prefix: "t3-app-open-" }); + for (const name of ["open", "code", "idea"]) { + yield* fs.writeFileString(path.join(bin, name), "#!/bin/sh\n"); + yield* fs.chmod(path.join(bin, name), 0o755); + } + const spawned: ChildProcess.StandardCommand[] = []; + yield* Effect.gen(function* () { + const launcher = yield* ExternalLauncher.ExternalLauncher; + for (const cwd of ["~/Applications/My App.app", "/Applications/My App.app"]) { + yield* launcher.launchEditor({ editor: "file-manager", cwd }); + } + yield* launcher.launchEditor({ editor: "vscode", cwd: "~/notes/todo.md:3" }); + yield* launcher.launchEditor({ + editor: "idea", + cwd: "~/notes/todo.md:3", + projectRoot: "~/notes", + }); + }).pipe( + Effect.provide( + layerTest({ + platform: "darwin", + env: { PATH: bin }, + onSpawn: (command) => spawned.push(command), + }), + ), + Effect.provideService(HostProcess.HomeDirectory, "/Users/server"), + ); + assert.deepEqual( + spawned.map((command) => command.args), + [ + ["/Users/server/Applications/My App.app"], + ["/Applications/My App.app"], + ["--goto", "/Users/server/notes/todo.md:3"], + ["/Users/server/notes", "--line", "3", "/Users/server/notes/todo.md"], + ], + ); + }).pipe(Effect.provide(NodeServices.layer)), ); it.effect.skipIf(windowsHost)( @@ -194,6 +206,8 @@ it.effect.skipIf(windowsHost)( const codePath = path.join(binDir, "code"); yield* fileSystem.writeFileString(codePath, "#!/bin/sh\n"); yield* fileSystem.chmod(codePath, 0o755); + const colonProject = path.join(project, "build:2026"); + yield* fileSystem.makeDirectory(colonProject); const file = `${path.join(project, "file.ts")}:3:7`; const spawned: ChildProcess.StandardCommand[] = []; @@ -201,6 +215,7 @@ it.effect.skipIf(windowsHost)( const launcher = yield* ExternalLauncher.ExternalLauncher; yield* launcher.launchEditor({ editor: "vscode", cwd: project }); yield* launcher.launchEditor({ editor: "vscode", cwd: file }); + yield* launcher.launchEditor({ editor: "vscode", cwd: colonProject }); }).pipe( Effect.provide( layerTest({ @@ -217,6 +232,7 @@ it.effect.skipIf(windowsHost)( [ ["--new-window", project], ["--goto", file], + ["--new-window", colonProject], ], ); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), @@ -295,6 +311,69 @@ it.effect.skipIf(windowsHost)( }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); +// Windows drive letters cannot resolve the POSIX xdg stubs. +it.effect.skipIf(windowsHost)( + "a Linux server without a display offers and opens the file manager on the desktop session's display", + () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const binDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-editors-" }); + for (const name of ["xdg-open", "xdg-mime"]) { + const filePath = path.join(binDir, name); + yield* fileSystem.writeFileString(filePath, "#!/bin/sh\n"); + yield* fileSystem.chmod(filePath, 0o755); + } + + const fileManager = (sessionEnv: ReadonlyArray) => + Effect.gen(function* () { + const spawned: ChildProcess.StandardCommand[] = []; + const result = yield* Effect.gen(function* () { + const launcher = yield* ExternalLauncher.ExternalLauncher; + const editors = yield* launcher.resolveAvailableEditors(); + const kind = yield* launcher.resolveFileManagerRevealKind(); + const opened = yield* launcher + .launchEditor({ editor: "file-manager", cwd: "/workspace" }) + .pipe(Effect.exit); + return { offered: editors.includes("file-manager"), kind, opened }; + }).pipe( + Effect.provide( + layerTest({ + platform: "linux", + env: { PATH: binDir }, + onSpawn: (command) => spawned.push(command), + spawnResult: (command) => + command.command === "busctl" + ? { stdout: JSON.stringify({ type: "as", data: sessionEnv }) } + : command.command === "xdg-mime" + ? { stdout: "org.kde.dolphin.desktop\n" } + : undefined, + }), + ), + Effect.provideService(HostProcess.Environment, { PATH: binDir }), + ); + const open = spawned.find((command) => command.command === "xdg-open"); + return { ...result, open: open && { args: open.args, env: open.options.env } }; + }); + + const session = yield* fileManager(["DISPLAY=:0", "HOME=/home/user"]); + assert.equal(session.offered, true); + assert.equal(session.kind, "files"); + assert.equal(Exit.isSuccess(session.opened), true); + assert.deepEqual(session.open, { + args: ["/workspace"], + env: { PATH: binDir, DISPLAY: ":0" }, + }); + + // No one is logged in to a desktop, so there is no screen to open it on. + const headless = yield* fileManager(["HOME=/home/user"]); + assert.equal(headless.offered, false); + assert.equal(headless.kind, undefined); + assert.equal(Exit.isFailure(headless.opened), true); + assert.equal(headless.open, undefined); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + it.effect("launches an installed editor with platform-safe arguments", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; @@ -338,6 +417,144 @@ it.effect("launches an installed editor with platform-safe arguments", () => }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); +const wslpathSpawner = (out: string | undefined) => + Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make((command) => + Effect.sync(() => { + if (!ChildProcess.isStandardCommand(command)) { + throw new Error("Expected a standard command"); + } + assert.equal(command.command, "wslpath"); + return makeMockDetachedHandle( + out === undefined ? { exitCode: 1 } : { exitCode: 0, stdout: `${out}\n` }, + ); + }), + ), + ); + +it.effect.each([ + // wslpath is mount-aware: a drive remapped in /etc/wsl.conf keeps its real letter. + { target: "/mnt/d/project", wslpath: "E:\\project", expected: "E:\\project" }, + // wslpath missing or failing falls back to the hand-built UNC path. + { + target: "/home/user/repo", + wslpath: undefined, + expected: "\\\\wsl.localhost\\Ubuntu-24.04\\home\\user\\repo", + }, + { target: "/", wslpath: undefined, expected: "\\\\wsl.localhost\\Ubuntu-24.04" }, +])("resolveWslFileManagerPath resolves $target to $expected", ({ target, wslpath, expected }) => + Effect.gen(function* () { + const result = yield* ExternalLauncher.resolveWslFileManagerPath(target, "Ubuntu-24.04"); + assert.equal(result, expected); + }).pipe(Effect.provide(wslpathSpawner(wslpath))), +); + +it.live.skipIf(!windowsHost)("opens Windows folders through the default shell handler", () => + Effect.gen(function* () { + let spawned: ChildProcess.StandardCommand | undefined; + yield* Effect.gen(function* () { + const launcher = yield* ExternalLauncher.ExternalLauncher; + yield* launcher.launchEditor({ + editor: "file-manager", + cwd: "C:/workspace [draft]/author's ‘left’ ‚low‛ $media/’; Write-Output injected; #", + }); + }).pipe( + Effect.provide( + layerTest({ + platform: "win32", + env: { PATH: "C:\\Windows", PATHEXT: ".EXE" }, + onSpawn: (command) => { + spawned = command; + }, + }), + ), + ); + assert.ok(spawned); + assert.equal(spawned.options.detached, false); + const source = Buffer.from(String(spawned.args.at(-1) ?? ""), "base64").toString("utf16le"); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const output = yield* spawner.string( + ChildProcess.make(spawned.command, [ + ...spawned.args.slice(0, -1), + Buffer.from( + `[Console]::OutputEncoding = [Text.UTF8Encoding]::new($false); function Invoke-Item { param($LiteralPath) Write-Output $LiteralPath }; ${source}`, + "utf16le", + ).toString("base64"), + ]), + ); + assert.equal( + output.trim(), + "C:\\workspace [draft]\\author's ‘left’ ‚low‛ $media\\’; Write-Output injected; #", + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + +for (const [replacement, verb, handler, target, expected] of [ + ["none", "none", "", "C:/repo/file.txt", 'explorer.exe /select,"C:\\repo\\file.txt"'], + ["Directory", "openindopus", "dopusrt.exe", "C:/repo/file.txt", "shell:C:\\repo"], + ["Folder", "openindopus", "dopusrt.exe", "C:/repo/file.txt", "shell:C:\\repo"], + [ + "Folder", + "open", + '"C:\\Windows\\explorer.exe"', + "C:/repo/file.txt", + 'explorer.exe /select,"C:\\repo\\file.txt"', + ], + [ + "Folder", + "open", + "ExplorerDelegate", + "C:/repo/file.txt", + 'explorer.exe /select,"C:\\repo\\file.txt"', + ], + [ + "Drive", + "openindopus", + "dopusrt.exe", + "C:/repo/file.txt", + 'explorer.exe /select,"C:\\repo\\file.txt"', + ], + ["Drive", "openindopus", "dopusrt.exe", "C:/file.txt", "shell:C:\\"], +] as const) { + it.live.skipIf(!windowsHost)( + `reveals ${target} with ${replacement}/${verb} handled by ${handler}`, + () => + Effect.gen(function* () { + let spawned: ChildProcess.StandardCommand | undefined; + yield* Effect.gen(function* () { + const launcher = yield* ExternalLauncher.ExternalLauncher; + yield* launcher.launchEditor({ + editor: "file-manager", + cwd: target, + reveal: true, + }); + }).pipe( + Effect.provide( + layerTest({ + platform: "win32", + env: { PATH: "C:\\Windows", PATHEXT: ".EXE" }, + onSpawn: (command) => { + spawned = command; + }, + }), + ), + ); + assert.ok(spawned); + const source = Buffer.from(String(spawned.args.at(-1) ?? ""), "base64").toString("utf16le"); + const stubs = `function Get-ItemProperty { param($LiteralPath, $ErrorAction) @{ '(default)' = if ($LiteralPath -eq 'Registry::HKEY_CLASSES_ROOT\\${replacement}\\shell') { '${verb}' } elseif ($LiteralPath -eq 'Registry::HKEY_CLASSES_ROOT\\${replacement}\\shell\\${verb}\\command') { '${handler}' } else { 'none' }; DelegateExecute = if ('${handler}' -eq 'ExplorerDelegate') { '{11dbb47c-a525-400b-9e80-a54615a090c0}' } else { '' } } }; function Invoke-Item { param($LiteralPath) Write-Output "shell:$LiteralPath" }; function Start-Process { param($FilePath, $ArgumentList) Write-Output "$FilePath $ArgumentList" }; `; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const output = yield* spawner.string( + ChildProcess.make(spawned.command, [ + ...spawned.args.slice(0, -1), + Buffer.from(stubs + source, "utf16le").toString("base64"), + ]), + ); + assert.equal(output.trim(), expected); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +} + it.effect.skipIf(windowsHost)( "reveals a WSL file in Windows File Explorer through its UNC path", () => @@ -358,7 +575,7 @@ it.effect.skipIf(windowsHost)( const editors = yield* launcher.resolveAvailableEditors(); yield* launcher.launchEditor({ editor: "file-manager", - cwd: "/home/t3/workspace [draft]/author's $media/clip|draft.mp4", + cwd: "/home/t3/workspace [draft]/author's ‘$media/clip|draft.mp4", reveal: true, }); return { kind, editors }; @@ -385,13 +602,85 @@ it.effect.skipIf(windowsHost)( assert.equal(spawned.options.detached, true); const encodedCommand = String(spawned.args[spawned.args.length - 1] ?? ""); const decodedCommand = Buffer.from(encodedCommand, "base64").toString("utf16le"); - assert.equal( + assert.include(decodedCommand, "'Directory', 'Folder'"); + assert.include( + decodedCommand, + "Invoke-Item -LiteralPath '\\\\wsl.localhost\\Ubuntu-24.04\\home\\t3\\workspace [draft]\\author''s ‘‘$media'", + ); + assert.include( decodedCommand, - "$ProgressPreference = 'SilentlyContinue'; $folderVerb = (Get-ItemProperty -LiteralPath 'Registry::HKEY_CLASSES_ROOT\\Directory\\shell' -ErrorAction SilentlyContinue).'(default)'; if ($folderVerb -and $folderVerb -ne 'none') { Invoke-Item -LiteralPath '\\\\wsl.localhost\\Ubuntu-24.04\\home\\t3\\workspace [draft]\\author''s $media' } else { Start-Process 'explorer.exe' -ArgumentList ('/select,\"' + '\\\\wsl.localhost\\Ubuntu-24.04\\home\\t3\\workspace [draft]\\author''s $media\\clip|draft.mp4' + '\"') }", + "'\\\\wsl.localhost\\Ubuntu-24.04\\home\\t3\\workspace [draft]\\author''s ‘‘$media\\clip|draft.mp4'", ); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); +const STOCK_FOLDER_VERB = + "\r\nHKEY_CLASSES_ROOT\\Directory\\shell\r\n (Default) REG_SZ none\r\n\r\n"; + +it.effect.each([ + ["selects the file with Explorer on stock Windows", { stdout: STOCK_FOLDER_VERB }, true], + [ + "keeps the PowerShell reveal for a replacement file manager", + { stdout: " (Default) REG_SZ open\r\n" }, + false, + ], + ["keeps the PowerShell reveal when the registry lookup fails", { exitCode: 1 }, false], +] as const)("%s", ([, registry, native]) => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const binDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-editors-" }); + yield* fileSystem.writeFileString(path.join(binDir, "explorer.EXE"), ""); + const spawned: ChildProcess.StandardCommand[] = []; + const verbatim: Array]> = []; + const exit = yield* Effect.gen(function* () { + const launcher = yield* ExternalLauncher.ExternalLauncher; + return yield* launcher + .launchEditor({ + editor: "file-manager", + cwd: "C:/Users/t3/my project/a,b.ts", + reveal: true, + }) + .pipe(Effect.exit); + }).pipe( + Effect.provideService(WindowsVerbatimSpawn, (command, args) => + Effect.sync(() => verbatim.push([command, args])), + ), + Effect.provide( + layerTest({ + platform: "win32", + env: { PATH: binDir, PATHEXT: ".COM;.EXE;.BAT;.CMD", SYSTEMROOT: "C:\\Windows" }, + onSpawn: (command) => spawned.push(command), + spawnResult: (command) => (command.command.endsWith("reg.exe") ? registry : undefined), + }), + ), + ); + + assert.deepEqual( + spawned.map((command) => command.command), + ["C:\\Windows\\System32\\reg.exe"], + ); + if (native) { + assert.isTrue(Exit.isSuccess(exit)); + assert.deepEqual(verbatim, [ + ["C:\\Windows\\explorer.exe", ['/select,"C:\\Users\\t3\\my project\\a,b.ts"']], + ]); + } else { + assert.deepEqual(verbatim, []); + // The PowerShell launch is reached; this host has no powershell.exe to run it. + assert.isTrue(Exit.isFailure(exit)); + if (Exit.isFailure(exit)) { + const error = Cause.squash(exit.cause); + assert.instanceOf(error, ExternalLauncher.ExternalLauncherCommandNotFoundError); + assert.equal( + (error as ExternalLauncher.ExternalLauncherCommandNotFoundError).command, + "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe", + ); + } + } + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + for (const wsl of [false, true]) { it.effect.skipIf(windowsHost)( "selects a file over D-Bus on " + (wsl ? "WSLg without interop PowerShell" : "Linux"), @@ -614,17 +903,32 @@ it.effect.skipIf(windowsHost)("does not offer the OpenZFS zed daemon as Zed on L ); yield* fs.chmod(zfsZed, 0o755); const editor = EDITORS.find((editor) => editor.id === "zed")!; - const resolve = resolveEditorCommand(editor, { HOME: home, PATH: binDir }).pipe( - Effect.provideService(HostProcess.Platform, "linux"), - Effect.provideService(Path.Path, withSystemBin(path, systemBin)), - ); - assert.deepEqual(yield* resolve, Option.none()); + const laterBin = path.join(home, "later-bin"); + const resolveOn = (platform: NodeJS.Platform) => + resolveEditorCommand(editor, { HOME: home, PATH: `${binDir}:${laterBin}` }).pipe( + Effect.provideService(HostProcess.Platform, platform), + Effect.provideService(Path.Path, withSystemBin(path, systemBin)), + ); + assert.deepEqual(yield* resolveOn("linux"), Option.none()); + assert.deepEqual(yield* resolveOn("darwin"), Option.none()); const zedEditor = path.join(home, ".local/bin/zed"); yield* fs.makeDirectory(path.dirname(zedEditor), { recursive: true }); yield* fs.writeFileString(zedEditor, "#!/bin/sh\necho 'Zed 0.200.0'\n"); yield* fs.chmod(zedEditor, 0o755); - assert.deepEqual(yield* resolve, Option.some({ command: zedEditor, baseArgs: [] })); + assert.deepEqual(yield* resolveOn("linux"), Option.some({ command: zedEditor, baseArgs: [] })); + + yield* fs.remove(zedEditor); + const laterZed = path.join(laterBin, "zed"); + yield* fs.makeDirectory(laterBin, { recursive: true }); + yield* fs.writeFileString(laterZed, "#!/bin/sh\necho 'Zed 0.200.0'\n"); + yield* fs.chmod(laterZed, 0o755); + for (const platform of ["linux", "darwin"] as const) { + assert.deepEqual( + yield* resolveOn(platform), + Option.some({ command: laterZed, baseArgs: [] }), + ); + } }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); @@ -647,7 +951,7 @@ it.effect.skipIf(windowsHost)("does not open Zed with the OpenZFS zed daemon on Effect.provide( layerTest({ platform: "linux", - env: { HOME: home, PATH: binDir }, + env: { HOME: home, PATH: binDir, DISPLAY: ":0" }, path: withSystemBin(path, systemBin), onSpawn: (command) => spawned.push(command), spawnResult: (command) => (command.args[0] === "--version" ? { exitCode: 1 } : undefined), @@ -687,7 +991,11 @@ it.effect.each([ yield* Effect.gen(function* () { const launcher = yield* ExternalLauncher.ExternalLauncher; assert.include(yield* launcher.resolveAvailableEditors(), editor); - yield* launcher.launchEditor({ editor, cwd: "/workspace with spaces/file.ts:12:4" }); + yield* launcher.launchEditor({ + editor, + cwd: "/workspace with spaces/file.ts:12:4", + projectRoot: "/workspace with spaces", + }); }).pipe( Effect.provide( layerTest({ @@ -703,6 +1011,7 @@ it.effect.each([ assert.ok(spawned); assert.equal(spawned.command, executable); assert.deepEqual(spawned.args, [ + "/workspace with spaces", "--line", "12", "--column", @@ -787,6 +1096,62 @@ it.effect("keeps the macOS VS Code Insiders bundle out of stable VS Code", () => }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); +it.effect.each([ + { + platform: "darwin", + scripts: "Library/Application Support/JetBrains/Toolbox/scripts", + suffix: "", + }, + { platform: "linux", scripts: ".local/share/JetBrains/Toolbox/scripts", suffix: "" }, + { platform: "win32", scripts: "JetBrains/Toolbox/scripts", suffix: ".cmd" }, + { + platform: "win32", + scripts: "JetBrains/Toolbox/scripts", + suffix: ".cmd", + install: "Programs/IntelliJ IDEA/bin/idea64.exe", + }, +] as const)( + "discovers a numbered JetBrains Toolbox script on $platform", + ({ platform, scripts, suffix, ...row }) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "t3-toolbox-numbered-" }); + const dir = path.join(home, scripts); + yield* fs.makeDirectory(dir, { recursive: true }); + for (const name of ["idea10", "idea2"]) { + yield* fs.writeFileString(path.join(dir, `${name}${suffix}`), "#!/bin/sh\n"); + yield* fs.chmod(path.join(dir, `${name}${suffix}`), 0o755); + } + const install = "install" in row ? path.join(home, row.install) : undefined; + if (install) { + yield* fs.makeDirectory(path.dirname(install), { recursive: true }); + yield* fs.writeFileString(install, ""); + } + + const editor = EDITORS.find((editor) => editor.id === "idea")!; + const resolved = yield* resolveEditorCommand(editor, { + HOME: home, + LOCALAPPDATA: home, + PATH: path.join(home, "empty"), + PATHEXT: ".COM;.EXE;.BAT;.CMD", + }).pipe( + Effect.provideService(HostProcess.Platform, platform), + // Keep IDEs installed on the host out of system locations such as /Applications. + Effect.provideService(FileSystem.FileSystem, { + ...fs, + stat: (file) => + fs.stat(file.startsWith(`${home}${path.sep}`) ? file : path.join(home, "missing")), + }), + ); + + assert.deepEqual( + resolved, + Option.some({ command: install ?? path.join(dir, `idea2${suffix}`), baseArgs: [] }), + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + for (const { platform, installPath, onPath } of [ { platform: "darwin", installPath: ".local/bin/agy", onPath: true }, { platform: "linux", installPath: ".local/bin/agy", onPath: false }, @@ -944,6 +1309,57 @@ it.effect( }, ); +it.effect.each(["Zed Nightly", "Zed Preview"])( + "launches stable Zed on Windows when %s comes first on PATH", + (channel) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-zed-channels-" }); + // A user folder naming Zed and a channel does not mark either install. + const programs = path.join(root, "Users", "zedpreview", "Programs"); + const prereleaseBin = path.join(programs, channel, "bin"); + const stableBin = path.join(programs, "Zed", "bin"); + for (const dir of [prereleaseBin, stableBin]) { + yield* fs.makeDirectory(dir, { recursive: true }); + yield* fs.writeFileString(path.join(dir, "zed.EXE"), ""); + } + const env = { PATH: `${prereleaseBin};${stableBin}`, PATHEXT: ".COM;.EXE;.BAT;.CMD" }; + + const launchZed = Effect.gen(function* () { + let spawned: ChildProcess.StandardCommand | undefined; + yield* Effect.gen(function* () { + const launcher = yield* ExternalLauncher.ExternalLauncher; + yield* launcher.launchEditor({ editor: "zed", cwd: "C:\\workspace" }); + }).pipe( + Effect.provide( + layerTest({ + platform: "win32", + env, + onSpawn: (command) => { + spawned = command; + }, + }), + ), + ); + assert.ok(spawned); + return spawned; + }); + + const stable = yield* launchZed; + assert.equal(stable.command, path.join(stableBin, "zed.EXE")); + assert.deepEqual(stable.args, ["C:\\workspace"]); + + env.PATH = `${stableBin};${prereleaseBin}`; + assert.equal((yield* launchZed).command, "zed"); + + env.PATH = `${prereleaseBin};${stableBin}`; + // With only the prerelease install left, it still opens by name. + yield* fs.remove(path.join(stableBin, "zed.EXE")); + assert.equal((yield* launchZed).command, "zed"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + const launchWindowsShimEditor = (cwd: string) => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; diff --git a/apps/server/src/process/externalLauncher.ts b/apps/server/src/process/externalLauncher.ts index 9b1a6971a98c..2bdb7bd3db4d 100644 --- a/apps/server/src/process/externalLauncher.ts +++ b/apps/server/src/process/externalLauncher.ts @@ -34,6 +34,7 @@ import * as ChildProcess from "effect/process/ChildProcess"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import { makeStaleWhileRevalidate } from "../environment/staleWhileRevalidate.ts"; +import { WindowsVerbatimSpawn } from "./windowsVerbatimSpawn.ts"; export { ExternalLauncherError, ExternalLauncherBrowserSpawnError, @@ -140,6 +141,7 @@ function resolveCommandEditorArgs( editor: (typeof EDITORS)[number], target: string, isDirectory: boolean, + projectRoot: string | undefined, ): ReadonlyArray { const parsedTarget = parseTargetPathAndPosition(target); @@ -147,13 +149,16 @@ function resolveCommandEditorArgs( case "direct-path": return [target]; case "goto": - if (Option.isSome(parsedTarget)) return ["--goto", target]; + // An existing folder wins over a position: `build:2026` is a legal folder name. // Without a window flag, VS Code's window settings may replace the user's last active window. - return isDirectory ? ["--new-window", target] : [target]; + if (isDirectory) return ["--new-window", target]; + return Option.isSome(parsedTarget) ? ["--goto", target] : [target]; case "line-column": return Option.match(parsedTarget, { onNone: () => [target], onSome: ({ path, line, column }) => [ + // Only positioned targets get the root: a folder target would open as a second project. + ...(projectRoot === undefined ? [] : [projectRoot]), "--line", line, // JetBrains IDEs count `--column` from 0; targets count columns from 1. @@ -178,11 +183,16 @@ function encodeUtf16LeBase64(input: string): string { } function escapePowerShellStringLiteral(input: string): string { - return `'${input.replaceAll("'", "''")}'`; + // PowerShell also recognises typographic apostrophes as single-quote delimiters. + return `'${input.replace(/['\u2018-\u201b]/g, "$&$&")}'`; +} + +function resolveWindowsRoot(env: NodeJS.ProcessEnv): string { + return env.SYSTEMROOT || env.windir || String.raw`C:\Windows`; } function resolvePowerShellPath(env: NodeJS.ProcessEnv = {}): string { - return `${env.SYSTEMROOT || env.windir || String.raw`C:\Windows`}\\System32\\WindowsPowerShell\\v1.0\\powershell.exe`; + return `${resolveWindowsRoot(env)}\\System32\\WindowsPowerShell\\v1.0\\powershell.exe`; } function resolveWslPowerShellPath(): string { @@ -361,11 +371,41 @@ const fileManagerRevealKindForPlatform = Effect.fn( return hasGraphicalLinuxSession(env) ? "files" : undefined; }); -function resolveWslFileManagerPath(target: string, distroName: string): string { +function wslUncFallbackPath(target: string, distroName: string): string { const relativePath = target.replace(/^\/+/, "").replaceAll("/", "\\"); return `\\\\wsl.localhost\\${distroName}${relativePath.length > 0 ? `\\${relativePath}` : ""}`; } +// wslpath -w is mount-aware, so a drive remapped in /etc/wsl.conf resolves to the +// real letter and a Linux path resolves to the \\wsl.localhost UNC path. Fall back +// to the UNC form by hand when wslpath is missing or fails. +export const resolveWslFileManagerPath = Effect.fn("externalLauncher.resolveWslFileManagerPath")( + function* ( + target: string, + distroName: string, + ): Effect.fn.Return { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + return yield* spawner + .spawn(ChildProcess.make("wslpath", ["-w", target], { stdin: "ignore", stderr: "ignore" })) + .pipe( + Effect.flatMap((handle) => + Effect.all([handle.stdout.pipe(Stream.decodeText(), Stream.mkString), handle.exitCode], { + concurrency: "unbounded", + }), + ), + Effect.map(([stdout, exitCode]) => { + const translated = stdout.trim(); + return exitCode === 0 && translated.length > 0 + ? translated + : wslUncFallbackPath(target, distroName); + }), + Effect.scoped, + Effect.timeout("2 seconds"), + Effect.orElseSucceed(() => wslUncFallbackPath(target, distroName)), + ); + }, +); + function buildBrowserLaunch( target: string, platform: NodeJS.Platform, @@ -433,16 +473,23 @@ const resolveBrowserLaunch = Effect.fn("externalLauncher.resolveBrowserLaunch")( return buildBrowserLaunch(target, platform, env); }); +// Editors and the file manager open on the desktop session, so a boot-started +// server discovers and launches them with that session's display. +const readEditorEnv = Effect.fn("externalLauncher.readEditorEnv")(function* () { + const env = { ...(yield* readBrowserLaunchEnv), ...(yield* readCommandLookupEnv) }; + return hasGraphicalLinuxSession(env) ? env : { ...env, ...(yield* readGraphicalSessionEnv()) }; +}); + const resolveAvailableEditors = Effect.fn("externalLauncher.resolveAvailableEditors")(function* () { const platform = yield* HostProcess.Platform; - const env = { ...(yield* readBrowserLaunchEnv), ...(yield* readCommandLookupEnv) }; + const env = yield* readEditorEnv(); return yield* buildAvailableEditors(platform, env).pipe(withPathDirectoryListings); }); const resolveFileManagerRevealKind = Effect.fn("externalLauncher.resolveFileManagerRevealKind")( function* () { const platform = yield* HostProcess.Platform; - const env = { ...(yield* readBrowserLaunchEnv), ...(yield* readCommandLookupEnv) }; + const env = yield* readEditorEnv(); return yield* fileManagerRevealKindForPlatform(platform, env); }, ); @@ -468,7 +515,7 @@ const resolveEditorLaunch = Effect.fn("resolveEditorLaunch")(function* ( FileSystem.FileSystem | Path.Path | ChildProcessSpawner.ChildProcessSpawner > { const platform = yield* HostProcess.Platform; - const env = { ...(yield* readBrowserLaunchEnv), ...(yield* readCommandLookupEnv) }; + const env = yield* readEditorEnv(); yield* Effect.annotateCurrentSpan({ "externalLauncher.editor": input.editor, "externalLauncher.cwd": input.cwd, @@ -498,7 +545,10 @@ const resolveEditorLaunch = Effect.fn("resolveEditorLaunch")(function* ( editor: editorDef.id, target: input.cwd, command, - args: [...baseArgs, ...resolveCommandEditorArgs(editorDef, input.cwd, isDirectory)], + args: [ + ...baseArgs, + ...resolveCommandEditorArgs(editorDef, input.cwd, isDirectory, input.projectRoot), + ], }; } @@ -511,31 +561,42 @@ const resolveEditorLaunch = Effect.fn("resolveEditorLaunch")(function* ( return yield* new ExternalLauncherUnsupportedEditorError({ editor: input.editor }); } - const target = expandHomePath(input.cwd, yield* HostProcess.HomeDirectory); + const target = input.cwd; if (input.reveal === true) { return yield* resolveFileManagerRevealLaunch(target, platform, env, command); } + if (platform === "win32") { + // ShellExecute honours replacement folder handlers that explorer.exe bypasses. + return { + ...powerShellFileManagerLaunch( + target, + `Invoke-Item -LiteralPath ${escapePowerShellStringLiteral(normalizeWindowsFileManagerPath(target))}`, + resolvePowerShellPath(env), + ), + detached: WINDOWS_POWERSHELL_DETACHED, + }; + } + return { editor: editorDef.id, target, command, args: command === "explorer.exe" && env.WSL_DISTRO_NAME !== undefined - ? [resolveWslFileManagerPath(target, env.WSL_DISTRO_NAME)] - : platform === "win32" - ? // Explorer interprets / as a switch and silently opens a fallback folder. - [normalizeWindowsFileManagerPath(target)] - : [target], + ? [yield* resolveWslFileManagerPath(target, env.WSL_DISTRO_NAME)] + : [target], }; }); -// A replacement manager registers a default Directory shell verb; stock Windows -// uses "none". Explorer needs its /select argument passed verbatim through -// PowerShell because Node's argument quoting changes how Explorer parses it. +// Replacement managers can register under Directory, Drive, or Folder. Explorer +// needs its /select argument passed verbatim through PowerShell because Node's +// argument quoting changes how Explorer parses it. function buildFileManagerRevealPowerShellSource(target: string, parent: string): string { const literal = escapePowerShellStringLiteral(target); - return `$ProgressPreference = 'SilentlyContinue'; $folderVerb = (Get-ItemProperty -LiteralPath 'Registry::HKEY_CLASSES_ROOT\\Directory\\shell' -ErrorAction SilentlyContinue).'(default)'; if ($folderVerb -and $folderVerb -ne 'none') { Invoke-Item -LiteralPath ${escapePowerShellStringLiteral(parent)} } else { Start-Process 'explorer.exe' -ArgumentList ('/select,"' + ${literal} + '"') }`; + const folderClass = /^[a-z]:\\$/i.test(parent) ? "Drive" : "Directory"; + // Explorer's ordinary open/explore handlers may also be explicit defaults. + return String.raw`$ProgressPreference = 'SilentlyContinue'; $folderVerb = '${folderClass}', 'Folder' | ForEach-Object { $key = "Registry::HKEY_CLASSES_ROOT\$_\shell"; $verb = (Get-ItemProperty -LiteralPath $key -ErrorAction SilentlyContinue).'(default)'; if ($verb -and $verb -ne 'none') { $handler = Get-ItemProperty -LiteralPath "$key\$verb\command" -ErrorAction SilentlyContinue; if ($handler.DelegateExecute -ne '{11dbb47c-a525-400b-9e80-a54615a090c0}' -and $handler.'(default)' -notmatch '^\s*"?(?:[^"]*[\\/])?explorer\.exe(?:["\s]|$)') { $verb } } }; if ($folderVerb) { Invoke-Item -LiteralPath ${escapePowerShellStringLiteral(parent)} } else { Start-Process 'explorer.exe' -ArgumentList ('/select,"' + ${literal} + '"') }`; } function powerShellFileManagerLaunch( @@ -587,13 +648,18 @@ const resolveFileManagerRevealLaunch = Effect.fn("resolveFileManagerRevealLaunch shouldUseWindowsHostFromWsl(platform, env) && env.WSL_DISTRO_NAME !== undefined ) { - const explorerTarget = resolveWslFileManagerPath(target, env.WSL_DISTRO_NAME); if (yield* isCommandAvailable(WSL_POWERSHELL_COMMAND, { env })) { // WSL names may contain quotes that Explorer's /select cannot encode. // Open the parent through the shell so replacement managers still apply. const path = yield* Path.Path; // .NET path parsing rejects characters that are legal in WSL filenames. - const parent = resolveWslFileManagerPath(path.dirname(target), env.WSL_DISTRO_NAME); + const [explorerTarget, parent] = yield* Effect.all( + [ + resolveWslFileManagerPath(target, env.WSL_DISTRO_NAME), + resolveWslFileManagerPath(path.dirname(target), env.WSL_DISTRO_NAME), + ], + { concurrency: "unbounded" }, + ); const source = explorerTarget.includes('"') ? `Invoke-Item -LiteralPath ${escapePowerShellStringLiteral(parent)}` : buildFileManagerRevealPowerShellSource(explorerTarget, parent); @@ -608,7 +674,7 @@ const resolveFileManagerRevealLaunch = Effect.fn("resolveFileManagerRevealLaunch editor: "file-manager", target, command: "explorer.exe", - args: [resolveWslFileManagerPath(path.dirname(target), env.WSL_DISTRO_NAME)], + args: [yield* resolveWslFileManagerPath(path.dirname(target), env.WSL_DISTRO_NAME)], }; } @@ -659,18 +725,72 @@ const tryRevealLinuxItem = Effect.fn("externalLauncher.tryRevealLinuxItem")( }), ); +// Stock Windows answers "none"; any other verb may be a replacement file manager, which only +// the PowerShell reveal honors. Reading stdout is about 600 ms faster than spawner.string here. +const hasStockWindowsFolderVerb = Effect.fn("externalLauncher.hasStockWindowsFolderVerb")( + function* () { + const env = yield* readBrowserLaunchEnv; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const handle = yield* spawner.spawn( + ChildProcess.make( + `${resolveWindowsRoot(env)}\\System32\\reg.exe`, + ["query", "HKCR\\Directory\\shell", "/ve"], + { stdin: "ignore", stderr: "ignore" }, + ), + ); + const [stdout, exitCode] = yield* Effect.all( + [handle.stdout.pipe(Stream.decodeText(), Stream.mkString), handle.exitCode], + { concurrency: "unbounded" }, + ); + return exitCode === 0 && /REG_SZ\s+none\s*$/im.test(stdout); + }, + Effect.scoped, + Effect.timeout("2 seconds"), + Effect.orElseSucceed(() => false), +); + +// Explorer parses /select itself, so it needs the argument verbatim rather than Node's quoting. +const revealWindowsItem = Effect.fn("externalLauncher.revealWindowsItem")(function* ( + target: string, +) { + const command = `${resolveWindowsRoot(yield* readBrowserLaunchEnv)}\\explorer.exe`; + const args = [`/select,"${normalizeWindowsFileManagerPath(target)}"`]; + yield* (yield* WindowsVerbatimSpawn)(command, args).pipe( + Effect.mapError( + (cause) => + new ExternalLauncherEditorSpawnError({ + editor: "file-manager", + target, + command, + args, + cause, + }), + ), + ); +}); + const launchEditor = Effect.fn("externalLauncher.launchEditor")(function* ( input: LaunchEditorInput, + hasStockFolderVerb: Effect.Effect, ) { - const launch = yield* resolveEditorLaunch(input); - if ( - input.editor === "file-manager" && - input.reveal === true && - launch.command === "xdg-open" && - (yield* HostProcess.Platform) === "linux" && - (yield* tryRevealLinuxItem(launch.target)) - ) { - return; + // Clients send `~/` host paths as written, and launches run without a shell to expand them. + const home = yield* HostProcess.HomeDirectory; + const cwd = expandHomePath(input.cwd, home); + const projectRoot = + input.projectRoot === undefined ? undefined : expandHomePath(input.projectRoot, home); + const launch = yield* resolveEditorLaunch({ ...input, cwd, projectRoot }); + if (input.editor === "file-manager" && input.reveal === true) { + const platform = yield* HostProcess.Platform; + if ( + platform === "linux" && + launch.command === "xdg-open" && + (yield* tryRevealLinuxItem(launch.target)) + ) { + return; + } + if (platform === "win32" && (yield* hasStockFolderVerb)) { + return yield* revealWindowsItem(launch.target); + } } yield* launchEditorProcess(launch); }); @@ -831,6 +951,12 @@ export const make = Effect.gen(function* () { ), EDITOR_DISCOVERY_CACHE_TTL, ); + const cachedStockWindowsFolderVerb = yield* makeStaleWhileRevalidate( + hasStockWindowsFolderVerb().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ), + EDITOR_DISCOVERY_CACHE_TTL, + ); // Forwarders retain the underlying launcher and discovery spans. return ExternalLauncher.of({ @@ -848,7 +974,9 @@ export const make = Effect.gen(function* () { ), launchEditor: Effect.fnUntraced( function* (input) { - return yield* provideCommandResolutionServices(launchEditor(input)); + return yield* provideCommandResolutionServices( + launchEditor(input, cachedStockWindowsFolderVerb), + ); }, Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), ), diff --git a/apps/server/src/process/providerProcesses.test.ts b/apps/server/src/process/providerProcesses.test.ts new file mode 100644 index 000000000000..26bad1cec9ac --- /dev/null +++ b/apps/server/src/process/providerProcesses.test.ts @@ -0,0 +1,127 @@ +// @effect-diagnostics nodeBuiltinImport:off -- real processes stand in for servers and CLIs. +import * as NodeChildProcess from "node:child_process"; +import * as NodeCrypto from "node:crypto"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, describe, expect, it } from "@effect/vitest"; +import * as AcpSessionRuntime from "@t3tools/provider-acp/server/AcpSessionRuntime"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Context from "effect/Context"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; + +import * as ServerConfig from "../config.ts"; +import * as ProviderProcesses from "./providerProcesses.ts"; + +// Real child processes stand in for servers and CLIs; only PIDs captured at spawn are signalled. +const ownedProcess = Effect.gen(function* () { + const ready = yield* Deferred.make(); + const exited = yield* Deferred.make(); + const child = yield* Effect.acquireRelease( + Effect.sync(() => { + const child = NodeChildProcess.spawn( + process.execPath, + ["-e", 'process.stdin.resume(); process.send("ready");'], + { stdio: ["pipe", "ignore", "ignore", "ipc"] }, + ); + child.once("message", () => Deferred.doneUnsafe(ready, Effect.void)); + child.once("error", (error) => Deferred.doneUnsafe(ready, Effect.die(error))); + child.once("exit", () => Deferred.doneUnsafe(exited, Effect.void)); + return child; + }), + (child) => + Effect.sync(() => child.kill("SIGKILL")).pipe(Effect.andThen(Deferred.await(exited))), + ); + yield* Deferred.await(ready); + const pid = child.pid; + assert(pid !== undefined); + const controller = AcpSessionRuntime.makePosixProcessTreeController("linux"); + const identity = controller.identity(pid); + assert(identity !== undefined); + const boot = NodeFS.readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim(); + return { + pid, + startTime: `${boot}:${identity.startTime}`, + pgid: 0, + running: () => { + const observed = controller.identity(pid); + return observed !== undefined && !AcpSessionRuntime.posixProcessIsZombie(observed); + }, + stop: Effect.sync(() => child.kill("SIGKILL")).pipe(Effect.andThen(Deferred.await(exited))), + }; +}); + +describe.runIf(HostProcess.Platform.defaultValue() === "linux")("ProviderProcesses", () => { + it.live("reaps only recorded CLIs of dead servers before startup completes", () => + Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + const directory = NodePath.join(config.stateDir, "provider-processes"); + NodeFS.mkdirSync(directory, { recursive: true }); + type Identity = { readonly pid: number; readonly startTime: string }; + const record = (owner: Identity, processes: ReadonlyArray) => { + const instanceId = NodeCrypto.randomUUID(); + NodeFS.writeFileSync( + NodePath.join(directory, `${instanceId}.json`), + JSON.stringify({ + version: 1, + instanceId, + hostname: NodeOS.hostname(), + stateDir: config.stateDir, + owner: { pid: owner.pid, startTime: owner.startTime }, + processes: processes.map(({ pid, startTime }) => ({ pid, startTime, pgid: 0 })), + }), + ); + }; + const reused = (identity: Identity) => ({ + ...identity, + startTime: identity.startTime.replace(/:\d+$/, ":1"), + }); + const deadOwner = yield* ownedProcess; + const liveOwner = yield* ownedProcess; + const reusedOwner = yield* ownedProcess; + const liveProvider = yield* ownedProcess; + const reusedProvider = yield* ownedProcess; + const orphan = yield* ownedProcess; + const orphanOfReusedOwner = yield* ownedProcess; + const unrecorded = yield* ownedProcess; + record(liveOwner, [liveProvider]); + record(deadOwner, [reused(reusedProvider)]); + record(deadOwner, [orphan]); + record(reused(reusedOwner), [orphanOfReusedOwner]); + yield* deadOwner.stop; + + const context = yield* Layer.build(ProviderProcesses.layer); + expect(Context.get(context, ProviderProcesses.ProviderProcesses).hasOtherLiveOwner).toBe( + true, + ); + + expect({ + orphan: orphan.running(), + orphanOfReusedOwner: orphanOfReusedOwner.running(), + liveProvider: liveProvider.running(), + reusedProvider: reusedProvider.running(), + reusedOwner: reusedOwner.running(), + unrecorded: unrecorded.running(), + }).toEqual({ + orphan: false, + orphanOfReusedOwner: false, + liveProvider: true, + reusedProvider: true, + reusedOwner: true, + unrecorded: true, + }); + }).pipe( + Effect.scoped, + Effect.provideService(HostProcess.Platform, "linux"), + Effect.provide( + ServerConfig.layerTest(process.cwd(), { prefix: "t3-restart-ownership-" }).pipe( + Layer.provideMerge(NodeServices.layer), + ), + ), + ), + ); +}); diff --git a/apps/server/src/process/providerProcesses.ts b/apps/server/src/process/providerProcesses.ts new file mode 100644 index 000000000000..4fe9a47637a0 --- /dev/null +++ b/apps/server/src/process/providerProcesses.ts @@ -0,0 +1,140 @@ +// @effect-diagnostics nodeBuiltinImport:off -- the SDK's synchronous spawn callback must record ownership before releasing the child. +import * as NodeCrypto from "node:crypto"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import * as AcpSessionRuntime from "@t3tools/provider-acp/server/AcpSessionRuntime"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; + +import * as ServerConfig from "../config.ts"; + +const ProcessIdentity = Schema.Struct({ + pid: Schema.Int.check(Schema.isBetween({ minimum: 2, maximum: Number.MAX_SAFE_INTEGER })), + startTime: Schema.String, +}); +const OwnershipRecord = Schema.Struct({ + version: Schema.Literal(1), + instanceId: Schema.String, + hostname: Schema.String, + stateDir: Schema.String, + owner: ProcessIdentity, + processes: Schema.Array(Schema.Struct({ ...ProcessIdentity.fields, pgid: Schema.Int })), +}); +const decodeRecord = Schema.decodeUnknownSync(Schema.fromJsonString(OwnershipRecord)); + +// A concurrent exit can remove the record. Incomplete/unknown records never authorize a signal. +const readRecord = (recordPath: string) => { + try { + return decodeRecord(NodeFS.readFileSync(recordPath, "utf8")); + } catch { + return undefined; + } +}; + +export const ProviderProcesses = Context.Reference<{ + readonly register?: (pid: number) => () => void; + readonly hasOtherLiveOwner: boolean; +}>("t3/process/ProviderProcesses", { defaultValue: () => ({ hasOtherLiveOwner: false }) }); + +const make = Effect.gen(function* () { + const platform = yield* HostProcess.Platform; + if (platform !== "linux") return { hasOtherLiveOwner: false }; + const { stateDir } = yield* ServerConfig.ServerConfig; + const directory = NodePath.join(stateDir, "provider-processes"); + const hostname = NodeOS.hostname(); + const boot = NodeFS.readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim(); + const controller = AcpSessionRuntime.makePosixProcessTreeController(platform); + const identity = (pid: number) => { + const observed = controller.identity(pid); + if (observed === undefined || AcpSessionRuntime.posixProcessIsZombie(observed)) + return undefined; + return { ...observed, startTime: `${boot}:${observed.startTime}` }; + }; + const matches = (expected: typeof ProcessIdentity.Type) => + identity(expected.pid)?.startTime === expected.startTime; + const kill = (survivor: typeof ProcessIdentity.Type) => { + try { + controller.signalProcess(survivor.pid, "SIGKILL"); + } catch {} + }; + const owner = identity(process.pid); + if (owner === undefined) return yield* Effect.die(new Error("Cannot identify provider owner")); + NodeFS.mkdirSync(directory, { recursive: true }); + + const writeRecord = (processes: (typeof OwnershipRecord.Type)["processes"]) => { + const instanceId = NodeCrypto.randomUUID(); + const recordPath = NodePath.join(directory, `${instanceId}.json`); + const pendingPath = `${recordPath}.pending`; + NodeFS.writeFileSync( + pendingPath, + JSON.stringify({ + version: 1, + instanceId, + hostname, + stateDir, + owner: { pid: owner.pid, startTime: owner.startTime }, + processes, + } satisfies typeof OwnershipRecord.Type), + ); + NodeFS.renameSync(pendingPath, recordPath); + return () => NodeFS.rmSync(recordPath, { force: true }); + }; + // Publish the server identity even before its first provider. Another server + // using this home must not stop the shared systemd slice while we are alive. + const forgetOwner = writeRecord([]); + yield* Effect.addFinalizer(() => Effect.sync(forgetOwner)); + let hasOtherLiveOwner = false; + + // Only immutable per-launch records are shared between server instances. A + // live owner's records belong to that server, even when it uses this home. + for (const file of NodeFS.readdirSync(directory)) { + if (!file.endsWith(".json")) continue; + const recordPath = NodePath.join(directory, file); + const record = readRecord(recordPath); + if (record === undefined) continue; + if (record.hostname !== hostname || record.stateDir !== stateDir) continue; + if (matches(record.owner)) { + if (record.owner.pid !== process.pid) hasOtherLiveOwner = true; + continue; + } + let survived = false; + for (const survivor of record.processes) { + if (survivor.pid === process.pid || !matches(survivor)) continue; + // Never infer ownership from argv, executable names, or group membership. + // As in the ACP controller, starttime blocks reuse between observations. + // Node has no pidfd_send_signal, so the final read-to-signal race remains. + kill(survivor); + for (let attempt = 0; matches(survivor) && !survived; attempt++) { + survived = attempt === 100; + if (!survived) yield* Effect.sleep("10 millis"); + } + } + // Like a failed slice stop, an unkillable CLI must not keep the server down; + // its record stays so the next start retries. + if (survived) { + yield* Effect.logWarning("Could not stop a provider left by a previous server", { file }); + continue; + } + NodeFS.rmSync(recordPath, { force: true }); + } + + return { + hasOtherLiveOwner, + register: (pid: number) => { + const provider = identity(pid); + if (provider === undefined || provider.ppid !== process.pid) { + throw new Error("Cannot identify spawned provider"); + } + // A rename keeps startup from seeing a partial write. The child is gated + // until this returns, so an abrupt death cannot leave an unrecorded CLI. + return writeRecord([{ pid, startTime: provider.startTime, pgid: provider.pgid }]); + }, + }; +}); + +export const layer = Layer.effect(ProviderProcesses, make); diff --git a/apps/server/src/process/windowsVerbatimSpawn.ts b/apps/server/src/process/windowsVerbatimSpawn.ts new file mode 100644 index 000000000000..15075c22455b --- /dev/null +++ b/apps/server/src/process/windowsVerbatimSpawn.ts @@ -0,0 +1,29 @@ +// @effect-diagnostics nodeBuiltinImport:off - Effect's spawner has no windowsVerbatimArguments option. +import * as NodeChildProcess from "node:child_process"; + +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; + +export type WindowsVerbatimSpawner = ( + command: string, + args: ReadonlyArray, +) => Effect.Effect; + +const spawnDetachedVerbatim: WindowsVerbatimSpawner = (command, args) => + Effect.callback((resume) => { + const child = NodeChildProcess.spawn(command, args, { + detached: true, + stdio: "ignore", + windowsVerbatimArguments: true, + }); + child.once("error", (error) => resume(Effect.fail(error))); + child.once("spawn", () => { + child.unref(); + resume(Effect.void); + }); + }); + +export const WindowsVerbatimSpawn = Context.Reference( + "t3/process/WindowsVerbatimSpawn", + { defaultValue: () => spawnDetachedVerbatim }, +); diff --git a/apps/server/src/processRunner.test.ts b/apps/server/src/processRunner.test.ts index 3754f25a4c77..6768fa23c763 100644 --- a/apps/server/src/processRunner.test.ts +++ b/apps/server/src/processRunner.test.ts @@ -2,6 +2,8 @@ import { isWindowsCommandNotFound } from "@t3tools/provider-core/server/snapshot import { describe, expect, it } from "@effect/vitest"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Cause from "effect/Cause"; import * as Layer from "effect/Layer"; import * as PlatformError from "effect/PlatformError"; import * as Sink from "effect/Sink"; @@ -143,6 +145,29 @@ describe("runProcess", () => { }), ); + it.effect("reports child startup before delivering cancellation during acquisition", () => + Effect.gen(function* () { + let started = false; + const spawner = makeSpawner(() => + Effect.withFiber((fiber) => + Effect.sync(() => { + fiber.interruptUnsafe(); + return makeHandle({}); + }), + ), + ); + const result = yield* runWith(spawner)({ + command: "git", + args: ["restore"], + onSpawn: () => { + started = true; + }, + }).pipe(Effect.exit); + expect(Exit.isFailure(result) && Cause.hasInterruptsOnly(result.cause)).toBe(true); + expect(started).toBe(true); + }).pipe(Effect.scoped), + ); + it.effect("fails when output exceeds max buffer in default mode", () => Effect.gen(function* () { const spawner = makeSpawner(() => Effect.succeed(makeHandle({ stdout: "x".repeat(2048) }))); diff --git a/apps/server/src/processRunner.ts b/apps/server/src/processRunner.ts index a345c67eca88..4c7efd3cc994 100644 --- a/apps/server/src/processRunner.ts +++ b/apps/server/src/processRunner.ts @@ -26,6 +26,8 @@ export interface ProcessRunInput { readonly stdin?: string | undefined; /** Receives every stdout chunk, including bytes beyond the buffered output limit. */ readonly onStdoutChunk?: ((chunk: Uint8Array) => void) | undefined; + /** Runs after successful spawn, before cancellation can interrupt acquisition. */ + readonly onSpawn?: (() => void) | undefined; readonly maxOutputBytes?: number | undefined; readonly outputMode?: "error" | "truncate" | undefined; readonly truncatedMarker?: string | undefined; @@ -290,7 +292,7 @@ const runProcessCore = Effect.fn("processRunner.runProcessCore")(function* ( cause, }); - const child = yield* spawner + const spawn = spawner .spawn( ChildProcess.make(spawnCommand.command, spawnCommand.args, { ...((input.spawnCwd ?? input.cwd) ? { cwd: input.spawnCwd ?? input.cwd } : {}), @@ -309,6 +311,12 @@ const runProcessCore = Effect.fn("processRunner.runProcessCore")(function* ( // Synchronous spawn throws arrive as defects; recover them too so callers see typed spawn failures. Effect.catchDefect((cause) => Effect.fail(toSpawnError(cause))), ); + const child = yield* input.onSpawn === undefined + ? spawn + : spawn.pipe( + Effect.tap(() => Effect.sync(() => input.onSpawn?.())), + Effect.uninterruptible, + ); const stdin = input.stdin; const onStdoutChunk = input.onStdoutChunk; diff --git a/apps/server/src/project/AgentSessionImporter.test.ts b/apps/server/src/project/AgentSessionImporter.test.ts index b05a0a0504cd..18c0946144d4 100644 --- a/apps/server/src/project/AgentSessionImporter.test.ts +++ b/apps/server/src/project/AgentSessionImporter.test.ts @@ -1,3 +1,4 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; import { expect, it } from "@effect/vitest"; import { type AgentSessionSource, @@ -15,24 +16,27 @@ import { type OrchestrationV2ProviderThread, } from "@t3tools/contracts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; -import * as KeyedLock from "@t3tools/shared/KeyedLock"; import * as AcpRegistryRuntimeCoordinator from "@t3tools/provider-acp-registry/server/AcpRegistryRuntimeCoordinator"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as Path from "effect/Path"; import * as Stream from "effect/Stream"; import * as SqlClient from "effect/sql/SqlClient"; import * as TestClock from "effect/testing/TestClock"; +import * as ServerConfig from "../config.ts"; import * as EventSink from "../orchestration-v2/EventSink.ts"; import * as EventStore from "../orchestration-v2/EventStore.ts"; import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; +import * as ProjectStore from "../orchestration-v2/ProjectStore.ts"; import * as ProviderAdapterRegistry from "../orchestration-v2/ProviderAdapterRegistry.ts"; import * as ThreadCommandExecutor from "../orchestration-v2/ThreadCommandExecutor.ts"; import { @@ -92,6 +96,7 @@ function candidate( { role: "assistant" as const, text: "Fixed", createdAt: timestamp }, ], }, + worktree: null as { path: string; branch: string | null } | null, } satisfies AgentSessionScanner.AgentSessionRecentThread; } @@ -179,14 +184,18 @@ const seedOwner = Effect.fn(function* ( }); const makeImporter = Effect.fn(function* ( - candidates: ReadonlyArray>, + candidates: ReadonlyArray, beforeTransaction: Effect.Effect = Effect.void, getAdapter?: ProviderAdapterRegistry.ProviderAdapterRegistryV2Shape["get"], - threadCommands?: KeyedLock.KeyedLock, + threadCommands?: ThreadCommandExecutor.ThreadCommandExecutor["Service"], settings: Layer.Layer< ServerSettings.ServerSettingsService, ServerSettingsError > = ServerSettings.layerTest(), + realScan?: { + readonly workspaceRoot: string; + readonly scanner: AgentSessionScanner.AgentSessionScanner["Service"]; + }, ) { const sql = yield* SqlClient.SqlClient; const events = yield* EventStore.EventStoreV2; @@ -228,15 +237,17 @@ const makeImporter = Effect.fn(function* ( ProviderSessionRuntime.ProviderSessionRuntimeRepository, yield* ProviderSessionRuntime.ProviderSessionRuntimeRepository, ), - Layer.mock(AgentSessionScanner.AgentSessionScanner)({ - recentThreads: () => Stream.fromIterable(candidates), - }), + realScan === undefined + ? Layer.mock(AgentSessionScanner.AgentSessionScanner)({ + recentThreads: () => Stream.fromIterable(candidates), + }) + : Layer.succeed(AgentSessionScanner.AgentSessionScanner, realScan.scanner), Layer.mock(ProjectService.ProjectService)({ getById: () => Effect.succeedSome({ id: projectId, title: "Project", - workspaceRoot: "/workspace/project", + workspaceRoot: realScan?.workspaceRoot ?? "/workspace/project", defaultModelSelection: null, scripts: [], createdAt: timestamp, @@ -259,7 +270,7 @@ const makeImporter = Effect.fn(function* ( ), Layer.succeed( ThreadCommandExecutor.ThreadCommandExecutor, - threadCommands ?? (yield* KeyedLock.make()), + threadCommands ?? (yield* ThreadCommandExecutor.make), ), ), ), @@ -304,6 +315,7 @@ it.effect( ); const unowned = candidate(CODEX_MODEL_SELECTION, "unowned"); unowned.thread.omittedMessageCount = 1; + unowned.worktree = { path: "/workspace/project/.claude/worktrees/feat", branch: "feat" }; const unownedPi = candidate(PI_MODEL_SELECTION, "unowned-pi", "pi"); const importer = yield* makeImporter([ active, @@ -356,6 +368,8 @@ it.effect( expect(notices).toEqual([]); } expect(records.thread.historyOrigin).toBe("v1_import"); + expect(records.thread.worktreePath).toBe(source.worktree?.path ?? null); + expect(records.thread.branch).toBe(source.worktree?.branch ?? null); expect(records.thread.activeProviderThreadId).toBe(records.providerThreads[0]?.id); expect(records.providerThreads[0]?.nativeThreadRef?.nativeId).toBe(nativeId(source)); expect(Option.getOrThrow(yield* runtimes.getByThreadId({ threadId }))).toMatchObject({ @@ -364,12 +378,52 @@ it.effect( source.source.provider === "claudeAgent" ? { threadId, resume: source.source.providerSessionId } : { threadId: nativeId(source) }, - runtimePayload: { importedTranscripts: [source.source] }, + runtimePayload: { + cwd: source.worktree?.path ?? "/workspace/project", + importedTranscripts: [source.source], + }, }); } }).pipe(Effect.scoped, Effect.provide(persistence)), ); +it.effect("tolerates a missing transcript owner but propagates failed ownership reads", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const runtimes = yield* ProviderSessionRuntime.ProviderSessionRuntimeRepository; + const source = candidate(CODEX_MODEL_SELECTION, "ownership-read"); + const threadId = importId(source); + yield* runtimes.upsert({ + threadId: ThreadId.make("missing-owner"), + providerName: "codex", + providerInstanceId: source.source.providerInstanceId, + adapterKey: "codex", + runtimeMode: "full-access", + status: "stopped", + lastSeenAt: timestamp, + resumeCursor: null, + runtimePayload: { importedTranscripts: [source.source] }, + }); + yield* runtimes.recordImportedTranscript({ + threadId: ThreadId.make("missing-owner"), + source: source.source, + }); + const importer = yield* makeImporter([source]); + expect(yield* importer.run).toEqual({ importedCount: 1, skippedCount: 0 }); + + yield* sql`UPDATE orchestration_v2_projection_threads SET payload_json = '{}' WHERE thread_id = ${threadId}`; + expect(yield* Effect.flip(importer.run)).toMatchObject({ + _tag: "AgentSessionScanError", + operation: "read-projects", + cause: { + _tag: "OrchestratorProjectionError", + threadId, + cause: { _tag: "ProjectionStoreReadError" }, + }, + }); + }).pipe(Effect.scoped, Effect.provide(persistence)), +); + it.effect("rechecks ownership after the import reaches the commit transaction", () => Effect.gen(function* () { const enteringTransaction = yield* Deferred.make(); @@ -429,8 +483,8 @@ it.effect( expect(yield* runtimes.list()).toEqual([]); yield* sql`DROP TRIGGER fail_runtime`; - expect(yield* importer.run).toEqual({ importedCount: 1, skippedCount: 0 }); - expect(yield* importer.run).toEqual({ importedCount: 1, skippedCount: 0 }); + expect(yield* importer.run).toEqual({ importedCount: 0, skippedCount: 0 }); + expect(yield* importer.run).toEqual({ importedCount: 0, skippedCount: 0 }); expect(yield* events.read({ threadId }).pipe(Stream.runCollect)).toEqual(committed); expect(importer.published).toHaveLength(6); expect( @@ -446,6 +500,82 @@ it.effect( }).pipe(Effect.scoped, Effect.provide(persistence)), ); +it.effect("counts only new threads when re-importing cached and recovered transcripts", () => + Effect.gen(function* () { + const events = yield* EventStore.EventStoreV2; + const runtimes = yield* ProviderSessionRuntime.ProviderSessionRuntimeRepository; + const cached = candidate( + CLAUDE_MODEL_SELECTION, + "020e7f00-dce6-406b-8ec6-ed90498945d6", + "claudeAgent", + ); + const recovered = candidate(CODEX_MODEL_SELECTION, "recovered"); + const initial = yield* makeImporter([cached, recovered]); + expect(yield* initial.run).toEqual({ importedCount: 2, skippedCount: 0 }); + const before = yield* events.read({ threadId: importId(recovered) }).pipe(Stream.runCollect); + // The file changed on disk but still holds the imported messages. + recovered.source.size += 100; + recovered.source.mtimeMs += 1; + const duplicate = { ...recovered.source, filePath: "/tmp/duplicate.jsonl" }; + const fresh = candidate(CODEX_MODEL_SELECTION, "fresh"); + const repeated = yield* makeImporter([ + { _tag: "AlreadyImported", source: cached.source }, + recovered, + { _tag: "Duplicate", source: duplicate }, + fresh, + { _tag: "Skipped" }, + ]); + + expect(yield* repeated.run).toEqual({ importedCount: 1, skippedCount: 1 }); + expect(yield* events.read({ threadId: importId(recovered) }).pipe(Stream.runCollect)).toEqual( + before, + ); + expect( + new Set( + repeated.published.flatMap((stored) => ("event" in stored ? [stored.event.threadId] : [])), + ), + ).toEqual(new Set([importId(fresh)])); + expect( + Option.getOrThrow(yield* runtimes.getByThreadId({ threadId: importId(recovered) })), + ).toMatchObject({ + runtimePayload: { importedTranscripts: [recovered.source, duplicate] }, + }); + }).pipe(Effect.scoped, Effect.provide(persistence)), +); + +it.effect("does not certify new native history after a partial import retry", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const runtimes = yield* ProviderSessionRuntime.ProviderSessionRuntimeRepository; + const original = candidate(CODEX_MODEL_SELECTION, "partial-retry"); + const threadId = importId(original); + const importer = yield* makeImporter([original]); + yield* sql`CREATE TEMP TRIGGER fail_runtime BEFORE INSERT ON provider_session_runtime BEGIN SELECT RAISE(ABORT, 'test runtime failure'); END`; + expect(yield* importer.run).toEqual({ importedCount: 0, skippedCount: 1 }); + yield* sql`DROP TRIGGER fail_runtime`; + expect(yield* runtimes.list()).toEqual([]); + const continued = candidate(CODEX_MODEL_SELECTION, original.source.providerSessionId); + continued.source.size = 200; + continued.source.mtimeMs = 3; + continued.thread.messages.push( + { role: "user", text: "Continue outside T3", createdAt: timestamp }, + { role: "assistant", text: "Native continuation", createdAt: timestamp }, + ); + // The thread holds only the original messages, so the continued transcript stays unrecorded. + expect(yield* (yield* makeImporter([continued])).run).toEqual({ + importedCount: 0, + skippedCount: 1, + }); + expect(yield* runtimes.list()).toEqual([]); + expect( + (yield* projections.getThreadRecords(threadId, ["messages"])).messages.map( + (message) => message.text, + ), + ).toEqual(["Fix it", "Fixed"]); + }).pipe(Effect.scoped, Effect.provide(persistence)), +); + it.effect("does not record a changed transcript as imported without its new messages", () => Effect.gen(function* () { const projections = yield* ProjectionStore.ProjectionStoreV2; @@ -685,7 +815,7 @@ const importDuringCommand = Effect.fn(function* ( const projections = yield* ProjectionStore.ProjectionStoreV2; const sink = yield* EventSink.EventSinkV2; const threadId = yield* seedOwner(candidate(CODEX_MODEL_SELECTION, name)); - const lock = yield* KeyedLock.make(); + const lock = yield* ThreadCommandExecutor.make; const loaded = yield* Deferred.make(); const importerCommitting = yield* Deferred.make(); const commandWritten = yield* Deferred.make(); @@ -829,3 +959,95 @@ it.effect("imports under a policy ceiling in the project's default, kept under t ); }).pipe(Effect.scoped, Effect.provide(persistence)), ); + +it.effect("repeated imports reach linked-worktree sessions behind the scan budget", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-import-worktree-budget-" }); + const repo = path.join(root, "repo"); + const worktree = path.join(root, "feature"); + const worktreeGitDir = path.join(repo, ".git", "worktrees", "feature"); + yield* fs.makeDirectory(worktreeGitDir, { recursive: true }); + yield* fs.makeDirectory(worktree); + yield* fs.writeFileString(path.join(repo, ".git", "config"), "[core]\n"); + yield* fs.writeFileString(path.join(worktreeGitDir, "commondir"), "../..\n"); + yield* fs.writeFileString(path.join(worktreeGitDir, "HEAD"), "ref: refs/heads/feature\n"); + yield* fs.writeFileString(path.join(worktree, ".git"), `gitdir: ${worktreeGitDir}\n`); + + const claudeHome = path.join(root, "claude"); + const codexHome = path.join(root, "codex"); + const sessions = path.join(codexHome, "sessions", "2026", "09", "01"); + yield* fs.makeDirectory(sessions, { recursive: true }); + const nowMs = DateTime.toEpochMillis(now); + yield* TestClock.setTime(nowMs); + // One more session than the 100-transcript scan budget, all run in the linked worktree. + for (let index = 0; index < 101; index++) { + const file = path.join(sessions, `rollout-${String(index).padStart(3, "0")}.jsonl`); + const records = [ + { type: "session_meta", payload: { id: `session-${index}`, cwd: worktree } }, + { type: "event_msg", payload: { type: "user_message", message: `Request ${index}` } }, + { + type: "response_item", + payload: { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "Done" }], + }, + }, + ]; + yield* fs.writeFileString( + file, + records.map((record) => JSON.stringify(record) + "\n").join(""), + ); + const modifiedAtSeconds = nowMs / 1000 - (index + 1); + yield* fs.utimes(file, modifiedAtSeconds, modifiedAtSeconds); + } + + const scannerContext = yield* Layer.build( + AgentSessionScanner.layer.pipe( + Layer.provide( + Layer.mergeAll( + ServerConfig.layerTest(claudeHome, { prefix: "t3-import-budget-config-" }), + ServerSettings.layerTest({ + providerInstances: { + [ProviderInstanceId.make("claudeAgent")]: { + driver: ProviderDriverKind.make("claudeAgent"), + config: { homePath: claudeHome }, + }, + [ProviderInstanceId.make("codex")]: { + driver: ProviderDriverKind.make("codex"), + config: { homePath: codexHome }, + }, + // Keeps the default Pi instance off the developer's real ~/.pi. + [ProviderInstanceId.make("pi")]: { + driver: ProviderDriverKind.make("pi"), + enabled: false, + config: {}, + }, + }, + }), + Layer.mock(ProjectStore.ProjectStoreV2)({ listShells: () => Effect.succeed([]) }), + Layer.succeed( + ProviderSessionRuntime.ProviderSessionRuntimeRepository, + yield* ProviderSessionRuntime.ProviderSessionRuntimeRepository, + ), + ), + ), + ), + ); + const importer = yield* makeImporter([], Effect.void, undefined, undefined, undefined, { + workspaceRoot: repo, + scanner: Context.get(scannerContext, AgentSessionScanner.AgentSessionScanner), + }); + + expect(yield* importer.run).toEqual({ importedCount: 100, skippedCount: 1 }); + expect(yield* importer.run).toEqual({ importedCount: 1, skippedCount: 0 }); + const oldest = yield* ProjectionStore.ProjectionStoreV2.pipe( + Effect.flatMap((projections) => + projections.getThread(ThreadId.make("import:codex:session-100")), + ), + ); + expect(oldest.worktreePath).toBe(worktree); + }).pipe(Effect.scoped, Effect.provide(Layer.merge(persistence, NodeServices.layer))), +); diff --git a/apps/server/src/project/AgentSessionImporter.ts b/apps/server/src/project/AgentSessionImporter.ts index 9c80ebf1049b..0f8736986a2d 100644 --- a/apps/server/src/project/AgentSessionImporter.ts +++ b/apps/server/src/project/AgentSessionImporter.ts @@ -35,6 +35,7 @@ import * as Stream from "effect/Stream"; import * as EventSink from "../orchestration-v2/EventSink.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; +import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; import * as ProviderAdapterRegistry from "../orchestration-v2/ProviderAdapterRegistry.ts"; import * as ThreadCommandExecutor from "../orchestration-v2/ThreadCommandExecutor.ts"; import * as ProviderSessionRuntime from "../persistence/ProviderSessionRuntime.ts"; @@ -42,6 +43,10 @@ import * as ServerSettings from "../serverSettings.ts"; import * as AgentSessionScanner from "./AgentSessionScanner.ts"; import * as ProjectService from "./ProjectService.ts"; +const isProjectionStoreThreadNotFoundError = Schema.is( + ProjectionStore.ProjectionStoreThreadNotFoundError, +); + const IMPORT_EVENT_PREFIX = "agent-session-import:v2"; const CLAUDE_SESSION_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; @@ -242,18 +247,21 @@ const make = Effect.gen(function* () { (cause) => new AgentSessionScanError({ operation: "read-projects", cause }), ), ); - const completedSources = runtimeRows.flatMap((runtime) => { - const payload = AgentSessionScanner.decodeImportedTranscriptPayload(runtime.runtimePayload); - if ( - Option.isNone(payload) || - payload.value.cwd === undefined || - normalizeProjectPathForComparison(payload.value.cwd) !== - normalizeProjectPathForComparison(project.workspaceRoot) - ) { - return []; - } - return payload.value.importedTranscripts ?? []; - }); + // The thread's project owns its transcripts: a linked worktree import records the worktree as cwd. + const completedSources = (yield* Effect.forEach(runtimeRows, (runtime) => { + const imported = AgentSessionScanner.decodeImportedTranscriptPayload( + runtime.runtimePayload, + ).pipe(Option.flatMapNullishOr((payload) => payload.importedTranscripts)); + if (Option.isNone(imported) || imported.value.length === 0) return Effect.succeed([]); + return orchestrator.getThreadRecords(runtime.threadId, []).pipe( + Effect.map(({ thread }) => (thread.projectId === input.projectId ? imported.value : [])), + Effect.catch((cause) => + isProjectionStoreThreadNotFoundError(cause.cause) + ? Effect.succeed([]) + : Effect.fail(new AgentSessionScanError({ operation: "read-projects", cause })), + ), + ); + })).flat(); const outcomes = scanner.recentThreads(project.workspaceRoot, completedSources); const importedThreadIds = new Set(); let importedCount = 0; @@ -271,7 +279,6 @@ const make = Effect.gen(function* () { ); if (outcome._tag === "AlreadyImported") { importedThreadIds.add(threadId); - importedCount += 1; return; } if (outcome._tag === "Duplicate") { @@ -309,11 +316,13 @@ const make = Effect.gen(function* () { thread.source === "claudeAgent" ? { threadId, resume: thread.providerSessionId } : { threadId: nativeThreadId }, - runtimePayload: { cwd: project.workspaceRoot }, + runtimePayload: { cwd: outcome.worktree?.path ?? project.workspaceRoot }, }, { onConflict: "ignore" }, ); - const existing = yield* Effect.option(orchestrator.getThreadRecords(threadId, [])); + const existing = yield* Effect.option( + orchestrator.getThreadRecords(threadId, ["messages"]), + ); if (Option.isSome(existing)) { if (existing.value.thread.projectId !== input.projectId) { return yield* new AgentSessionThreadProjectConflictError({ @@ -325,23 +334,23 @@ const make = Effect.gen(function* () { if (existing.value.thread.historyOrigin !== "v1_import") { return yield* new AgentSessionThreadModifiedError({ threadId }); } - // Other recorded transcripts mean this one changed after the import, and the - // thread does not hold its new messages, so it must stay unrecorded. - const recorded = Option.flatMap(yield* runtimes.getByThreadId({ threadId }), (row) => - AgentSessionScanner.decodeImportedTranscriptPayload(row.runtimePayload), - ).pipe(Option.flatMapNullishOr((payload) => payload.importedTranscripts)); + // A transcript can change after the import, so record it only when the + // thread already holds exactly its messages. + const stored = existing.value.messages; if ( - Option.isSome(recorded) && - recorded.value.length > 0 && - !recorded.value.some((imported) => - AgentSessionScanner.sameTranscriptIdentity(imported, source), + stored.length !== thread.messages.length || + stored.some( + (message, index) => + message.role !== thread.messages[index]?.role || + message.text !== thread.messages[index]?.text, ) ) { return false; } yield* persistRuntime; yield* runtimes.recordImportedTranscript({ threadId, source }); - return true; + importedThreadIds.add(threadId); + return null; } const model = thread.model ?? DEFAULT_MODEL_BY_PROVIDER[driver] ?? DEFAULT_MODEL; @@ -362,8 +371,8 @@ const make = Effect.gen(function* () { modelSelection: { instanceId: thread.providerInstanceId, model }, runtimeMode, interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, - branch: null, - worktreePath: null, + branch: outcome.worktree?.branch ?? null, + worktreePath: outcome.worktree?.path ?? null, linkedPullRequest: null, branchPullRequest: null, activeProviderThreadId: providerThreadId, diff --git a/apps/server/src/project/AgentSessionScanner.test.ts b/apps/server/src/project/AgentSessionScanner.test.ts index df8ef8aaf427..ce75e456158a 100644 --- a/apps/server/src/project/AgentSessionScanner.test.ts +++ b/apps/server/src/project/AgentSessionScanner.test.ts @@ -5,6 +5,7 @@ import * as NodeChildProcess from "node:child_process"; import * as NodeFS from "node:fs"; import * as NodeOS from "node:os"; import * as NodeSqlite from "node:sqlite"; +import * as NodeZlib from "node:zlib"; import { describe, expect, it } from "@effect/vitest"; import { AgentSessionImportSource, @@ -174,6 +175,21 @@ const writeTranscript = Effect.fn("AgentSessionScanner.test.writeTranscript")(fu yield* fileSystem.utimes(input.filePath, seconds, seconds); }); +const writeCompressedTranscript = Effect.fn("AgentSessionScanner.test.writeCompressedTranscript")( + function* (input: { + readonly filePath: string; + readonly contents: string; + readonly mtimeMs: number; + }) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* fileSystem.makeDirectory(path.dirname(input.filePath), { recursive: true }); + yield* fileSystem.writeFile(input.filePath, NodeZlib.zstdCompressSync(input.contents)); + const seconds = input.mtimeMs / 1000; + yield* fileSystem.utimes(input.filePath, seconds, seconds); + }, +); + const claudeSessionLine = (cwd: string) => `${JSON.stringify({ type: "user", cwd, sessionId: "s1" })}\n${JSON.stringify({ type: "assistant" })}\n`; @@ -326,6 +342,30 @@ const awaitGateReader = (gatePath: string) => (handle) => Effect.promise(() => handle.close()), ); +/** + * Offers each of `filePaths` to `finished` once its `cat` child has exited. A + * deadline fires for every read still in flight, so a test must wait for the + * unblocked reads before it moves the clock. + */ +const layerReportingProcessRunner = ( + filePaths: ReadonlySet, + finished: Queue.Queue, +) => + Layer.effect( + ProcessRunner.ProcessRunner, + Effect.gen(function* () { + const runner = yield* ProcessRunner.ProcessRunner; + return ProcessRunner.ProcessRunner.of({ + run: (input) => { + const filePath = input.args.at(-1) ?? ""; + return filePaths.has(filePath) + ? runner.run(input).pipe(Effect.ensuring(Queue.offer(finished, filePath))) + : runner.run(input); + }, + }); + }), + ).pipe(Layer.provide(ProcessRunner.layer), Layer.provide(NodeServices.layer)); + it.layer(NodeServices.layer)("AgentSessionScanner", (it) => { describe("scan", () => { it.effect("reads Claude project cwds from transcripts, newest first", () => @@ -869,7 +909,7 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => { }), ); - it.effect("skips linked git worktrees and reports the origin of real checkouts", () => + it.effect("folds linked git worktrees into their checkout and reports real origins", () => Effect.gen(function* () { const path = yield* Path.Path; const fileSystem = yield* FileSystem.FileSystem; @@ -905,8 +945,56 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => { path.join(submodule, ".git"), `gitdir: ${submoduleGitDir}\n`, ); + // A repository whose sessions all ran in a linked worktree is still offered. + const worktreeOnly = yield* makeTempDir("t3code-workspace-worktree-only-"); + const worktreeOnlyCheckout = yield* makeTempDir("t3code-workspace-worktree-only-wt-"); + const worktreeOnlyGitDir = path.join(worktreeOnly, ".git", "worktrees", "wt"); + yield* fileSystem.makeDirectory(worktreeOnlyGitDir, { recursive: true }); + yield* fileSystem.writeFileString( + path.join(worktreeOnly, ".git", "config"), + '[remote "origin"]\n\turl = git@github.com:pingdotgg/other.git\n', + ); + yield* fileSystem.writeFileString(path.join(worktreeOnlyGitDir, "commondir"), "../..\n"); + yield* fileSystem.writeFileString( + path.join(worktreeOnlyCheckout, ".git"), + `gitdir: ${worktreeOnlyGitDir}\n`, + ); - for (const [index, cwd] of [repo, worktree, plain, noRemote, submodule].entries()) { + // Not offered: the config is unreadable, so the repository could be bare; and + // the main checkout is reached through a symlink into T3's own state. + const configBaseDir = yield* fileSystem.realPath( + yield* makeTempDir("t3code-scanner-base-"), + ); + const links = yield* makeTempDir("t3code-workspace-links-"); + const unknown = yield* makeTempDir("t3code-workspace-unknown-"); + yield* fileSystem.makeDirectory(path.join(configBaseDir, "state-repo")); + yield* fileSystem.symlink(path.join(configBaseDir, "state-repo"), path.join(links, "repo")); + const hiddenCheckouts: Array = []; + for (const [main, config] of [ + [unknown, null], + [path.join(links, "repo"), "[core]\n"], + ] as const) { + const gitDir = path.join(main, ".git", "worktrees", "wt"); + yield* fileSystem.makeDirectory(gitDir, { recursive: true }); + if (config !== null) { + yield* fileSystem.writeFileString(path.join(main, ".git", "config"), config); + } + yield* fileSystem.writeFileString(path.join(gitDir, "commondir"), "../..\n"); + const checkout = yield* makeTempDir("t3code-workspace-hidden-wt-"); + yield* fileSystem.writeFileString(path.join(checkout, ".git"), `gitdir: ${gitDir}\n`); + hiddenCheckouts.push(checkout); + } + + const sessionDirectories = [ + repo, + worktree, + plain, + noRemote, + submodule, + worktreeOnlyCheckout, + ...hiddenCheckouts, + ]; + for (const [index, cwd] of sessionDirectories.entries()) { yield* writeTranscript({ filePath: path.join(claudeHomePath, "projects", `-slug-${index}`, "a.jsonl"), contents: claudeSessionLine(cwd), @@ -914,23 +1002,34 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => { }); } - const result = yield* runScan({ claudeHomePath, codexHomePath }); + const result = yield* runScan({ claudeHomePath, codexHomePath, configBaseDir }); expect( - result.candidates.map((candidate) => ({ path: candidate.path, git: candidate.git })), + result.candidates.map((candidate) => ({ + path: candidate.path, + git: candidate.git, + threadCount: candidate.threadCount, + })), ).toEqual([ + { + path: worktreeOnly, + git: { remoteKey: "github.com/pingdotgg/other", repository: "pingdotgg/other" }, + threadCount: 1, + }, { path: submodule, git: { remoteKey: "gitlab.com/pingdotgg/libs/vendor", repository: "pingdotgg/libs/vendor", }, + threadCount: 1, }, - { path: noRemote, git: { remoteKey: null, repository: null } }, - { path: plain, git: null }, + { path: noRemote, git: { remoteKey: null, repository: null }, threadCount: 1 }, + { path: plain, git: null, threadCount: 1 }, { path: repo, git: { remoteKey: "github.com/pingdotgg/t3code", repository: "pingdotgg/t3code" }, + threadCount: 2, }, ]); }), @@ -1005,6 +1104,70 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => { }), ); + it.effect("imports a checkout's sessions from the linked worktrees the scan counts", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const fileSystem = yield* FileSystem.FileSystem; + const nowMs = Date.parse("2026-08-24T12:00:00.000Z"); + yield* TestClock.setTime(nowMs); + const claudeHomePath = yield* makeTempDir("t3code-claude-home-"); + const codexHomePath = yield* makeTempDir("t3code-codex-home-"); + const repo = yield* makeTempDir("t3code-workspace-repo-"); + yield* fileSystem.makeDirectory(path.join(repo, ".git")); + yield* fileSystem.writeFileString(path.join(repo, ".git", "config"), "[core]\n"); + const worktreeGitDir = path.join(repo, ".git", "worktrees", "feature"); + yield* fileSystem.makeDirectory(worktreeGitDir, { recursive: true }); + yield* fileSystem.writeFileString(path.join(worktreeGitDir, "commondir"), "../..\n"); + yield* fileSystem.writeFileString( + path.join(worktreeGitDir, "HEAD"), + "ref: refs/heads/feature\n", + ); + const worktree = path.join(repo, ".claude", "worktrees", "feature"); + yield* fileSystem.makeDirectory(worktree, { recursive: true }); + yield* fileSystem.writeFileString( + path.join(worktree, ".git"), + `gitdir: ${worktreeGitDir}\n`, + ); + + for (const [index, [cwd, sessionId]] of [ + [repo, "at-root"], + [worktree, "in-worktree"], + ].entries()) { + yield* writeTranscript({ + filePath: path.join(claudeHomePath, "projects", `-slug-${index}`, `${sessionId}.jsonl`), + contents: `${JSON.stringify({ + type: "user", + cwd, + sessionId, + message: { role: "user", content: "Fix the project" }, + })}\n${JSON.stringify({ + type: "assistant", + sessionId, + message: { role: "assistant", content: [{ type: "text", text: "Done" }] }, + })}\n`, + mtimeMs: nowMs - (index + 1) * 60 * 60 * 1000, + }); + } + + const input = { claudeHomePath, codexHomePath }; + const { candidates } = yield* runScan(input); + expect(candidates.map(({ path, threadCount }) => ({ path, threadCount }))).toEqual([ + { path: repo, threadCount: 2 }, + ]); + const outcomes = yield* runRecentThreadOutcomes({ ...input, workspaceRoot: repo }); + expect( + outcomes.flatMap((outcome) => + outcome._tag === "Importable" + ? [{ sessionId: outcome.thread.providerSessionId, worktree: outcome.worktree }] + : [], + ), + ).toEqual([ + { sessionId: "at-root", worktree: null }, + { sessionId: "in-worktree", worktree: { path: worktree, branch: "feature" } }, + ]); + }), + ); + it.effect("excludes sandboxes under the configured worktrees dir without .t3 in the path", () => Effect.gen(function* () { const path = yield* Path.Path; @@ -1161,6 +1324,32 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => { const result = yield* runScan({ claudeHomePath, codexHomePath }); + expect(result.candidates.map(({ path, threadCount }) => ({ path, threadCount }))).toEqual([ + { path: workspace, threadCount: 1 }, + ]); + }), + ); + it.effect("discovers projects whose Codex rollouts are zstd-compressed", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const claudeHomePath = yield* makeTempDir("t3code-claude-home-"); + const codexHomePath = yield* makeTempDir("t3code-codex-home-"); + const workspace = yield* makeTempDir("t3code-workspace-"); + yield* writeCompressedTranscript({ + filePath: path.join( + codexHomePath, + "sessions", + "2026", + "05", + "03", + "rollout-2026-05-03T00-00-00-019a5f00-0000-7000-8000-00000000000b.jsonl.zst", + ), + contents: codexRolloutLine(workspace), + mtimeMs: Date.parse("2026-05-03T00:00:00.000Z"), + }); + + const result = yield* runScan({ claudeHomePath, codexHomePath }); + expect(result.candidates.map(({ path, threadCount }) => ({ path, threadCount }))).toEqual([ { path: workspace, threadCount: 1 }, ]); @@ -1169,6 +1358,66 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => { }); describe("recentThreads", () => { + it.effect("imports Codex conversations stored only as zstd-compressed rollouts", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const nowMs = Date.parse("2026-08-24T12:00:00.000Z"); + yield* TestClock.setTime(nowMs); + const claudeHomePath = yield* makeTempDir("t3code-claude-home-"); + const codexHomePath = yield* makeTempDir("t3code-codex-home-"); + const workspace = yield* makeTempDir("t3code-workspace-"); + yield* writeCompressedTranscript({ + filePath: path.join( + codexHomePath, + "sessions", + "2026", + "08", + "20", + "rollout-2026-08-20T10-00-00-019a5f00-0000-7000-8000-00000000000c.jsonl.zst", + ), + contents: [ + encodeTranscriptRecord({ + type: "session_meta", + payload: { id: "019a5f00-0000-7000-8000-00000000000c", cwd: workspace }, + }), + encodeTranscriptRecord({ + type: "event_msg", + timestamp: "2026-08-20T10:00:00.000Z", + payload: { type: "user_message", message: "Compressed prompt" }, + }), + encodeTranscriptRecord({ + type: "response_item", + timestamp: "2026-08-20T10:01:00.000Z", + payload: { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "Compressed answer" }], + }, + }), + ].join("\n"), + mtimeMs: nowMs - 4 * 24 * 60 * 60 * 1000, + }); + + const threads = yield* runRecentThreads({ + claudeHomePath, + codexHomePath, + workspaceRoot: workspace, + }); + + expect( + threads.map((thread) => ({ + id: thread.providerSessionId, + messages: thread.messages.map((message) => message.text), + })), + ).toEqual([ + { + id: "019a5f00-0000-7000-8000-00000000000c", + messages: ["Compressed prompt", "Compressed answer"], + }, + ]); + }), + ); + it.effect("imports recent Claude, Codex, and Pi sessions for the selected project only", () => Effect.gen(function* () { const path = yield* Path.Path; @@ -1775,6 +2024,144 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => { }), ); + it.effect("ignores sessions without visible messages without spending the import budget", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const nowMs = Date.parse("2026-08-24T12:00:00.000Z"); + yield* TestClock.setTime(nowMs); + const claudeHomePath = yield* makeTempDir("t3code-claude-home-"); + const codexHomePath = yield* makeTempDir("t3code-codex-home-"); + const workspace = yield* makeTempDir("t3code-workspace-"); + const codexRollout = (name: string) => + path.join(codexHomePath, "sessions", "2026", "08", "24", `rollout-${name}.jsonl`); + // Newer empty sessions would exhaust the 100-transcript import budget before any real one. + const emptySessions = Array.from({ length: 100 }, (_, index) => + index % 2 === 0 + ? { + filePath: codexRollout(`empty-${index}`), + records: [ + { + type: "session_meta", + payload: { id: `codex-empty-${index}`, cwd: workspace, source: "vscode" }, + }, + { + type: "event_msg", + payload: { + type: "thread_settings_applied", + thread_settings: { cwd: workspace }, + }, + }, + ], + } + : { + filePath: path.join( + claudeHomePath, + "projects", + "-slug", + `claude-empty-${index}.jsonl`, + ), + records: [ + { type: "system", subtype: "informational", content: "Status", cwd: workspace }, + { + type: "user", + isMeta: true, + message: { + role: "user", + content: "Caveat", + }, + cwd: workspace, + }, + ], + }, + ); + const transcripts = [ + ...emptySessions, + { + // Without session_meta there is no resumable session ID. + filePath: codexRollout("unusable"), + records: [ + { type: "turn_context", payload: { cwd: workspace } }, + { + type: "event_msg", + payload: { type: "user_message", message: "Import this prompt" }, + }, + ], + }, + { + filePath: codexRollout("real"), + records: [ + { type: "session_meta", payload: { id: "real-session", cwd: workspace } }, + { + type: "event_msg", + payload: { type: "user_message", message: "Import this prompt" }, + }, + ], + }, + ]; + for (const [index, { filePath, records }] of transcripts.entries()) { + yield* writeTranscript({ + filePath, + contents: records.map((record) => encodeTranscriptRecord(record)).join("\n"), + mtimeMs: nowMs - index * 60_000, + }); + } + + const outcomes = yield* runRecentThreadOutcomes({ + claudeHomePath, + codexHomePath, + workspaceRoot: workspace, + }); + + expect( + outcomes.map((outcome) => + outcome._tag === "Importable" ? outcome.thread.providerSessionId : outcome._tag, + ), + ).toEqual(["Skipped", "real-session"]); + }), + ); + + it.effect("skips a rewound Codex rollout whose inherited history is unavailable", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const nowMs = Date.parse("2026-08-24T12:00:00.000Z"); + yield* TestClock.setTime(nowMs); + const claudeHomePath = yield* makeTempDir("t3code-claude-home-"); + const codexHomePath = yield* makeTempDir("t3code-codex-home-"); + const workspace = yield* makeTempDir("t3code-workspace-"); + const threadId = "019a0000-0000-7000-8000-000000000001"; + const rolloutId = "019a0000-0000-7000-8000-000000000002"; + // The ancestor rollout is missing, and this rollout added no messages of its own. + yield* writeTranscript({ + filePath: path.join( + codexHomePath, + "sessions", + "2026", + "08", + "24", + `rollout-2026-08-24T11-00-00-${threadId}_${rolloutId}.jsonl`, + ), + contents: encodeTranscriptRecord({ + type: "session_meta", + payload: { + id: threadId, + cwd: workspace, + history_mode: "paginated", + history_base: { thread_id: threadId, end_ordinal_exclusive: 3, end_byte_offset: 512 }, + }, + }), + mtimeMs: nowMs, + }); + + const outcomes = yield* runRecentThreadOutcomes({ + claudeHomePath, + codexHomePath, + workspaceRoot: workspace, + }); + + expect(outcomes).toEqual([{ _tag: "Skipped" }]); + }), + ); + it.effect("skips excessive records without blocking an older valid transcript", () => Effect.gen(function* () { const path = yield* Path.Path; @@ -2498,28 +2885,11 @@ describe.skipIf(HostProcess.Platform.defaultValue() === "win32")( }); } - // The deadline fires for every read still in flight, so on a slow runner the - // neighbours' `cat` children must have exited before the clock moves. - const neighbourConfigs = new Set( - [repos[0], repos[2]].map((repo) => path.join(repo, ".git", "config")), - ); const finishedNeighbourReads = yield* Queue.unbounded(); - const processRunner = Layer.effect( - ProcessRunner.ProcessRunner, - Effect.gen(function* () { - const runner = yield* ProcessRunner.ProcessRunner; - return ProcessRunner.ProcessRunner.of({ - run: (input) => { - const filePath = input.args.at(-1) ?? ""; - return neighbourConfigs.has(filePath) - ? runner - .run(input) - .pipe(Effect.ensuring(Queue.offer(finishedNeighbourReads, filePath))) - : runner.run(input); - }, - }); - }), - ).pipe(Layer.provide(ProcessRunner.layer), Layer.provide(NodeServices.layer)); + const processRunner = layerReportingProcessRunner( + new Set([repos[0], repos[2]].map((repo) => path.join(repo, ".git", "config"))), + finishedNeighbourReads, + ); for (let attempt = 1; attempt <= 2; attempt++) { const scan = yield* Effect.forkChild( @@ -2628,6 +2998,12 @@ describe.skipIf(HostProcess.Platform.defaultValue() === "win32")( }); } + const finishedPointerReads = yield* Queue.unbounded(); + const processRunner = layerReportingProcessRunner( + new Set([path.join(worktree, ".git")]), + finishedPointerReads, + ); + // One scanner across scans: later scans skip the files that blocked. yield* Effect.gen(function* () { const scanner = yield* AgentSessionScanner.AgentSessionScanner; @@ -2636,22 +3012,24 @@ describe.skipIf(HostProcess.Platform.defaultValue() === "win32")( ); const first = yield* Effect.forkChild(scanPaths); // Pointer files are read first, then the commondir files they lead to. - for (const gatePath of [ - path.join(pointerCheckout, ".git"), - path.join(worktreeGitDir, "commondir"), - ]) { - yield* Effect.scoped( - Effect.gen(function* () { - yield* awaitGateReader(gatePath); - yield* TestClock.adjust("2 seconds"); - }), - ); - } + yield* Effect.scoped( + Effect.gen(function* () { + yield* awaitGateReader(path.join(pointerCheckout, ".git")); + yield* Queue.take(finishedPointerReads); + yield* TestClock.adjust("2 seconds"); + }), + ); + yield* Effect.scoped( + Effect.gen(function* () { + yield* awaitGateReader(path.join(worktreeGitDir, "commondir")); + yield* TestClock.adjust("2 seconds"); + }), + ); expect(yield* Fiber.join(first)).toEqual([pointerCheckout]); for (let attempt = 2; attempt <= LIBUV_POOL_SIZE + 1; attempt++) { expect(yield* scanPaths, `scan ${attempt}`).toEqual([pointerCheckout]); } - }).pipe(Effect.provide(layerScannerTest({ claudeHomePath, codexHomePath }))); + }).pipe(Effect.provide(layerScannerTest({ claudeHomePath, codexHomePath, processRunner }))); // Hangs if the scans left libuv pool threads stuck in open(). yield* fileSystem.readFileString(path.join(worktree, ".git")); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), @@ -3235,6 +3613,7 @@ describe("parseAgentSessionTranscript", () => { "\nGitHub\n", "# AGENTS.md instructions for /tmp/project\n\n\nRules\n", '\nTab\n', + '{"page_id":null}', "Fix the import titles", ].join("\n"); const thread = AgentSessionScanner.parseAgentSessionTranscript({ @@ -3255,6 +3634,27 @@ describe("parseAgentSessionTranscript", () => { expect(thread?.messages.map((message) => message.text)).toEqual([prompt]); }); + it.each([ + "Explain in this transcript", + "Keep this incomplete example", + ])("keeps Codex page markup that is not a complete leading block: %s", (prompt) => { + const thread = AgentSessionScanner.parseAgentSessionTranscript({ + contents: [ + encodeTranscriptRecord({ type: "session_meta", payload: { id: "codex-session" } }), + encodeTranscriptRecord({ + type: "event_msg", + payload: { type: "user_message", message: prompt }, + }), + ].join("\n"), + source: "codex", + providerInstanceId: ProviderInstanceId.make("codex"), + fallbackSessionId: "fallback", + lastActiveAtMs: Date.parse("2026-08-25T08:00:00.000Z"), + }); + + expect(thread?.title).toBe(prompt); + }); + it("skips sessions without a visible user message", () => { const thread = AgentSessionScanner.parseAgentSessionTranscript({ contents: JSON.stringify({ diff --git a/apps/server/src/project/AgentSessionScanner.ts b/apps/server/src/project/AgentSessionScanner.ts index ee3f2d2b3ab3..7a89f9c687de 100644 --- a/apps/server/src/project/AgentSessionScanner.ts +++ b/apps/server/src/project/AgentSessionScanner.ts @@ -1,5 +1,6 @@ import * as NodeOS from "node:os"; import * as NodeSqlite from "node:sqlite"; +import * as NodeZlib from "node:zlib"; import { AgentSessionImportSource, @@ -17,11 +18,13 @@ import { import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; +import * as NodeStream from "@effect/platform-node/NodeStream"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import * as Pull from "effect/Pull"; import * as Schema from "effect/Schema"; import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; @@ -177,7 +180,11 @@ const decodeCodexHistoryBase = Schema.decodeUnknownOption( ); // Original rollouts end in their thread ID, rewound ones in `_`. const CODEX_ROLLOUT_ID_PATTERN = - /-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:_([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}))?\.jsonl$/i; + /-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:_([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}))?\.jsonl(?:\.zst)?$/i; +// Codex zstd-compresses older rollouts; their byte limits and offsets count decompressed bytes. +const COMPRESSED_TRANSCRIPT_SUFFIX = ".jsonl.zst"; +const isCompressedTranscript = (filePath: string) => + filePath.endsWith(COMPRESSED_TRANSCRIPT_SUFFIX); // Codex records a chat's project only here; moving a chat leaves its rollout cwd unchanged. const CODEX_STATE_DATABASE = "state_5.sqlite"; const decodeCodexSessionIndexEntry = Schema.decodeUnknownOption( @@ -216,6 +223,8 @@ export type AgentSessionRecentThread = readonly _tag: "Importable"; readonly thread: AgentSessionThread; readonly source: AgentSessionImportSource; + /** The root's linked worktree the session ran in, or null when it ran in the root. */ + readonly worktree: { readonly path: string; readonly branch: string | null } | null; } | { readonly _tag: "AlreadyImported"; readonly source: AgentSessionImportSource } | { readonly _tag: "Duplicate"; readonly source: AgentSessionImportSource } @@ -360,7 +369,7 @@ function foldClaudeFields(fields: ClaudeFields, record: DecodedTranscriptRecord) // Codex records setup context as user text ahead of the real request. Skip // these leading blocks when deriving a title; imported messages keep them. const CODEX_TITLE_PREAMBLE = - /^(?:\s*(?:<(recommended_plugins|environment_context|user_instructions|INSTRUCTIONS|in-app-browser-context)(?:\s[^>]*)?>[\s\S]*?<\/\1>|# AGENTS\.md instructions\b[^\n]*|## My request for Codex:))+/; + /^(?:\s*(?:<(recommended_plugins|environment_context|user_instructions|INSTRUCTIONS|in-app-browser-context|external_codex_apps_open_page)(?:\s[^>]*)?>[\s\S]*?<\/\1>|# AGENTS\.md instructions\b[^\n]*|## My request for Codex:))+/; // Codex appends an entry per rename; like Codex, the newest nonempty name wins. function parseCodexThreadNames(contents: string): ReadonlyMap { @@ -438,7 +447,8 @@ export function parseAgentSessionTranscript( input.source === "claudeAgent" ? records.reduce(foldClaudeFields, emptyClaudeFields) : emptyClaudeFields; - return parseAgentSessionRecords(input, records, claudeFields, 0); + const thread = parseAgentSessionRecords(input, records, claudeFields, 0); + return thread === "empty" ? null : thread; } function parseAgentSessionRecords( @@ -446,7 +456,7 @@ function parseAgentSessionRecords( records: ReadonlyArray, claudeFields: ClaudeFields, droppedMessageCount: number, -): AgentSessionThread | null { +): AgentSessionThread | "empty" | null { const fallbackTimestamp = DateTime.formatIso(DateTime.makeUnsafe(input.lastActiveAtMs)); // Claude filenames are session IDs. Codex rollout filenames include extra // timestamp text, so only transcript metadata can provide a resumable ID. @@ -639,6 +649,7 @@ function parseAgentSessionRecords( }); } + if (messages.length === 0 && droppedMessageCount === 0) return "empty"; const visibleMessages = messages.map( ({ codexResponseUser: _codexResponseUser, ...message }) => message, ); @@ -793,10 +804,7 @@ type TranscriptIdentity = Pick< const fileIdKey = (inode: TranscriptIdentity["inode"]) => (inode === null ? null : String(inode)); -export function sameTranscriptIdentity( - left: TranscriptIdentity, - right: TranscriptIdentity, -): boolean { +function sameTranscriptIdentity(left: TranscriptIdentity, right: TranscriptIdentity): boolean { // Unsafe legacy numbers may already have lost file ID precision. if ( (typeof left.inode === "number" && !Number.isSafeInteger(left.inode)) || @@ -861,6 +869,33 @@ export const make = Effect.gen(function* () { const statOption = (target: string) => fileSystem.stat(target).pipe(Effect.asSome, Effect.orElseSucceed(Option.none)); + const transcriptChunkReader = Effect.fn("AgentSessionScanner.transcriptChunkReader")(function* ( + file: FileSystem.File, + filePath: string, + ) { + if (!isCompressedTranscript(filePath)) return (size: number) => file.readAlloc(size); + const pull = yield* Stream.toPull( + fileSystem + .stream(filePath) + .pipe(NodeStream.pipeThroughSimple(() => NodeZlib.createZstdDecompress())), + ); + let pending: Array = []; + let done = false; + // Untraced: called once per chunk inside the caller's traced read. + return Effect.fnUntraced(function* (size: number) { + while (pending.length === 0) { + if (done) return Option.none(); + const next = yield* pull.pipe(Pull.catchDone(() => Effect.undefined)); + if (next === undefined) done = true; + else pending = [...next]; + } + const chunk = pending.shift(); + if (chunk === undefined) return Option.none(); + if (chunk.byteLength > size) pending.unshift(chunk.subarray(size)); + return Option.some(chunk.subarray(0, size)); + }); + }); + const directoryIdentity = Effect.fn("AgentSessionScanner.directoryIdentity")(function* ( target: string, knownStats?: FileSystem.File.Info, @@ -876,6 +911,20 @@ export const make = Effect.gen(function* () { return `path:${normalizeProjectPathForComparison(realPath)}`; }); + /** Stats a directory that may be offered as a project, or none when it is missing or excluded. */ + const projectDirectoryStats = Effect.fn("AgentSessionScanner.projectDirectoryStats")(function* ( + directory: string, + ) { + if (isExcludedProjectPath(directory)) return Option.none(); + const stats = yield* statOption(directory); + if (Option.isNone(stats) || stats.value.type !== "Directory") return Option.none(); + const realPath = yield* fileSystem + .realPath(directory) + .pipe(Effect.orElseSucceed(() => directory)); + // Symlinks can disguise excluded worktree paths; re-check their targets. + return isExcludedProjectPath(realPath) ? Option.none() : stats; + }); + // Paths whose own read outlived the deadline, mapped to when they may be read // again. Without this every later scan would wait out the deadline again for // each phase that touches them. @@ -929,17 +978,26 @@ export const make = Effect.gen(function* () { ) { type GitIdentity = | { readonly _tag: "Repository"; readonly git: AgentSessionProjectGit | null } - | { readonly _tag: "Worktree" } + | { + readonly _tag: "Worktree"; + readonly gitDir: string; + readonly mainCheckout: string | null; + readonly git: AgentSessionProjectGit | null; + } | { readonly _tag: "NotGit" }; const identities = new Map(); - const repositories: Array<{ directory: string; gitDir: string; linkedWorktree: boolean }> = []; + const repositories: Array<{ + directory: string; + gitDir: string; + worktreeGitDir: string | null; + }> = []; const pointers: Array<{ directory: string; gitPath: string }> = []; for (const directory of directories) { const gitPath = path.join(directory, ".git"); const gitStats = yield* statOption(gitPath); if (Option.isNone(gitStats)) identities.set(directory, { _tag: "NotGit" }); else if (gitStats.value.type === "Directory") { - repositories.push({ directory, gitDir: gitPath, linkedWorktree: false }); + repositories.push({ directory, gitDir: gitPath, worktreeGitDir: null }); } else pointers.push({ directory, gitPath }); } @@ -954,7 +1012,7 @@ export const make = Effect.gen(function* () { const gitDir = path.resolve(directory, target); if (/[\\/]worktrees[\\/][^\\/]+[\\/]?$/.test(gitDir)) { linkedWorktrees.push({ directory, gitDir }); - } else repositories.push({ directory, gitDir, linkedWorktree: false }); + } else repositories.push({ directory, gitDir, worktreeGitDir: null }); } const commonDirTexts = yield* readGitFiles( @@ -962,38 +1020,100 @@ export const make = Effect.gen(function* () { ); for (const { directory, gitDir } of linkedWorktrees) { const commonDir = (commonDirTexts.get(path.join(gitDir, "commondir")) ?? "").trim(); - if (commonDir.length === 0) identities.set(directory, { _tag: "Worktree" }); - else + if (commonDir.length === 0) { + identities.set(directory, { _tag: "Worktree", gitDir, mainCheckout: null, git: null }); + } else repositories.push({ directory, gitDir: path.resolve(gitDir, commonDir), - linkedWorktree: true, + worktreeGitDir: gitDir, }); } const configTexts = yield* readGitFiles( repositories.map(({ gitDir }) => path.join(gitDir, "config")), ); - for (const { directory, gitDir, linkedWorktree } of repositories) { + for (const { directory, gitDir, worktreeGitDir } of repositories) { const configText = configTexts.get(path.join(gitDir, "config")) ?? ""; + const originUrl = parseOriginUrlFromGitConfig(configText); + const git: AgentSessionProjectGit = { + remoteKey: + originUrl === null ? null : sourceControlDefinitions.repositoryKeyFromRemote(originUrl), + repository: repositoryPathFromRemoteUrl(originUrl), + }; // Bare repositories have no main checkout, so offer their linked worktrees independently. - if (linkedWorktree && !isBareGitConfig(configText)) { - identities.set(directory, { _tag: "Worktree" }); + if (worktreeGitDir !== null && !isBareGitConfig(configText)) { + identities.set(directory, { + _tag: "Worktree", + gitDir: worktreeGitDir, + // Only a common dir named .git sits inside its main checkout, and an + // unreadable config can't rule out a bare repository. + mainCheckout: + configText.length > 0 && path.basename(gitDir) === ".git" ? path.dirname(gitDir) : null, + git, + }); continue; } - const originUrl = parseOriginUrlFromGitConfig(configText); - identities.set(directory, { - _tag: "Repository", - git: { - remoteKey: - originUrl === null ? null : sourceControlDefinitions.repositoryKeyFromRemote(originUrl), - repository: repositoryPathFromRemoteUrl(originUrl), - }, - }); + identities.set(directory, { _tag: "Repository", git }); } return identities; }); + /** Maps each directory to the project it counts toward: a linked worktree counts toward its main checkout. */ + const resolveDirectoryProjects = Effect.fn("AgentSessionScanner.resolveDirectoryProjects")( + function* (resolvedDirectories: ReadonlyArray) { + // Stat every directory first so one batch reads all git identities. + const directories = new Map>(); + for (const resolved of resolvedDirectories) { + if (directories.has(resolved)) continue; + directories.set(resolved, yield* projectDirectoryStats(resolved)); + } + const directoryGitIdentities = yield* readGitIdentities( + Array.from(directories).flatMap(([directory, stats]) => + Option.isSome(stats) ? [directory] : [], + ), + ); + const projects = new Map< + string, + { + readonly key: string; + readonly path: string; + readonly git: AgentSessionProjectGit | null; + readonly worktreeGitDir: string | null; + } | null + >(); + for (const [resolved, stats] of directories) { + const gitIdentity = directoryGitIdentities.get(resolved); + if (Option.isNone(stats) || gitIdentity === undefined) { + projects.set(resolved, null); + } else if (gitIdentity._tag === "Worktree") { + const { mainCheckout } = gitIdentity; + const mainStats = + mainCheckout === null ? Option.none() : yield* projectDirectoryStats(mainCheckout); + projects.set( + resolved, + mainCheckout !== null && Option.isSome(mainStats) + ? { + key: yield* directoryIdentity(mainCheckout, mainStats.value), + path: mainCheckout, + git: gitIdentity.git, + worktreeGitDir: gitIdentity.gitDir, + } + : null, + ); + } else { + projects.set(resolved, { + key: yield* directoryIdentity(resolved, stats.value), + path: resolved, + git: gitIdentity._tag === "Repository" ? gitIdentity.git : null, + worktreeGitDir: null, + }); + } + } + return projects; + }, + ); + const readCwd = Effect.fn("AgentSessionScanner.readCwd")(function* ( source: AgentSessionSource, transcript: TranscriptCandidate, @@ -1013,11 +1133,15 @@ export const make = Effect.gen(function* () { fileSystem.open(transcript.filePath, { flag: "r" }).pipe( Effect.flatMap((file) => Effect.gen(function* () { + const readChunk = yield* transcriptChunkReader(file, transcript.filePath); + const size = isCompressedTranscript(transcript.filePath) + ? Number.POSITIVE_INFINITY + : transcript.identity.size; const decoder = new TextDecoder(); let remaining = ""; let bytesRead = 0; let recordsRead = 0; - const maxBytes = Math.min(MAX_TRANSCRIPT_SCAN_BYTES, transcript.identity.size); + const maxBytes = Math.min(MAX_TRANSCRIPT_SCAN_BYTES, size); const reserveRecord = () => { if ( recordsRead === MAX_METADATA_RECORDS_PER_TRANSCRIPT || @@ -1049,7 +1173,7 @@ export const make = Effect.gen(function* () { ); budget.operationsRemaining -= 1; budget.bytesRemaining -= readSize; - const next = yield* file.readAlloc(readSize); + const next = yield* readChunk(readSize); if (Option.isNone(next)) { return readLastRecord(); } @@ -1066,7 +1190,7 @@ export const make = Effect.gen(function* () { } } - if (bytesRead < transcript.identity.size) { + if (bytesRead < size) { budget.truncated = true; return null; } @@ -1083,8 +1207,12 @@ export const make = Effect.gen(function* () { source: AgentSessionSource, prefix?: { readonly endByteOffset: number; readonly historyBytes: number }, ) { - const readBytes = prefix?.endByteOffset ?? expected.size; - if (readBytes > expected.size || readBytes > MAX_IMPORTED_TRANSCRIPT_BYTES) return null; + const compressed = isCompressedTranscript(filePath); + const readBytes = + prefix?.endByteOffset ?? (compressed ? MAX_IMPORTED_TRANSCRIPT_BYTES : expected.size); + if ((!compressed && readBytes > expected.size) || readBytes > MAX_IMPORTED_TRANSCRIPT_BYTES) { + return null; + } return yield* Effect.scoped( fileSystem.open(filePath, { flag: "r" }).pipe( @@ -1093,6 +1221,7 @@ export const make = Effect.gen(function* () { if (!sameTranscriptIdentity(expected, transcriptIdentity(filePath, yield* file.stat))) { return null; } + const readChunk = yield* transcriptChunkReader(file, filePath); const records: Array = []; const claudeHistory = createClaudeHistoryWindow(); let cwd = Option.none(); @@ -1153,10 +1282,12 @@ export const make = Effect.gen(function* () { }; while (bytesRead < readBytes) { - const next = yield* file.readAlloc( + const next = yield* readChunk( Math.min(TRANSCRIPT_PREFIX_BYTES, readBytes - bytesRead), ); if (Option.isNone(next)) { + // A compressed rollout's decompressed length is only known at its end. + if (compressed && prefix === undefined) break; return null; } @@ -1374,7 +1505,12 @@ export const make = Effect.gen(function* () { } const directory = path.join(sessionsDir, year, month, day); for (const entry of (yield* readDirectory(directory)).toSorted().toReversed()) { - if (!entry.startsWith("rollout-") || !entry.endsWith(".jsonl")) continue; + if ( + !entry.startsWith("rollout-") || + !(entry.endsWith(".jsonl") || isCompressedTranscript(entry)) + ) { + continue; + } if (operationsRemaining <= 0) { truncated = true; break; @@ -1695,60 +1831,27 @@ export const make = Effect.gen(function* () { git: AgentSessionProjectGit | null; } >(); - // Resolve every candidate directory first so one batch reads all git identities. - const directories = new Map>(); - for (const candidate of raw) { - const expanded = expandHomePath(candidate.cwd.trim(), homeDir); - if (!path.isAbsolute(expanded)) continue; - const resolved = path.resolve(expanded); - if (directories.has(resolved) || isExcludedProjectPath(resolved)) continue; - const stats = yield* statOption(resolved); - if (Option.isNone(stats) || stats.value.type !== "Directory") { - directories.set(resolved, Option.none()); - continue; - } - const realPath = yield* fileSystem - .realPath(resolved) - .pipe(Effect.orElseSucceed(() => resolved)); - // Symlinks can disguise excluded worktree paths; re-check their targets. - directories.set(resolved, isExcludedProjectPath(realPath) ? Option.none() : stats); - } - const directoryGitIdentities = yield* readGitIdentities( - Array.from(directories).flatMap(([directory, stats]) => - Option.isSome(stats) ? [directory] : [], - ), + const directoryProjects = yield* resolveDirectoryProjects( + raw.flatMap((candidate) => { + const expanded = expandHomePath(candidate.cwd.trim(), homeDir); + return path.isAbsolute(expanded) ? [path.resolve(expanded)] : []; + }), ); - const directoryKeys = new Map(); - const gitIdentities = new Map(); - for (const candidate of raw) { const expanded = expandHomePath(candidate.cwd.trim(), homeDir); if (!path.isAbsolute(expanded)) continue; - const resolved = path.resolve(expanded); - const stats = directories.get(resolved); - if (stats === undefined || Option.isNone(stats)) continue; - let key = directoryKeys.get(resolved); - if (key === undefined) { - const gitIdentity = directoryGitIdentities.get(resolved); - if (gitIdentity === undefined || gitIdentity._tag === "Worktree") { - key = ""; - } else { - key = yield* directoryIdentity(resolved, stats.value); - gitIdentities.set(key, gitIdentity._tag === "Repository" ? gitIdentity.git : null); - } - directoryKeys.set(resolved, key); - } - if (key === "") continue; + const project = directoryProjects.get(path.resolve(expanded)); + if (project === undefined || project === null) continue; - const existing = merged.get(key); + const existing = merged.get(project.key); if (!existing) { - merged.set(key, { - path: resolved, + merged.set(project.key, { + path: project.path, sources: [candidate.source], threadCount: candidate.threadCount, lastActiveAtMs: candidate.lastActiveAtMs, - git: gitIdentities.get(key) ?? null, + git: project.git, }); continue; } @@ -1829,6 +1932,45 @@ export const make = Effect.gen(function* () { } const { candidates } = cachedCandidates; + // The scan counts a linked worktree's sessions toward its main checkout, so import them with it. + const candidateDirectories = candidates.flatMap((candidate) => { + const expanded = expandHomePath(candidate.cwd.trim(), homeDir); + return path.isAbsolute(expanded) ? [path.resolve(expanded)] : []; + }); + const directoryProjects = yield* resolveDirectoryProjects(candidateDirectories); + const belongsToRoot = Effect.fn("AgentSessionScanner.belongsToRoot")(function* ( + resolved: string, + ) { + if ((yield* directoryIdentity(resolved)) === rootIdentity) return true; + if (!directoryProjects.has(resolved)) { + const project = (yield* resolveDirectoryProjects([resolved])).get(resolved) ?? null; + directoryProjects.set(resolved, project); + } + return directoryProjects.get(resolved)?.key === rootIdentity; + }); + const worktreesByDirectory = new Map(); + const rootWorktree = Effect.fn("AgentSessionScanner.rootWorktree")(function* ( + resolved: string, + ) { + const known = worktreesByDirectory.get(resolved); + if (known !== undefined) return known; + const gitDir = + (yield* directoryIdentity(resolved)) === rootIdentity + ? null + : (directoryProjects.get(resolved)?.worktreeGitDir ?? null); + let worktree: { path: string; branch: string | null } | null = null; + if (gitDir !== null) { + const headPath = path.join(gitDir, "HEAD"); + const head = (yield* readGitFiles([headPath])).get(headPath) ?? ""; + worktree = { + path: resolved, + branch: /^ref: refs\/heads\/(.+)$/.exec(head.trim())?.[1] ?? null, + }; + } + worktreesByDirectory.set(resolved, worktree); + return worktree; + }); + const eligibleTranscripts: Array<{ readonly candidate: RawCandidate; readonly transcript: RawCandidate["transcripts"][number] & { readonly mtimeMs: number }; @@ -1837,7 +1979,7 @@ export const make = Effect.gen(function* () { const expanded = expandHomePath(candidate.cwd.trim(), homeDir); if (!path.isAbsolute(expanded)) continue; const resolved = path.resolve(expanded); - if ((yield* directoryIdentity(resolved)) !== rootIdentity) continue; + if (!(yield* belongsToRoot(resolved))) continue; for (const transcript of candidate.transcripts) { if ( @@ -1968,12 +2110,10 @@ export const make = Effect.gen(function* () { projectCwd(snapshot.cwd.value, transcript.projectRoots).trim(), homeDir, ); - if ( - !path.isAbsolute(expandedCwd) || - (yield* directoryIdentity(path.resolve(expandedCwd))) !== rootIdentity - ) { + if (!path.isAbsolute(expandedCwd) || !(yield* belongsToRoot(path.resolve(expandedCwd)))) { return Option.some({ _tag: "Skipped" }); } + const worktree = yield* rootWorktree(path.resolve(expandedCwd)); const inheritedRecords = candidate.source === "codex" @@ -1987,7 +2127,12 @@ export const make = Effect.gen(function* () { { source: candidate.source, providerInstanceId: candidate.providerInstanceId, - fallbackSessionId: path.basename(transcript.filePath, ".jsonl"), + fallbackSessionId: path.basename( + transcript.filePath, + isCompressedTranscript(transcript.filePath) + ? COMPRESSED_TRANSCRIPT_SUFFIX + : ".jsonl", + ), lastActiveAtMs: transcript.mtimeMs, }, inheritedRecords === null @@ -1996,6 +2141,15 @@ export const make = Effect.gen(function* () { snapshot.claudeFields, snapshot.droppedMessageCount, ); + if (parsedThread === "empty") { + // null means an unreadable history base, which may hold the conversation. + if (candidate.source === "codex" && inheritedRecords === null) { + return Option.some({ _tag: "Skipped" }); + } + // Never recorded as imported, so charging them would starve real sessions every run. + transcriptsRemaining += 1; + return Option.none(); + } if (parsedThread === null) { return Option.some({ _tag: "Skipped" }); } @@ -2024,6 +2178,7 @@ export const make = Effect.gen(function* () { thread: savedTitle === undefined ? parsedThread : { ...parsedThread, title: savedTitle }, source, + worktree, }); }).pipe(importReadLock.withPermits(1)), ), diff --git a/apps/server/src/project/ManagedProjectFolders.test.ts b/apps/server/src/project/ManagedProjectFolders.test.ts index 011b54a6851b..07da49c73253 100644 --- a/apps/server/src/project/ManagedProjectFolders.test.ts +++ b/apps/server/src/project/ManagedProjectFolders.test.ts @@ -1,7 +1,13 @@ import { assert, it } from "@effect/vitest"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; -import { CommandId, ProjectId, ThreadId, VcsProcessSpawnError } from "@t3tools/contracts"; +import { + CommandId, + ProjectId, + ThreadId, + VcsProcessSpawnError, + VcsProcessTimeoutError, +} from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; @@ -174,6 +180,44 @@ it.effect("offers a Scratch folder when Git is not installed", () => ), ); +it.effect("offers the Scratch folder once a Git check that timed out is asked again", () => + withScratch( + ({ baseDir }) => + Effect.gen(function* () { + const path = yield* Path.Path; + const scratch = yield* ManagedProjectFolders.ManagedProjectFolders; + assert.isTrue(Option.isNone(yield* scratch.scratchRoot)); + assert.equal(yield* requireRoot, path.resolve(baseDir, "scratch")); + }), + { + vcsProcess: Layer.effect( + VcsProcess.VcsProcess, + Effect.map(VcsProcess.make, (real) => { + let timedOut = false; + return VcsProcess.VcsProcess.of({ + run: (input) => + Effect.suspend(() => { + if (timedOut) return real.run(input); + timedOut = true; + return Effect.fail( + new VcsProcessTimeoutError({ + operation: input.operation, + command: input.command, + cwd: input.cwd, + timeoutMs: 5_000, + }), + ); + }), + }); + }), + ).pipe( + Layer.provide(ProcessRunner.layer), + Layer.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), + ), + }, + ), +); + it.effect("creates one Scratch project, even for concurrent first requests", () => withScratch(() => Effect.gen(function* () { @@ -274,6 +318,15 @@ it.effect("gives each Scratch thread its own folder, named from its message", () ); assert.equal(path.dirname(pasted), root); assert.isAtMost(path.basename(pasted).length, 80); + + const accented = Option.getOrThrow( + yield* scratch.folderForThread({ + projectId, + threadId: ThreadId.make("thread-diacritics"), + text: "tìm kiếm skill", + }), + ); + assert.match(path.basename(accented), /^\d{4}-\d{2}-\d{2}-tim-kiem-skill-/); }), ), ); diff --git a/apps/server/src/project/ManagedProjectFolders.ts b/apps/server/src/project/ManagedProjectFolders.ts index 27316997077b..d526c381fb40 100644 --- a/apps/server/src/project/ManagedProjectFolders.ts +++ b/apps/server/src/project/ManagedProjectFolders.ts @@ -6,7 +6,7 @@ import { type ThreadId, VcsProcessSpawnError, } from "@t3tools/contracts"; -import { newProjectFolderName } from "@t3tools/shared/path"; +import { foldLatinDiacritics, newProjectFolderName } from "@t3tools/shared/path"; import * as Cause from "effect/Cause"; import * as Context from "effect/Context"; import * as SecureRandom from "effect/crypto/SecureRandom"; @@ -66,6 +66,10 @@ export class ScratchProjectError extends Schema.TaggedError } } +/** Where No project threads keep their folders, whether or not this T3 home allows them. */ +export const scratchFolderRoot = (baseDir: string, path: Path.Path) => + path.resolve(baseDir, "scratch"); + export type ScratchError = ScratchUnavailableError | ScratchFolderError | ScratchProjectError; // The spawner checks cwd before spawning, so NotFound from spawn itself means a missing executable. @@ -129,7 +133,7 @@ export class ManagedProjectFolders extends Context.Service< >()("t3/project/ManagedProjectFolders") {} const folderWords = (text: string) => - text + foldLatinDiacritics(text) .toLowerCase() .split(/[^a-z0-9]+/) .filter(Boolean) @@ -255,7 +259,7 @@ const make = Effect.gen(function* () { Effect.flatMap( (repositoryRoot): Effect.Effect> => { if (repositoryRoot === null) { - return Effect.succeed(Result.succeed(path.resolve(config.baseDir, "scratch"))); + return Effect.succeed(Result.succeed(scratchFolderRoot(config.baseDir, path))); } const unavailable = new ScratchUnavailableError({ baseDir: config.baseDir, @@ -264,15 +268,18 @@ const make = Effect.gen(function* () { return Effect.logWarning(unavailable.message).pipe(Effect.as(Result.fail(unavailable))); }, ), - Effect.catchCause((cause) => - Cause.hasInterrupts(cause) - ? Effect.interrupt - : Effect.succeed(Result.fail(new ScratchUnavailableError({ baseDir: config.baseDir }))), - ), ), - Duration.infinity, + // A failed git check (say, a timeout on a busy machine) is not an answer; ask again next time. + Exit.match({ onSuccess: () => Duration.infinity, onFailure: () => Duration.zero }), + ); + const scratchProbe = probe.pipe( + Effect.catchCause((cause) => + Cause.hasInterrupts(cause) + ? Effect.interrupt + : Effect.succeed(Result.fail(new ScratchUnavailableError({ baseDir: config.baseDir }))), + ), + Effect.onInterrupt(() => invalidate), ); - const scratchProbe = probe.pipe(Effect.onInterrupt(() => invalidate)); // A symlinked scratch folder lets the No project live at either the link or its target. const scratchProjectRoot = (root: string) => fileSystem.realPath(root).pipe( diff --git a/apps/server/src/project/ProjectCloneTracker.test.ts b/apps/server/src/project/ProjectCloneTracker.test.ts index 0d0a614e467a..77abcf87a81d 100644 --- a/apps/server/src/project/ProjectCloneTracker.test.ts +++ b/apps/server/src/project/ProjectCloneTracker.test.ts @@ -343,6 +343,9 @@ describe("parseGitCloneProgressLine", () => { expect( parseGitCloneProgressLine("Receiving objects: 45% (4500/10000), 12.30 MiB | 5.00 MiB/s"), ).toEqual({ stage: "receiving", percent: 45, detail: "12.30 MiB | 5.00 MiB/s" }); + expect( + parseGitCloneProgressLine("Receiving objects: 100% (5/5), 40.00 MiB | 1.40 MiB/s, done."), + ).toEqual({ stage: "receiving", percent: 100, detail: "40.00 MiB | 1.40 MiB/s" }); expect(parseGitCloneProgressLine("Resolving deltas: 100% (700/700), done.")).toEqual({ stage: "resolving", percent: 100, diff --git a/apps/server/src/project/ProjectFaviconResolver.test.ts b/apps/server/src/project/ProjectFaviconResolver.test.ts index ec548db308b7..143db2a86629 100644 --- a/apps/server/src/project/ProjectFaviconResolver.test.ts +++ b/apps/server/src/project/ProjectFaviconResolver.test.ts @@ -201,6 +201,17 @@ it.layer(layerTest)("ProjectFaviconResolverLive", (it) => { expect(resolved).not.toBeNull(); expect(resolved).toBe((yield* Path.Path).join(cwd, "public", "brand", "logo.svg")); + + const paddedCwd = yield* makeTempDir; + yield* writeTextFile( + paddedCwd, + "index.html", + '', + ); + yield* writeTextFile(paddedCwd, "brand/custom.svg", "custom"); + expect(yield* resolver.resolvePath(paddedCwd)).toBe( + (yield* Path.Path).join(paddedCwd, "brand", "custom.svg"), + ); }), ); diff --git a/apps/server/src/project/ProjectFaviconResolver.ts b/apps/server/src/project/ProjectFaviconResolver.ts index c63fac30595f..e09d4aad362a 100644 --- a/apps/server/src/project/ProjectFaviconResolver.ts +++ b/apps/server/src/project/ProjectFaviconResolver.ts @@ -144,7 +144,8 @@ export const make = Effect.gen(function* () { const projectFileLoader = yield* T3ProjectFileLoader.T3ProjectFileLoader; const resolveIconHref = (href: string): ReadonlyArray => { - const clean = href.replace(/^\//, ""); + // Browsers strip surrounding whitespace from URL attributes; file paths keep theirs. + const clean = href.trim().replace(/^\//, ""); return [path.join("public", clean), clean]; }; diff --git a/apps/server/src/project/ProjectService.deletion.test.ts b/apps/server/src/project/ProjectService.deletion.test.ts index 28b1c40541dd..25e5ee54a273 100644 --- a/apps/server/src/project/ProjectService.deletion.test.ts +++ b/apps/server/src/project/ProjectService.deletion.test.ts @@ -28,6 +28,7 @@ import * as ProjectStore from "../orchestration-v2/ProjectStore.ts"; import * as ThreadCommandExecutor from "../orchestration-v2/ThreadCommandExecutor.ts"; import { planThreadDeletion } from "../orchestration-v2/ThreadDeletion.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as ProjectSettingsOverrideSweep from "../provider/ProjectSettingsOverrideSweep.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; import * as ProjectEnrichmentService from "./ProjectEnrichmentService.ts"; @@ -501,9 +502,13 @@ it.effect("deletes a project without force once its imported threads were delete }).pipe(Effect.provide(layerDatabase)), ); -it.effect( - "clears a deleted project's settings overrides so a machine-disabled instance cannot stay alive", - () => +it.effect.each([ + { policy: "no", worktreeCleanup: undefined }, + { policy: "its off", worktreeCleanup: { mode: "off" } as const }, + { policy: "its legacy", worktreeCleanup: null }, +])( + "clears a deleted project's settings overrides, keeping $policy worktree cleanup policy, so a machine-disabled instance cannot stay alive", + ({ worktreeCleanup }) => Effect.gen(function* () { const projectId = ProjectId.make("project:overrides-cleanup"); const instanceId = ProviderInstanceId.make("codex_work"); @@ -514,7 +519,11 @@ it.effect( const serverSettings = yield* ServerSettings.ServerSettingsService; yield* serverSettings.updateSettings({ projectSettingsOverrides: { - [projectId]: { providerInstances: { [instanceId]: { enabled: true } } }, + [projectId]: { + providerInstances: { [instanceId]: { enabled: true } }, + defaultAutoPull: true, + ...(worktreeCleanup === undefined ? {} : { worktreeCleanup }), + }, }, }); @@ -523,8 +532,35 @@ it.effect( projectId, }); - const settingsAfterDelete = yield* serverSettings.getSettings; - assert.isUndefined(settingsAfterDelete.projectSettingsOverrides[projectId]); + const entry = (yield* serverSettings.getSettings).projectSettingsOverrides[projectId]; + if (worktreeCleanup === undefined) { + assert.isUndefined(entry); + } else { + assert.deepStrictEqual(entry, { worktreeCleanup, providerInstances: {} }); + } + + const liveProjectId = ProjectId.make("project:still-live"); + const missingProjectId = ProjectId.make("project:missing"); + yield* seedProject(liveProjectId); + // Older deletions or interrupted best-effort cleanup can leave the original overrides on restart. + yield* serverSettings.updateSettings({ + projectSettingsOverrides: { + [projectId]: { + providerInstances: { [instanceId]: { enabled: true } }, + defaultAutoPull: true, + ...(worktreeCleanup === undefined ? {} : { worktreeCleanup }), + }, + [liveProjectId]: { defaultAutoPull: true }, + [missingProjectId]: { worktreeCleanup: { mode: "off" } }, + }, + }); + yield* ProjectSettingsOverrideSweep.sweep; + assert.deepStrictEqual((yield* serverSettings.getSettings).projectSettingsOverrides, { + [liveProjectId]: { defaultAutoPull: true }, + ...(worktreeCleanup === undefined + ? {} + : { [projectId]: { worktreeCleanup, providerInstances: {} } }), + }); }).pipe(Effect.provide(layerServices)); }).pipe(Effect.provide(layerDatabase)), ); diff --git a/apps/server/src/project/ProjectService.test.ts b/apps/server/src/project/ProjectService.test.ts index 1517bed31ba2..492379beecc2 100644 --- a/apps/server/src/project/ProjectService.test.ts +++ b/apps/server/src/project/ProjectService.test.ts @@ -9,6 +9,7 @@ import { CommandId, type Project, ProjectId, ProviderInstanceId } from "@t3tools import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Metric from "effect/Metric"; import * as Option from "effect/Option"; @@ -30,12 +31,14 @@ import * as ProjectStore from "../orchestration-v2/ProjectStore.ts"; import * as RuntimeLayer from "../orchestration-v2/runtimeLayer.ts"; import * as ThreadCommandExecutor from "../orchestration-v2/ThreadCommandExecutor.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as ProcessRunner from "../processRunner.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; import * as ProjectEnrichmentService from "./ProjectEnrichmentService.ts"; import * as ProjectFaviconResolver from "./ProjectFaviconResolver.ts"; import * as ProjectService from "./ProjectService.ts"; import * as RepositoryIdentityResolver from "./RepositoryIdentityResolver.ts"; +import * as T3ProjectFileLoader from "./T3ProjectFileLoader.ts"; const layerWorkspacePaths = Layer.succeed(WorkspacePaths.WorkspacePaths, { normalizeWorkspaceRoot: (workspaceRoot) => Effect.succeed(workspaceRoot.replace(/\/$/, "")), @@ -300,7 +303,14 @@ it.layer(layerTest)("ProjectService", (it) => { }); assert.isFalse(reset.autoPull); assert.isNull(reset.projectIcon); - assert.equal(reset.faviconPath, hydratedCreated.faviconPath); + assert.equal( + (yield* waitForProject( + service, + projectId, + (project) => project.faviconPath === hydratedCreated.faviconPath, + )).faviconPath, + hydratedCreated.faviconPath, + ); assert.isNull(reset.defaultThreadEnvMode); assert.deepEqual( (yield* service.snapshot).projects.map((project) => project.id), @@ -543,6 +553,12 @@ it.layer(layerTest)("ProjectService", (it) => { title: "Delete retry", workspaceRoot: "/work/delete-retry", }); + // Enrichment landing between the two deletes would change the project the retry reads. + yield* waitForProject( + service, + projectId, + (project) => project.faviconPath !== null && project.repositoryIdentity !== null, + ); const input = { commandId: CommandId.make("command:delete-retry:delete"), projectId }; const deleted = yield* service.delete(input); assert.deepEqual(yield* service.delete(input), deleted); @@ -725,6 +741,112 @@ it.effect("invalidates workspace-derived metadata when a project moves", () => }), ); +it.effect("resolves a restored icon immediately after reselecting its saved path", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "project-icon-reselect-" }); + const iconPath = `${cwd}/icon.svg`; + const metadata = Layer.merge( + Layer.succeed(RepositoryIdentityResolver.RepositoryIdentityResolver, { + resolve: () => Effect.succeed(null), + }), + ProjectFaviconResolver.layer.pipe( + Layer.provide(layerWorkspacePaths), + Layer.provide(T3ProjectFileLoader.layer), + Layer.provide(NodeServices.layer), + ), + ); + yield* Effect.gen(function* () { + const service = yield* ProjectService.ProjectService; + const resolver = yield* ProjectFaviconResolver.ProjectFaviconResolver; + const projectId = ProjectId.make("project:icon-reselect"); + yield* service.create({ + commandId: CommandId.make("command:icon-reselect:create"), + projectId, + title: "Icon reselection", + workspaceRoot: cwd, + }); + yield* service.update({ + commandId: CommandId.make("command:icon-reselect:missing"), + projectId, + faviconPath: "icon.svg", + }); + assert.isNull(yield* resolver.resolvePath(cwd, "icon.svg")); + yield* fileSystem.writeFileString(iconPath, ''); + assert.isNull(yield* resolver.resolvePath(cwd, "icon.svg")); + yield* service.update({ + commandId: CommandId.make("command:icon-reselect:restored"), + projectId, + faviconPath: "icon.svg", + }); + assert.equal(yield* resolver.resolvePath(cwd, "icon.svg"), iconPath); + }).pipe(Effect.provide(layerTestFor(metadata))); + }).pipe(Effect.provide(NodeServices.layer)), +); + +it.effect("refuses a worktree container as a new project and as a new workspace root", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const processRunner = yield* ProcessRunner.ProcessRunner; + const fixture = yield* fileSystem.makeTempDirectoryScoped({ prefix: "project-container-" }); + const git = (cwd: string, ...args: ReadonlyArray) => + processRunner.run({ + command: "git", + args: ["-C", cwd, "-c", "user.email=t@t", "-c", "user.name=t", ...args], + }); + const source = `${fixture}/source`; + const container = `${fixture}/container`; + yield* fileSystem.makeDirectory(source); + yield* fileSystem.makeDirectory(container); + yield* git(source, "init", "-b", "develop"); + yield* fileSystem.writeFileString(`${source}/README.md`, "readme\n"); + yield* git(source, "add", "README.md"); + yield* git(source, "commit", "-m", "init"); + yield* git(fixture, "clone", "--bare", source, `${container}/.bare`); + yield* fileSystem.writeFileString(`${container}/.git`, "gitdir: ./.bare\n"); + yield* git(container, "worktree", "add", "develop", "develop"); + + yield* Effect.gen(function* () { + const service = yield* ProjectService.ProjectService; + // As cloned the container is bare; Initialize Git or a config edit clears core.bare. + for (const coreBare of ["true", "false"]) { + yield* git(container, "config", "core.bare", coreBare); + const rejected = yield* service + .create({ + commandId: CommandId.make(`command:container:create:${coreBare}`), + projectId: ProjectId.make(`project:container:${coreBare}`), + title: "Container", + workspaceRoot: container, + }) + .pipe(Effect.flip); + assert.equal(rejected._tag, "ProjectInvalidError", `core.bare=${coreBare}`); + assert.include(rejected.message, "Add one of its worktrees"); + assert.isTrue(Option.isNone(yield* service.getByWorkspaceRoot(container))); + } + + const projectId = ProjectId.make("project:container:worktree"); + const worktree = yield* service.create({ + commandId: CommandId.make("command:container:worktree"), + projectId, + title: "Worktree", + workspaceRoot: `${container}/develop`, + }); + const moved = yield* service + .update({ + commandId: CommandId.make("command:container:move"), + projectId, + workspaceRoot: container, + }) + .pipe(Effect.flip); + assert.equal(moved._tag, "ProjectInvalidError"); + assert.equal( + Option.getOrThrow(yield* service.getById(projectId)).workspaceRoot, + worktree.workspaceRoot, + ); + }).pipe(Effect.provide(layerTest)); + }).pipe(Effect.provide(ProcessRunner.layer.pipe(Layer.provideMerge(NodeServices.layer)))), +); + it.effect("serializes two projects claiming the same workspace root", () => Effect.gen(function* () { const eventSink = yield* EventSink.EventSinkV2; diff --git a/apps/server/src/project/ProjectService.ts b/apps/server/src/project/ProjectService.ts index 7080a9ff2bdf..1db031456f2d 100644 --- a/apps/server/src/project/ProjectService.ts +++ b/apps/server/src/project/ProjectService.ts @@ -33,8 +33,11 @@ import * as ProjectStore from "../orchestration-v2/ProjectStore.ts"; import * as ThreadCommandExecutor from "../orchestration-v2/ThreadCommandExecutor.ts"; import { planThreadDeletion } from "../orchestration-v2/ThreadDeletion.ts"; import * as ServerSettings from "../serverSettings.ts"; +import { deletedProjectOverridePatch } from "../provider/ProjectSettingsOverrideSweep.ts"; import { makeIsManagedWorktreePath } from "../worktreesDirectory.ts"; +import * as ProcessRunner from "../processRunner.ts"; import * as ProjectEnrichmentService from "./ProjectEnrichmentService.ts"; +import { makeIsWorktreeContainer } from "./worktreeContainer.ts"; import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; export interface ProjectCreateInput extends ProjectCreatePayload { @@ -182,6 +185,9 @@ export const make = Effect.gen(function* () { const serverSettings = yield* ServerSettings.ServerSettingsService; const fileSystem = yield* FileSystem.FileSystem; const isManagedWorktreePath = yield* makeIsManagedWorktreePath; + const isWorktreeContainer = yield* makeIsWorktreeContainer.pipe( + Effect.provide(ProcessRunner.layer), + ); const projectLocks = yield* KeyedLock.make(); const workspaceLocks = yield* KeyedLock.make(); @@ -244,6 +250,17 @@ export const make = Effect.gen(function* () { ), ); + const rejectWorktreeContainer = Effect.fn("ProjectService.rejectWorktreeContainer")(function* ( + projectId: ProjectId, + workspaceRoot: string, + ) { + if (!(yield* isWorktreeContainer(workspaceRoot))) return; + return yield* new ProjectInvalidError({ + projectId, + detail: `${workspaceRoot} holds a repository's worktrees but is not a checkout itself. Add one of its worktrees as the project instead.`, + }); + }); + const commit = Effect.fn("ProjectService.commit")(function* (command: ProjectCommand) { const { projectId } = command; const dispatchError = (cause: unknown) => @@ -383,6 +400,7 @@ export const make = Effect.gen(function* () { workspaceRoot: input.workspaceRoot, createIfMissing: input.createWorkspaceRootIfMissing ?? false, }); + yield* rejectWorktreeContainer(input.projectId, workspaceRoot); yield* commit({ type: "project.create", commandId: input.commandId, @@ -410,6 +428,9 @@ export const make = Effect.gen(function* () { projectId: input.projectId, workspaceRoot: input.workspaceRoot, }); + if (workspaceRoot !== previousRoot) { + yield* rejectWorktreeContainer(input.projectId, workspaceRoot); + } yield* commit({ type: "project.meta.update", commandId: input.commandId, @@ -429,7 +450,7 @@ export const make = Effect.gen(function* () { ...(input.pinned === undefined ? {} : { pinned: input.pinned }), ...(input.archived === undefined ? {} : { archived: input.archived }), }); - if (workspaceRoot !== previousRoot) { + if (workspaceRoot !== previousRoot || input.faviconPath !== undefined) { yield* projectEnrichment.invalidate([previousRoot, workspaceRoot]); } return yield* readCommitted(input.projectId); @@ -517,7 +538,7 @@ export const make = Effect.gen(function* () { projectThreads, (thread) => threadCommands - .withLock(thread.id, deleteChildThread(input, thread.id)) + .withDeletionLock(threadCommands.withLock(thread.id, deleteChildThread(input, thread.id))) .pipe( Effect.mapError( (cause) => @@ -530,7 +551,14 @@ export const make = Effect.gen(function* () { // Best-effort cleanup releases disabled instances kept alive by overrides; crashes have no replay to revisit deletion. const clearProjectSettingsOverrides = (projectId: ProjectId) => - serverSettings.updateSettings({ projectSettingsOverrides: { [projectId]: null } }).pipe( + serverSettings.getSettings.pipe( + Effect.flatMap(({ projectSettingsOverrides }) => + serverSettings.updateSettings({ + projectSettingsOverrides: { + [projectId]: deletedProjectOverridePatch(projectSettingsOverrides[projectId]), + }, + }), + ), Effect.asVoid, Effect.catchCause((cause) => Effect.logWarning( diff --git a/apps/server/src/project/ProjectSetupScriptRunner.test.ts b/apps/server/src/project/ProjectSetupScriptRunner.test.ts index 4eb466200bc9..fd40f24d5a90 100644 --- a/apps/server/src/project/ProjectSetupScriptRunner.test.ts +++ b/apps/server/src/project/ProjectSetupScriptRunner.test.ts @@ -1,16 +1,20 @@ import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; -import { describe, expect, it } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, describe, expect, it } from "@effect/vitest"; import { ProjectId, TerminalNotRunningError, type TerminalEvent, type TerminalSessionSnapshot, } from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Queue from "effect/Queue"; +import * as Stream from "effect/Stream"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; import { TestClock } from "effect/testing"; import * as ServerSettings from "../serverSettings.ts"; @@ -38,6 +42,7 @@ const openSnapshot = (history: string): TerminalSessionSnapshot => ({ }); const makeHarness = Effect.fn(function* (options: { + readonly command?: string; readonly history?: string; readonly opening?: Deferred.Deferred; readonly shell?: string | null; @@ -88,7 +93,7 @@ const makeHarness = Effect.fn(function* (options: { { id: "setup", name: "Setup", - command: "vp i", + command: options.command ?? "vp i", icon: "configure", runOnWorktreeCreate: true, }, @@ -102,6 +107,7 @@ const makeHarness = Effect.fn(function* (options: { const setup = Effect.fn(function* ( options: { + readonly command?: string; readonly history?: string; readonly shell?: string | null; } = {}, @@ -123,6 +129,32 @@ describe("ProjectSetupScriptRunner", () => { }), ); + it.effect("types an escaped Nushell wrapper once its line editor starts", () => + Effect.gen(function* () { + const command = + 'print "quoted"; print \'C:\\tools\'; print "$env.PATH"\nexit 7 # comment\r\n\t'; + const escaped = + 'print \\"quoted\\"; print \'C:\\\\tools\'; print \\"$env.PATH\\"\\u{a}exit 7 # comment\\u{d}\\u{a}\\u{9}'; + const { writes, emit } = yield* setup({ shell: "/opt/homebrew/bin/nu", command }); + yield* TestClock.adjust("1 second"); + expect(yield* Queue.size(writes)).toBe(0); + yield* emit({ ...event, type: "output", data: "> \x1b[?2004h" }); + const written = yield* Queue.take(writes); + const sentinel = written.match(/__T3_SETUP_DONE___[0-9a-f]{32}:/)?.[0]; + expect(sentinel).toBeDefined(); + expect(written).toBe( + ` ${[ + "$env.LAST_EXIT_CODE = 0", + "let __t3f = try { mktemp --tmpdir-path $env.T3CODE_WORKTREE_PATH --suffix .nu .t3-setup-XXXXXXXX } catch {|e| print -e $e.msg; null }", + `if $__t3f == null { $env.LAST_EXIT_CODE = 1 } else { try { [$"rm --permanent ($__t3f | to nuon)" "${escaped}"] | str join (char newline) | save --force --raw $__t3f; ^$nu.current-exe --login --commands $"cd $env.T3CODE_WORKTREE_PATH; source ($__t3f | to nuon)" } catch {|e| if $e != null and $e.exit_code? == null { print -e $e.msg }; if $env.LAST_EXIT_CODE == 0 { $env.LAST_EXIT_CODE = 1 } } }`, + "let __t3c = $env.LAST_EXIT_CODE", + "try { if $__t3f != null and ($__t3f | path exists) { rm --permanent $__t3f } } catch {|e| print -e $e.msg }", + `print $"\\n${sentinel}($__t3c)"`, + ].join("; ")}\r`, + ); + }), + ); + it.effect("shows a startup prompt and types only once the line editor starts", () => Effect.gen(function* () { const { result, writes, outputLines, emit } = yield* setup(); @@ -136,7 +168,19 @@ describe("ProjectSetupScriptRunner", () => { yield* emit({ ...event, type: "activity", hasRunningSubprocess: false, label: "zsh" }); yield* emit({ ...event, type: "output", data: "\r\n% \x1b[?2004h" }); - expect(yield* Queue.take(writes)).toMatch(/^ \( vp i\r\);/); + expect(yield* Queue.take(writes)).toContain("vp i"); + }), + ); + + it.effect("drops whole escape sequences from output lines", () => + Effect.gen(function* () { + const { outputLines, emit } = yield* setup(); + yield* emit({ + ...event, + type: "output", + data: "\x1b7\x1b[24;0fProgress: [ 40%]\x1b8\r\n\x1b%G\x1b#8\x1b[4:3mdone\x1bM\r\n", + }); + expect(yield* Queue.takeAll(outputLines)).toEqual(["Progress: [ 40%]", "done"]); }), ); @@ -144,7 +188,7 @@ describe("ProjectSetupScriptRunner", () => { Effect.gen(function* () { const { writes } = yield* setup(); yield* TestClock.adjust("3 seconds"); - expect(yield* Queue.take(writes)).toMatch(/^ \( vp i\r\);/); + expect(yield* Queue.take(writes)).toContain("vp i"); }), ); @@ -166,7 +210,7 @@ describe("ProjectSetupScriptRunner", () => { yield* emit({ ...event, type: "restarted", snapshot: openSnapshot("") }); // The restart lands inside a quiet window, so the next full window decides. yield* TestClock.adjust("6 seconds"); - expect(yield* Queue.take(writes)).toMatch(/^ \( vp i\r\);/); + expect(yield* Queue.take(writes)).toContain("vp i"); }), ); @@ -199,3 +243,110 @@ describe("ProjectSetupScriptRunner", () => { }), ); }); + +const runObservedSetup = Effect.fnUntraced(function* (command: string) { + const { writes, outputLines, emit, completion } = yield* setup({ command }); + yield* emit({ ...event, type: "output", data: "\x1b[?2004h" }); + const submitted = yield* Queue.take(writes); + const cwd = yield* HostProcess.WorkingDirectory; + const env = yield* HostProcess.Environment; + + // Enter becomes a newline at the terminal boundary; transport assertions below + // cover line-editor submissions and canonical TTY limits that bash -c bypasses. + const child = yield* Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const handle = yield* spawner.spawn( + ChildProcess.make( + "/bin/bash", + ["--noprofile", "--norc", "-c", submitted.replaceAll("\r", "\n")], + { + cwd, + env: { PATH: env.PATH }, + }, + ), + ); + return yield* Effect.all( + { + stdout: handle.stdout.pipe(Stream.decodeText(), Stream.mkString), + stderr: handle.stderr.pipe(Stream.decodeText(), Stream.mkString), + exitCode: handle.exitCode, + }, + { concurrency: "unbounded" }, + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)); + yield* emit({ ...event, type: "output", data: submitted + "\n" }); + yield* emit({ ...event, type: "output", data: child.stdout + child.stderr }); + yield* emit({ ...event, type: "closed" }); + return { + submitted, + lines: yield* Queue.clear(outputLines), + exitCode: (yield* completion).exitCode, + }; +}); + +describe.skipIf(HostProcess.Platform.defaultValue() === "win32")("POSIX setup completion", () => { + it.live("submits one line while preserving multiline shell syntax and literal text", () => + Effect.gen(function* () { + const result = yield* runObservedSetup( + [ + "printf '%s\\n' \"it's ready\" 'literal \\n $(printf wrong) `printf wrong` café'", + "cat <<'END'", + "$literal \\path 雪", + "END", + "printf '%s\\n' done # trailing comment", + ].join("\r\n"), + ); + assert.equal(result.submitted.at(-1), "\r"); + assert.notMatch(result.submitted.slice(0, -1), /[\r\n]/); + assert.deepEqual(result.lines, [ + "it's ready", + "literal \\n $(printf wrong) `printf wrong` café", + "$literal \\path 雪", + "done", + ]); + assert.equal(result.exitCode, 0); + }), + ); + + it.live("completes long scripts without exceeding canonical terminal line limits", () => + Effect.gen(function* () { + const expected = Array.from( + { length: 120 }, + (_, index) => `line-${index}: café 雪 'quoted' \\path`, + ); + const command = expected + .map((line) => `printf '%s\\n' "${line.replaceAll("\\", "\\\\")}"`) + .join("\n"); + const utf8 = new TextEncoder(); + assert.isAbove(utf8.encode(command).byteLength, 4_096); + + const result = yield* runObservedSetup(command); + assert.deepEqual(result.lines, expected); + assert.equal(result.exitCode, 0); + assert.equal(result.submitted.at(-1), "\r"); + assert.notInclude(result.submitted, "\n"); + // The runner can write before the shell switches the terminal out of canonical mode. + for (const line of result.submitted.slice(0, -1).split("\r")) { + assert.isBelow(utf8.encode(line).byteLength, 1_024); + } + }), + ); + + it.live("preserves a trailing backslash-newline even when losing it would still exit zero", () => + Effect.gen(function* () { + const result = yield* runObservedSetup("printf '%s\\n' direct\\\n"); + assert.deepEqual(result.lines, ["direct"]); + assert.equal(result.exitCode, 0); + }), + ); + + it.live.each([ + { name: "an explicit exit", command: "exit 7", exitCode: 7 }, + { name: "a syntax error", command: "if then", exitCode: 2 }, + ])("reports a nonzero completion for $name", ({ command, exitCode }) => + Effect.gen(function* () { + const result = yield* runObservedSetup(command); + assert.equal(result.exitCode, exitCode); + }), + ); +}); diff --git a/apps/server/src/project/ProjectSetupScriptRunner.ts b/apps/server/src/project/ProjectSetupScriptRunner.ts index 8e7c3105b36b..60084f648a18 100644 --- a/apps/server/src/project/ProjectSetupScriptRunner.ts +++ b/apps/server/src/project/ProjectSetupScriptRunner.ts @@ -145,16 +145,14 @@ function completionSentinelPattern(token: string): RegExp { return new RegExp(`${COMPLETION_SENTINEL_PREFIX}_${token}:(-?\\d+)`); } +// ECMA-48: CSI, OSC, then any other escape as intermediates (0x20-0x2F) and one final byte. function stripTerminalControl(text: string): string { return text - .replace( - /\x1b\[[0-9;?]*[ -/]*[@-~]|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)|\x1b[()][A-Za-z0-9]|\x1b[=>]/g, - "", - ) + .replace(/\x1b\[[0-?]*[ -/]*[@-~]|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)|\x1b[ -/]*[0-~]/g, "") .replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, ""); } -// Line editors (zsh, fish, bash 5.1+) enable bracketed paste when they start reading a command. +// Line editors (zsh, fish, nu, bash 5.1+) enable bracketed paste when they start reading a command. const BRACKETED_PASTE_ON = "\x1b[?2004h"; const BRACKETED_PASTE_OFF = "\x1b[?2004l"; // Longer than one subprocess poll plus its ps timeout, so a startup prompt is seen as busy first. @@ -166,17 +164,68 @@ function lineEditorReading(text: string, previous: boolean): boolean { return on === off ? previous : on > off; } -type CompletionShell = "posix" | "fish" | "powershell"; +type CompletionShell = "posix" | "fish" | "nushell" | "powershell"; function resolveCompletionShell(shell: string): CompletionShell | null { const name = (shell.split(/[/\\]/).at(-1) ?? shell).toLowerCase().replace(/\.exe$/, ""); if (name === "cmd") return null; if (name === "fish") return "fish"; + if (name === "nu") return "nushell"; if (name === "pwsh" || name === "powershell") return "powershell"; return "posix"; } -// A block with its own closing line protects the sentinel from trailing comments, heredocs and scripts reading stdin. +const utf8 = new TextEncoder(); +// Canonical TTY input (before a line editor starts reading) caps a line below 1 KiB on Linux. +const POSIX_MAX_LINE_BYTES = 1_023; +const POSIX_APPEND = '__t3_command="$__t3_command"'; + +function encodePosixScript(text: string): string { + let encoded = ""; + for (const character of text) { + const code = character.charCodeAt(0); + if (character === "'") { + encoded += "'\\''"; + } else if (character === "\\" || code <= 0x1f || code === 0x7f) { + encoded += `\\0${code.toString(8).padStart(3, "0")}`; + } else { + encoded += character; + } + } + return encoded; +} + +// Line editors run each submitted line on its own, so every line must be a complete command. +function wrapPosixCommand(command: string, sentinel: string): string { + const normalized = command.replace(/\r\n/g, "\n"); + // The `_` suffix keeps trailing newlines; the subshell contains exits and syntax errors. + const evaluate = (payload: string) => + `( __t3_command=$(printf '%b_' ${payload}); eval "\${__t3_command%_}" ); printf '\\n${sentinel}%s\\n' "$?"`; + const inline = evaluate(`'${encodePosixScript(normalized)}'`); + if (utf8.encode(` ${inline}\r`).byteLength <= POSIX_MAX_LINE_BYTES) return inline; + + const maxChunkBytes = POSIX_MAX_LINE_BYTES - utf8.encode(` ${POSIX_APPEND}''\r`).byteLength; + const lines: string[] = []; + let prefix = "__t3_command="; + let chunk = ""; + let chunkBytes = 0; + for (const character of normalized) { + const encoded = encodePosixScript(character); + const bytes = utf8.encode(encoded).byteLength; + if (chunkBytes + bytes > maxChunkBytes) { + lines.push(`${prefix}'${chunk}'`); + prefix = POSIX_APPEND; + chunk = ""; + chunkBytes = 0; + } + chunk += encoded; + chunkBytes += bytes; + } + lines.push(`${prefix}'${chunk}'`, `${evaluate('"$__t3_command"')}; unset __t3_command`); + // The leading space gives every line the history suppression a short command gets. + return lines.join("\r "); +} + function wrapCommandForCompletion( command: string, shell: CompletionShell, @@ -188,8 +237,22 @@ function wrapCommandForCompletion( return `$global:LASTEXITCODE = $null; & {\r${body}\r}; if ($null -ne $LASTEXITCODE) { $__t3c = $LASTEXITCODE } elseif ($?) { $__t3c = 0 } else { $__t3c = 1 }; Write-Host "${sentinel}$__t3c"`; case "fish": return `begin\r${body}\rend; printf '\\n${sentinel}%s\\n' $status`; + case "nushell": { + const escaped = command.replace(/["\\\x00-\x1f]/g, (char) => + char === '"' || char === "\\" ? `\\${char}` : `\\u{${char.charCodeAt(0).toString(16)}}`, + ); + // Nu resolves imports before running cd; source a file rooted in the worktree. + return [ + "$env.LAST_EXIT_CODE = 0", + "let __t3f = try { mktemp --tmpdir-path $env.T3CODE_WORKTREE_PATH --suffix .nu .t3-setup-XXXXXXXX } catch {|e| print -e $e.msg; null }", + `if $__t3f == null { $env.LAST_EXIT_CODE = 1 } else { try { [$"rm --permanent ($__t3f | to nuon)" "${escaped}"] | str join (char newline) | save --force --raw $__t3f; ^$nu.current-exe --login --commands $"cd $env.T3CODE_WORKTREE_PATH; source ($__t3f | to nuon)" } catch {|e| if $e != null and $e.exit_code? == null { print -e $e.msg }; if $env.LAST_EXIT_CODE == 0 { $env.LAST_EXIT_CODE = 1 } } }`, + "let __t3c = $env.LAST_EXIT_CODE", + "try { if $__t3f != null and ($__t3f | path exists) { rm --permanent $__t3f } } catch {|e| print -e $e.msg }", + `print $"\\n${sentinel}($__t3c)"`, + ].join("; "); + } case "posix": - return `( ${body}\r); printf '\\n${sentinel}%s\\n' "$?"`; + return wrapPosixCommand(command, sentinel); } } @@ -517,9 +580,9 @@ export const make = Effect.gen(function* () { // Return once the terminal exists so callers can show it while a startup prompt waits. // PowerShell and cmd have no line-editor signal. yield* ( - completionShell === "posix" || completionShell === "fish" - ? readiness.wait - : Effect.succeed("ready" as const) + completionShell === "powershell" || completionShell === null + ? Effect.succeed("ready" as const) + : readiness.wait ).pipe( Effect.flatMap((shell) => shell === "exited" || commandLine === null diff --git a/apps/server/src/project/RepositoryIdentityResolver.test.ts b/apps/server/src/project/RepositoryIdentityResolver.test.ts index 4604da894a86..50d749c3fb1b 100644 --- a/apps/server/src/project/RepositoryIdentityResolver.test.ts +++ b/apps/server/src/project/RepositoryIdentityResolver.test.ts @@ -5,8 +5,10 @@ import { expect, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; +import { vi } from "vite-plus/test"; import * as ProcessRunner from "../processRunner.ts"; +import { sourceControlDefinitions } from "../sourceControl/sourceControlDefinitions.ts"; import * as RepositoryIdentityResolver from "./RepositoryIdentityResolver.ts"; const normalizePathSeparators = (value: string) => value.replaceAll("\\", "/"); @@ -70,6 +72,30 @@ it.layer(NodeServices.layer)("RepositoryIdentityResolverLive", (it) => { }).pipe(Effect.provide(RepositoryIdentityResolver.layer)), ); + it.effect("keeps a literal .git in an Azure DevOps repository name", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-repository-identity-azure-git-suffix-test-", + }); + + yield* git(cwd, ["init"]); + yield* git(cwd, [ + "remote", + "add", + "origin", + "https://dev.azure.com/Acme/Platform/_git/Tool.git", + ]); + + const resolver = yield* RepositoryIdentityResolver.RepositoryIdentityResolver; + const identity = yield* resolver.resolve(cwd); + + expect(identity?.displayName).toBe("Acme/Platform/_git/Tool.git"); + expect(identity?.name).toBe("Tool.git"); + expect(sourceControlDefinitions.repositorySelector(identity)).toBe("Tool.git"); + }).pipe(Effect.provide(RepositoryIdentityResolver.layer)), + ); + it.effect("resolves the fetch remote of a partial clone", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; @@ -94,33 +120,53 @@ it.layer(NodeServices.layer)("RepositoryIdentityResolverLive", (it) => { }).pipe(Effect.provide(RepositoryIdentityResolver.layer)), ); - it.effect("returns the git top-level root path when resolving from a nested workspace", () => + it.effect("resolves a remote whose URL contains a space", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const repoRoot = yield* fileSystem.makeTempDirectoryScoped({ - prefix: "t3-repository-identity-nested-root-test-", + const cwd = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-repository-identity-spaced-url-test-", }); - const nestedWorkspace = path.join(repoRoot, "packages", "web"); - yield* fileSystem.makeDirectory(nestedWorkspace, { recursive: true }); - yield* git(repoRoot, ["init"]); - yield* git(repoRoot, ["remote", "add", "origin", "git@github.com:T3Tools/t3code.git"]); + yield* git(cwd, ["init"]); + yield* git(cwd, ["remote", "add", "origin", "ssh://git@git.example.test/team repos/app.git"]); const resolver = yield* RepositoryIdentityResolver.RepositoryIdentityResolver; - const identity = yield* resolver.resolve(nestedWorkspace); - const resolvedIdentityRoot = - identity?.rootPath === undefined ? "" : NodeFS.realpathSync.native(identity.rootPath); - const resolvedRepoRoot = NodeFS.realpathSync.native(repoRoot); - - expect(identity).not.toBeNull(); - expect(identity?.canonicalKey).toBe("github.com/t3tools/t3code"); - expect(normalizeResolvedPath(resolvedIdentityRoot)).toBe( - normalizeResolvedPath(resolvedRepoRoot), - ); + const identity = yield* resolver.resolve(cwd); + expect(identity?.locator.remoteUrl).toBe("ssh://git@git.example.test/team repos/app.git"); + expect(identity?.locator.remoteName).toBe("origin"); }).pipe(Effect.provide(RepositoryIdentityResolver.layer)), ); + it.effect.each(["repo", "repo "] as const)( + "returns the git top-level root path when resolving from a nested workspace in %j", + (rootName) => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const tempDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-repository-identity-nested-root-test-", + }); + const repoRoot = path.join(tempDir, rootName); + const nestedWorkspace = path.join(repoRoot, "packages", "web"); + + yield* fileSystem.makeDirectory(nestedWorkspace, { recursive: true }); + yield* git(repoRoot, ["init"]); + yield* git(repoRoot, ["remote", "add", "origin", "git@github.com:T3Tools/t3code.git"]); + + const resolver = yield* RepositoryIdentityResolver.RepositoryIdentityResolver; + const identity = yield* resolver.resolve(nestedWorkspace); + const resolvedIdentityRoot = + identity?.rootPath === undefined ? "" : NodeFS.realpathSync.native(identity.rootPath); + const resolvedRepoRoot = NodeFS.realpathSync.native(repoRoot); + + expect(identity).not.toBeNull(); + expect(identity?.canonicalKey).toBe("github.com/t3tools/t3code"); + expect(normalizeResolvedPath(resolvedIdentityRoot)).toBe( + normalizeResolvedPath(resolvedRepoRoot), + ); + }).pipe(Effect.provide(RepositoryIdentityResolver.layer)), + ); + it.effect.each(["bare clone", "dot-bare"] as const)( "resolves a %s to its Git directory and the same identity as a checkout", (layout) => @@ -303,6 +349,95 @@ it.layer(NodeServices.layer)("RepositoryIdentityResolverLive", (it) => { }).pipe(Effect.provide(RepositoryIdentityResolver.layer)), ); + it.effect("uses the fork as the identity when gh repo set-default points at it", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-repository-identity-gh-default-test-", + }); + + yield* git(cwd, ["init"]); + yield* git(cwd, ["remote", "add", "origin", "git@github.com:julius/t3code-fork.git"]); + yield* git(cwd, ["remote", "add", "upstream", "git@github.com:T3Tools/t3code.git"]); + yield* git(cwd, ["config", "remote.origin.gh-resolved", "base"]); + + const resolver = yield* RepositoryIdentityResolver.RepositoryIdentityResolver; + const identity = yield* resolver.resolve(cwd); + + expect(identity?.canonicalKey).toBe("github.com/julius/t3code-fork"); + expect(identity?.locator.remoteName).toBe("origin"); + }).pipe(Effect.provide(RepositoryIdentityResolver.layer)), + ); + + it.effect("follows an OWNER/REPO mark only to a remote that fetches that repository", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-repository-identity-gh-parent-test-", + }); + const resolver = yield* RepositoryIdentityResolver.RepositoryIdentityResolver; + const resolveFresh = () => resolver.resolve(cwd, { refresh: true }); + + yield* git(cwd, ["init"]); + yield* git(cwd, ["remote", "add", "fork", "https://github.com/julius/t3code-fork.git"]); + yield* git(cwd, ["remote", "add", "main", "https://github.com/T3Tools/t3code.git"]); + yield* git(cwd, ["remote", "add", "upstream", "https://github.com/other/t3code.git"]); + yield* git(cwd, ["config", "remote.fork.gh-resolved", "t3tools/t3code"]); + const parent = yield* resolveFresh(); + expect(parent?.canonicalKey).toBe("github.com/t3tools/t3code"); + expect(parent?.locator).toEqual({ + source: "git-remote", + remoteName: "main", + remoteUrl: "https://github.com/T3Tools/t3code.git", + }); + + // No remote fetches the marked repository, and several marks leave gh's choice unknown. + yield* git(cwd, ["config", "remote.fork.gh-resolved", "someone/else"]); + expect((yield* resolveFresh())?.locator.remoteName).toBe("upstream"); + yield* git(cwd, ["config", "remote.fork.gh-resolved", "base"]); + yield* git(cwd, ["config", "remote.main.gh-resolved", "base"]); + expect((yield* resolveFresh())?.locator.remoteName).toBe("upstream"); + + // The same path on another HTTPS port is a different server. + yield* git(cwd, ["config", "--unset", "remote.fork.gh-resolved"]); + yield* git(cwd, ["config", "--unset", "remote.main.gh-resolved"]); + yield* git(cwd, ["remote", "add", "backup", "https://ghe.example.com:9443/acme/web.git"]); + yield* git(cwd, ["remote", "add", "ghe-fork", "https://ghe.example.com:8443/me/web.git"]); + yield* git(cwd, ["remote", "add", "ghe-parent", "https://ghe.example.com:8443/acme/web.git"]); + yield* git(cwd, ["config", "remote.ghe-fork.gh-resolved", "acme/web"]); + expect((yield* resolveFresh())?.locator.remoteName).toBe("ghe-parent"); + }).pipe(Effect.provide(RepositoryIdentityResolver.layer)), + ); + + it.effect("reads the project's gh default when the server inherited GIT_DIR", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-repository-identity-gh-default-git-dir-", + }); + const other = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-repository-identity-inherited-git-dir-", + }); + + yield* git(cwd, ["init"]); + yield* git(cwd, ["remote", "add", "origin", "git@github.com:julius/t3code-fork.git"]); + yield* git(cwd, ["remote", "add", "upstream", "git@github.com:T3Tools/t3code.git"]); + yield* git(cwd, ["config", "remote.origin.gh-resolved", "base"]); + yield* git(other, ["init"]); + yield* git(other, ["config", "remote.upstream.gh-resolved", "base"]); + + vi.stubEnv("GIT_DIR", path.join(other, ".git")); + yield* Effect.addFinalizer(() => Effect.sync(() => vi.unstubAllEnvs())); + + const resolver = yield* RepositoryIdentityResolver.RepositoryIdentityResolver; + const identity = yield* resolver.resolve(cwd); + + expect(identity?.canonicalKey).toBe("github.com/julius/t3code-fork"); + expect(identity?.locator.remoteName).toBe("origin"); + }).pipe(Effect.provide(RepositoryIdentityResolver.layer)), + ); + it.effect("uses the last remote path segment as the repository name for nested groups", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; diff --git a/apps/server/src/project/RepositoryIdentityResolver.ts b/apps/server/src/project/RepositoryIdentityResolver.ts index cdf9dc5e1b30..826529e704b2 100644 --- a/apps/server/src/project/RepositoryIdentityResolver.ts +++ b/apps/server/src/project/RepositoryIdentityResolver.ts @@ -1,5 +1,7 @@ import type { RepositoryIdentity, SourceControlProviderError } from "@t3tools/contracts"; +import { parseGitRemoteHost } from "@t3tools/source-control-core/shared/definition"; import { repositoryPathFromRemoteUrl } from "@t3tools/source-control-core/server/SourceControlProvider"; +import { canonicalRepositoryPath, normalizeGitRemoteUrl } from "@t3tools/shared/git"; import * as Cache from "effect/Cache"; import * as Context from "effect/Context"; import * as Data from "effect/Data"; @@ -12,7 +14,7 @@ import * as Path from "effect/Path"; import * as ProcessRunner from "../processRunner.ts"; import { sourceControlDefinitions } from "../sourceControl/sourceControlDefinitions.ts"; -import { explicitBareRepositoryEnv } from "../vcs/VcsProcess.ts"; +import { explicitBareRepositoryEnv, unsetInheritedRepositoryEnv } from "../vcs/VcsProcess.ts"; import * as HostProcess from "@t3tools/shared/HostProcess"; const DEFAULT_REPOSITORY_IDENTITY_CACHE_CAPACITY = 512; @@ -50,11 +52,10 @@ function isLocalRemoteUrl(url: string): boolean { function parseRemoteFetchUrls(stdout: string): Map { const remotes = new Map(); - for (const line of stdout.split("\n")) { - const trimmed = line.trim(); - if (trimmed.length === 0) continue; + for (const line of stdout.split(/\r?\n/)) { + // Git uses a tab before the URL and one space before the direction; both can occur in URLs. // Partial clones append the promisor filter, e.g. `(fetch) [blob:none]`. - const match = /^(\S+)\s+(\S+)\s+\((fetch|push)\)(?:\s+\[[^\]]+\])?$/.exec(trimmed); + const match = /^(\S+)\t(.+) \((fetch|push)\)(?:\s+\[[^\]]+\])?$/.exec(line); if (!match) continue; const [, remoteName = "", remoteUrl = "", direction = ""] = match; if ( @@ -70,9 +71,43 @@ function parseRemoteFetchUrls(stdout: string): Map { return remotes; } +// `gh repo set-default` marks one remote with `base`, or with OWNER/REPO when it picked a +// parent repository. gh reads PRs from that repository, so the identity follows the mark when +// a remote fetches it. The locator must name a real remote: review checkouts fetch through it. +function pickMarkedRemote( + remotes: ReadonlyMap, + resolved: string, +): { readonly remoteName: string; readonly remoteUrl: string } | null { + const marks = resolved.split("\n").flatMap((line) => { + const match = /^remote\.(.+)\.gh-resolved\s+(\S+)$/u.exec(line.trim()); + const remoteUrl = match?.[1] === undefined ? undefined : remotes.get(match[1]); + return match?.[1] && match[2] && remoteUrl + ? [{ remoteName: match[1], remoteUrl, value: match[2] }] + : []; + }); + // gh's choice is ambiguous with several marks; GitHubSourceControlProvider ignores them too. + const [mark, ...others] = marks; + if (!mark || others.length > 0) return null; + if (mark.value === "base") return { remoteName: mark.remoteName, remoteUrl: mark.remoteUrl }; + // Compare hosts with HTTP ports kept: one hostname can serve separate forges on two ports. + const host = parseGitRemoteHost(mark.remoteUrl)?.host; + const markedPath = mark.value.toLowerCase(); + const [remoteName, remoteUrl] = + [...remotes.entries()].find( + ([, url]) => + parseGitRemoteHost(url)?.host === host && + normalizeGitRemoteUrl(url).split("/").slice(1).join("/") === markedPath, + ) ?? []; + return remoteName && remoteUrl ? { remoteName, remoteUrl } : null; +} + function pickPrimaryRemote( remotes: ReadonlyMap, + resolved: string, ): { readonly remoteName: string; readonly remoteUrl: string } | null { + const marked = pickMarkedRemote(remotes, resolved); + if (marked) return marked; + for (const preferredRemoteName of ["upstream", "origin"] as const) { const remoteUrl = remotes.get(preferredRemoteName); if (remoteUrl) { @@ -88,7 +123,13 @@ function pickPrimaryRemote( function repositoryPathOf(canonicalKey: string, remoteUrl: string): string { const canonicalPath = canonicalKey.split("/").slice(1).join("/"); const displayPath = repositoryPathFromRemoteUrl(remoteUrl); - return displayPath?.toLowerCase() === canonicalPath ? displayPath : canonicalPath; + if (displayPath === null) return canonicalPath; + // A host that names repositories literally keeps the `.git` the display path drops. + return ( + [displayPath, `${displayPath}.git`].find( + (path) => canonicalRepositoryPath(path) === canonicalRepositoryPath(canonicalPath), + ) ?? canonicalPath + ); } function buildRepositoryOrigin( @@ -137,12 +178,17 @@ class RepositoryIdentityCacheKey extends Data.Class<{ readonly isBare: boolean; }> {} +const gitEnv = (platform: NodeJS.Platform): NodeJS.ProcessEnv => ({ + ...unsetInheritedRepositoryEnv(process.env, platform), + ...explicitBareRepositoryEnv(process.env, platform), +}); + const resolveRepositoryIdentityCacheKey = Effect.fn("RepositoryIdentityResolver.resolveCacheKey")( function* (cwd: string) { const processRunner = yield* ProcessRunner.ProcessRunner; const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const env = explicitBareRepositoryEnv(process.env, yield* HostProcess.Platform); + const env = gitEnv(yield* HostProcess.Platform); // `git -C` cannot enter a missing folder or a file, so every probe below would fail. const isFolder = yield* fileSystem.stat(cwd).pipe( @@ -165,7 +211,8 @@ const resolveRepositoryIdentityCacheKey = Effect.fn("RepositoryIdentityResolver. return null; } if (topLevelResult.value.code === 0) { - const rootPath = topLevelResult.value.stdout.trim(); + // Paths may end in spaces, so strip only git's line terminator. + const rootPath = topLevelResult.value.stdout.replace(/\r?\n$/, ""); return rootPath.length > 0 ? new RepositoryIdentityCacheKey({ rootPath, isBare: false }) : null; @@ -209,9 +256,7 @@ const resolveRepositoryIdentityCacheKey = Effect.fn("RepositoryIdentityResolver. if (bareResult._tag === "None" || bareResult.value.timedOut) return null; if (bareResult.value.code !== 0) continue; - const [isBare, rootPath = ""] = bareResult.value.stdout - .split("\n") - .map((line) => line.trim()); + const [isBare, rootPath = ""] = bareResult.value.stdout.split(/\r?\n/); // A checkout's .git directory contains remotes but is not a project root. return isBare === "true" && rootPath.length > 0 ? new RepositoryIdentityCacheKey({ rootPath, isBare: true }) @@ -227,20 +272,31 @@ const resolveRepositoryIdentityFromCacheKey = Effect.fn( cacheKey: RepositoryIdentityCacheKey, ): Effect.fn.Return { const processRunner = yield* ProcessRunner.ProcessRunner; - const remoteResult = yield* processRunner - .run({ - command: "git", - args: ["-C", cacheKey.rootPath, ...(cacheKey.isBare ? ["--git-dir=."] : []), "remote", "-v"], - env: explicitBareRepositoryEnv(process.env, yield* HostProcess.Platform), - timeoutBehavior: "timedOutResult", - }) - .pipe(Effect.option); + const env = gitEnv(yield* HostProcess.Platform); + const runGit = (args: ReadonlyArray) => + processRunner + .run({ + command: "git", + args: ["-C", cacheKey.rootPath, ...(cacheKey.isBare ? ["--git-dir=."] : []), ...args], + env, + timeoutBehavior: "timedOutResult", + }) + .pipe(Effect.option); + const [remoteResult, resolvedResult] = yield* Effect.all( + [runGit(["remote", "-v"]), runGit(["config", "--get-regexp", "^remote\\..*\\.gh-resolved$"])], + { concurrency: "unbounded" }, + ); if (remoteResult._tag === "None" || remoteResult.value.code !== 0) { return null; } const remotes = parseRemoteFetchUrls(remoteResult.value.stdout); - const remote = pickPrimaryRemote(remotes); + // Exits 1 when no remote has a mark. + const resolved = + resolvedResult._tag === "Some" && resolvedResult.value.code === 0 + ? resolvedResult.value.stdout + : ""; + const remote = pickPrimaryRemote(remotes, resolved); return remote ? buildRepositoryIdentity({ ...remote, diff --git a/apps/server/src/project/gitCloneProgress.ts b/apps/server/src/project/gitCloneProgress.ts index f1d75b532bc7..b74ecf6da84e 100644 --- a/apps/server/src/project/gitCloneProgress.ts +++ b/apps/server/src/project/gitCloneProgress.ts @@ -16,7 +16,10 @@ const STAGE_PREFIXES: ReadonlyArray = [ [/^Checking out files/, "checkout"], ]; -const PERCENT = /:\s+(\d+)%\s+\((\d+)\/(\d+)\)(?:,\s*(.*?))?\s*(?:,\s*done\.)?\s*$/; +// Linear on hostile remote output: the tail is trimmed in code (optional `\s*`/`.*?` groups backtrack +// cubically), and `s` lets it span U+2028 so a failed `$` cannot retry at every later counter. +const PERCENT = /:\s+(\d+)%\s+\((\d+)\/(\d+)\)(.*)$/s; +const DONE_SUFFIX = ", done."; /** Pass each bare-CR progress redraw as a separate line. */ export function parseGitCloneProgressLine(line: string): GitCloneProgressLine | null { @@ -26,8 +29,11 @@ export function parseGitCloneProgressLine(line: string): GitCloneProgressLine | const stage = stageEntry[1]; const match = PERCENT.exec(trimmed); if (!match) return { stage, percent: null, detail: null }; + let tail = (match[4] ?? "").trimEnd(); + if (tail.endsWith(DONE_SUFFIX)) tail = tail.slice(0, -DONE_SUFFIX.length).trimEnd(); + if (tail.length > 0 && !tail.startsWith(",")) return { stage, percent: null, detail: null }; const percent = Number(match[1]); - const rawDetail = match[4]?.trim() ?? ""; + const rawDetail = tail.slice(1).trim(); const detail = rawDetail.length > 0 && rawDetail !== "done." ? rawDetail : null; return { stage, diff --git a/apps/server/src/project/worktreeContainer.test.ts b/apps/server/src/project/worktreeContainer.test.ts new file mode 100644 index 000000000000..b10f8db93dab --- /dev/null +++ b/apps/server/src/project/worktreeContainer.test.ts @@ -0,0 +1,94 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; + +import * as ProcessRunner from "../processRunner.ts"; +import { makeIsWorktreeContainer } from "./worktreeContainer.ts"; + +const git = Effect.fn("worktreeContainerTest.git")(function* ( + cwd: string, + ...args: ReadonlyArray +) { + const processRunner = yield* ProcessRunner.ProcessRunner; + return yield* processRunner.run({ + command: "git", + args: ["-C", cwd, "-c", "user.email=t@t", "-c", "user.name=t", ...args], + }); +}); + +it.layer(ProcessRunner.layer.pipe(Layer.provideMerge(NodeServices.layer)))( + "isWorktreeContainer", + (it) => { + it.effect("is true only for a folder of worktrees around metadata that is no checkout", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const isWorktreeContainer = yield* makeIsWorktreeContainer; + const fixture = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-container-test-" }); + + const source = path.join(fixture, "source"); + yield* fileSystem.makeDirectory(source); + yield* git(source, "init", "-b", "develop"); + yield* fileSystem.writeFileString(path.join(source, "README.md"), "readme\n"); + yield* git(source, "add", "README.md"); + yield* git(source, "commit", "-m", "init"); + + const makeContainer = Effect.fn(function* (name: string, coreBare: boolean) { + const root = path.join(fixture, name); + yield* fileSystem.makeDirectory(root); + yield* git(fixture, "clone", "--bare", source, path.join(root, ".bare")); + yield* fileSystem.writeFileString(path.join(root, ".git"), "gitdir: ./.bare\n"); + yield* git(root, "worktree", "add", "develop", "develop"); + yield* git(root, "config", "core.bare", String(coreBare)); + return root; + }); + const makeSeparateGitDirCheckout = Effect.fn(function* (name: string) { + const root = path.join(fixture, name); + yield* fileSystem.makeDirectory(root); + yield* git(root, "init", "-b", "develop", "--separate-git-dir", ".metadata", "."); + return root; + }); + + const container = yield* makeContainer("container", false); + assert.isTrue(yield* isWorktreeContainer(container), "container"); + assert.isFalse(yield* isWorktreeContainer(path.join(container, "develop")), "its worktree"); + + const linked = path.join(fixture, "link"); + yield* fileSystem.symlink(container, linked); + assert.isTrue(yield* isWorktreeContainer(linked), "container through a symlink"); + + // The issue's steps leave core.bare set; Initialize Git would clear it later. + const bare = yield* makeContainer("bare", true); + assert.isTrue(yield* isWorktreeContainer(bare), "core.bare container"); + + // Git allows an orphan worktree before the first commit: no index, yet a valid checkout. + const fresh = yield* makeSeparateGitDirCheckout("fresh"); + yield* git(fresh, "worktree", "add", "--orphan", "-b", "feature", "../fresh-feature"); + assert.isFalse(yield* isWorktreeContainer(fresh), "fresh checkout with an orphan worktree"); + + const checkout = yield* makeSeparateGitDirCheckout("checkout"); + yield* fileSystem.writeFileString(path.join(checkout, "README.md"), "readme\n"); + yield* git(checkout, "add", "README.md"); + yield* git(checkout, "commit", "-m", "init"); + yield* git(checkout, "worktree", "add", "worktrees/feature", "-b", "feature"); + assert.isFalse(yield* isWorktreeContainer(checkout), "checkout with a nested worktree"); + + const outside = path.join(fixture, "outside"); + yield* git( + fixture, + "clone", + "--no-checkout", + "--separate-git-dir", + path.join(outside, ".metadata"), + source, + outside, + ); + yield* git(outside, "worktree", "add", "../outside-feature", "-b", "feature"); + assert.isFalse(yield* isWorktreeContainer(outside), "worktrees outside the folder"); + }), + ); + }, +); diff --git a/apps/server/src/project/worktreeContainer.ts b/apps/server/src/project/worktreeContainer.ts new file mode 100644 index 000000000000..f20e9c4375ca --- /dev/null +++ b/apps/server/src/project/worktreeContainer.ts @@ -0,0 +1,95 @@ +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; + +import * as ProcessRunner from "../processRunner.ts"; +import { explicitBareRepositoryEnv, unsetInheritedRepositoryEnv } from "../vcs/VcsProcess.ts"; + +const parseGitfileTarget = (contents: string): string | undefined => { + const firstLine = contents.split(/\r?\n/u, 1)[0] ?? ""; + if (!firstLine.startsWith("gitdir:")) return undefined; + const target = firstLine.slice("gitdir:".length).trim(); + return target === "" ? undefined : target; +}; + +const splitNul = (stdout: string) => stdout.split("\0").filter((entry) => entry !== ""); + +/** + * Whether a folder only holds a repository's metadata and worktrees: its `.git` file points at + * metadata inside it, a linked worktree lives inside it, and either the repository is bare or it + * has commits but no index. A checkout whose tracked files are all deleted keeps its index, and a + * fresh one has no commit. Anything this cannot establish answers false. + */ +export const makeIsWorktreeContainer = Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const processRunner = yield* ProcessRunner.ProcessRunner; + const platform = yield* HostProcess.Platform; + const env = { + ...unsetInheritedRepositoryEnv(process.env, platform), + ...explicitBareRepositoryEnv(process.env, platform), + }; + + const realPath = (target: string) => fileSystem.realPath(target).pipe(Effect.option); + const isInside = (parent: string, child: string) => { + const relative = path.relative(parent, child); + return ( + relative !== "" && + relative !== ".." && + !relative.startsWith(`..${path.sep}`) && + !path.isAbsolute(relative) + ); + }; + + return Effect.fn("ProjectService.isWorktreeContainer")(function* (workspaceRoot: string) { + // A `.git` directory is an ordinary checkout, so only a gitfile costs any Git process. + const gitfileTarget = yield* fileSystem.readFileString(path.join(workspaceRoot, ".git")).pipe( + Effect.map(parseGitfileTarget), + Effect.orElseSucceed(() => undefined), + ); + if (gitfileTarget === undefined) return false; + const root = yield* realPath(workspaceRoot); + const gitDir = yield* realPath(path.resolve(workspaceRoot, gitfileTarget)); + if (root._tag === "None" || gitDir._tag === "None") return false; + if (!isInside(root.value, gitDir.value)) return false; + + const git = (args: ReadonlyArray) => + processRunner + .run({ + command: "git", + args: ["-C", workspaceRoot, ...args], + env, + timeoutBehavior: "timedOutResult", + }) + .pipe( + Effect.map((result) => (result.code === 0 && !result.timedOut ? result.stdout : null)), + Effect.orElseSucceed(() => null), + ); + + // Initializing Git in a bare container clears core.bare, so it is refused while still bare. + const isBare = yield* git(["rev-parse", "--is-bare-repository"]); + if (isBare === null) return false; + if (isBare.trim() !== "true") { + const hasCommit = yield* git(["rev-parse", "--quiet", "--verify", "HEAD^{commit}"]); + if (hasCommit === null) return false; + const hasIndex = yield* fileSystem + .exists(path.join(gitDir.value, "index")) + .pipe(Effect.orElseSucceed(() => true)); + if (hasIndex) return false; + } + + const worktrees = yield* git(["worktree", "list", "--porcelain", "-z"]); + if (worktrees === null) return false; + // The first entry is the main worktree, which Git reports as the metadata directory here. + const linkedWorktreePaths = splitNul(worktrees) + .filter((field) => field.startsWith("worktree ")) + .slice(1) + .map((field) => field.slice("worktree ".length)); + for (const worktreePath of linkedWorktreePaths) { + const worktree = yield* realPath(worktreePath); + if (worktree._tag === "Some" && isInside(root.value, worktree.value)) return true; + } + return false; + }); +}); diff --git a/apps/server/src/provider/AcpRegistryProbe.test.ts b/apps/server/src/provider/AcpRegistryProbe.test.ts index 77f2a1ae31d4..30915b31d474 100644 --- a/apps/server/src/provider/AcpRegistryProbe.test.ts +++ b/apps/server/src/provider/AcpRegistryProbe.test.ts @@ -66,6 +66,22 @@ describe("ACP Registry probe", () => { { value: "gpt-5.4", name: "Duplicate" }, ], }, + { + id: "effort", + name: "Effort", + category: "thought_level", + type: "select", + currentValue: "high", + options: [{ value: "high", name: "High" }], + }, + { + id: "mode", + name: "Mode", + category: "mode", + type: "select", + currentValue: "ask", + options: [{ value: "ask", name: "Ask" }], + }, ], }, modelConfigId: "model", @@ -88,7 +104,24 @@ describe("ACP Registry probe", () => { ], models: [{ id: "gpt-5.4", name: "GPT-5.4", description: null }], currentModelId: "gpt-5.4", - configOptions: [], + configOptions: [ + { + id: "effort", + label: "Effort", + type: "select", + options: [{ id: "high", label: "High" }], + currentValue: "high", + role: "reasoning", + }, + { + id: "mode", + label: "Mode", + type: "select", + options: [{ id: "ask", label: "Ask" }], + currentValue: "ask", + }, + ], + modelOptionIds: ["effort"], sessionManagement: { canList: true, canLoad: true, @@ -377,7 +410,6 @@ describe("ACP Registry probe", () => { }).pipe( Effect.provideService(AcpRegistrySupport.AcpRegistryCatalog, catalog), Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), - Effect.scoped, ); }); diff --git a/apps/server/src/provider/AgentContextBudget.test.ts b/apps/server/src/provider/AgentContextBudget.test.ts index 8cdd52d202f1..2a7d02fc2817 100644 --- a/apps/server/src/provider/AgentContextBudget.test.ts +++ b/apps/server/src/provider/AgentContextBudget.test.ts @@ -35,6 +35,7 @@ import { ProjectCloneTracker } from "../project/ProjectCloneTracker.ts"; import { ProjectService } from "../project/ProjectService.ts"; import { ProjectSetupScriptRunner } from "../project/ProjectSetupScriptRunner.ts"; import { PullRequestReviewGuideService } from "../pullRequest/PullRequestReviewGuideService.ts"; +import { PullRequestService } from "../pullRequest/PullRequestService.ts"; import { ScheduledTaskService } from "../scheduledTasks/ScheduledTaskService.ts"; import { SecretRequests } from "../secrets/SecretRequests.ts"; import { ServerSettingsService } from "../serverSettings.ts"; @@ -43,6 +44,7 @@ import { GitVcsDriver } from "../vcs/GitVcsDriver.ts"; import { VcsStatusBroadcaster } from "../vcs/VcsStatusBroadcaster.ts"; import { buildCodexAdditionalContext } from "./CodexDeveloperInstructions.ts"; import { DocumentText } from "./DocumentText.ts"; +import { EnvironmentThemeService } from "../environmentTheme.ts"; import { ProviderRegistry } from "./ProviderRegistry.ts"; import budget from "./agentContextBudget.json" with { type: "json" }; @@ -51,6 +53,7 @@ const toolHandlerServices = Layer.mergeAll( Layer.mock(AssetSigner)({}), Layer.mock(DeviceService)({}), Layer.mock(DocumentText)({}), + Layer.mock(EnvironmentThemeService)({}), Layer.mock(GitVcsDriver)({}), Layer.mock(GitWorkflowService)({}), Layer.mock(ManagedProjectFolders)({ namedProjectsRoot: "" }), @@ -68,6 +71,7 @@ const toolHandlerServices = Layer.mergeAll( Layer.mock(ProviderAdapterRegistryV2)({}), Layer.mock(ProviderRegistry)({}), Layer.mock(PullRequestReviewGuideService)({}), + Layer.mock(PullRequestService)({}), Layer.mock(RemoteDelegation)({}), Layer.mock(ScheduledTaskService)({}), Layer.mock(SecretRequests)({}), @@ -114,13 +118,16 @@ const layerAdvertisedTools = McpHttpServer.layerTools.pipe( ); // Claude Code loads these tools up front instead of behind ToolSearch, outside this budget. -it.effect("only request_secret asks Claude Code to load it up front", () => +it.effect("only request_secret and send_to_user ask Claude Code to load them up front", () => Effect.gen(function* () { const mcp = yield* McpServer.McpServer; const alwaysLoad = mcp.tools .filter(({ tool }) => tool._meta?.["anthropic/alwaysLoad"] !== undefined) .map(({ tool }) => [tool.name, tool._meta]); - expect(alwaysLoad).toEqual([["request_secret", { "anthropic/alwaysLoad": true }]]); + expect(alwaysLoad).toEqual([ + ["request_secret", { "anthropic/alwaysLoad": true }], + ["send_to_user", { "anthropic/alwaysLoad": true }], + ]); }).pipe(Effect.provide(layerAdvertisedTools)), ); diff --git a/apps/server/src/provider/AntigravityInstallation.test.ts b/apps/server/src/provider/AntigravityInstallation.test.ts index f1c58250fd4b..dcd3e4003009 100644 --- a/apps/server/src/provider/AntigravityInstallation.test.ts +++ b/apps/server/src/provider/AntigravityInstallation.test.ts @@ -22,6 +22,7 @@ import * as ChildProcess from "effect/process/ChildProcess"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import * as AntigravityInstallation from "./AntigravityInstallation.ts"; +import * as ModelManifest from "./ModelManifest.ts"; import { type AntigravityReleaseAsset } from "./antigravityRelease.ts"; const serverContents = "antigravity runtime\n"; @@ -127,6 +128,12 @@ interface HarnessOptions { readonly validate?: AntigravityInstallation.AntigravityInstallationOptions["validate"]; readonly useDefaultValidation?: boolean; readonly connectTimeout?: boolean; + readonly manifest?: Effect.Effect; + readonly native?: boolean; + readonly nativeVersion?: string; + readonly nativeManifest?: (manifest: { version: string; url: string; sha512: string }) => unknown; + readonly beforeManifest?: Effect.Effect; + readonly updateChecksEnabled?: Effect.Effect; } const makeHarness = Effect.fn("test.makeAntigravityInstallation")(function* ( @@ -134,6 +141,7 @@ const makeHarness = Effect.fn("test.makeAntigravityInstallation")(function* ( ) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; + const digest = yield* Digest.Digest; const baseDir = options.baseDir ?? (yield* fs.makeTempDirectoryScoped({ prefix: "t3-agy-test-" })); const platform = options.platform ?? hostPlatform; @@ -169,7 +177,10 @@ const makeHarness = Effect.fn("test.makeAntigravityInstallation")(function* ( }); const installation = yield* AntigravityInstallation.makeAntigravityInstallation({ baseDir, - releaseAsset: asset, + ...(options.native ? {} : { releaseAsset: asset }), + ...(options.updateChecksEnabled === undefined + ? {} + : { updateChecksEnabled: options.updateChecksEnabled }), ...(options.useDefaultValidation ? {} : { @@ -186,11 +197,34 @@ const makeHarness = Effect.fn("test.makeAntigravityInstallation")(function* ( Effect.provideService(HostProcess.Platform, platform), Effect.provideService(HostProcess.Architecture, arch), Effect.provideService(HostProcess.Environment, { PATH: options.path ?? "" }), + Effect.provideService(ModelManifest.ModelManifest, { + current: options.manifest ?? Effect.succeed(ModelManifest.BUNDLED_MODEL_MANIFEST), + refresh: Effect.succeed(ModelManifest.BUNDLED_MODEL_MANIFEST), + forceRefresh: Effect.succeed(ModelManifest.BUNDLED_MODEL_MANIFEST), + refreshOnDemand: Effect.succeed(ModelManifest.BUNDLED_MODEL_MANIFEST), + refreshInBackground: Effect.void, + }), Effect.provideService( HttpClient.HttpClient, HttpClient.make((request) => Effect.suspend(() => { requests.push(request.url); + if (request.url.includes("/manifests/")) { + return Effect.gen(function* () { + yield* options.beforeManifest ?? Effect.void; + const version = options.nativeVersion ?? "1.5.7"; + const folder = platform === "win32" ? "windows-x64" : "linux-x64"; + const manifest = { + version, + url: `https://storage.googleapis.com/antigravity-public/antigravity-cli/${version}-123/${folder}/cli_${platform === "win32" ? "windows_x64.exe" : "linux_x64.tar.gz"}`, + sha512: Hex.encode(yield* digest.digest(Digest.sha512, archive).pipe(Effect.orDie)), + }; + return HttpClientResponse.fromWeb( + request, + new Response(JSON.stringify(options.nativeManifest?.(manifest) ?? manifest)), + ); + }); + } if (options.connectTimeout) { return Effect.fail( new HttpClientError.HttpClientError({ @@ -255,6 +289,256 @@ const expectPreviousRelease = Effect.fn("test.expectPreviousAntigravityRelease") }); it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("Antigravity installation", (it) => { + it.effect.each([...new Set(["win32", hostPlatform])] as NodeJS.Platform[])( + "migrates the legacy managed runtime to the native CLI on %s and retains the old release", + (platform) => + Effect.gen(function* () { + const archive = + platform === "win32" + ? Buffer.from(serverContents) + : Buffer.from( + "H4sIAAAAAAACE+3RMQ7CQAwEQNe8Ik8wgYj3XIFQCiiOCxK/56CJRA9CYqZZrxsXLpc2n2q5ze0en5LdYZpe2b1n5rhf5+d+m+MuY8j4guXaSu3n4z+V9f9DXXo5HzcBAAAAAAAAAADAz3sA56lbdQAoAAA=", + "base64", + ); + const { installation, validations, requests, fs, baseDir, stagingReleased } = + yield* makeHarness({ native: true, archive, platform, previous: true }); + yield* installation.start; + expect(yield* terminalState(installation)).toMatchObject({ + phase: "succeeded", + installedVersion: "1.5.7", + }); + yield* Deferred.await(stagingReleased); + const resolved = yield* installation.resolve(); + expect(resolved).toMatchObject({ source: "managed", version: "1.5.7", harnessPath: null }); + expect(validations[0]?.executable.harnessPath).toBeNull(); + expect(yield* fs.readFileString(resolved.executablePath)).toBe(serverContents); + expect(requests.some((url) => url.includes("/manifests/"))).toBe(true); + const restarted = yield* makeHarness({ + baseDir, + native: true, + archive, + platform, + nativeVersion: "1.5.6", + }); + expect(yield* restarted.installation.start).toMatchObject({ + phase: "verifying", + message: "Checking the installed runtime.", + }); + expect(yield* terminalState(restarted.installation)).toMatchObject({ + phase: "succeeded", + installedVersion: "1.5.7", + }); + expect(restarted.requests.filter((url) => url.includes("storage.googleapis.com"))).toEqual( + [], + ); + expect((yield* restarted.installation.resolve()).executablePath).toBe( + resolved.executablePath, + ); + }), + ); + + // These tar fixtures exercise the POSIX extraction path; Windows downloads one executable. + it.effect.skipIf(hostPlatform === "win32")( + "rejects unsafe native tar members before activation", + () => + Effect.gen(function* () { + for (const archive of [ + "H4sIAAAAAAACE+3OMQqDQBAFUI/iCYxZJPdZMIRtFHQN5PYOKbW2EN5j4P9iYCZPtXyW/C3111ylD69h+Gc45rk/U4r1NnXdI2be6lrG90XPbWvNS5xvAAAAAAAAAAAA4D523kNDtgAoAAA=", + "H4sIAAAAAAACE+3RMQ7CMBAEQNe8Ii8IBhzxHhcIpYDCcSLxewwNiJ4IiZlmb6+54vp+m691PJe8jPUWviI2x2F4ZvOZMe7Ta37sdzGlQ+hiWME81Vza+fCf3p7flbmVy2kTAAAAAAAAAAAA+Hl3jhn2/wAoAAA=", + "H4sIAAAAAAACE+3TPQrCQBAG0K09RY4wRoPn2UJCClOsG8Hbu6YJ2CtI3mvmp5li+PJcp7Hkx1Sf6VuiuQzDWpvPGtGft/69P0Z/itRF+oHlXnNp59M+5e3/XVnacLseEnv8v/zLv/wDAAAAAAAAAPyZF/c+VksAKAAA", + ]) { + const { installation, validations } = yield* makeHarness({ + native: true, + archive: Buffer.from(archive, "base64"), + platform: "linux", + previous: true, + }); + yield* installation.start; + expect((yield* terminalState(installation)).phase).toBe("failed"); + expect(validations).toEqual([]); + yield* expectPreviousRelease(installation); + } + }), + ); + + it.effect.each([true, false])( + "respects background update checks (%s) while allowing explicit installation", + (enabled) => + Effect.gen(function* () { + const { installation, requests } = yield* makeHarness({ + native: true, + archive: Buffer.from(serverContents), + platform: "win32", + updateChecksEnabled: Effect.succeed(enabled), + }); + yield* installation.checkForUpdate; + if (enabled) { + yield* installation.changes.pipe( + Stream.filter((value) => value.version === "1.5.7"), + Stream.runHead, + ); + yield* installation.state; + expect(requests).toHaveLength(1); + } else { + yield* installation.changes.pipe(Stream.runHead); + expect(requests).toEqual([]); + } + yield* installation.start; + expect((yield* terminalState(installation)).phase).toBe("succeeded"); + }), + ); + + it.effect("keeps withdrawn native releases from activating or launching", () => + Effect.gen(function* () { + let withdrawn = false; + const manifest = Effect.sync(() => + withdrawn + ? ({ + ...ModelManifest.BUNDLED_MODEL_MANIFEST, + compatibility: [ + { + driver: "antigravity", + t3CodeRange: ">=0.0.42", + ranges: [{ range: ">=1.5.7", status: "broken" }], + }, + ], + } satisfies ModelManifest.ModelManifestData) + : ModelManifest.BUNDLED_MODEL_MANIFEST, + ); + const first = yield* makeHarness({ + native: true, + archive: Buffer.from(serverContents), + platform: "win32", + previous: true, + manifest, + validate: () => + Effect.sync(() => { + withdrawn = true; + }), + }); + yield* first.installation.start; + expect((yield* terminalState(first.installation)).phase).toBe("failed"); + yield* expectPreviousRelease(first.installation); + withdrawn = false; + const retry = yield* makeHarness({ + baseDir: first.baseDir, + native: true, + archive: Buffer.from(serverContents), + platform: "win32", + manifest, + }); + yield* retry.installation.start; + expect((yield* terminalState(retry.installation)).phase).toBe("succeeded"); + withdrawn = true; + expect(yield* retry.installation.resolve().pipe(Effect.flip)).toMatchObject({ + operation: "resolve", + }); + expect(yield* retry.installation.start.pipe(Effect.flip)).toMatchObject({ + operation: "start", + }); + const recovery = yield* makeHarness({ + baseDir: first.baseDir, + native: true, + nativeVersion: "1.5.6", + archive: Buffer.from("recovery runtime"), + platform: "win32", + manifest, + }); + yield* recovery.installation.checkForUpdate; + expect(yield* recovery.installation.state).toMatchObject({ + installedVersion: "1.5.7", + version: "1.5.6", + }); + yield* recovery.installation.start; + expect(yield* terminalState(recovery.installation)).toMatchObject({ + phase: "succeeded", + installedVersion: "1.5.6", + }); + }), + ); + + it.effect("discovers subsequent native releases from Google's CLI channel", () => + Effect.gen(function* () { + const first = yield* makeHarness({ + native: true, + archive: Buffer.from(serverContents), + platform: "win32", + }); + yield* first.installation.start; + expect((yield* terminalState(first.installation)).phase).toBe("succeeded"); + const initialPath = (yield* first.installation.resolve()).executablePath; + const next = yield* makeHarness({ + baseDir: first.baseDir, + native: true, + nativeVersion: "1.5.8", + archive: Buffer.from("next runtime"), + platform: "win32", + }); + yield* next.installation.start; + expect(yield* terminalState(next.installation)).toMatchObject({ + phase: "succeeded", + installedVersion: "1.5.8", + }); + const nextPath = (yield* next.installation.resolve()).executablePath; + expect(nextPath).not.toBe(initialPath); + const republished = yield* makeHarness({ + baseDir: first.baseDir, + native: true, + nativeVersion: "1.5.8", + archive: Buffer.from("republished runtime"), + platform: "win32", + }); + yield* republished.installation.start; + expect((yield* terminalState(republished.installation)).phase).toBe("succeeded"); + expect((yield* republished.installation.resolve()).executablePath).not.toBe(nextPath); + const offline = yield* makeHarness({ + baseDir: first.baseDir, + native: true, + nativeManifest: () => ({}), + archive: Buffer.from("unused"), + platform: "win32", + }); + yield* offline.installation.start; + expect(yield* terminalState(offline.installation)).toMatchObject({ + phase: "succeeded", + installedVersion: "1.5.8", + }); + expect(offline.requests.filter((url) => url.includes("storage.googleapis.com"))).toEqual([]); + }), + ); + + it.effect.each([ + { + name: "invalid checksum", + change: (manifest: { version: string; url: string; sha512: string }) => ({ + ...manifest, + sha512: "0".repeat(128), + }), + }, + { + name: "foreign download", + change: (manifest: { version: string; url: string; sha512: string }) => ({ + ...manifest, + url: "https://example.com/agy.exe", + }), + }, + ])("rejects a native $name and preserves the legacy release", ({ change }) => + Effect.gen(function* () { + const { installation, validations } = yield* makeHarness({ + native: true, + nativeManifest: change, + archive: Buffer.from(serverContents), + platform: "win32", + previous: true, + }); + const result = yield* Effect.result(installation.start); + if (result._tag === "Success") + expect((yield* terminalState(installation)).phase).toBe("failed"); + expect(validations).toEqual([]); + yield* expectPreviousRelease(installation); + }), + ); + it.effect("reports missing Node before downloading the standalone provider runtime", () => Effect.gen(function* () { const { installation, requests, validations } = yield* makeHarness(); @@ -768,10 +1052,11 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("Antigravity inst const windowsDirectory = path.join(baseDir, `windows-${name}`); yield* fs.makeDirectory(windowsDirectory); yield* fs.writeFileString(path.join(windowsDirectory, name), "cli"); - expect(yield* windows.resolve("agy", { PATH: windowsDirectory })).toMatchObject({ - executablePath: path.join(windowsDirectory, name), - harnessPath: null, - }); + const selected = yield* windows.resolve("agy", { PATH: windowsDirectory }); + expect(selected.harnessPath).toBeNull(); + expect((yield* fs.realPath(selected.executablePath)).toLowerCase()).toBe( + (yield* fs.realPath(path.join(windowsDirectory, name))).toLowerCase(), + ); } }), ); @@ -810,6 +1095,11 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("Antigravity inst expect(yield* installation.resolve(executableName)).toMatchObject({ source: "override", }); + expect( + yield* installation + .resolve(`~/external/${executableName}`) + .pipe(Effect.provideService(HostProcess.HomeDirectory, baseDir)), + ).toMatchObject({ source: "override", executablePath: externalExecutable }); yield* fs.remove(externalHarness); expect(yield* installation.resolve(externalExecutable).pipe(Effect.flip)).toMatchObject({ operation: "resolve", @@ -859,14 +1149,29 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("Antigravity inst it.effect("keeps leased releases available while new sessions resolve the new release", () => Effect.gen(function* () { - const { installation, fs, stagingReleased } = yield* makeHarness({ previous: true }); + const manifestEntered = yield* Deferred.make(); + const releaseManifest = yield* Deferred.make(); + const { installation, fs, stagingReleased } = yield* makeHarness({ + native: true, + platform: "win32", + archive: Buffer.from(serverContents), + previous: true, + beforeManifest: Deferred.succeed(manifestEntered, undefined).pipe( + Effect.andThen(Deferred.await(releaseManifest)), + ), + }); const processScope = yield* Scope.make(); const oldExecutable = yield* installation.acquire().pipe(Scope.provide(processScope)); - yield* installation.start; + const starting = yield* installation.start.pipe(Effect.forkScoped); + yield* Deferred.await(manifestEntered); + const concurrent = yield* installation.acquire().pipe(Scope.provide(processScope)); + expect(concurrent.executablePath).toBe(oldExecutable.executablePath); + yield* Deferred.succeed(releaseManifest, undefined); + yield* Fiber.join(starting); expect((yield* terminalState(installation)).phase).toBe("succeeded"); yield* Deferred.await(stagingReleased); const current = yield* installation.resolve(); - expect(current.version).toBe("fixture-new"); + expect(current.version).toBe("1.5.7"); expect(current.executablePath).not.toBe(oldExecutable.executablePath); expect(yield* fs.readFileString(oldExecutable.executablePath)).toBe(serverContents); expect(yield* installation.remove().pipe(Effect.flip)).toMatchObject({ operation: "remove" }); diff --git a/apps/server/src/provider/AntigravityInstallation.ts b/apps/server/src/provider/AntigravityInstallation.ts index 74b5cbf981b9..feb412867f2e 100644 --- a/apps/server/src/provider/AntigravityInstallation.ts +++ b/apps/server/src/provider/AntigravityInstallation.ts @@ -1,4 +1,9 @@ // @effect-diagnostics nodeBuiltinImport:off - Effect has no incremental digest or free-space query. +import { expandHomePath } from "@t3tools/provider-core/server/pathExpansion"; +import * as ModelManifest from "./ModelManifest.ts"; +import { resolveProviderCompatibility } from "./providerCompatibility.ts"; +import { cliArchiveTarCommand } from "@t3tools/shared/cliRelease"; +import { compareSemverVersions } from "@t3tools/shared/semver"; import * as EffectNodeStream from "@effect/platform-node/NodeStream"; import { ProviderDriverKind, type ProviderInstallState } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; @@ -29,20 +34,27 @@ import * as NodeFSP from "node:fs/promises"; import * as EffectAcpErrors from "effect-acp/errors"; import * as ServerConfig from "../config.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import { + ANTIGRAVITY_CLI_MANIFEST_BASE, + antigravityCliTarget, + parseAntigravityCliRelease, + type AntigravityCliReleaseAsset, +} from "./antigravityCliRelease.ts"; import { openZipArchive } from "../zipArchive.ts"; import { makeAntigravityAcpRuntime } from "./acp/AntigravityAcpSupport.ts"; import { buildAntigravityAcpSpawnInput, prepareAntigravityProfile, } from "./antigravityAuthSupport.ts"; -import { - resolveAntigravityReleaseAsset, - type AntigravityReleaseAsset, -} from "./antigravityRelease.ts"; +import { type AntigravityReleaseAsset } from "./antigravityRelease.ts"; const DRIVER = ProviderDriverKind.make("antigravity"); const DOWNLOAD_TIMEOUT = "45 minutes"; const VALIDATION_TIMEOUT = "90 seconds"; +const MAX_DOWNLOAD_BYTES = 512 * 1024 * 1024; +const MAX_EXECUTABLE_BYTES = 512 * 1024 * 1024; +const UPDATE_CHECK_INTERVAL = 60 * 60 * 1000; const FREE_SPACE_MARGIN = 256 * 1024 * 1024; const RECORD_MAX_BYTES = 8 * 1024; const RELEASE_RECORD = ".install-complete.json"; @@ -55,7 +67,7 @@ const InstalledRelease = Schema.Struct({ releaseId: ReleaseId, version: Schema.String, executable: Schema.Struct({ name: Schema.String, bytes: Schema.Number }), - harness: Schema.Struct({ name: Schema.String, bytes: Schema.Number }), + harness: Schema.NullOr(Schema.Struct({ name: Schema.String, bytes: Schema.Number })), }); type InstalledRelease = typeof InstalledRelease.Type; const encodeActiveRelease = Schema.encodeEffect(Schema.fromJsonString(ActiveRelease)); @@ -102,6 +114,7 @@ interface AntigravityInstallationService { ) => Effect.Effect; readonly state: Effect.Effect; readonly changes: Stream.Stream; + readonly checkForUpdate: Effect.Effect; readonly remove: ( protectedBinaryPaths?: ReadonlyArray, ) => Effect.Effect; @@ -115,14 +128,22 @@ export class AntigravityInstallation extends Context.Service< AntigravityInstallation, Effect.gen(function* () { const config = yield* ServerConfig.ServerConfig; - return yield* makeAntigravityInstallation({ baseDir: config.baseDir }); + const settings = yield* ServerSettings.ServerSettingsService; + return yield* makeAntigravityInstallation({ + baseDir: config.baseDir, + updateChecksEnabled: settings.getSettings.pipe( + Effect.map((value) => value.enableProviderUpdateChecks), + Effect.orElseSucceed(() => false), + ), + }); }), ); } export interface AntigravityInstallationOptions { readonly baseDir: string; - readonly releaseAsset?: AntigravityReleaseAsset | null; + readonly releaseAsset?: AntigravityReleaseAsset | AntigravityCliReleaseAsset | null; + readonly updateChecksEnabled?: Effect.Effect; readonly validate?: ( executable: AntigravityExecutable, expectedVersion: string, @@ -161,6 +182,7 @@ function isRunning(state: ProviderInstallState) { export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.make")(function* ( options: AntigravityInstallationOptions, ) { + const manifestService = yield* ModelManifest.ModelManifest; const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const random = yield* SecureRandom.SecureRandom; @@ -170,10 +192,11 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma const platform = yield* HostProcess.Platform; const arch = yield* HostProcess.Architecture; const environment = yield* HostProcess.Environment; - const releaseAsset = + let releaseAsset = options.releaseAsset ?? null; + const supported = options.releaseAsset === undefined - ? resolveAntigravityReleaseAsset(platform, arch) - : options.releaseAsset; + ? antigravityCliTarget(platform, arch) !== null + : options.releaseAsset !== null; const names = executableNames(platform); const managedDirectory = path.join( options.baseDir, @@ -191,13 +214,36 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma operationId: null, phase: "idle", downloadedBytes: 0, - totalBytes: releaseAsset?.archiveBytes ?? null, + totalBytes: + releaseAsset !== null && "archiveBytes" in releaseAsset ? releaseAsset.archiveBytes : null, version: releaseAsset?.version ?? null, installedVersion: null, canRemove: false, message: null, }); + const isWithdrawn = Effect.fn("AntigravityInstallation.isWithdrawn")(function* (version: string) { + const manifest = yield* manifestService.current; + const status = ( + resolveProviderCompatibility(manifest.compatibility, DRIVER, version) ?? + resolveProviderCompatibility( + ModelManifest.BUNDLED_MODEL_MANIFEST.compatibility, + DRIVER, + version, + ) + )?.status; + return status === "broken" || status === "unsupported"; + }); + const ensureEligible = Effect.fn("AntigravityInstallation.ensureEligible")(function* ( + version: string, + ) { + if (yield* isWithdrawn(version)) + return yield* installationError( + "verify", + "This Antigravity release was withdrawn. Check for updates and try again.", + ); + }); + const readRecord = Effect.fn("AntigravityInstallation.readRecord")(function* ( filePath: string, schema: Schema.Codec, @@ -233,15 +279,20 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma const record = yield* readRecord(path.join(directory, RELEASE_RECORD), InstalledRelease); if ( record.releaseId !== releaseId || - record.executable.name !== names.executable || - record.harness.name !== names.harness || + (record.harness === null + ? record.executable.name !== (platform === "win32" ? "agy.exe" : "antigravity") + : record.executable.name !== names.executable || record.harness.name !== names.harness) || !Number.isSafeInteger(record.executable.bytes) || record.executable.bytes <= 0 || - !Number.isSafeInteger(record.harness.bytes) || - record.harness.bytes <= 0 || + (record.harness !== null && + (!Number.isSafeInteger(record.harness.bytes) || record.harness.bytes <= 0)) || !record.version.trim() || - !(yield* executableFile(path.join(directory, names.executable), record.executable.bytes)) || - !(yield* executableFile(path.join(directory, names.harness), record.harness.bytes)) + !(yield* executableFile( + path.join(directory, record.executable.name), + record.executable.bytes, + )) || + (record.harness !== null && + !(yield* executableFile(path.join(directory, record.harness.name), record.harness.bytes))) ) { return yield* installationError( "resolve", @@ -249,8 +300,8 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma ); } return { - executablePath: path.join(directory, names.executable), - harnessPath: path.join(directory, names.harness), + executablePath: path.join(directory, record.executable.name), + harnessPath: record.harness === null ? null : path.join(directory, record.harness.name), source: "managed", version: record.version, managedVersionDirectory: directory, @@ -263,6 +314,16 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma ) { if (!(yield* executableFile(candidate))) return null; const executablePath = yield* fs.realPath(candidate); + const realVersions = yield* fs.realPath(versionsDirectory).pipe(Effect.option); + const directory = path.dirname(executablePath); + if ( + Option.isSome(realVersions) && + path.dirname(directory) === realVersions.value && + /^[a-f0-9]{64}$/u.test(path.basename(directory)) + ) { + const installed = yield* completedRelease(path.basename(directory)); + return { ...installed, source }; + } // Keep launcher symlinks intact: version-manager shims dispatch by their invoked name. if (isAntigravityCli(candidate, executablePath)) { return { @@ -273,18 +334,8 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma managedVersionDirectory: null, } satisfies AntigravityExecutable; } - const directory = path.dirname(executablePath); const harnessPath = path.join(directory, names.harness); if (!(yield* executableFile(harnessPath))) return null; - const realVersions = yield* fs.realPath(versionsDirectory).pipe(Effect.option); - if ( - Option.isSome(realVersions) && - path.dirname(directory) === realVersions.value && - /^[a-f0-9]{64}$/u.test(path.basename(directory)) - ) { - const installed = yield* completedRelease(path.basename(directory)); - return { ...installed, executablePath, harnessPath, source } satisfies AntigravityExecutable; - } return { executablePath, harnessPath, @@ -329,7 +380,7 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma "AntigravityInstallation.resolve", )( function* (binaryPath?: string, processEnvironment?: NodeJS.ProcessEnv) { - const override = binaryPath?.trim(); + const override = expandHomePath(binaryPath?.trim() ?? "", yield* HostProcess.HomeDirectory); if (override) { for (const candidate of yield* commandCandidates(override, processEnvironment)) { const selected = yield* fromExternal(candidate, "override"); @@ -342,7 +393,13 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma } if (yield* fs.exists(activePath)) { const active = yield* readRecord(activePath, ActiveRelease); - return yield* completedRelease(active.releaseId); + const installed = yield* completedRelease(active.releaseId); + if (yield* isWithdrawn(installed.version)) + return yield* installationError( + "resolve", + `Antigravity ${installed.version} no longer works with this T3 Code release. Update it in Antigravity setup.`, + ); + return installed; } for (const candidate of yield* commandCandidates(names.executable, processEnvironment)) { const selected = yield* fromExternal(candidate, "path"); @@ -350,7 +407,7 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma } return yield* installationError( "resolve", - releaseAsset + supported ? "Antigravity ACP runtime is not installed. Install it in this environment's provider settings or set a custom ACP executable path. The Antigravity app, IDE, and CLI are separate installations." : `Google does not publish an Antigravity runtime for ${platform}-${arch}. Use a supported environment or a custom executable.`, ); @@ -439,8 +496,10 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma }); const initialized = yield* runtime.initialize(); if ( - initialized.agentInfo?.name !== "antigravity-acp" || - initialized.agentInfo.version !== expectedVersion || + (executable.harnessPath === null + ? initialized.agentInfo?.name !== "Antigravity ACP Server" + : initialized.agentInfo?.name !== "antigravity-acp") || + initialized.agentInfo?.version !== expectedVersion || // Antigravity 1.1.1 may report protocol 2 with legacy ACP; choose wire format from the response shape. (initialized.protocolVersion !== 1 && initialized.protocolVersion !== 2) || initialized.agentCapabilities?.loadSession !== true || @@ -471,7 +530,12 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma ); const install = Effect.fn("AntigravityInstallation.install")( - function* (asset: AntigravityReleaseAsset) { + function* (asset: AntigravityReleaseAsset | AntigravityCliReleaseAsset) { + yield* ensureEligible(asset.version); + const cli = "sha512" in asset; + // Existing installation records use 64-character IDs, including native SHA-512 prefixes. + const releaseId = cli ? asset.sha512.slice(0, 64) : asset.sha256; + const archiveBytes = cli ? null : asset.archiveBytes; yield* resolveNodeExecutable("Antigravity", environment).pipe( Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, path), @@ -480,7 +544,7 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma installationError("verify", nodeRuntimeUnavailableMessage("Antigravity"), cause), ), ); - if (platform === "linux" && arch === "arm64") { + if (!cli && platform === "linux" && arch === "arm64") { const pageSize = yield* spawner .string(ChildProcess.make("getconf", ["PAGESIZE"], { stdin: "ignore", stderr: "ignore" })) .pipe( @@ -496,19 +560,19 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma SubscriptionRef.update(state, (current) => ({ ...current, phase, message })); yield* fs.makeDirectory(versionsDirectory, { recursive: true }); yield* SubscriptionRef.update(state, (current) => ({ ...current, canRemove: true })); - const destination = path.join(versionsDirectory, asset.sha256); + const destination = path.join(versionsDirectory, releaseId); const activate = Effect.fn("AntigravityInstallation.activate")( function* () { + yield* ensureEligible(asset.version); const pointerDirectory = yield* fs.makeTempDirectoryScoped({ directory: managedDirectory, prefix: "active.json.", }); const pointerPath = path.join(pointerDirectory, "contents.tmp"); - yield* fs.writeFileString( - pointerPath, - yield* encodeActiveRelease({ releaseId: asset.sha256 }), - { flag: "wx", mode: 0o600 }, - ); + yield* fs.writeFileString(pointerPath, yield* encodeActiveRelease({ releaseId }), { + flag: "wx", + mode: 0o600, + }); yield* fs.rename(pointerPath, activePath); yield* SubscriptionRef.update( state, @@ -532,7 +596,7 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma ); if (yield* fs.exists(destination)) { - const existing = yield* completedRelease(asset.sha256); + const existing = yield* completedRelease(releaseId); if (existing.version !== asset.version) { return yield* installationError( "verify", @@ -552,8 +616,9 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma try: () => NodeFSP.statfs(versionsDirectory, { bigint: true }), catch: (cause) => installationError("download", "Could not check free disk space.", cause), }).pipe(Effect.option); - const required = - asset.archiveBytes + asset.executable.bytes + asset.harness.bytes + FREE_SPACE_MARGIN; + const required = cli + ? MAX_DOWNLOAD_BYTES + MAX_EXECUTABLE_BYTES + FREE_SPACE_MARGIN + : asset.archiveBytes + asset.executable.bytes + asset.harness.bytes + FREE_SPACE_MARGIN; if ( Option.isSome(available) && available.value.bavail * available.value.bsize < BigInt(required) @@ -570,7 +635,7 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma const archivePath = path.join(staging, "download.zip"); const pairDirectory = path.join(staging, "runtime"); yield* fs.makeDirectory(pairDirectory); - const hash = NodeCrypto.createHash("sha256"); + const hash = NodeCrypto.createHash(cli ? "sha512" : "sha256"); let downloadedBytes = 0; let lastProgressAt = yield* Clock.currentTimeMillis; yield* Effect.gen(function* () { @@ -584,7 +649,8 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma if ( identityBody && contentLength !== undefined && - Number(contentLength) !== asset.archiveBytes + archiveBytes !== null && + Number(contentLength) !== archiveBytes ) { return yield* installationError( "download", @@ -595,15 +661,15 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma Stream.tap((chunk) => Effect.gen(function* () { downloadedBytes += chunk.byteLength; - if (downloadedBytes > asset.archiveBytes) { + if (downloadedBytes > (archiveBytes ?? MAX_DOWNLOAD_BYTES)) { return yield* installationError( "download", - "The Antigravity download exceeded the pinned release size.", + "The Antigravity download exceeded its allowed size.", ); } hash.update(chunk); const now = yield* Clock.currentTimeMillis; - if (now - lastProgressAt >= 250 || downloadedBytes === asset.archiveBytes) { + if (now - lastProgressAt >= 250 || downloadedBytes === archiveBytes) { lastProgressAt = now; yield* SubscriptionRef.update(state, (current) => ({ ...current, @@ -626,98 +692,154 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma : cause, ), ); - if (downloadedBytes !== asset.archiveBytes || hash.digest("hex") !== asset.sha256) { + if ( + (archiveBytes !== null && downloadedBytes !== archiveBytes) || + hash.digest("hex") !== (cli ? asset.sha512 : asset.sha256) + ) { return yield* installationError( "download", - "The Antigravity download failed its size or SHA-256 check. Nothing was installed.", + "The Antigravity download failed its size or checksum check. Nothing was installed.", ); } yield* report("extracting", "Extracting the verified runtime."); - yield* Effect.gen(function* () { - const archive = yield* openZipArchive(archivePath, (detail, cause) => - installationError("extract", detail, cause), - ); - if (archive.entryCount !== 2) { - return yield* installationError( - "extract", - "The archive must contain exactly the Antigravity executable and its harness.", - ); - } - const seen = new Set(); - for (;;) { - const entry = yield* archive.next; - if (!entry) break; - const expected = [asset.executable, asset.harness].find( - (file) => file.name === entry.fileName, - ); - const unixType = (entry.externalFileAttributes >>> 16) & 0o170000; + if (cli) { + const executablePath = path.join(pairDirectory, asset.executable.name); + if (asset.format === "executable") { + yield* fs.copyFile(archivePath, executablePath); + } else { + const tar = cliArchiveTarCommand(platform, environment); + const list = yield* spawner.string(ChildProcess.make(tar, ["-tzf", archivePath])); + const types = yield* spawner.string(ChildProcess.make(tar, ["-tvzf", archivePath])); if ( - !expected || - seen.has(entry.fileName) || - entry.fileName.includes("/") || - entry.fileName.includes("\\") || - (unixType !== 0 && unixType !== 0o100000) || - (entry.externalFileAttributes & 0x10) !== 0 || - (entry.generalPurposeBitFlag & 1) !== 0 || - ![0, 8].includes(entry.compressionMethod) || - entry.uncompressedSize !== expected.bytes + list.trim() !== asset.executable.name || + types.trim().split(/\r?\n/u).length !== 1 || + !types.startsWith("-") ) { return yield* installationError( "extract", - "The archive contains an unexpected, unsafe, or incorrectly sized member.", + "The Antigravity archive must contain one regular executable.", ); } - seen.add(entry.fileName); yield* Effect.gen(function* () { - const readable = yield* archive.streamEntry(entry); + const child = yield* spawner.spawn( + ChildProcess.make(tar, ["-xOzf", archivePath, asset.executable.name], { + stdout: "pipe", + stderr: "pipe", + }), + ); let extractedBytes = 0; - yield* EffectNodeStream.fromReadable({ - evaluate: () => readable, - onError: wrapFailure("extract", "Could not extract the Antigravity runtime."), - }).pipe( + yield* child.stdout.pipe( Stream.tap((chunk) => Effect.gen(function* () { extractedBytes += chunk.byteLength; - if (extractedBytes > expected.bytes) { + if (extractedBytes > MAX_EXECUTABLE_BYTES) return yield* installationError( "extract", - "An archive member exceeded its pinned size.", + "The Antigravity executable exceeded its size limit.", ); - } }), ), - Stream.run( - fs.sink(path.join(pairDirectory, entry.fileName), { flag: "wx", mode: 0o700 }), - ), + Stream.run(fs.sink(executablePath, { flag: "wx", mode: 0o700 })), ); - if (extractedBytes !== expected.bytes) { - return yield* installationError("extract", "An archive member was truncated."); - } + if ((yield* child.exitCode) !== 0 || extractedBytes === 0) + return yield* installationError( + "extract", + "Could not extract the Antigravity executable.", + ); }).pipe(Effect.scoped); } - if (!seen.has(asset.executable.name) || !seen.has(asset.harness.name)) { - return yield* installationError( - "extract", - "The archive is missing the Antigravity executable or its harness.", + } else + yield* Effect.gen(function* () { + const archive = yield* openZipArchive(archivePath, (detail, cause) => + installationError("extract", detail, cause), ); - } - }).pipe(Effect.scoped); + if (archive.entryCount !== 2) { + return yield* installationError( + "extract", + "The archive must contain exactly the Antigravity executable and its harness.", + ); + } + const seen = new Set(); + for (;;) { + const entry = yield* archive.next; + if (!entry) break; + const expected = [asset.executable, asset.harness].find( + (file) => file.name === entry.fileName, + ); + const unixType = (entry.externalFileAttributes >>> 16) & 0o170000; + if ( + !expected || + seen.has(entry.fileName) || + entry.fileName.includes("/") || + entry.fileName.includes("\\") || + (unixType !== 0 && unixType !== 0o100000) || + (entry.externalFileAttributes & 0x10) !== 0 || + (entry.generalPurposeBitFlag & 1) !== 0 || + ![0, 8].includes(entry.compressionMethod) || + entry.uncompressedSize !== expected.bytes + ) { + return yield* installationError( + "extract", + "The archive contains an unexpected, unsafe, or incorrectly sized member.", + ); + } + seen.add(entry.fileName); + yield* Effect.gen(function* () { + const readable = yield* archive.streamEntry(entry); + let extractedBytes = 0; + yield* EffectNodeStream.fromReadable({ + evaluate: () => readable, + onError: wrapFailure("extract", "Could not extract the Antigravity runtime."), + }).pipe( + Stream.tap((chunk) => + Effect.gen(function* () { + extractedBytes += chunk.byteLength; + if (extractedBytes > expected.bytes) { + return yield* installationError( + "extract", + "An archive member exceeded its pinned size.", + ); + } + }), + ), + Stream.run( + fs.sink(path.join(pairDirectory, entry.fileName), { flag: "wx", mode: 0o700 }), + ), + ); + if (extractedBytes !== expected.bytes) { + return yield* installationError("extract", "An archive member was truncated."); + } + }).pipe(Effect.scoped); + } + if (!seen.has(asset.executable.name) || !seen.has(asset.harness.name)) { + return yield* installationError( + "extract", + "The archive is missing the Antigravity executable or its harness.", + ); + } + }).pipe(Effect.scoped); yield* fs.remove(archivePath); const executablePath = path.join(pairDirectory, asset.executable.name); - const harnessPath = path.join(pairDirectory, asset.harness.name); + const harnessPath = cli ? null : path.join(pairDirectory, asset.harness.name); if (platform !== "win32") { yield* fs.chmod(executablePath, 0o755); - yield* fs.chmod(harnessPath, 0o755); + if (harnessPath !== null) yield* fs.chmod(harnessPath, 0o755); } - if (platform === "darwin" && arch === "x64") { + if (!cli && platform === "darwin" && arch === "x64") { // Google signs the Intel build with the hardened runtime but no JIT entitlement, so // libffi's writable+executable trampolines spin forever. Re-sign ad hoc after the hash check. const exitCode = yield* spawner .spawn( ChildProcess.make( "/usr/bin/codesign", - ["--force", "--sign", "-", executablePath, harnessPath], + [ + "--force", + "--sign", + "-", + executablePath, + ...(harnessPath === null ? [] : [harnessPath]), + ], { stdout: "ignore", stderr: "ignore" }, ), ) @@ -744,13 +866,16 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma asset.version, ).pipe(Effect.scoped, Effect.timeout(VALIDATION_TIMEOUT)); const record: InstalledRelease = { - releaseId: asset.sha256, + releaseId, version: asset.version, executable: { name: asset.executable.name, bytes: Number((yield* fs.stat(executablePath)).size), }, - harness: { name: asset.harness.name, bytes: Number((yield* fs.stat(harnessPath)).size) }, + harness: + cli || harnessPath === null + ? null + : { name: asset.harness.name, bytes: Number((yield* fs.stat(harnessPath)).size) }, }; yield* fs.writeFileString( path.join(pairDirectory, RELEASE_RECORD), @@ -759,7 +884,7 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma ); yield* fs.rename(pairDirectory, destination).pipe( Effect.catch((cause) => - completedRelease(asset.sha256).pipe( + completedRelease(releaseId).pipe( Effect.flatMap((existing) => existing.version === asset.version ? validate(existing, asset.version).pipe( @@ -794,31 +919,119 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma ), ); - const start = gate - .withPermit( + let lastUpdateCheck: number | undefined; + const refreshRelease = Effect.fn("AntigravityInstallation.refreshRelease")(function* ( + explicit = false, + ) { + if (options.releaseAsset !== undefined || !supported) return; + if (!explicit && !(yield* options.updateChecksEnabled ?? Effect.succeed(true))) return; + const now = yield* Clock.currentTimeMillis; + if (!explicit && lastUpdateCheck !== undefined && now - lastUpdateCheck < UPDATE_CHECK_INTERVAL) + return; + lastUpdateCheck = now; + const musl = + platform === "linux" && + (yield* fs.exists( + arch === "arm64" ? "/lib/libc.musl-aarch64.so.1" : "/lib/libc.musl-x86_64.so.1", + )); + const target = antigravityCliTarget(platform, arch, musl); + const latest = yield* http.get(`${ANTIGRAVITY_CLI_MANIFEST_BASE}/${target}.json`).pipe( + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap((response) => response.json), + Effect.map((json) => parseAntigravityCliRelease(json, platform, arch, musl)), + Effect.timeout("10 seconds"), + Effect.orElseSucceed(() => null), + ); + if (latest === null) return; + if (yield* isWithdrawn(latest.version)) { + releaseAsset = null; + return; + } + releaseAsset = latest; + yield* SubscriptionRef.updateEffect(state, (value) => + isRunning(value) + ? Effect.succeed(value) + : Effect.gen(function* () { + const keepInstalled = + value.installedVersion !== null && + compareSemverVersions(value.installedVersion, latest.version) > 0 && + !(yield* isWithdrawn(value.installedVersion)); + return { + ...value, + version: keepInstalled ? value.installedVersion : latest.version, + totalBytes: null, + }; + }), + ); + }); + + const start = Effect.gen(function* () { + const current = yield* SubscriptionRef.get(state); + if (isRunning(current)) return current; + yield* refreshRelease(true); + return yield* gate.withPermit( Effect.gen(function* () { const current = yield* SubscriptionRef.get(state); if (isRunning(current)) return current; - if (!releaseAsset) { + if (!supported) { return yield* installationError( "start", `Google does not publish an Antigravity runtime for ${platform}-${arch}. Use a supported remote environment or a custom executable.`, ); } + const asset = + releaseAsset !== null && !(yield* isWithdrawn(releaseAsset.version)) + ? releaseAsset + : null; + const installed = yield* resolve().pipe(Effect.option); + const reuse = + Option.isSome(installed) && + installed.value.source === "managed" && + installed.value.harnessPath === null && + (asset === null || compareSemverVersions(installed.value.version!, asset.version) > 0) + ? installed.value + : undefined; + if (asset === null && reuse === undefined) + return yield* installationError( + "start", + "Could not find a native Antigravity CLI release. Check the network connection and try again.", + ); const operationId = yield* random.uuidV4(); const next: ProviderInstallState = { driver: DRIVER, operationId, - phase: "downloading", + phase: reuse === undefined ? "downloading" : "verifying", downloadedBytes: 0, - totalBytes: releaseAsset.archiveBytes, - version: releaseAsset.version, + totalBytes: asset !== null && "archiveBytes" in asset ? asset.archiveBytes : null, + version: reuse?.version ?? asset?.version ?? null, installedVersion: current.installedVersion, canRemove: current.canRemove, - message: "Downloading Google's official Antigravity runtime.", + message: + reuse === undefined + ? "Downloading Google's official Antigravity runtime." + : "Checking the installed runtime.", }; yield* SubscriptionRef.set(state, next); - const work = install(releaseAsset).pipe( + const work = ( + reuse === undefined + ? install(asset!) + : validate(reuse, reuse.version!).pipe( + Effect.scoped, + Effect.timeout(VALIDATION_TIMEOUT), + Effect.andThen(ensureEligible(reuse.version!)), + Effect.andThen( + SubscriptionRef.update(state, (value) => ({ + ...value, + phase: "succeeded" as const, + installedVersion: reuse.version, + message: null, + })), + ), + ) + ).pipe( + Effect.mapError( + wrapFailure("verify", "Could not verify the installed Antigravity runtime."), + ), Effect.onExit((exit) => Exit.isFailure(exit) ? SubscriptionRef.update(state, (value) => { @@ -849,8 +1062,8 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma running = { operationId, fiber }; return next; }).pipe(Effect.uninterruptible), - ) - .pipe(Effect.mapError(wrapFailure("start", "Could not start the Antigravity installation."))); + ); + }).pipe(Effect.mapError(wrapFailure("start", "Could not start the Antigravity installation."))); const cancel = Effect.fn("AntigravityInstallation.cancel")(function* (operationId: string) { return yield* gate.withPermit( @@ -958,6 +1171,7 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma cancel, state: SubscriptionRef.get(state), changes: SubscriptionRef.changes(state), + checkForUpdate: refreshRelease().pipe(Effect.ignore), remove, }); }); diff --git a/apps/server/src/provider/AntigravityProvider.test.ts b/apps/server/src/provider/AntigravityProvider.test.ts index 791ed748de2e..a67627382785 100644 --- a/apps/server/src/provider/AntigravityProvider.test.ts +++ b/apps/server/src/provider/AntigravityProvider.test.ts @@ -291,561 +291,519 @@ describe("Antigravity model catalog", () => { it.layer(layerTest)("Antigravity provider snapshots", (it) => { it.effect("does not probe or run helper safety checks while disabled", () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeHarness({ enabled: false }); - const snapshot = yield* harness.provider.snapshot.refresh; - expect(snapshot).toMatchObject({ - enabled: false, + Effect.gen(function* () { + const harness = yield* makeHarness({ enabled: false }); + const snapshot = yield* harness.provider.snapshot.refresh; + expect(snapshot).toMatchObject({ + enabled: false, + installed: false, + status: "disabled", + auth: { status: "unknown" }, + models: [], + setup: { canAuthenticate: true, canInstall: true }, + showInteractionModeToggle: false, + supportsConversationRollback: false, + supportsTextGeneration: false, + }); + expect(yield* Ref.get(harness.probeCalls)).toBe(0); + expect(yield* Ref.get(harness.safetyCalls)).toBe(0); + }), + ); + + it.effect("treats initialize as installation proof, not account or model discovery", () => + Effect.gen(function* () { + for (const auth of [ + undefined, + { type: "oauth-personal", label: "Google account" }, + { type: "oauth-business", label: "Gemini Enterprise" }, + { type: "gemini-api-key", label: "Gemini API key" }, + { type: "agent-platform", label: "Agent Platform" }, + ]) { + const harness = yield* makeHarness(auth ? { auth } : {}); + expect(yield* harness.provider.snapshot.getSnapshot).toMatchObject({ installed: false, - status: "disabled", + status: "warning", auth: { status: "unknown" }, models: [], - setup: { canAuthenticate: true, canInstall: true }, - showInteractionModeToggle: false, - supportsConversationRollback: false, - supportsTextGeneration: false, }); - expect(yield* Ref.get(harness.probeCalls)).toBe(0); - expect(yield* Ref.get(harness.safetyCalls)).toBe(0); - }), - ), - ); - - it.effect("treats initialize as installation proof, not account or model discovery", () => - Effect.scoped( - Effect.gen(function* () { - for (const auth of [ - undefined, - { type: "oauth-personal", label: "Google account" }, - { type: "oauth-business", label: "Gemini Enterprise" }, - { type: "gemini-api-key", label: "Gemini API key" }, - { type: "agent-platform", label: "Agent Platform" }, - ]) { - const harness = yield* makeHarness(auth ? { auth } : {}); - expect(yield* harness.provider.snapshot.getSnapshot).toMatchObject({ - installed: false, - status: "warning", - auth: { status: "unknown" }, - models: [], - }); - yield* harness.initialize; - const snapshot = yield* harness.provider.snapshot.getSnapshot; - expect(snapshot).toMatchObject({ - installed: true, - status: "warning", - version: "agy_acp_server_20260818_01_RC01", - auth: { status: "unknown", ...(auth ? { type: auth.type } : {}) }, - message: - auth?.type === "gemini-api-key" || auth?.type === "agent-platform" - ? `Antigravity is installed. Connect with ${auth.label} to load models.` - : "Antigravity is installed. Google account access is not checked yet.", - models: [], - }); - expect(yield* Ref.get(harness.probeCalls)).toBe(1); - } - }), - ), + yield* harness.initialize; + const snapshot = yield* harness.provider.snapshot.getSnapshot; + expect(snapshot).toMatchObject({ + installed: true, + status: "warning", + version: "agy_acp_server_20260818_01_RC01", + auth: { status: "unknown", ...(auth ? { type: auth.type } : {}) }, + message: + auth?.type === "gemini-api-key" || auth?.type === "agent-platform" + ? `Antigravity is installed. Connect with ${auth.label} to load models.` + : "Antigravity is installed. Google account access is not checked yet.", + models: [], + }); + expect(yield* Ref.get(harness.probeCalls)).toBe(1); + } + }), ); it.effect("does not offer Auto, so Auto threads fall back to Supervised", () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeHarness(); - yield* harness.initialize; - // Antigravity has no automatic review; the shared ACP policy would approve every request. - expect((yield* harness.provider.snapshot.getSnapshot).supportedRuntimeModes).toEqual([ - "approval-required", - "auto-accept-edits", - "full-access", - ]); - }), - ), + Effect.gen(function* () { + const harness = yield* makeHarness(); + yield* harness.initialize; + // Antigravity has no automatic review; the shared ACP policy would approve every request. + expect((yield* harness.provider.snapshot.getSnapshot).supportedRuntimeModes).toEqual([ + "approval-required", + "auto-accept-edits", + "full-access", + ]); + }), ); it.effect("publishes session metadata and native commands without another health probe", () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeHarness(); - yield* harness.initialize; - const nextReady = yield* Stream.toPull( - harness.provider.snapshot.streamChanges.pipe( - Stream.filter( - (snapshot) => - snapshot.auth.status === "authenticated" && snapshot.slashCommands.length > 0, - ), + Effect.gen(function* () { + const harness = yield* makeHarness(); + yield* harness.initialize; + const nextReady = yield* Stream.toPull( + harness.provider.snapshot.streamChanges.pipe( + Stream.filter( + (snapshot) => + snapshot.auth.status === "authenticated" && snapshot.slashCommands.length > 0, ), - ); - yield* harness.provider.onSessionStarted(started, "/workspace"); - yield* harness.provider.onAvailableCommands(commands, "/workspace"); - const [snapshot] = yield* nextReady; - expect(snapshot).toMatchObject({ - status: "ready", - auth: { status: "authenticated", type: "oauth-personal" }, - supportsTextGeneration: true, - }); - expect(snapshot.models).toEqual(buildAntigravityModelsFromSession(sessionSetupResult)); - expect(snapshot.slashCommands).toEqual(expectedCommands); - expect((yield* harness.provider.snapshotForCwd("/workspace")).slashCommands).toEqual( - expectedCommands, - ); - expect(yield* Ref.get(harness.probeCalls)).toBe(1); - }), - ), + ), + ); + yield* harness.provider.onSessionStarted(started, "/workspace"); + yield* harness.provider.onAvailableCommands(commands, "/workspace"); + const [snapshot] = yield* nextReady; + expect(snapshot).toMatchObject({ + status: "ready", + auth: { status: "authenticated", type: "oauth-personal" }, + supportsTextGeneration: true, + }); + expect(snapshot.models).toEqual(buildAntigravityModelsFromSession(sessionSetupResult)); + expect(snapshot.slashCommands).toEqual(expectedCommands); + expect((yield* harness.provider.snapshotForCwd("/workspace")).slashCommands).toEqual( + expectedCommands, + ); + expect(yield* Ref.get(harness.probeCalls)).toBe(1); + }), ); it.effect("clears all account metadata on sign-out and authentication failure", () => - Effect.scoped( - Effect.gen(function* () { - const auth = { type: "gemini-api-key", label: "Gemini API key" }; - const harness = yield* makeHarness({ auth }); - yield* harness.initialize; - for (const clear of [harness.provider.onSignedOut, harness.provider.onAuthRequired]) { - yield* harness.provider.onSessionStarted(started, "/workspace"); - yield* harness.provider.onAvailableCommands(commands, "/workspace"); - yield* clear; - expect(yield* harness.provider.snapshot.getSnapshot).toMatchObject({ - installed: true, - status: "warning", - auth: { status: "unauthenticated", type: auth.type, label: auth.label }, - message: "Connect with Gemini API key to use Antigravity and load models.", - models: [], - slashCommands: [], - skills: [], - workspaceSnapshots: [], - supportsTextGeneration: false, - }); - yield* harness.provider.onAvailableCommands(commands, "/workspace"); - yield* harness.provider.onConfigOptionsUpdated([modelConfig]); - expect((yield* harness.provider.snapshotForCwd("/workspace")).slashCommands).toEqual([]); - expect((yield* harness.provider.snapshot.getSnapshot).models).toEqual([]); - } - const refreshed = yield* harness.provider.snapshot.refresh; - expect(refreshed.auth).toEqual({ - status: "unauthenticated", - type: auth.type, - label: auth.label, - }); - expect(refreshed.message).toBe( - "Connect with Gemini API key to use Antigravity and load models.", - ); - expect(refreshed.supportsTextGeneration).toBe(false); - }), - ), - ); - - it.effect("replaces live model choices and accepts an empty catalog", () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeHarness(); - yield* harness.initialize; + Effect.gen(function* () { + const auth = { type: "gemini-api-key", label: "Gemini API key" }; + const harness = yield* makeHarness({ auth }); + yield* harness.initialize; + for (const clear of [harness.provider.onSignedOut, harness.provider.onAuthRequired]) { yield* harness.provider.onSessionStarted(started, "/workspace"); yield* harness.provider.onAvailableCommands(commands, "/workspace"); - const before = yield* harness.provider.snapshot.getSnapshot; - const configOptions = [ - { - ...modelConfig, - currentValue: "gemini-3.8-flash-high", - options: modelOptions.slice(0, 3), - }, - ]; - const nextSnapshot = yield* Stream.toPull( - harness.provider.snapshot.streamChanges.pipe( - Stream.filter((snapshot) => snapshot.models.length === 3), - ), - ); - yield* harness.provider.onConfigOptionsUpdated(configOptions); - expect((yield* nextSnapshot)[0]).toMatchObject({ - models: buildAntigravityModelsFromSession({ configOptions }), - auth: before.auth, - workspaceSnapshots: before.workspaceSnapshots, - slashCommands: expectedCommands, + yield* clear; + expect(yield* harness.provider.snapshot.getSnapshot).toMatchObject({ + installed: true, + status: "warning", + auth: { status: "unauthenticated", type: auth.type, label: auth.label }, + message: "Connect with Gemini API key to use Antigravity and load models.", + models: [], + slashCommands: [], + skills: [], + workspaceSnapshots: [], + supportsTextGeneration: false, }); - yield* harness.provider.onConfigOptionsUpdated([]); + yield* harness.provider.onAvailableCommands(commands, "/workspace"); + yield* harness.provider.onConfigOptionsUpdated([modelConfig]); + expect((yield* harness.provider.snapshotForCwd("/workspace")).slashCommands).toEqual([]); expect((yield* harness.provider.snapshot.getSnapshot).models).toEqual([]); - }), - ), + } + const refreshed = yield* harness.provider.snapshot.refresh; + expect(refreshed.auth).toEqual({ + status: "unauthenticated", + type: auth.type, + label: auth.label, + }); + expect(refreshed.message).toBe( + "Connect with Gemini API key to use Antigravity and load models.", + ); + expect(refreshed.supportsTextGeneration).toBe(false); + }), + ); + + it.effect("replaces live model choices and accepts an empty catalog", () => + Effect.gen(function* () { + const harness = yield* makeHarness(); + yield* harness.initialize; + yield* harness.provider.onSessionStarted(started, "/workspace"); + yield* harness.provider.onAvailableCommands(commands, "/workspace"); + const before = yield* harness.provider.snapshot.getSnapshot; + const configOptions = [ + { + ...modelConfig, + currentValue: "gemini-3.8-flash-high", + options: modelOptions.slice(0, 3), + }, + ]; + const nextSnapshot = yield* Stream.toPull( + harness.provider.snapshot.streamChanges.pipe( + Stream.filter((snapshot) => snapshot.models.length === 3), + ), + ); + yield* harness.provider.onConfigOptionsUpdated(configOptions); + expect((yield* nextSnapshot)[0]).toMatchObject({ + models: buildAntigravityModelsFromSession({ configOptions }), + auth: before.auth, + workspaceSnapshots: before.workspaceSnapshots, + slashCommands: expectedCommands, + }); + yield* harness.provider.onConfigOptionsUpdated([]); + expect((yield* harness.provider.snapshot.getSnapshot).models).toEqual([]); + }), ); it.effect("does not guess a version before a custom launcher's first handshake", () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeHarness(); - yield* Deferred.succeed(harness.initialProbe, { - ...initializeResult, - agentInfo: { ...initializeResult.agentInfo, version: "unknown" }, - }); - yield* harness.initialUpdate; - expect((yield* harness.provider.snapshot.getSnapshot).version).toBeNull(); - }), - ), + Effect.gen(function* () { + const harness = yield* makeHarness(); + yield* Deferred.succeed(harness.initialProbe, { + ...initializeResult, + agentInfo: { ...initializeResult.agentInfo, version: "unknown" }, + }); + yield* harness.initialUpdate; + expect((yield* harness.provider.snapshot.getSnapshot).version).toBeNull(); + }), ); it.effect("does not replace a handshake version with an unknown custom-launcher probe", () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeHarness(); - yield* harness.initialize; - yield* harness.provider.onSessionStarted({ - ...started, - initializeResult: { - ...initializeResult, - agentInfo: { ...initializeResult.agentInfo, version: "1.4.0" }, - }, - }); - yield* Ref.set( - harness.probe, - Effect.succeed({ - ...initializeResult, - agentInfo: { ...initializeResult.agentInfo, version: "unknown" }, - }), - ); - expect((yield* harness.provider.snapshot.refresh).version).toBe("1.4.0"); - }), - ), + Effect.gen(function* () { + const harness = yield* makeHarness(); + yield* harness.initialize; + yield* harness.provider.onSessionStarted({ + ...started, + initializeResult: { + ...initializeResult, + agentInfo: { ...initializeResult.agentInfo, version: "1.4.0" }, + }, + }); + yield* Ref.set( + harness.probe, + Effect.succeed({ + ...initializeResult, + agentInfo: { ...initializeResult.agentInfo, version: "unknown" }, + }), + ); + expect((yield* harness.provider.snapshot.refresh).version).toBe("1.4.0"); + }), ); it.effect("replaces one account's catalog instead of combining accounts", () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeHarness(); - yield* harness.initialize; - yield* harness.provider.onSessionStarted(started); - yield* harness.provider.onSignedOut; - yield* harness.provider.onSessionStarted({ - ...started, - sessionSetupResult: { - sessionId: "new-account-session", - configOptions: [ - { ...modelConfig, currentValue: "gemini-pro-agent", options: [modelOptions[9]!] }, - ], - }, - }); - expect( - (yield* harness.provider.snapshot.getSnapshot).models.map((model) => model.slug), - ).toEqual(["gemini-pro-agent"]); - }), - ), + Effect.gen(function* () { + const harness = yield* makeHarness(); + yield* harness.initialize; + yield* harness.provider.onSessionStarted(started); + yield* harness.provider.onSignedOut; + yield* harness.provider.onSessionStarted({ + ...started, + sessionSetupResult: { + sessionId: "new-account-session", + configOptions: [ + { ...modelConfig, currentValue: "gemini-pro-agent", options: [modelOptions[9]!] }, + ], + }, + }); + expect( + (yield* harness.provider.snapshot.getSnapshot).models.map((model) => model.slug), + ).toEqual(["gemini-pro-agent"]); + }), ); it.effect("retains known account metadata when a local health check fails", () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeHarness(); - yield* harness.initialize; + Effect.gen(function* () { + const harness = yield* makeHarness(); + yield* harness.initialize; + yield* harness.provider.onSessionStarted(started, "/workspace"); + yield* harness.provider.onAvailableCommands(commands, "/workspace"); + yield* Ref.set( + harness.probe, + Effect.fail(EffectAcpErrors.AcpRequestError.internalError("probe failed")), + ); + const snapshot = yield* harness.provider.snapshot.refresh; + expect(snapshot).toMatchObject({ + installed: true, + status: "error", + auth: { status: "authenticated" }, + }); + expect(snapshot.models).toEqual(buildAntigravityModelsFromSession(sessionSetupResult)); + expect(snapshot.slashCommands).toEqual(expectedCommands); + expect(snapshot.workspaceSnapshots?.[0]?.cwd).toBe("/workspace"); + }), + ); + + it.effect("distinguishes missing executables from a failed installed executable", () => + Effect.gen(function* () { + const harness = yield* makeHarness(); + yield* harness.initialize; + const failures = [ + { + error: new EffectAcpErrors.AcpSpawnError({ cause: { code: "ENOENT" } }), + installed: false, + }, + { + error: new EffectAcpErrors.AcpSpawnError({ cause: { code: "EACCES" } }), + installed: true, + }, + { + error: new ProviderSetupError({ + instanceId, + operation: "resolve", + detail: "Antigravity is not installed.", + }), + installed: false, + }, + ]; + for (const { error, installed } of failures) { yield* harness.provider.onSessionStarted(started, "/workspace"); yield* harness.provider.onAvailableCommands(commands, "/workspace"); - yield* Ref.set( - harness.probe, - Effect.fail(EffectAcpErrors.AcpRequestError.internalError("probe failed")), - ); + yield* Ref.set(harness.probe, Effect.fail(error)); const snapshot = yield* harness.provider.snapshot.refresh; expect(snapshot).toMatchObject({ - installed: true, + installed, status: "error", auth: { status: "authenticated" }, }); - expect(snapshot.models).toEqual(buildAntigravityModelsFromSession(sessionSetupResult)); - expect(snapshot.slashCommands).toEqual(expectedCommands); - expect(snapshot.workspaceSnapshots?.[0]?.cwd).toBe("/workspace"); - }), - ), - ); - - it.effect("distinguishes missing executables from a failed installed executable", () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeHarness(); - yield* harness.initialize; - const failures = [ - { - error: new EffectAcpErrors.AcpSpawnError({ cause: { code: "ENOENT" } }), - installed: false, - }, - { - error: new EffectAcpErrors.AcpSpawnError({ cause: { code: "EACCES" } }), - installed: true, - }, - { - error: new ProviderSetupError({ - instanceId, - operation: "resolve", - detail: "Antigravity is not installed.", - }), - installed: false, - }, - ]; - for (const { error, installed } of failures) { - yield* harness.provider.onSessionStarted(started, "/workspace"); - yield* harness.provider.onAvailableCommands(commands, "/workspace"); - yield* Ref.set(harness.probe, Effect.fail(error)); - const snapshot = yield* harness.provider.snapshot.refresh; - expect(snapshot).toMatchObject({ - installed, - status: "error", - auth: { status: "authenticated" }, - }); - expect(snapshot.models).toHaveLength(installed ? 11 : 0); - expect(snapshot.slashCommands).toHaveLength(installed ? 2 : 0); - expect(snapshot.workspaceSnapshots).toHaveLength(installed ? 1 : 0); - expect(snapshot.supportsTextGeneration).toBe(installed); - } - }), - ), + expect(snapshot.models).toHaveLength(installed ? 11 : 0); + expect(snapshot.slashCommands).toHaveLength(installed ? 2 : 0); + expect(snapshot.workspaceSnapshots).toHaveLength(installed ? 1 : 0); + expect(snapshot.supportsTextGeneration).toBe(installed); + } + }), ); it.effect("does not let an old health result restore a signed-out account", () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeHarness(); - yield* harness.initialize; - yield* harness.provider.onSessionStarted(started); - const releaseProbe = yield* Deferred.make(); - const probeEntered = yield* Deferred.make(); - yield* Ref.set( - harness.probe, - Deferred.succeed(probeEntered, undefined).pipe( - Effect.andThen(Deferred.await(releaseProbe)), - ), - ); - const refresh = yield* harness.provider.snapshot.refresh.pipe(Effect.forkChild); - yield* Deferred.await(probeEntered); - yield* harness.provider.onSignedOut; - yield* Deferred.succeed(releaseProbe, initializeResult); - const snapshot = yield* Fiber.join(refresh); - expect(snapshot).toMatchObject({ - auth: { status: "unauthenticated" }, - models: [], - supportsTextGeneration: false, - }); - }), - ), + Effect.gen(function* () { + const harness = yield* makeHarness(); + yield* harness.initialize; + yield* harness.provider.onSessionStarted(started); + const releaseProbe = yield* Deferred.make(); + const probeEntered = yield* Deferred.make(); + yield* Ref.set( + harness.probe, + Deferred.succeed(probeEntered, undefined).pipe( + Effect.andThen(Deferred.await(releaseProbe)), + ), + ); + const refresh = yield* harness.provider.snapshot.refresh.pipe(Effect.forkChild); + yield* Deferred.await(probeEntered); + yield* harness.provider.onSignedOut; + yield* Deferred.succeed(releaseProbe, initializeResult); + const snapshot = yield* Fiber.join(refresh); + expect(snapshot).toMatchObject({ + auth: { status: "unauthenticated" }, + models: [], + supportsTextGeneration: false, + }); + }), ); it.effect("exposes helper support only when the supplied safety check allows it", () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeHarness({ safe: false }); - yield* harness.initialize; - yield* harness.provider.onSessionStarted(started); - expect((yield* harness.provider.snapshot.getSnapshot).supportsTextGeneration).toBe(false); - yield* Ref.set(harness.safety, Effect.succeed(true)); - yield* harness.provider.snapshot.refresh; - expect((yield* harness.provider.snapshot.getSnapshot).supportsTextGeneration).toBe(true); - }), - ), + Effect.gen(function* () { + const harness = yield* makeHarness({ safe: false }); + yield* harness.initialize; + yield* harness.provider.onSessionStarted(started); + expect((yield* harness.provider.snapshot.getSnapshot).supportsTextGeneration).toBe(false); + yield* Ref.set(harness.safety, Effect.succeed(true)); + yield* harness.provider.snapshot.refresh; + expect((yield* harness.provider.snapshot.getSnapshot).supportsTextGeneration).toBe(true); + }), ); it.effect("keeps discovered workspace skills through session and command updates", () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeHarness(); - yield* harness.initialize; - const skills = [ - { - name: "deploy", - description: "Ship it", - path: "/workspace/.agent/skills/deploy", - enabled: true, - }, - ]; - const discovered = yield* harness.provider.snapshotForCwd("/workspace", skills); - expect(discovered.skills).toEqual(skills); - yield* harness.provider.onSessionStarted(started, "/workspace"); - yield* harness.provider.onAvailableCommands(commands, "/workspace"); - const after = yield* harness.provider.snapshot.getSnapshot; - expect( - after.workspaceSnapshots?.find((entry) => entry.cwd === "/workspace")?.skills, - ).toEqual(skills); - expect((yield* harness.provider.snapshotForCwd("/workspace")).skills).toEqual(skills); - - const rescanned = [ - ...skills, - { name: "review", path: "/workspace/.agent/skills/review", enabled: true }, - ]; - yield* harness.provider.snapshotForCwd("/workspace", rescanned); - yield* harness.provider.onSessionStarted(started, "/workspace"); - expect( - (yield* harness.provider.snapshot.getSnapshot).workspaceSnapshots?.find( - (entry) => entry.cwd === "/workspace", - )?.skills, - ).toEqual(rescanned); - }), - ), + Effect.gen(function* () { + const harness = yield* makeHarness(); + yield* harness.initialize; + const skills = [ + { + name: "deploy", + description: "Ship it", + path: "/workspace/.agent/skills/deploy", + enabled: true, + }, + ]; + const discovered = yield* harness.provider.snapshotForCwd("/workspace", skills); + expect(discovered.skills).toEqual(skills); + yield* harness.provider.onSessionStarted(started, "/workspace"); + yield* harness.provider.onAvailableCommands(commands, "/workspace"); + const after = yield* harness.provider.snapshot.getSnapshot; + expect(after.workspaceSnapshots?.find((entry) => entry.cwd === "/workspace")?.skills).toEqual( + skills, + ); + expect((yield* harness.provider.snapshotForCwd("/workspace")).skills).toEqual(skills); + + const rescanned = [ + ...skills, + { name: "review", path: "/workspace/.agent/skills/review", enabled: true }, + ]; + yield* harness.provider.snapshotForCwd("/workspace", rescanned); + yield* harness.provider.onSessionStarted(started, "/workspace"); + expect( + (yield* harness.provider.snapshot.getSnapshot).workspaceSnapshots?.find( + (entry) => entry.cwd === "/workspace", + )?.skills, + ).toEqual(rescanned); + }), ); it.effect("bounds workspace metadata without starting sessions for workspace lookup", () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeHarness(); - yield* harness.initialize; - yield* harness.provider.onSessionStarted(started); - for (let index = 0; index < 35; index++) { - yield* harness.provider.onAvailableCommands(commands, `/workspace-${index}`); - } - const snapshot = yield* harness.provider.snapshotForCwd("/workspace-34"); - expect(snapshot.workspaceSnapshots).toHaveLength(32); - expect(snapshot.workspaceSnapshots?.[0]?.cwd).toBe("/workspace-3"); - expect(snapshot.slashCommands).toEqual(expectedCommands); - expect(yield* Ref.get(harness.probeCalls)).toBe(1); - }), - ), + Effect.gen(function* () { + const harness = yield* makeHarness(); + yield* harness.initialize; + yield* harness.provider.onSessionStarted(started); + for (let index = 0; index < 35; index++) { + yield* harness.provider.onAvailableCommands(commands, `/workspace-${index}`); + } + const snapshot = yield* harness.provider.snapshotForCwd("/workspace-34"); + expect(snapshot.workspaceSnapshots).toHaveLength(32); + expect(snapshot.workspaceSnapshots?.[0]?.cwd).toBe("/workspace-3"); + expect(snapshot.slashCommands).toEqual(expectedCommands); + expect(yield* Ref.get(harness.probeCalls)).toBe(1); + }), ); it.effect( "publishes limits, retains a successful read on failure, and clears them on sign-out", () => - Effect.scoped( - Effect.gen(function* () { - const limits: ServerProviderUsageLimits = { - checkedAt: "2026-10-09T12:00:00.000Z", - windows: [ - { - id: "3p-weekly", - kind: "weekly", - label: "Claude & GPT (shared) · Weekly", - usedPercent: 75, - }, - ], - }; - const usage = yield* Ref.make(limits); - const harness = yield* makeHarness({ probeUsage: Ref.get(usage) }); - const published = yield* harness.nextPublish((snapshot) => !!snapshot.usageLimits); - yield* harness.initialize; - expect((yield* published)?.usageLimits).toEqual(limits); - // The second health check is the one the failed background read triggers. - const checks = yield* Ref.make(0); - const failedReadApplied = yield* Deferred.make(); - yield* Ref.set( - harness.probe, - Ref.updateAndGet(checks, (count) => count + 1).pipe( - Effect.tap((count) => - count === 2 ? Deferred.succeed(failedReadApplied, undefined) : Effect.void, - ), - Effect.as(initializeResult), - ), - ); - yield* Ref.set(usage, { - checkedAt: "2026-10-09T12:01:00.000Z", - windows: [], - unavailable: { reason: "probeFailed" }, - }); - yield* harness.provider.snapshot.refresh; - yield* Deferred.await(failedReadApplied); - expect((yield* harness.provider.snapshot.getSnapshot).usageLimits).toEqual(limits); - yield* harness.provider.onSignedOut; - expect((yield* harness.provider.snapshot.getSnapshot).usageLimits).toBeUndefined(); - yield* Ref.set(usage, { - checkedAt: "2026-10-09T12:02:00.000Z", - windows: [], - unavailable: { reason: "unsupported" }, - }); - const unsupported = yield* harness.nextPublish( - (snapshot) => snapshot.usageLimits?.unavailable?.reason === "unsupported", - ); - yield* harness.provider.snapshot.refresh; - expect((yield* unsupported)?.usageLimits?.windows).toEqual([]); - }), - ), - ); - - it.effect("a quota read finishing after sign-out cannot restore account limits", () => - Effect.scoped( Effect.gen(function* () { - const readStarted = yield* Deferred.make(); - const readResult = yield* Deferred.make(); - const usage = yield* Ref.make>( - Effect.succeed({ checkedAt: "2026-10-09T12:00:00.000Z", windows: [] }), - ); - const harness = yield* makeHarness({ probeUsage: Ref.get(usage).pipe(Effect.flatten) }); + const limits: ServerProviderUsageLimits = { + checkedAt: "2026-10-09T12:00:00.000Z", + windows: [ + { + id: "3p-weekly", + kind: "weekly", + label: "Claude & GPT (shared) · Weekly", + usedPercent: 75, + }, + ], + }; + const usage = yield* Ref.make(limits); + const harness = yield* makeHarness({ probeUsage: Ref.get(usage) }); + const published = yield* harness.nextPublish((snapshot) => !!snapshot.usageLimits); yield* harness.initialize; - yield* Ref.set(harness.probe, Effect.succeed(initializeResult)); + expect((yield* published)?.usageLimits).toEqual(limits); + // The second health check is the one the failed background read triggers. + const checks = yield* Ref.make(0); + const failedReadApplied = yield* Deferred.make(); yield* Ref.set( - usage, - Deferred.succeed(readStarted, undefined).pipe(Effect.andThen(Deferred.await(readResult))), + harness.probe, + Ref.updateAndGet(checks, (count) => count + 1).pipe( + Effect.tap((count) => + count === 2 ? Deferred.succeed(failedReadApplied, undefined) : Effect.void, + ), + Effect.as(initializeResult), + ), ); - const refresh = yield* harness.provider.snapshot.refresh.pipe(Effect.forkChild); - yield* Deferred.await(readStarted); - yield* harness.provider.onSignedOut; - yield* Deferred.succeed(readResult, { + yield* Ref.set(usage, { checkedAt: "2026-10-09T12:01:00.000Z", - windows: [ - { id: "gemini-5h", kind: "session", label: "Gemini · 5-hour", usedPercent: 10 }, - ], + windows: [], + unavailable: { reason: "probeFailed" }, }); - yield* Fiber.join(refresh); + yield* harness.provider.snapshot.refresh; + yield* Deferred.await(failedReadApplied); + expect((yield* harness.provider.snapshot.getSnapshot).usageLimits).toEqual(limits); + yield* harness.provider.onSignedOut; expect((yield* harness.provider.snapshot.getSnapshot).usageLimits).toBeUndefined(); + yield* Ref.set(usage, { + checkedAt: "2026-10-09T12:02:00.000Z", + windows: [], + unavailable: { reason: "unsupported" }, + }); + const unsupported = yield* harness.nextPublish( + (snapshot) => snapshot.usageLimits?.unavailable?.reason === "unsupported", + ); + yield* harness.provider.snapshot.refresh; + expect((yield* unsupported)?.usageLimits?.windows).toEqual([]); }), - ), + ); + + it.effect("a quota read finishing after sign-out cannot restore account limits", () => + Effect.gen(function* () { + const readStarted = yield* Deferred.make(); + const readResult = yield* Deferred.make(); + const usage = yield* Ref.make>( + Effect.succeed({ checkedAt: "2026-10-09T12:00:00.000Z", windows: [] }), + ); + const harness = yield* makeHarness({ probeUsage: Ref.get(usage).pipe(Effect.flatten) }); + yield* harness.initialize; + yield* Ref.set(harness.probe, Effect.succeed(initializeResult)); + yield* Ref.set( + usage, + Deferred.succeed(readStarted, undefined).pipe(Effect.andThen(Deferred.await(readResult))), + ); + const refresh = yield* harness.provider.snapshot.refresh.pipe(Effect.forkChild); + yield* Deferred.await(readStarted); + yield* harness.provider.onSignedOut; + yield* Deferred.succeed(readResult, { + checkedAt: "2026-10-09T12:01:00.000Z", + windows: [{ id: "gemini-5h", kind: "session", label: "Gemini · 5-hour", usedPercent: 10 }], + }); + yield* Fiber.join(refresh); + expect((yield* harness.provider.snapshot.getSnapshot).usageLimits).toBeUndefined(); + }), ); it.effect("reports ready and starts sessions while one shared quota read is pending", () => - Effect.scoped( - Effect.gen(function* () { - const reads = yield* Ref.make(0); - const readStarted = yield* Deferred.make(); - const readResult = yield* Deferred.make(); - const harness = yield* makeHarness({ - probeUsage: Ref.update(reads, (count) => count + 1).pipe( - Effect.andThen(Deferred.succeed(readStarted, undefined)), - Effect.andThen(Deferred.await(readResult)), - ), - }); - const published = yield* harness.nextPublish((snapshot) => !!snapshot.usageLimits); - yield* harness.initialize; - yield* Deferred.await(readStarted); - yield* harness.provider.onSessionStarted(started, "/workspace"); - yield* harness.provider.onSessionStarted(started, "/other"); - const ready = yield* harness.provider.snapshot.getSnapshot; - expect(ready).toMatchObject({ status: "ready", auth: { status: "authenticated" } }); - expect(ready.usageLimits).toBeUndefined(); - expect(yield* Ref.get(reads)).toBe(1); - const limits = { - checkedAt: "2026-10-09T12:00:00.000Z", - windows: [ - { id: "gemini-5h", kind: "session", label: "Gemini · 5-hour", usedPercent: 10 }, - ], - } satisfies ServerProviderUsageLimits; - yield* Deferred.succeed(readResult, limits); - expect((yield* published)?.usageLimits).toEqual(limits); - }), - ), + Effect.gen(function* () { + const reads = yield* Ref.make(0); + const readStarted = yield* Deferred.make(); + const readResult = yield* Deferred.make(); + const harness = yield* makeHarness({ + probeUsage: Ref.update(reads, (count) => count + 1).pipe( + Effect.andThen(Deferred.succeed(readStarted, undefined)), + Effect.andThen(Deferred.await(readResult)), + ), + }); + const published = yield* harness.nextPublish((snapshot) => !!snapshot.usageLimits); + yield* harness.initialize; + yield* Deferred.await(readStarted); + yield* harness.provider.onSessionStarted(started, "/workspace"); + yield* harness.provider.onSessionStarted(started, "/other"); + const ready = yield* harness.provider.snapshot.getSnapshot; + expect(ready).toMatchObject({ status: "ready", auth: { status: "authenticated" } }); + expect(ready.usageLimits).toBeUndefined(); + expect(yield* Ref.get(reads)).toBe(1); + const limits = { + checkedAt: "2026-10-09T12:00:00.000Z", + windows: [{ id: "gemini-5h", kind: "session", label: "Gemini · 5-hour", usedPercent: 10 }], + } satisfies ServerProviderUsageLimits; + yield* Deferred.succeed(readResult, limits); + expect((yield* published)?.usageLimits).toEqual(limits); + }), ); it.effect("a quota read for the previous account is dropped after a new sign-in", () => - Effect.scoped( - Effect.gen(function* () { - const oldReadStarted = yield* Deferred.make(); - const newLimits = { - checkedAt: "2026-10-09T12:01:00.000Z", - windows: [{ id: "3p-weekly", kind: "weekly", label: "Weekly", usedPercent: 40 }], - } satisfies ServerProviderUsageLimits; - const usage = yield* Ref.make>( - Deferred.succeed(oldReadStarted, undefined).pipe(Effect.andThen(Effect.never)), - ); - const harness = yield* makeHarness({ probeUsage: Ref.get(usage).pipe(Effect.flatten) }); - const published = yield* harness.nextPublish((snapshot) => !!snapshot.usageLimits); - yield* harness.initialize; - yield* Deferred.await(oldReadStarted); - yield* Ref.set(usage, Effect.succeed(newLimits)); - yield* harness.provider.onSignedIn; - yield* harness.provider.onSessionStarted(started); - expect((yield* published)?.usageLimits).toEqual(newLimits); - // Switching again must not republish the previous account's limits, even if the read fails. - const failed = { - checkedAt: "2026-10-09T12:02:00.000Z", - windows: [], - unavailable: { reason: "probeFailed" }, - } satisfies ServerProviderUsageLimits; - yield* Ref.set(usage, Effect.succeed(failed)); - const republished = yield* harness.nextPublish( - (snapshot) => snapshot.usageLimits?.unavailable?.reason === "probeFailed", - ); - yield* harness.provider.onSignedIn; - expect((yield* harness.provider.snapshot.getSnapshot).usageLimits).toBeUndefined(); - yield* harness.provider.onSessionStarted(started); - expect((yield* republished)?.usageLimits).toEqual(failed); - }), - ), + Effect.gen(function* () { + const oldReadStarted = yield* Deferred.make(); + const newLimits = { + checkedAt: "2026-10-09T12:01:00.000Z", + windows: [{ id: "3p-weekly", kind: "weekly", label: "Weekly", usedPercent: 40 }], + } satisfies ServerProviderUsageLimits; + const usage = yield* Ref.make>( + Deferred.succeed(oldReadStarted, undefined).pipe(Effect.andThen(Effect.never)), + ); + const harness = yield* makeHarness({ probeUsage: Ref.get(usage).pipe(Effect.flatten) }); + const published = yield* harness.nextPublish((snapshot) => !!snapshot.usageLimits); + yield* harness.initialize; + yield* Deferred.await(oldReadStarted); + yield* Ref.set(usage, Effect.succeed(newLimits)); + yield* harness.provider.onSignedIn; + yield* harness.provider.onSessionStarted(started); + expect((yield* published)?.usageLimits).toEqual(newLimits); + // Switching again must not republish the previous account's limits, even if the read fails. + const failed = { + checkedAt: "2026-10-09T12:02:00.000Z", + windows: [], + unavailable: { reason: "probeFailed" }, + } satisfies ServerProviderUsageLimits; + yield* Ref.set(usage, Effect.succeed(failed)); + const republished = yield* harness.nextPublish( + (snapshot) => snapshot.usageLimits?.unavailable?.reason === "probeFailed", + ); + yield* harness.provider.onSignedIn; + expect((yield* harness.provider.snapshot.getSnapshot).usageLimits).toBeUndefined(); + yield* harness.provider.onSessionStarted(started); + expect((yield* republished)?.usageLimits).toEqual(failed); + }), ); }); diff --git a/apps/server/src/provider/AntigravityProvider.ts b/apps/server/src/provider/AntigravityProvider.ts index ed047331e73e..b140c7511ca1 100644 --- a/apps/server/src/provider/AntigravityProvider.ts +++ b/apps/server/src/provider/AntigravityProvider.ts @@ -1,3 +1,4 @@ +import { resolveProviderProbeTimeoutMs } from "@t3tools/provider-core/server/snapshotProbe"; import { ANTIGRAVITY_DEFAULT_MODEL, ProviderDriverKind, @@ -40,7 +41,7 @@ import { antigravityThinkingConfig } from "./acp/AntigravityAcpSupport.ts"; const EMPTY_MODEL_CAPABILITIES = createModelCapabilities({ optionDescriptors: [] }); const MAX_WORKSPACE_SNAPSHOTS = 32; -const HEALTH_CHECK_TIMEOUT = "90 seconds"; +const HEALTH_CHECK_TIMEOUT_MS = 90_000; const SIGN_IN_MESSAGE = "Sign in with Google to use Antigravity."; const AUTH_UNCHECKED_MESSAGE = "Antigravity is installed. Google account access is not checked yet."; @@ -155,6 +156,7 @@ interface AntigravityProviderState { } interface AntigravityProviderOptions { + readonly environment?: NodeJS.ProcessEnv; readonly stampIdentity: (snapshot: ServerProviderDraft) => Effect.Effect; readonly probe: Effect.Effect< EffectAcpSchema.InitializeResponse, @@ -271,10 +273,12 @@ export const makeAntigravityProvider = Effect.fn("makeAntigravityProvider")(func const checkProvider = Effect.fn("checkAntigravityProvider")(function* () { if (!settings.enabled) return yield* getSnapshot; const before = yield* SubscriptionRef.get(metadata); - const result = yield* options.probe.pipe( - Effect.timeoutOption(HEALTH_CHECK_TIMEOUT), - Effect.result, + const timeoutMs = resolveProviderProbeTimeoutMs( + "antigravity", + HEALTH_CHECK_TIMEOUT_MS, + options.environment, ); + const result = yield* options.probe.pipe(Effect.timeoutOption(timeoutMs), Effect.result); const initialized = Result.isSuccess(result) && Option.isSome(result.success) ? result.success.value : undefined; const version = initialized?.agentInfo?.version; @@ -289,7 +293,7 @@ export const makeAntigravityProvider = Effect.fn("makeAntigravityProvider")(func ? "Antigravity ACP runtime is not installed or its executable could not be found." : failure ? "Antigravity could not complete its local health check." - : `Antigravity did not respond to its local health check within ${HEALTH_CHECK_TIMEOUT}.`; + : `Antigravity did not respond to its local health check within ${timeoutMs / 1_000} seconds.`; const supportsTextGeneration = initialized !== undefined ? yield* options.supportsTextGeneration : false; const updatedAt = DateTime.formatIso(yield* DateTime.now); diff --git a/apps/server/src/provider/ClaudeCapabilitiesProbe.test.ts b/apps/server/src/provider/ClaudeCapabilitiesProbe.test.ts index bf3c9318877b..b89cffa42387 100644 --- a/apps/server/src/provider/ClaudeCapabilitiesProbe.test.ts +++ b/apps/server/src/provider/ClaudeCapabilitiesProbe.test.ts @@ -14,10 +14,13 @@ import { buildClaudeCapabilitiesProbeQueryOptions, probeClaudeCapabilities, probeClaudeWorkspaceSnapshot, + spawnClaudeProbeThroughEnv, } from "./ClaudeProvider.ts"; vi.mock("@anthropic-ai/claude-agent-sdk", { spy: true }); +const hostPath = process.env.PATH; + const decodeClaudeSettings = Schema.decodeEffect(ClaudeSettings); it("drops strict MCP mode, and only that, next to an administrator's managed-mcp.json", () => { @@ -27,6 +30,7 @@ it("drops strict MCP mode, and only that, next to an administrator's managed-mcp environment: { HOME: "/home/user" }, cwd: undefined, managedMcpConfig: true, + spawnVia: undefined, }); // Claude exits before initializing when handed --strict-mcp-config here. @@ -35,6 +39,29 @@ it("drops strict MCP mode, and only that, next to an administrator's managed-mcp assert.equal(options.env?.ENABLE_CLAUDEAI_MCP_SERVERS, "false"); }); +// A shell trampoline would not do: dash drops the exported function, and the launcher exits 127. +it.live.skipIf(HostProcess.Platform.defaultValue() !== "linux")( + "starts a Bash launcher that calls an exported function in the probe's workspace", + () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const workspace = yield* fileSystem.realPath(yield* fileSystem.makeTempDirectoryScoped()); + yield* fileSystem.writeFileString(`${workspace}/claude`, '#!/bin/bash\nt3_launch "$@"\n'); + yield* fileSystem.chmod(`${workspace}/claude`, 0o755); + const child = spawnClaudeProbeThroughEnv({ + command: "claude", + args: ["--probe"], + cwd: workspace, + env: { PATH: `.:${hostPath}`, "BASH_FUNC_t3_launch%%": '() { echo "$PWD $@"; }' }, + // @effect-diagnostics-next-line abortControllerInEffect:off - The launcher exits on its own. + signal: new AbortController().signal, + }); + assert.deepEqual(child.spawnargs.slice(0, 3), ["/usr/bin/env", "-C", workspace]); + const output = yield* Effect.promise(() => child.stdout.setEncoding("utf8").toArray()); + assert.equal(output.join(""), `${workspace} --probe\n`); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + it.effect("reports a deleted worktree's scan as finished, not pending", () => Effect.gen(function* () { const snapshot = yield* probeClaudeWorkspaceSnapshot( diff --git a/apps/server/src/provider/ClaudeCodeModelRoutes.test.ts b/apps/server/src/provider/ClaudeCodeModelRoutes.test.ts new file mode 100644 index 000000000000..2c06148cac68 --- /dev/null +++ b/apps/server/src/provider/ClaudeCodeModelRoutes.test.ts @@ -0,0 +1,67 @@ +import { assert, describe, it } from "@effect/vitest"; +import { type ModelSelection, ProviderInstanceId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Option from "effect/Option"; +import * as TestClock from "effect/testing/TestClock"; + +import { makeClaudeCodeModelRoutes } from "./ClaudeCodeModelRoutes.ts"; +import { + BUNDLED_CLAUDE_MODEL_CATALOG, + resolveClaudeCatalogApiModelId, +} from "./ClaudeModelCatalog.ts"; + +describe("Claude Code model routes", () => { + it.effect( + "a turn that starts before the first probe settles waits and then gets its routes", + () => + Effect.gen(function* () { + const haiku = { + instanceId: ProviderInstanceId.make("claudeAgent"), + model: "claude-haiku-4-5", + } satisfies ModelSelection; + const routes = yield* makeClaudeCodeModelRoutes( + () => Effect.succeed("same settings"), + "/state", + ); + const early = yield* routes + .route(BUNDLED_CLAUDE_MODEL_CATALOG, "/project") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* routes.record([ + { value: "haiku", resolvedModel: "us.anthropic.claude-haiku-4-5-20251001-v1:0" }, + ]); + yield* routes.settle; + assert.strictEqual( + resolveClaudeCatalogApiModelId(yield* Fiber.join(early), haiku), + "haiku", + ); + + // A first check that never reaches the probe still releases waiting turns. + const unprobed = yield* makeClaudeCodeModelRoutes( + () => Effect.succeed("same settings"), + "/state", + ); + const waiting = yield* unprobed + .route(BUNDLED_CLAUDE_MODEL_CATALOG, "/project") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* unprobed.settle; + assert.strictEqual(yield* Fiber.join(waiting), BUNDLED_CLAUDE_MODEL_CATALOG); + + const stuck = yield* makeClaudeCodeModelRoutes( + () => Effect.succeed("same settings"), + "/state", + ); + const fallback = yield* stuck + .route(BUNDLED_CLAUDE_MODEL_CATALOG, "/project") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust("2 minutes"); + assert.strictEqual(yield* Fiber.join(fallback), BUNDLED_CLAUDE_MODEL_CATALOG); + // Later turns, steers and effort changes do not wait for the stuck probe again. + const later = yield* stuck + .route(BUNDLED_CLAUDE_MODEL_CATALOG, "/project") + .pipe(Effect.timeoutOption("1 second"), Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust("1 second"); + assert.deepStrictEqual(yield* Fiber.join(later), Option.some(BUNDLED_CLAUDE_MODEL_CATALOG)); + }), + ); +}); diff --git a/apps/server/src/provider/ClaudeCodeModelRoutes.ts b/apps/server/src/provider/ClaudeCodeModelRoutes.ts new file mode 100644 index 000000000000..6a9e7a48e64c --- /dev/null +++ b/apps/server/src/provider/ClaudeCodeModelRoutes.ts @@ -0,0 +1,116 @@ +import { + type ModelInfo as ClaudeModelInfo, + resolveSettings as resolveClaudeSettings, + type SettingSource, + type Settings as ClaudeSdkSettings, +} from "@anthropic-ai/claude-agent-sdk"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; + +import { claudeLocalSettingsDirectory } from "./Drivers/ClaudeHome.ts"; +import { type ClaudeModelCatalog, withClaudeCodeModelRoutes } from "./ClaudeModelCatalog.ts"; + +// Env names that pick the backend, its region, or the model a family alias resolves to. +const MODEL_SETTINGS_ENV_PREFIXES = [ + "ANTHROPIC_", + "CLAUDE_CODE_USE_", + "AWS_", + "CLOUD_ML_", + "VERTEX_", +]; + +/** Equal keys mean Claude Code resolves model ids the same way under both settings. */ +function claudeModelSettingsKey(settings: ClaudeSdkSettings): string { + const sortedEntries = (record: Readonly> | undefined) => + Object.entries(record ?? {}).toSorted(([a], [b]) => a.localeCompare(b)); + return JSON.stringify([ + settings.availableModels ?? null, + sortedEntries(settings.modelOverrides), + sortedEntries(settings.env).filter(([name]) => + MODEL_SETTINGS_ENV_PREFIXES.some((prefix) => name.startsWith(prefix)), + ), + ]); +} + +const NO_MODEL_SETTINGS_KEY = claudeModelSettingsKey({}); + +class ClaudeModelSettingsReadError extends Schema.TaggedError()( + "ClaudeModelSettingsReadError", + { cause: Schema.Defect() }, +) {} + +export const resolveClaudeModelSettingsKey = Effect.fn("resolveClaudeModelSettingsKey")(function* ( + settingSources: ReadonlyArray, + cwd: string | null, +) { + // The SDK reads the local file from cwd, Claude Code from the main checkout's root. No key, so + // no routes, when that root file touches model settings. + const localDirectory = cwd === null ? null : yield* claudeLocalSettingsDirectory(cwd); + if (localDirectory !== null && localDirectory !== cwd) { + const rootLocal = yield* Effect.tryPromise({ + try: () => resolveClaudeSettings({ cwd: localDirectory, settingSources: ["local"] }), + catch: (cause) => new ClaudeModelSettingsReadError({ cause }), + }).pipe( + Effect.map(({ sources }) => sources.find(({ source }) => source === "local")?.settings), + Effect.orElseSucceed(() => undefined), + ); + if (claudeModelSettingsKey(rootLocal ?? {}) !== NO_MODEL_SETTINGS_KEY) return undefined; + } + return yield* Effect.tryPromise({ + try: () => + resolveClaudeSettings({ + ...(cwd === null ? {} : { cwd }), + settingSources: [...settingSources], + }), + catch: (cause) => new ClaudeModelSettingsReadError({ cause }), + }).pipe( + Effect.map(({ effective }) => claudeModelSettingsKey(effective)), + Effect.orElseSucceed(() => undefined), + ); +}); + +type ReportedModel = Pick; + +/** + * Claude Code's reported model ids, applied to a turn only where its cwd resolves model settings + * as the probe's cwd did. `settle` releases turns waiting for the first probe, whatever its outcome. + */ +export const makeClaudeCodeModelRoutes = Effect.fn("makeClaudeCodeModelRoutes")(function* ( + settingsKey: (cwd: string | null) => Effect.Effect, + probeCwd: string, +) { + const firstProbeSettled = yield* Deferred.make(); + const reported = yield* Ref.make<{ + readonly models: ReadonlyArray; + readonly settingsKey: string | undefined; + }>({ models: [], settingsKey: undefined }); + return { + record: Effect.fn("ClaudeCodeModelRoutes.record")(function* ( + models: ReadonlyArray, + ) { + yield* Ref.set(reported, { models, settingsKey: yield* settingsKey(probeCwd) }); + }), + settle: Deferred.succeed(firstProbeSettled, undefined).pipe(Effect.asVoid), + route: Effect.fn("ClaudeCodeModelRoutes.route")(function* ( + base: ClaudeModelCatalog, + cwd: string | null, + launchSettings?: ClaudeSdkSettings, + ) { + // Leave room for the sequential version and capability probes before falling back once; + // an expired wait counts as settled so later calls do not wait again. + yield* Deferred.await(firstProbeSettled).pipe( + Effect.timeoutOption("2 minutes"), + Effect.andThen(Deferred.succeed(firstProbeSettled, undefined)), + ); + if (claudeModelSettingsKey(launchSettings ?? {}) !== NO_MODEL_SETTINGS_KEY) return base; + const { models, settingsKey: probeKey } = yield* Ref.get(reported); + const routed = withClaudeCodeModelRoutes(base, models); + // Project settings can point an alias at another model, so routes hold only where the + // cwd resolves the same model settings as the probe's. + if (routed === base || probeKey === undefined) return base; + return (yield* settingsKey(cwd)) === probeKey ? routed : base; + }), + }; +}); diff --git a/apps/server/src/provider/ClaudeModelCatalog.test.ts b/apps/server/src/provider/ClaudeModelCatalog.test.ts index c8c8abdfcf2b..bbcc02f90132 100644 --- a/apps/server/src/provider/ClaudeModelCatalog.test.ts +++ b/apps/server/src/provider/ClaudeModelCatalog.test.ts @@ -1,8 +1,9 @@ import { assert, describe, it } from "@effect/vitest"; -import { ProviderDriverKind, ProviderInstanceId } from "@t3tools/contracts"; +import { type ModelSelection, ProviderDriverKind, ProviderInstanceId } from "@t3tools/contracts"; import { resolveProviderCatalog, type ModelManifestData } from "./ModelManifest.ts"; import { + BUNDLED_CLAUDE_MODEL_CATALOG, capClaudeCatalogEffort, type ClaudeModelCatalog, formatClaudeVersionUpgradeMessage, @@ -16,6 +17,7 @@ import { resolveClaudeModelSlug, resolveClaudeUpdateRequiredModels, scopeClaudeModelCatalog, + withClaudeCodeModelRoutes, } from "./ClaudeModelCatalog.ts"; const manifest = (): ModelManifestData => ({ @@ -135,6 +137,13 @@ describe("Claude model catalog", () => { formatClaudeVersionUpgradeMessage(catalog, "3.1.9"), "Claude Code v3.1.9 is too old for Claude Synthetic Next. Upgrade to v3.2.0 or newer to access it.", ); + // Claude Code before 2.1.321 does not know claude-fable-5-2 and runs it with a 200K window. + assert.notInclude( + resolveClaudeModelsForVersion(BUNDLED_CLAUDE_MODEL_CATALOG, "2.1.320").map( + (model) => model.slug, + ), + "claude-fable-5-2", + ); }); it("resolves aliases and declarative adapter mappings", () => { @@ -233,6 +242,65 @@ describe("Claude model catalog", () => { ); }); + // Rows Claude Code 2.1.310 reports from initialization for each configuration. + it("sends the model id Claude Code reports when it serves a catalog model under another id", () => { + const apiModel = ( + catalog: ClaudeModelCatalog, + model: string, + options?: ModelSelection["options"], + ) => + resolveClaudeCatalogApiModelId(catalog, { + instanceId: ProviderInstanceId.make("claudeAgent"), + model, + ...(options ? { options } : {}), + }); + + // Bedrock with ANTHROPIC_DEFAULT_*_MODEL and an availableModels allowlist of their ids. + const configuredBedrock = withClaudeCodeModelRoutes(BUNDLED_CLAUDE_MODEL_CATALOG, [ + { value: "default", resolvedModel: "global.anthropic.claude-opus-5-5" }, + { value: "sonnet", resolvedModel: "global.anthropic.claude-sonnet-5-5[1m]" }, + { value: "opus", resolvedModel: "global.anthropic.claude-opus-5-5" }, + { value: "haiku", resolvedModel: "global.anthropic.claude-haiku-5-5" }, + ]); + assert.strictEqual(apiModel(configuredBedrock, "claude-haiku-5-5"), "haiku"); + assert.strictEqual(apiModel(configuredBedrock, "haiku-5.5"), "haiku"); + assert.strictEqual(apiModel(configuredBedrock, "claude-sonnet-5-5"), "sonnet"); + assert.strictEqual(apiModel(configuredBedrock, "claude-opus-5-5"), "opus"); + assert.strictEqual(apiModel(configuredBedrock, "claude-opus-5"), "claude-opus-5"); + + // Bedrock with no model settings: Claude Code's `haiku` is Haiku 4.5 there. + const defaultBedrock = withClaudeCodeModelRoutes(BUNDLED_CLAUDE_MODEL_CATALOG, [ + { + value: "us.anthropic.claude-opus-4-6-v1", + resolvedModel: "us.anthropic.claude-opus-4-6-v1", + }, + { + value: "us.anthropic.claude-opus-4-6-v1[1m]", + resolvedModel: "us.anthropic.claude-opus-4-6-v1[1m]", + }, + { value: "haiku", resolvedModel: "us.anthropic.claude-haiku-4-5-20251001-v1:0" }, + ]); + assert.strictEqual( + apiModel(defaultBedrock, "claude-opus-4-6", [{ id: "contextWindow", value: "1m" }]), + "us.anthropic.claude-opus-4-6-v1[1m]", + ); + assert.strictEqual( + apiModel(defaultBedrock, "claude-opus-4-6", [{ id: "contextWindow", value: "200k" }]), + "us.anthropic.claude-opus-4-6-v1", + ); + assert.strictEqual(apiModel(defaultBedrock, "claude-haiku-4-5"), "haiku"); + assert.strictEqual(apiModel(defaultBedrock, "claude-haiku-5-5"), "claude-haiku-5-5"); + + // First-party ids already are what Claude Code runs, even when an alias points elsewhere. + const firstParty = withClaudeCodeModelRoutes(BUNDLED_CLAUDE_MODEL_CATALOG, [ + { value: "default", resolvedModel: "claude-opus-5-5" }, + { value: "opus", resolvedModel: "claude-opus-5-5" }, + { value: "claude-fable-5-1[1m]", resolvedModel: "claude-fable-5-1[1m]" }, + { value: "haiku", resolvedModel: "claude-haiku-4-5-20251001" }, + ]); + assert.strictEqual(firstParty, BUNDLED_CLAUDE_MODEL_CATALOG); + }); + it("marks effort options that would run above the cap without changing ids or defaults", () => { const effortModel = (slug: string) => ({ slug, diff --git a/apps/server/src/provider/ClaudeModelCatalog.ts b/apps/server/src/provider/ClaudeModelCatalog.ts index ceeb94a355e8..e3d4ea4397c0 100644 --- a/apps/server/src/provider/ClaudeModelCatalog.ts +++ b/apps/server/src/provider/ClaudeModelCatalog.ts @@ -32,6 +32,8 @@ export interface ClaudeCatalogModel { readonly model: ServerProviderModel; readonly runtime: ClaudeCodeProfile; readonly compatibility: ClaudeCodeCompatibility; + /** What to send Claude Code instead of the slug, from its own model list. */ + readonly apiModel?: string; } export interface ClaudeModelCatalog { @@ -221,7 +223,7 @@ export function claudeReportedModels( return models; } -function resolveClaudeCatalogModel( +export function resolveClaudeCatalogModel( catalog: ClaudeModelCatalog, slugOrAlias: string | null | undefined, ): ClaudeCatalogModel | undefined { @@ -235,6 +237,44 @@ function resolveClaudeCatalogModel( ); } +// Cloud backends decorate first-party ids: Bedrock adds a region prefix and a +// version ("us.anthropic.claude-opus-4-6-v1"), Vertex adds "@". +function stripClaudeBackendDecoration(wireModel: string): string { + return wireModel + .replace(/^(?:[a-z-]+\.)?anthropic\./, "") + .replace(/-v\d+(?::\d+)?$/, "") + .replace(/@[^@]*$/, ""); +} + +/** + * On Bedrock, Vertex and similar backends Claude Code does not accept the first-party slug: it + * either sends it verbatim or, when `availableModels` excludes it, silently runs the default + * model instead. Its model list names the id that does work, often a family alias that + * `ANTHROPIC_DEFAULT_*_MODEL` points at a backend id. + */ +export function withClaudeCodeModelRoutes( + catalog: ClaudeModelCatalog, + reportedModels: ReadonlyArray>, +): ClaudeModelCatalog { + const apiModels = new Map(); + for (const reported of reportedModels) { + if (reported.value === "default") continue; + const wireModel = stripClaudeContextWindowSuffix(reported.resolvedModel ?? reported.value); + const firstPartyModel = stripClaudeBackendDecoration(wireModel); + if (firstPartyModel === wireModel) continue; + const entry = resolveClaudeCatalogModel(catalog, firstPartyModel); + if (!entry || apiModels.has(entry.model.slug)) continue; + apiModels.set(entry.model.slug, stripClaudeContextWindowSuffix(reported.value)); + } + if (apiModels.size === 0) return catalog; + return { + models: catalog.models.map((entry) => { + const apiModel = apiModels.get(entry.model.slug); + return apiModel ? { ...entry, apiModel } : entry; + }), + }; +} + export function resolveClaudeModelSlug(catalog: ClaudeModelCatalog, slugOrAlias: string): string { return resolveClaudeCatalogModel(catalog, slugOrAlias)?.model.slug ?? slugOrAlias; } @@ -506,7 +546,7 @@ export function resolveClaudeCatalogApiModelId( modelSelection: ModelSelection, ): string { const entry = resolveClaudeCatalogModel(catalog, modelSelection.model); - const slug = entry?.model.slug ?? modelSelection.model; + const slug = entry?.apiModel ?? entry?.model.slug ?? modelSelection.model; const descriptors = getProviderOptionDescriptors({ caps: entry?.model.capabilities ?? EMPTY_CAPABILITIES, selections: modelSelection.options, diff --git a/apps/server/src/provider/ClaudeProvider.ts b/apps/server/src/provider/ClaudeProvider.ts index eafd128d610f..b657a2b813fe 100644 --- a/apps/server/src/provider/ClaudeProvider.ts +++ b/apps/server/src/provider/ClaudeProvider.ts @@ -1,3 +1,7 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off -- the SDK's spawn hook returns a Node child process. +import * as NodeChildProcess from "node:child_process"; + +import { resolveProviderProbeTimeoutMs } from "@t3tools/provider-core/server/snapshotProbe"; import { type ClaudeSettings, type ModelCapabilities, @@ -17,6 +21,9 @@ import * as Result from "effect/Result"; import * as Schema from "effect/Schema"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; import { createModelCapabilities } from "@t3tools/shared/model"; +import { envTrampoline, supportsEnvChdir } from "@t3tools/shared/envTrampoline"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as SpawnWorker from "../process/SpawnWorker.ts"; import { resolveSpawnCommand } from "@t3tools/shared/shell"; import { query as claudeQuery, @@ -26,6 +33,7 @@ import { type SDKControlGetUsageResponse, type SDKUserMessage, type SettingSource, + type SpawnOptions as ClaudeSpawnOptions, } from "@anthropic-ai/claude-agent-sdk"; import { @@ -170,11 +178,16 @@ function formatClaudeSubscriptionAuthLabel(subscriptionType: string): string { return `Claude ${subscriptionLabel} Subscription`; } +function hasApiKey(apiKeySource: string | undefined): boolean { + return Boolean(apiKeySource) && apiKeySource !== "none"; +} + function claudeAuthMetadata(input: { readonly subscriptionType: string | undefined; readonly authMethod: string | undefined; + readonly apiKeySource: string | undefined; }): { readonly type: string; readonly label: string } | undefined { - if (normalizeClaudeAuthMethod(input.authMethod) === "apiKey") { + if (normalizeClaudeAuthMethod(input.authMethod) === "apiKey" || hasApiKey(input.apiKeySource)) { return { type: "apiKey", label: "Claude API Key", @@ -209,8 +222,7 @@ function claudeAuthStatus( } if (capabilities.tokenSource !== "none") return "authenticated"; // API keys can authenticate with tokenSource: none; also check key and account fields. - const hasApiKey = Boolean(capabilities.apiKeySource) && capabilities.apiKeySource !== "none"; - return hasApiKey || capabilities.email || capabilities.subscriptionType + return hasApiKey(capabilities.apiKeySource) || capabilities.email || capabilities.subscriptionType ? "authenticated" : "unauthenticated"; } @@ -226,14 +238,49 @@ const CLAUDE_CAPABILITIES_PROBE_SETTING_SOURCES = [ "local", ] as const satisfies ReadonlyArray; +/** + * The SDK's own spawn stats the executable and hands libuv the workspace as cwd, so a mount + * that stops answering freezes the server. `env` enters the cwd and finds the CLI in the child. + * A shell would not do: dash drops the exported Bash functions a launcher may call. + */ +export function spawnClaudeProbeThroughEnv({ + command, + args, + cwd, + env, + signal, +}: ClaudeSpawnOptions) { + const launch = envTrampoline(command, args, cwd) ?? { command, args, cwd }; + return NodeChildProcess.spawn(launch.command, launch.args, { + ...("cwd" in launch ? { cwd: launch.cwd } : {}), + env, + signal, + // The probe ignores stderr. + stdio: ["pipe", "pipe", "ignore"], + }); +} + +/** Windows runs CreateProcess on the calling thread, so the probe launches from a launcher thread. */ +function spawnClaudeProbeOffThread({ command, args, cwd, env, signal }: ClaudeSpawnOptions) { + const child = SpawnWorker.spawnPiped(command, args, { cwd, env, signal, windowsHide: true }); + // The probe ignores stderr; drain it so a full pipe cannot stall the CLI. + child.stderr.resume(); + return child; +} + export function buildClaudeCapabilitiesProbeQueryOptions(input: { readonly executablePath: string; readonly abortController: AbortController; readonly environment: NodeJS.ProcessEnv; readonly cwd: string | undefined; readonly managedMcpConfig: boolean; + readonly spawnVia: "env" | "launcher-thread" | undefined; }): ClaudeQueryOptions { return { + ...(input.spawnVia === "env" ? { spawnClaudeCodeProcess: spawnClaudeProbeThroughEnv } : {}), + ...(input.spawnVia === "launcher-thread" + ? { spawnClaudeCodeProcess: spawnClaudeProbeOffThread } + : {}), persistSession: false, pathToClaudeCodeExecutable: input.executablePath, abortController: input.abortController, @@ -463,6 +510,13 @@ const probeClaudeCapabilities = ( ...(yield* readClaudeSettingsEnv(claudeSettings, cwd, claudeEnvironment)), }) : 0; + const platform = yield* HostProcess.Platform; + const spawnVia = + platform === "win32" + ? "launcher-thread" + : platform === "linux" && (yield* Effect.promise(supportsEnvChdir)) + ? "env" + : undefined; return yield* Effect.tryPromise({ try: async () => { const q = claudeQuery({ @@ -476,6 +530,7 @@ const probeClaudeCapabilities = ( environment: claudeEnvironment, cwd, managedMcpConfig, + spawnVia, }), }); const init = await q.initializationResult(); @@ -484,7 +539,9 @@ const probeClaudeCapabilities = ( catch: (cause) => new ClaudeCapabilitiesProbeSdkError({ operation: "initialize", cause }), }); }).pipe( - Effect.timeout(CAPABILITIES_PROBE_TIMEOUT_MS), + Effect.timeout( + resolveProviderProbeTimeoutMs("claude", CAPABILITIES_PROBE_TIMEOUT_MS, environment), + ), Effect.flatMap(({ q, init, graceMs }) => Effect.gen(function* () { // Usage has its own deadline so a slow optional request cannot discard initialization. @@ -651,7 +708,12 @@ export const checkClaudeProviderStatus = Effect.fn("checkClaudeProviderStatus")( ["--version"], providerProbeEnvironment(resolvedEnvironment), cwd, - ).pipe(Effect.timeoutOption(CLAUDE_VERSION_CHECK_TIMEOUT_MS), Effect.result); + ).pipe( + Effect.timeoutOption( + resolveProviderProbeTimeoutMs("claude", CLAUDE_VERSION_CHECK_TIMEOUT_MS, resolvedEnvironment), + ), + Effect.result, + ); if (Result.isFailure(versionProbe)) { const error = versionProbe.failure; @@ -756,7 +818,7 @@ export const checkClaudeProviderStatus = Effect.fn("checkClaudeProviderStatus")( auth: { status: "unknown" }, message: probeFailure ? Cause.isTimeoutError(probeFailure) - ? `Timed out after ${CAPABILITIES_PROBE_TIMEOUT_MS / 1_000}s while checking Claude account status.` + ? `Timed out after ${resolveProviderProbeTimeoutMs("claude", CAPABILITIES_PROBE_TIMEOUT_MS, resolvedEnvironment) / 1_000}s while checking Claude account status.` : "Claude Agent CLI failed while checking account status." : "Could not verify Claude authentication status from initialization result.", }, @@ -768,6 +830,7 @@ export const checkClaudeProviderStatus = Effect.fn("checkClaudeProviderStatus")( (yield* readClaudeSettingsEnv(claudeSettings, cwd, resolvedEnvironment)).ANTHROPIC_BASE_URL ?? resolvedEnvironment.ANTHROPIC_BASE_URL; const customApiEndpoint = customClaudeApiEndpointHost(baseUrl) !== undefined; + const endpoint = customApiEndpoint ? URL.parse(baseUrl?.trim() ?? "")?.origin : undefined; if (!customApiEndpoint && claudeAuthStatus(capabilities) === "unauthenticated") { const path = yield* Path.Path; const configDir = (yield* makeClaudeEnvironment(claudeSettings, resolvedEnvironment)) @@ -796,10 +859,13 @@ export const checkClaudeProviderStatus = Effect.fn("checkClaudeProviderStatus")( // A proxy's ANTHROPIC_AUTH_TOKEN reads as an API key; the endpoint is what the token is for. const authMetadata = apiProviderAuthMetadata(capabilities.apiProvider) ?? - (customApiEndpoint ? { type: "customEndpoint", label: "Custom API endpoint" } : undefined) ?? + (customApiEndpoint + ? { type: "customEndpoint", label: "Custom API endpoint", ...(endpoint ? { endpoint } : {}) } + : undefined) ?? claudeAuthMetadata({ subscriptionType: capabilities.subscriptionType, authMethod: capabilities.tokenSource, + apiKeySource: capabilities.apiKeySource, }); const usageCheckedAt = capabilities.usageCheckedAt ?? checkedAt; const usageLimits = !capabilities.usage diff --git a/apps/server/src/provider/CodexInstallation.test.ts b/apps/server/src/provider/CodexInstallation.test.ts index 98861629c79f..5536fe7a1704 100644 --- a/apps/server/src/provider/CodexInstallation.test.ts +++ b/apps/server/src/provider/CodexInstallation.test.ts @@ -41,6 +41,8 @@ const makeHarness = Effect.fn("test.makeCodexInstallation")(function* ( platform?: NodeJS.Platform; spawner?: ChildProcessSpawner.ChildProcessSpawner["Service"]; runtimeRenameFailures?: number; + /** Served by URL instead of the pinned package. */ + responses?: Readonly Response>>; } = {}, ) { const fs = yield* FileSystem.FileSystem; @@ -110,6 +112,8 @@ const makeHarness = Effect.fn("test.makeCodexInstallation")(function* ( HttpClient.make((request) => Effect.sync(() => { downloads++; + const respond = input.responses?.[request.url]; + if (respond) return HttpClientResponse.fromWeb(request, respond()); return Object.defineProperty( HttpClientResponse.fromWeb( request, @@ -529,4 +533,214 @@ it.effect("keeps an activated runtime when a later update fails verification", ( }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), ); -// Stands in for Windows' bsdtar: CRLF listings, and extraction writes the Windows layout. +const upstreamVersion = "0.163.0"; +const upstreamArchive = Buffer.from( + "H4sIAAAAAAACA+3WzY7CIBQF4D5Kw1oqtNQmPoh7rKStP9BQdMaYefcBE43T9RQ1nm8B7Q1Jk94cYN3peW026juZDvOqsrzO3nj2yofnUK9KwZKUJREcByet/2TymabuPby29S3/NIz0EIbWDC5u/sXf/PO8EAL5j+G/ew1vuP/TXrp2bpunnf/5KP+syCvkP4bJmg5vlf96JxuVbQej4+efi2qUf17g/h/Hhezl2RzdStmhM5osUz5Lyen+RljGF0XGiK/6H9UoF4pS2rpdCCr7fq/oRtqvTocVSjt77k2nr6vud0vyg6ABAAAAAAAAAAAAAABE8gvVbjvaACgAAA==", + "base64", +); +const upstreamReleaseUrl = `https://github.com/openai/codex/releases/download/rust-v${upstreamVersion}`; +const upstreamResponses = (checksum: string) => ({ + "https://registry.npmjs.org/@openai/codex/latest": () => + Response.json({ version: upstreamVersion }), + [`${upstreamReleaseUrl}/codex-package_SHA256SUMS`]: () => + new Response( + `${"0".repeat(64)} codex-package-x86_64-apple-darwin.tar.gz\n${checksum} codex-package-aarch64-apple-darwin.tar.gz\n`, + ), + [`${upstreamReleaseUrl}/codex-package-aarch64-apple-darwin.tar.gz`]: () => + new Response(upstreamArchive), +}); + +it.effect("updates a managed install to OpenAI's newest release and never downgrades it", () => + Effect.gen(function* () { + const digest = yield* Digest.Digest; + const checksum = Hex.encode( + yield* digest.digest(Digest.sha256, upstreamArchive).pipe(Effect.orDie), + ); + const h = yield* makeHarness({ responses: upstreamResponses(checksum) }); + yield* h.installation.checkForUpdate; + // A fresh install still gets the pinned release. + expect((yield* h.installation.state).version).toBe("0.156.1"); + yield* h.installation.start; + expect(yield* terminalState(h.installation)).toMatchObject({ + phase: "succeeded", + installedVersion: "0.156.1", + version: upstreamVersion, + }); + + yield* h.installation.start; + expect(yield* terminalState(h.installation)).toMatchObject({ + phase: "succeeded", + installedVersion: upstreamVersion, + version: upstreamVersion, + }); + expect((yield* h.installation.resolve()).version).toBe(upstreamVersion); + + // Without upstream information the older pin is not offered as a reinstall. + const restarted = yield* makeHarness({ baseDir: h.baseDir }); + expect(yield* restarted.installation.state).toMatchObject({ + installedVersion: upstreamVersion, + version: upstreamVersion, + }); + yield* restarted.installation.start; + expect(yield* terminalState(restarted.installation)).toMatchObject({ + phase: "succeeded", + installedVersion: upstreamVersion, + }); + expect(restarted.downloads()).toBe(0); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), +); + +it.effect("keeps the managed install when an update fails its published checksum", () => + Effect.gen(function* () { + const h = yield* makeHarness({ responses: upstreamResponses("f".repeat(64)) }); + yield* h.installation.start; + yield* terminalState(h.installation); + yield* h.installation.checkForUpdate; + yield* h.installation.start; + expect(yield* terminalState(h.installation)).toMatchObject({ + phase: "failed", + installedVersion: "0.156.1", + message: expect.stringContaining("SHA-256"), + }); + expect((yield* h.installation.resolve()).version).toBe("0.156.1"); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), +); + +it.effect("does not install a cached update the manifest has since withdrawn", () => + Effect.gen(function* () { + const digest = yield* Digest.Digest; + const checksum = Hex.encode( + yield* digest.digest(Digest.sha256, upstreamArchive).pipe(Effect.orDie), + ); + let manifest = ModelManifest.BUNDLED_MODEL_MANIFEST; + const h = yield* makeHarness({ + responses: upstreamResponses(checksum), + manifestCurrent: Effect.sync(() => manifest), + }); + yield* h.installation.start; + yield* terminalState(h.installation); + yield* h.installation.checkForUpdate; + expect((yield* h.installation.state).version).toBe(upstreamVersion); + manifest = { + ...manifest, + compatibility: [ + { + driver: "codex", + t3CodeRange: ">=0.0.42", + ranges: [ + { range: ">=0.160.0", status: "unsupported" }, + { range: ">=0.156.0 <0.160.0", status: "supported" }, + ], + }, + ], + }; + const downloads = h.downloads(); + yield* h.installation.start; + expect(yield* terminalState(h.installation)).toMatchObject({ + phase: "succeeded", + installedVersion: "0.156.1", + version: "0.156.1", + }); + expect(h.downloads()).toBe(downloads); + expect((yield* h.installation.resolve()).version).toBe("0.156.1"); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), +); + +it.effect("returns a managed install the manifest withdraws to the pinned release", () => + Effect.gen(function* () { + const digest = yield* Digest.Digest; + const checksum = Hex.encode( + yield* digest.digest(Digest.sha256, upstreamArchive).pipe(Effect.orDie), + ); + let manifest = ModelManifest.BUNDLED_MODEL_MANIFEST; + const h = yield* makeHarness({ + responses: upstreamResponses(checksum), + manifestCurrent: Effect.sync(() => manifest), + }); + yield* h.installation.checkForUpdate; + yield* h.installation.start; + yield* terminalState(h.installation); + yield* h.installation.start; + expect(yield* terminalState(h.installation)).toMatchObject({ + installedVersion: upstreamVersion, + }); + + manifest = { + ...manifest, + compatibility: [ + { + driver: "codex", + t3CodeRange: ">=0.0.42", + ranges: [ + { range: ">=0.160.0", status: "unsupported" }, + { range: ">=0.156.0 <0.160.0", status: "supported" }, + ], + }, + ], + }; + yield* h.installation.checkForUpdate; + expect(yield* h.installation.state).toMatchObject({ + installedVersion: upstreamVersion, + version: "0.156.1", + }); + yield* h.installation.start; + expect(yield* terminalState(h.installation)).toMatchObject({ + phase: "succeeded", + installedVersion: "0.156.1", + version: "0.156.1", + }); + expect((yield* h.installation.resolve()).version).toBe("0.156.1"); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), +); + +it.effect.each(["update", "reinstall"] as const)( + "rejects withdrawal during $0 validation", + (operation) => + Effect.gen(function* () { + const digest = yield* Digest.Digest; + const checksum = Hex.encode( + yield* digest.digest(Digest.sha256, upstreamArchive).pipe(Effect.orDie), + ); + let manifest = ModelManifest.BUNDLED_MODEL_MANIFEST; + let withdraw = operation === "update"; + const h = yield* makeHarness({ + responses: upstreamResponses(checksum), + manifestCurrent: Effect.sync(() => manifest), + options: { + validate: (_executable, version) => + Effect.sync(() => { + if (withdraw && version === upstreamVersion) + manifest = { + ...ModelManifest.BUNDLED_MODEL_MANIFEST, + compatibility: [ + { + driver: "codex", + t3CodeRange: ">=0.0.42", + ranges: [ + { range: ">=0.160.0", status: "unsupported" }, + { range: ">=0.156.0 <0.160.0", status: "supported" }, + ], + }, + ], + }; + }), + }, + }); + yield* h.installation.start; + yield* terminalState(h.installation); + yield* h.installation.checkForUpdate; + if (operation === "reinstall") { + yield* h.installation.start; + expect((yield* terminalState(h.installation)).phase).toBe("succeeded"); + withdraw = true; + } + yield* h.installation.start; + expect((yield* terminalState(h.installation)).phase).toBe("failed"); + if (operation === "update") { + expect((yield* h.installation.resolve()).version).toBe("0.156.1"); + } else { + expect((yield* h.installation.state).version).toBe("0.156.1"); + yield* Effect.flip(h.installation.resolve()); + } + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), +); diff --git a/apps/server/src/provider/CodexInstallation.ts b/apps/server/src/provider/CodexInstallation.ts index 8fdcfbebb328..c1e489ec0e1e 100644 --- a/apps/server/src/provider/CodexInstallation.ts +++ b/apps/server/src/provider/CodexInstallation.ts @@ -4,7 +4,7 @@ import { type ProviderInstallState, type ServerProviderCompatibilityAdvisory, } from "@t3tools/contracts"; -import { cliArchiveTarCommand } from "@t3tools/shared/cliRelease"; +import { cliArchiveTarCommand, parseChecksums } from "@t3tools/shared/cliRelease"; import { compareSemverVersions } from "@t3tools/shared/semver"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { resolveCommandPath, resolveSpawnCommand } from "@t3tools/shared/shell"; @@ -29,11 +29,18 @@ import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; import * as NodeCrypto from "node:crypto"; import * as ServerConfig from "../config.ts"; +import { selectManagedRelease } from "./managedRelease.ts"; import * as ModelManifest from "./ModelManifest.ts"; import { resolveProviderCompatibility } from "./providerCompatibility.ts"; const DRIVER = ProviderDriverKind.make("codex"); const STAGING_STALE_AFTER_MS = 24 * 60 * 60_000; +const UPDATE_CHECK_INTERVAL_MS = 60 * 60_000; +const UPDATE_CHECK_TIMEOUT = "20 seconds"; +const UPSTREAM_LATEST_URL = "https://registry.npmjs.org/@openai/codex/latest"; +const UPSTREAM_RELEASES_URL = "https://github.com/openai/codex/releases/download"; +// An upstream release has no recorded size; current packages are about 160 MB. +const UPSTREAM_MAX_ARCHIVE_BYTES = 1024 * 1024 * 1024; const Version = Schema.String.check(Schema.isPattern(/^\d+\.\d+\.\d+(?:-[a-zA-Z0-9.-]+)?$/u)); const ActiveRelease = Schema.Struct({ version: Version }); const InstalledRelease = Schema.Struct({ @@ -48,6 +55,7 @@ const PackageManifest = Schema.Struct({ entrypoint: Schema.String, }); const decodeVersion = Schema.decodeUnknownEffect(Version); +const decodeNpmLatest = Schema.decodeUnknownEffect(Schema.Struct({ version: Version })); const encodeRecord = Schema.encodeEffect(Schema.fromJsonString(InstalledRelease)); const encodeActive = Schema.encodeEffect(Schema.fromJsonString(ActiveRelease)); export interface CodexReleaseAsset { @@ -55,7 +63,8 @@ export interface CodexReleaseAsset { readonly target: string; readonly url: string; readonly sha256: string; - readonly archiveBytes: number; + /** Null for an upstream release, whose package checksum is published without a size. */ + readonly archiveBytes: number | null; } const RELEASES: Readonly> = { "darwin-arm64": { @@ -139,6 +148,11 @@ interface CodexInstallationService { ) => Effect.Effect; readonly state: Effect.Effect; readonly changes: Stream.Stream; + /** + * Looks up OpenAI's newest Codex release, at most hourly, and offers it + * through `state.version` when it is newer than the managed install. + */ + readonly checkForUpdate: Effect.Effect; readonly remove: ( protectedBinaryPaths?: ReadonlyArray, ) => Effect.Effect; @@ -186,15 +200,31 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function const activePath = path.join(managedDirectory, "active.json"); const executableName = platform === "win32" ? "codex.exe" : "codex"; const gate = yield* Semaphore.make(1); + const updateCheckGate = yield* Semaphore.make(1); let leases = 0; let running: { readonly operationId: string; readonly fiber: Fiber.Fiber } | undefined; + let upstream: CodexReleaseAsset | null = null; + let upstreamCheckedAt: number | null = null; + + // A local Codex is used as is, so only a managed install is ever updated. + const managedVersion = (current: ProviderInstallState) => + current.source === "managed" ? current.installedVersion : null; + // What `start` installs next. Clients compare `version` with + // `installedVersion` to offer an update. + const offer = (installedVersion: string | null) => { + const release = selectManagedRelease(installedVersion, asset, upstream); + return { + version: release?.version ?? installedVersion, + totalBytes: release?.archiveBytes ?? null, + }; + }; + const state = yield* SubscriptionRef.make({ driver: DRIVER, operationId: null, phase: "idle", downloadedBytes: 0, - totalBytes: asset?.archiveBytes ?? null, - version: asset?.version ?? null, + ...offer(null), installedVersion: null, executablePath: null, canRemove: false, @@ -279,6 +309,70 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function ) ); }); + const isSupported = (version: string) => + compatibility(version).pipe(Effect.map((advisory) => advisory?.status === "supported")); + // Forgets what the manifest withdrew after it was seen. Managed resolution + // refuses a withdrawn install, so the pin replaces it rather than a re-check. + const keptInstall = Effect.fn("CodexInstallation.keptInstall")(function* ( + installedVersion: string | null, + ) { + if (upstream && !(yield* isSupported(upstream.version))) upstream = null; + return installedVersion !== null && (yield* isSupported(installedVersion)) + ? installedVersion + : null; + }); + // npm names the latest stable release; its GitHub release lists the package + // checksums, so an upstream update stays SHA-256 verified. + const lookupUpstream = Effect.gen(function* () { + if (!asset) return null; + const latest = yield* http + .execute(HttpClientRequest.get(UPSTREAM_LATEST_URL)) + .pipe(Effect.flatMap(HttpClientResponse.filterStatusOk)); + const { version } = yield* latest.json.pipe(Effect.flatMap(decodeNpmLatest)); + // Managed resolution refuses anything else, so offering it would break sessions. + if (!(yield* isSupported(version))) return null; + const releaseUrl = `${UPSTREAM_RELEASES_URL}/rust-v${version}`; + const checksums = yield* http + .execute(HttpClientRequest.get(`${releaseUrl}/codex-package_SHA256SUMS`)) + .pipe( + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap((response) => response.text), + ); + const fileName = `codex-package-${asset.target}.tar.gz`; + const sha256 = parseChecksums(checksums).get(fileName); + if (!sha256) return null; + return { + version, + target: asset.target, + url: `${releaseUrl}/${fileName}`, + sha256, + archiveBytes: null, + } satisfies CodexReleaseAsset; + }); + // A check already in flight covers concurrent subscribers, including when it fails. + const checkForUpdate = updateCheckGate + .withPermitsIfAvailable(1)( + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis; + if (upstreamCheckedAt === null || now - upstreamCheckedAt >= UPDATE_CHECK_INTERVAL_MS) { + // A failed lookup counts too, and keeps whatever release was offered before. + upstreamCheckedAt = now; + upstream = yield* lookupUpstream.pipe( + Effect.timeout(UPDATE_CHECK_TIMEOUT), + Effect.orElseSucceed(() => upstream), + ); + } + // A running install already published the release it is installing. + yield* SubscriptionRef.updateEffect(state, (current) => + isRunning(current) + ? Effect.succeed(current) + : keptInstall(managedVersion(current)).pipe( + Effect.map((kept) => ({ ...current, ...offer(kept) })), + ), + ); + }), + ) + .pipe(Effect.ignore); const resolveManaged = Effect.fn("CodexInstallation.resolveManaged")( function* () { const active = yield* readRecord(activePath, ActiveRelease); @@ -464,6 +558,14 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function ); const install = Effect.fn("CodexInstallation.install")( function* (release: CodexReleaseAsset) { + const ensureEligible = Effect.fn("CodexInstallation.ensureEligible")(function* () { + if (release.version !== asset?.version && !(yield* isSupported(release.version))) { + return yield* installationError( + "verify", + "This Codex release was withdrawn. Check for updates and try again.", + ); + } + }); yield* decodeVersion(release.version); yield* fs.makeDirectory(managedDirectory, { recursive: true }); yield* SubscriptionRef.update(state, (current) => ({ ...current, canRemove: true })); @@ -480,12 +582,14 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function mode: 0o600, flag: "wx", }); + yield* ensureEligible(); yield* fs.rename(pointer, activePath); yield* SubscriptionRef.update( state, (current) => ({ ...current, + ...offer(release.version), phase: "succeeded", source: "managed", installedVersion: release.version, @@ -508,6 +612,7 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function "verify", "The existing managed Codex release differs from the official package. Remove it and reinstall.", ); + yield* ensureEligible(); yield* validate(existing, release.version).pipe( Effect.scoped, Effect.timeout("90 seconds"), @@ -523,6 +628,7 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function const runtime = path.join(staging, "runtime"); yield* fs.makeDirectory(runtime); const hash = NodeCrypto.createHash("sha256"); + const maxArchiveBytes = release.archiveBytes ?? UPSTREAM_MAX_ARCHIVE_BYTES; let downloadedBytes = 0; let lastProgressAt = yield* Clock.currentTimeMillis; const response = yield* http @@ -532,7 +638,7 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function Stream.tap((chunk) => Effect.gen(function* () { downloadedBytes += chunk.byteLength; - if (downloadedBytes > release.archiveBytes) + if (downloadedBytes > maxArchiveBytes) return yield* installationError( "download", "The Codex download exceeded the expected release size.", @@ -556,7 +662,10 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function ), }), ); - if (downloadedBytes !== release.archiveBytes || hash.digest("hex") !== release.sha256) + if ( + (release.archiveBytes !== null && downloadedBytes !== release.archiveBytes) || + hash.digest("hex") !== release.sha256 + ) return yield* installationError( "download", "The Codex download failed its size or SHA-256 check. Nothing was installed.", @@ -600,6 +709,7 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function }) satisfies ProviderInstallState, ); const executable = yield* fromDirectory(runtime, release.version, release.target); + yield* ensureEligible(); yield* validate(executable, release.version).pipe( Effect.scoped, Effect.timeout("90 seconds"), @@ -642,13 +752,54 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function ), ), ); + // Nothing newer is known, so "reinstall" re-checks the active runtime rather + // than replacing it with an older pinned release. + const reverify = Effect.fn("CodexInstallation.reverify")( + function* () { + const active = yield* readRecord(activePath, ActiveRelease); + const executable = yield* completedRelease(active.version); + yield* validate(executable, executable.version).pipe( + Effect.scoped, + Effect.timeout("90 seconds"), + ); + const kept = yield* keptInstall(executable.version); + if (kept === null) { + yield* SubscriptionRef.update(state, (current) => ({ ...current, ...offer(kept) })); + return yield* installationError( + "verify", + "This Codex release was withdrawn. Check for updates and try again.", + ); + } + yield* SubscriptionRef.update( + state, + (current) => + ({ + ...current, + ...offer(kept), + phase: "succeeded", + source: "managed", + installedVersion: executable.version, + executablePath: executable.executablePath, + message: null, + }) satisfies ProviderInstallState, + ); + }, + Effect.mapError( + wrapFailure( + "verify", + "The installed Codex runtime could not start. Remove it and reinstall.", + ), + ), + ); const start = gate .withPermit( Effect.gen(function* () { const current = yield* SubscriptionRef.get(state); if (isRunning(current)) return current; if (yield* reuseLocal()) return yield* SubscriptionRef.get(state); - if (!asset) { + const installedVersion = yield* keptInstall(managedVersion(current)); + const release = selectManagedRelease(installedVersion, asset, upstream); + if (!release && installedVersion === null) { return yield* installationError( "start", `OpenAI does not publish a Codex runtime for ${platform}-${arch}. Use a supported remote environment or a custom executable.`, @@ -658,14 +809,13 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function const next: ProviderInstallState = { ...current, operationId, - phase: "downloading", + phase: release ? "downloading" : "verifying", downloadedBytes: 0, - totalBytes: asset.archiveBytes, - version: asset.version, - message: "Downloading Codex.", + ...offer(installedVersion), + message: release ? "Downloading Codex." : "Checking Codex.", }; yield* SubscriptionRef.set(state, next); - const work = install(asset).pipe( + const work = (release ? install(release) : reverify()).pipe( Effect.onExit((exit) => Exit.isFailure(exit) ? SubscriptionRef.update(state, (value) => { @@ -745,6 +895,7 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function (current) => ({ ...current, + ...offer(null), operationId: null, phase: "idle", downloadedBytes: 0, @@ -796,11 +947,13 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function if (!(yield* fs.exists(activePath))) return; const active = yield* readRecord(activePath, ActiveRelease); const installed = yield* completedRelease(active.version); + const kept = yield* keptInstall(installed.version); yield* SubscriptionRef.update( state, (current) => ({ ...current, + ...offer(kept), installedVersion: installed.version, executablePath: installed.executablePath, source: "managed", @@ -832,6 +985,7 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function cancel, state: SubscriptionRef.get(state), changes: SubscriptionRef.changes(state), + checkForUpdate, remove, }); }); diff --git a/apps/server/src/provider/CodexProvider.test.ts b/apps/server/src/provider/CodexProvider.test.ts index c9a992bcf516..a70ea15a7357 100644 --- a/apps/server/src/provider/CodexProvider.test.ts +++ b/apps/server/src/provider/CodexProvider.test.ts @@ -1,11 +1,150 @@ import { assert, it } from "@effect/vitest"; +import * as Cause from "effect/Cause"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Queue from "effect/Queue"; +import * as Scope from "effect/Scope"; +import * as Sink from "effect/Sink"; +import * as Stream from "effect/Stream"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; +import * as TestClock from "effect/testing/TestClock"; +import * as HostProcess from "@t3tools/shared/HostProcess"; import { applyPreferredCodexDefaultModel, + codexCustomEndpoint, codexRoutesSpendNativeQuota, + makeCodexSkillsProbe, mapCodexModelCapabilities, } from "./CodexProvider.ts"; +function makeFakeCodexAppServers() { + const helpers: Array<{ + readonly codexHome: unknown; + readonly crash: Effect.Effect; + readonly requests: Array; + closed: boolean; + }> = []; + const spawner = ChildProcessSpawner.make((command) => + Effect.gen(function* () { + const output = yield* Queue.unbounded(); + const exited = yield* Deferred.make(); + const crash = Deferred.succeed(exited, ChildProcessSpawner.ExitCode(1)).pipe( + Effect.andThen(Queue.end(output)), + Effect.asVoid, + ); + const helper = { + codexHome: command._tag === "StandardCommand" ? command.options.env?.CODEX_HOME : undefined, + crash, + requests: [] as Array, + closed: false, + }; + helpers.push(helper); + yield* Effect.addFinalizer(() => Effect.sync(() => (helper.closed = true))); + const respond = (id: unknown, result: unknown) => + Queue.offer(output, new TextEncoder().encode(`${JSON.stringify({ id, result })}\n`)); + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(900_000_000 + helpers.length), + exitCode: Deferred.await(exited), + isRunning: Effect.map(Deferred.isDone(exited), (done) => !done), + kill: () => crash, + unref: Effect.succeed(Effect.void), + stdin: Sink.forEach((chunk: string | Uint8Array) => + Effect.forEach( + (typeof chunk === "string" ? chunk : new TextDecoder().decode(chunk)) + .split("\n") + .filter(Boolean), + (line) => { + const frame = JSON.parse(line) as { + readonly id?: number; + readonly method: string; + readonly params?: { readonly cwds: ReadonlyArray }; + }; + if (frame.id === undefined) return Effect.void; + if (frame.method === "initialize") { + return respond(frame.id, { + codexHome: "/codex-home", + platformFamily: "unix", + platformOs: "linux", + userAgent: "codex/0.0.0", + }); + } + helper.requests.push(frame.params); + return respond(frame.id, { + data: (frame.params?.cwds ?? []).map((cwd) => ({ + cwd, + errors: [], + skills: [ + { + name: `skill-in-${cwd}`, + path: `${cwd}/SKILL.md`, + enabled: true, + description: "", + scope: "repo", + }, + ], + })), + }); + }, + { discard: true }, + ), + ), + stdout: Stream.fromQueue(output), + stderr: Stream.empty, + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }); + }), + ); + return { helpers, spawner }; +} + +it.effect("reuses one Codex skills helper per launch until it idles out", () => + Effect.gen(function* () { + const { helpers, spawner } = makeFakeCodexAppServers(); + const providerScope = yield* Scope.make(); + const probe = yield* makeCodexSkillsProbe().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Scope.provide(providerScope), + ); + const read = (cwd: string, homePath = "/home-a") => + probe({ binaryPath: "codex", homePath, cwd, environment: {} }).pipe( + Effect.map((skills) => skills.map((skill) => skill.name)), + ); + + assert.deepStrictEqual(yield* read("/alpha"), ["skill-in-/alpha"]); + assert.deepStrictEqual(yield* read("/beta"), ["skill-in-/beta"]); + assert.strictEqual(helpers.length, 1); + assert.deepStrictEqual(helpers[0]!.requests, [ + { cwds: ["/alpha"], forceReload: true }, + { cwds: ["/beta"], forceReload: true }, + ]); + + // Another home is another launch identity; the previous helper retires at once. + assert.deepStrictEqual(yield* read("/alpha", "/home-b"), ["skill-in-/alpha"]); + assert.strictEqual(helpers.length, 2); + assert.strictEqual(helpers[0]!.closed, true); + assert.strictEqual(helpers[1]!.codexHome, "/home-b"); + + // A helper that exits is replaced within the same read. + yield* helpers[1]!.crash; + assert.deepStrictEqual(yield* read("/beta", "/home-b"), ["skill-in-/beta"]); + assert.strictEqual(helpers.length, 3); + + yield* TestClock.adjust("29 seconds"); + assert.strictEqual(helpers[2]!.closed, false); + yield* TestClock.adjust("1 second"); + assert.strictEqual(helpers[2]!.closed, true); + assert.deepStrictEqual(yield* read("/alpha", "/home-b"), ["skill-in-/alpha"]); + assert.strictEqual(helpers.length, 4); + + yield* Scope.close(providerScope, Exit.void); + assert.strictEqual(helpers[3]!.closed, true); + }).pipe(Effect.provideService(HostProcess.Platform, "linux")), +); + it("maps current Codex model capability fields", () => { const capabilities = mapCodexModelCapabilities({ additionalSpeedTiers: [], @@ -16,6 +155,7 @@ it("maps current Codex model capability fields", () => { id: "gpt-test", isDefault: true, model: "gpt-test", + availableAccessPrograms: { cyber: ["standard", "daybreakBlue"] }, defaultServiceTier: "flex", serviceTiers: [ { @@ -65,6 +205,16 @@ it("maps current Codex model capability fields", () => { ], currentValue: "flex", }, + { + id: "cyberAccessProgram", + label: "Cyber Access", + type: "select", + options: [ + { id: "standard", label: "Standard", isDefault: true }, + { id: "daybreakBlue", label: "Daybreak Blue" }, + ], + currentValue: "standard", + }, ]); }); @@ -181,4 +331,12 @@ it("reports native quota only for an OpenAI login on a native route", () => { name, ); } + // The custom endpoint is what a usage-limit hub is matched against. + assert.strictEqual(codexCustomEndpoint(hub), "https://hub.example.com"); + assert.strictEqual( + codexCustomEndpoint({ openai_base_url: "http://127.0.0.1:8317/v1" }), + "http://127.0.0.1:8317", + ); + assert.strictEqual(codexCustomEndpoint({ openai_base_url: "https://chatgpt.com/x" }), undefined); + assert.strictEqual(codexCustomEndpoint({}), undefined); }); diff --git a/apps/server/src/provider/CodexProvider.ts b/apps/server/src/provider/CodexProvider.ts index 0068e899e011..7dcf9079f82b 100644 --- a/apps/server/src/provider/CodexProvider.ts +++ b/apps/server/src/provider/CodexProvider.ts @@ -1,11 +1,14 @@ +import { resolveProviderProbeTimeoutMs } from "@t3tools/provider-core/server/snapshotProbe"; import { pipe } from "effect/Function"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as Equal from "effect/Equal"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import * as RcMap from "effect/RcMap"; import * as Result from "effect/Result"; import * as Schema from "effect/Schema"; import * as Schedule from "effect/Schedule"; @@ -56,7 +59,6 @@ import { import * as HostProcess from "@t3tools/shared/HostProcess"; import packageJson from "../../package.json" with { type: "json" }; const isCodexAppServerSpawnError = Schema.is(CodexErrors.CodexAppServerSpawnError); -const RATE_LIMITS_PROBE_TIMEOUT_MS = 3_000; type CodexRateLimitsProbe = | { @@ -82,17 +84,33 @@ const CodexRoutingConfig = Schema.Struct({ }); const decodeCodexRoutingConfig = Schema.decodeUnknownOption(CodexRoutingConfig); -function routesToNativeOpenai(config: unknown): boolean { +const CodexCyberAccessConfig = Schema.Struct({ + daybreak: Schema.optionalKey(Schema.NullOr(Schema.Boolean)), + features: Schema.optionalKey( + Schema.NullOr( + Schema.Struct({ + api_key_cyber_access_programs: Schema.optionalKey(Schema.NullOr(Schema.Boolean)), + }), + ), + ), +}); +const decodeCodexCyberAccessConfig = Schema.decodeUnknownOption(CodexCyberAccessConfig); + +// Null is Codex's auth-dependent OpenAI default; undefined is a route that cannot be read. +function codexRouteBaseUrl(config: unknown): string | null | undefined { const decoded = decodeCodexRoutingConfig(config); - if (Option.isNone(decoded)) return false; + if (Option.isNone(decoded)) return undefined; const routing = decoded.value; const providerId = routing.model_provider ?? "openai"; + if (providerId === "openai") return routing.openai_base_url || null; const provider = routing.model_providers?.[providerId]; - if (providerId !== "openai" && !provider) return false; - const baseUrl = - providerId === "openai" ? routing.openai_base_url || undefined : provider?.base_url; - // A provider without an override uses Codex's auth-dependent OpenAI default. - if (baseUrl == null) return true; + return provider ? (provider.base_url ?? null) : undefined; +} + +function routesToNativeOpenai(config: unknown): boolean { + const baseUrl = codexRouteBaseUrl(config); + if (baseUrl === undefined) return false; + if (baseUrl === null) return true; if (!URL.canParse(baseUrl)) return false; const url = new URL(baseUrl); return ( @@ -101,6 +119,15 @@ function routesToNativeOpenai(config: unknown): boolean { ); } +/** The origin of the custom endpoint Codex sends model requests to, if it uses one. */ +export function codexCustomEndpoint(config: unknown): string | undefined { + const baseUrl = codexRouteBaseUrl(config); + const url = baseUrl ? URL.parse(baseUrl) : null; + return url && /^https?:$/.test(url.protocol) && !routesToNativeOpenai(config) + ? url.origin + : undefined; +} + // A stored login does not mean this provider's traffic spends that login's quota. export const codexRoutesSpendNativeQuota = ( account: CodexSchema.V2GetAccountResponse, @@ -128,11 +155,15 @@ const CODEX_PRESENTATION = { export interface CodexAppServerProviderSnapshot { readonly account: CodexSchema.V2GetAccountResponse; + readonly endpoint?: string | undefined; readonly nativeUsage?: boolean; readonly rateLimits?: CodexRateLimitsProbe; readonly version: string | undefined; readonly models: ReadonlyArray | undefined; readonly serviceTier?: string | undefined; + /** Undefined when `config/read` failed, so T3 cannot know Codex's default program. */ + readonly daybreakEnabled?: boolean | undefined; + readonly cyberAccessProgramsUnavailable?: boolean | undefined; readonly skills: ReadonlyArray; } @@ -153,6 +184,12 @@ const SERVICE_TIER_DESCRIPTIONS: Readonly> = { ultrafast: "Even faster, more expensive", }; +const CYBER_ACCESS_PROGRAM_LABELS: Readonly> = { + standard: "Standard", + daybreakBlue: "Daybreak Blue", + daybreakRed: "Daybreak Red", +}; + function reasoningEffortLabel(reasoningEffort: string): string { return REASONING_EFFORT_LABELS[reasoningEffort] ?? reasoningEffort; } @@ -281,6 +318,25 @@ export function mapCodexModelCapabilities( currentValue: defaultServiceTier, }); } + const cyberAccessPrograms = model.availableAccessPrograms?.cyber ?? []; + if (cyberAccessPrograms.length > 0) { + const hasStandard = cyberAccessPrograms.includes("standard"); + const defaultProgram = hasStandard ? "standard" : cyberAccessPrograms[0]; + optionDescriptors.push({ + id: "cyberAccessProgram", + label: "Cyber Access", + type: "select", + // Standard stays selectable as an opt-out; Codex rejects it visibly when the login cannot use it. + options: (hasStandard ? cyberAccessPrograms : ["standard", ...cyberAccessPrograms]).map( + (program) => ({ + id: program, + label: CYBER_ACCESS_PROGRAM_LABELS[program] ?? program, + ...(program === defaultProgram ? { isDefault: true } : {}), + }), + ), + ...(defaultProgram === undefined ? {} : { currentValue: defaultProgram }), + }); + } return createModelCapabilities({ optionDescriptors, @@ -320,37 +376,95 @@ export function applyPreferredCodexDefaultModel( }); } -function applyConfiguredCodexServiceTier( +function mapCodexOptionDescriptors( models: ReadonlyArray, - serviceTier: string | undefined, + update: ( + descriptors: ReadonlyArray, + model: ServerProviderModel, + ) => ReadonlyArray, ): ReadonlyArray { - if (!serviceTier) return models; return models.map((model) => { if (!model.capabilities?.optionDescriptors) return model; return { ...model, capabilities: { ...model.capabilities, - optionDescriptors: model.capabilities.optionDescriptors.map((descriptor) => { - if (descriptor.id !== "serviceTier" || descriptor.type !== "select") return descriptor; - const value = descriptor.options.find( - (option) => - option.id === serviceTier || (serviceTier === "fast" && option.id === "priority"), - )?.id; - if (!value) return descriptor; - return { - ...descriptor, - currentValue: value, - options: descriptor.options.map(({ isDefault: _isDefault, ...option }) => - option.id === value ? { ...option, isDefault: true } : option, - ), - }; - }), + optionDescriptors: update(model.capabilities.optionDescriptors, model), }, }; }); } +function withConfiguredSelectDefault( + models: ReadonlyArray, + descriptorId: string, + pick: (options: ReadonlyArray<{ readonly id: string }>) => string | undefined, +): ReadonlyArray { + return mapCodexOptionDescriptors(models, (descriptors) => + descriptors.map((descriptor) => { + if (descriptor.id !== descriptorId || descriptor.type !== "select") return descriptor; + const value = pick(descriptor.options); + if (!value) return descriptor; + return { + ...descriptor, + currentValue: value, + options: descriptor.options.map(({ isDefault: _isDefault, ...option }) => + option.id === value ? { ...option, isDefault: true } : option, + ), + }; + }), + ); +} + +function applyConfiguredCodexServiceTier( + models: ReadonlyArray, + serviceTier: string | undefined, +): ReadonlyArray { + if (!serviceTier) return models; + return withConfiguredSelectDefault( + models, + "serviceTier", + (options) => + options.find( + (option) => + option.id === serviceTier || (serviceTier === "fast" && option.id === "priority"), + )?.id, + ); +} + +/** Mirrors Codex: `daybreak = true` prefers a Daybreak program, Blue before Red. */ +function applyConfiguredCodexCyberAccess( + models: ReadonlyArray, + snapshot: Pick< + CodexAppServerProviderSnapshot, + "daybreakEnabled" | "cyberAccessProgramsUnavailable" + >, +): ReadonlyArray { + // Without a descriptor no turn sends a program; without a default only an explicit choice goes. + const offered = mapCodexOptionDescriptors(models, (descriptors, model) => + descriptors.flatMap((descriptor): ReadonlyArray => { + if (descriptor.id !== "cyberAccessProgram" || descriptor.type !== "select") { + return [descriptor]; + } + if (model.isCustom || snapshot.cyberAccessProgramsUnavailable === true) return []; + if (snapshot.daybreakEnabled !== undefined) return [descriptor]; + const { currentValue: _currentValue, ...withoutDefault } = descriptor; + return [ + { + ...withoutDefault, + options: descriptor.options.map(({ isDefault: _isDefault, ...option }) => option), + }, + ]; + }), + ); + if (snapshot.daybreakEnabled !== true) return offered; + return withConfiguredSelectDefault(offered, "cyberAccessProgram", (options) => + ["daybreakBlue", "daybreakRed"].find((program) => + options.some((option) => option.id === program), + ), + ); +} + /** * Codex has no static default capability set, so a bare custom slug borrows * the first built-in's descriptors; an entry with its own capabilities keeps @@ -571,11 +685,7 @@ const readCodexRateLimits = ( client: CodexClient.CodexAppServerClient["Service"], deadlineMs: number, ) => - withCodexDiscoveryDeadline( - client.request("account/rateLimits/read", null), - deadlineMs, - RATE_LIMITS_PROBE_TIMEOUT_MS, - ).pipe( + withCodexDiscoveryDeadline(client.request("account/rateLimits/read", null), deadlineMs).pipe( Effect.map((response): CodexRateLimitsProbe => response ? { @@ -594,6 +704,7 @@ const readCodexRateLimits = ( ); const probeCodexAppServerProvider = Effect.fn("probeCodexAppServerProvider")(function* (input: { + readonly probeTimeoutMs: number; readonly binaryPath: string; readonly homePath?: string; readonly launchArgs?: string; @@ -618,8 +729,8 @@ const probeCodexAppServerProvider = Effect.fn("probeCodexAppServerProvider")(fun } // Optional discovery must finish before the outer timeout can discard the account result. - const deadlineMs = spawnedAtMs + CODEX_AUTH_PROBE_TIMEOUT_MS - CODEX_DISCOVERY_MARGIN_MS; - const [skillsResponse, models, { serviceTier, nativeUsage, rateLimits }] = yield* Effect.all( + const deadlineMs = spawnedAtMs + input.probeTimeoutMs - CODEX_DISCOVERY_MARGIN_MS; + const [skillsResponse, models, { config, nativeUsage, rateLimits }] = yield* Effect.all( [ withCodexDiscoveryDeadline(client.request("skills/list", { cwds: [input.cwd] }), deadlineMs), accountResponse.account?.type === "chatgpt" @@ -648,7 +759,7 @@ const probeCodexAppServerProvider = Effect.fn("probeCodexAppServerProvider")(fun : Effect.undefined; return readUsage.pipe( Effect.map((rateLimits) => ({ - serviceTier: config?.config.service_tier ?? undefined, + config: config?.config, nativeUsage, rateLimits, })), @@ -659,13 +770,20 @@ const probeCodexAppServerProvider = Effect.fn("probeCodexAppServerProvider")(fun { concurrency: "unbounded" }, ); + const cyberAccess = Option.getOrUndefined(decodeCodexCyberAccessConfig(config)); return { account: accountResponse, + endpoint: config ? codexCustomEndpoint(config) : undefined, nativeUsage, ...(rateLimits ? { rateLimits } : {}), version, models, - serviceTier, + serviceTier: config?.service_tier ?? undefined, + daybreakEnabled: config === undefined ? undefined : cyberAccess?.daybreak === true, + // An unread config cannot rule out an API key with the programs turned off. + cyberAccessProgramsUnavailable: + accountResponse.account?.type === "apiKey" && + (config === undefined || cyberAccess?.features?.api_key_cyber_access_programs === false), skills: skillsResponse ? parseCodexSkillsListResponse(skillsResponse, input.cwd) : [], } satisfies CodexAppServerProviderSnapshot; }); @@ -696,7 +814,12 @@ const probeCodexCliVersion = Effect.fn("probeCodexCliVersion")( if (result.code !== 0) return null; return CODEX_CLI_VERSION_PATTERN.exec(result.stdout)?.[1] ?? null; }, - Effect.timeoutOption(Duration.millis(DEFAULT_TIMEOUT_MS)), + (effect, input) => + effect.pipe( + Effect.timeoutOption( + resolveProviderProbeTimeoutMs("codex", DEFAULT_TIMEOUT_MS, input.environment), + ), + ), Effect.map(Option.getOrNull), Effect.catch((cause) => Effect.logDebug("Codex --version fallback failed.", { cause }).pipe(Effect.as(null)), @@ -718,6 +841,52 @@ export const probeCodexSkillsForCwd = Effect.fn("probeCodexSkillsForCwd")(functi return parseCodexSkillsListResponse(skillsResponse, input.cwd); }); +// Long enough for a burst of workspace switches; short because the helper holds the user's +// binary open, and Windows refuses to overwrite or delete a running executable. +const CODEX_SKILLS_HELPER_IDLE_TTL = "30 seconds"; + +/** Reuses one app-server per launch identity; each read passes its cwd and rescans from disk. */ +export const makeCodexSkillsProbe = Effect.fn("makeCodexSkillsProbe")(function* () { + const helpers = yield* RcMap.make({ + lookup: (launch: { + readonly binaryPath: string; + readonly homePath?: string; + readonly launchArgs?: string; + readonly environment: NodeJS.ProcessEnv; + }) => + withCodexAppServerClient({ ...launch, cwd: process.cwd() }).pipe( + Effect.map(({ client }) => client), + ), + idleTimeToLive: CODEX_SKILLS_HELPER_IDLE_TTL, + }); + return Effect.fn("CodexSkillsProbe.probe")(function* (input: { + readonly binaryPath: string; + readonly homePath?: string; + readonly launchArgs?: string; + readonly cwd: string; + readonly environment?: NodeJS.ProcessEnv; + }) { + const { cwd, ...rest } = input; + const launch = { + ...rest, + environment: providerProbeEnvironment(rest.environment ?? process.env), + }; + // Retire helpers for a previous binary, home, credentials, or launch options. + const stale = Array.from(yield* RcMap.keys(helpers)).filter( + (key) => !Equal.equals(key, launch), + ); + yield* Effect.forEach(stale, (key) => RcMap.invalidate(helpers, key), { discard: true }); + const skillsResponse = yield* RcMap.get(helpers, launch).pipe( + Effect.flatMap((client) => client.request("skills/list", { cwds: [cwd], forceReload: true })), + Effect.scoped, + // RcMap keeps a failed spawn, and a dead helper keeps failing; drop it so the retry respawns. + Effect.onError(() => RcMap.invalidate(helpers, launch)), + Effect.retry({ times: 1 }), + ); + return parseCodexSkillsListResponse(skillsResponse, cwd); + }); +}); + const emptyCodexModelsFromSettings = (codexSettings: CodexSettings): ServerProvider["models"] => appendCustomCodexModels([], codexSettings.customModels); @@ -760,7 +929,12 @@ const makePendingCodexProvider = Effect.fn("CodexProvider.makePendingCodexProvid }); }); -function accountProbeStatus({ account, rateLimits }: CodexAppServerProviderSnapshot): { +const quotePath = Schema.encodeSync(Schema.fromJsonString(Schema.String)); + +function accountProbeStatus( + { account, rateLimits }: CodexAppServerProviderSnapshot, + codexHome: string | undefined, +): { readonly status: Exclude; readonly auth: ServerProvider["auth"]; readonly message?: string; @@ -788,7 +962,9 @@ function accountProbeStatus({ account, rateLimits }: CodexAppServerProviderSnaps return { status: "error", auth: { status: "unauthenticated" }, - message: "Codex CLI is not authenticated. Run `codex login` and try again.", + message: codexHome + ? `Codex CLI is not authenticated. Run \`codex login\` with CODEX_HOME set to ${quotePath(codexHome)} and try again.` + : "Codex CLI is not authenticated. Run `codex login` and try again.", }; } @@ -798,6 +974,7 @@ function accountProbeStatus({ account, rateLimits }: CodexAppServerProviderSnaps export const checkCodexProviderStatus = Effect.fn("checkCodexProviderStatus")(function* ( codexSettings: CodexSettings, probe: (input: { + readonly probeTimeoutMs: number; readonly binaryPath: string; readonly homePath?: string; readonly launchArgs?: string; @@ -837,7 +1014,13 @@ export const checkCodexProviderStatus = Effect.fn("checkCodexProviderStatus")(fu }); } + const probeTimeoutMs = resolveProviderProbeTimeoutMs( + "codex", + CODEX_AUTH_PROBE_TIMEOUT_MS, + resolvedEnvironment, + ); const probeResult = yield* probe({ + probeTimeoutMs, binaryPath: codexSettings.binaryPath, homePath: codexSettings.homePath, launchArgs: resolveCodexLaunchArgs(codexSettings.launchArgs, resolvedEnvironment), @@ -846,7 +1029,7 @@ export const checkCodexProviderStatus = Effect.fn("checkCodexProviderStatus")(fu ...(managedAuth ? { skipNativeUsage: true } : {}), }).pipe( // Process teardown can outlast the work budget without discarding a completed snapshot. - Effect.timeoutOption(Duration.millis(CODEX_AUTH_PROBE_TIMEOUT_MS)), + Effect.timeoutOption(Duration.millis(probeTimeoutMs)), Effect.scoped, Effect.result, ); @@ -903,7 +1086,12 @@ export const checkCodexProviderStatus = Effect.fn("checkCodexProviderStatus")(fu const snapshot = probeResult.success.value; const accountStatus = managedAuth ? { status: "ready" as const, auth: managedAuth, message: undefined } - : accountProbeStatus(snapshot); + : accountProbeStatus( + snapshot, + codexSettings.homePath + ? expandHomePath(codexSettings.homePath, yield* HostProcess.HomeDirectory) + : undefined, + ); // Sign-in guidance still applies when Codex itself says it needs an OpenAI login. const usageLimits = !snapshot.nativeUsage && !accountStatus.message @@ -930,6 +1118,7 @@ export const checkCodexProviderStatus = Effect.fn("checkCodexProviderStatus")(fu appendCustomCodexModels(snapshot.models ?? [], codexSettings.customModels), applyPreferredCodexDefaultModel, (models) => applyConfiguredCodexServiceTier(models, snapshot.serviceTier), + (models) => applyConfiguredCodexCyberAccess(models, snapshot), ), skills: snapshot.skills, slashCommands: [ @@ -949,7 +1138,9 @@ export const checkCodexProviderStatus = Effect.fn("checkCodexProviderStatus")(fu installed: true, version: snapshot.version ?? null, status: snapshot.models === undefined ? "warning" : accountStatus.status, - auth: accountStatus.auth, + auth: snapshot.endpoint + ? { ...accountStatus.auth, endpoint: snapshot.endpoint } + : accountStatus.auth, ...(accountStatus.message ? { message: accountStatus.message } : snapshot.models === undefined diff --git a/apps/server/src/provider/CodexThreadRevert.test.ts b/apps/server/src/provider/CodexThreadRevert.test.ts index e48caf094027..f26871915472 100644 --- a/apps/server/src/provider/CodexThreadRevert.test.ts +++ b/apps/server/src/provider/CodexThreadRevert.test.ts @@ -68,7 +68,6 @@ it.effect("finds the revert boundary across pages of newest-first turns", () => ], }), ), - Effect.scoped, ), ); @@ -87,6 +86,5 @@ it.effect("rejects repeated cursors instead of looping or reverting incomplete h entries: [...page(1, null, 3, ["newest"], "again"), ...page(2, "again", 2, [], "again")], }), ), - Effect.scoped, ), ); diff --git a/apps/server/src/provider/DocumentText.ts b/apps/server/src/provider/DocumentText.ts index 83aa014257ae..d0e53121ae33 100644 --- a/apps/server/src/provider/DocumentText.ts +++ b/apps/server/src/provider/DocumentText.ts @@ -1,6 +1,7 @@ // @effect-diagnostics nodeBuiltinImport:off -- the isolated parser process is an external runtime boundary. import * as NodeChildProcess from "node:child_process"; import * as NodeURL from "node:url"; +import * as Cache from "effect/Cache"; import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; @@ -48,13 +49,17 @@ const make = Effect.gen(function* () { const isExecutable = yield* HostProcess.IsExecutable; const environment = yield* HostProcess.Environment; const clock = yield* Clock.Clock; - const lock = yield* Semaphore.make(1); + // A large DOCX worker peaks near 270 MiB RSS, so two slots bound parser memory near 540 MiB. + const lock = yield* Semaphore.make(2); const failed = new Map(); + // A cancelled extraction can still be publishing while its retry runs in the other slot. + let attempt = 0; // In a bundle import.meta.url is bin.mjs; development and tests use this source module. const entry = NodeURL.fileURLToPath( new URL(import.meta.url.endsWith(".ts") ? "../bin.ts" : "./bin.mjs", import.meta.url), ); - const extract = Effect.fn("DocumentText.extract")(function* (path: string, extension: string) { + const extract = Effect.fn("DocumentText.extract")(function* (path: string) { + const extension = path.split(".").at(-1)!.toLowerCase(); const sidecar = `${path}.txt`; if (yield* fs.exists(sidecar)) return sidecar; if ((failed.get(path) ?? 0) > clock.currentTimeMillisUnsafe()) return; @@ -94,7 +99,7 @@ const make = Effect.gen(function* () { return Effect.logWarning("Document text extraction failed.", { extension }); }), ); - const pending = `${sidecar}.part`; + const pending = `${sidecar}.${attempt++}.part`; yield* fs .writeFileString(pending, content) .pipe( @@ -104,13 +109,14 @@ const make = Effect.gen(function* () { ); return sidecar; }, lock.withPermits(1)); + const extracting = yield* Cache.make({ capacity: Infinity, timeToLive: 0, lookup: extract }); const ensure = Effect.fn("DocumentText.ensure")( function* (path: string) { const extension = path.split(".").at(-1)?.toLowerCase(); if (!extension || !extensions.has(extension)) return; const sidecar = `${path}.txt`; if (yield* fs.exists(sidecar)) return sidecar; - return yield* extract(path, extension); + return yield* Cache.get(extracting, path); }, Effect.orElseSucceed(() => undefined), ); diff --git a/apps/server/src/provider/Drivers/AntigravityDriver.test.ts b/apps/server/src/provider/Drivers/AntigravityDriver.test.ts index 3747ff032d29..447745fc673a 100644 --- a/apps/server/src/provider/Drivers/AntigravityDriver.test.ts +++ b/apps/server/src/provider/Drivers/AntigravityDriver.test.ts @@ -383,7 +383,7 @@ it.layer(layerTest)("AntigravityDriver", (it) => { const helper = { fs: { readTextFile: false, writeTextFile: false }, terminal: false, - session: { notices: {} }, + session: { notices: {}, compaction: {} }, }; expect( (yield* h.readRequests) @@ -397,7 +397,7 @@ it.layer(layerTest)("AntigravityDriver", (it) => { { fs: { readTextFile: true, writeTextFile: false }, terminal: false, - session: { notices: {} }, + session: { notices: {}, compaction: {} }, elicitation: { form: {} }, }, ]); @@ -426,7 +426,13 @@ it.layer(layerTest)("AntigravityDriver", (it) => { modelSelection, runtimePolicy: cliPolicy, }); - expect(h.launches.at(-1)?.args).toEqual(["acp"]); + expect(h.launches.at(-1)?.args).toEqual([ + "acp", + "--gemini-dir", + h.profileDirectory, + "--app-data-dir", + "antigravity-acp", + ]); expect( (yield* h.readRequests) .slice(requestsBefore) @@ -438,7 +444,7 @@ it.layer(layerTest)("AntigravityDriver", (it) => { .map((request) => request.params?.value), ).toEqual(["default"]); } - }).pipe(Effect.scoped), + }), ); it.effect.skipIf(windowsHost)( @@ -458,7 +464,7 @@ it.layer(layerTest)("AntigravityDriver", (it) => { expect(h.acquisitions).toHaveLength(2); expect(h.releases).toEqual([h.first.version, h.signedOut.version]); yield* h.assertClosed; - }).pipe(Effect.scoped), + }), ); it.effect.skipIf(windowsHost)("keeps the previous catalog when executable resolution fails", () => @@ -475,7 +481,7 @@ it.layer(layerTest)("AntigravityDriver", (it) => { expect(h.acquisitions).toHaveLength(2); expect(h.releases).toEqual([h.first.version]); yield* h.assertClosed; - }).pipe(Effect.scoped), + }), ); it.effect.skipIf(windowsHost)( @@ -496,7 +502,7 @@ it.layer(layerTest)("AntigravityDriver", (it) => { expect(after.models).toEqual(before.models); expect(after.auth).toEqual(before.auth); yield* h.assertClosed; - }).pipe(Effect.scoped), + }), ); it.effect.skipIf(windowsHost)( @@ -533,7 +539,7 @@ it.layer(layerTest)("AntigravityDriver", (it) => { expect(after.auth).toEqual(before.auth); expect(JSON.stringify(after)).not.toContain("sk-secret-123"); yield* h.assertClosed; - }).pipe(Effect.scoped), + }), ); it.effect.skipIf(windowsHost)( @@ -571,6 +577,6 @@ it.layer(layerTest)("AntigravityDriver", (it) => { ); expect(yield* fs.exists(directories.runtimeTemp)).toBe(false); expect(yield* fs.exists(legacyRoot)).toBe(false); - }).pipe(Effect.scoped), + }), ); }); diff --git a/apps/server/src/provider/Drivers/AntigravityDriver.ts b/apps/server/src/provider/Drivers/AntigravityDriver.ts index 7bb1f25b67ce..75bca5acc87b 100644 --- a/apps/server/src/provider/Drivers/AntigravityDriver.ts +++ b/apps/server/src/provider/Drivers/AntigravityDriver.ts @@ -434,6 +434,7 @@ export const AntigravityDriver: ProviderDriver< Effect.provideService(HttpClient.HttpClient, httpClient), ); const provider = yield* makeAntigravityProvider(settings, { + environment: processEnvironment, stampIdentity: classifyModels, probe, probeUsage: usage.probe, @@ -462,6 +463,11 @@ export const AntigravityDriver: ProviderDriver< makeRuntime, withProcess: authFlow.withProcess, defaultModel, + supportsNativeGoals: provider.snapshot.getSnapshot.pipe( + Effect.map( + (snapshot) => snapshot.slashCommands?.some((command) => command.name === "goal") === true, + ), + ), profileDirectory, onSessionStarted: provider.onSessionStarted, onSessionEvent: (event) => { diff --git a/apps/server/src/provider/Drivers/AntigravitySkills.test.ts b/apps/server/src/provider/Drivers/AntigravitySkills.test.ts index 5060d8a24e5e..c81b38a00dbc 100644 --- a/apps/server/src/provider/Drivers/AntigravitySkills.test.ts +++ b/apps/server/src/provider/Drivers/AntigravitySkills.test.ts @@ -273,15 +273,43 @@ it.layer(NodeServices.layer)("discoverAntigravitySkills", (it) => { }), ); - it.effect("rejects an oversized skill instead of returning an incomplete catalog", () => + it.effect("does not spend the scan budget on skill bodies", () => Effect.gen(function* () { const path = yield* Path.Path; const input = yield* makeWorkspace(); - const skillPath = yield* writeSkill( - path.join(input.cwd, ".agents", "skills", "oversized"), - `---\nname: oversized\ndescription: Read a large skill.\n---\n${"x".repeat(1_000_000)}`, + const root = path.join(input.userHome, ".gemini", "config", "skills"); + const expected = []; + for (let index = 0; index < 9; index += 1) { + const name = `large-${index}`; + const description = `Large skill ${index}. ${"y".repeat(2_000)}`; + const skillPath = yield* writeSkill( + path.join(root, name), + `---\nname: ${name}\ndescription: ${description}\n---\n${"x".repeat(950_000)}`, + ); + expected.push({ name, description, path: skillPath, scope: "user", enabled: true }); + } + + assert.deepEqual(yield* discoverAntigravitySkills(input), expected); + }), + ); + + it.effect("rejects frontmatter that does not close within the per-skill limit", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const input = yield* makeWorkspace(); + const root = path.join(input.cwd, ".agents", "skills"); + const largeBodyPath = yield* writeSkill( + path.join(root, "large-body"), + `---\nname: large-body\n---\n${"x".repeat(2_000_000)}`, ); + assert.deepEqual(yield* discoverAntigravitySkills(input), [ + { name: "large-body", path: largeBodyPath, scope: "project", enabled: true }, + ]); + const skillPath = yield* writeSkill( + path.join(root, "oversized"), + `---\nname: oversized\ndescription: ${"x".repeat(1_000_000)}\n---\n`, + ); const result = yield* discoverAntigravitySkills(input).pipe(Effect.result); assert.equal(result._tag, "Failure"); if (result._tag === "Failure") { @@ -291,7 +319,7 @@ it.layer(NodeServices.layer)("discoverAntigravitySkills", (it) => { }), ); - it.effect("bounds the total read size across skills", () => + it.effect("bounds the total frontmatter read size across skills", () => Effect.gen(function* () { const path = yield* Path.Path; const input = yield* makeWorkspace(); @@ -299,7 +327,7 @@ it.layer(NodeServices.layer)("discoverAntigravitySkills", (it) => { for (let index = 0; index < 9; index += 1) { yield* writeSkill( path.join(root, `large-${index}`), - `---\nname: large-${index}\n---\n${"x".repeat(900_000)}`, + `---\nname: large-${index}\ndescription: ${"x".repeat(900_000)}\n---\n`, ); } diff --git a/apps/server/src/provider/Drivers/AntigravitySkills.ts b/apps/server/src/provider/Drivers/AntigravitySkills.ts index 19e9d8d293f4..96e1fe61109f 100644 --- a/apps/server/src/provider/Drivers/AntigravitySkills.ts +++ b/apps/server/src/provider/Drivers/AntigravitySkills.ts @@ -37,6 +37,8 @@ export function antigravityUserSkillDirectories( const MAX_SKILL_BYTES = 1_000_000; const MAX_SCAN_BYTES = 8_000_000; const MAX_SCAN_ENTRIES = 10_000; +// Typical frontmatter fits in one chunk, so most skills cost about this much of the scan budget. +const SKILL_READ_CHUNK_BYTES = 1024; const SkillFrontmatter = Schema.Struct({ name: Schema.optional(Schema.NullOr(Schema.String)), @@ -106,7 +108,29 @@ function skillPathSortKey(entry: string) { ); } -const readSkill = Effect.fn("readAntigravitySkill")(function* ( +const DASH = 0x2d; + +/** Finds the byte offset after the second `---`, as parseSkillFrontmatter's indexOf calls do. */ +function makeFrontmatterEndFinder() { + let bytesSeen = 0; + let dashes = 0; + let markers = 0; + return (chunk: Uint8Array) => { + for (const byte of chunk) { + bytesSeen += 1; + dashes = byte === DASH ? dashes + 1 : 0; + if (dashes === 3) { + dashes = 0; + markers += 1; + if (markers === 2) return bytesSeen; + } + } + return undefined; + }; +} + +/** Reads through the frontmatter only, so skill bodies do not spend the scan budget. */ +const readSkillFrontmatter = Effect.fn("readAntigravitySkill")(function* ( skillPath: string, budget: ScanBudget, ) { @@ -115,26 +139,27 @@ const readSkill = Effect.fn("readAntigravitySkill")(function* ( if (info?.type !== "File") return undefined; const byteLimit = Math.min(MAX_SKILL_BYTES, budget.remainingBytes); - if (info.size > BigInt(byteLimit)) { - return yield* new AntigravitySkillsProbeError({ - reason: "scan-budget-exhausted", - path: skillPath, - }); - } + const findFrontmatterEnd = makeFrontmatterEndFinder(); + let frontmatterEnd: number | undefined; const chunks = yield* readIfPresent( - fileSystem.stream(skillPath, { bytesToRead: byteLimit + 1 }).pipe(Stream.runCollect), + fileSystem + .stream(skillPath, { bytesToRead: byteLimit + 1, chunkSize: SKILL_READ_CHUNK_BYTES }) + .pipe( + Stream.takeUntil((chunk) => (frontmatterEnd = findFrontmatterEnd(chunk)) !== undefined), + Stream.runCollect, + ), skillPath, ); if (chunks === undefined) return undefined; const bytes = Buffer.concat(chunks); - if (bytes.byteLength > byteLimit) { + budget.remainingBytes -= bytes.byteLength; + if ((frontmatterEnd ?? bytes.byteLength) > byteLimit) { return yield* new AntigravitySkillsProbeError({ reason: "scan-budget-exhausted", path: skillPath, }); } - budget.remainingBytes -= bytes.byteLength; - return bytes.toString("utf8"); + return bytes.toString("utf8", 0, frontmatterEnd); }); /** Keep read errors typed so workspace snapshots cannot cache partial results. */ @@ -187,7 +212,7 @@ export const discoverAntigravitySkills = Effect.fn("discoverAntigravitySkills")( if (skillFileName !== undefined) { if (!skillFileName.endsWith(".md")) return; const skillPath = path.join(directory, skillFileName); - const contents = yield* readSkill(skillPath, budget); + const contents = yield* readSkillFrontmatter(skillPath, budget); if (contents === undefined) return; const skill = parseSkillFrontmatter(contents, skillFileName); if (!skill || skillsByName.has(skill.name)) return; diff --git a/apps/server/src/provider/Drivers/AntigravityUsage.ts b/apps/server/src/provider/Drivers/AntigravityUsage.ts index 5cb55ed77ac0..2eb9744a3e42 100644 --- a/apps/server/src/provider/Drivers/AntigravityUsage.ts +++ b/apps/server/src/provider/Drivers/AntigravityUsage.ts @@ -15,7 +15,11 @@ import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import { resolveAntigravityInstanceDirectories } from "../antigravityAuthSupport.ts"; -import { makeAntigravityUsageCache, readAntigravityUsage } from "./antigravityUsageReader.ts"; +import { + antigravityRateModel, + makeAntigravityUsageCache, + readAntigravityUsage, +} from "./antigravityUsageReader.ts"; export class AntigravityUsage extends Context.Service< AntigravityUsage, @@ -123,6 +127,7 @@ export const layer = Layer.effect(AntigravityUsage, make); export const antigravityUsageReader: ProviderUsageReader = { kind: "scan", provider: "antigravity", + rateModel: antigravityRateModel, scan: ({ instances, windowStartMs }) => AntigravityUsage.pipe(Effect.flatMap((usage) => usage.scan({ instances, windowStartMs }))), }; diff --git a/apps/server/src/provider/Drivers/ClaudeDriver.ts b/apps/server/src/provider/Drivers/ClaudeDriver.ts index af20b7675494..d75b1cad8640 100644 --- a/apps/server/src/provider/Drivers/ClaudeDriver.ts +++ b/apps/server/src/provider/Drivers/ClaudeDriver.ts @@ -36,6 +36,10 @@ import { import { makeManagedServerProvider } from "@t3tools/provider-core/server/managedProvider"; import * as ModelCatalog from "@t3tools/provider-core/server/ModelCatalog"; import { resolveClaudeModelCatalog } from "../ClaudeModelCatalog.ts"; +import { + makeClaudeCodeModelRoutes, + resolveClaudeModelSettingsKey, +} from "../ClaudeCodeModelRoutes.ts"; import { defaultProviderContinuationIdentity, type ProviderDriver, @@ -146,6 +150,23 @@ export const ClaudeDriver: ProviderDriver 0 || + processEnv.CLAUDE_CONFIG_DIR !== (yield* HostProcess.Environment).CLAUDE_CONFIG_DIR; + // Only a successful probe records routes, so a failed one never sends turns back to slugs + // the backend rejects. + const modelRoutes = yield* makeClaudeCodeModelRoutes( + (cwd) => + resolveClaudeModelSettingsKey( + customHome ? ["project", "local"] : ["user", "project", "local"], + cwd, + ).pipe( + Effect.provideService(FileSystem.FileSystem, fileSystem), + Effect.provideService(Path.Path, path), + ), + stateDir, + ); const orchestrationAdapter = yield* createClaudeAdapterV2( { instanceId, @@ -159,6 +180,7 @@ export const ClaudeDriver: ProviderDriver snapshot.applyUsageLimits(update), modelCatalog, + routeModelCatalog: modelRoutes.route, }, ).pipe( Effect.mapError( @@ -175,6 +197,7 @@ export const ClaudeDriver: ProviderDriver Ref.set(lastGoodCapabilities, capabilities)), + Effect.tap(({ capabilities }) => + Ref.set(lastGoodCapabilities, capabilities).pipe( + Effect.andThen(modelRoutes.record(capabilities.models ?? [])), + ), + ), + Effect.ensuring(modelRoutes.settle), Effect.catch((error) => Ref.getAndSet(lastGoodCapabilities, undefined).pipe( Effect.flatMap((capabilities) => @@ -262,6 +290,8 @@ export const ClaudeDriver: ProviderDriver { }), ); + it.effect("follows the WinGet launcher to the package's native binary", () => + Effect.gen(function* () { + expect( + yield* resolveClaudeSdkExecutablePath("claude", {}).pipe( + withWindowsResolution({ resolvedCommand: WINGET_SHIM, existingFiles: [WINGET_EXE] }), + ), + ).toBe(WINGET_EXE); + }), + ); + it.effect("follows .bat and .ps1 launcher shims the same way", () => Effect.gen(function* () { for (const shim of [`${NPM_DIR}\\claude.bat`, `${NPM_DIR}\\claude.ps1`]) { diff --git a/apps/server/src/provider/Drivers/ClaudeExecutable.ts b/apps/server/src/provider/Drivers/ClaudeExecutable.ts index 5e4cbb4db6ef..5797066446b9 100644 --- a/apps/server/src/provider/Drivers/ClaudeExecutable.ts +++ b/apps/server/src/provider/Drivers/ClaudeExecutable.ts @@ -13,10 +13,14 @@ const WINDOWS_SHIM_EXTENSIONS: ReadonlySet = new Set([".cmd", ".bat", ". export const isWindowsClaudeLauncherShimPath = (filePath: string): boolean => WINDOWS_SHIM_EXTENSIONS.has(NodePath.win32.extname(filePath).toLowerCase()); -/** New packages provide bin/claude.exe; older packages need a JavaScript runtime for cli.js. */ -const NPM_PACKAGE_ENTRY_CANDIDATES = [ +/** + * npm: new packages provide bin/claude.exe; older ones need a JavaScript runtime for cli.js. + * WinGet: Links\claude.cmd forwards to the package's native executable under ..\Packages. + */ +const PACKAGE_ENTRY_CANDIDATES = [ ["node_modules", "@anthropic-ai", "claude-code", "bin", "claude.exe"], ["node_modules", "@anthropic-ai", "claude-code", "cli.js"], + ["..", "Packages", "Anthropic.ClaudeCode_Microsoft.Winget.Source_8wekyb3d8bbwe", "claude.exe"], ] as const; export type ExecutableFileCheck = (filePath: string) => boolean; @@ -53,7 +57,7 @@ export const resolveClaudeSdkExecutablePath = Effect.fn("resolveClaudeSdkExecuta } const shimDirectory = NodePath.win32.dirname(resolved); - for (const entrySegments of NPM_PACKAGE_ENTRY_CANDIDATES) { + for (const entrySegments of PACKAGE_ENTRY_CANDIDATES) { const candidate = NodePath.win32.join(shimDirectory, ...entrySegments); if (isFile(candidate)) { return candidate; diff --git a/apps/server/src/provider/Drivers/ClaudeHome.ts b/apps/server/src/provider/Drivers/ClaudeHome.ts index 8042a7f16e49..1dfb94bb91a1 100644 --- a/apps/server/src/provider/Drivers/ClaudeHome.ts +++ b/apps/server/src/provider/Drivers/ClaudeHome.ts @@ -100,6 +100,49 @@ export const hasClaudeManagedMcpConfig = Effect.fn("hasClaudeManagedMcpConfig")( return yield* fileSystem.exists(managedMcpConfigPath).pipe(Effect.orElseSucceed(() => false)); }); +// A worktree's .git file points at
/.git/worktrees/, whose commondir leads back to +//
/.git. A bare repository has no main checkout, so its worktree keeps its own file. +// Untraced because claudeLocalSettingsDirectory already owns the span. +const mainCheckoutRoot = Effect.fnUntraced(function* (checkout: string) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + return yield* fileSystem.readFileString(path.join(checkout, ".git")).pipe( + Effect.flatMap((pointer) => { + const gitDir = /^gitdir:\s*(.+)$/m.exec(pointer)?.[1]?.trim(); + if (gitDir === undefined) return Effect.succeed(checkout); + const worktreeGitDir = path.resolve(checkout, gitDir); + return fileSystem.readFileString(path.join(worktreeGitDir, "commondir")).pipe( + Effect.map((commonDir) => { + const common = path.resolve(worktreeGitDir, commonDir.trim()); + return path.basename(common) === ".git" ? path.dirname(common) : checkout; + }), + ); + }), + Effect.orElseSucceed(() => checkout), + ); +}); + +// Claude Code keeps .claude/settings.local.json at the repository root, or in cwd outside a +// repository, on Windows, or when the root is home. Its owner check is skipped, so callers must +// stay safe when Claude Code ignores the root file. +export const claudeLocalSettingsDirectory = Effect.fn("claudeLocalSettingsDirectory")(function* ( + cwd: string, +) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + if ((yield* HostProcess.Platform) === "win32") return cwd; + for (let directory = cwd; ; directory = path.dirname(directory)) { + const isCheckout = yield* fileSystem + .exists(path.join(directory, ".git")) + .pipe(Effect.orElseSucceed(() => false)); + if (isCheckout) { + const root = yield* mainCheckoutRoot(directory); + return root === (yield* HostProcess.HomeDirectory) ? cwd : root; + } + if (path.dirname(directory) === directory) return cwd; + } +}); + export const makeClaudeContinuationGroupKey = Effect.fn("makeClaudeContinuationGroupKey")( function* ( config: Pick, diff --git a/apps/server/src/provider/Drivers/ClaudeMaintenance.test.ts b/apps/server/src/provider/Drivers/ClaudeMaintenance.test.ts index 6fdf4c52e2a9..33f9f901c873 100644 --- a/apps/server/src/provider/Drivers/ClaudeMaintenance.test.ts +++ b/apps/server/src/provider/Drivers/ClaudeMaintenance.test.ts @@ -24,6 +24,8 @@ import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; import * as HttpClient from "effect/http/HttpClient"; import * as HttpClientResponse from "effect/http/HttpClientResponse"; +import * as Sink from "effect/Sink"; +import * as ChildProcess from "effect/process/ChildProcess"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import { makeClaudeMaintenanceResolution } from "./ClaudeMaintenance.ts"; @@ -77,6 +79,13 @@ const fixture = Effect.fn("ClaudeMaintenance.test.fixture")(function* ( : platform === "darwin" ? "/Library/Application Support/ClaudeCode" : "/etc/claude-code"; + const npmPath = path.join(directory, "bin/npm"); + if (install === "npm") { + yield* fs.makeDirectory(path.dirname(npmPath), { recursive: true }); + yield* fs.writeFileString(npmPath, "#!/bin/sh\n"); + yield* fs.chmod(npmPath, 0o755); + } + const npmProbes: Array> = []; let installationReads = 0; const testFs = FileSystem.makeNoop({ ...fs, @@ -105,11 +114,37 @@ const fixture = Effect.fn("ClaudeMaintenance.test.fixture")(function* ( managed, managedDirectory, settingsPath, + npmProbes, installationReads: () => installationReads, - makeResolution: makeClaudeMaintenanceResolution({ binaryPath, env: { PATH: "" } }).pipe( + makeResolution: makeClaudeMaintenanceResolution({ + binaryPath, + env: { PATH: install === "npm" ? path.dirname(npmPath) : "" }, + }).pipe( Effect.provideService( ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => Effect.die("version checks must not run a subprocess")), + ChildProcessSpawner.make((command) => { + if (!ChildProcess.isStandardCommand(command) || command.command !== npmPath) { + return Effect.die("version checks may only ask npm"); + } + npmProbes.push(command.args.map(String)); + return Effect.succeed( + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(1)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + stdin: Sink.drain, + stdout: Stream.encodeText( + Stream.make(JSON.stringify({ "@anthropic-ai/claude-code": { latest: "2.1.316" } })), + ), + stderr: Stream.empty, + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }), + ); + }), ), Effect.provideService(FileSystem.FileSystem, testFs), Effect.provideService(HostProcess.Platform, platform), @@ -233,13 +268,18 @@ it.layer(NodeServices.layer)("ClaudeMaintenance", (it) => { status: row.advisory, latestVersion: row.expected, }); - expect(requests).toHaveLength(1); - expect(requests[0]).toMatch( - row.npm ? /registry\.npmjs\.org/ : new RegExp(`claude-code-releases/${row.channel}$`), - ); + // npm installs ask `npm outdated`, which applies npm's release-age settings. + const lookups = row.npm ? f.npmProbes : requests; + expect(lookups).toHaveLength(1); + if (row.npm) { + expect(requests).toEqual([]); + expect(f.npmProbes[0]?.[0]).toBe("outdated"); + } else { + expect(requests[0]).toMatch(new RegExp(`claude-code-releases/${row.channel}$`)); + } yield* enrich; - expect(requests).toHaveLength(1); - }).pipe(Effect.scoped), + expect(lookups).toHaveLength(1); + }), ); it.effect.each([ @@ -297,7 +337,7 @@ it.layer(NodeServices.layer)("ClaudeMaintenance", (it) => { ); yield* check; expect(requests).toHaveLength(3); - }).pipe(Effect.scoped), + }), ); it.effect.each([ @@ -317,7 +357,7 @@ it.layer(NodeServices.layer)("ClaudeMaintenance", (it) => { ), ); expect((yield* resolve()).update?.lockKey).toBe(row.lockKey); - }).pipe(Effect.scoped), + }), ); it.effect("automatic refresh rereads the native channel while preserving both caches", () => @@ -363,6 +403,7 @@ it.layer(NodeServices.layer)("ClaudeMaintenance", (it) => { }).pipe( Effect.provide( Layer.mock(ProviderHost.ProviderHost)({ + awaitServerReady: Effect.void, paths: { cwd: f.directory, baseDir: f.directory, @@ -403,7 +444,7 @@ it.layer(NodeServices.layer)("ClaudeMaintenance", (it) => { ]); expect(f.installationReads()).toBe(1); } - }).pipe(Effect.scoped), + }), ); it.effect("bounds stalled feed bodies and skips disabled update checks", () => @@ -433,6 +474,6 @@ it.layer(NodeServices.layer)("ClaudeMaintenance", (it) => { yield* TestClock.adjust("4 seconds"); expect((yield* Fiber.join(pending)).versionAdvisory?.status).toBe("unknown"); expect(requests).toBe(1); - }).pipe(Effect.scoped), + }), ); }); diff --git a/apps/server/src/provider/Drivers/ClaudeSkillDispatch.ts b/apps/server/src/provider/Drivers/ClaudeSkillDispatch.ts index 6ce141eec733..bdf2ed1cae61 100644 --- a/apps/server/src/provider/Drivers/ClaudeSkillDispatch.ts +++ b/apps/server/src/provider/Drivers/ClaudeSkillDispatch.ts @@ -22,12 +22,14 @@ export interface ClaudeSkillDispatch { export function planClaudeSkillDispatch( prompt: string, skills: ReadonlyMap, + /** Mentions before this offset, such as inline handoff history, stay plain text. */ + mentionsFrom = 0, ): ClaudeSkillDispatch | undefined { const mentions = [...prompt.matchAll(SKILL_MENTION_PATTERN)].flatMap((match) => { const name = match[2] ?? ""; const skill = skills.get(name); - if (!skill) return []; const start = (match.index ?? 0) + (match[1]?.length ?? 0); + if (!skill || start < mentionsFrom) return []; return [{ name, skill, start, end: (match.index ?? 0) + match[0].length }]; }); const last = mentions.at(-1); diff --git a/apps/server/src/provider/Drivers/ClaudeSkills.test.ts b/apps/server/src/provider/Drivers/ClaudeSkills.test.ts index 8b9661a779ae..349294baa8c8 100644 --- a/apps/server/src/provider/Drivers/ClaudeSkills.test.ts +++ b/apps/server/src/provider/Drivers/ClaudeSkills.test.ts @@ -3,10 +3,19 @@ import { assert, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; +import * as HostProcess from "@t3tools/shared/HostProcess"; -import { discoverClaudeSkills } from "./ClaudeSkills.ts"; +import { discoverClaudeSkills as discoverSkills } from "./ClaudeSkills.ts"; import { planClaudeSkillDispatch } from "./ClaudeSkillDispatch.ts"; +// Windows makes the managed directory fixture-controlled without reading host policy or skills. +const discoverClaudeSkills = ( + ...[config, cwd, environment = {}]: Parameters +) => + discoverSkills(config, cwd, environment).pipe( + Effect.provideService(HostProcess.Platform, "win32"), + ); + const writeSkill = Effect.fn(function* ( skillsDir: string, directoryName: string, @@ -578,6 +587,96 @@ it.layer(NodeServices.layer)("discoverClaudeSkills", (it) => { }), ); + it.effect("discovers inline plugins from CLAUDE_CODE_PLUGIN_DIRS over installed namesakes", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const workspace = yield* fs.makeTempDirectoryScoped({ prefix: "t3-claude-skills-" }); + const configDir = path.join(workspace, "config"); + const named = path.join(workspace, "named"); + const unnamed = path.join(workspace, "unnamed"); + const inherited = path.join(workspace, "inherited"); + const muted = path.join(workspace, "muted"); + const installed = path.join(workspace, "installed-acme"); + yield* fs.makeDirectory(path.join(named, ".claude-plugin"), { recursive: true }); + yield* fs.writeFileString( + path.join(named, ".claude-plugin", "plugin.json"), + '{"name":"acme"}', + ); + yield* writeSkill(path.join(named, "skills"), "qa", "---\ndescription: QA a ticket\n---\n"); + yield* writeSkill(path.join(installed, "skills"), "old", "# Old"); + yield* writeSkill(path.join(unnamed, "skills"), "lint", "# Lint"); + yield* writeSkill(path.join(muted, "skills"), "hidden", "# Hidden"); + const quiet = path.join(workspace, "quiet"); + yield* fs.makeDirectory(path.join(quiet, ".claude-plugin"), { recursive: true }); + yield* fs.writeFileString( + path.join(quiet, ".claude-plugin", "plugin.json"), + '{"name":"quiet","defaultEnabled":false}', + ); + yield* writeSkill(path.join(quiet, "skills"), "hush", "# Hush"); + const bundle = path.join(workspace, "bundle"); + yield* fs.makeDirectory(path.join(bundle, "tools", ".claude-plugin"), { recursive: true }); + yield* fs.writeFileString( + path.join(bundle, "tools", ".claude-plugin", "plugin.json"), + '{"name":"tools"}', + ); + yield* writeSkill(path.join(bundle, "tools", "skills"), "fmt", "# Fmt"); + yield* writeSkill(path.join(bundle, "loose", "skills"), "ignored", "# Ignored"); + yield* writeSkill(path.join(inherited, "skills"), "stale", "# Stale"); + yield* writeSkill(path.join(workspace, "relative", "skills"), "skipped", "# Skipped"); + yield* fs.makeDirectory(path.join(configDir, "plugins"), { recursive: true }); + yield* fs.writeFileString( + path.join(configDir, "plugins", "installed_plugins.json"), + JSON.stringify({ plugins: { "acme@shop": [{ scope: "user", installPath: installed }] } }), + ); + yield* fs.writeFileString( + path.join(configDir, "settings.json"), + JSON.stringify({ + enabledPlugins: { "acme@shop": true, "muted@inline": false }, + env: { + CLAUDE_CODE_PLUGIN_DIRS: [named, "relative", ` ${unnamed} `, muted, quiet, bundle].join( + ";", + ), + }, + }), + ); + + const skills = yield* discoverClaudeSkills({ homePath: configDir }, workspace, { + CLAUDE_CODE_PLUGIN_DIRS: inherited, + }); + assert.deepEqual( + skills.map(({ name, scope, description }) => ({ name, scope, description })), + [ + { name: "acme:qa", scope: "user", description: "QA a ticket" }, + { name: "tools:fmt", scope: "user", description: undefined }, + { name: "unnamed:lint", scope: "user", description: undefined }, + ], + ); + + // Managed enabledPlugins entries lock their plugin name against inline copies. + const programFiles = path.join(workspace, "program-files"); + yield* fs.makeDirectory(path.join(programFiles, "ClaudeCode"), { recursive: true }); + yield* fs.writeFileString( + path.join(programFiles, "ClaudeCode", "managed-settings.json"), + '{"enabledPlugins":{"acme@shop":true}}', + ); + assert.deepEqual( + (yield* discoverClaudeSkills({ homePath: path.join(workspace, "empty") }, workspace, { + CLAUDE_CODE_PLUGIN_DIRS: [named, unnamed].join(";"), + ProgramFiles: programFiles, + })).map((skill) => skill.name), + ["unnamed:lint"], + ); + + assert.deepEqual( + (yield* discoverClaudeSkills({ homePath: path.join(workspace, "empty") }, workspace, { + CLAUDE_CODE_PLUGIN_DIRS: inherited, + })).map((skill) => skill.name), + ["inherited:stale"], + ); + }), + ); + it.effect("discovers user and project skills with frontmatter metadata", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -688,7 +787,7 @@ it.layer(NodeServices.layer)("discoverClaudeSkills", (it) => { }), ); - it.effect("prefers user skills on name collisions even with a stray .agents copy", () => + it.effect("prefers managed then user skills on collisions, even with a stray .agents copy", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -723,6 +822,62 @@ it.layer(NodeServices.layer)("discoverClaudeSkills", (it) => { description: "User deploy.", }, ]); + + const programFiles = path.join(tempDir, "Program Files"); + const programData = path.join(tempDir, "ProgramData"); + yield* writeSkill( + path.join(programData, "ClaudeCode", ".claude", "skills"), + "legacy-only", + "# Legacy skills must not be discovered", + ); + yield* fs.writeFileString( + path.join(programData, "ClaudeCode", "managed-settings.json"), + '{"skillOverrides":{"deploy":"off"}}', + ); + for (const ProgramFiles of [undefined, "", programFiles]) { + const legacySkills = yield* discoverClaudeSkills({ homePath: configDir }, workspace, { + PROGRAMDATA: programData, + ...(ProgramFiles !== undefined ? { ProgramFiles } : {}), + }); + assert.deepEqual( + legacySkills, + skills.map((skill) => ({ ...skill, enabled: false })), + ); + } + yield* writeSkill( + path.join(programFiles, "ClaudeCode", ".claude", "skills"), + "deploy", + ["---", "name: deploy", "description: Managed deploy.", "---"].join("\n"), + ); + yield* fs.writeFileString( + path.join(configDir, "settings.json"), + '{"skillOverrides":{"deploy":"off"}}', + ); + yield* fs.writeFileString( + path.join(programFiles, "ClaudeCode", "managed-settings.json"), + '{"skillOverrides":{"deploy":"on"}}', + ); + const managedSkills = yield* discoverClaudeSkills({ homePath: configDir }, workspace, { + ProgramFiles: programFiles, + PROGRAMDATA: programData, + }); + + assert.deepEqual(managedSkills, [ + { + name: "deploy", + path: path.join(programFiles, "ClaudeCode", ".claude", "skills", "deploy", "SKILL.md"), + enabled: true, + scope: "user", + description: "Managed deploy.", + }, + ]); + assert.deepEqual( + yield* discoverClaudeSkills({ homePath: configDir }, workspace, { + PROGRAMFILES: ` ${programFiles} `, + PROGRAMDATA: programData, + }), + managedSkills, + ); }), ); diff --git a/apps/server/src/provider/Drivers/ClaudeSkills.ts b/apps/server/src/provider/Drivers/ClaudeSkills.ts index c24c4afca9e7..4fdd3565d11b 100644 --- a/apps/server/src/provider/Drivers/ClaudeSkills.ts +++ b/apps/server/src/provider/Drivers/ClaudeSkills.ts @@ -106,19 +106,41 @@ function parseSkillFrontmatter(contents: string): SkillFrontmatter { }; } -function claudeManagedSettingsPath( +// A caller-supplied environment is a plain object, without Windows' case-insensitive keys. +function windowsEnvironmentValue(environment: NodeJS.ProcessEnv, name: string) { + const key = name.toLowerCase(); + return Object.entries(environment) + .find(([candidate]) => candidate.toLowerCase() === key)?.[1] + ?.trim(); +} + +function claudeManagedDirectory( path: Path.Path, platform: NodeJS.Platform, environment: NodeJS.ProcessEnv, ): string | undefined { if (platform === "darwin") { - return "/Library/Application Support/ClaudeCode/managed-settings.json"; + return "/Library/Application Support/ClaudeCode"; } if (platform === "win32") { - const programData = environment.PROGRAMDATA?.trim(); - return programData ? path.join(programData, "ClaudeCode", "managed-settings.json") : undefined; + const programFiles = windowsEnvironmentValue(environment, "ProgramFiles"); + return programFiles ? path.join(programFiles, "ClaudeCode") : undefined; } - return "/etc/claude-code/managed-settings.json"; + return "/etc/claude-code"; +} + +function claudeManagedSettingsPaths( + path: Path.Path, + platform: NodeJS.Platform, + environment: NodeJS.ProcessEnv, +): ReadonlyArray { + const managedDirectory = claudeManagedDirectory(path, platform, environment); + const programData = + platform === "win32" ? windowsEnvironmentValue(environment, "ProgramData") : undefined; + return [ + ...(programData ? [path.join(programData, "ClaudeCode", "managed-settings.json")] : []), + ...(managedDirectory ? [path.join(managedDirectory, "managed-settings.json")] : []), + ]; } /** Claude Code settings files, lowest precedence first. */ @@ -130,7 +152,6 @@ function claudeSettingsPaths( environment: NodeJS.ProcessEnv, repositoryRoot?: string, ): ReadonlyArray { - const managedPath = claudeManagedSettingsPath(path, platform, environment); const root = repositoryRoot !== undefined && repositoryRoot !== cwd ? repositoryRoot : undefined; return [ path.join(configDirPath, "settings.json"), @@ -141,7 +162,7 @@ function claudeSettingsPaths( ] : []), ...(root ? [path.join(root, ".claude", "settings.local.json")] : []), - ...(managedPath ? [managedPath] : []), + ...claudeManagedSettingsPaths(path, platform, environment), ]; } @@ -250,6 +271,8 @@ const readSkillSettings = Effect.fn("readSkillSettings")(function* ( const platform = yield* HostProcess.Platform; const overridesByName = new Map(); const enabledPlugins = new Map(); + const managedPluginNames = new Set(); + const managedPaths = new Set(claudeManagedSettingsPaths(path, platform, environment)); let syncedPluginsHidden = false; const repositoryRoot = cwd === undefined ? undefined : yield* findRepositoryRoot(cwd); const projectSettingsPath = @@ -285,6 +308,9 @@ const readSkillSettings = Effect.fn("readSkillSettings")(function* ( for (const [key, enabled] of Object.entries(parsed?.enabledPlugins ?? {})) { if (typeof enabled === "boolean") { enabledPlugins.set(key, enabled); + if (managedPaths.has(settingsPath) && key.includes("@")) { + managedPluginNames.add(pluginNameKey(key.slice(0, key.indexOf("@")))); + } } else { // Extended formats replace lower-precedence values without enabling the plugin here. enabledPlugins.delete(key); @@ -296,7 +322,12 @@ const readSkillSettings = Effect.fn("readSkillSettings")(function* ( } } - return { skillOverrides: overridesByName, enabledPlugins, syncedPluginsHidden }; + return { + skillOverrides: overridesByName, + enabledPlugins, + managedPluginNames, + syncedPluginsHidden, + }; }); const InstalledPlugins = Schema.fromJsonString( @@ -313,6 +344,7 @@ const PluginManifest = Schema.fromJsonString( Schema.Struct({ name: Schema.optional(Schema.NonEmptyString), skills: Schema.optional(Schema.Union([Schema.String, Schema.Array(Schema.String)])), + defaultEnabled: Schema.optional(Schema.Unknown), }), ); @@ -433,10 +465,66 @@ const resolveSyncedPluginBucket = Effect.fn("resolveSyncedPluginBucket")(functio return `${organizationId.toLowerCase()}_${accountId.toLowerCase()}`; }); +/** A plugin folder itself, or else the manifest-bearing plugins directly inside it. */ +const expandInlinePluginDirectory = Effect.fn("expandInlinePluginDirectory")(function* ( + directory: string, +) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const exists = (candidate: string) => + fileSystem.exists(candidate).pipe(Effect.orElseSucceed(() => false)); + for (const component of [path.join(".claude-plugin", "plugin.json"), "skills", "SKILL.md"]) { + if (yield* exists(path.join(directory, component))) return [directory]; + } + const children = yield* fileSystem + .readDirectory(directory) + .pipe(Effect.orElseSucceed((): ReadonlyArray => [])); + return yield* Effect.filter( + [...children].sort().map((child) => path.join(directory, child)), + (child) => exists(path.join(child, ".claude-plugin", "plugin.json")), + ); +}); + +/** Plugin folders Claude Code loads inline from CLAUDE_CODE_PLUGIN_DIRS, and their plugin names. */ +const discoverInlinePluginSkillRoots = Effect.fn("discoverInlinePluginSkillRoots")(function* ( + pluginDirectories: string | undefined, + enabledPlugins: ReadonlyMap, + managedPluginNames: ReadonlySet, +) { + const path = yield* Path.Path; + const home = yield* HostProcess.HomeDirectory; + const platform = yield* HostProcess.Platform; + const roots: Array = []; + const names = new Set(); + const seen = new Set(); + for (const entry of (pluginDirectories ?? "").split(platform === "win32" ? ";" : ":")) { + const directory = expandHomePath(entry.trim(), home); + // Claude skips relative entries rather than resolving them against its cwd. + if (!directory || !path.isAbsolute(directory)) continue; + for (const installPath of yield* expandInlinePluginDirectory(path.resolve(directory))) { + if (seen.has(installPath)) continue; + seen.add(installPath); + const manifest = yield* readPluginManifest(installPath); + const prefix = manifest?.name ?? path.basename(installPath); + // A `@inline` setting wins; otherwise the manifest's `defaultEnabled` decides. + const setting = enabledPlugins.get(`${prefix}@inline`); + if (setting === false || (setting === undefined && manifest?.defaultEnabled === false)) { + continue; + } + // Managed enabledPlugins entries, true or false, lock the name against inline copies. + if (managedPluginNames.has(pluginNameKey(prefix))) continue; + names.add(pluginNameKey(prefix)); + roots.push(...(yield* pluginSkillRoots(installPath, "user", prefix, manifest))); + } + } + return { roots, names }; +}); + const discoverPluginSkillRoots = Effect.fn("discoverPluginSkillRoots")(function* ( configDirPath: string, cwd: string | undefined, enabledPlugins: ReadonlyMap, + inlinePluginNames: ReadonlySet, syncedBucket: string | undefined, environment: NodeJS.ProcessEnv, ) { @@ -454,7 +542,8 @@ const discoverPluginSkillRoots = Effect.fn("discoverPluginSkillRoots")(function* Effect.orElseSucceed(() => undefined), ); const roots: Array = []; - const claimedPluginNames = new Set(); + // An enabled inline plugin replaces installed and synced plugins of the same name. + const claimedPluginNames = new Set(inlinePluginNames); // The CLI's process.cwd() resolves symlinks; registry projectPath values remain lexical. const workspace = cwd === undefined @@ -494,7 +583,7 @@ const discoverPluginSkillRoots = Effect.fn("discoverPluginSkillRoots")(function* const installPath = path.resolve(cwd ?? ".", install.installPath); const manifest = yield* readPluginManifest(installPath); const prefix = manifest?.name ?? key.split("@")[0]; - if (!prefix) continue; + if (!prefix || inlinePluginNames.has(pluginNameKey(prefix))) continue; // A registry record whose install is gone loads nothing and claims no namespace. if (Option.isSome(existing)) claimedPluginNames.add(pluginNameKey(prefix)); roots.push(...(yield* pluginSkillRoots(installPath, scope, prefix, manifest))); @@ -560,6 +649,7 @@ const resolveClaudeConfigDirPath = Effect.fn("resolveClaudeConfigDirPath")(funct const SettingsEnv = Schema.Struct({ ANTHROPIC_BASE_URL: Schema.optional(Schema.String), CLAUDE_CODE_ENTRYPOINT: Schema.optional(Schema.String), + CLAUDE_CODE_PLUGIN_DIRS: Schema.optional(Schema.String), // Unquoted JSON values must not fail the decode and drop ANTHROPIC_BASE_URL with them. CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY: Schema.optional(Schema.Unknown), CLAUDE_CODE_GATEWAY_MODEL_DISCOVERY_TIMEOUT_MS: Schema.optional(Schema.Unknown), @@ -627,7 +717,7 @@ export function claudeLaunchArgDirectories(launchArgs?: string): ReadonlyArray & Partial>, cwd?: string, @@ -637,11 +727,9 @@ export const discoverClaudeSkills = Effect.fn("discoverClaudeSkills")(function* const path = yield* Path.Path; const env = environment ?? process.env; const configDirPath = yield* resolveClaudeConfigDirPath(config, env, cwd); - const { skillOverrides, enabledPlugins, syncedPluginsHidden } = yield* readSkillSettings( - configDirPath, - cwd, - env, - ); + const { skillOverrides, enabledPlugins, managedPluginNames, syncedPluginsHidden } = + yield* readSkillSettings(configDirPath, cwd, env); + const managedDirectory = claudeManagedDirectory(path, yield* HostProcess.Platform, env); // Most installs have no synced folder; skip reading `.claude.json` for them. const hasSyncedPlugins = !syncedPluginsHidden && @@ -651,8 +739,20 @@ export const discoverClaudeSkills = Effect.fn("discoverClaudeSkills")(function* const syncedBucket = hasSyncedPlugins ? yield* resolveSyncedPluginBucket(config, configDirPath, env) : undefined; + // Claude applies its settings `env` over the inherited environment. + const inlinePluginDirectories = + (yield* readClaudeSettingsEnv(config, cwd, env)).CLAUDE_CODE_PLUGIN_DIRS ?? + env.CLAUDE_CODE_PLUGIN_DIRS; + const inlinePlugins = yield* discoverInlinePluginSkillRoots( + inlinePluginDirectories, + enabledPlugins, + managedPluginNames, + ); const roots: ReadonlyArray = [ + ...(managedDirectory + ? [{ directory: path.join(managedDirectory, ".claude", "skills"), scope: "user" as const }] + : []), { directory: path.join(configDirPath, "skills"), scope: "user" }, ...(cwd ? [{ directory: path.join(cwd, ".claude", "skills"), scope: "project" as const }] : []), ...claudeLaunchArgDirectories(config.launchArgs) @@ -661,7 +761,15 @@ export const discoverClaudeSkills = Effect.fn("discoverClaudeSkills")(function* directory: path.join(path.resolve(cwd ?? ".", directory), ".claude", "skills"), scope: "project" as const, })), - ...(yield* discoverPluginSkillRoots(configDirPath, cwd, enabledPlugins, syncedBucket, env)), + ...inlinePlugins.roots, + ...(yield* discoverPluginSkillRoots( + configDirPath, + cwd, + enabledPlugins, + inlinePlugins.names, + syncedBucket, + env, + )), ]; const skillsByName = new Map(); diff --git a/apps/server/src/provider/Drivers/CodexDriver.test.ts b/apps/server/src/provider/Drivers/CodexDriver.test.ts index 597be8562311..1462d051346f 100644 --- a/apps/server/src/provider/Drivers/CodexDriver.test.ts +++ b/apps/server/src/provider/Drivers/CodexDriver.test.ts @@ -42,6 +42,7 @@ import { CodexDriver } from "./CodexDriver.ts"; import * as CodexAdapterV2 from "../../orchestration-v2/Adapters/CodexAdapterV2.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import * as PortScanner from "../../preview/PortScanner.ts"; import * as ProviderCredentialStore from "../ProviderCredentialStore.ts"; import * as ProviderHostLive from "../ProviderHostLive.ts"; @@ -51,6 +52,7 @@ const layerDeps = ServerConfig.layerTest(process.cwd(), { Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), Layer.provideMerge(IdAllocator.layer), Layer.provideMerge(McpProviderSessions.layer), + Layer.provideMerge(PortScanner.layerTest), Layer.provideMerge( Layer.mock(CodexAdapterV2.CodexAppServerClientFactory)({ open: () => Effect.die("Maintenance resolution must not open a Codex session"), @@ -374,10 +376,7 @@ it.layer(layerTest)("CodexDriver", (it) => { lockKey: "codex-native", env: { CODEX_HOME: sharedHome }, }); - }).pipe( - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn), - Effect.scoped, - ), + }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn)), ); it.effect.skipIf(windowsHost)( @@ -431,10 +430,7 @@ it.layer(layerTest)("CodexDriver", (it) => { env: { CODEX_HOME: installHome }, }); } - }).pipe( - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn), - Effect.scoped, - ), + }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn)), ); it.effect.skipIf(windowsHost)("derives the standalone update home from either command path", () => @@ -478,7 +474,7 @@ it.layer(layerTest)("CodexDriver", (it) => { CODEX_HOME: fixture.home, }); } - }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn), Effect.scoped), + }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn)), ); // regression: #23937 `codex update` installs the native channel's release, which can lag npm. @@ -556,10 +552,7 @@ it.layer(layerTest)("CodexDriver", (it) => { maintenanceCapabilities: capabilities, }).status, ).not.toBe("behind_latest"); - }).pipe( - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn), - Effect.scoped, - ), + }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn)), ); it.effect("uses each instance's environment CODEX_HOME for continuation", () => @@ -577,7 +570,36 @@ it.layer(layerTest)("CodexDriver", (it) => { }); expect(instance.continuationIdentity.continuationKey).toBe(`codex:home:${codexHome}`); } - }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn), Effect.scoped), + }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn)), + ); + + it.effect.skipIf(windowsHost)( + "shares state from the environment CODEX_HOME in a shadow home", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const tempDir = yield* fs + .makeTempDirectoryScoped({ prefix: "t3-codex-driver-" }) + .pipe(Effect.flatMap((directory) => fs.realPath(directory))); + const codexHome = NodePath.join(tempDir, "custom-codex"); + const shadowHome = NodePath.join(tempDir, "shadow"); + yield* fs.makeDirectory(codexHome, { recursive: true }); + yield* fs.writeFileString(NodePath.join(codexHome, "config.toml"), 'model = "custom"\n'); + const instance = yield* CodexDriver.create({ + instanceId: ProviderInstanceId.make("env-shadow"), + displayName: "env-shadow", + enabled: false, + environment: [{ name: "CODEX_HOME", value: codexHome, sensitive: false }], + config: { ...CodexDriver.defaultConfig(), shadowHomePath: shadowHome }, + }).pipe(Effect.provideService(HostProcess.HomeDirectory, NodePath.join(tempDir, "user"))); + expect(instance.continuationIdentity.continuationKey).toBe(`codex:home:${codexHome}`); + expect(yield* fs.readLink(NodePath.join(shadowHome, "config.toml"))).toBe( + NodePath.join(codexHome, "config.toml"), + ); + expect(yield* fs.readLink(NodePath.join(shadowHome, "sessions"))).toBe( + NodePath.join(codexHome, "sessions"), + ); + }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn)), ); for (const fixture of [ @@ -653,10 +675,7 @@ it.layer(layerTest)("CodexDriver", (it) => { } else { expect(update).toBeNull(); } - }).pipe( - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn), - Effect.scoped, - ), + }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn)), ); } @@ -710,6 +729,7 @@ it.layer(layerTest)("CodexDriver", (it) => { const shimDir = NodePath.join(tempDir, fixture.dataRoot, "shims"); const npmPrefix = NodePath.join(tempDir, "mise", "installs", "node", "24.13.0"); const npmBin = NodePath.join(npmPrefix, "bin"); + const npmPath = NodePath.join(npmBin, "npm"); const npmEntry = NodePath.join( npmPrefix, "lib", @@ -719,7 +739,7 @@ it.layer(layerTest)("CodexDriver", (it) => { "bin", "codex.js", ); - for (const file of [brewPath, misePath, npmEntry]) { + for (const file of [brewPath, misePath, npmPath, npmEntry]) { yield* fs.makeDirectory(NodePath.dirname(file), { recursive: true }); yield* fs.writeFileString(file, "#!/bin/sh\n"); yield* fs.chmod(file, 0o755); @@ -734,12 +754,20 @@ it.layer(layerTest)("CodexDriver", (it) => { ].join(NodePath.delimiter); const probes: Array> = []; const metadataSpawner = ChildProcessSpawner.make((command) => { - if (!ChildProcess.isStandardCommand(command) || command.command !== brewPath) { + const isNpmOutdated = + ChildProcess.isStandardCommand(command) && + command.command === npmPath && + command.args[0] === "outdated"; + if ( + !ChildProcess.isStandardCommand(command) || + (command.command !== brewPath && !isNpmOutdated) + ) { return Effect.die("Provider resolution must not execute a provider or updater"); } - probes.push(command.args.map(String)); - const stdout = - command.args[0] === "--prefix" + if (!isNpmOutdated) probes.push(command.args.map(String)); + const stdout = isNpmOutdated + ? JSON.stringify({ "@openai/codex": { latest: "0.153.4" } }) + : command.args[0] === "--prefix" ? brewPrefix : JSON.stringify({ formulae: [{ versions: { stable: "2026.9.1" } }] }); return Effect.succeed( @@ -805,7 +833,7 @@ it.layer(layerTest)("CodexDriver", (it) => { } else { expect(capabilities.update).toBeNull(); } - }).pipe(Effect.scoped), + }), { skip: windowsHost }, ); @@ -856,6 +884,6 @@ it.layer(layerTest)("CodexDriver", (it) => { yield* TestClock.adjust("250 millis"); yield* Queue.take(probes); - }).pipe(Effect.scoped), + }), ); }); diff --git a/apps/server/src/provider/Drivers/CodexDriver.ts b/apps/server/src/provider/Drivers/CodexDriver.ts index c9cbb937c2eb..03245ba46d6c 100644 --- a/apps/server/src/provider/Drivers/CodexDriver.ts +++ b/apps/server/src/provider/Drivers/CodexDriver.ts @@ -1,3 +1,4 @@ +import { resolveProviderProbeTimeoutMs } from "@t3tools/provider-core/server/snapshotProbe"; import { CodexSettings, ProviderDriverKind } from "@t3tools/contracts"; import * as Digest from "effect/crypto/Digest"; import * as SecureRandom from "effect/crypto/SecureRandom"; @@ -28,7 +29,7 @@ import * as ResetCreditCoordinator from "../resetCreditCoordinator.ts"; import { checkCodexProviderStatus, makePendingCodexProvider, - probeCodexSkillsForCwd, + makeCodexSkillsProbe, withCodexAppServerClient, } from "../CodexProvider.ts"; import { resolveCodexLaunchArgs } from "../codexLaunchArgs.ts"; @@ -189,11 +190,11 @@ export const CodexDriver: ProviderDriver< yield* HostProcess.Environment, ); const environmentHome = processEnv.CODEX_HOME; - const homeLayout = yield* resolveCodexHomeLayout( - !config.homePath.trim() && !config.shadowHomePath.trim() && environmentHome?.trim() + const homeConfig = + !config.homePath.trim() && environmentHome?.trim() ? { ...config, homePath: environmentHome } - : config, - ); + : config; + const homeLayout = yield* resolveCodexHomeLayout(homeConfig); const continuationIdentity = codexContinuationIdentity(homeLayout); const stampIdentity = withInstanceIdentity({ instanceId, @@ -244,10 +245,13 @@ export const CodexDriver: ProviderDriver< accentColor, environment, enabled, - config, + config: homeConfig, }, { onUsageLimits: (update) => snapshot.applyUsageLimits(update), + models: Effect.suspend(() => snapshot.getSnapshot).pipe( + Effect.map((value) => value.models), + ), compatibility: installation.compatibility, }, ).pipe( @@ -317,22 +321,21 @@ export const CodexDriver: ProviderDriver< processEnv, snapshot.getSnapshot.pipe(Effect.map((value) => value.models)), ); + const probeSkills = yield* makeCodexSkillsProbe().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); const snapshotForCwd = (cwd: string) => !effectiveConfig.enabled ? snapshot.getSnapshot : Effect.all([ snapshot.getSnapshot, - probeCodexSkillsForCwd({ + probeSkills({ binaryPath: effectiveConfig.binaryPath, homePath: effectiveConfig.homePath, launchArgs: resolveCodexLaunchArgs(effectiveConfig.launchArgs, processEnv), cwd, environment: processEnv, - }).pipe( - Effect.scoped, - Effect.timeout("20 seconds"), - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), - ), + }).pipe(Effect.timeout(resolveProviderProbeTimeoutMs("codex", 20_000, processEnv))), ]).pipe( Effect.map(([machineSnapshot, skills]) => ({ ...machineSnapshot, skills })), Effect.mapError( diff --git a/apps/server/src/provider/Drivers/CodexManagedProvider.ts b/apps/server/src/provider/Drivers/CodexManagedProvider.ts index ab5e7add556a..f182245f8840 100644 --- a/apps/server/src/provider/Drivers/CodexManagedProvider.ts +++ b/apps/server/src/provider/Drivers/CodexManagedProvider.ts @@ -242,8 +242,10 @@ export const makeManagedCodexProvider = Effect.fn("makeManagedCodexProvider")(fu const resolveRuntime = runtime.auth.controller.withAccess!(runtime.resolve); // Launch settings bind the refresh command to the signed-in profile. The registry // already wraps openSession in withAccess, so resolve without re-entering it. + const models = snapshot.getSnapshot.pipe(Effect.map((value) => value.models)); const orchestrationAdapter = yield* createCodexAdapterV2(input, { onUsageLimits: (update) => snapshot.applyUsageLimits(update), + models, resolveRuntime: runtime.resolve, }).pipe( Effect.mapError( @@ -259,7 +261,7 @@ export const makeManagedCodexProvider = Effect.fn("makeManagedCodexProvider")(fu const nativeGeneration = yield* makeCodexTextGeneration( config, undefined, - snapshot.getSnapshot.pipe(Effect.map((value) => value.models)), + models, resolveRuntime, ); const protect = (operation: string, effect: Effect.Effect) => diff --git a/apps/server/src/provider/Drivers/antigravityUsageReader.ts b/apps/server/src/provider/Drivers/antigravityUsageReader.ts index 4d473f50c9d6..532233f7facc 100644 --- a/apps/server/src/provider/Drivers/antigravityUsageReader.ts +++ b/apps/server/src/provider/Drivers/antigravityUsageReader.ts @@ -129,18 +129,19 @@ const MODEL_IDS: Record = { }; // Effort and routing variants share the base model's price. -function withRateModel(record: UsageRecord): UsageRecord { - const base = record.model.replace( - /-(?:extra-low|low|medium|high|tiered|control|thinking|agent)$/, - "", - ); - const rateModel = /^gemini-3-flash-(?:agent(?:-[ab])?|[ab])$/.test(record.model) +export const antigravityRateModel = (model: string) => { + const base = model.replace(/-(?:extra-low|low|medium|high|tiered|control|thinking|agent)$/, ""); + return /^gemini-3-flash-(?:agent(?:-[ab])?|[ab])$/.test(model) ? "gemini-3.5-flash" - : record.model === "gemini-3.7-flash-safety-le" + : model === "gemini-3.7-flash-safety-le" ? "gemini-3.7-flash" : base === "gemini-3-flash" ? "gemini-3-flash-preview" : base; +}; + +function withRateModel(record: UsageRecord): UsageRecord { + const rateModel = antigravityRateModel(record.model); return rateModel === record.model ? record : { ...record, rateModel }; } diff --git a/apps/server/src/provider/Drivers/claudeUsage.test.ts b/apps/server/src/provider/Drivers/claudeUsage.test.ts index 6f457f4f4cf2..bc10ba5b7b29 100644 --- a/apps/server/src/provider/Drivers/claudeUsage.test.ts +++ b/apps/server/src/provider/Drivers/claudeUsage.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "@effect/vitest"; +import { parseRateTable, priceUsage } from "../../usage/usagePricing.ts"; import { claudeUsageFormat, parseClaudeLine } from "./claudeUsage.ts"; function claudeLine(overrides: { @@ -115,7 +116,8 @@ describe("Claude advisor iterations", () => { model: "claude-fable-5-1", input_tokens: 54204, cache_read_input_tokens: 0, - cache_creation_input_tokens: 0, + cache_creation_input_tokens: 500, + cache_creation: { ephemeral_5m_input_tokens: 200, ephemeral_1h_input_tokens: 300 }, output_tokens: 4229, }, { @@ -160,10 +162,11 @@ describe("Claude advisor iterations", () => { totals: { uncachedInputTokens: 54204, cachedInputTokens: 0, - cacheCreationTokens: 0, + cacheCreationTokens: 500, outputTokens: 4229, reasoningTokens: 0, }, + cacheCreation1hTokens: 300, dedupeKey: expect.any(String), }, ]; @@ -216,3 +219,44 @@ describe("Claude advisor iterations", () => { expect(parse(unusable).map((record) => record.model)).toEqual(["claude-opus-5-5"]); }); }); + +describe("Claude cache write pricing", () => { + it("prices one-hour cache writes at the one-hour rate and the rest at the five-minute rate", () => { + const model = "claude-opus-4-5-20251101"; + const rates = parseRateTable({ + [model]: { + litellm_provider: "anthropic", + input_cost_per_token: 5e-6, + output_cost_per_token: 25e-6, + cache_read_input_token_cost: 5e-7, + cache_creation_input_token_cost: 6.25e-6, + cache_creation_input_token_cost_above_1hr: 1e-5, + }, + }); + const [record] = parseClaudeLine( + JSON.stringify({ + type: "assistant", + timestamp: "2026-08-07T04:05:13.944Z", + sessionId: "s", + message: { + id: "msg_ttl", + model, + usage: { + input_tokens: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 1_000_000, + cache_creation: { + ephemeral_5m_input_tokens: 400_000, + ephemeral_1h_input_tokens: 600_000, + }, + output_tokens: 0, + }, + }, + }), + ); + + if (record === undefined) throw new Error("expected a usage record"); + // 0.4M x $6.25/M + 0.6M x $10/M + expect(priceUsage(rates, record).costUsd).toBeCloseTo(8.5, 9); + }); +}); diff --git a/apps/server/src/provider/Drivers/claudeUsage.ts b/apps/server/src/provider/Drivers/claudeUsage.ts index acb697d2941d..713b35884292 100644 --- a/apps/server/src/provider/Drivers/claudeUsage.ts +++ b/apps/server/src/provider/Drivers/claudeUsage.ts @@ -30,6 +30,13 @@ export function parseClaudeLine(line: string): readonly UsageRecord[] { return parseClaudeRecord(parsed); } +function cacheCreation1h(usage: Record): { cacheCreation1hTokens?: number } { + const split = usage["cache_creation"]; + if (typeof split !== "object" || split === null) return {}; + const tokens = tokenCount((split as Record)["ephemeral_1h_input_tokens"]); + return tokens > 0 ? { cacheCreation1hTokens: tokens } : {}; +} + function parseClaudeRecord(parsed: unknown): readonly UsageRecord[] { if (typeof parsed !== "object" || parsed === null) return []; @@ -70,6 +77,7 @@ function parseClaudeRecord(parsed: unknown): readonly UsageRecord[] { // Anthropic folds thinking tokens into output and does not break them out. reasoningTokens: 0, }, + ...cacheCreation1h(usageRecord), reportedCostUsd: typeof cost === "number" && Number.isFinite(cost) ? cost : null, speed: usageRecord["speed"] === "fast" ? "fast" : "standard", dedupeKey, @@ -101,6 +109,7 @@ function parseClaudeRecord(parsed: unknown): readonly UsageRecord[] { outputTokens: tokenCount(advisor["output_tokens"]), reasoningTokens: 0, }, + ...cacheCreation1h(advisor), reportedCostUsd: null, speed: executor.speed, dedupeKey: dedupeKey === null ? null : `${dedupeKey}:advisor:${index}`, diff --git a/apps/server/src/provider/Drivers/codexUsage.test.ts b/apps/server/src/provider/Drivers/codexUsage.test.ts index 74812bafb509..1cdbe05d0766 100644 --- a/apps/server/src/provider/Drivers/codexUsage.test.ts +++ b/apps/server/src/provider/Drivers/codexUsage.test.ts @@ -450,6 +450,61 @@ describe("parseCodexLine", () => { expect(state.malformedRecords).toBe(0); }); + it("bills request-ledger usage through resumed context counters and compaction", () => { + const state = initialCodexScanState(); + const request = (response_id: string, usage: unknown) => + JSON.stringify({ + type: "token_usage_record", + timestamp: "2026-08-01T05:17:45.000Z", + payload: { response_id, usage }, + }); + parseCodexLine(sessionMeta, state); + parseCodexLine(turnContext, state); + parseCodexLine(counted(counters(100, 20), counters(1000, 200)), state); + // Resume restores older context counters, while the request ledger continues. + expect( + parseCodexLine(request("response-1", counters(80, 10)), state)?.totals.outputTokens, + ).toBe(10); + expect(parseCodexLine(counted(counters(80, 10), counters(580, 110)), state)).toBeNull(); + // Rate-limit refreshes repeat the snapshot without another request. + expect(parseCodexLine(counted(counters(80, 10), counters(580, 110)), state)).toBeNull(); + expect( + parseCodexLine(request("compaction", counters(90, 15)), state)?.totals.outputTokens, + ).toBe(15); + parseCodexLine(turnContext, state); + expect( + parseCodexLine(counted({ ...counters(0, 0), total_tokens: 40 }, counters(580, 110)), state), + ).toBeNull(); + expect(parseCodexLine(request("response-2", counters(30, 5)), state)?.totals.outputTokens).toBe( + 5, + ); + expect(parseCodexLine(counted(counters(30, 5), counters(610, 115)), state)).toBeNull(); + expect(state.malformedRecords).toBe(0); + // A later turn from an older CLI can still report usage through snapshots alone. + parseCodexLine(turnContext, state); + expect( + parseCodexLine(counted(counters(20, 2), counters(630, 117)), state)?.totals.outputTokens, + ).toBe(2); + expect( + parseCodexLine(request("interrupted", counters(40, 4)), state)?.totals.outputTokens, + ).toBe(4); + parseCodexLine(turnContext, state); + expect( + parseCodexLine(counted(counters(10, 1), counters(680, 122)), state)?.totals.outputTokens, + ).toBe(1); + expect(parseCodexLine(request("", counters(15, 3)), state)).toBeNull(); + parseCodexLine(turnContext, state); + const transition = parseCodexRecords(request("upgraded", counters(25, 5)), state); + expect(transition.map((record) => record.totals.outputTokens)).toEqual([3, 5]); + expect(parseCodexLine(counted(counters(25, 5), counters(720, 130)), state)).toBeNull(); + expect(parseCodexLine(request("", counters(10, 1)), state)).toBeNull(); + expect( + parseCodexLine(counted(counters(10, 1), counters(730, 131)), state)?.totals.outputTokens, + ).toBe(1); + expect(parseCodexLine(counted(counters(10, 1), counters(730, 131)), state)).toBeNull(); + expect(state.malformedRecords).toBe(0); + }); + it("does not bill a context-window estimate or lose the following real request", () => { const state = initialCodexScanState(); parseCodexLine(turnContext, state); @@ -498,6 +553,134 @@ describe("parseCodexLine", () => { expect(codexUsageFormat.readMalformedRecords?.(state)).toBe(0); }); + const later = (line: string) => line.replace("05:17:49.919Z", "07:17:49.919Z"); + it.each([ + { + name: "bills a later request that continues from an earlier snapshot", + tail: [ + later(counted(counters(20, 4), counters(130, 26))), + later(counted(counters(5, 1), counters(135, 27))), + ], + processed: [120, 12, 35, 24, 6], + skipped: 0, + }, + { + name: "does not bill a re-emitted earlier snapshot again", + tail: [counted(counters(10, 2), counters(110, 22))], + processed: [120, 12, 35, 0], + skipped: 1, + }, + ])("$name when cumulative counters rewind", ({ tail, processed, skipped }) => { + const state = initialCodexScanState(); + parseCodexLine(turnContext, state); + const billed = [ + counted(counters(100, 20), counters(100, 20)), + counted(counters(10, 2), counters(110, 22)), + counted(counters(30, 5), counters(140, 27)), + ...tail, + ].map((line) => { + const record = parseCodexLine(line, state); + return record === null ? 0 : totalTokens(record.totals); + }); + + expect({ processed: billed, skipped: codexUsageFormat.readMalformedRecords?.(state) }).toEqual({ + processed, + skipped, + }); + }); + + const at = (time: string, line: string) => line.replace("05:17:49.919Z", `${time}.000Z`); + const anonymousRequest = (time: string, usage: unknown) => + JSON.stringify({ + type: "token_usage_record", + timestamp: `2026-08-01T${time}.000Z`, + payload: { usage }, + }); + it.each([ + { + name: "does not bill an older snapshot re-emitted after requests without cumulative counters", + lines: [ + at("10:00:00", counted(counters(100, 20), counters(100, 20))), + at("10:00:05", counted(counters(20, 4))), + at("10:00:10", counted(counters(30, 6))), + at("10:00:05", counted(counters(20, 4), counters(120, 24))), + ], + processed: [120, 24, 36, 0], + skipped: 1, + }, + { + name: "bills a rewind that follows a request without cumulative counters", + lines: [ + at("10:00:00", counted(counters(100, 20))), + at("10:00:10", counted(counters(20, 4), counters(60, 12))), + ], + processed: [120, 24], + skipped: 0, + }, + { + name: "keeps the baseline when it rejects an older snapshot, so the replay that follows is not billed", + lines: [ + at("10:00:00", counted(counters(100, 20), counters(100, 20))), + at("10:00:05", counted(counters(20, 4), counters(120, 24))), + at("10:00:10", counted(counters(30, 6), counters(150, 30))), + at("10:00:05", counted(counters(20, 4), counters(120, 24))), + at("10:00:10", counted(counters(30, 6), counters(150, 30))), + ], + processed: [120, 24, 36, 0, 0], + skipped: 1, + }, + { + name: "keeps the baseline when it rejects a lower snapshot from the baseline's own time", + lines: [ + at("10:00:00", counted(counters(100, 20), counters(100, 20))), + at("10:00:00", counted(counters(20, 4), counters(120, 24))), + at("10:00:00", counted(counters(30, 6), counters(150, 30))), + at("10:00:00", counted(counters(20, 4), counters(120, 24))), + at("10:00:00", counted(counters(30, 6), counters(150, 30))), + ], + processed: [120, 24, 36, 0, 0], + skipped: 1, + }, + { + name: "does not bill replayed snapshots against a cached baseline of unknown age", + lines: [ + at("10:00:00", counted(counters(100, 20), counters(100, 20))), + at("10:00:05", counted(counters(20, 4), counters(120, 24))), + at("10:00:10", counted(counters(30, 6), counters(150, 30))), + at("10:00:00", counted(counters(100, 20), counters(100, 20))), + at("10:00:05", counted(counters(20, 4), counters(120, 24))), + ], + undatedAfter: 3, + processed: [120, 24, 36, 0, 0], + skipped: 1, + }, + { + name: "warns about the anonymous request a rewound snapshot does not report", + lines: [ + at("10:00:00", counted(counters(100, 20), counters(100, 20))), + anonymousRequest("10:00:05", counters(40, 8)), + anonymousRequest("10:00:06", counters(20, 4)), + at("10:00:07", counted(counters(20, 4), counters(60, 12))), + at("10:00:08", counted(counters(10, 2), counters(70, 14))), + ], + processed: [120, 0, 0, 24, 12], + skipped: 1, + }, + ])("$name", ({ lines, undatedAfter, processed, skipped }) => { + const state = initialCodexScanState(); + parseCodexLine(turnContext, state); + const billed = lines.map((line, index) => { + if (index === undatedAfter) delete state.latestCumulativeAtMs; + const record = parseCodexLine(line, state); + return record === null ? 0 : totalTokens(record.totals); + }); + + expect({ processed: billed, skipped: codexUsageFormat.readMalformedRecords?.(state) }).toEqual({ + processed, + skipped, + }); + }); + it("ignores a replay of the current sequence's first request without rewinding the baseline", () => { const state = initialCodexScanState(); parseCodexLine(turnContext, state); diff --git a/apps/server/src/provider/Drivers/codexUsage.ts b/apps/server/src/provider/Drivers/codexUsage.ts index 90f7c78b849f..a2977d3080fe 100644 --- a/apps/server/src/provider/Drivers/codexUsage.ts +++ b/apps/server/src/provider/Drivers/codexUsage.ts @@ -54,9 +54,15 @@ function readCodexTokenUsage(value: unknown): CodexTokenUsage | null { return isCodexTokenUsage(usage) ? usage : null; } +// Every move of the baseline dates it, so an older snapshot re-emitted later is never a rewind. +function dateCodexBaseline(state: CodexScanState, timestampMs: number) { + state.latestCumulativeAtMs = Math.max(state.latestCumulativeAtMs ?? timestampMs, timestampMs); +} + // A zeroed baseline rebases the counters, so the sequence start measured against it is forgotten. -function rebaseCodexUsage(state: CodexScanState, cumulative: CodexTokenUsage) { +function rebaseCodexUsage(state: CodexScanState, cumulative: CodexTokenUsage, timestampMs: number) { state.lastCumulativeUsage = cumulative; + dateCodexBaseline(state, timestampMs); delete state.unreportedRequests; if (cumulative.input_tokens + cumulative.output_tokens === 0) delete state.sequenceStartSignature; } @@ -111,6 +117,18 @@ function unreportedRequests( ); } +// A rewound snapshot vouches only for the request it reports, so the anonymous requests read before +// it can be neither billed nor ruled out as that request's prefix. +function discardedRequests( + pending: readonly CodexUnreportedRequest[] | undefined, + last: CodexTokenUsage, +) { + const final = pending?.at(-1); + if (pending === undefined || final === undefined) return 0; + const reported = CODEX_USAGE_FIELDS.every((key) => final.usage[key] === last[key]); + return pending.length - (reported ? 1 : 0); +} + /** * Codex `token_count` events carry no model or service tier, so both are carried forward: the model * from the most recent `turn_context`, the tier from the most recent `thread_settings_applied`. @@ -121,12 +139,14 @@ export const CodexScanState = Schema.Struct({ sessionId: Schema.mutableKey(Schema.String), lastUsageSignature: Schema.mutableKey(Schema.NullOr(Schema.String)), lastCumulativeUsage: Schema.mutableKey(Schema.NullOr(CodexTokenUsage)), - // Optional so scan states cached before this field existed stay valid. + // Optional so scan states cached before these fields existed stay valid. sequenceStartSignature: Schema.mutableKey(Schema.optionalKey(Schema.String)), + latestCumulativeAtMs: Schema.mutableKey(Schema.optionalKey(Schema.Finite)), // `token_usage_record` usage since the last cumulative snapshot, one entry per record. unreportedRequests: Schema.mutableKey( Schema.optionalKey(Schema.mutable(Schema.Array(CodexUnreportedRequest))), ), + requestLedgerActive: Schema.mutableKey(Schema.optionalKey(Schema.Boolean)), malformedRecords: Schema.mutableKey(TokenCount), sawSessionMeta: Schema.mutableKey(Schema.Boolean), modelProvider: Schema.mutableKey(Schema.String), @@ -181,8 +201,8 @@ function isForkCopy(state: CodexScanState, timestampMs: number): boolean { } /** - * Cumulative counters validate request deltas. Older events without cumulative counters use - * normalized consecutive-request deduplication. + * Identified request-ledger entries report spending independently of context snapshots. + * Legacy snapshots validate request deltas against cumulative counters. */ export function parseCodexLine(line: string, state: CodexScanState): readonly UsageRecord[] { let parsed: unknown; @@ -221,6 +241,7 @@ function parseCodexRecord(parsed: unknown, state: CodexScanState): readonly Usag } if (record["type"] === "turn_context") { + state.requestLedgerActive = false; if (typeof payloadRecord["model"] === "string") state.model = payloadRecord["model"]; // Model carry is useful for usage, but history recovery needs this context's own identity. state.turnId = @@ -232,13 +253,32 @@ function parseCodexRecord(parsed: unknown, state: CodexScanState): readonly Usag return []; } - // Several of these may sit between two snapshots, so they only explain a gap in the counters. + // Request ledgers include spending that context snapshots can omit. if (record["type"] === "token_usage_record") { const timestampMs = parseTimestampMs(record["timestamp"]); const usage = readCodexTokenUsage(payloadRecord["usage"]); if (timestampMs === null || usage === null || state.model.length === 0) return []; // Copied history is already billed in the parent. if (isForkCopy(state, timestampMs)) return []; + const responseId = payloadRecord["response_id"]; + if (typeof responseId === "string" && responseId.length > 0) { + const pending = state.unreportedRequests ?? []; + state.requestLedgerActive = true; + for (const request of pending) { + state.lastCumulativeUsage = advanceCodexUsage(state.lastCumulativeUsage, request.usage); + } + state.lastCumulativeUsage = advanceCodexUsage(state.lastCumulativeUsage, usage); + dateCodexBaseline(state, timestampMs); + delete state.unreportedRequests; + return [ + ...pending.map((request) => codexRequestRecord(state, request.usage, request)), + { + ...codexRequestRecord(state, usage, codexAttribution(state, timestampMs)), + dedupeKey: JSON.stringify(["codex", state.sessionId, responseId]), + }, + ]; + } + state.requestLedgerActive = false; (state.unreportedRequests ??= []).push({ ...codexAttribution(state, timestampMs), usage }); return []; } @@ -270,18 +310,26 @@ function parseCodexRecord(parsed: unknown, state: CodexScanState): readonly Usag if (isForkCopy(state, timestampMs)) { // Copied history is already billed in the parent, but establishes the child's baseline. const signature = last === null ? null : JSON.stringify(last); - if (cumulative !== null) rebaseCodexUsage(state, cumulative); + if (cumulative !== null) rebaseCodexUsage(state, cumulative, timestampMs); else if (!hasCumulative && last !== null && signature !== state.lastUsageSignature) { state.lastCumulativeUsage = advanceCodexUsage(state.lastCumulativeUsage, last); + dateCodexBaseline(state, timestampMs); } state.lastUsageSignature = signature; return []; } + // Context snapshots can rewind on resume and omit compaction spending; identified requests are authoritative. + if (state.requestLedgerActive) { + if (cumulative !== null) rebaseCodexUsage(state, cumulative, timestampMs); + return []; + } + const previous = state.lastCumulativeUsage; + const latestCumulativeAtMs = state.latestCumulativeAtMs; const pending = state.unreportedRequests; // A valid snapshot remains the next baseline even when its paired request is malformed. - if (cumulative !== null) rebaseCodexUsage(state, cumulative); + if (cumulative !== null) rebaseCodexUsage(state, cumulative, timestampMs); if (last === null || (hasCumulative && cumulative === null)) { state.malformedRecords += 1; return []; @@ -316,12 +364,20 @@ function parseCodexRecord(parsed: unknown, state: CodexScanState): readonly Usag // Only the same event is a replay; a later request can restart with identical counters. if (restarted && sequenceStart === state.sequenceStartSignature) { state.lastCumulativeUsage = previous; + // The retained baseline keeps its own age; one of unknown age must not take the replay's. + if (latestCumulativeAtMs === undefined) delete state.latestCumulativeAtMs; return []; } - // Forks and paginated continuations can inherit totals without copying a counter snapshot. - // Only bill the latest request, and require the unseen prefix to be valid usage too. + // Forks and paginated continuations can inherit totals without copying a counter snapshot, and + // a later request can continue from an earlier snapshot; a re-emitted old snapshot is not later. + // Only bill the latest request, and require the unseen prefix to be valid usage too. A baseline + // of unknown age (a scan state cached before every move dated it) cannot order the snapshot. + const lowered = + previous !== null && CODEX_USAGE_FIELDS.some((key) => cumulative[key] < previous[key]); + const rewound = + lowered && latestCumulativeAtMs !== undefined && timestampMs > latestCumulativeAtMs; const reconciled = - previous === null + previous === null || rewound ? readCodexTokenUsage( Object.fromEntries(CODEX_USAGE_FIELDS.map((key) => [key, cumulative[key] - last[key]])), ) !== null @@ -329,10 +385,24 @@ function parseCodexRecord(parsed: unknown, state: CodexScanState): readonly Usag if (!restarted && !reconciled) { const bridged = unreportedRequests(previous, cumulative, last, pending); if (bridged === null) { + // A rejected snapshot from before the baseline's time is a replay, not the next baseline. + // At the baseline's own time only lower counters are one; zeroed counters still rebase. + const replayed = + latestCumulativeAtMs !== undefined && + (timestampMs < latestCumulativeAtMs || + (timestampMs === latestCumulativeAtMs && + lowered && + cumulative.input_tokens + cumulative.output_tokens > 0)); + if (replayed) { + state.lastCumulativeUsage = previous; + if (pending !== undefined) state.unreportedRequests = pending; + } state.malformedRecords += 1; return []; } unreported = bridged; + } else if (!restarted && rewound) { + state.malformedRecords += discardedRequests(pending, last); } if (restarted) state.sequenceStartSignature = sequenceStart; } else { @@ -344,6 +414,7 @@ function parseCodexRecord(parsed: unknown, state: CodexScanState): readonly Usag return []; } state.lastCumulativeUsage = advanced; + dateCodexBaseline(state, timestampMs); } state.lastUsageSignature = signature; @@ -418,6 +489,7 @@ export const codexUsageFormat: TranscriptUsageFormat = { session_id: true, model_provider: true, turn_id: true, + response_id: true, model: true, thread_settings: { service_tier: true }, forked_from_id: true, @@ -481,10 +553,7 @@ export const codexUsageReader: ProviderUsageReader< const path = yield* Path.Path; const environmentHome = environment.CODEX_HOME?.trim(); const layout = yield* resolveCodexHomeLayout( - config.setupMode !== "managed" && - !config.homePath.trim() && - !config.shadowHomePath.trim() && - environmentHome + config.setupMode !== "managed" && !config.homePath.trim() && environmentHome ? { ...config, homePath: environmentHome } : config, ); diff --git a/apps/server/src/provider/EventNdjsonLogger.test.ts b/apps/server/src/provider/EventNdjsonLogger.test.ts index eed20381b46c..0bb55b9bcb39 100644 --- a/apps/server/src/provider/EventNdjsonLogger.test.ts +++ b/apps/server/src/provider/EventNdjsonLogger.test.ts @@ -255,6 +255,65 @@ it.layer(NodeServices.layer)("EventNdjsonLogger", (it) => { }), ); + it.effect("drops Cursor streaming deltas while keeping its lifecycle updates", () => + Effect.gen(function* () { + const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-")); + const basePath = NodePath.join(tempDir, "events.log"); + const threadId = ThreadId.make("cursor-deltas"); + const cursorUpdate = (update: object) => ({ + provider: "cursor", + kind: "protocol", + event: { + direction: "incoming", + stage: "decoded", + payload: { type: "interaction.update", runId: "run-1", update }, + }, + }); + const toolCall = { type: "shell", args: { command: "ls" } }; + const transient = [ + { type: "text-delta", text: "Hel" }, + { type: "thinking-delta", text: "Hmm" }, + { type: "token-delta", tokens: 3 }, + { type: "shell-output-delta", event: { stdout: "a.ts\n" } }, + { type: "partial-tool-call", callId: "call-1", modelCallId: "m-1", toolCall }, + { + type: "tool-call-delta", + callId: "call-1", + modelCallId: "m-1", + taskUpdate: { type: "text-delta", text: "Sub" }, + }, + ].map(cursorUpdate); + const retained = [ + { type: "tool-call-started", callId: "call-1", modelCallId: "m-1", toolCall }, + { + type: "tool-call-delta", + callId: "call-1", + modelCallId: "m-1", + taskUpdate: { type: "tool-call-completed", callId: "call-2", toolCall }, + }, + { type: "tool-call-completed", callId: "call-1", modelCallId: "m-1", toolCall }, + { type: "turn-ended" }, + ].map(cursorUpdate); + + try { + const store = yield* makeEventNdjsonLogStore(basePath, { batchWindowMs: 0 }); + const native = store.logger("native"); + for (const event of [...transient, ...retained]) { + yield* native.write(event, threadId); + } + yield* store.close(); + + const records = NodeFS.readFileSync(ownedLogPath(basePath, "cursor-deltas"), "utf8") + .trim() + .split("\n") + .map((line) => decodeUnknownJson(parseLogLine(line).payload)); + assert.deepEqual(records, retained); + } finally { + NodeFS.rmSync(tempDir, { recursive: true, force: true }); + } + }), + ); + it.effect("bounds oversized records while retaining failure details", () => Effect.gen(function* () { const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-")); diff --git a/apps/server/src/provider/EventNdjsonLogger.ts b/apps/server/src/provider/EventNdjsonLogger.ts index 6a1d3844b237..5a7b3578578f 100644 --- a/apps/server/src/provider/EventNdjsonLogger.ts +++ b/apps/server/src/provider/EventNdjsonLogger.ts @@ -63,6 +63,13 @@ const transientNativeMethods = new Set([ "turn/diff/updated", ]); const transientAcpUpdates = new Set(["agent_message_chunk", "agent_thought_chunk"]); +const transientCursorUpdates = new Set([ + "text-delta", + "thinking-delta", + "token-delta", + "shell-output-delta", + "partial-tool-call", +]); export type EventNdjsonStream = "native" | "canonical" | "orchestration"; @@ -219,6 +226,18 @@ export function shouldPersistProviderEvent(stream: EventNdjsonStream, event: unk } } + if (nativeType === "interaction.update") { + let update = Reflect.get(nativeEvent, "update"); + if (typeof update !== "object" || update === null) return true; + // Subagent updates arrive wrapped; keep the wrapper when it carries a lifecycle record. + if (Reflect.get(update, "type") === "tool-call-delta") { + update = Reflect.get(update, "taskUpdate"); + if (typeof update !== "object" || update === null) return true; + } + const updateType = Reflect.get(update, "type"); + return typeof updateType !== "string" || !transientCursorUpdates.has(updateType); + } + const payload = Reflect.get(nativeEvent, "payload"); if (typeof payload !== "object" || payload === null) return true; diff --git a/apps/server/src/provider/GrokProvider.test.ts b/apps/server/src/provider/GrokProvider.test.ts index be899ae54003..3f28649f498b 100644 --- a/apps/server/src/provider/GrokProvider.test.ts +++ b/apps/server/src/provider/GrokProvider.test.ts @@ -639,7 +639,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("readGrokAccount" ), ); expect(limits.windows[0]?.usedPercent).toBe(37); - }).pipe(Effect.scoped), + }), ); it.effect( @@ -749,7 +749,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("readGrokAccount" message: "Grok could not read usage limits.", }); } - }).pipe(Effect.scoped), + }), ); it.effect("sanitizes HTTP failures and malformed billing responses, keeping the account", () => diff --git a/apps/server/src/provider/ModelManifest.test.ts b/apps/server/src/provider/ModelManifest.test.ts index e94ce85bf533..0b4c4c062dca 100644 --- a/apps/server/src/provider/ModelManifest.test.ts +++ b/apps/server/src/provider/ModelManifest.test.ts @@ -266,7 +266,7 @@ const layerService = (input: { ); describe("ModelManifest service", () => { - it.live("prefers a fetched manifest over the bundle and caches it to disk", () => + it.effect("prefers a fetched manifest over the bundle and caches it to disk", () => Effect.gen(function* () { const service = yield* ModelManifest.make; const refreshed = yield* service.refresh; @@ -285,29 +285,32 @@ describe("ModelManifest service", () => { ), ); - it.live("ignores older remote edits without replacing the current manifest or disk cache", () => { - let remote = { ...REMOTE_MANIFEST, updatedAt: "2000-01-01T00:00:00Z" }; - return Effect.gen(function* () { - const service = yield* ModelManifest.make; - assert.deepStrictEqual(yield* service.refresh, ModelManifest.BUNDLED_MODEL_MANIFEST); - assert.deepStrictEqual(yield* service.current, ModelManifest.BUNDLED_MODEL_MANIFEST); + it.effect( + "ignores older remote edits without replacing the current manifest or disk cache", + () => { + let remote = { ...REMOTE_MANIFEST, updatedAt: "2000-01-01T00:00:00Z" }; + return Effect.gen(function* () { + const service = yield* ModelManifest.make; + assert.deepStrictEqual(yield* service.refresh, ModelManifest.BUNDLED_MODEL_MANIFEST); + assert.deepStrictEqual(yield* service.current, ModelManifest.BUNDLED_MODEL_MANIFEST); - remote = { ...REMOTE_MANIFEST, updatedAt: REMOTE_UPDATED_AT }; - assert.deepStrictEqual(yield* service.forceRefresh, REMOTE_MANIFEST); - remote = { ...REMOTE_MANIFEST, updatedAt: ModelManifest.BUNDLED_MODEL_MANIFEST.updatedAt! }; - assert.deepStrictEqual(yield* service.forceRefresh, REMOTE_MANIFEST); - const rebooted = yield* ModelManifest.make; - assert.deepStrictEqual(yield* rebooted.current, REMOTE_MANIFEST); - }).pipe( - Effect.scoped, - Effect.provide( - layerService({ - prefix: "model-manifest-stale-fetch-test", - response: () => Response.json(remote), - }), - ), - ); - }); + remote = { ...REMOTE_MANIFEST, updatedAt: REMOTE_UPDATED_AT }; + assert.deepStrictEqual(yield* service.forceRefresh, REMOTE_MANIFEST); + remote = { ...REMOTE_MANIFEST, updatedAt: ModelManifest.BUNDLED_MODEL_MANIFEST.updatedAt! }; + assert.deepStrictEqual(yield* service.forceRefresh, REMOTE_MANIFEST); + const rebooted = yield* ModelManifest.make; + assert.deepStrictEqual(yield* rebooted.current, REMOTE_MANIFEST); + }).pipe( + Effect.scoped, + Effect.provide( + layerService({ + prefix: "model-manifest-stale-fetch-test", + response: () => Response.json(remote), + }), + ), + ); + }, + ); it.effect("preserves the last-good remote cache when later payloads are invalid", () => { let responseIndex = 0; @@ -364,7 +367,7 @@ describe("ModelManifest service", () => { ); }); - it.live("drops a disk cache of a manifest older than the bundled one", () => + it.effect("drops a disk cache of a manifest older than the bundled one", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -400,7 +403,7 @@ describe("ModelManifest service", () => { ), ); - it.live("does not fetch when provider update checks are disabled", () => + it.effect("does not fetch when provider update checks are disabled", () => Effect.gen(function* () { let fetchCount = 0; const service = yield* ModelManifest.make.pipe( diff --git a/apps/server/src/provider/ProjectSettingsOverrideSweep.ts b/apps/server/src/provider/ProjectSettingsOverrideSweep.ts index aa7d20527149..06f61f9ad8bf 100644 --- a/apps/server/src/provider/ProjectSettingsOverrideSweep.ts +++ b/apps/server/src/provider/ProjectSettingsOverrideSweep.ts @@ -1,32 +1,41 @@ -import { type ProjectId, type ServerSettings } from "@t3tools/contracts"; +import * as NodeUtil from "node:util"; + +import { type ProjectSettingsOverrides } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as ProjectStore from "../orchestration-v2/ProjectStore.ts"; import * as Settings from "../serverSettings.ts"; -const findOrphanedProjectIds = ( - settings: Pick, - liveProjectIds: ReadonlySet, -): ReadonlyArray => - Object.keys(settings.projectSettingsOverrides) - .map((projectId) => projectId as ProjectId) - .filter((projectId) => !liveProjectIds.has(projectId)); +export function deletedProjectOverridePatch(entry: ProjectSettingsOverrides | undefined) { + // Keep cleanup policy for deleted threads; omitting providerInstances would preserve obsolete references in the merge. + return entry?.worktreeCleanup !== undefined + ? { worktreeCleanup: entry.worktreeCleanup, providerInstances: {} } + : null; +} export const sweep = Effect.gen(function* () { const serverSettings = yield* Settings.ServerSettingsService; const projectStore = yield* ProjectStore.ProjectStoreV2; const settings = yield* serverSettings.getSettings; - const liveShells = yield* projectStore.listShells(); - const liveProjectIds = new Set(liveShells.map((shell) => shell.id)); + const projects = yield* projectStore.list({ includeDeleted: true }); + const projectsById = new Map( + projects.map((project) => [project.projectId, project]), + ); - const orphanedProjectIds = findOrphanedProjectIds(settings, liveProjectIds); - if (orphanedProjectIds.length === 0) return; + const overrides = Object.entries(settings.projectSettingsOverrides).flatMap( + ([projectId, entry]) => { + const project = projectsById.get(projectId); + if (project?.deletedAt === null) return []; + const patch = project ? deletedProjectOverridePatch(entry) : null; + if (NodeUtil.isDeepStrictEqual(entry, patch)) return []; + return [[projectId, patch] as const]; + }, + ); + if (overrides.length === 0) return; yield* serverSettings.updateSettings({ - projectSettingsOverrides: Object.fromEntries( - orphanedProjectIds.map((projectId) => [projectId, null]), - ), + projectSettingsOverrides: Object.fromEntries(overrides), }); }).pipe( Effect.asVoid, diff --git a/apps/server/src/provider/ProviderAuthService.test.ts b/apps/server/src/provider/ProviderAuthService.test.ts index 5023dc5bf47e..603df5e7e4bc 100644 --- a/apps/server/src/provider/ProviderAuthService.test.ts +++ b/apps/server/src/provider/ProviderAuthService.test.ts @@ -743,7 +743,7 @@ it.effect("queued logout prompts resolve the current controller after provider r assert.isTrue(yield* Fiber.join(queued)); assert.isTrue(replacementLoggedOut); assert.strictEqual(harness.actions.filter((action) => action === "native-logout").length, 1); - }).pipe(Effect.scoped), + }), ); it.effect.each(["start", "logout", "prompt"] as const)( @@ -814,7 +814,7 @@ it.effect.each(["start", "logout", "prompt"] as const)( const blocked = yield* Effect.flip(harness.service.logout({ instanceId })); assert.include(blocked.detail, "shared sign-in"); assert.equal(replacementMutations, 0); - }).pipe(Effect.scoped), + }), ); it.effect.each(["selection", "drain"] as const)( @@ -855,5 +855,5 @@ it.effect.each(["selection", "drain"] as const)( assert.notInclude(harness.actions, "stop:peer-persisted"); assert.isFalse(harness.sessions.has(ThreadId.make("target-draining"))); assert.include(harness.actions, "native-logout"); - }).pipe(Effect.scoped), + }), ); diff --git a/apps/server/src/provider/ProviderHostLive.ts b/apps/server/src/provider/ProviderHostLive.ts index 6398210599b6..fed07efc035e 100644 --- a/apps/server/src/provider/ProviderHostLive.ts +++ b/apps/server/src/provider/ProviderHostLive.ts @@ -4,11 +4,13 @@ import * as Digest from "effect/crypto/Digest"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import { resolveAttachmentPath } from "../attachmentStore.ts"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; import * as ServerConfig from "../config.ts"; +import * as ServerLifecycleEvents from "../serverLifecycleEvents.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as DocumentText from "./DocumentText.ts"; import * as ProviderCredentialStore from "./ProviderCredentialStore.ts"; @@ -22,6 +24,10 @@ export const layer = Layer.effect( const secrets = yield* ServerSecretStore.ServerSecretStore; const digest = yield* Digest.Digest; const fileSystem = yield* FileSystem.FileSystem; + // Optional so standalone provider layers in tests probe at once. + const lifecycleEvents = yield* Effect.serviceOption( + ServerLifecycleEvents.ServerLifecycleEvents, + ); return ProviderHost.ProviderHost.of({ paths: { cwd: config.cwd, @@ -37,6 +43,10 @@ export const layer = Layer.effect( subscribe: serverSettings.subscribeChanges, }, shouldRunBackgroundWork: backgroundPolicy.shouldRunScopeWork, + ...Option.match(lifecycleEvents, { + onNone: () => ({}), + onSome: (events) => ({ awaitServerReady: events.awaitReady }), + }), resolveAttachmentPath: (attachment) => resolveAttachmentPath({ attachmentsDir: config.attachmentsDir, attachment }), // DocumentText owns the path-resolution span. diff --git a/apps/server/src/provider/ProviderInstanceRegistry.test.ts b/apps/server/src/provider/ProviderInstanceRegistry.test.ts index ab35fe4d9ae5..ce6416332bae 100644 --- a/apps/server/src/provider/ProviderInstanceRegistry.test.ts +++ b/apps/server/src/provider/ProviderInstanceRegistry.test.ts @@ -212,7 +212,8 @@ describe("ProviderInstanceRegistry — multi-instance codex slice", () => { ); const layerBase = ProviderHostLive.layer.pipe(Layer.provideMerge(layerBaseDeps)); const layerTest = ProviderOrchestrationAdapterInfrastructure.layer.pipe( - Layer.provide(Layer.merge(PortScanner.layerTest, SqlitePersistence.layerMemory)), + Layer.provide(SqlitePersistence.layerMemory), + Layer.provideMerge(PortScanner.layerTest), Layer.provideMerge(layerBase), ); @@ -346,7 +347,7 @@ describe("ProviderInstanceRegistry — multi-instance codex slice", () => { }).pipe(Effect.provide(layerTest)), ); - it.live("treats an explicit in-config enabled:false as disabling despite the envelope", () => + it.effect("treats an explicit in-config enabled:false as disabling despite the envelope", () => Effect.gen(function* () { const staleId = ProviderInstanceId.make("codex_stale"); const configMap: ProviderInstanceConfigMap = { @@ -370,7 +371,7 @@ describe("ProviderInstanceRegistry — multi-instance codex slice", () => { }).pipe(Effect.provide(layerTest)), ); - it.live("lists a blocked driver's instances as unavailable without running them", () => + it.effect("lists a blocked driver's instances as unavailable without running them", () => Effect.gen(function* () { const instanceId = ProviderInstanceId.make("codex"); const reason = "Example Corp's policy does not allow the Codex provider."; @@ -862,7 +863,7 @@ describe("ProviderInstanceRegistry — multi-instance codex slice", () => { ); } - it.live( + it.effect( "shadows instances whose driver is not registered in this build without failing boot", () => Effect.gen(function* () { @@ -990,7 +991,7 @@ describe("ProviderInstanceRegistry secret environment", () => { { name: "BASE_URL", value: "https://example.test", sensitive: false }, ], ]); - }).pipe(Effect.scoped); + }); }); // An empty secret must not let a driver fall back to a credential the server inherited. @@ -1013,7 +1014,7 @@ describe("ProviderInstanceRegistry secret environment", () => { yield* mutator.reconcile({ ...fixture.configMap }); expect(fixture.reads).toHaveLength(readsAtStartup); expect(yield* registry.listUnavailable).toHaveLength(1); - }).pipe(Effect.scoped); + }); }); it.effect("does not read the secrets of an instance its config decodes as disabled", () => { @@ -1025,7 +1026,7 @@ describe("ProviderInstanceRegistry secret environment", () => { expect(fixture.reads).toEqual([]); expect(fixture.invalidated).toEqual([]); expect(fixture.created).toHaveLength(1); - }).pipe(Effect.scoped); + }); }); it.effect("re-reads secrets on reload and rebuilds only when they changed", () => { @@ -1048,7 +1049,7 @@ describe("ProviderInstanceRegistry secret environment", () => { "sk-live", "sk-rotated", ]); - }).pipe(Effect.scoped); + }); }); it.effect("reads a config field typed SettingsSecretValue and rebuilds when it rotates", () => { @@ -1068,7 +1069,7 @@ describe("ProviderInstanceRegistry secret environment", () => { { apiKey: "gw-live", model: "m" }, { apiKey: "gw-rotated", model: "m" }, ]); - }).pipe(Effect.scoped); + }); }); it.effect.each([ @@ -1084,7 +1085,7 @@ describe("ProviderInstanceRegistry secret environment", () => { return Effect.gen(function* () { yield* fixture.make; expect(fixture.configs).toEqual([expect.objectContaining({ [field]: "provider-live" })]); - }).pipe(Effect.scoped); + }); }); it.effect("keeps a working instance live when a reload cannot read its secret", () => { @@ -1098,7 +1099,7 @@ describe("ProviderInstanceRegistry secret environment", () => { expect(yield* registry.getInstance(instanceId)).toBeDefined(); expect(yield* registry.listUnavailable).toEqual([]); expect(fixture.created).toHaveLength(1); - }).pipe(Effect.scoped); + }); }); it.effect("starts without waiting for secrets to be read", () => { @@ -1115,6 +1116,6 @@ describe("ProviderInstanceRegistry secret environment", () => { yield* PubSub.take(changes); expect(yield* registry.getInstance(instanceId)).toBeDefined(); expect(yield* registry.listUnavailable).toEqual([]); - }).pipe(Effect.scoped); + }); }); }); diff --git a/apps/server/src/provider/ProviderInstanceRegistry.ts b/apps/server/src/provider/ProviderInstanceRegistry.ts index 3f4df0eb35d1..ae51e8009645 100644 --- a/apps/server/src/provider/ProviderInstanceRegistry.ts +++ b/apps/server/src/provider/ProviderInstanceRegistry.ts @@ -92,9 +92,20 @@ const withoutSecretReferences = ( typeof variable.value === "string" ? [{ ...variable, value: variable.value }] : [], ); +interface CheckedSecret { + readonly reference: SecretReference; + readonly value: string; +} + +const sameReference = (a: SecretReference, b: SecretReference) => + a.provider === b.provider && a.reference === b.reference; + +// `checked` supplies its reference's value instead of the resolver, which a concurrent refresh +// may have re-read since the check. const readInstanceSecrets = Effect.fn("ProviderInstanceRegistry.readInstanceSecrets")(function* ( driver: AnyProviderDriver | undefined, entry: ProviderInstanceConfig, + checked?: CheckedSecret, ) { const variables = entry.environment ?? []; const configFields = configSecretReferences(driver, entry); @@ -102,8 +113,15 @@ const readInstanceSecrets = Effect.fn("ProviderInstanceRegistry.readInstanceSecr ...secretReferencesIn(variables), ...configFields.map(({ reference }) => reference), ]; - const values = - references.length === 0 ? [] : yield* (yield* SecretResolver.SecretResolver).read(references); + const isChecked = (reference: SecretReference) => + checked !== undefined && sameReference(reference, checked.reference); + const toRead = references.filter((reference) => !isChecked(reference)); + const read = + toRead.length === 0 ? [] : yield* (yield* SecretResolver.SecretResolver).read(toRead); + let readIndex = 0; + const values = references.map((reference) => + isChecked(reference) ? checked?.value : read[readIndex++], + ); const unreadable: Array = []; let index = 0; // An empty secret reads as unreadable, so a driver cannot fall back to an inherited credential. @@ -130,6 +148,36 @@ const readInstanceSecrets = Effect.fn("ProviderInstanceRegistry.readInstanceSecr return { secrets, unreadable }; }); +// Replaces only the checked reference's value in the secrets a live instance already holds, so +// another reference whose read failed earlier is not read again and does not park the instance. +const carryCheckedSecret = ( + driver: AnyProviderDriver | undefined, + entry: ProviderInstanceConfig, + current: InstanceSecrets, + { reference: checked, value }: CheckedSecret, +): InstanceSecrets => { + const matches = (reference: SecretReference) => sameReference(reference, checked); + // A live instance resolved every variable in order, so slots line up with the entry's variables + // by position; names may repeat. + const variables = entry.environment ?? []; + const replacedSlots = configSecretReferences(driver, entry).filter(({ reference }) => + matches(reference), + ); + return { + environment: current.environment.map((variable, index) => { + const source = variables[index]?.value; + return source !== undefined && typeof source !== "string" && matches(source) + ? { ...variable, value } + : variable; + }), + config: current.config.map((slot) => + replacedSlots.some(({ field, key }) => field === slot.field && key === slot.key) + ? { ...slot, value } + : slot, + ), + }; +}; + /** Conflicting saved flags must preserve any explicit disable. */ const resolveEntryEnabled = (entry: ProviderInstanceConfig, typedConfig: unknown): boolean => { const rawConfigEnabled = providerInstanceConfigEnabledFlag(entry.config); @@ -161,6 +209,7 @@ const buildEntry = Effect.fn("ProviderInstanceRegistry.buildEntry")(function* { - const { driversById, parentScope, instanceId, rawInstanceId, entry, deferSecrets } = input; + const { + driversById, + parentScope, + instanceId, + rawInstanceId, + entry, + deferSecrets, + carriedSecrets, + } = input; const blocked = input.blockedReason(entry.driver); if (blocked !== undefined) { return { @@ -235,12 +292,14 @@ const buildEntry = Effect.fn("ProviderInstanceRegistry.buildEntry")(function* 0) { @@ -311,9 +370,10 @@ const makeReconcile = (input: { configMap: ProviderInstanceConfigMap, rebuild?: ReadonlySet, deferSecrets?: boolean, + carried?: ReadonlyMap, ) => Effect.Effect) => { const { state, driversById, blockedReason, parentScope } = input; - return (configMap, rebuild = new Set(), deferSecrets = false) => + return (configMap, rebuild = new Set(), deferSecrets = false, carried = new Map()) => Effect.gen(function* () { const previousEntries = yield* Ref.get(state.entries); const previousUnavailable = yield* Ref.get(state.unavailable); @@ -365,6 +425,7 @@ const makeReconcile = (input: { const instanceId = ProviderInstanceId.make(rawInstanceId); const kept = previousEntries.has(instanceId) && !replacedIds.has(instanceId); return kept || + carried.has(instanceId) || parkedSnapshot(instanceId, entry) !== undefined || !(yield* isEntryEnabled(driversById, entry)) ? [] @@ -406,6 +467,7 @@ const makeReconcile = (input: { rawInstanceId, entry, deferSecrets, + carriedSecrets: carried.get(instanceId), }); if (result.kind === "live") { builtEntries.set(instanceId, result.live); @@ -503,44 +565,87 @@ export const makeProviderInstanceRegistry = Effect.fn( ); // Rebuilds an instance only when it was unavailable or its secrets changed, so a refresh does - // not restart providers whose credentials are the same. - const reloadSecrets: ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutatorShape["reloadSecrets"] = - Effect.fn("ProviderInstanceRegistry.reloadSecrets")( - function* (instanceId) { - const configMap = yield* Ref.get(configMapRef); - const targets = yield* Effect.filter(Object.entries(configMap), ([rawInstanceId, entry]) => - (instanceId === undefined || rawInstanceId === instanceId) && - secretReferencesOf(driversById, entry).length > 0 - ? isEntryEnabled(driversById, entry) - : Effect.succeed(false), - ); - if (targets.length === 0) return; + // not restart providers whose credentials are the same. Without `checked` the cached values are + // dropped first; with it, only that reference takes the checked value and the rest carry over. + const reloadWhere = Effect.fn("ProviderInstanceRegistry.reloadWhere")( + function* ( + select: (rawInstanceId: string, entry: ProviderInstanceConfig) => boolean, + checked?: CheckedSecret, + ) { + const configMap = yield* Ref.get(configMapRef); + const targets = yield* Effect.filter(Object.entries(configMap), ([rawInstanceId, entry]) => + select(rawInstanceId, entry) && secretReferencesOf(driversById, entry).length > 0 + ? isEntryEnabled(driversById, entry) + : Effect.succeed(false), + ); + if (targets.length === 0) return []; + if (checked === undefined) { const resolver = yield* SecretResolver.SecretResolver; const references = targets.flatMap(([, entry]) => secretReferencesOf(driversById, entry)); yield* resolver.invalidate(references); yield* resolver.read(references); - const live = yield* Ref.get(entries); - const rebuild = new Set(); - for (const [rawInstanceId, entry] of targets) { - const current = live.get(ProviderInstanceId.make(rawInstanceId)); - if (current === undefined) { - rebuild.add(ProviderInstanceId.make(rawInstanceId)); - continue; + } + const live = yield* Ref.get(entries); + const rebuild = new Set(); + const carried = new Map(); + for (const [rawInstanceId, entry] of targets) { + const instanceId = ProviderInstanceId.make(rawInstanceId); + const current = live.get(instanceId); + if (current === undefined) { + rebuild.add(instanceId); + if (checked !== undefined) { + const { secrets, unreadable } = yield* readInstanceSecrets( + driversById.get(entry.driver), + entry, + checked, + ); + if (unreadable.length === 0) carried.set(instanceId, secrets); } - const { secrets, unreadable } = yield* readInstanceSecrets( + continue; + } + if (checked !== undefined) { + const secrets = carryCheckedSecret( driversById.get(entry.driver), - entry, + current.entry, + current.secrets, + checked, ); - // A failed re-read keeps the working instance and the value it already holds. - if (unreadable.length === 0 && !Equal.equals(secrets, current.secrets)) { + if (!Equal.equals(secrets, current.secrets)) { rebuild.add(current.instance.instanceId); + carried.set(current.instance.instanceId, secrets); } + continue; } - yield* reconcileWithR(configMap, rebuild); - }, - (effect) => semaphore.withPermits(1)(effect), - Effect.provideContext(driverContext), - ); + const { secrets, unreadable } = yield* readInstanceSecrets( + driversById.get(entry.driver), + entry, + ); + // A failed re-read keeps the working instance and the value it already holds. + if (unreadable.length === 0 && !Equal.equals(secrets, current.secrets)) { + rebuild.add(current.instance.instanceId); + } + } + yield* reconcileWithR(configMap, rebuild, false, carried); + return targets.map(([rawInstanceId]) => ProviderInstanceId.make(rawInstanceId)); + }, + (effect) => semaphore.withPermits(1)(effect), + Effect.provideContext(driverContext), + ); + const reloadSecrets: ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutatorShape["reloadSecrets"] = + (instanceId) => + reloadWhere( + (rawInstanceId) => instanceId === undefined || rawInstanceId === instanceId, + undefined, + ).pipe(Effect.asVoid); + const applySecret: ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutatorShape["applySecret"] = + (reference, value) => + reloadWhere( + (_, entry) => + secretReferencesOf(driversById, entry).some((candidate) => + sameReference(candidate, reference), + ), + { reference, value }, + ); // Reading secrets can wait on a vault unlock, so the server does not wait for it to start. yield* semaphore.withPermits(1)( @@ -592,6 +697,7 @@ export const makeProviderInstanceRegistry = Effect.fn( const mutator: ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutatorShape = { reconcile, reloadSecrets, + applySecret, }; return { registry, mutator }; diff --git a/apps/server/src/provider/ProviderInstanceRegistryHydration.test.ts b/apps/server/src/provider/ProviderInstanceRegistryHydration.test.ts index dc44f0324a6b..02bc92f4b450 100644 --- a/apps/server/src/provider/ProviderInstanceRegistryHydration.test.ts +++ b/apps/server/src/provider/ProviderInstanceRegistryHydration.test.ts @@ -165,6 +165,7 @@ describe("layerSettingsWatcher", () => { const mutator = Layer.succeed(ProviderInstanceRegistryMutator, { reconcile: () => Queue.offer(reconciled, undefined), reloadSecrets: () => Effect.sync(() => void reloads++), + applySecret: () => Effect.succeed([]), }); yield* Layer.build( layerSettingsWatcher(Stream.fromQueue(changes), initial).pipe(Layer.provide(mutator)), diff --git a/apps/server/src/provider/ProviderInstanceRegistryMutator.ts b/apps/server/src/provider/ProviderInstanceRegistryMutator.ts index 35eb52d3f290..911a055d7962 100644 --- a/apps/server/src/provider/ProviderInstanceRegistryMutator.ts +++ b/apps/server/src/provider/ProviderInstanceRegistryMutator.ts @@ -1,5 +1,9 @@ /** Reconcile closes removed/replaced scopes before creating replacements: at most one live instance per id. */ -import type { ProviderInstanceConfigMap, ProviderInstanceId } from "@t3tools/contracts"; +import type { + ProviderInstanceConfigMap, + ProviderInstanceId, + SecretReference, +} from "@t3tools/contracts"; import * as Context from "effect/Context"; import type * as Effect from "effect/Effect"; @@ -7,6 +11,14 @@ export interface ProviderInstanceRegistryMutatorShape { readonly reconcile: (configMap: ProviderInstanceConfigMap) => Effect.Effect; /** Re-reads the secrets of every instance, or one, and rebuilds those whose secrets changed. */ readonly reloadSecrets: (instanceId?: ProviderInstanceId) => Effect.Effect; + /** + * Rebuilds the instances that read `reference` with `value`, keeping the other secrets a live + * instance holds, and returns them. A parked instance reads its other secrets from the resolver. + */ + readonly applySecret: ( + reference: SecretReference, + value: string, + ) => Effect.Effect>; } export class ProviderInstanceRegistryMutator extends Context.Service< diff --git a/apps/server/src/provider/ProviderRegistry.test.ts b/apps/server/src/provider/ProviderRegistry.test.ts index 311695db2a31..c42d78b237d4 100644 --- a/apps/server/src/provider/ProviderRegistry.test.ts +++ b/apps/server/src/provider/ProviderRegistry.test.ts @@ -17,6 +17,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as PubSub from "effect/PubSub"; +import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; @@ -35,6 +36,8 @@ import { ServerSettingsError, ServerProvider, type ServerProviderSlashCommand, + type ServerProviderUpdateState, + type ServerProviderVersionAdvisoryStatus, type ServerSettings as ContractServerSettings, } from "@t3tools/contracts"; import { HttpClient, HttpClientResponse } from "effect/http"; @@ -44,7 +47,11 @@ import { deepMerge } from "@t3tools/shared/Struct"; import { createModelCapabilities } from "@t3tools/shared/model"; import { applyServerSettingsPatch } from "@t3tools/shared/serverSettings"; -import { checkCodexProviderStatus, type CodexAppServerProviderSnapshot } from "./CodexProvider.ts"; +import { + checkCodexProviderStatus, + mapCodexModelCapabilities, + type CodexAppServerProviderSnapshot, +} from "./CodexProvider.ts"; import { BUNDLED_CLAUDE_MODEL_CATALOG } from "./ClaudeModelCatalog.ts"; import { checkClaudeProviderStatus } from "./ClaudeProvider.ts"; import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; @@ -334,6 +341,86 @@ const awaitPersistedProvider = ( Effect.forkScoped, ); +const codexUpdateSnapshot = (advisory: ServerProviderVersionAdvisoryStatus) => + ({ + instanceId: ProviderInstanceId.make("codex"), + driver: ProviderDriverKind.make("codex"), + status: "ready", + enabled: true, + installed: true, + auth: { status: "authenticated" }, + checkedAt: "2026-06-10T00:00:00.000Z", + version: "1.0.0", + models: [], + slashCommands: [], + skills: [], + versionAdvisory: { + status: advisory, + currentVersion: "1.0.0", + latestVersion: advisory === "unknown" ? null : advisory === "current" ? "1.0.0" : "2.0.0", + updateCommand: null, + canUpdate: true, + checkedAt: "2026-06-10T00:00:00.000Z", + message: null, + }, + }) as const satisfies ServerProvider; + +const makeUpdateState = (status: ServerProviderUpdateState["status"]) => + ({ + status, + startedAt: "2026-06-10T00:00:00.000Z", + finishedAt: status === "queued" || status === "running" ? null : "2026-06-10T00:01:00.000Z", + message: `update ${status}`, + output: null, + }) satisfies ServerProviderUpdateState; + +// One Codex instance whose probe returns whatever `snapshot` currently holds. +const buildRegistryOverSnapshot = Effect.fn("buildRegistryOverSnapshot")(function* ( + snapshot: Ref.Ref, +) { + const driver = ProviderDriverKind.make("codex"); + const instance = { + instanceId: ProviderInstanceId.make("codex"), + driverKind: driver, + continuationIdentity: { driverKind: driver, continuationKey: "codex:instance:codex" }, + displayName: undefined, + enabled: true, + snapshot: { + resolveMaintenance: () => + Effect.succeed( + makeManualOnlyProviderMaintenanceCapabilities({ provider: driver, packageName: null }), + ), + getSnapshot: Ref.get(snapshot), + refresh: Ref.get(snapshot), + streamChanges: Stream.empty, + applyUsageLimits: () => Effect.void, + }, + orchestrationAdapter: {} as ProviderInstance["orchestrationAdapter"], + textGeneration: {} as ProviderInstance["textGeneration"], + } satisfies ProviderInstance; + const layerInstanceRegistry = Layer.succeed(ProviderInstanceRegistry.ProviderInstanceRegistry, { + getInstance: (instanceId) => + Effect.succeed(instanceId === instance.instanceId ? instance : undefined), + listInstances: Effect.succeed([instance]), + listUnavailable: Effect.succeed([]), + streamChanges: Stream.empty, + subscribeChanges: Effect.flatMap(PubSub.unbounded(), PubSub.subscribe), + }); + const scope = yield* Scope.make(); + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); + return yield* Layer.build( + ProviderRegistry.layer.pipe( + Layer.provide(layerUnusedMutator), + Layer.provideMerge(layerInstanceRegistry), + Layer.provideMerge( + ServerConfig.layerTest(process.cwd(), { prefix: "t3-provider-registry-update-state-" }), + ), + Layer.provideMerge(layerBackgroundPolicyAlwaysRun), + Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), + ), + ).pipe(Scope.provide(scope)); +}); + const layerTestNodeServices = Layer.mergeAll( ProviderLatestVersions.layer, NodeServices.layer, @@ -418,6 +505,20 @@ it.layer( }), ); + it.effect("allows a slow successful Codex probe with a configured budget", () => + Effect.gen(function* () { + const fiber = yield* checkCodexProviderStatus( + defaultCodexSettings, + () => Effect.sleep("22 seconds").pipe(Effect.as(makeCodexProbeSnapshot())), + { T3CODE_CODEX_PROBE_TIMEOUT_MS: "60000" }, + ).pipe(Effect.forkChild); + yield* TestClock.adjust("22 seconds"); + const status = yield* Fiber.join(fiber); + assert.strictEqual(status.status, "ready"); + assert.strictEqual(status.auth.status, "authenticated"); + }), + ); + it.effect("labels the plan the usage read reports over the cached login plan", () => Effect.gen(function* () { const statusWithUsagePlan = (planType: string | null) => @@ -466,6 +567,25 @@ it.layer( }), ); + it.effect("names the instance's CODEX_HOME in the sign-in hint for a custom home", () => + Effect.gen(function* () { + const homePath = "/srv/codex-work-home"; + const status = yield* checkCodexProviderStatus( + decodeCodexSettings({ binaryPath: "codex", homePath }), + () => + Effect.succeed( + makeCodexProbeSnapshot({ + account: { account: null, requiresOpenaiAuth: true }, + }), + ), + ); + + assert.strictEqual(status.auth.status, "unauthenticated"); + assert.include(status.message ?? "", "CODEX_HOME"); + assert.include(status.message ?? "", homePath); + }), + ); + it.effect("returns ready with unknown auth when app-server does not require OpenAI auth", () => Effect.gen(function* () { const status = yield* checkCodexProviderStatus(defaultCodexSettings, () => @@ -484,6 +604,84 @@ it.layer( }), ); + it.effect("offers cyber access only on catalog models that advertise it", () => + Effect.gen(function* () { + const catalogModel = (slug: string, cyber?: Array<"standard" | "daybreakBlue">) => ({ + slug, + name: slug, + isCustom: false, + capabilities: mapCodexModelCapabilities({ + additionalSpeedTiers: [], + ...(cyber ? { availableAccessPrograms: { cyber } } : {}), + defaultReasoningEffort: "medium", + description: slug, + displayName: slug, + hidden: false, + id: slug, + isDefault: false, + model: slug, + supportedReasoningEfforts: [], + }), + }); + const defaultPrograms = (snapshot: Partial) => + checkCodexProviderStatus(decodeCodexSettings({ customModels: ["my-model"] }), () => + Effect.succeed( + makeCodexProbeSnapshot({ + models: [ + catalogModel("both", ["standard", "daybreakBlue"]), + catalogModel("daybreak-only", ["daybreakBlue"]), + catalogModel("plain"), + ], + ...snapshot, + }), + ), + ).pipe( + Effect.map((status) => + Object.fromEntries( + status.models.map((model) => { + const descriptor = model.capabilities?.optionDescriptors?.find( + (option) => option.id === "cyberAccessProgram", + ); + return [ + model.slug, + descriptor?.type === "select" + ? (descriptor.currentValue ?? "no default") + : null, + ]; + }), + ), + ), + ); + + assert.deepStrictEqual(yield* defaultPrograms({ daybreakEnabled: false }), { + both: "standard", + "daybreak-only": "daybreakBlue", + plain: null, + "my-model": null, + }); + assert.deepStrictEqual(yield* defaultPrograms({ daybreakEnabled: true }), { + both: "daybreakBlue", + "daybreak-only": "daybreakBlue", + plain: null, + "my-model": null, + }); + // An unread config keeps the choice but cannot know Codex's default. + assert.deepStrictEqual(yield* defaultPrograms({}), { + both: "no default", + "daybreak-only": "no default", + plain: null, + "my-model": null, + }); + assert.deepStrictEqual( + yield* defaultPrograms({ + account: { account: { type: "apiKey" }, requiresOpenaiAuth: false }, + cyberAccessProgramsUnavailable: true, + }), + { both: null, "daybreak-only": null, plain: null, "my-model": null }, + ); + }), + ); + it.effect("reads the version of a Codex CLI too old to run app-server (#20351)", () => Effect.gen(function* () { if ((yield* HostProcess.Platform) === "win32") return; @@ -591,6 +789,86 @@ it.layer( }), ); + it.effect("shows Codex usage that answers after three seconds inside the probe budget", () => + Effect.gen(function* () { + const usageRequested = yield* Deferred.make(); + const spawner = ChildProcessSpawner.make(() => + Effect.gen(function* () { + const scope = yield* Effect.scope; + const output = yield* Queue.unbounded(); + const send = (message: unknown) => + Queue.offer(output, encoder.encode(`${JSON.stringify(message)}\n`)); + const results: Record = { + initialize: { + userAgent: "codex_cli_rs/0.167.0", + codexHome: "/nonexistent-codex-home", + platformFamily: "unix", + platformOs: "linux", + }, + "account/read": { + account: { type: "chatgpt", email: "test@example.com", planType: "pro" }, + requiresOpenaiAuth: true, + }, + "config/read": { config: {}, origins: {} }, + "skills/list": { data: [] }, + "model/list": { data: [] }, + }; + const answer = (line: string) => { + const { id, method } = JSON.parse(line) as { id?: number; method?: string }; + if (id === undefined || method === undefined) return Effect.void; + if (method !== "account/rateLimits/read") { + return send({ id, result: results[method] ?? {} }); + } + return Deferred.succeed(usageRequested, undefined).pipe( + Effect.andThen(Effect.sleep("4 seconds")), + Effect.andThen( + send({ + id, + result: { + rateLimits: { + limitId: "codex", + planType: "pro", + primary: { usedPercent: 42, windowDurationMins: 300, resetsAt: 2e9 }, + }, + rateLimitsByLimitId: null, + rateLimitResetCredits: null, + accountId: null, + }, + }), + ), + Effect.forkIn(scope), + ); + }; + const decoder = new TextDecoder(); + let buffered = ""; + return ChildProcessSpawner.makeHandle({ + ...mockHandle({ stdout: "", stderr: "", code: 0 }), + exitCode: Effect.never, + isRunning: Effect.succeed(true), + stdin: Sink.forEach((chunk: Uint8Array) => { + buffered += decoder.decode(chunk, { stream: true }); + const lines = buffered.split("\n"); + buffered = lines.pop() ?? ""; + return Effect.forEach(lines.filter(Boolean), answer, { discard: true }); + }), + stdout: Stream.fromQueue(output), + }); + }), + ); + const fiber = yield* checkCodexProviderStatus(defaultCodexSettings).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.forkChild, + ); + yield* Deferred.await(usageRequested); + yield* TestClock.adjust("25 seconds"); + const status = yield* Fiber.join(fiber); + + assert.strictEqual(status.auth.status, "authenticated"); + assert.isUndefined(status.usageLimits?.unavailable); + assert.strictEqual(status.usageLimits?.windows[0]?.usedPercent, 42); + }), + ); + it.effect("does not run --version when the Codex executable cannot start", () => Effect.gen(function* () { if ((yield* HostProcess.Platform) === "win32") return; @@ -1031,7 +1309,7 @@ it.layer( ); }); - it("retains stale OpenCode models when a refresh fails", () => { + it("retains stale OpenCode models when a refresh fails or lists none", () => { const previousProvider = { instanceId: ProviderInstanceId.make("opencode"), driver: ProviderDriverKind.make("opencode"), @@ -1081,6 +1359,21 @@ it.layer( ProviderRegistry.mergeProviderSnapshot(previousProvider, refreshedProvider).skills, previousProvider.skills, ); + + const emptyCatalogProvider = { + ...refreshedProvider, + status: "warning", + message: "OpenCode 2 is running, but it did not list any models yet.", + slashCommands: [], + skills: [], + } satisfies ServerProvider; + const mergedEmptyCatalog = ProviderRegistry.mergeProviderSnapshot( + previousProvider, + emptyCatalogProvider, + ); + assert.deepStrictEqual(mergedEmptyCatalog.models, [...previousProvider.models]); + assert.deepStrictEqual(mergedEmptyCatalog.slashCommands, previousProvider.slashCommands); + assert.deepStrictEqual(mergedEmptyCatalog.skills, previousProvider.skills); }); describe("Pi model inventories", () => { @@ -3179,6 +3472,67 @@ it.layer( }), ); + describe("update results across refreshes", () => { + const instanceId = ProviderInstanceId.make("codex"); + + it.effect.each([ + { status: "unchanged", advisory: "current", retired: true }, + { status: "failed", advisory: "current", retired: true }, + { status: "unchanged", advisory: "behind_latest", retired: false }, + { status: "failed", advisory: "behind_latest", retired: false }, + // The probe publishes `unknown` until the latest-version check lands. + { status: "unchanged", advisory: "unknown", retired: false }, + { status: "queued", advisory: "current", retired: false }, + { status: "running", advisory: "current", retired: false }, + { status: "succeeded", advisory: "current", retired: false }, + { status: "failed", advisory: "unknown", retired: false }, + ] as const)( + "a $status update result is retired: $retired when a refresh reports the provider $advisory", + ({ status, advisory, retired }) => + Effect.gen(function* () { + const snapshot = yield* Ref.make(codexUpdateSnapshot("behind_latest")); + const runtimeServices = yield* buildRegistryOverSnapshot(snapshot); + + yield* Effect.gen(function* () { + const registry = yield* ProviderRegistry.ProviderRegistry; + const state = makeUpdateState(status); + yield* registry.setProviderMaintenanceActionState({ + instanceId, + action: "update", + state, + }); + yield* Ref.set(snapshot, codexUpdateSnapshot(advisory)); + + const [refreshed] = yield* registry.refresh(ProviderDriverKind.make("codex")); + + expect(refreshed?.updateState).toEqual(retired ? undefined : state); + }).pipe(Effect.provide(runtimeServices)); + }), + ); + + it.effect.each(["failed", "unchanged"] as const)( + "keeps a %s recommended-version install after a current refresh", + (status) => + Effect.gen(function* () { + const snapshot = yield* Ref.make(codexUpdateSnapshot("current")); + const runtimeServices = yield* buildRegistryOverSnapshot(snapshot); + yield* Effect.gen(function* () { + const registry = yield* ProviderRegistry.ProviderRegistry; + const state = makeUpdateState(status); + const input = { + instanceId, + action: "update" as const, + state, + targetVersion: "0.9.0", + }; + yield* registry.setProviderMaintenanceActionState(input); + const [refreshed] = yield* registry.refresh(ProviderDriverKind.make("codex")); + expect(refreshed?.updateState).toEqual(state); + }).pipe(Effect.provide(runtimeServices)); + }), + ); + }); + it.effect("re-probes when settings change the codex binaryPath", () => Effect.gen(function* () { const firstMissing = `t3code_codex_first_`; @@ -3922,6 +4276,7 @@ it.layer( ); assert.strictEqual(status.status, "ready"); assert.strictEqual(status.auth.status, "authenticated"); + assert.strictEqual(status.auth.label, "Claude API Key"); }).pipe( Effect.provide( layerMockSpawner((args) => { diff --git a/apps/server/src/provider/ProviderRegistry.ts b/apps/server/src/provider/ProviderRegistry.ts index a9a978aba7bb..99a44678477c 100644 --- a/apps/server/src/provider/ProviderRegistry.ts +++ b/apps/server/src/provider/ProviderRegistry.ts @@ -4,6 +4,7 @@ import { isProviderWorkspaceSnapshotPending, ProviderDriverKind, type ProviderInstanceId, + type SecretReference, type ServerProvider, type ServerProviderUpdateState, } from "@t3tools/contracts"; @@ -74,6 +75,12 @@ export class ProviderRegistry extends Context.Service< instanceId?: ProviderInstanceId, ) => Effect.Effect>; + /** Rebuilds and refreshes only the instances that read a secret, with the value the caller just read. */ + readonly applySecretAndRefresh: ( + reference: SecretReference, + value: string, + ) => Effect.Effect>; + /** Only refreshed instances can verify an update; the published list still includes every row. */ readonly refreshAfterUpdate: ( provider: ProviderDriverKind, @@ -115,6 +122,7 @@ export class ProviderRegistry extends Context.Service< readonly instanceId: ProviderInstanceId; readonly action: ProviderMaintenanceActionKind; readonly state: ServerProviderUpdateState | null; + readonly targetVersion?: string | undefined; }) => Effect.Effect>; readonly streamChanges: Stream.Stream>; @@ -219,7 +227,8 @@ const shouldRetainMissingProviderModels = (provider: ServerProvider): boolean => const isAntigravity = provider.driver === ProviderDriverKind.make("antigravity"); const isCodex = provider.driver === ProviderDriverKind.make("codex"); - if (!isAntigravity && !isCodex && provider.driver !== ProviderDriverKind.make("opencode")) { + const isOpenCode = provider.driver === ProviderDriverKind.make("opencode"); + if (!isAntigravity && !isCodex && !isOpenCode) { return true; } @@ -235,9 +244,10 @@ const shouldRetainMissingProviderModels = (provider: ServerProvider): boolean => isAntigravity && provider.status === "warning" && provider.auth.status === "unknown"; const isPendingInitialProbe = provider.enabled && !provider.installed && provider.status === "warning"; + // A cold OpenCode server can finish its probe as a warning before it lists any models. const isIncompleteInstalledProbe = provider.installed && - (provider.status === "error" || (isCodex && provider.status === "warning")); + (provider.status === "error" || ((isCodex || isOpenCode) && provider.status === "warning")); return isPendingAntigravityAuthentication || isPendingInitialProbe || isIncompleteInstalledProbe; }; @@ -485,7 +495,10 @@ export const layer = Layer.effect( ReadonlyMap>> >(new Map()); const maintenanceActionStatesRef = yield* Ref.make< - ReadonlyMap + ReadonlyMap< + ProviderInstanceId, + { readonly update?: ServerProviderUpdateState; readonly targetVersion?: string | undefined } + > >(new Map()); const liveSubsRef = yield* Ref.make>( @@ -605,6 +618,21 @@ export const layer = Layer.effect( readonly publish?: boolean; }, ) { + // Latest-version status cannot resolve a failed recommended-version install. + if (provider.versionAdvisory?.status === "current") { + yield* Ref.update(maintenanceActionStatesRef, (previous) => { + const action = previous.get(provider.instanceId); + if ( + action?.targetVersion !== undefined || + (action?.update?.status !== "unchanged" && action?.update?.status !== "failed") + ) { + return previous; + } + const next = new Map(previous); + next.delete(provider.instanceId); + return next; + }); + } const providers = yield* upsertProviders([provider], options); yield* refreshCompatibility; return providers; @@ -615,21 +643,14 @@ export const layer = Layer.effect( readonly instanceId: ProviderInstanceId; readonly action: "update"; readonly state: ServerProviderUpdateState | null; + readonly targetVersion?: string | undefined; }) { yield* Ref.update(maintenanceActionStatesRef, (previous) => { - const previousActions = previous.get(input.instanceId); - const nextActions = { ...previousActions }; - if (input.state === null || input.state.status === "idle") { - delete nextActions[input.action]; - } else { - nextActions[input.action] = input.state; - } - const next = new Map(previous); - if (Object.keys(nextActions).length === 0) { + if (input.state === null || input.state.status === "idle") { next.delete(input.instanceId); } else { - next.set(input.instanceId, nextActions); + next.set(input.instanceId, { update: input.state, targetVersion: input.targetVersion }); } return next; }); @@ -752,6 +773,15 @@ export const layer = Layer.effect( return yield* instanceId === undefined ? refresh() : refreshInstance(instanceId); }); + const applySecretAndRefresh = Effect.fn("applySecretAndRefresh")(function* ( + reference: SecretReference, + value: string, + ) { + const instanceIds = yield* instanceRegistryMutator.applySecret(reference, value); + yield* Effect.forEach(instanceIds, refreshInstance, { discard: true }); + return yield* Ref.get(providersRef); + }); + const refreshAfterUpdate = Effect.fn("refreshAfterUpdate")(function* ( provider: ProviderDriverKind, selectedInstanceId: ProviderInstanceId, @@ -1193,6 +1223,8 @@ export const layer = Layer.effect( refreshInstance(instanceId).pipe(Effect.catchCause(recoverRefreshFailure)), reloadSecretsAndRefresh: (instanceId?: ProviderInstanceId) => reloadSecretsAndRefresh(instanceId).pipe(Effect.catchCause(recoverRefreshFailure)), + applySecretAndRefresh: (reference: SecretReference, value: string) => + applySecretAndRefresh(reference, value).pipe(Effect.catchCause(recoverRefreshFailure)), refreshWorkspaceSnapshot: (input) => refreshWorkspaceSnapshot(input).pipe(Effect.catchCause(recoverRefreshFailure)), refreshAfterUpdate, diff --git a/apps/server/src/provider/acp/AcpJsonRpcConnection.test.ts b/apps/server/src/provider/acp/AcpJsonRpcConnection.test.ts index f06c4f47570d..4c21e400f3c8 100644 --- a/apps/server/src/provider/acp/AcpJsonRpcConnection.test.ts +++ b/apps/server/src/provider/acp/AcpJsonRpcConnection.test.ts @@ -15,13 +15,16 @@ import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as Scope from "effect/Scope"; import * as Result from "effect/Result"; +import * as Scheduler from "effect/Scheduler"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; import { ChildProcessSpawner } from "effect/process"; import { describe, expect } from "vite-plus/test"; import * as AcpSessionRuntime from "@t3tools/provider-acp/server/AcpSessionRuntime"; import { AgentScope } from "@t3tools/shared/AgentScope"; import * as EffectAcpErrors from "effect-acp/errors"; +import type * as EffectAcpProtocol from "effect-acp/protocol"; const __dirname = NodePath.dirname(NodeURL.fileURLToPath(import.meta.url)); const mockAgentPath = NodePath.join(__dirname, "../../../scripts/acp-mock-agent.ts"); @@ -44,6 +47,136 @@ const setProcessEnv = (name: string, value: string) => { }; }; +const isPromptStarted = (event: AcpSessionRuntime.AcpSessionRequestLogEvent) => + event.method === "session/prompt" && event.status === "started"; + +const isOutgoing = (tag: string) => (event: EffectAcpProtocol.AcpProtocolLogEvent) => + event.direction === "outgoing" && + event.stage === "decoded" && + typeof event.payload === "object" && + event.payload !== null && + Reflect.get(event.payload, "tag") === tag; + +// The methods the mock agent read, in order, keeping only the given ones. +const readReceived = (requestLogPath: string, methods: ReadonlyArray) => + NodeFS.readFileSync(requestLogPath, "utf8") + .split("\n") + .filter((line) => line.length > 0) + .map((line) => (JSON.parse(line) as { readonly method?: string }).method) + .filter((method) => method !== undefined && methods.includes(method)); + +// Prompts, then cancels while the prompt is held on its way out: in its started request record, +// in its outgoing protocol log record, or by its fiber yielding every few operations. The cancel +// comes on dispatch or from inside the held record. Returns the prompt's exit +// and the order in which the agent read the prompt, the cancel and the request that ends it. +const promptThenCancel = (input: { + readonly cancelBehavior: "interrupt" | "wait-for-prompt"; + readonly holdPrompt: { + readonly in: "request-log" | "protocol-log" | "scheduler"; + readonly cancelOn: "dispatch" | "log"; + }; +}) => + Effect.gen(function* () { + const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "acp-runtime-")); + yield* Effect.addFinalizer(() => + Effect.sync(() => NodeFS.rmSync(tempDir, { recursive: true, force: true })), + ); + const requestLogPath = NodePath.join(tempDir, "requests.ndjson"); + const { holdPrompt } = input; + const promptHeld = yield* Deferred.make(); + const promptReleased = yield* Deferred.make(); + yield* Effect.addFinalizer(() => Deferred.succeed(promptReleased, undefined)); + const testScope = yield* Scope.Scope; + const runtimeMade = yield* Deferred.make(); + const cancelledFromLog = yield* Deferred.make>(); + const hold = ( + holdPrompt.cancelOn === "log" + ? Deferred.await(runtimeMade).pipe( + Effect.flatMap((made) => + made.cancel.pipe(Effect.forkIn(testScope, { startImmediately: true })), + ), + Effect.flatMap((fiber) => Deferred.succeed(cancelledFromLog, fiber)), + ) + : Effect.void + ).pipe( + Effect.andThen(Deferred.succeed(promptHeld, undefined)), + Effect.andThen(Deferred.await(promptReleased)), + ); + const runtime = yield* AcpSessionRuntime.make({ + ...mockRuntimeOptions, + spawn: { + ...mockRuntimeOptions.spawn, + env: { + T3_ACP_COMPLETE_FIRST_PROMPT_ON_CANCEL: "1", + T3_ACP_REQUEST_LOG_PATH: requestLogPath, + }, + }, + cancelBehavior: input.cancelBehavior, + interruptPromptOnCancel: false, + protocolLogging: { + logOutgoing: holdPrompt.in === "protocol-log", + logger: (event) => (isOutgoing("session/prompt")(event) ? hold : Effect.void), + }, + requestLogger: (event) => + holdPrompt.in === "request-log" && isPromptStarted(event) ? hold : Effect.void, + }); + yield* Deferred.succeed(runtimeMade, runtime); + const cancelReceived = yield* Deferred.make(); + yield* runtime.getEvents().pipe( + Stream.runForEach((event) => { + if (event._tag === "EventStreamBarrier") { + return Deferred.succeed(event.acknowledge, undefined); + } + if (event._tag === "ThoughtDelta" && event.text === "native-cancel-received") { + return Deferred.succeed(cancelReceived, undefined); + } + return Effect.void; + }), + Effect.forkChild, + ); + yield* runtime.start(); + const dispatched = yield* Deferred.make(); + const prompt = yield* runtime + .prompt({ prompt: [{ type: "text", text: "first" }] }, { dispatched }) + .pipe( + holdPrompt.in === "scheduler" + ? Effect.provideService(Scheduler.MaxOpsBeforeYield, 3) + : (effect) => effect, + Effect.forkChild, + ); + // A round trip: whatever was queued before it reaches the agent first. + const roundTrip = runtime.request("_test/environment", {}); + let cancel: Fiber.Fiber; + if (holdPrompt.in === "scheduler") { + yield* Deferred.await(dispatched); + cancel = yield* runtime.cancel.pipe(Effect.forkChild({ startImmediately: true })); + } else { + yield* Deferred.await(promptHeld); + if (holdPrompt.cancelOn === "dispatch") { + cancel = yield* Deferred.await(dispatched).pipe( + Effect.andThen(runtime.cancel), + Effect.forkChild({ startImmediately: true }), + ); + } else { + cancel = yield* Deferred.await(cancelledFromLog); + } + yield* roundTrip; + yield* Deferred.succeed(promptReleased, undefined); + } + yield* Deferred.await(cancelReceived); + yield* runtime.request("_test/finish-cancel", {}); + // The agent answers it only after reading every earlier line. + yield* roundTrip; + const received = readReceived(requestLogPath, [ + "session/prompt", + "session/cancel", + "_test/finish-cancel", + ]); + const promptExit = yield* Fiber.await(prompt); + yield* Fiber.join(cancel); + return { received, promptExit }; + }).pipe(Effect.scoped); + describe("AcpSessionRuntime", () => { it.effect.each(["session/new", "session/resume"] as const)( "buffers root metadata while %s startup is still pending", @@ -1239,4 +1372,104 @@ describe("AcpSessionRuntime", () => { Effect.ensuring(Effect.sync(() => NodeFS.rmSync(tempDir, { recursive: true, force: true }))), ); }); + + it.effect.each( + (["interrupt", "wait-for-prompt"] as const).flatMap((cancelBehavior) => + ( + [ + { in: "request-log", cancelOn: "dispatch" }, + { in: "protocol-log", cancelOn: "log" }, + { in: "scheduler", cancelOn: "dispatch" }, + ] as const + ).map((holdPrompt) => ({ cancelBehavior, holdPrompt })), + ), + )( + "sends session/prompt before a cancel issued while the prompt is held in the $holdPrompt.in ($cancelBehavior, cancel on $holdPrompt.cancelOn)", + ({ cancelBehavior, holdPrompt }) => + Effect.gen(function* () { + const run = yield* promptThenCancel({ cancelBehavior, holdPrompt }); + expect(run.received).toEqual(["session/prompt", "session/cancel", "_test/finish-cancel"]); + expect(run.promptExit).toEqual( + Exit.succeed({ stopReason: "cancelled", _meta: { nativeCancel: true } }), + ); + }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), + ); + + it.effect("sends a cancel after the cancel timeout when its prompt never goes out", () => + Effect.gen(function* () { + const runtime = yield* AcpSessionRuntime.make({ + ...mockRuntimeOptions, + interruptPromptOnCancel: false, + cancelTimeout: "5 seconds", + requestLogger: (event) => (isPromptStarted(event) ? Effect.never : Effect.void), + }); + yield* runtime.start(); + const dispatched = yield* Deferred.make(); + yield* runtime + .prompt({ prompt: [{ type: "text", text: "first" }] }, { dispatched }) + .pipe(Effect.forkChild); + yield* Deferred.await(dispatched); + const cancel = yield* runtime.cancel.pipe(Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust("5 seconds"); + yield* Fiber.join(cancel); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), + ); + + it.effect( + "sends a cancel before the next prompt when the prompt it waited for never went out", + () => + Effect.gen(function* () { + const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "acp-runtime-")); + yield* Effect.addFinalizer(() => + Effect.sync(() => NodeFS.rmSync(tempDir, { recursive: true, force: true })), + ); + const requestLogPath = NodePath.join(tempDir, "requests.ndjson"); + let promptsStarted = 0; + const runtime = yield* AcpSessionRuntime.make({ + ...mockRuntimeOptions, + spawn: { ...mockRuntimeOptions.spawn, env: { T3_ACP_REQUEST_LOG_PATH: requestLogPath } }, + interruptPromptOnCancel: false, + // The cancel spends a test-clock second in its outgoing record, the window in which the + // next prompt could overtake it. + protocolLogging: { + logOutgoing: true, + logger: (event) => + isOutgoing("session/cancel")(event) ? Effect.sleep("1 second") : Effect.void, + }, + requestLogger: (event) => + isPromptStarted(event) && promptsStarted++ === 0 ? Effect.never : Effect.void, + }); + yield* runtime.getEvents().pipe( + Stream.runForEach((event) => + event._tag === "EventStreamBarrier" + ? Deferred.succeed(event.acknowledge, undefined) + : Effect.void, + ), + Effect.forkChild, + ); + yield* runtime.start(); + const dispatched = yield* Deferred.make(); + const first = yield* runtime + .prompt({ prompt: [{ type: "text", text: "first" }] }, { dispatched }) + .pipe(Effect.forkChild); + yield* Deferred.await(dispatched); + const cancel = yield* runtime.cancel.pipe(Effect.forkChild({ startImmediately: true })); + const next = yield* runtime + .prompt({ prompt: [{ type: "text", text: "next" }] }) + .pipe(Effect.forkChild); + // The first prompt ends before it is queued, which releases the cancel's wait. + yield* Effect.sleep("1 second").pipe( + Effect.andThen(Fiber.interrupt(first)), + Effect.forkChild, + ); + yield* TestClock.adjust("1 second"); + yield* TestClock.adjust("1 second"); + yield* Fiber.join(cancel); + yield* Fiber.join(next); + expect(readReceived(requestLogPath, ["session/prompt", "session/cancel"])).toEqual([ + "session/cancel", + "session/prompt", + ]); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), + ); }); diff --git a/apps/server/src/provider/acp/AcpSessionRuntime.processTree.test.ts b/apps/server/src/provider/acp/AcpSessionRuntime.processTree.test.ts index 0c6bddbbf856..6b9afa7f30c5 100644 --- a/apps/server/src/provider/acp/AcpSessionRuntime.processTree.test.ts +++ b/apps/server/src/provider/acp/AcpSessionRuntime.processTree.test.ts @@ -1,6 +1,7 @@ // @effect-diagnostics nodeBuiltinImport:off import * as NodeChildProcess from "node:child_process"; import * as NodeFS from "node:fs"; +import * as NodeModule from "node:module"; import * as NodePath from "node:path"; import { describe, expect, it, vi } from "@effect/vitest"; @@ -813,6 +814,33 @@ describe("terminatePosixOwnedProcessTree", () => { }), ); + it.skipIf(HostProcess.Platform.defaultValue() !== "linux")( + "snapshots /proc when a process exits while its entry is being typed", + () => { + const fs = NodeModule.createRequire(import.meta.url)("node:fs") as typeof NodeFS; + const realReaddirSync = fs.readdirSync; + // Node lstat()s a dirent of unknown type; a process that exits meanwhile makes it throw. + fs.readdirSync = ((path: NodeFS.PathLike, options?: unknown) => { + if (path === "/proc" && typeof options === "object" && options !== null) { + throw Object.assign(new Error("ENOENT: no such file or directory, lstat '/proc/4563'"), { + code: "ENOENT", + syscall: "lstat", + path: "/proc/4563", + }); + } + return Reflect.apply(realReaddirSync, fs, [path, options]); + }) as typeof NodeFS.readdirSync; + NodeModule.syncBuiltinESMExports(); + try { + const snapshot = AcpSessionRuntime.makePosixProcessTreeController("linux").snapshot(); + expect(snapshot.some((entry) => entry.pid === process.pid)).toBe(true); + } finally { + fs.readdirSync = realReaddirSync; + NodeModule.syncBuiltinESMExports(); + } + }, + ); + it("fails closed where a stable POSIX identity provider is unavailable", () => { try { AcpSessionRuntime.makePosixProcessTreeController("darwin"); diff --git a/apps/server/src/provider/acp/AntigravityAcpSupport.test.ts b/apps/server/src/provider/acp/AntigravityAcpSupport.test.ts index 4996b6377272..dc755d1e7a5a 100644 --- a/apps/server/src/provider/acp/AntigravityAcpSupport.test.ts +++ b/apps/server/src/provider/acp/AntigravityAcpSupport.test.ts @@ -60,21 +60,25 @@ function makeModelRuntime( describe("Antigravity release gates", () => { it.each([ - { version: "1.4.1", filter: false, native: false, ownName: false }, - { version: "1.4.2", filter: true, native: false, ownName: false }, - { version: "1.4.4", filter: true, native: false, ownName: false }, - { version: "1.4.5", filter: true, native: true, ownName: false }, - { version: "1.4.6", filter: false, native: false, ownName: false }, - { version: "1.5.0", filter: false, native: false, ownName: false }, - { version: "1.5.1", filter: false, native: false, ownName: true }, - { version: "1.3.0", filter: false, native: false, ownName: false }, - { version: undefined, filter: false, native: false, ownName: false }, - { version: "unknown", filter: false, native: false, ownName: false }, + { version: "1.4.1", filter: false, native: false, ownName: false, advertisedAppend: false }, + { version: "1.4.2", filter: true, native: false, ownName: false, advertisedAppend: false }, + { version: "1.4.4", filter: true, native: false, ownName: false, advertisedAppend: false }, + { version: "1.4.5", filter: true, native: true, ownName: false, advertisedAppend: false }, + { version: "1.4.6", filter: false, native: false, ownName: false, advertisedAppend: false }, + { version: "1.5.0", filter: false, native: false, ownName: false, advertisedAppend: false }, + { version: "1.5.1", filter: false, native: false, ownName: true, advertisedAppend: false }, + { version: "1.5.7", filter: false, native: false, ownName: true, advertisedAppend: true }, + { version: "1.3.0", filter: false, native: false, ownName: false, advertisedAppend: false }, + { version: undefined, filter: false, native: false, ownName: false, advertisedAppend: false }, + { version: "unknown", filter: false, native: false, ownName: false, advertisedAppend: false }, ])( - "$version: workflow filter $filter, native reads and system prompt $native, own client name $ownName", - ({ version, filter, native, ownName }) => { + "$version: workflow filter $filter, qualified reads $native, advertised system append $advertisedAppend, own client name $ownName", + ({ version, filter, native, ownName, advertisedAppend }) => { const initialize = { protocolVersion: 1, + ...(advertisedAppend + ? { agentCapabilities: { _meta: { antigravity: { systemPrompt: { append: true } } } } } + : {}), ...(version === undefined ? {} : { agentInfo: { name: "antigravity-acp", version } }), }; // Helpers send no append, so only the workflow filter can apply. @@ -83,10 +87,12 @@ describe("Antigravity release gates", () => { ); const antigravity = { ...(filter ? { disabledTools: ["run_workflow"] } : {}), - ...(native ? { systemPrompt: { append: "" } } : {}), + ...(native || advertisedAppend + ? { systemPrompt: { append: "" } } + : {}), }; expect(antigravitySessionMeta(initialize, "")).toEqual( - filter || native ? { antigravity } : undefined, + filter || native || advertisedAppend ? { antigravity } : undefined, ); expect(isQualifiedAntigravityRelease(version)).toBe(native); expect(antigravityClientInfo(version).name).toBe(ownName ? "t3-code" : "zed"); diff --git a/apps/server/src/provider/acp/AntigravityAcpSupport.ts b/apps/server/src/provider/acp/AntigravityAcpSupport.ts index 66eab95c9fb3..7fd64e2534fa 100644 --- a/apps/server/src/provider/acp/AntigravityAcpSupport.ts +++ b/apps/server/src/provider/acp/AntigravityAcpSupport.ts @@ -12,6 +12,7 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; +import * as Predicate from "effect/Predicate"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; @@ -94,7 +95,15 @@ const OPEN_MODEL_ACCESS_RELEASE = [1, 5, 1] as const; */ export const antigravityAppendsSystemPrompt = ( initializeResult: EffectAcpSchema.InitializeResponse, -) => isQualifiedAntigravityRelease(initializeResult.agentInfo?.version); +) => { + if (isQualifiedAntigravityRelease(initializeResult.agentInfo?.version)) return true; + const extension = initializeResult.agentCapabilities?._meta?.antigravity; + return ( + Predicate.isObject(extension) && + Predicate.isObject(extension.systemPrompt) && + extension.systemPrompt.append === true + ); +}; export function antigravitySessionMeta( initializeResult: EffectAcpSchema.InitializeResponse, diff --git a/apps/server/src/provider/acp/AntigravitySessionFiles.test.ts b/apps/server/src/provider/acp/AntigravitySessionFiles.test.ts index 795e80b861e8..81885dc0c89b 100644 --- a/apps/server/src/provider/acp/AntigravitySessionFiles.test.ts +++ b/apps/server/src/provider/acp/AntigravitySessionFiles.test.ts @@ -31,7 +31,7 @@ it.layer(NodeServices.layer)("retireAntigravityRuntimeTempDirs", (it) => { yield* removeRetired; expect((yield* fs.readDirectory(parent)).toSorted()).toEqual(["other-instance"]); - }).pipe(Effect.scoped), + }), ); it.effect( diff --git a/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts b/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts index 09b8730bd600..52e96caa08ae 100644 --- a/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts +++ b/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts @@ -158,7 +158,7 @@ describe("watchAntigravityPromptOutcome", () => { code: "QUOTA_EXHAUSTED", retryable: true, }); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); it.effect("reads only this session's own unlinked log, and keeps a typed refusal", () => @@ -206,6 +206,6 @@ describe("watchAntigravityPromptOutcome", () => { code: "QUOTA_EXHAUSTED", retryable: true, }); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); }); diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index 486c4314927a..5b9a0efd649d 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { - "claudeSystemAppendBytes": 5597, - "codexAdditionalContextBytes": 7327, - "mcpToolListJsonBytes": 48131 + "claudeSystemAppendBytes": 5930, + "codexAdditionalContextBytes": 7324, + "mcpToolListJsonBytes": 49738 } diff --git a/apps/server/src/provider/antigravityAuthSupport.test.ts b/apps/server/src/provider/antigravityAuthSupport.test.ts index 0220f9fd153c..832f2961d836 100644 --- a/apps/server/src/provider/antigravityAuthSupport.test.ts +++ b/apps/server/src/provider/antigravityAuthSupport.test.ts @@ -118,7 +118,13 @@ describe("Antigravity process environment", () => { }); expect(spawn.command).toBe("/usr/bin/agy"); - expect(spawn.args).toEqual(["acp"]); + expect(spawn.args).toEqual([ + "acp", + "--gemini-dir", + profile.geminiHome, + "--app-data-dir", + "antigravity-acp", + ]); expect(spawn.env).not.toHaveProperty("ANTIGRAVITY_HARNESS_PATH"); expect(spawn.env).toMatchObject({ GEMINI_HOME: profile.geminiHome, PATH: "/usr/bin" }); }); diff --git a/apps/server/src/provider/antigravityAuthSupport.ts b/apps/server/src/provider/antigravityAuthSupport.ts index 6a926b12f7c9..8d78a504f9fb 100644 --- a/apps/server/src/provider/antigravityAuthSupport.ts +++ b/apps/server/src/provider/antigravityAuthSupport.ts @@ -467,7 +467,14 @@ export function buildAntigravityAcpSpawnInput(input: { // The agy CLI has no harness sibling and serves ACP through its `acp` subcommand. args: input.installation.harnessPath === null - ? ["acp"] + ? [ + "acp", + "--gemini-dir", + input.profile.geminiHome, + "--app-data-dir", + // The CLI joins this beneath gemini-dir, including when the value is absolute. + "antigravity-acp", + ] : input.profile.platform === "linux" ? ["--uid="] : [], diff --git a/apps/server/src/provider/antigravityCliRelease.test.ts b/apps/server/src/provider/antigravityCliRelease.test.ts new file mode 100644 index 000000000000..2e1e0010c22c --- /dev/null +++ b/apps/server/src/provider/antigravityCliRelease.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from "@effect/vitest"; +import { antigravityCliTarget, parseAntigravityCliRelease } from "./antigravityCliRelease.ts"; + +const manifest = { + version: "1.5.7", + url: "https://storage.googleapis.com/antigravity-public/antigravity-cli/1.5.7-123/darwin-arm/cli_mac_arm64.tar.gz", + sha512: "a".repeat(128), +}; + +describe("Antigravity CLI release manifests", () => { + it.each([ + ["darwin", "arm64", false, "darwin_arm64", "darwin-arm/cli_mac_arm64.tar.gz"], + ["darwin", "x64", false, "darwin_amd64", "darwin-x64/cli_mac_x64.tar.gz"], + ["linux", "arm64", false, "linux_arm64", "linux-arm/cli_linux_arm64.tar.gz"], + ["linux", "x64", false, "linux_amd64", "linux-x64/cli_linux_x64.tar.gz"], + ["linux", "arm64", true, "linux_arm64_musl", "linux-arm-musl/cli_linux_arm64_musl.tar.gz"], + ["linux", "x64", true, "linux_amd64_musl", "linux-x64-musl/cli_linux_x64_musl.tar.gz"], + ["win32", "arm64", false, "windows_arm64", "windows-arm/cli_windows_arm64.exe"], + ["win32", "x64", false, "windows_amd64", "windows-x64/cli_windows_x64.exe"], + ] as const)("selects Google's %s-%s release (musl %s)", (platform, arch, musl, target, file) => { + expect(antigravityCliTarget(platform, arch, musl)).toBe(target); + expect( + parseAntigravityCliRelease( + { + ...manifest, + url: `https://storage.googleapis.com/antigravity-public/antigravity-cli/1.5.7-123/${file}`, + }, + platform, + arch, + musl, + ), + ).toMatchObject({ + version: "1.5.7", + format: platform === "win32" ? "executable" : "tar.gz", + executable: { name: platform === "win32" ? "agy.exe" : "antigravity" }, + }); + }); + + it.each([ + { sha512: "invalid" }, + { version: "1.4.5" }, + { url: manifest.url.replace("https:", "http:") }, + { url: manifest.url.replace("storage.googleapis.com", "storage.googleapis.com.evil.test") }, + { url: manifest.url.replace("antigravity-public", "another-bucket") }, + { url: manifest.url.replace("darwin-arm", "linux-arm") }, + { url: `${manifest.url}?redirect=elsewhere` }, + { url: manifest.url.replace("1.5.7-123", "1.5.6-123") }, + ])("rejects invalid or mismatched release metadata: %j", (change) => { + expect(parseAntigravityCliRelease({ ...manifest, ...change }, "darwin", "arm64")).toBeNull(); + }); +}); diff --git a/apps/server/src/provider/antigravityCliRelease.ts b/apps/server/src/provider/antigravityCliRelease.ts new file mode 100644 index 000000000000..3e033beb37ac --- /dev/null +++ b/apps/server/src/provider/antigravityCliRelease.ts @@ -0,0 +1,60 @@ +import * as Schema from "effect/Schema"; +import { compareSemverVersions } from "@t3tools/shared/semver"; + +export const ANTIGRAVITY_CLI_MANIFEST_BASE = + "https://antigravity-cli-auto-updater-974169037036.us-central1.run.app/manifests"; + +const Manifest = Schema.Struct({ + version: Schema.String.check(Schema.isPattern(/^\d+\.\d+\.\d+$/u)), + url: Schema.String, + sha512: Schema.String.check(Schema.isPattern(/^[a-fA-F0-9]{128}$/u)), +}); +const decodeManifest = Schema.decodeUnknownOption(Manifest); + +export interface AntigravityCliReleaseAsset { + readonly version: string; + readonly url: string; + readonly sha512: string; + readonly executable: { readonly name: string }; + readonly format: "tar.gz" | "executable"; +} + +export function antigravityCliTarget(platform: NodeJS.Platform, arch: string, musl = false) { + if (!["darwin", "linux", "win32"].includes(platform) || !["x64", "arm64"].includes(arch)) { + return null; + } + const os = platform === "win32" ? "windows" : platform; + return `${os}_${arch === "x64" ? "amd64" : arch}${platform === "linux" && musl ? "_musl" : ""}`; +} + +export function parseAntigravityCliRelease( + value: unknown, + platform: NodeJS.Platform, + arch: string, + musl = false, +): AntigravityCliReleaseAsset | null { + const decoded = decodeManifest(value); + if (decoded._tag === "None" || antigravityCliTarget(platform, arch, musl) === null) return null; + const manifest = decoded.value; + if (compareSemverVersions(manifest.version, "1.5.0") < 0) return null; + const os = platform === "darwin" ? "mac" : platform === "win32" ? "windows" : "linux"; + const cpu = arch === "x64" ? "x64" : "arm64"; + const suffix = platform === "linux" && musl ? "_musl" : ""; + const extension = platform === "win32" ? "exe" : "tar.gz"; + const folder = `${platform === "win32" ? "windows" : platform}-${arch === "arm64" ? "arm" : "x64"}${suffix ? "-musl" : ""}`; + const prefix = `https://storage.googleapis.com/antigravity-public/antigravity-cli/${manifest.version}-`; + const rest = manifest.url.startsWith(prefix) ? manifest.url.slice(prefix.length) : ""; + if ( + !/^\d+\//u.test(rest) || + rest.replace(/^\d+\//u, "") !== `${folder}/cli_${os}_${cpu}${suffix}.${extension}` + ) { + return null; + } + return { + version: manifest.version, + url: manifest.url, + sha512: manifest.sha512.toLowerCase(), + executable: { name: platform === "win32" ? "agy.exe" : "antigravity" }, + format: platform === "win32" ? "executable" : "tar.gz", + }; +} diff --git a/apps/server/src/provider/antigravityRelease.ts b/apps/server/src/provider/antigravityRelease.ts index 255a92761994..54702d69935f 100644 --- a/apps/server/src/provider/antigravityRelease.ts +++ b/apps/server/src/provider/antigravityRelease.ts @@ -1,7 +1,3 @@ -// Hashes and sizes match the official ACP registry release. Like 1.4.2 through 1.4.4, it writes a -// run-workflow skill into the workspace unless T3 disables the tool (antigravitySessionMeta). -const ANTIGRAVITY_RELEASE_VERSION = "1.4.5"; - export interface AntigravityReleaseAsset { readonly version: string; readonly url: string; @@ -16,79 +12,3 @@ export interface AntigravityReleaseAsset { readonly bytes: number; }; } - -const releaseAssets = new Map([ - [ - "darwin-arm64", - { - version: ANTIGRAVITY_RELEASE_VERSION, - url: "https://dl.google.com/agy-extensions/releases/macos/agy-acp-server-1.4.5-darwin-arm64.zip", - sha256: "886fff7767d08994ca39eb02c3d01a7532c7325a94aa33aaf52d52068733284f", - archiveBytes: 97_923_855, - executable: { name: "agy_acp_server.par", bytes: 178_915_920 }, - harness: { name: "localharness_external", bytes: 122_957_936 }, - }, - ], - [ - "darwin-x64", - { - version: ANTIGRAVITY_RELEASE_VERSION, - url: "https://dl.google.com/agy-extensions/releases/macos/agy-acp-server-1.4.5-darwin-x86_64.zip", - sha256: "b39f3573c03b9acb9df028853f647527f1f3133f32bd77e78b796e48a205114a", - archiveBytes: 103_577_295, - executable: { name: "agy_acp_server.par", bytes: 183_127_600 }, - harness: { name: "localharness_external", bytes: 127_612_208 }, - }, - ], - [ - "linux-x64", - { - version: ANTIGRAVITY_RELEASE_VERSION, - url: "https://dl.google.com/agy-extensions/releases/linux/agy-acp-server-1.4.5-linux-x86_64.zip", - sha256: "923923a0d7dc8073f66cc6f78647a3107e3a7ee2b7e1a24df8b3927b5188013b", - archiveBytes: 321_783_181, - executable: { name: "agy_acp_server.par", bytes: 826_226_766 }, - harness: { name: "localharness_external", bytes: 133_926_136 }, - }, - ], - [ - "linux-arm64", - { - version: ANTIGRAVITY_RELEASE_VERSION, - url: "https://dl.google.com/agy-extensions/releases/linux/agy-acp-server-1.4.5-linux-arm64.zip", - sha256: "7d64246633ae094f67e3bbcd2d679e89048f91346d5339313336e7483f40f056", - archiveBytes: 307_829_008, - executable: { name: "agy_acp_server.par", bytes: 831_194_025 }, - harness: { name: "localharness_external", bytes: 127_512_832 }, - }, - ], - [ - "win32-x64", - { - version: ANTIGRAVITY_RELEASE_VERSION, - url: "https://dl.google.com/agy-extensions/releases/windows/agy-acp-server-1.4.5-windows-x86_64.zip", - sha256: "d24f4c73c9f96d4f44384e281c855a56c72d8eaa9c5b0fad7ccb1635c6b63f50", - archiveBytes: 100_423_655, - executable: { name: "agy_acp_server.exe", bytes: 55_839_864 }, - harness: { name: "localharness_external.exe", bytes: 147_843_736 }, - }, - ], - [ - "win32-arm64", - { - version: ANTIGRAVITY_RELEASE_VERSION, - url: "https://dl.google.com/agy-extensions/releases/windows/agy-acp-server-1.4.5-windows-arm64.zip", - sha256: "ab0a9aac8349e08ef289be6d16da8f89e3abe60eba5158032ef78c4a1220261c", - archiveBytes: 102_703_121, - executable: { name: "agy_acp_server.exe", bytes: 62_293_488 }, - harness: { name: "localharness_external.exe", bytes: 139_322_008 }, - }, - ], -]); - -export function resolveAntigravityReleaseAsset( - platform: NodeJS.Platform, - arch: string, -): AntigravityReleaseAsset | null { - return releaseAssets.get(`${platform}-${arch}`) ?? null; -} diff --git a/apps/server/src/provider/claudeUsageLimits.test.ts b/apps/server/src/provider/claudeUsageLimits.test.ts index f20d8b054363..b683c80f4f5e 100644 --- a/apps/server/src/provider/claudeUsageLimits.test.ts +++ b/apps/server/src/provider/claudeUsageLimits.test.ts @@ -199,6 +199,22 @@ describe("claudeUsageResponseToLimits", () => { ).toEqual([monthlyWindow]); }); + it("flags a $0 cap instead of reporting a monthly window", () => { + const response = { + rate_limits_available: true, + rate_limits: { + five_hour: null, + seven_day: null, + extra_usage: { is_enabled: true, used_credits: 0, monthly_limit: 0, utilization: null }, + }, + }; + expect(claudeUsageResponseToLimits({ checkedAt, response }).limits).toEqual({ + checkedAt, + windows: [], + zeroSpendCap: true, + }); + }); + it("omits disabled budgets and refuses invalid money without an enabled fallback", () => { const rateLimits = enterpriseResponse.rate_limits; const response = { diff --git a/apps/server/src/provider/claudeUsageLimits.ts b/apps/server/src/provider/claudeUsageLimits.ts index d0acf43bb496..ba1263ef704b 100644 --- a/apps/server/src/provider/claudeUsageLimits.ts +++ b/apps/server/src/provider/claudeUsageLimits.ts @@ -47,8 +47,20 @@ export const makeClaudeScopedLimitNames = Ref.make({ overageIncluded: undefined, }); +const SCOPED_WINDOW_PREFIX = "seven_day_"; +const scopeToken = (name: string) => name.toLowerCase().replace(/[^a-z0-9]+/g, "_"); + function scopedWindowId(displayName: string): string { - return `seven_day_${displayName.toLowerCase().replace(/[^a-z0-9]+/g, "_")}`; + return `${SCOPED_WINDOW_PREFIX}${scopeToken(displayName)}`; +} + +/** A model-scoped weekly allowance blocks only its own model; an unknown model is not blocked. */ +export function windowLimitsModel( + window: Pick, + model: string | null, +): boolean { + if (!window.id.startsWith(SCOPED_WINDOW_PREFIX)) return true; + return model !== null && scopeToken(model).includes(window.id.slice(SCOPED_WINDOW_PREFIX.length)); } function scopedWindow( @@ -202,6 +214,7 @@ export function claudeUsageResponseToLimits(input: { overageIncluded ??= entry.display_name; } const spendPercent = readSpendPercent(response.rate_limits); + const extra = response.rate_limits.extra_usage; if (spendPercent !== undefined) { windows.push({ id: "monthly_spend", @@ -211,7 +224,10 @@ export function claudeUsageResponseToLimits(input: { }); } return { - limits: makeUsageLimits({ checkedAt, windows }), + limits: { + ...makeUsageLimits({ checkedAt, windows }), + ...(extra?.is_enabled && extra.monthly_limit === 0 ? { zeroSpendCap: true } : {}), + }, names: { overageIncluded }, }; } diff --git a/apps/server/src/provider/codexLaunchArgs.test.ts b/apps/server/src/provider/codexLaunchArgs.test.ts index 8f4e1b7ca6d6..e74b29d44818 100644 --- a/apps/server/src/provider/codexLaunchArgs.test.ts +++ b/apps/server/src/provider/codexLaunchArgs.test.ts @@ -38,6 +38,15 @@ describe("codexExecLaunchArgs", () => { ); }); + it("drops launch args that enable user hooks", () => { + NodeAssert.deepStrictEqual( + codexExecLaunchArgs( + "--enable hooks --enable=hooks --enable codex_hooks --enable=codex_hooks --enable foo --disable hooks", + ), + ["--enable", "foo", "--disable", "hooks"], + ); + }); + it("does not pair value-taking flags with adjacent flags", () => { NodeAssert.deepStrictEqual(codexExecLaunchArgs("--config --strict-config --enable --disable"), [ "--strict-config", diff --git a/apps/server/src/provider/codexLaunchArgs.ts b/apps/server/src/provider/codexLaunchArgs.ts index 31ff37e31deb..e49314ee83e0 100644 --- a/apps/server/src/provider/codexLaunchArgs.ts +++ b/apps/server/src/provider/codexLaunchArgs.ts @@ -62,8 +62,17 @@ const filterLaunchArgs = (launchArgs: string | undefined, allowed: LaunchArgFilt return kept; }; -export const codexExecLaunchArgs = (launchArgs?: string) => - filterLaunchArgs(launchArgs, EXEC_FILTER); +const HOOK_FEATURES = new Set(["hooks", "codex_hooks"]); + +// Feature toggles beat config overrides, so a launch arg's hook enable would undo ours. +export const codexExecLaunchArgs = (launchArgs?: string) => { + const args = filterLaunchArgs(launchArgs, EXEC_FILTER); + return args.filter((arg, index) => { + if (arg.startsWith("--enable=")) return !HOOK_FEATURES.has(arg.slice("--enable=".length)); + if (arg === "--enable") return !HOOK_FEATURES.has(args[index + 1] ?? ""); + return !(args[index - 1] === "--enable" && HOOK_FEATURES.has(arg)); + }); +}; export const codexMcpListLaunchArgs = (launchArgs?: string) => filterLaunchArgs(launchArgs, MCP_LIST_FILTER); diff --git a/apps/server/src/provider/codexUsageLimits.test.ts b/apps/server/src/provider/codexUsageLimits.test.ts index b560aba5d3c2..3a5cae4a9d2c 100644 --- a/apps/server/src/provider/codexUsageLimits.test.ts +++ b/apps/server/src/provider/codexUsageLimits.test.ts @@ -1,12 +1,15 @@ +import * as DateTime from "effect/DateTime"; import { ProviderDriverKind, ProviderInstanceId } from "@t3tools/contracts"; import { exhaustedUsageWindow } from "@t3tools/shared/usageLimits"; import * as CodexErrors from "effect-codex-app-server/errors"; import { describe, expect, it } from "vite-plus/test"; import { + codexMessageResetAt, codexRateLimitsFailureMessage, codexRateLimitsToLimits, codexRateLimitsToUpdate, + codexTimeZone, mergeCodexRateLimits, } from "./codexUsageLimits.ts"; @@ -313,3 +316,61 @@ describe("codexUsageLimitResetAt", () => { ).toBeNull(); }); }); + +describe("codexMessageResetAt", () => { + const dhaka = DateTime.zoneMakeNamedUnsafe("Asia/Dhaka"); + const at = (iso: string) => DateTime.makeUnsafe(iso); + const failedAt = at("2026-11-07T03:00:00Z"); // 09:00 in Dhaka + + // Codex prints local time without seconds, so the parsed reset is the minute after. + it("reads Codex's same-day and dated reset text in Codex's zone", () => { + expect( + codexMessageResetAt("You’ve hit your usage limit. Try again at 3:37 PM.", failedAt, dhaka), + ).toBe("2026-11-07T09:38:00.000Z"); + expect( + codexMessageResetAt("Upgrade, or try again at Dec 1st, 2026 12:05 AM.", failedAt, dhaka), + ).toBe("2026-11-30T18:06:00.000Z"); + expect(codexMessageResetAt("Try again at 12:30 PM.", failedAt, dhaka)).toBe( + "2026-11-07T06:31:00.000Z", + ); + }); + + it("dates today's form by when Codex failed, across local midnight", () => { + expect(codexMessageResetAt("Try again at 11:59 PM.", at("2026-11-07T17:59:59Z"), dhaka)).toBe( + "2026-11-07T18:00:00.000Z", + ); + }); + + it("takes the later instant of a repeated DST hour", () => { + // 1:30 AM occurs twice in New York on Nov 1st, 2026: 05:30Z (EDT) and 06:30Z (EST). + const newYork = DateTime.zoneMakeNamedUnsafe("America/New_York"); + for (const iso of ["2026-11-01T05:15:00Z", "2026-11-01T06:15:00Z"]) { + expect(codexMessageResetAt("Try again at 1:30 AM.", at(iso), newYork)).toBe( + "2026-11-01T06:31:00.000Z", + ); + } + }); + + it("leaves the reset unknown without reset text or once it has passed", () => { + expect( + codexMessageResetAt("You’ve hit your usage limit. Try again later.", failedAt, dhaka), + ).toBeNull(); + expect( + codexMessageResetAt( + "exceeded retry limit, last status: 429 Too Many Requests", + failedAt, + dhaka, + ), + ).toBeNull(); + expect(codexMessageResetAt("Try again at 8:15 AM.", failedAt, dhaka)).toBeNull(); + }); +}); + +describe("codexTimeZone", () => { + it("uses Codex's TZ and falls back to the server's zone", () => { + const local = DateTime.zoneToString(DateTime.zoneMakeLocal()); + expect(DateTime.zoneToString(codexTimeZone({ TZ: ":Asia/Dhaka" }))).toBe("Asia/Dhaka"); + expect(DateTime.zoneToString(codexTimeZone({ TZ: "not/a-zone" }))).toBe(local); + expect(DateTime.zoneToString(codexTimeZone({}))).toBe(local); + }); +}); diff --git a/apps/server/src/provider/codexUsageLimits.ts b/apps/server/src/provider/codexUsageLimits.ts index d4f9c621119d..526852332033 100644 --- a/apps/server/src/provider/codexUsageLimits.ts +++ b/apps/server/src/provider/codexUsageLimits.ts @@ -201,3 +201,51 @@ export function codexUsageLimitResetAt( null, ); } + +const CODEX_RESET_TEXT = + /[Tt]ry again at (?:([A-Z][a-z]{2}) (\d{1,2})(?:st|nd|rd|th), (\d{4}) )?(\d{1,2}):(\d{2}) ([AP]M)\./; +const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; + +/** The zone Codex prints reset times in: its own `TZ`, else the server's. */ +export function codexTimeZone( + environment: Readonly>, +): DateTime.TimeZone { + return Option.getOrElse( + DateTime.zoneMakeNamed(environment.TZ?.replace(/^:/, "") ?? ""), + DateTime.zoneMakeLocal, + ); +} + +/** + * Codex writes a usage limit's reset into its error text in its own local time: + * "Try again at 3:37 PM." for today, else "try again at Nov 11th, 2026 3:37 PM.". + * It is the only reset left when a proxy such as CLIProxyAPI strips the rate-limit snapshot. + * `failedAt` is when Codex reported the failure, which fixes the date of today's form. + */ +export function codexMessageResetAt( + message: string, + failedAt: DateTime.DateTime, + zone: DateTime.TimeZone, +): string | null { + const match = CODEX_RESET_TEXT.exec(message); + if (!match) return null; + const [, month, day, year, hour, minute, meridiem] = match; + const monthIndex = month === undefined ? undefined : MONTHS.indexOf(month); + if (monthIndex === -1) return null; + const today = DateTime.toParts(DateTime.setZone(failedAt, zone)); + const reset = DateTime.makeZoned( + { + year: monthIndex === undefined ? today.year : Number(year), + month: monthIndex === undefined ? today.month : monthIndex + 1, + day: monthIndex === undefined ? today.day : Number(day), + hour: (Number(hour) % 12) + (meridiem === "PM" ? 12 : 0), + minute: Number(minute), + }, + // The repeated hour at a DST fall-back takes its later instant so a resume never fires early. + { timeZone: zone, adjustForTimeZone: true, disambiguation: "later" }, + ); + if (Option.isNone(reset)) return null; + // Codex drops the seconds, so the next minute is the first one past the reset. + const resetAt = DateTime.add(reset.value, { minutes: 1 }); + return DateTime.isGreaterThan(resetAt, failedAt) ? DateTime.formatIso(resetAt) : null; +} diff --git a/apps/server/src/provider/documentTextWorker.ts b/apps/server/src/provider/documentTextWorker.ts index 81008be757da..5957fdd3cd89 100644 --- a/apps/server/src/provider/documentTextWorker.ts +++ b/apps/server/src/provider/documentTextWorker.ts @@ -14,9 +14,10 @@ const CELL_END = Symbol("cell-end"); type TextPart = string | typeof CELL_START | typeof CELL_END | { fallback: string }; const TRUNCATED = "\n\n[Extracted text truncated at document extraction limits.]"; -// Parts the document never displays: tracked deletions and moves, field codes, phonetic guides, -// paragraph properties such as tab-stop definitions. +// Parts the document never displays: the XHTML head, tracked deletions and moves, field codes, +// phonetic guides, paragraph properties such as tab-stop definitions. const HIDDEN = new Set([ + "head", "script", "style", "binary-data", @@ -41,12 +42,43 @@ interface XmlNode { hasTextChoice?: boolean; } -function xml(source: string, odfSpaces = false, htmlEntities = false): XmlNode { +// ODF 1.3 Part 3 §6.1.2: paragraph character data turns TAB, CR and LF into spaces, collapses +// space runs and trims the ends; only , and keep literal ones. +function collapseOdfWhitespace(paragraph: XmlNode) { + let afterSpace = true; + let tail: { node: XmlNode; index: number; value: string } | undefined; + const visit = (node: XmlNode) => { + for (const [index, child] of node.children.entries()) { + if (typeof child !== "string") { + if (HIDDEN.has(child.name)) continue; + if (child.spaceCount === undefined && !["tab", "line-break", "p", "h"].includes(child.name)) + visit(child); + else { + afterSpace = false; + tail = undefined; + } + continue; + } + let value = child.replaceAll(/[ \t\r\n]+/g, " "); + if (afterSpace && value.startsWith(" ")) value = value.slice(1); + if (value) { + afterSpace = value.endsWith(" "); + tail = { node, index, value }; + } + node.children[index] = value; + } + }; + visit(paragraph); + if (tail && afterSpace) tail.node.children[tail.index] = tail.value.slice(0, -1); +} + +function xml(source: string, extension: string): XmlNode { + const odf = [".odt", ".ods", ".odp"].includes(extension); const root: XmlNode = { name: "root", attrs: {}, children: [] }; const stack = [root]; let nodes = 0; const parser = new SaxesParser(); - if (htmlEntities) Object.assign(parser.ENTITIES, characterEntities); + if (extension === ".epub") Object.assign(parser.ENTITIES, characterEntities); parser.on("opentag", (tag) => { if (++nodes > 200_000 || stack.length > 128) throw new Truncation("XML structure limit"); const node: XmlNode = { @@ -59,20 +91,25 @@ function xml(source: string, odfSpaces = false, htmlEntities = false): XmlNode { ), children: [], }; - if (odfSpaces && node.name === "s") node.spaceCount = repeatCount(node.attrs.c); + if (odf && node.name === "s") node.spaceCount = repeatCount(node.attrs.c); stack.at(-1)!.children.push(node); stack.push(node); }); parser.on("closetag", () => { const node = stack.pop()!; const parent = stack.at(-1)!; + if (odf && (node.name === "p" || node.name === "h")) collapseOdfWhitespace(node); + if (extension === ".docx" && node.name === "del") { + parent.children.pop(); + return; + } // Markup-compatibility fallbacks repeat their mc:Choice content, e.g. Word text boxes. // Keep one when no Choice carries text, such as a chart with a VML caption. if (node.name === "Fallback" && parent.hasTextChoice) { parent.children.pop(); return; } - if (!node.hasText) return; + if (!node.hasText && node.name !== "noBreakHyphen") return; parent.hasText = true; if (node.name === "Choice") parent.hasTextChoice = true; }); @@ -99,6 +136,7 @@ function* text(node: XmlNode): Generator { if (HIDDEN.has(node.name)) return; if (["tab", "cell"].includes(node.name)) yield "\t"; else if (["br", "line-break", "cr"].includes(node.name)) yield "\n"; + else if (node.name === "noBreakHyphen") yield "\u2011"; else if (node.spaceCount !== undefined) { yield " ".repeat(Math.min(MAX_OUTPUT, node.spaceCount)); if (node.spaceCount > MAX_OUTPUT) throw new Truncation("Space repeat limit"); @@ -108,6 +146,23 @@ function* text(node: XmlNode): Generator { else yield* text(child); } } +// SpreadsheetML `t` and `v` values are ST_Xstring: `_xHHHH_` is one UTF-16 code unit, and an +// escaped lead underscore (`_x005F_xHHHH_`) keeps the literal `_xHHHH_`. Decoded in one pass. +function* xstringText(node: XmlNode): Generator { + if (HIDDEN.has(node.name)) return; + if (node.name !== "t" && node.name !== "v") { + for (const child of children(node)) yield* xstringText(child); + return; + } + yield node.children + .filter((child) => typeof child === "string") + .join("") + .replaceAll( + /_x005[Ff]_(x[\dA-Fa-f]{4}_)|_x([\dA-Fa-f]{4})_/g, + (_, literal?: string, hex?: string) => + literal ? `_${literal}` : String.fromCharCode(Number.parseInt(hex!, 16)), + ); +} function* paragraphs(node: XmlNode): Generator { if (HIDDEN.has(node.name)) return; for (const child of node.children) { @@ -164,7 +219,7 @@ function odfValue(cell: XmlNode): string { } } function hasCellText(cell: XmlNode): boolean { - if (odfValue(cell).trim()) return true; + if (odfValue(cell).trim() || cell.name === "noBreakHyphen") return true; // Space markers can have enormous repeat counts; emptiness never requires expanding them. if (HIDDEN.has(cell.name) || cell.spaceCount !== undefined) return false; return cell.children.some((child) => @@ -342,6 +397,10 @@ function* rtf(source: string): Generator { ["ldblquote", "“"], ["rdblquote", "”"], ["bullet", "•"], + ["zwj", "\u200d"], + ["zwnj", "\u200c"], + ["ltrmark", "\u200e"], + ["rtlmark", "\u200f"], ["~", "\u00a0"], ["_", "\u2011"], ["-", ""], @@ -437,7 +496,7 @@ async function* document(path: string, extension: string): AsyncGenerator typeof child === "string") ?? ""; if (cell.attrs.t === "s") { const shared = strings[Number(raw)]; - if (shared) yield* inCell(text(shared)); - } else if (cell.attrs.t === "inlineStr") yield* inCell(text(cell)); + if (shared) yield* inCell(xstringText(shared)); + } else if (cell.attrs.t === "inlineStr") yield* inCell(xstringText(cell)); + else if (cell.attrs.t === "b" && (raw.trim() === "0" || raw.trim() === "1")) + yield* inCell([raw.trim() === "1" ? "true" : "false"]); else { - if (value && hasCellText(value)) yield* inCell(text(value)); + if (value && hasCellText(value)) yield* inCell(xstringText(value)); else for (const formula of all(cell, "f")) yield* inCell(text(formula)); const formatId = formats[Number(cell.attrs.s ?? 0)]?.attrs.numFmtId; const format = formatId ? customFormats.get(formatId) : undefined; diff --git a/apps/server/src/provider/managedRelease.ts b/apps/server/src/provider/managedRelease.ts new file mode 100644 index 000000000000..ca7c55e43b59 --- /dev/null +++ b/apps/server/src/provider/managedRelease.ts @@ -0,0 +1,29 @@ +import { compareSemverVersions, parseSemver } from "@t3tools/shared/semver"; + +// Older pins recorded non-semver versions such as `agy_acp_server_1.1.1`; any +// other release is an update from those. +const isNewer = (candidate: string, current: string) => + parseSemver(candidate) && parseSemver(current) + ? compareSemverVersions(candidate, current) > 0 + : candidate !== current; + +/** + * Picks the release a managed installation's `start` downloads. A fresh install + * gets T3 Code's pinned release; an existing one moves to the newest known + * release above it. Null means nothing newer is known, so `start` re-checks the + * installed runtime instead of downgrading it. + */ +export function selectManagedRelease( + installedVersion: string | null, + pinned: Release | null, + upstream: Release | null, +): Release | null { + if (installedVersion === null) return pinned; + let selected: Release | null = null; + for (const candidate of [pinned, upstream]) { + if (candidate && isNewer(candidate.version, selected?.version ?? installedVersion)) { + selected = candidate; + } + } + return selected; +} diff --git a/apps/server/src/provider/model-manifest.json b/apps/server/src/provider/model-manifest.json index 678ad71161c3..01727cc2680b 100644 --- a/apps/server/src/provider/model-manifest.json +++ b/apps/server/src/provider/model-manifest.json @@ -1,6 +1,6 @@ { "version": 1, - "updatedAt": "2026-11-10T17:25:47Z", + "updatedAt": "2026-11-14T15:58:47Z", "compatibility": [ { "driver": "codex", @@ -75,8 +75,9 @@ { "driver": "antigravity", "t3CodeRange": ">=0.0.42", - "recommendedRange": "=1.4.5 || =1.4.1", + "recommendedRange": ">=1.5.1 <2", "ranges": [ + { "range": ">=1.5.1 <2", "status": "supported" }, { "range": "=1.4.5", "status": "supported" }, { "range": "=1.4.1", "status": "supported" }, { "range": "=1.3.0", "status": "supported" }, @@ -94,8 +95,8 @@ "t3CodeRange": ">=0.0.46", "recommendedRange": ">=1.0.0", "ranges": [ - { "range": ">=0.80.5", "status": "supported" }, - { "range": "<0.80.5", "status": "unsupported" } + { "range": ">=0.84.3", "status": "supported" }, + { "range": "<0.84.3", "status": "unsupported" } ] }, { @@ -770,7 +771,7 @@ "aliases": ["fable", "fable-5.2", "claude-fable-5.2"], "status": "current", "profile": "fable-5", - "adapter": { "claudeCode": { "minVersion": "2.1.290" } } + "adapter": { "claudeCode": { "minVersion": "2.1.321" } } }, { "slug": "claude-fable-5-1", diff --git a/apps/server/src/provider/providerCompatibility.test.ts b/apps/server/src/provider/providerCompatibility.test.ts index 780c2d789731..ed722b7c5351 100644 --- a/apps/server/src/provider/providerCompatibility.test.ts +++ b/apps/server/src/provider/providerCompatibility.test.ts @@ -22,7 +22,6 @@ import { ProviderCompatibilityPolicy, resolveProviderCompatibility, } from "./providerCompatibility.ts"; -import { resolveAntigravityReleaseAsset } from "./antigravityRelease.ts"; import bundledManifest from "./model-manifest.json" with { type: "json" }; const decodeCompatibilityPolicies = Schema.decodeUnknownSync( @@ -133,19 +132,15 @@ describe("provider compatibility", () => { } }); - it.each([ - ["darwin", "arm64"], - ["win32", "x64"], - ] as const)( - "supports the managed Antigravity runtime for %s-%s in the bundled manifest", - (platform, arch) => { + it.each(["1.4.5", "1.5.7"])( + "supports the legacy and native Antigravity runtime %s in the bundled manifest", + (version) => { const policies = decodeCompatibilityPolicies(bundledManifest.compatibility); - const release = resolveAntigravityReleaseAsset(platform, arch); assert.strictEqual( resolveProviderCompatibility( policies, ProviderDriverKind.make("antigravity"), - `agy_acp_server_${release?.version}`, + `agy_acp_server_${version}`, )?.status, "supported", ); @@ -330,5 +325,5 @@ it.effect("a remote policy refresh preserves a newer health result on the regist ), ), ); - }).pipe(Effect.scoped), + }), ); diff --git a/apps/server/src/provider/providerInstallation.test.ts b/apps/server/src/provider/providerInstallation.test.ts index 8a3d3316ac32..d4c7467c62d0 100644 --- a/apps/server/src/provider/providerInstallation.test.ts +++ b/apps/server/src/provider/providerInstallation.test.ts @@ -95,6 +95,7 @@ const makeHarness = Effect.fn("providerInstallation.test.makeHarness")(function* }), state: Effect.succeed({ ...state, driver: ProviderDriverKind.make("codex") }), changes: Stream.succeed({ ...state, driver: ProviderDriverKind.make("codex") }), + checkForUpdate: Effect.void, remove: (paths) => Effect.sync(() => { protectedPaths = paths ?? []; @@ -103,6 +104,7 @@ const makeHarness = Effect.fn("providerInstallation.test.makeHarness")(function* }), Layer.mock(AntigravityInstallation.AntigravityInstallation)({ managedDirectory: "/unused-managed-runtime", + checkForUpdate: Effect.void, start: Effect.sync(() => { calls.push("start"); return { ...state, operationId: "install-1", phase: "downloading" as const }; diff --git a/apps/server/src/provider/providerInstallation.ts b/apps/server/src/provider/providerInstallation.ts index 7a628dfa5b31..e5e864517ba6 100644 --- a/apps/server/src/provider/providerInstallation.ts +++ b/apps/server/src/provider/providerInstallation.ts @@ -186,9 +186,21 @@ const make = Effect.fn("makeProviderInstallation")(function* () { .pipe(Effect.mapError(failure(input.instanceId))); }); + // Watching an installation is when its owner can act on an update, so the + // lookup runs then, in the background, unless update checks are off. const subscribe = (input: ProviderSetupInput) => Stream.unwrap( requireInstance(input.instanceId, "observe-install").pipe( + Effect.tap(({ installation }) => + settings.getSettings.pipe( + Effect.flatMap((current) => + current.enableProviderUpdateChecks + ? Effect.forkDetach(installation.checkForUpdate) + : Effect.void, + ), + Effect.ignore, + ), + ), Effect.map(({ installation }) => installation.changes), ), ); diff --git a/apps/server/src/provider/providerMaintenanceRunner.test.ts b/apps/server/src/provider/providerMaintenanceRunner.test.ts index 32bf49d5861a..985347a66e96 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.test.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.test.ts @@ -207,6 +207,7 @@ function makeRegistry( refresh: () => Ref.get(providersRef), refreshInstance: () => Ref.get(providersRef), reloadSecretsAndRefresh: () => Ref.get(providersRef), + applySecretAndRefresh: () => Ref.get(providersRef), refreshAfterUpdate: () => { const version = options?.installedVersionAfterUpdate; if (version === undefined) { diff --git a/apps/server/src/provider/providerMaintenanceRunner.ts b/apps/server/src/provider/providerMaintenanceRunner.ts index 80248acea577..9ac62dca7041 100644 --- a/apps/server/src/provider/providerMaintenanceRunner.ts +++ b/apps/server/src/provider/providerMaintenanceRunner.ts @@ -450,6 +450,7 @@ export const make = Effect.fn("ProviderMaintenanceRunner.make")(function* () { instanceId, action: "update", state, + targetVersion, }); const setQueuedState = setUpdateState( makeUpdateState({ diff --git a/apps/server/src/provider/testFixtures/grok-text-mock-agent.mjs b/apps/server/src/provider/testFixtures/grok-text-mock-agent.mjs index 357bdb7fe564..b8cb949a5fb7 100644 --- a/apps/server/src/provider/testFixtures/grok-text-mock-agent.mjs +++ b/apps/server/src/provider/testFixtures/grok-text-mock-agent.mjs @@ -5,12 +5,35 @@ const sessionId = "grok-text-session"; let currentModelId = "grok-mock-default"; const models = ["grok-mock-default", "grok-mock-alt"]; const write = (message) => process.stdout.write(`${JSON.stringify(message)}\n`); +const permissionRequestId = 9001; +let pendingPromptId; +const completePrompt = (id) => { + write({ + jsonrpc: "2.0", + method: "session/update", + params: { + sessionId, + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: process.env.T3_ACP_PROMPT_RESPONSE_TEXT ?? "" }, + }, + }, + }); + write({ jsonrpc: "2.0", id, result: { stopReason: "end_turn" } }); +}; for await (const line of NodeReadline.createInterface({ input: process.stdin })) { const request = JSON.parse(line); if (process.env.T3_ACP_REQUEST_LOG_PATH) { NodeFS.appendFileSync(process.env.T3_ACP_REQUEST_LOG_PATH, `${line}\n`); } + if (request.method === undefined) { + if (request.id === permissionRequestId && pendingPromptId !== undefined) { + completePrompt(pendingPromptId); + pendingPromptId = undefined; + } + continue; + } if (request.id === undefined) continue; let result; switch (request.method) { @@ -43,19 +66,25 @@ for await (const line of NodeReadline.createInterface({ input: process.stdin })) result = {}; break; case "session/prompt": - write({ - jsonrpc: "2.0", - method: "session/update", - params: { - sessionId, - update: { - sessionUpdate: "agent_message_chunk", - content: { type: "text", text: process.env.T3_ACP_PROMPT_RESPONSE_TEXT ?? "" }, + if (process.env.T3_ACP_REQUEST_PERMISSION) { + pendingPromptId = request.id; + write({ + jsonrpc: "2.0", + id: permissionRequestId, + method: "session/request_permission", + params: { + sessionId, + toolCall: { toolCallId: "tool-1", kind: "execute", title: "Run `git status`" }, + options: [ + { optionId: "allow-once", name: "Yes", kind: "allow_once" }, + { optionId: "reject-once", name: "No", kind: "reject_once" }, + ], }, - }, - }); - result = { stopReason: "end_turn" }; - break; + }); + continue; + } + completePrompt(request.id); + continue; default: write({ jsonrpc: "2.0", diff --git a/apps/server/src/provider/testUtils/providerRegistryMock.ts b/apps/server/src/provider/testUtils/providerRegistryMock.ts index 3ae8079414a9..ccf4979937b3 100644 --- a/apps/server/src/provider/testUtils/providerRegistryMock.ts +++ b/apps/server/src/provider/testUtils/providerRegistryMock.ts @@ -12,6 +12,7 @@ const makeProviderRegistryMock = ( refresh: () => Effect.succeed(providers), refreshInstance: () => Effect.succeed(providers), reloadSecretsAndRefresh: () => Effect.succeed(providers), + applySecretAndRefresh: () => Effect.succeed(providers), refreshAfterUpdate: () => Effect.succeed(providers), refreshWorkspaceSnapshot: () => Effect.succeed(providers), invalidateModelCaches: () => Effect.void, diff --git a/apps/server/src/provider/userInputAttachments.test.ts b/apps/server/src/provider/userInputAttachments.test.ts index aacc345b3cfe..4087601b58c0 100644 --- a/apps/server/src/provider/userInputAttachments.test.ts +++ b/apps/server/src/provider/userInputAttachments.test.ts @@ -7,6 +7,7 @@ import { describe, expect, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as Deferred from "effect/Deferred"; import * as Fiber from "effect/Fiber"; +import * as Queue from "effect/Queue"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as TestClock from "effect/testing/TestClock"; @@ -88,6 +89,114 @@ describe("question answer paths", () => { }).pipe(Effect.provide(layer)), ); + it.effect.each(["interrupted", "failed"])( + "extracts two documents at once and frees a slot when one is %s", + (outcome) => + Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + const [first, second, third] = ["first", "second", "third"].map((name) => + NodePath.join(config.attachmentsDir, `${name}.rtf`), + ); + NodeFS.writeFileSync(first!, String.raw`{\rtf1 First}`); + NodeFS.writeFileSync(second!, "invalid document"); + NodeFS.writeFileSync(third!, String.raw`{\rtf1 Third}`); + const started = yield* Queue.unbounded(); + const releases = new Map>(); + for (const path of [first!, second!, third!]) + releases.set(path, yield* Deferred.make()); + yield* Effect.gen(function* () { + const documents = yield* DocumentText.DocumentText; + const leaving = yield* documents.ensure(first!).pipe(Effect.forkChild); + const staying = yield* documents.ensure(first!).pipe(Effect.forkChild); + const freed = yield* documents.ensure(second!).pipe(Effect.forkChild); + expect(new Set([yield* Queue.take(started), yield* Queue.take(started)])).toEqual( + new Set([first, second]), + ); + const waiting = yield* documents.ensure(third!).pipe(Effect.forkChild); + yield* Fiber.interrupt(leaving); + if (outcome === "interrupted") yield* Fiber.interrupt(freed); + else { + yield* Deferred.succeed(releases.get(second!)!, undefined); + expect(yield* Fiber.join(freed)).toBeUndefined(); + } + expect(yield* Queue.take(started)).toBe(third); + yield* Deferred.succeed(releases.get(first!)!, undefined); + yield* Deferred.succeed(releases.get(third!)!, undefined); + expect(yield* Fiber.join(staying)).toBe(first + ".txt"); + expect(yield* Fiber.join(waiting)).toBe(third + ".txt"); + expect(yield* Queue.size(started)).toBe(0); + expect(NodeFS.existsSync(second + ".txt")).toBe(false); + expect( + NodeFS.readdirSync(config.attachmentsDir).filter((file) => file.endsWith(".part")), + ).toEqual([]); + }).pipe( + Effect.provide(Layer.fresh(DocumentText.layer)), + Effect.provideService(FileSystem.FileSystem, { + ...fs, + stat: (file) => + Effect.gen(function* () { + yield* Queue.offer(started, file); + yield* Deferred.await(releases.get(file)!); + return yield* fs.stat(file); + }), + }), + ); + }).pipe(Effect.provide(layer)), + ); + + it.effect("retries a document while its cancelled extraction is still publishing", () => + Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + const path = NodePath.join(config.attachmentsDir, "retried.rtf"); + NodeFS.writeFileSync(path, String.raw`{\rtf1 Retried}`); + const writing = yield* Deferred.make(); + const writeReleased = yield* Deferred.make(); + const renaming = yield* Deferred.make(); + const renameReleased = yield* Deferred.make(); + let writes = 0; + let renames = 0; + yield* Effect.gen(function* () { + const documents = yield* DocumentText.DocumentText; + const cancelled = yield* documents.ensure(path).pipe(Effect.forkChild); + yield* Deferred.await(writing); + const interrupting = yield* Fiber.interrupt(cancelled).pipe(Effect.forkChild); + const retry = yield* documents.ensure(path).pipe(Effect.forkChild); + yield* Deferred.await(renaming); + yield* Deferred.succeed(writeReleased, undefined); + yield* Fiber.join(interrupting); + yield* Deferred.succeed(renameReleased, undefined); + expect(yield* Fiber.join(retry)).toBe(path + ".txt"); + expect(NodeFS.readFileSync(path + ".txt", "utf8")).toBe("Retried"); + expect( + NodeFS.readdirSync(config.attachmentsDir).filter((file) => file.endsWith(".part")), + ).toEqual([]); + }).pipe( + Effect.provide(Layer.fresh(DocumentText.layer)), + Effect.provideService(FileSystem.FileSystem, { + ...fs, + writeFileString: (file, data) => + Effect.gen(function* () { + if (++writes === 1) { + yield* Deferred.succeed(writing, undefined); + yield* Deferred.await(writeReleased); + } + return yield* fs.writeFileString(file, data); + }), + rename: (from, to) => + Effect.gen(function* () { + if (++renames === 1) { + yield* Deferred.succeed(renaming, undefined); + yield* Deferred.await(renameReleased); + } + return yield* fs.rename(from, to); + }), + }), + ); + }).pipe(Effect.provide(layer)), + ); + it.effect.each(["spawn", "read"])("retries after a transient %s failure expires", (failure) => Effect.acquireUseRelease( Effect.promise(async () => { @@ -176,6 +285,11 @@ describe("question answer paths", () => { String.raw`{\rtf1\ansi A\enspace B\emspace C\qmspace D\~E\_F\-G}`, "A\u2002B\u2003C\u2005D\u00a0E\u2011FG", ], + [ + "joiner and direction marks", + String.raw`{\rtf1\ansi A\zwj B\zwnj C\ltrmark D\rtlmark E}`, + "A\u200dB\u200cC\u200eD\u200fE", + ], [ "text symbol fallback", String.raw`{\rtf1\ansi\uc10\u945\endash\emdash\enspace\emspace\qmspace\lquote\rquote\ldblquote\rdblquote\bullet Z}`, @@ -581,28 +695,32 @@ describe("question answer paths", () => { bounded: true, parts: { "word/document.xml": - 'Report & totalsNorth1234BeforeInnerAfterNeighbor', + 'Report & totalsNorth1234BeforeInnerAfterNeighbor', }, - expected: ["## Report & totals", "North\t12 34", "Before Inner After\tNeighbor"], + expected: [ + "## Report & totals", + "North\t12 34", + "Before Inner After\tNeighbor\t\u2011\n\u2011", + ], }, { ext: "docx", bounded: true, parts: { "word/document.xml": - 'FirstSecondThird', + 'FirstSecondThirdparty‑End', }, - expected: ["First\nSecond\nThird"], + expected: ["First\nSecond\nThird\u2011party\u2011End"], }, { ext: "docx", bounded: true, parts: { "word/document.xml": - 'Intro PAGEREF _Toc1 \\h 3Fee $500$700Moveddue', + 'Intro PAGEREF _Toc1 \\h 3Fee $500$700Moveddue', }, expected: ["Intro 3\nFee $700\tdue"], - absent: ["PAGEREF", "$500", "Moved"], + absent: ["PAGEREF", "$500", "Moved", "\u2011"], }, { ext: "odt", @@ -639,6 +757,18 @@ describe("question answer paths", () => { ], absent: ["SUM(0,0)", "[Extracted text truncated"], }, + { + ext: "xlsx", + bounded: true, + parts: { + "xl/workbook.xml": '', + "xl/_rels/workbook.xml.rels": + '', + "xl/worksheets/a.xml": + '1012', + }, + expected: ["true\tfalse\t1\t2"], + }, { ext: "xlsx", parts: { @@ -679,6 +809,22 @@ describe("question answer paths", () => { expected: ["## Sheet: Phonetics", "東京\t東京"], absent: ["とうきょう"], }, + { + ext: "xlsx", + bounded: true, + parts: { + "xl/workbook.xml": '', + "xl/_rels/workbook.xml.rels": + '', + "xl/sharedStrings.xml": + "first_x000D_\nsecond_x005F_x0041__x005F_x005F_x0041_Bold_x000D__x005F_x000D_&_xnothex_ cap_X0041_ _x000d_ tail", + "xl/worksheets/a.xml": + '01234_x005F_x005F_first_x000D_secondA1Total_x000D_due', + }, + expected: [ + "## Sheet: Escapes\nfirst second\t_x0041_\t_x005F_x0041_\tBold _x000D_\t&_xnothex_ cap_X0041_ tail\n_x005F_\tfirst second\tTotal due", + ], + }, { ext: "xlsx", bounded: true, @@ -782,9 +928,11 @@ describe("question answer paths", () => { ext: "odt", parts: { "content.xml": - 'Orchard19violetcrates', + '\n Orchard\n19violetcrates \n\t shipped todayDockGate\u00a0B SecondAconsole.log(1);\n B', }, - expected: ["## Orchard", "19 violet crates"], + expected: [ + "## Orchard\n19 violet crates shipped today\tDock\nGate\u00a0B\nSecond\nA B", + ], }, { ext: "ods", @@ -831,9 +979,9 @@ describe("question answer paths", () => { ext: "odp", parts: { "content.xml": - "FirstSecond", + "\n First\n\t slide Second", }, - expected: ["## Slide 1", "First", "## Slide 2", "Second"], + expected: ["## Slide 1\nFirst slide\n\n## Slide 2\nSecond"], }, { ext: "ods", @@ -928,13 +1076,13 @@ describe("question answer paths", () => { "OEBPS/book.opf": '', "OEBPS/a.xhtml": - '

Last chapter

Price 10 8

', + '

Last chapter

Price 10 8 old

', "OEBPS/b.xhtml": - "

First chapter

Larkspur

Direct div
Direct section
  • Direct item emphasis trailing
  • Account: A
    Balance: 731Currency: USD
    HeadingTotal
    Important value731
    Before
    Inner
    After
    Neighbor
    ", + 'My Novel

    First chapter

    Larkspur

    Direct div
    Direct section
  • Direct item emphasis trailing
  • Account: A
    Balance: 731Currency: USD
    HeadingTotal
    Important value731
    Before
    Inner
    After
    Neighbor
    ', }, expected: [ "## Chapter 1", - "# First chapter", + "\n# First chapter", "Larkspur", "Direct div", "Direct section", @@ -945,8 +1093,9 @@ describe("question answer paths", () => { "Before Inner After\tNeighbor", "## Chapter 2", "Last chapter", - "Price 10 8", + "Price 10 8 old", ], + absent: ["My Novel", "Hidden Author"], }, ]; for (const [caseIndex, item] of cases.entries()) { diff --git a/apps/server/src/pullRequest/PullRequestReadCache.test.ts b/apps/server/src/pullRequest/PullRequestReadCache.test.ts index 071f1d4978a7..99b3cdba22ff 100644 --- a/apps/server/src/pullRequest/PullRequestReadCache.test.ts +++ b/apps/server/src/pullRequest/PullRequestReadCache.test.ts @@ -1,5 +1,4 @@ import * as NodeV8 from "node:v8"; -import * as NodeVM from "node:vm"; import { assert, it } from "@effect/vitest"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; @@ -16,9 +15,6 @@ import * as KeyValueStore from "effect/persistence/KeyValueStore"; import * as TestClock from "effect/testing/TestClock"; import * as PullRequestReadCache from "./PullRequestReadCache.ts"; -NodeV8.setFlagsFromString("--expose-gc"); -const collectGarbage = NodeVM.runInNewContext("gc") as () => void; - const cacheLayer = (directory: string) => PullRequestReadCache.make.pipe(Effect.provide(KeyValueStore.layerFileSystem(directory))); @@ -203,13 +199,10 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("PR filesystem ca it.effect("does not keep a caller's data alive after its reads and invalidations", () => Effect.gen(function* () { const cache = yield* PullRequestReadCache.make; - const sweeps: Array> = []; + class SweepData extends Map {} // Like a sync sweep: an Effect.fn made per run that closes over that run's data. const sweep = Effect.fn("sweep")(function* (index: number) { - const groups = new Map( - Array.from({ length: 1_000 }, (_, id) => [`group-${id}`, id]), - ); - sweeps.push(new WeakRef(groups)); + const groups = new SweepData(Array.from({ length: 1_000 }, (_, id) => [`group-${id}`, id])); const read = Effect.fn("read")(function* (pr: string) { const size = groups.size; yield* cache.invalidate(pr); @@ -219,10 +212,8 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("PR filesystem ca yield* Effect.forEach(["pr-1", "pr-2"], read, { discard: true }); }); for (let index = 0; index < 5; index++) yield* sweep(index); - // WeakRef targets stay alive until the current job ends, so yield before collecting. - yield* Effect.yieldNow; - collectGarbage(); - assert.strictEqual(sweeps.filter((sweepData) => sweepData.deref() !== undefined).length, 0); + // Counts after a full GC. A WeakRef would keep its target alive until the job ends. + assert.strictEqual(NodeV8.queryObjects(SweepData, { format: "count" }), 0); }).pipe(Effect.provide(KeyValueStore.layerMemory)), ); diff --git a/apps/server/src/pullRequest/PullRequestReviewGuideService.test.ts b/apps/server/src/pullRequest/PullRequestReviewGuideService.test.ts index 3ee929f923a5..200ea36b7d64 100644 --- a/apps/server/src/pullRequest/PullRequestReviewGuideService.test.ts +++ b/apps/server/src/pullRequest/PullRequestReviewGuideService.test.ts @@ -390,208 +390,198 @@ it.layer(layerBase, { excludeTestServices: true })("PullRequestReviewGuideServic expected: ["local objects:", "the commit:", "branch feature:", "refs/pull/7/head:"], }, ])("refuses to write a guide for $name", ({ reportedHead, shadow, expected }) => - Effect.scoped( - Effect.gen(function* () { - const run = yield* startSingleBranchRun({ - repositoryName: "refused-head", - reportedHead, - refuseCommitFetch: false, - prepare: (repositories, workspace) => - git(workspace, ["fetch", "--quiet", "origin", "feature"]).pipe( - Effect.andThen( - shadow === "none" - ? Effect.void - : git(workspace, [shadow, repositories.headSha.slice(0, 12), "HEAD"]), - ), - Effect.asVoid, + Effect.gen(function* () { + const run = yield* startSingleBranchRun({ + repositoryName: "refused-head", + reportedHead, + refuseCommitFetch: false, + prepare: (repositories, workspace) => + git(workspace, ["fetch", "--quiet", "origin", "feature"]).pipe( + Effect.andThen( + shadow === "none" + ? Effect.void + : git(workspace, [shadow, repositories.headSha.slice(0, 12), "HEAD"]), ), - }); - const result = yield* run.settled; - assert.isFalse(yield* Deferred.isDone(run.stored), "a guide was written"); - assert.strictEqual(result.run?._tag, "Failed"); - const detail = result.run?._tag === "Failed" ? result.run.detail : ""; - for (const part of expected) assert.include(detail, part); - // Only the host's own head ref is tried. - assert.notInclude(detail, "refs/merge-requests/"); - assert.deepStrictEqual(run.observed, []); - assert.isFalse(run.fetchedRefs.some((ref) => ref.startsWith("-"))); - const fs = yield* FileSystem.FileSystem; - assert.isFalse(yield* fs.exists(`${run.repositories.root}/injected`)); - }), - ), + Effect.asVoid, + ), + }); + const result = yield* run.settled; + assert.isFalse(yield* Deferred.isDone(run.stored), "a guide was written"); + assert.strictEqual(result.run?._tag, "Failed"); + const detail = result.run?._tag === "Failed" ? result.run.detail : ""; + for (const part of expected) assert.include(detail, part); + // Only the host's own head ref is tried. + assert.notInclude(detail, "refs/merge-requests/"); + assert.deepStrictEqual(run.observed, []); + assert.isFalse(run.fetchedRefs.some((ref) => ref.startsWith("-"))); + const fs = yield* FileSystem.FileSystem; + assert.isFalse(yield* fs.exists(`${run.repositories.root}/injected`)); + }), ); it.effect("measures only the PR change using the renamed verified remote's fetch URL", () => - Effect.scoped( - Effect.gen(function* () { - const repositories = yield* makeRepositories; - const fork = `${repositories.root}/fork`; - yield* git(repositories.root, ["clone", "--quiet", repositories.remote, fork]); - yield* git(fork, ["branch", "stack-base", "main~1"]); - // The cached identity still names origin, which now belongs to the stale fork. - yield* git(repositories.workspace, ["remote", "rename", "origin", "upstream"]); - yield* git(repositories.workspace, ["remote", "add", "origin", fork]); - yield* git(repositories.workspace, [ - "config", - `url.${repositories.remote}.insteadOf`, - "review-upstream:", - ]); - yield* git(repositories.workspace, ["remote", "set-url", "upstream", "review-upstream:"]); - const changed = yield* Deferred.make(); - const driver = yield* GitVcsDriver.GitVcsDriver; - const stored = yield* Deferred.make(); - const context = yield* Layer.build( - makeLayer({ - ...repositories, - stored, - generate: (input) => - git(input.cwd, ["diff", "--name-only", `${input.baseCommit}...HEAD`]).pipe( - Effect.provideService(GitVcsDriver.GitVcsDriver, driver), - Effect.orDie, - Effect.flatMap((names) => Deferred.succeed(changed, names)), - Effect.as({ stops: [] }), - ), - }), - ); - const service = Context.get( - context, - PullRequestReviewGuideService.PullRequestReviewGuideService, - ); - yield* service.generate(reference); - assert.strictEqual(yield* Deferred.await(changed), "feature.txt"); - yield* Deferred.await(stored); - const result = yield* service.get(reference); - assert.strictEqual(result.guide?.headSha, repositories.headSha); - assert.deepStrictEqual(result.guide?.stops[0]?.files, [{ path: "feature.txt" }]); - }), - ), + Effect.gen(function* () { + const repositories = yield* makeRepositories; + const fork = `${repositories.root}/fork`; + yield* git(repositories.root, ["clone", "--quiet", repositories.remote, fork]); + yield* git(fork, ["branch", "stack-base", "main~1"]); + // The cached identity still names origin, which now belongs to the stale fork. + yield* git(repositories.workspace, ["remote", "rename", "origin", "upstream"]); + yield* git(repositories.workspace, ["remote", "add", "origin", fork]); + yield* git(repositories.workspace, [ + "config", + `url.${repositories.remote}.insteadOf`, + "review-upstream:", + ]); + yield* git(repositories.workspace, ["remote", "set-url", "upstream", "review-upstream:"]); + const changed = yield* Deferred.make(); + const driver = yield* GitVcsDriver.GitVcsDriver; + const stored = yield* Deferred.make(); + const context = yield* Layer.build( + makeLayer({ + ...repositories, + stored, + generate: (input) => + git(input.cwd, ["diff", "--name-only", `${input.baseCommit}...HEAD`]).pipe( + Effect.provideService(GitVcsDriver.GitVcsDriver, driver), + Effect.orDie, + Effect.flatMap((names) => Deferred.succeed(changed, names)), + Effect.as({ stops: [] }), + ), + }), + ); + const service = Context.get( + context, + PullRequestReviewGuideService.PullRequestReviewGuideService, + ); + yield* service.generate(reference); + assert.strictEqual(yield* Deferred.await(changed), "feature.txt"); + yield* Deferred.await(stored); + const result = yield* service.get(reference); + assert.strictEqual(result.guide?.headSha, repositories.headSha); + assert.deepStrictEqual(result.guide?.stops[0]?.files, [{ path: "feature.txt" }]); + }), ); it.effect("keeps excerpts despite the checkout's diff driver and a failing hook", () => - Effect.scoped( - Effect.gen(function* () { - const repositories = yield* makeRepositories; - const fs = yield* FileSystem.FileSystem; - yield* fs.writeFileString( - `${repositories.workspace}/.git/info/attributes`, - "feature.txt diff=blank\n", - ); - yield* git(repositories.workspace, ["config", "diff.blank.textconv", "true"]); - const hook = `${repositories.workspace}/.git/hooks/post-checkout`; - yield* fs.writeFileString(hook, "#!/bin/sh\nexit 1\n"); - yield* fs.chmod(hook, 0o755); - const stored = yield* Deferred.make(); - const context = yield* Layer.build( - makeLayer({ - ...repositories, - stored, - generate: () => - Effect.succeed({ - stops: [ - { - title: "The feature", - summary: "The change.", - attention: null, - excerpts: [{ path: "feature.txt", side: "new", start: 1, end: 3, caption: "" }], - files: [], - notes: [], - questions: [], - }, - ], - }), - }), - ); - const service = Context.get( - context, - PullRequestReviewGuideService.PullRequestReviewGuideService, - ); - const runs = yield* watchRuns; - yield* runs.generate(service, reference); - const result = yield* runs.settled(service, reference); - assert.notStrictEqual(result.run?._tag, "Failed", JSON.stringify(result.run)); - assert.deepStrictEqual(result.guide?.stops[0]?.excerpts, [ - { path: "feature.txt", side: "new", start: 1, end: 3, caption: "" }, - ]); - }), - ), + Effect.gen(function* () { + const repositories = yield* makeRepositories; + const fs = yield* FileSystem.FileSystem; + yield* fs.writeFileString( + `${repositories.workspace}/.git/info/attributes`, + "feature.txt diff=blank\n", + ); + yield* git(repositories.workspace, ["config", "diff.blank.textconv", "true"]); + const hook = `${repositories.workspace}/.git/hooks/post-checkout`; + yield* fs.writeFileString(hook, "#!/bin/sh\nexit 1\n"); + yield* fs.chmod(hook, 0o755); + const stored = yield* Deferred.make(); + const context = yield* Layer.build( + makeLayer({ + ...repositories, + stored, + generate: () => + Effect.succeed({ + stops: [ + { + title: "The feature", + summary: "The change.", + attention: null, + excerpts: [{ path: "feature.txt", side: "new", start: 1, end: 3, caption: "" }], + files: [], + notes: [], + questions: [], + }, + ], + }), + }), + ); + const service = Context.get( + context, + PullRequestReviewGuideService.PullRequestReviewGuideService, + ); + const runs = yield* watchRuns; + yield* runs.generate(service, reference); + const result = yield* runs.settled(service, reference); + assert.notStrictEqual(result.run?._tag, "Failed", JSON.stringify(result.run)); + assert.deepStrictEqual(result.guide?.stops[0]?.excerpts, [ + { path: "feature.txt", side: "new", start: 1, end: 3, caption: "" }, + ]); + }), ); it.effect("refuses generation when the verified remote disappeared despite a usable origin", () => - Effect.scoped( - Effect.gen(function* () { - const repositories = yield* makeRepositories; - const fork = `${repositories.root}/fork`; - yield* git(repositories.root, ["clone", "--quiet", repositories.remote, fork]); - yield* git(fork, ["branch", "stack-base", repositories.lowerSha]); - yield* git(repositories.workspace, ["remote", "set-url", "origin", fork]); - const started = yield* Deferred.make>(); - const stored = yield* Deferred.make(); - const generated = yield* Ref.make(false); - const driver = yield* GitVcsDriver.GitVcsDriver; - const context = yield* Layer.build( - makeLayer({ - ...repositories, - stored, - generate: () => Ref.set(generated, true).pipe(Effect.as({ stops: [] })), - }).pipe( - Layer.provide( - Layer.succeed(GitVcsDriver.GitVcsDriver, { - ...driver, - execute: (call) => - Effect.withFiber((fiber) => - Deferred.succeed(started, fiber).pipe(Effect.andThen(driver.execute(call))), - ), - }), - ), + Effect.gen(function* () { + const repositories = yield* makeRepositories; + const fork = `${repositories.root}/fork`; + yield* git(repositories.root, ["clone", "--quiet", repositories.remote, fork]); + yield* git(fork, ["branch", "stack-base", repositories.lowerSha]); + yield* git(repositories.workspace, ["remote", "set-url", "origin", fork]); + const started = yield* Deferred.make>(); + const stored = yield* Deferred.make(); + const generated = yield* Ref.make(false); + const driver = yield* GitVcsDriver.GitVcsDriver; + const context = yield* Layer.build( + makeLayer({ + ...repositories, + stored, + generate: () => Ref.set(generated, true).pipe(Effect.as({ stops: [] })), + }).pipe( + Layer.provide( + Layer.succeed(GitVcsDriver.GitVcsDriver, { + ...driver, + execute: (call) => + Effect.withFiber((fiber) => + Deferred.succeed(started, fiber).pipe(Effect.andThen(driver.execute(call))), + ), + }), ), - ); - const service = Context.get( - context, - PullRequestReviewGuideService.PullRequestReviewGuideService, - ); - const before = yield* service.get(reference); - yield* service.generate(reference); - // Join the background run so its failure is visible without polling service state. - yield* Fiber.await(yield* Deferred.await(started)); - const result = yield* service.get(reference); - assert.strictEqual(result.run?._tag, "Failed"); - assert.include( - result.run?._tag === "Failed" ? result.run.detail : "", - "no longer has a remote for the pull request's repository", - ); - assert.deepStrictEqual(result.guide, before.guide); - assert.isFalse(yield* Deferred.isDone(stored)); - assert.isFalse(yield* Ref.get(generated)); - }), - ), + ), + ); + const service = Context.get( + context, + PullRequestReviewGuideService.PullRequestReviewGuideService, + ); + const before = yield* service.get(reference); + yield* service.generate(reference); + // Join the background run so its failure is visible without polling service state. + yield* Fiber.await(yield* Deferred.await(started)); + const result = yield* service.get(reference); + assert.strictEqual(result.run?._tag, "Failed"); + assert.include( + result.run?._tag === "Failed" ? result.run.detail : "", + "no longer has a remote for the pull request's repository", + ); + assert.deepStrictEqual(result.guide, before.guide); + assert.isFalse(yield* Deferred.isDone(stored)); + assert.isFalse(yield* Ref.get(generated)); + }), ); it.effect("refuses a pull request from a repository the project is not a checkout of", () => - Effect.scoped( - Effect.gen(function* () { - const repositories = yield* makeRepositories; - const stored = yield* Deferred.make(); - const context = yield* Layer.build( - makeLayer({ - ...repositories, - stored, - identity: { - ...ACME_APP, - canonicalKey: "github.com/acme/other", - displayName: "acme/other", - name: "other", - }, - generate: () => - Effect.die("a guide must not be written from another repository's checkout"), - }), - ); - const service = Context.get( - context, - PullRequestReviewGuideService.PullRequestReviewGuideService, - ); - const result = yield* Effect.flip(service.generate(reference)); - assert.include(result.message, "in another repository"); - }), - ), + Effect.gen(function* () { + const repositories = yield* makeRepositories; + const stored = yield* Deferred.make(); + const context = yield* Layer.build( + makeLayer({ + ...repositories, + stored, + identity: { + ...ACME_APP, + canonicalKey: "github.com/acme/other", + displayName: "acme/other", + name: "other", + }, + generate: () => + Effect.die("a guide must not be written from another repository's checkout"), + }), + ); + const service = Context.get( + context, + PullRequestReviewGuideService.PullRequestReviewGuideService, + ); + const result = yield* Effect.flip(service.generate(reference)); + assert.include(result.message, "in another repository"); + }), ); }); diff --git a/apps/server/src/pullRequest/PullRequestService.test.ts b/apps/server/src/pullRequest/PullRequestService.test.ts index f1a0327c878c..9adc2bd6df81 100644 --- a/apps/server/src/pullRequest/PullRequestService.test.ts +++ b/apps/server/src/pullRequest/PullRequestService.test.ts @@ -1429,129 +1429,121 @@ it.effect( ); it.effect("publishes a merge for immediate settlement only after host confirmation", () => - Effect.scoped( - Effect.gen(function* () { - const mergedAt = "2026-09-03T02:00:00.000Z"; - let state: "open" | "merged" = "open"; - let confirmationFails = false; - const reference = { projectId: "p1" as ProjectId, repository: "acme/web", number: 1 }; - const service = yield* makeService({ - projects: [ - project({ id: "p1", title: "web", workspaceRoot: "/a", repository: "acme/web" }), - ], - providers: [ - fakePullRequestProvider(SourceControlProviderKind.make("github"), { - getChangeRequestSummary: () => - confirmationFails - ? Effect.fail( - new PullRequestProviderError({ - provider: SourceControlProviderKind.make("github"), - operation: "getChangeRequestSummary", - reason: "failed", - detail: "HTTP 504", - }), - ) - : Effect.succeed({ ...changeRequest(1, mergedAt), state, mergedAt }), - }), - ], - }); - const merges = yield* service.subscribeMerges; - const observedMerge = yield* Stream.runHead(merges).pipe( - Effect.forkChild({ startImmediately: true }), - ); + Effect.gen(function* () { + const mergedAt = "2026-09-03T02:00:00.000Z"; + let state: "open" | "merged" = "open"; + let confirmationFails = false; + const reference = { projectId: "p1" as ProjectId, repository: "acme/web", number: 1 }; + const service = yield* makeService({ + projects: [project({ id: "p1", title: "web", workspaceRoot: "/a", repository: "acme/web" })], + providers: [ + fakePullRequestProvider(SourceControlProviderKind.make("github"), { + getChangeRequestSummary: () => + confirmationFails + ? Effect.fail( + new PullRequestProviderError({ + provider: SourceControlProviderKind.make("github"), + operation: "getChangeRequestSummary", + reason: "failed", + detail: "HTTP 504", + }), + ) + : Effect.succeed({ ...changeRequest(1, mergedAt), state, mergedAt }), + }), + ], + }); + const merges = yield* service.subscribeMerges; + const observedMerge = yield* Stream.runHead(merges).pipe( + Effect.forkChild({ startImmediately: true }), + ); - assert.isUndefined(yield* service.runAction({ ...reference, action: "merge" })); - const queuedRefresh = Option.getOrThrow( - yield* Stream.runHead(service.subscribeRefreshes), - ).revision; - confirmationFails = true; - yield* service.runAction({ ...reference, action: "merge" }); - assert.isAbove( - Option.getOrThrow(yield* Stream.runHead(service.subscribeRefreshes)).revision, - queuedRefresh, - ); - confirmationFails = false; - state = "merged"; - // The event carries the host's merge time, not the moment the confirmation ran. - yield* TestClock.setTime(Date.parse(mergedAt) + 60_000); - yield* service.runAction({ - ...reference, - repository: " ACME/WEB ", - action: "merge", - mergeMethod: "merge", - }); + assert.isUndefined(yield* service.runAction({ ...reference, action: "merge" })); + const queuedRefresh = Option.getOrThrow( + yield* Stream.runHead(service.subscribeRefreshes), + ).revision; + confirmationFails = true; + yield* service.runAction({ ...reference, action: "merge" }); + assert.isAbove( + Option.getOrThrow(yield* Stream.runHead(service.subscribeRefreshes)).revision, + queuedRefresh, + ); + confirmationFails = false; + state = "merged"; + // The event carries the host's merge time, not the moment the confirmation ran. + yield* TestClock.setTime(Date.parse(mergedAt) + 60_000); + yield* service.runAction({ + ...reference, + repository: " ACME/WEB ", + action: "merge", + mergeMethod: "merge", + }); - assert.deepStrictEqual(Option.getOrThrow(yield* Fiber.join(observedMerge)), { - ...reference, - mergedAt, - }); - }), - ), + assert.deepStrictEqual(Option.getOrThrow(yield* Fiber.join(observedMerge)), { + ...reference, + mergedAt, + }); + }), ); it.effect("returns the host's merged reply as a summary and trusts it without another read", () => - Effect.scoped( - Effect.gen(function* () { - const mergedAt = "2026-09-03T02:00:00.000Z"; - const merged = { - number: 1, - title: "Change request 1", - url: "https://host/pull/1", - headBranch: "feat/1", - baseBranch: "main", - state: "merged" as const, - isDraft: false, - updatedAt: mergedAt, - mergedAt, - headSha: "abc123", - autoMergeEnabled: false, - }; - let summaryReads = 0; - const reference = { projectId: "p1" as ProjectId, repository: "acme/web", number: 1 }; - const service = yield* makeService({ - projects: [ - project({ id: "p1", title: "web", workspaceRoot: "/a", repository: "acme/web" }), - ], - providers: [ - fakePullRequestProvider(SourceControlProviderKind.make("github"), { - runAction: () => Effect.succeed(merged), - getChangeRequestSummary: () => - Effect.sync(() => { - summaryReads++; - return { ...changeRequest(1, mergedAt), state: "merged" as const, mergedAt }; - }), - }), - ], - }); - const merges = yield* service.subscribeMerges; - const observedMerge = yield* Stream.runHead(merges).pipe( - Effect.forkChild({ startImmediately: true }), - ); - const observedAt = Date.parse("2026-09-03T02:00:01.000Z"); - yield* TestClock.setTime(observedAt); - - // Clients that did not ask decode the success as void, so they must get nothing back. - assert.isUndefined(yield* service.runAction({ ...reference, action: "merge" })); - const result = yield* service.runAction({ - ...reference, - action: "merge", - replyWithSummary: true, - }); + Effect.gen(function* () { + const mergedAt = "2026-09-03T02:00:00.000Z"; + const merged = { + number: 1, + title: "Change request 1", + url: "https://host/pull/1", + headBranch: "feat/1", + baseBranch: "main", + state: "merged" as const, + isDraft: false, + updatedAt: mergedAt, + mergedAt, + headSha: "abc123", + autoMergeEnabled: false, + }; + let summaryReads = 0; + const reference = { projectId: "p1" as ProjectId, repository: "acme/web", number: 1 }; + const service = yield* makeService({ + projects: [project({ id: "p1", title: "web", workspaceRoot: "/a", repository: "acme/web" })], + providers: [ + fakePullRequestProvider(SourceControlProviderKind.make("github"), { + runAction: () => Effect.succeed(merged), + getChangeRequestSummary: () => + Effect.sync(() => { + summaryReads++; + return { ...changeRequest(1, mergedAt), state: "merged" as const, mergedAt }; + }), + }), + ], + }); + const merges = yield* service.subscribeMerges; + const observedMerge = yield* Stream.runHead(merges).pipe( + Effect.forkChild({ startImmediately: true }), + ); + const observedAt = Date.parse("2026-09-03T02:00:01.000Z"); + yield* TestClock.setTime(observedAt); - assert.deepStrictEqual(result, { - ...merged, - provider: SourceControlProviderKind.make("github"), - projectId: reference.projectId, - repository: reference.repository, - observedAt, - }); - assert.strictEqual(summaryReads, 0); - assert.deepStrictEqual(Option.getOrThrow(yield* Fiber.join(observedMerge)), { - ...reference, - mergedAt, - }); - }), - ), + // Clients that did not ask decode the success as void, so they must get nothing back. + assert.isUndefined(yield* service.runAction({ ...reference, action: "merge" })); + const result = yield* service.runAction({ + ...reference, + action: "merge", + replyWithSummary: true, + }); + + assert.deepStrictEqual(result, { + ...merged, + provider: SourceControlProviderKind.make("github"), + projectId: reference.projectId, + repository: reference.repository, + observedAt, + }); + assert.strictEqual(summaryReads, 0); + assert.deepStrictEqual(Option.getOrThrow(yield* Fiber.join(observedMerge)), { + ...reference, + mergedAt, + }); + }), ); it.effect("refreshes every reader before a queued merge confirmation finishes", () => @@ -2217,6 +2209,72 @@ it.effect("routes a hosted reference to another repository through a project on }), ); +it.effect("an uncached summary reaches the host despite a cached read in another project", () => + Effect.gen(function* () { + const caller = project({ + id: "caller", + title: "caller", + workspaceRoot: "/caller", + repository: "acme/other", + }); + const target = project({ + id: "gitlab", + title: "gitlab", + workspaceRoot: "/gitlab", + repository: "team/project", + provider: "gitlab", + }); + let missing = false; + let reads = 0; + const service = yield* makeService({ + projects: [caller, target], + providers: [ + fakePullRequestProvider(SourceControlProviderKind.make("github")), + fakePullRequestProvider(SourceControlProviderKind.make("gitlab"), { + getChangeRequestSummary: (input) => + Effect.suspend(() => { + reads++; + assert.strictEqual(input.cwd, target.workspaceRoot); + assert.strictEqual(input.repository, "team/project"); + assert.strictEqual(input.host, "gitlab.com"); + return missing + ? Effect.fail( + new PullRequestProviderError({ + provider: SourceControlProviderKind.make("gitlab"), + operation: "summary", + reason: "missing-tool", + detail: "The hosting CLI is unavailable.", + }), + ) + : Effect.succeed(changeRequest(42, "2026-07-02T00:00:00Z")); + }), + }), + ], + }); + const reference = { + projectId: caller.id, + host: "gitlab.com", + repository: "team/project", + number: 42, + allowStale: false, + }; + yield* service.summary(reference, { recoverTransientFailure: false }); + missing = true; + yield* service.summary(reference, { recoverTransientFailure: false }); + assert.strictEqual(reads, 1); + + const error = yield* service + .summary(reference, { recoverTransientFailure: false, bypassCache: true }) + .pipe(Effect.flip); + expect(error).toMatchObject({ + _tag: "PullRequestUnavailableError", + reason: "cli-missing", + provider: "gitlab", + }); + assert.strictEqual(reads, 2); + }), +); + it.effect("routes Azure reads and writes through the requested organization's checkout", () => Effect.gen(function* () { const seen: string[] = []; @@ -2673,6 +2731,40 @@ it.effect("does not revive old summaries when project epochs are evicted", () => }), ); +it.effect("a finished turn leaves another project's cached listing in place", () => + Effect.gen(function* () { + const reads: string[] = []; + const service = yield* makeService({ + projects: [ + project({ id: "p1", title: "web", workspaceRoot: "/a", repository: "acme/web" }), + project({ id: "p2", title: "api", workspaceRoot: "/b", repository: "acme/api" }), + ], + providers: [ + fakePullRequestProvider(SourceControlProviderKind.make("github"), { + listChangeRequests: ({ repository }) => { + reads.push(repository); + return Effect.succeed({ items: [], truncated: false, continues: false }); + }, + }), + ], + }); + const webListing = { state: "open" as const, projectId: "p1" as ProjectId }; + const everyListing = { state: "open" as const }; + + yield* service.list(webListing); + yield* service.list(everyListing); + reads.length = 0; + yield* service.refreshAfterTurn("p2" as ProjectId); + yield* service.list(webListing); + yield* service.list(everyListing); + assert.deepStrictEqual(reads, []); + + yield* service.refreshAfterTurn("p2" as ProjectId, { pushed: true }); + yield* service.list(everyListing); + assert.deepStrictEqual(reads.toSorted(), ["acme/api", "acme/web"]); + }), +); + it.effect("a refresh or a finished turn drops the provider's own change request reads", () => Effect.gen(function* () { const forgotten: Array<{ readonly repository: string; readonly number?: number } | undefined> = @@ -2766,56 +2858,52 @@ it.effect("a read during a refresh cannot keep the provider's old change request ); it.effect("close and reopen notify subscribed readers after invalidating their cached state", () => - Effect.scoped( - Effect.gen(function* () { - let hostCalls = 0; - let state: "open" | "closed" = "open"; - const reference = { projectId: "p1" as ProjectId, repository: "acme/web", number: 1 }; - const service = yield* makeService({ - projects: [ - project({ id: "p1", title: "web", workspaceRoot: "/a", repository: "acme/web" }), - ], - providers: [ - fakePullRequestProvider(SourceControlProviderKind.make("github"), { - getChangeRequestSummary: () => - Effect.succeed({ ...changeRequest(1, "2026-09-16T00:00:00.000Z"), state }), - runAction: (input) => - Effect.sync(() => { - state = input.action === "close" ? "closed" : "open"; - }), - listChangeRequests: () => { - hostCalls += 1; - return Effect.succeed({ items: [], truncated: false, continues: false }); - }, - }), - ], - }); - - yield* service.refreshAfterTurn(reference.projectId); - yield* service.list({ state: "open" }); - assert.strictEqual((yield* service.summary(reference)).state, "open"); - for (const action of ["close", "reopen"] as const) { - const refreshed = yield* service.subscribeRefreshes.pipe( - Stream.drop(1), - Stream.take(1), - Stream.mapEffect(() => - Effect.gen(function* () { - yield* service.list({ state: "open" }); - return yield* service.summary(reference); + Effect.gen(function* () { + let hostCalls = 0; + let state: "open" | "closed" = "open"; + const reference = { projectId: "p1" as ProjectId, repository: "acme/web", number: 1 }; + const service = yield* makeService({ + projects: [project({ id: "p1", title: "web", workspaceRoot: "/a", repository: "acme/web" })], + providers: [ + fakePullRequestProvider(SourceControlProviderKind.make("github"), { + getChangeRequestSummary: () => + Effect.succeed({ ...changeRequest(1, "2026-09-16T00:00:00.000Z"), state }), + runAction: (input) => + Effect.sync(() => { + state = input.action === "close" ? "closed" : "open"; }), - ), - Stream.runHead, - Effect.forkChild({ startImmediately: true }), - ); - yield* service.runAction({ ...reference, action }); - assert.strictEqual( - Option.getOrThrow(yield* Fiber.join(refreshed)).state, - action === "close" ? "closed" : "open", - ); - } - assert.strictEqual(hostCalls, 3); - }), - ), + listChangeRequests: () => { + hostCalls += 1; + return Effect.succeed({ items: [], truncated: false, continues: false }); + }, + }), + ], + }); + + yield* service.refreshAfterTurn(reference.projectId); + yield* service.list({ state: "open" }); + assert.strictEqual((yield* service.summary(reference)).state, "open"); + for (const action of ["close", "reopen"] as const) { + const refreshed = yield* service.subscribeRefreshes.pipe( + Stream.drop(1), + Stream.take(1), + Stream.mapEffect(() => + Effect.gen(function* () { + yield* service.list({ state: "open" }); + return yield* service.summary(reference); + }), + ), + Stream.runHead, + Effect.forkChild({ startImmediately: true }), + ); + yield* service.runAction({ ...reference, action }); + assert.strictEqual( + Option.getOrThrow(yield* Fiber.join(refreshed)).state, + action === "close" ? "closed" : "open", + ); + } + assert.strictEqual(hostCalls, 3); + }), ); it.effect("reads a host's repositories in one search, and files the rows back under each", () => @@ -2937,7 +3025,13 @@ it.effect("carries a slice on without sending the rows it already sent", () => cursors: { "github.com acme/web": "2026-07-02T00:00:00Z|1|3" }, }); - assert.deepStrictEqual(cursors, [{ updatedBefore: "2026-07-02T00:00:00Z", delivered: 1 }]); + assert.deepStrictEqual(cursors, [ + { + updatedBefore: "2026-07-02T00:00:00Z", + delivered: 1, + seenAt: [{ repository: "acme/web", number: 3 }], + }, + ]); assert.deepStrictEqual( result.entries.map((entry) => entry.number), [4], @@ -3163,6 +3257,40 @@ it.effect("reads the fresh diff when detail or summary discovers a changed revis }), ); +it.effect("keeps a withheld file's read out of the page's cached diff", () => + Effect.gen(function* () { + const provider = fakePullRequestProvider(SourceControlProviderKind.make("github"), { + getDiff: (input) => + Effect.succeed({ + patch: input.path === undefined ? "page" : `file ${input.path}`, + truncated: input.path === undefined, + nextCursor: null, + }), + }); + const service = yield* makeService({ + projects: [project({ id: "p1", title: "web", workspaceRoot: "/a", repository: "acme/web" })], + providers: [provider], + }); + const ref = { projectId: "p1" as ProjectId, repository: "acme/web", number: 10 }; + assert.strictEqual((yield* service.diff(ref)).patch, "page"); + assert.strictEqual( + (yield* service.diff({ ...ref, path: "sdk.patch" })).patch, + "file sdk.patch", + ); + assert.strictEqual((yield* service.diff(ref)).patch, "page"); + + // A host that would answer with its whole diff must not be cached as one file's. + const wholeDiffOnly = yield* makeService({ + projects: [project({ id: "p1", title: "web", workspaceRoot: "/a", repository: "acme/web" })], + providers: [ + { ...provider, capabilities: { ...provider.capabilities, withheldFileDiff: false } }, + ], + }); + const error = yield* Effect.flip(wholeDiffOnly.diff({ ...ref, path: "sdk.patch" })); + assert.strictEqual(error._tag, "PullRequestOperationError"); + }), +); + it.effect("keeps routed reads separate when the GitHub account changes", () => Effect.gen(function* () { for (const operation of ["summary", "detail", "checks", "diff", "filesViewed"] as const) { @@ -3372,6 +3500,7 @@ it.effect( Effect.succeed({ ...changeRequest(1, "2026-07-02T00:00:00Z"), isDraft: true, + rebaseRequired: false, reviewDecision: "approved", checksState: "passing", }), @@ -3381,6 +3510,7 @@ it.effect( deletions: 3, changedFiles: 5, mergeability: "conflicting", + rebaseRequired: true, }), }), ], @@ -3388,6 +3518,8 @@ it.effect( yield* service.summary(reference); const detail = yield* service.detail(reference); const summary = yield* service.summary(reference); + assert.strictEqual(detail.rebaseRequired, true); + assert.strictEqual(summary.rebaseRequired, true); assert.strictEqual(summary.isDraft, false); assert.deepStrictEqual(summary.author, detail.author); assert.strictEqual(summary.additions, 14); @@ -3399,6 +3531,27 @@ it.effect( }), ); +it.effect("preserves merge queue removal details from the provider", () => + Effect.gen(function* () { + const removal = { id: "RMQE_1", removedAt: "2026-07-02T00:00:00Z", reason: "CI failed" }; + const service = yield* makeService({ + projects: [project({ id: "p1", title: "web", workspaceRoot: "/a", repository: "acme/web" })], + providers: [ + fakePullRequestProvider(SourceControlProviderKind.make("github"), { + getChangeRequest: () => + Effect.succeed({ ...hostedChangeRequest("body"), mergeQueueRemoval: removal }), + }), + ], + }); + const detail = yield* service.detail({ + projectId: "p1" as ProjectId, + repository: "acme/web", + number: 1, + }); + assert.deepStrictEqual(detail.mergeQueueRemoval, removal); + }), +); + it.effect("notifies linked threads when detail changes draft status in either direction", () => Effect.gen(function* () { let isDraft = true; @@ -3467,19 +3620,31 @@ it.effect("deduplicates routed PR reports without dropping draft changes", () => }), ); -it.effect( - "reuses fresh detail checks without extending their refresh interval on detail cache hits", - () => +it.effect.each(["github", "azure-devops"])( + "reuses fresh %s detail checks without extending their refresh interval on detail cache hits", + (provider) => Effect.gen(function* () { let reads = 0; - const reference = { projectId: "p1" as ProjectId, repository: "acme/web", number: 1 }; + const repository = provider === "azure-devops" ? "web" : "acme/web"; + const reference = { projectId: "p1" as ProjectId, repository, number: 1 }; const service = yield* makeService({ projects: [ - project({ id: "p1", title: "web", workspaceRoot: "/a", repository: "acme/web" }), + project({ + id: "p1", + title: "web", + workspaceRoot: "/a", + repository, + provider, + ...(provider === "azure-devops" ? { host: "dev.azure.com" } : {}), + }), ], providers: [ - fakePullRequestProvider(SourceControlProviderKind.make("github"), { - getChangeRequest: () => Effect.succeed(hostedChangeRequest("Description")), + fakePullRequestProvider(SourceControlProviderKind.make(provider), { + getChangeRequest: () => + Effect.succeed({ + ...hostedChangeRequest("Description"), + checks: [{ name: "build", status: "success", description: null, url: null }], + }), getChangeRequestChecks: () => Effect.sync(() => { reads += 1; @@ -3509,6 +3674,37 @@ it.effect( }), ); +it.effect("asks Azure for checks when its detail came back with none", () => + Effect.gen(function* () { + let reads = 0; + const reference = { projectId: "p1" as ProjectId, repository: "web", number: 1 }; + const service = yield* makeService({ + projects: [ + project({ + id: "p1", + title: "web", + workspaceRoot: "/a", + repository: "web", + provider: "azure-devops", + }), + ], + providers: [ + fakePullRequestProvider(SourceControlProviderKind.make("azure-devops"), { + getChangeRequest: () => Effect.succeed(hostedChangeRequest("Description")), + getChangeRequestChecks: () => + Effect.sync(() => { + reads += 1; + return { state: "open" as const, checks: [] }; + }), + }), + ], + }); + yield* service.detail(reference); + yield* service.checks(reference); + expect(reads).toBe(1); + }), +); + it.effect("preserves check observation time on cache hits and advances it on refresh", () => Effect.gen(function* () { let reads = 0; @@ -4275,6 +4471,43 @@ it.effect("keeps viewed files itself for a host that keeps none of its own", () }), ); +it.effect("records shown revisions until the reader ticks the pushed version", () => + Effect.gen(function* () { + const asked: Array> = []; + const revisions = new Map([ + ["src/a.ts", "pushed-a"], + ["src/b.ts", "current-b"], + ]); + const service = yield* environmentViewedService(revisions, asked); + + yield* service.setFilesViewed({ + ...GITLAB_REFERENCE, + files: [ + { path: "src/a.ts", viewed: true, revision: "shown-a" }, + { path: "src/b.ts", viewed: true }, + ], + }); + assert.deepStrictEqual(asked, [["src/b.ts"]]); + yield* service.invalidate({ reference: GITLAB_REFERENCE }); + assert.deepStrictEqual((yield* service.filesViewed(GITLAB_REFERENCE)).files, [ + { path: "src/a.ts", state: "dismissed" }, + { path: "src/b.ts", state: "viewed" }, + ]); + + const reads = asked.length; + revisions.set("src/a.ts", "pushed-a-again"); + yield* service.setFilesViewed({ + ...GITLAB_REFERENCE, + files: [{ path: "src/a.ts", viewed: true, revision: "pushed-a-again" }], + }); + assert.strictEqual(asked.length, reads); + assert.deepStrictEqual((yield* service.filesViewed(GITLAB_REFERENCE)).files, [ + { path: "src/a.ts", state: "viewed" }, + { path: "src/b.ts", state: "viewed" }, + ]); + }), +); + it.effect("keeps a deleted file cleared, which the head has no version of at all", () => Effect.gen(function* () { const service = yield* environmentViewedService(new Map(), []); @@ -4323,6 +4556,26 @@ it.effect("leaves a mark alone when the host could not say what the head has of }), ); +it.effect("reports a file pushed to after the diff the reader ticked was drawn as changed", () => + Effect.gen(function* () { + const revisions = new Map([["src/a.ts", "shown-a"]]); + const service = yield* environmentViewedService(revisions, []); + + yield* service.filesViewed(GITLAB_REFERENCE); + yield* TestClock.adjust("61 seconds"); + revisions.set("src/a.ts", "pushed-a"); + yield* service.setFilesViewed({ + ...GITLAB_REFERENCE, + files: [{ path: "src/a.ts", viewed: true, revision: "shown-a" }], + }); + yield* service.invalidate({ reference: GITLAB_REFERENCE }); + + assert.deepStrictEqual((yield* service.filesViewed(GITLAB_REFERENCE)).files, [ + { path: "src/a.ts", state: "dismissed" }, + ]); + }), +); + it.effect("forgets what the head had of a marked file once a mutation moves the head", () => Effect.gen(function* () { const revisions = new Map([["src/a.ts", "blob-a"]]); diff --git a/apps/server/src/pullRequest/PullRequestService.ts b/apps/server/src/pullRequest/PullRequestService.ts index 86c18cc8e9e9..f8263a6a3fc0 100644 --- a/apps/server/src/pullRequest/PullRequestService.ts +++ b/apps/server/src/pullRequest/PullRequestService.ts @@ -229,8 +229,14 @@ export class PullRequestService extends Context.Service< ) => Effect.Effect; readonly summary: ( input: PullRequestRef, - options?: { readonly recoverTransientFailure?: boolean }, + options?: { + readonly recoverTransientFailure?: boolean; + /** Registration preflights must reach the provider despite a successful cached read. */ + readonly bypassCache?: boolean; + }, ) => Effect.Effect; + /** Whether the reference's host batches summary reads; false when it cannot be resolved. */ + readonly batchesReads: (input: PullRequestRef) => Effect.Effect; readonly stack: ( input: PullRequestRef, options?: { readonly includeDetails?: boolean }, @@ -253,7 +259,13 @@ export class PullRequestService extends Context.Service< readonly reportState: ( input: PullRequestReportStateInput, ) => Effect.Effect; - readonly refreshAfterTurn: (projectId: ProjectId) => Effect.Effect; + readonly subscribeLinkedThreadRefreshes: Stream.Stream; + /** Wakes linked-thread readers only; other pull request reads don't depend on thread links. */ + readonly notifyLinkedThreadsChanged: Effect.Effect; + readonly refreshAfterTurn: ( + projectId: ProjectId, + options?: { readonly pushed?: boolean }, + ) => Effect.Effect; /** Project and remote lookups inside `effect` read once per filter, e.g. across one sync sweep. */ readonly withSharedProjectReads: ( effect: Effect.Effect, @@ -593,6 +605,7 @@ function withRateLimitBackoff( ...(api.readsPublicHostWithoutCheckout === undefined ? {} : { readsPublicHostWithoutCheckout: api.readsPublicHostWithoutCheckout }), + ...(api.batchesReads === undefined ? {} : { batchesReads: api.batchesReads }), getViewer: options?.viewerAllowsPause === true ? interactive("getViewer", api.getViewer) @@ -800,6 +813,7 @@ export const make = Effect.gen(function* () { revision: 0, listings: true, }); + const linkedThreadRefreshes = yield* SubscriptionRef.make(0); const registry = yield* PullRequestProviderRegistry.PullRequestProviderRegistry; const requirementOf: UnavailableRequirementOf = (kind) => registry.get(kind)?.unavailableRequirement; @@ -1383,6 +1397,9 @@ export const make = Effect.gen(function* () { state: input.item.state, isDraft: input.item.isDraft, mergeability: input.item.mergeability, + ...(input.item.rebaseRequired === undefined + ? {} + : { rebaseRequired: input.item.rebaseRequired }), additions: input.item.additions, deletions: input.item.deletions, createdAt: input.item.createdAt, @@ -1634,7 +1651,18 @@ export const make = Effect.gen(function* () { filters: input.filters, ...(cursor === undefined ? {} - : { cursor: { updatedBefore: cursor.updatedBefore, delivered: cursor.delivered } }), + : { + cursor: { + updatedBefore: cursor.updatedBefore, + delivered: cursor.delivered, + seenAt: chunk.flatMap((project) => + (cursorOf(project)?.seenAt ?? []).map((number) => ({ + repository: project.repository, + number, + })), + ), + }, + }), }).pipe( observeRead, Effect.flatMap(({ value: page, observedAt }) => { @@ -1865,6 +1893,9 @@ export const make = Effect.gen(function* () { ...(changeRequest.mergeability === undefined ? {} : { mergeability: changeRequest.mergeability }), + ...(changeRequest.rebaseRequired === undefined + ? {} + : { rebaseRequired: changeRequest.rebaseRequired }), ...(changeRequest.stack === undefined ? {} : { stack: changeRequest.stack }), ...(changeRequest.headSha?.trim() ? { headSha: changeRequest.headSha.trim() } : {}), ...(changeRequest.autoMergeEnabled === undefined @@ -1955,10 +1986,7 @@ export const make = Effect.gen(function* () { provider: project.api.kind, capabilities: { ...capabilities, - dependencies: { - branchRelationships: true, - nativeMembership: false, - }, + dependencies: { branchRelationships: true, nativeMembership: false }, }, projectId: project.project.id, projectTitle: project.project.title, @@ -1972,6 +2000,9 @@ export const make = Effect.gen(function* () { state: changeRequest.state, isDraft: changeRequest.isDraft, mergeability: changeRequest.mergeability, + ...(changeRequest.rebaseRequired === undefined + ? {} + : { rebaseRequired: changeRequest.rebaseRequired }), additions: changeRequest.additions, deletions: changeRequest.deletions, changedFiles: changeRequest.changedFiles, @@ -2008,6 +2039,9 @@ export const make = Effect.gen(function* () { ...(changeRequest.workflowApprovalsRequired === undefined ? {} : { workflowApprovalsRequired: changeRequest.workflowApprovalsRequired }), + ...(changeRequest.mergeQueueRemoval === undefined + ? {} + : { mergeQueueRemoval: changeRequest.mergeQueueRemoval }), }), ), ), @@ -2315,23 +2349,31 @@ export const make = Effect.gen(function* () { const diffUncached: PullRequestService["Service"]["diff"] = (input) => requireProject(input).pipe( Effect.flatMap((project) => - project.api.capabilities.diff - ? project.api - .getDiff({ - cwd: project.project.workspaceRoot, - repository: project.repository, - host: project.host, - number: input.number, - ...(input.cursor === undefined ? {} : { cursor: input.cursor }), - ...(input.commit === undefined ? {} : { commit: input.commit }), - }) - .pipe(Effect.mapError(toPullRequestError("diff"))) - : Effect.fail( + input.path !== undefined && project.api.capabilities.withheldFileDiff !== true + ? Effect.fail( new PullRequestOperationError({ operation: "diff", - detail: "This host cannot provide a diff for a change request.", + detail: "This host cannot read one file of a change request's diff.", }), - ), + ) + : project.api.capabilities.diff + ? project.api + .getDiff({ + cwd: project.project.workspaceRoot, + repository: project.repository, + host: project.host, + number: input.number, + ...(input.path === undefined ? {} : { path: input.path }), + ...(input.cursor === undefined ? {} : { cursor: input.cursor }), + ...(input.commit === undefined ? {} : { commit: input.commit }), + }) + .pipe(Effect.mapError(toPullRequestError("diff"))) + : Effect.fail( + new PullRequestOperationError({ + operation: "diff", + detail: "This host cannot provide a diff for a change request.", + }), + ), ), ); @@ -2390,7 +2432,14 @@ export const make = Effect.gen(function* () { // Deliberately not `invalidatedByMutation`: ticking a file off says nothing about the // change request, and dropping a 300-file diff on every checkbox is the whole cost of // the feature. - Effect.ensuring(Effect.sync(() => bumpFilesViewedEpoch(ref))), + Effect.ensuring( + Effect.sync(() => { + bumpFilesViewedEpoch(ref); + if (input.files.some((file) => file.viewed && file.revision !== undefined)) { + bumpEpoch(fileRevisionEpochs, ref); + } + }), + ), ), ), ); @@ -3449,6 +3498,7 @@ export const make = Effect.gen(function* () { const filesViewedEpoch = (ref: PullRequestRef) => filesViewedEpochs.get(refScope(ref)) ?? 0; const bumpFilesViewedEpoch = (ref: PullRequestRef) => bumpEpoch(filesViewedEpochs, ref); + const fileRevisionEpochs = new Map(); let everyFileRevisionEpoch = 0; // Built after the epochs because it reads two of them: taken as an argument any higher, // `refEpoch` would be read while its `const` was still in its dead zone and this would throw. @@ -3469,7 +3519,8 @@ export const make = Effect.gen(function* () { toPullRequestError, runFork, refEpoch, - fileRevisionsEpoch: () => everyFileRevisionEpoch, + fileRevisionsEpoch: (ref) => + `${everyFileRevisionEpoch}:${fileRevisionEpochs.get(refScope(ref)) ?? 0}`, }); const filtersOfKey = ( @@ -3537,19 +3588,33 @@ export const make = Effect.gen(function* () { const summary: PullRequestService["Service"]["summary"] = (input, options) => { const key = refCacheKey(input); - const cached = persistedRead(input, "summary", summaryCodec, summaryUncached(input)); + const read = + options?.bypassCache === true + ? summaryUncached(input) + : persistedRead(input, "summary", summaryCodec, summaryUncached(input)); const held = lastGoodSummary.peek(key); - return held !== undefined && + return options?.bypassCache !== true && + held !== undefined && input.allowStale !== false && (options?.recoverTransientFailure !== false || held.state === "merged") ? Effect.succeed(held) - : cached.pipe( + : read.pipe( Effect.tap((value) => shouldReplaceHeldSummary(key, value) ? lastGoodSummary.record(key, value) : Effect.void, ), ); }; + const batchesReads: PullRequestService["Service"]["batchesReads"] = Effect.fn( + "PullRequestService.batchesReads", + )( + function* (input) { + const project = yield* requireProject(input); + return project.api.batchesReads === true; + }, + Effect.orElseSucceed(() => false), + ); + const stack: PullRequestService["Service"]["stack"] = (input, options) => persistedRead( input, @@ -3710,9 +3775,11 @@ export const make = Effect.gen(function* () { return detailUncached(refOfCacheKey(key)).pipe( Effect.tap( Effect.fn("PullRequestService.recordDetailStats")(function* (value: PullRequestDetail) { - // GitHub detail includes the same checks; the checks TTL counts from their observation. + // GitHub and Azure detail include the same checks; the checks TTL counts from their observation. + // Azure detail answers a failed policy read with no checks, which must not be cached. if ( - value.provider === "github" && + (value.provider === "github" || + (value.provider === "azure-devops" && value.checks.length > 0)) && value.observedAt !== undefined && !(yield* Cache.has(checksCache, key)) ) { @@ -3762,6 +3829,7 @@ export const make = Effect.gen(function* () { deletions: detail.deletions, changedFiles: detail.changedFiles, mergeability: detail.mergeability, + ...(detail.rebaseRequired === undefined ? {} : { rebaseRequired: detail.rebaseRequired }), headBranch: detail.headBranch, baseBranch: detail.baseBranch, closedAt: detail.closedAt, @@ -3973,16 +4041,26 @@ export const make = Effect.gen(function* () { const diffCache = yield* Cache.makeWith( (key: string) => { - const [reference, cursor, commit] = JSON.parse(key) as [string, string | null, string | null]; + const [reference, cursor, commit, , path] = JSON.parse(key) as [ + string, + string | null, + string | null, + string | null, + string | null, + ]; return diffUncached({ ...refOfCacheKey(reference), + ...(path === null ? {} : { path }), ...(cursor === null ? {} : { cursor }), ...(commit === null ? {} : { commit }), } as PullRequestDiffInput).pipe( observeRead, Effect.tap(({ value, observedAt }) => { // Keep the loader's epoch and read time if a refresh occurs during the provider read. - diffFiles.record(JSON.stringify([reference, cursor, commit]), value, observedAt); + // One withheld file is not the page's file list. + if (path === null) { + diffFiles.record(JSON.stringify([reference, cursor, commit]), value, observedAt); + } return Effect.void; }), ); @@ -4033,6 +4111,7 @@ export const make = Effect.gen(function* () { input.commit === undefined ? (lastGoodSummary.peek(refCacheKey(input))?.updatedAt ?? null) : null, + input.path ?? null, input.headSha ?? null, ]); const evictDiffs = Cache.entries(diffCache).pipe( @@ -4176,12 +4255,14 @@ export const make = Effect.gen(function* () { const refreshAfterTurn: PullRequestService["Service"]["refreshAfterTurn"] = Effect.fn( "PullRequestService.refreshAfterTurn", - )(function* (projectId) { + )(function* (projectId, options) { // Everything the provider holds, not just the project's repository: a turn can touch a // change request reached through another repository, and no epoch may move until those // reads are gone, or a reader in between keeps the old one under the new epoch. yield* forgetProviderReads(); - listingsEpoch = ++epochCounter; + const epoch = ++epochCounter; + // A push can open or move a PR; a plain turn waits for the listing TTL to spare the API quota. + if (options?.pushed === true) listingsEpoch = epoch; projectEpochs.delete(projectId); if (projectEpochs.size >= REF_EPOCH_CAPACITY) { const oldest = projectEpochs.keys().next().value; @@ -4190,11 +4271,16 @@ export const make = Effect.gen(function* () { projectEpochs.delete(oldest); } } - projectEpochs.set(projectId, listingsEpoch); + projectEpochs.set(projectId, epoch); yield* readCache.invalidate(`project:${projectId}`); yield* publishRefresh(); }); + const notifyLinkedThreadsChanged = SubscriptionRef.update( + linkedThreadRefreshes, + (revision) => revision + 1, + ); + const invalidatedByMutation = ( method: (input: I) => Effect.Effect, @@ -4384,6 +4470,7 @@ export const make = Effect.gen(function* () { return yield* listStats({ ...input, refs: refs.flatMap(Option.toArray) }); }), summary: credentialCached(summary), + batchesReads, stack: credentialCached(stack), subscribeMerges: PubSub.subscribe(mergedPullRequests).pipe( Effect.map((subscription) => Stream.fromSubscription(subscription)), @@ -4395,6 +4482,10 @@ export const make = Effect.gen(function* () { subscribeRefreshes: SubscriptionRef.changes(pullRequestRefreshes).pipe( Stream.filter((event) => event.revision > 0), ), + subscribeLinkedThreadRefreshes: SubscriptionRef.changes(linkedThreadRefreshes).pipe( + Stream.filter((revision) => revision > 0), + ), + notifyLinkedThreadsChanged, refreshAfterTurn, detail: credentialCached(detail), checks: credentialCached((input) => Cache.get(checksCache, refCacheKey(input))), diff --git a/apps/server/src/pullRequest/PullRequestWatchService.test.ts b/apps/server/src/pullRequest/PullRequestWatchService.test.ts new file mode 100644 index 000000000000..2c4f4d92e469 --- /dev/null +++ b/apps/server/src/pullRequest/PullRequestWatchService.test.ts @@ -0,0 +1,294 @@ +import { + ProjectId, + ProviderInstanceId, + ThreadId, + PullRequestOperationError, + PullRequestUnavailableError, + type PullRequestSummary, + type OrchestrationV2ServerCommand as OrchestrationCommand, + type OrchestrationProjectShell, + type ThreadPullRequestLink, + SourceControlProviderKind, +} from "@t3tools/contracts"; +import { describe, expect, it } from "@effect/vitest"; +import * as SecureRandom from "effect/crypto/SecureRandom"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Ref from "effect/Ref"; + +import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; +import { + type PullRequestTestThread, + v2PullRequestThread, +} from "../orchestration-v2/testkit/pullRequestFixtures.ts"; +import * as ProjectService from "../project/ProjectService.ts"; +import * as PullRequestService from "./PullRequestService.ts"; +import * as PullRequestWatchService from "./PullRequestWatchService.ts"; +import type { PullRequestTarget } from "./pullRequestTarget.ts"; + +const PROJECT_ID = ProjectId.make("project-1"); +const THREAD_ID = ThreadId.make("thread-1"); +const testCrypto = SecureRandom.make((size) => new Uint8Array(size).fill(7)); + +function makeProject( + repositoryIdentity: OrchestrationProjectShell["repositoryIdentity"] = { + canonicalKey: "github.com/t3tools/t3code", + locator: { + source: "git-remote", + remoteName: "origin", + remoteUrl: "git@github.com:T3Tools/T3Code.git", + }, + provider: SourceControlProviderKind.make("github"), + displayName: "T3Tools/T3Code", + owner: "T3Tools", + name: "T3Code", + }, +): OrchestrationProjectShell { + return { + id: PROJECT_ID, + title: "Project", + workspaceRoot: "/workspace/project", + defaultModelSelection: null, + scripts: [], + repositoryIdentity, + createdAt: "2026-08-01T00:00:00.000Z", + updatedAt: "2026-08-01T00:00:00.000Z", + }; +} + +function makeThread(pullRequests: ReadonlyArray): PullRequestTestThread { + return { + id: THREAD_ID, + projectId: PROJECT_ID, + title: "Thread", + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5" }, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + pullRequests, + createdAt: "2026-08-01T00:00:00.000Z", + updatedAt: "2026-08-20T00:00:00.000Z", + archivedAt: null, + settledOverride: null, + settledAt: null, + latestUserMessageAt: "2026-08-20T00:00:00.000Z", + }; +} + +function makeLink( + number: number, + overrides: Partial & { + readonly headBranch?: string; + readonly baseBranch?: string; + } = {}, +): ThreadPullRequestLink { + const { headBranch, baseBranch, ...rest } = overrides; + return { + host: "github.com", + repository: "t3tools/t3code", + number, + url: `https://github.com/t3tools/t3code/pull/${number}`, + source: "manual", + linkedAt: "2026-08-10T00:00:00.000Z", + snapshot: + headBranch === undefined + ? null + : { + state: "open", + title: `PR ${number}`, + headBranch, + baseBranch: baseBranch ?? "main", + isDraft: false, + updatedAt: null, + syncedAt: "2026-08-27T00:00:00.000Z", + }, + stack: null, + ...rest, + }; +} + +interface HarnessOptions { + readonly thread?: PullRequestTestThread | null; + readonly project?: OrchestrationProjectShell | null; + readonly readFailure?: PullRequestService.PullRequestError; + readonly readState?: PullRequestSummary["state"]; +} + +const makeHarness = Effect.fn("makePullRequestWatchHarness")(function* ( + options: HarnessOptions = {}, +) { + const commands = yield* Ref.make>([]); + const thread = options.thread === undefined ? makeThread([]) : options.thread; + const project = options.project === undefined ? makeProject() : options.project; + const dependencies = Layer.mergeAll( + Layer.mock(ProjectService.ProjectService)({ + getShell: () => Effect.succeed(Option.fromNullishOr(project)), + }), + Layer.mock(Orchestrator.OrchestratorV2)({ + getThreadShell: (id) => + Effect.succeed(id === thread?.id ? v2PullRequestThread(thread) : null), + dispatch: (command) => + Ref.update(commands, (recorded) => [...recorded, command]).pipe( + Effect.as({ sequence: 1, storedEvents: [] }), + ), + }), + Layer.mock(PullRequestService.PullRequestService)({ + summary: (input) => + options.readFailure !== undefined + ? Effect.fail(options.readFailure) + : Effect.succeed({ + ...input, + provider: SourceControlProviderKind.make("github"), + title: "Pull request", + url: `https://github.com/${input.repository}/pull/${input.number}`, + state: options.readState ?? "open", + headBranch: "feature", + baseBranch: "main", + updatedAt: "2026-08-20T00:00:00.000Z", + }), + }), + Layer.succeed(SecureRandom.SecureRandom, testCrypto), + ); + const service = yield* PullRequestWatchService.PullRequestWatchService.pipe( + Effect.provide(PullRequestWatchService.layer.pipe(Layer.provide(dependencies))), + ); + return { + commands, + setWatching: (watching: boolean, target: PullRequestTarget) => + service.setWatching({ threadId: THREAD_ID, target, watching }), + }; +}); + +describe("PullRequestWatchService", () => { + it.effect("links an unlinked watch even while the host has a transient read failure", () => + Effect.gen(function* () { + for (const options of [ + {}, + { + readFailure: new PullRequestOperationError({ + operation: "summary", + detail: "The host is temporarily unavailable.", + }), + }, + ]) { + const harness = yield* makeHarness(options); + const result = yield* harness.setWatching(true, { + url: "https://github.com/t3tools/t3code/pull/9", + }); + expect(result).toMatchObject({ number: 9, wasWatching: false }); + expect(yield* Ref.get(harness.commands)).toMatchObject([ + { + type: "thread.pull-request.watch", + number: 9, + watching: true, + link: { url: "https://github.com/t3tools/t3code/pull/9", source: "agent" }, + }, + ]); + } + }), + ); + + it.effect("reports a missing hosting CLI before saving a watch and still allows unwatch", () => + Effect.gen(function* () { + const missing = new PullRequestUnavailableError({ + reason: "cli-missing", + provider: SourceControlProviderKind.make("gitlab"), + requirement: "Install glab on the T3 server's PATH.", + }); + const harness = yield* makeHarness({ readFailure: missing }); + const target = { url: "https://gitlab.example.com/team/project/-/merge_requests/42" }; + const error = yield* harness.setWatching(true, target).pipe(Effect.flip); + expect(error).toEqual(missing); + expect(yield* Ref.get(harness.commands)).toEqual([]); + + yield* harness.setWatching(false, target); + expect(yield* Ref.get(harness.commands)).toMatchObject([ + { type: "thread.pull-request.watch", watching: false }, + ]); + }), + ); + + it.effect("rejects merged PRs and keeps an active watch usable without hosting access", () => + Effect.gen(function* () { + const freshMerged = yield* makeHarness({ readState: "merged" }); + const freshError = yield* freshMerged + .setWatching(true, { repository: "t3tools/t3code", number: 1 }) + .pipe(Effect.flip); + expect(freshError).toMatchObject({ _tag: "PullRequestNotOpenError", state: "merged" }); + expect(yield* Ref.get(freshMerged.commands)).toEqual([]); + + const watch = { + startedAt: "2026-08-20T00:00:00.000Z", + headSha: null, + failedChecks: [], + passed: false, + passedChecks: [], + remarksThrough: "2026-08-20T00:00:00.000Z", + remarkIds: [], + conflicting: false, + wakes: 0, + }; + const merged = makeLink(1, { headBranch: "done" }); + const harness = yield* makeHarness({ + readFailure: new PullRequestUnavailableError({ reason: "cli-missing" }), + thread: makeThread([ + { ...merged, snapshot: merged.snapshot && { ...merged.snapshot, state: "merged" } }, + makeLink(2, { headBranch: "idle" }), + makeLink(3, { headBranch: "watched", watch }), + ]), + }); + const error = yield* harness + .setWatching(true, { repository: "t3tools/t3code", number: 1 }) + .pipe(Effect.flip); + expect(error).toMatchObject({ _tag: "PullRequestNotOpenError", state: "merged" }); + expect( + yield* harness.setWatching(true, { repository: "t3tools/t3code", number: 3 }), + ).toMatchObject({ watching: true, wasWatching: true }); + expect( + yield* harness.setWatching(false, { repository: "t3tools/t3code", number: 3 }), + ).toMatchObject({ wasWatching: true }); + expect(yield* Ref.get(harness.commands)).toMatchObject([ + { type: "thread.pull-request.watch", number: 3, watching: true }, + { type: "thread.pull-request.watch", number: 3, watching: false }, + ]); + }), + ); + + it.effect("watches a pull request saved as closed, since it may have reopened", () => + Effect.gen(function* () { + const closed = makeLink(1, { headBranch: "closed" }); + const harness = yield* makeHarness({ + readState: "closed", + thread: makeThread([ + { ...closed, snapshot: closed.snapshot && { ...closed.snapshot, state: "closed" } }, + ]), + }); + yield* harness.setWatching(true, { repository: "t3tools/t3code", number: 1 }); + expect(yield* Ref.get(harness.commands)).toMatchObject([ + { type: "thread.pull-request.watch", number: 1, watching: true }, + ]); + }), + ); + + it.effect("refuses a watch from a subagent thread, whose parent owns the pull request", () => + Effect.gen(function* () { + const harness = yield* makeHarness({ + thread: { + ...makeThread([makeLink(1, { headBranch: "feature" })]), + lineage: { + rootThreadId: ThreadId.make("parent"), + parentThreadId: ThreadId.make("parent"), + relationshipToParent: "subagent", + }, + }, + }); + const error = yield* harness + .setWatching(true, { repository: "t3tools/t3code", number: 1 }) + .pipe(Effect.flip); + expect(error).toMatchObject({ _tag: "PullRequestWatchFromSubagentError" }); + expect(yield* Ref.get(harness.commands)).toEqual([]); + }), + ); +}); diff --git a/apps/server/src/pullRequest/PullRequestWatchService.ts b/apps/server/src/pullRequest/PullRequestWatchService.ts new file mode 100644 index 000000000000..6ed736075c01 --- /dev/null +++ b/apps/server/src/pullRequest/PullRequestWatchService.ts @@ -0,0 +1,180 @@ +import { + CommandId, + ThreadId, + type OrchestrationV2ThreadShell, + type PullRequestUnavailableError, +} from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; +import * as Context from "effect/Context"; +import * as SecureRandom from "effect/crypto/SecureRandom"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; + +import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; +import * as ProjectService from "../project/ProjectService.ts"; +import * as PullRequestService from "./PullRequestService.ts"; +import { + resolveTarget, + type PullRequestTarget, + type PullRequestTargetError, + type ResolvedTarget, +} from "./pullRequestTarget.ts"; +import { + threadPullRequestKeysEqual, + threadPullRequestsOf, +} from "../sourceControl/threadPullRequests.ts"; + +export class PullRequestWatchFailedError extends Schema.TaggedError()( + "PullRequestWatchFailedError", + { cause: Schema.Defect() }, +) { + override get message(): string { + return "Could not change whether the pull request is watched."; + } +} + +export class PullRequestNotOpenError extends Schema.TaggedError()( + "PullRequestNotOpenError", + { state: Schema.String }, +) { + override get message(): string { + return `The pull request is ${this.state}, so there is nothing to watch.`; + } +} + +export class PullRequestWatchFromSubagentError extends Schema.TaggedError()( + "PullRequestWatchFromSubagentError", + {}, +) { + override get message(): string { + return "This thread is a subagent, so it cannot watch pull requests. Its parent thread owns the pull request: finish your task and report back instead."; + } +} + +export class PullRequestWatchThreadNotFoundError extends Schema.TaggedError()( + "PullRequestWatchThreadNotFoundError", + { threadId: ThreadId }, +) { + override get message(): string { + return `Thread ${this.threadId} was not found.`; + } +} + +export class PullRequestWatchService extends Context.Service< + PullRequestWatchService, + { + readonly setWatching: (input: { + readonly threadId: ThreadId; + readonly target: PullRequestTarget; + readonly watching: boolean; + }) => Effect.Effect< + ResolvedTarget & { readonly watching: boolean; readonly wasWatching: boolean }, + | PullRequestWatchFailedError + | PullRequestNotOpenError + | PullRequestWatchFromSubagentError + | PullRequestWatchThreadNotFoundError + | PullRequestTargetError + | PullRequestUnavailableError + >; + } +>()("t3/pullRequest/PullRequestWatchService") {} + +const make = Effect.gen(function* () { + const engine = yield* Orchestrator.OrchestratorV2; + const projects = yield* ProjectService.ProjectService; + const pullRequests = yield* PullRequestService.PullRequestService; + const random = yield* SecureRandom.SecureRandom; + + const requireThread = Effect.fn("PullRequestWatchService.requireThread")(function* ( + threadId: ThreadId, + ) { + const thread = yield* engine + .getThreadShell(threadId) + .pipe(Effect.mapError((cause) => new PullRequestWatchFailedError({ cause }))); + if (thread === null || thread.deletedAt !== null) { + return yield* new PullRequestWatchThreadNotFoundError({ threadId }); + } + return thread; + }); + + // Keep the registration span stable across the service extraction. + const setWatching = Effect.fn("PullRequestsToolkit.setWatching")(function* (input: { + readonly threadId: ThreadId; + readonly target: PullRequestTarget; + readonly watching: boolean; + }) { + const { watching } = input; + const thread = yield* requireThread(input.threadId); + const project = yield* projects.getShell(thread.projectId).pipe( + Effect.map(Option.getOrUndefined), + Effect.mapError((cause) => new PullRequestWatchFailedError({ cause })), + ); + const target = yield* resolveTarget(input.target, project); + const watchedLink = (shell: OrchestrationV2ThreadShell) => + threadPullRequestsOf(shell).find( + (link) => link.source !== "stack-dismissed" && threadPullRequestKeysEqual(link, target), + ); + if (watching && thread.lineage.relationshipToParent === "subagent") { + return yield* new PullRequestWatchFromSubagentError(); + } + const before = watchedLink(thread); + if (watching && before?.snapshot?.state === "merged") { + return yield* new PullRequestNotOpenError({ state: "merged" }); + } + if (watching && before?.watch === undefined) { + yield* pullRequests + .summary( + { + projectId: thread.projectId, + host: target.host, + repository: target.repository, + number: target.number, + allowStale: false, + }, + { recoverTransientFailure: false, bypassCache: true }, + ) + .pipe( + Effect.flatMap((summary) => + summary.state === "merged" + ? Effect.fail(new PullRequestNotOpenError({ state: "merged" })) + : Effect.void, + ), + Effect.catchTags({ + // Host and transport errors can recover while the watch is active. + PullRequestOperationError: () => Effect.void, + }), + ); + } + const uuid = yield* random.uuidV4().pipe(Effect.orDie); + yield* engine + .dispatch({ + type: "thread.pull-request.watch", + commandId: CommandId.make(`server:mcp-pr-watch:${thread.id}:${uuid}`), + threadId: thread.id, + host: target.host, + repository: target.repository, + number: target.number, + watching, + ...(watching ? { link: { url: target.url, source: "agent" as const } } : {}), + }) + .pipe( + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.failCause(cause as Cause.Cause) + : Effect.fail(new PullRequestWatchFailedError({ cause })), + ), + ); + const after = yield* requireThread(thread.id); + return { + ...target, + watching: watchedLink(after)?.watch !== undefined, + wasWatching: before?.watch !== undefined, + }; + }); + + return PullRequestWatchService.of({ setWatching }); +}); + +export const layer = Layer.effect(PullRequestWatchService, make); diff --git a/apps/server/src/pullRequest/linkedThreads.test.ts b/apps/server/src/pullRequest/linkedThreads.test.ts index 8992a32ea10c..2dac78864158 100644 --- a/apps/server/src/pullRequest/linkedThreads.test.ts +++ b/apps/server/src/pullRequest/linkedThreads.test.ts @@ -1,10 +1,18 @@ +import { + ProjectId, + ThreadId, + type OrchestrationV2AppThread, + type OrchestrationV2DomainEvent, + type ThreadPullRequestLink, +} from "@t3tools/contracts"; import { assert, expect, it } from "@effect/vitest"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; -import { listLinkedPullRequestThreads } from "./linkedThreads.ts"; +import { listLinkedPullRequestThreads, makeLinkedThreadsChangeDetector } from "./linkedThreads.ts"; const encodePayload = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); @@ -209,3 +217,132 @@ it.effect( ); }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); + +it("signals linked-thread changes only when a linked answer could change", () => { + const now = DateTime.makeUnsafe("2026-11-12T00:00:00.000Z"); + const threadId = ThreadId.make("thread-a"); + const link: ThreadPullRequestLink = { + host: "github.com", + repository: "acme/web", + number: 7, + url: "https://github.com/acme/web/pull/7", + source: "agent", + linkedAt: "2026-11-12T00:00:00.000Z", + snapshot: null, + stack: null, + }; + const base = { + id: threadId, + projectId: ProjectId.make("project-a"), + title: "Thread A", + lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId }, + archivedAt: null, + settledOverride: null, + settledAt: null, + deletedAt: null, + updatedAt: now, + } as unknown as OrchestrationV2AppThread; + const event = ( + type: OrchestrationV2DomainEvent["type"], + payload: Partial, + ) => ({ type, threadId, payload: { ...base, ...payload } }) as OrchestrationV2DomainEvent; + const changed = makeLinkedThreadsChangeDetector(); + + expect( + [ + event("thread.metadata-updated", { title: "Unlinked rename" }), + event("thread.pull-request-synced", { pullRequests: [link] }), + event("thread.pull-request-synced", { + pullRequests: [{ ...link, watch: { since: "2026-11-12T00:00:00.000Z" } as never }], + }), + event("thread.visited", { pullRequests: [link] }), + event("thread.pinned", { pullRequests: [link], settledAt: now }), + { type: "thread.pull-request-link-updated", threadId, payload: link } as never, + event("thread.metadata-updated", { pullRequests: [link], title: "Renamed" }), + event("thread.archived", { pullRequests: [link], title: "Renamed", archivedAt: now }), + event("thread.pull-request-synced", { + pullRequests: [{ ...link, source: "stack-dismissed" }], + title: "Renamed", + archivedAt: now, + }), + event("thread.deleted", { pullRequests: [], deletedAt: now }), + ].map(changed), + ).toEqual([ + "changed", + "changed", + null, + null, + "changed", + null, + "changed", + "changed", + "changed", + null, + ]); + + const afterRestart = makeLinkedThreadsChangeDetector(); + expect(afterRestart(event("thread.created", { title: "New" }))).toBe(null); + expect( + makeLinkedThreadsChangeDetector()( + event("thread.deleted", { pullRequests: [], deletedAt: now }), + ), + ).toBe("changed"); +}); + +it("signals when activity reorders the threads linked to a pull request", () => { + const at = (second: number) => DateTime.makeUnsafe(`2026-11-12T00:00:0${second}.000Z`); + const link = { + host: "github.com", + repository: "acme/web", + number: 7, + url: "https://github.com/acme/web/pull/7", + source: "agent", + }; + const thread = (id: string, second: number, pullRequests: ReadonlyArray = [link]) => + ({ + type: "thread.pull-request-synced", + threadId: ThreadId.make(id), + payload: { + id: ThreadId.make(id), + projectId: ProjectId.make("project-a"), + title: id, + lineage: { rootThreadId: ThreadId.make(id) }, + pullRequests, + archivedAt: null, + settledOverride: null, + settledAt: null, + deletedAt: null, + updatedAt: at(second), + }, + }) as unknown as OrchestrationV2DomainEvent; + const activity = (id: string, second: number) => + ({ + type: "message.updated", + threadId: ThreadId.make(id), + occurredAt: at(second), + }) as unknown as OrchestrationV2DomainEvent; + const changed = makeLinkedThreadsChangeDetector(); + changed(thread("a", 1)); + changed(thread("b", 2)); + + expect( + [ + { ...thread("a", 3), type: "thread.pinned" } as OrchestrationV2DomainEvent, + activity("a", 4), + activity("b", 5), + activity("b", 5), + // Seen since startup without links, so its activity can't reorder an answer. + thread("unlinked", 5, []), + activity("unlinked", 6), + // Not seen since startup, so it may be linked from before the restart. + activity("persisted", 6), + ].map(changed), + ).toEqual(["reordered", "reordered", "reordered", null, "changed", null, "reordered"]); + + // Other threads linked to the pull request may have had no event since the restart. + const afterRestart = makeLinkedThreadsChangeDetector(); + afterRestart(thread("a", 2)); + expect( + afterRestart({ ...thread("a", 6), type: "thread.pinned" } as OrchestrationV2DomainEvent), + ).toBe("reordered"); +}); diff --git a/apps/server/src/pullRequest/linkedThreads.ts b/apps/server/src/pullRequest/linkedThreads.ts index 87ffae525eb9..1fb6b5f5dfeb 100644 --- a/apps/server/src/pullRequest/linkedThreads.ts +++ b/apps/server/src/pullRequest/linkedThreads.ts @@ -1,12 +1,18 @@ import { normalizeThreadPullRequestKey, + threadPullRequestKeyOf, threadPullRequestKeysEqual, + visibleThreadPullRequests, } from "../sourceControl/threadPullRequests.ts"; import { PullRequestLinkedThreadsResult, PullRequestOperationError, + type OrchestrationV2AppThread, + type OrchestrationV2DomainEvent, + type ThreadId, type ThreadPullRequestKey, } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; import * as Schema from "effect/Schema"; @@ -66,3 +72,67 @@ export const listLinkedPullRequestThreads = Effect.fn("listLinkedPullRequestThre }), ), ); + +// Mirrors the rows and filters of `listLinkedPullRequestThreads`. +function linkedKeys(thread: OrchestrationV2AppThread): ReadonlyArray { + if (thread.deletedAt !== null || thread.lineage.sideChat === true) return []; + return visibleThreadPullRequests(thread.pullRequests ?? []) + .map(threadPullRequestKeyOf) + .toSorted(); +} + +function linkedThreadsView( + thread: OrchestrationV2AppThread, + keys: ReadonlyArray, +): string | null { + if (keys.length === 0) return null; + const settled = thread.settledAt !== null || thread.settledOverride === "settled"; + return JSON.stringify([ + keys, + thread.title, + thread.projectId, + thread.archivedAt !== null, + settled, + ]); +} + +/** + * Tells how a domain event changed `pullRequests.linkedThreads` answers, so readers can refresh on + * change instead of polling. `reordered` means activity on a linked thread, or on one whose links + * are unknown since a restart, which can reorder answers. A thread's first full event after a + * restart counts as a change, since its earlier links are unknown. + */ +export function makeLinkedThreadsChangeDetector() { + const views = new Map(); + // Last activity of each thread with visible links. + const activeAt = new Map(); + const active = (threadId: ThreadId, at: DateTime.Utc): boolean => { + const previous = activeAt.get(threadId); + if (previous === undefined) return false; + const millis = DateTime.toEpochMillis(at); + if (millis <= previous) return false; + activeAt.set(threadId, millis); + return true; + }; + return (event: OrchestrationV2DomainEvent): "changed" | "reordered" | null => { + // The projection stamps every non-thread event as the thread's activity. A thread not seen + // since startup may be linked already, since history is not replayed. + if (!event.type.startsWith("thread.")) { + if (!views.has(event.threadId)) return "reordered"; + return active(event.threadId, event.occurredAt) ? "reordered" : null; + } + if (!("lineage" in event.payload)) return null; + const thread = event.payload; + const view = linkedThreadsView(thread, linkedKeys(thread)); + const known = views.has(event.threadId) || event.type === "thread.created"; + const previous = views.get(event.threadId) ?? null; + if (event.type === "thread.deleted") views.delete(event.threadId); + else views.set(event.threadId, view); + if (view === null) activeAt.delete(event.threadId); + if (!known || view !== previous) { + if (view !== null) activeAt.set(event.threadId, DateTime.toEpochMillis(thread.updatedAt)); + return "changed"; + } + return active(event.threadId, thread.updatedAt) ? "reordered" : null; + }; +} diff --git a/apps/server/src/pullRequest/pullRequestTarget.ts b/apps/server/src/pullRequest/pullRequestTarget.ts new file mode 100644 index 000000000000..4520b0523a36 --- /dev/null +++ b/apps/server/src/pullRequest/pullRequestTarget.ts @@ -0,0 +1,123 @@ +import type { OrchestrationProjectShell, SourceControlProviderKind } from "@t3tools/contracts"; +import { canonicalizeGitCafePullRequestUrl } from "@t3tools/source-control-definitions/registry"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +import { sourceControlDefinitions } from "../sourceControl/sourceControlDefinitions.ts"; +import { normalizeThreadPullRequestKey } from "../sourceControl/threadPullRequests.ts"; + +export interface PullRequestTarget { + readonly url?: string | undefined; + readonly repository?: string | undefined; + readonly number?: number | undefined; + readonly host?: string | undefined; +} + +export class PullRequestUrlInvalidError extends Schema.TaggedError()( + "PullRequestUrlInvalidError", + {}, +) { + override get message(): string { + return "This is not a recognised pull request URL. Pass repository and number instead."; + } +} + +export class PullRequestTargetIncompleteError extends Schema.TaggedError()( + "PullRequestTargetIncompleteError", + {}, +) { + override get message(): string { + return "Pass either url, or both repository and number."; + } +} + +export class PullRequestHostRequiredError extends Schema.TaggedError()( + "PullRequestHostRequiredError", + {}, +) { + override get message(): string { + return "This thread's project has no recognised remote. Pass host or url."; + } +} + +export class PullRequestUrlUnavailableError extends Schema.TaggedError()( + "PullRequestUrlUnavailableError", + {}, +) { + override get message(): string { + return "This thread's project is on a host with no known pull request URL shape. Pass url."; + } +} + +export type PullRequestTargetError = + | PullRequestUrlInvalidError + | PullRequestTargetIncompleteError + | PullRequestHostRequiredError + | PullRequestUrlUnavailableError; + +export interface ResolvedTarget { + readonly host: string; + readonly repository: string; + readonly number: number; + readonly url: string; +} + +function projectHostAndProvider(project: OrchestrationProjectShell | undefined): { + readonly host: string | null; + readonly kind: SourceControlProviderKind | null; +} { + const identity = project?.repositoryIdentity; + const kind = identity?.provider ?? null; + if (!identity || kind === null) return { host: null, kind: null }; + return { + host: sourceControlDefinitions.changeRequestHost(identity, kind), + kind, + }; +} + +export const resolveTarget = Effect.fn("PullRequestsToolkit.resolveTarget")(function* ( + input: PullRequestTarget, + project: OrchestrationProjectShell | undefined, +) { + if (input.url !== undefined) { + const parsed = sourceControlDefinitions.parseChangeRequestUrl(input.url)?.link; + if (parsed === undefined) { + return yield* new PullRequestUrlInvalidError({}); + } + return { + ...normalizeThreadPullRequestKey(parsed), + url: canonicalizeGitCafePullRequestUrl(input.url), + } satisfies ResolvedTarget; + } + if (input.repository === undefined || input.number === undefined) { + return yield* new PullRequestTargetIncompleteError({}); + } + const projectHost = projectHostAndProvider(project); + const host = (input.host ?? projectHost.host)?.toLowerCase(); + if (host === undefined) { + return yield* new PullRequestHostRequiredError({}); + } + const repository = input.repository.toLowerCase(); + // Without a URL, a public forge host says what shape its change request URLs take; + // any other host is taken to run the project's own forge. + const recognizedKind = sourceControlDefinitions.recognizeRemote(`https://${host}`)?.kind; + const kind = + sourceControlDefinitions.findByPublicHost(host)?.kind ?? + (recognizedKind === "gitcafe" ? recognizedKind : projectHost.kind); + const url = + kind === null + ? null + : sourceControlDefinitions.get(kind).changeRequestUrl({ + host, + repository, + number: input.number, + remoteUrl: project?.repositoryIdentity?.locator.remoteUrl, + }); + if (url === null) { + return yield* new PullRequestUrlUnavailableError({}); + } + return { + ...normalizeThreadPullRequestKey({ host, repository, number: input.number, url }), + url: canonicalizeGitCafePullRequestUrl(url), + } satisfies ResolvedTarget; +}); diff --git a/apps/server/src/pullRequest/pullRequestViewedFiles.ts b/apps/server/src/pullRequest/pullRequestViewedFiles.ts index 596808eb46a0..7558e8b6a166 100644 --- a/apps/server/src/pullRequest/pullRequestViewedFiles.ts +++ b/apps/server/src/pullRequest/pullRequestViewedFiles.ts @@ -30,7 +30,7 @@ const MAX_FILE_REVISION_PATHS = 1_000; interface FileRevisionsDependencies { readonly runFork: (effect: Effect.Effect) => unknown; readonly refEpoch: (ref: PullRequestRef) => number; - readonly fileRevisionsEpoch: () => number; + readonly fileRevisionsEpoch: (ref: PullRequestRef) => string; readonly toPullRequestError: ( operation: string, ) => (error: PullRequestProviderError) => PullRequestError; @@ -50,7 +50,7 @@ const makeFileRevisions = (dependencies: FileRevisionsDependencies) => { const fileRevisionsKey = (project: SupportedProject, ref: PullRequestRef) => [ refEpoch({ ...ref, host: project.host, repository: project.repository }), - fileRevisionsEpoch(), + fileRevisionsEpoch({ ...ref, host: project.host, repository: project.repository }), ref.projectId, project.repository.trim().toLowerCase(), ref.number, @@ -260,7 +260,9 @@ export const make = (dependencies: Dependencies) => { input: PullRequestSetFilesViewedInput, ): Effect.fn.Return { const viewer = yield* requiredViewerOf(project, "setFilesViewed"); - const cleared = input.files.filter((file) => file.viewed).map((file) => file.path); + const cleared = input.files + .filter((file) => file.viewed && file.revision === undefined) + .map((file) => file.path); const revisions = cleared.length === 0 ? null @@ -276,10 +278,10 @@ export const make = (dependencies: Dependencies) => { yield* filesViewedStore .set({ ...filesViewedScope(project, input.number, viewer), - // Missing paths have null baselines; empty revision means confirmed deletion. + // Shown revisions win over the head; missing paths have null baselines; empty means deleted. files: input.files.map((file) => ({ path: file.path, - revision: revisions?.get(file.path) ?? null, + revision: file.revision ?? revisions?.get(file.path) ?? null, viewed: file.viewed, })), viewedAt, diff --git a/apps/server/src/relay/AgentAwarenessRelay.test.ts b/apps/server/src/relay/AgentAwarenessRelay.test.ts index 85b9b44f4daf..10388e99fe7c 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.test.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.test.ts @@ -238,6 +238,7 @@ const makeTestRelay = Effect.fnUntraced(function* ( getThreadHistoryPage: () => Effect.die("unused"), getTimelinePage: () => Effect.die("Unused timeline read"), getMessageCount: () => Effect.die("unused message count"), + getMessagesById: () => Effect.die("unused messages by id"), getTurnItem: () => Effect.die("unused turn item read"), getThreadRecords: () => Effect.die("unused record read"), getThreadProjection: unused, @@ -377,6 +378,160 @@ describe("AgentAwarenessRelay", () => { }), ); + it.effect("publishes the parent as needing input when its subagent raises a secret card", () => + Effect.gen(function* () { + const now = DateTime.makeUnsafe(NOW); + const child = waitingChild({ + pendingRuntimeRequest: { + id: RuntimeRequestId.make("child-secret-card"), + kind: "user_input", + createdAt: now, + blocking: true, + userOnly: true, + }, + }); + const parent = shell({ status: "completed" }); + const events = yield* Queue.unbounded(); + const pulls = yield* Queue.unbounded(); + const { relay, publications } = yield* makeTestRelay({ + readShell: (threadId) => Effect.succeed(threadId === child.id ? child : parent), + domainEvents: Stream.fromEffectRepeat( + Queue.offer(pulls, undefined).pipe(Effect.andThen(Queue.take(events))), + ), + }); + yield* relay.start(); + yield* Queue.take(pulls); + const runId = RunId.make("run:child-secret-card"); + // Recording a secret card emits only this event for the child. + yield* Queue.offer(events, { + id: EventId.make("event:child-secret-card"), + type: "turn-item.updated", + threadId: CHILD_THREAD_ID, + runId, + occurredAt: now, + payload: { + id: TurnItemId.make("child-secret-card"), + threadId: CHILD_THREAD_ID, + runId, + nodeId: NodeId.make("node:child-secret-card"), + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 1, + status: "waiting", + title: "SMS code", + startedAt: now, + completedAt: null, + updatedAt: now, + type: "secret_request", + label: "SMS code", + reason: "The agent needs the code.", + secretStatus: "pending", + }, + }); + yield* Queue.take(pulls); + yield* relay.drain; + assert.equal(publications.length, 1); + assert.isTrue(publications[0]?.url.includes(`/threads/${THREAD_ID}/`) ?? false); + assert.equal(publications[0]?.state?.phase, "waiting_for_input"); + }), + ); + + it.effect( + "clears the parent's input wait when the user steers its subagent past a link card", + () => + Effect.gen(function* () { + const now = DateTime.makeUnsafe(NOW); + let child = waitingChild({ + pendingRuntimeRequest: { + id: RuntimeRequestId.make("child-link-card"), + kind: "user_input", + createdAt: now, + blocking: true, + userOnly: true, + }, + }); + const parent = shell({ status: "completed" }); + const events = yield* Queue.unbounded(); + const pulls = yield* Queue.unbounded(); + const { relay, publications } = yield* makeTestRelay({ + readShell: (threadId) => Effect.succeed(threadId === child.id ? child : parent), + domainEvents: Stream.fromEffectRepeat( + Queue.offer(pulls, undefined).pipe(Effect.andThen(Queue.take(events))), + ), + }); + yield* relay.start(); + yield* Queue.take(pulls); + const runId = RunId.make("run:child-link-card"); + const item = { + threadId: CHILD_THREAD_ID, + runId, + nodeId: NodeId.make("node:child-link-card"), + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + title: null, + startedAt: now, + updatedAt: now, + } as const; + yield* Queue.offer(events, { + id: EventId.make("event:child-link-card"), + type: "turn-item.updated", + threadId: CHILD_THREAD_ID, + runId, + occurredAt: now, + payload: { + ...item, + id: TurnItemId.make("child-link-card"), + ordinal: 1, + status: "waiting", + completedAt: null, + type: "link_request", + url: "https://example.com/login", + label: "Sign in", + reason: "The agent needs you to sign in.", + linkStatus: "pending", + }, + }); + yield* Queue.take(pulls); + yield* relay.drain; + assert.equal(publications[0]?.state?.phase, "waiting_for_input"); + + // Steering a running child records only these events for it; the + // projection then stops counting the link card as pending. + child = waitingChild({ pendingRuntimeRequest: null }); + yield* Queue.offer(events, { + id: EventId.make("event:child-steer"), + type: "turn-item.updated", + threadId: CHILD_THREAD_ID, + runId, + occurredAt: now, + payload: { + ...item, + id: TurnItemId.make("child-steer"), + ordinal: 2, + status: "completed", + completedAt: now, + createdBy: "user", + creationSource: "web", + type: "user_message", + messageId: MessageId.make("message:child-steer"), + inputIntent: "steer", + text: "Skip the login.", + attachments: [], + }, + }); + yield* Queue.take(pulls); + // The relay confirms a move into completed before publishing it. + yield* TestClock.adjust("5 seconds"); + yield* relay.drain; + assert.equal(publications.length, 2); + assert.equal(publications[1]?.state?.phase, "completed"); + }), + ); + it.effect("republishes a completed parent for its waiting child after a relink", () => Effect.gen(function* () { const parent = shell({ status: "completed" }); @@ -400,7 +555,7 @@ describe("AgentAwarenessRelay", () => { yield* relay.drain; assert.equal(publications.length, 2); assert.equal(publications[1]?.state?.phase, "waiting_for_approval"); - }).pipe(Effect.scoped), + }), ); it.effect("keeps a child's alert time when the parent's own request comes and goes", () => @@ -568,7 +723,7 @@ describe("AgentAwarenessRelay", () => { yield* relay.drain; assert.equal(publications.length, 3); assert.equal(publications[2]?.state, null); - }).pipe(Effect.scoped), + }), ); it.effect("holds a completion through a continuation gap, then publishes it once", () => @@ -598,6 +753,37 @@ describe("AgentAwarenessRelay", () => { }), ); + it.effect( + "clears a completion that landed while snoozed, even when processed after the wake", + () => + Effect.gen(function* () { + yield* TestClock.setTime(DateTime.toEpochMillis(DateTime.makeUnsafe(NOW))); + const { relay, currentShell, publications } = yield* makeTestRelay(); + const snooze = { + snoozedAt: DateTime.makeUnsafe(NOW), + snoozedUntil: DateTime.makeUnsafe("2026-09-04T13:00:00.000Z"), + }; + yield* Ref.set(currentShell, shell(snooze)); + yield* relay.publishThread(THREAD_ID); + yield* TestClock.adjust("2 hours"); + yield* Ref.set( + currentShell, + shell({ + ...snooze, + status: "completed", + latestRunCompletedAt: DateTime.makeUnsafe("2026-09-04T12:30:00.000Z"), + }), + ); + yield* relay.publishThread(THREAD_ID); + yield* TestClock.adjust("5 seconds"); + yield* relay.drain; + assert.deepEqual( + publications.map((publication) => publication.state?.phase ?? null), + ["running", null], + ); + }), + ); + it.effect("still publishes a completion when the thread settles during the hold", () => Effect.gen(function* () { const { relay, currentShell, publications } = yield* makeTestRelay(); @@ -1016,5 +1202,5 @@ it.effect("catches up after startup without polling after success", () => assert.equal(catchUp.activitySnapshotReads, 3); assert.equal(publications.length, 2); assert.equal(publications[1]?.state?.threadId, THREAD_ID); - }).pipe(Effect.scoped), + }), ); diff --git a/apps/server/src/relay/AgentAwarenessRelay.ts b/apps/server/src/relay/AgentAwarenessRelay.ts index c14558762183..ef2e34799742 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.ts @@ -16,7 +16,8 @@ import { import { type SubagentRequestAwareness, projectThreadAwarenessV2, - subagentAwaitsApproval, + subagentAwaitsUser, + turnItemCanChangeUserWait, } from "@t3tools/shared/agentAwareness"; import { turnItemUpdateCanEndBackgroundWork } from "@t3tools/shared/orchestrationV2PendingBackgroundWork"; import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker"; @@ -50,6 +51,7 @@ import { import { EnvironmentSigningKey } from "../cloud/environmentKeys.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import * as ThreadManagement from "../orchestration-v2/ThreadManagementService.ts"; +import { isSnoozed } from "../orchestration-v2/ThreadSettlementService.ts"; import * as ProjectService from "../project/ProjectService.ts"; import { forkParked } from "../serverActivation.ts"; @@ -116,6 +118,7 @@ function shouldPublishAgentAwarenessEvent( case "thread.active-reordered": case "thread.tags-updated": case "thread.visited": + case "thread.visit-recorded": case "thread.marked-unread": case "thread.runtime-mode-updated": case "thread.interaction-mode-updated": @@ -128,7 +131,11 @@ function shouldPublishAgentAwarenessEvent( case "provider-session.detached": case "provider-turn.updated": case "turn-item.updated": - return isTurnItemPayload(event.payload) && turnItemUpdateCanEndBackgroundWork(event.payload); + return ( + isTurnItemPayload(event.payload) && + (turnItemUpdateCanEndBackgroundWork(event.payload) || + turnItemCanChangeUserWait(event.payload)) + ); case "message.updated": case "plan.updated": case "checkpoint-scope.created": @@ -308,7 +315,7 @@ interface SubagentAlert { } const waitingApprovalId = (thread: AgentAwarenessThread | null) => - thread !== null && thread.archivedAt === null && subagentAwaitsApproval(thread) + thread !== null && thread.archivedAt === null && subagentAwaitsUser(thread) ? thread.pendingRuntimeRequest?.id : undefined; @@ -354,12 +361,27 @@ function resolveAgentAwarenessRelayPublishSnapshot(input: { state?.phase !== projectThreadAwarenessV2(awarenessInput)?.phase; return { projectId: input.thread.value.projectId, - state: sanitizeRelayAgentActivityState(state), + state: snoozedCompletion(input.thread.value, state) + ? null + : sanitizeRelayAgentActivityState(state), subagentWait: childDrivesWait ? input.pendingSubagentRequest : null, reason: "snapshot", }; } +// Checked at completion time, so a completion processed after the wake stays silent too. +function snoozedCompletion( + thread: AgentAwarenessThread, + state: { readonly phase: string } | null, +): boolean { + if (state?.phase !== "completed" || thread.latestRunCompletedAt == null) return false; + const completedAt = DateTime.toEpochMillis(thread.latestRunCompletedAt); + if (thread.snoozedAt != null && completedAt < DateTime.toEpochMillis(thread.snoozedAt)) { + return false; + } + return isSnoozed(thread, completedAt); +} + function terminalWorkSinceStart(thread: AgentAwarenessThread, startedAt: number): boolean { return ( thread.latestRunCompletedAt != null && @@ -377,7 +399,7 @@ export function resolveAgentAwarenessRelayActiveThreadIds(input: { return input.threads .filter((thread) => { // Publishing a waiting subagent republishes its top-level thread. - if (subagentAwaitsApproval(thread)) return true; + if (subagentAwaitsUser(thread)) return true; const project = projectById.get(thread.projectId); if (!project) { return false; @@ -758,7 +780,11 @@ export const make = Effect.gen(function* () { if (requestId !== entry.requestId) subagentRequests.set(childId, { ...entry, requestId }); // The newest wait is the one to announce. if (pending === null || DateTime.isGreaterThan(child.updatedAt, pending.updatedAt)) { - pending = { requestId, updatedAt: child.updatedAt }; + pending = { + requestId, + updatedAt: child.updatedAt, + waitsForInput: child.pendingRuntimeRequest?.kind === "user_input", + }; } } return pending; diff --git a/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.ts b/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.ts index 7d9de87ecfd9..ca77c9e26220 100644 --- a/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.ts +++ b/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.ts @@ -187,6 +187,8 @@ export class DesktopTelemetryReceiver extends Context.Service< readonly setDiagnosticsDemand: ( enabled: boolean, ) => Effect.Effect; + /** Tells the desktop parent that the server accepts requests, so it can skip readiness polling. */ + readonly announceServerReady: Effect.Effect; readonly requestDesktopUpdate: ( requestId: string, trackSwitch?: DesktopTrackSwitch, @@ -717,6 +719,9 @@ export const make = Effect.fn("resourceTelemetry.desktopTelemetryReceiver.make") health: Ref.get(health), subscribeHealth: subscribeBeforeSnapshotWithoutMutex(healthChanges, Ref.get(health)), setDiagnosticsDemand, + announceServerReady: sendControlMessage({ version: 1, type: "serverReady" }).pipe( + Effect.withSpan("DesktopTelemetryReceiver.announceServerReady"), + ), requestDesktopUpdate: Effect.fn("DesktopTelemetryReceiver.requestDesktopUpdate")( function (requestId, trackSwitch) { return sendControlMessage({ @@ -787,6 +792,7 @@ export const layerTest = ( setDiagnosticsDemand: Effect.fn("DesktopTelemetryReceiver.setDiagnosticsDemand")(function () { return Effect.void; }), + announceServerReady: Effect.void, requestDesktopUpdate: Effect.fn("DesktopTelemetryReceiver.requestDesktopUpdate")(function () { return Effect.void; }), diff --git a/apps/server/src/resourceTelemetry/ResourceTelemetry.test.ts b/apps/server/src/resourceTelemetry/ResourceTelemetry.test.ts index c12e2e44ed7e..8a3640977ddf 100644 --- a/apps/server/src/resourceTelemetry/ResourceTelemetry.test.ts +++ b/apps/server/src/resourceTelemetry/ResourceTelemetry.test.ts @@ -272,131 +272,127 @@ describe("ResourceTelemetry", () => { ); it.effect("rejects buffered snapshots from an earlier sidecar generation", () => - Effect.scoped( - Effect.gen(function* () { - const startedAt = DateTime.toEpochMillis(yield* DateTime.now); - const stale = nativeSnapshot({ - sequence: 100, - sampledAtUnixMs: startedAt + 1_000, - childCpuTimeMs: 100, - childWriteBytes: 1_000, - }); - const current = nativeSnapshot({ - sequence: 1, - sampledAtUnixMs: startedAt + 2_000, - childCpuTimeMs: 200, - childWriteBytes: 2_000, - }); - const nativeSnapshots = - yield* PubSub.unbounded(); - const layerNative = NativeTelemetryClient.layerTest({ - snapshots: Stream.fromPubSub(nativeSnapshots), - sampleNow: Effect.never, - health: Effect.succeed({ - status: "healthy", - hello: Option.none(), - lastSampleAt: Option.none(), - lastError: Option.none(), - restartCount: 1, - sampleIntervalMs: 1_000, - }), - }); - const layerTelemetry = ResourceTelemetry.layer.pipe( - Layer.provide( - Layer.mergeAll( - layerNative, - DesktopTelemetryReceiver.layerTest(), - ResourceAttribution.layer, - ), + Effect.gen(function* () { + const startedAt = DateTime.toEpochMillis(yield* DateTime.now); + const stale = nativeSnapshot({ + sequence: 100, + sampledAtUnixMs: startedAt + 1_000, + childCpuTimeMs: 100, + childWriteBytes: 1_000, + }); + const current = nativeSnapshot({ + sequence: 1, + sampledAtUnixMs: startedAt + 2_000, + childCpuTimeMs: 200, + childWriteBytes: 2_000, + }); + const nativeSnapshots = + yield* PubSub.unbounded(); + const layerNative = NativeTelemetryClient.layerTest({ + snapshots: Stream.fromPubSub(nativeSnapshots), + sampleNow: Effect.never, + health: Effect.succeed({ + status: "healthy", + hello: Option.none(), + lastSampleAt: Option.none(), + lastError: Option.none(), + restartCount: 1, + sampleIntervalMs: 1_000, + }), + }); + const layerTelemetry = ResourceTelemetry.layer.pipe( + Layer.provide( + Layer.mergeAll( + layerNative, + DesktopTelemetryReceiver.layerTest(), + ResourceAttribution.layer, ), - ); + ), + ); - yield* Effect.gen(function* () { - const telemetry = yield* ResourceTelemetry.ResourceTelemetry; - const subscription = yield* telemetry.subscribe; - const nextSnapshot = yield* Stream.runHead(subscription.changes).pipe(Effect.forkChild); + yield* Effect.gen(function* () { + const telemetry = yield* ResourceTelemetry.ResourceTelemetry; + const subscription = yield* telemetry.subscribe; + const nextSnapshot = yield* Stream.runHead(subscription.changes).pipe(Effect.forkChild); - yield* PubSub.publish(nativeSnapshots, nativeGeneration(stale, 0)); - yield* Effect.yieldNow; - expect((yield* telemetry.latest).health.scannedProcessCount).toBe(0); + yield* PubSub.publish(nativeSnapshots, nativeGeneration(stale, 0)); + yield* Effect.yieldNow; + expect((yield* telemetry.latest).health.scannedProcessCount).toBe(0); - yield* PubSub.publish(nativeSnapshots, nativeGeneration(current, 1)); - const received = yield* Fiber.join(nextSnapshot); + yield* PubSub.publish(nativeSnapshots, nativeGeneration(current, 1)); + const received = yield* Fiber.join(nextSnapshot); - expect(Option.isSome(received)).toBe(true); - expect(DateTime.toEpochMillis(Option.getOrThrow(received).readAt)).toBe( - current.sampledAtUnixMs, - ); - }).pipe(Effect.provide(layerTelemetry)); - }), - ), + expect(Option.isSome(received)).toBe(true); + expect(DateTime.toEpochMillis(Option.getOrThrow(received).readAt)).toBe( + current.sampledAtUnixMs, + ); + }).pipe(Effect.provide(layerTelemetry)); + }), ); it.effect("atomically subscribes while a desktop update is being rebuilt", () => - Effect.scoped( - Effect.gen(function* () { - const startedAt = DateTime.toEpochMillis(yield* DateTime.now); - const nextDesktopSnapshot = desktopSnapshot(startedAt + 1_000); - const desktopChanges = yield* PubSub.unbounded(); - const rebuildStarted = yield* Deferred.make(); - const finishRebuild = yield* Deferred.make(); - const attributionReads = yield* Ref.make(0); - const layerAttribution = Layer.succeed( - ResourceAttribution.ResourceAttribution, - ResourceAttribution.ResourceAttribution.of({ - record: () => Effect.void, - snapshot: Ref.updateAndGet(attributionReads, (count) => count + 1).pipe( - Effect.flatMap((count) => - count === 1 - ? Effect.void - : Deferred.succeed(rebuildStarted, undefined).pipe( - Effect.andThen(Deferred.await(finishRebuild)), - ), - ), - Effect.andThen( - Effect.succeed({ - readAt: DateTime.makeUnsafe(startedAt), - entries: [], - }), - ), + Effect.gen(function* () { + const startedAt = DateTime.toEpochMillis(yield* DateTime.now); + const nextDesktopSnapshot = desktopSnapshot(startedAt + 1_000); + const desktopChanges = yield* PubSub.unbounded(); + const rebuildStarted = yield* Deferred.make(); + const finishRebuild = yield* Deferred.make(); + const attributionReads = yield* Ref.make(0); + const layerAttribution = Layer.succeed( + ResourceAttribution.ResourceAttribution, + ResourceAttribution.ResourceAttribution.of({ + record: () => Effect.void, + snapshot: Ref.updateAndGet(attributionReads, (count) => count + 1).pipe( + Effect.flatMap((count) => + count === 1 + ? Effect.void + : Deferred.succeed(rebuildStarted, undefined).pipe( + Effect.andThen(Deferred.await(finishRebuild)), + ), ), - }), - ); - const layerNative = NativeTelemetryClient.layerTest({ - sampleNow: Effect.never, - health: Effect.succeed({ - status: "healthy", - hello: Option.none(), - lastSampleAt: Option.none(), - lastError: Option.none(), - restartCount: 0, - sampleIntervalMs: 1_000, - }), - }); - const layerDesktop = DesktopTelemetryReceiver.layerTest({ - changes: Stream.fromPubSub(desktopChanges), - }); - const layerTelemetry = ResourceTelemetry.layer.pipe( - Layer.provide(Layer.mergeAll(layerNative, layerDesktop, layerAttribution)), - ); + Effect.andThen( + Effect.succeed({ + readAt: DateTime.makeUnsafe(startedAt), + entries: [], + }), + ), + ), + }), + ); + const layerNative = NativeTelemetryClient.layerTest({ + sampleNow: Effect.never, + health: Effect.succeed({ + status: "healthy", + hello: Option.none(), + lastSampleAt: Option.none(), + lastError: Option.none(), + restartCount: 0, + sampleIntervalMs: 1_000, + }), + }); + const layerDesktop = DesktopTelemetryReceiver.layerTest({ + changes: Stream.fromPubSub(desktopChanges), + }); + const layerTelemetry = ResourceTelemetry.layer.pipe( + Layer.provide(Layer.mergeAll(layerNative, layerDesktop, layerAttribution)), + ); - yield* Effect.gen(function* () { - const telemetry = yield* ResourceTelemetry.ResourceTelemetry; - yield* Effect.yieldNow; - yield* PubSub.publish(desktopChanges, nextDesktopSnapshot); - yield* Deferred.await(rebuildStarted); + yield* Effect.gen(function* () { + const telemetry = yield* ResourceTelemetry.ResourceTelemetry; + yield* Effect.yieldNow; + yield* PubSub.publish(desktopChanges, nextDesktopSnapshot); + yield* Deferred.await(rebuildStarted); - const subscriptionFiber = yield* telemetry.subscribe.pipe(Effect.forkChild); - yield* Effect.yieldNow; - yield* Deferred.succeed(finishRebuild, undefined); - const subscription = yield* Fiber.join(subscriptionFiber); + const subscriptionFiber = yield* telemetry.subscribe.pipe(Effect.forkChild); + yield* Effect.yieldNow; + yield* Deferred.succeed(finishRebuild, undefined); + const subscription = yield* Fiber.join(subscriptionFiber); - expect(DateTime.toEpochMillis(subscription.latest.readAt)).toBe( - nextDesktopSnapshot.sampledAtUnixMs, - ); - expect(subscription.latest.power).toEqual(nextDesktopSnapshot.power); - }).pipe(Effect.provide(layerTelemetry)); - }), - ), + expect(DateTime.toEpochMillis(subscription.latest.readAt)).toBe( + nextDesktopSnapshot.sampledAtUnixMs, + ); + expect(subscription.latest.power).toEqual(nextDesktopSnapshot.power); + }).pipe(Effect.provide(layerTelemetry)); + }), ); }); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.interval.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.interval.test.ts index 40db56f86e95..4ff0cc506762 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.interval.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.interval.test.ts @@ -27,6 +27,7 @@ const dependencies = Layer.mergeAll( Layer.mock(ThreadLaunchService.ThreadLaunchService)({ launch: () => Effect.die("synthetic dispatch failure"), checkWorktreeBase: () => Effect.void, + isNonGitProject: () => Effect.succeed(false), }), Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ namedProjectsRoot: "/projects", diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.schedule.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.schedule.test.ts index b858dbfef49f..93e400fb15ce 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.schedule.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.schedule.test.ts @@ -189,6 +189,7 @@ it.effect("checks a new or changed worktree base branch, but not an unchanged on Scheduler.layer, ManagedPolicy.layerTest(), Layer.mock(ThreadLaunchService.ThreadLaunchService)({ + isNonGitProject: () => Effect.succeed(false), checkWorktreeBase: ({ projectId, workspaceStrategy }) => { const baseRef = workspaceStrategy.type === "worktree" ? workspaceStrategy.baseRef : ""; checked.push(baseRef); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index ee4385ff2e0a..1335e369484a 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -4,6 +4,7 @@ import * as NodeUtil from "node:util"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import { assert, it, vi } from "@effect/vitest"; import { + CommandId, ProjectId, ProviderInstanceId, ScheduledTaskError, @@ -29,6 +30,7 @@ import * as TestClock from "effect/testing/TestClock"; import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts"; import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts"; +import { OrchestratorDispatchError } from "../orchestration-v2/Orchestrator.ts"; import * as SecretRequests from "../secrets/SecretRequests.ts"; import * as ManagedProjectFolders from "../project/ManagedProjectFolders.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; @@ -663,6 +665,67 @@ it.effect("a fresh-session task launches one thread, then starts each later run }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); +it.effect("a refused dispatch records its refusal, and opaque causes stay generic", () => + Effect.gen(function* () { + const refusal = "Thread thread:refused was stopped."; + const refused = (cause: unknown) => + new OrchestratorDispatchError({ + commandId: CommandId.make("command:refused"), + commandType: "message.dispatch", + cause, + }); + const generic = refused(undefined).message; + const cases = [ + [refusal, refusal], + [" ", generic], + [new Error("private-nested-secret"), generic], + ] as const; + const failure = yield* Ref.make(undefined); + const layer = ScheduledTaskService.layer.pipe( + Layer.provide( + Layer.mergeAll( + ManagedPolicy.layerTest(), + Layer.mock(ThreadLaunchService.ThreadLaunchService)({}), + Layer.mock(ThreadManagementService.ThreadManagementService)({ + getThreadShell: () => Effect.succeed(threadShell(ProjectId.make("project:test"))), + sendToThread: () => + Ref.get(failure).pipe(Effect.flatMap((cause) => Effect.fail(refused(cause)))), + }), + Layer.mock(SecretRequests.SecretRequests)({}), + Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ + namedProjectsRoot: "/projects", + isScratchProject: () => Effect.succeed(false), + }), + NodeCrypto.layer, + Layer.succeed(Scheduler.Scheduler, { register: () => Effect.void }), + ), + ), + ); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* service.upsert({ + id: ScheduledTaskId.make("task:refused"), + title: "Refused dispatch", + prompt: "Check progress", + enabled: true, + schedule: { type: "interval", everyMs: 60_000 }, + projectId: ProjectId.make("project:test"), + threadId: ThreadId.make("thread:refused"), + workspaceStrategy: { type: "root" }, + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" }, + runtimeMode: "full-access", + interactionMode: "default", + }); + for (const [cause, expected] of cases) { + yield* Ref.set(failure, cause); + const result = yield* service.runNow({ id: task.id }); + assert.equal(result.task.lastRunStatus, "failed"); + assert.equal(result.task.lastRunError, expected); + } + }).pipe(Effect.provide(layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + it.effect("a first run whose thread was deleted before it bound removes the task", () => Effect.gen(function* () { const layer = ScheduledTaskService.layer.pipe( diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 496566e79b24..42a87670011b 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -53,6 +53,8 @@ import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts import * as ManagedProjectFolders from "../project/ManagedProjectFolders.ts"; import * as Metrics from "../observability/Metrics.ts"; import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts"; +import { OrchestratorDispatchError } from "../orchestration-v2/Orchestrator.ts"; +import { userFacingDispatchErrorMessage } from "../orchestration-v2/UserFacingErrors.ts"; import * as SecretRequests from "../secrets/SecretRequests.ts"; import * as ManagedPolicy from "../policy/ManagedPolicy.ts"; import * as DueHint from "../scheduling/DueHint.ts"; @@ -79,6 +81,7 @@ import { constantTimeEquals, verifyWebhookSignature } from "./webhookVerificatio export const WEBHOOK_ROUTE_PREFIX = "/api/hooks"; const isScheduledTaskError = Schema.is(ScheduledTaskError); +const isOrchestratorDispatchError = Schema.is(OrchestratorDispatchError); const PROMPT_TOO_LONG = "The filled-in prompt is too long."; const WEBHOOK_DELIVERY_RETENTION = 50; const WEBHOOK_DEDUPE_DELIVERY_RETENTION = 20; @@ -374,6 +377,14 @@ function errorMessage(error: unknown): string { return failure instanceof Error ? failure.message : String(failure); } +function dispatchErrorMessage(cause: Cause.Cause): string { + const failure = Cause.squash(cause); + // Direct orchestration refusals are user-facing; nested defects can contain private data. + return isOrchestratorDispatchError(failure) && typeof failure.cause === "string" + ? (userFacingDispatchErrorMessage(failure.cause) ?? errorMessage(failure)) + : errorMessage(failure); +} + function webhookPath(taskId: string, token: string): string { return `${WEBHOOK_ROUTE_PREFIX}/${encodeURIComponent(taskId)}/${token}`; } @@ -1108,7 +1119,7 @@ export const layer = Layer.effect( if (webhook !== undefined && runSucceeded) yield* Ref.set(webhook.dispatched, true); const completedAt = yield* localNow; const lastRunStatus = runSucceeded ? ("succeeded" as const) : ("failed" as const); - const lastRunError = runSucceeded ? null : errorMessage(result.cause); + const lastRunError = runSucceeded ? null : dispatchErrorMessage(result.cause); const threadId = runSucceeded ? result.value : null; // One transaction, so an edit cannot land between reading the schedule and // writing the next run computed from it. The origin is read first: it reads @@ -1451,11 +1462,18 @@ export const layer = Layer.effect( ? managedRuntimeModeDenial(managedPolicy.summary, input.runtimeMode) : undefined; if (modeDenial !== undefined) return yield* taskError(modeDenial, { taskId: id }); + // A folder that is not a Git repository, Scratch included, cannot hold a worktree. + const workspaceStrategy: ScheduledTask["workspaceStrategy"] = + input.workspaceStrategy.type === "worktree" && + ((yield* managedFolders.isScratchProject(input.projectId)) || + (yield* threadLaunch.isNonGitProject(input.projectId))) + ? { type: "root" } + : input.workspaceStrategy; // Runs launch unattended, so a new or changed worktree base must name // a real branch now rather than fail at its first run. An unchanged // one is left alone so a task whose branch was deleted can still be // edited or paused. - const strategy = input.workspaceStrategy; + const strategy = workspaceStrategy; const existingStrategy = existingTask?.workspaceStrategy; if ( threadId === null && @@ -1536,11 +1554,6 @@ export const layer = Layer.effect( return { token, secret, secretChanged }; }) : { token: null, secret: null, secretChanged: true }; - const workspaceStrategy: ScheduledTask["workspaceStrategy"] = - input.workspaceStrategy.type === "worktree" && - (yield* managedFolders.isScratchProject(input.projectId)) - ? { type: "root" } - : input.workspaceStrategy; const scheduleUnchanged = existingTask !== null && existingTask.enabled === input.enabled && @@ -1582,13 +1595,14 @@ export const layer = Layer.effect( return { task: (yield* findTask(id)) ?? task }; }); - const defaultWorkspaceStrategy: ScheduledTaskService["Service"]["defaultWorkspaceStrategy"] = ( - projectId, - ) => - threadLaunch.resolveDefaultWorktreeBase(projectId).pipe( - Effect.map((baseRef) => ({ type: "worktree" as const, baseRef, startFromOrigin: true })), - Effect.mapError((cause) => taskError(cause.message, { cause })), - ); + const defaultWorkspaceStrategy: ScheduledTaskService["Service"]["defaultWorkspaceStrategy"] = + Effect.fn("ScheduledTaskService.defaultWorkspaceStrategy")(function* (projectId) { + if (yield* threadLaunch.isNonGitProject(projectId)) return { type: "root" }; + return yield* threadLaunch.resolveDefaultWorktreeBase(projectId).pipe( + Effect.map((baseRef) => ({ type: "worktree" as const, baseRef, startFromOrigin: true })), + Effect.mapError((cause) => taskError(cause.message, { cause })), + ); + }); const setEnabled: ScheduledTaskService["Service"]["setEnabled"] = Effect.fn( "ScheduledTaskService.setEnabled", diff --git a/apps/server/src/secrets/SecretCheck.test.ts b/apps/server/src/secrets/SecretCheck.test.ts new file mode 100644 index 000000000000..473cab1dfc65 --- /dev/null +++ b/apps/server/src/secrets/SecretCheck.test.ts @@ -0,0 +1,363 @@ +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import { + ProviderDriverKind, + ProviderInstanceId, + SecretProviderId, + SecretProviderKind, + type SecretReference, + SettingsSecretValue, + type ServerProvider, +} from "@t3tools/contracts"; +import type { AnyProviderDriver, ProviderInstance } from "@t3tools/provider-core/server/driver"; +import { makeManualOnlyProviderMaintenanceCapabilities } from "@t3tools/provider-core/server/maintenanceResolver"; +import * as ProviderLatestVersions from "@t3tools/provider-core/server/ProviderLatestVersions"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; +import { defineSecretProviderDriver } from "@t3tools/secret-provider-core/server/driver"; +import { defineSecretProvider } from "@t3tools/secret-provider-core/shared/definition"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import { HttpClient, HttpClientResponse } from "effect/http"; +import * as Layer from "effect/Layer"; +import * as PubSub from "effect/PubSub"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; + +import * as ServerConfig from "../config.ts"; +import * as ModelManifest from "../provider/ModelManifest.ts"; +import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts"; +import * as ProviderInstanceRegistryMutator from "../provider/ProviderInstanceRegistryMutator.ts"; +import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; +import * as SecretCheck from "./SecretCheck.ts"; + +const FAKE_KIND = SecretProviderKind.make("fake"); +const secretProvider = SecretProviderId.make("personal"); +const wanted: SecretReference = { provider: secretProvider, reference: "fake://wanted" }; +const other: SecretReference = { provider: secretProvider, reference: "fake://other" }; +const locked: SecretReference = { provider: secretProvider, reference: "fake://locked" }; +const driverKind = ProviderDriverKind.make("fake-provider"); +const dependent = ProviderInstanceId.make("dependent"); +const unrelated = ProviderInstanceId.make("unrelated"); + +const providerSnapshot = (instanceId: ProviderInstanceId): ServerProvider => ({ + instanceId, + driver: driverKind, + status: "ready", + enabled: true, + installed: true, + auth: { status: "authenticated" }, + checkedAt: "2026-10-03T00:00:00.000Z", + version: "1.0.0", + models: [], + slashCommands: [], + skills: [], +}); + +/** + * A server whose `dependent` instance reads `dependentReads` and `unrelated` reads `other`, all + * through the real resolver, instance registry and provider registry. `vault` answers each read of + * a reference with the next queued value, repeating the last one, after `beforeRead`. + */ +const setup = Effect.fn("SecretCheckTest.setup")(function* ( + vault: Map>, + dependentReads: ReadonlyArray = [wanted], + options: { + readonly dependentConfig?: unknown; + readonly beforeRead?: (reference: string) => Effect.Effect; + } = {}, +) { + const reads: Array = []; + const created: Array<{ instanceId: ProviderInstanceId; secrets: ReadonlyArray }> = []; + const createdConfigs: Array = []; + const refreshed: Array = []; + + const secretDriver = defineSecretProviderDriver({ + kind: FAKE_KIND, + definition: defineSecretProvider({ + kind: FAKE_KIND, + label: "Fake", + configSchema: Schema.Struct({}), + isReference: (value) => value.startsWith("fake://"), + referencePlaceholder: "fake://item", + invalidReferenceMessage: "Use fake://item.", + }), + read: (_config, references) => + Effect.forEach(references, (reference) => + (options.beforeRead?.(reference) ?? Effect.void).pipe( + Effect.andThen( + Effect.sync(() => { + reads.push(reference); + const answers = vault.get(reference) ?? []; + return answers.length > 1 ? answers.shift() : answers[0]; + }), + ), + ), + ), + }); + + const driver: AnyProviderDriver = { + driverKind, + metadata: { displayName: "Fake" }, + configSchema: Schema.Struct({ + keys: Schema.Record(Schema.String, SettingsSecretValue).pipe( + Schema.withDecodingDefault(Effect.succeed({})), + ), + }), + defaultConfig: () => ({}), + create: (input) => + Effect.sync(() => { + created.push({ + instanceId: input.instanceId, + secrets: input.environment.map((variable) => variable.value), + }); + createdConfigs.push(input.config); + const snapshot = providerSnapshot(input.instanceId); + return { + instanceId: input.instanceId, + driverKind, + continuationIdentity: { driverKind, continuationKey: `fake:${input.instanceId}` }, + displayName: undefined, + enabled: input.enabled, + snapshot: { + resolveMaintenance: () => + Effect.succeed( + makeManualOnlyProviderMaintenanceCapabilities({ + provider: driverKind, + packageName: null, + }), + ), + getSnapshot: Effect.succeed(snapshot), + refresh: Effect.sync(() => { + refreshed.push(input.instanceId); + return snapshot; + }), + streamChanges: Stream.empty, + applyUsageLimits: () => Effect.void, + }, + orchestrationAdapter: {} as ProviderInstance["orchestrationAdapter"], + textGeneration: {} as ProviderInstance["textGeneration"], + } satisfies ProviderInstance; + }), + }; + const instanceConfig = (secrets: ReadonlyArray, config: unknown = {}) => ({ + driver: driverKind, + config, + environment: secrets.map((secret, index) => ({ + name: `API_KEY_${index}`, + value: secret, + sensitive: true, + })), + }); + + const resolverContext = yield* Layer.build( + SecretResolver.layer({ + drivers: [secretDriver], + secretProviders: Effect.succeed({ + [secretProvider]: { kind: FAKE_KIND, label: "Personal", config: {} }, + }), + }), + ); + const { registry, mutator } = yield* ProviderInstanceRegistry.makeProviderInstanceRegistry({ + drivers: [driver], + configMap: { + [dependent]: instanceConfig(dependentReads, options.dependentConfig), + [unrelated]: instanceConfig([other]), + }, + }).pipe(Effect.provide(resolverContext)); + // Secrets are read after startup, so wait for the change that applies them. + const changes = yield* registry.subscribeChanges; + yield* PubSub.take(changes); + + const services = yield* Layer.build( + SecretCheck.layer.pipe( + Layer.provideMerge(ProviderRegistry.layer), + Layer.provide( + Layer.mergeAll( + Layer.succeedContext(resolverContext), + Layer.succeed(ProviderInstanceRegistry.ProviderInstanceRegistry, registry), + Layer.succeed(ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutator, mutator), + ), + ), + ), + ); + const secretCheck = yield* SecretCheck.SecretCheck.pipe(Effect.provide(services)); + const resolver = yield* SecretResolver.SecretResolver.pipe(Effect.provide(resolverContext)); + return { secretCheck, registry, mutator, resolver, reads, created, createdConfigs, refreshed }; +}); + +const layerTest = Layer.mergeAll( + ServerConfig.layerTest(process.cwd(), { prefix: "t3-secret-check-" }), + ModelManifest.layerTest, + ProviderLatestVersions.layer, + Layer.succeed( + HttpClient.HttpClient, + HttpClient.make((request) => + Effect.succeed(HttpClientResponse.fromWeb(request, Response.json({ version: "0.0.0" }))), + ), + ), +).pipe(Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))); + +it.layer(layerTest)("SecretCheck", (it) => { + it.effect( + "a readable secret rebuilds and refreshes only the instances that read it, from that one read", + () => + Effect.gen(function* () { + const vault = new Map>([ + [wanted.reference, [undefined]], + [other.reference, ["other-key"]], + ]); + const server = yield* setup(vault); + assert.deepStrictEqual( + (yield* server.registry.listUnavailable).map((entry) => entry.instanceId), + [dependent], + ); + const createdBefore = server.created.length; + const readsBefore = server.reads.length; + + // Unlocked for the check's own read only: any second read would fail again. + vault.set(wanted.reference, ["wanted-key", undefined]); + const result = yield* server.secretCheck.check(wanted); + + assert.deepStrictEqual(result, { _tag: "Resolved" }); + assert.notInclude(JSON.stringify(result), "wanted-key"); + assert.deepStrictEqual(server.reads.slice(readsBefore), [wanted.reference]); + assert.deepStrictEqual(server.created.slice(createdBefore), [ + { instanceId: dependent, secrets: ["wanted-key"] }, + ]); + assert.deepStrictEqual(server.refreshed, [dependent]); + assert.deepStrictEqual(yield* server.registry.listUnavailable, []); + }), + ); + + it.effect("an unreadable secret fails naming its provider and leaves instances alone", () => + Effect.gen(function* () { + const vault = new Map>([ + [wanted.reference, [undefined]], + [other.reference, ["other-key"]], + ]); + const server = yield* setup(vault); + const createdBefore = server.created.length; + + const result = yield* server.secretCheck.check(wanted); + + assert.strictEqual(result._tag, "Failed"); + if (result._tag === "Failed") assert.include(result.detail, "Personal"); + assert.deepStrictEqual(server.created.slice(createdBefore), []); + assert.deepStrictEqual(server.refreshed, []); + assert.deepStrictEqual( + (yield* server.registry.listUnavailable).map((entry) => entry.instanceId), + [dependent], + ); + }), + ); + + it.effect("a failed check keeps the value the server already holds", () => + Effect.gen(function* () { + const vault = new Map>([ + [wanted.reference, ["wanted-key"]], + [locked.reference, ["locked-key", undefined]], + [other.reference, ["other-key"]], + ]); + const server = yield* setup(vault, [wanted, locked]); + const readsBefore = server.reads.length; + + assert.strictEqual((yield* server.secretCheck.check(locked))._tag, "Failed"); + + assert.deepStrictEqual(yield* server.resolver.read([locked]), ["locked-key"]); + assert.deepStrictEqual(server.reads.slice(readsBefore), [locked.reference]); + }), + ); + + it.effect("a check applies to an instance whose other secret failed a later refresh", () => + Effect.gen(function* () { + const vault = new Map>([ + [wanted.reference, ["wanted-old"]], + [locked.reference, ["locked-key", undefined]], + [other.reference, ["other-key"]], + ]); + const server = yield* setup(vault, [wanted, locked]); + // A Settings refresh while the vault is locked keeps the live instance and caches the failure. + yield* server.mutator.reloadSecrets(dependent); + vault.set(wanted.reference, ["wanted-new"]); + + assert.deepStrictEqual(yield* server.secretCheck.check(wanted), { _tag: "Resolved" }); + + assert.deepStrictEqual(server.created.at(-1), { + instanceId: dependent, + secrets: ["wanted-new", "locked-key"], + }); + assert.deepStrictEqual(yield* server.registry.listUnavailable, []); + }), + ); + + it.effect("a check applies the value it read when a refresh re-reads that secret first", () => + Effect.forEach( + // A live instance holding the old value, and one parked on a locked vault. + ["wanted-old", undefined], + (initial) => + Effect.gen(function* () { + const vault = new Map>([ + [wanted.reference, [initial]], + [other.reference, ["other-key"]], + ]); + const readStarted = yield* Deferred.make(); + const release = yield* Deferred.make(); + let gate: Effect.Effect = Effect.void; + const server = yield* setup(vault, [wanted], { + beforeRead: () => { + const current = gate; + gate = Effect.void; + return current; + }, + }); + // The check reads the rotated value; the refresh's read after it finds the vault locked. + vault.set(wanted.reference, ["wanted-new", undefined]); + gate = Deferred.succeed(readStarted, undefined).pipe( + Effect.andThen(Deferred.await(release)), + ); + + const check = yield* server.secretCheck + .check(wanted) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(readStarted); + // The refresh takes the registry lock, then waits on the resolver for the check's read. + const refresh = yield* server.mutator + .reloadSecrets(dependent) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.succeed(release, undefined); + + assert.deepStrictEqual(yield* Fiber.join(check), { _tag: "Resolved" }); + yield* Fiber.join(refresh); + assert.deepStrictEqual(server.created.at(-1), { + instanceId: dependent, + secrets: ["wanted-new"], + }); + assert.deepStrictEqual(yield* server.registry.listUnavailable, []); + }).pipe(Effect.scoped), + { discard: true }, + ), + ); + + it.effect("a check replaces only its own key of a record-valued config secret", () => + Effect.gen(function* () { + const vault = new Map>([ + [wanted.reference, ["wanted-old"]], + [locked.reference, ["locked-key"]], + [other.reference, ["other-key"]], + ]); + const server = yield* setup(vault, [], { + dependentConfig: { keys: { a: wanted, b: locked } }, + }); + vault.set(wanted.reference, ["wanted-new"]); + + assert.deepStrictEqual(yield* server.secretCheck.check(wanted), { _tag: "Resolved" }); + + assert.deepStrictEqual(server.created.at(-1), { instanceId: dependent, secrets: [] }); + assert.deepStrictEqual(server.createdConfigs.at(-1), { + keys: { a: "wanted-new", b: "locked-key" }, + }); + }), + ); +}); diff --git a/apps/server/src/secrets/SecretCheck.ts b/apps/server/src/secrets/SecretCheck.ts new file mode 100644 index 000000000000..f2a6241eeb7a --- /dev/null +++ b/apps/server/src/secrets/SecretCheck.ts @@ -0,0 +1,39 @@ +import type { SecretReference, SecretSourceCheckResult } from "@t3tools/contracts"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; + +import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; + +export class SecretCheck extends Context.Service< + SecretCheck, + { + /** + * Reads one secret fresh for the Settings "check" button and never returns the value. A + * readable secret also rebuilds the provider instances that read it, from that same read. + */ + readonly check: (reference: SecretReference) => Effect.Effect; + } +>()("t3/secrets/SecretCheck") {} + +const make = Effect.gen(function* () { + const resolver = yield* SecretResolver.SecretResolver; + const providerRegistry = yield* ProviderRegistry.ProviderRegistry; + + const check = Effect.fn("SecretCheck.check")(function* (reference: SecretReference) { + const [value] = yield* resolver.read([reference], { fresh: true }); + if (value === undefined || value.trim() === "") { + return { + _tag: "Failed", + detail: `Could not read this secret from "${yield* resolver.label(reference.provider)}". Check the provider's settings and that its store is unlocked.`, + } as const; + } + yield* providerRegistry.applySecretAndRefresh(reference, value); + return { _tag: "Resolved" } as const; + }); + + return SecretCheck.of({ check }); +}); + +export const layer = Layer.effect(SecretCheck, make); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 0807938a1d0e..ca859c8f1482 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -16,6 +16,7 @@ import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { EnvironmentHttpApi, type RepositoryIdentity } from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Cause from "effect/Cause"; import * as Duration from "effect/Duration"; import * as Deferred from "effect/Deferred"; @@ -28,17 +29,22 @@ import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder"; import * as BackgroundPolicy from "./background/BackgroundPolicy.ts"; import * as HostPowerMonitor from "./background/HostPowerMonitor.ts"; +import * as NonBlockingSpawner from "./process/NonBlockingSpawner.ts"; +import * as KeepAwake from "./background/KeepAwake.ts"; import * as ServerConfig from "./config.ts"; import { withUntracedRequests } from "./http.ts"; import * as ServerHttp from "./http.ts"; import { guardHttpResponseWriteErrors, + keepIdleConnections, rejectRequestsUntilServing, } from "./httpResponseErrorGuard.ts"; +import { resolveLauncherPath } from "./cloud/pinnedRuntime.ts"; import { fixPath } from "./os-jank.ts"; import * as Ws from "./ws.ts"; import * as AgentScopeLive from "./process/agentScope.ts"; import * as ExternalLauncher from "./process/externalLauncher.ts"; +import * as SpawnWorker from "./process/SpawnWorker.ts"; import * as NodePtyAdapter from "./terminal/NodePtyAdapter.ts"; import * as PullRequestHttp from "./pullRequest/http.ts"; import * as SpeechModelStore from "./speech/SpeechModelStore.ts"; @@ -73,6 +79,7 @@ import * as SourceControlBuiltInDrivers from "./sourceControl/builtInDrivers.ts" import * as SourceControlSignIn from "./sourceControl/SourceControlSignIn.ts"; import { BUILT_IN_SOURCE_CONTROL_DRIVER_LIST } from "./sourceControl/sourceControlDefinitions.ts"; import * as TextGeneration from "./textGeneration/TextGeneration.ts"; +import * as SecretCheck from "./secrets/SecretCheck.ts"; import * as SecretResolver from "./secrets/SecretResolver.ts"; import * as ProviderInstanceRegistryHydration from "./provider/ProviderInstanceRegistryHydration.ts"; import * as ProjectSettingsOverrideSweep from "./provider/ProjectSettingsOverrideSweep.ts"; @@ -125,6 +132,7 @@ import * as KnownWorkspaces from "./workspace/KnownWorkspaces.ts"; import * as WorkspaceFileSystem from "./workspace/WorkspaceFileSystem.ts"; import * as WorkspacePaths from "./workspace/WorkspacePaths.ts"; import * as GitVcsDriver from "./vcs/GitVcsDriver.ts"; +import * as VcsAutomaticFetchInterval from "./vcs/VcsAutomaticFetchInterval.ts"; import * as VcsDriverRegistry from "./vcs/VcsDriverRegistry.ts"; import * as VcsProjectConfig from "./vcs/VcsProjectConfig.ts"; import * as VcsProcess from "./vcs/VcsProcess.ts"; @@ -173,12 +181,14 @@ import * as ConnectReachability from "./cloud/ConnectReachability.ts"; import { pendingServiceUpdateExists } from "./cloud/updateHandoff.ts"; import * as RelayTracing from "./cloud/relayTracing.ts"; import * as CloudManagedEndpointRuntime from "./cloud/ManagedEndpointRuntime.ts"; +import * as RelayClientLedger from "./cloud/RelayClientLedger.ts"; import { MANAGED_TUNNEL_FIRST_REGISTRATION_JITTER, makeManagedTunnelRecovery, managedTunnelStartupAction, retryDesiredLinkReconcile, retryManagedTunnelRegistration, + watchManagedTunnelOrigin, } from "./cloud/managedTunnelStartup.ts"; import * as CloudCliTokenManager from "./cloud/CliTokenManager.ts"; import * as CloudCliState from "./cloud/CliState.ts"; @@ -223,6 +233,7 @@ import * as RelayClient from "@t3tools/shared/relayClient"; import { disableTailscaleServe, ensureTailscaleServe, + readTailscaleServeRootHolder, readTailscaleServeRoutesTo, readTailscaleStatus, } from "@t3tools/tailscale"; @@ -344,6 +355,9 @@ const layerRelayClient = Layer.unwrap( }), ).pipe(Layer.provide(ManagedPolicy.layer)); +// NodeServices and NodeHttpServer each output Node's own spawner, so each is followed by this one. +const layerSpawner = SpawnWorker.layerServer.pipe(Layer.provide(NodeServices.layer)); + const layerHttpServer = Layer.unwrap( Effect.gen(function* () { const config = yield* ServerConfig.ServerConfig; @@ -359,7 +373,7 @@ const layerHttpServer = Layer.unwrap( return NodeHttpServer.layer( () => { const server = rejectRequestsUntilServing( - guardHttpResponseWriteErrors(NodeHttp.createServer()), + keepIdleConnections(guardHttpResponseWriteErrors(NodeHttp.createServer())), ); if (config.privateNetworkOnly) { server.on("connection", (socket) => { @@ -380,12 +394,18 @@ const layerHttpServer = Layer.unwrap( // `threshold` when context takeover is disabled). websocket: { perMessageDeflate: true }, }, - ); + ).pipe(Layer.merge(layerSpawner)); }), -).pipe(Layer.provide(ManagedPolicy.layer), Layer.provide(NetService.layer)); +).pipe( + Layer.provide(ManagedPolicy.layer), + Layer.provide(NetService.layer), + // NodeHttpServer.layer also outputs NodeServices, whose spawner would shadow the wrapped one. + (layer) => NonBlockingSpawner.layer.pipe(Layer.provideMerge(layer)), +); const layerPlatformServices = DocumentText.layer.pipe( - Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), + Layer.provideMerge(NonBlockingSpawner.layer), + Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer, layerSpawner)), ); const layerPersistence = Layer.empty.pipe(Layer.provideMerge(SqlitePersistence.layerConfig)); @@ -481,7 +501,6 @@ const layerVcs = Layer.empty.pipe( Layer.provideMerge( VcsStatusBroadcaster.layer.pipe( Layer.provide(layerGitWorkflow), - Layer.provide(VcsStatusBroadcaster.layerAutomaticFetchInterval), // The orchestration runtime also consumes the broadcaster (run // finalization), so the policy cannot read the store from the runtime's // output. @@ -494,9 +513,14 @@ const layerVcs = Layer.empty.pipe( const layerCheckpointStore = CheckpointStore.layer.pipe(Layer.provide(layerVcsDriverRegistry)); -const layerPortScanner = PortScanner.layer.pipe(Layer.provide(ProcessRunner.layer)); +const layerPortScanner = PortScanner.layer.pipe( + Layer.provide(ProcessRunner.layer), + Layer.provide(layerBackground), +); -const layerTerminal = TerminalManager.layer.pipe( +const layerTerminal = TerminalManager.layer( + ProviderInstanceRegistryHydration.deriveProviderInstanceConfigMap, +).pipe( Layer.provide(layerPtyAdapter), Layer.provide(layerPortScanner), Layer.provide(layerNativeTelemetry), @@ -545,6 +569,14 @@ const layerCloudManagedEndpointRuntime = Layer.mergeAll( CloudManagedEndpointRuntime.layer.pipe( Layer.provide(ServerSecretStore.layer), Layer.provide(layerRelayClient), + Layer.provide( + Layer.unwrap( + Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + return RelayClientLedger.layer({ stateDir: config.stateDir }); + }), + ), + ), ), ); @@ -572,6 +604,7 @@ const layerScheduledTaskWebhookOrigin = Layer.effect( ); const layerOrchestrationV2Runtime = RuntimeLayer.layerProduction.pipe( + Layer.provide(UsageLimitSources.layerResetFallback), Layer.provide(layerScheduledTaskWebhookOrigin), Layer.provide(ProviderUsageAnalytics.layer), Layer.provide(layerCheckpointStore), @@ -659,6 +692,7 @@ const layerRuntimeCoreDependenciesBase = Layer.mergeAll( Layer.provide(layerServerRestart), ), layerThreadPullRequestWorker, + KeepAwake.layer, DelegatedTaskAnalytics.layer, Layer.effectDiscard( Effect.gen(function* () { @@ -695,7 +729,10 @@ const layerRuntimeCoreDependenciesBase = Layer.mergeAll( Layer.provideMerge(layerOrchestrationApplication), Layer.provideMerge(RuntimeLayer.layerEventInfrastructure), Layer.provideMerge(Layer.merge(ProjectStore.layer, ThreadSearch.layer)), + Layer.provideMerge(SecretCheck.layer), Layer.provideMerge(SecretResolver.layer), + // Above the settings it writes when a theme is applied. + Layer.provideMerge(EnvironmentTheme.layer), Layer.provideMerge(layerServerSettings), Layer.provideMerge(layerSourceControlProviderRegistry), Layer.provideMerge(layerGit), @@ -722,7 +759,9 @@ const layerRuntimeCoreDependenciesBase = Layer.mergeAll( Layer.provideMerge(layerPortScanner), Layer.provideMerge( Layer.mergeAll( - AntigravityInstallation.AntigravityInstallation.layer, + AntigravityInstallation.AntigravityInstallation.layer.pipe( + Layer.provide(layerServerSettings), + ), CodexInstallation.CodexInstallation.layer, ), ), @@ -782,8 +821,11 @@ const layerRuntimeDependencies = layerRuntimeCoreDependencies.pipe( Layer.provideMerge(AnalyticsService.layer.pipe(Layer.provide(ManagedPolicy.layer))), Layer.provideMerge(ExternalLauncher.layer), Layer.provideMerge(RemoteOpenTargets.layer), - Layer.provideMerge(DirectEndpoints.layer), + Layer.provideMerge(DirectEndpoints.layer.pipe(Layer.provide(layerServerEnvironment))), Layer.provideMerge(ServerLifecycleEvents.layer), + // GitVcsDriver.layer is memoized and some of its build sites live outside this file, so the + // fetch interval goes in context here, before the first build captures it. + Layer.provideMerge(VcsAutomaticFetchInterval.layer.pipe(Layer.provide(layerServerSettings))), Layer.provide(NetService.layer), ); @@ -880,6 +922,8 @@ const layerMakeServer = Layer.unwrap( const routesReady = yield* Deferred.make(); const layerLauncher = ServiceLauncherClient.layer; + // fixPath rewrites PATH, so pin the launcher the shell chose before it does. + const invokedAs = (yield* resolveLauncherPath) ?? (yield* HostProcess.InvokedAs); yield* fixPath({ shellEnvironmentPrepared: config.shellEnvironmentPrepared }); const layerHttpListening = Layer.effectDiscard( @@ -889,6 +933,17 @@ const layerMakeServer = Layer.unwrap( yield* startup.markHttpListening; }), ); + const layerDesktopReadySignal = Layer.effectDiscard( + Effect.gen(function* () { + const startup = yield* ServerRuntimeStartup.ServerRuntimeStartup; + const receiver = yield* DesktopTelemetryReceiver.DesktopTelemetryReceiver; + yield* startup.awaitCommandReady.pipe( + Effect.andThen(receiver.announceServerReady), + Effect.ignore, + Effect.forkScoped, + ); + }), + ); const layerRuntimeState = Layer.effectDiscard( Effect.gen(function* () { yield* Deferred.succeed(runtimeStateParked, undefined).pipe(Effect.orDie); @@ -917,6 +972,8 @@ const layerMakeServer = Layer.unwrap( ); }), ); + const tailscaleServeRootHolder = (target: { readonly host: string; readonly port: number }) => + readTailscaleServeRootHolder(config.tailscaleServePort, target); const layerTailscaleServe = config.tailscaleServeEnabled ? Layer.effectDiscard( Effect.acquireRelease( @@ -931,16 +988,31 @@ const layerMakeServer = Layer.unwrap( const localPort = address.port; const localHost = loopbackUrlHostForAddress(address); + const route = { + localPort, + urlHost: localHost, + localHost: localHost.replace(/^\[|\]$/gu, ""), + servePort: config.tailscaleServePort, + }; + const holder = yield* tailscaleServeRootHolder({ + host: route.localHost, + port: localPort, + }).pipe(Effect.orElseSucceed(() => "unreadable" as const)); + if (holder === "other" || holder === "unreadable") { + yield* Effect.logWarning( + holder === "other" + ? "Tailscale Serve left off; its port already serves another server or app" + : "Tailscale Serve left off; its current routes could not be read", + { localPort, servePort: config.tailscaleServePort }, + ); + return route; + } return yield* ensureTailscaleServe({ localPort, servePort: config.tailscaleServePort, localHost, }).pipe( - Effect.as({ - localPort, - localHost: localHost.replace(/^\[|\]$/gu, ""), - servePort: config.tailscaleServePort, - }), + Effect.as(route), Effect.tap(() => Effect.logInfo("Tailscale Serve configured", { localPort, @@ -948,11 +1020,13 @@ const layerMakeServer = Layer.unwrap( }), ), Effect.catch((cause) => - Effect.logWarning("Failed to configure Tailscale Serve", { - cause, - localPort, - servePort: config.tailscaleServePort, - }).pipe(Effect.as(null)), + Effect.logWarning( + cause._tag === "TailscaleCommandExitError" && + cause.stderrDiagnostic === "unsupported-target" + ? `Tailscale Serve left off; this Tailscale install can only proxy to localhost, not ${localHost}. Bind --host 127.0.0.1 or 0.0.0.0 instead` + : "Failed to configure Tailscale Serve", + { cause, localPort, servePort: config.tailscaleServePort }, + ).pipe(Effect.as(null)), ), ); }), @@ -1005,8 +1079,42 @@ const layerMakeServer = Layer.unwrap( ), ) : Effect.void, + ).pipe( + // Startup is not the only time the route can go stale, so whoever finds it on a dead + // port takes it back. + Effect.tap((configured) => + configured === null + ? Effect.void + : Effect.sleep("30 seconds").pipe( + Effect.andThen( + tailscaleServeRootHolder({ + host: configured.localHost, + port: configured.localPort, + }), + ), + Effect.flatMap((holder) => + holder === "dead" + ? ensureTailscaleServe({ + localPort: configured.localPort, + servePort: configured.servePort, + localHost: configured.urlHost, + }).pipe( + Effect.andThen( + Effect.logInfo("Tailscale Serve repaired", { + localPort: configured.localPort, + servePort: configured.servePort, + }), + ), + ) + : Effect.void, + ), + Effect.ignore, + Effect.forever, + Effect.forkScoped, + ), + ), ), - ) + ).pipe(Layer.provide(NetService.layer)) : Layer.empty; const layerCloudDesiredLinkReconcile = Layer.effectDiscard( Effect.gen(function* () { @@ -1071,6 +1179,28 @@ const layerMakeServer = Layer.unwrap( Stream.runForEach(recoverManagedTunnel), Effect.forkScoped, ); + yield* watchManagedTunnelOrigin({ + localOrigin, + readIngressOrigin: endpointRuntime.inspectConnector.pipe( + Effect.map((connector) => connector?.origin ?? null), + ), + // Like startup, a publish-only link never routes the tunnel to this host. + register: CloudCliState.readCliDesiredLinkMode.pipe( + Effect.flatMap((mode) => + mode === "publish_only" + ? Effect.void + : cloudLink + .registerManagedTunnelRecovery(localOrigin) + .pipe( + Effect.flatMap((registration) => + registration.status === "recovery_required" + ? endpointRuntime.requestRecovery(registration.config) + : Effect.void, + ), + ), + ), + ), + }).pipe(Effect.forkScoped); const wantsCliLink = hasCloudPublicConfig ? yield* CloudCliState.readCliDesiredCloudLink.pipe( Effect.catch((cause) => @@ -1218,6 +1348,7 @@ const layerMakeServer = Layer.unwrap( const layerServerApplication = Layer.mergeAll( layerRoutes, layerHttpListening, + layerDesktopReadySignal, layerRuntimeState.pipe(Layer.provide(layerLauncher)), layerTailscaleServe, layerCloudDesiredLinkReconcile, @@ -1242,6 +1373,7 @@ const layerMakeServer = Layer.unwrap( Layer.provide(VcsProcess.layer), Layer.provideMerge(AgentScopeLive.layer), Layer.provideMerge(layerPlatformServices), + Layer.provide(Layer.succeed(HostProcess.InvokedAs, invokedAs)), ); }), ).pipe( @@ -1267,9 +1399,10 @@ const layerMakeServer = Layer.unwrap( Layer.provide(VcsProcess.layer.pipe(Layer.provide(ProcessRunner.layer))), ); -// Keep the native TypeScript compiler from widening the CLI import requirements to any. -export const runServer: Effect.Effect< - never, - Layer.Error, - NodeServices.NodeServices | ServerConfig.ServerConfig -> = Layer.launch(layerMakeServer.pipe(Layer.provide(NodeCrypto.layer))); +export const runServer = Layer.launch( + // For the layers above whose spawner requirement is not met by one of them. + layerMakeServer.pipe( + Layer.provide(NodeCrypto.layer), + Layer.provide(NonBlockingSpawner.layer.pipe(Layer.provide(layerSpawner))), + ), +); diff --git a/apps/server/src/serverLifecycleEvents.ts b/apps/server/src/serverLifecycleEvents.ts index 7d394e4dc913..d90b7fa57bc3 100644 --- a/apps/server/src/serverLifecycleEvents.ts +++ b/apps/server/src/serverLifecycleEvents.ts @@ -1,5 +1,6 @@ import type { ServerLifecycleStreamEvent } from "@t3tools/contracts"; import * as Context from "effect/Context"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as PubSub from "effect/PubSub"; @@ -22,11 +23,14 @@ export class ServerLifecycleEvents extends Context.Service< readonly publish: (event: LifecycleEventInput) => Effect.Effect; readonly snapshot: Effect.Effect; readonly stream: Stream.Stream; + /** Completes once the first ready event is published. */ + readonly awaitReady: Effect.Effect; } >()("t3/serverLifecycleEvents") {} const make = Effect.gen(function* () { const pubsub = yield* PubSub.unbounded(); + const ready = yield* Deferred.make(); const state = yield* Ref.make({ sequence: 0, events: [], @@ -45,8 +49,14 @@ const make = Effect.gen(function* () { ...current.events.filter((entry) => entry.type !== nextEvent.type), ]; return [nextEvent, { sequence: nextSequence, events: nextEvents }] as const; - }).pipe(Effect.tap((event) => PubSub.publish(pubsub, event))), + }).pipe( + Effect.tap((event) => PubSub.publish(pubsub, event)), + Effect.tap((event) => + event.type === "ready" ? Deferred.succeed(ready, undefined) : Effect.void, + ), + ), snapshot: Ref.get(state), + awaitReady: Deferred.await(ready), get stream() { return Stream.fromPubSub(pubsub); }, diff --git a/apps/server/src/serverOwnership.ts b/apps/server/src/serverOwnership.ts index cd182e7f674f..3610e4e3ca23 100644 --- a/apps/server/src/serverOwnership.ts +++ b/apps/server/src/serverOwnership.ts @@ -28,6 +28,10 @@ import { compareExactServiceVersions, serviceStateActiveVersion, } from "./cloud/serviceProtocol.ts"; +import { + legacyOwnerStartedBeforeRecord, + legacyOwnerStartTimeCommand, +} from "./legacyServerOwner.ts"; import * as ProcessRunner from "./processRunner.ts"; import { clearPersistedServerRuntimeState, @@ -81,29 +85,20 @@ const legacyOwnerIsLive = Effect.fn("legacyOwnerIsLive")(function* ( if (!isProcessAlive(state.pid)) return false; const recordedAt = Date.parse(state.startedAt); if (!Number.isFinite(recordedAt)) return true; - const windows = (yield* HostProcess.Platform) === "win32"; + const platform = yield* HostProcess.Platform; const runner = yield* ProcessRunner.ProcessRunner; const result = yield* runner .run({ - command: windows ? "powershell.exe" : "ps", - args: windows - ? [ - "-NoProfile", - "-NonInteractive", - "-Command", - `(Get-Process -Id ${state.pid} -ErrorAction Stop).StartTime.ToUniversalTime().ToString('o')`, - ] - : ["-p", String(state.pid), "-o", "lstart="], - env: { LC_ALL: "C", TZ: "UTC" }, + ...legacyOwnerStartTimeCommand(state.pid, platform), timeout: Duration.seconds(2), maxOutputBytes: 16_384, }) .pipe(Effect.option); - if (Option.isNone(result) || result.value.code !== 0) return true; - const output = result.value.stdout.trim(); - const startedAt = Date.parse(windows ? output : `${output} UTC`); - // ps reports whole seconds. Unknown identity stays conservative. - return !Number.isFinite(startedAt) || startedAt <= recordedAt + 1_000; + return legacyOwnerStartedBeforeRecord( + Option.isSome(result) && result.value.code === 0 ? result.value.stdout : undefined, + recordedAt, + platform, + ); }); // Pre-lock services become invisible when another server replaces their runtime @@ -194,7 +189,8 @@ const openOwnershipLock = (stateDir: string) => } const db = new NodeSqlite.DatabaseSync(lockPath); try { - db.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;"); + // An idle service launcher probes this lock for a moment; wait out that probe. + db.exec("PRAGMA busy_timeout = 250; BEGIN EXCLUSIVE;"); const identity = NodeFS.statSync(lockPath, { bigint: true }); return { db, diff --git a/apps/server/src/serverRuntimeStartup.autoPull.test.ts b/apps/server/src/serverRuntimeStartup.autoPull.test.ts index ccb4183ea99d..708d57f065ac 100644 --- a/apps/server/src/serverRuntimeStartup.autoPull.test.ts +++ b/apps/server/src/serverRuntimeStartup.autoPull.test.ts @@ -37,172 +37,166 @@ import * as AnalyticsService from "./telemetry/AnalyticsService.ts"; import * as GitVcsDriver from "./vcs/GitVcsDriver.ts"; it.effect("parks automatic pull until activation without delaying command readiness", () => - Effect.scoped( - Effect.gen(function* () { - const activation = yield* Deferred.make(); - const prepared = yield* Deferred.make(); - const commitTrial = yield* Deferred.make(); - const statusCalled = yield* Deferred.make(); - const statusInterrupted = yield* Deferred.make(); - const cwd = "/auto-pull-project"; - const updatedAt = "2026-01-01T00:00:00.000Z"; - const snapshot = { - projects: [ - { - id: ProjectId.make("auto-pull-project"), - title: "Auto pull project", - workspaceRoot: cwd, - repositoryIdentity: null, - defaultModelSelection: null, - scripts: [], - createdAt: updatedAt, - updatedAt, - deletedAt: null, - }, - ], - updatedAt, - }; - const recovery = { - terminalizedRuns: 0, - stoppedSessions: 0, - closedRequests: 0, - retiredEffects: 0, - requeuedEffects: 0, - }; - const importSummary = { importedThreadCount: 0, importedMessageCount: 0 }; - const dependencies: Layer.Layer< - Layer.Services> - > = Layer.mergeAll( - Layer.mock(ServerConfig.ServerConfig)({ - ...(yield* ServerConfig.deriveServerPaths(cwd, undefined).pipe( - Effect.provide(Path.layer), - )), - baseDir: cwd, - dbPath: ":memory:", - logLevel: "Error", - traceMinLevel: "Info", - traceTimingEnabled: false, - traceBatchWindowMs: 200, - traceMaxBytes: 1024, - traceMaxFiles: 1, - otlpTracesUrl: undefined, - otlpMetricsUrl: undefined, - otlpLogsUrl: undefined, - otlpTracesExport: DEFAULT_SIGNAL_EXPORT, - otlpMetricsExport: DEFAULT_SIGNAL_EXPORT, - otlpLogsExport: DEFAULT_SIGNAL_EXPORT, - otelEnvironment: OtelEnvironment.none, - staticDir: undefined, - devAllowedOrigins: [], - desktopBootstrapToken: undefined, - logWebSocketEvents: false, - tailscaleServeEnabled: false, - tailscaleServePort: 443, - mode: "desktop", - cwd, - host: "localhost", - port: 3773, - devUrl: undefined, - noBrowser: true, - startupPresentation: "browser", - autoBootstrapProjectFromCwd: false, + Effect.gen(function* () { + const activation = yield* Deferred.make(); + const prepared = yield* Deferred.make(); + const commitTrial = yield* Deferred.make(); + const statusCalled = yield* Deferred.make(); + const statusInterrupted = yield* Deferred.make(); + const cwd = "/auto-pull-project"; + const updatedAt = "2026-01-01T00:00:00.000Z"; + const snapshot = { + projects: [ + { + id: ProjectId.make("auto-pull-project"), + title: "Auto pull project", + workspaceRoot: cwd, + repositoryIdentity: null, + defaultModelSelection: null, + scripts: [], + createdAt: updatedAt, + updatedAt, + deletedAt: null, + }, + ], + updatedAt, + }; + const recovery = { + terminalizedRuns: 0, + stoppedSessions: 0, + closedRequests: 0, + retiredEffects: 0, + requeuedEffects: 0, + }; + const importSummary = { importedThreadCount: 0, importedMessageCount: 0 }; + const dependencies: Layer.Layer< + Layer.Services> + > = Layer.mergeAll( + Layer.mock(ServerConfig.ServerConfig)({ + ...(yield* ServerConfig.deriveServerPaths(cwd, undefined).pipe(Effect.provide(Path.layer))), + baseDir: cwd, + dbPath: ":memory:", + logLevel: "Error", + traceMinLevel: "Info", + traceTimingEnabled: false, + traceBatchWindowMs: 200, + traceMaxBytes: 1024, + traceMaxFiles: 1, + otlpTracesUrl: undefined, + otlpMetricsUrl: undefined, + otlpLogsUrl: undefined, + otlpTracesExport: DEFAULT_SIGNAL_EXPORT, + otlpMetricsExport: DEFAULT_SIGNAL_EXPORT, + otlpLogsExport: DEFAULT_SIGNAL_EXPORT, + otelEnvironment: OtelEnvironment.none, + staticDir: undefined, + devAllowedOrigins: [], + desktopBootstrapToken: undefined, + logWebSocketEvents: false, + tailscaleServeEnabled: false, + tailscaleServePort: 443, + mode: "desktop", + cwd, + host: "localhost", + port: 3773, + devUrl: undefined, + noBrowser: true, + startupPresentation: "browser", + autoBootstrapProjectFromCwd: false, + }), + Layer.mock(Keybindings.Keybindings)({ start: Effect.void }), + Layer.mock(LegacyV1ThreadImporter.LegacyV1ThreadImporter)({ + pendingThreadCount: Effect.succeed(0), + reconcileShells: Effect.succeed(importSummary), + importPendingTranscripts: Effect.succeed(importSummary), + }), + Layer.mock(ProviderRuntimeRecovery.ProviderRuntimeRecoveryService)({ + recover: Effect.succeed(recovery), + prepareForShutdown: Effect.void, + reconcile: () => Effect.succeed(recovery), + }), + Layer.mock(Orchestrator.OrchestratorV2)({ recoverDelegatedTasks: Effect.void }), + Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ shutdown: Effect.void }), + Layer.mock(AgentAwarenessRelay.AgentAwarenessRelay)({ start: () => Effect.void }), + Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ runOnce: () => Effect.never }), + Layer.mock(ServerLifecycleEvents.ServerLifecycleEvents)({ + publish: (event) => Effect.succeed({ ...event, sequence: 1 }), + }), + Layer.mock(ServerSettings.ServerSettingsService)({ + start: Effect.void, + getSettings: Effect.succeed({ ...DEFAULT_SERVER_SETTINGS, defaultAutoPull: true }), + }), + Layer.mock(ServerEnvironment.ServerEnvironment)({ + getDescriptor: Effect.succeed({ + environmentId: EnvironmentId.make("auto-pull-environment"), + label: "Test environment", + platform: { os: "darwin", arch: "arm64" }, + serverVersion: "0.0.0-test", + capabilities: { repositoryIdentity: true }, }), - Layer.mock(Keybindings.Keybindings)({ start: Effect.void }), - Layer.mock(LegacyV1ThreadImporter.LegacyV1ThreadImporter)({ - pendingThreadCount: Effect.succeed(0), - reconcileShells: Effect.succeed(importSummary), - importPendingTranscripts: Effect.succeed(importSummary), - }), - Layer.mock(ProviderRuntimeRecovery.ProviderRuntimeRecoveryService)({ - recover: Effect.succeed(recovery), - prepareForShutdown: Effect.void, - reconcile: () => Effect.succeed(recovery), - }), - Layer.mock(Orchestrator.OrchestratorV2)({ recoverDelegatedTasks: Effect.void }), - Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ shutdown: Effect.void }), - Layer.mock(AgentAwarenessRelay.AgentAwarenessRelay)({ start: () => Effect.void }), - Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ runOnce: () => Effect.never }), - Layer.mock(ServerLifecycleEvents.ServerLifecycleEvents)({ - publish: (event) => Effect.succeed({ ...event, sequence: 1 }), - }), - Layer.mock(ServerSettings.ServerSettingsService)({ - start: Effect.void, - getSettings: Effect.succeed({ ...DEFAULT_SERVER_SETTINGS, defaultAutoPull: true }), - }), - Layer.mock(ServerEnvironment.ServerEnvironment)({ - getDescriptor: Effect.succeed({ - environmentId: EnvironmentId.make("auto-pull-environment"), - label: "Test environment", - platform: { os: "darwin", arch: "arm64" }, - serverVersion: "0.0.0-test", - capabilities: { repositoryIdentity: true }, - }), - }), - Layer.mock(ProjectStore.ProjectStoreV2)({ - listShells: () => Effect.succeed(snapshot.projects), - }), - Layer.mock(ProjectService.ProjectService)({ snapshot: Effect.succeed(snapshot) }), - Layer.mock(ThreadManagement.ThreadManagementService)({ - countThreads: () => Effect.succeed(0), - }), - Layer.mock(ThreadLaunch.ThreadLaunchService)({}), - Layer.mock(ServiceLauncherClient.ServiceLauncherClient)({ - managed: true, - protocol: SERVICE_LAUNCHER_PROTOCOL, - prepareTrial: () => - Deferred.succeed(prepared, undefined).pipe( - Effect.andThen(Deferred.await(commitTrial)), - Effect.as(undefined), - ), - }), - Layer.mock(GitVcsDriver.GitVcsDriver)({ - statusDetails: (root) => - Deferred.succeed(statusCalled, root).pipe( - Effect.andThen(Effect.never), - Effect.onInterrupt(() => Deferred.succeed(statusInterrupted, undefined)), - ), - }), - Layer.mock(AnalyticsService.AnalyticsService)({ record: () => Effect.void }), - // No provider logins or Connect link, so startup records no owners. - FileSystem.layerNoop({}), - Layer.mock(ServerSecretStore.ServerSecretStore)({ get: () => Effect.succeedNone }), - NodeCrypto.layer, - Layer.mock(EnvironmentAuth.EnvironmentAuth)({}), - Layer.mock(ExternalLauncher.ExternalLauncher)({}), - Layer.mock(HttpServer.HttpServer)({ - address: NetAddress.inetAddressFromIpStringUnsafe("127.0.0.1", 3773), - }), - Path.layer, - ); - - yield* Effect.gen(function* () { - const startup = yield* ServerRuntimeStartup.ServerRuntimeStartup; - yield* startup.markHttpListening; + }), + Layer.mock(ProjectStore.ProjectStoreV2)({ + listShells: () => Effect.succeed(snapshot.projects), + }), + Layer.mock(ProjectService.ProjectService)({ snapshot: Effect.succeed(snapshot) }), + Layer.mock(ThreadManagement.ThreadManagementService)({ + countThreads: () => Effect.succeed(0), + }), + Layer.mock(ThreadLaunch.ThreadLaunchService)({}), + Layer.mock(ServiceLauncherClient.ServiceLauncherClient)({ + managed: true, + protocol: SERVICE_LAUNCHER_PROTOCOL, + prepareTrial: () => + Deferred.succeed(prepared, undefined).pipe( + Effect.andThen(Deferred.await(commitTrial)), + Effect.as(undefined), + ), + }), + Layer.mock(GitVcsDriver.GitVcsDriver)({ + statusDetails: (root) => + Deferred.succeed(statusCalled, root).pipe( + Effect.andThen(Effect.never), + Effect.onInterrupt(() => Deferred.succeed(statusInterrupted, undefined)), + ), + }), + Layer.mock(AnalyticsService.AnalyticsService)({ record: () => Effect.void }), + // No provider logins or Connect link, so startup records no owners. + FileSystem.layerNoop({}), + Layer.mock(ServerSecretStore.ServerSecretStore)({ get: () => Effect.succeedNone }), + NodeCrypto.layer, + Layer.mock(EnvironmentAuth.EnvironmentAuth)({}), + Layer.mock(ExternalLauncher.ExternalLauncher)({}), + Layer.mock(HttpServer.HttpServer)({ + address: NetAddress.inetAddressFromIpStringUnsafe("127.0.0.1", 3773), + }), + Path.layer, + ); - // A reverted, awaited pull reaches statusDetails instead of prepareTrial. - // Race receipts and readiness so regressions and startup failures surface. - yield* Effect.raceFirst( - Effect.raceFirst(Deferred.await(prepared), Deferred.await(statusCalled)), - startup.awaitCommandReady, - ); - expect(yield* Deferred.isDone(activation)).toBe(false); - expect(yield* Deferred.isDone(statusCalled)).toBe(false); - expect(yield* Deferred.isDone(prepared)).toBe(true); + yield* Effect.gen(function* () { + const startup = yield* ServerRuntimeStartup.ServerRuntimeStartup; + yield* startup.markHttpListening; - yield* Deferred.succeed(commitTrial, undefined); - yield* Deferred.await(activation); - expect(yield* Deferred.await(statusCalled)).toBe(cwd); - yield* startup.awaitCommandReady; - }).pipe( - Effect.provide( - ServerRuntimeStartup.layerWithOptions({ - activate: Deferred.succeed(activation, undefined).pipe(Effect.asVoid), - awaitAuxiliaryParked: Effect.void, - }).pipe(Layer.provide(dependencies)), - ), - Effect.provideService(ServerActivation, Deferred.await(activation)), + yield* Effect.raceFirst( + Effect.raceFirst(Deferred.await(prepared), Deferred.await(statusCalled)), + startup.awaitCommandReady, ); - expect(yield* Deferred.isDone(statusInterrupted)).toBe(true); - }), - ), + expect(yield* Deferred.isDone(activation)).toBe(false); + expect(yield* Deferred.isDone(statusCalled)).toBe(false); + expect(yield* Deferred.isDone(prepared)).toBe(true); + + yield* Deferred.succeed(commitTrial, undefined); + yield* Deferred.await(activation); + expect(yield* Deferred.await(statusCalled)).toBe(cwd); + yield* startup.awaitCommandReady; + }).pipe( + Effect.provide( + ServerRuntimeStartup.layerWithOptions({ + activate: Deferred.succeed(activation, undefined).pipe(Effect.asVoid), + awaitAuxiliaryParked: Effect.void, + }).pipe(Layer.provide(dependencies)), + ), + Effect.provideService(ServerActivation, Deferred.await(activation)), + ); + expect(yield* Deferred.isDone(statusInterrupted)).toBe(true); + }), ); diff --git a/apps/server/src/serverRuntimeStartup.test.ts b/apps/server/src/serverRuntimeStartup.test.ts index a43dedb59d89..4fc30c524aec 100644 --- a/apps/server/src/serverRuntimeStartup.test.ts +++ b/apps/server/src/serverRuntimeStartup.test.ts @@ -1,6 +1,5 @@ import { assert, it } from "@effect/vitest"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; -import * as NodeFileSystem from "@effect/platform-node/NodeFileSystem"; import { DEFAULT_SERVER_SETTINGS, ProjectId, @@ -53,114 +52,115 @@ it.effect.each([ "disabled-default", "launch-failure", "project-failure", + "project-refused", ] as const)("keeps bootstrap launch failures separate from project failures: %s", (scenario) => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-bootstrap-startup-" }); - const paths = yield* ServerConfig.deriveServerPaths(baseDir, undefined).pipe( - Effect.provide(Path.layer), - ); - yield* fs.makeDirectory(paths.stateDir, { recursive: true }); - const projectId = ProjectId.make("bootstrap-project"); - const instanceId = ProviderInstanceId.make( - scenario === "disabled-default" ? "custom" : "codex", - ); - const events: ServerLifecycleStreamEvent[] = []; - const recovery = { - terminalizedRuns: 0, - stoppedSessions: 0, - closedRequests: 0, - retiredEffects: 0, - requeuedEffects: 0, - }; - const imports = { importedThreadCount: 0, importedMessageCount: 0 }; - const dependencies = Layer.mergeAll( - NodeCrypto.layer, - Path.layer, - Layer.mock(ServerConfig.ServerConfig)({ - ...paths, - baseDir, - logLevel: "Error", - traceMinLevel: "Info", - traceTimingEnabled: false, - traceBatchWindowMs: 200, - traceMaxBytes: 1024, - traceMaxFiles: 1, - otlpTracesUrl: undefined, - otlpMetricsUrl: undefined, - otlpLogsUrl: undefined, - otlpTracesExport: DEFAULT_SIGNAL_EXPORT, - otlpMetricsExport: DEFAULT_SIGNAL_EXPORT, - otlpLogsExport: DEFAULT_SIGNAL_EXPORT, - otelEnvironment: OtelEnvironment.none, - staticDir: undefined, - devAllowedOrigins: [], - desktopBootstrapToken: undefined, - logWebSocketEvents: false, - tailscaleServeEnabled: false, - tailscaleServePort: 443, - mode: "web", - cwd: "/bootstrap-project", - host: "localhost", - port: 3773, - devUrl: undefined, - noBrowser: true, - startupPresentation: "browser", - autoBootstrapProjectFromCwd: true, - }), - Layer.mock(Keybindings.Keybindings)({ start: Effect.void }), - Layer.mock(LegacyV1ThreadImporter.LegacyV1ThreadImporter)({ - pendingThreadCount: Effect.succeed(0), - reconcileShells: Effect.succeed(imports), - importPendingTranscripts: Effect.succeed(imports), - }), - Layer.mock(ProviderRuntimeRecovery.ProviderRuntimeRecoveryService)({ - recover: Effect.succeed(recovery), - prepareForShutdown: Effect.void, - reconcile: () => Effect.succeed(recovery), - }), - Layer.mock(Orchestrator.OrchestratorV2)({ recoverDelegatedTasks: Effect.void }), - Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ shutdown: Effect.void }), - Layer.mock(AgentAwarenessRelay.AgentAwarenessRelay)({ start: () => Effect.void }), - Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ runOnce: () => Effect.never }), - Layer.mock(ServerLifecycleEvents.ServerLifecycleEvents)({ - publish: (event) => - Effect.sync(() => { - const published = { ...event, sequence: events.length + 1 }; - events.push(published); - return published; - }), - }), - Layer.mock(ServerSettings.ServerSettingsService)({ - start: Effect.void, - getSettings: Effect.succeed({ - ...DEFAULT_SERVER_SETTINGS, - defaultModelSelection: - scenario === "disabled-default" ? { instanceId, model: "custom-model" } : null, - providerInstances: { - [instanceId]: { - driver: ProviderDriverKind.make("codex"), - enabled: false, - config: {}, - }, - }, + Effect.gen(function* () { + const projectId = ProjectId.make("bootstrap-project"); + const instanceId = ProviderInstanceId.make( + scenario === "disabled-default" ? "custom" : "codex", + ); + const events: ServerLifecycleStreamEvent[] = []; + const recovery = { + terminalizedRuns: 0, + stoppedSessions: 0, + closedRequests: 0, + retiredEffects: 0, + requeuedEffects: 0, + }; + const imports = { importedThreadCount: 0, importedMessageCount: 0 }; + const dependencies = Layer.mergeAll( + NodeCrypto.layer, + Path.layer, + Layer.mock(ServerConfig.ServerConfig)({ + ...(yield* ServerConfig.deriveServerPaths("/bootstrap-project", undefined).pipe( + Effect.provide(Path.layer), + )), + baseDir: "/bootstrap-project", + dbPath: ":memory:", + logLevel: "Error", + traceMinLevel: "Info", + traceTimingEnabled: false, + traceBatchWindowMs: 200, + traceMaxBytes: 1024, + traceMaxFiles: 1, + otlpTracesUrl: undefined, + otlpMetricsUrl: undefined, + otlpLogsUrl: undefined, + otlpTracesExport: DEFAULT_SIGNAL_EXPORT, + otlpMetricsExport: DEFAULT_SIGNAL_EXPORT, + otlpLogsExport: DEFAULT_SIGNAL_EXPORT, + otelEnvironment: OtelEnvironment.none, + staticDir: undefined, + devAllowedOrigins: [], + desktopBootstrapToken: undefined, + logWebSocketEvents: false, + tailscaleServeEnabled: false, + tailscaleServePort: 443, + mode: "web", + cwd: "/bootstrap-project", + host: "localhost", + port: 3773, + devUrl: undefined, + noBrowser: true, + startupPresentation: "browser", + autoBootstrapProjectFromCwd: true, + }), + Layer.mock(Keybindings.Keybindings)({ start: Effect.void }), + Layer.mock(LegacyV1ThreadImporter.LegacyV1ThreadImporter)({ + pendingThreadCount: Effect.succeed(0), + reconcileShells: Effect.succeed(imports), + importPendingTranscripts: Effect.succeed(imports), + }), + Layer.mock(ProviderRuntimeRecovery.ProviderRuntimeRecoveryService)({ + recover: Effect.succeed(recovery), + prepareForShutdown: Effect.void, + reconcile: () => Effect.succeed(recovery), + }), + Layer.mock(Orchestrator.OrchestratorV2)({ recoverDelegatedTasks: Effect.void }), + Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ shutdown: Effect.void }), + Layer.mock(AgentAwarenessRelay.AgentAwarenessRelay)({ start: () => Effect.void }), + Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ runOnce: () => Effect.never }), + Layer.mock(ServerLifecycleEvents.ServerLifecycleEvents)({ + publish: (event) => + Effect.sync(() => { + const published = { ...event, sequence: events.length + 1 }; + events.push(published); + return published; }), + }), + Layer.mock(ServerSettings.ServerSettingsService)({ + start: Effect.void, + getSettings: Effect.succeed({ + ...DEFAULT_SERVER_SETTINGS, + defaultModelSelection: + scenario === "disabled-default" ? { instanceId, model: "custom-model" } : null, + providerInstances: { + [instanceId]: { + driver: ProviderDriverKind.make("codex"), + enabled: false, + config: {}, + }, + }, }), - Layer.mock(ServerEnvironment.ServerEnvironment)({ - getDescriptor: Effect.succeed({ - environmentId: EnvironmentId.make("bootstrap-environment"), - label: "Test environment", - platform: { os: "linux", arch: "x64" }, - serverVersion: "0.0.0-test", - capabilities: { repositoryIdentity: true }, - }), + }), + Layer.mock(ServerEnvironment.ServerEnvironment)({ + getDescriptor: Effect.succeed({ + environmentId: EnvironmentId.make("bootstrap-environment"), + label: "Test environment", + platform: { os: "linux", arch: "x64" }, + serverVersion: "0.0.0-test", + capabilities: { repositoryIdentity: true }, }), - Layer.mock(ProjectStore.ProjectStoreV2)({}), - Layer.mock(ProjectService.ProjectService)({ - bootstrap: () => - scenario === "project-failure" - ? Effect.die("project bootstrap failed") + }), + Layer.mock(ProjectStore.ProjectStoreV2)({}), + Layer.mock(ProjectService.ProjectService)({ + bootstrap: () => + scenario === "project-failure" + ? Effect.die("project bootstrap failed") + : scenario === "project-refused" + ? Effect.fail( + new ProjectService.ProjectInvalidError({ projectId, detail: "refused folder" }), + ) : Effect.succeed({ created: true, project: { @@ -179,131 +179,127 @@ it.effect.each([ : null, }, }), - }), - Layer.mock(ThreadManagement.ThreadManagementService)({ - countThreads: () => Effect.succeed(0), - listActiveShellIndex: () => Effect.succeed([]), - }), - Layer.mock(ThreadLaunch.ThreadLaunchService)({ - launch: (input) => { - assert.equal(input.modelSelection.instanceId, instanceId); - return scenario === "launch-failure" - ? Effect.fail( - new ThreadLaunch.ThreadLaunchError({ - operation: "create-thread", - commandId: CommandId.make("bootstrap-command"), - projectId, - cause: "test launch failure", - }), - ) - : Effect.fail( - new ThreadManagement.ThreadManagementProviderInstanceDisabledError({ - projectId, - instanceId, - }), - ); - }, - }), - Layer.mock(ServiceLauncherClient.ServiceLauncherClient)({ - managed: false, - protocol: undefined, - prepareTrial: () => Effect.undefined, - }), - Layer.mock(GitVcsDriver.GitVcsDriver)({}), - Layer.mock(AnalyticsService.AnalyticsService)({}), - // No provider logins or Connect link, so startup records no owners. - FileSystem.layerNoop({}), - Layer.mock(ServerSecretStore.ServerSecretStore)({ get: () => Effect.succeedNone }), - Layer.mock(EnvironmentAuth.EnvironmentAuth)({}), - Layer.mock(ExternalLauncher.ExternalLauncher)({}), - Layer.mock(HttpServer.HttpServer)({ - address: NetAddress.inetAddressFromIpStringUnsafe("127.0.0.1", 3773), - }), - ); - yield* Effect.gen(function* () { - const startup = yield* ServerRuntimeStartup.ServerRuntimeStartup; - yield* startup.markHttpListening; - const ready = yield* Effect.exit(startup.awaitCommandReady); - assert.equal(Exit.isSuccess(ready), scenario !== "project-failure"); - const welcome = events.find((event) => event.type === "welcome"); - if (scenario === "project-failure") { - assert.isUndefined(welcome); - } else { - assert.isDefined(welcome); - if (welcome?.type === "welcome") { - assert.equal(welcome.payload.bootstrapProjectId, projectId); - assert.isUndefined(welcome.payload.bootstrapThreadId); - } + }), + Layer.mock(ThreadManagement.ThreadManagementService)({ + countThreads: () => Effect.succeed(0), + listActiveShellIndex: () => Effect.succeed([]), + }), + Layer.mock(ThreadLaunch.ThreadLaunchService)({ + launch: (input) => { + assert.equal(input.modelSelection.instanceId, instanceId); + return scenario === "launch-failure" + ? Effect.fail( + new ThreadLaunch.ThreadLaunchError({ + operation: "create-thread", + commandId: CommandId.make("bootstrap-command"), + projectId, + cause: "test launch failure", + }), + ) + : Effect.fail( + new ThreadManagement.ThreadManagementProviderInstanceDisabledError({ + projectId, + instanceId, + }), + ); + }, + }), + Layer.mock(ServiceLauncherClient.ServiceLauncherClient)({ + managed: false, + protocol: undefined, + prepareTrial: () => Effect.undefined, + }), + Layer.mock(GitVcsDriver.GitVcsDriver)({}), + Layer.mock(AnalyticsService.AnalyticsService)({}), + // No provider logins or Connect link, so startup records no owners. + FileSystem.layerNoop({}), + Layer.mock(ServerSecretStore.ServerSecretStore)({ get: () => Effect.succeedNone }), + Layer.mock(EnvironmentAuth.EnvironmentAuth)({}), + Layer.mock(ExternalLauncher.ExternalLauncher)({}), + Layer.mock(HttpServer.HttpServer)({ + address: NetAddress.inetAddressFromIpStringUnsafe("127.0.0.1", 3773), + }), + ); + yield* Effect.gen(function* () { + const startup = yield* ServerRuntimeStartup.ServerRuntimeStartup; + yield* startup.markHttpListening; + const ready = yield* Effect.exit(startup.awaitCommandReady); + assert.equal(Exit.isSuccess(ready), scenario !== "project-failure"); + const welcome = events.find((event) => event.type === "welcome"); + if (scenario === "project-failure") { + assert.isUndefined(welcome); + } else { + assert.isDefined(welcome); + if (welcome?.type === "welcome") { + assert.equal( + welcome.payload.bootstrapProjectId, + scenario === "project-refused" ? undefined : projectId, + ); + assert.isUndefined(welcome.payload.bootstrapThreadId); } - }).pipe( - Effect.provide(ServerRuntimeStartup.layerWithOptions().pipe(Layer.provide(dependencies))), - Effect.provideService(ServerActivation, Effect.never), - ); - }).pipe(Effect.provide(NodeFileSystem.layer)), - ), + } + }).pipe( + Effect.provide(ServerRuntimeStartup.layerWithOptions().pipe(Layer.provide(dependencies))), + Effect.provideService(ServerActivation, Effect.never), + ); + }), ); it.effect("interrupts the effect worker when awareness relay startup fails", () => - Effect.scoped( - Effect.gen(function* () { - const workerInterrupted = yield* Ref.make(false); - const workerFiberRef = yield* Ref.make | null>(null); + Effect.gen(function* () { + const workerInterrupted = yield* Ref.make(false); + const workerFiberRef = yield* Ref.make | null>(null); - const exit = yield* ServerRuntimeStartup.startEffectWorkerWithRelay({ - runWorker: Effect.never.pipe(Effect.ensuring(Ref.set(workerInterrupted, true))), - startRelay: Effect.yieldNow.pipe( - Effect.andThen(Effect.die("awareness relay startup failed")), - ), - workerFiberRef, - }).pipe(Effect.exit); + const exit = yield* ServerRuntimeStartup.startEffectWorkerWithRelay({ + runWorker: Effect.never.pipe(Effect.ensuring(Ref.set(workerInterrupted, true))), + startRelay: Effect.yieldNow.pipe( + Effect.andThen(Effect.die("awareness relay startup failed")), + ), + workerFiberRef, + }).pipe(Effect.exit); - assert.isTrue(Exit.isFailure(exit)); - assert.isTrue(yield* Ref.get(workerInterrupted)); - assert.isNull(yield* Ref.get(workerFiberRef)); - }), - ), + assert.isTrue(Exit.isFailure(exit)); + assert.isTrue(yield* Ref.get(workerInterrupted)); + assert.isNull(yield* Ref.get(workerFiberRef)); + }), ); it.effect("queues commands until startup signals readiness", () => - Effect.scoped( - Effect.gen(function* () { - const gate = yield* ServerRuntimeStartup.makeCommandGate; - const count = yield* Ref.make(0); - const queued = yield* gate - .enqueueCommand(Ref.updateAndGet(count, (value) => value + 1)) - .pipe(Effect.forkScoped); + Effect.gen(function* () { + const gate = yield* ServerRuntimeStartup.makeCommandGate; + const count = yield* Ref.make(0); + const queued = yield* gate + .enqueueCommand(Ref.updateAndGet(count, (value) => value + 1)) + .pipe(Effect.forkScoped); - yield* Effect.yieldNow; - assert.equal(yield* Ref.get(count), 0); - yield* gate.signalCommandReady; - assert.equal(yield* Fiber.join(queued), 1); - }), - ), + yield* Effect.yieldNow; + assert.equal(yield* Ref.get(count), 0); + yield* gate.signalCommandReady; + assert.equal(yield* Fiber.join(queued), 1); + }), ); it.effect("enqueueCommand fails queued work when readiness fails", () => - Effect.scoped( - Effect.gen(function* () { - const commandGate = yield* ServerRuntimeStartup.makeCommandGate; - const failure = yield* Deferred.make(); + Effect.gen(function* () { + const commandGate = yield* ServerRuntimeStartup.makeCommandGate; + const failure = yield* Deferred.make(); - const queuedCommandFiber = yield* commandGate - .enqueueCommand(Deferred.await(failure).pipe(Effect.as("should-not-run"))) - .pipe(Effect.forkScoped); + const queuedCommandFiber = yield* commandGate + .enqueueCommand(Deferred.await(failure).pipe(Effect.as("should-not-run"))) + .pipe(Effect.forkScoped); - yield* commandGate.failCommandReady( - new ServerRuntimeStartup.ServerRuntimeStartupError({ - mode: "web", - host: "127.0.0.1", - port: 3773, - cause: new Error("test startup failure"), - }), - ); + yield* commandGate.failCommandReady( + new ServerRuntimeStartup.ServerRuntimeStartupError({ + mode: "web", + host: "127.0.0.1", + port: 3773, + cause: new Error("test startup failure"), + }), + ); - const error = yield* Effect.flip(Fiber.join(queuedCommandFiber)); - assert.equal(error.message, "Server runtime startup failed before command readiness."); - }), - ), + const error = yield* Effect.flip(Fiber.join(queuedCommandFiber)); + assert.equal(error.message, "Server runtime startup failed before command readiness."); + }), ); it.effect("automatic pull only updates enabled, behind, clean default-branch checkouts", () => diff --git a/apps/server/src/serverRuntimeStartup.ts b/apps/server/src/serverRuntimeStartup.ts index 4ddf950f1dbb..81c020d3b563 100644 --- a/apps/server/src/serverRuntimeStartup.ts +++ b/apps/server/src/serverRuntimeStartup.ts @@ -271,12 +271,24 @@ const resolveAutoBootstrapWelcomeTargets = Effect.gen(function* () { if (serverConfig.autoBootstrapProjectFromCwd) { const threadModelSelection = getAutoBootstrapThreadModelSelection(); - const { project } = yield* projects.bootstrap({ - commandId: CommandId.make(yield* randomUUID), - projectId: ProjectId.make(yield* randomUUID), - title: path.basename(serverConfig.cwd) || "project", - workspaceRoot: serverConfig.cwd, - }); + const bootstrapped = yield* projects + .bootstrap({ + commandId: CommandId.make(yield* randomUUID), + projectId: ProjectId.make(yield* randomUUID), + title: path.basename(serverConfig.cwd) || "project", + workspaceRoot: serverConfig.cwd, + }) + .pipe( + // A folder the project rules refuse must not stop the server from starting there. + Effect.catchTags({ + ProjectInvalidError: (error) => + Effect.logWarning("Skipped bootstrap project", { detail: error.detail }).pipe( + Effect.as(undefined), + ), + }), + ); + if (bootstrapped === undefined) return {} satisfies AutoBootstrapWelcomeTargets; + const { project } = bootstrapped; const existingThread = (yield* threads.listActiveShellIndex()).find( (thread) => thread.projectId === project.id && isListedThreadLineage(thread.lineage), ); diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index 347b43dd6f44..873649cb168f 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -1,3 +1,4 @@ +import { deriveProviderInstanceConfigMap } from "./provider/ProviderInstanceRegistryHydration.ts"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { @@ -539,6 +540,17 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" (yield* serverSettings.getSettings).providerInstances[instanceId]?.displayName ?? "", ), ); + + // An inherited Object.prototype member is not an instance that already exists. + for (const name of ["constructor", "toString", "valueOf"]) { + const prototypeNamedId = ProviderInstanceId.make(name); + const created = yield* serverSettings.updateProviderInstance({ + operation: "create", + instanceId: prototypeNamedId, + instance: { driver: ProviderDriverKind.make("codex") }, + }); + assert.isTrue(Object.hasOwn(created.providerInstances, prototypeNamedId)); + } }).pipe(Effect.provide(layerServerSettings())), ); @@ -1457,7 +1469,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" { kind: "eas", id: "eas", - label: "EAS Simulator", + label: "Expo Simulators", accessToken: "expo-secret", maxSessionMinutes: 30, idleMinutes: 10, @@ -1774,7 +1786,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" { kind: "eas", id: "eas", - label: "EAS Simulator", + label: "Expo Simulators", accessToken: easReference, maxSessionMinutes: 30, idleMinutes: 10, @@ -1894,7 +1906,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" { kind: "eas", id: "eas", - label: "EAS Simulator", + label: "Expo Simulators", accessToken: personal("Expo/token"), maxSessionMinutes: 30, idleMinutes: 10, @@ -2063,6 +2075,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" }); const environment = yield* resolveProviderInstanceTerminalEnvironment({ + deriveProviderInstanceConfigMap, serverSettings, path, rawProviderInstanceId: instanceId, diff --git a/apps/server/src/serverSettings.ts b/apps/server/src/serverSettings.ts index 251becba12f6..4dc30d29a6cf 100644 --- a/apps/server/src/serverSettings.ts +++ b/apps/server/src/serverSettings.ts @@ -237,7 +237,7 @@ function ensureProviderInstanceMutationAllowed( ): Effect.Effect { if ( mutation.operation === "create" && - settings.providerInstances[mutation.instanceId] !== undefined + Object.hasOwn(settings.providerInstances, mutation.instanceId) ) { return Effect.fail( new ServerSettingsError({ diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index 9fe99cd2ea15..cdc54fdec972 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -30,7 +30,9 @@ import { readServiceState, registerServiceTask, serviceLauncherIsRunning, + serviceLauncherIsServing, serviceLauncherLockPath, + stateDirOwned, writeServiceState, } from "./serviceLauncher.ts"; import { @@ -46,12 +48,27 @@ import { } from "./cloud/serviceProtocol.ts"; const spawnedChildren = vi.hoisted((): Array => []); +const spawnHook = vi.hoisted(() => ({ + current: undefined as ((child: NodeChildProcess.ChildProcess) => void) | undefined, +})); + +const failRm = vi.hoisted(() => ({ path: undefined as string | undefined })); + +vi.mock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + const rm: typeof actual.rm = (path, options) => + path === failRm.path + ? Promise.reject(Object.assign(new Error("EACCES"), { code: "EACCES" })) + : actual.rm(path, options); + return { ...actual, rm }; +}); vi.mock("node:child_process", async (importOriginal) => { const actual = await importOriginal(); const spawn = (...args: Parameters) => { const child = actual.spawn(...args); spawnedChildren.push(child); + spawnHook.current?.(child); return child; }; return { ...actual, spawn: spawn as typeof actual.spawn }; @@ -191,6 +208,216 @@ const fakeServiceUnit = ( }); it.layer(NodeServices.layer)("service state persistence", (it) => { + it.effect.each([ + "startup", + "log write fails", + "home probe error", + "database probe error", + "pending update", + "update handoff", + "trial exit", + "failed trial start", + "stop", + ] as const)("waits for disk recovery during %s without losing the update lifecycle", (scenario) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-disk-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const databasePath = path.join(root, "userdata", "statev2.sqlite"); + let backupPath = path.join(root, "runtime", "db-backup", "disk-update"); + const pending = { + id: "disk-update", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + dbPath: databasePath, + status: "pending" as const, + }; + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); + yield* fs.writeFileString(databasePath, "before trial"); + const childSource = ` +const fs = require("node:fs"); +const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); +if (context.update?.status === "pending") { + if (context.childVersion !== context.update.targetVersion) throw new Error("wrong trial version"); + if (fs.readFileSync(${JSON.stringify(path.dirname(backupPath))} + "/" + context.update.id + "/database", "utf8") !== "before trial") { + throw new Error("missing original backup"); + } + if (${JSON.stringify(scenario)} === "trial exit") { + fs.writeFileSync(context.update.dbPath, "migrated"); + process.on("message", () => process.exit(1)); + process.send("migrated"); + } else { + process.send({ type: "prepared", updateId: context.update.id }); + process.on("message", (message) => { + if (message.type === "committed") process.send("ready"); + }); + } +} else if (${JSON.stringify(scenario)} === "update handoff" && context.update === undefined) { + process.on("message", (message) => { + if (message.type === "update-accepted") process.send("exiting"); + if (message === "exit") process.exit(0); + }); + process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${JSON.stringify(databasePath)} }); +} else { + process.send("ready"); +} +setInterval(() => {}, 1_000); +`; + for (const version of ["1.0.0", "1.1.0"]) { + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", version), + childSource, + ); + } + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + ...([ + "startup", + "log write fails", + "home probe error", + "database probe error", + "update handoff", + "stop", + ].includes(scenario) + ? {} + : { update: pending }), + }), + ); + let low = [ + "startup", + "log write fails", + "home probe error", + "database probe error", + "pending update", + "stop", + ].includes(scenario); + let recovered = false; + const blocked = Promise.withResolvers(); + const ready = Promise.withResolvers(); + const probing = Promise.withResolvers(); + const capacityResult = Promise.withResolvers(); + const actualStatfs = NodeFSP.statfs; + const stats = yield* Effect.promise(() => actualStatfs(root)); + yield* Effect.acquireRelease( + Effect.sync(() => { + vi.useFakeTimers({ + toFake: ["setInterval", "clearInterval", "setTimeout", "clearTimeout"], + }); + const capacity = vi.spyOn(NodeFSP, "statfs").mockImplementation(async (directory) => { + if (scenario === "stop" && directory === root) { + probing.resolve(); + await capacityResult.promise; + } + if ( + (scenario === "home probe error" && directory === root) || + (scenario === "database probe error" && directory === path.dirname(databasePath)) + ) { + throw new Error("statfs unavailable"); + } + if (scenario === "failed trial start" && !recovered && NodeFS.existsSync(backupPath)) + low = true; + return { ...stats, bavail: low ? 0 : 1024 ** 4, bsize: 1 }; + }); + const log = vi.spyOn(process.stderr, "write").mockImplementation((chunk) => { + if (String(chunk).includes("Waiting for free space")) { + blocked.resolve(); + if (scenario === "log write fails") { + throw Object.assign(new Error("no space left"), { code: "ENOSPC" }); + } + } + return true; + }); + spawnHook.current = (child) => + child.on("message", (message) => { + if (message === "ready") ready.resolve(); + if (message === "migrated" || message === "exiting") { + low = true; + child.send("exit"); + } + }); + return { capacity, log }; + }), + ({ capacity, log }) => + Effect.sync(() => { + capacity.mockRestore(); + log.mockRestore(); + spawnHook.current = undefined; + vi.useRealTimers(); + }), + ); + const beforeSpawns = spawnedChildren.length; + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { backupSpaceProbe: { canClone: async () => true } }, + ); + const running = launcher.run(); + yield* Effect.addFinalizer(() => + Effect.promise(async () => { + await launcher.stop("SIGTERM"); + await running; + }), + ); + if (scenario === "stop") { + yield* Effect.promise(() => probing.promise); + const stopping = launcher.stop("SIGTERM"); + capacityResult.resolve(); + yield* Effect.promise(() => stopping); + yield* Effect.promise(() => running); + assert.equal(spawnedChildren.length, beforeSpawns); + assert.equal(vi.getTimerCount(), 0); + return; + } + yield* Effect.promise(() => Promise.race([blocked.promise, ready.promise, running])); + assert.equal( + spawnedChildren.length - beforeSpawns, + ["trial exit", "update handoff"].includes(scenario) ? 1 : 0, + ); + if (scenario === "update handoff") { + const state = yield* Effect.promise(() => readServiceState(statePath)); + backupPath = path.join(root, "runtime", "db-backup", state.update!.id); + } + assert.equal( + yield* fs.exists(backupPath), + scenario === "trial exit" || scenario === "failed trial start", + ); + if (scenario === "trial exit") + assert.equal(yield* fs.readFileString(databasePath), "migrated"); + if (scenario === "failed trial start") { + yield* fs.remove(path.join(root, "runtime", "versions", "1.1.0"), { recursive: true }); + } + low = false; + recovered = true; + yield* Effect.promise(() => vi.advanceTimersByTimeAsync(30_000)); + yield* Effect.promise(() => Promise.race([ready.promise, running])); + const state = yield* Effect.promise(() => readServiceState(statePath)); + assert.equal( + state.activeVersion, + ["pending update", "update handoff"].includes(scenario) ? "1.1.0" : "1.0.0", + ); + assert.equal( + state.update?.status, + { + startup: undefined, + "log write fails": undefined, + "home probe error": undefined, + "database probe error": undefined, + "pending update": "committed", + "update handoff": "committed", + "trial exit": "rolled-back", + "failed trial start": "failed", + }[scenario], + ); + assert.equal(yield* fs.readFileString(databasePath), "before trial"); + assert.isFalse(yield* fs.exists(backupPath)); + }), + ); + it.effect( "loads optional service.env literally without changing inherited or managed variables", () => @@ -308,6 +535,70 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { }), ); + it.effect( + "repoints the systemd unit the manager loads when environment.d moves XDG_CONFIG_HOME", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-xdg-" }); + const home = path.join(root, "home"); + const managerUnitDir = path.join(home, ".config", "systemd", "user"); + const unitPath = path.join(managerUnitDir, "t3code.service"); + const render = (version: string) => + renderBootServiceUnit({ ...unitPlan(root, version), unitPath }); + yield* fs.makeDirectory(managerUnitDir, { recursive: true }); + yield* fs.writeFileString(unitPath, render("1.0.0")); + // Fake manager: busctl reports the unit search path, systemctl reloads. + const bin = path.join(root, "bin"); + yield* fs.makeDirectory(bin, { recursive: true }); + const unitPathJson = JSON.stringify({ + type: "as", + data: [`${managerUnitDir}.control`, managerUnitDir, "/usr/lib/systemd/user"], + }); + yield* fs.writeFileString( + path.join(bin, "busctl"), + `#!/bin/sh\nprintf '%s\\n' '${unitPathJson}'\n`, + ); + yield* fs.writeFileString( + path.join(bin, "systemctl"), + `#!/bin/sh\necho reload >> '${path.join(root, "reloads")}'\n`, + ); + yield* fs.chmod(path.join(bin, "busctl"), 0o755); + yield* fs.chmod(path.join(bin, "systemctl"), 0o755); + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", "1.1.0"), + "process.exit(0);\n", + ); + yield* Effect.promise(() => + writeServiceState(path.join(root, "runtime", "service-state.json"), { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.1.0", + update: { + id: "update-1", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + status: "committed", + }, + }), + ); + vi.stubEnv("PATH", bin); + vi.stubEnv("HOME", home); + // environment.d changes what services inherit, not where the manager reads units. + vi.stubEnv("XDG_CONFIG_HOME", path.join(home, "service-config")); + vi.stubEnv("T3_BOOT_SERVICE_UNIT", "t3code.service"); + + yield* Effect.promise(() => + expect(runLauncher(root)).rejects.toThrow("Active child exited unexpectedly (0)."), + ); + + assert.equal(yield* fs.readFileString(unitPath), render("1.1.0")); + assert.equal(yield* fs.readFileString(path.join(root, "reloads")), "reload\n"); + }), + ); + it.effect("keeps the service manager's append log bounded when it starts", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -382,6 +673,36 @@ process.exit(0); }), ); + it.effect("starts the runtime in a T3 home whose name ends in a space", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const parent = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-space-" }); + const root = path.join(parent, "T3 data "); + const versionDir = path.join(root, "runtime", "versions", "1.0.0"); + yield* writeFakeRuntime( + fs, + path, + versionDir, + ` +import { writeFileSync } from "node:fs"; +writeFileSync(new URL("./started", import.meta.url), ""); +process.exit(0); +`, + ); + yield* fs.writeFileString( + path.join(root, "runtime", "service-state.json"), + JSON.stringify({ protocol: 4, activeVersion: "1.0.0" }), + ); + + yield* Effect.promise(() => + expect(runLauncher(root)).rejects.toThrow("Active child exited unexpectedly (0)."), + ); + + assert.isTrue(yield* fs.exists(path.join(versionDir, "started"))); + }), + ); + it.effect("refuses non-install legacy documents without rewriting or spawning", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -681,17 +1002,25 @@ process.exit(0); }), ); - it.effect("stays idle instead of exiting when another server owns the T3 home", () => + it.effect("waits without exiting and restarts once the other owner releases the T3 home", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-refused-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-reclaim-" }); const statePath = path.join(root, "runtime", "service-state.json"); + const ownerPath = path.join(root, "userdata", "server-owner.sqlite"); + yield* fs.makeDirectory(path.dirname(ownerPath), { recursive: true }); yield* writeFakeRuntime( fs, path, path.join(root, "runtime", "versions", "1.0.0"), - `process.exit(${SERVER_EXIT_CODE_STATE_DIR_OWNED});\n`, + [ + `const db = new (require("node:sqlite").DatabaseSync)(${JSON.stringify(ownerPath)});`, + `try { db.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;"); }`, + `catch { process.exit(${SERVER_EXIT_CODE_STATE_DIR_OWNED}); }`, + 'process.send("owner");', + "setInterval(() => {}, 1_000);", + ].join("\n"), ); yield* Effect.promise(() => writeServiceState(statePath, { @@ -699,7 +1028,10 @@ process.exit(0); activeVersion: "1.0.0", }), ); + using otherOwner = new NodeSqlite.DatabaseSync(ownerPath); + otherOwner.exec("BEGIN EXCLUSIVE;"); const refused = Promise.withResolvers(); + const owned = Promise.withResolvers(); yield* Effect.acquireRelease( Effect.sync(() => vi.spyOn(process.stderr, "write").mockImplementation((chunk) => { @@ -709,15 +1041,221 @@ process.exit(0); ), (spy) => Effect.sync(() => spy.mockRestore()), ); - const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath))); + yield* Effect.acquireRelease( + Effect.sync(() => { + spawnHook.current = (child) => child.once("message", () => owned.resolve()); + }), + () => + Effect.sync(() => { + spawnHook.current = undefined; + }), + ); + const spawnsBefore = spawnedChildren.length; + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { stateDirOwnerPollMs: 10 }, + ); let completed = false; const running = launcher.run().finally(() => { completed = true; }); yield* Effect.promise(() => refused.promise); assert.isFalse(completed); + // An idle launcher must not make an SSH launch wait for a server it is not starting. + assert.isFalse(yield* Effect.promise(() => serviceLauncherIsServing(root))); + otherOwner.close(); + yield* Effect.promise(() => owned.promise); + assert.isTrue(yield* Effect.promise(() => serviceLauncherIsServing(root))); yield* Effect.promise(() => launcher.stop("SIGTERM")); yield* Effect.promise(() => running); + assert.strictEqual(spawnedChildren.length - spawnsBefore, 2); + }), + ); + + it.effect( + "keeps a pending update while owned and rolls a failed resumed trial back to the other owner's writes", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-reclaim-update-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const ownerPath = path.join(root, "userdata", "server-owner.sqlite"); + const databasePath = path.join(root, "userdata", "state.sqlite"); + const startsPath = path.join(root, "starts.log"); + yield* fs.makeDirectory(path.dirname(ownerPath), { recursive: true }); + yield* fs.writeFileString(databasePath, "before update"); + for (const version of ["1.0.0", "1.1.0"]) { + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", version), + [ + `const db = new (require("node:sqlite").DatabaseSync)(${JSON.stringify(ownerPath)});`, + `try { db.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;"); }`, + `catch { process.exit(${SERVER_EXIT_CODE_STATE_DIR_OWNED}); }`, + "const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);", + `require("node:fs").appendFileSync(${JSON.stringify(startsPath)}, context.childVersion + "\\n");`, + 'if (context.update?.status === "pending") {', + ` require("node:fs").writeFileSync(${JSON.stringify(databasePath)}, "migrated by failed trial");`, + " process.exit(1);", + "}", + 'process.send("active");', + "setInterval(() => {}, 1_000);", + ].join("\n"), + ); + } + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + update: { + id: "update-1", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + dbPath: databasePath, + status: "pending", + }, + }), + ); + using otherOwner = new NodeSqlite.DatabaseSync(ownerPath); + otherOwner.exec("BEGIN EXCLUSIVE;"); + const refused = Promise.withResolvers(); + const active = Promise.withResolvers(); + yield* Effect.acquireRelease( + Effect.sync(() => + vi.spyOn(process.stderr, "write").mockImplementation((chunk) => { + if (String(chunk).includes("owns this T3 home")) refused.resolve(); + return true; + }), + ), + (spy) => Effect.sync(() => spy.mockRestore()), + ); + yield* Effect.acquireRelease( + Effect.sync(() => { + spawnHook.current = (child) => + child.on("message", (message) => message === "active" && active.resolve()); + }), + () => + Effect.sync(() => { + spawnHook.current = undefined; + }), + ); + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { stateDirOwnerPollMs: 10 }, + ); + const running = launcher.run(); + yield* Effect.promise(() => refused.promise); + const idle = yield* Effect.promise(() => readServiceState(statePath)); + assert.strictEqual(idle.activeVersion, "1.0.0"); + assert.strictEqual(idle.update?.status, "pending"); + assert.isFalse(yield* fs.exists(path.join(root, "runtime", "db-backup", "update-1"))); + + yield* fs.writeFileString(databasePath, "written by the other owner"); + otherOwner.close(); + yield* Effect.promise(() => active.promise); + yield* Effect.promise(() => launcher.stop("SIGTERM")); + yield* Effect.promise(() => running); + + const state = yield* Effect.promise(() => readServiceState(statePath)); + assert.strictEqual(state.activeVersion, "1.0.0"); + assert.strictEqual(state.update?.status, "rolled-back"); + assert.strictEqual(yield* fs.readFileString(startsPath), "1.1.0\n1.0.0\n"); + assert.strictEqual(yield* fs.readFileString(databasePath), "written by the other owner"); + }), + ); + + it.effect("fails a pending update when its backup cannot be dropped while owned", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-stale-backup-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const ownerPath = path.join(root, "userdata", "server-owner.sqlite"); + const databasePath = path.join(root, "userdata", "state.sqlite"); + yield* fs.makeDirectory(path.dirname(ownerPath), { recursive: true }); + yield* fs.writeFileString(databasePath, "before update"); + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", "1.1.0"), + `process.exit(${SERVER_EXIT_CODE_STATE_DIR_OWNED});\n`, + ); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + update: { + id: "update-1", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + dbPath: databasePath, + status: "pending", + }, + }), + ); + const refused = Promise.withResolvers(); + yield* Effect.acquireRelease( + Effect.sync(() => + vi.spyOn(process.stderr, "write").mockImplementation((chunk) => { + if (String(chunk).includes("owns this T3 home")) refused.resolve(); + return true; + }), + ), + (spy) => Effect.sync(() => spy.mockRestore()), + ); + yield* Effect.acquireRelease( + Effect.sync(() => { + failRm.path = path.join(root, "runtime", "db-backup", "update-1"); + }), + () => + Effect.sync(() => { + failRm.path = undefined; + }), + ); + const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath))); + const running = launcher.run(); + yield* Effect.promise(() => refused.promise); + yield* Effect.promise(() => launcher.stop("SIGTERM")); + yield* Effect.promise(() => running); + + const state = yield* Effect.promise(() => readServiceState(statePath)); + assert.strictEqual(state.activeVersion, "1.0.0"); + assert.strictEqual(state.update?.status, "failed"); + }), + ); + + it.effect("treats the T3 home as owned while the lock or a pre-lock server holds it", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-owned-" }); + const stateDir = path.join(root, "userdata"); + const recordPath = path.join(stateDir, "server-runtime.json"); + yield* fs.makeDirectory(stateDir, { recursive: true }); + assert.isFalse(yield* Effect.promise(() => stateDirOwned(root))); + + using owner = new NodeSqlite.DatabaseSync(path.join(stateDir, "server-owner.sqlite")); + owner.exec("BEGIN EXCLUSIVE;"); + assert.isTrue(yield* Effect.promise(() => stateDirOwned(root))); + owner.close(); + + const record = { version: 1, pid: process.pid, port: 1, origin: "", startedAt: "" }; + yield* fs.writeFileString(recordPath, JSON.stringify(record)); + assert.isTrue(yield* Effect.promise(() => stateDirOwned(root))); + yield* fs.writeFileString(recordPath, JSON.stringify({ ...record, ownerId: "lock-aware" })); + assert.isFalse(yield* Effect.promise(() => stateDirOwned(root))); + // This process started after 1970, so its PID was reused since that record. + yield* fs.writeFileString( + recordPath, + JSON.stringify({ ...record, startedAt: "1970-01-01T00:00:00Z" }), + ); + assert.isFalse(yield* Effect.promise(() => stateDirOwned(root))); + yield* fs.writeFileString(recordPath, JSON.stringify({ ...record, pid: 2 ** 22 + 1 })); + assert.isFalse(yield* Effect.promise(() => stateDirOwned(root))); }), ); @@ -1433,6 +1971,56 @@ process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: paths.dat }), ); + it.effect("starts no trial when stopped during the pre-update backup", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-backup-stop-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const childSource = "setInterval(() => {}, 1000);\n"; + for (const version of ["1.0.0", "1.1.0"]) { + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", version), + childSource, + ); + } + const dbPath = path.join(root, "userdata", "state.sqlite"); + yield* fs.makeDirectory(path.dirname(dbPath), { recursive: true }); + yield* fs.writeFileString(dbPath, "database"); + const pending = { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + update: { + id: "update-1", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + dbPath, + status: "pending", + }, + } as const; + yield* Effect.promise(() => writeServiceState(statePath, pending)); + const childCount = spawnedChildren.length; + const launcher = new Launcher(root, pending); + + // Recovery is already queued, so this stop lands while the backup runs. + const running = launcher.run(); + yield* Effect.promise(() => launcher.stop("SIGTERM")); + yield* Effect.promise(() => running); + + // A trial started here arms its 30-minute prepared timer, which keeps the stopped launcher alive. + assert.deepEqual( + spawnedChildren.slice(childCount).map((child) => child.spawnargs.join(" ")), + [], + ); + assert.deepEqual(yield* Effect.promise(() => readServiceState(statePath)), pending); + const backupDir = path.join(root, "runtime", "db-backup", "update-1"); + assert.isFalse(yield* fs.exists(backupDir)); + assert.isFalse(yield* fs.exists(`${backupDir}.staging`)); + }), + ); + it.effect("records no update when the server exits during the backup check", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index e7c7b6734960..23263c7bd795 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -11,6 +11,7 @@ import * as NodeStreamPromises from "node:stream/promises"; import * as NodeUtil from "node:util"; import * as NodeZlib from "node:zlib"; import { acquireServiceSelectionLock } from "./cloud/serviceSelectionLock.ts"; +import { isNoCopyOnWrite, markNoCopyOnWrite } from "./noCopyOnWrite.ts"; import type { PendingServiceUpdate, @@ -32,6 +33,7 @@ import { serviceUnitLogPathOf, SERVICE_ENV_FILE, SERVICE_LAUNCHER_LOCK_FILE, + SERVICE_SERVING_LOCK_FILE, SERVICE_LAUNCHER_CONTEXT_ENV, SERVICE_LAUNCHER_PROTOCOL, SERVICE_STATE_FILE, @@ -39,7 +41,12 @@ import { SERVICE_BOOT_VERSION_FILE, SERVICE_STOP_MARKER_FILE, SERVICE_UNIT_ENV, + systemdUnitDirectoryOf, } from "./cloud/serviceProtocol.ts"; +import { + legacyOwnerStartedBeforeRecord, + legacyOwnerStartTimeCommand, +} from "./legacyServerOwner.ts"; const HANDOFF_DELAY_MS = 2_000; /** @@ -48,11 +55,15 @@ const HANDOFF_DELAY_MS = 2_000; */ const PREPARED_TIMEOUT_MS = 30 * 60_000; const TERMINATE_GRACE_MS = 5_000; +// Keep SQLite startup and journaling from turning a full filesystem into a restart loop. +const MIN_SERVICE_START_FREE_BYTES = 64 * 1024 ** 2; +const DISK_SPACE_RETRY_MS = 30_000; // A replacement started by a handoff waits up to two minutes for the stopping launcher. const LOCK_RETRY_MS = 500; const LOCK_ATTEMPTS = 240; // Match contracts' SERVER_EXIT_CODE_STATE_DIR_OWNED without depending on Effect. const STATE_DIR_OWNED_EXIT_CODE = 78; +const STATE_DIR_OWNER_POLL_MS = 30_000; type TerminalStatus = "committed" | "rolled-back" | "failed"; type ChildRole = "active" | "trial"; @@ -129,7 +140,34 @@ export async function registerServiceTask( } } -function serviceUnitFromEnvironment(env: NodeJS.ProcessEnv = process.env): ServiceUnit | undefined { +// The manager's environment can differ from the one it gives services (environment.d). +async function readSystemdUnitDirectory(): Promise { + try { + const { stdout } = await NodeUtil.promisify(NodeChildProcess.execFile)( + "busctl", + [ + "--user", + "--json=short", + "get-property", + "org.freedesktop.systemd1", + "/org/freedesktop/systemd1", + "org.freedesktop.systemd1.Manager", + "UnitPath", + ], + { timeout: 5_000 }, + ); + const { data } = JSON.parse(stdout) as { readonly data?: unknown }; + return Array.isArray(data) && data.every((entry) => typeof entry === "string") + ? systemdUnitDirectoryOf(data) + : undefined; + } catch { + return undefined; + } +} + +async function serviceUnitFromEnvironment( + env: NodeJS.ProcessEnv = process.env, +): Promise { const unitName = env[SERVICE_UNIT_ENV]?.trim(); if (!unitName) return undefined; if (unitName.endsWith(".xml")) { @@ -147,14 +185,18 @@ function serviceUnitFromEnvironment(env: NodeJS.ProcessEnv = process.env): Servi reload: () => Promise.resolve(), }; } - // The manager's own environment, which decides where it reads user units. const xdgConfigHome = env.XDG_CONFIG_HOME?.trim(); - const configHome = - xdgConfigHome && NodePath.isAbsolute(xdgConfigHome) - ? xdgConfigHome - : NodePath.join(home, ".config"); + const unitDir = + (await readSystemdUnitDirectory()) ?? + NodePath.join( + xdgConfigHome && NodePath.isAbsolute(xdgConfigHome) + ? xdgConfigHome + : NodePath.join(home, ".config"), + "systemd", + "user", + ); return { - path: NodePath.join(configHome, "systemd", "user", unitName), + path: NodePath.join(unitDir, unitName), reload: () => new Promise((resolve, reject) => { NodeChildProcess.spawn("systemctl", ["--user", "daemon-reload"], { @@ -318,14 +360,18 @@ async function syncDirectory(directory: string): Promise { } } -function runCp(args: ReadonlyArray): Promise { +function runCp(args: ReadonlyArray, signal?: AbortSignal): Promise { return new Promise((resolve) => { - NodeChildProcess.execFile("cp", args, (error) => resolve(error === null)); + NodeChildProcess.execFile("cp", args, { signal }, (error) => resolve(error === null)); }); } // macOS fs.copyFile clone flags fall back to full copies. -async function copyDatabaseFile(source: string, destination: string): Promise { +async function copyDatabaseFile( + source: string, + destination: string, + signal?: AbortSignal, +): Promise { // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime. const platform = process.platform; // cp -c replaces files and breaks hard links; copy into hard-linked destinations instead. @@ -338,15 +384,22 @@ async function copyDatabaseFile(source: string, destination: string): Promise { +async function compressDatabaseFile( + source: string, + destination: string, + signal: AbortSignal, +): Promise { await NodeStreamPromises.pipeline( NodeFS.createReadStream(source), NodeZlib.createZstdCompress({ params: { [NodeZlib.constants.ZSTD_c_compressionLevel]: 1 } }), NodeFS.createWriteStream(destination, { mode: 0o600 }), + { signal }, ); } @@ -452,6 +505,7 @@ export async function databaseBackupAdmission( const { readFreeBytes, canClone } = { ...filesystemBackupSpaceProbe, ...probe }; const backupRoot = NodePath.join(baseDir, "runtime", "db-backup"); await NodeFSP.mkdir(backupRoot, { recursive: true, mode: 0o700 }); + if (await isNoCopyOnWrite(dbPath)) await markNoCopyOnWrite(backupRoot); await removeAbandonedBackupCopies(backupRoot); const [size, free] = await Promise.all([databaseBytes(dbPath), readFreeBytes(backupRoot)]); if (free >= fullCopyBytes(size) + ONLINE_BACKUP_WAL_HEADROOM_BYTES) return { fullCopyFits: true }; @@ -561,6 +615,7 @@ async function backupDatabaseOnce( pending: PendingServiceUpdate, probe: BackupSpaceProbe, admittedOnCompressedBudget: boolean, + signal: AbortSignal, ): Promise { if ((await existingDatabaseBackup(baseDir, pending.id)) !== undefined) return; @@ -573,16 +628,17 @@ async function backupDatabaseOnce( await NodeFSP.mkdir(stagingDir, { recursive: true, mode: 0o700 }); try { for (const suffix of DB_FILE_SUFFIXES) { + signal.throwIfAborted(); const source = suffix === "" ? pending.dbPath : `${await restoreTarget(pending.dbPath)}${suffix}`; if (suffix !== "" && !(await pathExists(source))) continue; if (clone) { const destination = databaseBackupFile(stagingDir, suffix); - await copyDatabaseFile(source, destination); + await copyDatabaseFile(source, destination, signal); await syncFile(destination); } else { const destination = compressedBackupFile(stagingDir, suffix); - await compressDatabaseFile(source, destination); + await compressDatabaseFile(source, destination, signal); await syncFile(destination); } } @@ -849,13 +905,14 @@ export const serviceLauncherLockPath = (baseDir: string) => NodePath.join(baseDir, "runtime", SERVICE_LAUNCHER_LOCK_FILE); // An exclusive SQLite transaction, like the server's owner lock: the OS drops it when the holder dies. -async function tryAcquireLauncherLock( +async function tryAcquireSqliteLock( lockPath: string, + busyTimeoutMs = 0, ): Promise<{ readonly close: () => void } | undefined> { const { DatabaseSync } = await import("node:sqlite"); const db = new DatabaseSync(lockPath); try { - db.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;"); + db.exec(`PRAGMA busy_timeout = ${busyTimeoutMs}; BEGIN EXCLUSIVE;`); return db; } catch (cause) { db.close(); @@ -866,11 +923,18 @@ async function tryAcquireLauncherLock( export async function serviceLauncherIsRunning(lockPath: string): Promise { if (!NodeFS.existsSync(lockPath)) return false; - const lock = await tryAcquireLauncherLock(lockPath); + const lock = await tryAcquireSqliteLock(lockPath); lock?.close(); return lock === undefined; } +const servingLockPath = (baseDir: string) => + NodePath.join(baseDir, "runtime", SERVICE_SERVING_LOCK_FILE); + +/** Whether a launcher has, or is about to start, a server for this home. An idle one does not. */ +export const serviceLauncherIsServing = (baseDir: string): Promise => + serviceLauncherIsRunning(servingLockPath(baseDir)).catch(() => false); + /** Resolves once no launcher holds the lock; the caller bounds the wait through `signal`. */ export async function waitForServiceLauncherExit( lockPath: string, @@ -882,6 +946,50 @@ export async function waitForServiceLauncherExit( } } +// Mirror acquireServerOwnership: the lock, then a live pre-lock server-runtime.json owner; +// leave other failures for server startup to report. +export async function stateDirOwned(baseDir: string): Promise { + const stateDir = NodePath.join(baseDir, "userdata"); + const lock = await tryAcquireSqliteLock(NodePath.join(stateDir, "server-owner.sqlite")).catch( + () => null, + ); + if (lock === null) return false; + if (lock === undefined) return true; + lock.close(); + let record: unknown; + try { + record = JSON.parse( + await NodeFSP.readFile(NodePath.join(stateDir, "server-runtime.json"), "utf8"), + ); + } catch { + return false; + } + if (typeof record !== "object" || record === null || "ownerId" in record) return false; + const pid = "pid" in record ? record.pid : undefined; + if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0) return false; + try { + process.kill(pid, 0); + } catch (error) { + if (!(error instanceof Error && "code" in error && error.code === "EPERM")) return false; + } + const recordedAt = Date.parse( + "startedAt" in record && typeof record.startedAt === "string" ? record.startedAt : "", + ); + if (!Number.isFinite(recordedAt)) return true; + // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime. + const platform = process.platform; + const { command, args, env } = legacyOwnerStartTimeCommand(pid, platform); + const output = await new Promise((resolve) => + NodeChildProcess.execFile( + command, + args, + { env: { ...process.env, ...env }, timeout: 2_000, maxBuffer: 16_384 }, + (error, stdout) => resolve(error === null ? stdout : undefined), + ), + ); + return legacyOwnerStartedBeforeRecord(output, recordedAt, platform); +} + const stopMarkerPath = (baseDir: string) => NodePath.join(baseDir, "runtime", SERVICE_STOP_MARKER_FILE); const restartPendingPath = (baseDir: string) => @@ -895,6 +1003,7 @@ export class Launcher { #timer: NodeJS.Timeout | undefined; #transitions: Promise = Promise.resolve(); #stopRequested = false; + readonly #stopSignal = new AbortController(); #stopping = false; #done = false; readonly #completion = Promise.withResolvers(); @@ -903,8 +1012,11 @@ export class Launcher { #compressedBudgetUpdateId: string | undefined; readonly #serviceUnit: ServiceUnit | undefined; readonly #environment: Readonly; + readonly #stateDirOwnerPollMs: number; + #stateDirFreeProbes = 0; readonly #output: number | undefined; readonly #lockPath: string | undefined; + #servingLock: { readonly close: () => void } | undefined; readonly #stopChildOverIpc: boolean; constructor( @@ -914,6 +1026,7 @@ export class Launcher { readonly backupSpaceProbe?: Partial; readonly serviceUnit?: ServiceUnit; readonly environment?: Readonly; + readonly stateDirOwnerPollMs?: number; /** Launcher and server output, for managers that cannot redirect it. */ readonly output?: number; /** Makes this the only launcher for its home; a replacement waits for the one stopping. */ @@ -923,6 +1036,7 @@ export class Launcher { } = {}, ) { this.#baseDir = baseDir; + this.#stateDirOwnerPollMs = options.stateDirOwnerPollMs ?? STATE_DIR_OWNER_POLL_MS; this.#backupSpaceProbe = options.backupSpaceProbe; this.#serviceUnit = options.serviceUnit; this.#environment = { ...(options.environment ?? process.env) }; @@ -956,6 +1070,7 @@ export class Launcher { } } } finally { + this.#releaseServing(); process.off("SIGTERM", onSigterm); process.off("SIGINT", onSigint); } @@ -963,7 +1078,7 @@ export class Launcher { async #acquireLock(lockPath: string): Promise<{ readonly close: () => void } | undefined> { for (let attempt = 1; !this.#stopRequested; attempt++) { - const lock = await tryAcquireLauncherLock(lockPath); + const lock = await tryAcquireSqliteLock(lockPath); if (lock !== undefined) return lock; if (attempt >= LOCK_ATTEMPTS) { throw new Error("Another T3 Code service launcher still runs for this T3 home."); @@ -973,13 +1088,29 @@ export class Launcher { return undefined; } + // Advisory: without it an SSH launch may start its own server, as it does beside no service. + async #claimServing(): Promise { + // An SSH launch probes this lock for a moment; wait out that probe. + this.#servingLock ??= await tryAcquireSqliteLock(servingLockPath(this.#baseDir), 250).catch( + () => undefined, + ); + } + + #releaseServing(): void { + this.#servingLock?.close(); + this.#servingLock = undefined; + } + #terminate(child: NodeChildProcess.ChildProcess, signal: NodeJS.Signals = "SIGTERM") { return terminateChild(child, signal, this.#stopChildOverIpc); } + // Diagnostics are best-effort: a full log filesystem must not stop the launcher waiting for space. #log(line: string): void { - if (this.#output === undefined) process.stderr.write(line); - else NodeFS.writeSync(this.#output, line); + try { + if (this.#output === undefined) process.stderr.write(line); + else NodeFS.writeSync(this.#output, line); + } catch {} } #enqueue(transition: () => Promise): void { @@ -1011,10 +1142,13 @@ export class Launcher { return; } this.#stopRequested = true; + this.#stopSignal.abort(); this.#clearTimer(); this.#enqueue(async () => { // Let in-progress updates start replacements first; replacements observe the marker and own pre-activation tunnel cleanup. this.#stopping = true; + // An exit-78 transition queued before this one may have started the owner poll. + this.#clearTimer(); const child = this.#child?.process; this.#child = null; if (child !== undefined) await this.#terminate(child, signal); @@ -1035,6 +1169,55 @@ export class Launcher { void pruneRuntimeVersions(this.#baseDir, this.#state, launcherPaths).catch(() => undefined); } + async #serviceStartDiskSpaceBlocker(): Promise { + const databasePath = + this.#state.update?.status === "pending" + ? this.#state.update.dbPath + : NodePath.join(this.#baseDir, "userdata", "statev2.sqlite"); + const resolvedDatabasePath = await restoreTarget(databasePath).catch(() => databasePath); + const freeBytes = async (directory: string): Promise => { + try { + return await filesystemBackupSpaceProbe.readFreeBytes(directory); + } catch (cause) { + const parent = NodePath.dirname(directory); + if ((cause as NodeJS.ErrnoException).code === "ENOENT" && parent !== directory) { + return freeBytes(parent); + } + return Number.POSITIVE_INFINITY; + } + }; + const available = await Promise.all([ + freeBytes(this.#baseDir), + freeBytes(NodePath.dirname(resolvedDatabasePath)), + ]); + const free = Math.min(...available); + return free < MIN_SERVICE_START_FREE_BYTES + ? `T3 Code needs at least ${MIN_SERVICE_START_FREE_BYTES} bytes free before starting the server; ${free} bytes available.` + : undefined; + } + + async #deferForDiskSpace(retry: () => Promise): Promise { + const reason = await this.#serviceStartDiskSpaceBlocker(); + if (reason === undefined) return false; + // A stop during the probe must not arm a retry that outlives it. + if (this.#stopRequested) return true; + this.#clearTimer(); + this.#releaseServing(); + this.#log(`[service-launcher] ${reason} Waiting for free space before retrying.\n`); + this.#timer = setInterval( + () => + this.#enqueue(async () => { + if (this.#child !== null || this.#stopping) return; + if ((await this.#serviceStartDiskSpaceBlocker()) !== undefined) return; + this.#clearTimer(); + await this.#claimServing(); + await retry(); + }), + DISK_SPACE_RETRY_MS, + ); + return true; + } + async #selectState(next: ServiceState): Promise { const release = await acquireServiceSelectionLock(this.#statePath); try { @@ -1056,6 +1239,7 @@ export class Launcher { } async #recover(): Promise { + await this.#claimServing(); // Clear old stop markers to preserve future tunnels; clear restart-pending only for the awaited version to avoid CLI races. await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined); const restartPending = restartPendingPath(this.#baseDir); @@ -1110,17 +1294,22 @@ export class Launcher { } async #startTrial(pending: PendingServiceUpdate): Promise { + if (await this.#deferForDiskSpace(() => this.#startTrial(pending))) return; try { await backupDatabaseOnce( this.#baseDir, pending, { ...filesystemBackupSpaceProbe, ...this.#backupSpaceProbe }, this.#compressedBudgetUpdateId === pending.id, + this.#stopSignal.signal, ); } catch { + // A stop aborts the copy and leaves the update pending for the next start. + if (this.#stopRequested) return; await this.#returnToPrevious(pending, "failed", "db-backup-failed"); return; } + if (this.#stopRequested) return; try { await this.#startChild(pending.targetVersion, "trial", pending); } catch { @@ -1130,6 +1319,14 @@ export class Launcher { async #startChild(version: string, role: ChildRole, update?: ServiceUpdateRecord): Promise { if (this.#stopping) return; + if ( + await this.#deferForDiskSpace(() => + role === "trial" && update?.status === "pending" + ? this.#startTrial(update) + : this.#startChild(version, role, update), + ) + ) + return; if (!(await runtimeExists(this.#baseDir, version))) { throw new Error(`Selected t3@${version} runtime is missing or incomplete.`); } @@ -1277,8 +1474,6 @@ export class Launcher { this.#child = null; await this.#terminate(child.process); await finishOnlineBackup?.(); - // A stop during a minutes-long copy should not start a trial; restart resumes the pending update. - if (this.#stopRequested) return; await this.#startTrial(pending); } @@ -1393,30 +1588,69 @@ export class Launcher { await this.#startTrial(pending); return; } + if ( + await this.#deferForDiskSpace(() => + this.#startChild(child.version, "active", this.#state.update), + ) + ) + return; throw new Error(`Active child exited unexpectedly (${String(code ?? signal ?? "unknown")}).`); } - // Stay idle to avoid supervisor restart loops; never restore backups over another owner's possible writes. + // Stay idle until the lock frees to avoid supervisor restart loops. A pending update resumes its + // trial, since the other owner may have migrated the database past fromVersion. It stays pending + // only once its backup is gone, so no later trial, even after a launcher restart, restores it over + // the other owner's writes; exit 78 comes before the child opens the database. async #idleWhileStateDirOwned(): Promise { this.#clearTimer(); const pending = this.#state.update; if (pending?.status === "pending") { - const next: ServiceState = { - ...this.#state, - activeVersion: pending.fromVersion, - update: terminalUpdate({ pending, status: "failed", reason: "state-dir-owned" }), - }; - await this.#selectState(next); - await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined); - await this.#repointServiceUnit(next.activeVersion); + const discarded = await discardDatabaseBackup(this.#baseDir, pending.id).then( + () => true, + () => false, + ); + if (!discarded) { + const next: ServiceState = { + ...this.#state, + activeVersion: pending.fromVersion, + update: terminalUpdate({ pending, status: "failed", reason: "state-dir-owned" }), + }; + await this.#selectState(next); + await this.#repointServiceUnit(next.activeVersion); + } } - // Signal listeners alone do not keep Node alive. - this.#timer = setInterval(() => {}, 2_147_483_647); + this.#releaseServing(); this.#log( - "[service-launcher] Another T3 Code server owns this T3 home. Stop it, then restart this service.\n", + "[service-launcher] Another T3 Code server owns this T3 home. This service starts again once it stops.\n", + ); + this.#stateDirFreeProbes = 0; + // The interval also keeps Node alive: signal listeners alone do not. + this.#timer = setInterval( + () => this.#enqueue(() => this.#reclaimStateDir()), + this.#stateDirOwnerPollMs, ); } + async #reclaimStateDir(): Promise { + if (this.#stopRequested) { + this.#clearTimer(); + return; + } + if (this.#timer === undefined || this.#child !== null) return; + const owned = await stateDirOwned(this.#baseDir); + // A stop during the probe clears the timer. + if (this.#timer === undefined) return; + // A relaunching desktop app frees the home for a few seconds; leave it that gap. + this.#stateDirFreeProbes = owned ? 0 : this.#stateDirFreeProbes + 1; + if (this.#stateDirFreeProbes < 2) return; + this.#clearTimer(); + await this.#claimServing(); + this.#log("[service-launcher] The T3 home is free again. Starting the server.\n"); + const update = this.#state.update; + if (update?.status === "pending") await this.#startTrial(update); + else await this.#startChild(this.#state.activeVersion, "active", update); + } + async #returnToPrevious( pending: PendingServiceUpdate, status: "rolled-back" | "failed", @@ -1427,6 +1661,8 @@ export class Launcher { this.#child = null; await this.#terminate(child.process); } + if (await this.#deferForDiskSpace(() => this.#returnToPrevious(pending, status, reason))) + return; await restoreDatabaseBackup(this.#baseDir, pending); const outcome = terminalUpdate({ pending, status, reason }); const next: ServiceState = { @@ -1452,8 +1688,8 @@ export interface LauncherOptions { } export async function main(options: LauncherOptions = {}): Promise { - const baseDir = (options.baseDir ?? process.env.T3CODE_HOME)?.trim(); - if (baseDir === undefined || baseDir === "") { + const baseDir = options.baseDir ?? process.env.T3CODE_HOME; + if (baseDir === undefined || baseDir.trim() === "") { throw new Error("T3CODE_HOME is required by the T3 Code service launcher."); } const output = @@ -1461,6 +1697,8 @@ export async function main(options: LauncherOptions = {}): Promise { // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime. const windows = process.platform === "win32"; let logTimer: NodeJS.Timeout | undefined; + // A failed stderr write (ENOSPC on a redirected log) is emitted asynchronously; without a listener it is fatal. + process.stderr.on("error", () => undefined); try { // What the service manager started the launcher with. It locates the unit; service.env // changes only what the server gets. @@ -1480,7 +1718,7 @@ export async function main(options: LauncherOptions = {}): Promise { throw new Error(`Selected t3@${state.activeVersion} runtime is missing or incomplete.`); } } - const serviceUnit = serviceUnitFromEnvironment(managerEnvironment); + const serviceUnit = await serviceUnitFromEnvironment(managerEnvironment); const logPath = await serviceLogPath(options, serviceUnit); if (logPath !== undefined) { await boundServiceLog(logPath); diff --git a/apps/server/src/settingsSecrets.ts b/apps/server/src/settingsSecrets.ts index 22ea8ae88258..a38072d19813 100644 --- a/apps/server/src/settingsSecrets.ts +++ b/apps/server/src/settingsSecrets.ts @@ -235,7 +235,7 @@ function mapSecretFieldGroups( ) as EasDeviceHostConfig, ); - // Its own prefix: an EAS and a Simlock host may share an id. + // Its own prefix: an Expo and a Simlock host may share an id. const simlockDeviceHosts = settings.simlockDeviceHosts.map( (host) => mapFields( diff --git a/apps/server/src/sourceControl/SourceControlDiscovery.test.ts b/apps/server/src/sourceControl/SourceControlDiscovery.test.ts index 6f9582b2e09a..efa3aa00e595 100644 --- a/apps/server/src/sourceControl/SourceControlDiscovery.test.ts +++ b/apps/server/src/sourceControl/SourceControlDiscovery.test.ts @@ -175,7 +175,7 @@ it.effect("reports every hosting integration in registry order with its sign-in" }).pipe(Effect.provide(SourceControlDiscovery.layer.pipe(Layer.provide(layerDiscovery())))), ); -it.effect("explains a hosting CLI whose version probe failed instead of advising an install", () => +it.effect("explains a CLI whose version probe failed instead of advising an install", () => Effect.gen(function* () { const discovery = yield* SourceControlDiscovery.SourceControlDiscovery; const result = yield* discovery.discover; @@ -195,15 +195,23 @@ it.effect("explains a hosting CLI whose version probe failed instead of advising assert.strictEqual(authDetail("broken"), item("broken").installHint); assert.strictEqual(item("absent").installHint, "Install absent."); + + const git = result.versionControlSystems.find((entry) => entry.kind === "git"); + assert.isDefined(git); + assert.notInclude(git.installHint, "Install Git from"); + assert.include(git.installHint, "`git --version` exited with status 1"); }).pipe( Effect.provide( SourceControlDiscovery.layer.pipe( Layer.provide( - layerDiscovery(run, [ - cliDiscovery("slow", "slow-cli"), - cliDiscovery("broken", "broken-cli"), - cliDiscovery("absent", "absent-cli"), - ]), + layerDiscovery( + (input) => run(input.command === "git" ? { ...input, command: "broken-cli" } : input), + [ + cliDiscovery("slow", "slow-cli"), + cliDiscovery("broken", "broken-cli"), + cliDiscovery("absent", "absent-cli"), + ], + ), ), ), ), diff --git a/apps/server/src/sourceControl/SourceControlDiscovery.ts b/apps/server/src/sourceControl/SourceControlDiscovery.ts index 8311058c9787..e69f0d3761e6 100644 --- a/apps/server/src/sourceControl/SourceControlDiscovery.ts +++ b/apps/server/src/sourceControl/SourceControlDiscovery.ts @@ -15,8 +15,8 @@ import { refreshPathFromLoginShell } from "../os-jank.ts"; import * as VcsProcess from "../vcs/VcsProcess.ts"; import { detailFromCause, + failedProbeHint, firstNonEmptyLine, - unavailableInstallHint, } from "@t3tools/source-control-core/server/discovery"; import * as SourceControlProviderRegistry from "./SourceControlProviderRegistry.ts"; @@ -119,7 +119,7 @@ export const make = Effect.gen(function* () { ), Effect.catch((cause) => Effect.gen(function* () { - const installHint = yield* unavailableInstallHint(executable, input.installHint, cause); + const installHint = yield* failedProbeHint(input, cause); return { kind: input.kind, label: input.label, diff --git a/apps/server/src/sourceControl/SourceControlProviderRegistry.test.ts b/apps/server/src/sourceControl/SourceControlProviderRegistry.test.ts index c532d93e81ad..9d472fe6e77c 100644 --- a/apps/server/src/sourceControl/SourceControlProviderRegistry.test.ts +++ b/apps/server/src/sourceControl/SourceControlProviderRegistry.test.ts @@ -283,7 +283,7 @@ it.effect( url, ); } - }).pipe(Effect.scoped), + }), ); it.effect("does not refine a context whose server address is empty", () => diff --git a/apps/server/src/sourceControl/SourceControlRepositoryService.test.ts b/apps/server/src/sourceControl/SourceControlRepositoryService.test.ts index 4a0533da0b86..bebdeef9ece3 100644 --- a/apps/server/src/sourceControl/SourceControlRepositoryService.test.ts +++ b/apps/server/src/sourceControl/SourceControlRepositoryService.test.ts @@ -78,6 +78,7 @@ function layer(input: { Layer.provide( Layer.mock(GitVcsDriver.GitVcsDriver)({ execute: () => Effect.succeed(processOutput()), + invalidateRefs: () => Effect.void, ensureRemote: () => Effect.succeed("origin"), pushCurrentBranch: () => Effect.succeed({ @@ -220,7 +221,7 @@ it.effect.each(["--bare", "source.git"])( }); assert.strictEqual(result.cwd, destinationPath); assert.isTrue(yield* fs.exists(path.join(destinationPath, ".git"))); - }).pipe(Effect.provide(layer({ git: { execute: git.execute } }))); + }).pipe(Effect.provide(layer({ git }))); }).pipe( Effect.provide( GitVcsDriver.layer.pipe( @@ -235,6 +236,72 @@ it.effect.each(["--bare", "source.git"])( ), ); +it.effect.each([ + { readsWhen: "before git creates the repository", readBeforeInit: true }, + { readsWhen: "while HEAD is unborn", readBeforeInit: false }, +])("lists the cloned branches after a ref list was read $readsWhen", ({ readBeforeInit }) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const git = yield* GitVcsDriver.GitVcsDriver; + const parent = yield* fs.makeTempDirectoryScoped({ prefix: "t3-clone-refs-" }); + const source = path.join(parent, "source"); + const destinationPath = path.join(parent, "checkout"); + const run = (cwd: string, args: ReadonlyArray) => + git.execute({ operation: "test.git", cwd, args }); + yield* run(parent, ["init", "--initial-branch=staging", source]); + yield* run(source, [ + "-c", + "user.name=Test", + "-c", + "user.email=test@example.com", + "commit", + "--allow-empty", + "-m", + "init", + ]); + const refNames = git + .listRefs({ cwd: destinationPath }) + .pipe(Effect.map((result) => result.refs.map((ref) => ref.name))); + let duringClone: ReadonlyArray | null = null; + // Stands in for git clone so the read lands at a chosen point inside it. + const cloneInTwoSteps = Effect.gen(function* () { + yield* fs.makeDirectory(destinationPath).pipe(Effect.orDie); + if (readBeforeInit) duringClone = yield* refNames; + yield* run(parent, ["init", "--initial-branch=staging", destinationPath]); + if (!readBeforeInit) duringClone = yield* refNames; + return yield* run(destinationPath, ["pull", source, "staging"]); + }); + yield* Effect.gen(function* () { + const service = yield* SourceControlRepositoryService.SourceControlRepositoryService; + yield* service.cloneRepository({ remoteUrl: source, destinationPath }); + }).pipe( + Effect.provide( + layer({ + git: { + ...git, + execute: (input) => (input.args[0] === "clone" ? cloneInTwoSteps : git.execute(input)), + }, + }), + ), + ); + + assert.deepStrictEqual(duringClone, []); + assert.deepStrictEqual(yield* refNames, ["staging"]); + }).pipe( + Effect.provide( + GitVcsDriver.layer.pipe( + Layer.provide( + ServerConfig.layerTest(process.cwd(), { + prefix: "t3-clone-refs-config-", + }), + ), + Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), + ), + ), + ), +); + it.effect("reports clone progress from git's stderr and keeps its error text on failure", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/server/src/sourceControl/SourceControlRepositoryService.ts b/apps/server/src/sourceControl/SourceControlRepositoryService.ts index 3db65b0bd5f9..b4b2dc0e92fe 100644 --- a/apps/server/src/sourceControl/SourceControlRepositoryService.ts +++ b/apps/server/src/sourceControl/SourceControlRepositoryService.ts @@ -429,6 +429,8 @@ export const make = Effect.gen(function* () { ), ); + // A ref list read while the clone ran saw an unborn HEAD and no refs. + yield* git.invalidateRefs(prepared.destinationPath); return { cwd: prepared.destinationPath, remoteUrl, diff --git a/apps/server/src/speech/SpeechService.test.ts b/apps/server/src/speech/SpeechService.test.ts index 384d599ba3e7..551b406a6fa8 100644 --- a/apps/server/src/speech/SpeechService.test.ts +++ b/apps/server/src/speech/SpeechService.test.ts @@ -452,14 +452,19 @@ it.effect("honors exact aliases and phrase boundaries in batch and streamed text { term: "GitHub", aliases: [] }, { term: "McDonald’s", aliases: [] }, { term: "King’s Cross", aliases: [] }, + { term: "C#", aliases: [] }, + { term: "React", aliases: ["瑞安特"] }, + { term: "Kubernetes", aliases: [] }, ], }; const spoken = 'a test with a i and ay eye. Robert, Roberts. "the publisher" and the publisher. (git hub), git (hub). ' + - "git hub's API, git hub’s API, git’s hub, mac donald's menu, king's cross."; + "git hub's API, git hub’s API, git’s hub, mac donald's menu, king's cross. vitamin c and plan c. " + + "我用github和瑞安特,部署到kubernetis上。Robert用git hub。githubで管理する。"; const expected = "a test with AI and AI. Bob, Roberts. O'Reilly and the publisher. (GitHub), git (hub). " + - "GitHub's API, GitHub’s API, git’s hub, McDonald’s menu, King’s Cross."; + "GitHub's API, GitHub’s API, git’s hub, McDonald’s menu, King’s Cross. vitamin c and plan c. " + + "我用GitHub和React,部署到Kubernetes上。Bob用GitHub。GitHubで管理する。"; native.transcribe.mockResolvedValueOnce({ text: spoken }); native.finish.mockResolvedValueOnce(spoken); expect(yield* speech.transcribe(pcm(), options)).toBe(expected); diff --git a/apps/server/src/speech/customWords.ts b/apps/server/src/speech/customWords.ts index e788c9188a09..981632c46172 100644 --- a/apps/server/src/speech/customWords.ts +++ b/apps/server/src/speech/customWords.ts @@ -1,5 +1,11 @@ import type { SpeechCustomWords } from "@t3tools/contracts"; +// These scripts are written without spaces, so a neighbor in one of them still ends a word. +const UNSPACED_SCRIPTS = "\\p{scx=Han}\\p{scx=Hiragana}\\p{scx=Katakana}"; +const LETTER = `[[\\p{L}\\p{M}\\p{N}]--[${UNSPACED_SCRIPTS}]]`; +const WORD_CHARACTER = `[[\\p{L}\\p{M}\\p{N}\\p{Pc}]--[${UNSPACED_SCRIPTS}]]`; +const TOKEN = new RegExp(`${LETTER}(?:[^\\s${UNSPACED_SCRIPTS}]*${LETTER})?`, "gv"); + /** Fuzzy-matches preferred terms and rewrites aliases to their preferred spelling. */ export function makeSpeechTextCorrector( words: readonly string[], @@ -71,8 +77,14 @@ export function makeSpeechTextCorrector( const customWordKeys = (words: readonly string[]): CustomWordKey[] => words.flatMap((word) => { - const keys = [matchKey(word)]; - if (word.includes("&")) keys.push(matchKey(word.replaceAll("&", " and "))); + const sources = [word]; + if (word.includes("&")) sources.push(word.replaceAll("&", " and ")); + const keys = sources.flatMap((source) => { + const key = matchKey(source); + // A term that loses its symbols to one letter (C# -> c) would claim every standalone letter. + const lostSymbols = key !== source.replace(/\s+/gu, "").toLocaleLowerCase(); + return key.length < 2 && lostSymbols ? [] : [key]; + }); return [...new Set(keys)].filter(isAsciiKey).map((key) => ({ word, key })); }); @@ -106,7 +118,7 @@ export function makeSpeechTextCorrector( const apply = (text: string): string => { if (keys.length === 0) return text; // Leave surrounding punctuation in the source; only horizontal whitespace can join words. - const tokens = [...text.matchAll(/[\p{L}\p{M}\p{N}](?:\S*[\p{L}\p{M}\p{N}])?/gu)] + const tokens = [...text.matchAll(TOKEN)] .filter(([word]) => /^[\p{L}\p{M}\p{N}'’-]+$/u.test(word)) .map((match) => ({ value: match[0], @@ -162,10 +174,9 @@ export function makeSpeechTextCorrector( const lookup = new Map( spellings.map(({ spelling, term }) => [spelling.toLocaleLowerCase(), term]), ); - const word = "[\\p{L}\\p{M}\\p{N}\\p{Pc}]"; const pattern = new RegExp( - `(? escapePattern(spelling)).join("|")})(?!${word})`, - "giu", + `(? escapePattern(spelling)).join("|")})(?!${WORD_CHARACTER})`, + "giv", ); return (text: string) => { let output = ""; diff --git a/apps/server/src/speech/http.test.ts b/apps/server/src/speech/http.test.ts index d86f77fbdbd8..07f2fe6b99d5 100644 --- a/apps/server/src/speech/http.test.ts +++ b/apps/server/src/speech/http.test.ts @@ -269,7 +269,7 @@ effectIt.effect( handler(new Request("http://localhost/api/voice/models"), context), ); expect(yield* Effect.promise(() => listed.text())).not.toContain("/home/someone"); - }).pipe(Effect.scoped), + }), ); effectIt.effect.each([ @@ -361,5 +361,5 @@ effectIt.effect.each([ code: "invalid_request", reason, }); - }).pipe(Effect.scoped), + }), ); diff --git a/apps/server/src/speech/native.test.ts b/apps/server/src/speech/native.test.ts index 8712c63ed347..a41f917b2a68 100644 --- a/apps/server/src/speech/native.test.ts +++ b/apps/server/src/speech/native.test.ts @@ -1,4 +1,8 @@ // @effect-diagnostics nodeBuiltinImport:off - verifies isolated native worker processes and their exit events. +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + import { expect, it } from "vite-plus/test"; import { loadNativeSpeechModel } from "./native.ts"; @@ -155,6 +159,37 @@ it("kills a hung streaming feed without waiting for native cleanup", async () => await model.dispose(); await rejected; }); + +it("skips process exit teardown when the backend dies during native inference", async () => { + const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3code-speech-orphan-")); + const marker = NodePath.join(directory, "exit-teardown"); + // The exit hook stands in for native atexit teardown; the interval for the threadpool call. + const model = await loadNativeSpeechModel( + "unused.gguf", + new AbortController().signal, + streamingFixture(`async () => { + const { writeFileSync } = process.getBuiltinModule("node:fs"); + process.once("exit", () => writeFileSync(${JSON.stringify(marker)}, "ran")); + setInterval(() => {}, 1 << 30); + process.disconnect(); + return await new Promise(() => {}); + }`), + ); + try { + await model.begin({ timestamps: "none" }, false); + await expect(model.feed(new Float32Array(512).fill(0.25))).rejects.toThrow( + "Speech process exited", + ); + expect( + NodeFS.existsSync(marker), + "orphaned worker ran exit teardown while native inference was in flight", + ).toBe(false); + } finally { + await model.dispose(); + NodeFS.rmSync(directory, { recursive: true, force: true }); + } +}); + const vadFixture = ({ streaming = false, failVad = false, diff --git a/apps/server/src/speech/native.ts b/apps/server/src/speech/native.ts index 8331e1b5e0ec..70c5d21c6038 100644 --- a/apps/server/src/speech/native.ts +++ b/apps/server/src/speech/native.ts @@ -182,7 +182,8 @@ process.on("message", async (message) => { error: error instanceof Error ? error.stack ?? error.message : String(error) }); } }); -process.on("disconnect", () => process.exit(0)); +// Exit teardown under in-flight native inference segfaults, so terminate without it. +process.on("disconnect", () => process.kill(process.pid, "SIGKILL")); `; type Reply = { diff --git a/apps/server/src/speech/postProcessing.test.ts b/apps/server/src/speech/postProcessing.test.ts new file mode 100644 index 000000000000..3dd64c9b1bcc --- /dev/null +++ b/apps/server/src/speech/postProcessing.test.ts @@ -0,0 +1,81 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import { ProviderInstanceId, type ModelSelection } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Ref from "effect/Ref"; + +import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import * as TextGeneration from "../textGeneration/TextGeneration.ts"; +import * as SpeechPostProcessing from "./postProcessing.ts"; + +const textGenerationSelection = { + instanceId: ProviderInstanceId.make("fakeapi"), + model: "fake-model", +}; +const acpSelection = { instanceId: ProviderInstanceId.make("acp_local"), model: "default" }; + +it.effect("cleanup pinned to a provider that cannot generate text uses the text model", () => + Effect.gen(function* () { + const used = yield* Ref.make(null); + const layer = SpeechPostProcessing.layer.pipe( + Layer.provide( + Layer.succeed(TextGeneration.TextGeneration, { + generateTranscriptionPostProcessing: (input: { modelSelection: ModelSelection }) => + Ref.set(used, input.modelSelection).pipe(Effect.as({ transcription: "Hello, world." })), + } as unknown as TextGeneration.TextGeneration["Service"]), + ), + Layer.provide( + Layer.succeed(ProviderRegistry.ProviderRegistry, { + getProviders: Effect.succeed([ + { + instanceId: acpSelection.instanceId, + driver: "acpRegistry", + enabled: true, + installed: true, + supportsTextGeneration: false, + }, + ]), + } as unknown as ProviderRegistry.ProviderRegistry["Service"]), + ), + Layer.provide( + ServerSettings.layerTest({ + providerInstances: { + [acpSelection.instanceId]: { + driver: "acpRegistry", + enabled: true, + config: { source: "local", commandPath: "acp-agent" }, + }, + }, + textGenerators: { + [textGenerationSelection.instanceId]: { + kind: "openaiCompatible", + baseUrl: "http://127.0.0.1:18999/v1", + enabled: true, + }, + }, + textGenerationModelSelection: textGenerationSelection, + speechPostProcessingModelSelection: acpSelection, + }), + ), + Layer.provide(NodeServices.layer), + ); + yield* Effect.gen(function* () { + const postProcessing = yield* SpeechPostProcessing.SpeechPostProcessing; + yield* postProcessing.process({ + options: { + speechCustomWords: [], + speechPostProcessingPrompt: { mode: "default", customInstructions: "" }, + }, + transcript: "um hello world", + draft: { text: "", selection: { start: 0, end: 0 } }, + } as never); + }).pipe(Effect.provide(layer)); + const selection = yield* Ref.get(used); + assert.deepStrictEqual( + { instanceId: selection?.instanceId, model: selection?.model }, + textGenerationSelection, + ); + }), +); diff --git a/apps/server/src/speech/postProcessing.ts b/apps/server/src/speech/postProcessing.ts index 3208afd05728..92f4af17696c 100644 --- a/apps/server/src/speech/postProcessing.ts +++ b/apps/server/src/speech/postProcessing.ts @@ -8,6 +8,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import { resolveSpeechPostProcessingModelSelection } from "@t3tools/shared/serverSettings"; +import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as TextGeneration from "../textGeneration/TextGeneration.ts"; import { buildTranscriptionPostProcessingPrompt } from "../textGeneration/TranscriptionPostProcessing.ts"; @@ -27,11 +28,13 @@ const make = Effect.gen(function* () { const settingsService = yield* ServerSettings.ServerSettingsService; const fileSystem = yield* FileSystem.FileSystem; const textGeneration = yield* TextGeneration.TextGeneration; + const providerRegistry = yield* ProviderRegistry.ProviderRegistry; return SpeechPostProcessing.of({ process: Effect.fn("SpeechPostProcessing.process")( function* (input) { if (input.transcript.trim().length === 0) return { text: input.transcript }; const settings = yield* settingsService.getSettings; + const providers = yield* providerRegistry.getProviders; const { mode, customInstructions } = input.options.speechPostProcessingPrompt; const instructions = (mode === "custom" ? customInstructions.trim() : "") || @@ -46,7 +49,7 @@ const make = Effect.gen(function* () { input.draft, input.options.speechCorrectionWord, ), - modelSelection: resolveSpeechPostProcessingModelSelection(settings), + modelSelection: resolveSpeechPostProcessingModelSelection(settings, providers), }); const text = generated.transcription.trim(); return { diff --git a/apps/server/src/startupAccess.test.ts b/apps/server/src/startupAccess.test.ts index a48bb5263f8e..159778b3c7e0 100644 --- a/apps/server/src/startupAccess.test.ts +++ b/apps/server/src/startupAccess.test.ts @@ -21,6 +21,7 @@ const ipv4 = (address: string, internal = false): NodeOS.NetworkInterfaceInfo => it("resolves wildcard hosts to a concrete external interface when one is available", () => { const connectionString = resolveHeadlessConnectionString("0.0.0.0", 3773, { interfaces: { en0: [ipv4("192.168.1.42")], lo0: [ipv4("127.0.0.1", true)] }, + platform: "linux", }); expect(connectionString).toBe("http://192.168.1.42:3773"); @@ -34,16 +35,21 @@ it("skips public and container addresses for a private-network listener", () => eth1: [ipv4("10.0.0.5")], }; - expect(resolveHeadlessConnectionString("0.0.0.0", 3773, { interfaces })).toBe( + expect(resolveHeadlessConnectionString("0.0.0.0", 3773, { interfaces, platform: "linux" })).toBe( "http://203.0.113.7:3773", ); expect( - resolveHeadlessConnectionString("0.0.0.0", 3773, { interfaces, privateNetworkOnly: true }), + resolveHeadlessConnectionString("0.0.0.0", 3773, { + interfaces, + privateNetworkOnly: true, + platform: "linux", + }), ).toBe("http://10.0.0.5:3773"); expect( resolveHeadlessConnectionString("0.0.0.0", 3773, { interfaces: { eth0: [ipv4("203.0.113.7")] }, privateNetworkOnly: true, + platform: "linux", }), ).toBe("http://localhost:3773"); }); @@ -53,10 +59,21 @@ it("prefers a LAN address over a tailnet one for a private-network pairing URL", resolveHeadlessConnectionString("0.0.0.0", 3773, { interfaces: { tailscale0: [ipv4("100.101.102.103")], en0: [ipv4("192.168.1.42")] }, privateNetworkOnly: true, + platform: "linux", }), ).toBe("http://192.168.1.42:3773"); }); +it("skips NixOS container and systemd-nspawn veths for a private-network pairing URL", () => { + expect( + resolveHeadlessConnectionString("0.0.0.0", 3773, { + interfaces: { "ve-jellyfin": [ipv4("192.168.53.193")], eth0: [ipv4("192.168.0.18")] }, + privateNetworkOnly: true, + platform: "linux", + }), + ).toBe("http://192.168.0.18:3773"); +}); + it("targets the bound interface when the listener binds a specific address", () => { const hostFor = (ip: string) => loopbackUrlHostForAddress(NetAddress.inetAddressFromIpStringUnsafe(ip, 3773)); diff --git a/apps/server/src/startupAccess.ts b/apps/server/src/startupAccess.ts index f393faecebf9..1720300bd7be 100644 --- a/apps/server/src/startupAccess.ts +++ b/apps/server/src/startupAccess.ts @@ -7,6 +7,7 @@ import { isSharedAddressSpaceHost, isWildcardHost, } from "@t3tools/shared/hostClassification"; +import * as HostProcess from "@t3tools/shared/HostProcess"; import { QrCode } from "@t3tools/shared/qrCode"; import { buildPairingAppLink } from "@t3tools/shared/remote"; import * as Effect from "effect/Effect"; @@ -44,9 +45,19 @@ export const isLoopbackHost = (host: string | undefined): boolean => { * host bridge such as `br0` or Proxmox's `vmbr0` carries the LAN address, so * it stays listed. */ -export const VIRTUAL_INTERFACE = +const VIRTUAL_INTERFACE = /^(docker|br-|veth|virbr|vmnet|vboxnet|vEthernet|podman|cni|flannel|cali|lxcbr|lxdbr|bridge[1-9]\d\d)/; +/** + * systemd-nspawn's `ve-*`/`vz-*` and NixOS containers' `vb-*` name host-side + * veths after the machine. Windows aliases are user-renameable, so these + * prefixes only apply to Linux interface discovery. + */ +const LINUX_CONTAINER_INTERFACE = /^(ve-|vb-|vz-)/; + +export const isVirtualInterface = (name: string, platform: NodeJS.Platform): boolean => + VIRTUAL_INTERFACE.test(name) || (platform === "linux" && LINUX_CONTAINER_INTERFACE.test(name)); + /** * Loopback or private-range IPv4 (LAN, tailnet, CGNAT), not link-local, in * plain or IPv4-mapped IPv6 form. @@ -86,6 +97,7 @@ const resolveHeadlessConnectionHost = ( host: string | undefined, interfaces: NetworkInterfacesMap, privateNetworkOnly: boolean, + platform: NodeJS.Platform, ): string => { if (!host) { return "localhost"; @@ -96,7 +108,7 @@ const resolveHeadlessConnectionHost = ( } const interfaceEntries = Object.entries(interfaces).flatMap(([name, entries]) => - privateNetworkOnly && VIRTUAL_INTERFACE.test(name) ? [] : (entries ?? []), + privateNetworkOnly && isVirtualInterface(name, platform) ? [] : (entries ?? []), ); const externalIpv4Entries = interfaceEntries.filter( (entry) => @@ -128,12 +140,14 @@ export const resolveHeadlessConnectionString = ( options: { readonly interfaces?: NetworkInterfacesMap; readonly privateNetworkOnly?: boolean | undefined; - } = {}, + readonly platform: NodeJS.Platform; + }, ): string => { const connectionHost = resolveHeadlessConnectionHost( host, options.interfaces ?? NodeOS.networkInterfaces(), options.privateNetworkOnly === true, + options.platform, ); return `http://${formatHostForUrl(connectionHost)}:${port}`; }; @@ -196,10 +210,11 @@ export const issueHeadlessServeAccessInfo = Effect.fn("issueHeadlessServeAccessI const serverConfig = yield* ServerConfig.ServerConfig; const httpServer = yield* HttpServer.HttpServer; const serverAuth = yield* EnvironmentAuth.EnvironmentAuth; + const platform = yield* HostProcess.Platform; const connectionString = resolveHeadlessConnectionString( serverConfig.host, resolveListeningPort(httpServer.address, serverConfig.port), - { privateNetworkOnly: serverConfig.privateNetworkOnly }, + { privateNetworkOnly: serverConfig.privateNetworkOnly, platform }, ); const issued = yield* serverAuth.issueStartupPairingCredential(); diff --git a/apps/server/src/storageCleanup.test.ts b/apps/server/src/storageCleanup.test.ts index b933895c4de1..cf56bde07f21 100644 --- a/apps/server/src/storageCleanup.test.ts +++ b/apps/server/src/storageCleanup.test.ts @@ -2,6 +2,8 @@ import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Effect from "effect/Effect"; import * as Context from "effect/Context"; +import type * as Digest from "effect/crypto/Digest"; +import * as Base64Url from "effect/encoding/Base64Url"; import * as Deferred from "effect/Deferred"; import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; @@ -14,6 +16,8 @@ import * as Stream from "effect/Stream"; import { SCAN_CACHE_FILE_NAME } from "./usage/usageScanCache.ts"; import { TestClock } from "effect/testing"; import * as SqlClient from "effect/sql/SqlClient"; +import * as Sqlite from "./persistence/Sqlite.ts"; +import { checkpointRefForScopeOrdinal } from "./orchestration-v2/CheckpointService.ts"; import { DEFAULT_SERVER_SETTINGS, type WorktreeKeepWhen, @@ -26,17 +30,20 @@ import * as ServerConfig from "./config.ts"; import * as StorageCleanup from "./storageCleanup.ts"; import * as GitVcsDriver from "./vcs/GitVcsDriver.ts"; import * as GitManager from "./git/GitManager.ts"; +import * as GitWorkflowService from "./git/GitWorkflowService.ts"; import * as Settings from "./serverSettings.ts"; import * as ProjectionStore from "./orchestration-v2/ProjectionStore.ts"; import * as ProviderSessionManager from "./orchestration-v2/ProviderSessionManager.ts"; import * as ProjectStore from "./orchestration-v2/ProjectStore.ts"; import * as Orchestrator from "./orchestration-v2/Orchestrator.ts"; +import * as ThreadManagement from "./orchestration-v2/ThreadManagementService.ts"; import * as TerminalManager from "./terminal/Manager.ts"; import { CodexProviderCapabilitiesV2 } from "./orchestration-v2/Adapters/CodexAdapterV2.ts"; import { ServerActivation } from "./serverActivation.ts"; import { withWorkspaceLease } from "./workspace/workspaceLease.ts"; import { describe, expect, it } from "@effect/vitest"; import { + CheckpointScopeId, EventId, ProjectId, ProviderDriverKind, @@ -46,6 +53,7 @@ import { RuntimeRequestId, ThreadId, type OrchestrationV2DomainEvent, + type OrchestrationV2ServerCommand, type OrchestrationV2ThreadShell, } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; @@ -205,6 +213,13 @@ describe("V2 storage cleanup eligibility", () => { resetIn: -1_000, idle: false, }, + { + name: "never armed because its reset is days overdue", + armed: false, + autoResumeSetting: true, + resetIn: -5 * DAY_MS, + idle: true, + }, { name: "not armed with recovery settings off", armed: false, @@ -318,6 +333,7 @@ const cleanupFixture = ( worktreeAfterDays: 8, worktreeOnDelete: true, }, + sqlOverride?: SqlClient.SqlClient, ) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -350,6 +366,7 @@ const cleanupFixture = ( let sessionLiveChecks = 0; let onLiveCheck: Effect.Effect = Effect.void; let outboxStatus: string | null = null; + let scheduledThreadIds: string[] = []; let fullStatusCalls = 0; let gitInspections = 0; let snapshotReads = 0; @@ -419,6 +436,31 @@ const cleanupFixture = ( getThreadShell: (threadId: ThreadId) => Effect.sync(() => threads.find((thread) => thread.id === threadId) ?? null), } as unknown as ProjectionStore.ProjectionStoreV2["Service"]), + Effect.provideService(ThreadManagement.ThreadManagementService, { + getShellSnapshot: (input?: { threadIds?: ReadonlyArray }) => + Effect.sync(() => ({ + threads: threads.filter((thread) => input?.threadIds?.includes(thread.id) ?? true), + archivedThreads: [], + })), + dispatch: (command: OrchestrationV2ServerCommand) => + Effect.sync(() => { + if (command.type === "thread.metadata.update" && command.branch != null) { + const branch = command.branch; + threads = threads.map((thread) => + thread.id === command.threadId && thread.branch === command.expectedBranch + ? { ...thread, branch } + : thread, + ); + } + return { sequence: 0, storedEvents: [] }; + }), + } as unknown as ThreadManagement.ThreadManagementService["Service"]), + Effect.provideService(GitWorkflowService.GitWorkflowService, { + currentBranch: (cwd: string) => + command(cwd, ["branch", "--show-current"]).pipe( + Effect.map((result) => result.stdout.trim() || null), + ), + } as unknown as GitWorkflowService.GitWorkflowService["Service"]), Effect.provideService(ProviderSessionManager.ProviderSessionManagerV2, { isLive: () => Effect.sync(() => { @@ -429,39 +471,49 @@ const cleanupFixture = ( Effect.provideService(Orchestrator.OrchestratorV2, { streamDomainEvents: Stream.suspend(() => events), } as unknown as Orchestrator.OrchestratorV2["Service"]), - Effect.provideService(SqlClient.SqlClient, ((strings: TemplateStringsArray) => - Effect.sync(() => - openSessionCwd !== null && strings.join("").includes("projection_provider_sessions") - ? [ - { - payload_json: JSON.stringify({ - id: "session-1", - driver: "codex", - providerInstanceId: "codex", - status: "ready", - cwd: openSessionCwd, - model: null, - capabilities: CodexProviderCapabilitiesV2, - createdAt: "2026-06-01T12:00:00.000Z", - updatedAt: "2026-06-01T12:00:00.000Z", - lastError: null, - }), - }, - ] - : outboxStatus !== null && strings.join("").includes("orchestration_v2_effect_outbox") - ? [{ status: outboxStatus }] - : strings.join("").includes("deleted_at IS NOT NULL") - ? deletedThreads.map((thread) => ({ - payload_json: JSON.stringify(thread), - workspaceRoot: repo, - })) - : strings.join("").includes('AS "worktreePath"') + Effect.provideService( + SqlClient.SqlClient, + sqlOverride ?? + (((strings: TemplateStringsArray) => + Effect.sync(() => + strings.join("").includes("retained_checkpoint_threads") + ? [] + : openSessionCwd !== null && + strings.join("").includes("projection_provider_sessions") ? [ - ...threads.flatMap((thread) => thread.worktreePath ?? []), - ...storedWorktreePaths, - ].map((worktreePath) => ({ worktreePath })) - : [], - )) as unknown as SqlClient.SqlClient), + { + payload_json: JSON.stringify({ + id: "session-1", + driver: "codex", + providerInstanceId: "codex", + status: "ready", + cwd: openSessionCwd, + model: null, + capabilities: CodexProviderCapabilitiesV2, + createdAt: "2026-06-01T12:00:00.000Z", + updatedAt: "2026-06-01T12:00:00.000Z", + lastError: null, + }), + }, + ] + : outboxStatus !== null && + strings.join("").includes("orchestration_v2_effect_outbox") + ? [{ status: outboxStatus }] + : strings.join("").includes("deleted_at IS NOT NULL") + ? deletedThreads.map((thread) => ({ + payload_json: JSON.stringify(thread), + workspaceRoot: repo, + })) + : strings.join("").includes('AS "worktreePath"') + ? [ + ...threads.flatMap((thread) => thread.worktreePath ?? []), + ...storedWorktreePaths, + ].map((worktreePath) => ({ worktreePath })) + : strings.join("").includes("scheduled_tasks") + ? scheduledThreadIds.map((threadId) => ({ threadId })) + : [], + )) as unknown as SqlClient.SqlClient), + ), Effect.provideService(GitManager.GitManager, { invalidateStatus: () => Effect.void, } as unknown as GitManager.GitManager["Service"]), @@ -520,12 +572,16 @@ const cleanupFixture = ( setThreads: (next: typeof threads) => { threads = next; }, + threadBranches: () => threads.map((thread) => thread.branch), setOnLiveCheck: (hook: Effect.Effect) => { onLiveCheck = hook; }, setOutboxStatus: (status: string | null) => { outboxStatus = status; }, + setScheduledThreadIds: (next: string[]) => { + scheduledThreadIds = next; + }, setDeletedThreads: (next: OrchestrationV2ThreadShell[]) => { deletedThreads = next; }, @@ -593,17 +649,249 @@ const cleanupTestLayer = GitVcsDriver.layer.pipe( Layer.provideMerge(ServerConfig.layerTest(process.cwd(), { prefix: "storage-cleanup-" })), Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), ); +const annexWorktree = Effect.fnUntraced(function* ( + fixture: Effect.Success>, + worktree: string, +) { + const adminDir = (yield* fixture.command(worktree, ["rev-parse", "--absolute-git-dir"])).stdout; + yield* fixture.fs.remove(`${worktree}/.git`); + yield* fixture.fs.symlink(adminDir.trim(), `${worktree}/.git`); +}); + const runCleanupTest = ( effect: Effect.Effect< A, E, | FileSystem.FileSystem + | Digest.Digest | ServerConfig.ServerConfig | GitVcsDriver.GitVcsDriver | Path.Path | Scope.Scope >, -) => effect.pipe(Effect.provide(cleanupTestLayer), Effect.scoped); +) => effect.pipe(Effect.provide(cleanupTestLayer)); + +const legacyCheckpointRef = (threadId: string, turn: number) => + `refs/t3/checkpoints/${Base64Url.encode(threadId)}/turn/${turn}`; + +const checkpointCleanupFixture = Effect.fn("test.checkpointCleanupFixture")(function* () { + const sql = yield* SqlClient.SqlClient; + const git = yield* GitVcsDriver.GitVcsDriver; + const fixture = yield* cleanupFixture(DEFAULT_SERVER_SETTINGS.storageCleanup, sql); + const { repo, worktree, command } = fixture; + const timestamp = DateTime.formatIso(at(0)); + yield* sql`INSERT INTO projection_projects ${sql.insert({ + project_id: "project-1", + title: "Checkpoint cleanup", + workspace_root: repo, + scripts_json: "[]", + created_at: timestamp, + updated_at: timestamp, + deleted_at: timestamp, + })}`; + const addThread = Effect.fn("test.addCheckpointThread")(function* ( + id: string, + options: { deleted?: boolean; archived?: boolean; source?: string; legacyOnly?: boolean } = {}, + ) { + const scopeId = CheckpointScopeId.make(`scope:${id}`); + yield* sql`INSERT INTO orchestration_v2_projection_threads ${sql.insert({ + thread_id: id, + project_id: "project-1", + title: id, + default_provider: "codex", + runtime_mode: "full-access", + interaction_mode: "default", + created_at: timestamp, + updated_at: timestamp, + archived_at: options.archived ? timestamp : null, + deleted_at: options.deleted ? timestamp : null, + payload_json: JSON.stringify({ + worktreePath: worktree, + forkedFrom: options.source ? { type: "run", threadId: options.source } : null, + }), + })}`; + if (!options.legacyOnly) + yield* sql`INSERT INTO orchestration_v2_projection_checkpoint_scopes ${sql.insert({ + scope_id: scopeId, + thread_id: id, + node_id: `node:${id}`, + kind: "root_run", + ordinal_within_parent: 0, + advances_app_run_count: 1, + created_at: timestamp, + payload_json: JSON.stringify({ cwd: worktree }), + })}`; + const refs: string[] = options.legacyOnly + ? [legacyCheckpointRef(id, 0)] + : [ + yield* checkpointRefForScopeOrdinal({ scopeId, ordinalWithinScope: 0 }), + yield* checkpointRefForScopeOrdinal({ scopeId, ordinalWithinScope: 1 }), + legacyCheckpointRef(id, 0), + ]; + for (const ref of refs) yield* command(repo, ["update-ref", ref, "HEAD"]); + return refs; + }); + const listRefs = () => + git + .execute({ + operation: "cleanup-test.list-checkpoints", + cwd: repo, + args: ["for-each-ref", "--format=%(refname)", "refs/t3/"], + maxOutputBytes: 16 * 1024 * 1024, + }) + .pipe(Effect.map(({ stdout }) => stdout.trim().split("\n").filter(Boolean).sort())); + return { ...fixture, sql, addThread, listRefs }; +}); + +describe("checkpoint ref cleanup", () => { + it.live("prunes refs owned by deleted threads after their worktree is removed", () => + runCleanupTest( + Effect.gen(function* () { + const { service, repo, worktree, command, addThread, listRefs } = + yield* checkpointCleanupFixture(); + yield* addThread("deleted", { deleted: true }); + yield* addThread("imported-deleted", { deleted: true, legacyOnly: true }); + const retained = [ + ...(yield* addThread("active")), + ...(yield* addThread("archived", { archived: true })), + legacyCheckpointRef("unknown-owner", 0), + ]; + yield* command(repo, ["update-ref", retained.at(-1)!, "HEAD"]); + yield* command(repo, ["pack-refs", "--all"]); + yield* command(repo, ["worktree", "remove", worktree]); + yield* service.runNow; + expect(yield* listRefs()).toEqual(retained.sort()); + }).pipe(Effect.provide(Sqlite.layerMemory)), + ), + ); + + it.live("keeps deleted fork ancestry while a live or archived descendant needs it", () => + runCleanupTest( + Effect.gen(function* () { + const { service, sql, addThread, listRefs } = yield* checkpointCleanupFixture(); + const root = yield* addThread("root", { deleted: true }); + const intermediate = yield* addThread("intermediate", { deleted: true, source: "root" }); + const child = yield* addThread("child", { archived: true, source: "intermediate" }); + yield* service.runNow; + expect(yield* listRefs()).toEqual([...root, ...intermediate, ...child].sort()); + yield* sql`UPDATE orchestration_v2_projection_threads SET deleted_at = 'deleted' WHERE thread_id = 'child'`; + yield* service.runNow; + expect(yield* listRefs()).toEqual([]); + }).pipe(Effect.provide(Sqlite.layerMemory)), + ), + ); + + it.live("waits for a deleted thread's pending checkpoint effects", () => + runCleanupTest( + Effect.gen(function* () { + const { service, sql, addThread, listRefs } = yield* checkpointCleanupFixture(); + const refs = yield* addThread("capturing", { deleted: true }); + yield* sql`INSERT INTO orchestration_v2_effect_outbox ${sql.insert({ + effect_id: "capture", + command_id: "capture", + thread_id: "capturing", + effect_type: "checkpoint.capture", + payload_json: "{}", + status: "running", + available_at: "2026-01-01T00:00:00Z", + created_at: "2026-01-01T00:00:00Z", + updated_at: "2026-01-01T00:00:00Z", + })}`; + yield* service.runNow; + expect(yield* listRefs()).toEqual(refs.toSorted()); + yield* sql`UPDATE orchestration_v2_effect_outbox SET status = 'succeeded'`; + yield* service.runNow; + expect(yield* listRefs()).toEqual([]); + }).pipe(Effect.provide(Sqlite.layerMemory)), + ), + ); + + it.live("deletes a symbolic checkpoint ref without its target or neighbouring refs", () => + runCleanupTest( + Effect.gen(function* () { + const { service, repo, command, addThread, listRefs } = yield* checkpointCleanupFixture(); + const refs = yield* addThread("deleted", { deleted: true }); + yield* command(repo, ["symbolic-ref", refs[0]!, "refs/heads/main"]); + const retained = [ + "refs/t3/unrelated/0", + `${refs[1]}-other`, + refs[1]!.replace(/1$/, "2/nested"), + ]; + for (const ref of retained) yield* command(repo, ["update-ref", ref, "HEAD"]); + yield* service.runNow; + expect(yield* listRefs()).toEqual(retained.sort()); + yield* command(repo, ["show-ref", "--verify", "refs/heads/main"]); + }).pipe(Effect.provide(Sqlite.layerMemory)), + ), + ); + + it.live("cleans more checkpoint refs than default git output holds", () => + runCleanupTest( + Effect.gen(function* () { + const { service, repo, addThread, listRefs } = yield* checkpointCleanupFixture(); + const git = yield* GitVcsDriver.GitVcsDriver; + const prefix = (yield* addThread("many-checkpoints", { deleted: true }))[0]!.slice(0, -1); + yield* git.execute({ + operation: "cleanup-test.seed-checkpoints", + cwd: repo, + args: ["update-ref", "--stdin"], + stdin: Array.from( + { length: 12_283 }, + (_, ordinal) => `update ${prefix}${ordinal} HEAD\n`, + ).join(""), + }); + yield* service.runNow; + expect(yield* listRefs()).toEqual([]); + }).pipe(Effect.provide(Sqlite.layerMemory)), + ), + ); +}); + +it.live.each(["repository", "uppercase", "bare"] as const)( + "keeps a worktree holding an ignored nested %s", + (kind) => + runCleanupTest( + Effect.gen(function* () { + const { service, fs, repo, worktree, command, setPolicy } = yield* cleanupFixture(); + setPolicy("tracked-changes"); + const nested = `${worktree}/dist/nested`; + if (kind === "uppercase") { + yield* command(repo, ["worktree", "add", "-b", "nested", nested]); + } else { + yield* command(repo, [ + "clone", + "--no-hardlinks", + ...(kind === "bare" ? ["--bare"] : []), + repo, + nested, + ]); + } + if (kind === "uppercase") yield* fs.rename(`${nested}/.git`, `${nested}/.GIT`); + yield* fs.writeFileString(`${nested}/uncommitted.txt`, "irreplaceable work\n"); + expect((yield* command(worktree, ["status", "--porcelain"])).stdout).toBe(""); + const report = yield* service.runNow; + expect(report.entries.find((entry) => entry.path === worktree)).toMatchObject({ + outcome: "kept", + reason: "Contains a nested Git repository", + }); + expect(yield* fs.readFileString(`${nested}/uncommitted.txt`)).toBe("irreplaceable work\n"); + expect( + (yield* command(nested, [ + ...(kind === "uppercase" + ? [ + `--git-dir=${(yield* fs.readFileString(`${nested}/.GIT`)).trim().slice(8)}`, + "--work-tree=.", + ] + : []), + "-c", + "safe.bareRepository=all", + "rev-parse", + "--is-bare-repository", + ])).stdout.trim(), + ).toBe(String(kind === "bare")); + }), + ), +); const submoduleCleanupFixture = () => Effect.gen(function* () { @@ -629,11 +917,71 @@ const submoduleCleanupFixture = () => return { ...fixture, submodule, child, recordSubmodules }; }); -it.live("removes clean submodules recursively, including deinitialized repositories", () => +it.live("inspects a pnpm store excluded from reclaimed bytes for nested repositories", () => runCleanupTest( Effect.gen(function* () { - const { service, fs, repo, worktree, submodule, command, recordSubmodules } = + const { service, fs, repo, worktree, command, setPolicy } = yield* cleanupFixture(); + setPolicy("tracked-changes"); + const storeDir = yield* fs.makeTempDirectoryScoped({ prefix: "cleanup-store-" }); + const nested = `${worktree}/node_modules/.pnpm/nested`; + yield* command(repo, ["clone", "--no-hardlinks", repo, nested]); + yield* fs.writeFileString(`${nested}/uncommitted.txt`, "keep me"); + yield* fs.writeFileString( + `${worktree}/node_modules/.modules.yaml`, + JSON.stringify({ storeDir, virtualStoreDir: ".pnpm" }), + ); + const report = yield* service.runNow; + expect(report.entries.find((entry) => entry.path === worktree)).toMatchObject({ + outcome: "kept", + reason: "Contains a nested Git repository", + }); + expect(yield* fs.readFileString(`${nested}/uncommitted.txt`)).toBe("keep me"); + }), + ), +); + +it.live("keeps an ignored repository inside a registered submodule", () => + runCleanupTest( + Effect.gen(function* () { + const { service, fs, repo, worktree, submodule, command, setPolicy } = yield* submoduleCleanupFixture(); + setPolicy("tracked-changes"); + const nested = `${submodule}/dist/nested`; + yield* command(repo, ["clone", "--no-hardlinks", repo, nested]); + yield* fs.writeFileString(`${nested}/uncommitted.txt`, "keep me"); + const report = yield* service.runNow; + expect(report.entries.find((entry) => entry.path === worktree)).toMatchObject({ + outcome: "kept", + reason: "Contains a nested Git repository", + }); + expect(yield* fs.readFileString(`${nested}/uncommitted.txt`)).toBe("keep me"); + }), + ), +); + +it.live("removes clean submodules recursively, including deinitialized repositories", () => + runCleanupTest( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const git = yield* GitVcsDriver.GitVcsDriver; + let editedDuringClean = false; + const { service, repo, worktree, submodule, command, recordSubmodules, setPolicy } = + yield* submoduleCleanupFixture().pipe( + Effect.provideService(GitVcsDriver.GitVcsDriver, { + ...git, + execute: (input) => + git.execute(input).pipe( + Effect.tap(() => { + if (input.args[0] !== "clean") return Effect.void; + editedDuringClean = true; + return fs.writeFileString(`${input.cwd}/tracked.txt`, "late edit\n"); + }), + ), + }), + ); + setPolicy("tracked-changes"); + yield* fs.writeFileString(`${worktree}/notes.txt`, "untracked\n"); + yield* fs.writeFileString(`${worktree}/.env`, "ignored\n"); yield* command(submodule, [ "-c", "protocol.file.allow=always", @@ -642,10 +990,55 @@ it.live("removes clean submodules recursively, including deinitialized repositor repo, "stored", ]); + const legacy = `${worktree}/legacy`; + yield* command(repo, ["clone", "--no-hardlinks", repo, legacy]); + yield* command(worktree, ["submodule", "add", repo, "legacy"]); + expect((yield* fs.stat(`${legacy}/.git`)).type).toBe("Directory"); + yield* command(legacy, [ + "-c", + "protocol.file.allow=always", + "submodule", + "add", + repo, + "child", + ]); + yield* command(legacy, [ + "-c", + "protocol.file.allow=always", + "submodule", + "add", + repo, + "stored", + ]); + yield* command(legacy, [ + "-c", + "user.name=Cleanup test", + "-c", + "user.email=cleanup@example.test", + "commit", + "-am", + "record legacy children", + ]); + yield* command(legacy, ["push", "origin", "HEAD:legacy"]); + yield* command(legacy, ["submodule", "deinit", "stored"]); yield* recordSubmodules(); yield* command(submodule, ["submodule", "deinit", "stored"]); + const checkpoint = (yield* command(submodule, [ + "commit-tree", + "HEAD^{tree}", + "-p", + "HEAD", + "-m", + "checkpoint", + ])).stdout.trim(); + yield* command(submodule, [ + "update-ref", + "refs/t3/orchestration-v2/checkpoints/test", + checkpoint, + ]); const admin = (yield* command(worktree, ["rev-parse", "--absolute-git-dir"])).stdout.trim(); const report = yield* service.runNow; + expect(editedDuringClean).toBe(false); expect(report.entries.find((entry) => entry.path === worktree)).toMatchObject({ outcome: "removed", }); @@ -751,15 +1144,21 @@ it.live("keeps unpushed submodule history after checkout and deinitialization", it.live("keeps submodules when stored repository metadata is malformed", () => runCleanupTest( Effect.gen(function* () { - const { service, fs, worktree, submodule, child, command } = yield* submoduleCleanupFixture(); + const { service, fs, worktree, submodule, child, command, setPolicy } = + yield* submoduleCleanupFixture(); + setPolicy("tracked-changes"); const childGit = (yield* command(child, ["rev-parse", "--absolute-git-dir"])).stdout.trim(); yield* command(submodule, ["submodule", "deinit", "child"]); yield* fs.writeFileString(`${childGit}/HEAD`, "not a ref\n"); + yield* fs.writeFileString(`${worktree}/notes.txt`, "keep while the worktree stays\n"); expect( (yield* service.runNow).entries.find((entry) => entry.path === worktree), ).toMatchObject({ outcome: "kept", reason: expect.stringContaining("inspected") }); expect(yield* fs.exists(worktree)).toBe(true); expect(yield* fs.readFileString(`${childGit}/HEAD`)).toBe("not a ref\n"); + expect(yield* fs.readFileString(`${worktree}/notes.txt`)).toBe( + "keep while the worktree stays\n", + ); }), ), ); @@ -879,10 +1278,10 @@ describe("subagent worktree ownership", () => { ? "Thread is running or has pending work" : "Not settled", }); - }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), + }).pipe(Effect.provide(cleanupTestLayer)), ); - it.live.each([ + it.effect.each([ "parent-unsettled", "parent-resettled", "child-unsettled", @@ -891,6 +1290,7 @@ describe("subagent worktree ownership", () => { ] as const)("cancels inherited settlement during measurement (%s)", (change) => runCleanupTest( Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); const fixture = yield* pausedMeasurementFixture({ ...DEFAULT_SERVER_SETTINGS.storageCleanup, worktreeSettledAfterDays: 0, @@ -947,6 +1347,8 @@ describe("subagent worktree ownership", () => { it.effect.each([ "idle", "grandchild", + "branchless", + "branchless-running", "null-branch", "stale-branch", "isolated-owner", @@ -978,7 +1380,7 @@ describe("subagent worktree ownership", () => { }, }); const root = shell({ - branch: "feature", + branch: ["branchless", "branchless-running"].includes(scenario) ? null : "feature", worktreePath: fixture.worktree, settledAt: scenario === "child-settled-only" ? null : at(-20 * DAY_MS), ...(scenario === "isolated-owner" || scenario === "fork-owner" @@ -1020,7 +1422,7 @@ describe("subagent worktree ownership", () => { : "subagent", ...(scenario === "side-chat" ? { sideChat: true } : {}), }, - status: scenario === "running" ? "running" : "completed", + status: ["running", "branchless-running"].includes(scenario) ? "running" : "completed", pendingBackgroundTasks: scenario === "pending" ? [{ taskId: "task", kind: "command" }] : [], latestRunCompletedAt: scenario === "recent-activity" || scenario === "recent-inactive" @@ -1069,7 +1471,7 @@ describe("subagent worktree ownership", () => { outcome: removed ? "removed" : "kept", reason: removed ? "Settled for 20 days" - : scenario === "running" || scenario === "pending" + : ["running", "pending", "branchless-running"].includes(scenario) ? "Thread is running or has pending work" : shared ? "Shared by 2 threads" @@ -1081,12 +1483,60 @@ describe("subagent worktree ownership", () => { ? "Inactive for 2 of 8 days" : ["recent-grace", "recent-settle", "child-settled-now"].includes(scenario) ? "Settled moments ago" - : "Not settled", + : scenario === "branchless" + ? "Thread has no branch" + : "Not settled", }); - }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), + }).pipe(Effect.provide(cleanupTestLayer)), ); - it.live.each([ + it.effect("keeps a checkout two top-level threads store through a symlinked parent", () => + Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); + const fixture = yield* cleanupFixture({ + ...DEFAULT_SERVER_SETTINGS.storageCleanup, + worktreeSettledAfterDays: 8, + }); + const linkedParent = `${yield* fixture.fs.makeTempDirectoryScoped({ prefix: "cleanup-link-" })}/worktrees`; + yield* fixture.fs.symlink(fixture.config.worktreesDir, linkedParent); + const settled = { branch: "feature", settledAt: at(-20 * DAY_MS) }; + fixture.setThreads([ + shell({ ...settled, worktreePath: fixture.worktree }), + shell({ ...settled, id: ThreadId.make("other"), worktreePath: `${linkedParent}/feature` }), + ]); + const report = yield* fixture.service.runNow; + expect(report.entries).toEqual([ + expect.objectContaining({ outcome: "kept", reason: "Shared by 2 threads" }), + ]); + expect(yield* fixture.fs.exists(fixture.worktree)).toBe(true); + }).pipe(Effect.provide(cleanupTestLayer)), + ); + + it.effect("keeps a checkout two top-level threads store in different case on Windows", () => + Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); + const fixture = yield* cleanupFixture({ + ...DEFAULT_SERVER_SETTINGS.storageCleanup, + worktreeSettledAfterDays: 8, + }).pipe(Effect.provideService(HostProcess.Platform, "win32")); + const settled = { branch: "feature", settledAt: at(-20 * DAY_MS) }; + fixture.setThreads([ + shell({ ...settled, worktreePath: fixture.worktree }), + shell({ + ...settled, + id: ThreadId.make("other"), + worktreePath: fixture.worktree.toUpperCase(), + }), + ]); + const report = yield* fixture.service.runNow; + expect(report.entries).toEqual([ + expect.objectContaining({ outcome: "kept", reason: "Shared by 2 threads" }), + ]); + expect(yield* fixture.fs.exists(fixture.worktree)).toBe(true); + }).pipe(Effect.provide(cleanupTestLayer)), + ); + + it.effect.each([ "running", "pending", "activity", @@ -1098,6 +1548,7 @@ describe("subagent worktree ownership", () => { ] as const)("cancels removal when a child changes during measurement (%s)", (change) => runCleanupTest( Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); const fixture = yield* pausedMeasurementFixture(); const root = shell({ branch: "feature", worktreePath: fixture.worktree }); const child = { @@ -1187,7 +1638,29 @@ describe("deleted thread worktrees", () => { reason: "Thread was deleted", }); expect(yield* fixture.fs.exists(fixture.worktree)).toBe(false); - }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), + }).pipe(Effect.provide(cleanupTestLayer)), + ); + + it.effect("leaves a deleted owner's subagent on the owner's saved branch", () => + Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); + const fixture = yield* cleanupFixture({ + ...DEFAULT_SERVER_SETTINGS.storageCleanup, + worktreeAfterDays: null, + worktreeOnDelete: true, + }); + const owner = parent(fixture.worktree); + fixture.setDeletedThreads([owner]); + fixture.setThreads([subagent(owner)]); + yield* fixture.command(fixture.worktree, ["switch", "-c", "renamed"]); + expect((yield* fixture.service.runNow).entries[0]).toMatchObject({ + threadId: owner.id, + outcome: "kept", + reason: "Repository or branch changed", + }); + // The deleted owner cannot follow, so moving its subagent would split their records. + expect(fixture.threadBranches()).toEqual(["feature"]); + }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), ); it.effect("applies the remaining subagent's rules when deletion cleanup is off", () => @@ -1209,7 +1682,7 @@ describe("deleted thread worktrees", () => { reason: "Inactive for 30 days", }); expect(yield* fixture.fs.exists(fixture.worktree)).toBe(false); - }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), + }).pipe(Effect.provide(cleanupTestLayer)), ); // Releases one stopped-session event after the startup sweep and returns the snapshot reads it caused. @@ -1268,7 +1741,7 @@ describe("deleted thread worktrees", () => { const fixture = yield* cleanupFixture(); const reads = yield* stopSessionAfterStartupSweep(fixture, ThreadId.make("thread-1")); expect(reads, "a live thread's stopped session re-read every thread").toBe(0); - }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), + }).pipe(Effect.provide(cleanupTestLayer)), ); it.effect("removes a deleted thread's worktree once its session stops", () => @@ -1290,7 +1763,7 @@ describe("deleted thread worktrees", () => { ); expect(reads).toBeGreaterThan(0); expect(yield* fixture.fs.exists(fixture.worktree)).toBe(false); - }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), + }).pipe(Effect.provide(cleanupTestLayer)), ); it.effect("removes a deleted thread's worktree once a live thread's session in it stops", () => @@ -1320,7 +1793,7 @@ describe("deleted thread worktrees", () => { ); expect(reads).toBeGreaterThan(0); expect(yield* fixture.fs.exists(fixture.worktree)).toBe(false); - }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), + }).pipe(Effect.provide(cleanupTestLayer)), ); it.effect( @@ -1366,7 +1839,7 @@ describe("deleted thread worktrees", () => { ); expect(reads).toBeGreaterThan(0); expect(yield* fixture.fs.exists(fixture.worktree)).toBe(false); - }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), + }).pipe(Effect.provide(cleanupTestLayer)), ); it.effect( @@ -1399,7 +1872,7 @@ describe("deleted thread worktrees", () => { ); expect(reads).toBeGreaterThan(0); expect(yield* fixture.fs.exists(fixture.worktree)).toBe(false); - }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), + }).pipe(Effect.provide(cleanupTestLayer)), ); it.effect("does not sweep for a kept deleted worktree when a session elsewhere stops", () => @@ -1421,7 +1894,7 @@ describe("deleted thread worktrees", () => { ); expect(reads).toBe(0); expect(yield* fixture.fs.exists(fixture.worktree)).toBe(true); - }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), + }).pipe(Effect.provide(cleanupTestLayer)), ); it.effect( @@ -1451,7 +1924,7 @@ describe("deleted thread worktrees", () => { fixture.repo, ); expect(reads).toBe(0); - }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), + }).pipe(Effect.provide(cleanupTestLayer)), ); it.effect( @@ -1525,7 +1998,7 @@ describe("deleted thread worktrees", () => { yield* Deferred.await(delivered); yield* fixture.service.drain; expect(yield* fixture.fs.exists(fixture.worktree)).toBe(false); - }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), + }).pipe(Effect.provide(cleanupTestLayer)), ); it.effect("waits out the settle grace of a subagent that just settled", () => @@ -1546,7 +2019,7 @@ describe("deleted thread worktrees", () => { reason: "Settled moments ago", }); expect(yield* fixture.fs.exists(fixture.worktree)).toBe(true); - }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), + }).pipe(Effect.provide(cleanupTestLayer)), ); }); @@ -1635,7 +2108,7 @@ describe("orphan worktree reports", () => { ), ); - it.live.each(["nested", "direct"] as const)( + it.live.each(["nested", "direct", "annexed"] as const)( "reports a linked orphan's size without removing its files (%s)", (location) => runCleanupTest( @@ -1644,8 +2117,13 @@ describe("orphan worktree reports", () => { fixture.setThreads([]); const orphan = `${fixture.config.worktreesDir}/${location === "nested" ? "repository/" : ""}orphan`; yield* fixture.command(fixture.repo, ["worktree", "add", "-b", "orphan", orphan]); - const sizes = yield* Effect.forEach(yield* fixture.fs.readDirectory(orphan), (name) => - allocatedBytes(fixture.fs, `${orphan}/${name}`), + if (location === "annexed") yield* annexWorktree(fixture, orphan); + const sizes = yield* Effect.forEach( + // Links are not measured, and the annexed `.git` is one. + (yield* fixture.fs.readDirectory(orphan)).filter( + (name) => location !== "annexed" || name !== ".git", + ), + (name) => allocatedBytes(fixture.fs, `${orphan}/${name}`), ); const outside = yield* fixture.fs.makeTempDirectoryScoped(); yield* fixture.fs.writeFileString(`${outside}/external`, "outside data"); @@ -1755,6 +2233,57 @@ describe("orphan worktree reports", () => { ), ); + it.live.each([ + { home: "an installed", worktreeHome: false, customFolderOutcome: "removed" }, + { home: "a git worktree's dev", worktreeHome: true, customFolderOutcome: "kept" }, + ] as const)( + "on $home home, a custom worktrees folder ends $customFolderOutcome", + ({ worktreeHome, customFolderOutcome }) => + runCleanupTest( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const config = yield* ServerConfig.ServerConfig; + const checkout = yield* fs.makeTempDirectoryScoped({ prefix: "cleanup-dev-checkout-" }); + yield* fs.writeFileString(`${checkout}/.git`, "gitdir: /repo/.git/worktrees/dev\n"); + const baseDir = worktreeHome ? `${checkout}/.t3` : config.baseDir; + const fixture = yield* cleanupFixture().pipe( + Effect.provideService(ServerConfig.ServerConfig, { + ...config, + baseDir, + worktreesDir: `${baseDir}/worktrees`, + }), + ); + const customFolder = yield* fs.realPath( + yield* fs.makeTempDirectoryScoped({ prefix: "cleanup-real-worktrees-" }), + ); + const realWorktree = `${customFolder}/seeded`; + yield* fixture.command(fixture.repo, ["worktree", "add", "-b", "seeded", realWorktree]); + fixture.setSettings({ worktreesDirectory: customFolder }); + fixture.setThreads([ + shell({ branch: "feature", worktreePath: fixture.worktree }), + shell({ + id: ThreadId.make("seeded"), + lineage: { + rootThreadId: ThreadId.make("seeded"), + parentThreadId: null, + relationshipToParent: null, + }, + branch: "seeded", + worktreePath: realWorktree, + }), + ]); + + const { entries } = yield* fixture.service.runNow; + + expect(entries.find((entry) => entry.path === realWorktree)?.outcome).toBe( + customFolderOutcome, + ); + expect(yield* fs.exists(realWorktree)).toBe(customFolderOutcome === "kept"); + expect(yield* fs.exists(fixture.worktree)).toBe(false); + }), + ), + ); + it.live("skips symbolic links at both directory levels and disabled cleanup", () => runCleanupTest( Effect.gen(function* () { @@ -2067,7 +2596,8 @@ describe("storage cleanup reports and local file policies", () => { occurredAt: at(0), payload: { ...shell({ deletedAt: at(0) }), lastVisitedAt: null }, }); - expect(fixture.snapshotReads()).toBe(startupReads + 3 * Number(worktreeOnDelete)); + // Deletion always sweeps so the deleted thread's checkpoint refs are pruned. + expect(fixture.snapshotReads()).toBe(startupReads + 2 * Number(worktreeOnDelete) + 1); expect(fixture.sessionLiveChecks()).toBe(worktreeOnDelete ? 6 : 0); }), ), @@ -2107,6 +2637,114 @@ describe("storage cleanup reports and local file policies", () => { }), ), ); + it.effect("removes an idle No project thread's scratch folder past the retention", () => + runCleanupTest( + Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); + const fixture = yield* cleanupFixture(); + const scratchRoot = `${fixture.config.baseDir}/scratch`; + const folder = `${scratchRoot}/quiet-otter`; + yield* fixture.fs.makeDirectory(folder, { recursive: true }); + yield* fixture.fs.writeFileString(`${folder}/notes.md`, "scratch\n"); + fixture.setProjectRoots([scratchRoot]); + fixture.setStoredWorktreePaths([]); + // Only this rule lets cleanup delete untracked files, which is all a scratch folder has. + fixture.setPolicy("tracked-changes"); + fixture.setThreads([ + shell({ worktreePath: folder, settledAt: at(-20 * DAY_MS), status: "completed" }), + ]); + const report = yield* fixture.service.runNow; + expect(report.entries).toEqual([ + expect.objectContaining({ path: folder, outcome: "removed" }), + ]); + expect(yield* fixture.fs.exists(folder)).toBe(false); + }), + ), + ); + it.effect.each([ + { + name: "a Git repository in a subfolder", + policy: "tracked-changes", + repository: "clone/.git", + reason: "Contains a nested Git repository", + }, + { + name: "a Git repository of its own", + policy: "tracked-changes", + repository: ".git", + reason: "Contains a nested Git repository", + }, + { name: "files under the default keep rule", reason: "Has local files (notes.md)" }, + { + name: "a scheduled task", + policy: "tracked-changes", + scheduled: true, + reason: "Thread has a scheduled task", + }, + ] as const)( + "keeps an idle No project thread's scratch folder with $name", + ({ + policy, + repository, + scheduled, + reason, + }: { + readonly policy?: WorktreeKeepWhen; + readonly repository?: string; + readonly scheduled?: boolean; + readonly reason: string; + }) => + runCleanupTest( + Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); + const fixture = yield* cleanupFixture(); + const scratchRoot = `${fixture.config.baseDir}/scratch`; + const folder = `${scratchRoot}/quiet-otter`; + yield* fixture.fs.makeDirectory( + repository === undefined ? folder : `${folder}/${repository}`, + { recursive: true }, + ); + yield* fixture.fs.writeFileString(`${folder}/notes.md`, "scratch\n"); + // No thread owns this one, so it is listed and never removed. + const leftover = `${scratchRoot}/leftover`; + yield* fixture.fs.makeDirectory(leftover); + yield* fixture.fs.utimes(leftover, 1, 1); + fixture.setProjectRoots([scratchRoot]); + fixture.setStoredWorktreePaths([]); + if (policy !== undefined) fixture.setPolicy(policy); + if (scheduled) fixture.setScheduledThreadIds(["thread-1"]); + fixture.setThreads([ + shell({ worktreePath: folder, settledAt: at(-20 * DAY_MS), status: "completed" }), + ]); + const report = yield* fixture.service.runNow; + expect( + report.entries.map(({ path, outcome, reason }) => ({ path, outcome, reason })), + ).toEqual([ + { path: leftover, outcome: "kept", reason: "No thread uses this folder" }, + { path: folder, outcome: "kept", reason }, + ]); + expect(yield* fixture.fs.exists(`${folder}/notes.md`)).toBe(true); + }), + ), + ); + it.effect("keeps a thread's worktree folder that lost its .git link", () => + runCleanupTest( + Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); + const fixture = yield* cleanupFixture(); + fixture.setThreads([ + shell({ branch: "feature", worktreePath: fixture.worktree, archivedAt: at(-DAY_MS) }), + ]); + yield* fixture.fs.remove(`${fixture.worktree}/.git`); + yield* fixture.fs.makeDirectory(`${fixture.worktree}/node_modules`); + const report = yield* fixture.service.runNow; + expect(report.entries).toEqual([ + expect.objectContaining({ path: fixture.worktree, outcome: "kept" }), + ]); + expect(yield* fixture.fs.exists(`${fixture.worktree}/node_modules`)).toBe(true); + }), + ), + ); it.live("skips a stored worktree path whose parent is a regular file", () => runCleanupTest( Effect.gen(function* () { @@ -2122,18 +2760,30 @@ describe("storage cleanup reports and local file policies", () => { }), ), ); - it.live.each(["before-check", "while-measuring"] as const)( - "keeps a worktree protected by a missing project root (%s)", - (timing) => + it.live.each([ + ["before-check", "stored"], + ["while-measuring", "stored"], + ["before-check", "symlinked parent"], + ["while-measuring", "symlinked parent"], + ] as const)( + "keeps a worktree protected by a missing project root (%s, %s spelling)", + ([timing, spelling]) => runCleanupTest( Effect.gen(function* () { const fixture = yield* pausedMeasurementFixture(); + let worktree = fixture.worktree; + if (spelling === "symlinked parent") { + const linkedParent = `${yield* fixture.fs.makeTempDirectoryScoped({ prefix: "cleanup-link-" })}/worktrees`; + yield* fixture.fs.symlink(fixture.config.worktreesDir, linkedParent); + worktree = `${linkedParent}/feature`; + fixture.setThreads([shell({ branch: "feature", worktreePath: worktree })]); + } if (timing === "before-check") - fixture.setProjectRoots([fixture.repo, `${fixture.worktree}/removed-checkout`]); + fixture.setProjectRoots([fixture.repo, `${worktree}/removed-checkout`]); const cleanup = yield* fixture.service.runNow.pipe(Effect.forkChild); if (timing === "while-measuring") { yield* Deferred.await(fixture.measuring); - fixture.setProjectRoots([fixture.repo, `${fixture.worktree}/removed-checkout`]); + fixture.setProjectRoots([fixture.repo, `${worktree}/removed-checkout`]); } yield* Deferred.succeed(fixture.resume, undefined); const report = yield* Fiber.join(cleanup); @@ -2183,9 +2833,14 @@ describe("storage cleanup reports and local file policies", () => { Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const storeDir = yield* fs.makeTempDirectoryScoped({ prefix: "pnpm-store-" }); + const statted: string[] = []; const { service, worktree, setPolicy } = yield* cleanupFixture().pipe( Effect.provideService(FileSystem.FileSystem, { ...fs, + lstat: (target) => { + statted.push(target); + return fs.lstat(target); + }, stat: (target) => fs .stat(target) @@ -2226,6 +2881,9 @@ describe("storage cleanup reports and local file policies", () => { ); const report = yield* service.runNow; expect(report.entries[0]).toMatchObject({ outcome: "removed", bytes }); + expect(statted.includes(`${nodeModules}/.pnpm/pkg@1.0.0/index.js`)).toBe( + store !== "same-device", + ); }), ), ); @@ -2259,12 +2917,13 @@ describe("storage cleanup reports and local file policies", () => { }), ), ); - it.live.each([ + it.effect.each([ { days: 8, reason: "Inactive for 1 of 8 days" }, { days: null, reason: "No rules apply" }, ])("keeps an ineligible age/delete-only worktree without Git ($days)", ({ days, reason }) => runCleanupTest( Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); const fixture = yield* cleanupFixture({ ...DEFAULT_SERVER_SETTINGS.storageCleanup, worktreeAfterDays: days, @@ -2274,9 +2933,7 @@ describe("storage cleanup reports and local file policies", () => { shell({ branch: "feature", worktreePath: fixture.worktree, - latestUserMessageAt: DateTime.makeUnsafe( - DateTime.toEpochMillis(yield* DateTime.now) - DAY_MS, - ), + latestUserMessageAt: at(-DAY_MS), }), ]); yield* fixture.fs.writeFileString(`${fixture.worktree}/tracked.txt`, "local edit\n"); @@ -2320,7 +2977,18 @@ describe("storage cleanup reports and local file policies", () => { }), ), ); - it.live("still removes a worktree when measuring its size fails", () => + it.live("removes a worktree whose .git file git-annex replaced with a symlink", () => + runCleanupTest( + Effect.gen(function* () { + const fixture = yield* cleanupFixture(); + const { service, worktree, fs } = fixture; + yield* annexWorktree(fixture, worktree); + expect((yield* service.runNow).entries[0]).toMatchObject({ outcome: "removed" }); + expect(yield* fs.exists(worktree)).toBe(false); + }), + ), + ); + it.live("keeps a worktree when the inspection cannot read its directories", () => runCleanupTest( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -2331,13 +2999,48 @@ describe("storage cleanup reports and local file policies", () => { }), ); const report = yield* service.runNow; - expect(report.entries[0]).toMatchObject({ outcome: "removed", bytes: null, files: null }); + expect(report.entries[0]).toMatchObject({ + outcome: "kept", + reason: "Worktree could not be fully inspected", + bytes: null, + files: null, + }); expect(report.bytesFreed).toBe(0); - expect(yield* fs.exists(worktree)).toBe(false); + expect(yield* fs.exists(worktree)).toBe(true); }), ), ); - it.live.each([ + it.effect("removes a worktree owned by a repository other than the project root", () => + runCleanupTest( + Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); + const fixture = yield* cleanupFixture(); + const { fs, command, config } = fixture; + const oldRepo = yield* fs.realPath( + yield* fs.makeTempDirectoryScoped({ prefix: "cleanup-old-repo-" }), + ); + yield* command(oldRepo, ["init", "-b", "main"]); + yield* command(oldRepo, ["config", "user.email", "cleanup@example.test"]); + yield* command(oldRepo, ["config", "user.name", "Cleanup test"]); + yield* fs.writeFileString(`${oldRepo}/tracked.txt`, "original\n"); + yield* command(oldRepo, ["add", "."]); + yield* command(oldRepo, ["commit", "-m", "initial"]); + yield* fs.makeDirectory(`${config.worktreesDir}/old-repo`, { recursive: true }); + const oldWorktree = `${config.worktreesDir}/old-repo/feature`; + yield* command(oldRepo, ["worktree", "add", "-b", "feature", oldWorktree]); + fixture.setThreads([shell({ branch: "feature", worktreePath: oldWorktree })]); + const report = yield* fixture.service.runNow; + expect(report.entries.find((entry) => entry.path === oldWorktree)).toMatchObject({ + outcome: "removed", + }); + expect(yield* fs.exists(oldWorktree)).toBe(false); + expect((yield* command(oldRepo, ["worktree", "list", "--porcelain"])).stdout).not.toContain( + oldWorktree, + ); + }), + ), + ); + it.effect.each([ [ "its owner's branch is cleared", (worktree: string) => [shell({ branch: null, worktreePath: worktree, updatedAt: at(0) })], @@ -2374,6 +3077,7 @@ describe("storage cleanup reports and local file policies", () => { ] as const)("keeps a worktree when %s during size measurement", ([, threadsWhileMeasuring]) => runCleanupTest( Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); const fixture = yield* pausedMeasurementFixture(); const { fs, worktree } = fixture; fixture.setPolicy("tracked-changes"); @@ -2394,6 +3098,20 @@ describe("storage cleanup reports and local file policies", () => { }), ), ); + it.live("follows a branch switched while nothing watched the worktree", () => + runCleanupTest( + Effect.gen(function* () { + const { service, worktree, command, threadBranches } = yield* cleanupFixture(); + yield* command(worktree, ["switch", "-c", "renamed"]); + expect((yield* service.runNow).entries[0]).toMatchObject({ + outcome: "kept", + reason: "Repository or branch changed", + }); + expect(threadBranches()).toEqual(["renamed"]); + expect((yield* service.runNow).entries[0]).toMatchObject({ outcome: "removed" }); + }), + ), + ); it.live("does not measure a worktree kept for an open provider session", () => runCleanupTest( Effect.gen(function* () { @@ -2425,24 +3143,39 @@ describe("storage cleanup reports and local file policies", () => { }), ), ); + it.live("keeps a worktree for a live session in a missing folder under its stored alias", () => + runCleanupTest( + Effect.gen(function* () { + const fixture = yield* cleanupFixture(); + const linkedParent = `${yield* fixture.fs.makeTempDirectoryScoped({ prefix: "cleanup-link-" })}/worktrees`; + yield* fixture.fs.symlink(fixture.config.worktreesDir, linkedParent); + fixture.setThreads([shell({ branch: "feature", worktreePath: `${linkedParent}/feature` })]); + fixture.setOpenSessionCwd(`${linkedParent}/feature/removed-folder`); + const report = yield* fixture.service.runNow; + expect(report.entries[0]).toMatchObject({ + outcome: "kept", + reason: "Provider session is still open", + }); + expect(yield* fixture.fs.exists(fixture.worktree)).toBe(true); + }), + ), + ); it.live("compares the checked-out branch without a full status", () => runCleanupTest( Effect.gen(function* () { - const { service, fs, repo, worktree, command, setThreads, fullStatusCalls } = + const { service, fs, repo, worktree, command, fullStatusCalls, threadBranches } = yield* cleanupFixture(); const changed = { outcome: "kept", reason: "Repository or branch changed" }; yield* command(worktree, ["checkout", "--detach"]); expect((yield* service.runNow).entries[0]).toMatchObject(changed); yield* command(worktree, ["switch", "--orphan", "other"]); expect((yield* service.runNow).entries[0]).toMatchObject(changed); - yield* command(worktree, ["switch", "-c", "(feature)", "feature"]); - setThreads([shell({ branch: "(feature)", worktreePath: worktree })]); - expect((yield* service.runNow).entries[0]).toMatchObject(changed); - setThreads([shell({ branch: "feature", worktreePath: worktree })]); yield* command(worktree, ["tag", "feature", "feature"]); // Newer Git refuses to create a tag named HEAD, but still reads one. yield* command(worktree, ["update-ref", "refs/tags/HEAD", "refs/heads/feature"]); yield* command(worktree, ["switch", "feature"]); + expect((yield* service.runNow).entries[0]).toMatchObject(changed); + expect(threadBranches()).toEqual(["feature"]); const index = (yield* command(worktree, ["rev-parse", "--git-path", "index"])).stdout; yield* fs.writeFileString(`${index.trim()}.lock`, ""); const locked = (yield* service.runNow).entries[0]; @@ -2509,7 +3242,7 @@ describe("storage cleanup reports and local file policies", () => { }), ), ); - it.live.each([ + it.effect.each([ ["unchanged", "removed", "No commits beyond the project branch"], ["unmerged", "kept", "Has commits beyond the project branch"], ["detached", "kept", "Project branch is unavailable"], @@ -2517,6 +3250,7 @@ describe("storage cleanup reports and local file policies", () => { ] as const)("cleans archived local-only worktrees: %s", ([state, outcome, reason]) => runCleanupTest( Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); const { service, fs, repo, worktree, command, setThreads } = yield* cleanupFixture({ ...DEFAULT_SERVER_SETTINGS.storageCleanup, worktreeUnchanged: state !== "merged-only", @@ -2539,15 +3273,55 @@ describe("storage cleanup reports and local file policies", () => { }), ), ); + it.live("keeps an unsettled new thread's unchanged worktree after its first turn", () => + runCleanupTest( + Effect.gen(function* () { + const { service, fs, worktree, setThreads } = yield* cleanupFixture({ + ...DEFAULT_SERVER_SETTINGS.storageCleanup, + worktreeSettledAfterDays: 11, + worktreeOnMerge: true, + worktreeUnchanged: true, + }); + const now = yield* DateTime.now; + const minutesAgo = (minutes: number) => DateTime.subtract(now, { minutes }); + const thread = shell({ + branch: "feature", + worktreePath: worktree, + status: "completed", + createdAt: minutesAgo(40), + latestUserMessageAt: minutesAgo(40), + latestRunRequestedAt: minutesAgo(40), + latestRunStartedAt: minutesAgo(40), + latestRunCompletedAt: minutesAgo(20), + updatedAt: minutesAgo(20), + }); + setThreads([thread]); + + const report = yield* service.runNow; + expect(report.entries[0]).toMatchObject({ outcome: "kept" }); + expect(yield* fs.exists(worktree)).toBe(true); + + setThreads([{ ...thread, settledAt: minutesAgo(10), updatedAt: minutesAgo(10) }]); + const settled = yield* service.runNow; + expect(settled.entries[0]).toMatchObject({ + outcome: "removed", + reason: "No commits beyond the project branch", + }); + }), + ), + ); it.live( "discovers trunk without a remote HEAD or tracking ref and removes an integrated worktree", () => runCleanupTest( Effect.gen(function* () { - const { service, fs, repo, worktree, command } = yield* cleanupFixture({ + const { service, fs, repo, worktree, command, setThreads } = yield* cleanupFixture({ ...DEFAULT_SERVER_SETTINGS.storageCleanup, worktreeUnchanged: true, }); + setThreads([ + shell({ branch: "feature", worktreePath: worktree, archivedAt: at(-DAY_MS) }), + ]); const remote = yield* fs.makeTempDirectoryScoped({ prefix: "cleanup-remote-" }); yield* command(remote, ["init", "--bare", "-b", "trunk"]); yield* command(repo, ["remote", "add", "origin", remote]); @@ -2569,10 +3343,13 @@ describe("storage cleanup reports and local file policies", () => { () => runCleanupTest( Effect.gen(function* () { - const { service, fs, repo, worktree, command } = yield* cleanupFixture({ + const { service, fs, repo, worktree, command, setThreads } = yield* cleanupFixture({ ...DEFAULT_SERVER_SETTINGS.storageCleanup, worktreeUnchanged: true, }); + setThreads([ + shell({ branch: "feature", worktreePath: worktree, archivedAt: at(-DAY_MS) }), + ]); const remote = yield* fs.makeTempDirectoryScoped({ prefix: "cleanup-remote-" }); yield* command(remote, ["init", "--bare", "-b", "trunk"]); yield* command(repo, ["remote", "add", "origin", remote]); @@ -2597,10 +3374,11 @@ describe("storage cleanup reports and local file policies", () => { it.live("keeps the worktree when the remote default is unavailable even if main exists", () => runCleanupTest( Effect.gen(function* () { - const { service, fs, repo, worktree, command } = yield* cleanupFixture({ + const { service, fs, repo, worktree, command, setThreads } = yield* cleanupFixture({ ...DEFAULT_SERVER_SETTINGS.storageCleanup, worktreeUnchanged: true, }); + setThreads([shell({ branch: "feature", worktreePath: worktree, archivedAt: at(-DAY_MS) })]); const remote = yield* fs.makeTempDirectoryScoped({ prefix: "cleanup-remote-" }); yield* command(remote, ["init", "--bare", "-b", "trunk"]); yield* command(repo, ["remote", "add", "origin", remote]); @@ -2906,11 +3684,12 @@ describe("storage cleanup reports and local file policies", () => { }); describe("settled worktree rechecks", () => { - it.live.each(["autoResumeLimitedThreads", "snoozeLimitedThreads"] as const)( + it.effect.each(["autoResumeLimitedThreads", "snoozeLimitedThreads"] as const)( "keeps a usage-limited checkout when %s is enabled during measurement", (preference) => runCleanupTest( Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); const fixture = yield* pausedMeasurementFixture(); fixture.setSettings({ autoResumeLimitedThreads: false, snoozeLimitedThreads: false }); fixture.setThreads([ @@ -2928,16 +3707,20 @@ describe("settled worktree rechecks", () => { fixture.setSettings({ [preference]: true }); yield* Deferred.succeed(fixture.resume, undefined); const report = yield* Fiber.join(run); - expect(report.entries[0]?.outcome).toBe("kept"); + expect(report.entries[0]).toMatchObject({ + outcome: "kept", + reason: "Settings changed since check", + }); expect(yield* fixture.fs.exists(fixture.worktree)).toBe(true); }), ), ); - it.live.each([{ settledAt: null }, { settledAt: at(DAY_MS) }])( + it.effect.each([{ settledAt: null }, { settledAt: at(DAY_MS) }])( "keeps a checkout changed during measurement (%j)", (change) => runCleanupTest( Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); const fixture = yield* pausedMeasurementFixture(); const thread = shell({ branch: "feature", @@ -2965,9 +3748,10 @@ describe("settled worktree rechecks", () => { ); }); -it.live("keeps saved usage data when removing a settled checkout", () => +it.effect("keeps saved usage data when removing a settled checkout", () => runCleanupTest( Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); const fixture = yield* cleanupFixture(); fixture.setSettings({ storageCleanup: { ...DEFAULT_SERVER_SETTINGS.storageCleanup, worktreeSettledAfterDays: 0 }, diff --git a/apps/server/src/storageCleanup.ts b/apps/server/src/storageCleanup.ts index 503ced625f8c..60605c23eb8a 100644 --- a/apps/server/src/storageCleanup.ts +++ b/apps/server/src/storageCleanup.ts @@ -1,8 +1,12 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeFSP from "node:fs/promises"; import { + CheckpointScopeId, GitCommandError, OrchestrationV2AppThreadJson, OrchestrationV2ProviderSessionJson, } from "@t3tools/contracts"; +import * as Base64Url from "effect/encoding/Base64Url"; import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; import type { @@ -17,6 +21,7 @@ import type { ThreadId, } from "@t3tools/contracts"; import { resolveWorktreeCleanup } from "@t3tools/shared/projectSettings"; +import { resolveWorktreeT3Home } from "@t3tools/shared/devHome"; import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker"; import * as Context from "effect/Context"; import * as Deferred from "effect/Deferred"; @@ -37,18 +42,23 @@ import { parse as parseYaml } from "yaml"; import * as ServerConfig from "./config.ts"; import * as GitManager from "./git/GitManager.ts"; +import * as GitWorkflowService from "./git/GitWorkflowService.ts"; import * as ProjectStore from "./orchestration-v2/ProjectStore.ts"; +import { scratchFolderRoot } from "./project/ManagedProjectFolders.ts"; import * as Orchestrator from "./orchestration-v2/Orchestrator.ts"; import * as ProjectionStore from "./orchestration-v2/ProjectionStore.ts"; import * as ProviderSessionManager from "./orchestration-v2/ProviderSessionManager.ts"; +import { checkpointRefForScopeOrdinal } from "./orchestration-v2/CheckpointService.ts"; import { threadHasQueuedTurnStart } from "./orchestration-v2/ThreadSettlementService.ts"; +import { MAX_OVERDUE_ARM_MS } from "./orchestration-v2/UsageLimitRecoveryWorker.ts"; +import * as WorktreeBranchSync from "./orchestration-v2/WorktreeBranchSync.ts"; import { forkParked } from "./serverActivation.ts"; import * as Settings from "./serverSettings.ts"; import * as TerminalManager from "./terminal/Manager.ts"; import { isFilesystemRoot, managedWorktreesDirectories } from "./worktreesDirectory.ts"; import * as GitVcsDriver from "./vcs/GitVcsDriver.ts"; import { isNonRepositoryGitStderr } from "./vcs/GitVcsDriverCore.ts"; -import { removeLinkedWorktree } from "./vcs/worktreeRemoval.ts"; +import { isLinkedWorktree, removeLinkedWorktree } from "./vcs/worktreeRemoval.ts"; import { withWorkspaceLease } from "./workspace/workspaceLease.ts"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; @@ -77,6 +87,15 @@ class PnpmModulesManifestParseError extends Schema.TaggedError()( + "WorktreeDirectoryReadError", + { cause: Schema.Defect() }, +) { + override get message(): string { + return "Could not read a worktree directory."; + } +} + const DAY_MS = 86_400_000; // Give settle scripts time to open terminals that protect the worktree. const SETTLE_SCRIPT_GRACE_MS = 60_000; @@ -182,10 +201,12 @@ function heldForLimitRecovery( // An auto-resume holds until its continuation replaces the failed run. return recovery.autoResume || (recovery.snooze === true && resetMs > now); // The recovery worker arms this failure on a later tick; hold it until then. - // Like the worker, skip a reset that had already passed when the run ended. + // Like the worker, skip a reset that had already passed when the run ended + // or that is too overdue to arm. if ( !Number.isFinite(resetMs) || - resetMs <= DateTime.toEpochMillis(thread.latestRunCompletedAt ?? thread.updatedAt) + resetMs <= DateTime.toEpochMillis(thread.latestRunCompletedAt ?? thread.updatedAt) || + now - resetMs > MAX_OVERDUE_ARM_MS ) return false; return settings.autoResumeLimitedThreads || (settings.snoozeLimitedThreads && resetMs > now); @@ -305,6 +326,8 @@ const make = Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; const git = yield* GitVcsDriver.GitVcsDriver; const gitManager = yield* GitManager.GitManager; + const gitWorkflow = yield* GitWorkflowService.GitWorkflowService; + const reconcileBranch = yield* WorktreeBranchSync.makeReconcile; const terminals = yield* TerminalManager.TerminalManager; const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -331,20 +354,6 @@ const make = Effect.gen(function* () { !path.isAbsolute(relative) ); }; - const hasTerminal = (worktreePath: string) => - [...liveTerminals.values()] - .flatMap((entries) => [...entries.values()]) - .some((terminal) => { - if (terminal.status !== "starting" && terminal.status !== "running") return false; - const cwd = path.resolve(terminal.cwd); - return ( - (terminal.worktreePath !== null && - pathKey(terminal.worktreePath) === pathKey(worktreePath)) || - pathKey(cwd) === pathKey(worktreePath) || - inside(worktreePath, cwd) - ); - }); - // pnpm imports packages by clone or reflink when its store shares a device // with the worktree. Those files have one link but share blocks with the // store, so removing them frees almost nothing. A store on another device @@ -375,9 +384,48 @@ const make = Effect.gen(function* () { ); const measureWorktree = Effect.fn("storageCleanup.measureWorktree")( - function* (worktreePath: string) { + function* (worktreePath: string, inspectRepositories: boolean | "plain-folder" = false) { const root = yield* fs.realPath(worktreePath); const skipped = yield* sharedPnpmVirtualStore(root); + const incomplete = { bytes: null, reason: "worktree could not be fully inspected" }; + // A plain folder is not a checkout, so a repository at its root counts like one below it. + const repositories = new Set(inspectRepositories === "plain-folder" ? [] : [pathKey(root)]); + const moduleStores: string[] = []; + if (inspectRepositories === true && (yield* fs.exists(path.join(root, ".gitmodules")))) { + const submodules = yield* git.execute({ + operation: "StorageCleanup.nestedRepositories", + cwd: root, + args: [ + "submodule", + "foreach", + "--recursive", + "--quiet", + 'printf "%s\\0" "$toplevel/$sm_path"', + ], + maxOutputBytes: 64 * 1024, + }); + if (submodules.stdoutTruncated || submodules.stderrTruncated) return incomplete; + for (const submodule of submodules.stdout.split("\0").filter(Boolean)) { + repositories.add(pathKey(path.resolve(submodule))); + repositories.add(pathKey(path.resolve(submodule, ".git"))); + moduleStores.push(pathKey(path.resolve(submodule, ".git", "modules"))); + } + } + // Git status omits ignored repositories, and worktree remove deletes them. + // Registered submodules have their own recursive safety checks. + const isNestedRepository = (target: string, names: ReadonlyArray) => { + const key = pathKey(target); + if ( + repositories.has(key) || + moduleStores.some((store) => key === store || inside(store, key)) + ) + return false; + const markers = new Set(names.map((name) => name.toLowerCase())); + return ( + markers.has(".git") || + (markers.has("head") && markers.has("objects") && markers.has("refs")) + ); + }; const pending = [root]; let entries = 1; let bytes = 0; @@ -386,39 +434,57 @@ const make = Effect.gen(function* () { pending.splice(-8), // Untraced: this runs once per entry, and a worktree can hold millions. Effect.fnUntraced(function* (target) { - if (target === skipped) return { target, bytes: 0, names: [] }; + const empty = { target, bytes: 0, names: [], children: [] }; + const inStore = skipped !== null && (target === skipped || inside(skipped, target)); + if (inStore && !inspectRepositories) return empty; const stat = yield* fs.lstat(target); - if (stat.type === "SymbolicLink") return { target, bytes: 0, names: [] }; + if (stat.type === "SymbolicLink") return empty; if ((yield* fs.realPath(target)) !== target) return null; - if (stat.type === "Directory") - return { target, bytes: 0, names: yield* fs.readDirectory(target) }; + if (stat.type === "Directory") { + if (!inStore) { + const names = yield* fs.readDirectory(target); + return { target, bytes: 0, names, children: names }; + } + // The store is excluded from reclaimed bytes; only walk its directories. + const dirents = yield* Effect.tryPromise({ + try: () => NodeFSP.readdir(target, { withFileTypes: true }), + catch: (cause) => new WorktreeDirectoryReadError({ cause }), + }); + return { + target, + bytes: 0, + names: dirents.map((entry) => entry.name), + children: dirents.filter((entry) => entry.isDirectory()).map((entry) => entry.name), + }; + } // Removing a shared hardlink leaves its blocks allocated. - if (stat.type !== "File" || Option.getOrElse(stat.nlink, () => 1n) !== 1n) - return { target, bytes: 0, names: [] }; + if (stat.type !== "File" || Option.getOrElse(stat.nlink, () => 1n) !== 1n) return empty; const bytes = Option.match(stat.blocks, { onNone: () => Number(stat.size), onSome: (blocks) => Number(blocks * 512n), }); - return { target, bytes, names: [] }; + return { ...empty, bytes }; }), { concurrency: "unbounded" }, ); for (const result of inspected) { - if (result === null) return null; + if (result === null) return incomplete; + if (inspectRepositories && isNestedRepository(result.target, result.names)) + return { bytes: null, reason: "contains a nested Git repository" }; bytes += result.bytes; entries += result.names.length; - if (entries > 2_000_000) return null; - for (const name of result.names) { + if (entries > 2_000_000) return incomplete; + for (const name of result.children) { const child = path.join(result.target, name); - if (!inside(root, child)) return null; + if (!inside(root, child)) return incomplete; pending.push(child); } } } - return bytes; + return { bytes, reason: null }; }, Effect.timeout("30 seconds"), - Effect.orElseSucceed(() => null), + Effect.orElseSucceed(() => ({ bytes: null, reason: "worktree could not be fully inspected" })), ); // Large snapshots take seconds; use the reader worker instead of blocking the event loop. @@ -429,18 +495,21 @@ const make = Effect.gen(function* () { }); const containsProjectRoot = Effect.fn("StorageCleanup.containsProjectRoot")(function* ( - worktreePath: string, + worktreePaths: ReadonlyArray, projects: ReadonlyArray<{ readonly workspaceRoot: string }>, ) { + const contains = (projectPath: string) => + worktreePaths.some( + (worktreePath) => + pathKey(projectPath) === pathKey(worktreePath) || inside(worktreePath, projectPath), + ); for (const project of projects) { const projectPath = path.resolve(project.workspaceRoot); - if (pathKey(projectPath) === pathKey(worktreePath) || inside(worktreePath, projectPath)) - return true; + if (contains(projectPath)) return true; const realPath = yield* fs .realPath(projectPath) .pipe(Effect.orElseSucceed(() => projectPath)); - if (pathKey(realPath) === pathKey(worktreePath) || inside(worktreePath, realPath)) - return true; + if (contains(realPath)) return true; } return false; }); @@ -449,6 +518,7 @@ const make = Effect.gen(function* () { roots: ReadonlyArray, projects: ReadonlyArray<{ readonly workspaceRoot: string }>, entries: StorageCleanupReportEntry[], + scratchRoot: string | null, ) { const rows = yield* sql<{ worktreePath: string }>` SELECT DISTINCT json_extract(payload_json, '$.worktreePath') AS "worktreePath" @@ -487,12 +557,14 @@ const make = Effect.gen(function* () { (root) => !inside(root, managedRoot) && (pathKey(root) === pathKey(target) || inside(root, target)), ); - const directories = Effect.fnUntraced(function* (parent: string, managedRoot: string) { - if (inProject(parent, managedRoot)) return []; + // The scratch root is itself a project, so its folders skip the project check. + const directories = Effect.fnUntraced(function* (parent: string, managedRoot: string | null) { + const skipped = (target: string) => managedRoot !== null && inProject(target, managedRoot); + if (skipped(parent)) return []; const result: string[] = []; for (const name of yield* fs.readDirectory(parent).pipe(Effect.orElseSucceed(() => []))) { const target = path.join(parent, name); - if (!inside(parent, target) || inProject(target, managedRoot)) continue; + if (!inside(parent, target) || skipped(target)) continue; const directory = yield* Effect.gen(function* () { if (pathKey(yield* fs.realPath(target)) !== pathKey(target)) return false; return (yield* fs.stat(target)).type === "Directory"; @@ -507,7 +579,7 @@ const make = Effect.gen(function* () { let firstMeasurement = true; const reported = new Set(); const report = Effect.fnUntraced( - function* (worktreePath: string) { + function* (worktreePath: string, scratch = false) { const key = pathKey(worktreePath); if ( reported.has(key) || @@ -519,7 +591,7 @@ const make = Effect.gen(function* () { const modified = Option.getOrNull((yield* fs.stat(worktreePath)).mtime); if (modified !== null && modified.getTime() > now - ORPHAN_WORKTREE_GRACE_MS) return; const gitPath = path.join(worktreePath, ".git"); - const linked = (yield* fs.exists(gitPath)) && (yield* fs.stat(gitPath)).type === "File"; + const linked = (yield* fs.exists(gitPath)) && (yield* isLinkedWorktree(worktreePath)); reported.add(key); const remaining = measurementDeadline - (yield* Clock.currentTimeMillis); if (!orphanSizes.has(key) && (firstMeasurement || remaining > 0)) { @@ -527,7 +599,7 @@ const make = Effect.gen(function* () { firstMeasurement ? Effect.asSome : Effect.timeoutOption(remaining), ); firstMeasurement = false; - if (Option.isSome(measured)) orphanSizes.set(key, measured.value); + if (Option.isSome(measured)) orphanSizes.set(key, measured.value.bytes); } const bytes = orphanSizes.get(key) ?? null; entries.push({ @@ -535,7 +607,9 @@ const make = Effect.gen(function* () { outcome: "kept", reason: linked ? "No thread uses this worktree" - : "No thread uses this folder, and it isn't a git worktree", + : scratch + ? "No thread uses this folder" + : "No thread uses this folder, and it isn't a git worktree", path: worktreePath, projectPath: null, threadId: null, @@ -562,7 +636,7 @@ const make = Effect.gen(function* () { (worktree) => Effect.gen(function* () { if (pathKey(yield* fs.realPath(worktree)) !== pathKey(worktree)) return false; - return (yield* fs.stat(path.join(worktree, ".git"))).type === "File"; + return yield* isLinkedWorktree(worktree); }).pipe(Effect.orElseSucceed(() => false)), ); }); @@ -573,7 +647,7 @@ const make = Effect.gen(function* () { if (referenced.has(pathKey(projectDirectory))) return; const gitPath = path.join(projectDirectory, ".git"); if (yield* fs.exists(gitPath)) { - if ((yield* fs.stat(gitPath)).type === "File") yield* report(projectDirectory); + if (yield* isLinkedWorktree(projectDirectory)) yield* report(projectDirectory); return; } for (const worktreePath of yield* directories(projectDirectory, root)) { @@ -583,6 +657,8 @@ const make = Effect.gen(function* () { }).pipe(Effect.orElseSucceed(() => undefined)); } } + if (scratchRoot !== null) + for (const folder of yield* directories(scratchRoot, null)) yield* report(folder, true); for (const key of orphanSizes.keys()) { if (!reported.has(key)) orphanSizes.delete(key); } @@ -635,9 +711,24 @@ const make = Effect.gen(function* () { if (head === null) return true; const ref = head.exitCode === 0 ? head.stdout.trim() : ""; const name = ref.startsWith("refs/heads/") ? ref.slice("refs/heads/".length).trim() : null; - return (name?.startsWith("(") ? null : name) !== branch; + return name !== branch; }); + // Nothing watches an idle worktree, so its threads still record the branch it left. + // Record the checked-out one so the next sweep can apply the rules to it. + const followBranch = Effect.fn("StorageCleanup.followBranch")( + function* (worktreePath: string) { + const cwd = yield* fs.realPath(worktreePath); + yield* reconcileBranch({ cwd, refName: yield* gitWorkflow.currentBranch(cwd) }); + }, + (effect, worktreePath) => + effect.pipe( + Effect.catchCause((cause) => + Effect.logWarning("Failed to reconcile worktree branch", { cwd: worktreePath, cause }), + ), + ), + ); + // Git status and non-forced worktree removal both skip assume-unchanged and skip-worktree files. const hiddenTrackedChange = Effect.fn("StorageCleanup.hiddenTrackedChange")(function* ( cwd: string, @@ -731,6 +822,21 @@ const make = Effect.gen(function* () { return changed === undefined ? null : reason(changed.file); }); + // A worktree can outlive its project moving to another clone. Git removes a + // worktree through the repository that registered it, not through the + // project's current checkout, so resolve the owning repository first. + const worktreeRepository = Effect.fn("StorageCleanup.worktreeRepository")(function* ( + worktreePath: string, + ) { + const commonDir = yield* git.execute({ + operation: "StorageCleanup.worktreeRepository", + cwd: worktreePath, + args: ["rev-parse", "--git-common-dir"], + }); + const resolved = path.resolve(worktreePath, commonDir.stdout.trim()); + return path.basename(resolved) === ".git" ? path.dirname(resolved) : resolved; + }); + const localChangesReason = Effect.fn("StorageCleanup.localChangesReason")(function* ( cwd: string, rules: WorktreeCleanupRules, @@ -828,6 +934,8 @@ const make = Effect.gen(function* () { "rev-list", "--max-count=1", "HEAD", + // Checkpoint refs hold workspace snapshots, not commits a user could lose. + "--exclude=refs/t3/*", "--all", "--reflog", "--not", @@ -922,18 +1030,50 @@ const make = Effect.gen(function* () { let waiting = false; const kept: string[] = []; const roots: Array = []; - for (const directory of managedWorktreesDirectories( - serverSettings, - config.worktreesDir, - path, - yield* HostProcess.HomeDirectory, - )) { + // A git worktree's dev home is seeded from real data, so its threads and any + // copied settings can point at the developer's real worktrees. + const isWorktreeDevHome = + (yield* resolveWorktreeT3Home(config.baseDir)) === path.resolve(config.baseDir); + for (const directory of isWorktreeDevHome + ? [config.worktreesDir] + : managedWorktreesDirectories( + serverSettings, + config.worktreesDir, + path, + yield* HostProcess.HomeDirectory, + )) { const root = yield* fs.exists(directory).pipe( Effect.flatMap((exists) => (exists ? fs.realPath(directory) : Effect.succeed(null))), Effect.orElseSucceed(() => null), ); if (root !== null && !isFilesystemRoot(root, path)) roots.push(root); } + const scratchRoot = yield* fs + .realPath(scratchFolderRoot(config.baseDir, path)) + .pipe(Effect.orElseSucceed(() => null)); + // Nothing can bring a scratch folder's files back, so these hold whatever the retention says. + const scratchKeepReason = Effect.fn("StorageCleanup.scratchKeepReason")(function* ( + folder: string, + threadIds: ReadonlyArray, + rules: WorktreeCleanupRules, + ) { + const realParent = yield* fs.realPath(path.dirname(folder)); + if ((yield* fs.realPath(folder)) !== path.join(realParent, path.basename(folder))) + return "folder is a symbolic link"; + // Every file here is untracked, and only "tracked-changes" lets cleanup delete those. + if (rules.worktreeKeepWhen !== "tracked-changes") { + const file = (yield* fs.readDirectory(folder)).find((name) => name !== "node_modules"); + if (file !== undefined) return `has local files (${file})`; + } + // Each run of a scheduled task works on what its earlier runs left in the folder. + const scheduled = yield* sql<{ threadId: string }>` + SELECT thread_id AS "threadId" FROM scheduled_tasks + WHERE enabled = 1 AND thread_id IS NOT NULL + `.pipe(NodeSqliteClient.readOnly); + if (scheduled.some((task) => threadIds.includes(task.threadId))) + return "thread has a scheduled task"; + return null; + }); const worktreePathReason = Effect.fn("StorageCleanup.worktreePathReason")(function* ( worktreePath: string, ) { @@ -944,8 +1084,10 @@ const make = Effect.gen(function* () { return "worktree is a symbolic link"; if (!roots.some((root) => inside(root, realPath))) return "outside the managed worktree folder"; - if ((yield* fs.stat(path.join(worktreePath, ".git"))).type !== "File") - return "not a linked worktree"; + const linked = yield* isLinkedWorktree(worktreePath).pipe( + Effect.catchReason("PlatformError", "NotFound", () => Effect.succeed(false)), + ); + if (!linked) return "not a linked worktree"; return null; }); const deletedRows = yield* sql<{ payload_json: string; workspaceRoot: string }>` @@ -963,20 +1105,58 @@ const make = Effect.gen(function* () { const threadsById = new Map(snapshot.threads.map((thread) => [thread.id, thread])); const refreshedDefaultRefs = new Map>(); const defaultBranches = new Map(); - const groups = Map.groupBy( - snapshot.threads.filter((thread) => thread.worktreePath !== null), - (thread) => path.resolve(thread.worktreePath!), + // Threads may store one checkout through different symlinked spellings; compare real paths. + // Keys are only compared, never opened, so they take pathKey's Windows case folding. + const canonicalPaths = new Map(); + const canonical = Effect.fnUntraced(function* (target: string) { + const resolved = path.resolve(target); + const cached = canonicalPaths.get(resolved); + if (cached !== undefined) return cached; + const realPath = yield* fs.realPath(resolved).pipe(Effect.orElseSucceed(() => resolved)); + const key = pathKey(realPath); + canonicalPaths.set(resolved, key); + return key; + }); + // A cwd in a deleted folder cannot be resolved, so stored spellings are compared too. + const occupies = Effect.fnUntraced(function* ( + worktreePaths: ReadonlyArray, + target: string, + ) { + const targets = [path.resolve(target), yield* canonical(target)]; + return worktreePaths.some((worktree) => + targets.some((cwd) => pathKey(cwd) === pathKey(worktree) || inside(worktree, cwd)), + ); + }); + const hasTerminal = Effect.fnUntraced(function* (worktreePaths: ReadonlyArray) { + for (const terminal of [...liveTerminals.values()].flatMap((entries) => [ + ...entries.values(), + ])) { + if (terminal.status !== "starting" && terminal.status !== "running") continue; + if ( + (terminal.worktreePath !== null && + (yield* occupies(worktreePaths, terminal.worktreePath))) || + (yield* occupies(worktreePaths, terminal.cwd)) + ) + return true; + } + return false; + }); + const liveThreads = snapshot.threads.filter((thread) => thread.worktreePath !== null); + const liveKeys = yield* Effect.forEach(liveThreads, (thread) => + canonical(thread.worktreePath!), ); - const deletedByPath = Map.groupBy(deletedThreads, (thread) => - path.resolve(thread.worktreePath!), + const groups = Map.groupBy(liveThreads, (_thread, index) => liveKeys[index]!); + const deletedKeys = yield* Effect.forEach(deletedThreads, (thread) => + canonical(thread.worktreePath!), ); + const deletedByPath = Map.groupBy(deletedThreads, (_thread, index) => deletedKeys[index]!); const candidates = [ ...[...groups].map( - ([worktreePath, group]) => + ([worktreeKey, group]) => // A deleted thread still owns the worktree its remaining subagents share, // but only deletion cleanup acts on it; otherwise the live rules apply. deletedByPath - .get(worktreePath) + .get(worktreeKey) ?.find( (deleted) => resolveWorktreeCleanup(serverSettings, deleted.projectId).worktreeOnDelete && @@ -985,13 +1165,20 @@ const make = Effect.gen(function* () { worktreeOwner(group) ?? group[0]!, ), - ...deletedThreads.filter((thread) => !groups.has(path.resolve(thread.worktreePath!))), + ...deletedThreads.filter((_thread, index) => !groups.has(deletedKeys[index]!)), ]; for (const thread of candidates) { const settings = resolveWorktreeCleanup(serverSettings, thread.projectId); if (!worktreeCleanupEnabled(settings)) continue; const worktreePath = path.resolve(thread.worktreePath!); - const group = groups.get(worktreePath) ?? []; + const worktreeKey = yield* canonical(worktreePath); + const worktreePaths = [worktreePath, worktreeKey]; + const group = groups.get(worktreeKey) ?? []; + // No project threads have no branch; their folders sit directly in the scratch root. + const scratch = + thread.branch === null && + scratchRoot !== null && + pathKey(path.dirname(worktreeKey)) === pathKey(scratchRoot); const activityAt = Math.max(...group.map(storageCleanupActivityAt)); const deleted = "workspaceRoot" in thread; const settlementChain = deleted @@ -1038,14 +1225,12 @@ const make = Effect.gen(function* () { if (members.length > 1 && worktreeOwner(members) === undefined) return keep(`shared by ${members.length} threads`); if (project === undefined) return keep("project is unavailable"); - if ( - thread.branch === null || - !group.every((member) => storageCleanupThreadIdle(member, now, serverSettings)) - ) + if (!group.every((member) => storageCleanupThreadIdle(member, now, serverSettings))) return keep(waitForStop("thread is running or has pending work")); + if (thread.branch === null && !scratch) return keep("thread has no branch"); if (group.some((member) => member.branch !== thread.branch)) return keep("a subagent thread's branch differs from this worktree's branch"); - if (hasTerminal(worktreePath)) return keep(waitForStop("open terminal")); + if (yield* hasTerminal(worktreePaths)) return keep(waitForStop("open terminal")); if ( [...group, ...settlementChain.slice(1)].some( (member) => @@ -1068,6 +1253,9 @@ const make = Effect.gen(function* () { settledFor !== null && settings.worktreeSettledAfterDays !== null && settledFor >= settings.worktreeSettledAfterDays * DAY_MS; + // A worktree is unchanged before its first commit, so only finished threads qualify. + const unchangedApplies = + settings.worktreeUnchanged && (settledThread !== undefined || thread.archivedAt !== null); let eligible = deleted || old || settled; let removalReason = deleted ? "thread was deleted" @@ -1083,18 +1271,167 @@ const make = Effect.gen(function* () { : `settled for ${settledDays} of ${settings.worktreeSettledAfterDays} days` : settings.worktreeOnMerge ? "not merged" - : settings.worktreeUnchanged + : unchangedApplies ? "has commits beyond the default branch" - : "no rules apply"; + : settings.worktreeUnchanged + ? "not settled" + : "no rules apply"; + // The merge and unchanged rules read Git, which a scratch folder does not have. + if (scratch && !eligible) + return keep( + settings.worktreeAfterDays !== null || settings.worktreeSettledAfterDays !== null + ? notDueReason + : "no rules apply", + ); // Only Git-based rules can make a not-yet-due worktree eligible, so skip Git otherwise. - if (!eligible && !settings.worktreeUnchanged && !settings.worktreeOnMerge) - return keep(notDueReason); + if (!eligible && !unchangedApplies && !settings.worktreeOnMerge) return keep(notDueReason); + // Re-read this path's threads and sessions so new activity cancels removal; full per-worktree snapshots take seconds. + const blockedSinceCheck = Effect.fnUntraced(function* () { + const latestProjects = yield* projectStore.listShells(); + if (yield* containsProjectRoot(worktreePaths, [project, ...latestProjects])) + return "contains a project checkout"; + const bindings = yield* projections.getThreadWorktreePaths({ includeArchived: true }); + const bindingKeys = yield* Effect.forEach(bindings, (binding) => + canonical(binding.worktreePath), + ); + const latest = bindings.filter((_binding, index) => bindingKeys[index] === worktreeKey); + if (yield* hasTerminal(worktreePaths)) return waitForStop("open terminal"); + if (deleted) { + if ( + !resolveWorktreeCleanup(yield* settingsService.getSettings, thread.projectId) + .worktreeOnDelete + ) + return "thread or cleanup settings changed since check"; + // Durable deletion cleanup must succeed or be explicitly cancelled before removing its checkout. + const pendingCleanup = yield* sql<{ status: string }>` + SELECT status FROM orchestration_v2_effect_outbox + WHERE thread_id = ${thread.id} AND status NOT IN ('succeeded', 'cancelled') + `.pipe(NodeSqliteClient.readOnly); + // Failed effects never retry, so only running work can finish and free the path. + if (pendingCleanup.some((row) => row.status !== "failed")) + return waitForStop("thread deletion is still pending"); + if (pendingCleanup.length > 0) return "thread deletion is still pending"; + } + const latestShells = yield* Effect.forEach(latest, (entry) => + projections.getThreadShell(entry.threadId), + ); + const latestGroup = latestShells.filter((shell) => shell !== null); + const latestKeys = yield* Effect.forEach(latestGroup, (shell) => + shell.worktreePath === null ? Effect.succeed(null) : canonical(shell.worktreePath), + ); + const latestOwner = worktreeOwner(deleted ? [thread, ...latestGroup] : latestGroup); + if ( + latestGroup.length !== group.length || + latestOwner?.id !== thread.id || + latestOwner?.branch !== thread.branch || + latestGroup.some((shell, index) => { + const previous = group.find((member) => member.id === shell.id); + return ( + previous === undefined || + shell.branch !== thread.branch || + !Equal.equals(shell.settledAt, previous.settledAt) || + (shell.settledAt !== null && !Equal.equals(shell.updatedAt, previous.updatedAt)) || + !storageCleanupThreadIdle(shell, now, serverSettings) || + latestKeys[index] !== worktreeKey || + storageCleanupActivityAt(shell) !== storageCleanupActivityAt(previous) + ); + }) + ) + return waitForStop("thread activity or shared worktree changed since check"); + if (settlementChain.length > 1) { + const ancestors = yield* Effect.forEach(settlementChain.slice(1), (ancestor) => + projections.getThreadShell(ancestor.id), + ); + const latestSettlement = worktreeSettlementChain( + latestGroup.find((shell) => shell.id === thread.id)!, + new Map( + [...latestGroup, ...ancestors.filter((ancestor) => ancestor !== null)].map( + (ancestor) => [ancestor.id, ancestor], + ), + ), + now, + serverSettings, + ).at(-1); + if ( + latestSettlement?.id !== settledThread?.id || + !Equal.equals(latestSettlement?.settledAt, settledThread?.settledAt) || + !Equal.equals(latestSettlement?.updatedAt, settledThread?.updatedAt) + ) + return "parent settlement changed since check"; + } + // Sessions can outlive their run and can be shared across app threads. + const sessionRows = yield* sql<{ payload_json: string }>` + SELECT payload_json FROM orchestration_v2_projection_provider_sessions + WHERE status != 'stopped' + `.pipe(NodeSqliteClient.readOnly); + const sessions = yield* Effect.forEach(sessionRows, (row) => + decodeCleanupSession(row.payload_json), + ); + // Ready persisted sessions may have released their runtime; only live sessions protect the path. + for (const session of sessions) { + if ( + (yield* occupies(worktreePaths, session.cwd)) && + (yield* providerSessions.isLive(session.id)) + ) + return waitForStop("provider session is still open"); + } + return null; + }); + const settingsChangedSinceCheck = Effect.gen(function* () { + const currentSettings = yield* settingsService.getSettings; + const current = resolveWorktreeCleanup(currentSettings, thread.projectId); + return ( + currentSettings.autoResumeLimitedThreads !== serverSettings.autoResumeLimitedThreads || + currentSettings.snoozeLimitedThreads !== serverSettings.snoozeLimitedThreads || + Object.keys(settings).some( + (key) => + current[key as keyof typeof settings] !== settings[key as keyof typeof settings], + ) + ); + }); + if (scratch) { + const threadIds = group.map((member) => member.id); + const blocked = + (yield* scratchKeepReason(worktreePath, threadIds, settings)) ?? + (yield* blockedSinceCheck()); + if (blocked !== null) return keep(blocked); + const inspection = yield* measureWorktree(worktreePath, "plain-folder"); + if (inspection.reason !== null) return keep(inspection.reason); + return yield* withWorkspaceLease( + worktreePath, + Effect.gen(function* () { + // Walked again under the lease: a repository deleted with the folder is gone for good. + const finalInspection = yield* measureWorktree(worktreePath, "plain-folder"); + if (finalInspection.reason !== null) return keep(finalInspection.reason); + // The walk takes time, so the cheap checks run last, right before removal. + const blockedWhileMeasuring = + (yield* scratchKeepReason(worktreePath, threadIds, settings)) ?? + (yield* blockedSinceCheck()); + if (blockedWhileMeasuring !== null) return keep(blockedWhileMeasuring); + if (yield* settingsChangedSinceCheck) return keep("settings changed since check"); + yield* fs.remove(worktreePath, { recursive: true }); + entries.push({ + ...entry, + outcome: "removed", + reason: sentence(removalReason), + bytes: inspection.bytes, + }); + // The stored path stays so the next turn recreates the folder, empty. + yield* Effect.logInfo("storage cleanup removed scratch folder", { + threadId: thread.id, + }); + }), + ); + } const pathReason = yield* worktreePathReason(worktreePath); if (pathReason !== null) return keep(pathReason); - if (yield* containsProjectRoot(worktreePath, [project, ...snapshot.projects])) + if (yield* containsProjectRoot(worktreePaths, [project, ...snapshot.projects])) return keep("contains a project checkout"); - if (yield* branchChanged(worktreePath, thread.branch)) + if (yield* branchChanged(worktreePath, thread.branch)) { + // A deleted owner cannot follow, so its remaining subagents stay on its branch. + if (!deleted) yield* followBranch(worktreePath); return keep("repository or branch changed"); + } const changes = yield* localChangesReason(worktreePath, settings); if (changes !== null) return keep(changes); const head = yield* git.resolveCommit({ cwd: worktreePath, revision: "HEAD" }); @@ -1112,8 +1449,12 @@ const make = Effect.gen(function* () { let branch: string | null | undefined; let defaultRef: string; if (remote === null) { - if (!settings.worktreeUnchanged) - return keep("no remote is configured for merged-worktree cleanup"); + if (!unchangedApplies) + return keep( + settings.worktreeUnchanged + ? notDueReason + : "no remote is configured for merged-worktree cleanup", + ); const projectHead = yield* git.execute({ operation: "StorageCleanup.resolveProjectBranch", cwd: repositoryCwd, @@ -1172,7 +1513,7 @@ const make = Effect.gen(function* () { allowNonZeroExit: true, }); const integrated = ancestor.exitCode === 0; - eligible = integrated && settings.worktreeUnchanged; + eligible = integrated && unchangedApplies; if (eligible) removalReason = remote === null @@ -1191,101 +1532,16 @@ const make = Effect.gen(function* () { }); if (eligible) removalReason = "pull request was merged"; } - if (!eligible && remote === null) return keep("has commits beyond the project branch"); + if (!eligible && remote === null) + return keep(integrated ? notDueReason : "has commits beyond the project branch"); } if (!eligible) return keep(notDueReason); - // Re-read this path's threads and sessions so new activity cancels removal; full per-worktree snapshots take seconds. - const blockedSinceCheck = Effect.fnUntraced(function* () { - const latestProjects = yield* projectStore.listShells(); - if (yield* containsProjectRoot(worktreePath, [project, ...latestProjects])) - return "contains a project checkout"; - const latest = (yield* projections.getThreadWorktreePaths({ - includeArchived: true, - })).filter((entry) => path.resolve(entry.worktreePath) === worktreePath); - if (hasTerminal(worktreePath)) return waitForStop("open terminal"); - if (deleted) { - if ( - !resolveWorktreeCleanup(yield* settingsService.getSettings, thread.projectId) - .worktreeOnDelete - ) - return "thread or cleanup settings changed since check"; - // Durable deletion cleanup must succeed or be explicitly cancelled before removing its checkout. - const pendingCleanup = yield* sql<{ status: string }>` - SELECT status FROM orchestration_v2_effect_outbox - WHERE thread_id = ${thread.id} AND status NOT IN ('succeeded', 'cancelled') - `.pipe(NodeSqliteClient.readOnly); - // Failed effects never retry, so only running work can finish and free the path. - if (pendingCleanup.some((row) => row.status !== "failed")) - return waitForStop("thread deletion is still pending"); - if (pendingCleanup.length > 0) return "thread deletion is still pending"; - } - const latestShells = yield* Effect.forEach(latest, (entry) => - projections.getThreadShell(entry.threadId), - ); - const latestGroup = latestShells.filter((shell) => shell !== null); - const latestOwner = worktreeOwner(deleted ? [thread, ...latestGroup] : latestGroup); - if ( - latestGroup.length !== group.length || - latestOwner?.id !== thread.id || - latestOwner?.branch !== thread.branch || - latestGroup.some((shell) => { - const previous = group.find((member) => member.id === shell.id); - return ( - previous === undefined || - shell.branch !== thread.branch || - !Equal.equals(shell.settledAt, previous.settledAt) || - (shell.settledAt !== null && !Equal.equals(shell.updatedAt, previous.updatedAt)) || - !storageCleanupThreadIdle(shell, now, serverSettings) || - path.resolve(shell.worktreePath!) !== worktreePath || - storageCleanupActivityAt(shell) !== storageCleanupActivityAt(previous) - ); - }) - ) - return waitForStop("thread activity or shared worktree changed since check"); - if (settlementChain.length > 1) { - const ancestors = yield* Effect.forEach(settlementChain.slice(1), (ancestor) => - projections.getThreadShell(ancestor.id), - ); - const latestSettlement = worktreeSettlementChain( - latestGroup.find((shell) => shell.id === thread.id)!, - new Map( - [...latestGroup, ...ancestors.filter((ancestor) => ancestor !== null)].map( - (ancestor) => [ancestor.id, ancestor], - ), - ), - now, - serverSettings, - ).at(-1); - if ( - latestSettlement?.id !== settledThread?.id || - !Equal.equals(latestSettlement?.settledAt, settledThread?.settledAt) || - !Equal.equals(latestSettlement?.updatedAt, settledThread?.updatedAt) - ) - return "parent settlement changed since check"; - } - // Sessions can outlive their run and can be shared across app threads. - const sessionRows = yield* sql<{ payload_json: string }>` - SELECT payload_json FROM orchestration_v2_projection_provider_sessions - WHERE status != 'stopped' - `.pipe(NodeSqliteClient.readOnly); - const sessions = yield* Effect.forEach(sessionRows, (row) => - decodeCleanupSession(row.payload_json), - ); - // Ready persisted sessions may have released their runtime; only live sessions protect the path. - for (const session of sessions) { - const cwd = path.resolve(session.cwd); - if ( - (pathKey(cwd) === pathKey(worktreePath) || inside(worktreePath, cwd)) && - (yield* providerSessions.isLive(session.id)) - ) - return waitForStop("provider session is still open"); - } - return null; - }); const blocked = yield* blockedSinceCheck(); if (blocked !== null) return keep(blocked); - // Size is an estimate; provider startup must not wait for directory traversal. - const bytes = yield* measureWorktree(worktreePath); + // Inspect before the lease; provider startup must not wait for directory traversal. + const inspection = yield* measureWorktree(worktreePath, true); + if (inspection.reason !== null) return keep(inspection.reason); + const bytes = inspection.bytes; yield* withWorkspaceLease( worktreePath, Effect.gen(function* () { @@ -1302,35 +1558,25 @@ const make = Effect.gen(function* () { return keep("commit changed since check"); const finalChanges = yield* localChangesReason(worktreePath, settings); if (finalChanges !== null) return keep(finalChanges); - const currentSettings = yield* settingsService.getSettings; - const current = resolveWorktreeCleanup(currentSettings, thread.projectId); - if ( - currentSettings.autoResumeLimitedThreads !== - serverSettings.autoResumeLimitedThreads || - currentSettings.snoozeLimitedThreads !== serverSettings.snoozeLimitedThreads || - Object.keys(settings).some( - (key) => - current[key as keyof typeof settings] !== settings[key as keyof typeof settings], - ) - ) - return keep("settings changed since check"); - // Clean only untracked files. Ordinary worktrees retain Git's final tracked-edit check. - if (settings.worktreeKeepWhen === "tracked-changes") - yield* cleanupGit(worktreePath, ["clean", "-ffdx"], "git clean"); + if (yield* settingsChangedSinceCheck) return keep("settings changed since check"); const submodules = yield* submoduleRemoval(worktreePath, settings); if (submodules.reason !== null) return keep(submodules.reason); if (submodules.force) { const changes = yield* localChangesReason(worktreePath, settings); if (changes !== null) return keep(changes); } + const repository = yield* worktreeRepository(worktreePath); + // Forced removal already discards local files; do not widen its final-check window. + if (settings.worktreeKeepWhen === "tracked-changes" && !submodules.force) + yield* cleanupGit(worktreePath, ["clean", "-ffdx"], "git clean"); yield* removeLinkedWorktree({ operation: "StorageCleanup.cleanupGit", - cwd: project.workspaceRoot, + cwd: repository, worktreePath, // Git requires force for submodules; recursive checks above cannot make removal atomic. force: submodules.force, execute: git.execute, - run: (args) => cleanupGit(project.workspaceRoot, [...args], "git worktree remove"), + run: (args) => cleanupGit(repository, [...args], "git worktree remove"), }); entries.push({ ...entry, outcome: "removed", reason: sentence(removalReason), bytes }); yield* gitManager.invalidateStatus(project.workspaceRoot); @@ -1359,6 +1605,7 @@ const make = Effect.gen(function* () { roots, yield* projectStore.list({ includeDeleted: true }), entries, + scratchRoot, ); return { waiting, kept }; }); @@ -1402,6 +1649,115 @@ const make = Effect.gen(function* () { }); const reportRef = yield* SubscriptionRef.make(null); + const cleanCheckpointRefs = Effect.fn("StorageCleanup.cleanCheckpointRefs")(function* () { + // Forks display their source history even after its thread is deleted. Keep + // that ancestry, including deleted intermediate forks and archived children. + const rows = yield* sql<{ + threadId: string; + worktreePath: string | null; + workspaceRoot: string | null; + scopeId: string | null; + cwd: string | null; + }>` + WITH RECURSIVE retained_checkpoint_threads(thread_id) AS ( + SELECT thread_id FROM orchestration_v2_projection_threads WHERE deleted_at IS NULL + UNION + SELECT json_extract(t.payload_json, '$.forkedFrom.threadId') + FROM orchestration_v2_projection_threads t + JOIN retained_checkpoint_threads retained ON retained.thread_id = t.thread_id + WHERE json_extract(t.payload_json, '$.forkedFrom.type') = 'run' + ) + SELECT t.thread_id AS "threadId", + json_extract(t.payload_json, '$.worktreePath') AS "worktreePath", + p.workspace_root AS "workspaceRoot", s.scope_id AS "scopeId", + json_extract(s.payload_json, '$.cwd') AS cwd + FROM orchestration_v2_projection_threads t + LEFT JOIN projection_projects p ON p.project_id = t.project_id + LEFT JOIN orchestration_v2_projection_checkpoint_scopes s ON s.thread_id = t.thread_id + WHERE t.deleted_at IS NOT NULL + AND NOT EXISTS (SELECT 1 FROM retained_checkpoint_threads WHERE thread_id = t.thread_id) + AND NOT EXISTS ( + SELECT 1 FROM orchestration_v2_effect_outbox e + WHERE e.thread_id = t.thread_id AND e.status IN ('pending', 'running') + ) + `.pipe(NodeSqliteClient.readOnly); + const targets = new Map>(); + for (const row of rows) { + // Derive ownership from tombstones, never from absence in this database: + // another T3 home can own refs in the same Git repository. + const prefixes = [`refs/t3/checkpoints/${Base64Url.encode(row.threadId)}/turn/`]; + if (row.scopeId !== null) { + prefixes.push( + (yield* checkpointRefForScopeOrdinal({ + scopeId: CheckpointScopeId.make(row.scopeId), + ordinalWithinScope: 0, + })).slice(0, -1), + ); + } + // A worktree may already be gone; its refs live in the shared repository. + for (const cwd of [row.cwd, row.worktreePath, row.workspaceRoot]) { + if (cwd === null) continue; + const key = path.resolve(cwd); + const owned = targets.get(key) ?? new Set(); + for (const prefix of prefixes) owned.add(prefix); + targets.set(key, owned); + } + } + const repositories = new Map }>(); + for (const [cwd, prefixes] of targets) { + yield* Effect.gen(function* () { + if (!(yield* fs.exists(cwd))) return; + const result = yield* git.execute({ + operation: "StorageCleanup.checkpointRepository", + cwd, + args: ["rev-parse", "--path-format=absolute", "--git-common-dir"], + allowNonZeroExit: true, + }); + if (result.exitCode !== 0) return; + const commonDir = yield* fs.realPath(result.stdout.trim()); + const repository = repositories.get(commonDir) ?? { cwd, prefixes: new Set() }; + for (const prefix of prefixes) repository.prefixes.add(prefix); + repositories.set(commonDir, repository); + }).pipe( + Effect.catch((cause) => + Effect.logWarning("checkpoint repository inspection failed", { cause }), + ), + ); + } + for (const { cwd, prefixes } of repositories.values()) { + yield* Effect.gen(function* () { + const result = yield* git.execute({ + operation: "StorageCleanup.listCheckpointRefs", + cwd, + args: [ + "for-each-ref", + "--format=%(refname)", + "refs/t3/checkpoints/", + "refs/t3/orchestration-v2/checkpoints/", + ], + maxOutputBytes: 16 * 1024 * 1024, + }); + const refs = result.stdout.split("\n").filter((ref) => { + const ordinalStart = ref.lastIndexOf("/") + 1; + return ( + prefixes.has(ref.slice(0, ordinalStart)) && + /^(0|[1-9]\d*)$/.test(ref.slice(ordinalStart)) + ); + }); + if (refs.length === 0) return; + yield* git.execute({ + operation: "StorageCleanup.deleteCheckpointRefs", + cwd, + // Remove a symbolic checkpoint itself, never its branch/tag target. + args: ["update-ref", "--no-deref", "-z", "--stdin"], + stdin: refs.map((ref) => `delete ${ref}\0\0`).join(""), + }); + yield* Effect.logInfo("storage cleanup removed checkpoint refs", { refCount: refs.length }); + }).pipe( + Effect.catch((cause) => Effect.logWarning("checkpoint ref cleanup failed", { cause })), + ); + } + }); // Only deleted threads' worktrees wait on session stops: any stop while one waits, or a stop inside one kept for another reason. // A deletion stays pending until a sweep that started after it finishes. let deletedWorktreeWaits = true; @@ -1416,6 +1772,9 @@ const make = Effect.gen(function* () { const serverSettings = yield* settingsService.getSettings; const settings = serverSettings.storageCleanup; const entries: StorageCleanupReportEntry[] = []; + yield* cleanCheckpointRefs().pipe( + Effect.catch((cause) => Effect.logWarning("checkpoint ref cleanup failed", { cause })), + ); const deletedWorktrees = yield* cleanWorktrees(serverSettings, now, entries).pipe( Effect.catch((error) => Effect.gen(function* () { @@ -1594,6 +1953,7 @@ const make = Effect.gen(function* () { ) return Effect.void; } + if (event.type === "thread.deleted") return requestSweep; if (!anyWorktreePolicy(lastSettings, (rules) => rules.worktreeOnDelete)) return Effect.void; return event.type === "provider-session.updated" && event.payload.status === "error" ? providerSessions diff --git a/apps/server/src/telemetry/ProviderDimensions.test.ts b/apps/server/src/telemetry/ProviderDimensions.test.ts index d88010f600e3..00e59c11228c 100644 --- a/apps/server/src/telemetry/ProviderDimensions.test.ts +++ b/apps/server/src/telemetry/ProviderDimensions.test.ts @@ -76,3 +76,16 @@ it("drops input strings outside the schema's literals, since rejected calls are handoffSettings("t3_thread_launch", { workspaceStrategy: { type: "/home/me/secret" } }, {}), ).toEqual({ targetChosen: false, workspace: "root" }); }); + +it("records the binding the saved scheduled task resolved to, and none for failures", () => { + expect(handoffSettings("schedule_task", {}, { boundThreadId: null })).toEqual({ + bindToCurrentThread: false, + }); + expect( + handoffSettings( + "schedule_task", + { bindToCurrentThread: true }, + { _tag: "OrchestratorMcpFailure", code: "invalid_request", message: "Different project." }, + ), + ).toEqual({}); +}); diff --git a/apps/server/src/telemetry/ProviderDimensions.ts b/apps/server/src/telemetry/ProviderDimensions.ts index f81a22e844ae..8352ed0d15cc 100644 --- a/apps/server/src/telemetry/ProviderDimensions.ts +++ b/apps/server/src/telemetry/ProviderDimensions.ts @@ -128,9 +128,9 @@ function callerOrigin(input: { } /** - * Settings the agent chose for a handoff, read only from closed enums and - * booleans in the tool's arguments and result. Prompts, titles, and ids are - * never read. + * Settings for a handoff, recorded only as closed enums and booleans derived + * from the tool's arguments and result. Prompts, titles, and ids are never + * recorded. */ export function handoffSettings(tool: string, args: unknown, result: unknown): Fields { const input = asFields(args); @@ -166,8 +166,13 @@ export function handoffSettings(tool: string, args: unknown, result: unknown): F return defined({ delivery: stringField(output, "delivery") }); case "t3_thread_send_attachments": return { delivery: "auto" }; - case "schedule_task": - return { bindToCurrentThread: input.bindToCurrentThread !== false }; + case "schedule_task": { + // Omitting the input lets the server decide, so read the saved task. + const bound = output.boundThreadId; + return bound === null || typeof bound === "string" + ? { bindToCurrentThread: bound !== null } + : {}; + } default: return {}; } diff --git a/apps/server/src/terminal/Manager.test.ts b/apps/server/src/terminal/Manager.test.ts index 5c65b9d973ba..e7c273569a44 100644 --- a/apps/server/src/terminal/Manager.test.ts +++ b/apps/server/src/terminal/Manager.test.ts @@ -1,14 +1,20 @@ +import { deriveProviderInstanceConfigMap } from "../provider/ProviderInstanceRegistryHydration.ts"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; import { DEFAULT_TERMINAL_ID, + ProjectId, + ThreadId, + TerminalCwdNotFoundError, type TerminalAttachStreamEvent, type TerminalEvent, type TerminalOpenInput, ProviderDriverKind, ProviderInstanceId, - ProjectId, + retainedTerminalHistory, + terminalSequenceEnd, + terminalSequenceStateAfter, TerminalProviderInstanceNotFoundError, } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; @@ -26,6 +32,7 @@ import * as PlatformError from "effect/PlatformError"; import * as Path from "effect/Path"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import * as Ref from "effect/Ref"; +import * as Queue from "effect/Queue"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; @@ -33,12 +40,16 @@ import * as SynchronizedRef from "effect/SynchronizedRef"; import * as TestClock from "effect/testing/TestClock"; import { expect, vi } from "vite-plus/test"; +import * as ServerConfig from "../config.ts"; import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; import * as ProcessRunner from "../processRunner.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as ProjectService from "../project/ProjectService.ts"; import * as ProjectSetupScriptRunner from "../project/ProjectSetupScriptRunner.ts"; import * as TerminalManager from "./Manager.ts"; +import * as GitWorkflowService from "../git/GitWorkflowService.ts"; +import { makePrepareTerminalLaunch } from "../workspace/restoreMissingWorktree.ts"; +import { withWorkspaceLease } from "../workspace/workspaceLease.ts"; import { terminalOutputStream } from "./OutputProtocol.ts"; import * as PtyAdapter from "@t3tools/shared/PtyAdapter"; import packageJson from "../../package.json" with { type: "json" }; @@ -328,7 +339,6 @@ function retainedHistory(text: string, maxLines: number, maxBytes = Infinity): s if (terminated) lines.pop(); const retained = lines.slice(Math.max(0, lines.length - maxLines)).join("\n"); const capped = terminated ? `${retained}\n` : retained; - if (Buffer.byteLength(capped) <= maxBytes) return capped; const points = Array.from(capped); let start = points.length; let bytes = 0; @@ -338,7 +348,11 @@ function retainedHistory(text: string, maxLines: number, maxBytes = Infinity): s bytes += next; start -= 1; } - return points.slice(start).join(""); + const kept = points.slice(start).join(""); + if (maxLines <= 0 || maxBytes <= 0) return kept; + // A cut inside a control sequence drops the rest of that sequence too. + const state = terminalSequenceStateAfter(0, text, 0, text.length - kept.length); + return kept.slice(terminalSequenceEnd(state, kept)); } it("preserves line and byte limits across arbitrary chunks, Unicode, ANSI sequences, and clear", () => { @@ -356,6 +370,10 @@ it("preserves line and byte limits across arbitrary chunks, Unicode, ANSI sequen "\u001b[31m", "\u001b[0m", "\u001b]8;;url\u0007", + "\u001b]0;", + "\u001bP", + "\u0007", + "\u001b\\", "\ud83d", "\ude80", ]; @@ -366,7 +384,8 @@ it("preserves line and byte limits across arbitrary chunks, Unicode, ANSI sequen for (const maxBytes of [0, 3, 8, 64, Infinity]) { for (const maxLines of [0, 1, 3, 5, 5_000]) { - let expected = retainedHistory("before\ninitial\n", maxLines, maxBytes); + let full = "before\ninitial\n"; + let expected = retainedHistory(full, maxLines, maxBytes); const history = new TerminalManager.BoundedTerminalHistory( maxLines, "before\ninitial\n", @@ -377,18 +396,33 @@ it("preserves line and byte limits across arbitrary chunks, Unicode, ANSI sequen for (let step = 0; step < 300; step += 1) { if (step % 73 === 0) { history.clear(); - expected = ""; - expect(history.value()).toBe(expected); + full = ""; + expect(history.value()).toBe(""); } const chunk = nextFragment() + nextFragment(); history.append(chunk); - expected = retainedHistory(expected + chunk, maxLines, maxBytes); + full += chunk; + expected = retainedHistory(full, maxLines, maxBytes); expect(history.value()).toBe(expected); } } } }); +it("does not retain the tail of a control sequence whose introducer was trimmed", () => { + for (const [text, maxBytes, retained] of [ + ["prefix\u001b[31mOK", 6, "OK"], + ["prefix\u001b]0;HIDDEN\u0007OK", 10, "OK"], + ["prefix\u001b]0;🙂HIDDEN\u0007OK", 9, "OK"], + // libghostty leaves every string but OSC at the emoji's second byte, so HELLO was visible. + ["prefix\u001bP🙂HELLO\u001b\\OK", 9, "HELLO\u001b\\OK"], + ] as const) { + expect(new TerminalManager.BoundedTerminalHistory(5_000, text, maxBytes).value()).toBe( + retained, + ); + } +}); + it("bounds long partial lines and joins surrogate pairs across chunk boundaries", () => { const maxBytes = 65_539; let expected = ""; @@ -407,6 +441,41 @@ it("bounds long partial lines and joins surrogate pairs across chunk boundaries" } }); +it("restores bounded history across append compaction, clears and split Unicode", () => { + const maxBytes = 64; + const history = new TerminalManager.BoundedTerminalHistory(3, "", maxBytes); + let cursor: ReturnType["cursor"] | undefined; + let persisted = ""; + let compactions = 0; + const flush = () => { + const update = history.persistenceUpdate(cursor); + if (!update.append) compactions += 1; + const encoded = Buffer.from(update.data).toString(); + persisted = update.append ? persisted + encoded : encoded; + cursor = update.cursor; + expect(Buffer.byteLength(persisted)).toBeLessThanOrEqual(maxBytes * 2); + expect(retainedHistory(persisted, 3, maxBytes)).toBe(Buffer.from(history.value()).toString()); + }; + for (let index = 0; index < 100; index += 1) { + history.append(`line ${index} 名\n`); + flush(); + } + expect(compactions).toBeGreaterThan(1); + expect(compactions).toBeLessThan(50); + history.clear(); + flush(); + expect(persisted).toBe(""); + history.append("\ud83d"); + flush(); + history.append("\ude80"); + flush(); + expect(persisted).toBe("🚀"); + // More output than the retained window between flushes must replace the file. + history.append("z".repeat(maxBytes * 3)); + flush(); + expect(persisted).toBe("z".repeat(maxBytes)); +}); + it.layer( Layer.mergeAll( NodeServices.layer, @@ -415,6 +484,91 @@ it.layer( ), { excludeTestServices: true }, )("TerminalManager", (it) => { + it.effect.each([ + { method: "open", location: "managed" }, + { method: "attach", location: "managed" }, + { method: "open", location: "subdirectory" }, + { method: "open", location: "unmanaged" }, + { method: "open", location: "no-worktree" }, + ] as const)( + "recovers $method with a $location cwd without a cleanup gap", + ({ method, location }) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const { manager, ptyAdapter, baseDir } = yield* createManager(); + const worktreesDir = path.join(baseDir, "worktrees"); + const worktreePath = + location === "unmanaged" + ? path.join(baseDir, "missing") + : path.join(worktreesDir, "project", "feature"); + const cwd = location === "subdirectory" ? path.join(worktreePath, "src") : worktreePath; + const cleanupQueued = yield* Deferred.make>(); + let creations = 0; + const prepareLaunch = yield* makePrepareTerminalLaunch(() => + Effect.succeed({ + id: ThreadId.make("thread-1"), + projectId: ProjectId.make("project-1"), + worktreePath: location === "no-worktree" ? null : worktreePath, + branch: "feature", + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + Layer.mock(ServerConfig.ServerConfig)({ + worktreesDir, + } as ServerConfig.ServerConfig["Service"]), + ServerSettings.layerTest(), + Layer.mock(ProjectService.ProjectService)({ + getById: () => Effect.succeedSome({ workspaceRoot: baseDir } as never), + }), + Layer.mock(GitWorkflowService.GitWorkflowService)({ + pruneWorktrees: () => Effect.void, + listLocalBranchNames: () => Effect.succeed(["feature"]), + createWorktree: () => + Effect.gen(function* () { + creations++; + yield* fs.makeDirectory(cwd, { recursive: true }).pipe(Effect.orDie); + const cleanup = yield* withWorkspaceLease( + worktreePath, + Effect.sync(() => { + assert.equal(ptyAdapter.spawnInputs.length, 1); + }), + ).pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.succeed(cleanupQueued, cleanup); + return {} as never; + }), + }), + ), + ), + ); + const input = openInput({ + cwd, + worktreePath: location === "subdirectory" ? null : worktreePath, + }); + const launch = prepareLaunch(input).pipe( + Effect.flatMap(({ input, recover }) => + (method === "attach" + ? manager.attachStream(input, () => Effect.void).pipe(Effect.asVoid) + : manager.open(input).pipe(Effect.asVoid) + ).pipe(Effect.provideService(TerminalManager.TerminalCwdRecovery, recover)), + ), + ); + if (location === "unmanaged" || location === "no-worktree") { + const error = yield* launch.pipe(Effect.flip); + assert.deepEqual(error, new TerminalCwdNotFoundError({ cwd })); + assert.equal(creations, 0); + assert.equal(ptyAdapter.spawnInputs.length, 0); + return; + } + yield* launch; + yield* Fiber.join(yield* Deferred.await(cleanupQueued)); + assert.equal(creations, 1); + assert.equal(ptyAdapter.spawnInputs.length, 1); + assert.isTrue(yield* fs.exists(cwd)); + }), + ); + it.effect("spawns lazily and reuses running terminal per thread", () => Effect.gen(function* () { const { manager, ptyAdapter } = yield* createManager(); @@ -1043,6 +1197,168 @@ it.layer( }), ); + it.effect("clears a terminal that keeps printing while its history is saved", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + let printWhileSaving = Effect.void; + const printing = FileSystem.FileSystem.of({ + ...fs, + writeFileString: (filePath, data, options) => + printWhileSaving.pipe(Effect.andThen(fs.writeFileString(filePath, data, options))), + }); + const { manager, ptyAdapter, getEvents } = yield* createManager().pipe( + Effect.provideService(FileSystem.FileSystem, printing), + ); + yield* manager.open(openInput()); + const process = ptyAdapter.processes[0]!; + // Each save sees fresh output, so the history worker never goes idle. + printWhileSaving = Effect.gen(function* () { + const printed = yield* Deferred.make(); + const unsubscribe = yield* manager.subscribe((event) => + event.type === "output" ? Deferred.succeed(printed, undefined) : Effect.void, + ); + process.emitData("tick\n"); + yield* Deferred.await(printed); + unsubscribe(); + }); + + yield* manager.clear({ threadId: "thread-1", terminalId: DEFAULT_TERMINAL_ID }); + + expect((yield* getEvents).map((event) => event.type)).toContain("cleared"); + }), + ); + + it.effect("restores a log tail that starts inside a control sequence without its remainder", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + for (const [persisted, restored] of [ + ["\u001b]0;x\u0007ABCD", "ABCD"], + ["名\u001b]0;名\u0007ABCD", "ABCD"], + ["\u001b]0;x\u0007\u001b[31mABCD", "ABCD"], + ] as const) { + const { manager, logsDir } = yield* createManager(5, { historyByteLimit: 8 }); + const path = yield* historyLogPath(logsDir); + yield* fs.makeDirectory(logsDir, { recursive: true }); + yield* fs.writeFileString(path, persisted); + expect((yield* manager.open(openInput())).history).toBe(restored); + expect(yield* fs.readFileString(path)).toBe(restored); + } + }), + ); + + it.effect("persists small terminal updates without rewriting the retained history", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const writes = yield* Queue.unbounded(); + const maxBytes = 64 * 1024; + const { manager, ptyAdapter, logsDir } = yield* createManager(5_000, { + historyByteLimit: maxBytes, + }).pipe( + Effect.provideService(FileSystem.FileSystem, { + ...fs, + writeFileString: (path, data, options) => + fs + .writeFileString(path, data, options) + .pipe( + Effect.tap(() => + data.length > 0 ? Queue.offer(writes, Buffer.byteLength(data)) : Effect.void, + ), + ), + }), + ); + yield* manager.open(openInput()); + const process = ptyAdapter.processes[0]!; + process.emitData("a".repeat(maxBytes)); + let written = yield* Queue.take(writes); + for (let index = 0; index < 16; index += 1) { + process.emitData("b".repeat(1024)); + written += yield* Queue.take(writes); + } + expect(written).toBeLessThan(maxBytes * 2); + const expected = "a".repeat(maxBytes - 16 * 1024) + "b".repeat(16 * 1024); + const persisted = yield* fs.readFileString(yield* historyLogPath(logsDir)); + expect(Buffer.byteLength(persisted)).toBe(maxBytes + 16 * 1024); + for (const failure of [undefined, "write", "rename"] as const) { + let failed = false; + const fail = () => { + failed = true; + return Effect.fail( + PlatformError.systemError({ + _tag: "Unknown", + module: "FileSystem", + method: failure === "write" ? "writeFileString" : "rename", + cause: { code: "ENOSPC" }, + }), + ); + }; + const recovered = yield* createManager(5_000, { historyByteLimit: maxBytes }).pipe( + Effect.provideService(FileSystem.FileSystem, { + ...fs, + writeFileString: (file, data, options) => + failure === "write" && !failed && data === expected + ? fs.writeFileString(file, "partial", options).pipe(Effect.andThen(fail)) + : fs.writeFileString(file, data, options), + rename: (from, to) => (failure === "rename" && !failed ? fail() : fs.rename(from, to)), + }), + ); + const recoveredPath = yield* historyLogPath(recovered.logsDir); + yield* fs.makeDirectory(recovered.logsDir, { recursive: true }); + yield* fs.writeFileString(recoveredPath, persisted); + const snapshot = yield* recovered.manager.open(openInput()); + expect(snapshot.status).toBe("running"); + expect(snapshot.history).toBe(expected); + expect(failed).toBe(failure !== undefined); + expect(yield* fs.readFileString(recoveredPath)).toBe( + failure === undefined ? expected : persisted, + ); + expect(yield* fs.readDirectory(recovered.logsDir)).toHaveLength(1); + } + yield* manager.close({ threadId: "thread-1" }); + expect((yield* manager.open(openInput())).history).toBe(expected); + expect(yield* fs.readFileString(yield* historyLogPath(logsDir))).toBe(expected); + }), + ); + + it.effect("rewrites history after an append fails partway through", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const writes = yield* Queue.unbounded(); + let failAppend = false; + const { manager, ptyAdapter, logsDir } = yield* createManager().pipe( + Effect.provideService(FileSystem.FileSystem, { + ...fs, + writeFileString: (path, data, options) => { + if (options?.flag === "a" && failAppend) { + failAppend = false; + return fs.writeFileString(path, data.slice(0, 1), options).pipe( + // The file cannot also be a parent directory: fail after the partial append. + Effect.andThen(fs.writeFileString(`${path}/missing`, data)), + Effect.ensuring(Queue.offer(writes, undefined)), + ); + } + return fs + .writeFileString(path, data, options) + .pipe( + Effect.tap(() => (data.length > 0 ? Queue.offer(writes, undefined) : Effect.void)), + ); + }, + }), + ); + yield* manager.open(openInput()); + const process = ptyAdapter.processes[0]!; + process.emitData("start:"); + yield* Queue.take(writes); + failAppend = true; + process.emitData("middle"); + yield* Queue.take(writes); + expect(failAppend).toBe(false); + process.emitData(":end"); + yield* Queue.take(writes); + expect(yield* fs.readFileString(yield* historyLogPath(logsDir))).toBe("start:middle:end"); + yield* manager.close({ threadId: "thread-1" }); + }), + ); + it.effect("releases delivered output while the process queue stays busy", () => Effect.gen(function* () { // Observe retained payloads, since delivery and bounded history alone cannot detect a leak. @@ -1157,8 +1473,11 @@ it.layer( process.emitData("prompt "); process.emitData("\u001b[?2026$p\u001b[?2026;2$y\u001b[>q\u001b[?u\u001b[?31u"); + process.emitData("\u001b[=c\u001b[=0c\u001bZ"); process.emitData("\u001bP$q m\u001b\\\u001bP1$r0m\u001b\\"); process.emitData("\u001bP+q544e\u001b\\\u001bP1+r544e=1b\u001b\\"); + // BEL is DCS payload, not a terminator: only ST ends the string. + process.emitData("\u001bP$qm\u0007hidden\u001b\\\u001bP+q544e\u0007hidden\u001b\\"); process.emitData('\u001b[!p\u001b["p\u001b[4 q\u001b[u'); process.emitData("\u001b]0;x\u001b\u001b[?2026$p"); process.emitData("done\n"); @@ -1174,6 +1493,50 @@ it.layer( }), ); + // Ghostty executes 0x00-0x0F inside a CSI and ignores DEL and 0x10-0x1F; none change the query. + it.effect.each([ + { name: "NUL in DA", data: "A\u001b[0\u0000cB", history: "AB" }, + { name: "DEL in DA", data: "A\u001b[0\u007fcB", history: "AB" }, + { name: "NUL in DECRQM", data: "A\u001b[?2026\u0000$pB", history: "AB" }, + { name: "CR in DSR", data: "AB\u001b[5\rnC", history: "AB\rC" }, + { name: "DLE in DSR", data: "AB\u001b[5\u0010nC", history: "ABC" }, + // Ghostty reads non-ASCII there as UTF-8 bytes: it drops é and runs 0x80-0x9F as C1 controls. + { name: "é in DA", data: "A\u001b[0écB", history: "AB" }, + { name: "DECID byte in a CSI", data: "A\u001b[嚠B", history: "A\ufffdB" }, + { + name: "C1 DCS then C1 CSI in DSR", + data: "A\u001b[\u0090\u009b6nB", + history: "A\u001bP\u009cB", + }, + { name: "ESC in DSR", data: "A\u001b[5\u001b[6nB", history: "AB" }, + ])( + "strips a query with an embedded control but keeps what the terminal ran ($name)", + ({ data, history }) => + Effect.gen(function* () { + const { manager, ptyAdapter } = yield* createManager(); + yield* manager.open(openInput()); + ptyAdapter.processes[0]!.emitData(data); + yield* manager.close({ threadId: "thread-1" }); + + const reopened = yield* manager.open(openInput()); + const restored = reopened.history.replaceAll("\u0000", ""); + expect(restored.split("\u001b[?1049h")[0]).toBe(history); + }), + ); + + // Ghostty aborts a CSI at a non-ASCII code point and prints it and what follows. + it.effect("keeps a CSI interrupted by non-ASCII text in history", () => + Effect.gen(function* () { + const { manager, ptyAdapter } = yield* createManager(); + yield* manager.open(openInput()); + ptyAdapter.processes[0]!.emitData("A\u001b[界nB"); + yield* manager.close({ threadId: "thread-1" }); + + const reopened = yield* manager.open(openInput()); + expect(reopened.history.split("\u001b[?1049h")[0]).toBe("A\u001b[界nB"); + }), + ); + it.effect.each([ { name: "OSC", sequence: "\u001b]0;unfinished" }, { name: "DCS", sequence: "\u001bPunfinished" }, @@ -1225,6 +1588,21 @@ it.layer( }), ); + it.effect("keeps a decoded U+009C inside a stripped OSC query out of history", () => + Effect.gen(function* () { + const { manager, ptyAdapter } = yield* createManager(); + yield* manager.open(openInput()); + const process = ptyAdapter.processes[0]!; + // libghostty receives UTF-8 (c2 9c), which is OSC payload; only ESC \ ends this string. + process.emitData("A\u001b]11;?\u009cHIDDEN\u001b\\B\n"); + // A DCS string does end at U+009C there, so the text after it stays visible. + process.emitData("C\u001bP$qm\u009cSHOWN\u001b\\D\n"); + yield* manager.close({ threadId: "thread-1" }); + + expect((yield* manager.open(openInput())).history).toMatch(/^AB\nCSHOWN\u001b\\D\n/); + }), + ); + it.effect("preserves large chunked OSC 52 and DCS strings across reopen", () => Effect.gen(function* () { const { manager, ptyAdapter, logsDir } = yield* createManager(); @@ -1329,6 +1707,28 @@ it.layer( }), ); + // Case-insensitive filesystems resolve "Terminal_YQ.log" to thread "a"'s "terminal_YQ.log", + // so the mixed-case file stands in for that alias on case-sensitive hosts. + it.effect.each(["terminal_", "Terminal_"])( + "never resolves a %s legacy history name to another thread's history file", + (prefix) => + Effect.gen(function* () { + const { manager, logsDir } = yield* createManager(); + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const threadId = `${prefix}${Base64Url.encode("a")}`; + const otherThreadPath = path.join(logsDir, `${threadId}.log`); + yield* fs.makeDirectory(logsDir, { recursive: true }); + yield* fs.writeFileString(otherThreadPath, "thread a\n"); + + expect((yield* manager.open(openInput({ threadId }))).history).toBe(""); + yield* manager.close({ threadId, terminalId: DEFAULT_TERMINAL_ID, deleteHistory: true }); + yield* manager.close({ threadId, deleteHistory: true }); + + expect(yield* fs.readFileString(otherThreadPath)).toBe("thread a\n"); + }), + ); + it.effect("keeps reading and deleting history at an encoded path that already fit", () => Effect.gen(function* () { const threadId = "x".repeat(48); @@ -1540,6 +1940,41 @@ it.layer( }), ); + it.effect("keeps a startup failure in the snapshot every later attachment receives", () => + Effect.gen(function* () { + const { manager, ptyAdapter } = yield* createManager(); + ptyAdapter.spawnFailures.push(new Error("forkpty(3) failed.")); + const startupFailure = { + type: "snapshot", + snapshot: { + status: "error", + error: expect.stringContaining("Failed to spawn PTY process"), + }, + }; + const record = (events: TerminalAttachStreamEvent[]) => (event: TerminalAttachStreamEvent) => + Effect.sync(() => events.push(event)); + + const creating: TerminalAttachStreamEvent[] = []; + (yield* manager.attachStream(openInput(), record(creating)))(); + const resumed: TerminalAttachStreamEvent[] = []; + (yield* manager.attachStream({ ...openInput(), resume: true }, record(resumed)))(); + const observed: TerminalAttachStreamEvent[] = []; + (yield* manager.observeStream( + { threadId: "thread-1", terminalId: DEFAULT_TERMINAL_ID }, + record(observed), + ))(); + expect([creating, resumed, observed]).toMatchObject([ + [startupFailure], + [startupFailure], + [startupFailure], + ]); + + const restarted = yield* manager.open(openInput()); + expect(restarted.status).toBe("running"); + expect(restarted).not.toHaveProperty("error"); + }), + ); + it.effect("retries with fallback shells when preferred shell spawn fails", () => Effect.gen(function* () { const platform = yield* HostProcess.Platform; @@ -1577,6 +2012,59 @@ it.layer( }), ); + it.effect("starts an existing $SHELL whose path contains spaces before a dash", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const { join } = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3code-shell-path-" }); + const bin = join(root, "Jane -Doe Smith", ".local", "bin"); + yield* fs.makeDirectory(bin, { recursive: true }); + yield* fs.writeFileString(join(bin, "fish"), ""); + // Temp dirs are not POSIX-absolute on Windows, so stat maps a fixed POSIX prefix onto them. + const fixtureFs = FileSystem.FileSystem.of({ + ...fs, + stat: (filePath) => + fs.stat( + filePath.startsWith("/shell-fixture/") + ? join(root, filePath.slice("/shell-fixture/".length)) + : filePath, + ), + }); + const { manager, ptyAdapter } = yield* createManager(5, { + env: { SHELL: "/shell-fixture/Jane -Doe Smith/.local/bin/fish -l" }, + }).pipe( + Effect.provide(layerWithHostPlatform("darwin")), + Effect.provideService(FileSystem.FileSystem, fixtureFs), + ); + + yield* manager.open(openInput()); + + expect(ptyAdapter.spawnInputs.map((input) => input.shell)).toEqual([ + "/shell-fixture/Jane -Doe Smith/.local/bin/fish", + ]); + }), + ); + + it.effect("keeps missing or quoted $SHELL paths that contain spaces", () => + Effect.gen(function* () { + for (const [shell, expected] of [ + ["/Users/Jane Doe/.local/bin/fish", "/Users/Jane Doe/.local/bin/fish"], + ['"/opt/my -shell/fish" -l', "/opt/my -shell/fish"], + ]) { + const { manager, ptyAdapter } = yield* createManager(5, { + env: { SHELL: shell }, + }).pipe(Effect.provide(layerWithHostPlatform("darwin"))); + + yield* manager.open(openInput()); + + expect( + ptyAdapter.spawnInputs.map((input) => input.shell), + shell, + ).toEqual([expected]); + } + }), + ); + it.effect( "writes completion syntax for the fallback shell instead of the configured fish shell", () => @@ -2053,6 +2541,7 @@ it.layer( const path = yield* Path.Path; const serverSettings = yield* ServerSettings.ServerSettingsService; const environment = yield* TerminalManager.resolveProviderInstanceTerminalEnvironment({ + deriveProviderInstanceConfigMap, serverSettings, path, rawProviderInstanceId: "configured_home", @@ -2198,7 +2687,7 @@ it.layer( { type: "snapshot", snapshot: { status: "running", pid: ptyAdapter.processes[0]?.pid } }, ]); expect(ptyAdapter.processes).toHaveLength(1); - }).pipe(Effect.scoped), + }), ); it.effect("resyncs bounded history and follows close/reopen across a stalled stream", () => @@ -2272,7 +2761,7 @@ it.layer( ptyAdapter.processes.at(-1)!.emitData("new session only\n"); expect(yield* pull).toMatchObject([{ type: "output", data: "new session only\n" }]); expect(ptyAdapter.spawnInputs).toHaveLength(4); - }).pipe(Effect.scoped), + }), ); it.effect.each([ @@ -2319,7 +2808,58 @@ it.layer( yield* Deferred.await(restored); expect(process.resizeCalls.at(-1)).toEqual({ cols: 130, rows: 15 }); - }).pipe(Effect.scoped), + }), + ); + + it.effect("sends bounded attach streams only the history the client retains", () => + Effect.gen(function* () { + const { manager, ptyAdapter } = yield* createManager(); + const maxHistoryBytes = 9; + yield* manager.open(openInput()); + const process = ptyAdapter.processes[0]!; + const published = yield* Deferred.make(); + const unsubscribe = yield* manager.subscribe((event) => + event.type === "output" && event.data === "yz\n" + ? Deferred.succeed(published, undefined).pipe(Effect.asVoid) + : Effect.void, + ); + process.emitData("aaaa\nx名😀"); + process.emitData("yz\n"); + yield* Deferred.await(published); + unsubscribe(); + + const full = yield* Stream.toPull(manager.attachOutputStream(openInput(), "full")); + const bounded = yield* Stream.toPull( + manager.attachOutputStream({ ...openInput(), maxHistoryBytes }, "bounded"), + ); + const expectRetained = Effect.fnUntraced(function* () { + const [fullEvents, boundedEvents] = [yield* full, yield* bounded]; + const fullSnapshot = fullEvents.at(-1); + if (fullSnapshot?.type !== "snapshot") throw new Error("Expected a full snapshot"); + expect(boundedEvents.at(-1)).toEqual({ + ...fullSnapshot, + snapshot: { + ...fullSnapshot.snapshot, + history: retainedTerminalHistory(fullSnapshot.snapshot.history, maxHistoryBytes), + }, + }); + return fullSnapshot.snapshot.history; + }); + + expect(yield* expectRetained()).toBe("aaaa\nx名😀yz\n"); + // The cap starts inside 名, so the tail begins at the next code point. + expect(retainedTerminalHistory("aaaa\nx名😀yz\n", maxHistoryBytes)).toBe("😀yz\n"); + process.emitData("after\n"); + expect(yield* full).toEqual(yield* bounded); + + yield* publishPastOutputBudget(manager, process); + expect(yield* expectRetained()).toMatch(/last\n$/); + + process.emitExit({ exitCode: 0, signal: 0 }); + expect(yield* full).toEqual(yield* bounded); + yield* manager.attachStream({ ...openInput(), restartIfNotRunning: true }, () => Effect.void); + yield* expectRetained(); + }), ); it.effect("does not replay in-flight output already included in an observer snapshot", () => @@ -2356,7 +2896,7 @@ it.layer( expect(observed.filter((event) => event.type === "output")).toMatchObject([ { data: "after\n" }, ]); - }).pipe(Effect.scoped), + }), ); it.effect("observes terminal history and live output without changing the process", () => diff --git a/apps/server/src/terminal/Manager.ts b/apps/server/src/terminal/Manager.ts index 625b678446e1..d034af98ee1f 100644 --- a/apps/server/src/terminal/Manager.ts +++ b/apps/server/src/terminal/Manager.ts @@ -5,6 +5,10 @@ import { withWorkspaceLease } from "../workspace/workspaceLease.ts"; import { DEFAULT_TERMINAL_ID, DEFAULT_MAX_TERMINAL_BUFFER_BYTES, + retainedTerminalHistory, + TERMINAL_SEQUENCE_GROUND, + terminalSequenceEnd, + terminalSequenceStateAfter, TerminalCwdError, TerminalCwdNotDirectoryError, TerminalCwdNotFoundError, @@ -67,7 +71,7 @@ import * as ServerConfig from "../config.ts"; import { mergeProviderInstanceEnvironment } from "@t3tools/provider-core/server/instanceEnvironment"; import { resolveCodexHomeLayout } from "../provider/Drivers/CodexHomeLayout.ts"; import { makeClaudeEnvironment } from "../provider/Drivers/ClaudeHome.ts"; -import { deriveProviderInstanceConfigMap } from "../provider/ProviderInstanceRegistryHydration.ts"; +import type { deriveProviderInstanceConfigMap } from "../provider/ProviderInstanceRegistryHydration.ts"; import * as ServerSettings from "../serverSettings.ts"; import { increment, terminalSessionsTotal } from "../observability/Metrics.ts"; import { expandHomeVariable } from "@t3tools/provider-core/server/pathExpansion"; @@ -166,6 +170,10 @@ class TerminalProcessSignalError extends Schema.TaggedError Effect.Effect +>("t3/terminal/TerminalCwdRecovery", { defaultValue: () => (error) => Effect.fail(error) }) {} + export class TerminalManager extends Context.Service< TerminalManager, { @@ -308,6 +316,7 @@ interface TerminalSessionState { childCommandLabel: string | null; runtimeEnv: Record | null; stopOnSettle: boolean; + startupError: string | null; } interface TerminalGrid { @@ -403,6 +412,7 @@ function snapshot(session: TerminalSessionState): TerminalSessionSnapshot { sequence: session.eventSequence, cols: session.cols, rows: session.rows, + ...(session.startupError === null ? {} : { error: session.startupError }), }; } @@ -519,10 +529,10 @@ function defaultShellResolver(platform: NodeJS.Platform, env: NodeJS.ProcessEnv) return env.SHELL ?? "bash"; } -function normalizeShellCommand( +const normalizeShellCommand = Effect.fn("terminal.normalizeShellCommand")(function* ( value: string | undefined, platform: NodeJS.Platform, -): string | null { +) { if (!value) return null; const trimmed = value.trim(); if (trimmed.length === 0) return null; @@ -531,10 +541,24 @@ function normalizeShellCommand( return trimmed; } - const firstToken = trimmed.split(/\s+/g)[0]?.trim(); - if (!firstToken) return null; - return firstToken.replace(/^['"]|['"]$/g, ""); -} + const quoted = /^(['"])(.*?)\1(?:\s|$)/.exec(trimmed); + if (quoted) return quoted[2] || null; + + if (trimmed.startsWith("/") && /\s/.test(trimmed)) { + const fs = yield* FileSystem.FileSystem; + const ends = [...trimmed.matchAll(/\s+/g)].map((match) => match.index); + // Unquoted paths are ambiguous; prefer the longest existing file over flag syntax. + for (const end of [trimmed.length, ...ends.reverse()]) { + const candidate = trimmed.slice(0, end); + const info = yield* fs + .stat(candidate) + .pipe(Effect.catchTags({ PlatformError: () => Effect.succeed(null) })); + if (info?.type === "File") return candidate; + } + } + + return trimmed.split(/\s+(?=-)/)[0]?.replace(/^['"]|['"]$/g, "") || null; +}); function basenameForPlatform(command: string, platform: NodeJS.Platform): string { const normalized = @@ -610,13 +634,13 @@ function uniqueShellCandidates(candidates: Array): ShellC return ordered; } -function resolveShellCandidates( +const resolveShellCandidates = Effect.fn("terminal.resolveShellCandidates")(function* ( shellResolver: () => string, platform: NodeJS.Platform, env: NodeJS.ProcessEnv, -): ShellCandidate[] { +) { const requested = shellCandidateFromCommand( - normalizeShellCommand(shellResolver(), platform), + yield* normalizeShellCommand(shellResolver(), platform), platform, ); @@ -634,7 +658,7 @@ function resolveShellCandidates( return uniqueShellCandidates([ requested, - shellCandidateFromCommand(normalizeShellCommand(env.SHELL, platform), platform), + shellCandidateFromCommand(yield* normalizeShellCommand(env.SHELL, platform), platform), shellCandidateFromCommand("/bin/zsh", platform), shellCandidateFromCommand("/bin/bash", platform), shellCandidateFromCommand("/bin/sh", platform), @@ -642,7 +666,7 @@ function resolveShellCandidates( shellCandidateFromCommand("bash", platform), shellCandidateFromCommand("sh", platform), ]); -} +}); function isRetryableShellSpawnError(error: PtyAdapter.PtySpawnError): boolean { const queue: unknown[] = [error]; @@ -877,6 +901,20 @@ interface TerminalHistoryChunk { lineBreaks: number; } +interface TerminalHistoryPersistenceCursor { + readonly generation: number; + readonly offset: number; + readonly fileBytes: number; +} + +function countLineBreaks(data: string): number { + let lineBreaks = 0; + for (let index = data.indexOf("\n"); index !== -1; index = data.indexOf("\n", index + 1)) { + lineBreaks += 1; + } + return lineBreaks; +} + export class BoundedTerminalHistory { private readonly maxLines: number; private readonly maxBytes: number; @@ -886,7 +924,11 @@ export class BoundedTerminalHistory { private lineBreaks = 0; // Reading the old string's tail on each append can force chunk concatenation. private lastCodeUnit: number | undefined; + // The terminal parser's state at the first retained code unit, which is where a replay starts. + private sequenceState = TERMINAL_SEQUENCE_GROUND; private cachedValue: string | null = ""; + private generation = 0; + private appendedLength = 0; constructor(maxLines: number, initial: string, maxBytes = DEFAULT_HISTORY_BYTE_LIMIT) { this.maxLines = maxLines; @@ -896,6 +938,7 @@ export class BoundedTerminalHistory { append(text: string): void { if (text.length === 0) return; + this.appendedLength += text.length; this.cachedValue = null; if (this.maxBytes <= 0 || this.maxLines <= 0) { this.clear(); @@ -916,6 +959,8 @@ export class BoundedTerminalHistory { firstCode <= 0xdfff ) { // Joining a split surrogate changes its UTF-8 size from 3 to 4 bytes. + // The previous flush may have encoded the lone surrogate as a replacement character. + this.generation += 1; previous.data += text[0]; previous.byteLength += 1; this.byteLength += 1; @@ -945,10 +990,7 @@ export class BoundedTerminalHistory { private appendChunk(data: string): void { const byteLength = Buffer.byteLength(data); - let lineBreaks = 0; - for (let index = data.indexOf("\n"); index !== -1; index = data.indexOf("\n", index + 1)) { - lineBreaks += 1; - } + const lineBreaks = countLineBreaks(data); const previous = this.chunks.at(-1); if (previous && previous.data.length + data.length <= MAX_HISTORY_CHUNK_LENGTH) { previous.data += data; @@ -965,6 +1007,7 @@ export class BoundedTerminalHistory { private discardChunk(): void { const first = this.chunks[this.start]!; + this.sequenceState = terminalSequenceStateAfter(this.sequenceState, first.data); this.byteLength -= first.byteLength; this.lineBreaks -= first.lineBreaks; this.chunks[this.start++] = undefined; @@ -976,6 +1019,7 @@ export class BoundedTerminalHistory { this.discardChunk(); return; } + this.sequenceState = terminalSequenceStateAfter(this.sequenceState, first.data, 0, offset); first.data = first.data.slice(offset); first.byteLength -= byteLength; first.lineBreaks -= lineBreaks; @@ -1023,6 +1067,12 @@ export class BoundedTerminalHistory { } this.trimChunk(offset, bytes, lineBreaks); } + // A cut inside a control sequence would replay the rest of it as text. + while (this.sequenceState !== TERMINAL_SEQUENCE_GROUND && this.start < this.chunks.length) { + const first = this.chunks[this.start]!; + const dropped = first.data.slice(0, terminalSequenceEnd(this.sequenceState, first.data)); + this.trimChunk(dropped.length, Buffer.byteLength(dropped), countLineBreaks(dropped)); + } if ( this.start === this.chunks.length || (this.start > 2_048 && this.start * 2 >= this.chunks.length) @@ -1034,14 +1084,49 @@ export class BoundedTerminalHistory { } clear(): void { + this.generation += 1; + this.appendedLength = 0; this.chunks = []; this.start = 0; this.byteLength = 0; this.lineBreaks = 0; this.lastCodeUnit = undefined; + this.sequenceState = TERMINAL_SEQUENCE_GROUND; this.cachedValue = ""; } + persistenceUpdate(previous?: TerminalHistoryPersistenceCursor) { + const offset = this.appendedLength; + let data: string | undefined; + if (previous?.generation === this.generation) { + let remaining = offset - previous.offset; + const tail: string[] = []; + for (let index = this.chunks.length - 1; remaining > 0 && index >= this.start; index -= 1) { + const chunk = this.chunks[index]!.data; + const suffix = chunk.slice(Math.max(0, chunk.length - remaining)); + tail.push(suffix); + remaining -= suffix.length; + } + if (remaining === 0) data = tail.toReversed().join(""); + } + // Keep the on-disk log bounded while amortizing compaction across new output. + // Readers already cap the plain-text tail by bytes and lines. + const append = + data !== undefined && + previous !== undefined && + previous.fileBytes + Buffer.byteLength(data) <= this.maxBytes * 2; + const output = append && data !== undefined ? data : this.value(); + return { + data: output, + append, + cursor: { + generation: this.generation, + offset, + fileBytes: (append ? previous.fileBytes : 0) + Buffer.byteLength(output), + }, + }; + } + value(): string { if (this.cachedValue !== null) return this.cachedValue; this.cachedValue = this.chunks @@ -1106,7 +1191,7 @@ function shouldStripCsiSequence(body: string, finalByte: string): boolean { if (finalByte === "R" && /^[0-9;?]*$/.test(body)) { return true; } - if (finalByte === "c" && /^[>0-9;?]*$/.test(body)) { + if (finalByte === "c" && /^[>=0-9;?]*$/.test(body)) { return true; } // Strip DECRQM/DECRPM query traffic to prevent echoed replies during replay; $ guards preserve setters. @@ -1144,16 +1229,70 @@ function stripStringTerminator(value: string): string { return value; } -function findStringTerminatorIndex(input: string, start: number): number | null { +interface C1Effect { + /** ESC form of the sequence the code point starts, or "" when it returns to ground. */ + readonly next: string; + /** Its 0x9A byte ran DECID, which replies with device attributes. */ + readonly query: boolean; + /** What replays its visible effect from ground. */ + readonly safe: string; +} + +// Outside ground and OSC, libghostty parses a code point's UTF-8 bytes one at a time: 0x80-0x9F +// act as C1 controls, the rest are dropped. Bytes left over after a return to ground print U+FFFD. +function c1Effect(codePoint: number): C1Effect | undefined { + // A lone surrogate is encoded as U+FFFD, whose bytes are all dropped. + if (codePoint >= 0xd800 && codePoint <= 0xdfff) return undefined; + let next: string | undefined; + let query = false; + let safe = ""; + for (let shift = codePoint > 0xffff ? 12 : codePoint > 0x7ff ? 6 : 0; shift >= 0; shift -= 6) { + const byte = 0x80 | ((codePoint >> shift) & 0x3f); + if (next === "") { + safe += "\ufffd"; + } else if (next?.startsWith("\u001b]")) { + next += "\ufffd"; + } else if (byte === 0x9b) { + next = "\u001b["; + } else if (byte === 0x9d) { + next = "\u001b]"; + } else if (byte === 0x90) { + next = "\u001bP"; + } else if (byte === 0x98 || byte === 0x9e || byte === 0x9f) { + next = "\u001b_"; + } else if (byte < 0xa0) { + next = ""; + query = byte === 0x9a; + // Any other C1 control but ST runs as ESC plus the byte less 0x40. + safe = query || byte === 0x9c ? "" : `\u001b${String.fromCharCode(byte - 0x40)}`; + } + } + return next === undefined ? undefined : { next, query, safe }; +} + +// BEL ends only OSC; inside DCS, PM and APC it is payload. Non-ASCII is the reverse: libghostty +// keeps it inside OSC, while a C1 byte in it ends the other strings. +function findStringEnd( + input: string, + start: number, + isOsc: boolean, +): { end: number; c1?: { effect: C1Effect; index: number } } | null { for (let index = start; index < input.length; index += 1) { const codePoint = input.charCodeAt(index); - if (codePoint === 0x07 || codePoint === 0x9c || isControlSequenceCancel(codePoint)) { - return index + 1; + if ((isOsc && codePoint === 0x07) || isControlSequenceCancel(codePoint)) { + return { end: index + 1 }; } // Like xterm.js, an ESC that does not start ST ends the string and begins the next sequence. if (codePoint === 0x1b) { if (index + 1 >= input.length) return null; - return input.charCodeAt(index + 1) === 0x5c ? index + 2 : index; + return { end: input.charCodeAt(index + 1) === 0x5c ? index + 2 : index }; + } + if (codePoint >= 0x80 && !isOsc) { + const fullCodePoint = input.codePointAt(index) ?? codePoint; + const end = index + (fullCodePoint > 0xffff ? 2 : 1); + const effect = c1Effect(fullCodePoint); + if (effect) return { end, c1: { effect, index } }; + index = end - 1; } } return null; @@ -1199,6 +1338,8 @@ function sanitizeTerminalHistoryChunk( const pendingScanStart = Math.max(0, pendingControlSequence.length - 1); let visibleText = ""; let index = 0; + // Set when a C1 code point began the sequence at `index`: the ESC form that stands in for it. + let c1Introducer = ""; const append = (value: string) => { visibleText += value; @@ -1207,29 +1348,68 @@ function sanitizeTerminalHistoryChunk( while (index < input.length) { const codePoint = input.charCodeAt(index); - if (codePoint === 0x1b) { - const nextCodePoint = input.charCodeAt(index + 1); + if (c1Introducer.length > 0 || codePoint === 0x1b) { + // Empty for a sequence the input introduces itself, which starts with ESC at `index`. + const prefix = c1Introducer; + c1Introducer = ""; + const contentStart = prefix.length > 0 ? index : index + 2; + const nextCodePoint = (prefix || input).charCodeAt(prefix.length > 0 ? 1 : index + 1); if (Number.isNaN(nextCodePoint)) { return { visibleText, pendingControlSequence: input.slice(index) }; } if (nextCodePoint === 0x5b) { - let cursor = index + 2; + let cursor = contentStart; + // Inside a CSI Ghostty executes 0x00-0x0F and ignores DEL and 0x10-0x1F, which it would + // print in ground state. None join the parameters; only executed ones survive a strip. + let body = ""; + let executedControls = ""; while (cursor < input.length) { - const cancelled = isControlSequenceCancel(input.charCodeAt(cursor)); - if (cancelled || isCsiFinalByte(input.charCodeAt(cursor))) { - const sequence = input.slice(index, cursor + 1); - const body = input.slice(index + 2, cursor); + const cursorCodePoint = input.charCodeAt(cursor); + const cancelled = isControlSequenceCancel(cursorCodePoint); + if (cancelled || isCsiFinalByte(cursorCodePoint)) { if (cancelled || !shouldStripCsiSequence(body, input[cursor] ?? "")) { - append(sequence); + append(prefix + input.slice(index, cursor + 1)); + } else { + append(executedControls); } index = cursor + 1; break; } + if (cursorCodePoint >= 0x80) { + const fullCodePoint = input.codePointAt(cursor) ?? cursorCodePoint; + const end = cursor + (fullCodePoint > 0xffff ? 2 : 1); + const c1 = c1Effect(fullCodePoint); + if (c1) { + // The CSI ends here undispatched, so it is kept only when the terminal is back in + // ground with nothing to answer. + if (c1.next.length === 0 && !c1.query) { + append(prefix + input.slice(index, end)); + } else { + append(executedControls + c1.safe); + } + c1Introducer = c1.next; + index = end; + break; + } + cursor = end; + continue; + } + // An ESC abandons the CSI and starts the next sequence, which may be stripped. + if (cursorCodePoint === 0x1b) { + append(executedControls); + index = cursor; + break; + } + if (cursorCodePoint < 0x10) { + executedControls += input[cursor]; + } else if (cursorCodePoint >= 0x20 && cursorCodePoint !== 0x7f) { + body += input[cursor]; + } cursor += 1; } if (cursor >= input.length) { - return { visibleText, pendingControlSequence: input.slice(index) }; + return { visibleText, pendingControlSequence: prefix + input.slice(index) }; } continue; } @@ -1240,22 +1420,32 @@ function sanitizeTerminalHistoryChunk( nextCodePoint === 0x5e || nextCodePoint === 0x5f ) { - const terminatorIndex = findStringTerminatorIndex( + const stringEnd = findStringEnd( input, - Math.max(index + 2, pendingScanStart), + Math.max(contentStart, pendingScanStart), + nextCodePoint === 0x5d, ); - if (terminatorIndex === null) { - return { visibleText, pendingControlSequence: input.slice(index) }; + if (stringEnd === null) { + return { visibleText, pendingControlSequence: prefix + input.slice(index) }; } - const sequence = input.slice(index, terminatorIndex); - const content = stripStringTerminator(input.slice(index + 2, terminatorIndex)); + const { end, c1 } = stringEnd; + const content = stripStringTerminator( + prefix.slice(2) + input.slice(contentStart, c1?.index ?? end), + ); const strip = (nextCodePoint === 0x5d && shouldStripOscSequence(content)) || (nextCodePoint === 0x50 && shouldStripDcsSequence(content)); - if (!strip) { - append(sequence); + if (c1 === undefined || (!strip && c1.effect.next.length === 0 && !c1.effect.query)) { + if (!strip) { + append(prefix + input.slice(index, end)); + } + } else { + // ST stands in for a code point that cannot be replayed as it came. + const closed = strip ? "" : `${prefix}${input.slice(index, c1.index)}\u009c`; + append(closed + c1.effect.safe); + c1Introducer = c1.effect.next; } - index = terminatorIndex; + index = end; continue; } @@ -1263,7 +1453,10 @@ function sanitizeTerminalHistoryChunk( if (escapeSequenceEndIndex === null) { return { visibleText, pendingControlSequence: input.slice(index) }; } - append(input.slice(index, escapeSequenceEndIndex)); + // ESC Z (DECID) is the legacy primary device attributes query. + if (nextCodePoint !== 0x5a) { + append(input.slice(index, escapeSequenceEndIndex)); + } index = escapeSequenceEndIndex; continue; } @@ -1275,7 +1468,7 @@ function sanitizeTerminalHistoryChunk( index = nextSequenceStart; } - return { visibleText, pendingControlSequence: "" }; + return { visibleText, pendingControlSequence: c1Introducer }; } function legacySafeThreadId(threadId: string): string { @@ -1451,6 +1644,7 @@ interface TerminalManagerOptions { export const resolveProviderInstanceTerminalEnvironment = Effect.fn( "terminal.resolveProviderInstanceTerminalEnvironment", )(function* (input: { + readonly deriveProviderInstanceConfigMap: typeof deriveProviderInstanceConfigMap; readonly serverSettings: ServerSettings.ServerSettingsService["Service"]; readonly path: Path.Path; readonly rawProviderInstanceId: string; @@ -1460,7 +1654,7 @@ export const resolveProviderInstanceTerminalEnvironment = Effect.fn( const settings = yield* input.serverSettings.getSettings.pipe( Effect.mapError((cause) => new TerminalProviderEnvironmentError({ providerInstanceId, cause })), ); - const instance = deriveProviderInstanceConfigMap(settings)[providerInstanceId]; + const instance = input.deriveProviderInstanceConfigMap(settings)[providerInstanceId]; if (instance === undefined) { return yield* new TerminalProviderInstanceNotFoundError({ providerInstanceId }); } @@ -1490,7 +1684,9 @@ export const resolveProviderInstanceTerminalEnvironment = Effect.fn( }); /** @public */ -export const make = Effect.fn("TerminalManager.make")(function* () { +export const make = Effect.fn("TerminalManager.make")(function* ( + deriveConfigMap: typeof deriveProviderInstanceConfigMap, +) { const { terminalLogsDir, providerStatusCacheDir, baseDir } = yield* ServerConfig.ServerConfig; const ptyAdapter = yield* PtyAdapter.PtyAdapter; const portDiscovery = yield* PortScanner.PortDiscovery; @@ -1501,6 +1697,7 @@ export const make = Effect.fn("TerminalManager.make")(function* () { "terminal.resolveProviderInstanceEnvironment", )((rawProviderInstanceId: string, env: Record | undefined) => resolveProviderInstanceTerminalEnvironment({ + deriveProviderInstanceConfigMap: deriveConfigMap, serverSettings, path, rawProviderInstanceId, @@ -1680,14 +1877,19 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func ); }; - const legacyHistoryPath = (threadId: string) => - path.join(logsDir, `${legacySafeThreadId(threadId)}.log`); + const legacyHistoryName = (threadId: string) => `${legacySafeThreadId(threadId)}.log`; + // Legacy names in the current "terminal_" namespace would alias another thread's history, + // case-insensitively on some filesystems. + const hasLegacyHistoryName = (threadId: string) => + !legacySafeThreadId(threadId).toLowerCase().startsWith("terminal_"); const historyPaths = (threadId: string, terminalId: string) => [ ...new Set([ historyPath(threadId, terminalId), encodedHistoryPath(threadId, terminalId), - ...(terminalId === DEFAULT_TERMINAL_ID ? [legacyHistoryPath(threadId)] : []), + ...(terminalId === DEFAULT_TERMINAL_ID && hasLegacyHistoryName(threadId) + ? [path.join(logsDir, legacyHistoryName(threadId))] + : []), ]), ]; @@ -1934,6 +2136,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func yield* registerKillFiber(process, fiber); }); + const persistedHistory = new WeakMap(); const persistWorker = yield* makeKeyedCoalescingWorker< string, PersistHistoryRequest, @@ -1954,9 +2157,19 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func return; } + const update = request.history.persistenceUpdate( + request.immediate ? undefined : persistedHistory.get(request.history), + ); + if (update.append && update.data.length === 0) return; + // An append may partially succeed before failing. Retry from a full snapshot, + // never from an uncertain file offset. + persistedHistory.delete(request.history); yield* fileSystem - .writeFileString(historyPath(threadId, terminalId), request.history.value()) + .writeFileString(historyPath(threadId, terminalId), update.data, { + flag: update.append ? "a" : "w", + }) .pipe( + Effect.tap(() => Effect.sync(() => persistedHistory.set(request.history, update.cursor))), Effect.catch((error) => Effect.logWarning("failed to persist terminal history", { threadId, @@ -2003,6 +2216,19 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func const info = yield* file.stat; const limit = BigInt(historyByteLimit); const offset = info.size > limit ? info.size - limit : 0n; + // The tail can start inside a control sequence; only the bytes before it say which. + const decoder = new TextDecoder("utf-8", { ignoreBOM: true }); + let sequenceState = TERMINAL_SEQUENCE_GROUND; + const skipped = new Uint8Array(Math.min(Number(offset), 64 * 1024)); + for (let remaining = Number(offset); remaining > 0;) { + const read = Number(yield* file.read(skipped.subarray(0, remaining))); + if (read === 0) break; + remaining -= read; + sequenceState = terminalSequenceStateAfter( + sequenceState, + decoder.decode(skipped.subarray(0, read), { stream: true }), + ); + } yield* file.seek(offset, "start"); const bytes = new Uint8Array(Number(info.size - offset)); let length = 0; @@ -2016,8 +2242,15 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func // A tail read can start inside a UTF-8 code point. Skip its remaining bytes. while (start < length && ((bytes[start] ?? 0) & 0xc0) === 0x80) start += 1; } + sequenceState = terminalSequenceStateAfter( + sequenceState, + decoder.decode(bytes.subarray(0, start)), + ); + const history = new TextDecoder("utf-8", { ignoreBOM: true }).decode( + bytes.subarray(start, length), + ); return { - history: new TextDecoder("utf-8", { ignoreBOM: true }).decode(bytes.subarray(start, length)), + history: history.slice(terminalSequenceEnd(sequenceState, history)), truncated: offset > 0n, }; }); @@ -2047,17 +2280,15 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func const history = new BoundedTerminalHistory(historyLineLimit, raw, historyByteLimit); const capped = history.value(); if (migrating || truncated || capped !== raw) { - const write = migrating - ? Effect.gen(function* () { - const temporaryPath = yield* fileSystem.makeTempFileScoped({ - directory: logsDir, - prefix: ".history-", - }); - yield* fileSystem.writeFileString(temporaryPath, capped); - yield* fileSystem.rename(temporaryPath, nextPath); - }).pipe(Effect.scoped) - : fileSystem.writeFileString(nextPath, capped); - yield* write.pipe( + const write = Effect.gen(function* () { + const temporaryPath = yield* fileSystem.makeTempFileScoped({ + directory: logsDir, + prefix: ".history-", + }); + yield* fileSystem.writeFileString(temporaryPath, capped); + yield* fileSystem.rename(temporaryPath, nextPath); + }).pipe( + Effect.scoped, Effect.mapError( (cause) => new TerminalHistoryError({ @@ -2068,6 +2299,17 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func }), ), ); + yield* migrating + ? write + : write.pipe( + Effect.catch((error) => + Effect.logWarning("failed to compact terminal history", { + threadId, + terminalId, + error, + }), + ), + ); } if (migrating) { yield* fileSystem.remove(sourcePath, { force: true }).pipe( @@ -2119,7 +2361,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func yield* Effect.forEach( entries.filter( (name) => - name === `${legacySafeThreadId(threadId)}.log` || + (hasLegacyHistoryName(threadId) && name === legacyHistoryName(threadId)) || threadParts.some((part) => name === `${part}.log` || name.startsWith(`${part}_`)), ), (name) => @@ -2136,7 +2378,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func }); const assertValidCwd = Effect.fn("terminal.assertValidCwd")(function* (cwd: string) { - const stats = yield* fileSystem.stat(cwd).pipe( + const stat = fileSystem.stat(cwd).pipe( Effect.catchTags({ PlatformError: (cause) => cause.reason._tag === "NotFound" @@ -2144,6 +2386,14 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func : new TerminalCwdStatError({ cwd, cause }), }), ); + const stats = yield* stat.pipe( + Effect.catchTags({ + TerminalCwdNotFoundError: (error) => + Effect.flatMap(TerminalCwdRecovery, (recover) => + recover(error).pipe(Effect.andThen(stat)), + ), + }), + ); if (stats.type !== "Directory") { return yield* new TerminalCwdNotDirectoryError({ cwd }); } @@ -2432,6 +2682,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func const startingAt = yield* nowIso; yield* modifyManagerState((state) => { session.status = "starting"; + session.startupError = null; session.shell = null; session.cwd = input.cwd; session.worktreePath = input.worktreePath ?? null; @@ -2455,7 +2706,11 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func increment(terminalSessionsTotal, { lifecycle: eventType }).pipe( Effect.andThen( Effect.gen(function* () { - const shellCandidates = resolveShellCandidates(shellResolver, platform, baseEnv); + const shellCandidates = yield* resolveShellCandidates( + shellResolver, + platform, + baseEnv, + ).pipe(Effect.provideService(FileSystem.FileSystem, fileSystem)); const terminalEnv = createTerminalSpawnEnv( baseEnv, session.runtimeEnv, @@ -2548,6 +2803,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func { const error = startResult.failure; + const message = error.message; if (ptyProcess) { yield* startKillEscalation(ptyProcess, session.threadId, session.terminalId); } @@ -2555,6 +2811,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func yield* modifyManagerState((state) => { cleanupProcessHandles(session); session.status = "error"; + session.startupError = message; session.pid = null; session.process = null; session.hasRunningSubprocess = false; @@ -2572,7 +2829,6 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func yield* evictInactiveSessionsIfNeeded(); - const message = error.message; yield* publishEvent({ type: "error", threadId: session.threadId, @@ -2891,6 +3147,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func childCommandLabel: null, runtimeEnv: normalizedRuntimeEnv(input.env), stopOnSettle: input.stopOnSettle ?? false, + startupError: null, }; const createdSession = session; @@ -3236,20 +3493,41 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func registerCleanup, ); - const attachOutputStream: TerminalManager["Service"]["attachOutputStream"] = (input, viewerId) => - Stream.unwrap( + const attachOutputStream: TerminalManager["Service"]["attachOutputStream"] = ( + input, + viewerId, + ) => { + const { maxHistoryBytes } = input; + const retained = (listener: (event: TerminalAttachStreamEvent) => Effect.Effect) => + maxHistoryBytes === undefined + ? listener + : (event: TerminalAttachStreamEvent) => + listener( + "snapshot" in event + ? { + ...event, + snapshot: { + ...event.snapshot, + history: retainedTerminalHistory(event.snapshot.history, maxHistoryBytes), + }, + } + : event, + ); + return Stream.unwrap( Effect.acquireRelease( Effect.sync(() => retainViewerOnce(input, viewerId, null)), (release) => Effect.sync(release), ).pipe( Effect.as( terminalOutputStream( - (listener, registerCleanup) => attachStream(input, listener, registerCleanup, viewerId), - (listener, registerCleanup) => resyncStream(input, listener, registerCleanup), + (listener, registerCleanup) => + attachStream(input, retained(listener), registerCleanup, viewerId), + (listener, registerCleanup) => resyncStream(input, retained(listener), registerCleanup), ), ), ), ); + }; const metadataEventFromTerminalEvent = ( event: TerminalEvent, @@ -3400,7 +3678,12 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func session.pendingProcessEventIndex = 0; session.processEventDrainRunning = false; const eventStamp = advanceEventSequence(session); - yield* persistHistory(input.threadId, terminalId, session.history); + // A drain waits for the key to go idle, which never happens while the shell keeps + // printing; every reader of the file flushes it first. + yield* persistWorker.enqueue(toSessionKey(input.threadId, terminalId), { + history: session.history, + immediate: true, + }); yield* publishEvent({ type: "cleared", threadId: input.threadId, @@ -3509,4 +3792,5 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func }); }); -export const layer = Layer.effect(TerminalManager, make()).pipe(Layer.provide(ProcessRunner.layer)); +export const layer = (deriveConfigMap: typeof deriveProviderInstanceConfigMap) => + Layer.effect(TerminalManager, make(deriveConfigMap)).pipe(Layer.provide(ProcessRunner.layer)); diff --git a/apps/server/src/terminal/OutputProtocol.test.ts b/apps/server/src/terminal/OutputProtocol.test.ts index 9750f4181520..7ae536605e8f 100644 --- a/apps/server/src/terminal/OutputProtocol.test.ts +++ b/apps/server/src/terminal/OutputProtocol.test.ts @@ -9,80 +9,99 @@ import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Queue from "effect/Queue"; -import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; import { Rpc, RpcGroup, RpcMessage, RpcSerialization, RpcServer } from "effect/rpc"; import { terminalOutputStream, withOutputWindow } from "./OutputProtocol.ts"; +const eventWith = ( + field: "data" | "message" | "threadId", + text: string, +): TerminalAttachStreamEvent => + field === "message" + ? { threadId: "thread-1", terminalId: "default", type: "error", message: text } + : { + threadId: field === "threadId" ? text : "thread-1", + terminalId: "default", + type: "output", + data: field === "data" ? text : "x", + }; + describe("terminal output window", () => { it.effect.each([ - { tag: WS_METHODS.terminalAttach, size: 1, limit: 8 }, - { tag: WS_METHODS.terminalObserve, size: 1, limit: 8 }, - { tag: WS_METHODS.terminalAttach, size: 64 * 1024, limit: 1 }, - { tag: WS_METHODS.terminalObserve, size: 64 * 1024, limit: 1 }, - { tag: WS_METHODS.subscribeServerConfig, size: 1, limit: 8 }, - { tag: ORCHESTRATION_V2_WS_METHODS.subscribeShell, size: 1, limit: 8 }, - { tag: ORCHESTRATION_V2_WS_METHODS.subscribeThread, size: 64 * 1024, limit: 8 }, - { tag: WS_METHODS.subscribeTerminalMetadata, size: 1, limit: 1 }, - ])("limits $tag with $size-byte values to $limit pending chunks", ({ tag, size, limit }) => - Effect.gen(function* () { - const group = RpcGroup.make(Rpc.make(tag, { success: Schema.String, stream: true })); - const output = yield* Queue.unbounded(); - const responses = yield* Queue.unbounded(); - const receive = yield* Deferred.make[0]>(); - const protocol = yield* RpcServer.Protocol.make((write) => - Effect.gen(function* () { - yield* Deferred.succeed(receive, write); - const serialization = yield* RpcSerialization.RpcSerialization; - return { - disconnects: yield* Queue.unbounded(), - send: (_clientId, response) => Queue.offer(responses, response), - end: () => Effect.void, - clientIds: Effect.succeed(new Set([0])), - initialMessage: Effect.succeedNone, - supportsAck: true, - supportsTransferables: false, - supportsSpanPropagation: false, - supportsNotifications: true, - codecFor: serialization.codecFor, - }; - }), - ); - yield* RpcServer.make(group).pipe( - Effect.provide(group.toLayerHandler(tag, () => Stream.fromQueue(output))), - Effect.provideService(RpcServer.Protocol, withOutputWindow(protocol)), - Effect.forkScoped, - ); - const write = yield* Deferred.await(receive); - yield* write(0, { _tag: "Request", id: "1", tag, payload: null, headers: [] }); + { tag: WS_METHODS.terminalAttach, field: "data", size: 1, limit: 8 }, + { tag: WS_METHODS.terminalObserve, field: "data", size: 1, limit: 8 }, + { tag: WS_METHODS.terminalAttach, field: "data", size: 64 * 1024, limit: 1 }, + { tag: WS_METHODS.terminalObserve, field: "data", size: 64 * 1024, limit: 1 }, + { tag: WS_METHODS.terminalAttach, field: "message", size: 64 * 1024, limit: 1 }, + { tag: WS_METHODS.terminalAttach, field: "threadId", size: 64 * 1024, limit: 1 }, + { tag: WS_METHODS.subscribeServerConfig, field: "data", size: 1, limit: 8 }, + { tag: ORCHESTRATION_V2_WS_METHODS.subscribeShell, field: "data", size: 1, limit: 8 }, + { tag: ORCHESTRATION_V2_WS_METHODS.subscribeThread, field: "data", size: 64 * 1024, limit: 8 }, + { tag: WS_METHODS.subscribeTerminalMetadata, field: "data", size: 1, limit: 1 }, + ] as const)( + "limits $tag with $size-byte $field values to $limit pending chunks", + ({ tag, field, size, limit }) => + Effect.gen(function* () { + const group = RpcGroup.make( + Rpc.make(tag, { success: TerminalAttachStreamEventSchema, stream: true }), + ); + const output = yield* Queue.unbounded(); + const responses = yield* Queue.unbounded(); + const receive = yield* Deferred.make[0]>(); + const protocol = yield* RpcServer.Protocol.make((write) => + Effect.gen(function* () { + yield* Deferred.succeed(receive, write); + const serialization = yield* RpcSerialization.RpcSerialization; + return { + disconnects: yield* Queue.unbounded(), + send: (_clientId, response) => Queue.offer(responses, response), + end: () => Effect.void, + clientIds: Effect.succeed(new Set([0])), + initialMessage: Effect.succeedNone, + supportsAck: true, + supportsTransferables: false, + supportsSpanPropagation: false, + supportsNotifications: true, + codecFor: serialization.codecFor, + }; + }), + ); + yield* RpcServer.make(group).pipe( + Effect.provide(group.toLayerHandler(tag, () => Stream.fromQueue(output))), + Effect.provideService(RpcServer.Protocol, withOutputWindow(protocol)), + Effect.forkScoped, + ); + const write = yield* Deferred.await(receive); + yield* write(0, { _tag: "Request", id: "1", tag, payload: null, headers: [] }); - for (let index = 0; index < limit; index++) { - const value = String(index).repeat(size); - yield* Queue.offer(output, value); + for (let index = 0; index < limit; index++) { + const value = eventWith(field, String(index).repeat(size)); + yield* Queue.offer(output, value); + yield* TestClock.adjust(0); + assert.equal(yield* Queue.size(responses), 1); + assert.deepEqual(yield* Queue.take(responses), { + _tag: "Chunk", + requestId: "1", + values: [value], + }); + } + const resumedValue = eventWith(field, "i".repeat(size)); + yield* Queue.offer(output, resumedValue); yield* TestClock.adjust(0); - assert.equal(yield* Queue.size(responses), 1); - assert.deepEqual(yield* Queue.take(responses), { - _tag: "Chunk", - requestId: "1", - values: [value], - }); - } - yield* Queue.offer(output, "i".repeat(size)); - yield* TestClock.adjust(0); - assert.equal(yield* Queue.size(responses), 0); - yield* write(0, { _tag: "Ack", requestId: "1" }); - const resumed = yield* Queue.take(responses); - assert.equal(resumed._tag, "Chunk"); - if (resumed._tag === "Chunk") assert.deepEqual(resumed.values, ["i".repeat(size)]); + assert.equal(yield* Queue.size(responses), 0); + yield* write(0, { _tag: "Ack", requestId: "1" }); + const resumed = yield* Queue.take(responses); + assert.equal(resumed._tag, "Chunk"); + if (resumed._tag === "Chunk") assert.deepEqual(resumed.values, [resumedValue]); - yield* Queue.offer(output, "j"); - yield* TestClock.adjust(0); - assert.equal(yield* Queue.size(responses), 0); - yield* write(0, { _tag: "Interrupt", requestId: "1" }); - assert.equal((yield* Queue.take(responses))._tag, "Exit"); - }).pipe(Effect.provide(RpcSerialization.layerJson), Effect.scoped), + yield* Queue.offer(output, eventWith(field, "j")); + yield* TestClock.adjust(0); + assert.equal(yield* Queue.size(responses), 0); + yield* write(0, { _tag: "Interrupt", requestId: "1" }); + assert.equal((yield* Queue.take(responses))._tag, "Exit"); + }).pipe(Effect.provide(RpcSerialization.layerJson)), ); }); @@ -199,6 +218,9 @@ describe("terminal output stream", () => { Effect.forkScoped, ); const write = yield* Deferred.await(receive); + // Each event arrives after the batching window, so it is sent on its own. + const publish = (event: TerminalAttachStreamEvent) => + TestClock.adjust("2 millis").pipe(Effect.andThen(publisher.publish(event))); const takeChunk = Effect.fnUntraced(function* (queue: typeof stalled) { const response = yield* Queue.take(queue); assert.equal(response._tag, "Chunk"); @@ -215,7 +237,7 @@ describe("terminal output stream", () => { // Eight unacknowledged chunks fill only client 0's output window. for (let index = 0; index < 7; index++) { const event = output("before-stall-" + index); - yield* publisher.publish(event); + yield* publish(event); assert.deepEqual(yield* takeChunk(stalled), [event]); assert.deepEqual(yield* takeChunk(healthy), [event]); yield* write(1, { _tag: "Ack", requestId: "2" }); @@ -223,7 +245,7 @@ describe("terminal output stream", () => { assert.equal(publisher.listeners.size, 2); for (let index = 0; index < 100; index++) { const event = output(index + ":" + "界".repeat(1365)); - yield* publisher.publish(event); + yield* publish(event); assert.deepEqual(yield* takeChunk(healthy), [event]); yield* write(1, { _tag: "Ack", requestId: "2" }); if (index === 31) assert.equal(publisher.listeners.size, 2); @@ -238,11 +260,11 @@ describe("terminal output stream", () => { yield* write(0, { _tag: "Ack", requestId: "1" }); } const after = output("unique output after recovery"); - yield* publisher.publish(after); + yield* publish(after); assert.deepEqual(yield* takeChunk(healthy), [after]); assert.deepEqual(yield* takeChunk(stalled), [after]); assert.equal(yield* Queue.size(stalled), 0); - }).pipe(Effect.provide(RpcSerialization.layerJson), Effect.scoped), + }).pipe(Effect.provide(RpcSerialization.layerJson)), ); it.effect("interrupts recovery while its subscription is still waiting", () => @@ -271,7 +293,7 @@ describe("terminal output stream", () => { yield* Fiber.interrupt(recovery); yield* Deferred.await(interrupted); assert.equal(publisher.listeners.size, 0); - }).pipe(Effect.scoped), + }), ); it.effect("cleans listeners when cancelled during a successful acquisition handoff", () => @@ -305,10 +327,51 @@ describe("terminal output stream", () => { yield* Fiber.join(cancellation); assert.equal(publisher.listeners.size, 0); } + }), + ); + + it.effect("emits idle output immediately and batches a burst until a fixed deadline", () => + Effect.gen(function* () { + const publisher = makePublisher(); + const pull = yield* Stream.toPull(terminalOutputStream(attach(publisher, ""))); + assert.deepEqual(yield* pull, [snapshot("")]); + + yield* TestClock.adjust("10 millis"); + yield* publisher.publish(output("a")); + assert.deepEqual(yield* pull, [output("a")]); + + yield* publisher.publish(output("b")); + const burst = yield* pull.pipe(Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust("1 millis"); + assert.isUndefined(burst.pollUnsafe()); + // Arrivals inside the window never push the deadline back. + yield* publisher.publish(output("c")); + yield* publisher.publish(output("d")); + yield* TestClock.adjust("1 millis"); + assert.deepEqual(yield* Fiber.join(burst), [output("b"), output("c"), output("d")]); }).pipe(Effect.scoped), ); - it.effect("caps each delivered batch at the output window in UTF-8 bytes", () => + it.effect("flushes pending output early, in order, when a lifecycle event arrives", () => + Effect.gen(function* () { + const publisher = makePublisher(); + const pull = yield* Stream.toPull(terminalOutputStream(attach(publisher, ""))); + assert.deepEqual(yield* pull, [snapshot("")]); + + yield* publisher.publish(output("last words")); + const waiting = yield* pull.pipe(Effect.forkChild({ startImmediately: true })); + assert.isUndefined(waiting.pollUnsafe()); + const closed: TerminalAttachStreamEvent = { + type: "closed", + threadId: "thread-1", + terminalId: "default", + }; + yield* publisher.publish(closed); + assert.deepEqual(yield* Fiber.join(waiting), [output("last words"), closed]); + }).pipe(Effect.scoped), + ); + + it.effect("flushes full batches early and caps each at the output window in UTF-8 bytes", () => Effect.gen(function* () { const publisher = makePublisher(); const pull = yield* Stream.toPull(terminalOutputStream(attach(publisher, ""))); @@ -317,8 +380,9 @@ describe("terminal output stream", () => { const data = "界".repeat(1024); for (let index = 0; index < 1024; index++) yield* publisher.publish(output(data)); + // Twenty 3 KiB events fill a 64 KiB batch, so full batches skip the deadline. let delivered = 0; - while (delivered < 1024) { + while (delivered < 1020) { const batch = yield* pull; assert.isAtMost( batch.reduce( @@ -329,8 +393,12 @@ describe("terminal output stream", () => { ); delivered += batch.length; } - assert.equal(delivered, 1024); - }).pipe(Effect.scoped), + assert.equal(delivered, 1020); + const tail = yield* pull.pipe(Effect.forkChild({ startImmediately: true })); + assert.isUndefined(tail.pollUnsafe()); + yield* TestClock.adjust("2 millis"); + assert.lengthOf(yield* Fiber.join(tail), 4); + }), ); it.effect("charges snapshot history but never overflows on one snapshot", () => @@ -348,6 +416,6 @@ describe("terminal output stream", () => { assert.equal(publisher.listeners.size, 1); yield* publisher.publish(snapshot(history)); assert.equal(publisher.listeners.size, 0); - }).pipe(Effect.scoped), + }), ); }); diff --git a/apps/server/src/terminal/OutputProtocol.ts b/apps/server/src/terminal/OutputProtocol.ts index 11339fc2bfa1..84f6e4092341 100644 --- a/apps/server/src/terminal/OutputProtocol.ts +++ b/apps/server/src/terminal/OutputProtocol.ts @@ -6,6 +6,8 @@ import { } from "@t3tools/contracts"; import type * as Arr from "effect/Array"; import * as Cause from "effect/Cause"; +import * as Clock from "effect/Clock"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Latch from "effect/Latch"; import * as Stream from "effect/Stream"; @@ -13,6 +15,8 @@ import type { RpcServer } from "effect/rpc"; const MAX_PENDING_CHUNKS = 8; const MAX_PENDING_BYTES = 64 * 1024; +// Charge an envelope so tiny events cannot bypass the byte budget. +const EVENT_OVERHEAD = 128; const isFull = (sizes: number[]) => sizes.length >= MAX_PENDING_CHUNKS || sizes.reduce((total, size) => total + size, 0) >= MAX_PENDING_BYTES; @@ -29,6 +33,36 @@ const COUNT_WINDOWED_STREAMS: ReadonlySet = new Set([ ORCHESTRATION_V2_WS_METHODS.subscribeThread, ]); +const byteLengthOf = (value: unknown) => (typeof value === "string" ? Buffer.byteLength(value) : 0); + +// Charges every variable-size field a terminal event can carry, so the byte +// budgets below cannot be bypassed by a large payload in an uncharged field. +const queuedSize = (event: TerminalEvent | TerminalAttachStreamEvent) => + EVENT_OVERHEAD + + ("threadId" in event ? byteLengthOf(event.threadId) + byteLengthOf(event.terminalId) : 0) + + (event.type === "output" + ? byteLengthOf(event.data) + : event.type === "error" + ? byteLengthOf(event.message) + : event.type === "activity" + ? byteLengthOf(event.label) + : "snapshot" in event + ? byteLengthOf(event.snapshot.threadId) + + byteLengthOf(event.snapshot.terminalId) + + byteLengthOf(event.snapshot.history) + + byteLengthOf(event.snapshot.cwd) + + byteLengthOf(event.snapshot.worktreePath) + + byteLengthOf(event.snapshot.label) + : 0); + +// Byte-windowed streams carry TerminalAttachStreamEvent, whose encoded form is the +// event itself, so charge it like the output queue does instead of serializing twice. +const chunkSize = (values: ReadonlyArray) => + values.reduce( + (total, value) => total + queuedSize(value as TerminalAttachStreamEvent), + EVENT_OVERHEAD, + ); + interface Window { readonly bytes: boolean; readonly sizes: number[]; @@ -86,10 +120,7 @@ export function withOutputWindow( } else if (response._tag === "Chunk") { const window = windowFor(clientId, String(response.requestId)); if (window) { - window.sizes.push( - // @effect-diagnostics-next-line globalJsonInEffect:off - window.bytes ? Buffer.byteLength(JSON.stringify(response)) : 0, - ); + window.sizes.push(window.bytes ? chunkSize(response.values) : 0); if (!isFull(window.sizes)) { return send.pipe( Effect.andThen(() => @@ -108,8 +139,9 @@ export function withOutputWindow( } const MAX_QUEUED_OUTPUT = 8 * 1024 * 1024; -// Charge an envelope so tiny events cannot bypass the byte budget. -const EVENT_OVERHEAD = 128; +// Output that follows an emission this closely waits and rides along in one batch, +// so a PTY that delivers one line per callback (ConPTY) is not sent one frame per line. +const OUTPUT_WINDOW_NANOS = 2_000_000n; type Listener
    = (event: A) => Effect.Effect; // Acquisition must register cleanup synchronously with installing the listener. @@ -118,14 +150,6 @@ type Subscribe = ( registerCleanup: (stop: () => void) => void, ) => Effect.Effect<() => void, E>; -const queuedSize = (event: TerminalEvent | TerminalAttachStreamEvent) => - EVENT_OVERHEAD + - (event.type === "output" - ? Buffer.byteLength(event.data) - : "snapshot" in event - ? Buffer.byteLength(event.snapshot.history) - : 0); - export function terminalOutputStream( subscribe: Subscribe, resync?: Subscribe, @@ -133,11 +157,15 @@ export function terminalOutputStream { return Stream.unwrap( Effect.gen(function* () { + const clock = yield* Clock.Clock; const ready = yield* Latch.make(); let pending: Array = []; let sizes: number[] = []; let head = 0; let queued = 0; + // Pending events other than output; any of them flushes the window early. + let lifecycle = 0; + let deadline = 0n; let overflowed = false; let subscription = 0; let unsubscribe: (() => void) | null = null; @@ -149,7 +177,9 @@ export function terminalOutputStream lifecycle > 0 || queued >= MAX_PENDING_BYTES; yield* Effect.addFinalizer(() => Effect.sync(release)); const listen = (from: Subscribe) => @@ -159,16 +189,19 @@ export function terminalOutputStream Effect.sync(() => { if (id !== subscription) return; + const wasEmpty = head === pending.length; const size = queuedSize(event); pending.push(event); sizes.push(size); queued += size; + if (event.type !== "output") lifecycle += 1; // A single snapshot must fit even when history fills the budget. if (queued - sizes[head]! > maxQueued) { release(); overflowed = true; } - ready.openUnsafe(); + // The current or next pull handles the pending output deadline. + if (wasEmpty || overflowed || urgent()) ready.openUnsafe(); }), (stop) => { // Overflow or cancellation may invalidate acquisition before registration. @@ -185,6 +218,7 @@ export function terminalOutputStream => { @@ -209,9 +243,16 @@ export function terminalOutputStream= deadline || urgent()) { + deadline = now + OUTPUT_WINDOW_NANOS; + return Effect.succeed(takeBatch()); + } + return Effect.raceFirst(ready.await, Effect.sleep(Duration.nanos(deadline - now))).pipe( + Effect.andThen(pull), + ); }, ); diff --git a/apps/server/src/terminal/nodePtyFlushDataAndCleanUp.test.ts b/apps/server/src/terminal/nodePtyFlushDataAndCleanUp.test.ts new file mode 100644 index 000000000000..7dc0861c7f53 --- /dev/null +++ b/apps/server/src/terminal/nodePtyFlushDataAndCleanUp.test.ts @@ -0,0 +1,131 @@ +import * as NodeModule from "node:module"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; + +const requireCjs = NodeModule.createRequire(import.meta.url); +const { + WindowsPtyAgent, +}: { + WindowsPtyAgent: { + prototype: { + _flushDataAndCleanUp(this: FakeAgent): void; + _cleanUpProcess(this: FakeAgent): void; + }; + }; +} = requireCjs("node-pty/lib/windowsPtyAgent.js"); + +interface FakeAgent { + _useConptyDll: boolean; + _closeTimeout?: unknown; + _inSocket: { readable: boolean }; + _outSocket: { readable: boolean; destroy: () => void }; + _cleanUpProcess: () => void; +} + +// The sockets are stand-ins; the timer logic and _cleanUpProcess are node-pty's own (as patched by T3). +const makeAgent = (): FakeAgent => { + const agent: FakeAgent = { + _useConptyDll: false, + _inSocket: { readable: true }, + _outSocket: { readable: true, destroy: vi.fn() }, + _cleanUpProcess: () => WindowsPtyAgent.prototype._cleanUpProcess.call(agent), + }; + return agent; +}; + +const flush = (agent: FakeAgent) => WindowsPtyAgent.prototype._flushDataAndCleanUp.call(agent); + +// A minimal event loop that mirrors Node's phases: expired timers run first, then immediates (the check phase). +// node-pty calls the global timer functions at call time, so replacing them lets the test place a "poll phase" +// (delivery of already-buffered pipe data) exactly between the two. +interface Timer { + readonly id: number; + readonly dueAt: number; + readonly run: () => void; +} +let now = 0; +let nextId = 0; +let timers: Timer[] = []; +let immediates: Array<() => void> = []; + +const runExpiredTimers = () => { + for (const timer of timers.filter((t) => t.dueAt <= now).toSorted((a, b) => a.dueAt - b.dueAt)) { + timers = timers.filter((t) => t.id !== timer.id); + timer.run(); + } +}; +const runImmediates = () => { + const batch = immediates; + immediates = []; + for (const run of batch) run(); +}; +/** One loop iteration after `ms` of wall time; `poll` runs between the timers and check phases. */ +const loopIteration = (ms: number, poll?: () => void) => { + now += ms; + runExpiredTimers(); + poll?.(); + runImmediates(); +}; + +// ConPTY never closes the output pipe after the shell exits, so node-pty ends the session on a timer. +// T3 shortens it (patches/node-pty@1.2.0-beta.15.patch) so the terminal reports "exited" about 0.9 s sooner. +describe("node-pty Windows exit flush", () => { + beforeEach(() => { + now = 0; + nextId = 0; + timers = []; + immediates = []; + vi.stubGlobal("setTimeout", (run: () => void, ms: number) => { + const id = ++nextId; + timers.push({ id, dueAt: now + ms, run }); + return id; + }); + vi.stubGlobal("clearTimeout", (id: number) => { + timers = timers.filter((t) => t.id !== id); + }); + vi.stubGlobal("setImmediate", (run: () => void) => { + immediates.push(run); + return ++nextId; + }); + }); + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("closes the output socket 100 ms after the exit, not a second later", () => { + const agent = makeAgent(); + flush(agent); + + loopIteration(99); + expect(agent._outSocket.destroy).not.toHaveBeenCalled(); + + loopIteration(1); + expect(agent._outSocket.destroy).toHaveBeenCalledTimes(1); + expect(agent._outSocket.readable).toBe(false); + expect(agent._inSocket.readable).toBe(false); + }); + + // Timers run before the poll phase. After a main-thread stall longer than the window, output that is already + // buffered in the pipe is read only after the timer has expired, so the cleanup has to wait one loop turn for it. + it("keeps the socket open when buffered output is read right after a stalled timer expires", () => { + const agent = makeAgent(); + flush(agent); + + loopIteration(150, () => flush(agent)); + expect(agent._outSocket.destroy).not.toHaveBeenCalled(); + + loopIteration(99); + expect(agent._outSocket.destroy).not.toHaveBeenCalled(); + + loopIteration(1); + expect(agent._outSocket.destroy).toHaveBeenCalledTimes(1); + }); + + it("closes after a stall when no output was buffered", () => { + const agent = makeAgent(); + flush(agent); + + loopIteration(150); + expect(agent._outSocket.destroy).toHaveBeenCalledTimes(1); + }); +}); diff --git a/apps/server/src/testUtils/fakeWsl.ts b/apps/server/src/testUtils/fakeWsl.ts new file mode 100644 index 000000000000..d8f88fcde8db --- /dev/null +++ b/apps/server/src/testUtils/fakeWsl.ts @@ -0,0 +1,37 @@ +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; + +// `wslpath` honours the distro's configurable automount root, so the fake maps drives under a temp dir. +export const makeFakeWslHost = Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const mountRoot = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-wsl-mnt-" }); + const binDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-wsl-bin-" }); + const wslpath = path.join(binDir, "wslpath"); + yield* fileSystem + .writeFileString( + wslpath, + [ + "#!/bin/sh", + '[ "$1" = "-u" ] || exit 1', + "p=$(printf '%s' \"$2\" | tr '\\\\' '/')", + "drive=$(printf '%s' \"$p\" | cut -c1 | tr 'A-Z' 'a-z')", + `printf '%s/%s%s\\n' '${mountRoot}' "$drive" "\${p#?:}"`, + "", + ].join("\n"), + ) + .pipe(Effect.orDie); + yield* fileSystem.chmod(wslpath, 0o755).pipe(Effect.orDie); + const environment = { + ...HostProcess.Environment.defaultValue(), + PATH: `${binDir}:${HostProcess.Environment.defaultValue().PATH ?? ""}`, + }; + const underWsl = (effect: Effect.Effect) => + effect.pipe( + Effect.provideService(HostProcess.Platform, "linux"), + Effect.provideService(HostProcess.Environment, environment), + ); + return { mountRoot, underWsl }; +}); diff --git a/apps/server/src/textGeneration/AntigravityTextGeneration.test.ts b/apps/server/src/textGeneration/AntigravityTextGeneration.test.ts index 5cb6a1f8d4d8..5a410dbf342f 100644 --- a/apps/server/src/textGeneration/AntigravityTextGeneration.test.ts +++ b/apps/server/src/textGeneration/AntigravityTextGeneration.test.ts @@ -343,7 +343,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { { type: "text", text: expect.stringContaining("+handleRemoteCallback()") }, ]); yield* fixture.assertCleaned; - }).pipe(Effect.scoped), + }), ); it.effect.each(["tool_call", "tool_call_update"] as const)( @@ -362,7 +362,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { expect(error.detail).toContain("tool work"); expect(fixture.state.cancellations).toBe(1); yield* fixture.assertCleaned; - }).pipe(Effect.scoped), + }), ); it.effect("rejects native permission and choice requests instead of answering them", () => @@ -387,7 +387,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { expect(reply).toEqual({ outcome: { outcome: "cancelled" } }); expect(error.detail).toContain("permission or user input"); yield* fixture.assertCleaned; - }).pipe(Effect.scoped), + }), ); it.effect.each(["file", "terminal", "extension"] as const)( @@ -416,7 +416,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { const error = yield* Fiber.join(child).pipe(Effect.flip); expect(error.detail).toContain("tool or user input"); yield* fixture.assertCleaned; - }).pipe(Effect.scoped), + }), ); it.effect.each([ @@ -432,7 +432,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { .pipe(Effect.flip); expect(error.detail).toContain(detail); yield* fixture.assertCleaned; - }).pipe(Effect.scoped), + }), ); it.effect("does not return partial JSON after native cancellation", () => @@ -449,7 +449,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { .pipe(Effect.flip); expect(error.detail).toContain("cancelled"); yield* fixture.assertCleaned; - }).pipe(Effect.scoped), + }), ); it.effect.each([ @@ -470,7 +470,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { expect(error.detail).toBe(detail); expect(error.detail).not.toContain("Raw provider failure text."); yield* fixture.assertCleaned; - }).pipe(Effect.scoped), + }), ); it.effect.each([ @@ -517,7 +517,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { expect(error.detail).toContain("wrote files"); if (target !== undefined) expect(yield* fs.exists(target)).toBe(true); yield* fixture.assertCleaned; - }).pipe(Effect.scoped), + }), ); it.effect("uses the native default without sending T3's default selection as a model ID", () => @@ -530,7 +530,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { expect(result).toEqual({ title: "Repair login" }); expect(fixture.state.selectedModels).toEqual([]); yield* fixture.assertCleaned; - }).pipe(Effect.scoped), + }), ); it.effect.each([ @@ -549,6 +549,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { { name: "hooks.json", value: "{}", available: true }, { name: "hooks.json", value: '{"hooks":{}}', available: true }, { name: "mcp_config.json", value: '{"mcpServers":{}}', available: true }, + { name: "mcp_config.json", value: "", available: true }, ])("checks $name before starting a helper, available=$available", ({ name, value, available }) => Effect.gen(function* () { const fixture = yield* makeFixture(); @@ -562,7 +563,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { expect(Exit.isSuccess(result)).toBe(available); expect(fixture.state.workspaces.length).toBe(available ? 1 : 0); yield* fixture.assertCleaned; - }).pipe(Effect.scoped), + }), ); it.effect("does not launch during sign-out or before sign-in", () => @@ -575,7 +576,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { expect(error.detail).toBe("Sign in before starting Antigravity."); expect(fixture.state.workspaces).toEqual([]); yield* fixture.assertCleaned; - }).pipe(Effect.scoped), + }), ); it.effect.each(["caller", "sign-out"] as const)( @@ -596,7 +597,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { } expect(fixture.state.cancellations).toBe(1); yield* fixture.assertCleaned; - }).pipe(Effect.scoped), + }), ); it.effect("refuses unsafe native session IDs without deleting other files", () => @@ -609,7 +610,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { expect(fixture.state.prompts).toEqual([]); expect(yield* fixture.fs.readFileString(`${fixture.sessionBase}.db`)).toBe("helper session"); expect(fixture.state.closed).toEqual(fixture.state.workspaces); - }).pipe(Effect.scoped), + }), ); it.effect("keeps native sessions whose metadata belongs to a different workspace", () => @@ -627,7 +628,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { "keep this session", ); expect(yield* fixture.fs.readFileString(`${fixture.sessionBase}.meta`)).toBe(metadata); - }).pipe(Effect.scoped), + }), ); }); @@ -640,5 +641,5 @@ it.effect("retains filesystem services supplied only during construction", () => title: "Repair login", }); yield* fixture.assertCleaned; - }).pipe(Effect.scoped), + }), ); diff --git a/apps/server/src/textGeneration/AntigravityTextGeneration.ts b/apps/server/src/textGeneration/AntigravityTextGeneration.ts index b7e7cfaeb40e..b67bad85faf0 100644 --- a/apps/server/src/textGeneration/AntigravityTextGeneration.ts +++ b/apps/server/src/textGeneration/AntigravityTextGeneration.ts @@ -79,14 +79,20 @@ export const isAntigravityTextGenerationAvailable = Effect.fn( if (info.type !== "File" || info.size > 64_000n) { return false; } + // CLI 1.5.8 writes a 0-byte mcp_config.json before any server exists. const empty = yield* fs.readFileString(configurationPath).pipe( - Effect.flatMap(decodeConfiguration), - Effect.flatMap((configuration) => - decodeConfigurationObject( - configuration[name === "hooks.json" ? "hooks" : "mcpServers"] ?? configuration, - ), + Effect.flatMap((text) => + text.trim().length === 0 + ? Effect.succeed(true) + : decodeConfiguration(text).pipe( + Effect.flatMap((configuration) => + decodeConfigurationObject( + configuration[name === "hooks.json" ? "hooks" : "mcpServers"] ?? configuration, + ), + ), + Effect.map((configuration) => Object.keys(configuration).length === 0), + ), ), - Effect.map((configuration) => Object.keys(configuration).length === 0), Effect.orElseSucceed(() => false), ); if (!empty) return false; diff --git a/apps/server/src/textGeneration/ClaudeTextGeneration.test.ts b/apps/server/src/textGeneration/ClaudeTextGeneration.test.ts index 393b9ef632d6..cb5b6a90f114 100644 --- a/apps/server/src/textGeneration/ClaudeTextGeneration.test.ts +++ b/apps/server/src/textGeneration/ClaudeTextGeneration.test.ts @@ -234,6 +234,35 @@ it.layer(layerClaudeTextGenerationTest)("ClaudeTextGeneration", (it) => { }), ); + it.effect("passes --settings from the instance launch args to Claude text generation", () => + withFakeClaudeEnv( + { + output: JSON.stringify({ + structured_output: { + title: "Use instance settings", + }, + }), + argsMustContain: `"env":{"ANTHROPIC_VERTEX_PROJECT_ID":"vertex-project-a"}`, + claudeConfig: { + launchArgs: `--settings '{"env":{"ANTHROPIC_VERTEX_PROJECT_ID":"vertex-project-a"}}'`, + }, + }, + (textGeneration) => + Effect.gen(function* () { + const generated = yield* textGeneration.generateThreadTitle({ + cwd: process.cwd(), + message: "thread title", + modelSelection: { + instanceId: ProviderInstanceId.make("claudeAgent"), + model: SYNTHETIC_CLAUDE_STANDARD_MODEL, + }, + }); + + expect(generated.title).toBe(sanitizeThreadTitle("Use instance settings")); + }), + ), + ); + it.effect("retries without --no-session-persistence when the Claude CLI rejects it", () => withFakeClaudeEnv( { diff --git a/apps/server/src/textGeneration/ClaudeTextGeneration.ts b/apps/server/src/textGeneration/ClaudeTextGeneration.ts index 34e7415b3c9f..a523a3873a2c 100644 --- a/apps/server/src/textGeneration/ClaudeTextGeneration.ts +++ b/apps/server/src/textGeneration/ClaudeTextGeneration.ts @@ -1,7 +1,9 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Option from "effect/Option"; +import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; @@ -32,6 +34,7 @@ import { hasClaudeManagedMcpConfig, makeClaudeEnvironment, } from "../provider/Drivers/ClaudeHome.ts"; +import { readClaudeLaunchArgSettings } from "../orchestration-v2/Adapters/ClaudeAdapterV2.ts"; const CLAUDE_TIMEOUT_MS = 180_000; /** `--tools` takes bare names; patterns go in the allow list. */ @@ -97,9 +100,14 @@ export const makeClaudeTextGeneration = Effect.fn("makeClaudeTextGeneration")(fu claudeSettings: ClaudeSettings, environment?: NodeJS.ProcessEnv, modelCatalog: Effect.Effect = Effect.succeed(BUNDLED_CLAUDE_MODEL_CATALOG), + routeModelCatalog: ( + catalog: ClaudeModelCatalog, + cwd: string, + ) => Effect.Effect = Effect.succeed, ) { const commandSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; const claudeEnvironment = yield* makeClaudeEnvironment(claudeSettings, environment); const scopedModelCatalog = modelCatalog.pipe( Effect.map((catalog) => scopeClaudeModelCatalog(catalog, claudeSettings.customModels)), @@ -146,9 +154,19 @@ export const makeClaudeTextGeneration = Effect.fn("makeClaudeTextGeneration")(fu }: TextGenerationOperations.Request): Effect.fn.Return< S["Type"], TextGenerationError, - S["DecodingServices"] + S["DecodingServices"] | Scope.Scope > { - const catalog = yield* scopedModelCatalog; + const workingDirectory = + operation === "generateThreadTitle" + ? yield* fileSystem + .makeTempDirectoryScoped({ prefix: "t3code-claude-title-" }) + .pipe( + Effect.mapError((cause) => + normalizeCliError("claude", operation, cause, "Failed to create title directory"), + ), + ) + : cwd; + const catalog = yield* routeModelCatalog(yield* scopedModelCatalog, workingDirectory); const resolvedModelSelection = { ...modelSelection, model: resolveClaudeModelSlug(catalog, modelSelection.model), @@ -185,7 +203,18 @@ export const makeClaudeTextGeneration = Effect.fn("makeClaudeTextGeneration")(fu : undefined; const fastMode = fastModeDescriptor?.type === "boolean" ? fastModeDescriptor.currentValue : undefined; + const launchArgSettings = yield* readClaudeLaunchArgSettings({ + settings: claudeSettings, + cwd, + fileSystem, + path, + }).pipe( + Effect.mapError( + (cause) => new TextGenerationError({ operation, detail: cause.message, cause }), + ), + ); const settings = { + ...launchArgSettings, disableAllHooks: true, ...(typeof thinking === "boolean" ? { alwaysThinkingEnabled: thinking } : {}), ...(fastMode ? { fastMode: true } : {}), @@ -200,16 +229,6 @@ export const makeClaudeTextGeneration = Effect.fn("makeClaudeTextGeneration")(fu const runClaudeCommand = Effect.fn("runClaudeJson.runClaudeCommand")(function* ( noSessionPersistence: boolean, ) { - const workingDirectory = - operation === "generateThreadTitle" - ? yield* fileSystem - .makeTempDirectoryScoped({ prefix: "t3code-claude-title-" }) - .pipe( - Effect.mapError((cause) => - normalizeCliError("claude", operation, cause, "Failed to create title directory"), - ), - ) - : cwd; const managedMcpConfig = yield* hasClaudeManagedMcpConfig(); const spawnCommand = yield* resolveSpawnCommand( claudeSettings.binaryPath || "claude", @@ -344,5 +363,9 @@ export const makeClaudeTextGeneration = Effect.fn("makeClaudeTextGeneration")(fu ); }); - return TextGenerationOperations.fromRunner("ClaudeTextGeneration", runClaudeJson); + return TextGenerationOperations.fromRunner( + "ClaudeTextGeneration", + (request: TextGenerationOperations.Request) => + Effect.scoped(runClaudeJson(request)), + ); }); diff --git a/apps/server/src/textGeneration/CodexTextGeneration.test.ts b/apps/server/src/textGeneration/CodexTextGeneration.test.ts index 9e16eaeb205b..58dda4eb348c 100644 --- a/apps/server/src/textGeneration/CodexTextGeneration.test.ts +++ b/apps/server/src/textGeneration/CodexTextGeneration.test.ts @@ -1,5 +1,4 @@ import { - DEFAULT_TEXT_GENERATION_MODEL_BY_PROVIDER, ProjectId, ProviderDriverKind, type ModelSelection, @@ -59,6 +58,7 @@ interface FakeCodexInput { mcpServers?: ReadonlyArray<{ name: string; enabled: boolean }> | number; requiredCwd?: string; requireInstructions?: string; + requireMcpServersOff?: ReadonlyArray; } function makeFakeCodexBinary(dir: string, input: FakeCodexInput) { @@ -74,6 +74,7 @@ function makeFakeCodexBinary(dir: string, input: FakeCodexInput) { mcpServers: input.mcpServers ?? [], requiredCwd: input.requiredCwd ?? null, requireInstructions: input.requireInstructions ?? null, + requireMcpServersOff: input.requireMcpServersOff ?? [], stderr: input.stderr ?? null, output: input.output, exitCode: input.exitCode ?? 0, @@ -130,6 +131,33 @@ function makeFakeCodexBinary(dir: string, input: FakeCodexInput) { ' process.stderr.write(message + "\\n");', " process.exit(code);", "}", + "// Like Codex: a later mcp_servers={...} override replaces servers earlier -c flags defined.", + "const servers = new Map();", + "const offServers = new Set();", + "for (let index = 0; index < args.length; index += 1) {", + ' if (args[index] !== "-c" && args[index] !== "--config") continue;', + ' const value = args[index + 1] ?? "";', + ' if (value.startsWith("mcp_servers={")) {', + ' const named = [...value.matchAll(/"([^"]+)"=\\{enabled=false\\}/g)].map((match) => match[1]);', + " for (const name of named) offServers.add(name);", + " for (const name of [...servers.keys()]) {", + " if (named.includes(name)) servers.set(name, false);", + " else servers.delete(name);", + " }", + ' } else if (value.startsWith("mcp_servers.")) {', + ' const [name, key] = value.slice("mcp_servers.".length).split(/[.=]/);', + " if (value === `mcp_servers.${name}.enabled=false`) offServers.add(name);", + " // An enabled flag alone toggles a server another layer (config.toml) defines.", + ' if (key === "enabled" && !servers.has(name)) continue;', + ' servers.set(name, servers.get(name) === true || key === "url" || key === "command");', + " }", + "}", + "for (const [name, hasTransport] of servers) {", + ' if (!hasTransport) fail("Error loading config.toml: invalid transport in `mcp_servers." + name + "`", 12);', + "}", + "for (const name of check.requireMcpServersOff) {", + ' if (!offServers.has(name)) fail("MCP server left on: " + name, 13);', + "}", 'if (check.requiredCwd !== null && process.cwd() !== check.requiredCwd) fail("cannot inspect checkout", 10);', 'if (check.requireInstructions !== null && !instructions.includes(check.requireInstructions)) fail("missing writer task", 11);', "if (check.requireArg !== null && !originalArgs.includes(` ${check.requireArg} `)) {", @@ -366,7 +394,7 @@ it.layer(layerCodexTextGenerationTest)("CodexTextGeneration", (it) => { { name: "already-off", enabled: false }, ], requireArg: - '--config features.plugins=false --config features.apps=false --config mcp_servers={"augmenter"={enabled=false},"my.server"={enabled=false}}', + 'exec --config mcp_servers={"augmenter"={enabled=false},"my.server"={enabled=false}} --config developer_instructions="Use', }, (textGeneration) => Effect.gen(function* () { @@ -385,6 +413,52 @@ it.layer(layerCodexTextGenerationTest)("CodexTextGeneration", (it) => { ), ); + it.effect("turns off an MCP server that launch args define without dropping its transport", () => + withFakeCodexEnv( + { + output: JSON.stringify({ title: "Remove default shapes" }), + launchArgs: `-c 'mcp_servers.t3-code.url="http://127.0.0.1:1/mcp"' -c 'mcp_servers.t3-code.enabled=true'`, + mcpServers: [{ name: "t3-code", enabled: true }], + requireMcpServersOff: ["t3-code"], + requireArg: + "--config features.apps=false --config mcp_servers.t3-code.enabled=false --ephemeral", + }, + (textGeneration) => + Effect.gen(function* () { + const result = yield* textGeneration.generateThreadTitle({ + cwd: process.cwd(), + message: "Remove default shapes from characters", + modelSelection: DEFAULT_TEST_MODEL_SELECTION, + }); + expect(result.title).toBe("Remove default shapes"); + }), + ), + ); + + it.effect("keeps MCP servers off when launch args set their own mcp_servers table", () => + withFakeCodexEnv( + { + output: JSON.stringify({ title: "Remove default shapes" }), + launchArgs: `-c 'mcp_servers={"my.server"={command="echo",enabled=true}}'`, + mcpServers: [ + { name: "my.server", enabled: true }, + { name: "my server", enabled: true }, + ], + requireArg: + '--config features.apps=false --config mcp_servers={"my.server"={enabled=false},"my server"={enabled=false}} --config mcp_servers.my server.enabled=false --ephemeral', + }, + (textGeneration) => + Effect.gen(function* () { + const result = yield* textGeneration.generateThreadTitle({ + cwd: process.cwd(), + message: "Remove default shapes from characters", + modelSelection: DEFAULT_TEST_MODEL_SELECTION, + }); + expect(result.title).toBe("Remove default shapes"); + }), + ), + ); + it.effect("still generates when Codex cannot list its MCP servers", () => withFakeCodexEnv( { @@ -425,6 +499,26 @@ it.layer(layerCodexTextGenerationTest)("CodexTextGeneration", (it) => { ), ); + it.effect("keeps user hooks off over launch args that turn them on", () => + withFakeCodexEnv( + { + output: JSON.stringify({ title: "Hookless title" }), + launchArgs: "--config features.hooks=true --enable hooks", + requireArg: "--config features.codex_hooks=false --config features.hooks=false", + forbidArg: "--enable", + }, + (textGeneration) => + Effect.gen(function* () { + const result = yield* textGeneration.generateThreadTitle({ + cwd: process.cwd(), + message: "Describe this change", + modelSelection: DEFAULT_TEST_MODEL_SELECTION, + }); + expect(result.title).toBe("Hookless title"); + }), + ), + ); + it.effect("passes exec-safe launch args into codex exec", () => withFakeCodexEnv( { @@ -547,6 +641,8 @@ const PREFLIGHT_PRIMARY_STATES = [ "not installed, fallback disabled for project", "signed out", "signed out, fallback failed", + "cannot generate text", + "model cannot generate text", ] as const; it.effect.each(PREFLIGHT_PRIMARY_STATES)("text generation preflight: %s", (primaryState) => @@ -559,6 +655,11 @@ it.effect.each(PREFLIGHT_PRIMARY_STATES)("text generation preflight: %s", (prima model: "chat-model", options: [{ id: "effort", value: "high" }], }; + const threadDefault = { + instanceId: fallback, + model: "z-ai/glm-5.3-flash", + options: [{ id: "thinking", value: false }], + }; const calls: ModelSelection[] = []; const instances: ProviderInstance[] = [primary, fallback].map((instanceId) => { const driver = ProviderDriverKind.make(instanceId === primary ? "codex" : "claudeAgent"); @@ -586,6 +687,22 @@ it.effect.each(PREFLIGHT_PRIMARY_STATES)("text generation preflight: %s", (prima ...(primaryState === "signed out, fallback failed" && instanceId === fallback ? { status: "error", auth: { status: "unknown" } } : {}), + ...(primaryState === "cannot generate text" && instanceId === primary + ? { supportsTextGeneration: false } + : {}), + ...(primaryState === "model cannot generate text" && instanceId === primary + ? { + models: [ + { + slug: selection.model, + name: selection.model, + isCustom: false, + supportsTextGeneration: false, + capabilities: null, + }, + ], + } + : {}), ...(primaryState === "exhausted" && instanceId === primary ? { usageLimits: { @@ -627,8 +744,12 @@ it.effect.each(PREFLIGHT_PRIMARY_STATES)("text generation preflight: %s", (prima }), ServerSettings.layerTest({ providerInstances: { - [fallback]: { driver: ProviderDriverKind.make("claudeAgent"), config: {} }, + [fallback]: { + driver: ProviderDriverKind.make("claudeAgent"), + config: { customModels: [threadDefault.model] }, + }, }, + defaultModelSelection: threadDefault, ...(primaryState === "not installed, fallback disabled for project" ? { projectSettingsOverrides: { @@ -660,11 +781,7 @@ it.effect.each(PREFLIGHT_PRIMARY_STATES)("text generation preflight: %s", (prima primaryState === "not installed, fallback disabled for project" || primaryState === "signed out, fallback failed" ? selection - : { - instanceId: fallback, - model: - DEFAULT_TEXT_GENERATION_MODEL_BY_PROVIDER[ProviderDriverKind.make("claudeAgent")], - }, + : threadDefault, ]); }), ); diff --git a/apps/server/src/textGeneration/CodexTextGeneration.ts b/apps/server/src/textGeneration/CodexTextGeneration.ts index 4de5392502c8..fc741aaaeb26 100644 --- a/apps/server/src/textGeneration/CodexTextGeneration.ts +++ b/apps/server/src/textGeneration/CodexTextGeneration.ts @@ -55,6 +55,11 @@ const codexMcpServersOff = (names: ReadonlyArray): string | null => names.length === 0 ? null : `mcp_servers={${names.map((name) => `${JSON.stringify(name)}={enabled=false}`).join(",")}}`; +// Codex splits -c keys at the first "=" and on every dot, so only those names take a dotted flag. +const codexMcpServerOffFlags = (names: ReadonlyArray) => + names + .filter((name) => !name.includes(".") && !name.includes("=")) + .flatMap((name) => ["--config", `mcp_servers.${name}.enabled=false`]); const encodeJsonString = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); export const makeCodexTextGeneration = Effect.fn("makeCodexTextGeneration")(function* ( codexConfig: CodexSettings, @@ -291,20 +296,28 @@ export const makeCodexTextGeneration = Effect.fn("makeCodexTextGeneration")(func } : {}), }; - const mcpServersOff = yield* listEnabledMcpServers({ + const enabledMcpServers = yield* listEnabledMcpServers({ binary, cwd, env, launchArgs: codexMcpListLaunchArgs(launchArgs), - }).pipe(Effect.map(codexMcpServersOff)); + }); + const mcpServersOff = codexMcpServersOff(enabledMcpServers); + const execLaunchArgs = codexExecLaunchArgs(launchArgs); + // A launch-arg mcp_servers={...} table replaces ours and can re-enable dotted names. + const launchArgsSetMcpTable = execLaunchArgs.some((arg) => /\bmcp_servers\s*=/.test(arg)); const spawnCommand = yield* resolveSpawnCommand( binary, [ "exec", - ...codexExecLaunchArgs(launchArgs), + // Before launch args: a later table would replace servers their -c flags define. + ...(mcpServersOff ? ["--config", mcpServersOff] : []), + ...execLaunchArgs, // A user's MCP server could act on the text-generation prompt. ...CODEX_NO_PLUGIN_ARGS, - ...(mcpServersOff ? ["--config", mcpServersOff] : []), + ...(mcpServersOff && launchArgsSetMcpTable ? ["--config", mcpServersOff] : []), + // After launch args, so a launch arg's enabled=true cannot turn a server back on. + ...codexMcpServerOffFlags(enabledMcpServers), "--ephemeral", "--skip-git-repo-check", "-s", @@ -313,6 +326,11 @@ export const makeCodexTextGeneration = Effect.fn("makeCodexTextGeneration")(func model, "--config", `model_reasoning_effort="${reasoningEffort}"`, + // Current Codex gives hooks precedence over the historical codex_hooks alias. + "--config", + "features.codex_hooks=false", + "--config", + "features.hooks=false", ...(serviceTier ? ["--config", `service_tier="${serviceTier}"`] : []), "--output-schema", schemaPath, diff --git a/apps/server/src/textGeneration/GrokTextGeneration.test.ts b/apps/server/src/textGeneration/GrokTextGeneration.test.ts index 819b530561cb..542fe0e4a2b5 100644 --- a/apps/server/src/textGeneration/GrokTextGeneration.test.ts +++ b/apps/server/src/textGeneration/GrokTextGeneration.test.ts @@ -35,7 +35,7 @@ function makeAcpGrokWrapper(dir: string, env: Record): string { env, source: execScriptSource({ scriptPath: mockAgentPath, - expectedArgs: ["agent", "stdio"], + expectedArgs: ["--permission-mode", "default", "agent", "stdio"], }), }); } @@ -58,14 +58,19 @@ function withFakeAcpGrok( }).pipe(Effect.scoped); } -function readJsonRpcRequests( - filePath: string, -): ReadonlyArray<{ readonly method?: string; readonly params?: Record }> { +function readJsonRpcRequests(filePath: string): ReadonlyArray<{ + readonly id?: unknown; + readonly method?: string; + readonly params?: Record; +}> { return NodeFS.readFileSync(filePath, "utf8") .trim() .split("\n") .filter((line) => line.length > 0) - .map((line) => JSON.parse(line) as { method?: string; params?: Record }); + .map( + (line) => + JSON.parse(line) as { id?: unknown; method?: string; params?: Record }, + ); } it.layer(layerGrokTextGenerationTest)("GrokTextGeneration", (it) => { @@ -78,6 +83,7 @@ it.layer(layerGrokTextGenerationTest)("GrokTextGeneration", (it) => { return withFakeAcpGrok( { T3_ACP_REQUEST_LOG_PATH: requestLogPath, + T3_ACP_REQUEST_PERMISSION: "1", T3_ACP_PROMPT_RESPONSE_TEXT: JSON.stringify({ subject: "Add Grok provider", body: "Wire up the ACP runtime and headless text generation path.", @@ -110,6 +116,9 @@ it.layer(layerGrokTextGenerationTest)("GrokTextGeneration", (it) => { request.params?.modelId === "grok-mock-alt", ), ).toBe(true); + expect(requests.find((request) => request.id === 9001)).toMatchObject({ + result: { outcome: { outcome: "cancelled" } }, + }); }), ); }); diff --git a/apps/server/src/textGeneration/OpenCode2TextGeneration.test.ts b/apps/server/src/textGeneration/OpenCode2TextGeneration.test.ts index 869cd1b4fb5e..6cb1bfad71be 100644 --- a/apps/server/src/textGeneration/OpenCode2TextGeneration.test.ts +++ b/apps/server/src/textGeneration/OpenCode2TextGeneration.test.ts @@ -70,7 +70,7 @@ it.layer(layer)("OpenCode2TextGeneration", (it) => { files: [{ uri: NodeURL.pathToFileURL(screenshotPath).href, name: "screen.png" }], }); } - }).pipe(Effect.scoped), + }), ); it.effect("keeps a provider's own error message out of the caller-visible detail", () => @@ -153,7 +153,7 @@ it.layer(layer)("OpenCode2TextGeneration", (it) => { .pipe(Effect.flip); assert.equal(failure.detail, "OpenCode could not generate the text."); assert.deepEqual(failure.cause, refusal); - }).pipe(Effect.scoped), + }), ); it.effect("times out when the prompt request and its interrupt never answer", () => @@ -219,6 +219,6 @@ it.layer(layer)("OpenCode2TextGeneration", (it) => { yield* TestClock.adjust("5 seconds"); const failure = yield* Fiber.join(generation); assert.equal(failure.detail, "OpenCode did not finish generating in time."); - }).pipe(Effect.scoped), + }), ); }); diff --git a/apps/server/src/textGeneration/TextGeneration.ts b/apps/server/src/textGeneration/TextGeneration.ts index 3042c97191cb..818afb4fae55 100644 --- a/apps/server/src/textGeneration/TextGeneration.ts +++ b/apps/server/src/textGeneration/TextGeneration.ts @@ -9,10 +9,7 @@ import { } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; import { exhaustedUsageWindow } from "@t3tools/shared/usageLimits"; -import { - defaultTextGenerationModel, - fallbackTextGenerationModelSelection, -} from "@t3tools/shared/serverSettings"; +import { fallbackTextGenerationModelSelection } from "@t3tools/shared/serverSettings"; import * as ServerSettings from "../serverSettings.ts"; import type { ProviderTextGeneration } from "@t3tools/provider-core/server/textGeneration"; import { TextGenerationError } from "@t3tools/contracts"; @@ -111,8 +108,11 @@ export const make = Effect.gen(function* () { const unusable = textGeneratorCannotRun || (selected?.enabled === true && - selected.status === "error" && - (!selected.installed || selected.auth.status === "unauthenticated")); + (!isProviderTextGenerationCapable(selected) || + selected.models.find((model) => model.slug === selection.model) + ?.supportsTextGeneration === false || + (selected.status === "error" && + (!selected.installed || selected.auth.status === "unauthenticated")))); if (unusable || exhaustedUsageWindow(selected, now, selection.model)) { const settings = yield* serverSettings.getSettings.pipe( Effect.mapError( @@ -125,21 +125,25 @@ export const make = Effect.gen(function* () { ), ); selection = - fallbackTextGenerationModelSelection(settings.providerInstances, (instanceId, driver) => { - const provider = snapshots.find((candidate) => candidate.instanceId === instanceId); - return ( - instanceId !== selection.instanceId && - provider?.enabled === true && - resolveProjectProviderInstanceEnabled(settings, projectId ?? null, instanceId) && - provider.installed && - provider.auth.status !== "unauthenticated" && - provider.status !== "error" && - isProviderAvailable(provider) && - isProviderTextGenerationCapable(provider) && - (operation !== "generateReviewGuide" || INVESTIGATING_DRIVERS.has(driver)) && - !exhaustedUsageWindow(provider, now, defaultTextGenerationModel(driver)) - ); - }) ?? selection; + fallbackTextGenerationModelSelection( + settings.providerInstances, + (instanceId, driver, model) => { + const provider = snapshots.find((candidate) => candidate.instanceId === instanceId); + return ( + instanceId !== selection.instanceId && + provider?.enabled === true && + resolveProjectProviderInstanceEnabled(settings, projectId ?? null, instanceId) && + provider.installed && + provider.auth.status !== "unauthenticated" && + provider.status !== "error" && + isProviderAvailable(provider) && + isProviderTextGenerationCapable(provider) && + (operation !== "generateReviewGuide" || INVESTIGATING_DRIVERS.has(driver)) && + !exhaustedUsageWindow(provider, now, model) + ); + }, + { defaultModelSelection: settings.defaultModelSelection }, + ) ?? selection; } if (textGeneratorCannotRun && selection.instanceId === textGeneratorId) { return yield* new TextGenerationError({ diff --git a/apps/server/src/textGeneration/TextGenerationPresets.ts b/apps/server/src/textGeneration/TextGenerationPresets.ts index b53ecc543597..2dd3b9532129 100644 --- a/apps/server/src/textGeneration/TextGenerationPresets.ts +++ b/apps/server/src/textGeneration/TextGenerationPresets.ts @@ -3,7 +3,7 @@ import type { TextGenerationPolicy } from "@t3tools/provider-core/server/textGen export const conventionalCommitsTextGenerationPolicy: TextGenerationPolicy = { kind: "conventional_commits", commitInstructions: - "Use Conventional Commits when generating commit subjects. Prefer the narrowest accurate type and include a scope only when it is obvious from the diff.", + "Use Conventional Commits when generating commit subjects. Prefer the narrowest accurate type and include a scope only when it is obvious from the diff. Start the description after the colon with a lowercase word, never a proper noun or acronym, because commitlint rejects a capitalized description.", changeRequestInstructions: "Keep the change request title concise. Do not force Conventional Commit syntax into the title unless the repository already uses it.", inferRepositoryConventions: false, diff --git a/apps/server/src/usage/UsageLimitSources.test.ts b/apps/server/src/usage/UsageLimitSources.test.ts index 274e63b46c90..fc8a93f9e1bc 100644 --- a/apps/server/src/usage/UsageLimitSources.test.ts +++ b/apps/server/src/usage/UsageLimitSources.test.ts @@ -1,11 +1,21 @@ import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import { expect, it } from "@effect/vitest"; -import { SecretProviderId, ServerConfigStreamEvent, UsageLimitSourceId } from "@t3tools/contracts"; +import { + ProviderDriverKind, + SecretProviderId, + ServerConfigStreamEvent, + UsageLimitSourceId, + type ServerProviderUsageWindow, + type UsageLimitSourceSnapshot, +} from "@t3tools/contracts"; import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; import { HttpClient, HttpClientResponse } from "effect/http"; import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; @@ -26,6 +36,15 @@ it.effect("keeps malformed source URLs encodable for config subscribers", () => "hub.test:8317", "Usage limit source", ]); + // Only an http(s) hub URL names an endpoint an agent can be sent to. + expect(snapshots.map((source) => source.origin)).toEqual([ + undefined, + undefined, + undefined, + "https://hub.test:8317", + undefined, + ]); + expect(snapshots[3]?.endpoint).toBe("https://hub.test:8317/team-a"); expect(() => encodeConfigEvent({ version: 1, @@ -66,7 +85,7 @@ it.effect("keeps malformed source URLs encodable for config subscribers", () => }, [UsageLimitSourceId.make("valid-hub")]: { kind: "cliproxy", - url: "https://hub.test:8317", + url: "https://hub.test:8317/team-a/management.html", managementKey: "", enabled: true, }, @@ -87,7 +106,6 @@ it.effect("keeps malformed source URLs encodable for config subscribers", () => ), NodeCrypto.layer, ]), - Effect.scoped, ), ); @@ -154,6 +172,137 @@ it.effect("reads a management key from its secret provider", () => { ), NodeCrypto.layer, ]), - Effect.scoped, ); }); + +const NOW = Date.parse("2026-11-13T12:00:00.000Z"); +const at = (hours: number) => DateTime.formatIso(DateTime.makeUnsafe(NOW + hours * 3_600_000)); +const spent = (id: string, hours: number): ServerProviderUsageWindow => ({ + id, + kind: "weekly", + label: id, + usedPercent: 100, + resetsAt: at(hours), +}); +const hub = ( + accounts: ReadonlyArray<{ + driver?: string; + windows?: ReadonlyArray; + unavailable?: true; + }>, +): UsageLimitSourceSnapshot => ({ + id: UsageLimitSourceId.make("hub"), + kind: "cliproxy", + label: "hub", + checkedAt: at(0), + accounts: accounts.map((account, index) => ({ + id: `account-${index}`, + driver: ProviderDriverKind.make(account.driver ?? "claudeAgent"), + usageLimits: { + checkedAt: at(0), + windows: account.windows ?? [], + ...(account.unavailable ? { unavailable: { reason: "probeFailed" as const } } : {}), + }, + })), +}); + +it.each([ + { + name: "earliest refill when every account is spent, each at its latest spent window", + accounts: [ + { windows: [spent("five_hour", 1), spent("seven_day", 3)] }, + { windows: [spent("five_hour", 2)] }, + { driver: "codex", windows: [] }, + ], + model: "claude-haiku-5-5", + expected: at(2), + }, + { + name: "unknown while an account still has room", + accounts: [{ windows: [spent("five_hour", 1)] }, { windows: [] }], + model: "claude-haiku-5-5", + expected: null, + }, + { + name: "unknown while an account could not be read", + accounts: [{ windows: [spent("five_hour", 1)] }, { unavailable: true as const }], + model: "claude-haiku-5-5", + expected: null, + }, + { + name: "unknown when a spent window has no reset or already reset", + accounts: [{ windows: [{ ...spent("five_hour", 1), resetsAt: undefined }] }], + model: "claude-haiku-5-5", + expected: null, + }, + { + name: "a model-scoped weekly window blocks only its own model", + accounts: [{ windows: [spent("seven_day_fable", 5)] }], + model: "claude-haiku-5-5", + expected: null, + }, + { + name: "a model-scoped weekly window blocks its own model", + accounts: [{ windows: [spent("seven_day_fable", 5)] }], + model: "claude-fable-5-2", + expected: at(5), + }, + { + name: "no account of the driver", + accounts: [{ driver: "codex", windows: [spent("five_hour", 1)] }], + model: "claude-haiku-5-5", + expected: null, + }, +])("hub pool reset: $name", ({ accounts, model, expected }) => { + expect( + UsageLimitSources.hubPoolResetAt( + [hub(accounts)], + ProviderDriverKind.make("claudeAgent"), + model, + NOW, + ), + ).toBe(expected); +}); + +it.effect("the reset fallback refreshes its hub, and a stalled hub cannot hold the failure", () => + Effect.gen(function* () { + yield* TestClock.setTime(NOW); + const roomLeft = { ...hub([{ windows: [spent("five_hour", 2)] }]), origin: "http://hub.test" }; + const cached = { + ...roomLeft, + accounts: roomLeft.accounts.map((account) => ({ + ...account, + usageLimits: { + ...account.usageLimits, + windows: account.usageLimits.windows.map((window) => ({ ...window, usedPercent: 99 })), + }, + })), + }; + let current: ReadonlyArray = [cached]; + let refresh: Effect.Effect = Effect.sync(() => { + current = [roomLeft]; + }); + const lookup = UsageLimitSources.UsageLimitResetFallback.pipe( + Effect.flatMap((fallback) => + fallback.resetAt(ProviderDriverKind.make("claudeAgent"), "http://hub.test", null), + ), + Effect.provide( + UsageLimitSources.layerResetFallback.pipe( + Layer.provide( + Layer.mock(UsageLimitSources.UsageLimitSources)({ + current: Effect.sync(() => current), + refresh: Effect.suspend(() => refresh), + }), + ), + ), + ), + ); + // The last poll saw room; the account was spent after it. + expect(yield* lookup).toBe(at(2)); + current = [cached]; + refresh = Effect.never; + const stalled = yield* Effect.forkChild(lookup); + yield* TestClock.adjust("5 seconds"); + expect(yield* Fiber.join(stalled)).toBeNull(); + }), +); diff --git a/apps/server/src/usage/UsageLimitSources.ts b/apps/server/src/usage/UsageLimitSources.ts index 7d40b0fb9ece..d6f867d621d2 100644 --- a/apps/server/src/usage/UsageLimitSources.ts +++ b/apps/server/src/usage/UsageLimitSources.ts @@ -3,6 +3,7 @@ import { UsageLimitSourceError, type UsageLimitSourceConsumeResetCreditInput, type ProviderConsumeResetCreditResult, + type ProviderDriverKind, type ServerSettings, UsageLimitSourceConfig, settingsSecretLiteral, @@ -26,6 +27,7 @@ import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretReso import * as SecretResolverLayer from "../secrets/SecretResolver.ts"; import { resolveSettingsSecrets } from "../secrets/settingsSecretValue.ts"; import * as Settings from "../serverSettings.ts"; +import { windowLimitsModel } from "../provider/claudeUsageLimits.ts"; import { makeCliproxyApi } from "./cliproxyApi.ts"; export class UsageLimitSources extends Context.Service< @@ -79,7 +81,19 @@ export const make = Effect.gen(function* () { config: UsageLimitSourceConfig, ) { const checkedAt = DateTime.formatIso(yield* DateTime.now); - const base = { id, kind: config.kind, label: sourceLabel(id, config), checkedAt } as const; + const url = URL.parse(config.url); + const base = { + id, + kind: config.kind, + label: sourceLabel(id, config), + ...(url && /^https?:$/.test(url.protocol) + ? { + origin: url.origin, + endpoint: `${url.origin}${url.pathname.replace(/(\/management\.html)?\/*$/i, "")}`, + } + : {}), + checkedAt, + } as const; const managementKey = yield* readManagementKey(config); if (managementKey.length === 0) { return { ...base, accounts: [], error: missingManagementKey(config) }; @@ -198,3 +212,64 @@ export const make = Effect.gen(function* () { export const layer = Layer.effect(UsageLimitSources, make).pipe( Layer.provide(SecretResolverLayer.layer), ); + +/** A hub answers with a usage limit only when every account it routes the driver to is spent. */ +export function hubPoolResetAt( + sources: ReadonlyArray, + driver: ProviderDriverKind, + model: string | null, + now: number, +): string | null { + let earliest: number | null = null; + for (const source of sources) { + for (const account of source.accounts) { + if (account.driver !== driver) continue; + // An unread account may be the one still blocked, so its refill is unknown. + if (account.usageLimits.unavailable) return null; + const spent = account.usageLimits.windows.filter( + (window) => window.usedPercent >= 100 && windowLimitsModel(window, model), + ); + if (spent.length === 0) return null; + const resets = spent.map((window) => Date.parse(window.resetsAt ?? "")); + if (resets.some((reset) => !Number.isFinite(reset))) return null; + const recoversAt = Math.max(...resets); + if (recoversAt <= now) return null; + earliest = earliest === null ? recoversAt : Math.min(earliest, recoversAt); + } + } + return earliest === null ? null : DateTime.formatIso(DateTime.makeUnsafe(earliest)); +} + +export class UsageLimitResetFallback extends Context.Reference<{ + readonly resetAt: ( + driver: ProviderDriverKind, + endpoint: string, + model: string | null, + ) => Effect.Effect; +}>("t3/usage/UsageLimitResetFallback", { + defaultValue: () => ({ resetAt: () => Effect.succeed(null) }), +}) {} + +export const layerResetFallback = Layer.effect( + UsageLimitResetFallback, + Effect.gen(function* () { + const sources = yield* UsageLimitSources; + return { + resetAt: Effect.fn("UsageLimitResetFallback.resetAt")(function* (driver, endpoint, model) { + // Only the hub at the instance's endpoint serves its turns. + const atEndpoint = Effect.map(sources.current, (current) => + current.filter((source) => source.origin === endpoint), + ); + if ((yield* atEndpoint).length === 0) return null; + // The pool usually ran out after the last poll, so the cached accounts still show room. + yield* sources.refresh.pipe(Effect.timeout("5 seconds"), Effect.ignore); + return hubPoolResetAt( + yield* atEndpoint, + driver, + model, + DateTime.toEpochMillis(yield* DateTime.now), + ); + }), + }; + }), +); diff --git a/apps/server/src/usage/UsageService.test.ts b/apps/server/src/usage/UsageService.test.ts index 316edfae2502..01cbf3a9f52c 100644 --- a/apps/server/src/usage/UsageService.test.ts +++ b/apps/server/src/usage/UsageService.test.ts @@ -51,6 +51,7 @@ import * as AntigravityUsage from "../provider/Drivers/AntigravityUsage.ts"; import * as ProviderHostLive from "../provider/ProviderHostLive.ts"; import { totalTokens, type UsageRecord } from "@t3tools/provider-core/server/usage"; import * as UsageService from "./UsageService.ts"; +import { DESKTOP_HOST_SCOPE_ENV, physicalSourceIdOf } from "./usageTranscriptReader.ts"; import * as MuseSdk from "../../../../packages/provider-muse/src/server/sdk.ts"; import { MuseUsageHome } from "@t3tools/provider-muse/server"; import { makeScanCacheWriter, SCAN_CACHE_FILE_NAME } from "./usageScanCache.ts"; @@ -213,6 +214,7 @@ const layerService = (input: { Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), Layer.provideMerge(Layer.succeed(HostProcess.Platform, input.platform ?? "linux")), Layer.provideMerge(Layer.succeed(HostProcess.Hostname, input.hostname ?? "usage-test-host")), + Layer.provideMerge(Layer.succeed(HostProcess.HomeDirectory, input.home)), Layer.provideMerge(ServerSettings.layerTest(input.settings)), Layer.provideMerge( Layer.succeed( @@ -249,6 +251,7 @@ const layerService = (input: { ); const BEFORE_NARROW_WINDOW = Date.parse("2026-08-01T10:00:00Z") / 1000; +const NARROW_WINDOW_MTIME = Date.parse("2026-09-01T10:00:00Z") / 1000; const NARROW_WINDOW: UsageSummaryInput = { timeZone: "UTC", sinceDay: UsageDay.make("2026-09-01"), @@ -520,7 +523,74 @@ describe("Codex usage history", () => { }); describe("UsageService", () => { - it.live("counts compressed Codex rollouts across refresh, rewrites, and cache restarts", () => + it.effect("keeps pricing metadata paired with the rates used during a concurrent refresh", () => + Effect.gen(function* () { + const { home, settings } = yield* setup; + const sessions = NodePath.join(home, "codex", "sessions"); + yield* Effect.promise(async () => { + await NodeFSP.mkdir(sessions, { recursive: true }); + await NodeFSP.writeFile( + NodePath.join(sessions, "session.jsonl"), + [ + { type: "session_meta", payload: { id: "session", model_provider: "openai" } }, + { type: "turn_context", payload: { model: "gpt-6-astra", turn_id: "turn" } }, + { + type: "event_msg", + timestamp: "2026-08-01T10:00:00Z", + payload: { + type: "token_count", + info: { last_token_usage: { input_tokens: 0, output_tokens: 1 } }, + }, + }, + ] + .map((record) => encodeUnknownJsonString(record)) + .join("\n") + "\n", + ); + }); + let rate = 0; + yield* Effect.gen(function* () { + const resolving = yield* Deferred.make(); + const release = yield* Deferred.make(); + const service = yield* UsageService.make.pipe( + Effect.provideService(CodexUsageHistory.CodexUsageHistory, { + resolve: () => + Deferred.succeed(resolving, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.as(new Map()), + ), + }), + ); + const initialPricing = yield* service.refreshRates; + yield* TestClock.adjust(61_000); + const pending = yield* service.readSummary(WINDOW).pipe(Effect.forkChild); + yield* Deferred.await(resolving); + yield* service.refreshRates; + yield* Deferred.succeed(release, undefined); + const original = yield* Fiber.join(pending); + assert.strictEqual(original.buckets[0]?.costUsd, 1); + assert.strictEqual(original.pricing.fetchedAt, initialPricing.fetchedAt); + const updated = yield* service.readSummary(WINDOW); + assert.strictEqual(updated.buckets[0]?.costUsd, 2); + assert.notStrictEqual(updated.pricing.fetchedAt, initialPricing.fetchedAt); + }).pipe( + Effect.provide( + layerService({ + prefix: "concurrent-rates", + home, + settings, + ratesResponse: () => + Effect.succeed( + Response.json({ + "gpt-6-astra": { input_cost_per_token: 0, output_cost_per_token: ++rate }, + }), + ), + }), + ), + ); + }).pipe(Effect.scoped), + ); + + it.effect("counts compressed Codex rollouts across refresh, rewrites, and cache restarts", () => Effect.gen(function* () { const { home, settings } = yield* setup; const sessions = NodePath.join(home, "codex", "sessions"); @@ -577,17 +647,63 @@ describe("UsageService", () => { assert.strictEqual(totalOutputTokens(rewritten), 71); assert.deepStrictEqual((yield* service.readSummary(WINDOW)).buckets, rewritten.buckets); yield* service.awaitPersisted; + const { stateDir } = yield* ServerConfig.ServerConfig; + const cachePath = NodePath.join(stateDir, SCAN_CACHE_FILE_NAME); + const originalPath = (yield* Effect.promise(() => NodeFSP.realpath(compressedPath))).slice( + 0, + -4, + ); + yield* Effect.promise(async () => { + const document = decodeUnknownJsonString(await NodeFSP.readFile(cachePath, "utf8")) as { + files: Record< + string, + { cs: Record; r: unknown[]; t: unknown[]; mr: number } + >; + }; + const archived = document.files[await NodeFSP.realpath(compressedPath)]; + assert.ok(archived); + document.files[originalPath] = { + ...archived, + r: [], + t: [], + mr: 1, + cs: { ...archived.cs, malformedRecords: 1 }, + }; + await NodeFSP.writeFile(cachePath, encodeUnknownJsonString(document)); + }); const restarted = yield* UsageService.make; - assert.deepStrictEqual((yield* restarted.readSummary(WINDOW)).buckets, rewritten.buckets); + const restored = yield* restarted.readSummary(WINDOW); + assert.deepStrictEqual(restored.buckets, rewritten.buckets); + assert.strictEqual( + restored.sources.find((source) => source.fingerprint.provider === "codex") + ?.malformedRecords, + 0, + ); yield* restarted.awaitPersisted; + yield* Effect.promise(async () => { + const document = decodeUnknownJsonString(await NodeFSP.readFile(cachePath, "utf8")) as { + files: Record; + }; + assert.strictEqual(document.files[originalPath], undefined); + await NodeFSP.rm(compressedPath); + }); + const removed = yield* UsageService.make; + const retained = yield* removed.readSummary(WINDOW); + assert.deepStrictEqual(retained.buckets, rewritten.buckets); + assert.strictEqual( + retained.sources.find((source) => source.fingerprint.provider === "codex") + ?.malformedRecords, + 0, + ); + yield* removed.awaitPersisted; }).pipe( Effect.scoped, Effect.provide(layerService({ prefix: "usage-compressed-codex", home, settings })), ); - }).pipe(Effect.scoped), + }), ); - it.live("keeps the first host identity across recreation and explicit override removal", () => + it.effect("keeps the first host identity across recreation and explicit override removal", () => Effect.gen(function* () { const { transcript, settings, home } = yield* setup; yield* Effect.promise(() => NodeFSP.writeFile(transcript, claudeLine(1, 5))); @@ -615,10 +731,10 @@ describe("UsageService", () => { const source = summary.sources.find((source) => source.fingerprint.provider === "claude"); assert.strictEqual(source?.fingerprint.hostId, expected); } - }).pipe(Effect.scoped), + }), ); - it.live("concurrent initialization keeps one host identity across service recreation", () => + it.effect("concurrent initialization keeps one host identity across service recreation", () => Effect.gen(function* () { const { transcript, settings, home } = yield* setup; yield* Effect.promise(() => NodeFSP.writeFile(transcript, claudeLine(1, 5))); @@ -671,10 +787,10 @@ describe("UsageService", () => { const source = summary.sources.find((source) => source.fingerprint.provider === "claude"); assert.strictEqual(source?.fingerprint.hostId, hostId); } - }).pipe(Effect.scoped), + }), ); - it.live( + it.effect( "repairs initial Codex prices after unchanged-file retries, persists evidence and corrections, and honors later native tiers", () => Effect.gen(function* () { @@ -823,10 +939,106 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped), + }), ); - it.live( + it.effect( + "keeps resolved Codex tiers when a grown pre-v15 cache entry re-parses without history", + () => + Effect.gen(function* () { + const { home, settings } = yield* setup; + const sessions = NodePath.join(home, "codex", "sessions"); + const rolloutPath = NodePath.join(sessions, "tier.jsonl"); + const context = (turn_id: string) => ({ + type: "turn_context", + payload: { model: "gpt-6-astra", turn_id }, + }); + const usage = (output_tokens: number, total_output_tokens: number) => ({ + type: "event_msg", + timestamp: "2026-08-01T10:00:00Z", + payload: { + type: "token_count", + info: { + last_token_usage: { input_tokens: 0, output_tokens }, + total_token_usage: { input_tokens: 0, output_tokens: total_output_tokens }, + }, + }, + }); + const lines = (records: readonly unknown[]) => + records.map((record) => encodeUnknownJsonString(record)).join("\n") + "\n"; + const verify = (summary: UsageSummary, output: number, cost: number) => { + assert.strictEqual(totalOutputTokens(summary), output); + assert.strictEqual( + summary.buckets.reduce((sum, bucket) => sum + bucket.costUsd, 0), + cost, + ); + }; + yield* Effect.promise(async () => { + await NodeFSP.mkdir(sessions, { recursive: true }); + await NodeFSP.writeFile( + rolloutPath, + lines([ + { type: "session_meta", payload: { id: "tier-session", model_provider: "openai" } }, + context("ultra-turn"), + usage(1, 1), + // An equal request at the same time, from a turn priced at the standard tier. + context("standard-turn"), + usage(1, 2), + ]), + ); + }); + yield* Effect.gen(function* () { + const { stateDir } = yield* ServerConfig.ServerConfig; + yield* seedCodexHistory({ + id: "ultra-turn", + sessionId: "tier-session", + options: [{ id: "serviceTier", value: "ultrafast" }], + }); + yield* seedCodexHistory({ id: "standard-turn", sessionId: "tier-session", options: [] }); + const first = yield* UsageService.make; + verify(yield* first.readSummary(WINDOW), 2, 7); + yield* first.awaitPersisted; + + // Downgrade the saved cache to v14, whose clean Codex entries cannot resume. + yield* Effect.promise(async () => { + const cachePath = NodePath.join(stateDir, SCAN_CACHE_FILE_NAME); + const document = JSON.parse(await NodeFSP.readFile(cachePath, "utf8")); + await NodeFSP.rm(cachePath); + await NodeFSP.writeFile( + NodePath.join(stateDir, "usage-scan-cache-v14.json"), + encodeUnknownJsonString({ ...document, version: 14 }), + ); + await NodeFSP.appendFile(rolloutPath, lines([context("later-turn"), usage(2, 4)])); + }); + const sql = yield* SqlClient.SqlClient; + yield* sql`ALTER TABLE orchestration_events RENAME TO unavailable_events`; + const upgraded = yield* UsageService.make; + verify(yield* upgraded.readSummary(WINDOW), 4, 9); + yield* upgraded.awaitPersisted; + const restarted = yield* UsageService.make; + verify(yield* restarted.readSummary(WINDOW), 4, 9); + yield* sql`ALTER TABLE unavailable_events RENAME TO orchestration_events`; + }).pipe( + Effect.provide( + layerService({ + prefix: "usage-reparse-tier", + home, + settings, + ratesDocument: { + "gpt-6-astra": { + input_cost_per_token: 0, + output_cost_per_token: 1, + input_cost_per_token_ultrafast: 0, + output_cost_per_token_ultrafast: 6, + }, + }, + }), + ), + ); + }), + ); + + it.effect( "reads Pi per-model usage and auxiliary costs once across forks and repeated entries", () => Effect.gen(function* () { @@ -1012,10 +1224,80 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped), + }), + ); + + it.effect("prices Pi usage mapped onto a Pi model at that model's rate", () => + Effect.gen(function* () { + const { home, settings } = yield* setup; + const dir = NodePath.join(home, "pi", "sessions", "project"); + const message = (id: string, model: string) => ({ + type: "message", + id, + timestamp: "2026-08-01T10:00:00Z", + message: { + role: "assistant", + provider: "anthropic", + model, + usage: { input: 10, output: 5 }, + }, + }); + yield* Effect.promise(async () => { + await NodeFSP.mkdir(dir, { recursive: true }); + await NodeFSP.writeFile( + NodePath.join(dir, "session.jsonl"), + [ + { type: "session", id: "session" }, + message("released", "actual-model"), + message("preview", "actual-model-preview"), + ] + .map((line) => encodeUnknownJsonString(line)) + .join("\n") + "\n", + ); + }); + const summary = yield* Effect.gen(function* () { + const service = yield* UsageService.make; + return yield* service.readSummary(WINDOW); + }).pipe( + Effect.scoped, + Effect.provide( + layerService({ + prefix: "usage-pi-model-alias", + home, + ratesDocument: { + "actual-model": { input_cost_per_token: 1, output_cost_per_token: 4 }, + }, + settings: { + ...settings, + usageModelAliases: { "anthropic/actual-model-preview": "anthropic/actual-model" }, + }, + }), + ), + ); + assert.deepStrictEqual( + summary.buckets + .filter((bucket) => bucket.provider === "pi") + .map(({ model, records, unpricedRecords, costUsd, costSource }) => ({ + model, + records, + unpricedRecords, + costUsd, + costSource, + })), + [ + { + model: "anthropic/actual-model", + records: 2, + unpricedRecords: 0, + costUsd: 60, + costSource: "modelPriced", + }, + ], + ); + }), ); - it.live( + it.effect( "reads Muse inference history once, persists appended usage, and reprices unknown models", () => Effect.gen(function* () { @@ -1234,10 +1516,10 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped), + }), ); - it.live("returns other providers' usage while Muse home discovery is still pending", () => + it.effect("returns other providers' usage while Muse home discovery is still pending", () => Effect.gen(function* () { const { transcript, settings, home } = yield* setup; yield* Effect.promise(() => NodeFSP.writeFile(transcript, claudeLine(1, 5))); @@ -1294,73 +1576,108 @@ describe("UsageService", () => { assert.deepStrictEqual(museSources(refreshed), [ [NodePath.join(museHome, "sessions"), "ok", undefined], ]); - }).pipe(Effect.scoped), + }), ); - it.live("reports missing Pi history and skips malformed records without losing valid usage", () => + it.effect("identifies a transcript directory created after the server first read usage", () => Effect.gen(function* () { const { home, settings } = yield* setup; + const scope = "0123456789abcdef0123456789abcdef"; + const dir = NodePath.join(home, "pi", "sessions"); yield* Effect.gen(function* () { const service = yield* UsageService.make; - const missing = yield* service.readSummary(WINDOW); + const piFingerprint = (summary: UsageSummary) => + summary.sources.find((source) => source.fingerprint.provider === "pi")?.fingerprint; + const missing = piFingerprint(yield* service.readSummary(WINDOW)); + assert.strictEqual(missing?.resolvedHomePath, dir); + assert.isUndefined(missing?.physicalSourceId); + yield* Effect.promise(() => NodeFSP.mkdir(dir, { recursive: true })); assert.strictEqual( - missing.sources.find((source) => source.fingerprint.provider === "pi")?.status, - "missing", + piFingerprint(yield* service.readSummary(WINDOW))?.physicalSourceId, + physicalSourceIdOf(scope, yield* Effect.promise(() => NodeFSP.realpath(dir))), ); - const dir = NodePath.join(home, "pi", "sessions"); - yield* Effect.promise(async () => { - await NodeFSP.mkdir(dir, { recursive: true }); - await NodeFSP.writeFile( - NodePath.join(dir, "session.jsonl"), - [ - "{broken", - "null", - encodeUnknownJsonString({ type: "session", id: "valid" }), - encodeUnknownJsonString({ - type: "message", - id: "bad", - timestamp: "invalid", - message: { - role: "assistant", - provider: "openai", - model: "model", - usage: { input: 100 }, - }, - }), - encodeUnknownJsonString({ - type: "message", - id: "valid", - timestamp: "2026-08-01T10:00:00Z", - message: { - role: "assistant", - provider: "openai", - model: "model", - usage: { - input: -1, - cacheRead: "100", - output: 7, - reasoning: 50, - cost: { total: -1 }, - }, - }, - }), - ].join("\n") + "\n", - ); - }); - const summary = yield* service.readSummary(WINDOW); - const pi = summary.buckets.filter((bucket) => bucket.provider === "pi"); - assert.strictEqual(pi.length, 1); - assert.strictEqual(pi[0] && totalTokens(pi[0].totals), 7); - assert.strictEqual(pi[0]?.totals.reasoningTokens, 7); - assert.strictEqual(pi[0]?.costSource, "unpriced"); }).pipe( Effect.scoped, - Effect.provide(layerService({ prefix: "usage-pi-malformed", home, settings })), + Effect.provide( + layerService({ + prefix: "usage-physical-source-created", + home, + settings, + // The native Windows server reads identity from realpath alone. + platform: "win32", + environment: { [DESKTOP_HOST_SCOPE_ENV]: scope }, + }), + ), ); - }).pipe(Effect.scoped), + }), ); - it.live("discovers Pi history in its default home and directory overrides", () => + it.effect( + "reports missing Pi history and skips malformed records without losing valid usage", + () => + Effect.gen(function* () { + const { home, settings } = yield* setup; + yield* Effect.gen(function* () { + const service = yield* UsageService.make; + const missing = yield* service.readSummary(WINDOW); + assert.strictEqual( + missing.sources.find((source) => source.fingerprint.provider === "pi")?.status, + "missing", + ); + const dir = NodePath.join(home, "pi", "sessions"); + yield* Effect.promise(async () => { + await NodeFSP.mkdir(dir, { recursive: true }); + await NodeFSP.writeFile( + NodePath.join(dir, "session.jsonl"), + [ + "{broken", + "null", + encodeUnknownJsonString({ type: "session", id: "valid" }), + encodeUnknownJsonString({ + type: "message", + id: "bad", + timestamp: "invalid", + message: { + role: "assistant", + provider: "openai", + model: "model", + usage: { input: 100 }, + }, + }), + encodeUnknownJsonString({ + type: "message", + id: "valid", + timestamp: "2026-08-01T10:00:00Z", + message: { + role: "assistant", + provider: "openai", + model: "model", + usage: { + input: -1, + cacheRead: "100", + output: 7, + reasoning: 50, + cost: { total: -1 }, + }, + }, + }), + ].join("\n") + "\n", + ); + }); + const summary = yield* service.readSummary(WINDOW); + const pi = summary.buckets.filter((bucket) => bucket.provider === "pi"); + assert.strictEqual(pi.length, 1); + assert.strictEqual(pi[0] && totalTokens(pi[0].totals), 7); + assert.strictEqual(pi[0]?.totals.reasoningTokens, 7); + assert.strictEqual(pi[0]?.costSource, "unpriced"); + }).pipe( + Effect.scoped, + Effect.provide(layerService({ prefix: "usage-pi-malformed", home, settings })), + ); + }), + ); + + it.effect("discovers Pi history in its default home and directory overrides", () => Effect.gen(function* () { const { home, settings } = yield* setup; for (const [agentDir, sessionDir, relativeDir, launchArgs, instanceHome, platform] of [ @@ -1464,10 +1781,10 @@ describe("UsageService", () => { Effect.provideService(HostProcess.HomeDirectory, home), ); } - }).pipe(Effect.scoped), + }), ); - it.live("reads shared managed default and disabled extra account history once", () => + it.effect("reads shared managed default and disabled extra account history once", () => Effect.gen(function* () { const { home, settings } = yield* setup; const summary = yield* Effect.gen(function* () { @@ -1544,10 +1861,10 @@ describe("UsageService", () => { ).length, 1, ); - }).pipe(Effect.scoped), + }), ); - it.live("restores the Cursor account cache after a restart", () => + it.effect("restores the Cursor account cache after a restart", () => Effect.gen(function* () { const { settings, home } = yield* setup; yield* TestClock.setTime(CURSOR_NOW); @@ -1586,10 +1903,10 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped, Effect.provide(TestClock.layer())), + }), ); - it.live("reads the Cursor account of an API key or T3 sign-in instead of a CLI login", () => + it.effect("reads the Cursor account of an API key or T3 sign-in instead of a CLI login", () => Effect.gen(function* () { const { settings, home } = yield* setup; const signIn = new TextEncoder().encode( @@ -1638,10 +1955,10 @@ describe("UsageService", () => { ]); assert.strictEqual(cursorSource(summary)?.status, "ok"); } - }).pipe(Effect.scoped), + }), ); - it.live("does not show a previous Cursor account's history after the API key changes", () => + it.effect("does not show a previous Cursor account's history after the API key changes", () => Effect.gen(function* () { const { settings, home } = yield* setup; yield* TestClock.setTime(CURSOR_NOW); @@ -1680,10 +1997,10 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped, Effect.provide(TestClock.layer())), + }), ); - it.live("keeps a new Cursor key's refresh visible beside a failing key", () => + it.effect("keeps a new Cursor key's refresh visible beside a failing key", () => Effect.gen(function* () { const { settings, home } = yield* setup; const cursor = makeFakeCursor(); @@ -1765,10 +2082,10 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped), + }), ); - it.live("reports one Cursor source when two API keys belong to the same account", () => + it.effect("reports one Cursor source when two API keys belong to the same account", () => Effect.gen(function* () { const { settings, home } = yield* setup; yield* TestClock.setTime(CURSOR_NOW); @@ -1827,10 +2144,10 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped, Effect.provide(TestClock.layer())), + }), ); - it.live( + it.effect( "includes OpenCode history but does not substitute desktop usage for an unavailable Cursor account", () => Effect.gen(function* () { @@ -1884,10 +2201,47 @@ describe("UsageService", () => { ?.distinctSessions, 1, ); - }).pipe(Effect.scoped), + }), ); - it.live("includes host and merged OpenCode instance histories once across list aliases", () => + it.effect("counts unpriced tokens from the unpriced records of a mixed model", () => + Effect.gen(function* () { + const { settings, home } = yield* setup; + const message = (id: string, cost: number, input: number, output: number, minute: number) => + encodeUnknownJson({ + id, + sessionID: "session-1", + role: "assistant", + modelID: "acme-coder-1", + providerID: "acme", + time: { created: Date.parse(`2026-08-01T10:0${minute}:00Z`) }, + cost, + tokens: { input, output, reasoning: 0, cache: { read: 0, write: 0 } }, + }); + const priced = yield* message("msg_priced", 1, 60, 40, 0); + const unpriced = yield* message("msg_unpriced", 0, 9_000, 900, 1); + yield* Effect.promise(async () => { + const directory = NodePath.join(home, "opencode", "storage", "message", "session-1"); + await NodeFSP.mkdir(directory, { recursive: true }); + await NodeFSP.writeFile(NodePath.join(directory, "msg_priced.json"), priced); + await NodeFSP.writeFile(NodePath.join(directory, "msg_unpriced.json"), unpriced); + }); + const service = yield* UsageService.make.pipe( + Effect.provide(layerService({ prefix: "usage-service-unpriced-tokens", home, settings })), + ); + const summary = yield* service.readSummary(WINDOW); + const model = mergeUsage( + [{ environmentId: EnvironmentId.make("local"), label: "local", summary }], + summary.contractVersion, + ).models.find((entry) => entry.model === "acme-coder-1"); + + assert.strictEqual(model?.costUsd, 1); + assert.strictEqual(model?.totalTokens, 10_000); + assert.strictEqual(model?.unpricedTokens, 9_900); + }), + ); + + it.effect("includes host and merged OpenCode instance histories once across list aliases", () => Effect.gen(function* () { const { settings, home } = yield* setup; const hostRoot = NodePath.join(home, "opencode"); @@ -1963,10 +2317,10 @@ describe("UsageService", () => { .sort((a, b) => a - b), [5, 7, 11], ); - }).pipe(Effect.scoped), + }), ); - it.live("counts aliased OpenCode and Antigravity directories once", () => + it.effect("counts aliased OpenCode and Antigravity directories once", () => Effect.gen(function* () { const { settings, home } = yield* setup; const opencode = NodePath.join(home, "opencode-store"); @@ -2017,10 +2371,10 @@ describe("UsageService", () => { sourcesFor("antigravity")[0]?.fingerprint.resolvedHomePath, yield* Effect.promise(() => NodeFSP.realpath(conversations)), ); - }).pipe(Effect.scoped), + }), ); - it.live("reads Antigravity history from each instance's data directory", () => + it.effect("reads Antigravity history from each instance's data directory", () => Effect.gen(function* () { const { settings, home } = yield* setup; const first = NodePath.join(home, "antigravity-first"); @@ -2088,10 +2442,10 @@ describe("UsageService", () => { outputTokens, ); } - }).pipe(Effect.scoped), + }), ); - it.live("reads configured and disabled accounts once across shared and aliased homes", () => + it.effect("reads configured and disabled accounts once across shared and aliased homes", () => Effect.gen(function* () { const { transcript, settings, home } = yield* setup; const codexHome = NodePath.join(home, "codex-account"); @@ -2157,6 +2511,7 @@ describe("UsageService", () => { }, [ProviderInstanceId.make("codex-work")]: { driver: ProviderDriverKind.make("codex"), + config: { shadowHomePath: NodePath.join(home, "work-shadow") }, environment: [{ name: "CODEX_HOME", value: codexHome, sensitive: false }], }, [ProviderInstanceId.make("codex-alias")]: { @@ -2206,10 +2561,38 @@ describe("UsageService", () => { sources.filter((source) => source.fingerprint.provider === "codex").length, 1, ); - }).pipe(Effect.scoped), + }), + ); + + it.effect("keeps default-home history after the default account moves to a custom home", () => + Effect.gen(function* () { + const { transcript, settings, home } = yield* setup; + const defaultHome = NodePath.join(home, "default-claude"); + const defaultProjects = NodePath.join(defaultHome, "projects", "proj"); + yield* Effect.promise(async () => { + await NodeFSP.writeFile(transcript, claudeLine(1, 5)); + await NodeFSP.mkdir(defaultProjects, { recursive: true }); + await NodeFSP.writeFile( + NodePath.join(defaultProjects, "earlier.jsonl"), + claudeLine(2, 7, "claude-fable-5", "earlier-session"), + ); + }); + const summary = yield* UsageService.make.pipe( + Effect.flatMap((service) => service.readSummary(WINDOW)), + Effect.provide( + layerService({ + prefix: "usage-service-default-home-test", + home, + environment: { CLAUDE_CONFIG_DIR: defaultHome }, + settings, + }), + ), + ); + assert.strictEqual(totalOutputTokens(summary), 12); + }), ); - it.live( + it.effect( "uses explicit account settings before environment and default homes, then refreshes them", () => Effect.gen(function* () { @@ -2289,10 +2672,10 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped), + }), ); - it.live( + it.effect( "uses inherited home variables when explicit default accounts have no home settings", () => Effect.gen(function* () { @@ -2335,10 +2718,10 @@ describe("UsageService", () => { .resolvedHomePath, NodePath.join(home, "grok", "sessions"), ); - }).pipe(Effect.scoped), + }), ); - it.live.skipIf(HostProcess.Platform.defaultValue() === "win32" || process.getuid?.() === 0)( + it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32" || process.getuid?.() === 0)( "reports unreadable transcripts as partial and recovers once they can be read", () => Effect.gen(function* () { @@ -2363,10 +2746,79 @@ describe("UsageService", () => { }).pipe( Effect.provide(layerService({ prefix: "usage-service-unreadable", home, settings })), ); - }).pipe(Effect.scoped), + }), + ); + + it.effect( + "answers a provisional read from the last scan, then the exact read adds what changed", + () => + Effect.gen(function* () { + const { transcript, settings, home } = yield* setup; + yield* Effect.promise(() => NodeFSP.writeFile(transcript, claudeLine(1, 5))); + const provisional = { ...WINDOW, provisional: true }; + const claude = (summary: UsageSummary) => + summary.sources.find((source) => source.fingerprint.provider === "claude"); + yield* Effect.gen(function* () { + // With nothing persisted, the first answer is a full scan. + const service = yield* UsageService.make; + const loop = NodePath.join(NodePath.dirname(transcript), "loop.jsonl"); + yield* Effect.promise(() => NodeFSP.symlink(loop, loop, "junction")); + assert.strictEqual(claude(yield* service.readSummary(provisional))?.status, "partial"); + yield* Effect.promise(() => NodeFSP.rm(loop)); + // A scan that missed a path covers nothing, so this is a full scan too. + assert.isUndefined(claude(yield* service.readSummary(provisional))?.refreshing); + yield* service.awaitPersisted; + + yield* Effect.promise(() => NodeFSP.appendFile(transcript, claudeLine(2, 7))); + const restarted = yield* UsageService.make; + const saved = yield* restarted.readSummary(provisional); + assert.strictEqual(totalOutputTokens(saved), 5); + assert.isTrue(claude(saved)?.refreshing); + // A window that starts before the saved scan could only show part of its usage. + const wider = yield* restarted.readSummary({ + ...provisional, + sinceDay: UsageDay.make("2026-07-01"), + }); + assert.strictEqual(totalOutputTokens(wider), 12); + assert.isUndefined(claude(wider)?.refreshing); + const refreshed = yield* restarted.readSummary({ ...WINDOW, awaitRefresh: true }); + assert.strictEqual(totalOutputTokens(refreshed), 12); + assert.isUndefined(claude(refreshed)?.refreshing); + // Saved usage with nothing to show marks nothing refreshing, so it must not be the answer. + yield* Effect.promise(() => + NodeFSP.appendFile(transcript, claudeLine(3, 11, "claude-other")), + ); + const unseen = yield* restarted.readSummary({ + ...provisional, + modelFilter: { include: [{ model: "claude-other" }] }, + }); + assert.strictEqual(totalOutputTokens(unseen), 11); + yield* restarted.awaitPersisted; + + // A saved scan with an entry this server cannot read covers nothing. + const { stateDir } = yield* ServerConfig.ServerConfig; + const cachePath = NodePath.join(stateDir, SCAN_CACHE_FILE_NAME); + yield* Effect.promise(async () => { + const document = decodeUnknownJsonString(await NodeFSP.readFile(cachePath, "utf8")) as { + files: Record>; + }; + const [entry] = Object.values(document.files); + document.files[NodePath.join(NodePath.dirname(transcript), "broken.jsonl")] = { + ...entry, + r: "broken", + }; + await NodeFSP.writeFile(cachePath, JSON.stringify(document)); + }); + const reloaded = yield* UsageService.make; + assert.isUndefined(claude(yield* reloaded.readSummary(provisional))?.refreshing); + }).pipe( + Effect.scoped, + Effect.provide(layerService({ prefix: "usage-service-provisional", home, settings })), + ); + }), ); - it.live("filters original model IDs before aliases, pricing, buckets and session counts", () => + it.effect("filters original model IDs before aliases, pricing, buckets and session counts", () => Effect.gen(function* () { const { transcript, settings, home } = yield* setup; yield* Effect.promise(() => @@ -2468,10 +2920,10 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped), + }), ); - it.live("reprices unchanged transcripts when custom prices are added, edited, or removed", () => + it.effect("reprices unchanged transcripts when custom prices are added, edited, or removed", () => Effect.gen(function* () { const { transcript, settings, home } = yield* setup; yield* Effect.promise(() => NodeFSP.writeFile(transcript, claudeLine(1, 5, "example-model"))); @@ -2512,10 +2964,10 @@ describe("UsageService", () => { layerService({ prefix: "usage-service-price-overrides-test", home, settings }), ), ); - }).pipe(Effect.scoped), + }), ); - it.live( + it.effect( "retains Codex reconciliation and partial coverage through persisted restart, append and repair", () => Effect.gen(function* () { @@ -2635,10 +3087,10 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped), + }), ); - it.live( + it.effect( "merges partial Claude copies through tail replacement, newline completion and restart", () => Effect.gen(function* () { @@ -2694,10 +3146,10 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped), + }), ); - it.live( + it.effect( "keeps large-record totals and costs exact through append, dedupe, restart and cleanup", () => Effect.gen(function* () { @@ -2749,13 +3201,14 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped), + }), ); - it.live.each([4, 5, 6, 7, 8, 10])( + it.effect.each([4, 5, 6, 7, 8, 10, 11, 12, 13, 14])( "upgrades a v%s cache: reparses live Codex usage, keeps deleted rollouts, removes the old file", (version) => Effect.gen(function* () { + yield* TestClock.setTime(Date.parse("2026-08-02T12:00:00Z")); const { home, settings } = yield* setup; const sessions = NodePath.join(home, "codex", "sessions"); const rollout = (sessionId: string, outputTokens: number) => @@ -2853,7 +3306,7 @@ describe("UsageService", () => { 60 + 20 * (version === 4 ? 1 : 2), ); yield* upgraded.awaitPersisted; - assert.strictEqual(SCAN_CACHE_FILE_NAME, "usage-scan-cache-v12.json"); + assert.strictEqual(SCAN_CACHE_FILE_NAME, "usage-scan-cache-v17.json"); assert.deepStrictEqual( (yield* Effect.promise(() => NodeFSP.readdir(stateDir))) .filter((name) => name.startsWith("usage-scan-cache")) @@ -2879,10 +3332,10 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped), + }), ); - it.live("re-parses a clean v10 Codex rollout that grew after an interrupted request", () => + it.effect("re-parses a clean v10 Codex rollout that grew after an interrupted request", () => Effect.gen(function* () { const { home, settings } = yield* setup; const sessions = NodePath.join(home, "codex", "sessions"); @@ -2979,10 +3432,10 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped), + }), ); - it.live("preserves saved tokens, costs and sessions after transcript cleanup and restart", () => + it.effect("preserves saved tokens, costs and sessions after transcript cleanup and restart", () => Effect.gen(function* () { const { transcript, settings, home } = yield* setup; const alias = NodePath.join(home, "claude-alias"); @@ -3080,10 +3533,10 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped), + }), ); - it.live("credits the same copy of a duplicate after its transcripts are deleted", () => + it.effect("credits the same copy of a duplicate after its transcripts are deleted", () => Effect.gen(function* () { const { transcript, settings, home } = yield* setup; const dir = NodePath.dirname(transcript); @@ -3118,10 +3571,68 @@ describe("UsageService", () => { }).pipe( Effect.provide(layerService({ prefix: "usage-service-copy-order-test", home, settings })), ); - }).pipe(Effect.scoped), + }), + ); + + it.effect("credits the same copy of a duplicate from the last scan as an exact read does", () => + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse("2026-08-15T00:00:00Z")); + const { transcript, settings, home } = yield* setup; + const dir = NodePath.dirname(transcript); + const [first = "", second = ""] = yield* Effect.promise(async () => { + await NodeFSP.writeFile(NodePath.join(dir, "a.jsonl"), ""); + await NodeFSP.writeFile(NodePath.join(dir, "b.jsonl"), ""); + return (await NodeFSP.readdir(dir)).map((name) => NodePath.join(dir, name)); + }); + const forked = (line: string) => line.replace('"session-1"', '"session-2"'); + yield* Effect.promise(async () => { + await NodeFSP.writeFile(first, claudeLine(1, 5)); + await NodeFSP.utimes(first, BEFORE_NARROW_WINDOW, BEFORE_NARROW_WINDOW); + await NodeFSP.writeFile(second, forked(claudeLine(1, 5)) + forked(claudeLine(2, 7))); + }); + yield* Effect.gen(function* () { + const service = yield* UsageService.make; + // The narrow window saves the second copy before the wide one reads the first. + yield* service.readSummary(NARROW_WINDOW); + const exact = yield* service.readSummary(WINDOW); + yield* service.awaitPersisted; + const restarted = yield* UsageService.make; + const saved = yield* restarted.readSummary({ ...WINDOW, provisional: true }); + assert.isTrue(saved.sources.some((source) => source.refreshing === true)); + assert.deepStrictEqual(saved.buckets, exact.buckets); + }).pipe( + Effect.provide(layerService({ prefix: "usage-service-saved-order-test", home, settings })), + ); + + // A narrow scan that first saves the earlier copy still puts it before a later file it skipped. + yield* Effect.promise(async () => { + await NodeFSP.writeFile(first, forked(claudeLine(1, 5)) + forked(claudeLine(2, 7))); + await NodeFSP.utimes( + first, + Date.parse("2026-07-01T00:00:00Z") / 1000, + Date.parse("2026-07-01T00:00:00Z") / 1000, + ); + await NodeFSP.writeFile(second, claudeLine(1, 5)); + await NodeFSP.utimes(second, BEFORE_NARROW_WINDOW, BEFORE_NARROW_WINDOW); + }); + yield* Effect.gen(function* () { + const service = yield* UsageService.make; + yield* service.readSummary(WINDOW); + yield* Effect.promise(() => + NodeFSP.utimes(first, NARROW_WINDOW_MTIME, NARROW_WINDOW_MTIME), + ); + yield* service.readSummary(NARROW_WINDOW); + const saved = yield* service.readSummary({ ...WINDOW, provisional: true }); + assert.isTrue(saved.sources.some((source) => source.refreshing === true)); + const exact = yield* service.readSummary(WINDOW); + assert.deepStrictEqual(saved.buckets, exact.buckets); + }).pipe( + Effect.provide(layerService({ prefix: "usage-service-narrow-order-test", home, settings })), + ); + }), ); - it.live.skipIf(HostProcess.Platform.defaultValue() === "win32")( + it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( "keeps a newer cached read when a slower scan of another window finishes later", () => Effect.gen(function* () { @@ -3148,50 +3659,53 @@ describe("UsageService", () => { }).pipe( Effect.provide(layerService({ prefix: "usage-service-stale-read-test", home, settings })), ); - }).pipe(Effect.scoped), + }), ); - it.live("keeps an absent directory missing when its cached files only contain diagnostics", () => - Effect.gen(function* () { - const { transcript, settings, home } = yield* setup; - yield* Effect.promise(() => - NodeFSP.rm(NodePath.join(home, "claude", "projects"), { recursive: true }), - ); - yield* Effect.gen(function* () { - const { stateDir } = yield* ServerConfig.ServerConfig; - const mtimeMs = yield* Clock.currentTimeMillis; - const cache = makeScanCacheWriter()( - new Map([ - [ - transcript, - { - size: 0, - mtimeMs, - provider: "claude" as const, - records: [], - tailRecords: [], - malformedRecords: 2, - position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null }, - }, - ], - ]), - {}, + it.effect( + "keeps an absent directory missing when its cached files only contain diagnostics", + () => + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse("2026-08-02T12:00:00Z")); + const { transcript, settings, home } = yield* setup; + yield* Effect.promise(() => + NodeFSP.rm(NodePath.join(home, "claude", "projects"), { recursive: true }), ); - yield* Effect.promise(async () => { - await NodeFSP.mkdir(stateDir, { recursive: true }); - await NodeFSP.writeFile(NodePath.join(stateDir, SCAN_CACHE_FILE_NAME), cache); - }); - const summary = yield* (yield* UsageService.make).readSummary(WINDOW); - const source = summary.sources.find((source) => source.fingerprint.provider === "claude"); - assert.strictEqual(source?.status, "missing"); - assert.strictEqual(source?.scannedFiles, 0); - assert.strictEqual(source?.malformedRecords, 2); - }).pipe( - Effect.provide( - layerService({ prefix: "usage-service-missing-diagnostics", home, settings }), - ), - ); - }).pipe(Effect.scoped), + yield* Effect.gen(function* () { + const { stateDir } = yield* ServerConfig.ServerConfig; + const mtimeMs = yield* Clock.currentTimeMillis; + const cache = makeScanCacheWriter()( + new Map([ + [ + transcript, + { + size: 0, + mtimeMs, + provider: "claude" as const, + records: [], + tailRecords: [], + malformedRecords: 2, + position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null }, + }, + ], + ]), + {}, + ); + yield* Effect.promise(async () => { + await NodeFSP.mkdir(stateDir, { recursive: true }); + await NodeFSP.writeFile(NodePath.join(stateDir, SCAN_CACHE_FILE_NAME), cache); + }); + const summary = yield* (yield* UsageService.make).readSummary(WINDOW); + const source = summary.sources.find((source) => source.fingerprint.provider === "claude"); + assert.strictEqual(source?.status, "missing"); + assert.strictEqual(source?.scannedFiles, 0); + assert.strictEqual(source?.malformedRecords, 2); + }).pipe( + Effect.provide( + layerService({ prefix: "usage-service-missing-diagnostics", home, settings }), + ), + ); + }), ); it.effect("reports saved usage of a removed directory only for windows it reaches", () => @@ -3225,10 +3739,10 @@ describe("UsageService", () => { assert.deepStrictEqual(later.buckets, []); assert.strictEqual(later.sources[0]?.status, "missing"); assert.strictEqual(later.sources[0]?.scannedFiles, 0); - }).pipe(Effect.scoped, Effect.provide(TestClock.layer())), + }), ); - it.live("does not share an in-flight scan after custom prices change", () => + it.effect("does not share an in-flight scan after custom prices change", () => Effect.gen(function* () { const { transcript, settings, home } = yield* setup; yield* Effect.promise(() => NodeFSP.writeFile(transcript, claudeLine(1, 5, "example-model"))); @@ -3262,7 +3776,14 @@ describe("UsageService", () => { HttpClient.HttpClient, HttpClient.make((request) => Deferred.await(releaseRates).pipe( - Effect.as(HttpClientResponse.fromWeb(request, Response.json({}))), + Effect.as( + HttpClientResponse.fromWeb( + request, + Response.json({ + "example-model": { input_cost_per_token: 1e-6, output_cost_per_token: 1e-5 }, + }), + ), + ), ), ), ), @@ -3281,15 +3802,16 @@ describe("UsageService", () => { const original = yield* Fiber.join(first); const updated = yield* Fiber.join(second); - assert.strictEqual(original.buckets[0]?.costUsd, 0); + // Fetched rates price the first read, so scans must start while the fetch is pending. + assert.isAbove(original.buckets[0]?.costUsd ?? 0, 0); assert.closeTo(updated.buckets[0]?.costUsd ?? -1, 0.00006, 1e-12); }).pipe( Effect.provide(layerService({ prefix: "usage-service-price-race-test", home, settings })), ); - }).pipe(Effect.scoped), + }), ); - it.live.each(["network", "http", "invalidJson", "empty"] as const)( + it.effect.each(["network", "http", "invalidJson", "empty"] as const)( "recovers pricing through the GitHub API after a %s failure", (failure) => Effect.gen(function* () { @@ -3345,10 +3867,60 @@ describe("UsageService", () => { assert.strictEqual(cached.pricing.status, "cached"); assert.deepStrictEqual(cached.buckets, summary.buckets); assert.strictEqual(requests.length, 2); - }).pipe(Effect.scoped), + }), ); - it.live("does not orphan an in-flight scan when its first caller is interrupted", () => + it.effect("refetches cached rates early for a model they do not list", () => + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse("2026-08-02T00:00:00Z")); + const { transcript, settings, home } = yield* setup; + yield* Effect.promise(() => + NodeFSP.writeFile( + transcript, + claudeLine(1, 5, "claude-new-model") + claudeLine(2, 5, "claude-unknown-model"), + ), + ); + const requests: string[] = []; + let ratesDocument: Record = { + "claude-fable-5": { input_cost_per_token: 1e-5, output_cost_per_token: 5e-5 }, + }; + const serviceLayer = layerService({ + prefix: "usage-service-unlisted-model-test", + home, + settings, + onRatesFetch: (request) => requests.push(request.url), + ratesResponse: () => Effect.succeed(Response.json(ratesDocument)), + }); + const newModelCost = (summary: UsageSummary) => + summary.buckets.find((bucket) => bucket.model === "claude-new-model")?.costUsd; + + // Rates fetched for this scan are fresh: no second request for the models they lack. + const first = yield* (yield* UsageService.make.pipe( + Effect.provide(serviceLayer), + )).readSummary(WINDOW); + assert.strictEqual(newModelCost(first), 0); + assert.strictEqual(requests.length, 1); + + // An hour later LiteLLM lists the new model; a restarted server starts from the cached table. + ratesDocument = { + ...ratesDocument, + "claude-new-model": { input_cost_per_token: 2e-5, output_cost_per_token: 1e-4 }, + }; + yield* TestClock.adjust(Duration.hours(1)); + const service = yield* UsageService.make.pipe(Effect.provide(serviceLayer)); + const refreshed = yield* service.readSummary(WINDOW); + assert.strictEqual(refreshed.pricing.status, "fresh"); + assert.closeTo(newModelCost(refreshed) ?? -1, 10 * 2e-5 + 5 * 1e-4, 1e-12); + assert.strictEqual(requests.length, 2); + + // A model the fresh table still lacks does not refetch again before the TTL. + yield* TestClock.adjust(Duration.hours(1)); + yield* service.readSummary(WINDOW); + assert.strictEqual(requests.length, 2); + }), + ); + + it.effect("does not orphan an in-flight scan when its first caller is interrupted", () => Effect.gen(function* () { const { settings, home } = yield* setup; const service = yield* UsageService.make.pipe( @@ -3423,6 +3995,6 @@ describe("UsageService", () => { orphanedAt, `interruption left the next matching request pending at scheduler check ${orphanedAt}`, ); - }).pipe(Effect.scoped), + }), ); }); diff --git a/apps/server/src/usage/UsageService.ts b/apps/server/src/usage/UsageService.ts index 341b14d8c628..a59fbc66a02c 100644 --- a/apps/server/src/usage/UsageService.ts +++ b/apps/server/src/usage/UsageService.ts @@ -40,10 +40,17 @@ import type { UsageRecord, } from "@t3tools/provider-core/server/usage"; import { canonicalModelFilter, resolveModelAliases, UsageAggregator } from "./usageAggregation.ts"; -import { createOverrideRateTable, parseRateTable, type RateTable } from "./usagePricing.ts"; import { + createOverrideRateTable, + isUnlistedModel, + parseRateTable, + type RateTable, +} from "./usagePricing.ts"; +import { + DESKTOP_HOST_SCOPE_ENV, listTranscriptFiles, readDirectoryVolumeId, + readPhysicalSourceId, readTranscriptRecords, } from "./usageTranscriptReader.ts"; import { @@ -83,6 +90,12 @@ const scanReaders = BUILT_IN_USAGE_DRIVERS.flatMap((driver) => driver.usage?.kind === "scan" ? [{ driver, reader: driver.usage }] : [], ); +const providerRateModels = new Map( + BUILT_IN_USAGE_DRIVERS.flatMap(({ usage }) => + usage?.rateModel === undefined ? [] : [[usage.provider, usage.rateModel] as const], + ), +); + const transcriptFormats = new Map( transcriptReaders.map(({ reader }) => [reader.provider, reader.format] as const), ); @@ -92,6 +105,7 @@ interface TranscriptSource { readonly format: TranscriptUsageFormat; readonly dir: string; readonly volumeId: string; + readonly physicalSourceId?: string; readonly fileName?: string; } @@ -137,7 +151,10 @@ function sharedSessions( return shared; } const decodeCachedSources = Schema.decodeUnknownOption( - Schema.Struct({ sources: Schema.Record(Schema.String, CachedSource) }), + Schema.Struct({ + sources: Schema.Record(Schema.String, CachedSource), + coveredSince: Schema.optional(Schema.Record(Schema.String, Schema.Number)), + }), ); export class UsageService extends Context.Service< @@ -155,6 +172,24 @@ export const make = Effect.gen(function* () { const settingsService = yield* ServerSettings.ServerSettingsService; const httpClient = yield* HttpClient.HttpClient; const hostEnvironment = yield* HostProcess.Environment; + const hostPlatform = yield* HostProcess.Platform; + const hostScope = hostEnvironment[DESKTOP_HOST_SCOPE_ENV]?.trim(); + // A mount or a directory's on-disk name does not change under a running server. + // A miss is not cached: the directory may not exist yet. + const physicalSourceIds = new Map>(); + const physicalSourceId = (dir: string) => + Effect.promise(() => { + const cached = physicalSourceIds.get(dir); + if (cached !== undefined) return cached; + const id = readPhysicalSourceId(dir, hostScope, hostPlatform); + physicalSourceIds.set(dir, id); + void id.then((resolved) => { + if (resolved === undefined && physicalSourceIds.get(dir) === id) { + physicalSourceIds.delete(dir); + } + }); + return id; + }); const hostname = yield* HostProcess.Hostname; const readerContext = yield* Effect.context(); @@ -165,7 +200,15 @@ export const make = Effect.gen(function* () { const correctedBatch = (records: readonly UsageRecord[]) => records.some((record) => resolvedRecords.has(record)) ? records.map(correctedRecord) : records; const sourceCache = new Map(); + // Earliest window start a complete scan read each transcript directory from. + const coveredSince = new Map(); + const coverageKey = (source: TranscriptSource) => `${source.provider}\0${source.dir}`; let cacheDirty = false; + const cover = (source: TranscriptSource, windowStartMs: number) => { + if ((coveredSince.get(coverageKey(source)) ?? Infinity) <= windowStartMs) return; + coveredSince.set(coverageKey(source), windowStartMs); + cacheDirty = true; + }; const isWithinDirectory = (filePath: string, dir: string) => { const relative = path.relative(dir, filePath); return relative !== ".." && !relative.startsWith(".." + path.sep) && !path.isAbsolute(relative); @@ -235,27 +278,26 @@ export const make = Effect.gen(function* () { Effect.catchCause(() => Effect.succeed(null)), ); + // Outside ratesLock, which a slow fetch holds: a provisional answer prices with these. + const loadSavedRates = yield* Effect.cached( + fileSystem.readFileString(ratesCachePath).pipe( + Effect.flatMap((raw) => decodeRatesCache(raw)), + Effect.map((saved) => { + const parsed = parseRateTable(saved.document); + if (parsed.size === 0) return; + rates = parsed; + ratesFetchedAtMs = saved.fetchedAtMs; + ratesStatus = "cached"; + }), + Effect.ignoreCause, + ), + ); + const loadRates = Effect.fn("UsageService.loadRates")(function* (force: boolean) { const now = yield* Clock.currentTimeMillis; const maxAgeMs = force ? RATES_REFRESH_FLOOR_MS : RATES_TTL_MS; + yield* loadSavedRates; if (ratesFetchedAtMs !== null && now - ratesFetchedAtMs < maxAgeMs) return; - - if (ratesFetchedAtMs === null) { - const fromDisk = yield* fileSystem.readFileString(ratesCachePath).pipe( - Effect.flatMap((raw) => decodeRatesCache(raw)), - Effect.catchCause(() => Effect.succeed(null)), - ); - if (fromDisk !== null) { - const parsed = parseRateTable(fromDisk.document); - if (parsed.size > 0) { - rates = parsed; - ratesFetchedAtMs = fromDisk.fetchedAtMs; - ratesStatus = "cached"; - if (now - fromDisk.fetchedAtMs < maxAgeMs) return; - } - } - } - if (ratesLastFailedAtMs !== null && now - ratesLastFailedAtMs < RATES_REFRESH_FLOOR_MS) return; let fetched = yield* fetchRates(LITELLM_RATES_URL); @@ -281,6 +323,24 @@ export const make = Effect.gen(function* () { const ensureRates = (force: boolean) => ratesLock.withPermit(loadRates(force)); + // Rates are kept for RATES_TTL_MS, so a model released after the table was fetched would stay unpriced + // that long. A scan that meets a model the table does not list refetches early: once per model against + // a fresh table, with loadRates keeping RATES_REFRESH_FLOOR_MS between attempts. + const checkedUnlistedModels = new Set(); + const refreshForUnlistedModels = Effect.fn("UsageService.refreshForUnlistedModels")(function* ( + models: ReadonlySet, + ) { + const unlisted = [...models].filter( + (model) => !checkedUnlistedModels.has(model) && isUnlistedModel(rates, model), + ); + if (unlisted.length === 0) return; + yield* ensureRates(true); + const now = yield* Clock.currentTimeMillis; + // A failed fetch leaves the models unchecked, so a later scan tries again. + if (ratesFetchedAtMs === null || now - ratesFetchedAtMs >= RATES_REFRESH_FLOOR_MS) return; + for (const model of unlisted) checkedUnlistedModels.add(model); + }); + const refreshRates = ensureRates(true).pipe( Effect.map(pricing), Effect.withSpan("UsageService.refreshRates"), @@ -306,9 +366,8 @@ export const make = Effect.gen(function* () { > = Object.entries(settings.providerInstances) .filter(([, instance]) => instance.driver === driver.driverKind) .map(([id, instance]) => [ProviderInstanceId.make(id), instance, true] as const); - if (!Object.hasOwn(settings.providerInstances, driver.driverKind)) { - entries.push([ProviderInstanceId.make(driver.driverKind), {}, false]); - } + // History in the driver's default home stays counted after every instance moves elsewhere; readers dedupe shared directories. + entries.push([ProviderInstanceId.make(driver.driverKind), {}, false]); const decodeConfig = Schema.decodeUnknownOption(driver.configSchema); return yield* Effect.forEach( entries, @@ -375,11 +434,13 @@ export const make = Effect.gen(function* () { const key = `${provider}\0${dir}`; if (seen.has(key)) continue; seen.add(key); + const sourceId = yield* physicalSourceId(dir); dirs.push({ provider, format, dir, volumeId, + ...(sourceId === undefined ? {} : { physicalSourceId: sourceId }), ...(fileName === undefined ? {} : { fileName }), }); } @@ -396,19 +457,27 @@ export const make = Effect.gen(function* () { Effect.catchCause(() => Effect.succeed(null)), ); let document = yield* readDocument(scanCachePath); + const current = document !== null; for (const fileName of LEGACY_SCAN_CACHE_FILE_NAMES) { if (document !== null) break; document = yield* readDocument(path.join(config.stateDir, fileName)); cacheDirty = document !== null; } if (document === null) return; - for (const [path, entry] of decodeScanCache(document, transcriptFormats)) { - fileCache.set(path, entry); - } + const decoded = decodeScanCache(document, transcriptFormats); + for (const [path, entry] of decoded) fileCache.set(path, entry); + const saved = (document as { readonly files?: unknown }).files; + const complete = + typeof saved === "object" && saved !== null && Object.keys(saved).length === decoded.size; const sources = decodeCachedSources(document); if (Option.isSome(sources)) { for (const [key, source] of Object.entries(sources.value.sources)) sourceCache.set(key, source); + // Coverage describes the saved files, so it needs every one, in this version's format. + if (current && complete && fileCache.size > 0) { + for (const [key, since] of Object.entries(sources.value.coveredSince ?? {})) + coveredSince.set(key, since); + } } }), ); @@ -448,7 +517,10 @@ export const make = Effect.gen(function* () { if (cacheDirty) { cacheDirty = false; yield* Effect.sync(() => - writeScanCache(fileCache, { sources: Object.fromEntries(sourceCache) }), + writeScanCache(fileCache, { + sources: Object.fromEntries(sourceCache), + coveredSince: Object.fromEntries(coveredSince), + }), ).pipe( Effect.flatMap((contents) => writeCacheFile(scanCachePath, contents)), Effect.andThen( @@ -532,21 +604,44 @@ export const make = Effect.gen(function* () { failed: true, }; + const identity = (record: UsageRecord) => + encodeUsageRecordKey([ + record.provider, + record.sessionId, + record.timestampMs, + record.model, + record.rateModel, + record.totals, + record.reportedCostUsd, + record.dedupeKey, + ]); + if (!parsed.resumed && cached !== undefined && cached.provider === provider) { + // A whole-file re-parse re-emits saved requests unresolved. Keep their saved tier + // corrections, since the history that resolved them may no longer be available. + // Equal requests keep their own corrections by matching occurrences in order. + const saved = new Map(); + for (const previous of [...cached.records, ...cached.tailRecords]) { + const corrected = correctedRecord(previous); + const key = identity(corrected); + const occurrences = saved.get(key); + if (occurrences === undefined) saved.set(key, [corrected]); + else occurrences.push(corrected); + } + for (const record of [...parsed.records, ...parsed.tailRecords]) { + const corrected = saved.get(identity(record))?.shift(); + if ( + record.unresolvedServiceTierTurnId !== undefined && + corrected !== undefined && + corrected.unresolvedServiceTierTurnId === undefined + ) { + resolvedRecords.set(record, corrected); + } + } + } if (parsed.resumed && cached !== undefined && cached.tailRecords.length > 0) { // Completing a tentative line reparses it. Reuse its saved correction only when the // same usage is emitted again, without putting inferred tiers in the parser state. const reparsed = [...parsed.records, ...parsed.tailRecords]; - const identity = (record: UsageRecord) => - encodeUsageRecordKey([ - record.provider, - record.sessionId, - record.timestampMs, - record.model, - record.rateModel, - record.totals, - record.reportedCostUsd, - record.dedupeKey, - ]); for (const [index, previous] of cached.tailRecords.entries()) { const record = reparsed[index]; const corrected = correctedRecord(previous); @@ -585,6 +680,7 @@ export const make = Effect.gen(function* () { readonly provider: UsageProviderKind; readonly dir: string; readonly volumeId: string; + readonly physicalSourceId?: string; readonly hostId?: string; readonly status?: UsageSource["status"]; readonly message?: string; @@ -595,19 +691,33 @@ export const make = Effect.gen(function* () { readonly malformedRecords: number; }[] | null; + /** Every transcript path the walk listed, in walk order, whatever its mtime. */ + readonly walked?: readonly string[]; readonly refreshing?: true; } const scanTranscriptDir = Effect.fn("UsageService.scanTranscriptDir")(function* ( source: TranscriptSource, windowStartMs: number, + fromMemory: boolean, ) { - const { provider, format, dir, volumeId, fileName } = source; + const { provider, format, dir, volumeId, physicalSourceId, fileName } = source; + const identity = { + provider, + dir, + volumeId, + ...(physicalSourceId === undefined ? {} : { physicalSourceId }), + }; const exists = yield* fileSystem .exists(dir) .pipe(Effect.catchCause(() => Effect.succeed(false))); - if (!exists) return { provider, dir, volumeId, files: null } satisfies ScannedDir; - const { files, failedPaths } = yield* Effect.promise(() => + if (!exists) { + cover(source, windowStartMs); + return { ...identity, files: null } satisfies ScannedDir; + } + // scanSummary aggregates the saved files, as it does for a deleted directory. + if (fromMemory) return { ...identity, files: [] } satisfies ScannedDir; + const { files, walked, failedPaths } = yield* Effect.promise(() => listTranscriptFiles(dir, windowStartMs, fileName === undefined ? undefined : { fileName }), ); const read = yield* Effect.forEach( @@ -638,16 +748,23 @@ export const make = Effect.gen(function* () { } return { path, records, malformedRecords }; }); + const livePaths = new Set(files.map((file) => file.path)); + for (const file of read) { + if (file.failed || !file.path.endsWith(".jsonl.zst")) continue; + const originalPath = file.path.slice(0, -4); + // A successfully read archive replaces the original's saved records and diagnostics. + if (!livePaths.has(originalPath) && fileCache.delete(originalPath)) cacheDirty = true; + } const unread = failedPaths + read.filter((file) => file.failed).length; + if (unread === 0) cover(source, windowStartMs); yield* Effect.annotateCurrentSpan({ "usage.files": files.length, "usage.files_parsed": read.filter((file) => file.update !== undefined).length, }); return { - provider, - dir, - volumeId, + ...identity, files: parsedFiles, + walked, ...(unread > 0 ? { status: "partial", @@ -658,17 +775,13 @@ export const make = Effect.gen(function* () { }); const collectDirs = Effect.fn("UsageService.collectDirs")(function* ( + { dirs, pending }: Effect.Success>, windowStartMs: number, settings: ServerSettingsValue, retentionCutoffMs: number, awaitRefresh: boolean, + fromMemory: boolean, ) { - const { dirs, pending } = yield* resolveTranscriptDirs( - settings, - retentionCutoffMs, - awaitRefresh, - ).pipe(Effect.provideContext(readerContext)); - const scans = Effect.forEach( scanReaders, ({ driver, reader }) => @@ -679,14 +792,19 @@ export const make = Effect.gen(function* () { Effect.flatMap((sources) => Effect.forEach(sources, ({ volumeId, ...source }) => Effect.map( + // A reader's own volumeId names a remote account, not a local directory. volumeId === undefined - ? Effect.promise(() => readDirectoryVolumeId(source.dir)) - : Effect.succeed(volumeId), - (resolved): ScannedDir => ({ + ? Effect.all([ + Effect.promise(() => readDirectoryVolumeId(source.dir)), + physicalSourceId(source.dir), + ]) + : Effect.succeed<[string, string | undefined]>([volumeId, undefined]), + ([resolved, sourceId]): ScannedDir => ({ ...source, files: source.files?.map((file) => ({ ...file, malformedRecords: 0 })) ?? null, provider: reader.provider, volumeId: resolved, + ...(sourceId === undefined ? {} : { physicalSourceId: sourceId }), }), ), ), @@ -698,7 +816,7 @@ export const make = Effect.gen(function* () { // Scan independent sources together, directories sequentially to bound open files; keep order, since the first copy decides a duplicate's bucket. const [transcripts, scanDirs] = yield* Effect.all( - [Effect.forEach(dirs, (dir) => scanTranscriptDir(dir, windowStartMs)), scans], + [Effect.forEach(dirs, (dir) => scanTranscriptDir(dir, windowStartMs, fromMemory)), scans], { concurrency: "unbounded" }, ); const scanned: readonly ScannedDir[] = [...transcripts, ...pending, ...scanDirs.flat()]; @@ -708,6 +826,7 @@ export const make = Effect.gen(function* () { const scanSummary = Effect.fn("UsageService.scanSummary")(function* ( input: UsageSummaryInput, settings: ServerSettingsValue, + provisional: boolean, ) { if (input.sinceDay > input.untilDay) { return yield* new UsageReadError({ @@ -755,55 +874,145 @@ export const make = Effect.gen(function* () { const retentionCutoffMs = startedAtMs - CACHE_RETENTION_DAYS * 24 * 60 * 60 * 1000; - const [, scannedDirs] = yield* Effect.all( - [ - ensureRates(false), - collectDirs(windowStartMs, settings, retentionCutoffMs, input.awaitRefresh === true), - ], + const awaitRefresh = input.awaitRefresh === true; + const collect = Effect.gen(function* () { + const resolved = yield* resolveTranscriptDirs(settings, retentionCutoffMs, awaitRefresh).pipe( + Effect.provideContext(readerContext), + ); + // A window no complete scan has read would show partial totals, so every directory must cover it. + // Saved files are keyed by path, so two providers reading one directory keep only one parse. + const fromMemory = + provisional && + resolved.dirs.length > 0 && + new Set(resolved.dirs.map((source) => source.dir)).size === resolved.dirs.length && + resolved.dirs.every( + (source) => (coveredSince.get(coverageKey(source)) ?? Infinity) <= windowStartMs, + ); + const [, scanned] = yield* Effect.all( + [ + fromMemory ? loadSavedRates : provisional ? ensureRates(false) : Effect.void, + collectDirs( + resolved, + windowStartMs, + settings, + retentionCutoffMs, + awaitRefresh, + fromMemory, + ), + ], + { concurrency: 2 }, + ); + return { scannedDirs: scanned, fromMemory }; + }); + const [{ scannedDirs, fromMemory }] = yield* Effect.all( + [collect, provisional ? Effect.void : ensureRates(false)], { concurrency: 2 }, ); - const aggregator = new UsageAggregator({ - timeZone: input.timeZone, - sinceDay: input.sinceDay, - untilDay: input.untilDay, - resolution: input.resolution ?? "day", - ...hourlyWindow, - rates, - priceOverrides: createOverrideRateTable(settings.usagePriceOverrides), - modelAliases: resolveModelAliases(settings.usageModelAliases), - ...(input.modelFilter === undefined ? {} : { modelFilter: input.modelFilter }), - }); - + const priceOverrides = createOverrideRateTable(settings.usagePriceOverrides); + const modelAliases = resolveModelAliases(settings.usageModelAliases); const sources: UsageSource[] = []; // Deleted transcripts' saved usage still aggregates and deduplicates normally. const retainedSinceMs = Math.max(windowStartMs, retentionCutoffMs); + // Saved paths are files below a scanned root, so their folder decides; folders repeat thousands of times. + const withinDir = (dir: string) => { + const foldersWithin = new Map(); + return (filePath: string) => { + const folder = path.dirname(filePath); + let within = foldersWithin.get(folder); + if (within === undefined) { + within = isWithinDirectory(folder, dir); + foldersWithin.set(folder, within); + } + return within; + }; + }; const filesByDir = scannedDirs.map(({ provider, dir, files }) => { const retainedFiles = [...(files ?? [])]; const livePaths = new Set(retainedFiles.map((file) => file.path)); + const isRetained = withinDir(dir); for (const [filePath, entry] of fileCache) { if ( entry.provider !== provider || entry.mtimeMs < retainedSinceMs || livePaths.has(filePath) || - !isWithinDirectory(filePath, dir) + !isRetained(filePath) ) continue; retainedFiles.push({ path: filePath, - records: [...entry.records, ...entry.tailRecords], + records: + entry.tailRecords.length === 0 + ? entry.records + : [...entry.records, ...entry.tailRecords], malformedRecords: entry.malformedRecords, }); } return retainedFiles; }); + // Usage two transcripts share counts in the first one, so saved files take the order an exact + // scan reads them in: walked files in walk order, then the directory's deleted files. + for (const { provider, dir, walked } of fromMemory ? [] : scannedDirs) { + if (walked === undefined) continue; + const within = withinDir(dir); + const saved = [...fileCache.keys()]; + const members = new Set( + saved.filter( + (filePath) => fileCache.get(filePath)!.provider === provider && within(filePath), + ), + ); + if (members.size < 2) continue; + const listed = new Set(walked); + const target = [ + ...walked.filter((filePath) => members.has(filePath)), + ...saved.filter((filePath) => members.has(filePath) && !listed.has(filePath)), + ]; + let next = 0; + const ordered = saved.map((filePath) => (members.has(filePath) ? target[next++]! : filePath)); + if (ordered.every((filePath, index) => filePath === saved[index])) continue; + const entries = new Map(fileCache); + fileCache.clear(); + for (const filePath of ordered) fileCache.set(filePath, entries.get(filePath)!); + cacheDirty = true; + } + + // The models priced below, as UsageAggregator resolves them: an alias replaces rateModel. + const rateModels = new Set(); + for (const files of filesByDir) { + for (const file of files) { + for (const record of file.records) { + const alias = modelAliases.get(record.model); + const model = alias ?? record.model; + if (priceOverrides.has(model.trim())) continue; + rateModels.add(alias === undefined ? (record.rateModel ?? model) : model); + } + } + } + // A saved answer prices with saved rates and the full scan after it refetches. + if (!fromMemory) yield* refreshForUnlistedModels(rateModels); + + const summaryPricing = pricing(); + + const aggregator = new UsageAggregator({ + timeZone: input.timeZone, + sinceDay: input.sinceDay, + untilDay: input.untilDay, + resolution: input.resolution ?? "day", + ...hourlyWindow, + rates, + priceOverrides, + modelAliases, + rateModel: (provider, model) => providerRateModels.get(provider)?.(model), + ...(input.modelFilter === undefined ? {} : { modelFilter: input.modelFilter }), + }); + const windowEndMs = hourlyWindow?.untilTimeMs ?? Date.parse(`${input.untilDay}T00:00:00Z`) + 86400000 + MTIME_SLACK_MS; let resolvedAny = false; for (const { reader } of transcriptReaders) { - if (reader.resolveServiceTiers === undefined) continue; + if (reader.resolveServiceTiers === undefined || fromMemory) continue; const pending = new Set(); for (const [index, files] of filesByDir.entries()) { if (scannedDirs[index]?.provider !== reader.provider) continue; @@ -845,7 +1054,17 @@ export const make = Effect.gen(function* () { for (const [ index, - { provider, dir, volumeId, files, status, message, refreshing, hostId: sourceHostId }, + { + provider, + dir, + volumeId, + physicalSourceId, + files, + status, + message, + refreshing, + hostId: sourceHostId, + }, ] of scannedDirs.entries()) { let scannedFiles = 0; let skippedFiles = 0; @@ -853,6 +1072,7 @@ export const make = Effect.gen(function* () { // Distinct per directory. Buckets carry per-cell session counts, but a // session spans days and models, so clients total this figure instead. const sessionIds = new Set(); + let counted = false; for (const file of filesByDir[index] ?? []) { malformedRecords += file.malformedRecords; @@ -881,8 +1101,9 @@ export const make = Effect.gen(function* () { eventOccurrences.set(key, occurrence); usageRecord = { ...record, dedupeKey: key + ":" + occurrence }; } - if (aggregator.add(usageRecord, dir) && record.sessionId.length > 0) { - sessionIds.add(record.sessionId); + if (aggregator.add(usageRecord, dir)) { + counted = true; + if (record.sessionId.length > 0) sessionIds.add(record.sessionId); } } } @@ -890,7 +1111,13 @@ export const make = Effect.gen(function* () { const sourceMessage = message ?? (files === null ? "No transcript directory on this environment." : null); sources.push({ - fingerprint: { hostId: sourceHostId ?? hostId, provider, resolvedHomePath: dir, volumeId }, + fingerprint: { + hostId: sourceHostId ?? hostId, + provider, + resolvedHomePath: dir, + volumeId, + ...(physicalSourceId === undefined ? {} : { physicalSourceId }), + }, status: files === null && scannedFiles === 0 ? "missing" @@ -909,18 +1136,28 @@ export const make = Effect.gen(function* () { .filter(Boolean) .join(" ") : sourceMessage, - ...(refreshing ? { refreshing } : {}), + // A saved answer misses what changed since its scan, so the client must ask again. An + // empty source stays unmarked: it would only flash a loading row. + ...(refreshing || (fromMemory && counted && transcriptFormats.has(provider)) + ? { refreshing: true } + : {}), }); } const pruned = pruneScanCache(fileCache, retentionCutoffMs); - if (pruned > 0) cacheDirty = true; + if (pruned > 0) { + cacheDirty = true; + // Pruned files are no longer saved, so nothing before the cutoff stays covered. + for (const [key, since] of coveredSince) { + if (since < retentionCutoffMs) coveredSince.set(key, retentionCutoffMs); + } + } const aggregated = aggregator.finish(); const readAt = yield* DateTime.now; const finishedAtMs = yield* Clock.currentTimeMillis; - return { + const summary = { contractVersion: USAGE_CONTRACT_VERSION, readAt: DateTime.formatIso(readAt), timeZone: input.timeZone, @@ -928,15 +1165,32 @@ export const make = Effect.gen(function* () { untilDay: input.untilDay, buckets: aggregated.buckets, sources, - pricing: pricing(), + pricing: summaryPricing, scanDurationMs: Math.max(0, finishedAtMs - startedAtMs), ...(input.modelFilter === undefined ? {} : { modelFilter: input.modelFilter }), } satisfies UsageSummary; + return { summary, fromMemory }; + }); + + const readWindow = Effect.fn("UsageService.readWindow")(function* ( + input: UsageSummaryInput, + settings: ServerSettingsValue, + provisional: boolean, + ) { + const { summary, fromMemory } = yield* scanSummary(input, settings, provisional); + // With nothing marked refreshing the client would keep a saved answer as final. + return fromMemory && !summary.sources.some((source) => source.refreshing === true) + ? (yield* scanSummary(input, settings, false)).summary + : summary; }); const inflightScans = new Map>(); - const scanKey = (input: UsageSummaryInput, settings: ServerSettingsValue): string => + const scanKey = ( + input: UsageSummaryInput, + settings: ServerSettingsValue, + provisional: boolean, + ): string => JSON.stringify([ input.timeZone, input.sinceDay, @@ -946,6 +1200,7 @@ export const make = Effect.gen(function* () { input.untilTime ?? null, settings, input.awaitRefresh === true, + provisional, input.modelFilter ?? null, ]); @@ -956,7 +1211,8 @@ export const make = Effect.gen(function* () { const modelFilter = canonicalModelFilter(requestedFilter); const input = modelFilter === undefined ? unfiltered : { ...unfiltered, modelFilter }; const settings = yield* readSettings; - const key = scanKey(input, settings); + const provisional = input.provisional === true && input.awaitRefresh !== true; + const key = scanKey(input, settings, provisional); const deferred = yield* Effect.uninterruptible( Effect.gen(function* () { const existing = inflightScans.get(key); @@ -967,7 +1223,7 @@ export const make = Effect.gen(function* () { inflightScans.set(key, created); // Detach so caller departure cannot cancel shared scans; finished scans warm the cache. // Register writes before releasing waiters, but start writing after delivering their summary. - yield* scanSummary(input, settings).pipe( + yield* readWindow(input, settings, provisional).pipe( Effect.onExit((exit) => Effect.sync(() => inflightScans.delete(key)).pipe( Effect.andThen(Exit.isSuccess(exit) ? schedulePersist : Effect.void), diff --git a/apps/server/src/usage/cliproxyApi.test.ts b/apps/server/src/usage/cliproxyApi.test.ts index cf7d65b5df57..12db11364d61 100644 --- a/apps/server/src/usage/cliproxyApi.test.ts +++ b/apps/server/src/usage/cliproxyApi.test.ts @@ -423,6 +423,26 @@ describe("CLIProxyAPI built-in management API", () => { }), ); + it.effect("keeps a Claude account whose window reports no utilization", () => + Effect.gen(function* () { + const test = fixture({ + accounts: [{ ...accounts[0]!, provider: "claude" }], + upstream: () => ({ + status: 200, + body: { + five_hour: { utilization: null, resets_at: null }, + seven_day: { utilization: 50, resets_at: "2099-01-01T00:00:00Z" }, + }, + }), + }); + const result = yield* (yield* test.api).readAccounts(config); + expect(result[0]?.usageLimits.unavailable).toBeUndefined(); + expect( + result[0]?.usageLimits.windows.map((window) => [window.id, window.usedPercent]), + ).toEqual([["seven_day", 50]]); + }), + ); + const antigravitySummary = { buckets: [ { diff --git a/apps/server/src/usage/cliproxyApi.ts b/apps/server/src/usage/cliproxyApi.ts index 7f3888129b71..3ad5d3ec55d0 100644 --- a/apps/server/src/usage/cliproxyApi.ts +++ b/apps/server/src/usage/cliproxyApi.ts @@ -70,8 +70,9 @@ const CodexUsage = Schema.Struct({ }), ), }); +// The usage endpoint can report a window without a utilization, as the CLI's own schema allows. const ClaudeWindow = Schema.Struct({ - utilization: Schema.Number, + utilization: Schema.NullOr(Schema.Number), resets_at: Schema.NullOr(Schema.String), }); const ClaudeUsage = Schema.Struct({ diff --git a/apps/server/src/usage/usageAggregation.test.ts b/apps/server/src/usage/usageAggregation.test.ts index 625f7a9d63ec..58953fbcfafe 100644 --- a/apps/server/src/usage/usageAggregation.test.ts +++ b/apps/server/src/usage/usageAggregation.test.ts @@ -12,11 +12,13 @@ const rates: RateTable = new Map([ outputCostPerToken: 5e-5, cacheReadCostPerToken: 1e-6, cacheCreationCostPerToken: 1.25e-5, + cacheCreation1hCostPerToken: 2e-5, fast: { inputCostPerToken: 2e-5, outputCostPerToken: 1e-4, cacheReadCostPerToken: 2e-6, cacheCreationCostPerToken: 2.5e-5, + cacheCreation1hCostPerToken: 4e-5, }, ultrafast: null, contextTiers: [], @@ -139,6 +141,32 @@ describe("UsageAggregator", () => { } }); + it("keeps a live Grok turn over a saved pre-v17 maximum in either source order", () => { + const turn = (timestampMs: number, outputTokens: number) => + record({ + provider: "grok", + model: "grok-4.5", + timestampMs, + dedupeKey: `s:p:${timestampMs}:grok-4.5`, + reportedCostUsd: outputTokens / 10, + totals: { ...record().totals, uncachedInputTokens: 0, outputTokens }, + }); + const first = turn(Date.parse("2026-08-08T10:00:00Z"), 10); + const resumed = turn(Date.parse("2026-08-08T11:00:00Z"), 20); + const savedMaximum = { ...first, totals: resumed.totals, reportedCostUsd: 2 }; + + const expected = aggregate([first, resumed]).buckets; + + for (const order of [ + [first, resumed, savedMaximum], + [savedMaximum, first, resumed], + ]) { + const result = aggregate(order); + expect(result.duplicatesDropped).toBe(1); + expect(result.buckets).toEqual(expected); + } + }); + it("does not count a key whose first copy fell outside the window", () => { const result = aggregate([ record({ dedupeKey: "msg_1:", timestampMs: Date.parse("2026-07-01T12:00:00Z") }), @@ -249,6 +277,7 @@ describe("UsageAggregator", () => { untilDay: "2026-08-31", rates, modelAliases: resolveModelAliases({ "example-preview": "claude-fable-5" }), + rateModel: () => "claude-fable", }); aggregator.add(record()); aggregator.add(record({ model: "example-preview", rateModel: "example-preview-high" })); @@ -264,20 +293,24 @@ describe("UsageAggregator", () => { describe("resolveModelAliases", () => { it("follows chains to the final model and drops chains that enter a loop", () => { - expect( - resolveModelAliases({ + expect([ + ...resolveModelAliases({ "preview[1m]": "preview", preview: "example-model", + shared: "preview", + intoLoop: "loop", loop: "back", back: "loop", - intoLoop: "loop", + laterIntoLoop: "back", + deepIntoLoop: "intoLoop", self: "self", + validAfterLoops: "shared", }), - ).toEqual( - new Map([ - ["preview[1m]", "example-model"], - ["preview", "example-model"], - ]), - ); + ]).toEqual([ + ["preview[1m]", "example-model"], + ["preview", "example-model"], + ["shared", "example-model"], + ["validAfterLoops", "example-model"], + ]); }); }); diff --git a/apps/server/src/usage/usageAggregation.ts b/apps/server/src/usage/usageAggregation.ts index 0bdf1b8cc261..f4c39753548f 100644 --- a/apps/server/src/usage/usageAggregation.ts +++ b/apps/server/src/usage/usageAggregation.ts @@ -10,7 +10,7 @@ import type { } from "@t3tools/contracts"; import { EMPTY_TOTALS, type UsageRecord } from "@t3tools/provider-core/server/usage"; -import { cacheSavingsUsd, priceUsage, type RateTable } from "./usagePricing.ts"; +import { cacheSavingsUsd, lookupRate, priceUsage, type RateTable } from "./usagePricing.ts"; // en-CA gives ISO-ordered parts in the requested zone; Date getters are host-local. function makeDayFormatter(timeZone: string): (timestampMs: number) => string { @@ -58,6 +58,7 @@ interface MutableBucket { speedPremiumUsd: number; records: number; unpricedRecords: number; + unpricedTokens: number; providerReportedRecords: number; sessions: Set; } @@ -69,6 +70,7 @@ export interface AggregateOptions { readonly rates: RateTable; readonly priceOverrides?: RateTable; readonly modelAliases?: ReadonlyMap; + readonly rateModel?: (provider: UsageRecord["provider"], model: string) => string | undefined; readonly modelFilter?: UsageModelFilter; readonly resolution?: UsageResolution; readonly sinceTimeMs?: number; @@ -78,15 +80,26 @@ export interface AggregateOptions { export function resolveModelAliases( aliases: Readonly>, ): ReadonlyMap { - const resolved = new Map(); - for (const model of Object.keys(aliases)) { - const seen = new Set([model]); - let target = aliases[model]!; - while (Object.hasOwn(aliases, target) && !seen.has(target)) { - seen.add(target); + const models = Object.keys(aliases); + const memo = new Map(); + for (const model of models) { + if (memo.has(model)) continue; + const path: string[] = []; + let target = model; + while (Object.hasOwn(aliases, target) && !memo.has(target)) { + path.push(target); + // Null marks both the active walk and paths that end in a cycle. + memo.set(target, null); target = aliases[target]!; } - if (!Object.hasOwn(aliases, target)) resolved.set(model, target); + const terminal = Object.hasOwn(aliases, target) ? (memo.get(target) ?? null) : target; + for (const alias of path) memo.set(alias, terminal); + } + + const resolved = new Map(); + for (const model of models) { + const target = memo.get(model); + if (target !== null && target !== undefined) resolved.set(model, target); } return resolved; } @@ -146,12 +159,18 @@ export function mergeUsageCopies(held: UsageRecord, copy: UsageRecord): UsageRec b.cacheCreationTokens <= a.cacheCreationTokens && b.outputTokens <= a.outputTokens && b.reasoningTokens <= a.reasoningTokens && + (copy.cacheCreation1hTokens ?? 0) <= (held.cacheCreation1hTokens ?? 0) && reportedCostUsd === held.reportedCostUsd ) { return held; } + const cacheCreation1hTokens = Math.max( + held.cacheCreation1hTokens ?? 0, + copy.cacheCreation1hTokens ?? 0, + ); return { ...held, + ...(cacheCreation1hTokens > 0 ? { cacheCreation1hTokens } : {}), totals: { uncachedInputTokens: Math.max(a.uncachedInputTokens, b.uncachedInputTokens), cachedInputTokens: Math.max(a.cachedInputTokens, b.cachedInputTokens), @@ -163,6 +182,30 @@ export function mergeUsageCopies(held: UsageRecord, copy: UsageRecord): UsageRec }; } +/** + * Grok reports each turn once, complete, so real copies are equal. A larger one is a deleted + * transcript's pre-v17 maximum over turns that shared a prompt id, whichever source it came from. + */ +function smallestGrokCopy(held: UsageRecord, copy: UsageRecord): UsageRecord { + const a = held.totals; + const b = copy.totals; + const reportedCostUsd = + held.reportedCostUsd === null || copy.reportedCostUsd === null + ? (held.reportedCostUsd ?? copy.reportedCostUsd) + : Math.min(held.reportedCostUsd, copy.reportedCostUsd); + return { + ...held, + totals: { + uncachedInputTokens: Math.min(a.uncachedInputTokens, b.uncachedInputTokens), + cachedInputTokens: Math.min(a.cachedInputTokens, b.cachedInputTokens), + cacheCreationTokens: Math.min(a.cacheCreationTokens, b.cacheCreationTokens), + outputTokens: Math.min(a.outputTokens, b.outputTokens), + reasoningTokens: Math.min(a.reasoningTokens, b.reasoningTokens), + }, + reportedCostUsd, + }; +} + interface Contribution { record: UsageRecord; /** Null when the first copy fell outside the window. */ @@ -210,7 +253,12 @@ export class UsageAggregator { const held = this.#seen.get(record.dedupeKey); if (held !== undefined) { this.#duplicatesDropped += 1; - if (held.bucket !== null) held.record = mergeUsageCopies(held.record, record); + if (held.bucket !== null) { + held.record = + record.provider === "grok" + ? smallestGrokCopy(held.record, record) + : mergeUsageCopies(held.record, record); + } return false; } } @@ -283,16 +331,27 @@ export class UsageAggregator { this.#options.priceOverrides, ); bucket.records += 1; - if (priced.costSource === "unpriced") bucket.unpricedRecords += 1; + if (priced.costSource === "unpriced") { + bucket.unpricedRecords += 1; + bucket.unpricedTokens += + record.totals.uncachedInputTokens + + record.totals.cachedInputTokens + + record.totals.cacheCreationTokens + + record.totals.outputTokens; + } if (priced.costSource === "providerReported") bucket.providerReportedRecords += 1; } - // Aliases use the target's rate, replacing provider-specific rateModel. + // Aliases use the target's rate. Only an unpriced target goes through the provider's own rate naming. #mapModel(record: UsageRecord): UsageRecord { const model = this.#options.modelAliases?.get(record.model); if (model === undefined) return record; const { rateModel: _rateModel, ...rest } = record; - return { ...rest, model }; + const rateModel = + lookupRate(this.#options.rates, model) === null + ? this.#options.rateModel?.(record.provider, model) + : undefined; + return rateModel === undefined ? { ...rest, model } : { ...rest, model, rateModel }; } // Time-ordered records usually reuse the previous bucket; avoid rebuilding its key. @@ -330,6 +389,7 @@ export class UsageAggregator { speedPremiumUsd: 0, records: 0, unpricedRecords: 0, + unpricedTokens: 0, providerReportedRecords: 0, sessions: new Set(), }; @@ -377,6 +437,7 @@ export class UsageAggregator { costSource: resolveCostSource(bucket), records: bucket.records, unpricedRecords: bucket.unpricedRecords, + unpricedTokens: bucket.unpricedTokens, sessions: bucket.sessions.size, }); } diff --git a/apps/server/src/usage/usagePricing.ts b/apps/server/src/usage/usagePricing.ts index 4a04116db2df..a9ffc691643f 100644 --- a/apps/server/src/usage/usagePricing.ts +++ b/apps/server/src/usage/usagePricing.ts @@ -12,6 +12,8 @@ export interface TokenRates { readonly outputCostPerToken: number; readonly cacheReadCostPerToken: number; readonly cacheCreationCostPerToken: number; + /** Cache writes with a 1-hour TTL; the 5-minute rate when unpublished. */ + readonly cacheCreation1hCostPerToken: number; } interface SpeedRates extends TokenRates { @@ -37,20 +39,24 @@ export function createOverrideRateTable( overrides: Readonly>, ): RateTable { return new Map( - Object.entries(overrides).map(([model, prices]) => [ - model.trim(), - { - inputCostPerToken: prices.inputCostPerMillionTokens / 1_000_000, - outputCostPerToken: prices.outputCostPerMillionTokens / 1_000_000, - cacheReadCostPerToken: - (prices.cacheReadCostPerMillionTokens ?? prices.inputCostPerMillionTokens) / 1_000_000, - cacheCreationCostPerToken: - (prices.cacheWriteCostPerMillionTokens ?? prices.inputCostPerMillionTokens) / 1_000_000, - fast: null, - ultrafast: null, - contextTiers: [], - }, - ]), + Object.entries(overrides).map(([model, prices]) => { + const cacheCreationCostPerToken = + (prices.cacheWriteCostPerMillionTokens ?? prices.inputCostPerMillionTokens) / 1_000_000; + return [ + model.trim(), + { + inputCostPerToken: prices.inputCostPerMillionTokens / 1_000_000, + outputCostPerToken: prices.outputCostPerMillionTokens / 1_000_000, + cacheReadCostPerToken: + (prices.cacheReadCostPerMillionTokens ?? prices.inputCostPerMillionTokens) / 1_000_000, + cacheCreationCostPerToken, + cacheCreation1hCostPerToken: cacheCreationCostPerToken, + fast: null, + ultrafast: null, + contextTiers: [], + }, + ]; + }), ); } @@ -60,7 +66,7 @@ function finiteNumber(value: unknown): number | null { return typeof value === "number" && Number.isFinite(value) ? value : null; } -// Missing cache rates fall back to input pricing; Anthropic normally discounts reads and charges extra for writes. +// Missing cache rates fall back to input pricing, and a missing 1-hour write rate to the 5-minute one. function readTokenRates( entry: LiteLlmEntry, suffix: string, @@ -70,18 +76,28 @@ function readTokenRates( const input = finiteNumber(entry[`input_cost_per_token${suffix}`]); const output = finiteNumber(entry[`output_cost_per_token${suffix}`]) ?? outputFallback ?? null; if (input === null || output === null) return null; - const cacheRate = (name: string, field: "cacheReadCostPerToken" | "cacheCreationCostPerToken") => + const cacheRate = ( + name: string, + field: "cacheReadCostPerToken" | "cacheCreationCostPerToken" | "cacheCreation1hCostPerToken", + fallback = input, + ) => finiteNumber(entry[`${name}${suffix}`]) ?? (standard !== undefined && standard.inputCostPerToken > 0 ? (standard[field] / standard.inputCostPerToken) * input - : input); + : fallback); + const cacheCreationCostPerToken = cacheRate( + "cache_creation_input_token_cost", + "cacheCreationCostPerToken", + ); return { inputCostPerToken: input, outputCostPerToken: output, cacheReadCostPerToken: cacheRate("cache_read_input_token_cost", "cacheReadCostPerToken"), - cacheCreationCostPerToken: cacheRate( - "cache_creation_input_token_cost", - "cacheCreationCostPerToken", + cacheCreationCostPerToken, + cacheCreation1hCostPerToken: cacheRate( + "cache_creation_input_token_cost_above_1hr", + "cacheCreation1hCostPerToken", + cacheCreationCostPerToken, ), }; } @@ -92,6 +108,7 @@ function scaleTokenRates(rates: TokenRates, multiple: number): TokenRates { outputCostPerToken: rates.outputCostPerToken * multiple, cacheReadCostPerToken: rates.cacheReadCostPerToken * multiple, cacheCreationCostPerToken: rates.cacheCreationCostPerToken * multiple, + cacheCreation1hCostPerToken: rates.cacheCreation1hCostPerToken * multiple, }; } @@ -192,7 +209,8 @@ function sameTokenRates(a: TokenRates | null, b: TokenRates | null): boolean { a.inputCostPerToken === b.inputCostPerToken && a.outputCostPerToken === b.outputCostPerToken && a.cacheReadCostPerToken === b.cacheReadCostPerToken && - a.cacheCreationCostPerToken === b.cacheCreationCostPerToken + a.cacheCreationCostPerToken === b.cacheCreationCostPerToken && + a.cacheCreation1hCostPerToken === b.cacheCreation1hCostPerToken ); } @@ -292,6 +310,14 @@ export function lookupRate(table: RateTable, model: string): ModelRate | null { return rate; } +/** A priceable model the table has no rate for; a newer table may list it. */ +export function isUnlistedModel(table: RateTable, model: string): boolean { + const bareName = bareModelName(stripVariantSuffix(normalizeRateKey(model))); + return ( + bareName.length > 0 && !UNPRICEABLE_MODELS.has(bareName) && lookupRate(table, model) === null + ); +} + function resolveRate(table: RateTable, model: string): ModelRate | null { const key = stripVariantSuffix(normalizeRateKey(model)); const bareName = bareModelName(key); @@ -301,7 +327,7 @@ function resolveRate(table: RateTable, model: string): ModelRate | null { export type PricedRecord = Pick< UsageRecord, - "model" | "rateModel" | "totals" | "speed" | "reportedCostUsd" + "model" | "rateModel" | "totals" | "cacheCreation1hTokens" | "speed" | "reportedCostUsd" >; function recordRate(table: RateTable, record: PricedRecord): ModelRate | null { @@ -316,11 +342,15 @@ export interface PricedUsage { readonly speedPremiumUsd: number; } -function costByCategory(totals: UsageTokenTotals, rates: TokenRates): UsageCategoryCost { +function costByCategory(record: PricedRecord, rates: TokenRates): UsageCategoryCost { + const { totals } = record; + const oneHour = Math.min(record.cacheCreation1hTokens ?? 0, totals.cacheCreationTokens); return { input: totals.uncachedInputTokens * rates.inputCostPerToken, cacheRead: totals.cachedInputTokens * rates.cacheReadCostPerToken, - cacheWrite: totals.cacheCreationTokens * rates.cacheCreationCostPerToken, + cacheWrite: + (totals.cacheCreationTokens - oneHour) * rates.cacheCreationCostPerToken + + oneHour * rates.cacheCreation1hCostPerToken, output: totals.outputTokens * rates.outputCostPerToken, }; } @@ -352,13 +382,13 @@ export function priceUsage( return reported === null ? unsplit(0, "unpriced") : unsplit(reported, "providerReported"); } - const listCost = costByCategory(totals, ratesAt(rate, record.speed, totals)); + const listCost = costByCategory(record, ratesAt(rate, record.speed, totals)); const listCostUsd = sumCategories(listCost); if (reported !== null && listCostUsd <= 0) return unsplit(reported, "providerReported"); const premiumUsd = record.speed === "standard" ? 0 - : listCostUsd - sumCategories(costByCategory(totals, ratesAt(rate, "standard", totals))); + : listCostUsd - sumCategories(costByCategory(record, ratesAt(rate, "standard", totals))); const scale = reported === null ? 1 : reported / listCostUsd; return { costUsd: reported ?? listCostUsd, diff --git a/apps/server/src/usage/usageScanCache.test.ts b/apps/server/src/usage/usageScanCache.test.ts index 2866a295da8f..36864768eb52 100644 --- a/apps/server/src/usage/usageScanCache.test.ts +++ b/apps/server/src/usage/usageScanCache.test.ts @@ -65,7 +65,15 @@ describe("scan cache round trip", () => { [ "/a.jsonl", 100, - [record(), record({ dedupeKey: "msg_2:", model: "claude-opus-5-5", speed: "fast" })], + [ + record(), + record({ + dedupeKey: "msg_2:", + model: "claude-opus-5-5", + speed: "fast", + cacheCreation1hTokens: 8, + }), + ], ], [ "/b.jsonl", @@ -160,7 +168,7 @@ describe("scan cache round trip", () => { expect(restored).toEqual(original); }); - it.each([5, 6, 7, 8, 10])( + it.each([5, 6, 7, 8, 10, 11, 12, 13])( "retains v%s history and compatible positions but invalidates obsolete state", (version) => { const original = cacheWith([["/gone.jsonl", 100, [record({ speed: "fast" })]]]); @@ -217,11 +225,15 @@ describe("scan cache round trip", () => { }; const restored = decodeScanCache(JSON.parse(JSON.stringify(legacy)), TEST_FORMAT_MAP); - expect(restored.get("/gone.jsonl")).toEqual({ - ...original.get("/gone.jsonl"), - size: -1, - position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null }, - }); + expect(restored.get("/gone.jsonl")).toEqual( + version >= 13 + ? original.get("/gone.jsonl") + : { + ...original.get("/gone.jsonl"), + size: -1, + position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null }, + }, + ); expect(restored.get("/gone-codex.jsonl")).toEqual({ ...original.get("/gone-codex.jsonl"), size: -1, @@ -229,7 +241,7 @@ describe("scan cache round trip", () => { }); expect(restored.get("/clean-codex.jsonl")).toEqual({ ...cleanCodex, - size: version >= 8 ? cleanCodex.size : -1, + size: -1, position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null }, }); expect(restored.get("/gone-pi.jsonl")).toEqual( @@ -244,11 +256,82 @@ describe("scan cache round trip", () => { }, ); - it("re-parses a Claude transcript cached before partial copies were merged", () => { + it("re-parses a v14 Codex entry that skipped records and restarts a clean one when it grows", () => { + const codex = (malformedRecords: number): CachedFile => ({ + size: 10, + mtimeMs: 100, + provider: "codex", + records: [record({ provider: "codex", model: "gpt-6-astra", dedupeKey: null })], + tailRecords: [], + malformedRecords, + position: position({ + state: { ...initialCodexScanState(), model: "gpt-6-astra", sessionId: "session-a" }, + }), + }); + const original: ScanCache = new Map([ + ["/skipped-codex.jsonl", codex(1)], + ["/clean-codex.jsonl", codex(0)], + ]); + const v14 = { ...parse(makeScanCacheWriter()(original, {})), version: 14 }; + + const restored = decodeScanCache(JSON.parse(JSON.stringify(v14)), TEST_FORMAT_MAP); + expect(restored.get("/skipped-codex.jsonl")).toEqual({ + ...codex(1), + size: -1, + position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null }, + }); + // Served while unchanged, but its state predates the latest snapshot time, so it cannot resume. + expect(restored.get("/clean-codex.jsonl")).toEqual({ + ...codex(0), + position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null }, + }); + // The migrated entries persist without a state and load again under the current version. + expect(decodeScanCache(parse(makeScanCacheWriter()(restored, {})), TEST_FORMAT_MAP)).toEqual( + restored, + ); + }); + + it("restarts a grown v15 Codex entry, whose baseline time can be older than its totals", () => { + const request = { + input_tokens: 100, + cached_input_tokens: 0, + cache_write_input_tokens: 0, + output_tokens: 20, + reasoning_output_tokens: 0, + total_tokens: 120, + }; + const entry: CachedFile = { + size: 10, + mtimeMs: 100, + provider: "codex", + records: [record({ provider: "codex", model: "gpt-6-astra", dedupeKey: null })], + tailRecords: [], + malformedRecords: 0, + position: position({ + state: { + ...initialCodexScanState(), + model: "gpt-6-astra", + lastCumulativeUsage: request, + latestCumulativeAtMs: 1_786_000_000_000, + }, + }), + }; + const current = parse(makeScanCacheWriter()(new Map([["/codex.jsonl", entry]]), {})); + + expect( + decodeScanCache({ ...current, version: 15 }, TEST_FORMAT_MAP).get("/codex.jsonl"), + ).toEqual({ + ...entry, + position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null }, + }); + expect(decodeScanCache(current, TEST_FORMAT_MAP).get("/codex.jsonl")).toEqual(entry); + }); + + it("re-parses a Claude transcript cached before 1-hour cache writes were counted", () => { const original = cacheWith([["/claude.jsonl", 100, [record()]]]); - const v11 = { ...parse(makeScanCacheWriter()(original, {})), version: 11 }; + const v12 = { ...parse(makeScanCacheWriter()(original, {})), version: 12 }; - const restored = decodeScanCache(JSON.parse(JSON.stringify(v11)), TEST_FORMAT_MAP); + const restored = decodeScanCache(JSON.parse(JSON.stringify(v12)), TEST_FORMAT_MAP); expect(restored.get("/claude.jsonl")).toEqual({ ...original.get("/claude.jsonl"), size: -1, @@ -256,6 +339,29 @@ describe("scan cache round trip", () => { }); }); + it.each(["s:p:grok-4.5", "s:p:1786000000000:grok-4.5"])( + "re-parses v16 Grok transcripts with key %s", + (dedupeKey) => { + const original = cacheWith([ + ["/grok/updates.jsonl", 100, [record({ provider: "grok", model: "grok-4.5", dedupeKey })]], + ]); + original.set("/grok/updates.jsonl", { + ...original.get("/grok/updates.jsonl")!, + provider: "grok", + }); + const encoded = parse(makeScanCacheWriter()(original, {})); + const restored = decodeScanCache({ ...encoded, version: 16 }, TEST_FORMAT_MAP); + expect(restored.get("/grok/updates.jsonl")).toEqual({ + ...original.get("/grok/updates.jsonl"), + size: -1, + position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null }, + records: [ + record({ provider: "grok", model: "grok-4.5", dedupeKey: "s:p:1786000000000:grok-4.5" }), + ], + }); + }, + ); + it("rewrites only changed entries and still restores the whole cache", () => { const write = makeScanCacheWriter(); const cache = cacheWith([ diff --git a/apps/server/src/usage/usageScanCache.ts b/apps/server/src/usage/usageScanCache.ts index 4cd959c75aa2..843b414b97f3 100644 --- a/apps/server/src/usage/usageScanCache.ts +++ b/apps/server/src/usage/usageScanCache.ts @@ -10,7 +10,7 @@ import { CLAUDE_SYNTHETIC_MODEL } from "../provider/Drivers/claudeUsage.ts"; import { mergeUsageCopies } from "./usageAggregation.ts"; import { GUARD_LENGTH, type TranscriptParsePosition } from "./usageTranscriptReader.ts"; -const USAGE_SCAN_CACHE_VERSION = 12 as const; +const USAGE_SCAN_CACHE_VERSION = 17 as const; const SPEED_COMPATIBLE_SINCE_VERSION = 4; /** @@ -18,8 +18,13 @@ const SPEED_COMPATIBLE_SINCE_VERSION = 4; * replace it, dropping saved usage for deleted transcripts. Separate files keep * the newer one; older files are removed once it is saved. */ -export const SCAN_CACHE_FILE_NAME = "usage-scan-cache-v12.json"; +export const SCAN_CACHE_FILE_NAME = "usage-scan-cache-v17.json"; export const LEGACY_SCAN_CACHE_FILE_NAMES = [ + "usage-scan-cache-v16.json", + "usage-scan-cache-v15.json", + "usage-scan-cache-v14.json", + "usage-scan-cache-v13.json", + "usage-scan-cache-v12.json", "usage-scan-cache-v11.json", "usage-scan-cache-v10.json", "usage-scan-cache-v9.json", @@ -59,7 +64,8 @@ type SerializedRecord = readonly [ reportedCostUsd: number | null, speed: number, rateModelIndex?: number | null, - unresolvedServiceTierTurnId?: string, + unresolvedServiceTierTurnId?: string | null, + cacheCreation1hTokens?: number, ]; interface SerializedFile { @@ -118,9 +124,11 @@ function serializeFile(entry: CachedFile, tables: InternTables): SerializedFile record.rateModel === undefined ? null : intern(tables.models, tables.modelIndex, record.rateModel), - ...(record.unresolvedServiceTierTurnId === undefined - ? ([] as const) - : ([record.unresolvedServiceTierTurnId] as const)), + ...(record.cacheCreation1hTokens !== undefined + ? ([record.unresolvedServiceTierTurnId ?? null, record.cacheCreation1hTokens] as const) + : record.unresolvedServiceTierTurnId !== undefined + ? ([record.unresolvedServiceTierTurnId] as const) + : ([] as const)), ]; return { s: entry.size, @@ -211,6 +219,7 @@ export function decodeScanCache( speedIndex, rateModelIndex, unresolvedServiceTierTurnId, + cacheCreation1h = 0, ] = row as SerializedRecord; const speed = typeof speedIndex === "number" ? SPEEDS[speedIndex] : undefined; @@ -220,8 +229,8 @@ export function decodeScanCache( typeof timestampMs !== "number" || !Number.isFinite(timestampMs) || model === undefined || - (unresolvedServiceTierTurnId !== undefined && - typeof unresolvedServiceTierTurnId !== "string") || + (unresolvedServiceTierTurnId != null && typeof unresolvedServiceTierTurnId !== "string") || + !Number.isFinite(cacheCreation1h) || (rateModelIndex != null && rateModel === undefined) || !Number.isFinite(uncached) || !Number.isFinite(cached) || @@ -250,12 +259,22 @@ export function decodeScanCache( outputTokens: output, reasoningTokens: reasoning, }, + ...(cacheCreation1h > 0 ? { cacheCreation1hTokens: cacheCreation1h } : {}), reportedCostUsd: typeof reportedCostUsd === "number" ? reportedCostUsd : null, speed, - ...(version >= 7 && unresolvedServiceTierTurnId !== undefined + ...(version >= 7 && unresolvedServiceTierTurnId != null ? { unresolvedServiceTierTurnId } : {}), - dedupeKey: typeof dedupeKey === "string" ? dedupeKey : null, + dedupeKey: + typeof dedupeKey !== "string" + ? null + : // Saved copies of moved transcripts must still match the re-parsed v17 key. + provider === "grok" && + version < 17 && + dedupeKey.endsWith(`:${model}`) && + !dedupeKey.endsWith(`:${timestampMs}:${model}`) + ? `${dedupeKey.slice(0, -model.length)}${timestampMs}:${model}` + : dedupeKey, }); } return records; @@ -290,19 +309,22 @@ export function decodeScanCache( continue; } // Keep history for deleted rollouts, but re-parse incompatible reducer state. - // v9 and v11 repair skipped Codex requests; clean v8+ entries stay served while unchanged. - // v10 adds Claude advisor calls; before v12 a repeated Claude key kept its first, possibly partial, copy. + // v13 adds Claude 1-hour cache writes; v14 uses Codex request ledgers. + // v15 repairs skipped Codex requests whose counters rewound; clean v14 entries stay served. + // v17 keys Grok turns by time, restoring turns previously dropped as repeats. const legacy = - (entry.p === "claude" && version < 12) || + (entry.p === "claude" && version < 13) || + (entry.p === "grok" && version < 17) || (format.state !== undefined && (version < 6 || - (entry.p === "codex" && (version < 8 || (version < 11 && malformedRecords > 0))))); + (entry.p === "codex" && (version < 14 || (version < 15 && malformedRecords > 0))))); // A corrupt state disqualifies the entry: resuming with it would attach // appended usage to the wrong model or replay fork-copied history. if (!legacy && !isValidState.get(entry.p)?.(entry.cs)) continue; - // Before v11 a clean scan could end past an interrupted request whose usage its state dropped, - // so a grown file re-parses whole instead of resuming. - const resumable = !legacy && !(entry.p === "codex" && version < 11); + // Before v16 a Codex baseline can lack the time of its latest move, or carry an older one, + // and that time keeps a re-emitted older snapshot from billing twice. A grown file re-parses + // whole instead of resuming. + const resumable = !legacy && !(entry.p === "codex" && version < 16); const provider = entry.p; const records = decodeRecords(entry.r, provider); diff --git a/apps/server/src/usage/usageTranscriptReader.test.ts b/apps/server/src/usage/usageTranscriptReader.test.ts index 105884d81ccf..ca994dd3ac5b 100644 --- a/apps/server/src/usage/usageTranscriptReader.test.ts +++ b/apps/server/src/usage/usageTranscriptReader.test.ts @@ -13,6 +13,8 @@ import * as Schema from "effect/Schema"; import { TEST_FORMATS } from "./usageTestFormats.ts"; import { listTranscriptFiles, + physicalSourceIdOf, + readPhysicalSourceId, readTranscriptRecords, transcriptLineGate, } from "./usageTranscriptReader.ts"; @@ -99,6 +101,30 @@ describe("compressed transcripts", () => { assert.isAbove(listed.files.find((file) => file.path === compressed)?.size ?? 0, 0); }); + it("lists files in directory order when more directories wait than it reads at once", async () => { + for (let index = 0; index < 40; index += 1) { + const nested = NodePath.join(dir, `project-${index}`, "sessions"); + await NodeFSP.mkdir(nested, { recursive: true }); + await NodeFSP.writeFile(NodePath.join(nested, "a.jsonl"), ""); + await NodeFSP.writeFile(NodePath.join(nested, "..", "b.jsonl"), ""); + } + // Usage that two transcripts share counts in the first one listed, so the order is part of the result. + const inOrder = async (folder: string): Promise => { + const paths: string[] = []; + for (const entry of await NodeFSP.readdir(folder, { withFileTypes: true })) { + const child = NodePath.join(folder, entry.name); + if (entry.isDirectory()) paths.push(...(await inOrder(child))); + else paths.push(child); + } + return paths; + }; + const listed = await listTranscriptFiles(dir, 0); + assert.deepStrictEqual( + listed.files.map((file) => file.path), + await inOrder(dir), + ); + }); + it.each([ { parser: "native", streamingThresholdBytes: undefined }, { parser: "projected", streamingThresholdBytes: 1024 }, @@ -553,3 +579,49 @@ describe("transcriptLineGate", () => { } }); }); + +describe("readPhysicalSourceId", () => { + const scope = "0123456789abcdef0123456789abcdef"; + const windowsPath = "C:\\Users\\User\\.claude\\projects"; + + // One NTFS directory the Windows server reads natively and the WSL server + // through drvfs. The temp dir stands in for the drive's mount point. + async function drvfsFixture(extraMounts: (escapedMountPoint: string) => string = () => "") { + const mountPoint = NodePath.join(dir, "drive c"); + await NodeFSP.mkdir(NodePath.join(mountPoint, "Users", "User", ".claude", "projects"), { + recursive: true, + }); + const escaped = mountPoint.replaceAll(" ", "\\040"); + const mountInfoPath = NodePath.join(dir, "mountinfo"); + await NodeFSP.writeFile( + mountInfoPath, + [ + "24 1 8:32 / / rw,relatime shared:1 - ext4 /dev/sdc rw,discard", + `81 24 0:61 / ${escaped} rw,noatime shared:2 - 9p C:\\134 rw,dirsync,aname=drvfs;path=C:\\;uid=1000;gid=1000;symlinkroot=/mnt/,mmap,access=client,msize=65536,trans=fd`, + extraMounts(escaped), + ].join("\n"), + ); + return { mountPoint, mountInfoPath }; + } + + it("names a drvfs directory as the Windows server names it, whatever its typed case", async () => { + const { mountPoint, mountInfoPath } = await drvfsFixture(); + const wslId = await readPhysicalSourceId( + `${mountPoint}/users/user/.claude/projects`, + scope, + "linux", + mountInfoPath, + ); + assert.strictEqual(wslId, physicalSourceIdOf(scope, windowsPath)); + }); + + it("gives no identity to native or shadowing Linux mounts, or without a desktop scope", async () => { + const { mountPoint, mountInfoPath } = await drvfsFixture( + (escaped) => `90 81 8:48 / ${escaped}/Users rw,relatime - ext4 /dev/sdd rw`, + ); + const projects = `${mountPoint}/Users/User/.claude/projects`; + assert.isUndefined(await readPhysicalSourceId(projects, scope, "linux", mountInfoPath)); + assert.isUndefined(await readPhysicalSourceId(dir, scope, "linux", mountInfoPath)); + assert.isUndefined(await readPhysicalSourceId(projects, undefined, "linux", mountInfoPath)); + }); +}); diff --git a/apps/server/src/usage/usageTranscriptReader.ts b/apps/server/src/usage/usageTranscriptReader.ts index 38b80d36c866..5b92f48356aa 100644 --- a/apps/server/src/usage/usageTranscriptReader.ts +++ b/apps/server/src/usage/usageTranscriptReader.ts @@ -1,8 +1,11 @@ // @effect-diagnostics nodeBuiltinImport:off // Cold windows span ~1.4 GB/1,500 files; node:fs buffer streaming is ~10x cheaper than materializing files. +import * as NodeCrypto from "node:crypto"; +import * as NodeFS from "node:fs"; import * as NodeFSP from "node:fs/promises"; import * as NodePath from "node:path"; import * as NodeStringDecoder from "node:string_decoder"; +import * as NodeUtil from "node:util"; import * as NodeZlib from "node:zlib"; import type { @@ -41,8 +44,12 @@ export interface TranscriptParseResult { export const GUARD_LENGTH = 64; const STREAMING_THRESHOLD_BYTES = 8 * 1024 * 1024; +const READ_CHUNK_BYTES = 256 * 1024; const NEWLINE = 0x0a; const STAT_CONCURRENCY = 32; +const WALK_CONCURRENCY = 16; +// Measured about 3x faster than the promise API's stat across 20,000 transcripts. +const statFile = NodeUtil.promisify(NodeFS.stat); const CARRIAGE_RETURN = 0x0d; function selectUsageFields(fields: SelectedFields) { @@ -71,38 +78,54 @@ export async function listTranscriptFiles( root: string, sinceMs: number, options?: { readonly fileName?: string }, -): Promise<{ readonly files: readonly TranscriptFile[]; readonly failedPaths: number }> { +): Promise<{ + readonly files: readonly TranscriptFile[]; + readonly walked: readonly string[]; + readonly failedPaths: number; +}> { const fileName = options?.fileName; - const candidates: string[] = []; let failedPaths = 0; - const walk = async (dir: string): Promise => { + let reading = 0; + const waiting: Array<() => void> = []; + // Sibling directories read together; each returns its own list so the result keeps readdir order. + const walk = async (dir: string): Promise => { + if (reading < WALK_CONCURRENCY) reading += 1; + else await new Promise((resume) => waiting.push(resume)); let entries; try { entries = await NodeFSP.readdir(dir, { withFileTypes: true }); } catch { failedPaths += 1; - return; - } - for (const entry of entries) { - const child = NodePath.join(dir, entry.name); - if (entry.isDirectory()) await walk(child); - else if ( - fileName !== undefined - ? entry.name === fileName - : entry.name.endsWith(".jsonl") || entry.name.endsWith(".jsonl.zst") - ) { - candidates.push(child); - } + return []; + } finally { + // A waiting directory takes over this slot, so the count only drops when none waits. + const next = waiting.shift(); + if (next === undefined) reading -= 1; + else next(); } + const found = await Promise.all( + entries.map((entry) => { + const child = NodePath.join(dir, entry.name); + if (entry.isDirectory()) return walk(child); + return ( + fileName !== undefined + ? entry.name === fileName + : entry.name.endsWith(".jsonl") || entry.name.endsWith(".jsonl.zst") + ) + ? [child] + : []; + }), + ); + return found.flat(); }; - await walk(root); + const candidates = await walk(root); const found: Array = Array.from({ length: candidates.length }); const queue = candidates.entries(); const statQueued = async (): Promise => { for (const [index, path] of queue) { try { - const stats = await NodeFSP.stat(path); + const stats = await statFile(path); if (stats.mtimeMs >= sinceMs) { found[index] = { path, size: stats.size, mtimeMs: stats.mtimeMs }; } @@ -116,7 +139,7 @@ export async function listTranscriptFiles( await Promise.all( Array.from({ length: Math.min(STAT_CONCURRENCY, candidates.length) }, statQueued), ); - return { files: found.filter((file) => file !== undefined), failedPaths }; + return { files: found.filter((file) => file !== undefined), walked: candidates, failedPaths }; } export async function readDirectoryVolumeId(path: string): Promise { @@ -128,6 +151,137 @@ export async function readDirectoryVolumeId(path: string): Promise { } } +/** + * Set by the desktop app, from a value it persists, on its Windows server and + * the WSL server it manages, so both can name one Windows directory alike + * without merging separate machines that share a hostname and path. + */ +export const DESKTOP_HOST_SCOPE_ENV = "T3CODE_DESKTOP_HOST_SCOPE"; + +const MOUNT_FIELD_ESCAPE = /\\([0-7]{3})/g; +const WINDOWS_ROOT = /^(?:[A-Za-z]:|\\\\[^\\]+\\[^\\]+)\\?$/; + +const unescapeMountField = (field: string) => + field.replace(MOUNT_FIELD_ESCAPE, (_, octal: string) => + String.fromCharCode(Number.parseInt(octal, 8)), + ); + +const normalizeWindowsPath = (path: string) => + (/^[a-z]:/.test(path) ? path[0]!.toUpperCase() + path.slice(1) : path).replace(/\\$/, ""); + +/** + * The Windows directory behind the deepest mount holding `dir`, read from + * `/proc/self/mountinfo` so a custom automount root resolves and an ext4 + * mount under `/mnt` never passes for a drive. + */ +function windowsMountFor( + mountInfo: string, + dir: string, +): { readonly mountPoint: string; readonly windowsRoot: string } | undefined { + let deepest: { mountPoint: string; windowsRoot: string | undefined } | undefined; + for (const line of mountInfo.split("\n")) { + const fields = line.split(" "); + const separator = fields.indexOf("-", 6); + if (separator === -1) continue; + const mountPoint = unescapeMountField(fields[4] ?? ""); + const contains = + mountPoint === "/" || + dir === mountPoint || + dir.startsWith(mountPoint.endsWith("/") ? mountPoint : `${mountPoint}/`); + if (!contains || (deepest !== undefined && deepest.mountPoint.length > mountPoint.length)) { + continue; + } + // WSL2 mounts drives over 9p with `aname=drvfs;path=C:\`; WSL1 as `drvfs`. + const options = (fields[separator + 3] ?? "").split(/[,;]/); + const source = options.includes("aname=drvfs") + ? options.find((option) => option.startsWith("path="))?.slice("path=".length) + : fields[separator + 1] === "drvfs" + ? unescapeMountField(fields[separator + 2] ?? "") + : undefined; + const root = unescapeMountField(fields[3] ?? "/") + .split("/") + .filter(Boolean); + deepest = { + mountPoint, + windowsRoot: + source !== undefined && WINDOWS_ROOT.test(source) + ? [normalizeWindowsPath(source), ...root].join("\\") + : undefined, + }; + } + return deepest?.windowsRoot === undefined + ? undefined + : { mountPoint: deepest.mountPoint, windowsRoot: deepest.windowsRoot }; +} + +/** + * drvfs keeps the casing a path was typed with, while Windows reports the + * on-disk names. Read those names rather than folding case, which would join + * distinct entries of a case-sensitive directory. + */ +async function onDiskNames( + mountPoint: string, + parts: readonly string[], +): Promise { + const names: string[] = []; + let current = mountPoint; + for (const part of parts) { + let entries: string[]; + try { + entries = await NodeFSP.readdir(current); + } catch { + return undefined; + } + const folded = part.toUpperCase(); + const [name, ...others] = entries.includes(part) + ? [part] + : entries.filter((entry) => entry.toUpperCase() === folded); + if (name === undefined || others.length > 0) return undefined; + names.push(name); + current = NodePath.posix.join(current, name); + } + return names; +} + +export function physicalSourceIdOf(scope: string, windowsPath: string): string { + return NodeCrypto.createHash("sha256") + .update(`${scope}\0${windowsPath}`) + .digest("hex") + .slice(0, 32); +} + +/** + * Identifies a Windows directory the same way from the Windows server and from + * a WSL server reading it through drvfs. Directories that are not Windows + * backed, and servers the desktop app did not start, get none. + */ +export async function readPhysicalSourceId( + dir: string, + scope: string | undefined, + platform: NodeJS.Platform, + mountInfoPath = "/proc/self/mountinfo", +): Promise { + if (scope === undefined || scope.length === 0) return undefined; + try { + if (platform === "win32") { + // The native realpath reports on-disk casing; the JS one keeps the input's. + return physicalSourceIdOf(scope, normalizeWindowsPath(await NodeFSP.realpath(dir))); + } + if (platform !== "linux") return undefined; + const mount = windowsMountFor(await NodeFSP.readFile(mountInfoPath, "utf8"), dir); + if (mount === undefined) return undefined; + const names = await onDiskNames( + mount.mountPoint, + dir.slice(mount.mountPoint.length).split("/").filter(Boolean), + ); + return names === undefined + ? undefined + : physicalSourceIdOf(scope, [mount.windowsRoot, ...names].join("\\")); + } catch { + return undefined; + } +} + async function guardMatches( handle: NodeFSP.FileHandle, position: TranscriptParsePosition, @@ -147,6 +301,29 @@ async function guardMatches( } } +/** Plain files reuse one buffer, so a chunk is only valid until the next one is requested. */ +async function* readChunks( + handle: NodeFSP.FileHandle, + start: number, + compressed: boolean, +): AsyncGenerator { + if (compressed) { + // Compressed snapshots cache a full parse; decoded byte offsets cannot + // resume the compressed input. + yield* handle + .createReadStream({ start, autoClose: true, highWaterMark: READ_CHUNK_BYTES }) + .compose(NodeZlib.createZstdDecompress()); + return; + } + const buffer = Buffer.allocUnsafe(READ_CHUNK_BYTES); + for (let position = start; ;) { + const { bytesRead } = await handle.read(buffer, 0, buffer.length, position); + if (bytesRead === 0) return; + position += bytesRead; + yield buffer.subarray(0, bytesRead); + } +} + // Match markers in one latin1 alternation; one character per byte preserves exact ASCII offsets. export function transcriptLineGate(markers: readonly [string, ...string[]]) { if (markers.length === 1) { @@ -223,7 +400,8 @@ export async function readTranscriptRecords( const append = (segment: Buffer) => { if (!streaming && pendingBytes + segment.length <= streamingThresholdBytes) { - if (segment.length > 0) pendingChunks.push(segment); + // Copy: the next read overwrites the chunk this segment points into. + if (segment.length > 0) pendingChunks.push(Buffer.from(segment)); pendingBytes += segment.length; return; } @@ -254,15 +432,7 @@ export async function readTranscriptRecords( streaming = undefined; decoder = undefined; }; - const input = handle.createReadStream({ - start, - autoClose: compressed, - highWaterMark: 256 * 1024, - }); - // Compressed snapshots cache a full parse; decoded byte offsets cannot - // resume the compressed input. - const stream = compressed ? input.compose(NodeZlib.createZstdDecompress()) : input; - for await (const chunk of stream) { + for await (const chunk of readChunks(handle, start, compressed)) { let lineStart = 0; while (lineStart < chunk.length) { const newlineIndex = chunk.indexOf(NEWLINE, lineStart); diff --git a/apps/server/src/usage/usageTranscriptStreaming.test.ts b/apps/server/src/usage/usageTranscriptStreaming.test.ts index 856ba2ac5b9d..635b5ae4e6be 100644 --- a/apps/server/src/usage/usageTranscriptStreaming.test.ts +++ b/apps/server/src/usage/usageTranscriptStreaming.test.ts @@ -254,7 +254,7 @@ describe("large usage records", () => { outputTokens: 99, reasoningTokens: 10, }, - dedupeKey: "s1:p1:grok-4.5-build", + dedupeKey: "s1:p1:1785578400123:grok-4.5-build", reportedCostUsd: 0.25, }); }); diff --git a/apps/server/src/utils/subscribeBeforeSnapshot.test.ts b/apps/server/src/utils/subscribeBeforeSnapshot.test.ts index 70f0beda25ce..e37b969a123d 100644 --- a/apps/server/src/utils/subscribeBeforeSnapshot.test.ts +++ b/apps/server/src/utils/subscribeBeforeSnapshot.test.ts @@ -15,70 +15,66 @@ import { describe("subscribeBeforeSnapshot", () => { it.effect("atomically reads the initial snapshot before receiving later changes", () => - Effect.scoped( - Effect.gen(function* () { - const changes = yield* PubSub.sliding(1); - const latest = yield* Ref.make(1); - const mutex = yield* Semaphore.make(1); - const snapshotStarted = yield* Deferred.make(); - const finishSnapshot = yield* Deferred.make(); - const subscriptionFiber = yield* subscribeBeforeSnapshot( - changes, - Deferred.succeed(snapshotStarted, undefined).pipe( - Effect.andThen(Deferred.await(finishSnapshot)), - Effect.andThen(Ref.get(latest)), - ), - mutex, - ).pipe(Effect.forkChild); + Effect.gen(function* () { + const changes = yield* PubSub.sliding(1); + const latest = yield* Ref.make(1); + const mutex = yield* Semaphore.make(1); + const snapshotStarted = yield* Deferred.make(); + const finishSnapshot = yield* Deferred.make(); + const subscriptionFiber = yield* subscribeBeforeSnapshot( + changes, + Deferred.succeed(snapshotStarted, undefined).pipe( + Effect.andThen(Deferred.await(finishSnapshot)), + Effect.andThen(Ref.get(latest)), + ), + mutex, + ).pipe(Effect.forkChild); - yield* Deferred.await(snapshotStarted); - const publishFiber = yield* mutex - .withPermits(1)(Ref.set(latest, 2).pipe(Effect.andThen(PubSub.publish(changes, 2)))) - .pipe(Effect.forkChild); - yield* Deferred.succeed(finishSnapshot, undefined); + yield* Deferred.await(snapshotStarted); + const publishFiber = yield* mutex + .withPermits(1)(Ref.set(latest, 2).pipe(Effect.andThen(PubSub.publish(changes, 2)))) + .pipe(Effect.forkChild); + yield* Deferred.succeed(finishSnapshot, undefined); - const subscription = yield* Fiber.join(subscriptionFiber); - yield* Fiber.join(publishFiber); - const firstChange = yield* subscription.changes.pipe( - Stream.runHead, - Effect.timeout("1 second"), - ); + const subscription = yield* Fiber.join(subscriptionFiber); + yield* Fiber.join(publishFiber); + const firstChange = yield* subscription.changes.pipe( + Stream.runHead, + Effect.timeout("1 second"), + ); - expect(subscription.latest).toBe(1); - expect(firstChange).toEqual(Option.some(2)); - }), - ), + expect(subscription.latest).toBe(1); + expect(firstChange).toEqual(Option.some(2)); + }), ); it.effect("subscribes before an uncoordinated snapshot can change", () => - Effect.scoped( - Effect.gen(function* () { - const changes = yield* PubSub.sliding(1); - const latest = yield* Ref.make(1); - const snapshotStarted = yield* Deferred.make(); - const finishSnapshot = yield* Deferred.make(); - const subscriptionFiber = yield* subscribeBeforeSnapshotWithoutMutex( - changes, - Deferred.succeed(snapshotStarted, undefined).pipe( - Effect.andThen(Deferred.await(finishSnapshot)), - Effect.andThen(Ref.get(latest)), - ), - ).pipe(Effect.forkChild); + Effect.gen(function* () { + const changes = yield* PubSub.sliding(1); + const latest = yield* Ref.make(1); + const snapshotStarted = yield* Deferred.make(); + const finishSnapshot = yield* Deferred.make(); + const subscriptionFiber = yield* subscribeBeforeSnapshotWithoutMutex( + changes, + Deferred.succeed(snapshotStarted, undefined).pipe( + Effect.andThen(Deferred.await(finishSnapshot)), + Effect.andThen(Ref.get(latest)), + ), + ).pipe(Effect.forkChild); - yield* Deferred.await(snapshotStarted); - yield* Ref.set(latest, 2); - yield* PubSub.publish(changes, 2); - yield* Deferred.succeed(finishSnapshot, undefined); + yield* Deferred.await(snapshotStarted); + yield* Ref.set(latest, 2); + yield* PubSub.publish(changes, 2); + yield* Deferred.succeed(finishSnapshot, undefined); - const subscription = yield* Fiber.join(subscriptionFiber); - const firstChange = yield* subscription.changes.pipe( - Stream.runHead, - Effect.timeout("1 second"), - ); + const subscription = yield* Fiber.join(subscriptionFiber); + const firstChange = yield* subscription.changes.pipe( + Stream.runHead, + Effect.timeout("1 second"), + ); - expect(subscription.latest).toBe(2); - expect(firstChange).toEqual(Option.some(2)); - }), - ), + expect(subscription.latest).toBe(2); + expect(firstChange).toEqual(Option.some(2)); + }), ); }); diff --git a/apps/server/src/vcs/GitVcsDriver.test.ts b/apps/server/src/vcs/GitVcsDriver.test.ts index 89580e0937af..1acbf036fb52 100644 --- a/apps/server/src/vcs/GitVcsDriver.test.ts +++ b/apps/server/src/vcs/GitVcsDriver.test.ts @@ -1,3 +1,5 @@ +import * as NodeNet from "node:net"; +import * as SecureRandom from "effect/crypto/SecureRandom"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Cause from "effect/Cause"; @@ -173,6 +175,392 @@ const makeCheckpointFixture = Effect.fn("makeCheckpointFixture")(function* ( return { git, checkpointRef }; }); +it.effect.each(["interrupt", "timeout", "success"] as const)( + "checkpoint bulk staging cleans operation-owned packs after $0 and can retry", + (stop) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const liveProcess = yield* VcsProcess.VcsProcess; + const sandbox = yield* fs.makeTempDirectoryScoped({ prefix: "t3-bulk-checkpoint-" }); + const cwd = path.join(sandbox, "repo"); + yield* fs.makeDirectory(cwd); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + const originalCheckpoint = (yield* git(["rev-parse", checkpointRef])).stdout; + const originalIndex = yield* fs.readFile(path.join(cwd, ".git/index")); + const originalHead = (yield* git(["rev-parse", "HEAD"])).stdout; + yield* git(["config", "core.bigFileThreshold", "1m"]); + const crypto = yield* SecureRandom.SecureRandom; + const payload = yield* crypto.randomBytes(2 * 1024 * 1024); + yield* fs.writeFile(path.join(cwd, "a-media.bin"), payload); + + // Git keeps bulk-checkin's pack open while the next path's clean filter runs. + // The socket is a barrier, not a race against the speed of compression. + const connected = Promise.withResolvers(); + const sockets = new Set(); + let released = false; + const server = yield* Effect.acquireRelease( + Effect.sync(() => + NodeNet.createServer((socket) => { + sockets.add(socket); + connected.resolve(socket); + // Git can run the filter again while refreshing the staged entry. + if (released) socket.end(); + }), + ), + (server) => + Effect.sync(() => { + for (const socket of sockets) socket.destroy(); + server.close(); + }), + ); + yield* Effect.promise( + () => + new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }), + ); + const address = server.address(); + assert.isTrue(address !== null && typeof address !== "string"); + if (address === null || typeof address === "string") return; + const filter = path.join(sandbox, "filter.cjs"); + yield* fs.writeFileString( + filter, + `const s = require('node:net').connect(${address.port}, '127.0.0.1'); s.resume(); process.stdin.resume(); s.on('end', () => process.exit(0));`, + ); + yield* git(["config", "filter.barrier.clean", `"${process.execPath}" "${filter}"`]); + yield* git(["config", "filter.barrier.required", "true"]); + yield* fs.writeFileString(path.join(cwd, ".gitattributes"), "z-gate filter=barrier\n"); + yield* fs.writeFileString(path.join(cwd, "z-gate"), "gate\n"); + const sharedPack = path.join(cwd, ".git/objects/pack/tmp_pack_other_operation"); + yield* fs.writeFileString(sharedPack, "owned by another Git operation"); + let objectDirectory = path.join(cwd, ".git/objects"); + const captureDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe( + Effect.provideService(VcsProcess.VcsProcess, { + run: (input) => { + if (input.args.includes("add") && input.env?.GIT_OBJECT_DIRECTORY) + objectDirectory = input.env.GIT_OBJECT_DIRECTORY; + return liveProcess.run(input); + }, + }), + ); + const fiber = yield* captureDriver.checkpoints + .captureCheckpoint({ cwd, checkpointRef }) + .pipe(Effect.exit, Effect.forkScoped); + const socket = yield* Effect.promise(() => connected.promise).pipe( + Effect.raceFirst( + Fiber.join(fiber).pipe( + Effect.flatMap((result) => + Exit.isFailure(result) + ? Effect.failCause(result.cause) + : Effect.die("capture ended before the filter connected"), + ), + ), + ), + ); + const packs = (yield* fs.readDirectory(path.join(objectDirectory, "pack"))).filter( + (name) => name.startsWith("tmp_pack_") && name !== "tmp_pack_other_operation", + ); + assert.strictEqual(packs.length, 1); + assert.isAbove( + Number((yield* fs.stat(path.join(objectDirectory, "pack", packs[0]!))).size), + 1024 * 1024, + ); + // An overlapping capture publishes the same large blob while the first + // still owns an unfinished pack. Its objects/ref must survive either exit. + const concurrentRef = CheckpointRef.make("refs/t3/checkpoints/concurrent"); + const concurrentDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe( + Effect.provideService(VcsProcess.VcsProcess, { + run: (input) => + liveProcess.run({ ...input, args: ["-c", "filter.barrier.clean=cat", ...input.args] }), + }), + ); + yield* concurrentDriver.checkpoints.captureCheckpoint({ cwd, checkpointRef: concurrentRef }); + const concurrentOid = (yield* git(["rev-parse", concurrentRef])).stdout; + const sharedPackNames = (yield* fs.readDirectory(path.join(cwd, ".git/objects/pack"))).sort(); + const packName = sharedPackNames.find((name) => name.endsWith(".pack")); + assert.isDefined(packName); + const sharedPackBytes = yield* fs.readFile(path.join(cwd, ".git/objects/pack", packName!)); + if (stop === "interrupt") yield* Fiber.interrupt(fiber); + else if (stop === "timeout") yield* TestClock.adjust("30 seconds"); + released = true; + socket.end(); + const result = yield* Fiber.await(fiber); + if (stop === "interrupt") + assert.isTrue(Exit.isFailure(result) && Cause.hasInterruptsOnly(result.cause)); + else if (stop === "timeout") { + assert.isTrue(Exit.isSuccess(result)); + if (Exit.isSuccess(result)) { + assert.isTrue(Exit.isFailure(result.value)); + if (Exit.isFailure(result.value)) { + const error = Cause.findErrorOption(result.value.cause); + assert.isTrue(Option.isSome(error) && error.value._tag === "VcsProcessTimeoutError"); + } + } + } else assert.isTrue(Exit.isSuccess(result) && Exit.isSuccess(result.value)); + assert.deepEqual( + (yield* fs.readDirectory(path.join(cwd, ".git/objects/pack"))).sort(), + sharedPackNames, + ); + assert.deepEqual( + yield* fs.readFile(path.join(cwd, ".git/objects/pack", packName!)), + sharedPackBytes, + ); + assert.strictEqual((yield* git(["rev-parse", concurrentRef])).stdout, concurrentOid); + assert.isFalse(yield* fs.exists(path.join(objectDirectory, "pack", packs[0]!))); + assert.isFalse(yield* fs.exists(objectDirectory)); + assert.strictEqual(yield* fs.readFileString(sharedPack), "owned by another Git operation"); + if (stop !== "success") + assert.strictEqual((yield* git(["rev-parse", checkpointRef])).stdout, originalCheckpoint); + assert.deepEqual(yield* fs.readFile(path.join(cwd, ".git/index")), originalIndex); + assert.strictEqual((yield* git(["rev-parse", "HEAD"])).stdout, originalHead); + assert.deepEqual(yield* fs.readFile(path.join(cwd, "a-media.bin")), payload); + + yield* git(["config", "filter.barrier.clean", "cat"]); + yield* captureDriver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + assert.isFalse(yield* fs.exists(objectDirectory)); + const blob = (yield* git(["hash-object", "a-media.bin"])).stdout.trim(); + assert.strictEqual( + (yield* git(["rev-parse", `${checkpointRef}:a-media.bin`])).stdout.trim(), + blob, + ); + yield* fs.remove(sharedPack); + yield* git(["fsck", "--full"]); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect.each(["failure", "interrupt"] as const)( + "checkpoint publication $0 preserves existing refs and permits retry", + (stop) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-checkpoint-publish-" }); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + const previous = (yield* git(["show-ref"])).stdout; + yield* fs.writeFileString(path.join(cwd, "new.txt"), "new checkpoint content\n"); + const published = yield* Deferred.make(); + const captureDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe( + Effect.provideService(FileSystem.FileSystem, { + ...fs, + link: (source, destination) => + fs.link(source, destination).pipe( + Effect.andThen(Deferred.succeed(published, undefined)), + Effect.andThen( + stop === "interrupt" + ? Effect.never + : Effect.fail( + PlatformError.systemError({ + _tag: "BadResource", + module: "FileSystem", + method: "link", + }), + ), + ), + ), + }), + ); + const fiber = yield* captureDriver.checkpoints + .captureCheckpoint({ cwd, checkpointRef }) + .pipe(Effect.exit, Effect.forkScoped); + yield* Deferred.await(published); + if (stop === "interrupt") yield* Fiber.interrupt(fiber); + const result = yield* Fiber.await(fiber); + if (stop === "interrupt") + assert.isTrue(Exit.isFailure(result) && Cause.hasInterruptsOnly(result.cause)); + else { + assert.isTrue(Exit.isSuccess(result) && Exit.isFailure(result.value)); + if (Exit.isSuccess(result) && Exit.isFailure(result.value)) { + const error = Cause.findErrorOption(result.value.cause); + assert.isTrue( + Option.isSome(error) && + error.value._tag === "VcsCheckpointStorageError" && + error.value.stage === "publish", + ); + } + } + assert.strictEqual((yield* git(["show-ref"])).stdout, previous); + assert.deepEqual( + (yield* fs.readDirectory(path.join(cwd, ".git/objects"))).filter((name) => + name.startsWith("tmp_objdir-t3-checkpoint-"), + ), + [], + ); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + assert.strictEqual( + (yield* git(["show", `${checkpointRef}:new.txt`])).stdout, + "new checkpoint content\n", + ); + yield* git(["fsck", "--full"]); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect.each([ + { stop: "before git restore spawns", gate: "restore", index: "staged", file: "later\n" }, + { stop: "after git restore ran", gate: "clean", index: "head", file: "unstaged\n" }, +] as const)( + "checkpoint restore interrupted $stop leaves the $index index", + ({ gate, index, file }) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const liveProcess = yield* VcsProcess.VcsProcess; + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-restore-interrupt-" }); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + yield* fs.writeFileString(path.join(cwd, "file.txt"), "later\n"); + yield* git(["add", "file.txt"]); + const stagedTree = (yield* git(["write-tree"])).stdout; + const entered = yield* Deferred.make(); + const gatedDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe( + Effect.provideService(VcsProcess.VcsProcess, { + run: (input) => + input.operation === "GitVcsDriver.checkpoints.restoreCheckpoint" && + input.args.includes(gate) + ? Deferred.succeed(entered, undefined).pipe(Effect.andThen(Effect.never)) + : liveProcess.run(input), + }), + ); + const fiber = yield* gatedDriver.checkpoints + .restoreCheckpoint({ cwd, checkpointRef }) + .pipe(Effect.exit, Effect.forkScoped); + yield* Deferred.await(entered); + yield* Fiber.interrupt(fiber); + const result = yield* Fiber.await(fiber); + assert.isTrue(Exit.isFailure(result) && Cause.hasInterruptsOnly(result.cause)); + + assert.strictEqual( + (yield* git(["write-tree"])).stdout, + index === "head" ? (yield* git(["rev-parse", "HEAD^{tree}"])).stdout : stagedTree, + ); + assert.strictEqual(yield* fs.readFileString(path.join(cwd, "file.txt")), file); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect("checkpoint publication falls back to rename where hardlinks are unavailable", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* SecureRandom.SecureRandom; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-publish-no-links-" }); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + yield* git(["config", "core.bigFileThreshold", "1m"]); + yield* fs.writeFile(path.join(cwd, "media.bin"), yield* crypto.randomBytes(2 * 1024 * 1024)); + const captureDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe( + Effect.provideService(FileSystem.FileSystem, { + ...fs, + link: () => + Effect.fail( + PlatformError.systemError({ + _tag: "PermissionDenied", + module: "FileSystem", + method: "link", + }), + ), + }), + ); + yield* captureDriver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + const files = yield* fs.readDirectory(path.join(cwd, ".git/objects/pack")); + assert.isTrue(files.some((name) => name.endsWith(".pack"))); + assert.isTrue(yield* driver.checkpoints.hasCheckpointRef({ cwd, checkpointRef })); + assert.match((yield* git(["count-objects", "-v"])).stdout, /^garbage: 0$/m); + yield* git(["fsck", "--full"]); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( + "checkpoint publication preserves Git's shared object directory permissions", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-publish-shared-" }); + const git = (args: ReadonlyArray) => + driver.execute({ operation: "test.shared", cwd, args }); + yield* git(["init", "--shared=0660"]); + const checkpointRef = CheckpointRef.make("refs/t3/checkpoints/shared"); + yield* fs.writeFileString(path.join(cwd, "shared.txt"), "new shared object\n"); + const blob = (yield* git(["hash-object", "-w", "shared.txt"])).stdout.trim(); + const directory = path.join(cwd, ".git/objects", blob.slice(0, 2)); + const expectedMode = (yield* fs.stat(directory)).mode & 0o7777; + yield* fs.remove(directory, { recursive: true }); + assert.isFalse(yield* fs.exists(directory)); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + assert.strictEqual((yield* fs.stat(directory)).mode & 0o7777, expectedMode); + yield* git(["fsck", "--full"]); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect("checkpoint capture ignores an inherited object directory", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const sandbox = yield* fs.makeTempDirectoryScoped({ prefix: "t3-inherited-objects-" }); + const cwd = path.join(sandbox, "repo"); + yield* fs.makeDirectory(cwd); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + const unrelated = path.join(sandbox, "unrelated"); + yield* git(["init", "--bare", unrelated]); + const unrelatedObjects = path.join(unrelated, "objects"); + // VcsProcess clears inherited selectors from git children; only the driver sees this one. + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }).pipe( + Effect.provideService(HostProcess.Environment, { + ...(yield* HostProcess.Environment), + GIT_OBJECT_DIRECTORY: unrelatedObjects, + }), + ); + assert.strictEqual((yield* git(["show", `${checkpointRef}:file.txt`])).stdout, "unstaged\n"); + assert.deepEqual( + (yield* fs.readDirectory(unrelatedObjects)).filter( + (name) => name !== "info" && name !== "pack", + ), + [], + ); + yield* git(["fsck", "--full"]); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect.each(["sha1", "sha256"] as const)( + "checkpoint object isolation supports linked worktrees with $0 objects", + (format) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const sandbox = yield* fs.makeTempDirectoryScoped({ prefix: "t3-checkpoint-worktree-" }); + // A separator must remain part of the alternate's path, not split the list. + const platform = yield* HostProcess.Platform; + const cwd = path.join(sandbox, platform === "win32" ? "repo;space" : 'repo:space"\u0001'); + yield* fs.makeDirectory(cwd); + yield* driver.execute({ + operation: "test.init", + cwd, + args: ["init", `--object-format=${format}`], + }); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + const worktree = path.join(sandbox, "linked"); + yield* git(["worktree", "add", "-b", "linked", worktree]); + yield* fs.writeFileString(path.join(worktree, "untracked.txt"), "linked worktree\n"); + const originalIndex = yield* fs.readFile(path.join(cwd, ".git/index")); + yield* driver.checkpoints.captureCheckpoint({ cwd: worktree, checkpointRef }); + assert.strictEqual( + (yield* git(["show", `${checkpointRef}:untracked.txt`])).stdout, + "linked worktree\n", + ); + assert.deepEqual(yield* fs.readFile(path.join(cwd, ".git/index")), originalIndex); + yield* git(["fsck", "--full"]); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + it.effect("checkpoint capture skips untracked nested repositories without a commit", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; @@ -861,6 +1249,63 @@ it.effect.each([1_700_000_000, 1_700_000_000.9999])( }).pipe(Effect.scoped, Effect.provide(layerGitContract)), ); +it.effect("a branch switch landing after the snapshot belongs to the next checkpoint", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const liveProcess = yield* VcsProcess.VcsProcess; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-checkpoint-switch-race-" }); + const git = (args: ReadonlyArray) => + driver.execute({ operation: "checkpoint-test", cwd, args }); + yield* git(["init"]); + yield* git(["config", "user.name", "Test"]); + yield* git(["config", "user.email", "test@test.com"]); + yield* fileSystem.writeFileString(path.join(cwd, "f.txt"), "A\n"); + yield* git(["add", "."]); + yield* git(["commit", "-m", "a"]); + yield* git(["checkout", "-b", "b"]); + yield* fileSystem.writeFileString(path.join(cwd, "f.txt"), "B\n"); + yield* git(["commit", "-am", "b"]); + yield* git(["checkout", "-"]); + const [first, second, third] = ["1", "2", "3"].map((turn) => + CheckpointRef.make(`refs/t3/checkpoints/switch-race/${turn}`), + ) as [CheckpointRef, CheckpointRef, CheckpointRef]; + const racedDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe( + Effect.provideService(VcsProcess.VcsProcess, { + run: Effect.fn(function* (input: VcsProcess.VcsProcessInput) { + const result = yield* liveProcess.run(input); + if (input.args.includes("write-tree")) yield* git(["checkout", "b"]).pipe(Effect.orDie); + return result; + }), + }), + ); + const changedSince = (fromCheckpointRef: CheckpointRef, toCheckpointRef: CheckpointRef) => + Effect.gen(function* () { + const refs = { cwd, fromCheckpointRef, toCheckpointRef }; + return { + patch: yield* driver.checkpoints.diffCheckpoints({ ...refs, ignoreWhitespace: false }), + authored: yield* driver.checkpoints.listAuthoredPaths(refs), + }; + }); + + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef: first }); + yield* racedDriver.checkpoints.captureCheckpoint({ + cwd, + checkpointRef: second, + previousCheckpointRef: first, + }); + yield* driver.checkpoints.captureCheckpoint({ + cwd, + checkpointRef: third, + previousCheckpointRef: second, + }); + + assert.deepEqual(yield* changedSince(first, second), { patch: "", authored: null }); + assert.deepEqual(yield* changedSince(second, third), { patch: "", authored: null }); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + it.effect("checkpoint capture preserves racy edits made after resetting the index", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; @@ -1051,6 +1496,7 @@ it.effect.each(["committed", "unborn"] as const)( assert.deepStrictEqual(commands, [ "rev-parse", + ...(head === "committed" ? ["log"] : []), "config", ...(head === "committed" ? ["read-tree", "ls-files"] : []), "add", @@ -1116,6 +1562,8 @@ it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( "write-tree", "commit-tree", "update-ref", + // Without a parsed top level, capture cannot rule out submodules. + "submodule", ]); assert.strictEqual((yield* git(["show", `${checkpointRef}:file.txt`])).stdout, "unstaged\n"); assert.deepEqual(yield* fileSystem.readFile(path.join(cwd, ".git", "index")), originalIndex); @@ -1273,6 +1721,154 @@ it.effect("restores a checkpoint before the first commit without staging its fil }).pipe(Effect.scoped, Effect.provide(layerGitContract)), ); +it.effect("restores edits to skip-worktree files whose checkpoint content matches HEAD", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-skip-worktree-restore-" }); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + const skippedPath = path.join(cwd, "skipped.txt"); + yield* fileSystem.writeFileString(skippedPath, "original\n"); + yield* git(["add", "skipped.txt"]); + yield* git(["commit", "-m", "add skipped file"]); + yield* git(["update-index", "--skip-worktree", "skipped.txt"]); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + yield* fileSystem.writeFileString(skippedPath, "later\n"); + + assert.isTrue( + yield* driver.checkpoints.restoreCheckpoint({ cwd, checkpointRef, fallbackToHead: false }), + ); + + assert.strictEqual(yield* fileSystem.readFileString(skippedPath), "original\n"); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect("sparse restore rewrites manual skip-worktree files and keeps exclusions absent", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const liveProcess = yield* VcsProcess.VcsProcess; + // One byte per chunk, so the index listing splits inside every record. + const driver = yield* GitVcsDriver.makeVcsDriverShape().pipe( + Effect.provideService(VcsProcess.VcsProcess, { + run: (input) => { + const onStdoutChunk = input.onStdoutChunk; + return liveProcess.run( + onStdoutChunk === undefined + ? input + : { + ...input, + onStdoutChunk: (chunk) => { + for (let index = 0; index < chunk.length; index++) + onStdoutChunk(chunk.subarray(index, index + 1)); + }, + }, + ); + }, + }), + ); + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-sparse-restore-" }); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + for (const name of ["in/normal", "in/skipped", "in/s space", "out/deep/absent", "out/stray"]) { + yield* fs.makeDirectory(path.dirname(path.join(cwd, name)), { recursive: true }); + yield* fs.writeFileString(path.join(cwd, name), "original\n"); + } + yield* git(["add", "."]); + yield* git(["commit", "-m", "sparse fixture"]); + yield* git(["sparse-checkout", "set", "--cone", "--sparse-index", "in"]); + // Keeps Git from clearing the bit on files that are present. + yield* git(["config", "sparse.expectFilesOutsideOfPatterns", "true"]); + yield* git(["update-index", "--skip-worktree", "in/skipped", "in/s space"]); + const captureCwd = path.join(cwd, "in"); + yield* driver.checkpoints.captureCheckpoint({ cwd: captureCwd, checkpointRef }); + yield* fs.writeFileString(path.join(cwd, "in/skipped"), "later\n"); + yield* fs.writeFileString(path.join(cwd, "in/s space"), "later\n"); + yield* fs.writeFileString(path.join(cwd, "in/normal"), "later\n"); + + for (const restoreCwd of [captureCwd, cwd]) { + assert.isTrue( + yield* driver.checkpoints.restoreCheckpoint({ + cwd: restoreCwd, + checkpointRef, + fallbackToHead: false, + }), + ); + assert.strictEqual(yield* fs.readFileString(path.join(cwd, "in/skipped")), "original\n"); + assert.strictEqual(yield* fs.readFileString(path.join(cwd, "in/s space")), "original\n"); + assert.strictEqual(yield* fs.readFileString(path.join(cwd, "in/normal")), "original\n"); + assert.isFalse(yield* fs.exists(path.join(cwd, "out"))); + } + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect("restores and deletes checkpoints inside populated submodules", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const git = (cwd: string, args: ReadonlyArray) => + driver.execute({ operation: "test", cwd, args }); + const makeRepository = Effect.fn("makeRepository")(function* (prefix: string) { + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix }); + yield* git(cwd, ["init"]); + yield* git(cwd, ["config", "user.email", "test@test.com"]); + yield* git(cwd, ["config", "user.name", "Test"]); + yield* fileSystem.writeFileString(path.join(cwd, "file.txt"), "original\n"); + yield* git(cwd, ["add", "."]); + yield* git(cwd, ["commit", "-m", "initial"]); + return cwd; + }); + const nested = yield* makeRepository("t3-submodule-nested-"); + const library = yield* makeRepository("t3-submodule-library-"); + const addSubmodule = (cwd: string, url: string, name: string) => + git(cwd, ["-c", "protocol.file.allow=always", "submodule", "add", url, name]).pipe( + Effect.andThen(git(cwd, ["commit", "-m", `add ${name}`])), + ); + yield* addSubmodule(library, nested, "nested"); + const root = yield* makeRepository("t3-submodule-root-"); + yield* addSubmodule(root, library, "library"); + yield* git(root, [ + "-c", + "protocol.file.allow=always", + "submodule", + "update", + "--init", + "--recursive", + ]); + const files = ["file.txt", "library/file.txt", "library/nested/file.txt"].map((file) => + path.join(root, file), + ); + const checkpointRef = CheckpointRef.make("refs/t3/checkpoints/submodules"); + yield* fileSystem.writeFileString(files[1]!, "captured\n"); + yield* driver.checkpoints.captureCheckpoint({ cwd: root, checkpointRef }); + + for (const file of files) { + yield* fileSystem.writeFileString(file, "changed\n"); + yield* fileSystem.writeFileString(`${file}.added`, "new\n"); + } + assert.isTrue( + yield* driver.checkpoints.restoreCheckpoint({ + cwd: root, + checkpointRef, + fallbackToHead: false, + }), + ); + + assert.deepStrictEqual( + yield* Effect.forEach(files, (file) => fileSystem.readFileString(file)), + ["original\n", "captured\n", "original\n"], + ); + for (const file of files) assert.isFalse(yield* fileSystem.exists(`${file}.added`)); + + yield* driver.checkpoints.deleteCheckpointRefs({ cwd: root, checkpointRefs: [checkpointRef] }); + for (const cwd of [root, path.join(root, "library"), path.join(root, "library", "nested")]) { + const ref = yield* git(cwd, ["for-each-ref", "refs/t3"]); + assert.strictEqual(ref.stdout, ""); + } + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + // The real VCS process, spawning `command` wherever the driver runs git. const vcsProcessSpawning = (command: string) => Layer.effect( diff --git a/apps/server/src/vcs/GitVcsDriver.ts b/apps/server/src/vcs/GitVcsDriver.ts index dbc87e088d90..d1e9dae66f1f 100644 --- a/apps/server/src/vcs/GitVcsDriver.ts +++ b/apps/server/src/vcs/GitVcsDriver.ts @@ -4,6 +4,7 @@ import * as Context from "effect/Context"; import * as SecureRandom from "effect/crypto/SecureRandom"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; @@ -12,6 +13,7 @@ import { ChildProcessSpawner } from "effect/process"; import { GitCommandError, + VcsCheckpointStorageError, VcsProcessExitError, VcsUnsupportedOperationError, type VcsSwitchRefInput, @@ -37,6 +39,7 @@ import { } from "@t3tools/contracts"; import { isMissingExecutable } from "@t3tools/source-control-core/server/discovery"; import * as HostProcess from "@t3tools/shared/HostProcess"; +import { quoteGitPatchPath } from "@t3tools/shared/gitPatchPath"; import { makeGitVcsDriverCore, PATCH_RENDER_PREFIX_ARGS, @@ -121,7 +124,10 @@ export interface ExecuteGitProgress { } export interface CreateWorktreeProgress { - /** Git refuses existing paths; claimed paths belong to this call and can be removed on cancel. */ + /** + * Runs uninterruptibly once the worktree exists; from then on the caller owns its cleanup. + * For new-branch calls on Git supporting add --reason, cancellation before the claim removes what it made. + */ readonly onWorktreeClaimed?: (path: string) => Effect.Effect; readonly onCheckoutProgress?: (input: { percent: number; @@ -177,12 +183,16 @@ export interface GitPushResult { status: "pushed" | "skipped_up_to_date"; branch: string; upstreamBranch?: string | undefined; + /** Set when `upstreamBranch` is on a configured push remote instead of the tracked upstream. */ + pushRemote?: string | undefined; setUpstream?: boolean | undefined; } export interface GitPushTarget { remoteName: string; branchName: string; + /** The tracked upstream is current, yet pushes go to this configured push remote. */ + viaPushRemote: boolean; } export interface GitRangeContext { @@ -327,6 +337,8 @@ export class GitVcsDriver extends Context.Service< cwd: string, options?: GitRemoteStatusOptions, ) => Effect.Effect; + /** Refs, HEAD and shallow boundaries. Null when the snapshot cannot be read. */ + readonly readRefSnapshot: (cwd: string) => Effect.Effect; readonly prepareCommitContext: ( cwd: string, filePaths?: readonly string[], @@ -458,6 +470,8 @@ export class GitVcsDriver extends Context.Service< input: VcsSwitchRefInput, ) => Effect.Effect; readonly initRepo: (input: VcsInitInput) => Effect.Effect; + /** Drops cached refs after something outside this driver changed them, such as a clone into `cwd`. */ + readonly invalidateRefs: (cwd: string) => Effect.Effect; readonly listLocalBranchNames: (cwd: string) => Effect.Effect; } >()("t3/vcs/GitVcsDriver") {} @@ -467,6 +481,10 @@ const FLAGGED_INDEX_ENTRIES_MAX_BYTES = 1024 * 1024; const CHECKPOINT_RECOVERY_MAX_CANDIDATES = 64; const CHECKPOINT_RECOVERY_TIMEOUT = "5 seconds"; const GIT_CHECK_IGNORE_MAX_STDIN_BYTES = 256 * 1024; +// Reflog subjects that only point a ref at an existing commit. Anything else counts as creating +// one, so an unknown subject keeps its commit attributed. +const REF_MOVE_REFLOG_SUBJECT = + /^(?:$|(?:checkout|reset|branch|clone|fetch)\b|[^:]*\((?:start|finish|abort|reset)\): |(?:merge|pull|rebase|cherry-pick)\b[^()]*: fast-forward$)/i; const WORKSPACE_GIT_HARDENED_CONFIG_ARGS = [ "-c", "core.fsmonitor=false", @@ -557,6 +575,7 @@ const gitCommand = Effect.fn("GitVcsDriver.gitCommand")( args: ReadonlyArray, options?: { readonly stdin?: string; + readonly onSpawn?: () => void; readonly env?: NodeJS.ProcessEnv; readonly allowNonZeroExit?: boolean; readonly timeoutMs?: number; @@ -572,6 +591,7 @@ const gitCommand = Effect.fn("GitVcsDriver.gitCommand")( cwd, spawnCwd: globalThis.process.cwd(), ...(options?.stdin !== undefined ? { stdin: options.stdin } : {}), + ...(options?.onSpawn !== undefined ? { onSpawn: options.onSpawn } : {}), ...(options?.env !== undefined ? { env: options.env } : {}), ...(options?.allowNonZeroExit !== undefined ? { allowNonZeroExit: options.allowNonZeroExit } @@ -585,6 +605,10 @@ const gitCommand = Effect.fn("GitVcsDriver.gitCommand")( }), ); +const HEAD_MOVES_SCAN_LIMIT = 512; +const HEAD_MARK_TRAILER = "T3-Head-Moved-At"; +const HEAD_MARK_TRAILER_PATTERN = new RegExp(`^${HEAD_MARK_TRAILER}: (\\d+) (\\d+)$`, "m"); + export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* () { const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -639,6 +663,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( const execute: VcsDriver.VcsDriver["Service"]["execute"] = (input) => gitCommand(vcsProcess, input.operation, input.cwd, input.args, { ...(input.stdin !== undefined ? { stdin: input.stdin } : {}), + ...(input.onSpawn !== undefined ? { onSpawn: input.onSpawn } : {}), ...(input.env !== undefined ? { env: input.env } : {}), ...(input.allowNonZeroExit !== undefined ? { allowNonZeroExit: input.allowNonZeroExit } : {}), ...(input.timeoutMs !== undefined ? { timeoutMs: input.timeoutMs } : {}), @@ -912,6 +937,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( "--git-common-dir", "--git-path", "index", + "--show-toplevel", "--show-prefix", "--verify", "--quiet", @@ -922,13 +948,15 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( const headExists = result.exitCode === 0; const headChecked = headExists || result.exitCode === 1; const lines = result.stdout.split("\n"); - const [gitCommonDir = "", indexPath = "", prefix = ""] = lines; - if (headChecked && lines.length === (headExists ? 5 : 4) && lines.at(-1) === "") { + const [gitCommonDir = "", indexPath = "", topLevel = "", prefix = "", head = ""] = lines; + if (headChecked && lines.length === (headExists ? 6 : 5) && lines.at(-1) === "") { return { gitCommonDir: gitCommonDir.trim(), indexPath: indexPath.trim(), + topLevel: topLevel.trim(), prefix, headExists, + head: headExists ? head : undefined, }; } // Git before 2.31 echoes --path-format, and a path containing a newline spans lines. @@ -942,11 +970,26 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( return { gitCommonDir: fallbackCommonDir, indexPath: undefined, + topLevel: undefined, prefix: prefixResult.stdout.replace(/\n$/, ""), headExists: headChecked ? headExists : yield* hasHeadCommit(cwd), + head: undefined, }; }); + // $displaypath is relative to cwd, so "../" marks submodules outside a nested workspace. + const listSubmodules = Effect.fn("GitVcsDriver.listSubmodules")(function* (cwd: string) { + const result = yield* execute({ + operation: "GitVcsDriver.checkpoints.listSubmodules", + cwd, + args: ["submodule", "foreach", "--quiet", 'printf "%s\\0" "$displaypath"'], + }); + return result.stdout + .split("\0") + .filter((entry) => entry !== "" && !entry.startsWith("../")) + .map((entry) => path.join(cwd, entry)); + }); + // Git defaults and macOS writeout-only can leave empty refs after crashes; flush before publication. // Set fsync first: Git 2.36 ignores unknown batch without resetting the method. // Batch fully flushes refs and equals fsync on macOS HFS+/APFS and Windows NTFS/ReFS. @@ -965,6 +1008,64 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( // core.longpaths is disabled in the user's Git configuration. const longPaths = hostPlatform === "win32" ? ["-c", "core.longpaths=true"] : []; + // Newest first: the commit each HEAD move landed on, when, and Git's reason for it. + const readHeadMoves = (cwd: string) => + execute({ + operation: VcsProcess.CHECKPOINT_CAPTURE_OPERATION, + cwd, + args: [ + "-c", + "log.showSignature=false", + "log", + "--walk-reflogs", + "-n", + String(HEAD_MOVES_SCAN_LIMIT), + "--date=unix", + "--format=%H %gd %gs", + "HEAD", + "--", + ], + allowNonZeroExit: true, + }).pipe( + Effect.map((result) => + result.exitCode !== 0 + ? [] + : result.stdout.split("\n").flatMap((line) => { + const [, to, at, reason] = /^([0-9a-f]+) HEAD@\{(\d+)\} (.*)$/.exec(line) ?? []; + return to === undefined || at === undefined || reason === undefined + ? [] + : [{ to, at, reason }]; + }), + ), + Effect.orElseSucceed(() => []), + ); + + // A checkpoint taken after a branch switch has a second parent: the checkpoint before it with + // the switch applied, which is where that turn started. Other ranges start at `from` itself. + const resolveTurnStart = (input: VcsDriver.VcsListAuthoredPathsInput) => + execute({ + operation: "GitVcsDriver.checkpoints.resolveTurnStart", + cwd: input.cwd, + args: [ + "-c", + "log.showSignature=false", + "log", + "--no-walk=unsorted", + "--format=%H %P", + `${input.fromCheckpointRef}^{commit}`, + `${input.toCheckpointRef}^2`, + ], + allowNonZeroExit: true, + }).pipe( + Effect.map((result): string => { + const [fromLine = "", startLine = ""] = result.stdout.trimEnd().split("\n"); + const [start = "", , startedFrom] = startLine.split(" "); + return result.exitCode === 0 && startedFrom === fromLine.split(" ")[0] + ? start + : input.fromCheckpointRef; + }), + ); + const checkpoints: VcsDriver.VcsCheckpointOps = { captureCheckpoint: Effect.fn("GitVcsDriver.checkpoints.captureCheckpoint")(function* (input) { const operation = VcsProcess.CHECKPOINT_CAPTURE_OPERATION; @@ -979,15 +1080,56 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( "-c", "splitIndex.sharedIndexExpire=never", ]; - const { gitCommonDir, indexPath, prefix, headExists } = yield* resolveCaptureRepository( - input.cwd, - ); + const { gitCommonDir, indexPath, topLevel, prefix, headExists, head } = + yield* resolveCaptureRepository(input.cwd); + // Read before the snapshot and pinned to the same HEAD: a switch that lands while files are + // being staged then belongs to the next checkpoint instead of being applied to this one. + const headMoves = head === undefined ? [] : yield* readHeadMoves(input.cwd); + const lastMove = headMoves.at(0); + // Moves share a second, so the mark also counts how many of that second it has seen. + const headMark = + lastMove !== undefined && lastMove.to === head + ? `${lastMove.at} ${headMoves.filter((move) => move.at === lastMove.at).length}` + : undefined; const indexId = yield* random.uuidV4(); const tempIndexPath = path.join(gitCommonDir, `t3-checkpoint-index-${indexId}`); const hostEnv = yield* HostProcess.Environment; + // VcsProcess clears inherited object selectors, so every other command uses this store. + const sharedObjects = path.join(gitCommonDir, "objects"); + const quotedObjects = quoteGitPatchPath(sharedObjects); + // Isolate bulk-checkin packs so cleanup cannot remove another Git writer's objects. + const privateObjects = yield* Effect.acquireRelease( + fileSystem + .makeTempDirectory({ directory: sharedObjects, prefix: "tmp_objdir-t3-checkpoint-" }) + .pipe( + Effect.mapError( + (cause) => + new VcsCheckpointStorageError({ + operation, + cwd: input.cwd, + stage: "isolate", + cause, + }), + ), + ), + (directory) => + fileSystem.remove(directory, { recursive: true, force: true }).pipe( + Effect.catch((cause) => + Effect.logWarning("Could not remove private checkpoint objects", { + reason: cause.reason._tag, + }), + ), + ), + ); const commitEnv: NodeJS.ProcessEnv = { - ...hostEnv, GIT_INDEX_FILE: tempIndexPath, + GIT_OBJECT_DIRECTORY: privateObjects, + // A filter must not publish refs that depend on our disposable store. + GIT_QUARANTINE_PATH: privateObjects, + // Git accepts C-quoted alternate paths, including path-list separators. + GIT_ALTERNATE_OBJECT_DIRECTORIES: quotedObjects.startsWith('"') + ? quotedObjects + : `"${quotedObjects}"`, GIT_AUTHOR_NAME: "T3 Code", GIT_AUTHOR_EMAIL: "t3code@users.noreply.github.com", GIT_COMMITTER_NAME: "T3 Code", @@ -1291,6 +1433,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( GIT_INDEX_FILE: undefined, GIT_OBJECT_DIRECTORY: undefined, GIT_ALTERNATE_OBJECT_DIRECTORIES: undefined, + GIT_QUARANTINE_PATH: undefined, }; const exclusions: Array = []; for (const entry of candidates) { @@ -1332,8 +1475,137 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( }); } - const message = `t3 checkpoint ref=${input.checkpointRef}`; - // HEAD is the parent so listAuthoredPaths can see how HEAD moved between checkpoints. + // Switching branches rewrites files the turn never wrote, and only HEAD's reflog tells a + // switch from a reset. The trailer marks where that reflog stood for the next checkpoint. + const turnStart = + headMark === undefined || input.previousCheckpointRef === undefined + ? null + : yield* Effect.gen(function* () { + const previous = yield* execute({ + operation, + cwd: input.cwd, + args: [ + "-c", + "log.showSignature=false", + "log", + "--no-walk", + "--format=%H %T %ct %P%n%B", + `${input.previousCheckpointRef}^{commit}`, + ], + allowNonZeroExit: true, + }); + if (previous.exitCode !== 0) return null; + const [commit = "", tree = "", capturedAt = "", previousHead = ""] = previous.stdout + .slice(0, previous.stdout.indexOf("\n")) + .split(" "); + const [, markedAt, markedCount] = + HEAD_MARK_TRAILER_PATTERN.exec(previous.stdout) ?? []; + const oldestMarked = headMoves.findLastIndex((move) => move.at === markedAt); + // -1 when the scan ended inside the marked second, so the count cannot be placed. + const since = + oldestMarked === HEAD_MOVES_SCAN_LIMIT - 1 + ? -1 + : oldestMarked - Number(markedCount) + 1; + if (headMoves[since]?.to !== previousHead) return null; + const switches = headMoves + .slice(0, since) + .map((move, index) => ({ ...move, from: headMoves[index + 1]!.to })) + .filter( + (move) => + move.reason.startsWith("checkout: moving from ") && move.from !== move.to, + ) + .toReversed(); + const landedOn = switches.at(-1)?.to; + if (landedOn === undefined) return null; + + yield* execute({ + operation, + cwd: input.cwd, + args: [...indexConfig, "-c", "index.sparse=false", "read-tree", tree], + env: commitEnv, + }); + for (const { from, to } of switches) { + const changes = yield* execute({ + operation, + cwd: input.cwd, + args: [ + "-c", + "core.quotePath=true", + "diff-tree", + "-r", + "--no-renames", + from, + to, + ], + maxOutputBytes: VcsDriver.CHECKPOINT_DIFF_MAX_OUTPUT_BYTES, + outputMode: "error", + }); + // A switch succeeds only where the files it replaces were unmodified, so each + // changed path takes the destination's version. Mode 000000 drops the path. + const entries = changes.stdout + .split("\n") + // A switch leaves a submodule's own checkout where it was. + .filter((line) => line !== "" && !/^:(?:160000 |[0-7]{6} 160000 )/.test(line)) + .map((line) => + line.replace( + /^:[0-7]{6} ([0-7]{6}) [0-9a-f]+ ([0-9a-f]+) [A-Z]\t/, + "$1 $2\t", + ), + ); + if (entries.length === 0) continue; + yield* execute({ + operation, + cwd: input.cwd, + args: [ + ...indexConfig, + "-c", + "index.sparse=false", + "update-index", + "--index-info", + ], + stdin: `${entries.join("\n")}\n`, + env: commitEnv, + }); + } + const startTree = (yield* execute({ + operation, + cwd: input.cwd, + args: [...indexConfig, ...durableWrite, "write-tree"], + env: commitEnv, + })).stdout.trim(); + if (startTree === "" || startTree === tree) return null; + const startCommit = yield* execute({ + operation, + cwd: input.cwd, + args: [ + ...durableWrite, + "commit-tree", + startTree, + "-p", + landedOn, + "-p", + commit, + "-m", + `t3 checkpoint turn start ref=${input.checkpointRef}`, + ], + // listAuthoredPaths reads commits newer than the start as the turn's own. + env: { ...commitEnv, GIT_COMMITTER_DATE: `${capturedAt} +0000` }, + }); + return startCommit.stdout.trim() || null; + }).pipe( + Effect.catch((cause) => + Effect.logWarning("checkpoint branch switch turn start failed", { + cwd: input.cwd, + checkpointRef: input.checkpointRef, + cause: String(cause), + }).pipe(Effect.as(null)), + ), + ); + + const message = `t3 checkpoint ref=${input.checkpointRef}${ + headMark === undefined ? "" : `\n\n${HEAD_MARK_TRAILER}: ${headMark}` + }`; + // HEAD is the first parent so listAuthoredPaths can see how HEAD moved between checkpoints. const commitTreeResult = yield* execute({ operation, cwd: input.cwd, @@ -1341,7 +1613,8 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( ...durableWrite, "commit-tree", treeOid, - ...(headExists ? ["-p", "HEAD"] : []), + ...(headExists ? ["-p", head ?? "HEAD"] : []), + ...(turnStart === null ? [] : ["-p", turnStart]), "-m", message, ], @@ -1358,13 +1631,134 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( }); } + // Git writers have exited; bound publication by fanout directory. + yield* fileSystem.readDirectory(privateObjects).pipe( + Effect.flatMap((directories) => + Effect.forEach( + directories.filter( + (directory) => directory === "pack" || /^[0-9a-f]{2}$/.test(directory), + ), + (directory) => + Effect.gen(function* () { + const source = path.join(privateObjects, directory); + const destination = path.join(sharedObjects, directory); + // Most fanouts already exist; avoid constructing an EEXIST error + // for each of them on every capture. mkdir still handles the race. + if (!(yield* fileSystem.exists(destination))) + yield* fileSystem.makeDirectory(destination).pipe( + Effect.andThen( + Effect.gen(function* () { + // Git has already applied core.sharedRepository to the source. + // chmod bypasses umask, but never changes a pre-existing bucket. + const { mode } = yield* fileSystem.stat(source); + yield* fileSystem.chmod(destination, mode & 0o7777); + }), + ), + Effect.catchIf( + (cause) => cause.reason._tag === "AlreadyExists", + () => Effect.void, + ), + Effect.uninterruptible, + ); + const names = yield* fileSystem.readDirectory(source); + // Per-object spans would swamp the capture span for large workspaces. + const linkObject = Effect.fnUntraced(function* (name: string) { + const from = path.join(source, name); + const to = path.join(destination, name); + yield* fileSystem.link(from, to).pipe( + Effect.catchIf( + (cause) => + cause.reason._tag === "PermissionDenied" || + cause.reason._tag === "Unknown", + () => + Effect.gen(function* () { + if (yield* fileSystem.exists(to)) return; + // Git also falls back to rename without hardlinks; identical object names are interchangeable. + yield* fileSystem.rename(from, to); + }), + ), + Effect.catchIf( + (cause) => cause.reason._tag === "AlreadyExists", + () => Effect.void, + ), + ); + }); + if (directory !== "pack") { + yield* Effect.forEach( + names.filter((name) => /^(?:[0-9a-f]{38}|[0-9a-f]{62})$/.test(name)), + linkObject, + { discard: true }, + ); + return; + } + const files = new Set(names); + for (const name of names) { + if (!/^pack-(?:[0-9a-f]{40}|[0-9a-f]{64})\.pack$/.test(name)) continue; + const base = name.slice(0, -5); + if (!files.has(`${base}.idx`)) continue; + // Link metadata before the pack so an I/O error cannot strand a + // large unindexed pack. Defer cancellation across this short group + // of links. Never roll back shared links: a concurrent writer may + // already depend on them. Ref publication still happens last. + yield* Effect.forEach( + [".keep", ".promisor", ".rev", ".idx", ".pack"] + .map((extension) => `${base}${extension}`) + .filter((file) => files.has(file)), + linkObject, + { discard: true }, + ).pipe(Effect.uninterruptible); + } + }), + { concurrency: 16, discard: true }, + ), + ), + Effect.mapError( + (cause) => + new VcsCheckpointStorageError({ operation, cwd: input.cwd, stage: "publish", cause }), + ), + ); + yield* execute({ operation, cwd: input.cwd, args: [...durableWrite, "update-ref", input.checkpointRef, commitOid], }); }).pipe(Effect.ensuring(cleanupTempIndex)); - }), + + // The tree records only each submodule's HEAD, so submodules keep their own ref. + const hasSubmodules = + topLevel === undefined || + (yield* fileSystem + .exists(path.join(topLevel, ".gitmodules")) + .pipe(Effect.orElseSucceed(() => true))); + if (!hasSubmodules) return; + // A submodule that cannot be captured drops its ref, so restore leaves it alone. + yield* listSubmodules(input.cwd).pipe( + Effect.flatMap((submodules) => + Effect.forEach( + submodules, + (cwd) => + checkpoints + .captureCheckpoint({ cwd, checkpointRef: input.checkpointRef }) + .pipe( + Effect.catch((cause) => + Effect.logWarning("submodule checkpoint capture failed", { cwd, cause }).pipe( + Effect.andThen( + checkpoints + .deleteCheckpointRefs({ cwd, checkpointRefs: [input.checkpointRef] }) + .pipe(Effect.ignore), + ), + ), + ), + ), + { discard: true }, + ), + ), + Effect.catch((cause) => + Effect.logWarning("submodule checkpoint capture failed", { cwd: input.cwd, cause }), + ), + ); + }, Effect.scoped), hasCheckpointRef: (input) => resolveCheckpointCommit(input.cwd, input.checkpointRef).pipe( @@ -1389,23 +1783,16 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( cwd: input.cwd, args: ["ls-files", "--cached", `--with-tree=${commitOid}`, "-z", "--", "."], }); - // An empty index and checkpoint have nothing for git restore's pathspec to match. - if (tracked.stdout.length > 0) { - yield* execute({ - operation, - cwd: input.cwd, - args: [ - ...longPaths, - "restore", - "--source", - commitOid, - "--worktree", - "--staged", - "--", - ".", - ], - }); - } + const sparseConfig = + tracked.stdout.length > 0 + ? yield* execute({ + operation, + cwd: input.cwd, + args: ["config", "--bool", "core.sparseCheckout"], + allowNonZeroExit: true, + }) + : null; + const sparseCheckout = sparseConfig?.stdout.trim() === "true"; // Restoring away the last tracked file can remove a nested workspace directory. const recreateWorkspace = fileSystem.makeDirectory(input.cwd, { recursive: true }).pipe( Effect.mapError( @@ -1428,6 +1815,95 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( args: [...longPaths, "reset", "--quiet", "--", "."], }); }); + let mutationStarted = false; + const onSpawn = () => { + mutationStarted = true; + }; + // Queued commands preserve user staging; spawned children stop before this scope resets it. + yield* Effect.addFinalizer((exit) => + mutationStarted && Exit.hasInterrupts(exit) ? resetIndex.pipe(Effect.ignore) : Effect.void, + ); + // An empty index and checkpoint have nothing for git restore's pathspec to match. + if (tracked.stdout.length > 0) { + const restore = [...longPaths, "restore", "--source", commitOid, "--worktree", "--staged"]; + // restore leaves skip-worktree paths alone. Outside a sparse checkout every such bit was + // set by hand and capture recorded the file, so override them all. + yield* execute({ + operation, + cwd: input.cwd, + args: [...restore, ...(sparseCheckout ? [] : ["--ignore-skip-worktree-bits"]), "--", "."], + onSpawn, + }); + if (sparseCheckout) { + // Sparse exclusions stay excluded; only skip-worktree paths the rules select are manual. + const manual = yield* Effect.gen(function* () { + const skipped: string[] = []; + let recordStart = true; + let keep = false; + let record: Uint8Array[] = []; + yield* vcsProcess.run({ + operation, + command: "git", + cwd: input.cwd, + args: ["ls-files", "--full-name", "--sparse", "-v", "-z", "--", "."], + maxOutputBytes: 4_096, + outputMode: "truncate", + // The index can be huge; keep only the "S "/"s " records of the NUL-delimited list. + onStdoutChunk: (chunk) => { + let start = 0; + while (start < chunk.length) { + const end = chunk.indexOf(0, start); + if (recordStart) { + keep = chunk[start] === 83 || chunk[start] === 115; + record = []; + } + if (keep) { + record.push(Buffer.from(chunk.subarray(start, end === -1 ? undefined : end))); + if (end !== -1) { + const entry = Buffer.concat(record).subarray(2); + record = []; + // Sparse directory entries are exclusions; stdin cannot carry non-UTF-8. + if (entry.at(-1) !== 47 && NodeBuffer.isUtf8(entry)) + skipped.push(entry.toString("utf8")); + } + } + recordStart = end !== -1; + if (!recordStart) break; + start = end + 1; + } + }, + }); + if (skipped.length === 0) return skipped; + const stdin = skipped.join("\0") + "\0"; + // check-rules echoes a subset of its input, so the input size bounds its output. + const rules = yield* execute({ + operation, + cwd: input.cwd, + args: ["sparse-checkout", "check-rules", "-z"], + stdin, + maxOutputBytes: Buffer.byteLength(stdin), + outputMode: "error", + }); + return rules.stdout.split("\0").filter((entry) => entry.length > 0); + }).pipe( + // A Git without ls-files --sparse or check-rules keeps the plain restore's result. + Effect.orElseSucceed((): string[] => []), + ); + if (manual.length > 0) { + yield* execute({ + operation, + cwd: input.cwd, + args: [ + ...restore, + "--ignore-skip-worktree-bits", + "--pathspec-from-file=-", + "--pathspec-file-nul", + ], + stdin: manual.map((entry) => `:(top,literal)${entry}\0`).join(""), + }); + } + } + } yield* recreateWorkspace; const clean = execute({ operation, @@ -1435,6 +1911,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( args: [...longPaths, "clean", "-fd", "--", "."], env: { LC_ALL: "C" }, allowNonZeroExit: true, + onSpawn, }); let cleaned = yield* clean; // Removing a new .gitignore exposes the files it hid; clean again until nothing is left. @@ -1487,8 +1964,34 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( } yield* resetIndex; + // A submodule without this ref was not captured, and restoring it to HEAD would lose edits. + // The root is already restored, so a failing submodule must not hide that from the caller. + yield* listSubmodules(input.cwd).pipe( + Effect.flatMap((submodules) => + Effect.forEach( + submodules, + (cwd) => + checkpoints + .restoreCheckpoint({ + cwd, + checkpointRef: input.checkpointRef, + fallbackToHead: false, + }) + .pipe( + Effect.catch((cause) => + Effect.logWarning("submodule checkpoint restore failed", { cwd, cause }), + ), + ), + { discard: true }, + ), + ), + Effect.catch((cause) => + Effect.logWarning("submodule checkpoint restore failed", { cwd: input.cwd, cause }), + ), + ); + return true; - }), + }, Effect.scoped), diffCheckpoints: Effect.fn("GitVcsDriver.checkpoints.diffCheckpoints")(function* (input) { const operation = "GitVcsDriver.checkpoints.diffCheckpoints"; @@ -1502,15 +2005,13 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( "checkpoint.fallback_from_to_head": input.fallbackFromToHead, }); - let fromRevision: string = input.fromCheckpointRef; + let fromRevision = yield* resolveTurnStart(input); if (input.fallbackFromToHead === true) { const resolvedFromCommit = yield* resolveCheckpointCommit( input.cwd, input.fromCheckpointRef, ); - if (resolvedFromCommit) { - fromRevision = resolvedFromCommit; - } else { + if (!resolvedFromCommit) { const headCommit = yield* resolveHeadCommit(input.cwd); if (!headCommit) { return yield* new VcsProcessExitError({ @@ -1569,25 +2070,36 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( listAuthoredPaths: Effect.fn("GitVcsDriver.checkpoints.listAuthoredPaths")(function* (input) { const operation = "GitVcsDriver.checkpoints.listAuthoredPaths"; + const turnStart = yield* resolveTurnStart(input); const heads = yield* execute({ operation, cwd: input.cwd, args: [ "log", "--no-walk=unsorted", - "--format=%P %ct", - `${input.fromCheckpointRef}^{commit}`, + "--format=%ct %P", + `${turnStart}^{commit}`, `${input.toCheckpointRef}^{commit}`, ], }); const [fromLine = "", toLine = ""] = heads.stdout.trimEnd().split("\n"); - const [startHead = "", capturedAt = ""] = fromLine.split(" "); - const [endHead = ""] = toLine.split(" "); + const [capturedAt = "", startHead = ""] = fromLine.split(" "); + const [, endHead = ""] = toLine.split(" "); if (startHead === "" || endHead === "" || startHead === endHead) { return null; } - const listPaths = (args: ReadonlyArray) => + const listLines = (args: ReadonlyArray, stdin?: string) => + execute({ + operation, + cwd: input.cwd, + // log.showSignature prints verification text into the output. + args: ["-c", "log.showSignature=false", ...args], + ...(stdin === undefined ? {} : { stdin }), + maxOutputBytes: VcsDriver.CHECKPOINT_DIFF_MAX_OUTPUT_BYTES, + outputMode: "error", + }).pipe(Effect.map((result) => result.stdout.split("\n").filter((line) => line !== ""))); + const listPaths = (args: ReadonlyArray, stdin?: string) => execute({ operation, cwd: input.cwd, @@ -1603,26 +2115,73 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( "--no-renames", "--no-ext-diff", ], + ...(stdin === undefined ? {} : { stdin }), maxOutputBytes: VcsDriver.CHECKPOINT_DIFF_MAX_OUTPUT_BYTES, outputMode: "error", }).pipe(Effect.map((result) => result.stdout.split("\0"))); + + // A commit is an import when a fetch delivered it since the previous checkpoint and no + // reflog in any worktree shows this repository creating it. --all covers every worktree's + // HEAD, and with --walk-reflogs --since compares each entry's own time. + const listImportedCommits = Effect.gen(function* () { + const entries = (yield* listLines([ + "log", + "--walk-reflogs", + "--all", + `--since=@${capturedAt}`, + "--format=%H %gD %gs", + ])).map((line) => { + const [commit = "", selector = "", ...subject] = line.split(" "); + return { + commit, + fetched: selector.startsWith("refs/remotes/"), + subject: subject.join(" "), + }; + }); + // A push records commits this repository already had. + const fetchedTips = new Set( + entries + .filter((entry) => entry.fetched && entry.subject !== "update by push") + .map((entry) => entry.commit), + ); + if (fetchedTips.size === 0) return fetchedTips; + // Fetching another branch can deliver a commit that was made here and pushed. + const createdHere = new Set( + entries + .filter((entry) => !entry.fetched && !REF_MOVE_REFLOG_SUBJECT.test(entry.subject)) + .map((entry) => entry.commit), + ); + const fetched = yield* listLines( + ["rev-list", `--since=@${capturedAt}`, "--stdin"], + `${[...fetchedTips, `^${startHead}`].join("\n")}\n`, + ); + return new Set(fetched.filter((commit) => !createdHere.has(commit))); + }); + // Remerge diffs retain merge conflict fixes while excluding clean upstream changes. - const pathLists = yield* Effect.all( + const [recentCommits, ...pathLists] = yield* Effect.all( [ - listPaths(["diff", startHead, `${input.fromCheckpointRef}^{commit}`]), + listLines(["rev-list", `--since=@${capturedAt}`, `${startHead}..${endHead}`]), + listPaths(["diff", startHead, `${turnStart}^{commit}`]), listPaths(["diff", endHead, `${input.toCheckpointRef}^{commit}`]), listPaths(["log", "--format=", "--diff-merges=remerge", `${endHead}..${startHead}`]), - listPaths([ - "log", - "--format=", - "--diff-merges=remerge", - `--since=@${capturedAt}`, - `${startHead}..${endHead}`, - ]), ], { concurrency: "unbounded" }, ); - return new Set(pathLists.flat().filter((path) => path.length > 0)); + // Without readable reflogs every recent commit stays attributed, as before. + const importedCommits = + recentCommits.length === 0 + ? new Set() + : yield* listImportedCommits.pipe(Effect.orElseSucceed(() => new Set())); + const ownCommits = recentCommits.filter((commit) => !importedCommits.has(commit)); + const ownCommitPaths = + ownCommits.length === 0 + ? [] + : yield* listPaths( + ["log", "--no-walk=unsorted", "--stdin", "--format=", "--diff-merges=remerge"], + `${ownCommits.join("\n")}\n`, + ); + return new Set([...pathLists.flat(), ...ownCommitPaths].filter((path) => path.length > 0)); }), deleteCheckpointRefs: Effect.fn("GitVcsDriver.checkpoints.deleteCheckpointRefs")( @@ -1638,6 +2197,10 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( }), { discard: true }, ); + const submodules = yield* listSubmodules(input.cwd).pipe(Effect.orElseSucceed(() => [])); + for (const cwd of submodules) { + yield* checkpoints.deleteCheckpointRefs({ cwd, checkpointRefs: input.checkpointRefs }); + } }, ), }; diff --git a/apps/server/src/vcs/GitVcsDriverCore.test.ts b/apps/server/src/vcs/GitVcsDriverCore.test.ts index 0fce1ca55f52..9e5b20205fef 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.test.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.test.ts @@ -1,14 +1,18 @@ // @effect-diagnostics nodeBuiltinImport:off - realpathSync.native resolves Windows 8.3 short names, which the Effect realPath does not. +import * as NodeChildProcess from "node:child_process"; import * as NodeFS from "node:fs"; +import * as NodeNet from "node:net"; import * as NodePath from "node:path"; import * as NodeURL from "node:url"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { assert, it, describe } from "@effect/vitest"; +import { afterAll, vi } from "vite-plus/test"; import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as Deferred from "effect/Deferred"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; @@ -47,6 +51,7 @@ import { windowsLongPathConfigEnv, } from "./GitVcsDriverCore.ts"; import * as GitVcsDriver from "./GitVcsDriver.ts"; +import { VcsAutomaticFetchInterval } from "./VcsAutomaticFetchInterval.ts"; import { gitConfigEnv } from "./VcsProcess.ts"; const encodeGitCommandError = Schema.encodeEffect(Schema.fromJsonString(GitCommandError)); @@ -341,6 +346,20 @@ const git = ( return result.stdout.trim(); }); +const [, gitMajor, gitMinor] = + /(\d+)\.(\d+)/u.exec(NodeChildProcess.execFileSync("git", ["--version"], { encoding: "utf8" })) ?? + []; +const gitHasReftable = Number(gitMajor) > 2 || (Number(gitMajor) === 2 && Number(gitMinor) >= 45); + +const gitWithStdin = ( + cwd: string, + args: ReadonlyArray, + stdin: string, +): Effect.Effect => + Effect.flatMap(GitVcsDriver.GitVcsDriver, (driver) => + driver.execute({ operation: "GitVcsDriver.test.git", cwd, args, stdin, timeoutMs: 10_000 }), + ).pipe(Effect.map((result) => result.stdout.trim())); + // Git blocks file submodule transports (CVE-2022-39253) despite repo config; local fixtures need env permission. const withGitFileProtocol = (effect: Effect.Effect) => Effect.flatMap(HostProcess.Environment, (env) => @@ -357,17 +376,49 @@ const initRepoWithCommit = ( GitVcsDriver.GitVcsDriver | FileSystem.FileSystem | Path.Path > => Effect.gen(function* () { - const driver = yield* GitVcsDriver.GitVcsDriver; - yield* driver.initRepo({ cwd }); - yield* git(cwd, ["config", "user.email", "test@test.com"]); - yield* git(cwd, ["config", "user.name", "Test"]); + const fileSystem = yield* FileSystem.FileSystem; + const pathService = yield* Path.Path; + // Git commits whatever already sits in cwd, so only an empty one can take the shared copy. + const empty = + !(yield* fileSystem.exists(cwd)) || (yield* fileSystem.readDirectory(cwd)).length === 0; + if (empty) { + if (!repoTemplate) { + const template = yield* fileSystem.makeTempDirectory({ + prefix: "git-vcs-driver-template-", + }); + yield* commitInitialRepo(template); + repoTemplate = template; + } + yield* fileSystem.copy(repoTemplate, cwd); + } else { + yield* commitInitialRepo(cwd); + } + const head = yield* fileSystem.readFileString(pathService.join(cwd, ".git", "HEAD")); + return { initialBranch: head.trim().replace(/^ref: refs\/heads\//, "") }; + }); + +const commitInitialRepo = (cwd: string, initArgs: ReadonlyArray = []) => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const pathService = yield* Path.Path; + yield* git(cwd, ["init", ...initArgs]); + // Newer Git detaches auto-maintenance after commit; its lock file would race the template copy. + yield* fileSystem.writeFileString( + pathService.join(cwd, ".git", "config"), + "[user]\n\temail = test@test.com\n\tname = Test\n[maintenance]\n\tauto = false\n", + { flag: "a" }, + ); yield* writeTextFile(cwd, "README.md", "# test\n"); yield* git(cwd, ["add", "."]); yield* git(cwd, ["commit", "-m", "initial commit"]); - const initialBranch = yield* git(cwd, ["branch", "--show-current"]); - return { initialBranch }; }); +// Most tests start from the same one-commit repository; they copy it instead of spawning git. +let repoTemplate: string | undefined; +afterAll(() => { + if (repoTemplate) NodeFS.rmSync(repoTemplate, { recursive: true, force: true }); +}); + it.effect.each(["root", "nested", "worktree"] as const)( "skips clean filters while the %s index is locked and resumes after unlock", (location) => @@ -509,7 +560,10 @@ it.effect.each([ assert.equal(created.failure.exitCode, hookExitCode); } const metadataQueries = spawnedArgs.filter((args) => args.includes("--git-common-dir")); - assert.deepStrictEqual(metadataQueries, [["rev-parse", "--git-common-dir"]]); + assert.deepStrictEqual( + metadataQueries, + Array.from({ length: 2 }, () => ["rev-parse", "--git-common-dir"]), + ); assert.isFalse(spawnedArgs.some((args) => args.includes("symbolic-ref"))); invalidationFailure = "none"; @@ -572,7 +626,7 @@ it.effect.each(["branch", "detached", "nested", "aliased-root", "aliased-nested" newRefName: "feature/cold", }); - assert.equal(spawnedArgs.filter((args) => args.includes("--git-common-dir")).length, 1); + assert.equal(spawnedArgs.filter((args) => args.includes("--git-common-dir")).length, 2); assert.isFalse(spawnedArgs.some((args) => args.includes("--show-toplevel"))); assert.isFalse(spawnedArgs.some((args) => args.includes("symbolic-ref"))); spawnedArgs.length = 0; @@ -634,7 +688,10 @@ it.effect.each(["bare", "unborn"] as const)( assert.equal(result.failure.operation, "GitVcsDriver.createWorktree.createBranch"); assert.equal(result.failure.detail, "git branch failed"); } - assert.equal(spawnedArgs.filter((args) => args.includes("--git-common-dir")).length, 1); + assert.equal( + spawnedArgs.filter((args) => args.includes("--git-common-dir")).length, + state === "bare" ? 2 : 1, + ); assert.isFalse(spawnedArgs.some((args) => args[0] === "symbolic-ref")); spawnedArgs.length = 0; const refs = yield* driver.listRefs({ cwd }); @@ -649,6 +706,65 @@ it.effect.each(["bare", "unborn"] as const)( ).pipe(Effect.provide(layerTest)), ); +it.effect("reads the project's own repository when the server inherited GIT_DIR", () => + Effect.scoped( + Effect.gen(function* () { + const path = yield* Path.Path; + const cwd = yield* makeTmpDir(); + const other = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + yield* initRepoWithCommit(other); + yield* git(other, ["switch", "-c", "other-repository"]); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* Effect.addFinalizer(() => Effect.sync(() => vi.unstubAllEnvs())); + vi.stubEnv("GIT_DIR", path.join(other, ".git")); + assert.equal((yield* driver.statusDetailsLocal(cwd)).branch, initialBranch); + const refs = yield* driver.listRefs({ cwd }); + assert.isFalse(refs.refs.some((ref) => ref.name === "other-repository")); + }), + ).pipe(Effect.provide(layerTest)), +); + +it.effect("reads a branch whose name starts with a parenthesis as attached", () => + Effect.scoped( + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + yield* git(cwd, ["switch", "-c", "(topic"]); + const driver = yield* GitVcsDriver.GitVcsDriver; + const status = yield* driver.statusDetailsLocal(cwd, { includeDivergence: false }); + assert.equal(status.branch, "(topic"); + }), + ).pipe(Effect.provide(layerTest)), +); + +it.effect.each(["an existing", "a new"] as const)( + "follows a checkout of %s branch made outside the driver after refs were cached", + (branch) => + Effect.scoped( + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + if (branch === "an existing") yield* git(cwd, ["branch", "feature/outside"]); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* driver.listRefs({ cwd }); + + yield* git( + cwd, + branch === "an existing" + ? ["switch", "feature/outside"] + : ["switch", "-c", "feature/outside"], + ); + + const refs = (yield* driver.listRefs({ cwd })).refs; + const previous = refs.find((ref) => ref.name === initialBranch); + assert.isTrue(refs.find((ref) => ref.name === "feature/outside")?.current); + assert.isFalse(previous?.current); + assert.isNull(previous?.worktreePath ?? null); + }), + ).pipe(Effect.provide(layerTest)), +); + it.effect.each(["on a branch", "detached", "unborn"] as const)( "reads only the repository root for review requests that ignore the branch when %s", (state) => @@ -1008,6 +1124,287 @@ describe("pullCurrentBranch", () => { }; }); + it.effect("resolves a status waiter when Pull claims its queued fetch", () => + Effect.scoped( + Effect.gen(function* () { + const { cwd, remoteHead } = yield* cloneBehindRemote; + const blockers: string[] = []; + for (let index = 0; index < 4; index++) { + const blocker = yield* cloneBehindRemote; + blockers.push(NodeFS.realpathSync.native(blocker.cwd)); + } + const targetPath = NodeFS.realpathSync.native(cwd); + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const release = yield* Deferred.make(); + const blockersStarted = yield* Deferred.make(); + const targetEntered = yield* Deferred.make(); + let running = 0; + let targetFetches = 0; + let waiterSucceeded = false; + const spawner = ChildProcessSpawner.make((command) => + Effect.gen(function* () { + const handle = yield* delegate.spawn(command); + if (!ChildProcess.isStandardCommand(command) || !command.args.includes("fetch")) + return handle; + const commandCwd = String(command.options.cwd); + if (commandCwd === targetPath) targetFetches++; + if (blockers.includes(commandCwd)) { + running++; + if (running === 4) yield* Deferred.succeed(blockersStarted, undefined); + yield* Deferred.await(release); + } + return handle; + }), + ); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options); + if (options.name === "GitVcsDriver.fetchRemoteForStatus.wait") { + const attribute = span.attribute.bind(span); + span.attribute = (key, value) => { + attribute(key, value); + if (key === "statusFetch.joining") Deferred.doneUnsafe(targetEntered, Effect.void); + }; + const end = span.end.bind(span); + span.end = (endTime, exit) => { + waiterSucceeded = Exit.isSuccess(exit); + end(endTime, exit); + }; + } + return span; + }, + }); + yield* Effect.gen(function* () { + for (const blocker of blockers) { + yield* driver.statusDetailsRemote(blocker).pipe(Effect.forkChild); + } + yield* Deferred.await(blockersStarted); + const status = yield* driver + .statusDetailsRemote(cwd) + .pipe(Effect.withTracer(tracer), Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(targetEntered); + const pull = yield* driver.pullCurrentBranch(cwd); + const refreshed = yield* Fiber.join(status); + assert.equal(pull.status, "pulled"); + assert.equal(yield* git(cwd, ["rev-parse", "HEAD"]), remoteHead); + assert.isTrue(waiterSucceeded); + assert.equal(refreshed.behindCount, 0); + assert.equal(targetFetches, 0); + }).pipe(Effect.ensuring(Deferred.succeed(release, undefined))); + }), + ).pipe(Effect.provide(layerTest)), + ); + + const claimedFetchFallbacks = ["diverged", "interrupted", "upstream-only"] as const; + + it.effect.each(claimedFetchFallbacks)( + "falls back to the claimed status fetch when Pull is %s", + (failure) => + Effect.scoped( + Effect.gen(function* () { + const { cwd, updater, remoteHead } = yield* cloneBehindRemote; + if (failure === "upstream-only") { + yield* git(updater, ["push", "origin", "HEAD:refs/heads/other"]); + yield* writeTextFile(cwd, ".git/refs/remotes/origin/other.lock", ""); + } + if (failure === "diverged") { + yield* writeTextFile(cwd, "local.txt", "local\n"); + yield* git(cwd, ["add", "local.txt"]); + yield* git(cwd, ["commit", "-m", "local commit"]); + } + const blockers: string[] = []; + for (let index = 0; index < 4; index++) { + const blocker = yield* cloneBehindRemote; + blockers.push(NodeFS.realpathSync.native(blocker.cwd)); + } + const targetPath = NodeFS.realpathSync.native(cwd); + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const release = yield* Deferred.make(); + const blockersStarted = yield* Deferred.make(); + const targetEntered = yield* Deferred.make(); + const targetFetchStarted = yield* Deferred.make(); + const pullBeforeFetch = yield* Deferred.make(); + const releasePull = yield* Deferred.make(); + let running = 0; + let targetFetches = 0; + let waiterSucceeded = false; + let targetFetchExitCode: number | undefined; + const spawner = ChildProcessSpawner.make((command) => + Effect.gen(function* () { + if (ChildProcess.isStandardCommand(command)) { + const commandCwd = NodeFS.realpathSync.native(String(command.options.cwd)); + if ( + commandCwd === targetPath && + failure === "interrupted" && + command.args[0] === "rev-parse" && + command.args[1] === "HEAD" && + command.args.length === 2 + ) { + yield* Deferred.succeed(pullBeforeFetch, undefined); + yield* Deferred.await(releasePull); + } + } + const handle = yield* delegate.spawn(command); + if (!ChildProcess.isStandardCommand(command) || !command.args.includes("fetch")) + return handle; + const commandCwd = String(command.options.cwd); + if (commandCwd === targetPath) { + targetFetches++; + yield* Deferred.succeed(targetFetchStarted, undefined); + return ChildProcessSpawner.makeHandle({ + ...handle, + exitCode: handle.exitCode.pipe( + Effect.tap((exitCode) => + Effect.sync(() => { + targetFetchExitCode = exitCode; + }), + ), + ), + }); + } + if (blockers.includes(commandCwd)) { + running++; + if (running === 4) yield* Deferred.succeed(blockersStarted, undefined); + yield* Deferred.await(release); + } + return handle; + }), + ); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options); + if (options.name === "GitVcsDriver.fetchRemoteForStatus.wait") { + const attribute = span.attribute.bind(span); + span.attribute = (key, value) => { + attribute(key, value); + if (key === "statusFetch.joining") + Deferred.doneUnsafe(targetEntered, Effect.void); + }; + const end = span.end.bind(span); + span.end = (endTime, exit) => { + waiterSucceeded = Exit.isSuccess(exit); + end(endTime, exit); + }; + } + return span; + }, + }); + yield* Effect.gen(function* () { + for (const blocker of blockers) { + yield* driver.statusDetailsRemote(blocker).pipe(Effect.forkChild); + } + yield* Deferred.await(blockersStarted); + const status = yield* driver + .statusDetailsRemote(cwd) + .pipe(Effect.withTracer(tracer), Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(targetEntered); + if (failure === "interrupted") { + const pull = yield* driver.pullCurrentBranch(cwd).pipe(Effect.forkChild); + yield* Deferred.await(pullBeforeFetch); + yield* Fiber.interrupt(pull); + const exit = yield* Fiber.await(pull); + assert.isTrue(Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)); + } else if (failure === "upstream-only") { + assert.equal((yield* driver.pullCurrentBranch(cwd)).status, "pulled"); + yield* (yield* FileSystem.FileSystem).remove( + `${cwd}/.git/refs/remotes/origin/other.lock`, + ); + } else { + const error = yield* Effect.flip(driver.pullCurrentBranch(cwd)); + assert.equal(error.operation, "GitVcsDriver.pullCurrentBranch.pull"); + } + assert.equal(targetFetches, 0); + yield* Deferred.succeed(release, undefined); + yield* Deferred.await(targetFetchStarted); + assert.isTrue((yield* Fiber.join(status)).isRepo); + assert.isTrue(waiterSucceeded); + assert.equal(targetFetches, 1); + assert.equal(targetFetchExitCode, 0); + if (failure === "upstream-only") { + assert.equal(yield* git(cwd, ["rev-parse", "refs/remotes/origin/other"]), remoteHead); + } + }).pipe( + Effect.ensuring( + Effect.all([ + Deferred.succeed(release, undefined), + Deferred.succeed(releasePull, undefined), + ]), + ), + ); + }), + ).pipe(Effect.provide(layerTest)), + ); + + it.effect("waits for a running status fetch before starting Pull", () => + Effect.scoped( + Effect.gen(function* () { + const { cwd, remoteHead } = yield* cloneBehindRemote; + const targetPath = NodeFS.realpathSync.native(cwd); + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const fetchStarted = yield* Deferred.make(); + const releaseFetch = yield* Deferred.make(); + const pullResolvedCommonDir = yield* Deferred.make(); + const pullStarted = yield* Deferred.make(); + const events: string[] = []; + const spawner = ChildProcessSpawner.make((command) => + Effect.gen(function* () { + if (!ChildProcess.isStandardCommand(command)) return yield* delegate.spawn(command); + const target = NodeFS.realpathSync.native(String(command.options.cwd)) === targetPath; + if (target && command.args.includes("pull")) { + events.push("pull-spawn"); + yield* Deferred.succeed(pullStarted, undefined); + } + const handle = yield* delegate.spawn(command); + if (target && command.args.includes("fetch")) { + yield* Deferred.succeed(fetchStarted, undefined); + yield* Deferred.await(releaseFetch); + yield* handle.exitCode; + events.push("fetch-exit"); + } + return handle; + }), + ); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options); + if (options.name === "resolveGitCommonDir") { + const end = span.end.bind(span); + span.end = (endTime, exit) => { + end(endTime, exit); + Deferred.doneUnsafe(pullResolvedCommonDir, Effect.void); + }; + } + return span; + }, + }); + yield* Effect.gen(function* () { + yield* driver.statusDetailsRemote(cwd).pipe(Effect.forkChild); + yield* Deferred.await(fetchStarted); + const pull = yield* driver + .pullCurrentBranch(cwd) + .pipe(Effect.withTracer(tracer), Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(pullResolvedCommonDir); + assert.isFalse(yield* Deferred.isDone(pullStarted)); + assert.isUndefined(pull.pollUnsafe()); + yield* Deferred.succeed(releaseFetch, undefined); + const result = yield* Fiber.join(pull); + assert.equal(result.status, "pulled"); + assert.equal(yield* git(cwd, ["rev-parse", "HEAD"]), remoteHead); + assert.deepEqual(events, ["fetch-exit", "pull-spawn"]); + }).pipe(Effect.ensuring(Deferred.succeed(releaseFetch, undefined))); + }), + ).pipe(Effect.provide(layerTest)), + ); + it.effect("still fetches other branches on the first pull", () => Effect.gen(function* () { const driver = yield* GitVcsDriver.GitVcsDriver; @@ -1167,6 +1564,48 @@ describe("pullCurrentBranch", () => { ), ), ); + + // A slow smudge filter (LFS) can hold the checkout past 30 seconds; killing it there leaves a + // half-updated working tree. The automatic pulls and pull request refresh use these defaults. + it.effect.each(["pull", "merge"] as const)( + "lets a slow %s checkout finish instead of killing it at 30 seconds", + (subcommand) => + Effect.gen(function* () { + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const { cwd, remoteHead } = yield* cloneBehindRemote; + yield* git(cwd, ["fetch", "origin"]); + const checkoutStarted = yield* Deferred.make(); + const releaseCheckout = yield* Deferred.make(); + const spawner = ChildProcessSpawner.make((command) => + Effect.gen(function* () { + const handle = yield* delegate.spawn(command); + if (!ChildProcess.isStandardCommand(command) || command.args[0] !== subcommand) { + return handle; + } + yield* Deferred.succeed(checkoutStarted, undefined); + return ChildProcessSpawner.makeHandle({ + ...handle, + exitCode: Deferred.await(releaseCheckout).pipe(Effect.andThen(handle.exitCode)), + }); + }), + ); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + + const move = + subcommand === "pull" + ? Effect.asVoid(driver.pullCurrentBranch(cwd)) + : Effect.asVoid(driver.refreshCheckedOutBranch({ cwd, targetCommit: remoteHead })); + const moving = yield* move.pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(checkoutStarted); + yield* TestClock.adjust("60 seconds"); + yield* Deferred.succeed(releaseCheckout, undefined); + + yield* Fiber.join(moving); + assert.equal(yield* git(cwd, ["rev-parse", "HEAD"]), remoteHead); + }).pipe(Effect.provide(layerTest)), + ); }); it.effect("retries an in-flight ref snapshot invalidated by a mutation", () => @@ -1412,6 +1851,53 @@ it.effect("reads worktrees from git that rejects worktree list -z", () => ), ); +it.effect("commits selected files with git that rejects --pathspec-from-file", () => + Effect.scoped( + Effect.gen(function* () { + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + // Git before 2.25 has no `add --pathspec-from-file` and exits with a usage error. + const oldGitSpawner = ChildProcessSpawner.make((command) => { + if (!ChildProcess.isStandardCommand(command)) { + return Effect.die("expected a standard Git command"); + } + if (command.args.at(-1) === "version") { + return Effect.succeed(makeSuccessfulHandle("git version 2.24.4\n")); + } + if (command.args.some((arg) => String(arg).startsWith("--pathspec-from-file"))) { + return Effect.succeed(makeFailedHandle("error: unknown option `pathspec-from-file=-'")); + } + return delegate.spawn(command); + }); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, oldGitSpawner), + ); + yield* Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const selected = ["selected[1].txt", "-dash.txt", "with space.txt"]; + for (const file of selected) yield* writeTextFile(cwd, file, "selected\n"); + yield* writeTextFile(cwd, "left.txt", "left\n"); + + yield* driver.commit(cwd, "Selected", "", { stage: { filePaths: selected } }); + + assert.sameMembers( + (yield* git(cwd, ["show", "--pretty=", "--name-only", "-z", "HEAD"])) + .split("\0") + .filter(Boolean), + selected, + ); + assert.equal(yield* git(cwd, ["status", "--porcelain"]), "?? left.txt"); + }).pipe(Effect.provideService(GitVcsDriver.GitVcsDriver, driver)); + }), + ).pipe( + Effect.provide( + layerServerConfig.pipe( + Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), + ), + ), + ), +); + it.effect("does not report a separate git dir as the worktree of its branch", () => Effect.scoped( Effect.gen(function* () { @@ -1629,6 +2115,37 @@ it.effect("does not fetch for status when the branch tracks a local branch", () ).pipe(Effect.provide(layerTest)), ); +it.effect("does not fetch for status when the automatic fetch interval is zero", () => + Effect.scoped( + Effect.gen(function* () { + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const fetches: Array> = []; + const spawner = ChildProcessSpawner.make((command) => { + if (ChildProcess.isStandardCommand(command) && command.args.includes("fetch")) { + fetches.push(command.args.map(String)); + } + return delegate.spawn(command); + }); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provideService(VcsAutomaticFetchInterval, Effect.succeed(Duration.zero)), + ); + const cwd = yield* makeTmpDir(); + const remote = yield* makeTmpDir("git-vcs-driver-remote-"); + const { initialBranch } = yield* initRepoWithCommit(cwd); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["push", "--set-upstream", "origin", initialBranch]); + + yield* driver.statusDetails(cwd); + const status = yield* driver.statusDetailsRemote(cwd); + + assert.equal(status.upstreamRef, `origin/${initialBranch}`); + assert.deepStrictEqual(fetches, []); + }), + ).pipe(Effect.provide(layerTest)), +); + it.effect("lets a worktree checkout that is still working run past five minutes", () => Effect.gen(function* () { const checkoutStarted = yield* Deferred.make(); @@ -1637,7 +2154,8 @@ it.effect("lets a worktree checkout that is still working run past five minutes" Effect.gen(function* () { if (!ChildProcess.isStandardCommand(command)) return yield* Effect.die("expected Git command"); - if (!command.args.includes("worktree")) return makeSuccessfulHandle(""); + if (!command.args.includes("worktree") || command.args.includes("-h")) + return makeSuccessfulHandle(""); yield* Deferred.succeed(checkoutStarted, undefined); return ChildProcessSpawner.makeHandle({ ...makeSuccessfulHandle(""), @@ -1744,6 +2262,53 @@ it.effect("cancels a slow submodule checkout without reporting best-effort succe ), ); +// regression: #18537 the fetch before creating a worktree outgrew the 30s default +it.effect.each(["healthy", "hung"] as const)("bounds a %s remote fetch at five minutes", (fetch) => + Effect.gen(function* () { + const started = yield* Deferred.make(); + const finished = yield* Deferred.make(); + const spawner = ChildProcessSpawner.make((command) => + Effect.succeed( + ChildProcess.isStandardCommand(command) && command.args[0] === "fetch" + ? ChildProcessSpawner.makeHandle({ + ...makeSuccessfulHandle(""), + exitCode: Deferred.succeed(started, undefined).pipe( + Effect.andThen(Deferred.await(finished)), + Effect.as(ChildProcessSpawner.ExitCode(0)), + ), + }) + : makeNonRepositoryHandle(), + ), + ); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + const fetching = yield* driver + .fetchRemote({ cwd: yield* makeTmpDir(), remoteName: "origin" }) + .pipe(Effect.result, Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(started); + yield* TestClock.adjust("31 seconds"); + if (fetch === "healthy") yield* Deferred.succeed(finished, undefined); + else { + yield* TestClock.adjust("268 seconds"); + assert.isUndefined(fetching.pollUnsafe()); + yield* TestClock.adjust("1 second"); + } + + const result = yield* Fiber.join(fetching); + assert.equal( + Result.isFailure(result) ? result.failure.detail : "fetched", + fetch === "healthy" ? "fetched" : "Git command timed out.", + ); + }).pipe( + Effect.provide( + layerServerConfig.pipe( + Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), + ), + ), + ), +); + describe("rejected git refs", () => { const remoteUrl = "https://user:token-123@git.example.invalid/org/secret-repo.git"; const failedPush = `error: failed to push some refs to '${remoteUrl}'`; @@ -1875,9 +2440,12 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { yield* git(cwd, ["init"]); yield* git(cwd, ["config", "user.email", "test@test.com"]); yield* git(cwd, ["config", "user.name", "Test"]); + // Selected paths add up to more than Linux's default 2 MiB argument limit. + const dir = "d".repeat(200); + yield* fs.makeDirectory(path.join(cwd, dir)); const files = Array.from( { length: 6_000 }, - (_, index) => `file-${String(index).padStart(5, "0")}-${"x".repeat(160)}.txt`, + (_, index) => `${dir}/file-${String(index).padStart(5, "0")}-${"x".repeat(160)}.txt`, ); yield* Effect.forEach(files, (file) => fs.writeFileString(path.join(cwd, file), "before\n"), { concurrency: 16, @@ -1903,6 +2471,10 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.isNotNull(context); assert.isAbove(context!.stagedSummary.length, 1_000_000); assert.lengthOf(context!.stagedSummary.split("\n"), files.length); + const selected = files.slice(1); + yield* driver.commit(cwd, "selected", "", { stage: { filePaths: selected } }); + assert.equal(yield* git(cwd, ["log", "-1", "--pretty=%s"]), "selected"); + assert.equal(yield* git(cwd, ["status", "--porcelain"]), `M ${files[0]}`); }), ); describe("structured errors", () => { @@ -1938,17 +2510,39 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); - // regression: #8645 name the cause of a failed git command - it.effect("names the branch conflict behind a failed worktree add", () => + it.effect("fails with a GitCommandError when spawn throws", () => Effect.gen(function* () { - const parent = yield* makeTmpDir(); - const pathService = yield* Path.Path; - const cwd = pathService.join(parent, "repo"); - const fileSystem = yield* FileSystem.FileSystem; - yield* fileSystem.makeDirectory(cwd); + const cwd = yield* makeTmpDir(); const driver = yield* GitVcsDriver.GitVcsDriver; - yield* initRepoWithCommit(cwd); - const firstWorktree = pathService.join(parent, "first"); + + // Over the OS argument limit, Node's spawn throws E2BIG synchronously. + const error = yield* driver + .execute({ + operation: "GitVcsDriver.test.spawnThrow", + cwd, + args: ["status", "x".repeat(4 * 1024 * 1024)], + }) + .pipe(Effect.flip); + + assert.deepInclude(error, { + _tag: "GitCommandError", + operation: "GitVcsDriver.test.spawnThrow", + detail: "Failed to spawn Git process.", + }); + }), + ); + + // regression: #8645 name the cause of a failed git command + it.effect("names the branch conflict behind a failed worktree add", () => + Effect.gen(function* () { + const parent = yield* makeTmpDir(); + const pathService = yield* Path.Path; + const cwd = pathService.join(parent, "repo"); + const fileSystem = yield* FileSystem.FileSystem; + yield* fileSystem.makeDirectory(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* initRepoWithCommit(cwd); + const firstWorktree = pathService.join(parent, "first"); yield* git(cwd, ["worktree", "add", firstWorktree, "-b", "shared-branch"]); const error = yield* driver @@ -2011,6 +2605,7 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const spawner = ChildProcessSpawner.make((command) => { if ( ChildProcess.isStandardCommand(command) && + !command.args.includes("-h") && (command.args.includes("worktree") || command.args[0] === "branch" || command.args[0] === "checkout") @@ -2045,6 +2640,55 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("explains a non-fast-forward push from a host with a translated Git", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const remote = yield* makeTmpDir("git-remote-"); + const other = yield* makeTmpDir("git-other-"); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["push", "--set-upstream", "origin", initialBranch]); + yield* git(other, ["clone", "--branch", initialBranch, remote, "."]); + yield* git(other, [ + "-c", + "user.email=t@t", + "-c", + "user.name=T", + "commit", + "--allow-empty", + "-m", + "remote", + ]); + yield* git(other, ["push", "origin", initialBranch]); + yield* git(cwd, ["commit", "--allow-empty", "-m", "local"]); + const locales: Array = []; + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const spawner = ChildProcessSpawner.make((command) => { + if (ChildProcess.isStandardCommand(command) && command.args.includes("push")) { + locales.push(command.options.env?.LANGUAGE); + } + return delegate.spawn(command); + }); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provide(layerServerConfig), + ); + const hostEnv = yield* HostProcess.Environment; + + const error = yield* driver + .pushCurrentBranch(cwd, null) + .pipe( + Effect.provideService(HostProcess.Environment, { ...hostEnv, LANGUAGE: "de" }), + Effect.flip, + ); + + assert.equal(error.reason, "push_rejected_non_fast_forward"); + assert.isNotEmpty(locales); + for (const locale of locales) assert.equal(locale, "C"); + }), + ); + it.effect("lets a slow remote branch checkout finish without blocking config writes", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -2093,6 +2737,31 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect( + "switches a remote ref to its tracking branch when a tag shares the branch name", + () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const remote = yield* makeTmpDir("git-remote-"); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["push", "origin", "HEAD:refs/heads/v1.2"]); + yield* git(cwd, ["fetch", "origin"]); + yield* git(cwd, ["branch", "--track", "v1.2", "origin/v1.2"]); + yield* git(cwd, ["tag", "v1.2"]); + assert.equal(yield* git(cwd, ["branch", "--show-current"]), initialBranch); + + const result = yield* (yield* GitVcsDriver.GitVcsDriver).switchRef({ + cwd, + refName: "origin/v1.2", + }); + + assert.equal(result.refName, "v1.2"); + assert.equal(yield* git(cwd, ["symbolic-ref", "HEAD"]), "refs/heads/v1.2"); + }), + ); + it.effect.each([ // Git cannot tell a missing revision from a missing file in these. { label: "an argument that may be a path", args: ["log", "-1", "missing"] }, @@ -2387,9 +3056,22 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const { initialBranch } = yield* initRepoWithCommit(cwd); const driver = yield* GitVcsDriver.GitVcsDriver; yield* git(cwd, ["checkout", "-b", "feature/commits"]); - for (let index = 0; index < 102; index++) { - yield* git(cwd, ["commit", "--allow-empty", "-m", `feature ${index}`]); - } + const parent = yield* git(cwd, ["rev-parse", "HEAD"]); + yield* gitWithStdin( + cwd, + ["fast-import", "--quiet"], + Array.from({ length: 102 }, (_, index) => { + const message = `feature ${index}`; + return [ + "commit refs/heads/feature/commits", + `committer Test ${1_700_000_000 + index} +0000`, + `data ${Buffer.byteLength(message)}`, + message, + ...(index === 0 ? [`from ${parent}`] : []), + "", + ].join("\n"); + }).join(""), + ); const result = yield* driver.getReviewBranchCommits({ cwd, baseRef: initialBranch }); assert.strictEqual(result.truncated, true); @@ -2572,7 +3254,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }); const source = dirty.sources.find((source) => source.kind === "working-tree")!; assert.deepStrictEqual(source.files, [ - { path: "untracked.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "untracked.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ]); assert.include(source.diff, "+untracked content"); }), @@ -2678,7 +3366,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.isTrue(metadata.sources[0]!.patchesDeferred); assert.isEmpty(metadata.sources[0]!.diff); assert.deepStrictEqual(metadata.sources[0]!.files, [ - { path: "README.md", previousPath: null, additions: 1, deletions: 1 }, + { + path: "README.md", + previousPath: null, + changeType: "change", + additions: 1, + deletions: 1, + }, ]); }), ); @@ -2724,6 +3418,35 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("keeps the review working beside a nested repository with no commits", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* git(cwd, ["init", "scaffold"]); + const committed = `${cwd}/committed`; + yield* (yield* FileSystem.FileSystem).makeDirectory(committed); + yield* initRepoWithCommit(committed); + yield* writeTextFile(cwd, "README.md", "# tracked change\n"); + yield* writeTextFile(cwd, "untracked.txt", "untracked contents\n"); + + const preview = yield* driver.getReviewDiffPreview({ cwd, ignoreWhitespace: false }); + const diff = preview.sources.find((source) => source.kind === "working-tree")?.diff ?? ""; + + assert.include(diff, "+# tracked change"); + assert.include(diff, "+untracked contents"); + assert.notInclude(diff, "scaffold"); + assert.include(diff, "diff --git a/committed b/committed"); + + const status = yield* driver.statusDetailsLocal(cwd, { includeBranchChanges: true }); + assert.deepStrictEqual(status.branchChanges, { + baseRef: null, + insertions: 3, + deletions: 1, + }); + }), + ); + it.effect("keeps untracked filenames with pathspec magic in the review", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -2748,7 +3471,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }); const scopedSource = scoped.sources.find((source) => source.kind === "working-tree")!; assert.deepStrictEqual(scopedSource.files, [ - { path: ":(exclude)after.ts", previousPath: null, additions: 1, deletions: 0 }, + { + path: ":(exclude)after.ts", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ]); assert.include(scopedSource.diff, "+literal pathspec contents"); assert.notInclude(scopedSource.diff, "ordinary.ts"); @@ -2771,7 +3500,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const source = preview.sources.find((candidate) => candidate.kind === "working-tree")!; assert.deepStrictEqual(source.files, [ - { path: "excluded/new.ts", previousPath: null, additions: 1, deletions: 0 }, + { + path: "excluded/new.ts", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ]); assert.include(source.diff, "+outside the cone"); }), @@ -2814,7 +3549,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const source = preview.sources.find((candidate) => candidate.kind === "working-tree")!; assert.deepStrictEqual(source.files, [ - { path: "after.ts", previousPath: "before.ts", additions: 1, deletions: 1 }, + { + path: "after.ts", + previousPath: "before.ts", + changeType: "rename-changed", + additions: 1, + deletions: 1, + }, ]); const scoped = yield* driver.getReviewDiffPreview({ cwd, @@ -2934,7 +3675,15 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const preview = yield* driver.getReviewDiffPreview({ cwd }); assert.deepStrictEqual( preview.sources.find((source) => source.kind === "working-tree")!.files, - [{ path: "untracked.txt", previousPath: null, additions: 1, deletions: 0 }], + [ + { + path: "untracked.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, + ], ); assert.strictEqual(yield* fs.readFileString(path.join(cwd, ".git/filter-runs")), ""); assert.deepStrictEqual(yield* fs.readFile(indexPath), originalIndex); @@ -2974,8 +3723,20 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const preview = yield* driver.getReviewDiffPreview({ cwd }); const dirty = preview.sources.find((source) => source.kind === "working-tree")!; assert.deepStrictEqual(dirty.files, [ - { path: "tracked.txt", previousPath: null, additions: 1, deletions: 1 }, - { path: "untracked.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "tracked.txt", + previousPath: null, + changeType: "change", + additions: 1, + deletions: 1, + }, + { + path: "untracked.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ]); assert.include(dirty.diff, "-before"); assert.include(dirty.diff, "+after!"); @@ -3039,14 +3800,44 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { } assert.deepStrictEqual(branch.files, [ - { path: "a-large.txt", previousPath: null, additions: 4000, deletions: 0 }, - { path: "untracked.txt", previousPath: null, additions: 4000, deletions: 0 }, - { path: "z-last.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "a-large.txt", + previousPath: null, + changeType: "new", + additions: 4000, + deletions: 0, + }, + { + path: "untracked.txt", + previousPath: null, + changeType: "new", + additions: 4000, + deletions: 0, + }, + { path: "z-last.txt", previousPath: null, changeType: "new", additions: 1, deletions: 0 }, ]); assert.deepStrictEqual(dirty.files, [ - { path: "a-large.txt", previousPath: null, additions: 4000, deletions: 4000 }, - { path: "untracked.txt", previousPath: null, additions: 4000, deletions: 0 }, - { path: "z-last.txt", previousPath: null, additions: 1, deletions: 1 }, + { + path: "a-large.txt", + previousPath: null, + changeType: "change", + additions: 4000, + deletions: 4000, + }, + { + path: "untracked.txt", + previousPath: null, + changeType: "new", + additions: 4000, + deletions: 0, + }, + { + path: "z-last.txt", + previousPath: null, + changeType: "change", + additions: 1, + deletions: 1, + }, ]); }), ); @@ -3057,10 +3848,12 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const { initialBranch } = yield* initRepoWithCommit(cwd); const driver = yield* GitVcsDriver.GitVcsDriver; yield* writeTextFile(cwd, "mode-only.sh", "echo unchanged\n"); - yield* git(cwd, ["add", "mode-only.sh"]); + yield* writeTextFile(cwd, "removed.txt", "removed\n"); + yield* git(cwd, ["add", "mode-only.sh", "removed.txt"]); yield* git(cwd, ["commit", "-m", "add executable candidate"]); yield* git(cwd, ["checkout", "-b", "feature/paths"]); yield* git(cwd, ["mv", "README.md", "renamed.md"]); + yield* git(cwd, ["rm", "removed.txt"]); yield* writeTextFile(cwd, "[literal].txt", "literal\n"); yield* writeTextFile(cwd, " leading.txt", "whitespace path\n"); yield* writeTextFile(cwd, "l.txt", "other\n"); @@ -3100,12 +3893,28 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.deepInclude(branch.files ?? [], { path: "renamed.md", previousPath: "README.md", + changeType: "rename-pure", + additions: 0, + deletions: 0, + }); + assert.deepInclude(branch.files ?? [], { + path: "removed.txt", + previousPath: null, + changeType: "deleted", + additions: 0, + deletions: 1, + }); + assert.deepInclude(branch.files ?? [], { + path: "mode-only.sh", + previousPath: null, + changeType: "change", additions: 0, deletions: 0, }); assert.deepInclude(branch.files ?? [], { path: "binary.dat", previousPath: null, + changeType: "new", additions: 0, deletions: 0, }); @@ -3113,6 +3922,7 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.deepInclude(branch.files ?? [], { path: "tab\tand\nnewline.txt", previousPath: null, + changeType: "new", additions: 1, deletions: 0, }); @@ -3131,8 +3941,14 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const preview = yield* driver.getReviewDiffPreview({ cwd }); const dirty = preview.sources.find((source) => source.kind === "working-tree")!; assert.deepStrictEqual(dirty.files, [ - { path: "staged.txt", previousPath: null, additions: 1, deletions: 0 }, - { path: "untracked.txt", previousPath: null, additions: 1, deletions: 0 }, + { path: "staged.txt", previousPath: null, changeType: "new", additions: 1, deletions: 0 }, + { + path: "untracked.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ]); assert.include(dirty.diff, "b/staged.txt"); assert.include(dirty.diff, "b/untracked.txt"); @@ -3316,6 +4132,77 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("loads working-tree contents as Git diffs them under the ident attribute", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + const fileSystem = yield* FileSystem.FileSystem; + const filePath = (yield* Path.Path).join(cwd, "README.md"); + const committed = "$Id$\nfirst\nsecond\n"; + yield* writeTextFile(cwd, ".gitattributes", "README.md ident\n"); + yield* writeTextFile(cwd, "README.md", committed); + yield* writeTextFile(cwd, "PLAIN.md", "plain\n"); + yield* git(cwd, ["add", "."]); + yield* git(cwd, ["commit", "-m", "add ident keyword"]); + // Checkout smudges the keyword to `$Id: $`. + yield* fileSystem.remove(filePath); + yield* git(cwd, ["checkout-index", "--force", "README.md"]); + const smudged = yield* fileSystem.readFileString(filePath); + assert.match(smudged, /^\$Id: [0-9a-f]{40} \$\n/); + yield* writeTextFile(cwd, "README.md", smudged.replace("first", "edited")); + yield* writeTextFile(cwd, "PLAIN.md", "$Id: literal $\n"); + const disk = yield* fileSystem.readFile(filePath); + + const contents = yield* driver.getReviewDiffFileContents( + makeReviewDiffFileContentsInput(cwd), + ); + const plain = yield* driver.getReviewDiffFileContents( + makeReviewDiffFileContentsInput(cwd, { oldPath: "PLAIN.md", newPath: "PLAIN.md" }), + ); + + assert.deepStrictEqual(contents, { + oldContents: committed, + newContents: "$Id$\nedited\nsecond\n", + }); + assert.strictEqual(plain.newContents, "$Id: literal $\n"); + yield* writeTextFile(cwd, "README.md", "$Id: x $Id: y $\n"); + const sharedDollar = yield* driver.getReviewDiffFileContents( + makeReviewDiffFileContentsInput(cwd), + ); + assert.strictEqual(sharedDollar.newContents, "$Id$Id: y $\n"); + yield* git(cwd, ["config", "filter.unrelated.clean", "touch filter-ran; cat"]); + yield* writeTextFile(cwd, ".gitattributes", "README.md ident text eol=lf\n"); + yield* writeTextFile( + cwd, + "README.md", + smudged.replace("first", "edited").replaceAll("\n", "\r\n"), + ); + const crlf = yield* driver.getReviewDiffFileContents(makeReviewDiffFileContentsInput(cwd)); + assert.strictEqual(crlf.newContents, "$Id$\r\nedited\r\nsecond\r\n"); + yield* fileSystem.writeFile(filePath, disk); + yield* writeTextFile(cwd, ".gitattributes", "README.md ident=set\n"); + const literal = yield* driver.getReviewDiffFileContents( + makeReviewDiffFileContentsInput(cwd), + ); + assert.strictEqual(literal.newContents, smudged.replace("first", "edited")); + yield* writeTextFile(cwd, ".gitattributes", "README.md ident filter=probe\n"); + yield* git(cwd, ["config", "filter.probe.clean", "touch filter-ran; cat"]); + const filtered = yield* driver.getReviewDiffFileContents( + makeReviewDiffFileContentsInput(cwd), + ); + assert.strictEqual(filtered.newContents, literal.newContents); + yield* writeTextFile(cwd, ".gitattributes", "README.md ident filter=unspecified\n"); + yield* git(cwd, ["config", "filter.unspecified.clean", "touch filter-ran; cat"]); + const reservedFilter = yield* driver.getReviewDiffFileContents( + makeReviewDiffFileContentsInput(cwd), + ); + assert.strictEqual(reservedFilter.newContents, literal.newContents); + assert.isFalse(yield* fileSystem.exists((yield* Path.Path).join(cwd, "filter-ran"))); + assert.deepStrictEqual(yield* fileSystem.readFile(filePath), disk); + }), + ); + it.effect("Changes combines commits, uncommitted edits, and untracked files", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -3332,7 +4219,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const clean = yield* driver.getReviewDiffPreview({ cwd, baseRef: initialBranch }); assert.deepStrictEqual(sourceFiles(clean, "working-tree"), []); assert.deepStrictEqual(sourceFiles(clean, "branch-range"), [ - { path: "README.md", previousPath: null, additions: 1, deletions: 0 }, + { + path: "README.md", + previousPath: null, + changeType: "change", + additions: 1, + deletions: 0, + }, ]); yield* writeTextFile(cwd, "README.md", "# test\ncommitted\ndirty\n"); @@ -3340,14 +4233,26 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const dirty = yield* driver.getReviewDiffPreview({ cwd, baseRef: initialBranch }); const changes = dirty.sources.find((source) => source.kind === "branch-range")!; assert.deepStrictEqual(changes.files, [ - { path: "README.md", previousPath: null, additions: 2, deletions: 0 }, - { path: "new.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "README.md", + previousPath: null, + changeType: "change", + additions: 2, + deletions: 0, + }, + { path: "new.txt", previousPath: null, changeType: "new", additions: 1, deletions: 0 }, ]); assert.strictEqual(changes.diff.match(/^diff --git a\/README\.md /gm)?.length, 1); assert.include(changes.diff, "+dirty"); assert.deepStrictEqual(sourceFiles(dirty, "working-tree"), [ - { path: "README.md", previousPath: null, additions: 1, deletions: 0 }, - { path: "new.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "README.md", + previousPath: null, + changeType: "change", + additions: 1, + deletions: 0, + }, + { path: "new.txt", previousPath: null, changeType: "new", additions: 1, deletions: 0 }, ]); }), ); @@ -3376,7 +4281,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const changes = preview.sources.find((source) => source.kind === "branch-range")!; assert.strictEqual(changes.baseRef, "origin/main"); assert.deepStrictEqual(changes.files, [ - { path: "unpushed.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "unpushed.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ]); }), ); @@ -3415,7 +4326,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const changes = preview.sources.find((source) => source.kind === "branch-range")!; assert.strictEqual(changes.baseRef, baseRef); assert.deepStrictEqual(changes.files, [ - { path: "feature.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "feature.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ]); } @@ -3449,8 +4366,20 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const changes = preview.sources.find((source) => source.kind === "branch-range")!; assert.strictEqual(changes.baseRef, "refs/remotes/upstream/main"); assert.deepStrictEqual(changes.files, [ - { path: "feature.txt", previousPath: null, additions: 1, deletions: 0 }, - { path: "upstream.txt", previousPath: null, additions: 1, deletions: 1 }, + { + path: "feature.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, + { + path: "upstream.txt", + previousPath: null, + changeType: "change", + additions: 1, + deletions: 1, + }, ]); const contents = yield* driver.getReviewDiffFileContents( makeReviewDiffFileContentsInput(cwd, { @@ -3505,7 +4434,11 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { // Base config reads start only after status completes; hold every one until the rename lands. const spawner = ChildProcessSpawner.make((command) => Effect.gen(function* () { - if (ChildProcess.isStandardCommand(command) && command.args[0] === "config") { + if ( + ChildProcess.isStandardCommand(command) && + command.args[0] === "config" && + !command.args.includes("status.showUntrackedFiles") + ) { if (!(yield* Ref.getAndSet(renameClaimed, true))) { yield* git(cwd, ["branch", "-m", "t3/temporary", "feature/named"]).pipe( Effect.provideService(GitVcsDriver.GitVcsDriver, outer), @@ -3562,7 +4495,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const changes = preview.sources.find((source) => source.kind === "branch-range")!; assert.strictEqual(changes.baseRef, baseRef); assert.deepStrictEqual(changes.files, [ - { path: "README.md", previousPath: null, additions: 1, deletions: 0 }, + { + path: "README.md", + previousPath: null, + changeType: "change", + additions: 1, + deletions: 0, + }, ]); const scoped = yield* driver.getReviewDiffPreview({ cwd, @@ -3611,7 +4550,15 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const explicit = yield* driver.getReviewDiffPreview({ cwd, baseRef: initialBranch }); assert.deepStrictEqual( explicit.sources.find((source) => source.kind === "branch-range")!.files, - [{ path: "detached.txt", previousPath: null, additions: 1, deletions: 0 }], + [ + { + path: "detached.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, + ], ); const implicit = yield* driver.getReviewDiffPreview({ cwd }); assert.deepStrictEqual( @@ -3641,7 +4588,7 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const changes = preview.sources.find((source) => source.kind === "branch-range")!; assert.isNull(changes.baseRef); assert.deepStrictEqual(changes.files, [ - { path: "new.txt", previousPath: null, additions: 2, deletions: 0 }, + { path: "new.txt", previousPath: null, changeType: "new", additions: 2, deletions: 0 }, ]); const status = yield* driver.statusDetailsLocal(cwd, { includeBranchChanges: true }); assert.deepStrictEqual(status.branchChanges, { @@ -3672,7 +4619,15 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const preview = yield* driver.getReviewDiffPreview({ cwd, baseRef: initialBranch }); assert.deepStrictEqual( preview.sources.find((source) => source.kind === "branch-range")!.files, - [{ path: "feature.txt", previousPath: null, additions: 1, deletions: 0 }], + [ + { + path: "feature.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, + ], ); } }), @@ -3704,6 +4659,25 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("reports a local upstream without a slash and its divergence", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const initialBranch = yield* git(cwd, ["symbolic-ref", "--short", "HEAD"]); + yield* git(cwd, ["branch", "feature"]); + yield* git(cwd, ["commit", "--allow-empty", "-m", "advance base"]); + yield* git(cwd, ["checkout", "feature"]); + yield* git(cwd, ["branch", `--set-upstream-to=${initialBranch}`]); + const driver = yield* GitVcsDriver.GitVcsDriver; + + const remoteStatus = yield* driver.statusDetailsRemote(cwd, { refreshUpstream: false }); + assert.equal(remoteStatus.upstreamRef, initialBranch); + assert.equal(remoteStatus.hasUpstream, true); + assert.equal(remoteStatus.aheadCount, 0); + assert.equal(remoteStatus.behindCount, 1); + }), + ); + it.effect("does not open a project folder laid out as a bare repository", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -3779,6 +4753,7 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { { path: "repeated.txt", previousPath: null, + changeType: "change", additions: Number(additions), deletions: Number(removals), }, @@ -3829,6 +4804,29 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("reads Changes totals without rewriting the real index for stat-dirty files", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* writeTextFile(cwd, "tracked.txt", "same content\n"); + yield* git(cwd, ["add", "tracked.txt"]); + yield* git(cwd, ["commit", "-m", "add tracked file"]); + yield* fileSystem.utimes(path.join(cwd, "tracked.txt"), 1_000_000_000, 1_000_000_000); + const indexPath = path.join(cwd, ".git", "index"); + const originalIndex = yield* fileSystem.readFile(indexPath); + + const status = yield* driver.statusDetailsLocal(cwd, { includeBranchChanges: true }); + + assert.strictEqual(status.branchChanges?.insertions, 0); + assert.strictEqual(status.branchChanges?.deletions, 0); + // Writing the index takes index.lock, which makes concurrent user Git writes fail. + assert.deepStrictEqual(yield* fileSystem.readFile(indexPath), originalIndex); + }), + ); + it.effect("reads Changes totals with untracked files when requested", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -3925,6 +4923,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { yield* git(cwd, ["checkout", "-b", "feature/many-untracked"]); yield* writeTextFile(cwd, ".git/info/exclude", "ignored.txt\n"); yield* writeTextFile(cwd, "bulk[1]/ignored.txt", "ignored\n"); + const clean = yield* driver.statusDetailsLocal(cwd, { includeBranchChanges: true }); + assert.deepStrictEqual(clean.branchChanges, { + baseRef: "main", + insertions: 0, + deletions: 0, + }); + assert.deepStrictEqual(untrackedScanRoots, []); yield* Effect.forEach( Array.from({ length: 1_001 }, (_, index) => `bulk[1]/${index}.txt`), (file) => writeTextFile(cwd, file, "one\ntwo\n"), @@ -3949,13 +4954,15 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { deletions: 0, }); assert.deepStrictEqual(untrackedScanRoots, [ - scenario === "normal" ? [":(top,literal)bulk[1]/"] : [], + scenario === "normal" || scenario === "no" ? [":(top,literal)bulk[1]/"] : [], ]); - if (scenario === "normal") { + if (scenario === "normal" || scenario === "no") { assert.deepStrictEqual( status.workingTree.files.map((file) => file.path), ["bulk[1]/"], ); + } else if (scenario === "all") { + assert.lengthOf(status.workingTree.files, 1_001); } } }), @@ -4050,7 +5057,52 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { insertions: 1, deletions: 1, }); - }), + const preview = yield* driver.getReviewDiffPreview({ cwd, metadataOnly: true }); + assert.deepStrictEqual( + preview.sources.find((source) => source.kind === "working-tree")?.files, + [ + { + path: "asset.data", + previousPath: null, + changeType: "deleted", + additions: 0, + deletions: 1, + }, + { + path: "ordinary.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, + ], + ); + }), + ); + + it.effect("keeps an oversized untracked file out of the review diff", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + const fileSystem = yield* FileSystem.FileSystem; + const pathService = yield* Path.Path; + yield* writeTextFile(cwd, "README.md", "# tracked change\n"); + yield* writeTextFile(cwd, "ordinary.txt", "new\n"); + yield* writeTextFile(cwd, "large.data", ""); + // Sparse: the logical size is over the budget without allocating it. + yield* fileSystem.truncate(pathService.join(cwd, "large.data"), 257 * 1024 * 1024); + + const preview = yield* driver.getReviewDiffPreview({ cwd, metadataOnly: true }); + + const source = preview.sources.find((source) => source.kind === "working-tree")!; + assert.isUndefined(source.files); + assert.isUndefined(source.patchesDeferred); + assert.isTrue(source.truncated); + assert.include(source.diff, "+# tracked change"); + assert.include(source.diff, "b/ordinary.txt"); + assert.notInclude(source.diff, "large.data"); + }), ); it.effect("reports changes to a file named HEAD", () => @@ -4188,7 +5240,7 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); - it.effect("reports remote status on unborn HEAD without failing", () => + it.effect("reports status on unborn HEAD without failing or dropping its upstream", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); const driver = yield* GitVcsDriver.GitVcsDriver; @@ -4202,6 +5254,32 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.equal(status.hasUpstream, false); assert.equal(status.aheadCount, 0); assert.equal(status.behindCount, 0); + + const seed = yield* makeTmpDir(); + const remote = yield* makeTmpDir("git-remote-"); + yield* initRepoWithCommit(seed); + yield* git(remote, ["init", "--bare"]); + yield* git(seed, ["push", remote, `HEAD:refs/heads/${initialBranch}`]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["fetch", "origin"]); + yield* git(cwd, ["config", `branch.${initialBranch}.remote`, "origin"]); + yield* git(cwd, ["config", `branch.${initialBranch}.merge`, `refs/heads/${initialBranch}`]); + assert.equal((yield* driver.statusDetailsLocal(cwd)).hasUpstream, true); + }), + ); + + it.effect("reports no branch while a clone has not checked out HEAD yet", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* driver.initRepo({ cwd }); + // `git clone` points HEAD here until checkout; status then prints `branch.head (unknown)`. + yield* (yield* FileSystem.FileSystem).writeFileString( + NodePath.join(cwd, ".git", "HEAD"), + "ref: refs/heads/.invalid\n", + ); + + assert.equal((yield* driver.statusDetailsLocal(cwd)).branch, null); }), ); @@ -4272,6 +5350,100 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("a ref snapshot detects branch identity, ref and ancestry changes", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const remote = yield* makeTmpDir("git-vcs-driver-remote-"); + const updater = yield* makeTmpDir("git-vcs-driver-updater-"); + const { initialBranch } = yield* initRepoWithCommit(cwd); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["push", "-u", "origin", initialBranch]); + yield* git(updater, ["clone", remote, "."]); + yield* git(updater, ["config", "user.email", "test@test.com"]); + yield* git(updater, ["config", "user.name", "Test"]); + const driver = yield* GitVcsDriver.GitVcsDriver; + const snapshot = driver.readRefSnapshot(cwd); + + // Newer Git can create origin/HEAD on the first fetch. + yield* git(cwd, ["fetch", "origin"]); + const initial = yield* snapshot; + assert.isNotNull(initial); + yield* git(cwd, ["fetch", "origin"]); + assert.equal(yield* snapshot, initial); + + yield* git(cwd, ["branch", "same-commit"]); + const beforeCheckout = yield* snapshot; + const refsBeforeCheckout = yield* git(cwd, ["show-ref", "--head"]); + yield* git(cwd, ["checkout", "same-commit"]); + assert.equal(yield* git(cwd, ["show-ref", "--head"]), refsBeforeCheckout); + const afterCheckout = yield* snapshot; + assert.isNotNull(afterCheckout); + assert.notEqual(afterCheckout, beforeCheckout); + yield* git(cwd, ["checkout", "--detach"]); + assert.equal(yield* git(cwd, ["show-ref", "--head"]), refsBeforeCheckout); + const detached = yield* snapshot; + assert.isNotNull(detached); + assert.notEqual(detached, beforeCheckout); + assert.notEqual(detached, afterCheckout); + assert.equal(yield* snapshot, detached); + yield* git(cwd, ["checkout", initialBranch]); + assert.equal(yield* snapshot, beforeCheckout); + yield* git(cwd, ["branch", "-d", "same-commit"]); + + yield* writeTextFile(updater, "remote.txt", "remote\n"); + yield* git(updater, ["add", "remote.txt"]); + yield* git(updater, ["commit", "-m", "remote commit"]); + yield* git(updater, ["push", "origin", initialBranch]); + assert.equal(yield* snapshot, initial); + yield* git(cwd, ["fetch", "origin"]); + const fetched = yield* snapshot; + assert.notEqual(fetched, initial); + + const shallow = yield* makeTmpDir("git-vcs-driver-shallow-"); + yield* git(shallow, ["clone", "--depth=1", NodeURL.pathToFileURL(remote).href, "."]); + const shallowRefs = yield* git(shallow, ["show-ref", "--head"]); + const shallowSnapshot = yield* driver.readRefSnapshot(shallow); + assert.isNotNull(shallowSnapshot); + const shallowWorktree = yield* makeTmpDir("git-vcs-driver-shallow-worktree-"); + yield* git(shallow, ["worktree", "add", "--detach", shallowWorktree]); + const worktreeSnapshot = yield* driver.readRefSnapshot(shallowWorktree); + assert.isNotNull(worktreeSnapshot); + yield* git(shallow, ["fetch", "--unshallow", "origin"]); + assert.equal(yield* git(shallow, ["show-ref", "--head"]), shallowRefs); + const unshallowSnapshot = yield* driver.readRefSnapshot(shallow); + const unshallowWorktreeSnapshot = yield* driver.readRefSnapshot(shallowWorktree); + assert.isNotNull(unshallowSnapshot); + assert.isNotNull(unshallowWorktreeSnapshot); + assert.notEqual(unshallowSnapshot, shallowSnapshot); + assert.notEqual(unshallowWorktreeSnapshot, worktreeSnapshot); + + yield* git(cwd, ["checkout", "--detach"]); + yield* git(cwd, ["commit", "--allow-empty", "-m", "detached"]); + assert.notEqual(yield* snapshot, fetched); + + const head = yield* git(cwd, ["rev-parse", "HEAD"]); + yield* driver.execute({ + operation: "GitVcsDriver.test.git", + cwd, + args: ["update-ref", "--stdin"], + stdin: Array.from( + { length: 16_000 }, + (_, index) => `create refs/heads/snapshot-${String(index).padStart(5, "0")} ${head}\n`, + ).join(""), + }); + yield* git(cwd, ["pack-refs", "--all"]); + const large = yield* snapshot; + assert.isNotNull(large); + assert.isAbove(Buffer.byteLength(large ?? "", "utf8"), 1_000_000); + assert.equal(yield* snapshot, large); + yield* git(cwd, ["update-ref", "refs/heads/snapshot-00000", "HEAD~1"]); + assert.notEqual(yield* snapshot, large); + + assert.isNull(yield* driver.readRefSnapshot(yield* makeTmpDir())); + }), + ); + it.effect("does not start Git auto-maintenance from background upstream fetches", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -4300,6 +5472,37 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("leaves FETCH_HEAD to the user's own pulls in background upstream fetches", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const remote = yield* makeTmpDir("git-vcs-driver-remote-"); + const updater = yield* makeTmpDir("git-vcs-driver-updater-"); + const { initialBranch } = yield* initRepoWithCommit(cwd); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["push", "-u", "origin", initialBranch]); + + yield* git(updater, ["clone", remote, "."]); + yield* git(updater, ["config", "user.email", "test@test.com"]); + yield* git(updater, ["config", "user.name", "Test"]); + yield* writeTextFile(updater, "remote.txt", "remote\n"); + yield* git(updater, ["add", "remote.txt"]); + yield* git(updater, ["commit", "-m", "remote commit"]); + yield* git(updater, ["push", "origin", initialBranch]); + const fetchHeadPath = (yield* Path.Path).join(cwd, ".git", "FETCH_HEAD"); + assert.equal(NodeFS.existsSync(fetchHeadPath), false); + + const status = yield* (yield* GitVcsDriver.GitVcsDriver).statusDetailsRemote(cwd); + + assert.equal(status.behindCount, 1); + assert.equal( + NodeFS.existsSync(fetchHeadPath), + false, + "background status fetch wrote .git/FETCH_HEAD, which an overlapping `git pull` merges from", + ); + }), + ); + it.effect("uses origin HEAD for default-branch detection with a non-origin upstream", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -4333,6 +5536,83 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("does not report a default branch the remote deleted after the clone", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const remote = yield* makeTmpDir("git-vcs-driver-origin-"); + yield* initRepoWithCommit(cwd); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["branch", "-M", "development"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["push", "origin", "development", "development:main"]); + yield* git(cwd, ["remote", "set-head", "origin", "development"]); + yield* git(remote, ["--git-dir=.", "symbolic-ref", "HEAD", "refs/heads/main"]); + yield* git(cwd, ["push", "origin", "--delete", "development"]); + yield* git(cwd, ["fetch", "--prune", "origin"]); + + const driver = yield* GitVcsDriver.GitVcsDriver; + const defaultBranch = yield* driver.resolveDefaultBranchName(cwd, "origin"); + + assert.notEqual(defaultBranch, "development"); + for (const refName of ["origin/HEAD", "HEAD~1", "main^", "@{-1}", "-", ":/fix"]) + assert.isNull(yield* driver.resolveWorktreeBaseRemote({ cwd, refName })); + assert.deepEqual( + yield* driver.resolveWorktreeBaseRemote({ cwd, refName: "feature/a\u00a0b" }), + { remoteName: "origin", branchName: "feature/a\u00a0b" }, + ); + }), + ); + + it.effect.each(["checkout", "bare-backed worktree"] as const)( + "uses the only remote's HEAD for default-branch detection without origin in a %s", + (layout) => + Effect.gen(function* () { + const driver = yield* GitVcsDriver.GitVcsDriver; + const path = yield* Path.Path; + const source = yield* makeTmpDir(); + const remote = yield* makeTmpDir("git-vcs-driver-github-"); + yield* initRepoWithCommit(source); + yield* git(remote, ["init", "--bare"]); + yield* git(source, ["branch", "-M", "trunk"]); + yield* git(source, ["remote", "add", "github", remote]); + yield* git(source, ["push", "-u", "github", "trunk"]); + let cwd = source; + if (layout === "bare-backed worktree") { + const container = yield* makeTmpDir("git-vcs-driver-bare-"); + const gitDirArgs = ["--git-dir", path.join(container, "repo.git")]; + cwd = path.join(container, "trunk"); + yield* git(container, ["init", "--bare", "repo.git"]); + yield* git(container, [...gitDirArgs, "config", "user.email", "test@test.com"]); + yield* git(container, [...gitDirArgs, "config", "user.name", "Test"]); + yield* git(container, [...gitDirArgs, "remote", "add", "github", remote]); + yield* git(container, [...gitDirArgs, "fetch", "github"]); + yield* git(container, [ + ...gitDirArgs, + "worktree", + "add", + "--track", + "-b", + "trunk", + cwd, + "github/trunk", + ]); + } + yield* git(cwd, [ + "symbolic-ref", + "refs/remotes/github/HEAD", + "refs/remotes/github/trunk", + ]); + + assert.isTrue((yield* driver.statusDetailsLocal(cwd)).isDefaultBranch); + const remoteStatus = yield* driver.statusDetailsRemote(cwd); + assert.equal(remoteStatus.defaultBranch, "trunk"); + assert.isTrue(remoteStatus.isDefaultBranch); + + yield* git(cwd, ["remote", "add", "origin", remote]); + assert.isNull((yield* driver.statusDetailsRemote(cwd)).defaultBranch); + }), + ); + it.effect("makes background upstream status fetches non-interactive", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -4418,25 +5698,21 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); - it.effect("reports staged file counts on unborn HEAD without failing", () => + it.effect("reports net file counts from the empty tree on unborn HEAD", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); const driver = yield* GitVcsDriver.GitVcsDriver; yield* driver.initRepo({ cwd }); - yield* git(cwd, ["config", "user.email", "test@test.com"]); - yield* git(cwd, ["config", "user.name", "Test"]); yield* writeTextFile(cwd, "initial.ts", "// first file\n"); yield* git(cwd, ["add", "initial.ts"]); + yield* writeTextFile(cwd, "initial.ts", "// edited after staging\n"); - const status = yield* driver.statusDetails(cwd); + const status = yield* driver.statusDetailsLocal(cwd); assert.equal(status.isRepo, true); - assert.equal(status.workingTree.files.length, 1); - const file = status.workingTree.files[0]; - if (file) { - assert.equal(file.path, "initial.ts"); - assert.equal(file.insertions, 1); - } + assert.deepStrictEqual(status.workingTree.files, [ + { path: "initial.ts", insertions: 1, deletions: 0 }, + ]); }), ); }); @@ -4610,13 +5886,14 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const { initialBranch } = yield* initRepoWithCommit(cwd); const driver = yield* GitVcsDriver.GitVcsDriver; const oid = yield* git(cwd, ["rev-parse", "HEAD"]); - for (let index = 0; index <= 100; index += 1) { - yield* git(cwd, [ - "update-ref", - `refs/heads/${index === 0 ? "topic" : `topic-${index}`}`, - oid, - ]); - } + yield* gitWithStdin( + cwd, + ["update-ref", "--stdin"], + Array.from( + { length: 101 }, + (_, index) => `create refs/heads/${index === 0 ? "topic" : `topic-${index}`} ${oid}\n`, + ).join(""), + ); const result = yield* driver .renameBranch({ cwd, oldBranch: initialBranch, newBranch: "topic" }) .pipe(Effect.result); @@ -4688,6 +5965,58 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect.each(["untracked", "other-remote"] as const)( + "refuses a remote switch that collides with a local branch (%s)", + (tracking) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const remote = yield* makeTmpDir("git-remote-"); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "upstream", remote]); + yield* git(cwd, ["push", "upstream", "HEAD:refs/heads/feature"]); + yield* git(cwd, ["fetch", "upstream"]); + yield* git(cwd, ["checkout", "--no-track", "-b", "feature"]); + yield* git(cwd, ["commit", "--allow-empty", "-m", "Local work"]); + if (tracking === "other-remote") { + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["fetch", "origin"]); + yield* git(cwd, ["branch", "--set-upstream-to=origin/feature", "feature"]); + } + const featureHead = yield* git(cwd, ["rev-parse", "feature"]); + const upstream = yield* git(cwd, [ + "for-each-ref", + "--format=%(upstream)", + "refs/heads/feature", + ]); + yield* git(cwd, ["checkout", initialBranch]); + const head = yield* git(cwd, ["rev-parse", "HEAD"]); + yield* writeTextFile(cwd, "README.md", "Uncommitted work\n"); + const driver = yield* GitVcsDriver.GitVcsDriver; + + const result = yield* driver + .switchRef({ cwd, refName: "upstream/feature" }) + .pipe(Effect.result); + assert.isTrue(Result.isFailure(result)); + assert.equal(yield* git(cwd, ["branch", "--show-current"]), initialBranch); + assert.equal(yield* git(cwd, ["rev-parse", "HEAD"]), head); + assert.equal(yield* git(cwd, ["rev-parse", "feature"]), featureHead); + assert.equal( + yield* git(cwd, ["for-each-ref", "--format=%(upstream)", "refs/heads/feature"]), + upstream, + ); + assert.equal( + yield* (yield* FileSystem.FileSystem).readFileString( + (yield* Path.Path).join(cwd, "README.md"), + ), + "Uncommitted work\n", + ); + // Selecting the existing local branch remains an explicit, attached checkout. + assert.equal((yield* driver.switchRef({ cwd, refName: "feature" })).refName, "feature"); + assert.equal(yield* git(cwd, ["symbolic-ref", "HEAD"]), "refs/heads/feature"); + }), + ); + it.effect("rejects a missing branch without restoring a matching dirty file", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -4791,6 +6120,172 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }); describe("worktree operations", () => { + it.effect.each(["current", "legacy"] as const)( + "preserves another creator's worktree on its new branch at the same path (%s)", + (version) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const path = NodePath.join(yield* makeTmpDir("git-worktrees-"), "contested"); + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const reachedAdd = yield* Deferred.make(); + let paused = false; + const spawner = ChildProcessSpawner.make((command) => + Effect.gen(function* () { + if ( + ChildProcess.isStandardCommand(command) && + command.args.includes("worktree") && + command.args.includes("add") + ) { + if (command.args.includes("-h")) { + return version === "legacy" + ? makeFailedHandle("usage: git worktree add [--lock]") + : yield* delegate.spawn(command); + } + if (!paused) { + paused = true; + yield* Deferred.succeed(reachedAdd, undefined); + return yield* Effect.never; + } + } + return yield* delegate.spawn(command); + }), + ); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provide(layerServerConfig), + ); + const creator = yield* driver + .createWorktree({ + cwd, + path, + refName: initialBranch, + newRefName: "contested", + }) + .pipe(Effect.forkChild); + yield* Deferred.await(reachedAdd); + yield* git(cwd, ["worktree", "add", "--", path, "contested"]); + yield* writeTextFile(path, "owned-by-B.txt", "keep this work\n"); + yield* Fiber.interrupt(creator); + assert.include(yield* git(cwd, ["worktree", "list", "--porcelain"]), path); + assert.equal(yield* git(path, ["branch", "--show-current"]), "contested"); + assert.equal( + yield* (yield* FileSystem.FileSystem).readFileString( + NodePath.join(path, "owned-by-B.txt"), + ), + "keep this work\n", + ); + const created = yield* driver.createWorktree({ + cwd, + path: `${path}-retry`, + refName: initialBranch, + newRefName: "retry", + }); + assert.equal(yield* git(created.worktree.path, ["branch", "--show-current"]), "retry"); + assert.notInclude(yield* git(cwd, ["worktree", "list", "--porcelain"]), "locked"); + }), + ); + + it.effect.each(["main", "linked"] as const)( + "removes its worktree and branch when cancelled before the claim (%s)", + (checkout) => + Effect.gen(function* () { + const repo = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(repo); + const cwd = + checkout === "main" + ? repo + : NodePath.join(yield* makeTmpDir("git-linked-"), "checkout"); + if (checkout === "linked") { + yield* git(repo, ["config", "worktree.useRelativePaths", "true"]); + yield* git(repo, ["worktree", "add", "--detach", cwd]); + } + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.GitVcsDriver; + const worktreePath = path.join(yield* makeTmpDir("git-worktrees-"), "cancelled ü"); + const hookPath = path.join(repo, ".git/hooks/post-checkout"); + const connected = Promise.withResolvers(); + const sockets = new Set(); + const server = yield* Effect.acquireRelease( + Effect.sync(() => + NodeNet.createServer((socket) => { + sockets.add(socket); + connected.resolve(); + }), + ), + (server) => + Effect.sync(() => { + for (const socket of sockets) socket.destroy(); + server.close(); + }), + ); + yield* Effect.promise( + () => + new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }), + ); + const address = server.address(); + assert.isTrue(address !== null && typeof address !== "string"); + if (address === null || typeof address === "string") return; + const gate = path.join(cwd, "hook-gate.cjs"); + yield* fs.writeFileString( + gate, + `const s = require('node:net').connect(${address.port}, '127.0.0.1'); s.on('connect', () => process.stderr.write('Updating files: 50% (1/1000)\\n')); s.resume();`, + ); + yield* writeTextFile( + repo, + ".git/hooks/post-checkout", + `#!/bin/sh\nexec "${process.execPath.replaceAll("\\", "/")}" "${gate.replaceAll("\\", "/")}"\n`, + ); + yield* fs.chmod(hookPath, 0o755); + const hookStarted = yield* Deferred.make(); + let claims = 0; + const input = { + cwd, + path: worktreePath, + refName: initialBranch, + newRefName: "t3/cancel", + }; + const worktrees = git(cwd, ["worktree", "list", "--porcelain"]); + + const cancelled = yield* driver + .createWorktree(input, { + progress: { + onCheckoutProgress: ({ total }) => + total === 1000 ? Deferred.succeed(hookStarted, undefined) : Effect.void, + onWorktreeClaimed: () => Effect.sync(() => void claims++), + }, + }) + .pipe(Effect.forkChild); + yield* Deferred.await(hookStarted); + yield* Effect.promise(() => connected.promise); + assert.include(yield* worktrees, worktreePath); + yield* Fiber.interrupt(cancelled); + + assert.equal(claims, 0); + assert.notInclude(yield* worktrees, worktreePath); + assert.equal(yield* git(cwd, ["branch", "--list", "t3/cancel"]), ""); + assert.isFalse(yield* fs.exists(worktreePath)); + + // A retry gets the same path and branch, and a claimed worktree outlives a later cancel. + yield* fs.remove(hookPath); + const claimed = yield* Deferred.make(); + const retry = yield* driver + .createWorktree(input, { + progress: { onWorktreeClaimed: () => Deferred.succeed(claimed, undefined) }, + }) + .pipe(Effect.forkChild); + yield* Deferred.await(claimed); + yield* Fiber.interrupt(retry); + assert.include(yield* worktrees, worktreePath); + assert.notInclude(yield* worktrees, "locked"); + assert.equal(yield* git(worktreePath, ["branch", "--show-current"]), "t3/cancel"); + }), + ); + it.effect("uses parallel checkout without skipping filters or hooks", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -4847,6 +6342,19 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { `${expected}\n`, ); } + yield* writeTextFile( + cwd, + ".git/hooks/post-checkout", + '#!/bin/sh\ngit worktree unlock "$PWD" 2>/dev/null || :\ngit worktree lock --reason user-kept "$PWD"\n', + ); + const lockedPath = path.join(yield* makeTmpDir("git-worktrees-"), "hook-locked"); + yield* driver.createWorktree({ + cwd, + path: lockedPath, + refName: initialBranch, + newRefName: "feature/hook-locked", + }); + assert.include(yield* git(cwd, ["worktree", "list", "--porcelain"]), "locked user-kept"); }), ); @@ -5157,6 +6665,44 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("gives parallel renames to one generated name distinct branches", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const path = yield* Path.Path; + const worktrees = yield* makeTmpDir("git-worktrees-"); + const driver = yield* GitVcsDriver.GitVcsDriver; + // A multi-model send launches one worktree per model with the same message, so their + // generated names often match and the renames can land together. + const temporaryBranches = ["t3/aaaa1111", "t3/bbbb2222", "t3/cccc3333"]; + yield* Effect.forEach(temporaryBranches, (branch) => + driver.createWorktree({ + cwd, + path: path.join(worktrees, branch.replace("/", "-")), + refName: initialBranch, + newRefName: branch, + }), + ); + + const renamed = yield* Effect.forEach( + temporaryBranches, + (branch) => + driver.renameBranch({ + cwd: path.join(worktrees, branch.replace("/", "-")), + oldBranch: branch, + newBranch: "feat/add-search", + }), + { concurrency: "unbounded" }, + ); + + assert.deepStrictEqual(renamed.map((result) => result.branch).toSorted(), [ + "feat/add-search", + "feat/add-search-1", + "feat/add-search-2", + ]); + }), + ); + it.effect.each(["release/v2", "refs/heads/release/v2", "deadbeef"])( "records the branch name when creating a worktree from %s", (baseRef) => @@ -5276,7 +6822,8 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const cwd = yield* makeTmpDir(); const { initialBranch } = yield* initRepoWithCommit(cwd); - yield* git(cwd, ["submodule", "add", submoduleRepo, "shared"]); + // A file URL transfers a pack like a network remote; a plain path hardlinks silently. + yield* git(cwd, ["submodule", "add", NodeURL.pathToFileURL(submoduleRepo).href, "shared"]); yield* git(cwd, ["commit", "-m", "add submodule"]); const recordedHead = yield* git(submoduleRepo, ["rev-parse", "HEAD"]); yield* writeTextFile(submoduleRepo, "SHARED.md", "# newer remote default\n"); @@ -5351,7 +6898,11 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { yield* git(pathService.join(worktreePath, "shared"), ["status", "--porcelain"]), "", ); - if (progress) assert.isNotEmpty(yield* Ref.get(lines)); + if (progress) { + assert.isTrue( + (yield* Ref.get(lines)).some((line) => line.startsWith("Receiving objects")), + ); + } }).pipe(withGitFileProtocol), ); @@ -5360,12 +6911,16 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const fileSystem = yield* FileSystem.FileSystem; const pathService = yield* Path.Path; + const submoduleRepo = yield* makeTmpDir("git-submodule-"); + yield* initRepoWithCommit(submoduleRepo); + const cwd = yield* makeTmpDir(); const { initialBranch } = yield* initRepoWithCommit(cwd); - yield* writeTextFile( - cwd, - ".gitmodules", - '[submodule "missing"]\n\tpath = missing\n\turl = /nonexistent/repo.git\n', + yield* git(cwd, ["submodule", "add", NodeURL.pathToFileURL(submoduleRepo).href, "shared"]); + yield* fileSystem.writeFileString( + pathService.join(cwd, ".gitmodules"), + (yield* fileSystem.readFileString(pathService.join(cwd, ".gitmodules"))) + + '[submodule "missing"]\n\tpath = missing\n\turl = /nonexistent/repo.git\n', ); yield* git(cwd, ["add", "."]); const gitlink = yield* git(cwd, ["rev-parse", "HEAD"]); @@ -5398,7 +6953,9 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.lengthOf(results, 1); assert.isFalse(results[0]!.ok); assert.include(results[0]!.detail, "GitVcsDriver.createWorktree.updateSubmodules"); - }), + // A submodule that did clone must not be left as a bare gitdir that reads as modified. + assert.equal(yield* git(worktreePath, ["status", "--porcelain"]), ""); + }).pipe(withGitFileProtocol), ); it.effect("creates worktrees under the configured worktrees directory", () => @@ -5424,6 +6981,22 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { ); assert.equal(yield* fileSystem.exists(created.worktree.path), true); + yield* git(cwd, ["worktree", "remove", "--force", created.worktree.path]); + yield* git(cwd, ["branch", "-D", "feature/custom-dir"]); + yield* fileSystem.makeDirectory(created.worktree.path); + const leftover = pathService.join(created.worktree.path, "notes.txt"); + yield* fileSystem.writeFileString(leftover, "my notes"); + const recreated = yield* driver.createWorktree( + { cwd, path: null, refName: initialBranch, newRefName: "feature/custom-dir" }, + { worktreesDirectory }, + ); + assert.notEqual(recreated.worktree.path, created.worktree.path); + assert.equal( + yield* git(recreated.worktree.path, ["branch", "--show-current"]), + "feature/custom-dir", + ); + assert.deepEqual(yield* fileSystem.readDirectory(created.worktree.path), ["notes.txt"]); + const error = yield* driver .createWorktree( { cwd, path: null, refName: initialBranch, newRefName: "feature/relative-dir" }, @@ -5704,6 +7277,41 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect.skipIf(!symlinksSupported)( + "removes a worktree whose .git file git-annex replaced with a symlink, keeping the symlink when refused", + () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const pathService = yield* Path.Path; + const worktreePath = pathService.join(yield* makeTmpDir("git-worktrees-"), "annexed"); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* driver.createWorktree({ + cwd, + path: worktreePath, + refName: initialBranch, + newRefName: "feature/annexed", + }); + const gitFile = pathService.join(worktreePath, ".git"); + const adminDir = (yield* git(worktreePath, ["rev-parse", "--absolute-git-dir"])).trim(); + NodeFS.rmSync(gitFile); + NodeFS.symlinkSync(adminDir, gitFile); + yield* writeTextFile(worktreePath, "notes.txt", "draft\n"); + + const refused = yield* Effect.result(driver.removeWorktree({ cwd, path: worktreePath })); + + assert.isTrue(Result.isFailure(refused)); + assert.isTrue(NodeFS.lstatSync(gitFile).isSymbolicLink()); + assert.equal(NodeFS.realpathSync(gitFile), NodeFS.realpathSync(adminDir)); + NodeFS.rmSync(pathService.join(worktreePath, "notes.txt")); + + yield* driver.removeWorktree({ cwd, path: worktreePath }); + + assert.isFalse(NodeFS.existsSync(worktreePath)); + assert.isFalse(NodeFS.existsSync(adminDir)); + }), + ); + it.effect("leaves a folder alone that only shares the worktree's short name", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -5828,6 +7436,112 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }); describe("remote operations", () => { + it.effect.each([ + { + configuration: "ordinary", + fetchRefspecs: ["+refs/heads/*:refs/remotes/origin/*"], + addsMapping: false, + }, + { + configuration: "restricted", + fetchRefspecs: [ + "+refs/heads/main:refs/remotes/origin/main", + "+refs/heads/team/*:refs/remotes/origin/team/*", + "+refs/heads/release/release-*:refs/remotes/origin/release/release-*", + ], + addsMapping: true, + }, + ])( + "sets a fetched branch's upstream with $configuration fetch configuration", + ({ fetchRefspecs, addsMapping }) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const remote = yield* makeTmpDir("git-remote-"); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["config", "--unset-all", "remote.origin.fetch"]); + for (const refspec of fetchRefspecs) { + yield* git(cwd, ["config", "--add", "remote.origin.fetch", refspec]); + } + const remoteBranch = "feature/example-change"; + const branch = "review/example-change"; + const mapping = `+refs/heads/${remoteBranch}*:refs/remotes/origin/${remoteBranch}*`; + yield* git(cwd, [ + "push", + remote, + "HEAD:refs/heads/main", + `HEAD:refs/heads/${remoteBranch}`, + ]); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* driver.fetchRemoteTrackingBranch({ cwd, remoteName: "origin", remoteBranch }); + yield* git(cwd, ["checkout", "--no-track", "-b", branch, `origin/${remoteBranch}`]); + + const input = { cwd, branch, remoteName: "origin", remoteBranch }; + yield* driver.setBranchUpstream(input); + yield* driver.setBranchUpstream(input); + + assert.equal( + yield* git(cwd, ["rev-parse", "--abbrev-ref", `${branch}@{upstream}`]), + `origin/${remoteBranch}`, + ); + assert.equal(yield* git(cwd, ["config", `branch.${branch}.remote`]), "origin"); + assert.equal( + yield* git(cwd, ["config", `branch.${branch}.merge`]), + `refs/heads/${remoteBranch}`, + ); + assert.deepEqual( + (yield* git(cwd, ["config", "--get-all", "remote.origin.fetch"])).split("\n"), + addsMapping ? [...fetchRefspecs, mapping] : fetchRefspecs, + ); + // A merged PR's head is deleted; later fetches of the remote must keep working. + yield* git(remote, ["--git-dir", remote, "branch", "-D", remoteBranch]); + yield* git(cwd, ["fetch", "origin"]); + }), + ); + + it.effect.each(["ordinary", "restricted"] as const)( + "retries upstream configuration with a config lock and %s fetch mappings", + (configuration) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + yield* git(cwd, ["remote", "add", "origin", "https://example.invalid/repo.git"]); + if (configuration === "restricted") { + yield* git(cwd, [ + "config", + "remote.origin.fetch", + `+refs/heads/${initialBranch}:refs/remotes/origin/${initialBranch}`, + ]); + } + yield* git(cwd, ["branch", "review/pr-head"]); + yield* git(cwd, ["update-ref", "refs/remotes/origin/pr-head", "HEAD"]); + const lockPath = (yield* Path.Path).join(cwd, ".git", "config.lock"); + yield* (yield* FileSystem.FileSystem).writeFileString(lockPath, ""); + const { driver, failures } = yield* makeDriverReleasingLock( + configuration === "restricted" + ? (args) => args[0] === "config" && args[1] === "--add" + : (args) => args.includes("--set-upstream-to"), + lockPath, + ); + + yield* TestClock.withLive( + driver.setBranchUpstream({ + cwd, + branch: "review/pr-head", + remoteName: "origin", + remoteBranch: "pr-head", + }), + ); + + assert.equal(yield* Ref.get(failures), 1); + assert.equal( + yield* git(cwd, ["rev-parse", "--abbrev-ref", "review/pr-head@{upstream}"]), + "origin/pr-head", + ); + }), + ); + it.effect("ensureRemote reuses a partial clone remote and avoids its name", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -5963,39 +7677,82 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); - it.effect("tracks a remote start point once another process releases the config lock", () => - Effect.gen(function* () { - const cwd = yield* makeTmpDir(); - const { initialBranch } = yield* initRepoWithCommit(cwd); - const remote = yield* makeTmpDir("git-remote-"); - yield* git(remote, ["init", "--bare"]); - yield* git(cwd, ["remote", "add", "origin", remote]); - yield* git(cwd, ["push", "origin", initialBranch]); - const fileSystem = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const lockPath = path.join(cwd, ".git", "config.lock"); - yield* fileSystem.writeFileString(lockPath, ""); - const { driver, failures } = yield* makeDriverReleasingLock( - (args) => args.some((arg) => arg.startsWith("--set-upstream-to")), - lockPath, - ); + it.effect.each(["createWorktree", "switchRef"] as const)( + "tracks a remote start point once another process releases the config lock: %s", + (operation) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const remote = yield* makeTmpDir("git-remote-"); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["push", "origin", `${initialBranch}:refs/heads/feature/tracked`]); + yield* git(cwd, ["fetch", "origin"]); + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const lockPath = path.join(cwd, ".git", "config.lock"); + yield* fileSystem.writeFileString(lockPath, ""); + const { driver, failures } = yield* makeDriverReleasingLock( + (args) => args.some((arg) => arg.startsWith("--set-upstream-to")), + lockPath, + ); - // The upstream retry waits on the real clock; the lock is already gone by then. - yield* TestClock.withLive( - driver.createWorktree({ - cwd, - path: path.join(yield* makeTmpDir("git-worktrees-"), "tracked"), - refName: `origin/${initialBranch}`, - newRefName: "feature/tracked", - }), - ); + // The upstream retry waits on the real clock; the lock is already gone by then. + if (operation === "createWorktree") { + yield* TestClock.withLive( + driver.createWorktree({ + cwd, + path: path.join(yield* makeTmpDir("git-worktrees-"), "tracked"), + refName: "origin/feature/tracked", + newRefName: "feature/tracked", + }), + ); + } else { + const switched = yield* TestClock.withLive( + driver.switchRef({ cwd, refName: "origin/feature/tracked" }), + ); + assert.equal(switched.refName, "feature/tracked"); + } - assert.equal(yield* Ref.get(failures), 1); - assert.equal( - yield* git(cwd, ["rev-parse", "--abbrev-ref", "feature/tracked@{upstream}"]), - `origin/${initialBranch}`, - ); - }), + assert.equal(yield* Ref.get(failures), 1); + assert.equal( + yield* git(cwd, ["rev-parse", "--abbrev-ref", "feature/tracked@{upstream}"]), + "origin/feature/tracked", + ); + }), + ); + + it.effect( + "leaves no untracked branch behind when the config stays locked during a switch", + () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const remote = yield* makeTmpDir("git-remote-"); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "upstream", remote]); + yield* git(cwd, ["push", "upstream", `${initialBranch}:refs/heads/feature`]); + yield* git(cwd, ["fetch", "upstream"]); + const fileSystem = yield* FileSystem.FileSystem; + const lockPath = (yield* Path.Path).join(cwd, ".git", "config.lock"); + yield* fileSystem.writeFileString(lockPath, ""); + const driver = yield* GitVcsDriver.GitVcsDriver; + + // The upstream retry waits on the real clock before giving up. + yield* TestClock.withLive(driver.switchRef({ cwd, refName: "upstream/feature" })).pipe( + Effect.flip, + ); + assert.equal(yield* git(cwd, ["branch", "--show-current"]), initialBranch); + assert.equal(yield* git(cwd, ["branch", "--list", "feature"]), ""); + + yield* fileSystem.remove(lockPath); + const switched = yield* driver.switchRef({ cwd, refName: "upstream/feature" }); + assert.equal(switched.refName, "feature"); + assert.equal( + yield* git(cwd, ["rev-parse", "--abbrev-ref", "feature@{upstream}"]), + "upstream/feature", + ); + }), ); it.effect("fetches a remote branch into a tracking branch past another process's lock", () => @@ -6234,28 +7991,42 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); - it.effect("prepares selected files without staging and commits only those files", () => - Effect.gen(function* () { - const cwd = yield* makeTmpDir(); - yield* initRepoWithCommit(cwd); - const driver = yield* GitVcsDriver.GitVcsDriver; - - yield* writeTextFile(cwd, "a.txt", "a\n"); - yield* writeTextFile(cwd, "b.txt", "b\n"); - - const context = yield* driver.prepareCommitContext(cwd, ["a.txt"]); - assert.include(context?.stagedSummary ?? "", "a.txt"); - assert.notInclude(context?.stagedSummary ?? "", "b.txt"); - assert.equal(yield* git(cwd, ["diff", "--cached", "--name-only"]), ""); + it.effect.each(["root", "aliased-nested"] as const)( + "prepares selected files without staging and commits only those files from %s", + (location) => + Effect.gen(function* () { + const sandbox = yield* makeTmpDir(); + const path = yield* Path.Path; + const fileSystem = yield* FileSystem.FileSystem; + const repository = path.join(sandbox, "repo"); + const nested = path.join(repository, "sub", "dir"); + yield* fileSystem.makeDirectory(nested, { recursive: true }); + yield* initRepoWithCommit(repository); + const cwd = location === "aliased-nested" ? path.join(sandbox, "link") : repository; + if (location === "aliased-nested") NodeFS.symlinkSync(nested, cwd, "junction"); + const driver = yield* GitVcsDriver.GitVcsDriver; + const selected = "sub/dir/a.txt"; + yield* writeTextFile(repository, selected, "a\n"); + yield* writeTextFile(repository, "b.txt", "b\n"); - const commit = yield* driver.commit(cwd, "Add a", "", { stage: { filePaths: ["a.txt"] } }); - assert.match(commit.commitSha, /^[a-f0-9]{40}$/); - assert.equal(yield* git(cwd, ["log", "-1", "--pretty=%s"]), "Add a"); + const context = yield* driver.prepareCommitContext(cwd, [selected]); + assert.include(context?.stagedSummary ?? "", "a.txt"); + assert.notInclude(context?.stagedSummary ?? "", "b.txt"); + assert.equal(yield* git(repository, ["diff", "--cached", "--name-only"]), ""); - const status = yield* git(cwd, ["status", "--porcelain"]); - assert.include(status, "?? b.txt"); - assert.notInclude(status, "a.txt"); - }), + const commit = yield* driver.commit(cwd, "Add a", "", { + stage: { filePaths: [selected] }, + }); + assert.match(commit.commitSha, /^[a-f0-9]{40}$/); + assert.equal(yield* git(repository, ["log", "-1", "--pretty=%s"]), "Add a"); + assert.equal( + yield* git(repository, ["diff-tree", "--no-commit-id", "--name-only", "-r", "HEAD"]), + selected, + ); + const status = yield* git(repository, ["status", "--porcelain"]); + assert.include(status, "?? b.txt"); + assert.notInclude(status, "a.txt"); + }), ); it.effect("stops a selected-file commit when another process holds the index lock", () => @@ -6328,30 +8099,49 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { ); it.effect.each([ - { location: "root", conflict: false }, - { location: "nested", conflict: false }, - { location: "worktree", conflict: false }, - { location: "root", conflict: true }, - { location: "nested", conflict: true }, - { location: "worktree", conflict: true }, + { location: "root", conflict: false, encoding: "utf8" }, + { location: "nested", conflict: false, encoding: "utf8" }, + { location: "worktree", conflict: false, encoding: "utf8" }, + { location: "root", conflict: true, encoding: "utf8" }, + { location: "nested", conflict: true, encoding: "utf8" }, + { location: "worktree", conflict: true, encoding: "utf8" }, + { location: "root", conflict: true, encoding: "utf16le" }, + { location: "root", conflict: true, encoding: "utf16le", large: true }, ])( - "rejects selected-file merge commits without mutation ($location, conflict: $conflict)", - ({ location, conflict }) => + "rejects selected-file merge commits without mutation ($location, conflict: $conflict, $encoding)", + ({ location, conflict, encoding, large }) => Effect.gen(function* () { let cwd = yield* makeTmpDir(); const { initialBranch } = yield* initRepoWithCommit(cwd); + const writeReadme = (text: string) => + Effect.sync(() => + NodeFS.writeFileSync( + NodePath.join(cwd, "README.md"), + encoding === "utf16le" ? Buffer.from(`\ufeff${text}`, "utf16le") : text, + ), + ); + if (encoding === "utf16le") { + yield* writeTextFile( + cwd, + ".gitattributes", + "README.md text working-tree-encoding=UTF-16\n", + ); + yield* writeReadme("base\n"); + yield* git(cwd, ["add", "."]); + yield* git(cwd, ["commit", "-m", "Declare UTF-16 encoding"]); + } const driver = yield* GitVcsDriver.GitVcsDriver; const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; yield* git(cwd, ["checkout", "-b", "feature"]); yield* writeTextFile(cwd, "feature.txt", "feature\n"); - if (conflict) yield* writeTextFile(cwd, "README.md", "feature version\n"); + if (conflict) yield* writeReadme("feature version\n"); yield* git(cwd, ["add", "."]); yield* git(cwd, ["commit", "-m", "Feature"]); const featureHead = yield* git(cwd, ["rev-parse", "HEAD"]); yield* git(cwd, ["checkout", initialBranch]); yield* writeTextFile(cwd, "main.txt", "main\n"); - if (conflict) yield* writeTextFile(cwd, "README.md", "main version\n"); + if (conflict) yield* writeReadme("main version\n"); yield* git(cwd, ["add", "."]); yield* git(cwd, ["commit", "-m", "Main"]); const mainHead = yield* git(cwd, ["rev-parse", "HEAD"]); @@ -6400,7 +8190,20 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.equal(yield* git(cwd, ["rev-parse", "HEAD"]), mainHead); assert.equal(yield* git(cwd, ["ls-files", "--unmerged"]), unmerged); - if (conflict) yield* writeTextFile(cwd, "README.md", "resolved version\n"); + const resolvedText = `Heading\n=======\n\nresolved${large ? "x".repeat(17 * 1024 * 1024) : ""}\n`; + if (conflict) { + const unresolved = yield* driver + .commit(commitCwd, "Unresolved merge", "", { stage: {} }) + .pipe(Effect.result); + assert.isTrue(Result.isFailure(unresolved), "committed an unresolved conflict"); + assert.equal(yield* git(cwd, ["rev-parse", "HEAD"]), mainHead); + assert.equal(yield* git(cwd, ["ls-files", "--unmerged"]), unmerged); + // A lone "=======" is a heading underline, not a conflict boundary. + yield* writeReadme(resolvedText); + assert.isNotNull( + yield* driver.prepareCommitContext(commitCwd, undefined, { includePatch: false }), + ); + } yield* driver.commit(commitCwd, "Finish merge", "", { stage: {} }); assert.equal( yield* git(cwd, ["show", "-s", "--format=%P", "HEAD"]), @@ -6409,11 +8212,155 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.isFalse(NodeFS.existsSync(path.join(gitDir, "MERGE_HEAD"))); assert.equal(yield* git(cwd, ["show", "HEAD:feature.txt"]), "feature"); assert.equal(yield* git(cwd, ["show", "HEAD:unrelated.txt"]), "unrelated"); - if (conflict) - assert.equal(yield* git(cwd, ["show", "HEAD:README.md"]), "resolved version"); + if (large) { + assert.equal( + yield* git(cwd, ["cat-file", "-s", "HEAD:README.md"]), + String(Buffer.byteLength(resolvedText)), + ); + } else if (conflict) + assert.equal( + yield* git(cwd, ["show", "HEAD:README.md"]), + "Heading\n=======\n\nresolved", + ); }), ); + it.effect("reports a merge resolved to HEAD's tree as committable", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + // A branch named MERGE_HEAD is not a merge in progress. + yield* git(cwd, ["branch", "MERGE_HEAD"]); + assert.isNull(yield* driver.prepareCommitContext(cwd, undefined, { includePatch: false })); + yield* git(cwd, ["branch", "-D", "MERGE_HEAD"]); + yield* git(cwd, ["checkout", "-b", "feature"]); + yield* writeTextFile(cwd, "README.md", "feature version\n"); + yield* git(cwd, ["commit", "-am", "Feature"]); + yield* git(cwd, ["checkout", initialBranch]); + yield* writeTextFile(cwd, "README.md", "main version\n"); + yield* git(cwd, ["commit", "-am", "Main"]); + const merge = yield* driver.execute({ + operation: "test.merge", + cwd, + args: ["merge", "feature"], + allowNonZeroExit: true, + }); + assert.equal(merge.exitCode, 1); + yield* writeTextFile(cwd, "README.md", "main version\n"); + + assert.isNotNull( + yield* driver.prepareCommitContext(cwd, undefined, { includePatch: false }), + ); + }), + ); + + it.effect.each([ + { operation: "cherry-pick", head: "CHERRY_PICK_HEAD", refs: "files" }, + { operation: "revert", head: "REVERT_HEAD", refs: "files" }, + // Reftable keeps CHERRY_PICK_HEAD and REVERT_HEAD as refs with no file under .git. + { operation: "cherry-pick", head: "CHERRY_PICK_HEAD", refs: "reftable" }, + { operation: "revert", head: "REVERT_HEAD", refs: "reftable" }, + ])( + "rejects selected-file commits during a $operation ($refs)", + ({ operation, head, refs }, ctx) => + Effect.gen(function* () { + if (refs === "reftable" && !gitHasReftable) + return ctx.skip("Needs Git 2.45 or later for reftable."); + const cwd = yield* makeTmpDir(); + if (refs === "reftable") yield* commitInitialRepo(cwd, ["--ref-format=reftable"]); + else yield* initRepoWithCommit(cwd); + // Reftable leaves .git/HEAD as a stub, so ask Git for the branch. + const initialBranch = yield* git(cwd, ["symbolic-ref", "--short", "HEAD"]); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* git(cwd, ["checkout", "-b", "other"]); + yield* writeTextFile(cwd, "README.md", "other version\n"); + yield* writeTextFile(cwd, "picked.txt", "picked\n"); + yield* git(cwd, ["add", "."]); + yield* git(cwd, ["commit", "-m", "Other"]); + if (operation === "cherry-pick") yield* git(cwd, ["checkout", initialBranch]); + yield* writeTextFile(cwd, "README.md", "main version\n"); + yield* git(cwd, ["commit", "-am", "Main"]); + const originalHead = yield* git(cwd, ["rev-parse", "HEAD"]); + yield* git(cwd, ["branch", head]); + const result = yield* driver.execute({ + operation: `test.${operation}`, + cwd, + args: [operation, operation === "cherry-pick" ? "other" : "HEAD~1"], + allowNonZeroExit: true, + }); + assert.equal(result.exitCode, 1); + yield* writeTextFile(cwd, "README.md", "resolved version\n"); + yield* git(cwd, ["add", "README.md"]); + const selected = operation === "cherry-pick" ? "picked.txt" : "README.md"; + + const commit = yield* driver + .commit(cwd, "Partial", "", { stage: { filePaths: [selected] } }) + .pipe(Effect.result); + + assert.isTrue(Result.isFailure(commit), `Selected-file ${operation} commit succeeded`); + assert.equal(yield* git(cwd, ["rev-parse", "HEAD"]), originalHead); + assert.equal( + yield* git(cwd, [ + "-c", + "core.warnAmbiguousRefs=false", + "rev-parse", + "--verify", + "--symbolic-full-name", + head, + ]), + head, + ); + }), + ); + + it.effect("rejects selected-file commits during a multi-commit no-commit cherry-pick", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + const path = yield* Path.Path; + yield* writeTextFile(cwd, "a.txt", "base a\n"); + yield* writeTextFile(cwd, "b.txt", "base b\n"); + yield* git(cwd, ["add", "."]); + yield* git(cwd, ["commit", "-m", "Base"]); + yield* git(cwd, ["checkout", "-b", "feature"]); + yield* writeTextFile(cwd, "a.txt", "feature a\n"); + yield* writeTextFile(cwd, "b.txt", "feature b\n"); + yield* git(cwd, ["commit", "-am", "Feature one"]); + yield* writeTextFile(cwd, "later.txt", "later\n"); + yield* git(cwd, ["add", "."]); + yield* git(cwd, ["commit", "-m", "Feature two"]); + yield* git(cwd, ["checkout", initialBranch]); + yield* writeTextFile(cwd, "a.txt", "main a\n"); + yield* writeTextFile(cwd, "b.txt", "main b\n"); + yield* git(cwd, ["commit", "-am", "Main"]); + const head = yield* git(cwd, ["rev-parse", "HEAD"]); + const started = yield* driver.execute({ + operation: "test.sequencer", + cwd, + args: ["cherry-pick", "--no-commit", "feature~1", "feature"], + allowNonZeroExit: true, + }); + assert.equal(started.exitCode, 1); + yield* writeTextFile(cwd, "a.txt", "resolved a\n"); + const gitDir = yield* git(cwd, ["rev-parse", "--absolute-git-dir"]); + assert.isFalse(NodeFS.existsSync(path.join(gitDir, "CHERRY_PICK_HEAD"))); + const names = ["index", "sequencer/todo", "sequencer/head"]; + const before = names.map((name) => NodeFS.readFileSync(path.join(gitDir, name))); + + const error = yield* driver + .commit(cwd, "Partial", "", { stage: { filePaths: ["a.txt"] } }) + .pipe(Effect.flip); + assert.include(error.detail, "Cannot commit selected files during a cherry-pick or revert"); + assert.deepEqual( + names.map((name) => NodeFS.readFileSync(path.join(gitDir, name))), + before, + ); + assert.equal(yield* git(cwd, ["rev-parse", "HEAD"]), head); + }), + ); + it.effect("treats selected file paths literally", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -6441,6 +8388,53 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("keeps other files conflicted through a selected-file commit", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + // git stash pop leaves README.md conflicted with no merge in progress. + yield* writeTextFile(cwd, "README.md", "stashed version\n"); + yield* git(cwd, ["stash"]); + yield* writeTextFile(cwd, "README.md", "committed version\n"); + yield* git(cwd, ["commit", "-am", "Change README"]); + const pop = yield* driver.execute({ + operation: "test.stashPop", + cwd, + args: ["stash", "pop"], + allowNonZeroExit: true, + }); + assert.equal(pop.exitCode, 1); + yield* writeTextFile(cwd, "other.txt", "other\n"); + const head = yield* git(cwd, ["rev-parse", "HEAD"]); + const unmerged = yield* git(cwd, ["ls-files", "--unmerged"]); + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const nested = path.join(cwd, "nested"); + yield* fs.makeDirectory(nested); + // A formatting hook stages its edit into the temporary index that --only commits from. + yield* writeTextFile( + cwd, + ".git/hooks/pre-commit", + "#!/bin/sh\necho formatted >> other.txt\ngit add other.txt\n", + ); + yield* fs.chmod(path.join(cwd, ".git/hooks/pre-commit"), 0o755); + + yield* driver.commit(cwd, "Add other", "", { stage: { filePaths: ["other.txt"] } }); + + assert.equal(yield* git(cwd, ["rev-parse", "HEAD~1"]), head); + assert.equal(yield* git(cwd, ["show", "--pretty=", "--name-only", "HEAD"]), "other.txt"); + assert.equal(yield* git(cwd, ["show", "HEAD:other.txt"]), "other\nformatted"); + assert.equal(yield* git(cwd, ["show", ":other.txt"]), "other\nformatted"); + assert.equal(yield* git(cwd, ["ls-files", "--unmerged"]), unmerged); + const all = yield* driver.prepareCommitContext(nested).pipe(Effect.flip); + assert.equal( + all.detail, + "Cannot commit with unresolved merge conflicts in README.md. Resolve them, then commit.", + ); + }), + ); + const commitWithHooks = (hooks: Record) => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -6735,7 +8729,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const preview = yield* driver.getReviewDiffPreview({ cwd: worktreePath }); const changes = preview.sources.find((source) => source.kind === "branch-range")!; assert.deepStrictEqual(changes.files, [ - { path: "feature.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "feature.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ]); assert.include(changes.diff, "+feature work"); assert.notInclude(changes.diff, "upstream-only work"); @@ -6900,6 +8900,92 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("republishes a branch whose upstream was deleted", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const remote = yield* makeTmpDir("git-remote-"); + yield* initRepoWithCommit(cwd); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* (yield* GitVcsDriver.GitVcsDriver).createRef({ + cwd, + refName: "feature/push", + }); + yield* (yield* GitVcsDriver.GitVcsDriver).switchRef({ + cwd, + refName: "feature/push", + }); + yield* writeTextFile(cwd, "feature.txt", "feature\n"); + yield* git(cwd, ["add", "-A"]); + yield* (yield* GitVcsDriver.GitVcsDriver).commit(cwd, "Add feature", ""); + + yield* (yield* GitVcsDriver.GitVcsDriver).pushCurrentBranch(cwd, null); + + yield* git(cwd, ["push", "origin", "--delete", "feature/push"]); + yield* writeTextFile(cwd, "followup.txt", "followup\n"); + yield* git(cwd, ["add", "-A"]); + yield* (yield* GitVcsDriver.GitVcsDriver).commit(cwd, "Add followup", ""); + const republished = yield* (yield* GitVcsDriver.GitVcsDriver).pushCurrentBranch(cwd, null); + assert.deepInclude(republished, { status: "pushed", branch: "feature/push" }); + assert.equal( + yield* git(remote, ["--git-dir=.", "rev-parse", "refs/heads/feature/push"]), + yield* git(cwd, ["rev-parse", "HEAD"]), + ); + + const fork = yield* makeTmpDir("git-fork-"); + yield* git(fork, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "fork", fork]); + yield* git(cwd, ["remote", "add", "feature", fork]); + yield* git(cwd, ["push", "-u", "fork", "feature/push"]); + yield* git(cwd, ["push", "fork", "--delete", "feature/push"]); + yield* writeTextFile(cwd, "fork.txt", "fork\n"); + yield* git(cwd, ["add", "-A"]); + yield* (yield* GitVcsDriver.GitVcsDriver).commit(cwd, "Add fork change", ""); + assert.deepInclude( + yield* (yield* GitVcsDriver.GitVcsDriver).resolvePushTarget(cwd, { + branch: "feature/push", + hasUpstream: false, + }), + { remoteName: "fork", branchName: "feature/push" }, + ); + const forkRepublished = yield* (yield* GitVcsDriver.GitVcsDriver).pushCurrentBranch( + cwd, + null, + ); + assert.deepInclude(forkRepublished, { upstreamBranch: "fork/feature/push" }); + assert.equal( + yield* git(fork, ["--git-dir=.", "rev-parse", "refs/heads/feature/push"]), + yield* git(cwd, ["rev-parse", "HEAD"]), + ); + + yield* git(cwd, ["config", "branch.feature/push.remote", "."]); + yield* git(cwd, ["config", "branch.feature/push.merge", "refs/heads/push"]); + yield* writeTextFile(cwd, "local.txt", "local\n"); + yield* git(cwd, ["add", "-A"]); + yield* (yield* GitVcsDriver.GitVcsDriver).commit(cwd, "Add local change", ""); + const localUpstreamGone = yield* (yield* GitVcsDriver.GitVcsDriver).pushCurrentBranch( + cwd, + null, + ); + assert.deepInclude(localUpstreamGone, { upstreamBranch: "origin/feature/push" }); + + yield* git(cwd, ["push", "fork", "HEAD:refs/heads/release"]); + yield* git(cwd, ["branch", "-u", "fork/release"]); + yield* git(cwd, ["push", "fork", "--delete", "release"]); + yield* writeTextFile(cwd, "release.txt", "release\n"); + yield* git(cwd, ["add", "-A"]); + yield* (yield* GitVcsDriver.GitVcsDriver).commit(cwd, "Add release change", ""); + yield* (yield* GitVcsDriver.GitVcsDriver).pushCurrentBranch(cwd, null); + assert.equal( + yield* (yield* GitVcsDriver.GitVcsDriver).readConfigValue( + cwd, + "branch.feature/push.t3-merge-base", + ), + "release", + ); + }), + ); + it.effect("rebases a pushed branch's pulls when branch.autoSetupRebase asks for it", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -7003,6 +9089,16 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { yield* git(originRemote, ["--git-dir=.", "rev-parse", "refs/heads/feature/push"]), publishedHead, ); + + const pushOnlyRemote = yield* makeTmpDir("git-push-only-remote-"); + yield* git(pushOnlyRemote, ["init", "--bare"]); + yield* git(cwd, ["remote", "set-url", "--push", "origin", pushOnlyRemote]); + const pushedToPushUrl = yield* driver.pushCurrentBranch(cwd, null); + assert.deepInclude(pushedToPushUrl, { status: "pushed", branch }); + assert.equal( + yield* git(pushOnlyRemote, ["--git-dir=.", "rev-parse", "refs/heads/feature/push"]), + head, + ); }), ); @@ -7046,14 +9142,15 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { ); it.effect.each([ - { baseBranch: "dev", local: false }, - { baseBranch: "dev", local: true }, - { baseBranch: "feature/base", local: true }, - { baseBranch: "upstream", local: true }, - { baseBranch: "upstream/feature/x", local: true }, + { baseBranch: "dev", local: false, branch: "feature/x", deleted: false }, + { baseBranch: "dev", local: true, branch: "feature/x", deleted: false }, + { baseBranch: "feature/base", local: true, branch: "feature/x", deleted: false }, + { baseBranch: "upstream", local: true, branch: "feature/x", deleted: false }, + { baseBranch: "upstream/feature/x", local: true, branch: "feature/x", deleted: false }, + { baseBranch: "dev", local: false, branch: "origin/dev", deleted: true }, ])( - "publishes a branch tracking $baseBranch (local=$local) under its own name, not the base", - ({ baseBranch, local }) => + "publishes $branch tracking $baseBranch (local=$local, deleted=$deleted) without updating the base", + ({ baseBranch, local, branch, deleted }) => Effect.gen(function* () { const cwd = yield* makeTmpDir(); const remote = yield* makeTmpDir("git-remote-"); @@ -7078,55 +9175,78 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { "checkout", "--track", "-b", - "feature/x", + branch, local ? `refs/heads/${baseBranch}` : `refs/remotes/upstream/${baseBranch}`, ]); assert.equal( - yield* driver.readConfigValue(cwd, "branch.feature/x.remote"), + yield* driver.readConfigValue(cwd, `branch.${branch}.remote`), local ? "." : "upstream", ); yield* writeTextFile(cwd, "feature.txt", "feature\n"); yield* git(cwd, ["add", "-A"]); yield* driver.commit(cwd, "Add feature", ""); + if (deleted) { + yield* git(cwd, ["push", "upstream", "--delete", baseBranch]); + } + const publishBranch = branch === "origin/dev" ? "dev" : branch; + assert.deepInclude( + yield* driver.resolvePushTarget(cwd, { branch, hasUpstream: !deleted }), + { remoteName: "origin", branchName: publishBranch }, + ); const pushed = yield* driver.pushCurrentBranch(cwd, null); assert.deepInclude(pushed, { status: "pushed", - branch: "feature/x", - upstreamBranch: "origin/feature/x", + branch, + upstreamBranch: `origin/${publishBranch}`, setUpstream: true, }); assert.equal( - yield* git(remote, ["--git-dir=.", "log", "-1", "--pretty=%s", "feature/x"]), + yield* git(remote, ["--git-dir=.", "log", "-1", "--pretty=%s", publishBranch]), "Add feature", ); + if (publishBranch !== baseBranch) { + assert.equal( + yield* git(remote, ["--git-dir=.", "rev-parse", `refs/heads/${baseBranch}`]), + devSha, + ); + } assert.equal( - yield* git(remote, ["--git-dir=.", "rev-parse", `refs/heads/${baseBranch}`]), - devSha, - ); - assert.equal( - yield* git(upstream, ["--git-dir=.", "rev-parse", `refs/heads/${baseBranch}`]), - upstreamDevSha, + yield* git(upstream, [ + "--git-dir=.", + "for-each-ref", + "--format=%(objectname)", + `refs/heads/${baseBranch}`, + ]), + deleted ? "" : upstreamDevSha, ); assert.equal( - yield* git(cwd, ["rev-parse", "--abbrev-ref", "@{upstream}"]), - "origin/feature/x", + yield* git(cwd, ["rev-parse", "--symbolic-full-name", "@{upstream}"]), + `refs/remotes/origin/${publishBranch}`, ); assert.equal( - yield* driver.readConfigValue(cwd, "branch.feature/x.t3-merge-base"), + yield* driver.readConfigValue(cwd, `branch.${branch}.t3-merge-base`), baseBranch, ); assert.equal( - yield* driver.readConfigValue(cwd, "branch.feature/x.t3-merge-base-ref"), + yield* driver.readConfigValue(cwd, `branch.${branch}.t3-merge-base-ref`), local ? `refs/heads/${baseBranch}` : `refs/remotes/upstream/${baseBranch}`, ); assert.equal(yield* git(cwd, ["rev-parse", `refs/heads/${baseBranch}`]), upstreamDevSha); + if (deleted) return; + const preview = yield* driver.getReviewDiffPreview({ cwd }); const changes = preview.sources.find((source) => source.kind === "branch-range")!; assert.deepStrictEqual(changes.files, [ - { path: "feature.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "feature.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ]); assert.equal( changes.baseRef, @@ -7271,6 +9391,32 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { yield* git(remote, ["--git-dir=.", "log", "-1", "--pretty=%s", "effect-atom"]), "Add alias update", ); + + yield* git(cwd, ["push", "my-org/upstream", "--delete", "effect-atom"]); + assert.deepInclude( + yield* driver.resolvePushTarget(cwd, { + branch: "upstream/effect-atom", + hasUpstream: false, + }), + { remoteName: "my-org/upstream", branchName: "effect-atom" }, + ); + assert.deepInclude(yield* driver.pushCurrentBranch(cwd, null), { + upstreamBranch: "my-org/upstream/effect-atom", + setUpstream: true, + }); + assert.equal( + yield* git(remote, ["--git-dir=.", "rev-parse", "refs/heads/effect-atom"]), + yield* git(cwd, ["rev-parse", "HEAD"]), + ); + assert.equal( + yield* git(remote, [ + "--git-dir=.", + "for-each-ref", + "--format=%(refname)", + "refs/heads/upstream/", + ]), + "", + ); }), ); @@ -7309,6 +9455,196 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.notEqual(originMain.exitCode, 0); }), ); + + it.effect.each(["branch.topic.pushRemote", "remote.pushDefault"] as const)( + "pushes a branch tracking its upstream head to the %s remote", + (configKey) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const upstreamRemote = yield* makeTmpDir("git-upstream-remote-"); + const forkRemote = yield* makeTmpDir("git-fork-remote-"); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* git(upstreamRemote, ["init", "--bare"]); + yield* git(forkRemote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "upstream", upstreamRemote]); + yield* git(cwd, ["remote", "add", "fork", forkRemote]); + yield* git(cwd, ["checkout", "-b", "topic"]); + yield* git(cwd, ["push", "upstream", "topic"]); + yield* git(cwd, ["push", "fork", "topic"]); + yield* git(cwd, ["branch", "--set-upstream-to=upstream/topic"]); + yield* git(cwd, ["config", configKey, "fork"]); + const upstreamTopicSha = yield* git(cwd, ["rev-parse", "HEAD"]); + yield* writeTextFile(cwd, "fork.txt", "fork\n"); + yield* git(cwd, ["add", "-A"]); + yield* driver.commit(cwd, "Add fork update", ""); + + const pushed = yield* driver.pushCurrentBranch(cwd, null); + + assert.deepInclude(pushed, { + status: "pushed", + branch: "topic", + upstreamBranch: "fork/topic", + pushRemote: "fork", + setUpstream: false, + }); + assert.equal( + yield* git(forkRemote, ["--git-dir=.", "log", "-1", "--pretty=%s", "topic"]), + "Add fork update", + ); + assert.equal( + yield* git(upstreamRemote, ["--git-dir=.", "rev-parse", "topic"]), + upstreamTopicSha, + ); + assert.equal( + yield* git(cwd, ["rev-parse", "--abbrev-ref", "@{upstream}"]), + "upstream/topic", + ); + }), + ); + + it.effect.each(["deleted", "rewound"] as const)( + "pushes an up-to-date tracked branch to the push remote when its branch there is %s", + (forkState) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const upstreamRemote = yield* makeTmpDir("git-upstream-remote-"); + const forkRemote = yield* makeTmpDir("git-fork-remote-"); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* git(upstreamRemote, ["init", "--bare"]); + yield* git(forkRemote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "upstream", upstreamRemote]); + yield* git(cwd, ["remote", "add", "fork", forkRemote]); + yield* git(cwd, ["checkout", "-b", "topic"]); + yield* writeTextFile(cwd, "upstream.txt", "upstream\n"); + yield* git(cwd, ["add", "-A"]); + yield* driver.commit(cwd, "Add upstream update", ""); + yield* git(cwd, ["push", "-u", "upstream", "topic"]); + yield* git(cwd, ["config", "remote.pushDefault", "fork"]); + // Another checkout changes the fork after this one last fetched it, + // leaving a local fork/topic that still contains HEAD. + yield* git(cwd, ["push", "fork", "topic"]); + yield* git( + forkRemote, + forkState === "deleted" + ? ["--git-dir=.", "update-ref", "-d", "refs/heads/topic"] + : ["--git-dir=.", "update-ref", "refs/heads/topic", "topic~1"], + ); + + const pushed = yield* driver.pushCurrentBranch(cwd, null); + + assert.deepInclude(pushed, { + status: "pushed", + upstreamBranch: "fork/topic", + pushRemote: "fork", + }); + assert.equal( + yield* git(forkRemote, ["--git-dir=.", "log", "-1", "--pretty=%s", "topic"]), + "Add upstream update", + ); + assert.equal( + yield* git(cwd, ["rev-parse", "--abbrev-ref", "@{upstream}"]), + "upstream/topic", + ); + }), + ); + + it.effect.each([ + { pushUrls: [], status: "skipped_up_to_date" }, + { pushUrls: ["mirror"], status: "pushed" }, + { pushUrls: ["fork", "mirror"], status: "pushed" }, + ] as const)( + "skips an up-to-date tracked branch only when every push URL has it (pushurl $pushUrls)", + ({ pushUrls, status }) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const upstreamRemote = yield* makeTmpDir("git-upstream-remote-"); + const remotes = { + fork: yield* makeTmpDir("git-fork-remote-"), + mirror: yield* makeTmpDir("git-mirror-remote-"), + }; + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* git(upstreamRemote, ["init", "--bare"]); + yield* git(remotes.fork, ["init", "--bare"]); + yield* git(remotes.mirror, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "upstream", upstreamRemote]); + yield* git(cwd, ["remote", "add", "fork", remotes.fork]); + yield* git(cwd, ["checkout", "-b", "topic"]); + yield* git(cwd, ["push", "-u", "upstream", "topic"]); + yield* git(cwd, ["push", "fork", "topic"]); + yield* git(cwd, ["config", "remote.pushDefault", "fork"]); + for (const name of pushUrls) { + yield* git(cwd, ["remote", "set-url", "--add", "--push", "fork", remotes[name]]); + } + + const pushed = yield* driver.pushCurrentBranch(cwd, null); + + assert.deepInclude(pushed, { status, upstreamBranch: "fork/topic", pushRemote: "fork" }); + const head = yield* git(cwd, ["rev-parse", "HEAD"]); + for (const name of pushUrls) { + assert.equal( + yield* git(remotes[name], ["--git-dir=.", "rev-parse", "refs/heads/topic"]), + head, + ); + } + assert.equal( + yield* git(cwd, ["rev-parse", "--abbrev-ref", "@{upstream}"]), + "upstream/topic", + ); + }), + ); + + it.effect("pushes an up-to-date branch whose tracked remote has a separate push URL", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const fetchRemote = yield* makeTmpDir("git-fetch-remote-"); + const pushRemote = yield* makeTmpDir("git-push-remote-"); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* git(fetchRemote, ["init", "--bare"]); + yield* git(pushRemote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", fetchRemote]); + yield* git(cwd, ["checkout", "-b", "topic"]); + yield* git(cwd, ["push", "-u", "origin", "topic"]); + yield* git(cwd, ["remote", "set-url", "--push", "origin", pushRemote]); + + const pushed = yield* driver.pushCurrentBranch(cwd, null); + + assert.deepInclude(pushed, { status: "pushed", upstreamBranch: "origin/topic" }); + assert.equal( + yield* git(pushRemote, ["--git-dir=.", "rev-parse", "refs/heads/topic"]), + yield* git(cwd, ["rev-parse", "HEAD"]), + ); + }), + ); + + it.effect("skips an up-to-date branch whose differently named upstream is its base", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const upstreamRemote = yield* makeTmpDir("git-upstream-remote-"); + const forkRemote = yield* makeTmpDir("git-fork-remote-"); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* git(upstreamRemote, ["init", "--bare"]); + yield* git(forkRemote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "upstream", upstreamRemote]); + yield* git(cwd, ["remote", "add", "fork", forkRemote]); + yield* git(cwd, ["push", "upstream", "HEAD:refs/heads/release"]); + yield* git(cwd, ["fetch", "upstream"]); + yield* git(cwd, ["checkout", "-b", "topic", "--track", "upstream/release"]); + yield* git(cwd, ["config", "remote.pushDefault", "fork"]); + + const pushed = yield* driver.pushCurrentBranch(cwd, null); + + assert.deepInclude(pushed, { + status: "skipped_up_to_date", + upstreamBranch: "upstream/release", + }); + assert.equal(yield* git(forkRemote, ["--git-dir=.", "branch", "--list", "topic"]), ""); + }), + ); }); }); diff --git a/apps/server/src/vcs/GitVcsDriverCore.ts b/apps/server/src/vcs/GitVcsDriverCore.ts index e04bd48a8c84..da1794188b0b 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.ts @@ -1,7 +1,10 @@ import * as Cache from "effect/Cache"; +import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as Data from "effect/Data"; +import * as Deferred from "effect/Deferred"; import * as Digest from "effect/crypto/Digest"; +import * as SecureRandom from "effect/crypto/SecureRandom"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; @@ -26,6 +29,7 @@ import { type GitCommandFailureReason, T3_PROJECT_FILE_NAME, type ReviewBranchCommitsInput, + type ReviewDiffChangeType, type ReviewDiffFileContentsInput, type ReviewDiffPreviewInput, type ReviewDiffFileStat, @@ -42,7 +46,12 @@ import { resolveProjectFileBackedSetting } from "@t3tools/shared/projectSettings import { gitCommandDuration, gitCommandsTotal, withMetrics } from "../observability/Metrics.ts"; import * as GitVcsDriver from "./GitVcsDriver.ts"; import { removeEmptyDirectory, type RemoveEmptyDirectoryError } from "./removeEmptyDirectory.ts"; -import { explicitBareRepositoryEnv, gitConfigEnv } from "./VcsProcess.ts"; +import { VcsAutomaticFetchInterval } from "./VcsAutomaticFetchInterval.ts"; +import { + explicitBareRepositoryEnv, + gitConfigEnv, + unsetInheritedRepositoryEnv, +} from "./VcsProcess.ts"; import { makeWorktreeCheckoutCache } from "./worktreeCheckoutCache.ts"; import { removeLinkedWorktree } from "./worktreeRemoval.ts"; import { resolveWorktreesDirectory } from "../worktreesDirectory.ts"; @@ -63,11 +72,19 @@ const branchBaseConfigKey = (branch: string) => `branch.${branch}.t3-merge-base` * CLI's key, which branches cut before then still carry instead. */ const legacyBranchBaseConfigKey = (branch: string) => `branch.${branch}.gh-merge-base`; +// Differently named upstreams are bases, not publish targets; pushing HEAD there would write features to shared branches. +const isAliasOfUpstreamHead = (branch: string, upstreamBranch: string, upstreamRef: string) => + branch === upstreamBranch || + (branch.endsWith(`/${upstreamBranch}`) && upstreamRef.endsWith(`/${branch}`)); const gitProcesses = Semaphore.makeUnsafe(8); const backgroundGitProcesses = Semaphore.makeUnsafe(2); +// The fetch before `git worktree add` downloads everything the remote gained since +// the last fetch; on busy repositories that alone outlasts the 30s default. +const FETCH_REMOTE_TIMEOUT_MS = Duration.toMillis(Duration.minutes(5)); const WORKTREE_INDEX_REFRESH_TIMEOUT_MS = Duration.toMillis(Duration.minutes(5)); const WORKTREE_REMOVE_TIMEOUT_MS = Duration.toMillis(Duration.minutes(5)); const DEFAULT_MAX_OUTPUT_BYTES = 1_000_000; +const REF_SNAPSHOT_MAX_OUTPUT_BYTES = 16 * 1024 * 1024; const OUTPUT_TRUNCATED_MARKER = "\n\n[truncated]"; const PREPARED_COMMIT_PATCH_MAX_OUTPUT_BYTES = 49_000; const RANGE_COMMIT_SUMMARY_MAX_OUTPUT_BYTES = 19_000; @@ -77,13 +94,15 @@ const REVIEW_DIFF_PATCH_MAX_OUTPUT_BYTES = 120_000; const REVIEW_METADATA_MAX_OUTPUT_BYTES = 16 * 1024 * 1024; // Temporary-index git add scales poorly (50k files took ~30 s); report incomplete totals above this cap. const REVIEW_UNTRACKED_MAX_FILES = 5_000; -// git diff --numstat reads every untracked byte (~5 ms/MB), so status totals skip larger untracked input. -const UNTRACKED_TOTALS_MAX_BYTES = 256 * 1024 * 1024; +// git diff reads every untracked byte (~5 ms/MB), so larger untracked input stays out of the review index. +const REVIEW_UNTRACKED_MAX_BYTES = 256 * 1024 * 1024; // The listing walks the whole untracked tree before the cap applies, so a folder whose listing // failed (too many files, output limit, timeout) is not listed again on every status poll. const UNTRACKED_TOTALS_RETRY_MIN_MS = 60_000; const UNTRACKED_TOTALS_RETRY_MAX_MS = 30 * 60_000; const REVIEW_DIFF_FILE_MAX_OUTPUT_BYTES = 1024 * 1024; +// Git's ident clean step (convert.c ident_to_git): `$Id:` up to the next `$` on the same line. +const GIT_IDENT_KEYWORD = /\$Id:[^$\n]*\$/g; const REVIEW_BRANCH_COMMIT_LIMIT = 100; const REVIEW_BRANCH_SUBJECT_LIMIT = 300; const REVIEW_BRANCH_COMMIT_MAX_OUTPUT_BYTES = 128 * 1024; @@ -187,6 +206,7 @@ interface GitRepositoryPaths { } interface GitRefsSnapshot { + readonly defaultBranch: string | null; readonly localBranches: ReadonlyArray; readonly remoteBranches: ReadonlyArray; readonly hasPrimaryRemote: boolean; @@ -214,18 +234,41 @@ function parseBranchAb(value: string): { ahead: number; behind: number } { }; } +function rawChangeType(status: string): ReviewDiffChangeType { + switch (status[0]) { + case "A": + return "new"; + case "D": + return "deleted"; + case "R": + return status === "R100" ? "rename-pure" : "rename-changed"; + default: + return "change"; + } +} + +// Parses `--numstat -z`, optionally preceded by `--raw` records that carry each file's change type. function parseReviewNumstat(stdout: string): ReviewDiffFileStat[] { const fields = stdout.split("\0"); + const changeTypes = new Map(); const files: ReviewDiffFileStat[] = []; for (let index = 0; index < fields.length; index++) { const field = fields[index]!; + if (field.startsWith(":")) { + const status = field.slice(field.lastIndexOf(" ") + 1); + if (/^[RC]/.test(status)) index++; + changeTypes.set(fields[++index]!, rawChangeType(status)); + continue; + } const match = /^(\d+|-)\t(\d+|-)\t([\s\S]*)$/.exec(field); if (!match) continue; const previousPath = match[3] === "" ? fields[++index]! : null; const path = previousPath !== null ? fields[++index]! : match[3]!; + const changeType = changeTypes.get(path); files.push({ path, previousPath, + ...(changeType === undefined ? {} : { changeType }), additions: match[1] === "-" ? 0 : Number(match[1]), deletions: match[2] === "-" ? 0 : Number(match[2]), }); @@ -326,6 +369,14 @@ function splitNullSeparatedPaths(input: string, truncated: boolean): string[] { return parts.filter((value) => value.length > 0); } +// Parses "git version 2.24.3 (Apple Git-128)" or "git version 2.45.1.windows.1". +const gitSupportsPathspecFromFile = (versionOutput: string): boolean => { + const match = /^git version (\d+)\.(\d+)/.exec(versionOutput); + if (match === null) return false; + const major = Number(match[1]); + return major > 2 || (major === 2 && Number(match[2]) >= 25); +}; + export function splitNullSeparatedGitStdoutPaths( result: Pick, ): string[] { @@ -547,7 +598,7 @@ function parseDefaultBranchFromRemoteHeadRef(value: string, remoteName: string): return null; } const refName = trimmed.slice(prefix.length).trim(); - return refName.length > 0 ? refName : null; + return refName.length > 0 && refName !== "HEAD" ? refName : null; } function isMissingGitCwdError(error: GitCommandError): boolean { @@ -581,6 +632,10 @@ export function isNonRepositoryGitStderr(stderr: string): boolean { const normalized = stderr.toLowerCase(); return NON_REPOSITORY_STDERR_PHRASES.some((phrase) => normalized.includes(phrase)); } +const UNTRACKED_FILES_SHOWN_CONFIG_VALUES = new Set(["normal", "all", "true", "yes", "on", "1"]); +function untrackedFilesHiddenByConfig(value: string | null): boolean { + return value !== null && !UNTRACKED_FILES_SHOWN_CONFIG_VALUES.has(value.toLowerCase()); +} function isUnbornHeadStderr(stderr: string): boolean { const normalized = stderr.toLowerCase(); return ( @@ -965,9 +1020,22 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const { worktreesDir, worktreeCheckoutCacheDir } = yield* ServerConfig.ServerConfig; const isServerInternalPath = yield* makeIsServerInternalPath; const digest = yield* Digest.Digest; + const random = yield* SecureRandom.SecureRandom; const hostPlatform = yield* HostProcess.Platform; const serviceScope = yield* Effect.scope; - const statusFetches = new Map>(); + const automaticFetchInterval = yield* VcsAutomaticFetchInterval; + type StatusFetchState = { + phase: "queued" | "running" | "claimed"; + claimed: Deferred.Deferred; + finished: Deferred.Deferred; + }; + const statusFetches = new Map< + string, + { + state: StatusFetchState; + fiber: Fiber.Fiber; + } + >(); const statusFetchProcesses = yield* Semaphore.make(4); const executeRaw: GitVcsDriver.GitVcsDriver["Service"]["execute"] = Effect.fnUntraced( @@ -993,8 +1061,10 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* }), ), ); + const hostEnv = yield* HostProcess.Environment; const baseEnv = { - ...(yield* HostProcess.Environment), + ...hostEnv, + ...unsetInheritedRepositoryEnv(hostEnv, hostPlatform), // Concurrent status polls must not take index.lock to save refreshed indexes. GIT_OPTIONAL_LOCKS: "0", ...input.env, @@ -1003,6 +1073,12 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const env = { ...baseEnv, ...explicitBareRepositoryEnv(baseEnv, hostPlatform) }; const spawnEnv = { ...env, ...windowsLongPathConfigEnv(hostPlatform, env) }; const resolved = yield* resolveSpawnCommand("git", [], { env: spawnEnv }); + const spawnError = (cause: unknown) => + new GitCommandError({ + ...gitCommandContext(commandInput), + detail: "Failed to spawn Git process.", + cause, + }); // cmd wrappers truncate multiline commits; leave git.cmd handling to Node lookup. const executable = resolved.shell ? "git" : resolved.command; const child = yield* commandSpawner @@ -1018,14 +1094,9 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* }), ) .pipe( - Effect.mapError( - (cause) => - new GitCommandError({ - ...gitCommandContext(commandInput), - detail: "Failed to spawn Git process.", - cause, - }), - ), + Effect.mapError(spawnError), + // Synchronous spawn throws (such as E2BIG) arrive as defects. + Effect.catchDefect((cause) => Effect.fail(spawnError(cause))), ); const [stdout, stderr, exitCode] = yield* Effect.all( @@ -1380,6 +1451,8 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* "--quiet", "--no-tags", "--no-auto-gc", + // FETCH_HEAD is written without a lock; overlapping a user's `git pull` makes that pull fail. + "--no-write-fetch-head", "--", remoteName, ]; @@ -1389,11 +1462,14 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const fiber = yield* Effect.uninterruptible( Effect.gen(function* () { const pending = statusFetches.get(key); - if (pending) return pending; + if (pending) return pending.fiber; + const claimed = yield* Deferred.make(); + const finished = yield* Deferred.make(); + const state: StatusFetchState = { phase: "queued", claimed, finished }; // Frequent poll deadlines and disconnected clients must not interrupt a // healthy writer. A separate safety limit prevents a hung transport from // occupying its worker slot indefinitely. - const started = yield* executeGit("GitVcsDriver.fetchRemoteForStatus", fetchCwd, args, { + const fetch = executeGit("GitVcsDriver.fetchRemoteForStatus", fetchCwd, args, { env: STATUS_UPSTREAM_REFRESH_ENV, fallbackErrorDetail: "Background Git fetch exited with a non-zero status.", timeoutMs: Duration.toMillis(STATUS_UPSTREAM_FETCH_TIMEOUT), @@ -1406,18 +1482,39 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ), Effect.tapError(() => recordStatusRemoteRefreshFailure(key)), Effect.tapCause((cause) => Effect.logWarning("Background Git fetch failed", cause)), - statusFetchProcesses.withPermits(1), + ); + const started = yield* Effect.raceFirst( + Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + if (state.phase !== "queued") return false; + state.phase = "running"; + yield* restore(fetch); + return true; + }), + ).pipe(statusFetchProcesses.withPermits(1)), + Deferred.await(claimed).pipe(Effect.as(false)), + ).pipe( + Effect.flatMap((completed) => + completed + ? Effect.void + : Deferred.await(finished).pipe( + Effect.flatMap((succeeded) => + succeeded ? Effect.void : fetch.pipe(statusFetchProcesses.withPermits(1)), + ), + ), + ), Effect.onExit(() => Effect.sync(() => { - statusFetches.delete(key); + if (statusFetches.get(key)?.state === state) statusFetches.delete(key); }), ), Effect.forkIn(serviceScope, { startImmediately: false, uninterruptible: false }), ); - statusFetches.set(key, started); + statusFetches.set(key, { state, fiber: started }); return started; }), ); + yield* Effect.annotateCurrentSpan("statusFetch.joining", true); const result = yield* Fiber.join(fiber).pipe( Effect.timeoutOption(STATUS_UPSTREAM_REFRESH_TIMEOUT), ); @@ -1486,23 +1583,23 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* : null; }); + const readCurrentBranch = (operation: string, cwd: string) => + executeGit(operation, cwd, ["symbolic-ref", "--quiet", "HEAD"], { + timeoutMs: 5_000, + allowNonZeroExit: true, + }).pipe( + Effect.map((result) => (result.exitCode === 0 ? branchNameFromHeadRef(result.stdout) : null)), + ); + const resolveRepositoryPathsUncached = Effect.fn("resolveRepositoryPathsUncached")(function* ( cwd: string, ) { const gitCommonDir = yield* readGitCommonDir(cwd); if (gitCommonDir === null) return null; - const [worktreeRoot, currentBranchResult] = yield* Effect.all( + const [worktreeRoot, currentBranch] = yield* Effect.all( [ readWorktreeRoot(cwd), - executeGit( - "GitVcsDriver.resolveRepositoryPaths.currentBranch", - cwd, - ["symbolic-ref", "--quiet", "HEAD"], - { - timeoutMs: 5_000, - allowNonZeroExit: true, - }, - ), + readCurrentBranch("GitVcsDriver.resolveRepositoryPaths.currentBranch", cwd), ], { concurrency: 2 }, ); @@ -1514,8 +1611,6 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* Effect.map((physicalCwd) => path.relative(worktreeRoot, physicalCwd) === ""), Effect.orElseSucceed(() => undefined), ); - const currentBranch = - currentBranchResult.exitCode === 0 ? branchNameFromHeadRef(currentBranchResult.stdout) : null; return { gitCommonDir, @@ -1571,17 +1666,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const path = yield* Path.Path; const fetchCwd = path.basename(gitCommonDir) === ".git" ? path.dirname(gitCommonDir) : gitCommonDir; - return yield* executeGit( - "GitVcsDriver.statusDetails.defaultBranch", - fetchCwd, - ["--git-dir", gitCommonDir, "symbolic-ref", "refs/remotes/origin/HEAD"], - { allowNonZeroExit: true }, - ).pipe( - Effect.map((result) => { - if (result.exitCode !== 0) return null; - return parseDefaultBranchFromRemoteHeadRef(result.stdout, "origin"); - }), - ); + return yield* resolvePrimaryDefaultBranchName(fetchCwd, ["--git-dir", gitCommonDir]); }), { capacity: 2_048, @@ -1689,9 +1774,42 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ), }); + const readRefSnapshot: GitVcsDriver.GitVcsDriver["Service"]["readRefSnapshot"] = Effect.fn( + "GitVcsDriver.readRefSnapshot", + )( + function* (cwd) { + const result = yield* executeGit( + "GitVcsDriver.readRefSnapshot", + cwd, + ["show-ref", "--head"], + { + allowNonZeroExit: true, + maxOutputBytes: REF_SNAPSHOT_MAX_OUTPUT_BYTES, + }, + ); + if (result.exitCode !== 0) return null; + const symbolicHead = yield* executeGit( + "GitVcsDriver.readRefSnapshot", + cwd, + ["symbolic-ref", "--quiet", "HEAD"], + { allowNonZeroExit: true }, + ); + // Exit 1 means detached HEAD; other failures require a full status read. + if (symbolicHead.exitCode !== 0 && symbolicHead.exitCode !== 1) return null; + // Deepening a shallow clone changes ancestry without moving refs. + const shallowPath = path.join(yield* resolveGitCommonDir(cwd), "shallow"); + const shallow = (yield* fileSystem.exists(shallowPath)) + ? yield* fileSystem.readFileString(shallowPath) + : ""; + return `${result.stdout}\0${symbolicHead.stdout}\0${shallow}`; + }, + Effect.orElseSucceed(() => null), + ); + const refreshStatusUpstreamIfStale = Effect.fn("refreshStatusUpstreamIfStale")(function* ( cwd: string, ) { + if (Duration.isZero(yield* automaticFetchInterval)) return; const headRef = yield* runGitStdout( "GitVcsDriver.refreshStatusUpstreamIfStale.headRef", cwd, @@ -1717,11 +1835,20 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const resolveDefaultBranchName = ( cwd: string, remoteName: string, + gitDirArgs: ReadonlyArray = [], ): Effect.Effect => executeGit( "GitVcsDriver.resolveDefaultBranchName", cwd, - ["symbolic-ref", `refs/remotes/${remoteName}/HEAD`], + // Unlike `symbolic-ref`, this fails when the remote deleted the branch HEAD points at. + [ + ...gitDirArgs, + "rev-parse", + "--verify", + "--quiet", + "--symbolic-full-name", + `refs/remotes/${remoteName}/HEAD`, + ], { allowNonZeroExit: true }, ).pipe( Effect.map((result) => { @@ -1755,8 +1882,11 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const originRemoteExists = (cwd: string): Effect.Effect => remoteExists({ cwd, remoteName: "origin" }); - const listRemoteNames = (cwd: string): Effect.Effect, GitCommandError> => - runGitStdout("GitVcsDriver.listRemoteNames", cwd, ["remote"]).pipe( + const listRemoteNames = ( + cwd: string, + gitDirArgs: ReadonlyArray = [], + ): Effect.Effect, GitCommandError> => + runGitStdout("GitVcsDriver.listRemoteNames", cwd, [...gitDirArgs, "remote"]).pipe( Effect.map(parseRemoteNamesInGitOrder), ); @@ -1779,6 +1909,18 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* return firstRemote ?? null; }); + const resolvePrimaryDefaultBranchName = Effect.fn("GitVcsDriver.resolvePrimaryDefaultBranchName")( + function* (cwd: string, gitDirArgs: ReadonlyArray = []) { + const originDefault = yield* resolveDefaultBranchName(cwd, "origin", gitDirArgs); + if (originDefault !== null) return originDefault; + const remoteNames = yield* listRemoteNames(cwd, gitDirArgs); + const [firstRemote] = remoteNames; + return firstRemote === undefined || remoteNames.includes("origin") + ? null + : yield* resolveDefaultBranchName(cwd, firstRemote, gitDirArgs); + }, + ); + const resolvePrimaryRemoteName = (cwd: string) => findPrimaryRemoteName(cwd).pipe( Effect.filterOrFail( @@ -1795,7 +1937,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ), ); - const resolvePushRemoteName = Effect.fn("resolvePushRemoteName")(function* ( + const resolveConfiguredPushRemoteName = Effect.fn("resolveConfiguredPushRemoteName")(function* ( cwd: string, refName: string, ) { @@ -1815,13 +1957,65 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ["config", "--get", "remote.pushDefault"], true, ).pipe(Effect.map((stdout) => stdout.trim())); - if (pushDefaultRemote.length > 0) { - return pushDefaultRemote; + return pushDefaultRemote.length > 0 ? pushDefaultRemote : null; + }); + + const resolvePushRemoteName = Effect.fn("resolvePushRemoteName")(function* ( + cwd: string, + refName: string, + fallbackRemoteName: string | null = null, + ) { + const configuredRemote = yield* resolveConfiguredPushRemoteName(cwd, refName); + if (configuredRemote) { + return configuredRemote; + } + + if (fallbackRemoteName) { + return fallbackRemoteName; } return yield* resolvePrimaryRemoteName(cwd).pipe(Effect.orElseSucceed(() => null)); }); + const remoteRefContainsHead = (cwd: string, remoteRef: string) => + executeGit( + "GitVcsDriver.pushCurrentBranch.publishContainsHead", + cwd, + ["merge-base", "--is-ancestor", "HEAD", `refs/remotes/${remoteRef}`], + { allowNonZeroExit: true }, + ).pipe( + Effect.map((result) => result.exitCode === 0), + Effect.orElseSucceed(() => false), + ); + + // A fetch reads only the remote's first url, while a push writes every push URL (pushurl, else + // each url). A fetched ref is evidence of the push destination only when those are the same one. + const pushesOnlyToFetchUrl = Effect.fn("GitVcsDriver.pushesOnlyToFetchUrl")( + function* (cwd: string, remoteName: string) { + const [fetchUrl, pushUrls] = yield* Effect.all([ + runGitStdout("GitVcsDriver.pushesOnlyToFetchUrl.fetchUrl", cwd, [ + "remote", + "get-url", + "--", + remoteName, + ]), + runGitStdout("GitVcsDriver.pushesOnlyToFetchUrl.pushUrls", cwd, [ + "remote", + "get-url", + "--push", + "--all", + "--", + remoteName, + ]), + ]); + // Preserve whitespace in URLs and reject ambiguous multiline fetch URLs. + return ( + fetchUrl.endsWith("\n") && !fetchUrl.slice(0, -1).includes("\n") && pushUrls === fetchUrl + ); + }, + Effect.orElseSucceed(() => false), + ); + const ensureRemote: GitVcsDriver.GitVcsDriver["Service"]["ensureRemote"] = Effect.fn( "ensureRemote", )(function* (input) { @@ -2083,10 +2277,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* branch = branchNameFromHeadRef(branchResult.stdout); } const currentUpstreamRef = yield* readCurrentUpstreamRef(cwd); - let upstreamRef = - currentUpstreamRef === null - ? null - : (parseUpstreamRefByFirstSeparator(currentUpstreamRef)?.upstreamRef ?? null); + let upstreamRef = currentUpstreamRef; // Trimming a remote that starts with Unicode whitespace can leave a leading slash. if (currentUpstreamRef?.startsWith("/")) { upstreamRef = (yield* resolveCurrentUpstream(cwd, currentUpstreamRef))?.upstreamRef ?? null; @@ -2114,7 +2305,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ); } - const defaultBranch = yield* resolveDefaultBranchName(cwd, "origin"); + const defaultBranch = yield* resolvePrimaryDefaultBranchName(cwd); const isDefaultBranch = branch !== null && (branch === defaultBranch || @@ -2146,11 +2337,18 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* options?: GitVcsDriver.GitLocalStatusOptions, ) { const includeDivergence = options?.includeDivergence !== false; + // Hidden untracked files could not be committed; `all` stays as configured. + const untrackedFilesHidden = untrackedFilesHiddenByConfig( + yield* readConfigValue(cwd, "status.showUntrackedFiles").pipe( + Effect.orElseSucceed(() => null), + ), + ); const statusArgs = [ "status", "--porcelain=2", "--branch", "-z", + ...(untrackedFilesHidden ? ["--untracked-files=normal"] : []), ...(includeDivergence ? [] : ["--no-ahead-behind"]), ]; const indexResult = yield* executeGitWithStableDiagnostics( @@ -2207,41 +2405,25 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* isMissingGitCwdError(error) ? Effect.succeed(null) : Effect.fail(error), }), ); + // Porcelain diff rewrites indexes despite GIT_OPTIONAL_LOCKS=0; plumbing with -M preserves rename detection. + const readNumstat = (ref: string) => + executeGitWithStableDiagnostics( + "GitVcsDriver.statusDetails.numstat", + cwd, + ["diff-index", "-M", "--numstat", "-z", ref, "--"], + { allowNonZeroExit: true, maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES }, + ); // diff-index does not read status output, so both spawn together; a non-repository discards it. const [statusResult, numstatResult, defaultBranch, hasPrimaryRemote] = yield* Effect.all( [ readStatus, - // Porcelain diff rewrites indexes despite GIT_OPTIONAL_LOCKS=0; plumbing with -M preserves rename detection. - executeGitWithStableDiagnostics( - "GitVcsDriver.statusDetails.numstat", - cwd, - ["diff-index", "-M", "--numstat", "-z", "HEAD", "--"], - { allowNonZeroExit: true, maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES }, - ).pipe( + readNumstat("HEAD").pipe( + Effect.filterOrElse( + (result) => result.exitCode === 0 || !isUnbornHeadStderr(result.stderr), + () => Effect.flatMap(readEmptyTreeHash(cwd), readNumstat), + ), Effect.flatMap((result) => { if (result.exitCode === 0) return Effect.succeed(parseReviewNumstat(result.stdout)); - if (isUnbornHeadStderr(result.stderr)) { - return Effect.map( - Effect.all([ - runGitStdoutWithOptions( - "GitVcsDriver.statusDetails.numstat.unborn", - cwd, - ["diff-files", "--numstat", "-z"], - { maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES }, - ), - runGitStdoutWithOptions( - "GitVcsDriver.statusDetails.numstat.unborn.staged", - cwd, - ["diff", "--cached", "--numstat", "-z"], - { maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES }, - ), - ]), - ([unstagedStdout, stagedStdout]) => [ - ...parseReviewNumstat(stagedStdout), - ...parseReviewNumstat(unstagedStdout), - ], - ); - } return Effect.fail( new GitCommandError({ ...gitCommandContext({ @@ -2260,7 +2442,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ), statusCacheKey ? Cache.get(defaultBranchCache, statusCacheKey).pipe(Effect.orElseSucceed(() => null)) - : resolveDefaultBranchName(cwd, "origin").pipe(Effect.orElseSucceed(() => null)), + : resolvePrimaryDefaultBranchName(cwd).pipe(Effect.orElseSucceed(() => null)), statusCacheKey ? Cache.get(primaryRemoteExistsCache, statusCacheKey).pipe( Effect.orElseSucceed(() => false), @@ -2298,6 +2480,10 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* let refName: string | null = null; let upstreamRef: string | null = null; + // Git omits branch.ab when the upstream ref is gone, e.g. deleted after its PR merged, + // and also for an unborn HEAD, which can still track a fetched upstream. + let upstreamRefExists = false; + let isUnbornHead = false; let aheadCount = 0; let behindCount = 0; let aheadOfDefaultCount = 0; @@ -2308,9 +2494,14 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const statusRecords = statusStdout.split("\0"); for (let index = 0; index < statusRecords.length; index++) { const line = statusRecords[index]!; + if (line === "# branch.oid (initial)") { + isUnbornHead = true; + continue; + } if (line.startsWith("# branch.head ")) { const value = line.slice("# branch.head ".length).trim(); - refName = value.startsWith("(") ? null : value; + // `(unknown)` is printed while HEAD names no resolvable branch, e.g. mid-`git clone`. + refName = value === "(detached)" || value === "(unknown)" ? null : value; continue; } if (line.startsWith("# branch.upstream ")) { @@ -2318,11 +2509,13 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* upstreamRef = value.length > 0 ? value : null; continue; } - if (includeDivergence && line.startsWith("# branch.ab ")) { - const value = line.slice("# branch.ab ".length).trim(); - const parsed = parseBranchAb(value); - aheadCount = parsed.ahead; - behindCount = parsed.behind; + if (line.startsWith("# branch.ab ")) { + upstreamRefExists = true; + if (includeDivergence) { + const parsed = parseBranchAb(line.slice("# branch.ab ".length).trim()); + aheadCount = parsed.ahead; + behindCount = parsed.behind; + } continue; } if (line.length > 0 && !line.startsWith("#")) { @@ -2341,6 +2534,8 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* } } + if (!upstreamRefExists && !isUnbornHead) upstreamRef = null; + const isDefaultBranch = refName !== null && (refName === defaultBranch || @@ -2465,6 +2660,18 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* })), ); + // git add takes --pathspec-from-file from Git 2.25; older Git keeps the argument form. + let pathspecFromFileSupported: boolean | undefined; + const supportsPathspecFromFile = Effect.fn("GitVcsDriver.supportsPathspecFromFile")(function* ( + cwd: string, + ) { + if (pathspecFromFileSupported !== undefined) return pathspecFromFileSupported; + const version = yield* executeGit("GitVcsDriver.version", cwd, ["version"]).pipe(Effect.option); + if (Option.isNone(version)) return false; + pathspecFromFileSupported = gitSupportsPathspecFromFile(version.value.stdout); + return pathspecFromFileSupported; + }); + const stageCommitChanges = Effect.fnUntraced(function* ( cwd: string, filePaths?: readonly string[], @@ -2473,6 +2680,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const config = env ? ["-c", "core.splitIndex=false", "-c", "splitIndex.sharedIndexExpire=never"] : []; + let preserveUnselectedConflicts = false; if (filePaths && filePaths.length > 0) { if (env) { const head = yield* executeGit( @@ -2489,43 +2697,188 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* { env }, ); } else { - const mergeHeadPath = yield* runGitStdout("GitVcsDriver.commit.mergeHead", cwd, [ + const inProgressHeads = [ + { head: "MERGE_HEAD", operation: "merge" }, + { head: "CHERRY_PICK_HEAD", operation: "cherry-pick" }, + { head: "REVERT_HEAD", operation: "revert" }, + // A conflicted multi-commit `cherry-pick --no-commit` writes only the sequencer; reset + // would clear its unmerged entries, so `--continue` would walk past the conflict. + { head: "sequencer", operation: "cherry-pick or revert" }, + ] as const; + const headPaths = yield* runGitStdout("GitVcsDriver.commit.mergeHead", cwd, [ "rev-parse", - "--git-path", - "MERGE_HEAD", + ...inProgressHeads.flatMap(({ head }) => ["--git-path", head]), ]); - const isMerging = yield* fileSystem.exists(path.resolve(cwd, mergeHeadPath.trim())).pipe( + const resolvedHeadPaths = headPaths.trim().split("\n"); + // Reftable keeps CHERRY_PICK_HEAD and REVERT_HEAD as refs with no file under .git. A + // same-named branch resolves to its full name; an ambiguity warning would blank stdout. + const operationRefs = new Set(["CHERRY_PICK_HEAD", "REVERT_HEAD"]); + const isOperationRef = (head: string) => + runGitStdout( + "GitVcsDriver.commit.operationRef", + cwd, + [ + "-c", + "core.warnAmbiguousRefs=false", + "rev-parse", + "--verify", + "--quiet", + "--symbolic-full-name", + head, + ], + true, + ).pipe(Effect.map((ref) => ref.trim() === head)); + const inProgress = yield* Effect.findFirst(inProgressHeads, ({ head }, index) => + fileSystem.exists(path.resolve(cwd, resolvedHeadPaths[index] ?? "")).pipe( + Effect.filterOrElse( + (exists) => exists || !operationRefs.has(head), + () => isOperationRef(head), + ), + ), + ).pipe( Effect.mapError( (cause) => new GitCommandError({ operation: "GitVcsDriver.commit.mergeHead", command: "git commit", cwd, - detail: "Could not check whether a merge is in progress.", + detail: "Could not check whether a merge, cherry-pick or revert is in progress.", cause, }), ), ); - // Like git commit --only, reject partial merges before reset can discard their ancestry. - if (isMerging) { + // Like git commit --only, reject partial commits before reset can discard their state. + if (Option.isSome(inProgress)) { + const { operation } = inProgress.value; return yield* new GitCommandError({ operation: "GitVcsDriver.commit", command: "git commit", cwd, - detail: - "Cannot commit selected files during a merge. Commit all files to finish the merge.", + detail: `Cannot commit selected files during a ${operation}. Commit all files to finish the ${operation}.`, }); } - yield* runGit("GitVcsDriver.commit.reset", cwd, ["reset", "--quiet"]); + const selectedFiles = new Set(filePaths); + const unmergedPaths = yield* runGitStdoutWithOptions( + "GitVcsDriver.commit.unmergedPaths", + cwd, + ["-c", "diff.relative=false", "diff", "--cached", "--name-only", "--diff-filter=U", "-z"], + { maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES }, + ); + // Reset drops every conflict stage, so keep the index while unselected files are conflicted. + preserveUnselectedConflicts = unmergedPaths + .split("\0") + .some((file) => file.length > 0 && !selectedFiles.has(file)); + if (!preserveUnselectedConflicts) { + yield* runGit("GitVcsDriver.commit.reset", cwd, ["reset", "--quiet"]); + } } + // Status paths are repository-relative, while cwd can be a subdirectory. + const pathspecs = filePaths.map((file) => `:(top,literal)${file}`); + const fromStdin = yield* supportsPathspecFromFile(cwd); yield* runGit( "GitVcsDriver.commit.addSelected", cwd, - // Status paths are repository-relative, while cwd can be a subdirectory. - [...config, "add", "-A", "--", ...filePaths.map((file) => `:(top,literal)${file}`)], - env ? { env } : {}, + fromStdin + ? [...config, "add", "-A", "--pathspec-from-file=-", "--pathspec-file-nul"] + : [...config, "add", "-A", "--", ...pathspecs], + { + ...(env ? { env } : {}), + ...(fromStdin ? { stdin: pathspecs.map((pathspec) => `${pathspec}\0`).join("") } : {}), + }, ); } else { + // add -A marks unmerged paths resolved, so refuse while one still has conflict markers. + const unmergedPaths = (yield* runGitStdoutWithOptions( + "GitVcsDriver.commit.unmergedPaths", + cwd, + ["-c", "diff.relative=false", "diff", "--name-only", "--diff-filter=U", "-z"], + env ? { env } : {}, + )) + .split("\0") + .filter((file) => file.length > 0); + if (unmergedPaths.length > 0) { + const check = yield* executeGit( + "GitVcsDriver.commit.conflictMarkers", + cwd, + [ + "-c", + "diff.relative=false", + "diff", + "--check", + "--no-color", + "--", + ...unmergedPaths.map((file) => `:(top,literal)${file}`), + ], + { + ...(env ? { env } : {}), + allowNonZeroExit: true, + maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES, + appendTruncationMarker: true, + }, + ); + const reports = check.stdout.split("\n"); + const conflicted: string[] = []; + let emptyTree: string | undefined; + for (const file of unmergedPaths) { + const markerLines = new Set( + reports.flatMap((line) => { + const match = line.startsWith(`${file}:`) + ? /^(\d+): leftover conflict marker$/.exec(line.slice(file.length + 1)) + : null; + return match ? [Number(match[1]) - 1] : []; + }), + ); + if (markerLines.size === 0) continue; + emptyTree ??= yield* readEmptyTreeHash(cwd); + // --check numbers lines after working-tree-encoding and clean filters; a diff from the + // empty tree prints that view. Read it in full so large resolved files can still commit. + const fileDiff = yield* runGitStdoutWithOptions( + "GitVcsDriver.commit.conflictMarkers", + cwd, + [ + "-c", + "diff.relative=false", + "diff", + "--no-color", + "--no-ext-diff", + "--no-textconv", + "--text", + emptyTree, + "--", + `:(top,literal)${file}`, + ], + { ...(env ? { env } : {}), maxOutputBytes: Number.POSITIVE_INFINITY }, + ).pipe( + Effect.mapError( + (cause) => + new GitCommandError({ + operation: "GitVcsDriver.commit.conflictMarkers", + command: "git commit", + cwd, + detail: `Could not check ${file} for conflict markers.`, + cause, + }), + ), + ); + const diffLines = fileDiff.split("\n"); + const lines = diffLines + .slice(diffLines.findIndex((line) => line.startsWith("@@")) + 1) + .filter((line) => line.startsWith("+")) + .map((line) => line.slice(1)); + // Git also reports a lone "=======", which is valid content such as a heading underline. + const reported = (marker: string) => + lines.some((line, index) => markerLines.has(index) && line.startsWith(marker)); + if (reported("<") && reported(">")) conflicted.push(file); + } + if (conflicted.length > 0) { + return yield* new GitCommandError({ + operation: "GitVcsDriver.commit", + command: "git commit", + cwd, + detail: `Cannot commit with unresolved merge conflicts in ${conflicted.join(", ")}. Resolve them, then commit.`, + }); + } + } yield* runGit( "GitVcsDriver.commit.addAll", cwd, @@ -2533,6 +2886,9 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* env ? { env } : {}, ); } + return preserveUnselectedConflicts && filePaths + ? filePaths.map((file) => `:(top,literal)${file}`) + : undefined; }); const prepareCommitContext: GitVcsDriver.GitVcsDriver["Service"]["prepareCommitContext"] = @@ -2544,7 +2900,9 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* cwd, ["rev-parse", "--git-path", "index", "--git-path", "objects"], )).split("\n"); - const indexPath = path.resolve(cwd, indexValue.trim()); + // Git's relative paths use the physical cwd, even when the project uses a symlink. + const physicalCwd = yield* fileSystem.realPath(cwd); + const indexPath = path.resolve(physicalCwd, indexValue.trim()); const directory = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-commit-index-", }); @@ -2562,7 +2920,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const quoted = (entry: string) => entry.startsWith('"') ? entry : `"${entry.replace(/[\\"]/g, "\\$&")}"`; const alternates = [ - path.resolve(cwd, objectsValue.trim()), + path.resolve(physicalCwd, objectsValue.trim()), ...objectDirectories .split("\n") .flatMap((line) => @@ -2615,7 +2973,15 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* { env, maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES }, ).pipe(Effect.map((stdout) => stdout.trim())); if (stagedSummary.length === 0) { - return null; + // Like git commit, a merge resolved to HEAD's tree still records its merge commit. + const mergeHeadPath = yield* runGitStdout( + "GitVcsDriver.prepareCommitContext.mergeHead", + cwd, + ["rev-parse", "--git-path", "MERGE_HEAD"], + ); + if (!(yield* fileSystem.exists(path.resolve(physicalCwd, mergeHeadPath.trim())))) { + return null; + } } // A user-supplied message needs change detection, but no patch for generation. @@ -2667,14 +3033,26 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* body, options?: GitVcsDriver.GitCommitOptions, ) { - if (options?.stage !== undefined) { - yield* stageCommitChanges(cwd, options.stage.filePaths); - } + const partialCommitPaths = + options?.stage !== undefined + ? yield* stageCommitChanges(cwd, options.stage.filePaths) + : undefined; const args = ["commit", "-m", subject]; const trimmedBody = body.trim(); if (trimmedBody.length > 0) { args.push("-m", trimmedBody); } + const onlyFromStdin = partialCommitPaths ? yield* supportsPathspecFromFile(cwd) : false; + const onlyPathspecArgs = onlyFromStdin + ? ["--pathspec-from-file=-", "--pathspec-file-nul"] + : ["--", ...(partialCommitPaths ?? [])]; + const onlyPathspecStdin = + partialCommitPaths && onlyFromStdin + ? { stdin: partialCommitPaths.map((pathspec) => `${pathspec}\0`).join("") } + : {}; + if (partialCommitPaths) { + args.push("--only", ...onlyPathspecArgs); + } const progress = options?.progress?.onOutputLine === undefined ? options?.progress @@ -2687,9 +3065,19 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* }; yield* executeGit("GitVcsDriver.commit.commit", cwd, args, { maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES, + ...onlyPathspecStdin, ...(options?.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}), ...(progress ? { progress } : {}), }).pipe(Effect.asVoid); + if (partialCommitPaths) { + // --only runs hooks on a temporary index, so a hook's `git add` never reaches the real one. + yield* runGit( + "GitVcsDriver.commit.syncSelectedIndex", + cwd, + ["reset", "--quiet", "HEAD", ...onlyPathspecArgs], + onlyPathspecStdin, + ); + } const commitSha = yield* runGitStdout("GitVcsDriver.commit.revParseHead", cwd, [ "rev-parse", "HEAD", @@ -2730,10 +3118,40 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* hasUpstream: boolean, ) { if (!hasUpstream) { - const remoteName = yield* resolvePushRemoteName(cwd, branch); + // A deleted upstream keeps its config: republish to it when it was this branch's own, + // otherwise it was the base, so keep that before publishing. + let goneUpstream: { remoteName: string; branchName: string } | null = null; + let goneBaseUpstream: { remoteName: string; branchName: string; upstreamRef: string } | null = + null; + const mergeRef = yield* readConfigValue(cwd, `branch.${branch}.merge`); + const mergeBranch = mergeRef?.startsWith("refs/heads/") + ? mergeRef.slice("refs/heads/".length) + : null; + const configuredRemoteName = yield* readConfigValue(cwd, `branch.${branch}.remote`); + if (mergeBranch && configuredRemoteName && configuredRemoteName !== ".") { + if (isAliasOfUpstreamHead(branch, mergeBranch, `${configuredRemoteName}/${mergeBranch}`)) { + goneUpstream = { remoteName: configuredRemoteName, branchName: mergeBranch }; + } else { + goneBaseUpstream = { + remoteName: configuredRemoteName, + branchName: mergeBranch, + upstreamRef: `${configuredRemoteName}/${mergeBranch}`, + }; + } + } + const remoteName = yield* resolvePushRemoteName(cwd, branch, goneUpstream?.remoteName); if (!remoteName) return null; - const branchName = yield* resolvePublishBranchName(cwd, branch); - return { remoteName, branchName, publishes: true, upstream: null }; + // A gone local upstream still gets the full local name, as below. + const branchName = + goneUpstream?.branchName ?? + (configuredRemoteName === "." ? branch : yield* resolvePublishBranchName(cwd, branch)); + return { + remoteName, + branchName, + publishes: true, + upstream: goneBaseUpstream, + pushRemote: null, + }; } if ((yield* readConfigValue(cwd, `branch.${branch}.remote`)) === ".") { @@ -2751,21 +3169,27 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* branchName: branch, publishes: true, upstream: { remoteName: ".", branchName: localUpstream, upstreamRef: localUpstream }, + pushRemote: null, }; } const upstream = yield* resolveCurrentUpstream(cwd).pipe(Effect.orElseSucceed(() => null)); if (!upstream) return null; - // Differently named upstreams are bases, not publish targets; pushing HEAD there would write features to shared branches. - const isAliasOfUpstreamHead = - branch === upstream.branchName || - (branch.endsWith(`/${upstream.branchName}`) && upstream.upstreamRef.endsWith(`/${branch}`)); - if (isAliasOfUpstreamHead) { + if (isAliasOfUpstreamHead(branch, upstream.branchName, upstream.upstreamRef)) { + // A fork workflow pushes to the configured push remote and keeps the upstream for pulls, as plain `git push` does. + const configuredPushRemote = yield* resolveConfiguredPushRemoteName(cwd, branch).pipe( + Effect.orElseSucceed(() => null), + ); + const pushRemote = + configuredPushRemote && configuredPushRemote !== upstream.remoteName + ? configuredPushRemote + : null; return { - remoteName: upstream.remoteName, + remoteName: pushRemote ?? upstream.remoteName, branchName: upstream.branchName, publishes: false, upstream, + pushRemote, }; } const publishRemoteName = yield* resolvePushRemoteName(cwd, branch).pipe( @@ -2777,6 +3201,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* branchName, publishes: true, upstream, + pushRemote: null, }; }); @@ -2785,7 +3210,11 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* )(function* (cwd, current) { const destination = yield* resolvePushDestination(cwd, current.branch, current.hasUpstream); return destination - ? { remoteName: destination.remoteName, branchName: destination.branchName } + ? { + remoteName: destination.remoteName, + branchName: destination.branchName, + viaPushRemote: destination.pushRemote !== null, + } : null; }); @@ -2808,7 +3237,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const requestedRemoteName = options?.remoteName?.trim() || null; if (requestedRemoteName) { const publishBranch = yield* resolvePublishBranchName(cwd, branch); - yield* runGit( + yield* executeGitWithStableDiagnostics( "GitVcsDriver.pushCurrentBranch.pushWithRequestedRemote", cwd, ["push", "--", requestedRemoteName, `HEAD:refs/heads/${publishBranch}`], @@ -2823,16 +3252,41 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* } const hasNoLocalDelta = details.aheadCount === 0 && details.behindCount === 0; - if (hasNoLocalDelta && details.hasUpstream) { - return { - status: "skipped_up_to_date" as const, - branch, - ...(details.upstreamRef ? { upstreamBranch: details.upstreamRef } : {}), - }; - } - const destination = yield* resolvePushDestination(cwd, branch, details.hasUpstream); - if (hasNoLocalDelta) { + if (hasNoLocalDelta && details.hasUpstream) { + if (!destination?.pushRemote) { + // The tracking ref reflects the fetch URL, so it says nothing about a separate push URL. + if (!destination || (yield* pushesOnlyToFetchUrl(cwd, destination.remoteName))) { + return { + status: "skipped_up_to_date" as const, + branch, + ...(details.upstreamRef ? { upstreamBranch: details.upstreamRef } : {}), + }; + } + } else { + const { pushRemote, branchName } = destination; + const pushRemoteRef = `${pushRemote}/${branchName}`; + // The local ref can be stale when the branch was deleted or rewound elsewhere. + const pushRemoteHasHead = + (yield* pushesOnlyToFetchUrl(cwd, pushRemote)) && + (yield* fetchRemoteTrackingBranch({ + cwd, + remoteName: pushRemote, + remoteBranch: branchName, + }).pipe( + Effect.andThen(remoteRefContainsHead(cwd, pushRemoteRef)), + Effect.orElseSucceed(() => false), + )); + if (pushRemoteHasHead) { + return { + status: "skipped_up_to_date" as const, + branch, + upstreamBranch: pushRemoteRef, + pushRemote, + }; + } + } + } else if (hasNoLocalDelta) { const comparableBaseBranch = yield* resolveBaseBranchForNoUpstream(cwd, branch).pipe( Effect.orElseSucceed(() => null), ); @@ -2845,21 +3299,13 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* } // The configured diff base can contain HEAD while the publish branch is still behind. - const publishContainsHead = yield* executeGit( - "GitVcsDriver.pushCurrentBranch.publishContainsHead", - cwd, - [ - "merge-base", - "--is-ancestor", - "HEAD", - `refs/remotes/${destination.remoteName}/${destination.branchName}`, - ], - { allowNonZeroExit: true }, - ).pipe( - Effect.map((result) => result.exitCode === 0), - Effect.orElseSucceed(() => false), - ); - if (publishContainsHead) { + if ( + (yield* remoteRefContainsHead( + cwd, + `${destination.remoteName}/${destination.branchName}`, + )) && + (yield* pushesOnlyToFetchUrl(cwd, destination.remoteName)) + ) { return { status: "skipped_up_to_date" as const, branch, @@ -2902,7 +3348,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* }); } } - yield* runGit( + yield* executeGitWithStableDiagnostics( upstream ? "GitVcsDriver.pushCurrentBranch.pushOwnBranch" : "GitVcsDriver.pushCurrentBranch.pushWithUpstream", @@ -2919,7 +3365,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* } if (destination?.upstream) { - yield* runGit( + yield* executeGitWithStableDiagnostics( "GitVcsDriver.pushCurrentBranch.pushUpstream", cwd, ["push", "--", destination.remoteName, `HEAD:refs/heads/${destination.branchName}`], @@ -2928,12 +3374,19 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* return { status: "pushed" as const, branch, - upstreamBranch: destination.upstream.upstreamRef, + ...(destination.pushRemote + ? { + upstreamBranch: `${destination.remoteName}/${destination.branchName}`, + pushRemote: destination.pushRemote, + } + : { upstreamBranch: destination.upstream.upstreamRef }), setUpstream: false, }; } - yield* runGit("GitVcsDriver.pushCurrentBranch.push", cwd, ["push"], { timeoutMs: null }); + yield* executeGitWithStableDiagnostics("GitVcsDriver.pushCurrentBranch.push", cwd, ["push"], { + timeoutMs: null, + }); return { status: "pushed" as const, branch, @@ -2968,12 +3421,31 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* }); } const remote = yield* readConfigValue(cwd, `branch.${refName}.remote`); + // The upstream-only retry leaves other remote-tracking refs stale, so a claimed status + // fetch must still run after it. + let fetchedUpstreamOnly = false; if (remote) { const gitCommonDir = yield* resolveGitCommonDir(cwd); const pending = statusFetches.get(`${gitCommonDir}\0${remote}`); // The status wait can finish while its fetch still owns reference locks. // Let our existing writer finish before starting the pull's fetch. - if (pending) yield* Fiber.await(pending); + if (pending) { + const claimed = yield* Effect.uninterruptible( + Effect.gen(function* () { + if (pending.state.phase !== "queued") return false; + pending.state.phase = "claimed"; + yield* Effect.addFinalizer((exit) => + Deferred.succeed( + pending.state.finished, + Exit.isSuccess(exit) && !fetchedUpstreamOnly, + ), + ); + yield* Deferred.succeed(pending.state.claimed, undefined); + return true; + }), + ); + if (!claimed) yield* Fiber.await(pending.fiber); + } } const beforeSha = yield* runGitStdout( "GitVcsDriver.pullCurrentBranch.beforeSha", @@ -2984,8 +3456,9 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* // After a successful pull, HEAD before and after decides the result, so its output is not // needed. A large fast-forward's diffstat can exceed the output cap, which must not fail a // pull Git applied. + // A deadline would kill git mid-checkout (e.g. a slow LFS smudge) and strand a half-updated tree. const options = { - timeoutMs: 30_000, + timeoutMs: null, appendTruncationMarker: true, allowNonZeroExit: true, }; @@ -3026,6 +3499,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* currentSha === beforeSha ) { args = ["pull", "--ff-only", "--", remote, mergeRef]; + fetchedUpstreamOnly = true; result = yield* executeGitWithStableDiagnostics( "GitVcsDriver.pullCurrentBranch.pull", cwd, @@ -3054,11 +3528,14 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const refreshed = yield* readStatusDetailsLocal(cwd); return { - status: beforeSha.length > 0 && beforeSha === afterSha ? "skipped_up_to_date" : "pulled", + status: + beforeSha.length > 0 && beforeSha === afterSha + ? ("skipped_up_to_date" as const) + : ("pulled" as const), refName, upstreamRef: refreshed.upstreamRef, }; - }); + }, Effect.scoped); const readRangeContext: GitVcsDriver.GitVcsDriver["Service"]["readRangeContext"] = Effect.fn( "readRangeContext", @@ -3135,7 +3612,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const prepareReviewIndex = Effect.fn("prepareReviewIndex")(function* ( cwd: string, untrackedPaths: ReadonlyArray, - maxBytes?: number, + allowPartial: boolean, ) { const [stagedDeletionsStdout, indexValue] = yield* Effect.all( [ @@ -3163,21 +3640,42 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* { concurrency: 2 }, ); const stagedDeletions = new Set(stagedDeletionsStdout.split("\0").filter(Boolean)); - const pathsToAdd = untrackedPaths.filter((relativePath) => !stagedDeletions.has(relativePath)); - if (pathsToAdd.length === 0) return undefined; - if (maxBytes !== undefined) { - const sizes = yield* Effect.forEach( - pathsToAdd, - (relativePath) => - fileSystem.lstat(path.resolve(cwd, relativePath)).pipe( - Effect.map((info) => (info.type === "File" ? Number(info.size) : 0)), - Effect.orElseSucceed(() => 0), - ), - { concurrency: 16 }, - ); - // null: over the byte budget, so the caller reports totals as unavailable. - if (sizes.reduce((total, size) => total + size, 0) > maxBytes) return null; + // Staged deletions never enter the index, so their worktree copies must not spend the budget. + // A nested repository (listed with a trailing slash) with no commits cannot be indexed: + // Git before 2.53 fails the add, and later versions accept it and then fail every diff. + const candidates = yield* Effect.filter( + untrackedPaths.filter((relativePath) => !stagedDeletions.has(relativePath)), + (relativePath) => + relativePath.endsWith("/") + ? executeGit( + "GitVcsDriver.readUnifiedWorkingTreeReviewDiff.nestedRepositoryHead", + path.resolve(cwd, relativePath), + ["rev-parse", "--verify", "--quiet", "HEAD"], + { allowNonZeroExit: true }, + ).pipe(Effect.map((result) => result.exitCode !== 1)) + : Effect.succeed(true), + { concurrency: 16 }, + ); + const sizes = yield* Effect.forEach( + candidates, + (relativePath) => + fileSystem.lstat(path.resolve(cwd, relativePath)).pipe( + Effect.map((info) => (info.type === "File" ? Number(info.size) : 0)), + Effect.orElseSucceed(() => 0), + Effect.map((size) => ({ relativePath, size })), + ), + { concurrency: 16 }, + ); + // Smallest first, so which files fit does not depend on how their names sort. + const pathsToAdd: string[] = []; + let bytes = 0; + for (const { relativePath, size } of sizes.toSorted((left, right) => left.size - right.size)) { + bytes += size; + if (bytes > REVIEW_UNTRACKED_MAX_BYTES) break; + pathsToAdd.push(relativePath); } + const omitted = pathsToAdd.length < candidates.length; + if (pathsToAdd.length === 0 || (omitted && !allowPartial)) return { env: undefined, omitted }; const indexPath = path.isAbsolute(indexValue.trim()) ? indexValue.trim() @@ -3230,7 +3728,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ], { env, stdin: `${pathsToAdd.join("\0")}\0` }, ); - return env; + return { env, omitted }; }); const readEmptyTreeHash = Effect.fn("readEmptyTreeHash")(function* (cwd: string) { @@ -3246,8 +3744,8 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const prepareUntrackedReviewIndexOrReason = Effect.fn("prepareUntrackedReviewIndex")(function* ( cwd: string, pathArgs: ReadonlyArray, - onlyPath?: string, - maxBytes?: number, + onlyPath: string | undefined, + allowPartial: boolean, ) { const untracked = yield* executeGit( "GitVcsDriver.review.listUntracked", @@ -3266,10 +3764,10 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* (candidate) => onlyPath === undefined || candidate === onlyPath || candidate === `${onlyPath}/`, ); - if (paths.length === 0) return { env: undefined }; + if (paths.length === 0) return { env: undefined, omitted: false }; if (paths.length > REVIEW_UNTRACKED_MAX_FILES) return { unavailable: "Too many untracked files to count." }; - const env = yield* prepareReviewIndex(cwd, paths, maxBytes).pipe( + const index = yield* prepareReviewIndex(cwd, paths, allowPartial).pipe( Effect.catchTags({ PlatformError: (cause) => Effect.fail( @@ -3283,15 +3781,16 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ), }), ); - if (env === null) return { unavailable: "Untracked files are too large to count." }; - return { env }; + if (index.omitted && !allowPartial) + return { unavailable: "Untracked files are too large to count." }; + return index; }); const prepareUntrackedReviewIndex = ( cwd: string, pathArgs: ReadonlyArray, onlyPath?: string, ) => - prepareUntrackedReviewIndexOrReason(cwd, pathArgs, onlyPath).pipe( + prepareUntrackedReviewIndexOrReason(cwd, pathArgs, onlyPath, true).pipe( Effect.map((result) => ("unavailable" in result ? null : result)), ); @@ -3382,13 +3881,11 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* detail: remembered.detail, }); } - // Empty status can mean untracked files are hidden; retain the full scan. - const untracked = yield* prepareUntrackedReviewIndexOrReason( - cwd, - pathspecs, - undefined, - UNTRACKED_TOTALS_MAX_BYTES, - ); + // Status always lists untracked files, so no roots means there are none to scan. + const untracked = + roots.length === 0 + ? { env: undefined, omitted: false } + : yield* prepareUntrackedReviewIndexOrReason(cwd, pathspecs, undefined, false); if ("unavailable" in untracked) { const count = remembered?.roots === rootsKey ? remembered.count + 1 : 1; untrackedTotalsFailures.delete(cwd); @@ -3416,7 +3913,8 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* executeGit( "GitVcsDriver.readBranchChangeTotals", cwd, - [...REVIEW_DIFF_ARGS, "--numstat", "-z", ref, "--"], + // Plumbing: porcelain diff refreshes and rewrites the real index, taking index.lock. + ["diff-index", "-M", "--numstat", "-z", ref, "--"], { allowNonZeroExit: true, maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES, @@ -3642,7 +4140,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const stat = yield* executeGit( "GitVcsDriver.getReviewDiffPreview.commitStat", cwd, - [...diffArgs, "--numstat", "-z", commit.parent, commit.sha, "--", ...pathArgs], + [...diffArgs, "--raw", "--numstat", "-z", commit.parent, commit.sha, "--", ...pathArgs], { maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES }, ); const files = parseReviewNumstat(stat.stdout); @@ -3692,7 +4190,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ref: string, env?: NodeJS.ProcessEnv, ) { - const args = [...diffArgs, "--numstat", "-z"]; + const args = [...diffArgs, "--raw", "--numstat", "-z"]; const result = yield* executeGit( "GitVcsDriver.getReviewDiffPreview.stat", cwd, @@ -3738,9 +4236,11 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* }); const [dirtyTrackedResult, baseResult] = yield* Effect.gen(function* () { const untracked = input.trackedOnly - ? { env: undefined } + ? { env: undefined, omitted: false } : yield* prepareUntrackedReviewIndex(cwd, pathArgs, input.file?.path); - const metadataOnly = input.metadataOnly === true && !input.file && untracked !== null; + // Untracked files left out make the file list partial, so the bounded patch is sent instead. + const complete = untracked !== null && !untracked.omitted; + const metadataOnly = input.metadataOnly === true && !input.file && complete; const [dirty, base] = review.mergeBase === dirtyRef ? yield* readTrackedDiff(dirtyRef, untracked?.env, metadataOnly).pipe( @@ -3753,7 +4253,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ], { concurrency: 2 }, ); - if (untracked !== null) return [dirty, base] as const; + if (complete) return [dirty, base] as const; const incomplete = (ref: string | null, result: typeof dirty) => ref === null ? result : { ...result, files: undefined, stdoutTruncated: true }; return [incomplete(dirtyRef, dirty), incomplete(review.mergeBase, base)] as const; @@ -3911,7 +4411,43 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* if (bytes.includes(0)) { return yield* fileError("fs.readFile", `Cannot expand binary file '${input.newPath}'.`); } - return new TextDecoder("utf-8", { ignoreBOM: true }).decode(bytes); + const contents = new TextDecoder("utf-8", { ignoreBOM: true }).decode(bytes); + if (!contents.includes("$Id:")) return contents; + const collapsed = contents.replace(GIT_IDENT_KEYWORD, () => "$Id$"); + if (collapsed === contents) return contents; + // check-attr reports both boolean ident and literal ident=set as "set". + const attribute = yield* runGitStdout( + "GitVcsDriver.getReviewDiffFileContents.workingTree.ident", + repositoryRoot, + ["check-attr", "-z", "ident", "filter", "--", input.newPath], + ); + const attributes = attribute.split("\0"); + if (attributes[2] !== "set" || attributes[5] !== "unspecified") return contents; + + // Literal filter=unspecified is indistinguishable from an absent filter in check-attr. + const filters = yield* executeGit( + "GitVcsDriver.getReviewDiffFileContents.workingTree.filters", + repositoryRoot, + ["config", "--name-only", "--get-regexp", "^filter\\.unspecified\\."], + { allowNonZeroExit: true }, + ); + if (filters.exitCode !== 1 || filters.stdout.length > 0) return contents; + // Probe a nonoverlapping keyword so ident conversion runs once on the actual file. + const [canonicalHash, collapsedHash] = yield* Effect.all([ + runGitStdoutWithOptions( + "GitVcsDriver.getReviewDiffFileContents.workingTree.canonicalHash", + repositoryRoot, + ["hash-object", "--stdin", `--path=${input.newPath}`], + { stdin: "$Id: probe $" }, + ), + runGitStdoutWithOptions( + "GitVcsDriver.getReviewDiffFileContents.workingTree.collapsedHash", + repositoryRoot, + ["hash-object", "--stdin", `--path=${input.newPath}`], + { stdin: "$Id$" }, + ), + ]); + return canonicalHash === collapsedHash ? collapsed : contents; }); const getReviewDiffFileContents = Effect.fn("getReviewDiffFileContents")(function* ( @@ -4109,7 +4645,14 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* executeGit( "GitVcsDriver.listRefs.defaultRef", fetchCwd, - [...gitDirArgs, "symbolic-ref", "refs/remotes/origin/HEAD"], + [ + ...gitDirArgs, + "rev-parse", + "--verify", + "--quiet", + "--symbolic-full-name", + "refs/remotes/origin/HEAD", + ], { timeoutMs: 5_000, allowNonZeroExit: true, @@ -4137,7 +4680,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* } const defaultBranch = defaultRefResult.exitCode === 0 - ? defaultRefResult.stdout.trim().replace(/^refs\/remotes\/origin\//, "") + ? parseDefaultBranchFromRemoteHeadRef(defaultRefResult.stdout, "origin") : null; const parsedWorktreeEntries = worktreeListResult.exitCode === 0 @@ -4212,6 +4755,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* : left.ref.name.localeCompare(right.ref.name); return { + defaultBranch, localBranches: localBranches.toSorted(byRecencyThenName).map(({ ref }) => ref), remoteBranches: remoteBranches.toSorted(byRecencyThenName).map(({ ref }) => ref), hasPrimaryRemote: remoteNames.length > 0, @@ -4340,22 +4884,51 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* }; } - const snapshot = yield* resolveListRefsSnapshot( - repositoryPaths.gitCommonDir, - input.refresh === true, + // Both caches outlive a checkout made outside T3 Code, so this checkout's branch is read + // on every call and replaces whatever the cached snapshot claims for it. + const [snapshot, currentBranch] = yield* Effect.all( + [ + resolveListRefsSnapshot(repositoryPaths.gitCommonDir, input.refresh === true), + readCurrentBranch("GitVcsDriver.listRefs.currentBranch", input.cwd).pipe( + Effect.orElseSucceed(() => repositoryPaths.currentBranch), + ), + ], + { concurrency: 2 }, ); - const hasCurrentWorktreeBranch = - repositoryPaths.worktreeRoot !== null && - snapshot.localBranches.some((ref) => ref.worktreePath === repositoryPaths.worktreeRoot); - const localBranches = snapshot.localBranches.map((ref) => ({ - ...ref, - worktreePath: - ref.worktreePath ?? - (ref.name === repositoryPaths.currentBranch ? repositoryPaths.worktreeRoot : null), - current: hasCurrentWorktreeBranch - ? ref.worktreePath === repositoryPaths.worktreeRoot - : ref.name === repositoryPaths.currentBranch, - })); + const { worktreeRoot } = repositoryPaths; + // The snapshot misses a branch created outside T3 Code until it refreshes. An unborn HEAD + // names a branch with no ref yet, so the ref must exist before it is listed. + const currentBranchUnlisted = + currentBranch !== null && !snapshot.localBranches.some((ref) => ref.name === currentBranch); + const unlistedCurrentBranch: ReadonlyArray = + currentBranchUnlisted && + (yield* branchExists(input.cwd, currentBranch).pipe(Effect.orElseSucceed(() => false))) + ? [ + { + name: currentBranch, + current: true, + isRemote: false, + isDefault: currentBranch === snapshot.defaultBranch, + worktreePath: worktreeRoot, + }, + ] + : []; + const listedBranches = snapshot.localBranches.map((ref) => { + const current = ref.name === currentBranch; + return { + ...ref, + worktreePath: + worktreeRoot === null + ? ref.worktreePath + : current + ? worktreeRoot + : ref.worktreePath === worktreeRoot + ? null + : ref.worktreePath, + current, + }; + }); + const localBranches = [...unlistedCurrentBranch, ...listedBranches]; const combinedBranches = input.includeMatchingRemoteRefs ? [...localBranches, ...snapshot.remoteBranches] : dedupeRemoteBranchesWithLocalMatches([...localBranches, ...snapshot.remoteBranches]); @@ -4435,6 +5008,40 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* yield* refresh; }); + // Git writes the lock reason during registration, before checkout or user hooks can run. + const releaseWorktreeRegistration = Effect.fn("releaseWorktreeRegistration")(function* ( + cwd: string, + reason: string, + remove: boolean, + ) { + const commonDir = yield* readGitCommonDir(cwd); + if (commonDir === null) return; + const registrations = path.join(commonDir, "worktrees"); + const entries = yield* fileSystem.readDirectory(registrations).pipe( + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + () => Effect.succeed([]), + ), + ); + for (const entry of entries) { + const adminDir = path.join(registrations, entry); + const locked = yield* fileSystem + .readFileString(path.join(adminDir, "locked")) + .pipe(Effect.orElseSucceed(() => "")); + if (locked.replace(/\r?\n$/, "") !== reason) continue; + // gitdir can be relative to the registration with worktree.useRelativePaths enabled. + const gitdir = yield* fileSystem.readFileString(path.join(adminDir, "gitdir")); + const registered = path.dirname(path.resolve(adminDir, gitdir.replace(/\r?\n$/, ""))); + yield* runGit("GitVcsDriver.createWorktree.unlock", cwd, [ + "worktree", + "unlock", + "--", + registered, + ]); + if (remove) yield* removeWorktree({ cwd, path: registered, force: true }); + } + }); + const createWorktree: GitVcsDriver.GitVcsDriver["Service"]["createWorktree"] = Effect.fn( "createWorktree", )(function* (input, options) { @@ -4467,10 +5074,32 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* if (Buffer.byteLength(prefix + character) > 23) break; prefix += character; } - const directoryName = `${prefix}-${Hex.encode(hash).slice(0, 16)}`; - worktreePath = path.join(parentDir, repoName, directoryName); + const basePath = path.join(parentDir, repoName, `${prefix}-${Hex.encode(hash).slice(0, 16)}`); + // The name is stable per branch, so a folder left by an earlier worktree must not block it. + worktreePath = basePath; + for ( + let suffix = 2; + yield* fileSystem.exists(worktreePath).pipe(Effect.orElseSucceed(() => false)); + suffix++ + ) { + worktreePath = `${basePath}-${suffix}`; + } } const newRefName = input.newRefName; + // Older Git still creates worktrees normally, but cannot prove ownership for cancellation. + const addHelp = newRefName + ? yield* executeGit( + "GitVcsDriver.createWorktree.capabilities", + input.cwd, + ["worktree", "add", "-h"], + { allowNonZeroExit: true }, + ) + : null; + const lockReason = + addHelp && `${addHelp.stdout}\n${addHelp.stderr}`.includes("--reason") + ? `t3-create-${yield* random.uuidV4()}` + : undefined; + if (newRefName) { // Kept apart from the checkout so the shared-config lock only covers the tracking write. yield* createBranchAt( @@ -4504,6 +5133,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* `checkout.workers=${checkoutWorkers}`, "worktree", "add", + ...(lockReason ? ["--lock", "--reason", lockReason] : []), "--", worktreePath, checkoutRef, @@ -4531,30 +5161,66 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* : {}), }, ); + // Set inside the claim's uninterruptible region: a pending cancel still surfaces as it ends. + let claimed = false; + const onWorktreeClaimed = progress?.onWorktreeClaimed; yield* ( fastCheckout ? checkoutCache - .create({ cwd: input.cwd, worktreePath, refName: checkoutRef, checkoutWorkers }) + .create({ + cwd: input.cwd, + worktreePath, + refName: checkoutRef, + checkoutWorkers, + lockReason, + }) .pipe(Effect.flatMap((cloned) => (cloned ? Effect.void : stockCheckout))) : stockCheckout ).pipe( - Effect.onError(() => - newRefName - ? deleteLocalBranch({ cwd: input.cwd, refName: newRefName, force: true }).pipe( + Effect.andThen( + fastCheckout ? checkoutCache.schedule(worktreePath, cacheGeneration) : Effect.void, + ), + Effect.andThen( + onWorktreeClaimed + ? Effect.uninterruptible( + onWorktreeClaimed(worktreePath).pipe( + Effect.andThen( + Effect.sync(() => { + claimed = true; + }), + ), + ), + ) + : Effect.void, + ), + Effect.onError((cause) => + newRefName && !claimed + ? // A failed hook keeps its worktree, as Git does; only a cancel abandons it. + (Cause.hasInterrupts(cause) + ? lockReason + ? releaseWorktreeRegistration(input.cwd, lockReason, true) + : Effect.void + : Effect.void + ).pipe( + Effect.andThen( + deleteLocalBranch({ cwd: input.cwd, refName: newRefName, force: true }), + ), + Effect.catch((cause) => + Effect.logWarning("could not clean up a failed worktree or its branch", { cause }), + ), + ) + : Effect.void, + ), + Effect.onExit(() => + lockReason + ? releaseWorktreeRegistration(input.cwd, lockReason, false).pipe( Effect.catch((cause) => - Effect.logWarning("could not remove the branch of a failed worktree", { cause }), + Effect.logWarning("could not unlock a created worktree", { cause }), ), ) : Effect.void, ), ); - if (fastCheckout) { - yield* checkoutCache.schedule(worktreePath, cacheGeneration); - } - - if (progress?.onWorktreeClaimed) { - yield* progress.onWorktreeClaimed(worktreePath); - } // worktree add leaves submodules empty, potentially omitting project skills, tools or source. const hasSubmodules = yield* fileSystem @@ -4589,6 +5255,61 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* yield* progress.onSubmodulesStarted(); } const onSubmoduleLine = progress?.onSubmoduleLine; + const updateSubmodules = (paths: readonly string[]) => + runGit( + "GitVcsDriver.createWorktree.updateSubmodules", + worktreePath, + [ + "submodule", + "update", + "--init", + ...(submoduleMode.value === "recursive" ? ["--recursive"] : []), + // Git reports transfer progress to a pipe only when forced. + ...(onSubmoduleLine ? ["--progress"] : []), + // A clone left by a failed update already has HEAD at the gitlink, so Git would skip it. + ...(paths.length > 0 ? ["--force", "--", ...paths] : []), + ], + { + // Cloning scales with remote size; let it finish or be cancelled, like worktree add. + timeoutMs: null, + ...(onSubmoduleLine + ? { + env: { LANGUAGE: "C" }, + progress: { onStdoutLine: onSubmoduleLine, onStderrLine: onSubmoduleLine }, + // Progress for many submodules can pass the output cap; truncate instead of failing. + appendTruncationMarker: true, + keepLineCallbacksAfterTruncation: true, + } + : {}), + }, + ); + // One failed clone stops Git before it checks out any submodule, leaving the cloned ones + // as empty working trees that read as modified; check those out on their own. + const checkOutClonedSubmodules = Effect.gen(function* () { + // Records are "submodule..path\n". + const submodulePaths = splitNullSeparatedPaths( + yield* runGitStdout("GitVcsDriver.createWorktree.listSubmodules", worktreePath, [ + "config", + "-z", + "--file", + ".gitmodules", + "--get-regexp", + "^submodule\\..*\\.path$", + ]), + false, + ).flatMap((record) => { + const newline = record.indexOf("\n"); + return newline === -1 ? [] : [record.slice(newline + 1)]; + }); + const clonedPaths = yield* Effect.filter(submodulePaths, (submodulePath) => + fileSystem + .exists(path.join(worktreePath, submodulePath, ".git")) + .pipe(Effect.orElseSucceed(() => false)), + ); + if (clonedPaths.length > 0) { + yield* updateSubmodules(clonedPaths); + } + }); // Registering submodule URLs writes the shared config; the clones that follow do not. yield* runGit( "GitVcsDriver.createWorktree.initSubmodules", @@ -4596,31 +5317,23 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ["submodule", "init"], { writesSharedConfig: true }, ).pipe( - Effect.andThen( - runGit( - "GitVcsDriver.createWorktree.updateSubmodules", - worktreePath, - submoduleMode.value === "recursive" - ? ["submodule", "update", "--init", "--recursive"] - : ["submodule", "update", "--init"], - { - // Cloning scales with remote size; let it finish or be cancelled, like worktree add. - timeoutMs: null, - ...(onSubmoduleLine - ? { - env: { LANGUAGE: "C" }, - progress: { onStdoutLine: onSubmoduleLine, onStderrLine: onSubmoduleLine }, - } - : {}), - }, - ), - ), + Effect.andThen(updateSubmodules([])), Effect.matchEffect({ onFailure: (cause) => Effect.logWarning("worktree submodule checkout failed; submodules may be incomplete", { worktreePath, cause, }).pipe( + Effect.andThen( + checkOutClonedSubmodules.pipe( + Effect.catch((retryCause) => + Effect.logWarning("could not check out the submodules that did clone", { + worktreePath, + cause: retryCause, + }), + ), + ), + ), Effect.andThen( progress?.onSubmodulesFinished ? progress.onSubmodulesFinished({ ok: false, detail: cause.message }) @@ -4773,7 +5486,8 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ? ["merge", "--ff-only", input.targetCommit] : ["reset", "--merge", input.targetCommit], { - timeoutMs: 30_000, + // Killing a slow checkout partway through strands a half-updated working tree. + timeoutMs: null, fallbackErrorDetail: "git failed to move the checkout onto the pull request head", }, ); @@ -4786,6 +5500,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const options = { env: STATUS_UPSTREAM_REFRESH_ENV, fallbackErrorDetail: `git fetch ${input.remoteName} failed`, + timeoutMs: FETCH_REMOTE_TIMEOUT_MS, }; // Another fetch in the same repository, such as the background status // poll, can win the race for a remote-tracking ref. The loser fails, but @@ -4911,7 +5626,14 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* remoteName: "origin", branchName: input.refName, }; - return remoteNames.includes(remoteName) ? { remoteName, branchName } : null; + // `/HEAD` is the remote's default-branch pointer, not a branch it can serve, and + // a revision expression such as `HEAD~1`, `@{-1}` or `:/text` is no branch name at all. + return remoteNames.includes(remoteName) && + branchName !== "HEAD" && + !branchName.startsWith("-") && + !/[\x00-\x20\x7f~^:?*[\\]|\.\.|@\{/.test(branchName) + ? { remoteName, branchName } + : null; }); const fetchWorktreeBaseCommit: GitVcsDriver.GitVcsDriver["Service"]["fetchWorktreeBaseCommit"] = @@ -4974,19 +5696,51 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ]); }); - const setBranchUpstream: GitVcsDriver.GitVcsDriver["Service"]["setBranchUpstream"] = (input) => - executeGitWithStableDiagnostics( - "GitVcsDriver.setBranchUpstream", - input.cwd, - [ + const setBranchUpstream: GitVcsDriver.GitVcsDriver["Service"]["setBranchUpstream"] = Effect.fn( + "setBranchUpstream", + )( + function* (input) { + const args = [ "branch", "--set-upstream-to", - `${input.remoteName}/${input.remoteBranch}`, + `refs/remotes/${input.remoteName}/${input.remoteBranch}`, "--", input.branch, - ], - { writesSharedConfig: true }, - ).pipe(retryWhileConfigLocked, Effect.asVoid); + ]; + const result = yield* executeGitWithStableDiagnostics( + "GitVcsDriver.setBranchUpstream", + input.cwd, + args, + { allowNonZeroExit: true }, + ); + if (result.exitCode === 0) return; + // Git requires a configured fetch mapping even when the tracking ref already exists. The + // mapping is a glob because an exact source fails every later fetch once the branch is deleted. + if ( + /^fatal: cannot set up tracking information; starting point '.+' is not a branch$/m.test( + result.stderr, + ) + ) { + yield* executeGitWithStableDiagnostics( + "GitVcsDriver.setBranchUpstream.addFetchMapping", + input.cwd, + [ + "config", + "--add", + `remote.${input.remoteName}.fetch`, + `+refs/heads/${input.remoteBranch}*:refs/remotes/${input.remoteName}/${input.remoteBranch}*`, + ], + ).pipe(retryWhileConfigLocked); + } + yield* executeGitWithStableDiagnostics( + "GitVcsDriver.setBranchUpstream", + input.cwd, + args, + ).pipe(retryWhileConfigLocked); + }, + // Keep mapping detection and both writes under the same lock across sibling worktrees. + (effect, input) => withSharedConfigLock(input.cwd, effect), + ); /** * Unlinks links under `target` without following them and removes the @@ -5107,22 +5861,28 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* if (input.oldBranch === input.newBranch) { return { branch: input.newBranch }; } - const targetBranch = input.exactName - ? input.newBranch - : yield* resolveAvailableBranchName(input.cwd, input.newBranch); - - yield* executeGit( - "GitVcsDriver.renameBranch", + // Parallel launches often generate the same name; picking it under the lock that serializes + // the renames keeps a second launch from choosing a name the first is about to take. + return yield* withSharedConfigLock( input.cwd, - ["branch", "-m", "--", input.oldBranch, targetBranch], - { - timeoutMs: 10_000, - fallbackErrorDetail: "git branch rename failed", - writesSharedConfig: true, - }, - ); + Effect.gen(function* () { + const targetBranch = input.exactName + ? input.newBranch + : yield* resolveAvailableBranchName(input.cwd, input.newBranch); + + yield* executeGit( + "GitVcsDriver.renameBranch", + input.cwd, + ["branch", "-m", "--", input.oldBranch, targetBranch], + { + timeoutMs: 10_000, + fallbackErrorDetail: "git branch rename failed", + }, + ); - return { branch: targetBranch }; + return { branch: targetBranch }; + }), + ); }); const switchRef: GitVcsDriver.GitVcsDriver["Service"]["switchRef"] = Effect.fn("switchRef")( @@ -5156,7 +5916,8 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ? yield* executeGit( "GitVcsDriver.switchRef.localTrackingBranch", input.cwd, - ["for-each-ref", "--format=%(refname:short)\t%(upstream:short)", "refs/heads"], + // `:short` prefixes a name a tag shares with `heads/`, so names are unambiguous here. + ["for-each-ref", "--format=%(refname:lstrip=2)\t%(upstream)", "refs/heads"], { timeoutMs: 5_000, allowNonZeroExit: true, @@ -5164,7 +5925,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ).pipe( Effect.map((result) => result.exitCode === 0 - ? parseTrackingBranchByUpstreamRef(result.stdout, input.refName) + ? parseTrackingBranchByUpstreamRef(result.stdout, `refs/remotes/${input.refName}`) : null, ), ) @@ -5191,30 +5952,42 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const checkoutArgs = localInputExists ? ["checkout", input.refName] - : remoteExists && !localTrackingBranch && localTrackedBranchTargetExists - ? ["checkout", input.refName] - : newTrackingBranch - ? ["checkout", newTrackingBranch] - : remoteExists && !localTrackingBranch - ? ["checkout", "--track", input.refName] - : remoteExists && localTrackingBranch - ? ["checkout", localTrackingBranch] - : ["checkout", input.refName]; + : newTrackingBranch + ? ["checkout", newTrackingBranch] + : remoteExists && !localTrackingBranch + ? ["checkout", "--track", input.refName] + : remoteExists && localTrackingBranch + ? ["checkout", localTrackingBranch] + : ["checkout", input.refName]; yield* refuseOptionLikeRefName("GitVcsDriver.switchRef", input.cwd, checkoutArgs.at(-1)); // Created before the checkout so the shared-config lock is not held while files are written. // Not rolled back on failure: Git may already have switched HEAD to it (post-checkout hooks). if (newTrackingBranch) { - yield* executeGit( + const startRef = `refs/remotes/${input.refName}`; + yield* executeGitWithStableDiagnostics( "GitVcsDriver.switchRef.createTrackingBranch", input.cwd, - ["branch", "--track", "--", newTrackingBranch, input.refName], - { - timeoutMs: 10_000, - fallbackErrorDetail: "git branch create failed", - writesSharedConfig: true, - }, + ["branch", "--no-track", "--", newTrackingBranch, startRef], + { fallbackErrorDetail: "git branch create failed" }, + ); + // An untracked branch of this name would make every later switch to the remote detach. + yield* executeGitWithStableDiagnostics( + "GitVcsDriver.switchRef.trackRemote", + input.cwd, + ["branch", `--set-upstream-to=${startRef}`, "--", newTrackingBranch], + { writesSharedConfig: true }, + ).pipe( + retryWhileConfigLocked, + Effect.tapError(() => + executeGit( + "GitVcsDriver.switchRef.removeUntrackedBranch", + input.cwd, + ["update-ref", "-d", `refs/heads/${newTrackingBranch}`, startRef], + { allowNonZeroExit: true }, + ), + ), ); } @@ -5351,6 +6124,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* statusDetails, statusDetailsLocal, statusDetailsRemote, + readRefSnapshot, prepareCommitContext, commit: Effect.fnUntraced(function* (cwd, subject, body, options) { return yield* withListRefsInvalidation(cwd, commit(cwd, subject, body, options)); @@ -5440,6 +6214,13 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* return yield* withListRefsInvalidation(input.cwd, switchRef(input)); }), initRepo: initRepoWithListRefsInvalidation, + invalidateRefs: Effect.fn("GitVcsDriver.invalidateRefs")(function* (cwd) { + // First, so a cached "not a repository" cannot end the ref invalidation early. + const cacheKey = normalizeRepositoryPathsCacheKey(cwd); + yield* Cache.invalidate(repositoryPathsRefreshCache, cacheKey); + yield* Cache.invalidate(repositoryPathsCache, cacheKey); + return yield* withListRefsInvalidation(cwd, Effect.void); + }), listLocalBranchNames, listWorktreePaths, }); diff --git a/apps/server/src/vcs/VcsAutomaticFetchInterval.ts b/apps/server/src/vcs/VcsAutomaticFetchInterval.ts new file mode 100644 index 000000000000..0f2c900b3ba1 --- /dev/null +++ b/apps/server/src/vcs/VcsAutomaticFetchInterval.ts @@ -0,0 +1,26 @@ +import * as Context from "effect/Context"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import { DEFAULT_AUTOMATIC_GIT_FETCH_INTERVAL } from "@t3tools/contracts"; +import { resolveServerBackgroundActivitySettings } from "@t3tools/shared/backgroundActivitySettings"; + +import * as ServerSettings from "../serverSettings.ts"; + +export class VcsAutomaticFetchInterval extends Context.Reference>( + "t3/vcs/VcsAutomaticFetchInterval", + { defaultValue: () => Effect.succeed(DEFAULT_AUTOMATIC_GIT_FETCH_INTERVAL) }, +) {} + +export const layer = Layer.effect( + VcsAutomaticFetchInterval, + Effect.gen(function* () { + const serverSettings = yield* ServerSettings.ServerSettingsService; + return serverSettings.getSettings.pipe( + Effect.map( + (settings) => resolveServerBackgroundActivitySettings(settings).automaticGitFetchInterval, + ), + Effect.orElseSucceed(() => DEFAULT_AUTOMATIC_GIT_FETCH_INTERVAL), + ); + }), +); diff --git a/apps/server/src/vcs/VcsDriver.ts b/apps/server/src/vcs/VcsDriver.ts index b382922c5ab2..383db4302543 100644 --- a/apps/server/src/vcs/VcsDriver.ts +++ b/apps/server/src/vcs/VcsDriver.ts @@ -14,6 +14,8 @@ import * as VcsProcess from "./VcsProcess.ts"; export interface VcsCaptureCheckpointInput { readonly cwd: string; readonly checkpointRef: CheckpointRef; + /** The checkpoint this one follows, so a turn that switched branches starts where it landed. */ + readonly previousCheckpointRef?: CheckpointRef; } export interface VcsRestoreCheckpointInput { diff --git a/apps/server/src/vcs/VcsProcess.test.ts b/apps/server/src/vcs/VcsProcess.test.ts index b87dbf5e7825..18eaebe6e095 100644 --- a/apps/server/src/vcs/VcsProcess.test.ts +++ b/apps/server/src/vcs/VcsProcess.test.ts @@ -92,7 +92,7 @@ describe("VcsProcess.run", () => { yield* fs.remove(lock); yield* git(restore); expect(yield* fs.readFileString(file)).toBe("before\n"); - }).pipe(provideLive, Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(provideLive, Effect.provide(NodeServices.layer)), ); it.effect.each([ diff --git a/apps/server/src/vcs/VcsProcess.ts b/apps/server/src/vcs/VcsProcess.ts index c379caf48847..da2f988ce1ed 100644 --- a/apps/server/src/vcs/VcsProcess.ts +++ b/apps/server/src/vcs/VcsProcess.ts @@ -33,6 +33,7 @@ export interface VcsProcessInput { readonly spawnCwd?: string; readonly stdin?: string; readonly onStdoutChunk?: (chunk: Uint8Array) => void; + readonly onSpawn?: () => void; readonly env?: NodeJS.ProcessEnv; readonly allowNonZeroExit?: boolean; readonly timeoutMs?: number; @@ -134,6 +135,29 @@ export const gitConfigEnv = ( export const explicitBareRepositoryEnv = (env: NodeJS.ProcessEnv, platform: NodeJS.Platform) => gitConfigEnv(env, "safe.bareRepository", "explicit", platform); +const REPOSITORY_SELECTORS = new Set([ + "GIT_DIR", + "GIT_WORK_TREE", + "GIT_COMMON_DIR", + "GIT_INDEX_FILE", + "GIT_OBJECT_DIRECTORY", + "GIT_ALTERNATE_OBJECT_DIRECTORIES", +]); + +/** + * Unsets repository selectors the server inherited, such as a Git hook's `GIT_DIR`, so Git finds + * the repository from cwd. Spread it before a command's own env, which may still set them. + */ +export const unsetInheritedRepositoryEnv = ( + inherited: NodeJS.ProcessEnv, + platform: NodeJS.Platform, +): NodeJS.ProcessEnv => + Object.fromEntries( + Object.keys(inherited) + .filter((name) => REPOSITORY_SELECTORS.has(platform === "win32" ? name.toUpperCase() : name)) + .map((name) => [name, undefined]), + ); + const classifyNonZeroExit = ( stderr: string, classifyFailure: VcsProcessInput["classifyFailure"], @@ -209,7 +233,11 @@ export const make = Effect.gen(function* () { ...(input.spawnCwd !== undefined ? { spawnCwd: input.spawnCwd } : {}), ...(input.stdin !== undefined ? { stdin: input.stdin } : {}), ...(input.onStdoutChunk !== undefined ? { onStdoutChunk: input.onStdoutChunk } : {}), + ...(input.onSpawn !== undefined ? { onSpawn: input.onSpawn } : {}), env: { + ...(input.command === "git" + ? unsetInheritedRepositoryEnv(process.env, hostPlatform) + : {}), // Failures are read by English patterns; without LANG, macOS Git follows the system language. // LANGUAGE leaves LC_CTYPE, which hooks inherit, alone. ...(input.command === "git" ? { LANGUAGE: "C" } : {}), diff --git a/apps/server/src/vcs/VcsStatusBroadcaster.test.ts b/apps/server/src/vcs/VcsStatusBroadcaster.test.ts index e70afb2ddde0..08fad4ae56af 100644 --- a/apps/server/src/vcs/VcsStatusBroadcaster.test.ts +++ b/apps/server/src/vcs/VcsStatusBroadcaster.test.ts @@ -28,6 +28,7 @@ import type { import { GitCommandError, GitManagerError } from "@t3tools/contracts"; import { applyGitStatusStreamEvent } from "@t3tools/shared/git"; +import * as VcsAutomaticFetchInterval from "./VcsAutomaticFetchInterval.ts"; import * as VcsStatusBroadcaster from "./VcsStatusBroadcaster.ts"; import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; import * as GitWorkflowService from "../git/GitWorkflowService.ts"; @@ -71,15 +72,18 @@ const remoteStatusWithPr: VcsStatusRemoteResult = { function layerTestFor(state: { currentLocalStatus: VcsStatusLocalResult; currentRemoteStatus: VcsStatusRemoteResult | null; + currentDivergence?: Pick; localStatusCalls: number; remoteStatusCalls: number; localInvalidationCalls: number; remoteInvalidationCalls: number; remoteStatusRefreshUpstreamValues?: Array; + localStatusBranchChanges?: Array; + refSnapshot?: string | null; remoteStatusError?: GitManagerError; backgroundWorkEnabled?: boolean; automaticFetchInterval?: Effect.Effect; - beforeRemoteStatus?: Effect.Effect; + beforeRemoteStatus?: Effect.Effect; beforeLocalStatus?: Effect.Effect; beforeCurrentBranch?: Effect.Effect; beforeInvalidate?: Effect.Effect; @@ -100,15 +104,19 @@ function layerTestFor(state: { Layer.provide(layerBackgroundPolicy(() => state.backgroundWorkEnabled !== false)), Layer.provide( Layer.succeed( - VcsStatusBroadcaster.VcsAutomaticFetchInterval, + VcsAutomaticFetchInterval.VcsAutomaticFetchInterval, state.automaticFetchInterval ?? Effect.succeed(Duration.seconds(30)), ), ), Layer.provide( Layer.mock(GitWorkflowService.GitWorkflowService)({ - localStatus: () => + localStatus: (_input, options) => Effect.suspend(() => { state.localStatusCalls += 1; + const includeBranchChanges = options?.includeBranchChanges !== false; + state.localStatusBranchChanges?.push(includeBranchChanges); + const { branchChanges: _branchChanges, ...withoutBranchChanges } = + state.currentLocalStatus; if (state.failLocalStatus) { return (state.onLocalStatusFailure ?? Effect.void).pipe( Effect.andThen( @@ -123,7 +131,7 @@ function layerTestFor(state: { ); } return (state.beforeLocalStatus ?? Effect.void).pipe( - Effect.as(state.currentLocalStatus), + Effect.as(includeBranchChanges ? state.currentLocalStatus : withoutBranchChanges), ); }), repositoryRoot: () => Effect.sync(() => (state.currentLocalStatus.isRepo ? "/repo" : null)), @@ -133,6 +141,18 @@ function layerTestFor(state: { : (state.beforeCurrentBranch ?? Effect.void).pipe( Effect.andThen(Effect.sync(() => state.currentLocalStatus.refName)), ), + readRefSnapshot: () => + Effect.sync(() => (state.refSnapshot === undefined ? "refs" : state.refSnapshot)), + upstreamDivergence: () => + Effect.sync(() => + state.currentRemoteStatus?.hasUpstream && state.currentLocalStatus.refName !== null + ? { + refName: state.currentLocalStatus.refName, + aheadCount: (state.currentDivergence ?? state.currentRemoteStatus).aheadCount, + behindCount: (state.currentDivergence ?? state.currentRemoteStatus).behindCount, + } + : null, + ), remoteStatus: (_input, options) => Effect.suspend(() => { return (state.beforeRemoteStatus ?? Effect.void).pipe( @@ -239,7 +259,7 @@ describe("VcsStatusBroadcaster", () => { yield* broadcaster.refreshLocalStatus("/repo"); assert.strictEqual(state.localStatusCalls, 3); - }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(layerTestFor(state))); }); it.effect( @@ -334,10 +354,84 @@ describe("VcsStatusBroadcaster", () => { ]); assert.deepEqual(yield* branches, [{ refName: null, isRepo: false }]); assert.equal(state.remoteStatusCalls, 0); - }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(layerTestFor(state))); }, ); + it.effect("publishes the new branch before pending detailed status completes", () => { + const state = newState(); + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const branches = yield* Stream.toPull(broadcaster.streamBranchStatus({ cwd: "/repo" })); + yield* branches; + const reading = yield* Deferred.make(); + const resume = yield* Deferred.make(); + state.beforeLocalStatus = Deferred.succeed(reading, undefined).pipe( + Effect.andThen(Deferred.await(resume)), + ); + const details = yield* Stream.runHead( + broadcaster.streamStatus({ cwd: "/repo", includeRemote: false }), + ).pipe(Effect.forkScoped); + yield* Deferred.await(reading); + state.currentLocalStatus = { ...baseLocalStatus, refName: "feature/next" }; + const update = yield* branches.pipe(Effect.forkScoped({ startImmediately: true })); + yield* broadcaster.refreshBranchStatus({ cwd: "/repo" }); + assert.deepEqual( + update.pollUnsafe(), + Exit.succeed([{ refName: "feature/next", isRepo: true }]), + ); + state.beforeLocalStatus = Effect.void; + yield* Deferred.succeed(resume, undefined); + yield* Fiber.join(details); + yield* broadcaster.getStatus({ cwd: "/repo" }); + state.currentLocalStatus = { ...baseLocalStatus, refName: "feature/latest" }; + yield* broadcaster.refreshBranchStatus({ cwd: "/repo" }); + assert.deepEqual(yield* branches, [{ refName: "feature/latest", isRepo: true }]); + }).pipe(Effect.provide(layerTestFor(state))); + }); + + it.effect("invalidates totals for a refresh arriving after detailed demand closes", () => { + const state = newState(); + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const detailScope = yield* Scope.make(); + const details = yield* Stream.toPull( + broadcaster.streamStatus({ cwd: "/repo", includeRemote: false }), + ).pipe(Scope.provide(detailScope)); + yield* details; + const reading = yield* Deferred.make(); + const resume = yield* Deferred.make(); + state.currentLocalStatus = { + ...baseLocalStatus, + refName: "feature/next", + workingTree: { files: [], insertions: 1, deletions: 0 }, + }; + state.beforeLocalStatus = Deferred.succeed(reading, undefined).pipe( + Effect.andThen(Deferred.await(resume)), + ); + yield* broadcaster.refreshBranchStatus({ cwd: "/repo" }); + yield* Deferred.await(reading); + yield* Scope.close(detailScope, Exit.void); + state.currentLocalStatus = { + ...state.currentLocalStatus, + workingTree: { files: [], insertions: 2, deletions: 0 }, + }; + const trailing = yield* broadcaster + .refreshBranchStatus({ cwd: "/repo" }) + .pipe(Effect.forkScoped({ startImmediately: true })); + state.beforeLocalStatus = Effect.void; + yield* Deferred.succeed(resume, undefined); + yield* Fiber.join(trailing); + const reopened = yield* Stream.runHead( + broadcaster.streamStatus({ cwd: "/repo", includeRemote: false }), + ); + assert.deepEqual( + reopened, + Option.some({ _tag: "snapshot" as const, local: state.currentLocalStatus, remote: null }), + ); + }).pipe(Effect.provide(layerTestFor(state))); + }); + it.effect("upgrades an in-flight branch refresh when detailed demand arrives", () => { const state: Parameters[0] = { currentLocalStatus: baseLocalStatus, @@ -371,7 +465,7 @@ describe("VcsStatusBroadcaster", () => { yield* Fiber.join(detailsFiber), Option.some({ _tag: "snapshot" as const, local: state.currentLocalStatus, remote: null }), ); - }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(layerTestFor(state))); }); it.effect("publishes terminal branch switches and validates cached status reads", () => { @@ -528,6 +622,48 @@ describe("VcsStatusBroadcaster", () => { }, ); + it.effect.each([0, 2])( + "refreshes merge divergence while a lookup ends with %s commits behind", + (behindAfterLookup) => { + const state: Parameters[0] = { + currentLocalStatus: baseLocalStatus, + currentRemoteStatus: { ...baseRemoteStatus, aheadCount: 2, behindCount: 3 }, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + backgroundWorkEnabled: false, + }; + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + yield* broadcaster.getStatus({ cwd: "/repo" }); + const started = yield* Deferred.make(); + const release = yield* Deferred.make(); + state.beforeRemoteStatus = Deferred.succeed(started, undefined).pipe( + Effect.andThen(Deferred.await(release)), + ); + const lookup = yield* broadcaster.refreshPullRequestStatus("/repo").pipe(Effect.forkScoped); + yield* Deferred.await(started); + state.currentRemoteStatus = { ...baseRemoteStatus, aheadCount: 4, behindCount: 0 }; + yield* broadcaster.refreshLocalStatus("/repo"); + const status = yield* broadcaster.getStatus({ cwd: "/repo" }); + assert.equal(status.aheadCount, 4); + assert.equal(status.behindCount, 0); + assert.equal(state.remoteStatusCalls, 1); + state.currentDivergence = { aheadCount: 4, behindCount: behindAfterLookup }; + state.currentRemoteStatus = + behindAfterLookup > 0 + ? { ...baseRemoteStatus, ...state.currentDivergence } + : { ...baseRemoteStatus, aheadCount: 2, behindCount: 3 }; + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(lookup); + const afterLookup = yield* broadcaster.getStatus({ cwd: "/repo" }); + assert.equal(afterLookup.aheadCount, 4); + assert.equal(afterLookup.behindCount, behindAfterLookup); + }).pipe(Effect.provide(layerTestFor(state))); + }, + ); + it.effect("a local-only refresh after a branch switch leaves the PR refresh able to read", () => { const state: Parameters[0] = { currentLocalStatus: baseLocalStatus, @@ -620,6 +756,7 @@ describe("VcsStatusBroadcaster", () => { Layer.mock(GitWorkflowService.GitWorkflowService)({ localStatus: () => Effect.succeed(localStatus), currentBranch: () => Effect.succeed(localStatus.refName), + readRefSnapshot: () => Effect.succeed("refs"), remoteStatus: () => Effect.succeed(remoteStatus), invalidateLocalStatus: () => Effect.void, invalidateRemoteStatus: () => Effect.void, @@ -716,6 +853,7 @@ describe("VcsStatusBroadcaster", () => { Layer.mock(GitWorkflowService.GitWorkflowService)({ localStatus: () => Effect.succeed(baseLocalStatus), currentBranch: () => Effect.succeed(baseLocalStatus.refName), + readRefSnapshot: () => Effect.succeed("refs"), remoteStatus: () => Effect.gen(function* () { remoteReads += 1; @@ -747,7 +885,7 @@ describe("VcsStatusBroadcaster", () => { assert.deepStrictEqual(refreshed, remoteStatusWithPr); const final = yield* broadcaster.getStatus({ cwd: "/repo" }); assert.deepStrictEqual(final.pr, remoteStatusWithPr.pr); - }).pipe(Effect.provide(layer), Effect.scoped); + }).pipe(Effect.provide(layer)); }); it.effect("an initial status read cannot overwrite an explicit refresh", () => { @@ -762,6 +900,7 @@ describe("VcsStatusBroadcaster", () => { Layer.mock(GitWorkflowService.GitWorkflowService)({ localStatus: () => Effect.succeed(baseLocalStatus), currentBranch: () => Effect.succeed(baseLocalStatus.refName), + readRefSnapshot: () => Effect.succeed("refs"), remoteStatus: () => Effect.gen(function* () { remoteReads += 1; @@ -791,7 +930,7 @@ describe("VcsStatusBroadcaster", () => { (yield* broadcaster.getStatus({ cwd: "/repo" })).pr, remoteStatusWithPr.pr, ); - }).pipe(Effect.provide(layer), Effect.scoped); + }).pipe(Effect.provide(layer)); }); it.effect("keeps the published local snapshot when a refresh remote read fails", () => { @@ -815,6 +954,7 @@ describe("VcsStatusBroadcaster", () => { return state.currentLocalStatus; }), currentBranch: () => Effect.succeed(state.currentLocalStatus.refName), + readRefSnapshot: () => Effect.succeed("refs"), remoteStatus: () => Effect.suspend(() => { state.remoteStatusCalls += 1; @@ -941,6 +1081,49 @@ describe("VcsStatusBroadcaster", () => { }, ); + it.effect("publishes a checkout's branch before returning, ahead of the remote read", () => { + const state: Parameters[0] = { + currentLocalStatus: baseLocalStatus, + currentRemoteStatus: baseRemoteStatus, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + }; + + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + yield* broadcaster.getStatus({ cwd: "/repo" }); + const subscribed = yield* Deferred.make(); + const localPublished = yield* Deferred.make(); + const remotePublished = yield* Deferred.make(); + yield* broadcaster.streamStatus({ cwd: "/repo", includeRemote: false }).pipe( + Stream.runForEach((event) => { + if (event._tag === "snapshot") return Deferred.succeed(subscribed, undefined); + if (event._tag === "localUpdated") return Deferred.succeed(localPublished, event.local); + return event.remote === null + ? Effect.void + : Deferred.succeed(remotePublished, event.remote); + }), + Effect.forkScoped, + ); + yield* Deferred.await(subscribed); + + const releaseRemote = yield* Deferred.make(); + state.beforeRemoteStatus = Deferred.await(releaseRemote); + state.currentLocalStatus = { ...baseLocalStatus, refName: "main" }; + state.currentRemoteStatus = remoteStatusWithPr; + yield* broadcaster.refreshAfterCheckout("/repo"); + + assert.equal((yield* Deferred.await(localPublished)).refName, "main"); + assert.isFalse(yield* Deferred.isDone(remotePublished)); + + yield* Deferred.succeed(releaseRemote, undefined); + assert.deepStrictEqual(yield* Deferred.await(remotePublished), remoteStatusWithPr); + assert.equal((yield* broadcaster.getStatus({ cwd: "/repo" })).pr?.number, 2978); + }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }); + it.effect("a remote poll refreshes Changes totals when divergence counts stay the same", () => { const state: Parameters[0] = { currentLocalStatus: { @@ -977,6 +1160,116 @@ describe("VcsStatusBroadcaster", () => { yield* Deferred.await(polled); const status = yield* broadcaster.getStatus({ cwd: "/repo" }); assert.deepStrictEqual(status.branchChanges, state.currentLocalStatus.branchChanges); + }).pipe(Effect.provide(layerTestFor(state))); + }); + + it.effect("reads Changes totals only while a subscriber shows them", () => { + const branchChanges = { baseRef: "origin/main", insertions: 3, deletions: 1 }; + const state: Parameters[0] = { + currentLocalStatus: { ...baseLocalStatus, branchChanges }, + currentRemoteStatus: baseRemoteStatus, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + localStatusBranchChanges: [], + }; + + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const paletteEvents = yield* Queue.unbounded(); + yield* broadcaster.streamStatus({ cwd: "/repo", includeRemote: false }).pipe( + Stream.runForEach((event) => Queue.offer(paletteEvents, event)), + Effect.forkScoped, + ); + const paletteSnapshot = yield* Queue.take(paletteEvents); + assert.strictEqual(paletteSnapshot._tag, "snapshot"); + if (paletteSnapshot._tag === "snapshot") { + assert.isUndefined(paletteSnapshot.local.branchChanges); + } + yield* broadcaster.refreshLocalStatus("/repo"); + assert.deepStrictEqual(state.localStatusBranchChanges, [false, false]); + + const headerScope = yield* Scope.make(); + const headerEvents = yield* Queue.unbounded(); + yield* broadcaster.streamStatus({ cwd: "/repo" }).pipe( + Stream.runForEach((event) => Queue.offer(headerEvents, event)), + Effect.forkIn(headerScope), + ); + const headerSnapshot = yield* Queue.take(headerEvents); + assert.strictEqual(headerSnapshot._tag, "snapshot"); + if (headerSnapshot._tag === "snapshot") { + assert.deepStrictEqual(headerSnapshot.local.branchChanges, branchChanges); + } + const paletteUpdate = yield* Queue.take(paletteEvents); + assert.strictEqual(paletteUpdate._tag, "localUpdated"); + if (paletteUpdate._tag === "localUpdated") { + assert.deepStrictEqual(paletteUpdate.local.branchChanges, branchChanges); + } + // The header's first remote poll also rereads local status. + while ((yield* Queue.take(headerEvents))._tag !== "remoteUpdated") {} + const turnEnd = yield* broadcaster.refreshLocalStatus("/repo"); + assert.deepStrictEqual(turnEnd.branchChanges, branchChanges); + + yield* Scope.close(headerScope, Exit.void); + yield* broadcaster.refreshLocalStatus("/repo"); + assert.deepStrictEqual(state.localStatusBranchChanges, [ + false, + false, + true, + true, + true, + false, + ]); + }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }); + + it.effect("a light refresh that a header joins mid-read publishes Changes totals", () => { + const branchChanges = { baseRef: "origin/main", insertions: 3, deletions: 1 }; + const state: Parameters[0] = { + currentLocalStatus: { ...baseLocalStatus, branchChanges }, + currentRemoteStatus: baseRemoteStatus, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + localStatusBranchChanges: [], + fileSystem: FileSystem.makeNoop({ realPath: (cwd) => Effect.succeed(cwd) }), + }; + + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const subscribeHeader = Effect.fn("subscribeHeader")(function* (scope: Scope.Scope) { + const events = yield* Queue.unbounded(); + yield* broadcaster.streamStatus({ cwd: "/repo" }).pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkIn(scope, { startImmediately: true }), + ); + return events; + }); + const firstHeader = yield* Scope.make(); + const firstEvents = yield* subscribeHeader(firstHeader); + while ((yield* Queue.take(firstEvents))._tag !== "remoteUpdated") {} + yield* Scope.close(firstHeader, Exit.void); + + const readStarted = yield* Deferred.make(); + const finishRead = yield* Deferred.make(); + state.beforeLocalStatus = Deferred.succeed(readStarted, undefined).pipe( + Effect.andThen(Deferred.await(finishRead)), + ); + const turnEnd = yield* broadcaster.refreshLocalStatus("/repo").pipe(Effect.forkScoped); + yield* Deferred.await(readStarted); + // Starting immediately registers the header's demand before it waits behind the read. + const secondEvents = yield* subscribeHeader(yield* Scope.make()); + yield* Deferred.succeed(finishRead, undefined); + const snapshot = yield* Queue.take(secondEvents); + assert.strictEqual(snapshot._tag, "snapshot"); + if (snapshot._tag === "snapshot") { + assert.deepStrictEqual(snapshot.local.branchChanges, branchChanges); + } + + assert.deepStrictEqual((yield* Fiber.join(turnEnd)).branchChanges, branchChanges); + assert.deepStrictEqual(state.localStatusBranchChanges?.slice(-2), [false, true]); }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); }); @@ -989,6 +1282,7 @@ describe("VcsStatusBroadcaster", () => { branchChanges: { baseRef: "origin/main", insertions: 745, deletions: 24 }, }; let cachedLocal: VcsStatusLocalResult | undefined; + let refSnapshot = "before"; const readLocalStatus = () => Effect.sync(() => (cachedLocal ??= local)); const layerTest = VcsStatusBroadcaster.layer.pipe( Layer.provideMerge(NodeServices.layer), @@ -1006,6 +1300,7 @@ describe("VcsStatusBroadcaster", () => { }), currentBranch: () => Effect.succeed(baseLocalStatus.refName), invalidateRemoteStatus: () => Effect.void, + readRefSnapshot: () => Effect.sync(() => refSnapshot), remoteStatus: () => Deferred.succeed(fetchStarted, undefined).pipe( Effect.andThen(Deferred.await(finishFetch)), @@ -1024,6 +1319,7 @@ describe("VcsStatusBroadcaster", () => { ...baseLocalStatus, branchChanges: { baseRef: "origin/main", insertions: 0, deletions: 0 }, }; + refSnapshot = "after"; yield* Deferred.succeed(finishFetch, undefined); const status = yield* Fiber.join(refresh); assert.deepStrictEqual(status.branchChanges, local.branchChanges); @@ -1031,6 +1327,59 @@ describe("VcsStatusBroadcaster", () => { }); }); + it.effect("an explicit refresh re-reads local totals only after refs moved", () => { + const totals = (insertions: number): VcsStatusLocalResult => ({ + ...baseLocalStatus, + branchChanges: { baseRef: "origin/main", insertions, deletions: 0 }, + }); + const state: Parameters[0] = { + currentLocalStatus: totals(745), + currentRemoteStatus: remoteStatusWithPr, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + }; + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const subscribed = yield* Deferred.make(); + const events = yield* broadcaster.streamStatus({ cwd: "/repo", includeRemote: false }).pipe( + Stream.tap((event) => + event._tag === "snapshot" ? Deferred.succeed(subscribed, undefined) : Effect.void, + ), + Stream.filter((event) => event._tag !== "snapshot"), + Stream.take(4), + Stream.runCollect, + Effect.forkScoped, + ); + yield* Deferred.await(subscribed); + + state.localStatusCalls = 0; + yield* broadcaster.refreshStatus("/repo"); + assert.equal(state.localStatusCalls, 1); + + state.localStatusCalls = 0; + state.currentRemoteStatus = baseRemoteStatus; + state.beforeRemoteStatus = Effect.sync(() => { + state.currentLocalStatus = totals(0); + state.refSnapshot = "moved"; + }); + const moved = yield* broadcaster.refreshStatus("/repo"); + assert.equal(state.localStatusCalls, 2); + assert.deepStrictEqual(moved.branchChanges, totals(0).branchChanges); + assert.deepStrictEqual( + [...(yield* Fiber.join(events))].map((event) => event._tag), + // The first refresh adds the totals this palette-style subscription skipped. + ["localUpdated", "remoteUpdated", "remoteUpdated", "localUpdated"], + ); + + state.localStatusCalls = 0; + state.refSnapshot = null; + yield* broadcaster.refreshStatus("/repo"); + assert.equal(state.localStatusCalls, 2); + }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }); + it.effect("publishes local status while an earlier lookup holds the remote lock", () => { const state: Parameters[0] = { currentLocalStatus: baseLocalStatus, @@ -1077,7 +1426,7 @@ describe("VcsStatusBroadcaster", () => { const status = yield* Fiber.join(refresh); assert.isTrue(status.hasWorkingTreeChanges); assert.deepStrictEqual(status.pr, remoteStatusWithPr.pr); - }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(layerTestFor(state))); }); it.effect("publishes local changes before a remote error and propagates that error", () => { @@ -1139,7 +1488,7 @@ describe("VcsStatusBroadcaster", () => { assert.isTrue(cached.hasWorkingTreeChanges); assert.deepStrictEqual(cached.pr, remoteStatusWithPr.pr); assert.deepStrictEqual(yield* Deferred.await(published), state.currentLocalStatus); - }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(layerTestFor(state))); }); it.effect.each(["getStatus", "refreshStatus", "autoPull", "autoPullFailure"] as const)( @@ -1225,7 +1574,7 @@ describe("VcsStatusBroadcaster", () => { assert.isNull(status.pr); assert.equal(status.aheadCount, 0); assert.equal(status.behindCount, 0); - }).pipe(Effect.provide(layerTest), Effect.scoped); + }).pipe(Effect.provide(layerTest)); }, ); @@ -1275,7 +1624,7 @@ describe("VcsStatusBroadcaster", () => { assert.deepStrictEqual(cached.pr, state.currentRemoteStatus.pr); assert.equal(cached.aheadCount, 2); assert.equal(state.remoteStatusCalls, 2); - }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(layerTestFor(state))); }); it.effect("a terminal checkout publishes the new branch without waiting for a poll", () => @@ -1307,7 +1656,10 @@ describe("VcsStatusBroadcaster", () => { const next = applyGitStatusStreamEvent(clientStates.at(-1) ?? null, event); clientStates.push(next); yield* Deferred.succeed(subscribed, undefined); - if (next.refName === "repro") yield* Deferred.succeed(checkedOut, undefined); + // The refresh ends with the new branch's remote, after its local update. + if (event._tag === "remoteUpdated" && event.remote && next.refName === "repro") { + yield* Deferred.succeed(checkedOut, undefined); + } }), ).pipe(Effect.forkScoped); yield* Deferred.await(subscribed); @@ -1322,12 +1674,12 @@ describe("VcsStatusBroadcaster", () => { assert.deepStrictEqual( clientStates.filter((status) => status.refName === "repro").map((status) => status.pr), - [null], + [null, null], ); // A checkout must not trigger a network fetch. assert.deepStrictEqual(state.remoteStatusRefreshUpstreamValues.slice(1), [false]); }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); it.effect("a terminal checkout updates branch-only subscribers without full status", () => @@ -1360,18 +1712,58 @@ describe("VcsStatusBroadcaster", () => { assert.deepEqual(yield* branches, [{ refName: "repro", isRepo: true }]); assert.equal(state.localStatusCalls, 0); }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); - it.effect("a checkout read that fails while index.lock is held retries until it succeeds", () => + it.effect("a terminal checkout updates the branch before pending detailed status completes", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const gitDir = yield* fs.makeTempDirectoryScoped({ prefix: "vcs-head-watch-" }); const headPath = path.join(gitDir, "HEAD"); yield* fs.writeFileString(headPath, "ref: refs/heads/main\n"); + const state: Parameters[0] = { + currentLocalStatus: { ...baseLocalStatus, refName: "main" }, + currentRemoteStatus: null, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + headPath, + }; + + yield* Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const branches = yield* Stream.toPull(broadcaster.streamBranchStatus({ cwd: "/repo" })); + yield* branches; + const details = yield* Stream.toPull( + broadcaster.streamStatus({ cwd: "/repo", includeRemote: false }), + ); + yield* details; + + const resume = yield* Deferred.make(); + state.beforeLocalStatus = Deferred.await(resume); + state.currentLocalStatus = { ...baseLocalStatus, refName: "repro" }; + const lockPath = path.join(gitDir, "HEAD.lock"); + yield* fs.writeFileString(lockPath, "ref: refs/heads/repro\n"); + yield* fs.rename(lockPath, headPath); + + assert.deepEqual(yield* branches, [{ refName: "repro", isRepo: true }]); + state.currentLocalStatus = { ...baseLocalStatus, refName: "second" }; + yield* fs.writeFileString(lockPath, "ref: refs/heads/second\n"); + yield* fs.rename(lockPath, headPath); + assert.deepEqual(yield* branches, [{ refName: "second", isRepo: true }]); + yield* Deferred.succeed(resume, undefined); + }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("a checkout read that fails while index.lock is held retries until it succeeds", () => + Effect.gen(function* () { + const events = yield* Queue.unbounded(); + const headPath = "/repo/.git/HEAD"; const failed = yield* Deferred.make(); - const state = { + const state: Parameters[0] = { currentLocalStatus: { ...baseLocalStatus, isDefaultRef: true, refName: "main" }, currentRemoteStatus: null, localStatusCalls: 0, @@ -1379,6 +1771,12 @@ describe("VcsStatusBroadcaster", () => { localInvalidationCalls: 0, remoteInvalidationCalls: 0, headPath, + fileSystem: FileSystem.makeNoop({ + realPath: (target) => Effect.succeed(target), + exists: () => Effect.succeed(true), + watch: (_target, options) => + Stream.fromQueue(events).pipe(Stream.onStart(Effect.sync(() => options?.onReady?.()))), + }), failLocalStatus: false, onLocalStatusFailure: Deferred.succeed(failed, undefined).pipe(Effect.asVoid), }; @@ -1402,9 +1800,7 @@ describe("VcsStatusBroadcaster", () => { // A terminal commit holds index.lock across the checkout, so the HEAD event's read fails. state.failLocalStatus = true; state.currentLocalStatus = { ...baseLocalStatus, isDefaultRef: false, refName: "repro" }; - const lockPath = path.join(gitDir, "HEAD.lock"); - yield* fs.writeFileString(lockPath, "ref: refs/heads/repro\n"); - yield* fs.rename(lockPath, headPath); + yield* Queue.offer(events, { _tag: "Update", path: "HEAD" }); yield* Deferred.await(failed); const failedCalls = state.localStatusCalls; @@ -1414,7 +1810,7 @@ describe("VcsStatusBroadcaster", () => { yield* Deferred.await(checkedOut); assert.strictEqual(state.localStatusCalls, failedCalls + 1); }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }), ); it.effect("subscribers share one HEAD watch that ends with the last of them", () => @@ -1456,7 +1852,7 @@ describe("VcsStatusBroadcaster", () => { yield* subscribe; assert.strictEqual(state.headPathCalls, 2); }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); it.effect("a HEAD watch that ended restarts for the next subscriber and keeps earlier ones", () => @@ -1524,7 +1920,51 @@ describe("VcsStatusBroadcaster", () => { yield* checkout("two"); yield* Deferred.await(first.sawRef("two")); }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("a HEAD watch ends when its git directory is removed", () => + Effect.gen(function* () { + const gitDir = "/repo/.git/worktrees/feature"; + const events = yield* Queue.unbounded(); + const closed = yield* Deferred.make(); + let gitDirExists = true; + const state: Parameters[0] = { + currentLocalStatus: { ...baseLocalStatus, refName: "main" }, + currentRemoteStatus: null, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + headPath: `${gitDir}/HEAD`, + fileSystem: FileSystem.makeNoop({ + realPath: (target) => Effect.succeed(target), + exists: (target) => Effect.succeed(target !== gitDir || gitDirExists), + watch: (_target, options) => + Stream.fromQueue(events).pipe( + Stream.onStart(Effect.sync(() => options?.onReady?.())), + Stream.ensuring(Deferred.succeed(closed, undefined)), + ), + }), + }; + + yield* Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const branches = yield* Stream.toPull(broadcaster.streamBranchStatus({ cwd: "/repo" })); + assert.deepEqual(yield* branches, [{ refName: "main", isRepo: true }]); + + // Every checkout removes HEAD.lock, which leaves the watch running. + state.currentLocalStatus = { ...baseLocalStatus, refName: "other" }; + yield* Queue.offer(events, { _tag: "Remove", path: "HEAD.lock" }); + yield* Queue.offer(events, { _tag: "Update", path: "HEAD" }); + assert.deepEqual(yield* branches, [{ refName: "other", isRepo: true }]); + + // `git worktree remove` deletes the watched directory; Windows keeps reporting it. + gitDirExists = false; + yield* Queue.offer(events, { _tag: "Remove", path: gitDir }); + yield* Deferred.await(closed); + }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }), ); it.effect("releases remote demand while passive observers remain", () => { @@ -1623,6 +2063,92 @@ describe("VcsStatusBroadcaster", () => { assert.isTrue(Option.isSome(yield* Deferred.poll(remoteInterrupted))); }).pipe(Effect.provide(layerTest)); }); + + it.effect("refreshes requested during a refresh share one follow-up refresh", () => { + const state: Parameters[0] = { + currentLocalStatus: baseLocalStatus, + currentRemoteStatus: baseRemoteStatus, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + }; + + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const fetchStarted = yield* Deferred.make(); + const finishFetch = yield* Deferred.make(); + state.beforeRemoteStatus = Deferred.succeed(fetchStarted, undefined).pipe( + Effect.andThen(Deferred.await(finishFetch)), + ); + const first = yield* broadcaster.refreshStatus("/repo").pipe(Effect.forkScoped); + yield* Deferred.await(fetchStarted); + // Every read after the first sees a PR, so a caller that reused the + // first refresh would get a status from before its own request. + state.beforeRemoteStatus = Effect.sync(() => { + state.currentRemoteStatus = remoteStatusWithPr; + }); + const later = yield* Effect.all( + [broadcaster.refreshStatus("/repo"), broadcaster.refreshStatus("/repo")], + { concurrency: "unbounded" }, + ).pipe(Effect.forkScoped({ startImmediately: true })); + yield* Deferred.succeed(finishFetch, undefined); + + assert.isNull((yield* Fiber.join(first)).pr); + const [second, third] = yield* Fiber.join(later); + assert.deepStrictEqual(second.pr, remoteStatusWithPr.pr); + assert.deepStrictEqual(third.pr, remoteStatusWithPr.pr); + assert.equal(state.remoteStatusCalls, 2); + + yield* broadcaster.refreshStatus("/repo"); + assert.equal(state.remoteStatusCalls, 3); + }).pipe(Effect.provide(layerTestFor(state))); + }); + + it.effect("a failed refresh releases interrupted waiters and runs the shared follow-up", () => { + const state: Parameters[0] = { + currentLocalStatus: baseLocalStatus, + currentRemoteStatus: remoteStatusWithPr, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + }; + + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const started = yield* Deferred.make(); + const release = yield* Deferred.make(); + state.beforeRemoteStatus = Deferred.succeed(started, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.andThen( + Effect.fail( + new GitManagerError({ + operation: "VcsStatusBroadcaster.test", + cwd: "/repo", + detail: "remote status failed", + }), + ), + ), + ); + const first = yield* broadcaster.refreshStatus("/repo").pipe(Effect.exit, Effect.forkScoped); + yield* Deferred.await(started); + const interrupted = yield* broadcaster + .refreshStatus("/repo") + .pipe(Effect.forkScoped({ startImmediately: true })); + const remaining = yield* broadcaster + .refreshStatus("/repo") + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Fiber.interrupt(interrupted); + state.beforeRemoteStatus = Effect.void; + yield* Deferred.succeed(release, undefined); + assert.isTrue(Exit.isFailure(yield* Fiber.join(first))); + assert.deepStrictEqual((yield* Fiber.join(remaining)).pr, remoteStatusWithPr.pr); + assert.equal(state.remoteStatusCalls, 1); + yield* broadcaster.refreshStatus("/repo"); + assert.equal(state.remoteStatusCalls, 2); + }).pipe(Effect.provide(layerTestFor(state))); + }); }); const newState = (): Parameters[0] => ({ @@ -1664,7 +2190,7 @@ it.effect("detail join during invalidation must get fresh totals", () => { yield* Fiber.join(snapshotFiber), Option.some({ _tag: "snapshot" as const, local: state.currentLocalStatus, remote: null }), ); - }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(layerTestFor(state))); }); it.effect("serializes branch reads with detailed refresh publication", () => { const state = newState(); @@ -1687,10 +2213,12 @@ it.effect("serializes branch reads with detailed refresh publication", () => { yield* Fiber.join(older); yield* Fiber.join(newer); state.branchRead = () => Effect.succeed("new"); - const result = yield* b.getStatus({ cwd: "/repo" }); - assert.equal(result.refName, "new"); + const result = yield* b + .streamStatus({ cwd: "/repo", includeRemote: false }) + .pipe(Stream.runHead, Effect.flatMap(Effect.fromOption)); + assert.equal(result._tag === "snapshot" && result.local.refName, "new"); assert.equal(state.localStatusCalls, 1); - }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(layerTestFor(state))); }); it.effect("publishes overlapping branch reads in checkout order", () => { @@ -1721,7 +2249,7 @@ it.effect("publishes overlapping branch reads in checkout order", () => { { refName: "old", isRepo: true }, { refName: "new", isRepo: true }, ]); - }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(layerTestFor(state))); }); it.effect("does not replay older queued branches after a new subscriber snapshot", () => { @@ -1754,5 +2282,5 @@ it.effect("does not replay older queued branches after a new subscriber snapshot yield* b.refreshBranchStatus({ cwd: "/repo" }); assert.deepEqual(yield* Queue.take(values), { refName: "latest", isRepo: true }); yield* Fiber.join(subscriber); - }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(layerTestFor(state))); }); diff --git a/apps/server/src/vcs/VcsStatusBroadcaster.ts b/apps/server/src/vcs/VcsStatusBroadcaster.ts index ffe685ef60d5..b691b9d2978a 100644 --- a/apps/server/src/vcs/VcsStatusBroadcaster.ts +++ b/apps/server/src/vcs/VcsStatusBroadcaster.ts @@ -25,7 +25,6 @@ import type { VcsStatusSubscriptionInput, } from "@t3tools/contracts"; import { DEFAULT_AUTOMATIC_GIT_FETCH_INTERVAL } from "@t3tools/contracts"; -import { resolveServerBackgroundActivitySettings } from "@t3tools/shared/backgroundActivitySettings"; import { mergeGitStatusParts } from "@t3tools/shared/git"; import * as KeyedLock from "@t3tools/shared/KeyedLock"; import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; @@ -34,6 +33,7 @@ import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; import * as GitWorkflowService from "../git/GitWorkflowService.ts"; import * as ProjectStore from "../orchestration-v2/ProjectStore.ts"; import * as ServerSettings from "../serverSettings.ts"; +import { VcsAutomaticFetchInterval } from "./VcsAutomaticFetchInterval.ts"; const VCS_STATUS_REFRESH_FAILURE_BASE_DELAY = Duration.seconds(30); const VCS_STATUS_REFRESH_FAILURE_MAX_DELAY = Duration.minutes(15); @@ -143,8 +143,12 @@ interface CachedRemoteStatus extends CachedValue { readonly branch: string | null; } +interface CachedLocalStatus extends CachedValue { + readonly includesBranchChanges: boolean; +} + interface CachedVcsStatus { - readonly local: CachedValue | null; + readonly local: CachedLocalStatus | null; readonly remote: CachedRemoteStatus | null; } @@ -160,24 +164,6 @@ interface ActiveRemotePoller { readonly demandCwds: Ref.Ref>; } -export class VcsAutomaticFetchInterval extends Context.Reference>( - "t3/vcs/VcsAutomaticFetchInterval", - { defaultValue: () => Effect.succeed(DEFAULT_AUTOMATIC_GIT_FETCH_INTERVAL) }, -) {} - -export const layerAutomaticFetchInterval = Layer.effect( - VcsAutomaticFetchInterval, - Effect.gen(function* () { - const serverSettings = yield* ServerSettings.ServerSettingsService; - return serverSettings.getSettings.pipe( - Effect.map( - (settings) => resolveServerBackgroundActivitySettings(settings).automaticGitFetchInterval, - ), - Effect.orElseSucceed(() => DEFAULT_AUTOMATIC_GIT_FETCH_INTERVAL), - ); - }), -); - export class VcsAutoPullPolicy extends Context.Reference<{ readonly isEnabled: (cwd: string) => Effect.Effect; }>("t3/vcs/VcsAutoPullPolicy", { @@ -236,6 +222,11 @@ export class VcsStatusBroadcaster extends Context.Service< readonly refreshLocalStatus: ( cwd: string, ) => Effect.Effect; + /** + * For a checkout someone is waiting on. Returns once the new branch is published, so the + * caller's reply does not outrun it, and finishes the slower remote read in the background. + */ + readonly refreshAfterCheckout: (cwd: string) => Effect.Effect; readonly refreshStatus: ( cwd: string, options?: { readonly localOnly?: boolean }, @@ -286,17 +277,29 @@ export const make = Effect.gen(function* () { const detailedSubscribers = new Map(); const branches = new Map< string, - { readonly refName: string | null; readonly isRepo: boolean; readonly revision: number } + { + readonly refName: string | null; + readonly isRepo: boolean; + readonly revision: number; + readonly observation: number; + } >(); let branchRevision = 0; + // A slow detailed read must not replace a newer HEAD-only observation. + let branchObservation = 0; const publishBranch = Effect.fn("VcsStatusBroadcaster.publishBranch")(function* ( cwd: string, { refName, isRepo }: { readonly refName: string | null; readonly isRepo: boolean }, + observation: number, ) { const previous = branches.get(cwd); - if (previous?.refName === refName && previous.isRepo === isRepo) return; + if (previous && previous.observation > observation) return; + if (previous?.refName === refName && previous.isRepo === isRepo) { + branches.set(cwd, { ...previous, observation }); + return; + } const revision = ++branchRevision; - branches.set(cwd, { refName, isRepo, revision }); + branches.set(cwd, { refName, isRepo, revision, observation }); yield* PubSub.publish(branchChangesPubSub, { cwd, refName, isRepo, revision }); }); const localRefreshLocks = yield* KeyedLock.make(); @@ -316,9 +319,51 @@ export const make = Effect.gen(function* () { return yield* Ref.get(cacheRef).pipe(Effect.map((cache) => cache.get(cwd) ?? null)); }); + // Only full subscribers and direct reads show Changes totals. Command palette rows subscribe + // without remote status and skip them, which halves the git processes a burst of rows spawns. + const branchChangesDemand = new Map(); + const wantsBranchChanges = (cwd: string) => branchChangesDemand.has(cwd); + const acquireBranchChangesDemand = (cwd: string) => + Effect.acquireRelease( + Effect.sync(() => { + branchChangesDemand.set(cwd, (branchChangesDemand.get(cwd) ?? 0) + 1); + }), + () => + Effect.sync(() => { + const count = (branchChangesDemand.get(cwd) ?? 1) - 1; + if (count > 0) branchChangesDemand.set(cwd, count); + else branchChangesDemand.delete(cwd); + }), + ); + + const readLocalStatus = Effect.fn("VcsStatusBroadcaster.readLocalStatus")(function* ( + cwd: string, + ) { + const includeBranchChanges = wantsBranchChanges(cwd); + const local = yield* workflow.localStatus({ cwd }, { includeBranchChanges }); + // A subscriber that shows totals arrived during the read and must not receive it without them. + if (!includeBranchChanges && wantsBranchChanges(cwd)) { + return { + local: yield* workflow.localStatus({ cwd }, { includeBranchChanges: true }), + includesBranchChanges: true, + }; + } + return { local, includesBranchChanges: includeBranchChanges }; + }); + const updateCachedLocalStatus = Effect.fn("VcsStatusBroadcaster.updateCachedLocalStatus")( - function* (cwd: string, local: VcsStatusLocalResult, options?: { publish?: boolean }) { - const nextLocal = { fingerprint: fingerprintStatusPart(local), value: local }; + function* ( + cwd: string, + read: { readonly local: VcsStatusLocalResult; readonly includesBranchChanges: boolean }, + observation: number, + options?: { publish?: boolean }, + ) { + const { local, includesBranchChanges } = read; + const nextLocal = { + fingerprint: fingerprintStatusPart(local), + value: local, + includesBranchChanges, + }; const [localChanged, remoteCleared] = yield* Ref.modify(cacheRef, (cache) => { const previous = cache.get(cwd) ?? { local: null, remote: null }; const remote = previous.remote?.branch === local.refName ? previous.remote : null; @@ -327,7 +372,7 @@ export const make = Effect.gen(function* () { new Map(cache).set(cwd, { local: nextLocal, remote }), ] as const; }); - yield* publishBranch(cwd, local); + yield* publishBranch(cwd, local, observation); if (options?.publish && remoteCleared) { yield* PubSub.publish(changesPubSub, { cwd, @@ -366,21 +411,22 @@ export const make = Effect.gen(function* () { const loadLocalStatus = Effect.fn("VcsStatusBroadcaster.loadLocalStatus")(function* ( cwd: string, + options?: { publish?: boolean }, ) { - const local = yield* workflow.localStatus({ cwd }); - return yield* updateCachedLocalStatus(cwd, local); + const observation = ++branchObservation; + return yield* updateCachedLocalStatus(cwd, yield* readLocalStatus(cwd), observation, options); }); const getOrLoadLocalStatusUnlocked = Effect.fn("VcsStatusBroadcaster.getOrLoadLocalStatus")( function* (cwd: string) { const cached = yield* getCachedStatus(cwd); if (cached?.local) { - const branch = yield* workflow.currentBranch(cwd); - if (branch === cached.local.value.refName) return cached.local.value; + if (cached.local.includesBranchChanges || !wantsBranchChanges(cwd)) { + const branch = yield* workflow.currentBranch(cwd); + if (branch === cached.local.value.refName) return cached.local.value; + } yield* workflow.invalidateLocalStatus(cwd); - return yield* updateCachedLocalStatus(cwd, yield* workflow.localStatus({ cwd }), { - publish: true, - }); + return yield* loadLocalStatus(cwd, { publish: true }); } return yield* loadLocalStatus(cwd); }, @@ -403,6 +449,7 @@ export const make = Effect.gen(function* () { "VcsStatusBroadcaster.getStatus", )(function* (input) { const cwd = yield* withFileSystem(normalizeCwd(input.cwd)); + yield* acquireBranchChangesDemand(cwd); yield* getOrLoadLocalStatus(cwd); const cached = yield* getCachedStatus(cwd); if (cached?.local && cached.remote) { @@ -419,14 +466,18 @@ export const make = Effect.gen(function* () { return mergeGitStatusParts(parts.local, parts.remote); }), ); - }); + }, Effect.scoped); // A refresh that started after a request saw every change that request could mean, so queued // callers share it instead of each running git status again. const localRefreshRequests = new Map(); const localRefreshResults = new Map< string, - { readonly startedAt: number; readonly local: VcsStatusLocalResult } + { + readonly startedAt: number; + readonly local: VcsStatusLocalResult; + readonly includesBranchChanges: boolean; + } >(); const refreshLocalStatusCore = Effect.fn("VcsStatusBroadcaster.refreshLocalStatusCore")( function* (cwd: string) { @@ -436,12 +487,19 @@ export const make = Effect.gen(function* () { cwd, Effect.gen(function* () { const shared = localRefreshResults.get(cwd); - if (shared !== undefined && shared.startedAt >= requestedAt) return shared.local; + if ( + shared !== undefined && + shared.startedAt >= requestedAt && + (shared.includesBranchChanges || !wantsBranchChanges(cwd)) + ) { + return shared.local; + } const startedAt = localRefreshRequests.get(cwd) ?? requestedAt; yield* workflow.invalidateLocalStatus(cwd); - const local = yield* workflow.localStatus({ cwd }); - localRefreshResults.set(cwd, { startedAt, local }); - return yield* updateCachedLocalStatus(cwd, local, { publish: true }); + const observation = ++branchObservation; + const read = yield* readLocalStatus(cwd); + localRefreshResults.set(cwd, { startedAt, ...read }); + return yield* updateCachedLocalStatus(cwd, read, observation, { publish: true }); }), ); }, @@ -451,7 +509,30 @@ export const make = Effect.gen(function* () { "VcsStatusBroadcaster.refreshLocalStatus", )(function* (rawCwd) { const cwd = yield* withFileSystem(normalizeCwd(rawCwd)); - return yield* refreshLocalStatusCore(cwd); + const local = yield* refreshLocalStatusCore(cwd); + const cached = yield* getCachedStatus(cwd); + // Divergence is local Git data; a slow forge lookup must not delay turn-end updates. + if (cached?.remote?.value?.hasUpstream) { + const divergence = yield* workflow.upstreamDivergence(cwd); + const latest = yield* getCachedStatus(cwd); + if (divergence && latest?.remote?.value && latest.remote.branch === divergence.refName) { + const value = { + ...latest.remote.value, + aheadCount: divergence.aheadCount, + behindCount: divergence.behindCount, + }; + yield* updateCachedRemoteStatus( + cwd, + { + branch: divergence.refName, + value, + fingerprint: fingerprintStatusPart(value), + }, + { publish: true }, + ); + } + } + return local; }); const readRemoteStatus = Effect.fn("VcsStatusBroadcaster.readRemoteStatus")(function* ( @@ -461,7 +542,18 @@ export const make = Effect.gen(function* () { const branch = yield* workflow.currentBranch(cwd); // GitManager's remote TTL cache is keyed by cwd, not the checked-out branch. yield* workflow.invalidateRemoteStatus(cwd); - const value = yield* workflow.remoteStatus({ cwd }, options); + const previousRemote = (yield* getCachedStatus(cwd))?.remote; + const remote = yield* workflow.remoteStatus({ cwd }, options); + const latest = yield* getCachedStatus(cwd); + // An overlapping turn refresh and fetch can both move divergence during the forge lookup. + const refreshed = + remote?.hasUpstream && latest?.remote !== previousRemote && latest?.remote?.branch === branch + ? yield* workflow.upstreamDivergence(cwd) + : null; + const value = + remote && refreshed?.refName === branch + ? { ...remote, aheadCount: refreshed.aheadCount, behindCount: refreshed.behindCount } + : remote; const currentBranch = yield* workflow.currentBranch(cwd); const cached = yield* getCachedStatus(cwd); if (branch !== currentBranch || cached?.local?.value.refName !== currentBranch) { @@ -551,33 +643,63 @@ export const make = Effect.gen(function* () { ); }); + const refreshRemoteAndLocalTotals = Effect.fn("VcsStatusBroadcaster.refreshRemoteAndLocalTotals")( + function* (cwd: string, policyCwd: string, refsBeforeLocalRead: string | null) { + return yield* withRemoteWriteLock( + cwd, + Effect.gen(function* () { + const refreshUpstream = !Duration.isZero(yield* automaticFetchInterval); + // Bump the PR lookup epoch after earlier remote writers have finished. + yield* workflow.invalidateStatus(cwd); + const remoteRead = yield* readRemoteStatus(cwd, { refreshUpstream }); + // Without a fetch, the cached behind count cannot justify an automatic pull. + const pulled = refreshUpstream + ? yield* maybeAutoPull(cwd, remoteRead, [policyCwd]) + : null; + if (pulled !== null) return mergeGitStatusParts(pulled.local, pulled.remote); + yield* updateCachedRemoteStatus(cwd, remoteRead, { publish: true }); + // A fetch, ours or not, can move the base used for local Changes totals. + if ( + refsBeforeLocalRead === null || + (yield* workflow.readRefSnapshot(cwd)) !== refsBeforeLocalRead + ) { + yield* refreshLocalStatusCore(cwd); + } + const parts = yield* getStatusParts(cwd); + return mergeGitStatusParts(parts.local, parts.remote); + }), + ); + }, + ); + const refreshStatus: VcsStatusBroadcaster["Service"]["refreshStatus"] = Effect.fn( "VcsStatusBroadcaster.refreshStatus", )(function* (rawCwd, options) { const cwd = yield* withFileSystem(normalizeCwd(rawCwd)); + yield* acquireBranchChangesDemand(cwd); + const refsBeforeLocalRead = options?.localOnly ? null : yield* workflow.readRefSnapshot(cwd); yield* refreshLocalStatusCore(cwd); if (options?.localOnly) { // Never cache a remote here: readers take `value: null` as "read and found no PR". const parts = yield* getStatusParts(cwd); return mergeGitStatusParts(parts.local, parts.remote); } - return yield* withRemoteWriteLock( - cwd, - Effect.gen(function* () { - const refreshUpstream = !Duration.isZero(yield* automaticFetchInterval); - // Bump the PR lookup epoch after earlier remote writers have finished. - yield* workflow.invalidateStatus(cwd); - const remoteRead = yield* readRemoteStatus(cwd, { refreshUpstream }); - // Fetch can move the base used for local Changes totals. - yield* refreshLocalStatusCore(cwd); - // Without a fetch, the cached behind count cannot justify an automatic pull. - const pulled = refreshUpstream ? yield* maybeAutoPull(cwd, remoteRead, [rawCwd]) : null; - if (pulled !== null) return mergeGitStatusParts(pulled.local, pulled.remote); - yield* updateCachedRemoteStatus(cwd, remoteRead, { publish: true }); - const parts = yield* getStatusParts(cwd); - return mergeGitStatusParts(parts.local, parts.remote); - }), + return yield* refreshRemoteAndLocalTotals(cwd, rawCwd, refsBeforeLocalRead); + }, Effect.scoped); + + const refreshAfterCheckout: VcsStatusBroadcaster["Service"]["refreshAfterCheckout"] = Effect.fn( + "VcsStatusBroadcaster.refreshAfterCheckout", + )(function* (rawCwd) { + const cwd = yield* withFileSystem(normalizeCwd(rawCwd)); + const localDone = yield* Deferred.make(); + // Owned by the broadcaster so a client disconnecting mid-switch still leaves status correct. + yield* refreshLocalStatusCore(cwd).pipe( + Effect.ensuring(Deferred.succeed(localDone, undefined)), + Effect.andThen(refreshRemoteAndLocalTotals(cwd, rawCwd, null)), + Effect.ignoreCause({ log: true }), + Effect.forkIn(broadcasterScope), ); + yield* Deferred.await(localDone); }); const refreshPullRequestStatus: VcsStatusBroadcaster["Service"]["refreshPullRequestStatus"] = @@ -765,7 +887,66 @@ export const make = Effect.gen(function* () { } }); - // Serialize branch publication and cache invalidation with detailed reads and refreshes. + // A request made during a refresh needs a read that starts after it, so overlapping requests + // share one follow-up. Keyed by raw cwd because auto-pull policy depends on it. + // Missing means idle; null means running with no follow-up queued. + type SharedRefresh = Deferred.Deferred; + const queuedRefreshesRef = yield* Ref.make(new Map()); + + const runQueuedRefreshes = Effect.fn("VcsStatusBroadcaster.runQueuedRefreshes")(function* ( + rawCwd: string, + first: SharedRefresh, + ) { + let next: SharedRefresh | null = first; + while (next !== null) { + yield* Deferred.into(refreshStatus(rawCwd), next); + next = yield* Ref.modify(queuedRefreshesRef, (queued) => { + const followUp = queued.get(rawCwd) ?? null; + const nextQueued = new Map(queued); + if (followUp === null) { + nextQueued.delete(rawCwd); + } else { + nextQueued.set(rawCwd, null); + } + return [followUp, nextQueued] as const; + }); + } + }); + + const shareQueuedRefreshStatus: VcsStatusBroadcaster["Service"]["refreshStatus"] = Effect.fn( + "VcsStatusBroadcaster.shareQueuedRefreshStatus", + )(function* (rawCwd, options) { + if (options?.localOnly) return yield* refreshStatus(rawCwd, options); + const requested = yield* Deferred.make(); + // Registering and forking together keeps an interrupt from leaving an entry with no worker. + const shared = yield* Effect.uninterruptible( + Effect.gen(function* () { + const followUp = yield* Ref.modify( + queuedRefreshesRef, + (queued): readonly [SharedRefresh | undefined, Map] => { + if (!queued.has(rawCwd)) return [undefined, new Map(queued).set(rawCwd, null)]; + const followUp = queued.get(rawCwd) ?? requested; + return [followUp, new Map(queued).set(rawCwd, followUp)]; + }, + ); + if (followUp !== undefined) return followUp; + // The broadcaster scope owns the work, so one caller's interrupt does not cancel it for the rest. + yield* runQueuedRefreshes(rawCwd, requested).pipe(Effect.forkIn(broadcasterScope)); + return requested; + }), + ); + return yield* Deferred.await(shared); + }); + + const readHeadBranch = Effect.fn("VcsStatusBroadcaster.readHeadBranch")(function* (cwd: string) { + const observation = ++branchObservation; + const refName = yield* workflow.currentBranch(cwd); + // A null branch is a detached HEAD or a checkout that is gone, such as a removed worktree. + const isRepo = refName !== null || (yield* workflow.repositoryRoot(cwd)) !== null; + return { observation, refName, isRepo }; + }); + + // Keep cache invalidation serialized with detailed reads and refreshes. const refreshBranchStatusCore = Effect.fn("VcsStatusBroadcaster.refreshBranchStatusCore")( (cwd: string) => localRefreshLocks.withLock( @@ -773,15 +954,11 @@ export const make = Effect.gen(function* () { Effect.gen(function* () { if (detailedSubscribers.has(cwd)) { const local = yield* getOrLoadLocalStatusUnlocked(cwd); - return { - refName: local.refName, - isRepo: local.isRepo, - revision: branches.get(cwd)?.revision ?? 0, - }; + return ( + branches.get(cwd) ?? { refName: local.refName, isRepo: local.isRepo, revision: 0 } + ); } - const refName = yield* workflow.currentBranch(cwd); - // A null branch is a detached HEAD or a checkout that is gone, such as a removed worktree. - const isRepo = refName !== null || (yield* workflow.repositoryRoot(cwd)) !== null; + const { observation, refName, isRepo } = yield* readHeadBranch(cwd); // HEAD cannot certify working-tree totals, even when it has not moved. localRefreshResults.delete(cwd); yield* workflow.invalidateLocalStatus(cwd); @@ -790,58 +967,73 @@ export const make = Effect.gen(function* () { return previous ? new Map(cache).set(cwd, { ...previous, local: null }) : cache; }); if (detailedSubscribers.has(cwd)) { - const local = yield* workflow.localStatus({ cwd }); - yield* updateCachedLocalStatus(cwd, local, { publish: true }); - return { - refName: local.refName, - isRepo: local.isRepo, - revision: branches.get(cwd)?.revision ?? 0, - }; + const local = yield* loadLocalStatus(cwd, { publish: true }); + return ( + branches.get(cwd) ?? { refName: local.refName, isRepo: local.isRepo, revision: 0 } + ); } - yield* publishBranch(cwd, { refName, isRepo }); - return { refName, isRepo, revision: branches.get(cwd)?.revision ?? 0 }; + yield* publishBranch(cwd, { refName, isRepo }, observation); + return branches.get(cwd) ?? { refName, isRepo, revision: 0 }; }), ), ); // A checkout outside T3 Code, such as `git switch` in the terminal, calls no refresh RPC. // Rereading remote too, without a fetch, sends the new branch with its own PR and counts. - const refreshCheckedOutRef = Effect.fn("VcsStatusBroadcaster.refreshCheckedOutRef")(function* ( + const publishCheckedOutRef = Effect.fn("VcsStatusBroadcaster.publishCheckedOutRef")(function* ( cwd: string, ) { - if (!detailedSubscribers.has(cwd)) { - yield* refreshBranchStatusCore(cwd); - } else if ((yield* getCachedStatus(cwd))?.remote) { - yield* refreshRemoteStatus(cwd, { refreshUpstream: false }); - } else { - yield* refreshLocalStatusCore(cwd); - } + if (!detailedSubscribers.has(cwd)) return yield* refreshBranchStatusCore(cwd); + const { observation, refName, isRepo } = yield* readHeadBranch(cwd); + yield* publishBranch(cwd, { refName, isRepo }, observation); }); + const reconcileCheckedOutRef = Effect.fn("VcsStatusBroadcaster.reconcileCheckedOutRef")( + function* (cwd: string) { + if (!detailedSubscribers.has(cwd)) return; + if ((yield* getCachedStatus(cwd))?.remote) { + yield* refreshRemoteStatus(cwd, { refreshUpstream: false }); + } else { + yield* refreshLocalStatusCore(cwd); + } + }, + ); + + const retryCheckedOutRef = (cwd: string, refresh: Effect.Effect) => + refresh.pipe( + Effect.retry(CHECKOUT_REFRESH_RETRY), + Effect.catchCause((cause) => + Effect.logWarning("VCS checked-out ref refresh failed", { + cwdLength: cwd.length, + ...remoteRefreshFailureDiagnostics(cause), + }), + ), + ); + // Runs for the subscription lifetime, so tracing would leave a span open indefinitely. const watchHead = Effect.fnUntraced(function* (cwd: string, ready: Deferred.Deferred) { const headPath = yield* workflow.headPath(cwd); if (headPath === null) return; + const headDirectory = path.dirname(headPath); // Git replaces HEAD by renaming HEAD.lock, which a watch on the file itself would lose. yield* fs - .watch(path.dirname(headPath), { + .watch(headDirectory, { onReady: () => Deferred.doneUnsafe(ready, Exit.void), }) .pipe( + // On Windows a watch outlives its deleted directory and reports it ~150k times a second. + Stream.takeUntilEffect((event) => + event._tag === "Remove" + ? fs.exists(headDirectory).pipe(Effect.map((exists) => !exists)) + : Effect.succeed(false), + ), Stream.filter((event) => path.basename(event.path) === "HEAD"), // A rebase moves HEAD many times; refresh once for whatever arrived during a refresh. Stream.buffer({ capacity: 1, strategy: "sliding" }), - Stream.runForEach(() => - refreshCheckedOutRef(cwd).pipe( - Effect.retry(CHECKOUT_REFRESH_RETRY), - Effect.catchCause((cause) => - Effect.logWarning("VCS checked-out ref refresh failed", { - cwdLength: cwd.length, - ...remoteRefreshFailureDiagnostics(cause), - }), - ), - ), - ), + Stream.mapEffect(() => retryCheckedOutRef(cwd, publishCheckedOutRef(cwd))), + // Detailed status waits behind its own reads; the branch keeps up on the stage above. + Stream.buffer({ capacity: 1, strategy: "sliding" }), + Stream.runForEach(() => retryCheckedOutRef(cwd, reconcileCheckedOutRef(cwd))), ); }); @@ -917,6 +1109,7 @@ export const make = Effect.gen(function* () { else detailedSubscribers.set(cwd, remaining); }), ); + if (input.includeRemote !== false) yield* acquireBranchChangesDemand(cwd); const subscription = yield* PubSub.subscribe(changesPubSub); // Watch before reading so a checkout in between still publishes. yield* Effect.acquireRelease(retainHeadWatcher(cwd), () => releaseHeadWatcher(cwd)); @@ -952,8 +1145,21 @@ export const make = Effect.gen(function* () { const refreshBranchStatus: VcsStatusBroadcaster["Service"]["refreshBranchStatus"] = Effect.fn( "VcsStatusBroadcaster.refreshBranchStatus", )(function* (input) { - const { refName, isRepo } = yield* refreshBranchStatusCore( - yield* withFileSystem(normalizeCwd(input.cwd)), + const cwd = yield* withFileSystem(normalizeCwd(input.cwd)); + if (!detailedSubscribers.has(cwd)) { + const { refName, isRepo } = yield* refreshBranchStatusCore(cwd); + return { refName, isRepo }; + } + const { observation, refName, isRepo } = yield* readHeadBranch(cwd); + yield* publishBranch(cwd, { refName, isRepo }, observation); + yield* refreshBranchStatusCore(cwd).pipe( + Effect.catchCause((cause) => + Effect.logWarning("VCS branch refresh failed", { + cwdLength: cwd.length, + ...remoteRefreshFailureDiagnostics(cause), + }), + ), + Effect.forkIn(broadcasterScope), ); return { refName, isRepo }; }); @@ -987,7 +1193,8 @@ export const make = Effect.gen(function* () { ), getStatus, refreshLocalStatus, - refreshStatus, + refreshAfterCheckout, + refreshStatus: shareQueuedRefreshStatus, refreshPullRequestStatus, streamStatus, }); diff --git a/apps/server/src/vcs/worktreeCheckoutCache.ts b/apps/server/src/vcs/worktreeCheckoutCache.ts index a72d6d859fc5..8a51189fe6d6 100644 --- a/apps/server/src/vcs/worktreeCheckoutCache.ts +++ b/apps/server/src/vcs/worktreeCheckoutCache.ts @@ -511,6 +511,7 @@ export const makeWorktreeCheckoutCache = Effect.fn("WorktreeCheckoutCache.make") readonly worktreePath: string; readonly refName: string; readonly checkoutWorkers: string; + readonly lockReason?: string | undefined; }) { if (!(yield* gitSupported)) return false; const worktreePath = path.resolve(input.cwd, input.worktreePath); @@ -613,13 +614,14 @@ export const makeWorktreeCheckoutCache = Effect.fn("WorktreeCheckoutCache.make") if (moved === null || (yield* fileSystem.exists(target.indexHook))) return null; return target; }); - const rollback = git("rollback", input.cwd, [ - "worktree", - "remove", - "--force", - "--", - worktreePath, - ]).pipe( + const rollback = ( + input.lockReason + ? git("unlockRollback", input.cwd, ["worktree", "unlock", "--", worktreePath]) + : Effect.void + ).pipe( + Effect.andThen( + git("rollback", input.cwd, ["worktree", "remove", "--force", "--", worktreePath]), + ), Effect.flatMap((removed) => removed === null ? Effect.logWarning("could not remove a partially created worktree", { worktreePath }) @@ -633,6 +635,7 @@ export const makeWorktreeCheckoutCache = Effect.fn("WorktreeCheckoutCache.make") "worktree", "add", "--no-checkout", + ...(input.lockReason ? ["--lock", "--reason", input.lockReason] : []), "--", worktreePath, input.refName, diff --git a/apps/server/src/vcs/worktreeRemoval.ts b/apps/server/src/vcs/worktreeRemoval.ts index f1140cdcc3a4..b48994873b28 100644 --- a/apps/server/src/vcs/worktreeRemoval.ts +++ b/apps/server/src/vcs/worktreeRemoval.ts @@ -9,6 +9,42 @@ import type * as GitVcsDriver from "./GitVcsDriver.ts"; const INDEX_LIST_MAX_BYTES = 64 * 1024 * 1024; +/** + * git-annex replaces a linked worktree's `.git` file with a symlink to its admin + * directory, so its symlinks into `.git/annex` resolve. Git then refuses to remove it. + */ +const annexedAdminDir = Effect.fn("WorktreeRemoval.annexedAdminDir")( + function* (worktreePath: string) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const adminDir = path.resolve( + worktreePath, + yield* fileSystem.readLink(path.join(worktreePath, ".git")), + ); + const pointsBackTo = path.dirname( + path.resolve( + adminDir, + (yield* fileSystem.readFileString(path.join(adminDir, "gitdir"))).trim(), + ), + ); + return (yield* fileSystem.realPath(pointsBackTo)) === (yield* fileSystem.realPath(worktreePath)) + ? Option.some(adminDir) + : Option.none(); + }, + Effect.orElseSucceed(() => Option.none()), +); + +export const isLinkedWorktree = Effect.fn("WorktreeRemoval.isLinkedWorktree")(function* ( + worktreePath: string, +) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + return ( + (yield* fileSystem.stat(path.join(worktreePath, ".git"))).type === "File" || + Option.isSome(yield* annexedAdminDir(worktreePath)) + ); +}); + /** * Git for Windows before 2.54 follows NTFS junctions; newer Git leaves them behind. * On Windows, mirror Git's removal checks, then delete with Node's `rm`, which @@ -33,6 +69,39 @@ export const removeLinkedWorktree = Effect.fn("WorktreeRemoval.removeLinkedWorkt // Git refuses to remove a worktree with untracked files unless forced, but // its check honors `status.showUntrackedFiles=no` and would delete them. const statusConfig = ["-c", "status.showUntrackedFiles=normal"]; + // Git's own refusals, worded the way it prints them. + const refuse = (message: string, cause?: unknown) => + new GitCommandError({ + operation: input.operation, + command: "git worktree remove", + cwd: input.cwd, + detail: `fatal: ${message}`, + ...(cause === undefined ? {} : { cause }), + }); + const annexed = yield* annexedAdminDir(worktreePath); + if (Option.isSome(annexed)) { + const adminDir = annexed.value; + const writeGitFile = fileSystem.writeFileString(gitFile, `gitdir: ${adminDir}\n`); + // A kept worktree's annexed files resolve only through the symlink; a gitdir + // file is the fallback when it cannot be recreated. + yield* Effect.addFinalizer(() => + fileSystem.exists(worktreePath).pipe( + Effect.flatMap((kept) => + kept + ? fileSystem.remove(gitFile, { force: true }).pipe( + Effect.andThen(fileSystem.symlink(adminDir, gitFile)), + Effect.catch(() => writeGitFile), + ) + : Effect.void, + ), + Effect.ignore, + ), + ); + yield* fileSystem.remove(gitFile).pipe( + Effect.andThen(writeGitFile), + Effect.mapError((cause) => refuse(`failed to replace '${gitFile}'`, cause)), + ); + } const linked = (yield* HostProcess.isWindows) && (yield* fileSystem.stat(gitFile).pipe( @@ -50,15 +119,6 @@ export const removeLinkedWorktree = Effect.fn("WorktreeRemoval.removeLinkedWorkt ]); } - // Git's own refusals, worded the way it prints them. - const refuse = (message: string, cause?: unknown) => - new GitCommandError({ - operation: input.operation, - command: "git worktree remove", - cwd: input.cwd, - detail: `fatal: ${message}`, - ...(cause === undefined ? {} : { cause }), - }); const gitOutput = (cwd: string, args: ReadonlyArray, env?: NodeJS.ProcessEnv) => input .execute({ @@ -153,4 +213,4 @@ export const removeLinkedWorktree = Effect.fn("WorktreeRemoval.removeLinkedWorkt ); // The folder is gone, so Git only drops the worktree's metadata. return yield* input.run(["worktree", "remove", "--force", "--force", "--", worktreePath]); -}); +}, Effect.scoped); diff --git a/apps/server/src/workspace/IgnorePatterns.test.ts b/apps/server/src/workspace/IgnorePatterns.test.ts new file mode 100644 index 000000000000..f10345566eca --- /dev/null +++ b/apps/server/src/workspace/IgnorePatterns.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "@effect/vitest"; + +import { parseIgnoreFile } from "./IgnorePatterns.ts"; + +const decide = (contents: string, path: string, isFolder = false) => + parseIgnoreFile(contents).decide(path, isFolder); + +describe("parseIgnoreFile", () => { + it("lets the last matching pattern decide, including negations", () => { + expect(decide("*keep*\n!*keep*\n", "sub/keep-link.ts")).toBe("included"); + expect(decide("!*keep*\n*keep*\n", "sub/keep-link.ts")).toBe("ignored"); + expect(decide("# comment\n\nother\n", "keep.ts")).toBeUndefined(); + }); + + it("anchors patterns that contain a slash and matches bare names at any depth", () => { + expect(decide("link", "a/b/link")).toBe("ignored"); + expect(decide("/link", "a/link")).toBeUndefined(); + expect(decide("/link", "link")).toBe("ignored"); + expect(decide("a/link", "x/a/link")).toBeUndefined(); + expect(decide("a/*.ts", "a/b/c.ts")).toBeUndefined(); + expect(decide("link", "blink")).toBeUndefined(); + expect(decide("link", "a/blink")).toBeUndefined(); + }); + + it("supports globstars, classes, escapes, and folder-only patterns", () => { + expect(decide("**/gen/*.ts", "gen/a.ts")).toBe("ignored"); + expect(decide("a/**/b", "a/x/y/b")).toBe("ignored"); + expect(decide("a/**/b", "a/b")).toBe("ignored"); + expect(decide("foo/**/bar", "foo/xbar")).toBeUndefined(); + expect(decide("foo/**/bar", "foo/x/ybar")).toBeUndefined(); + expect(decide("out/**", "out/x/y")).toBe("ignored"); + expect(decide("link-[0-9]", "link-7")).toBe("ignored"); + expect(decide("link-[!0-9]", "link-7")).toBeUndefined(); + expect(decide("link-?", "link-/")).toBeUndefined(); + expect(decide("\\!bang\n\\#hash\nstar\\*", "!bang")).toBe("ignored"); + expect(decide("\\!bang\n\\#hash\nstar\\*", "#hash")).toBe("ignored"); + expect(decide("\\!bang\n\\#hash\nstar\\*", "starx")).toBeUndefined(); + expect(decide("dist/", "dist")).toBeUndefined(); + expect(decide("dist/", "dist", true)).toBe("ignored"); + }); + + it("matches adversarial patterns in linear time", () => { + // A backtracking regex takes seconds to reject this pair; the vitest timeout catches that. + const pattern = `${"a*".repeat(20)}b\n`; + expect(decide(pattern, `${"a".repeat(200)}c`)).toBeUndefined(); + expect(decide(pattern.repeat(50), `${"a".repeat(1_000)}c`)).toBeUndefined(); + expect(decide(pattern, `${"a".repeat(200)}b`)).toBe("ignored"); + }); + + it("parses unclosed classes in linear time", () => { + // Rescanning the rest of the line for each `[` stalls for over a minute here; linear takes ms. + expect(decide("[".repeat(30_000), "[".repeat(30_000))).toBe("ignored"); + expect(decide(`${"[".repeat(30_000)}]`, "x")).toBeUndefined(); + expect(decide("[a][", "a[")).toBe("ignored"); + }); +}); diff --git a/apps/server/src/workspace/IgnorePatterns.ts b/apps/server/src/workspace/IgnorePatterns.ts new file mode 100644 index 000000000000..1180b02316ef --- /dev/null +++ b/apps/server/src/workspace/IgnorePatterns.ts @@ -0,0 +1,170 @@ +/** + * Gitignore-syntax matching for `.ignore` files, which git cannot evaluate for us. Patterns come + * from the repository, so each runs as an NFA over the path (O(pattern × path)): a backtracking + * regex stalls for seconds on `a*a*a*…b`. POSIX `[:class:]` names are not supported. + */ + +type Token = + | { readonly type: "char"; readonly char: string } + | { readonly type: "one" } + | { readonly type: "class"; readonly negated: boolean; readonly ranges: ReadonlyArray } + | { readonly type: "star" } + | { readonly type: "any" } + /** A leading or inner `**` segment: nothing, or anything ending in `/`. */ + | { readonly type: "folders" }; + +interface Pattern { + readonly tokens: ReadonlyArray; + readonly negated: boolean; + readonly folderOnly: boolean; +} + +export type IgnoreDecision = "ignored" | "included"; + +function parseClass(chars: ReadonlyArray, start: number) { + let index = start + 1; + const negated = chars[index] === "!" || chars[index] === "^"; + if (negated) index += 1; + const ranges: string[] = []; + for (let first = true; index < chars.length; first = false) { + let char = chars[index]!; + if (char === "]" && !first) + return { token: { type: "class", negated, ranges } as const, end: index }; + if (char === "\\" && index + 1 < chars.length) char = chars[++index]!; + if (chars[index + 1] === "-" && index + 2 < chars.length && chars[index + 2] !== "]") { + let last = chars[index + 2]!; + index += 2; + if (last === "\\" && index + 1 < chars.length) last = chars[++index]!; + ranges.push(char, last); + } else { + ranges.push(char, char); + } + index += 1; + } + return undefined; +} + +function parseLine(rawLine: string): Pattern | undefined { + let line = rawLine.endsWith("\r") ? rawLine.slice(0, -1) : rawLine; + // Trailing spaces are dropped unless escaped. + while (line.endsWith(" ") && !line.endsWith("\\ ")) line = line.slice(0, -1); + if (line === "" || line.startsWith("#")) return undefined; + const negated = line.startsWith("!"); + if (negated) line = line.slice(1); + else if (line.startsWith("\\!") || line.startsWith("\\#")) line = line.slice(1); + const folderOnly = line.endsWith("/") && !line.endsWith("\\/"); + if (folderOnly) line = line.slice(0, -1); + if (line === "") return undefined; + const anchored = line.slice(0, -1).includes("/"); + if (line.startsWith("/")) line = line.slice(1); + + const chars = Array.from(line); + const tokens: Token[] = anchored ? [] : [{ type: "folders" }]; + // A `[` that never closes leaves every later `[` unclosed too, so one failed scan settles them + // all; rescanning the suffix for each would make `[[[[…` quadratic. + let classesClose = true; + for (let index = 0; index < chars.length; index += 1) { + const char = chars[index]!; + if (char === "*" && chars[index + 1] === "*") { + const atSegmentStart = index === 0 || chars[index - 1] === "/"; + const next = chars[index + 2]; + if (atSegmentStart && next === "/") { + tokens.push({ type: "folders" }); + index += 2; + continue; + } + if (atSegmentStart && next === undefined) { + tokens.push({ type: "any" }); + index += 1; + continue; + } + } + if (char === "*") tokens.push({ type: "star" }); + else if (char === "?") tokens.push({ type: "one" }); + else if (char === "[") { + const parsed = classesClose ? parseClass(chars, index) : undefined; + if (!parsed) classesClose = false; + if (parsed) { + tokens.push(parsed.token); + index = parsed.end; + } else tokens.push({ type: "char", char }); + } else if (char === "\\" && index + 1 < chars.length) { + tokens.push({ type: "char", char: chars[++index]! }); + } else tokens.push({ type: "char", char }); + } + return { tokens, negated, folderOnly }; +} + +/** + * States are token positions. A `folders` token that has consumed part of a segment is `~position` + * instead: it may only finish after the next `/`, so an inner globstar never matches `xbar` as `bar`. + */ +function matches(tokens: ReadonlyArray, path: ReadonlyArray): boolean { + let states = new Set(); + const add = (set: Set, position: number) => { + // Wildcards also match nothing, so reaching one reaches the token after it. + for (; position <= tokens.length && !set.has(position); position += 1) { + set.add(position); + const type = tokens[position]?.type; + if (type !== "star" && type !== "any" && type !== "folders") return; + } + }; + add(states, 0); + for (const char of path) { + const next = new Set(); + for (const state of states) { + const position = state < 0 ? ~state : state; + const token = tokens[position]; + if (!token) continue; + switch (token.type) { + case "char": + if (token.char === char) add(next, position + 1); + break; + case "one": + if (char !== "/") add(next, position + 1); + break; + case "class": { + let inRange = false; + for (let index = 0; index < token.ranges.length && !inRange; index += 2) { + inRange = token.ranges[index]! <= char && char <= token.ranges[index + 1]!; + } + if (char !== "/" && inRange !== token.negated) add(next, position + 1); + break; + } + case "star": + if (char !== "/") add(next, position); + break; + case "any": + add(next, position); + break; + case "folders": + if (char === "/") add(next, position); + else next.add(~position); + break; + } + } + if (next.size === 0) return false; + states = next; + } + return states.has(tokens.length); +} + +export interface IgnoreFile { + /** The last pattern that matches decides; `undefined` when none does. */ + readonly decide: (relativePath: string, isFolder: boolean) => IgnoreDecision | undefined; +} + +export function parseIgnoreFile(contents: string): IgnoreFile { + const patterns = contents.split("\n").flatMap((line) => parseLine(line) ?? []); + return { + decide: (relativePath, isFolder) => { + const path = Array.from(relativePath); + for (let index = patterns.length - 1; index >= 0; index -= 1) { + const pattern = patterns[index]!; + if (pattern.folderOnly && !isFolder) continue; + if (matches(pattern.tokens, path)) return pattern.negated ? "included" : "ignored"; + } + return undefined; + }, + }; +} diff --git a/apps/server/src/workspace/WorkspaceEntries.test.ts b/apps/server/src/workspace/WorkspaceEntries.test.ts index a037b5a172e6..ecf150129942 100644 --- a/apps/server/src/workspace/WorkspaceEntries.test.ts +++ b/apps/server/src/workspace/WorkspaceEntries.test.ts @@ -433,6 +433,20 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { }); }), ); + + it.effect("finds ignored files that sort past the listing limit", () => + Effect.gen(function* () { + const cwd = yield* makeTempDir({ git: true }); + yield* writeTextFile(cwd, ".gitignore", "*.secret\n"); + for (let index = 1; index <= 205; index += 1) { + yield* writeTextFile(cwd, `file-${String(index).padStart(3, "0")}.secret`, "s"); + } + + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; + const listed = yield* workspaceEntries.listIgnoredFiles({ cwd, query: "file-205" }); + expect(listed.files).toContainEqual({ path: "file-205.secret", byteLength: 1 }); + }), + ); }); describe("search", () => { @@ -457,17 +471,31 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { }), ); - it.effect("matches square brackets in the query literally", () => + it.effect("matches glob metacharacters in the query literally", () => Effect.gen(function* () { const cwd = yield* makeTempDir({ prefix: "t3code-workspace-brackets-" }); yield* writeTextFile(cwd, "src/app/game/[id]/index.tsx"); + yield* writeTextFile(cwd, "src/foo{bar}.ts"); + yield* writeTextFile(cwd, "src/main.ts"); + yield* writeTextFile(cwd, "src/notes.md"); - for (const query of ["game/[i", "game/[id]"]) { - const result = yield* searchWorkspaceEntries({ cwd, query, limit: 10 }); - expect(result.entries.map((entry) => entry.path)).toContain( - "src/app/game/[id]/index.tsx", + const search = (query: string) => + searchWorkspaceEntries({ cwd, query, limit: 10 }).pipe( + Effect.map((result) => result.entries.map((entry) => entry.path)), ); + + for (const query of ["game/[i", "game/[id]"]) { + expect(yield* search(query)).toContain("src/app/game/[id]/index.tsx"); } + expect(yield* search("foo{bar}.ts")).toContain("src/foo{bar}.ts"); + expect((yield* search("*.{ts,tsx}")).toSorted()).toEqual([ + "src/app/game/[id]/index.tsx", + "src/foo{bar}.ts", + "src/main.ts", + ]); + expect((yield* search("*.{ts}")).toSorted()).toEqual(["src/foo{bar}.ts", "src/main.ts"]); + expect(yield* search("src/{main}.ts")).toEqual(["src/main.ts"]); + expect(yield* search("src/{foo{bar},notes}.md")).toEqual(["src/notes.md"]); }), ); @@ -479,6 +507,11 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { yield* writeTextFile(cwd, "ignored.txt", "ignore me"); yield* writeTextFile(cwd, ".convex/local-storage/data.json", "{}"); yield* writeTextFile(cwd, "convex/UOoS-l/convex_local_storage/modules/data.json", "{}"); + yield* writeTextFile(cwd, ".git/info/exclude", "info-excluded.txt\n"); + yield* writeTextFile(cwd, "info-excluded.txt", "ignore me"); + yield* writeTextFile(cwd, ".git/configured-excludes", "configured-excluded.txt\n"); + yield* git(cwd, ["config", "core.excludesFile", ".git/configured-excludes"]); + yield* writeTextFile(cwd, "configured-excluded.txt", "ignore me"); const result = yield* searchWorkspaceEntries({ cwd, query: "", limit: 100 }); const paths = result.entries.map((entry) => entry.path); @@ -486,6 +519,8 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { expect(paths).toContain("src"); expect(paths).toContain("src/keep.ts"); expect(paths).not.toContain("ignored.txt"); + expect(paths).not.toContain("info-excluded.txt"); + expect(paths).not.toContain("configured-excluded.txt"); expect(paths.some((entryPath) => entryPath.startsWith(".convex/"))).toBe(false); expect(paths.some((entryPath) => entryPath.startsWith("convex/"))).toBe(false); }), @@ -511,6 +546,133 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { }), ); + it.effect("indexes symlinked files and directories inside the workspace", () => + Effect.gen(function* () { + if ((yield* HostProcess.Platform) === "win32") return; + const cwd = yield* makeTempDir({ prefix: "t3code-workspace-symlinks-" }); + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* writeTextFile(cwd, "src/index.ts", "export {};"); + yield* fileSystem.symlink("src", path.join(cwd, "src-link")); + yield* fileSystem.symlink("src/index.ts", path.join(cwd, "index-link.ts")); + const symlinkEntries = [ + { kind: "directory", path: "src-link" }, + { kind: "file", path: "index-link.ts" }, + ]; + + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; + const searched = yield* searchWorkspaceEntries({ cwd, query: "link", limit: 100 }); + const listed = yield* workspaceEntries.list({ cwd }); + + for (const entry of symlinkEntries) { + expect(searched.entries).toContainEqual(expect.objectContaining(entry)); + expect(listed.entries).toContainEqual(expect.objectContaining(entry)); + } + }), + ); + it.effect("skips symlinks that ignore rules hide or that leave the workspace", () => + Effect.gen(function* () { + if ((yield* HostProcess.Platform) === "win32") return; + const cwd = yield* makeTempDir({ prefix: "t3code-workspace-symlinks-", git: true }); + const outside = yield* makeTempDir({ prefix: "t3code-workspace-symlinks-outside-" }); + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* writeTextFile(cwd, "src/index.ts", "export {};"); + yield* writeTextFile(cwd, ".ignore", "custom-link\n"); + yield* fileSystem.symlink("src", path.join(cwd, "custom-link")); + yield* fileSystem.symlink("src/index.ts", path.join(cwd, "tracked-link.ts")); + yield* fileSystem.symlink("src", path.join(cwd, "ignored-link")); + yield* fileSystem.symlink("src", path.join(cwd, "kept-link")); + yield* fileSystem.symlink(outside, path.join(cwd, "outside-link")); + yield* git(cwd, ["add", "tracked-link.ts"]); + yield* writeTextFile(cwd, ".gitignore", "tracked-link.ts\nignored-link\n"); + // A submodule path makes `git check-ignore` refuse the whole batch without --no-index. + const submodule = path.join(cwd, "vendor/sub"); + yield* writeTextFile(submodule, "a.ts", "export {};"); + yield* git(submodule, ["init"]); + yield* git(submodule, ["add", "a.ts"]); + yield* git(submodule, [ + "-c", + "user.name=t", + "-c", + "user.email=t@t", + "commit", + "-m", + "init", + ]); + yield* git(cwd, ["add", "vendor/sub"]); + yield* fileSystem.symlink("a.ts", path.join(submodule, "sublink.ts")); + + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; + const searched = yield* searchWorkspaceEntries({ cwd, query: "link", limit: 100 }); + const listed = yield* workspaceEntries.list({ cwd }); + + for (const entries of [searched.entries, listed.entries]) { + const paths = entries.map((entry) => entry.path); + expect(paths).toContain("kept-link"); + expect(paths).toContain("vendor/sub/sublink.ts"); + expect(paths).not.toContain("custom-link"); + expect(paths).not.toContain("tracked-link.ts"); + expect(paths).not.toContain("ignored-link"); + expect(paths).not.toContain("outside-link"); + } + }), + ); + it.effect("lets the nearest .ignore rule decide a symlink, and git's rules still hide it", () => + Effect.gen(function* () { + if ((yield* HostProcess.Platform) === "win32") return; + const cwd = yield* makeTempDir({ prefix: "t3code-workspace-symlinks-", git: true }); + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* writeTextFile(cwd, ".gitignore", "*keep*\n"); + yield* writeTextFile(cwd, ".ignore", "!*keep*\n*hide*\n"); + yield* writeTextFile(cwd, "sub/.ignore", "!*hide*\n"); + yield* writeTextFile(cwd, "keep-real.ts", "export {};"); + yield* writeTextFile(cwd, "sub/hide-real.ts", "export {};"); + yield* fileSystem.symlink("keep-real.ts", path.join(cwd, "keep-link.ts")); + yield* fileSystem.symlink("hide-real.ts", path.join(cwd, "sub/hide-link.ts")); + + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; + const searched = yield* searchWorkspaceEntries({ cwd, query: "link", limit: 100 }); + const listed = yield* workspaceEntries.list({ cwd }); + + for (const entries of [searched.entries, listed.entries]) { + const paths = entries.map((entry) => entry.path); + expect(paths).toContain("sub/hide-link.ts"); + expect(paths).not.toContain("keep-link.ts"); + } + // The real files are shown or hidden by the same rules. + const listedPaths = listed.entries.map((entry) => entry.path); + expect(listedPaths).toContain("sub/hide-real.ts"); + expect(listedPaths).not.toContain("keep-real.ts"); + }), + ); + it.effect("matches .ignore rules against whole names and treats folder links as links", () => + Effect.gen(function* () { + if ((yield* HostProcess.Platform) === "win32") return; + const cwd = yield* makeTempDir({ prefix: "t3code-workspace-symlinks-", git: true }); + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* writeTextFile(cwd, "src/index.ts", "export {};"); + yield* writeTextFile(cwd, ".gitignore", "blink\n"); + // `!link` must not re-include `blink`, and `folder-link/` does not apply to a link. + yield* writeTextFile(cwd, ".ignore", "!link\nfolder-link/\n"); + yield* fileSystem.symlink("src", path.join(cwd, "blink")); + yield* fileSystem.symlink("src", path.join(cwd, "folder-link")); + yield* writeTextFile(cwd, "sub/blink", "regular"); + + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; + const searched = yield* searchWorkspaceEntries({ cwd, query: "link", limit: 100 }); + const listed = yield* workspaceEntries.list({ cwd }); + + for (const entries of [searched.entries, listed.entries]) { + const paths = entries.map((entry) => entry.path); + expect(paths).not.toContain("blink"); + expect(paths).not.toContain("sub/blink"); + expect(paths).toContain("folder-link"); + } + }), + ); it.effect("finds a path inside a gitignored nested git worktree only by its exact path", () => Effect.gen(function* () { const cwd = yield* makeTempDir({ prefix: "t3code-workspace-worktree-", git: true }); @@ -526,6 +688,13 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { yield* git(cwd, ["commit", "-m", "init"], env); yield* git(cwd, ["worktree", "add", "-b", "x", ".worktrees/x"]); yield* writeTextFile(cwd, ".worktrees/x/src/foo.py", "print(1)"); + const expected = [".worktrees/x/src/foo.py"]; + // Windows file names cannot contain a newline. + if ((yield* HostProcess.Platform) !== "win32") { + yield* git(cwd, ["worktree", "add", "--detach", ".worktrees/line\nname"]); + yield* writeTextFile(cwd, ".worktrees/line\nname/src/foo.py", "print(2)"); + expected.unshift(".worktrees/line\nname/src/foo.py"); + } const exact = yield* searchWorkspaceEntries({ cwd, @@ -533,10 +702,14 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { limit: 25, kind: "file", }); - expect(exact.entries).toEqual([{ path: ".worktrees/x/src/foo.py", kind: "file" }]); + expect(exact.entries.toSorted((a, b) => (a.path < b.path ? -1 : 1))).toEqual( + expected.map((path) => ({ path, kind: "file" })), + ); const partial = yield* searchWorkspaceEntries({ cwd, query: "src/fo", limit: 25 }); - expect(partial.entries.map((entry) => entry.path)).not.toContain(".worktrees/x/src/foo.py"); + for (const path of expected) { + expect(partial.entries.map((entry) => entry.path)).not.toContain(path); + } }), ); }); @@ -577,6 +750,8 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { yield* writeTextFile(cwd, ".gitignore", "ignored.txt\n"); yield* writeTextFile(cwd, "src/keep.ts", "square\nSquare\n"); yield* writeTextFile(cwd, "ignored.txt", "Square\n"); + yield* writeTextFile(cwd, ".git/info/exclude", "info-excluded.txt\n"); + yield* writeTextFile(cwd, "info-excluded.txt", "Square\n"); const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; const result = yield* workspaceEntries.searchContents({ @@ -799,10 +974,17 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { if (created.ok) { const finder = created.value; finders += 1; - const scanFiles = finder.scanFiles.bind(finder); + // The synchronous rescan and grep wait on fff's picker lock on the event loop. finder.scanFiles = () => { + throw new Error("refresh must not rescan on the JS thread"); + }; + finder.grep = () => { + throw new Error("search must not grep on the JS thread"); + }; + const scanFilesAsync = finder.scanFilesAsync.bind(finder); + finder.scanFilesAsync = () => { rescans += 1; - return scanFiles(); + return scanFilesAsync(); }; const waitForIndexReady = finder.waitForIndexReady.bind(finder); finder.waitForIndexReady = (timeoutMs) => @@ -832,6 +1014,68 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { expect({ finders, rescans }).toEqual({ finders: 1, rescans: 1 }); }), ); + + it.effect("serves content search once a first scan that outlasted the wait finishes", () => + Effect.gen(function* () { + const cwd = yield* makeTempDir({ prefix: "t3code-workspace-content-slow-scan-" }); + yield* writeTextFile(cwd, "src/notes.ts", "const needle = 1;\n"); + // Real finders; only the initial wait and the progress it reads report a scan still running. + let scanning = true; + let finders = 0; + let settle = (): Promise => Promise.resolve(); + const createFinder = FileFinder.create.bind(FileFinder); + vi.spyOn(FileFinder, "create").mockImplementation((options) => { + const created = createFinder(options); + if (!created.ok) return created; + const finder = created.value; + finders += 1; + const waitForIndexReady = finder.waitForIndexReady.bind(finder); + settle = () => waitForIndexReady(15_000); + finder.waitForIndexReady = (timeoutMs) => + scanning ? Promise.resolve({ ok: true, value: false }) : waitForIndexReady(timeoutMs); + const getScanProgress = finder.getScanProgressAsync.bind(finder); + finder.getScanProgressAsync = async () => { + const progress = await getScanProgress(); + return scanning && progress.ok + ? { + ok: true, + value: { ...progress.value, isScanning: true, isWarmupComplete: false }, + } + : progress; + }; + return created; + }); + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; + const searchNeedle = workspaceEntries + .searchContents({ + cwd, + query: "needle", + limit: 100, + caseSensitive: true, + wholeWord: false, + useRegex: false, + }) + .pipe( + Effect.map((result): unknown => result.matches.map((match) => match.path)), + Effect.catchTags({ + WorkspaceSearchIndexScanTimedOut: (error) => Effect.succeed(error._tag), + }), + ); + + const whileScanning = yield* Effect.all([searchNeedle, searchNeedle], { + concurrency: "unbounded", + }); + scanning = false; + yield* Effect.promise(() => settle()); + const afterScan = yield* searchNeedle; + + expect({ whileScanning, afterScan, finders }).toEqual({ + whileScanning: ["WorkspaceSearchIndexScanTimedOut", "WorkspaceSearchIndexScanTimedOut"], + afterScan: ["src/notes.ts"], + finders: 1, + }); + }), + ); }); describe("browse", () => { diff --git a/apps/server/src/workspace/WorkspaceEntries.ts b/apps/server/src/workspace/WorkspaceEntries.ts index cac706f7ca4a..ad1d2c2ae0eb 100644 --- a/apps/server/src/workspace/WorkspaceEntries.ts +++ b/apps/server/src/workspace/WorkspaceEntries.ts @@ -179,7 +179,6 @@ export const make = Effect.gen(function* () { function* ( cause: | WorkspaceSearchIndex.WorkspaceSearchIndexCreateFailed - | WorkspaceSearchIndex.WorkspaceSearchIndexScanTimedOut | WorkspaceSearchIndex.WorkspaceSearchIndexRefreshFailed, ) { yield* Effect.logWarning("Failed to refresh workspace search index", { @@ -197,7 +196,6 @@ export const make = Effect.gen(function* () { Effect.provide(workspaceSearchIndexes.get(indexKey)), Effect.catchTags({ WorkspaceSearchIndexCreateFailed: recoverRefreshFailure, - WorkspaceSearchIndexScanTimedOut: recoverRefreshFailure, WorkspaceSearchIndexRefreshFailed: recoverRefreshFailure, }), ); @@ -307,23 +305,30 @@ export const make = Effect.gen(function* () { ) { return []; } - const listed = yield* vcsProcess - .run({ - operation: "WorkspaceEntries.listWorktrees", - command: "git", - args: ["worktree", "list", "--porcelain"], - cwd, - allowNonZeroExit: true, - timeoutMs: 5_000, - }) - .pipe(Effect.orElseSucceed(() => undefined)); - if (!listed || listed.exitCode !== 0) return []; + const listWorktrees = (args: ReadonlyArray) => + vcsProcess + .run({ + operation: "WorkspaceEntries.listWorktrees", + command: "git", + args: ["worktree", "list", "--porcelain", ...args], + cwd, + allowNonZeroExit: true, + timeoutMs: 5_000, + }) + .pipe( + Effect.map((result) => (result.exitCode === 0 ? result.stdout : undefined)), + Effect.orElseSucceed(() => undefined), + ); + // Only NUL framing keeps a newline in a worktree path. Pre-2.36 Git rejects `-z`. + const listed = + (yield* listWorktrees(["-z"])) ?? (yield* listWorktrees([]))?.replaceAll("\n", "\0"); + if (listed === undefined) return []; const realCwd = yield* Effect.promise(() => NodeFSP.realpath(cwd).catch(() => cwd)); - const worktreeRoots = listed.stdout.split(/\n\s*\n/).flatMap((record) => { - const lines = record.split("\n"); - const worktree = lines.find((line) => line.startsWith("worktree ")); + const worktreeRoots = listed.split("\0\0").flatMap((record) => { + const fields = record.split("\0"); + const worktree = fields.find((field) => field.startsWith("worktree ")); // A `prunable` record's directory is gone; another checkout may now sit there. - if (worktree === undefined || lines.some((line) => line.startsWith("prunable"))) return []; + if (worktree === undefined || fields.some((field) => field.startsWith("prunable"))) return []; const root = worktree.slice("worktree ".length); const relativeRoot = path.relative(realCwd, root); return relativeRoot === "" || @@ -576,9 +581,13 @@ export const make = Effect.gen(function* () { }) .pipe(Effect.orElseSucceed(() => undefined)); if (!result || result.exitCode !== 0) return { files: [], truncated: false }; + // Filter before the cap so files sorting past it stay reachable by search. + const query = input.query?.toLowerCase() ?? ""; const paths = result.stdout .split("\0") - .filter((entry) => entry.length > 0 && !entry.endsWith("/")) + .filter( + (entry) => entry.length > 0 && !entry.endsWith("/") && entry.toLowerCase().includes(query), + ) .toSorted(); const files = yield* Effect.promise(() => Promise.all( diff --git a/apps/server/src/workspace/WorkspaceFileSystem.test.ts b/apps/server/src/workspace/WorkspaceFileSystem.test.ts index 3d31cd6b63e1..6e9ee4406df2 100644 --- a/apps/server/src/workspace/WorkspaceFileSystem.test.ts +++ b/apps/server/src/workspace/WorkspaceFileSystem.test.ts @@ -12,6 +12,7 @@ import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as ServerConfig from "../config.ts"; +import { makeFakeWslHost } from "../testUtils/fakeWsl.ts"; import * as VcsDriverRegistry from "../vcs/VcsDriverRegistry.ts"; import * as VcsProcess from "../vcs/VcsProcess.ts"; import * as KnownWorkspaces from "./KnownWorkspaces.ts"; @@ -43,6 +44,11 @@ const injectShortReads = Effect.fn("injectShortReads")(function* (target: string yield* Effect.addFinalizer(() => Effect.sync(() => open.mockImplementation(native.open))); }); +const whitespaceFilePaths = + HostProcess.Platform.defaultValue() === "win32" + ? [" report.txt"] + : [" report.txt", "report.txt ", "\treport.txt\t", " "]; + const UNKNOWN_WORKSPACE_PREFIX = "t3code-unknown-workspace-"; const layerKnownWorkspacesTest = Layer.mock(KnownWorkspaces.KnownWorkspaces)({ assertContains: (cwd) => @@ -91,6 +97,14 @@ const writeTextFile = Effect.fn("writeTextFile")(function* ( yield* fileSystem.writeFileString(absolutePath, contents).pipe(Effect.orDie); }); +const makeWslHost = Effect.gen(function* () { + const { mountRoot, underWsl } = yield* makeFakeWslHost; + const service = yield* underWsl( + WorkspaceFileSystem.make.pipe(Effect.provide(layerKnownWorkspacesTest)), + ); + return { mountRoot, service, underWsl }; +}); + it.layer(layerTest, { excludeTestServices: true })("WorkspaceFileSystemLive", (it) => { describe("getMetadata", () => { it.effect( @@ -176,6 +190,31 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceFileSystemLive", (i expect(statted).toEqual([]); }), ); + + it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( + "reports a Windows drive path under WSL as the file it maps to", + () => + Effect.gen(function* () { + const { mountRoot, service, underWsl } = yield* makeWslHost; + yield* writeTextFile(mountRoot, "c/Users/me/Documents/notes.md", "# Notes\n"); + + const { entries } = yield* underWsl( + service.getMetadata({ + paths: [ + "C:\\Users\\me\\Documents\\notes.md", + "C:/Users/me/Documents/notes.md", + "C:/Users/me/Documents/notes\u0000.md", + ], + }), + ); + + expect(entries).toEqual([ + { kind: "file", byteLength: 8 }, + { kind: "file", byteLength: 8 }, + null, + ]); + }), + ); }); describe("readFile", () => { @@ -196,6 +235,26 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceFileSystemLive", (i byteLength: 26, truncated: false, }); + + const bomContents = "\uFEFFexport const answer = 42;\n"; + yield* writeTextFile(cwd, "bom.ts", bomContents); + const bomRead = yield* workspaceFileSystem.readFile({ cwd, relativePath: "bom.ts" }); + yield* workspaceFileSystem.writeFile({ + cwd, + relativePath: "bom.ts", + contents: bomRead.contents.replace("42", "43"), + }); + const fs = yield* FileSystem.FileSystem; + const saved = yield* fs.readFile(`${cwd}/bom.ts`); + expect([...saved]).toEqual([...new TextEncoder().encode(bomContents.replace("42", "43"))]); + + yield* writeTextFile(cwd, "report.txt", "ordinary file"); + for (const relativePath of whitespaceFilePaths) { + yield* writeTextFile(cwd, relativePath, "whitespace file"); + const read = yield* workspaceFileSystem.readFile({ cwd, relativePath }); + expect(read.contents).toBe("whitespace file"); + expect(read.relativePath).toBe(relativePath); + } }), ); @@ -250,28 +309,38 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceFileSystemLive", (i byteLength: 9, truncated: false, }); + + for (const relativePath of whitespaceFilePaths) { + yield* writeTextFile(outsideDir, relativePath, "whitespace file"); + const requestedPath = path.join(outsideDir, relativePath); + const read = yield* workspaceFileSystem.readFile({ cwd, relativePath: requestedPath }); + expect(read.contents).toBe("whitespace file"); + expect(read.relativePath).toBe(requestedPath); + } }), ); - it.effect("reads ~/ paths from the server home outside the workspace", () => - Effect.gen(function* () { - const workspaceFileSystem = yield* WorkspaceFileSystem.WorkspaceFileSystem; - const cwd = yield* makeTempDir; - const home = yield* makeTempDir; - yield* writeTextFile(home, ".claude/CLAUDE.md", "# Notes\n"); - yield* writeTextFile(cwd, "~/.claude/CLAUDE.md", "Workspace decoy\n"); - - const result = yield* workspaceFileSystem - .readFile({ cwd, relativePath: "~/.claude/CLAUDE.md" }) - .pipe(Effect.provideService(HostProcess.HomeDirectory, home)); - - expect(result).toEqual({ - relativePath: "~/.claude/CLAUDE.md", - contents: "# Notes\n", - byteLength: 8, - truncated: false, - }); - }), + it.effect.each(["~/.claude/CLAUDE.md", "~\\.claude/CLAUDE.md"])( + "reads %s from the server home outside the workspace", + (relativePath) => + Effect.gen(function* () { + const workspaceFileSystem = yield* WorkspaceFileSystem.WorkspaceFileSystem; + const cwd = yield* makeTempDir; + const home = yield* makeTempDir; + yield* writeTextFile(home, ".claude/CLAUDE.md", "# Notes\n"); + yield* writeTextFile(cwd, "~/.claude/CLAUDE.md", "Workspace decoy\n"); + + const result = yield* workspaceFileSystem + .readFile({ cwd, relativePath }) + .pipe(Effect.provideService(HostProcess.HomeDirectory, home)); + + expect(result).toEqual({ + relativePath, + contents: "# Notes\n", + byteLength: 8, + truncated: false, + }); + }), ); it.effect("does not read server state files", () => @@ -330,6 +399,24 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceFileSystemLive", (i }), ); + it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( + "reads a Windows drive path under WSL from where wslpath mounts it", + () => + Effect.gen(function* () { + const { mountRoot, service, underWsl } = yield* makeWslHost; + const cwd = yield* makeTempDir; + yield* writeTextFile(mountRoot, "c/Users/me/Documents/notes.md", "# Notes\n"); + + for (const relativePath of [ + "C:\\Users\\me\\Documents\\notes.md", + "C:/Users/me/Documents/notes.md", + ]) { + const result = yield* underWsl(service.readFile({ cwd, relativePath })); + expect(result).toMatchObject({ contents: "# Notes\n", truncated: false }); + } + }), + ); + it.effect("refuses a network path outside the workspace", () => Effect.gen(function* () { const workspaceFileSystem = yield* WorkspaceFileSystem.WorkspaceFileSystem; @@ -457,14 +544,17 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceFileSystemLive", (i }), ); - it.effect("rejects binary files without leaking their contents into the error", () => + it.effect.each([ + { name: "a NUL byte", bytes: [0x61, 0, 0x62] }, + { name: "invalid UTF-8", bytes: [0x63, 0x61, 0x66, 0xe9] }, + ])("rejects files with $name without leaking their contents", ({ bytes }) => Effect.gen(function* () { const workspaceFileSystem = yield* WorkspaceFileSystem.WorkspaceFileSystem; const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; const cwd = yield* makeTempDir; const absolutePath = path.join(cwd, "asset.bin"); - yield* fileSystem.writeFile(absolutePath, Uint8Array.from([0x61, 0, 0x62])); + yield* fileSystem.writeFile(absolutePath, Uint8Array.from(bytes)); const error = yield* workspaceFileSystem .readFile({ cwd, relativePath: "asset.bin" }) @@ -560,6 +650,21 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceFileSystemLive", (i expect(yield* fileSystem.readFileString(path.join(cwd, "src/main.ts"))).toBe( "export const edited = true;\n", ); + + yield* writeTextFile(cwd, "report.txt", "ordinary file"); + for (const relativePath of whitespaceFilePaths) { + yield* writeTextFile(cwd, relativePath, "whitespace file"); + const written = yield* workspaceFileSystem.writeFile({ + cwd, + relativePath, + contents: "edited", + }); + expect(yield* fileSystem.readFileString(path.join(cwd, "report.txt"))).toBe( + "ordinary file", + ); + expect(yield* fileSystem.readFileString(path.join(cwd, relativePath))).toBe("edited"); + expect(written.relativePath).toBe(relativePath); + } }).pipe( Effect.provide( Layer.fresh(WorkspaceFileSystem.layer).pipe( @@ -582,17 +687,13 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceFileSystemLive", (i const path = yield* Path.Path; const fileSystem = yield* FileSystem.FileSystem; - const error = yield* workspaceFileSystem - .writeFile({ - cwd, - relativePath: "../escape.md", - contents: "# nope\n", - }) - .pipe(Effect.flip); + for (const relativePath of ["../escape.md", "../escape.md ", " folder/../../escape.md"]) { + const error = yield* workspaceFileSystem + .writeFile({ cwd, relativePath, contents: "# nope\n" }) + .pipe(Effect.flip); - expect(error.message).toContain( - "Workspace file path must be relative to the project root: ../escape.md", - ); + expect(error).toBeInstanceOf(WorkspacePaths.WorkspacePathOutsideRootError); + } const escapedPath = path.resolve(cwd, "..", "escape.md"); const escapedStat = yield* fileSystem diff --git a/apps/server/src/workspace/WorkspaceFileSystem.ts b/apps/server/src/workspace/WorkspaceFileSystem.ts index 2f3823f97e47..494472e4ce68 100644 --- a/apps/server/src/workspace/WorkspaceFileSystem.ts +++ b/apps/server/src/workspace/WorkspaceFileSystem.ts @@ -19,6 +19,7 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import * as Schema from "effect/Schema"; import { makeIsServerInternalPath } from "../auth/serverInternalPaths.ts"; @@ -26,6 +27,7 @@ import * as KnownWorkspaces from "./KnownWorkspaces.ts"; import * as WorkspaceEntries from "./WorkspaceEntries.ts"; import * as WorkspacePaths from "./WorkspacePaths.ts"; import { fileHeaderMimeType } from "./fileHeaderMimeType.ts"; +import { toWslDrivePath } from "./wslDrivePath.ts"; const PROJECT_READ_FILE_MAX_BYTES = 1024 * 1024; @@ -133,6 +135,27 @@ export class WorkspaceFileSystem extends Context.Service< } >()("t3/workspace/WorkspaceFileSystem") {} +const UTF8_BOM = "\uFEFF"; + +async function startsWithUtf8Bom(filePath: string): Promise { + const handle = await NodeFSP.open( + filePath, + // O_NONBLOCK keeps a FIFO with no writer from hanging the save. + NodeFS.constants.O_RDONLY | + (NodeFS.constants.O_NOFOLLOW ?? 0) | + (NodeFS.constants.O_NONBLOCK ?? 0), + ).catch(() => null); + if (!handle) return false; + try { + const { bytesRead, buffer } = await handle.read(Buffer.alloc(3), 0, 3, 0); + return bytesRead === 3 && buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf; + } catch { + return false; + } finally { + await handle.close(); + } +} + /** @public */ export const make = Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; @@ -141,9 +164,18 @@ export const make = Effect.gen(function* () { const knownWorkspaces = yield* KnownWorkspaces.KnownWorkspaces; const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; const isServerInternalPath = yield* makeIsServerInternalPath; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + + const toHostPath = (requestedPath: string) => + toWslDrivePath(requestedPath).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); const metadataForPath = Effect.fnUntraced(function* (requestedPath: string) { - const expandedPath = expandHomePath(requestedPath, yield* HostProcess.HomeDirectory); + const expandedPath = expandHomePath( + yield* toHostPath(requestedPath), + yield* HostProcess.HomeDirectory, + ); // Chips ask as soon as an agent's message renders, and touching a network // path makes Windows sign in to its host with the user's NTLM credentials. if (!path.isAbsolute(expandedPath) || isNetworkPath(expandedPath)) return null; @@ -227,10 +259,11 @@ export const make = Effect.gen(function* () { const resolveReadTarget = Effect.fn("WorkspaceFileSystem.resolveReadTarget")(function* ( input: ProjectReadFileInput, ) { - const requestedPath = input.relativePath.trim(); - const hostPath = requestedPath.startsWith("~/") - ? expandHomePath(requestedPath, yield* HostProcess.HomeDirectory) - : requestedPath; + const requestedPath = input.relativePath; + const hostPath = + requestedPath.startsWith("~/") || requestedPath.startsWith("~\\") + ? expandHomePath(requestedPath, yield* HostProcess.HomeDirectory) + : yield* toHostPath(requestedPath); if ( isNetworkPath(hostPath) && !KnownWorkspaces.isWithinRoot(path, path.resolve(hostPath), path.resolve(input.cwd)) @@ -424,22 +457,22 @@ export const make = Effect.gen(function* () { offset += bytesRead; } const fileBytes = buffer.subarray(0, offset); - if (fileBytes.includes(0)) { - return yield* new WorkspaceBinaryFileError({ + const truncated = stat.size > PROJECT_READ_FILE_MAX_BYTES; + const binaryFileError = () => + new WorkspaceBinaryFileError({ workspaceRoot: input.cwd, relativePath: input.relativePath, resolvedPath: realTargetPath, }); - } - - const truncated = stat.size > PROJECT_READ_FILE_MAX_BYTES; - return { - relativePath: target.relativePath, - // Streaming decode drops a character the byte limit split, instead of ending in U+FFFD. - contents: new TextDecoder("utf-8").decode(fileBytes, { stream: truncated }), - byteLength: stat.size, - truncated, - }; + if (fileBytes.includes(0)) return yield* binaryFileError(); + // Fatal decoding rejects invalid UTF-8, since U+FFFD contents would corrupt the file on save. + // Streaming decode drops a character the byte limit split, instead of rejecting it. + const contents = yield* Effect.try({ + try: () => + new TextDecoder("utf-8", { fatal: true }).decode(fileBytes, { stream: truncated }), + catch: binaryFileError, + }); + return { relativePath: target.relativePath, contents, byteLength: stat.size, truncated }; }), (handle) => handle === null @@ -550,6 +583,12 @@ export const make = Effect.gen(function* () { const replacesFile = yield* fileSystem .exists(realTargetPath) .pipe(Effect.orElseSucceed(() => false)); + // readFile decodes the BOM away, so an editor save would otherwise drop it from the file. + const keepsBom = + replacesFile && + !input.contents.startsWith(UTF8_BOM) && + (yield* Effect.promise(() => startsWithUtf8Bom(realTargetPath))); + const contents = keepsBom ? UTF8_BOM + input.contents : input.contents; yield* fileSystem.makeDirectory(path.dirname(realTargetPath), { recursive: true }).pipe( Effect.mapError( @@ -567,7 +606,7 @@ export const make = Effect.gen(function* () { yield* Effect.tryPromise({ // A symlink swapped in after the check is not followed. Windows lacks O_NOFOLLOW. try: () => - NodeFSP.writeFile(realTargetPath, input.contents, { + NodeFSP.writeFile(realTargetPath, contents, { flag: NodeFS.constants.O_WRONLY | NodeFS.constants.O_CREAT | diff --git a/apps/server/src/workspace/WorkspacePaths.ts b/apps/server/src/workspace/WorkspacePaths.ts index ea183ed43e1a..e630643c71be 100644 --- a/apps/server/src/workspace/WorkspacePaths.ts +++ b/apps/server/src/workspace/WorkspacePaths.ts @@ -175,15 +175,14 @@ const make = Effect.gen(function* () { const resolveRelativePathWithinRoot: WorkspacePaths["Service"]["resolveRelativePathWithinRoot"] = Effect.fn("WorkspacePaths.resolveRelativePathWithinRoot")(function* (input) { - const normalizedInputPath = input.relativePath.trim(); - if (path.isAbsolute(normalizedInputPath)) { + if (path.isAbsolute(input.relativePath)) { return yield* new WorkspacePathOutsideRootError({ workspaceRoot: input.workspaceRoot, relativePath: input.relativePath, }); } - const absolutePath = path.resolve(input.workspaceRoot, normalizedInputPath); + const absolutePath = path.resolve(input.workspaceRoot, input.relativePath); // Only the host separator becomes `/`: on POSIX `\` is a file name character. const relativeToRoot = path .relative(input.workspaceRoot, absolutePath) diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts index 2f9ac05bf9e3..6c4e873f12b4 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts @@ -4,17 +4,26 @@ import { type GrepCursor, type GrepOptions, type GrepResult, + type WatchEvent, } from "@ff-labs/fff-node"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { afterEach, expect, it } from "@effect/vitest"; import * as HostProcess from "@t3tools/shared/HostProcess"; +import { PROJECT_FAVICON_EXTENSIONS, isProjectFaviconPath } from "@t3tools/shared/projectFavicon"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; +import * as TestClock from "effect/testing/TestClock"; import { vi } from "vite-plus/test"; +import * as VcsProcess from "../vcs/VcsProcess.ts"; import * as WorkspaceSearchIndex from "./WorkspaceSearchIndex.ts"; +const testLayer = VcsProcess.layer.pipe(Layer.provideMerge(NodeServices.layer)); + afterEach(() => { vi.restoreAllMocks(); }); @@ -32,17 +41,122 @@ function fileItem(relativePath: string): FileItem { }; } -it.effect("filters image searches before applying the result limit", () => +it.live("image searches match the image subset of a file search without fetching every file", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const vcsProcess = yield* VcsProcess.VcsProcess; + const writeFiles = Effect.fn(function* (root: string, relativePaths: ReadonlyArray) { + for (const relativePath of relativePaths) { + yield* fileSystem.makeDirectory(path.dirname(path.join(root, relativePath)), { + recursive: true, + }); + yield* fileSystem.writeFileString(path.join(root, relativePath), ""); + yield* fileSystem.utimes(path.join(root, relativePath), 0, 0); + } + }); + + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-image-search-" }); + yield* vcsProcess.run({ operation: "test.gitInit", command: "git", args: ["init"], cwd }); + yield* fileSystem.writeFileString(path.join(cwd, ".git/info/exclude"), "hidden/\n"); + yield* writeFiles(cwd, [ + ...Array.from({ length: 220 }, (_, index) => `assets/source-${index}.ts`), + ...PROJECT_FAVICON_EXTENSIONS.flatMap((extension) => [ + `assets/logo${extension}`, + `other/logo${extension.toUpperCase()}`, + ]), + "assets/logo.ts", + "assets/logotype.png", + "assetss/logotype.png", + "assets/images/logo.PNG", + "assets/foo{bar}.png", + "assets/[id].png", + "hidden/logo.png", + ]); + + const fileSearch = vi.spyOn(FileFinder.prototype, "fileSearchAsync"); + const searchIndex = yield* WorkspaceSearchIndex.make(cwd); + yield* searchIndex.search("", 200, undefined, true); + const nativeResult = yield* Effect.promise( + (): ReturnType => fileSearch.mock.results[0]?.value, + ); + if (!nativeResult.ok) return yield* Effect.die(nativeResult.error); + expect(nativeResult.value.items.length).toBeLessThanOrEqual(201); + + // The image glob must not change fff's parse of fuzzy paths, globs, negations or locations. + for (const query of [ + "", + "logo", + "lgo", + "*.png", + "!*.png", + "assets/**", + "!assets/", + "assets*/", + "/assets", + "assets/", + "logo:12", + "/assets/logotype.png", + "assets/images/", + "logo *.svg !other", + "foo{bar}.png", + "[id].png", + "*.{png,svg}", + "hidden/", + ]) { + const ordinary = yield* searchIndex.search(query, 1000, "file"); + const expected = ordinary.entries.filter((entry) => isProjectFaviconPath(entry.path)); + for (const limit of [1, 200]) { + const images = yield* searchIndex.search(query, limit, "directory", true); + expect(images, `query=${query} limit=${limit}`).toEqual({ + entries: expected.slice(0, limit), + truncated: expected.length > limit, + }); + } + } + + // Past 15,000 matches fff drops its filename fallback, which the image glob would otherwise + // reorder: "x/logo/logo.png" must keep its place relative to "a/xxlogo.png". + const largeCwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-image-rank-" }); + yield* writeFiles(largeCwd, [ + ...Array.from({ length: 14_998 }, (_, index) => `b/logo-${index}.png`), + "x/logo/source.ts", + "a/xxlogo.png", + "x/logo/logo.png", + ]); + const largeIndex = yield* WorkspaceSearchIndex.make(largeCwd); + const pairPaths = ["a/xxlogo.png", "x/logo/logo.png"]; + const ordinary = yield* largeIndex.search("logo", 20_000, "file"); + const images = yield* largeIndex.search("logo", 20_000, "file", true); + expect(images.entries).toHaveLength(15_000); + expect(images.entries.filter((entry) => pairPaths.includes(entry.path))).toEqual( + ordinary.entries.filter((entry) => pairPaths.includes(entry.path)), + ); + }).pipe(Effect.scoped, Effect.provide(testLayer)), +); + +it.effect("fetches past a first page of Git-ignored results", () => Effect.scoped( Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-search-ignored-" }); + const vcsProcess = yield* VcsProcess.VcsProcess; + yield* vcsProcess.run({ operation: "test.gitInit", command: "git", args: ["init"], cwd }); + yield* fs.writeFileString(path.join(cwd, ".git", "info", "exclude"), "hidden/\n"); + yield* fs.makeDirectory(path.join(cwd, "hidden")); + yield* fs.writeFileString(path.join(cwd, "hidden", "a.ts"), ""); + const items = [ - ...Array.from({ length: 200 }, (_, index) => fileItem(`src/file-${index}.ts`)), - fileItem("public/icon.svg"), + fileItem("hidden/a.ts"), + fileItem("hidden/b.ts"), + fileItem("hidden/c.ts"), + fileItem("src/a.ts"), ]; - const fileSearch = vi.fn(() => ({ + const fileSearch = vi.fn(async (_query: string, options?: { pageSize?: number }) => ({ ok: true as const, value: { - items, + items: items.slice(0, options?.pageSize), scores: [], totalMatched: items.length, totalFiles: items.length, @@ -51,20 +165,51 @@ it.effect("filters image searches before applying the result limit", () => const finder = { destroy: vi.fn(), waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), - fileSearch, + getScanProgressAsync: async () => ({ + ok: true as const, + value: { isScanning: false }, + }), + watch: () => ({ ok: true as const, value: () => undefined }), + fileSearchAsync: fileSearch, } as unknown as FileFinder; vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); - const searchIndex = yield* WorkspaceSearchIndex.make("/workspace/project"); - const resultWithoutKind = yield* searchIndex.search("", 200, undefined, true); - const resultWithDirectoryKind = yield* searchIndex.search("", 200, "directory", true); + const searchIndex = yield* WorkspaceSearchIndex.make(cwd); + const result = yield* searchIndex.search("a", 1, "file", false); - expect(resultWithoutKind.entries).toEqual([{ kind: "file", path: "public/icon.svg" }]); - expect(resultWithDirectoryKind.entries).toEqual([{ kind: "file", path: "public/icon.svg" }]); - expect(fileSearch).toHaveBeenCalledTimes(2); - expect(fileSearch).toHaveBeenCalledWith("", { pageSize: 25_002 }); + expect(result).toEqual({ entries: [{ kind: "file", path: "src/a.ts" }], truncated: false }); }), - ), + ).pipe(Effect.provide(testLayer)), +); + +// A backslash is a file name character on Linux and macOS, so only Windows may treat it as a separator. +it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( + "keeps a backslash in a POSIX folder name as part of the search result path", + () => + Effect.scoped( + Effect.gen(function* () { + const items = [fileItem("dir\\one/report.txt")]; + const finder = { + destroy: vi.fn(), + waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), + getScanProgressAsync: async () => ({ + ok: true as const, + value: { isScanning: false }, + }), + watch: () => ({ ok: true as const, value: () => undefined }), + fileSearchAsync: vi.fn(async () => ({ + ok: true as const, + value: { items, scores: [], totalMatched: 1, totalFiles: 1 }, + })), + } as unknown as FileFinder; + vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); + + const searchIndex = yield* WorkspaceSearchIndex.make("/workspace/project"); + const result = yield* searchIndex.search("report", 50, "file"); + + expect(result.entries).toEqual([{ kind: "file", path: "dir\\one/report.txt" }]); + }), + ).pipe(Effect.provide(testLayer)), ); it.effect("continues whole-word searches after a filtered grep page", () => @@ -103,7 +248,7 @@ it.effect("continues whole-word searches after a filtered grep page", () => filteredFileCount: 1, nextCursor: cursor, }); - const grep = vi.fn((_query: string, options?: GrepOptions) => + const grep = vi.fn(async (_query: string, options?: GrepOptions) => options?.cursor ? { ok: true as const, value: grepResult("needle", [[0, 6]], null) } : { @@ -114,7 +259,12 @@ it.effect("continues whole-word searches after a filtered grep page", () => const finder = { destroy: vi.fn(), waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), - grep, + getScanProgressAsync: async () => ({ + ok: true as const, + value: { isScanning: false }, + }), + watch: () => ({ ok: true as const, value: () => undefined }), + grepAsync: grep, } as unknown as FileFinder; vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); @@ -141,7 +291,250 @@ it.effect("continues whole-word searches after a filtered grep page", () => expect(grep).toHaveBeenCalledTimes(2); expect(grep.mock.calls[1]?.[1]?.cursor).toBe(nextCursor); }), - ), + ).pipe(Effect.provide(testLayer)), +); + +function fakeSymlinkFinder( + directories: ReadonlyArray, + options: { + readonly indexed?: ReadonlyArray<{ + readonly path: string; + readonly kind: "file" | "directory"; + }>; + readonly watchFailures?: number; + } = {}, +) { + const watchers: Array<(events: ReadonlyArray) => void> = []; + let watchFailures = options.watchFailures ?? 0; + // fff's pageIndex is an offset into the results, not a page number. + const directorySearch = vi.fn( + async (_query: string, searchOptions?: { pageIndex?: number; pageSize?: number }) => { + const offset = searchOptions?.pageIndex ?? 0; + const items = directories.slice(offset, offset + (searchOptions?.pageSize ?? 100)); + return { + ok: true as const, + value: { + items: items.map((relativePath) => ({ relativePath })), + totalMatched: directories.length, + }, + }; + }, + ); + const indexedItems = (options.indexed ?? []).map((entry) => ({ + type: entry.kind, + item: { relativePath: entry.path, fileName: entry.path.split("/").at(-1) }, + })); + const finder = { + destroy: vi.fn(), + waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), + // fff's isScanning waits on the picker lock on the JS thread. + isScanning: () => { + throw new Error("isScanning must not run on the JS thread"); + }, + getScanProgressAsync: async () => ({ + ok: true as const, + value: { isScanning: false }, + }), + watch: (callback: (events: ReadonlyArray) => void) => { + if (watchFailures > 0) { + watchFailures -= 1; + throw new Error("File system watcher is not ready"); + } + watchers.push(callback); + return { ok: true as const, value: () => undefined }; + }, + directorySearchAsync: directorySearch, + fileSearchAsync: async () => { + const items = indexedItems.flatMap((entry) => (entry.type === "file" ? [entry.item] : [])); + return { ok: true as const, value: { items, scores: [], totalMatched: items.length } }; + }, + mixedSearchAsync: async () => ({ + ok: true as const, + value: { items: indexedItems, totalMatched: indexedItems.length }, + }), + } as unknown as FileFinder; + vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); + return { + directorySearch, + notifyWatchers: (events: ReadonlyArray) => + watchers.forEach((watcher) => watcher(events)), + }; +} + +const listedPaths = (searchIndex: WorkspaceSearchIndex.WorkspaceSearchIndex["Service"]) => + searchIndex.list().pipe(Effect.map((result) => result.entries.map((entry) => entry.path))); + +it.effect("reads symlinks again after the watcher reports a change", () => + Effect.scoped( + Effect.gen(function* () { + if ((yield* HostProcess.Platform) === "win32") return; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-symlink-watch-" }); + const outside = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-outside-" }); + yield* fileSystem.writeFileString(path.join(cwd, "a.ts"), ""); + yield* fileSystem.symlink("a.ts", path.join(cwd, "old-link.ts")); + const { directorySearch, notifyWatchers } = fakeSymlinkFinder([]); + + const searchIndex = yield* WorkspaceSearchIndex.make(cwd); + const concurrent = yield* Effect.all( + Array.from({ length: 8 }, () => listedPaths(searchIndex)), + { concurrency: "unbounded" }, + ); + yield* fileSystem.remove(path.join(cwd, "old-link.ts")); + yield* fileSystem.symlink("a.ts", path.join(cwd, "new-link.ts")); + const beforeNotice = yield* listedPaths(searchIndex); + // Editing a file cannot move a link, so it does not cost a rescan. + notifyWatchers([{ kind: "modified", path: path.join(cwd, "a.ts") }]); + const afterEdit = yield* listedPaths(searchIndex); + notifyWatchers([{ kind: "created", path: path.join(cwd, "new-link.ts") }]); + const afterNotice = yield* listedPaths(searchIndex); + // fff reports a link replaced in place as modified. + yield* fileSystem.remove(path.join(cwd, "new-link.ts")); + yield* fileSystem.symlink(outside, path.join(cwd, "new-link.ts")); + notifyWatchers([{ kind: "modified", path: path.join(cwd, "new-link.ts") }]); + const afterReplace = yield* listedPaths(searchIndex); + + expect(concurrent).toEqual(Array.from({ length: 8 }, () => ["old-link.ts"])); + expect(beforeNotice).toEqual(["old-link.ts"]); + expect(afterEdit).toEqual(["old-link.ts"]); + expect(afterNotice).toEqual(["new-link.ts"]); + expect(afterReplace).toEqual([]); + expect(directorySearch).toHaveBeenCalledTimes(3); + }), + ).pipe(Effect.provide(testLayer)), +); + +it.effect("answers from the last symlink scan while a slow rescan runs", () => + Effect.scoped( + Effect.gen(function* () { + if ((yield* HostProcess.Platform) === "win32") return; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-symlink-stale-" }); + yield* fileSystem.writeFileString(path.join(cwd, "a.ts"), ""); + yield* fileSystem.symlink("a.ts", path.join(cwd, "old-link.ts")); + const { notifyWatchers } = fakeSymlinkFinder([]); + // Holding git's ignore check open keeps the rescan from finishing. + const gate = yield* Deferred.make(); + let holdIgnoreChecks = false; + const vcsProcess = yield* VcsProcess.VcsProcess; + const searchIndex = yield* WorkspaceSearchIndex.make(cwd).pipe( + Effect.provideService(VcsProcess.VcsProcess, { + ...vcsProcess, + run: (input) => + holdIgnoreChecks && input.operation === "WorkspaceSearchIndex.listGitIgnored" + ? Deferred.await(gate).pipe(Effect.andThen(vcsProcess.run(input))) + : vcsProcess.run(input), + }), + ); + expect(yield* listedPaths(searchIndex)).toEqual(["old-link.ts"]); + + holdIgnoreChecks = true; + yield* fileSystem.symlink("a.ts", path.join(cwd, "new-link.ts")); + notifyWatchers([{ kind: "created", path: path.join(cwd, "new-link.ts") }]); + const duringRescan = yield* Effect.forkChild(listedPaths(searchIndex)); + yield* TestClock.adjust("1 second"); + expect(yield* Fiber.join(duringRescan)).toEqual(["old-link.ts"]); + + yield* Deferred.succeed(gate, undefined); + expect(yield* listedPaths(searchIndex)).toEqual(["new-link.ts", "old-link.ts"]); + }), + ).pipe(Effect.provide(testLayer)), +); + +it.effect("keeps symlinks once the watcher subscribes after starting late", () => + Effect.scoped( + Effect.gen(function* () { + if ((yield* HostProcess.Platform) === "win32") return; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-symlink-watch-" }); + yield* fileSystem.writeFileString(path.join(cwd, "a.ts"), ""); + yield* fileSystem.symlink("a.ts", path.join(cwd, "link.ts")); + const { directorySearch } = fakeSymlinkFinder([], { watchFailures: 2 }); + + const searchIndex = yield* WorkspaceSearchIndex.make(cwd); + const scansAfterRound: number[] = []; + for (let round = 0; round < 5; round += 1) { + expect(yield* listedPaths(searchIndex)).toEqual(["link.ts"]); + expect(yield* listedPaths(searchIndex)).toEqual(["link.ts"]); + scansAfterRound.push(directorySearch.mock.calls.length); + yield* TestClock.adjust("1 minute"); + } + + // An unwatched scan serves reads only briefly; the third has a watcher and is kept. + expect(scansAfterRound).toEqual([1, 2, 3, 3, 3]); + }), + ).pipe(Effect.provide(testLayer)), +); + +it.effect( + "lists a symlink the watcher indexed once, with the link's kind, and drops rejected ones", + () => + Effect.scoped( + Effect.gen(function* () { + if ((yield* HostProcess.Platform) === "win32") return; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-symlink-dupe-" }); + const outside = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-outside-" }); + yield* fileSystem.makeDirectory(path.join(cwd, "src")); + yield* fileSystem.symlink("src", path.join(cwd, "src-link")); + // A link the scan rejects stays out even when the watcher indexed it. + yield* fileSystem.symlink(outside, path.join(cwd, "outside-link")); + fakeSymlinkFinder([], { + indexed: [ + { path: "outside-link", kind: "file" }, + { path: "src-link", kind: "file" }, + { path: "link-file.ts", kind: "file" }, + ], + }); + + const searchIndex = yield* WorkspaceSearchIndex.make(cwd); + const searched = yield* searchIndex.search("link", 10); + const files = yield* searchIndex.search("link", 1, "file"); + const listed = yield* searchIndex.list(); + + expect(searched.entries).toEqual([ + { path: "src-link", kind: "directory" }, + { path: "link-file.ts", kind: "file" }, + ]); + // The dropped link does not take the only slot. + expect(files).toEqual({ + entries: [{ path: "link-file.ts", kind: "file" }], + truncated: false, + }); + expect(listed.entries).toEqual([ + { path: "link-file.ts", kind: "file" }, + { path: "src-link", kind: "directory" }, + ]); + }), + ).pipe(Effect.provide(testLayer)), +); + +it.effect("finds symlinks in folders past the first page of the index", () => + Effect.scoped( + Effect.gen(function* () { + if ((yield* HostProcess.Platform) === "win32") return; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-symlink-pages-" }); + yield* fileSystem.makeDirectory(path.join(cwd, "late")); + yield* fileSystem.writeFileString(path.join(cwd, "late/a.ts"), ""); + yield* fileSystem.symlink("a.ts", path.join(cwd, "late/link.ts")); + // Folders fff lists but that are gone by the time they are read are skipped. + const firstPage = Array.from({ length: 1_000 }, (_, index) => `missing-${index}`); + const { directorySearch } = fakeSymlinkFinder([...firstPage, "late"]); + + const searchIndex = yield* WorkspaceSearchIndex.make(cwd); + + expect(yield* listedPaths(searchIndex)).toEqual(["late", "late/link.ts"]); + expect(directorySearch.mock.calls.map(([, options]) => options?.pageIndex)).toEqual([ + 0, 1_000, + ]); + }), + ).pipe(Effect.provide(testLayer)), ); // fff reads HOME on Linux and macOS but the user profile on Windows, so a temp HOME cannot stand in there. @@ -172,5 +565,5 @@ it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( expect(result).toBe("WorkspaceSearchIndexCreateFailed"); }), - ).pipe(Effect.provide(NodeServices.layer)), + ).pipe(Effect.provide(testLayer)), ); diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.ts b/apps/server/src/workspace/WorkspaceSearchIndex.ts index 3b2e90b576c5..7813055e9b83 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.ts @@ -1,22 +1,28 @@ +// @effect-diagnostics nodeBuiltinImport:off - readdir with file types avoids an lstat per child +import * as NodeFSP from "node:fs/promises"; import * as NodeModule from "node:module"; import * as NodeSea from "node:sea"; import * as NodeURL from "node:url"; import type { DirItem, - DirSearchResult, FileItem, FileFinder as FileFinderType, GrepCursor, MixedItem, - MixedSearchResult, Result, - SearchResult, + WatchEvent, + WatchUnsubscribe, } from "@ff-labs/fff-node"; +import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as LayerMap from "effect/LayerMap"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; import type { @@ -27,7 +33,12 @@ import type { ProjectSearchContentsResult, ProjectSearchEntriesResult, } from "@t3tools/contracts"; -import { isProjectFaviconPath } from "@t3tools/shared/projectFavicon"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import { PROJECT_FAVICON_EXTENSIONS, isProjectFaviconPath } from "@t3tools/shared/projectFavicon"; +import { scoreSubsequenceMatch } from "@t3tools/shared/searchRanking"; + +import * as VcsProcess from "../vcs/VcsProcess.ts"; +import { type IgnoreDecision, type IgnoreFile, parseIgnoreFile } from "./IgnorePatterns.ts"; // fff-node's native library stays external; single-executables require disk packages instead of importing them. const requireForFff = NodeModule.createRequire(import.meta.url); @@ -38,6 +49,14 @@ const fffEntry = NodeSea.isSea() : NodeURL.fileURLToPath(import.meta.resolve("@ff-labs/fff-node")); const { FileFinder } = requireForFff(fffEntry) as typeof import("@ff-labs/fff-node"); +// A glob intersects existing extension constraints; fff ORs bare `*.ext` constraints together. +const IMAGE_SEARCH_CONSTRAINT = `**/*.{${PROJECT_FAVICON_EXTENSIONS.map((extension) => + extension.slice(1).replace(/[a-z]/g, (letter) => `[${letter}${letter.toUpperCase()}]`), +).join(",")}}`; + +// fff 0.11 disables filename fallback above this match count per base/overflow segment. +const FFF_FILENAME_FALLBACK_MAX_MATCHES = 15_000; + const WORKSPACE_INDEX_MAX_ENTRIES = 25_000; const WORKSPACE_INDEX_PAGE_SIZE = WORKSPACE_INDEX_MAX_ENTRIES + 2; const WORKSPACE_INDEX_SCAN_TIMEOUT = "15 seconds"; @@ -45,6 +64,13 @@ const WORKSPACE_INDEX_SCAN_TIMEOUT_MS = 15_000; const WORKSPACE_INDEX_IDLE_TTL = "15 minutes"; const CONTENT_SEARCH_TIME_BUDGET_MS = 250; const CONTENT_SEARCH_MAX_MATCHES_PER_FILE = 100; +const SYMLINK_SCAN_CONCURRENCY = 16; +// A scan no watcher can invalidate is reused this long, so typing does not rescan every folder. +const UNWATCHED_SYMLINK_SCAN_TTL_MS = 2_000; +// A rescan that outlasts this leaves the read on the previous scan. +const SYMLINK_RESCAN_WAIT = "5 millis"; +// Each page is a synchronous native call; small pages let searches run between them. +const SYMLINK_DIRECTORY_PAGE_SIZE = 1_000; export class WorkspaceSearchIndexCreateFailed extends Schema.TaggedError()( "WorkspaceSearchIndexCreateFailed", @@ -123,13 +149,17 @@ export class WorkspaceSearchIndex extends Context.Service< ) => Effect.Effect; readonly searchContents: ( input: Omit, - ) => Effect.Effect; + ) => Effect.Effect< + ProjectSearchContentsResult, + WorkspaceSearchIndexScanTimedOut | WorkspaceSearchIndexSearchFailed + >; readonly refresh: () => Effect.Effect; } >()("t3/workspace/WorkspaceSearchIndex") {} -function toPosixPath(input: string): string { - return input.replaceAll("\\", "/"); +// Only the host separator becomes `/`: on POSIX `\` is a file name character. +function toPosixPath(input: string, hostIsWindows: boolean): string { + return hostIsWindows ? input.replaceAll("\\", "/") : input; } function trimDirectorySeparator(input: string): string { @@ -141,8 +171,8 @@ function parentPathOf(input: string): string | undefined { return separatorIndex === -1 ? undefined : input.slice(0, separatorIndex); } -function toProjectEntry(item: MixedItem): ProjectEntry | null { - const normalizedPath = trimDirectorySeparator(toPosixPath(item.item.relativePath)); +function toProjectEntry(item: MixedItem, hostIsWindows: boolean): ProjectEntry | null { + const normalizedPath = trimDirectorySeparator(toPosixPath(item.item.relativePath, hostIsWindows)); if (!normalizedPath) { return null; } @@ -153,23 +183,34 @@ function toProjectEntry(item: MixedItem): ProjectEntry | null { }; } -function toFileEntry(item: FileItem): ProjectEntry | null { - const normalizedPath = trimDirectorySeparator(toPosixPath(item.relativePath)); +function toFileEntry(item: FileItem, hostIsWindows: boolean): ProjectEntry | null { + const normalizedPath = trimDirectorySeparator(toPosixPath(item.relativePath, hostIsWindows)); return normalizedPath ? { path: normalizedPath, kind: "file" } : null; } -function toDirectoryEntry(item: DirItem): ProjectEntry | null { - const normalizedPath = trimDirectorySeparator(toPosixPath(item.relativePath)); +function toDirectoryEntry(item: DirItem, hostIsWindows: boolean): ProjectEntry | null { + const normalizedPath = trimDirectorySeparator(toPosixPath(item.relativePath, hostIsWindows)); return normalizedPath ? { path: normalizedPath, kind: "directory" } : null; } +function isLoneFuzzyPath(query: string): boolean { + const trimmed = query.trim(); + return ( + !/\s/u.test(trimmed) && + (trimmed.startsWith("/") || (trimmed.endsWith("/") && !/^[*!\\]/u.test(trimmed))) + ); +} + +type SearchPage = { readonly items: ReadonlyArray; readonly totalMatched: number }; + function mapFileSearchResult( - result: SearchResult, + result: SearchPage, limit: number, + hostIsWindows: boolean, imageOnly = false, ): ProjectSearchEntriesResult { const entries = result.items.flatMap((item) => { - const entry = toFileEntry(item); + const entry = toFileEntry(item, hostIsWindows); return entry && (!imageOnly || isProjectFaviconPath(entry.path)) ? [entry] : []; }); return { @@ -179,11 +220,12 @@ function mapFileSearchResult( } function mapDirectorySearchResult( - result: DirSearchResult, + result: SearchPage, limit: number, + hostIsWindows: boolean, ): ProjectSearchEntriesResult { const entries = result.items.flatMap((item) => { - const entry = toDirectoryEntry(item); + const entry = toDirectoryEntry(item, hostIsWindows); return entry ? [entry] : []; }); const rootDirectoryCount = result.items.some((item) => item.relativePath.length === 0) ? 1 : 0; @@ -194,12 +236,13 @@ function mapDirectorySearchResult( } function mapMixedSearchResult( - result: MixedSearchResult, + result: SearchPage, limit: number, + hostIsWindows: boolean, ): { readonly entries: ProjectEntry[]; readonly truncated: boolean } { const entries: ProjectEntry[] = []; for (const item of result.items) { - const entry = toProjectEntry(item); + const entry = toProjectEntry(item, hostIsWindows); if (entry) { entries.push(entry); } @@ -291,6 +334,190 @@ function withDirectoryAncestors(entries: ReadonlyArray): ProjectEn return [...entryByPath.values()]; } +interface ScannedDirectory { + readonly directory: string; + readonly links: ReadonlyArray; + /** Every symlink in the folder, including ones dropped for leaving the workspace. */ + readonly linkPaths: ReadonlyArray; + /** A nested repository (or the workspace's own), whose ignore rules own what is below it. */ + readonly isRepository: boolean; + readonly ignoreFile: string | undefined; +} + +const joinEntryPath = (directory: string, name: string) => + directory ? `${directory}/${name}` : name; + +/** + * fff skips symlinks, and `followSymlinks` drops file links, dedupes folder links, and walks out + * of the workspace. So links are read from the folders fff already indexed (ignore rules applied), + * without descending into them, and kept only when they resolve inside the workspace. + */ +const scanDirectory = Effect.fn("WorkspaceSearchIndex.scanDirectory")(function* ( + fileSystem: FileSystem.FileSystem, + path: Path.Path, + cwd: string, + root: string, + directory: string, +) { + // Effect's readDirectory returns bare names; one lstat per child would cost far more. + const children = yield* Effect.promise(() => + NodeFSP.readdir(path.join(cwd, directory), { withFileTypes: true }).catch(() => []), + ); + const links: ProjectEntry[] = []; + const linkPaths: string[] = []; + let isRepository = false; + let ignoreFile: string | undefined; + for (const child of children) { + if (child.name === ".git") isRepository = true; + if (child.name === ".ignore" && child.isFile()) { + ignoreFile = yield* fileSystem + .readFileString(path.join(cwd, directory, child.name)) + .pipe(Effect.orElseSucceed(() => undefined)); + } + if (!child.isSymbolicLink()) continue; + const entryPath = joinEntryPath(directory, child.name); + linkPaths.push(entryPath); + const absolutePath = path.join(cwd, entryPath); + const target = yield* Effect.all([ + fileSystem.realPath(absolutePath), + fileSystem.stat(absolutePath), + ]).pipe(Effect.orElseSucceed(() => undefined)); + if (!target) continue; + const relative = path.relative(root, target[0]); + if ( + relative === ".." || + relative.startsWith(`..${path.sep}`) || + path.isAbsolute(relative) || + relative.split(path.sep).includes(".git") || + (target[1].type !== "Directory" && target[1].type !== "File") + ) { + continue; + } + links.push({ path: entryPath, kind: target[1].type === "Directory" ? "directory" : "file" }); + } + return { directory, links, linkPaths, isRepository, ignoreFile } satisfies ScannedDirectory; +}); + +/** Whether `entryPath` is `directory` itself or below it. */ +const isWithin = (entryPath: string, directory: string) => + directory === "" || entryPath.startsWith(`${directory}/`); + +/** + * fff reads `.ignore` files alongside git's rules, and like ripgrep the nearest one that matches + * decides, ahead of any `.gitignore`. `ignoreFiles` is ordered deepest folder first. + */ +function decideByIgnoreFiles( + entry: ProjectEntry, + ignoreFiles: ReadonlyArray<{ readonly directory: string; readonly file: IgnoreFile }>, +): IgnoreDecision | undefined { + for (const { directory, file } of ignoreFiles) { + if (!isWithin(entry.path, directory) || entry.path === directory) continue; + const relative = directory ? entry.path.slice(directory.length + 1) : entry.path; + // Like git and ripgrep, a link is never a folder to a folder-only rule, whatever it targets. + const decision = file.decide(relative, false); + if (decision) return decision; + } + return undefined; +} + +/** Places each symlink before the first index result that matches the query less closely. */ +function mergeSymlinkMatches( + indexed: ProjectSearchEntriesResult, + linkPaths: ReadonlySet, + symlinks: ReadonlyArray, + query: string, + limit: number, +): ProjectSearchEntriesResult { + if (linkPaths.size === 0) return indexed; + // fff's watcher indexes links created after its scan, under a kind of its own choosing, so a + // path it already returned takes the link's kind (or drops out of a kind-filtered search). + const symlinkByPath = new Map(symlinks.map((entry) => [entry.path, entry])); + const score = (entry: ProjectEntry) => + scoreSubsequenceMatch(entry.path.toLowerCase(), query) ?? Number.POSITIVE_INFINITY; + const indexedPaths = new Set(indexed.entries.map((entry) => entry.path)); + const pending = symlinks + .flatMap((entry) => { + const entryScore = score(entry); + return Number.isFinite(entryScore) && !indexedPaths.has(entry.path) + ? [{ entry, score: entryScore }] + : []; + }) + .toSorted((left, right) => left.score - right.score); + const entries: ProjectEntry[] = []; + let nextPending = 0; + for (const indexedEntry of indexed.entries) { + const entry = linkPaths.has(indexedEntry.path) + ? symlinkByPath.get(indexedEntry.path) + : indexedEntry; + if (!entry) continue; + const entryScore = score(entry); + while (nextPending < pending.length && pending[nextPending]!.score < entryScore) { + entries.push(pending[nextPending++]!.entry); + } + entries.push(entry); + } + for (; nextPending < pending.length; nextPending += 1) entries.push(pending[nextPending]!.entry); + return { + entries: entries.slice(0, limit), + truncated: indexed.truncated || entries.length > limit, + }; +} + +// fff applies .gitignore files only; Git also ignores paths from .git/info/exclude and the global excludes file. +const listGitIgnoredPaths = Effect.fn("WorkspaceSearchIndex.listGitIgnoredPaths")(function* ( + vcsProcess: VcsProcess.VcsProcess["Service"], + cwd: string, + hostIsWindows: boolean, +) { + const result = yield* vcsProcess + .run({ + operation: "WorkspaceSearchIndex.listGitIgnoredPaths", + command: "git", + args: [ + "-c", + "core.fsmonitor=false", + "ls-files", + "--others", + "--ignored", + "--exclude-standard", + "--directory", + "-z", + ], + cwd, + allowNonZeroExit: true, + timeoutMs: 10_000, + maxOutputBytes: 16 * 1024 * 1024, + }) + .pipe(Effect.orElseSucceed(() => undefined)); + if (!result || result.exitCode !== 0 || result.stdoutTruncated) return new Set(); + return new Set( + result.stdout + .split("\0") + .filter((entry) => entry.length > 0) + .map((entry) => trimDirectorySeparator(toPosixPath(entry, hostIsWindows))), + ); +}); + +function mixedItemPath(item: MixedItem): string { + return item.item.relativePath; +} + +function isIgnoredPath( + ignoredPaths: ReadonlySet, + relativePath: string, + hostIsWindows: boolean, +): boolean { + if (ignoredPaths.size === 0) return false; + let candidate: string | undefined = trimDirectorySeparator( + toPosixPath(relativePath, hostIsWindows), + ); + while (candidate) { + if (ignoredPaths.has(candidate)) return true; + candidate = parentPathOf(candidate); + } + return false; +} + const createFinder = Effect.fn("WorkspaceSearchIndex.createFinder")(function* ( cwd: string, variant: WorkspaceSearchIndexVariant, @@ -346,6 +573,9 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( cwd: string, variant: WorkspaceSearchIndexVariant = "paths", ) { + const hostIsWindows = yield* HostProcess.isWindows; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; const finder = yield* Effect.acquireRelease(createFinder(cwd, variant), (finder) => Effect.try({ try: () => finder.destroy(), @@ -353,6 +583,8 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( }).pipe(Effect.orDie), ); let initialScanTimedOut = false; + const vcsProcess = yield* VcsProcess.VcsProcess; + let ignoredPaths = yield* listGitIgnoredPaths(vcsProcess, cwd, hostIsWindows); yield* waitForIndexReady( cwd, finder, @@ -363,32 +595,41 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( cause, }), ).pipe( + // The scan keeps running on this finder; failing here would destroy it and cache the failure. Effect.catchTags({ - WorkspaceSearchIndexScanTimedOut: (error) => - variant === "paths" - ? Effect.sync(() => { - initialScanTimedOut = true; - }) - : Effect.fail(error), + WorkspaceSearchIndexScanTimedOut: () => + Effect.sync(() => { + initialScanTimedOut = true; + }), }), ); - const hasIncompleteInitialScan = () => initialScanTimedOut && finder.isScanning(); + // fff's isScanning takes the picker lock too, so progress is read off the JS thread. + const isScanning = Effect.promise(async () => { + try { + const progress = await finder.getScanProgressAsync(); + return !progress.ok || progress.value.isScanning; + } catch { + return true; + } + }); + const hasIncompleteInitialScan = () => (initialScanTimedOut ? isScanning : Effect.succeed(false)); const runSearch = Effect.fn("WorkspaceSearchIndex.runSearch")(function* ( query: string, pageSize: number, - operation: "directorySearch" | "fileSearch" | "grep" | "mixedSearch", - execute: () => Result, + operation: "directorySearch" | "fileSearch" | "getScanProgress" | "grep" | "mixedSearch", + execute: () => Promise>, ): Effect.fn.Return { - const result = yield* Effect.try({ + // fff's searches wait on its picker lock, which its watcher can hold for minutes. + const result = yield* Effect.tryPromise({ try: execute, catch: (cause) => new WorkspaceSearchIndexSearchFailed({ cwd, queryLength: query.length, pageSize, - reason: `FileFinder.${operation} threw unexpectedly.`, + reason: `FileFinder.${operation} rejected unexpectedly.`, cause, }), }); @@ -403,15 +644,268 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( return result.value; }); + const root = yield* fileSystem.realPath(cwd).pipe(Effect.orElseSucceed(() => cwd)); + + const listIndexedDirectories = Effect.fn("WorkspaceSearchIndex.listIndexedDirectories")( + function* () { + const directories: string[] = []; + // fff's pageIndex is an offset into the results, not a page number. + for (let pageIndex = 0; ; pageIndex += SYMLINK_DIRECTORY_PAGE_SIZE) { + const result = yield* Effect.promise(async () => { + try { + return await finder.directorySearchAsync("", { + pageIndex, + pageSize: SYMLINK_DIRECTORY_PAGE_SIZE, + }); + } catch { + return undefined; + } + }); + if (!result?.ok) return directories; + for (const item of result.value.items) { + const directory = trimDirectorySeparator(toPosixPath(item.relativePath, hostIsWindows)); + if (directory) directories.push(directory); + } + if (result.value.items.length < SYMLINK_DIRECTORY_PAGE_SIZE) return directories; + } + }, + ); + + /** Ignore rules belong to the repository that owns a path, so each one answers for its own. */ + const listGitIgnored = Effect.fn("WorkspaceSearchIndex.listGitIgnored")(function* ( + repository: string, + entries: ReadonlyArray, + ) { + const relativePaths = entries.map((entry) => + repository ? entry.path.slice(repository.length + 1) : entry.path, + ); + const repositoryCwd = path.join(cwd, repository); + const result = yield* vcsProcess + .run({ + operation: "WorkspaceSearchIndex.listGitIgnored", + command: "git", + // fff applies ignore rules to tracked paths too, so the index is not consulted. + args: ["-c", "core.fsmonitor=false", "check-ignore", "--no-index", "-z", "--stdin"], + cwd: repositoryCwd, + stdin: `${relativePaths.join("\0")}\0`, + allowNonZeroExit: true, + timeoutMs: 10_000, + maxOutputBytes: 16 * 1024 * 1024, + }) + .pipe(Effect.orElseSucceed(() => undefined)); + // Exit 1 means nothing matched. + if (result?.exitCode === 1) return []; + if (result?.exitCode !== 0 || result.stdoutTruncated) { + // Outside git no rules apply (git exits before reading stdin, so the cause is lost). + // Any other failure hides the repository's links rather than leaking ones it ignores. + const insideRepository = yield* vcsProcess + .run({ + operation: "WorkspaceSearchIndex.listGitIgnored", + command: "git", + args: ["rev-parse", "--git-dir"], + cwd: repositoryCwd, + timeoutMs: 10_000, + }) + .pipe( + Effect.as(true), + Effect.orElseSucceed(() => false), + ); + return insideRepository ? entries.map((entry) => entry.path) : []; + } + return result.stdout + .split("\0") + .filter(Boolean) + .map((relativePath) => (repository ? `${repository}/${relativePath}` : relativePath)); + }); + + const scanSymlinkEntries = Effect.fn("WorkspaceSearchIndex.scanSymlinkEntries")(function* () { + const indexComplete = !(yield* isScanning); + const directories = yield* listIndexedDirectories(); + const scanned = yield* Effect.forEach( + ["", ...directories], + (directory) => scanDirectory(fileSystem, path, cwd, root, directory), + { concurrency: SYMLINK_SCAN_CONCURRENCY }, + ); + const candidates = scanned.flatMap((directory) => directory.links); + const linkPaths = new Set(scanned.flatMap((directory) => directory.linkPaths)); + if (candidates.length === 0) return { entries: candidates, linkPaths, indexComplete }; + + const ignoreFiles = scanned + .flatMap(({ directory, ignoreFile }) => + ignoreFile === undefined ? [] : [{ directory, file: parseIgnoreFile(ignoreFile) }], + ) + .toSorted((left, right) => right.directory.length - left.directory.length); + const decisions = new Map( + candidates.map((entry) => [entry.path, decideByIgnoreFiles(entry, ignoreFiles)]), + ); + const repositories = [ + "", + ...scanned.flatMap(({ directory, isRepository }) => + isRepository && directory ? [directory] : [], + ), + ].toSorted((left, right) => right.length - left.length); + const entriesByRepository = Map.groupBy( + candidates.filter((entry) => decisions.get(entry.path) === undefined), + (entry) => + repositories.find( + (repository) => isWithin(entry.path, repository) && entry.path !== repository, + ) ?? "", + ); + const ignored = new Set( + (yield* Effect.forEach( + entriesByRepository, + ([repository, entries]) => listGitIgnored(repository, entries), + { concurrency: 4 }, + )).flat(), + ); + return { + entries: candidates.filter((entry) => + decisions.get(entry.path) === undefined + ? !ignored.has(entry.path) + : decisions.get(entry.path) === "included", + ), + linkPaths, + indexComplete, + }; + }); + + // One scan runs at a time and outlives the reads that started it. Only the first read waits for + // it in full: after that a change or expiry marks the result stale, and reads wait briefly for + // the rescan before answering from the last finished one, so file churn cannot stall search. + type SymlinkScan = Effect.Success>; + let lastSymlinkScan: SymlinkScan | undefined; + let lastSymlinkScanExpiresAt: number | undefined; + let pendingSymlinkScan: Deferred.Deferred | undefined; + let symlinkScanStale = false; + let knownLinkPaths: ReadonlySet = new Set(); + const invalidateSymlinkScan = () => { + symlinkScanStale = true; + }; + const isKnownLink = (absolutePath: string) => + [cwd, root].some((base) => + knownLinkPaths.has(toPosixPath(path.relative(base, absolutePath), hostIsWindows)), + ); + const onWatchBatch = (events: ReadonlyArray) => { + // fff reports a link replaced in place as `modified`, so edits to known links count too. + const editsOnly = events.every( + (event) => + event.kind === "modified" && + !/(^|[\\/])\.(git)?ignore$/.test(event.path) && + !isKnownLink(event.path), + ); + if (!editsOnly) invalidateSymlinkScan(); + }; + // fff refuses subscriptions until its watcher is up, which can trail the initial scan, so + // each read retries until one is in place. + let unwatch: WatchUnsubscribe | undefined; + yield* Effect.addFinalizer(() => Effect.sync(() => unwatch?.())); + const ensureWatching = () => { + if (unwatch) return true; + try { + const result = finder.watch(onWatchBatch); + if (result.ok) unwatch = result.value; + } catch { + // Not ready yet; the next read tries again. + } + return unwatch !== undefined; + }; + const indexScope = yield* Effect.scope; + const startSymlinkScan = Effect.fn("WorkspaceSearchIndex.startSymlinkScan")(function* () { + symlinkScanStale = false; + // Subscribing before the scan starts means no change during it goes unseen. + const watched = ensureWatching(); + const done = yield* Deferred.make(); + pendingSymlinkScan = done; + yield* scanSymlinkEntries().pipe( + Effect.tap((result) => + Effect.gen(function* () { + lastSymlinkScan = result; + knownLinkPaths = result.linkPaths; + // Without a watcher, or before fff has seen every folder, the result is kept only briefly. + lastSymlinkScanExpiresAt = + watched && result.indexComplete + ? undefined + : (yield* Clock.currentTimeMillis) + UNWATCHED_SYMLINK_SCAN_TTL_MS; + }), + ), + Effect.onExit((exit) => + Effect.sync(() => { + pendingSymlinkScan = undefined; + }).pipe(Effect.andThen(Deferred.done(done, exit))), + ), + Effect.forkIn(indexScope), + ); + return done; + }); + const listSymlinkEntries = Effect.fn("WorkspaceSearchIndex.listSymlinkEntries")(function* () { + if ( + lastSymlinkScanExpiresAt !== undefined && + (yield* Clock.currentTimeMillis) >= lastSymlinkScanExpiresAt + ) { + symlinkScanStale = true; + } + const pending = + pendingSymlinkScan ?? + (lastSymlinkScan === undefined || symlinkScanStale ? yield* startSymlinkScan() : undefined); + let result = lastSymlinkScan; + if (pending && result === undefined) { + result = yield* Deferred.await(pending); + } else if (pending) { + const fresh = yield* Deferred.await(pending).pipe(Effect.timeoutOption(SYMLINK_RESCAN_WAIT)); + if (Option.isSome(fresh)) result = fresh.value; + } + if (result === undefined) + return yield* Effect.die("The symlink scan finished without a result."); + return { + // Git's ignore rules hide a link the same way searchVisible hides the files it indexes. + entries: result.entries.filter( + (entry) => !isIgnoredPath(ignoredPaths, entry.path, hostIsWindows), + ), + linkPaths: result.linkPaths, + }; + }); + + // Grows the first page until hidden entries can no longer crowd visible ones out of it. + const searchVisible = Effect.fn("WorkspaceSearchIndex.searchVisible")(function* ( + query: string, + pageSize: number, + operation: "directorySearch" | "fileSearch" | "mixedSearch", + execute: (pageSize: number) => Promise & Metadata>>, + pathOf: (item: I) => string, + ): Effect.fn.Return & Metadata, WorkspaceSearchIndexSearchFailed> { + let size = pageSize; + while (true) { + const fetchSize = size; + const result = yield* runSearch(query, fetchSize, operation, () => execute(fetchSize)); + if (ignoredPaths.size === 0) return result; + const items = result.items.filter( + (item) => !isIgnoredPath(ignoredPaths, pathOf(item), hostIsWindows), + ); + if ( + items.length >= pageSize || + result.items.length >= result.totalMatched || + fetchSize >= WORKSPACE_INDEX_PAGE_SIZE + ) { + return { + ...result, + items, + totalMatched: result.totalMatched - (result.items.length - items.length), + }; + } + size = Math.min(WORKSPACE_INDEX_PAGE_SIZE, fetchSize * 4); + } + }); + const refresh: WorkspaceSearchIndex["Service"]["refresh"] = Effect.fn( "WorkspaceSearchIndex.refresh", )(function* () { - const result = yield* Effect.try({ - try: () => finder.scanFiles(), + // scanFiles waits on fff's picker lock, which its watcher can hold for seconds. + const result = yield* Effect.tryPromise({ + try: () => finder.scanFilesAsync(), catch: (cause) => new WorkspaceSearchIndexRefreshFailed({ cwd, - reason: "FileFinder.scanFiles threw unexpectedly.", + reason: "FileFinder.scanFilesAsync rejected unexpectedly.", cause, }), }); @@ -433,17 +927,26 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( cause, }), ).pipe(Effect.catchTags({ WorkspaceSearchIndexScanTimedOut: () => Effect.void })); + ignoredPaths = yield* listGitIgnoredPaths(vcsProcess, cwd, hostIsWindows); + invalidateSymlinkScan(); }); const list: WorkspaceSearchIndex["Service"]["list"] = Effect.fn("WorkspaceSearchIndex.list")( function* () { - const incompleteBeforeQuery = hasIncompleteInitialScan(); - const result = yield* runSearch("", WORKSPACE_INDEX_PAGE_SIZE, "mixedSearch", () => - finder.mixedSearch("", { pageSize: WORKSPACE_INDEX_PAGE_SIZE }), + const incompleteBeforeQuery = yield* hasIncompleteInitialScan(); + const result = yield* searchVisible( + "", + WORKSPACE_INDEX_PAGE_SIZE, + "mixedSearch", + (pageSize) => finder.mixedSearchAsync("", { pageSize }), + mixedItemPath, ); - const mapped = mapMixedSearchResult(result, WORKSPACE_INDEX_MAX_ENTRIES); - const sortedEntries = withDirectoryAncestors(mapped.entries).toSorted((left, right) => - left.path.localeCompare(right.path), + const mapped = mapMixedSearchResult(result, WORKSPACE_INDEX_MAX_ENTRIES, hostIsWindows); + const symlinks = yield* listSymlinkEntries(); + // fff's watcher can index a link the scan rejected; only accepted links are listed. + const indexed = mapped.entries.filter((entry) => !symlinks.linkPaths.has(entry.path)); + const sortedEntries = withDirectoryAncestors([...indexed, ...symlinks.entries]).toSorted( + (left, right) => left.path.localeCompare(right.path), ); const entries = sortedEntries.slice(0, WORKSPACE_INDEX_MAX_ENTRIES); return { @@ -454,38 +957,108 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( }, ); - const search: WorkspaceSearchIndex["Service"]["search"] = Effect.fn( - "WorkspaceSearchIndex.search", - )(function* (rawQuery, limit, kind, imageOnly) { - // fff parses any word containing "[" as a glob and has no escape for it; fuzzy - // matching is subsequence-based, so dropping "[" still matches "[id]" folders. - const query = rawQuery.replaceAll("[", ""); - const pageSize = imageOnly ? WORKSPACE_INDEX_PAGE_SIZE : Math.max(1, limit + 1); - const incompleteBeforeQuery = hasIncompleteInitialScan(); + const searchNames = Effect.fn("WorkspaceSearchIndex.searchNames")(function* ( + query: string, + limit: number, + kind: ProjectEntryKind | undefined, + imageOnly: boolean | undefined, + ) { + const { entries: allSymlinks, linkPaths } = yield* listSymlinkEntries(); + const symlinks = allSymlinks.filter((entry) => + imageOnly + ? entry.kind === "file" && isProjectFaviconPath(entry.path) + : kind === undefined || entry.kind === kind, + ); + // Index results that are filtered-out links get dropped in the merge, so fetch past them. + const indexLimit = limit + linkPaths.size - symlinks.length; + const pageSize = Math.max(1, indexLimit + 1); + const merge = (indexed: ProjectSearchEntriesResult) => + mergeSymlinkMatches(indexed, linkPaths, symlinks, query, limit); if (kind === "file" || imageOnly) { - const result = yield* runSearch(query, pageSize, "fileSearch", () => - finder.fileSearch(query, { pageSize }), + const searchFiles = (fileQuery: string, size: number) => + searchVisible( + query, + size, + "fileSearch", + (fetchSize) => finder.fileSearchAsync(fileQuery, { pageSize: fetchSize }), + (item: FileItem) => item.relativePath, + ); + // Adding a constraint changes fff's lone-path parse; offset pages can skip tied matches. + if (imageOnly && isLoneFuzzyPath(query)) { + const result = yield* searchFiles(query, WORKSPACE_INDEX_PAGE_SIZE); + return merge(mapFileSearchResult(result, indexLimit, hostIsWindows, true)); + } + let result = yield* searchFiles( + imageOnly ? `${query} ${IMAGE_SEARCH_CONSTRAINT}` : query, + pageSize, ); - const mapped = mapFileSearchResult(result, limit, imageOnly); - return { ...mapped, truncated: incompleteBeforeQuery || mapped.truncated }; + if ( + imageOnly && + query.trim() && + result.totalFiles > FFF_FILENAME_FALLBACK_MAX_MATCHES && + result.totalMatched > 0 + ) { + const unfiltered = yield* runSearch(query, 1, "fileSearch", () => + finder.fileSearchAsync(query, { pageSize: 1 }), + ); + if (unfiltered.totalMatched > FFF_FILENAME_FALLBACK_MAX_MATCHES) { + result = yield* searchFiles(query, WORKSPACE_INDEX_PAGE_SIZE); + } + } + return merge(mapFileSearchResult(result, indexLimit, hostIsWindows, imageOnly)); } if (kind === "directory") { - const result = yield* runSearch(query, pageSize, "directorySearch", () => - finder.directorySearch(query, { pageSize }), + const result = yield* searchVisible( + query, + pageSize, + "directorySearch", + (size) => finder.directorySearchAsync(query, { pageSize: size }), + (item) => item.relativePath, ); - const mapped = mapDirectorySearchResult(result, limit); - return { ...mapped, truncated: incompleteBeforeQuery || mapped.truncated }; + return merge(mapDirectorySearchResult(result, indexLimit, hostIsWindows)); } - const result = yield* runSearch(query, pageSize, "mixedSearch", () => - finder.mixedSearch(query, { pageSize }), + const result = yield* searchVisible( + query, + pageSize, + "mixedSearch", + (size) => finder.mixedSearchAsync(query, { pageSize: size }), + mixedItemPath, ); - const mapped = mapMixedSearchResult(result, limit); + return merge(mapMixedSearchResult(result, indexLimit, hostIsWindows)); + }); + + const search: WorkspaceSearchIndex["Service"]["search"] = Effect.fn( + "WorkspaceSearchIndex.search", + )(function* (rawQuery, limit, kind, imageOnly) { + // fff parses any word containing "[" or "{" as a glob and has no escape for them; fuzzy + // matching is subsequence-based, so dropping them still matches "[id]" and "a{b}" paths. + // "{" is dropped only when the query finds nothing as a glob, so "*.{ts,tsx}" keeps working. + const query = rawQuery.replaceAll("[", ""); + const incompleteBeforeQuery = yield* hasIncompleteInitialScan(); + let mapped = yield* searchNames(query, limit, kind, imageOnly); + if (mapped.entries.length === 0 && query.includes("{")) { + mapped = yield* searchNames(query.replaceAll("{", ""), limit, kind, imageOnly); + } return { ...mapped, truncated: incompleteBeforeQuery || mapped.truncated }; }); const searchContents: WorkspaceSearchIndex["Service"]["searchContents"] = Effect.fn( "WorkspaceSearchIndex.searchContents", )(function* (input) { + if (initialScanTimedOut) { + const progress = yield* runSearch(input.query, input.limit, "getScanProgress", () => + finder.getScanProgressAsync(), + ); + // Files the first scan has not indexed yet would read as "No results found". + if (!progress.isWarmupComplete) { + return yield* new WorkspaceSearchIndexScanTimedOut({ + cwd, + timeout: WORKSPACE_INDEX_SCAN_TIMEOUT, + }); + } + initialScanTimedOut = false; + } + let { searchQuery, regexMode } = buildContentSearchQuery(input); const deadline = performance.now() + CONTENT_SEARCH_TIME_BUDGET_MS; // Grep cursors advance per file; whole-word filtering needs enough raw candidates before advancing. @@ -500,7 +1073,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( do { const remainingTimeBudgetMs = Math.max(1, Math.ceil(deadline - performance.now())); const result = yield* runSearch(input.query, input.limit, "grep", () => - finder.grep(searchQuery, { + finder.grepAsync(searchQuery, { mode: regexMode ? "regex" : "plain", smartCase: !input.caseSensitive && !regexMode, maxMatchesPerFile: Math.min(CONTENT_SEARCH_MAX_MATCHES_PER_FILE, rawPageSize), @@ -518,13 +1091,14 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( } for (const match of result.items) { + if (isIgnoredPath(ignoredPaths, match.relativePath, hostIsWindows)) continue; const matchRanges = mapContentMatchRanges(match.lineContent, match.matchRanges).filter( (range) => !input.wholeWord || isWholeWordRange(match.lineContent, range), ); // fff reports hits past its 512-char lineContent cap with no ranges, so whole words can't be verified there. if (input.wholeWord && matchRanges.length === 0) continue; matches.push({ - path: toPosixPath(match.relativePath), + path: toPosixPath(match.relativePath, hostIsWindows), lineNumber: match.lineNumber, lineContent: match.lineContent, matchRanges, diff --git a/apps/server/src/workspace/restoreMissingWorktree.ts b/apps/server/src/workspace/restoreMissingWorktree.ts new file mode 100644 index 000000000000..dee23025dbf4 --- /dev/null +++ b/apps/server/src/workspace/restoreMissingWorktree.ts @@ -0,0 +1,261 @@ +import { ThreadId, type ProjectId, type TerminalCwdNotFoundError } from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; + +import * as ServerConfig from "../config.ts"; +import * as GitWorkflowService from "../git/GitWorkflowService.ts"; +import { scratchFolderRoot } from "../project/ManagedProjectFolders.ts"; +import * as ProjectService from "../project/ProjectService.ts"; +import { makeIsManagedWorktreePath } from "../worktreesDirectory.ts"; +import { isWithinRoot } from "./KnownWorkspaces.ts"; +import { withWorkspaceLease } from "./workspaceLease.ts"; +import { TerminalCwdRecovery } from "../terminal/Manager.ts"; + +export class WorktreeRecreateError extends Schema.TaggedError()( + "WorktreeRecreateError", + { + worktreePath: Schema.String, + branch: Schema.String, + cause: Schema.Defect(), + }, +) { + override get message(): string { + return `Could not recreate the worktree for branch '${this.branch}' at ${this.worktreePath}.`; + } +} + +class WorktreeRecreateNoBaseError extends Schema.TaggedError()( + "WorktreeRecreateNoBaseError", + { worktreePath: Schema.String, branch: Schema.String }, +) { + override get message(): string { + return `Could not recreate branch '${this.branch}' at ${this.worktreePath}: the project workspace has no commit to start it from.`; + } +} + +// `git worktree remove` skips pnpm junctions on Windows, leaving these behind without a checkout. +const LEFTOVER_WORKTREE_ENTRIES = new Set(["node_modules", ".next"]); + +interface WorktreeThread { + readonly id: ThreadId; + readonly projectId: ProjectId; + readonly worktreePath: string | null; + readonly branch: string | null; +} + +export const makeRestoreMissingWorktree = Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const gitWorkflow = yield* GitWorkflowService.GitWorkflowService; + const projects = yield* ProjectService.ProjectService; + const isManagedWorktreePath = yield* makeIsManagedWorktreePath; + const serverConfig = yield* ServerConfig.ServerConfig; + + // Storage cleanup prunes idle No project folders, which have no branch to restore from. + // An empty folder, only directly inside the No project's own root, lets the thread run again. + const recreateScratchFolder = Effect.fn("restoreMissingWorktree.recreateScratchFolder")( + function* (thread: WorktreeThread, folder: string) { + if (path.resolve(folder) !== folder || (yield* fileSystem.exists(folder))) return; + const project = Option.getOrUndefined(yield* projects.getById(thread.projectId)); + if (project === undefined || path.dirname(folder) !== path.resolve(project.workspaceRoot)) + return; + const root = scratchFolderRoot(serverConfig.baseDir, path); + const realRoot = yield* fileSystem.realPath(root).pipe(Effect.orElseSucceed(() => root)); + if (path.dirname(folder) !== root && path.dirname(folder) !== realRoot) return; + yield* Effect.logWarning("recreating missing scratch folder", { + threadId: thread.id, + folder, + }); + yield* fileSystem.makeDirectory(folder, { recursive: true }); + }, + // Turn start reports a folder that is still missing. + Effect.orElseSucceed(() => undefined), + ); + + // Only folders holding nothing but leftovers are cleared, never user files. + const isLeftoverFolder = (worktreePath: string) => + fileSystem.exists(path.join(worktreePath, ".git")).pipe( + Effect.flatMap((hasGit) => + hasGit + ? Effect.succeed(false) + : fileSystem.readDirectory(worktreePath).pipe( + Effect.flatMap((entries) => + Effect.forEach(entries, (entry) => + LEFTOVER_WORKTREE_ENTRIES.has(entry) + ? fileSystem + .stat(path.join(worktreePath, entry)) + .pipe(Effect.map((info) => info.type === "Directory")) + : Effect.succeed(false), + ), + ), + Effect.map((leftovers) => leftovers.every(Boolean)), + ), + ), + Effect.orElseSucceed(() => false), + ); + + const resolveRecreateBase = Effect.fn("restoreMissingWorktree.resolveRecreateBase")(function* ( + cwd: string, + remoteName: string | null, + ) { + // Recovery can follow an external checkout, before the status cache expires. + yield* gitWorkflow.invalidateLocalStatus(cwd); + const { refName: currentBranch } = yield* gitWorkflow.localStatus({ cwd }); + if (currentBranch !== null) { + const refName = `refs/heads/${currentBranch}`; + return { refName, baseRefName: refName }; + } + const remoteHead = `refs/remotes/${remoteName}/HEAD`; + if (remoteName !== null && (yield* gitWorkflow.hasCommit({ cwd, refName: remoteHead }))) { + const { commitSha } = yield* gitWorkflow.resolveRemoteTrackingCommit({ + cwd, + refName: "HEAD", + fallbackRemoteName: remoteName, + }); + // Start from the commit so the thread branch does not track the default branch. + return { refName: commitSha, baseRefName: remoteHead }; + } + if (yield* gitWorkflow.hasCommit({ cwd, refName: "HEAD" })) { + return { refName: "HEAD", baseRefName: undefined }; + } + return null; + }); + + const recreateMissingWorktree = Effect.fn("restoreMissingWorktree.recreate")(function* (input: { + readonly cwd: string; + readonly worktreePath: string; + readonly branch: string; + }) { + const { cwd, worktreePath, branch } = input; + yield* gitWorkflow.pruneWorktrees({ cwd }); + const localBranchNames = yield* gitWorkflow.listLocalBranchNames(cwd); + if (localBranchNames.includes(branch)) { + yield* gitWorkflow.createWorktree({ cwd, refName: branch, path: worktreePath }); + return; + } + const remoteName = yield* gitWorkflow + .resolvePrimaryRemoteName(cwd) + .pipe(Effect.orElseSucceed(() => null)); + // Push keeps the tracking ref current, so recovery reads it without fetching. + const pushedRef = remoteName === null ? null : `refs/remotes/${remoteName}/${branch}`; + if (pushedRef !== null && (yield* gitWorkflow.hasCommit({ cwd, refName: pushedRef }))) { + yield* gitWorkflow.createWorktree({ + cwd, + refName: pushedRef, + newRefName: branch, + path: worktreePath, + }); + return; + } + const base = yield* resolveRecreateBase(cwd, remoteName); + if (base === null) { + return yield* new WorktreeRecreateNoBaseError({ worktreePath, branch }); + } + yield* gitWorkflow.createWorktree({ + cwd, + refName: base.refName, + newRefName: branch, + ...(base.baseRefName === undefined ? {} : { baseRefName: base.baseRefName }), + path: worktreePath, + }); + }); + + return Effect.fn("restoreMissingWorktree")(function* (thread: WorktreeThread) { + const { worktreePath, branch } = thread; + if (worktreePath === null) return; + if (branch === null) return yield* recreateScratchFolder(thread, worktreePath); + const exists = yield* fileSystem.exists(worktreePath).pipe(Effect.orElseSucceed(() => true)); + const leftover = exists && (yield* isLeftoverFolder(worktreePath)); + if ((exists && !leftover) || !(yield* isManagedWorktreePath(worktreePath))) return; + const project = yield* projects.getById(thread.projectId).pipe( + Effect.map(Option.getOrUndefined), + Effect.orElseSucceed(() => undefined), + ); + if (project === undefined) return; + yield* Effect.logWarning("recreating missing worktree", { + threadId: thread.id, + worktreePath, + branch, + }); + // `git worktree add` refuses a non-empty target. + yield* (leftover ? fileSystem.remove(worktreePath, { recursive: true }) : Effect.void).pipe( + Effect.andThen(recreateMissingWorktree({ cwd: project.workspaceRoot, worktreePath, branch })), + Effect.mapError((cause) => + cause._tag === "WorktreeRecreateNoBaseError" + ? cause + : new WorktreeRecreateError({ worktreePath, branch, cause }), + ), + ); + }); +}); + +export class ThreadWorktreeRestore extends Context.Service< + ThreadWorktreeRestore, + { + readonly restore: ( + thread: WorktreeThread, + ) => Effect.Effect; + } +>()("t3/workspace/restoreMissingWorktree/ThreadWorktreeRestore") {} + +// Turn start restores under the same lease, so concurrent restores see each other's worktree. +export const layerThreadWorktreeRestore = Layer.effect( + ThreadWorktreeRestore, + Effect.gen(function* () { + const path = yield* Path.Path; + const restore = yield* makeRestoreMissingWorktree; + return { + restore: Effect.fn("ThreadWorktreeRestore.restore")(function* (thread: WorktreeThread) { + return yield* thread.worktreePath === null + ? restore(thread) + : withWorkspaceLease(path.resolve(thread.worktreePath), restore(thread)); + }), + }; + }), +); + +// Worktree setup uses terminals, so recovery must stay outside TerminalManager. +export const makePrepareTerminalLaunch = Effect.fn( + "restoreMissingWorktree.makePrepareTerminalLaunch", +)(function* ( + getThreadShell: (threadId: ThreadId) => Effect.Effect, +) { + const path = yield* Path.Path; + const restoreMissingWorktree = yield* makeRestoreMissingWorktree; + return Effect.fn("restoreMissingWorktree.prepareTerminalLaunch")(function* < + Input extends { + readonly threadId: string; + readonly cwd?: string | undefined; + readonly worktreePath?: string | null | undefined; + }, + >(input: Input) { + const thread = yield* getThreadShell(ThreadId.make(input.threadId)).pipe( + Effect.orElseSucceed(() => null), + ); + const worktreePath = thread?.worktreePath ?? null; + if ( + thread === null || + worktreePath === null || + !isWithinRoot(path, path.resolve(input.cwd ?? worktreePath), worktreePath) + ) { + return { input, recover: TerminalCwdRecovery.defaultValue() }; + } + const recover = (error: TerminalCwdNotFoundError) => + restoreMissingWorktree(thread).pipe( + Effect.catch((cause) => + Effect.logWarning("failed to recreate missing worktree", { + threadId: thread.id, + worktreePath, + cause, + }).pipe(Effect.andThen(Effect.fail(error))), + ), + ); + // The lease must cover the worktree root, even when the terminal opens in a subdirectory. + return { input: { ...input, worktreePath }, recover }; + }); +}); diff --git a/apps/server/src/workspace/workspaceLease.ts b/apps/server/src/workspace/workspaceLease.ts index 8ad6e41b859c..90e6124bb186 100644 --- a/apps/server/src/workspace/workspaceLease.ts +++ b/apps/server/src/workspace/workspaceLease.ts @@ -1,4 +1,7 @@ +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import type * as FileSystem from "effect/FileSystem"; +import type * as Path from "effect/Path"; import * as Semaphore from "effect/Semaphore"; const leases = new Map(); @@ -20,3 +23,92 @@ export const withWorkspaceLease = ( ), ); }); + +// A restore rewrites files under its directory, so it must exclude provider +// startup in any overlapping workspace: symlink aliases, parents, and children. +// Kept apart from withWorkspaceLease because that lease is non-reentrant and +// restores call Git work that leases the project root. +interface Reservation { + readonly cwd: string; + readonly kind: "restore" | "turn-start"; +} + +const reservations = new Set(); +let reservationReleased = Deferred.makeUnsafe(); + +// path.relative folds win32 drive-letter case and treats a POSIX backslash as +// part of a name, unlike a raw prefix check. +const contains = (path: Path.Path, parent: string, child: string) => { + const relative = path.relative(parent, child); + return ( + relative === "" || + (!path.isAbsolute(relative) && relative !== ".." && !relative.startsWith(`..${path.sep}`)) + ); +}; + +const conflicts = (path: Path.Path, a: Reservation, b: Reservation) => + (a.kind === "restore" || b.kind === "restore") && + (contains(path, a.cwd, b.cwd) || contains(path, b.cwd, a.cwd)); + +const reserve = (path: Path.Path, reservation: Reservation): Effect.Effect => + Effect.suspend(() => { + for (const held of reservations) { + if (conflicts(path, held, reservation)) { + return Deferred.await(reservationReleased).pipe(Effect.andThen(reserve(path, reservation))); + } + } + reservations.add(reservation); + return Effect.void; + }); + +const release = (reservation: Reservation) => + Effect.sync(() => { + if (!reservations.delete(reservation)) return; + const released = reservationReleased; + reservationReleased = Deferred.makeUnsafe(); + Deferred.doneUnsafe(released, Effect.void); + }); + +// A worktree removed by storage cleanup is recreated inside the reserved +// section, so resolve its nearest existing ancestor: a symlink alias above it +// must still canonicalize the same on both sides. Walks at most the path depth. +const canonicalize = ( + fileSystem: FileSystem.FileSystem, + path: Path.Path, + target: string, +): Effect.Effect => + fileSystem.realPath(target).pipe( + Effect.catchReason("PlatformError", "NotFound", () => { + const parent = path.dirname(target); + return parent === target + ? Effect.succeed(target) + : canonicalize(fileSystem, path, parent).pipe( + Effect.map((resolved) => path.join(resolved, path.basename(target))), + ); + }), + Effect.orElseSucceed(() => target), + ); + +const withReservation = + (kind: Reservation["kind"]) => + ( + fileSystem: FileSystem.FileSystem, + path: Path.Path, + cwd: string, + effect: Effect.Effect, + ): Effect.Effect => + canonicalize(fileSystem, path, cwd).pipe( + Effect.flatMap((canonical) => { + const reservation: Reservation = { cwd: canonical, kind }; + return reserve(path, reservation).pipe( + Effect.andThen(effect), + Effect.ensuring(release(reservation)), + ); + }), + ); + +/** Exclusive over every overlapping workspace while checkpoint files are restored. */ +export const withCheckpointRestoreReservation = withReservation("restore"); + +/** Waits out overlapping restores; turn startups never wait on each other. */ +export const withTurnStartReservation = withReservation("turn-start"); diff --git a/apps/server/src/workspace/wslDrivePath.ts b/apps/server/src/workspace/wslDrivePath.ts new file mode 100644 index 000000000000..188e1c85f45e --- /dev/null +++ b/apps/server/src/workspace/wslDrivePath.ts @@ -0,0 +1,29 @@ +import * as HostProcess from "@t3tools/shared/HostProcess"; +import { isWindowsDrivePath } from "@t3tools/shared/path"; +import * as Effect from "effect/Effect"; +import * as ChildProcess from "effect/process/ChildProcess"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; + +// WSL mounts Windows drives under a configurable automount root, so only wslpath knows where. +// No WSL env check: servers started by systemd lack WSL_DISTRO_NAME, and off WSL the spawn just fails. +// Untraced: it runs for every chip path, and only drive paths on Linux do any work. +export const toWslDrivePath = Effect.fnUntraced(function* (requestedPath: string) { + // spawn throws on a NUL byte instead of failing. + if (!isWindowsDrivePath(requestedPath) || requestedPath.includes("\0")) return requestedPath; + if ((yield* HostProcess.Platform) !== "linux") return requestedPath; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const output = yield* spawner + .string( + ChildProcess.make("wslpath", ["-u", requestedPath], { + env: yield* HostProcess.Environment, + stdin: "ignore", + stderr: "ignore", + }), + ) + .pipe( + Effect.timeout("1 second"), + Effect.orElseSucceed(() => ""), + ); + const linuxPath = output.endsWith("\n") ? output.slice(0, -1) : output; + return linuxPath.startsWith("/") ? linuxPath : requestedPath; +}); diff --git a/apps/server/src/ws.test.ts b/apps/server/src/ws.test.ts index 8384a5320473..d2dcdf3cbdb1 100644 --- a/apps/server/src/ws.test.ts +++ b/apps/server/src/ws.test.ts @@ -6,6 +6,7 @@ import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; import { + AuthAccessReadScope, AuthOrchestrationOperateScope, AuthOrchestrationReadScope, AuthSessionId, @@ -25,6 +26,7 @@ import { WsRpcGroup, WsServerRefreshProvidersRpc, WsSubscribeServerConfigRpc, + type AuthAccessStreamEvent, type OrchestrationV2AppThread, type OrchestrationV2DomainEvent, type OrchestrationV2StoredEvent, @@ -138,6 +140,7 @@ import * as TriageService from "./triage/TriageService.ts"; import * as TextGenerators from "./textGeneration/TextGenerators.ts"; import * as WorkspacePaths from "./workspace/WorkspacePaths.ts"; import * as ServerSecretStore from "./auth/ServerSecretStore.ts"; +import * as SecretCheck from "./secrets/SecretCheck.ts"; import * as ServerConfigModule from "./config.ts"; import * as Keybindings from "./keybindings.ts"; import { USAGE_LIMITS_COMMAND } from "@t3tools/shared/usageLimits"; @@ -152,6 +155,7 @@ import * as EventSink from "./orchestration-v2/EventSink.ts"; import * as LegacyV1ThreadImporter from "./orchestration-v2/legacy/LegacyV1ThreadImporter.ts"; import * as ThreadCommandExecutor from "./orchestration-v2/ThreadCommandExecutor.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; +import { projectIdentitiesDigest } from "./orchestration-v2/ShellStream.ts"; import * as OrchestrationEventStore from "./persistence/OrchestrationEventStore.ts"; import * as SqlitePersistence from "./persistence/Sqlite.ts"; import * as ExternalLauncher from "./process/externalLauncher.ts"; @@ -236,7 +240,7 @@ it.effect("resends a reveal kind and remote targets that outlast the snapshot ti assert.deepEqual(remote.config.remoteOpenTargets, targets); assert.equal(remote.config.shellRevealInFileManagerKind, "file-explorer"); } - }).pipe(Effect.scoped), + }), ); it.effect("runs editor, remote open target and direct endpoint discovery side by side", () => @@ -275,25 +279,23 @@ const ReplayTestLayer = OrchestrationEventStore.layer.pipe( Layer.provideMerge(SqlitePersistence.layerMemory), ); -it.live.each(["thread", "shell"] as const)( - "%s resume skips unknown types, synchronizes, and receives the live tail", - (scope) => - Effect.gen(function* () { - const store = yield* OrchestrationEventStore.OrchestrationEventStore; - const sql = yield* SqlClient.SqlClient; - const now = "2026-01-05T00:00:00.000Z"; - const threadId = ThreadId.make("thread:replay"); - const projectId = ProjectId.make("project:replay"); - const project = { - id: projectId, - title: "Replay", - workspaceRoot: "/tmp/replay", - defaultModelSelection: null, - scripts: [], - createdAt: now, - updatedAt: now, - } satisfies OrchestrationProjectShell; - const insert = (version: number, unknown = false) => sql<{ sequence: number }>` +const resumeSkipsUnknownTypes = (scope: "thread" | "shell") => + Effect.gen(function* () { + const store = yield* OrchestrationEventStore.OrchestrationEventStore; + const sql = yield* SqlClient.SqlClient; + const now = "2026-01-05T00:00:00.000Z"; + const threadId = ThreadId.make("thread:replay"); + const projectId = ProjectId.make("project:replay"); + const project = { + id: projectId, + title: "Replay", + workspaceRoot: "/tmp/replay", + defaultModelSelection: null, + scripts: [], + createdAt: now, + updatedAt: now, + } satisfies OrchestrationProjectShell; + const insert = (version: number, unknown = false) => sql<{ sequence: number }>` INSERT INTO orchestration_events ( event_id, aggregate_kind, stream_id, stream_version, event_type, occurred_at, actor_kind, payload_json, metadata_json, application_event_version @@ -308,85 +310,91 @@ it.live.each(["thread", "shell"] as const)( )}, '{}', 2 ) RETURNING sequence `; - const [before] = yield* insert(0); - yield* insert(1, true); - const [after] = yield* insert(2); - const [highWater] = yield* insert(3, true); - const updates = yield* PubSub.unbounded(); - let liveSequence = 0; - const consume = ( - stream: Stream.Stream, - ) => - stream.pipe( - Stream.tap((item) => - Effect.gen(function* () { - if (item.kind !== "synchronized") return; - const [live] = yield* insert(4); - liveSequence = live!.sequence; - const events = yield* store - .readApplicationEvents({ - afterSequence: highWater!.sequence, - throughSequence: liveSequence, - }) - .pipe(Stream.runCollect); - yield* store.publishCommitted(events); - }), - ), - Stream.takeUntil((item) => item.sequence === liveSequence), - Stream.map((item) => ({ kind: item.kind as string, sequence: item.sequence })), - Stream.runCollect, - ); - const subscription = Effect.gen(function* () { - return scope === "thread" - ? yield* subscribeOrchestrationV2Thread({ - threadId, - afterSequence: before!.sequence, - requestCompletionMarker: true, - }).pipe(Effect.flatMap(consume)) - : yield* subscribeOrchestrationV2Shell({ - afterSequence: before!.sequence, - requestCompletionMarker: true, - }).pipe(Effect.flatMap(consume)); - }); - const items = yield* subscription.pipe( - Effect.provideService(ThreadManagement.ThreadManagementService, { - ensureLegacyTranscript: () => Effect.void, - getShellSnapshot: () => Effect.die("shell resume must not load a full snapshot"), - listActiveSubagentThreadIds: () => Effect.succeed([]), - streamStoredEventsFrom: ( - input: Parameters< - ThreadManagement.ThreadManagementService["Service"]["streamStoredEventsFrom"] - >[0], - ) => { - assert.equal(input?.afterSequence, highWater!.sequence); - return store - .streamApplicationEvents({ afterSequence: input?.afterSequence ?? 0 }) - .pipe( - Stream.filter((event): event is OrchestrationV2StoredEvent => "event" in event), - ); - }, - } as unknown as ThreadManagement.ThreadManagementService["Service"]), - Effect.provideService(ProjectStore.ProjectStoreV2, { - listShells: () => Effect.succeed([]), - } as unknown as ProjectStore.ProjectStoreV2["Service"]), - Effect.provideService(ProjectService.ProjectService, { - getShell: () => Effect.succeedSome(project), - } as unknown as ProjectService.ProjectService["Service"]), - Effect.provideService(ProjectEnrichment.ProjectEnrichmentService, { - subscribeChanges: PubSub.subscribe(updates), - } as ProjectEnrichment.ProjectEnrichmentService["Service"]), - ); - assert.deepEqual( - items.map((item) => item.kind), - scope === "thread" - ? ["event", "synchronized", "event"] - : ["project.updated", "synchronized", "project.updated"], - ); - assert.deepEqual( - items.flatMap((item) => (item.sequence === undefined ? [] : [item.sequence])), - [after!.sequence, liveSequence], + const [before] = yield* insert(0); + yield* insert(1, true); + const [after] = yield* insert(2); + const [highWater] = yield* insert(3, true); + const updates = yield* PubSub.unbounded(); + let liveSequence = 0; + const consume = ( + stream: Stream.Stream, + ) => + stream.pipe( + Stream.tap((item) => + Effect.gen(function* () { + if (item.kind !== "synchronized") return; + const [live] = yield* insert(4); + liveSequence = live!.sequence; + const events = yield* store + .readApplicationEvents({ + afterSequence: highWater!.sequence, + throughSequence: liveSequence, + }) + .pipe(Stream.runCollect); + yield* store.publishCommitted(events); + }), + ), + Stream.takeUntil((item) => item.sequence === liveSequence), + Stream.map((item) => ({ kind: item.kind as string, sequence: item.sequence })), + Stream.runCollect, ); - }).pipe(Effect.scoped, Effect.provide(ReplayTestLayer)), + const subscription = Effect.gen(function* () { + return scope === "thread" + ? yield* subscribeOrchestrationV2Thread({ + threadId, + afterSequence: before!.sequence, + requestCompletionMarker: true, + }).pipe(Effect.flatMap(consume)) + : yield* subscribeOrchestrationV2Shell({ + afterSequence: before!.sequence, + requestCompletionMarker: true, + }).pipe(Effect.flatMap(consume)); + }); + const items = yield* subscription.pipe( + Effect.provideService(ThreadManagement.ThreadManagementService, { + ensureLegacyTranscript: () => Effect.void, + getShellSnapshot: () => Effect.die("shell resume must not load a full snapshot"), + listActiveSubagentThreadIds: () => Effect.succeed([]), + streamStoredEventsFrom: ( + input: Parameters< + ThreadManagement.ThreadManagementService["Service"]["streamStoredEventsFrom"] + >[0], + ) => { + assert.equal(input?.afterSequence, highWater!.sequence); + return store + .streamApplicationEvents({ afterSequence: input?.afterSequence ?? 0 }) + .pipe(Stream.filter((event): event is OrchestrationV2StoredEvent => "event" in event)); + }, + } as unknown as ThreadManagement.ThreadManagementService["Service"]), + Effect.provideService(ProjectStore.ProjectStoreV2, { + listShells: () => Effect.succeed([]), + } as unknown as ProjectStore.ProjectStoreV2["Service"]), + Effect.provideService(ProjectService.ProjectService, { + getShell: () => Effect.succeedSome(project), + } as unknown as ProjectService.ProjectService["Service"]), + Effect.provideService(ProjectEnrichment.ProjectEnrichmentService, { + subscribeChanges: PubSub.subscribe(updates), + } as ProjectEnrichment.ProjectEnrichmentService["Service"]), + ); + assert.deepEqual( + items.map((item) => item.kind), + scope === "thread" + ? ["event", "synchronized", "event"] + : ["project.updated", "synchronized", "project.updated"], + ); + assert.deepEqual( + items.flatMap((item) => (item.sequence === undefined ? [] : [item.sequence])), + [after!.sequence, liveSequence], + ); + }).pipe(Effect.scoped, Effect.provide(ReplayTestLayer)); + +it.effect("thread resume skips unknown types, synchronizes, and receives the live tail", () => + resumeSkipsUnknownTypes("thread"), +); + +// Shell updates batch through `Stream.groupedWithin`, whose window needs real time to close. +it.live("shell resume skips unknown types, synchronizes, and receives the live tail", () => + resumeSkipsUnknownTypes("shell"), ); it.effect("coalesces streaming replay before synchronization and the live tail", () => @@ -711,6 +719,8 @@ it.effect("shell snapshots omit idle children and older settled threads only on shell("live", "parent", "running"), ...Array.from({ length: 25 }, (_, index) => shell(`recent-${index}`, null, null, 1)), shell("old", null, null, 30), + shell("remote-parent", null, null, 40), + shell("remote-child-parent", "parent", null), ]; const firstSnapshot = (input: { readonly omitIdleSubagents?: boolean; @@ -727,6 +737,7 @@ it.effect("shell snapshots omit idle children and older settled threads only on pinnedAt: null, snoozedUntil: null, mayBeLive: thread.activityRunStatus !== null, + hasRemoteTasks: thread.id.startsWith("remote-"), })), ), readShellSnapshot: (options?: { readonly threadIds?: ReadonlyArray }) => @@ -770,15 +781,17 @@ it.effect("shell snapshots omit idle children and older settled threads only on const withoutChildren = yield* firstSnapshot({ omitIdleSubagents: true }); assert.deepEqual( withoutChildren?.threads.map((thread) => thread.id), - ["parent", "live", ...recent, "old"], + ["parent", "live", ...recent, "old", "remote-parent", "remote-child-parent"], ); + // A linked environment's clients nest its task under its parent, so that parent stays: an + // older settled thread or an idle subagent. const withoutOld = yield* firstSnapshot({ omitIdleSubagents: true, omitOlderSettled: true }); assert.deepEqual( withoutOld?.threads.map((thread) => thread.id), - ["parent", "live", ...recent], + ["parent", "live", ...recent, "remote-parent", "remote-child-parent"], ); assert.deepEqual(withoutOld?.olderSettled?.counts, [ - { projectId: ProjectId.make("project"), count: 1 }, + { projectId: ProjectId.make("project"), count: 2 }, ]); }).pipe(Effect.scoped, Effect.provide(ReplayTestLayer)), ); @@ -791,6 +804,7 @@ const layerUnusedConfigRpcServices = Layer.mergeAll( }), Layer.mock(GiteaCli.GiteaCli)({ baseUrl: Option.none() }), Layer.mock(ServerSecretStore.ServerSecretStore)({}), + Layer.mock(SecretCheck.SecretCheck)({}), Layer.mock(WorkspacePaths.WorkspacePaths)({}), Layer.mock(TraceDiagnostics.TraceDiagnostics)({}), Layer.mock(TriageService.TriageService)({}), @@ -1137,10 +1151,133 @@ it.effect.each([ Layer.mock(ServerBrowser.ServerBrowser)({}), ), ), - Effect.scoped, ), ); +it.effect("access changes made while the snapshot loads reach the client after it", () => + Effect.gen(function* () { + const pairingChanges = yield* PubSub.unbounded(); + const sessionChanges = yield* PubSub.unbounded(); + const subscribed = yield* Queue.unbounded(); + const subscribe = (changes: PubSub.PubSub) => + Stream.unwrap( + PubSub.subscribe(changes).pipe( + Effect.tap(() => Queue.offer(subscribed, undefined)), + Effect.map(Stream.fromSubscription), + ), + ); + const revokedElsewhere = AuthSessionId.make("revoked-elsewhere"); + const responses = yield* Queue.unbounded(); + const server = yield* RpcServer.makeNoSerialization(WsRpcGroup, { + ...WS_RPC_SERVER_OPTIONS, + onFromServer: (response) => + response._tag === "Chunk" + ? Effect.forEach( + response.values, + (event) => Queue.offer(responses, event as AuthAccessStreamEvent), + { + discard: true, + }, + ) + : Effect.void, + }).pipe( + Effect.provide( + layerWsRpc( + { + sessionId: AuthSessionId.make("current"), + subject: "current", + method: "browser-session-cookie", + scopes: [], + }, + {}, + yield* ServerBrowser.ServerBrowser, + () => Effect.void, + { secretReferences: true }, + ).pipe( + Layer.merge(RpcAuthorization.layer([AuthAccessReadScope])), + Layer.provide( + Layer.mergeAll( + layerUnusedConfigRpcServices, + Layer.mock(ExternalLauncher.ExternalLauncher)({}), + Layer.mock(ProviderRegistry.ProviderRegistry)({}), + Layer.mock(UsageLimitSources.UsageLimitSources)({}), + Layer.mock(RemoteOpenTargets.RemoteOpenTargets)({}), + Layer.mock(DirectEndpoints.DirectEndpoints)({}), + Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ + namedProjectsRoot: "/unused", + }), + Layer.mock(ScratchThreadPromotion.ScratchThreadPromotion)({}), + Layer.mock(ServerEnvironment.ServerEnvironment)({}), + Layer.mock(EnvironmentAuth.EnvironmentAuth)({ + listPairingLinks: () => Effect.succeed([]), + listConnectDevices: () => Effect.succeed([]), + // Another client revokes a session after the snapshot read it. + listClientSessions: () => + PubSub.publish(sessionChanges, { + type: "clientRemoved", + sessionId: revokedElsewhere, + }).pipe(Effect.as([])), + }), + Layer.mock(PairingGrantStore.PairingGrantStore)({ + streamChanges: subscribe(pairingChanges), + }), + Layer.mock(SessionStore.SessionStore)({ + cookieName: "auth-access-test", + legacyCookieName: undefined, + streamChanges: subscribe(sessionChanges), + }), + ), + ), + ), + ), + ); + const requestId = RpcMessage.RequestId("auth-access"); + yield* server.write(0, { + _tag: "Request", + id: requestId, + tag: WS_METHODS.subscribeAuthAccess, + payload: {}, + headers: Headers.empty, + }); + const snapshot = yield* Queue.take(responses); + assert.equal(snapshot.type, "snapshot"); + yield* server.write(0, { _tag: "Ack", requestId }); + yield* Queue.take(subscribed); + yield* Queue.take(subscribed); + + // A later change arrives either way, so a dropped revocation fails as a mismatch. + yield* PubSub.publish(pairingChanges, { type: "pairingLinkRemoved", id: "later" }); + const next = yield* Queue.take(responses); + assert.deepEqual(next, { + version: 1, + revision: 2, + type: "clientRemoved", + payload: { sessionId: revokedElsewhere }, + }); + yield* server.write(0, { _tag: "Ack", requestId }); + assert.equal((yield* Queue.take(responses)).type, "pairingLinkRemoved"); + yield* server.write(0, { _tag: "Ack", requestId }); + + // Device trust has no incremental event, so a change re-sends the whole snapshot. + yield* PubSub.publish(pairingChanges, { type: "connectDevicesChanged" }); + assert.deepEqual(yield* Queue.take(responses), { + version: 1, + revision: 4, + type: "snapshot", + payload: { pairingLinks: [], clientSessions: [], connectDevices: [] }, + }); + yield* server.write(0, { _tag: "Interrupt", requestId, interruptors: [] }); + }).pipe( + Effect.provide( + Layer.mergeAll( + Layer.fresh(layerConfigSources), + Layer.mock(PreviewAutomationBroker.PreviewAutomationBroker)({}), + Layer.mock(ServerBrowser.ServerBrowser)({}), + ), + ), + ), +); + it.effect("on-demand clients get provider config without workspace snapshots", () => Effect.gen(function* () { const instanceId = ProviderInstanceId.make("claudeAgent"); @@ -1175,6 +1312,7 @@ it.effect("on-demand clients get provider config without workspace snapshots", ( ], }; let scansCwd = true; + let registryProviders: ReadonlyArray = [provider]; const providerChanges = yield* Queue.unbounded>(); const pulled = yield* Queue.unbounded(); const responses = yield* Queue.unbounded(); @@ -1185,11 +1323,12 @@ it.effect("on-demand clients get provider config without workspace snapshots", ( resolveFileManagerRevealKind: () => Effect.undefined, }), Layer.mock(ProviderRegistry.ProviderRegistry)({ - getProviders: Effect.succeed([provider]), + getProviders: Effect.sync(() => registryProviders), streamChanges: Stream.fromQueue(providerChanges).pipe( Stream.tap(() => Queue.offer(pulled, undefined)), ), - refreshWorkspaceSnapshot: () => Effect.succeed([provider]), + refresh: () => Effect.sync(() => registryProviders), + refreshWorkspaceSnapshot: () => Effect.sync(() => registryProviders), }), Layer.mock(ProjectService.ProjectService)({ workspaceScanDirectory: (cwd) => Effect.succeed(cwd), @@ -1211,6 +1350,7 @@ it.effect("on-demand clients get provider config without workspace snapshots", ( Layer.mock(UsageLimitSources.UsageLimitSources)({ current: Effect.succeed([]), streamChanges: Stream.make([]), + refresh: Effect.void, }), Layer.mock(RemoteOpenTargets.RemoteOpenTargets)({ resolveTargets: () => Effect.succeed([]), @@ -1366,6 +1506,121 @@ it.effect("on-demand clients get provider config without workspace snapshots", ( }, ]); yield* server.write(0, { _tag: "Interrupt", requestId: configRequest, interruptors: [] }); + + // A client that takes digests and deltas, against a catalog of over 200 KB. + const heavy: ServerProvider = { + ...provider, + slashCommands: Array.from({ length: 250 }, (_, index) => ({ + name: `command-${index}`, + description: "d".repeat(400), + })), + skills: Array.from({ length: 200 }, (_, index) => ({ + name: `skill-${index}`, + description: "d".repeat(500), + path: `/home/user/.claude/skills/skill-${index}/SKILL.md`, + enabled: true, + })), + }; + registryProviders = [heavy]; + const lightRequest = RpcMessage.RequestId("light-config"); + yield* server.write(0, { + _tag: "Request", + id: lightRequest, + tag: WS_METHODS.subscribeServerConfig, + payload: { + workspaceSnapshotsOnDemand: true, + providerCatalogsOnDemand: true, + providerCatalogDigests: true, + providerStatusDeltas: true, + }, + headers: Headers.empty, + }); + const light = (yield* Queue.take(responses)) as ServerConfigStreamEvent; + if (light.type !== "snapshot") return yield* Effect.die("Missing light snapshot"); + const digest = light.config.providers[0]?.catalogDigest; + assert.isString(digest); + assert.deepEqual( + [light.config.providers[0]?.slashCommands, light.config.providers[0]?.skills], + [[], []], + ); + // Budget for the first config a phone decodes: the catalog stays out of it. + assert.isBelow(JSON.stringify(light.config.providers).length, 1024); + yield* server.write(0, { _tag: "Ack", requestId: lightRequest }); + + // A health check sends the provider without the lists the client already holds. + yield* Queue.offer(providerChanges, [{ ...heavy, checkedAt: "2026-10-25T00:10:00.000Z" }]); + yield* Queue.take(pulled); + yield* TestClock.adjust("1 second"); + const delta = (yield* Queue.take(responses)) as ServerConfigStreamEvent; + if (delta.type !== "providerStatusDeltas") return yield* Effect.die("Expected provider deltas"); + assert.deepEqual( + delta.payload.providers.map((entry) => [ + entry.checkedAt, + entry.catalogDigest, + entry.models, + entry.slashCommands, + entry.skills, + ]), + [["2026-10-25T00:10:00.000Z", digest, undefined, undefined, undefined]], + ); + yield* server.write(0, { _tag: "Ack", requestId: lightRequest }); + + // A changed description changes the digest, so the client loads the lists again. + yield* Queue.offer(providerChanges, [ + { + ...heavy, + checkedAt: "2026-10-25T00:20:00.000Z", + skills: heavy.skills.map((skill, index) => + index === 0 ? { ...skill, description: "changed" } : skill, + ), + }, + ]); + yield* Queue.take(pulled); + yield* TestClock.adjust("1 second"); + const redescribed = (yield* Queue.take(responses)) as ServerConfigStreamEvent; + if (redescribed.type !== "providerStatusDeltas") { + return yield* Effect.die("Expected provider deltas"); + } + assert.notEqual(redescribed.payload.providers[0]?.catalogDigest, digest); + yield* server.write(0, { _tag: "Ack", requestId: lightRequest }); + + // A caller that only waits for a machine refresh gets no provider list back. + yield* server.write(0, { + _tag: "Request", + id: RpcMessage.RequestId("refresh-omitted"), + tag: WS_METHODS.serverRefreshProviders, + payload: { workspaceSnapshotsOnDemand: true, omitProviders: true }, + headers: Headers.empty, + }); + assert.deepEqual(yield* Queue.take(responses), { providers: [] }); + + // The cwd snapshot brings the lists the digest stands for, under the same digest. + yield* server.write(0, { + _tag: "Request", + id: RpcMessage.RequestId("refresh-machine-catalog"), + tag: WS_METHODS.serverRefreshProviders, + payload: { + instanceId, + cwd: "/b", + workspaceSnapshotsOnDemand: true, + providerCatalogsOnDemand: true, + omitProviders: true, + machineCatalog: true, + }, + headers: Headers.empty, + }); + const loaded = (yield* Queue.take(responses)) as { + readonly providers: ReadonlyArray; + }; + assert.equal(loaded.providers[0]?.catalogDigest, digest); + assert.deepEqual(loaded.providers[0]?.slashCommands[0], { name: "command-0" }); + assert.deepEqual(loaded.providers[0]?.skills[0], { + name: "skill-0", + path: "/home/user/.claude/skills/skill-0/SKILL.md", + enabled: true, + }); + assert.lengthOf(loaded.providers[0]?.workspaceSnapshots ?? [], 1); + yield* server.write(0, { _tag: "Interrupt", requestId: lightRequest, interruptors: [] }); }).pipe( Effect.provide( Layer.mergeAll( @@ -1374,7 +1629,6 @@ it.effect("on-demand clients get provider config without workspace snapshots", ( Layer.mock(ServerBrowser.ServerBrowser)({}), ), ), - Effect.scoped, ), ); @@ -1546,7 +1800,6 @@ it.effect("a missing thread worktree gets its project's skills in the cwd snapsh Layer.mock(ServerBrowser.ServerBrowser)({}), ), ), - Effect.scoped, ), ); @@ -1620,7 +1873,6 @@ it.effect("a git action that fails partway still refreshes the repository status Layer.mock(ServerBrowser.ServerBrowser)({}), ), ), - Effect.scoped, ), ); @@ -1708,7 +1960,6 @@ it.effect("a publish whose push fails still refreshes the new remote", () => Layer.mock(ServerBrowser.ServerBrowser)({}), ), ), - Effect.scoped, ), ); @@ -1864,41 +2115,123 @@ it.effect.each(["thread", "shell"] as const)( }).pipe(Effect.scoped, Effect.provide(ReplayTestLayer)), ); -// A thread idle since before the boundary missed nothing; resuming must not resend its body. -it.effect("thread resume at its own high-water below the pruned boundary skips the snapshot", () => - Effect.gen(function* () { - const sql = yield* SqlClient.SqlClient; - const now = "2026-10-13T00:00:00.000Z"; - const threadId = ThreadId.make("thread:idle"); - const [last] = yield* sql<{ sequence: number }>` - INSERT INTO orchestration_events ( - event_id, aggregate_kind, stream_id, stream_version, event_type, - occurred_at, actor_kind, payload_json, metadata_json, application_event_version - ) VALUES ( - 'idle:0', 'thread', ${threadId}, 0, 'provider-session.detached', - ${now}, 'server', ${JSON.stringify({ providerSessionId: "session:idle", detachedAt: now })}, - '{}', 2 - ) RETURNING sequence - `; - yield* sql` - INSERT INTO orchestration_v2_projection_metadata ( - projection_name, schema_version, last_sequence, updated_at - ) VALUES (${OrchestrationEventStore.EVENT_LOG_PRUNED_THROUGH}, 2, ${last!.sequence + 10}, ${now}) - `; - const exit = yield* subscribeOrchestrationV2Thread({ - threadId, - afterSequence: last!.sequence, - }).pipe( - Effect.provideService(ThreadManagement.ThreadManagementService, { - ensureLegacyTranscript: () => Effect.void, - getThreadSnapshot: () => Effect.die("snapshot"), - streamStoredEventsFrom: () => Stream.empty, - } as unknown as ThreadManagement.ThreadManagementService["Service"]), - Effect.exit, - ); - // A snapshot load dies with "snapshot", so success means the resume replayed nothing. - assert.isTrue(Exit.isSuccess(exit)); - }).pipe(Effect.scoped, Effect.provide(ReplayTestLayer)), +// A cursor at the high-water missed nothing, even below the boundary; resuming must not resend +// a snapshot. The shell cases are a client that just loaded the HTTP snapshot of a pruned +// database: it gets the marker, and the project identities only when it does not name them. +it.effect.each([ + { scope: "thread", knowsIdentities: false, isRepository: true, kinds: [] }, + { + scope: "shell", + knowsIdentities: false, + isRepository: true, + kinds: ["snapshot", "synchronized"], + }, + { scope: "shell", knowsIdentities: true, isRepository: true, kinds: ["synchronized"] }, + // The client keeps a cached identity over an absent one, so "resolved to none" still goes out. + { + scope: "shell", + knowsIdentities: true, + isRepository: false, + kinds: ["snapshot", "synchronized"], + }, +] as const)( + "$scope resume at its own high-water below the pruned boundary skips the snapshot (knows identities: $knowsIdentities, repository: $isRepository)", + ({ scope, knowsIdentities, isRepository, kinds }) => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const now = "2026-10-13T00:00:00.000Z"; + const threadId = ThreadId.make("thread:idle"); + const [last] = yield* sql<{ sequence: number }>` + INSERT INTO orchestration_events ( + event_id, aggregate_kind, stream_id, stream_version, event_type, + occurred_at, actor_kind, payload_json, metadata_json, application_event_version + ) VALUES ( + 'idle:0', 'thread', ${threadId}, 0, 'provider-session.detached', + ${now}, 'server', ${JSON.stringify({ providerSessionId: "session:idle", detachedAt: now })}, + '{}', 2 + ) RETURNING sequence + `; + yield* sql` + INSERT INTO orchestration_v2_projection_metadata ( + projection_name, schema_version, last_sequence, updated_at + ) VALUES (${OrchestrationEventStore.EVENT_LOG_PRUNED_THROUGH}, 2, ${last!.sequence + 10}, ${now}) + `; + const identity = isRepository + ? ({ + canonicalKey: "github.com/t3/idle", + locator: { + source: "git-remote", + remoteName: "origin", + remoteUrl: "git@github.com:t3/idle", + }, + } as const) + : null; + const project = { + id: ProjectId.make("project:idle"), + title: "Idle", + workspaceRoot: "/tmp/idle", + defaultModelSelection: null, + scripts: [], + createdAt: now, + updatedAt: now, + } satisfies OrchestrationProjectShell; + const subscription = Effect.gen(function* () { + if (scope === "thread") { + return yield* subscribeOrchestrationV2Thread({ + threadId, + afterSequence: last!.sequence, + }).pipe(Effect.as([])); + } + const stream = yield* subscribeOrchestrationV2Shell({ + afterSequence: last!.sequence, + requestCompletionMarker: true, + ...(knowsIdentities + ? { + knownProjectIdentities: projectIdentitiesDigest([ + { ...project, repositoryIdentity: identity }, + ]), + } + : {}), + }); + return yield* stream.pipe( + Stream.takeUntil((item) => item.kind === "synchronized"), + Stream.map((item) => item.kind), + Stream.runCollect, + ); + }); + const exit = yield* subscription.pipe( + Effect.provideService(ThreadManagement.ThreadManagementService, { + ensureLegacyTranscript: () => Effect.void, + getThreadSnapshot: () => Effect.die("snapshot"), + listActiveShellIndex: () => Effect.die("snapshot"), + readShellSnapshot: () => Effect.die("snapshot"), + listActiveSubagentThreadIds: () => Effect.succeed([]), + streamStoredEventsFrom: () => Stream.empty, + } as unknown as ThreadManagement.ThreadManagementService["Service"]), + Effect.provideService(ProjectStore.ProjectStoreV2, { + listShells: () => Effect.succeed([project]), + } as unknown as ProjectStore.ProjectStoreV2["Service"]), + Effect.provideService( + ProjectService.ProjectService, + {} as ProjectService.ProjectService["Service"], + ), + Effect.provideService(ProjectEnrichment.ProjectEnrichmentService, { + subscribeChanges: PubSub.subscribe(yield* PubSub.unbounded()), + getAvailable: () => + Effect.succeed({ + repositoryIdentity: identity, + faviconPath: null, + repositoryIdentityResolved: true, + }), + } as unknown as ProjectEnrichment.ProjectEnrichmentService["Service"]), + Effect.exit, + ); + // A snapshot load dies with "snapshot", so success means the resume replayed nothing. + assert.deepEqual( + Exit.isFailure(exit) ? Cause.squash(exit.cause) : Array.from(exit.value), + kinds, + ); + }).pipe(Effect.scoped, Effect.provide(ReplayTestLayer)), ); const acpDriver = ProviderDriverKind.make("acpRegistry"); @@ -2130,6 +2463,5 @@ it.effect("ACP session RPCs list a thread missing its history and repair it on i Layer.mock(ServerBrowser.ServerBrowser)({}), ), ), - Effect.scoped, ), ); diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index efaf134f329f..43f11665c0e9 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -1,4 +1,6 @@ import { OrchestrationDispatchCommandError } from "@t3tools/contracts"; +// @effect-diagnostics-next-line nodeBuiltinImport:off -- A synchronous digest inside the pure provider mapping. +import * as NodeCrypto from "node:crypto"; import * as Context from "effect/Context"; import * as SecureRandom from "effect/crypto/SecureRandom"; import * as Orchestrator from "./orchestration-v2/Orchestrator.ts"; @@ -83,6 +85,7 @@ import { type ServerConfigStreamEvent, type ServerLifecycleStreamEvent, type ServerLowDiskSpace, + serverProviderDeltas, type FilesystemBrowseFailure, FilesystemBrowseError, AssetWorkspaceContextNotFoundError, @@ -135,6 +138,8 @@ import { shellStreamItemFromThreadShell, shellStreamItemsFromInitialSnapshot, loadSubscriberShellSnapshot, + canResumeShellFrom, + resumeIdentityRoots, shellStreamItemsFromResumeSnapshot, shouldPublishShellApplicationEvent, skipUnchangedThreadShells, @@ -170,6 +175,7 @@ import * as ThreadSearch from "./orchestration-v2/ThreadSearch.ts"; import * as OrchestrationEventStore from "./persistence/OrchestrationEventStore.ts"; import { userFacingDispatchErrorMessage } from "./orchestration-v2/UserFacingErrors.ts"; import * as ProviderRegistry from "./provider/ProviderRegistry.ts"; +import * as SecretCheck from "./secrets/SecretCheck.ts"; import * as AcpRegistrySessions from "./provider/AcpRegistrySessions.ts"; import * as ProviderInstanceRegistry from "./provider/ProviderInstanceRegistry.ts"; import * as AcpRegistrySupport from "@t3tools/provider-acp-registry/server/AcpRegistrySupport"; @@ -187,6 +193,7 @@ import { invalidateSettingsSecrets } from "./sourceControl/sourceControlHostSecr import * as DatabaseStorage from "./persistence/DatabaseStorage.ts"; import * as StorageCleanup from "./storageCleanup.ts"; import * as TerminalManager from "./terminal/Manager.ts"; +import { makePrepareTerminalLaunch } from "./workspace/restoreMissingWorktree.ts"; import { terminalOutputStream, withOutputWindow } from "./terminal/OutputProtocol.ts"; import * as ServerBrowser from "./preview/ServerBrowser.ts"; import * as DeviceService from "./device/DeviceService.ts"; @@ -476,26 +483,67 @@ function workspaceSnapshotOrCatalog( ]; } -// On-demand catalog clients read full entries from the cwd snapshot they fetch. Names still -// change the config when the machine catalog does, so the client refetches, and skill names -// and labels keep skill links working where no snapshot is fetched. +// What an on-demand catalog client keeps of the machine catalog: full entries come from the +// cwd snapshot it fetches, and skill names and labels keep skill links working without one. +function machineCatalogNames(provider: Pick) { + return { + slashCommands: provider.slashCommands.map(({ name }) => ({ name })), + skills: provider.skills.map(({ description: _d, shortDescription: _s, ...skill }) => skill), + }; +} + +// Stable across restarts, so a client's cached lists stay current until the catalog changes. +// It covers the full entries: a changed description or input hint must reach the client too. +const machineCatalogDigest = (provider: Pick) => + NodeCrypto.createHash("sha256") + .update(JSON.stringify([provider.slashCommands, provider.skills])) + .digest("hex") + .slice(0, 16); + +// Names, or the digest that stands for them, still change the config when the machine catalog +// does, so the client refetches its cwd snapshot. function withoutWorkspaceSnapshots( providers: ReadonlyArray, catalogsPerCwd: ReadonlySet = new Set(), + catalogDigests = false, ): ServerProvider[] { - return providers.map(({ workspaceSnapshots: _workspaceSnapshots, ...provider }) => - catalogsPerCwd.has(provider.instanceId) + return providers.map(({ workspaceSnapshots: _workspaceSnapshots, ...provider }) => { + if (!catalogsPerCwd.has(provider.instanceId)) return provider; + return catalogDigests ? { ...provider, - slashCommands: provider.slashCommands.map(({ name }) => ({ name })), - skills: provider.skills.map( - ({ description: _d, shortDescription: _s, ...skill }) => skill, - ), + slashCommands: [], + skills: [], + catalogDigest: machineCatalogDigest(provider), } - : provider, - ); + : { ...provider, ...machineCatalogNames(provider) }; + }); } +// Deltas are taken from the list this client last received, which a late snapshot replaces. +const withProviderStatusDeltas = (sent: ReadonlyArray) => + Stream.mapAccum( + () => sent, + ( + previous, + event: ServerConfigStreamEvent, + ): readonly [ReadonlyArray, ReadonlyArray] => { + if (event.type === "snapshot") return [event.config.providers, [event]]; + if (event.type !== "providerStatuses") return [previous, [event]]; + const { providers } = event.payload; + return [ + providers, + [ + { + version: 1, + type: "providerStatusDeltas", + payload: { providers: serverProviderDeltas(previous, providers) }, + }, + ], + ]; + }, + ); + function unexpectedCompatibilityError(error: never): never { throw new Error(`Unhandled compatibility error: ${String(error)}`); } @@ -709,16 +757,11 @@ const isOtherWsRpc = (rpc: ServerWsRpc): rpc is Exclude const serverWsRpcs = Array.from(ServerWsRpcGroup.requests.values()); const SplitWsRpcGroup = RpcGroup.make(...serverWsRpcs.filter(isSplitWsRpc)); const OtherWsRpcGroup = RpcGroup.make(...serverWsRpcs.filter(isOtherWsRpc)); -// Replaying each intervening event costs a shell refetch; past this gap a -// single O(active-threads) snapshot is cheaper and bounded. -const SHELL_RESUME_MAX_GAP = 1_000; -// Row count alone does not bound replay memory: a few events with large tool -// payloads can decode to gigabytes. Before replaying, sum the serialized -// payload bytes of the range in SQL and reset with a snapshot past this budget. -const ORCHESTRATION_REPLAY_PAYLOAD_BUDGET_BYTES = 8 * 1024 * 1024; - function toAuthAccessStreamEvent( - change: PairingGrantStore.BootstrapCredentialChange | SessionStore.SessionCredentialChange, + change: Exclude< + PairingGrantStore.BootstrapCredentialChange | SessionStore.SessionCredentialChange, + { readonly type: "connectDevicesChanged" } + >, revision: number, currentSessionId: AuthSessionId, ): AuthAccessStreamEvent { @@ -853,36 +896,6 @@ function readClientOwners( ); } -const canReplayPersistedRange = Effect.fnUntraced(function* ( - afterSequence: number, - headSequence: number, - maxGap: number, -) { - const applicationEvents = yield* OrchestrationEventStore.OrchestrationEventStore; - - const replayGap = headSequence - afterSequence; - if (replayGap < 0 || replayGap > maxGap) { - return false; - } - if (afterSequence < (yield* applicationEvents.prunedThroughSequence)) return false; - const stats = yield* applicationEvents.getReplayStats({ - afterSequence, - throughSequence: headSequence, - }); - if (stats.rawPayloadBytes > ORCHESTRATION_REPLAY_PAYLOAD_BUDGET_BYTES) { - yield* Effect.logDebug("orchestration replay replaced by snapshot", { - afterSequence, - headSequence, - replayGap, - eventCount: stats.eventCount, - payloadBytes: stats.rawPayloadBytes, - payloadBudgetBytes: ORCHESTRATION_REPLAY_PAYLOAD_BUDGET_BYTES, - }); - return false; - } - return true; -}); - const enrichProjectShells = Effect.fn("ws.orchestrationV2.enrichProjectShells")( (projects: ReadonlyArray) => Effect.flatMap(ProjectEnrichmentService.ProjectEnrichmentService, (projectEnrichment) => @@ -1153,6 +1166,7 @@ export const subscribeOrchestrationV2Shell = Effect.fn("ws.orchestrationV2.subsc function* (input: { readonly afterSequence?: number; readonly httpSnapshotFallback?: true; + readonly knownProjectIdentities?: string; readonly requestCompletionMarker?: boolean; readonly omitIdleSubagents?: boolean; readonly omitOlderSettled?: boolean; @@ -1381,7 +1395,7 @@ export const subscribeOrchestrationV2Shell = Effect.fn("ws.orchestrationV2.subsc }); }); const highWater = yield* applicationEvents.latestApplicationSequence; - if (!(yield* canReplayPersistedRange(input.afterSequence, highWater, SHELL_RESUME_MAX_GAP))) { + if (!(yield* canResumeShellFrom(applicationEvents, input.afterSequence, highWater))) { return yield* fromSnapshot; } @@ -1400,12 +1414,22 @@ export const subscribeOrchestrationV2Shell = Effect.fn("ws.orchestrationV2.subsc skipUnknownEventTypes: true, }) .pipe(Stream.filter(filterSubagentTranscript), Stream.runCollect); - if (input.afterSequence < (yield* applicationEvents.prunedThroughSequence)) { + if ( + input.afterSequence < highWater && + input.afterSequence < (yield* applicationEvents.prunedThroughSequence) + ) { return yield* fromSnapshot; } const replay = toShellStream(Stream.fromIterable(replayed)); return composeShellStreamWithEnrichment({ - initial: initialEnrichmentItems(loaded), + initial: initialEnrichmentItems({ + snapshot: loaded.snapshot, + resolvedRepositoryIdentityRoots: resumeIdentityRoots({ + projects: loaded.snapshot.projects, + resolvedRepositoryIdentityRoots: loaded.resolvedRepositoryIdentityRoots, + knownProjectIdentities: input.knownProjectIdentities, + }), + }), tail: Stream.concat(Stream.concat(replay, completionMarker), liveFrom(highWater)), enrichment: enrichmentRefreshes, }); @@ -1535,6 +1559,9 @@ export const layerWsRpc = ( const vcsProvisioning = yield* VcsProvisioningService.VcsProvisioningService; const vcsStatusBroadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; const terminalManager = yield* TerminalManager.TerminalManager; + const prepareTerminalLaunch = yield* makePrepareTerminalLaunch( + threadManagement.getThreadShell, + ); const previewManager = yield* PreviewManager.PreviewManager; const portDiscovery = yield* PortScanner.PortDiscovery; const providerRegistry = yield* ProviderRegistry.ProviderRegistry; @@ -1578,6 +1605,7 @@ export const layerWsRpc = ( ); const serverAuth = yield* EnvironmentAuth.EnvironmentAuth; const sourceControlDiscovery = yield* SourceControlDiscovery.SourceControlDiscovery; + const secretCheck = yield* SecretCheck.SecretCheck; const otlpEndpointCheck = yield* OtlpEndpointCheck.OtlpEndpointCheck; const textGenerators = yield* TextGenerators.TextGenerators; const sourceControlRepositories = @@ -1797,6 +1825,7 @@ export const layerWsRpc = ( Effect.all({ pairingLinks: serverAuth.listPairingLinks(), clientSessions: serverAuth.listClientSessions(currentSessionId), + connectDevices: serverAuth.listConnectDevices(), }).pipe( Effect.mapError( (error) => @@ -1811,6 +1840,7 @@ export const layerWsRpc = ( readonly workspaceSnapshotsOnDemand: boolean; readonly deferNetworkDiscovery: boolean; readonly catalogsPerCwd?: ReadonlySet; + readonly catalogDigests?: boolean; }) { const keybindingsConfig = yield* keybindings.loadConfigState; const currentProviders = yield* providerRegistry.getProviders; @@ -1818,7 +1848,11 @@ export const layerWsRpc = ( ? withUsageLimitsCommands(currentProviders, yield* usageLimitSources.current) : currentProviders; const providers = options.workspaceSnapshotsOnDemand - ? withoutWorkspaceSnapshots(commandProviders, options.catalogsPerCwd) + ? withoutWorkspaceSnapshots( + commandProviders, + options.catalogsPerCwd, + options.catalogDigests, + ) : commandProviders; const settings = ServerSettings.redactServerSettingsForClient( yield* serverSettings.getSettings, @@ -1899,6 +1933,7 @@ export const layerWsRpc = ( }), newProjectsRoot: managedFolders.namedProjectsRoot, ...(lastEventLoopStall === undefined ? {} : { lastEventLoopStall }), + serverTime: DateTime.formatIso(yield* DateTime.now), }; }); @@ -2495,12 +2530,15 @@ export const layerWsRpc = ( providers = yield* providerRegistry.refreshInstance(instance.instanceId); } } + const { instanceId, cwd } = input; + if (input.omitProviders === true && (instanceId === undefined || cwd === undefined)) { + return { providers: [] }; + } if (input.workspaceSnapshotsOnDemand !== true) return { providers }; const commandProviders = input.usageLimitsCommand === true ? withUsageLimitsCommands(providers, yield* usageLimitSources.current) : providers; - const { instanceId, cwd } = input; if (instanceId === undefined || cwd === undefined) { return { providers: withoutWorkspaceSnapshots(commandProviders) }; } @@ -2510,14 +2548,23 @@ export const layerWsRpc = ( return { providers: commandProviders .filter((provider) => provider.instanceId === instanceId) - .map((provider) => ({ - ...provider, - // Callers read only the cwd snapshot; the config stream carries the rest. - models: [], - slashCommands: [], - skills: [], - workspaceSnapshots: workspaceSnapshotOrCatalog(provider, cwd, catalogOnDemand), - })), + .map((provider) => { + const catalog = + catalogOnDemand && input.machineCatalog === true + ? machineCatalogNames(provider) + : undefined; + return { + ...provider, + // Callers read only the cwd snapshot; the config stream carries the rest. + models: [], + slashCommands: catalog?.slashCommands ?? [], + skills: catalog?.skills ?? [], + ...(catalog === undefined + ? {} + : { catalogDigest: machineCatalogDigest(provider) }), + workspaceSnapshots: workspaceSnapshotOrCatalog(provider, cwd, catalogOnDemand), + }; + }), }; }), [WS_METHODS.mcpAppsCallTool]: (input) => mcpAppRequests.callTool(input), @@ -2655,6 +2702,10 @@ export const layerWsRpc = ( Effect.annotateCurrentSpan({ "orchestration_v2.thread_id": input.threadId }).pipe( Effect.andThen(peerLinkRequests.answer(input)), ), + [WS_METHODS.serverGetSettings]: (_input) => + Effect.map(serverSettings.getSettings, (settings) => + ServerSettings.redactServerSettingsForClient(settings, settingsClient), + ), [WS_METHODS.serverUpdateSettings]: /* Request instrumentation already owns the span; keep this helper in that context. */ Effect.fnUntraced(function* ({ patch, providerInstanceMutation }) { @@ -2670,6 +2721,7 @@ export const layerWsRpc = ( return ServerSettings.redactServerSettingsForClient(settings, settingsClient); }), [WS_METHODS.serverDiscoverSourceControl]: (_input) => sourceControlDiscovery.discover, + [WS_METHODS.serverCheckSecretSource]: (input) => secretCheck.check(input.source), [WS_METHODS.serverGetTraceDiagnostics]: (_input) => TraceDiagnostics.readTraceDiagnostics({ traceFilePath: config.serverTracePath, @@ -2824,6 +2876,8 @@ export const layerWsRpc = ( pullRequests.subscribeRefreshes.pipe( Stream.map((event) => (input.scoped ? event : event.revision)), ), + [WS_METHODS.pullRequestsSubscribeLinkedThreadRefreshes]: () => + pullRequests.subscribeLinkedThreadRefreshes, [WS_METHODS.pullRequestsReviewerCandidates]: (input) => withPullRequestViewer(input, pullRequests.reviewerCandidates(input)), [WS_METHODS.pullRequestsRequestReviewers]: (input) => @@ -2994,7 +3048,7 @@ export const layerWsRpc = ( new ProjectMutationError({ commandId: mutation.commandId, message: - cause._tag === "ProjectNotEmptyError" + cause._tag === "ProjectNotEmptyError" || cause._tag === "ProjectInvalidError" ? cause.message : "Failed to mutate project.", cause, @@ -3174,19 +3228,52 @@ export const layerWsRpc = ( [WS_METHODS.vcsCreateWorktree]: (input) => gitWorkflow.createWorktree(input).pipe(Effect.tap(() => refreshGitStatus(input.cwd))), [WS_METHODS.vcsRemoveWorktree]: (input) => - gitWorkflow.removeWorktree(input).pipe(Effect.tap(() => refreshGitStatus(input.cwd))), + gitWorkflow.removeWorktree(input).pipe( + Effect.tap(() => refreshGitStatus(input.cwd)), + // Subscribers of the removed checkout learn it is no longer a repository. + Effect.tap(() => + vcsStatusBroadcaster + .refreshLocalStatus(input.path) + .pipe(Effect.ignoreCause({ log: true }), Effect.forkDetach, Effect.asVoid), + ), + ), [WS_METHODS.vcsCreateRef]: (input) => - gitWorkflow.createRef(input).pipe(Effect.tap(() => refreshGitStatus(input.cwd))), + gitWorkflow + .createRef(input) + .pipe( + Effect.tap(() => + input.switchRef + ? vcsStatusBroadcaster.refreshAfterCheckout(input.cwd) + : refreshGitStatus(input.cwd), + ), + ), [WS_METHODS.vcsSwitchRef]: (input) => - gitWorkflow.switchRef(input).pipe(Effect.tap(() => refreshGitStatus(input.cwd))), + gitWorkflow + .switchRef(input) + .pipe(Effect.tap(() => vcsStatusBroadcaster.refreshAfterCheckout(input.cwd))), [WS_METHODS.vcsInit]: (input) => vcsProvisioning.initRepository(input).pipe(Effect.tap(() => refreshGitStatus(input.cwd))), [WS_METHODS.reviewGetBranchCommits]: (input) => review.getBranchCommits(input), [WS_METHODS.reviewGetDiffPreview]: (input) => review.getDiffPreview(input), [WS_METHODS.reviewGetDiffFileContents]: (input) => review.getDiffFileContents(input), - [WS_METHODS.terminalOpen]: (input) => terminalManager.open(input), + [WS_METHODS.terminalOpen]: (input) => + prepareTerminalLaunch(input).pipe( + Effect.flatMap((launch) => + terminalManager + .open(launch.input) + .pipe(Effect.provideService(TerminalManager.TerminalCwdRecovery, launch.recover)), + ), + ), [WS_METHODS.terminalAttach]: (input, metadata) => - terminalManager.attachOutputStream(input, terminalViewerId(metadata)), + Stream.unwrap( + prepareTerminalLaunch(input).pipe( + Effect.map((launch) => + terminalManager + .attachOutputStream(launch.input, terminalViewerId(metadata)) + .pipe(Stream.provideService(TerminalManager.TerminalCwdRecovery, launch.recover)), + ), + ), + ), [WS_METHODS.terminalWrite]: (input) => terminalManager.write(input), [WS_METHODS.terminalResize]: (input, metadata) => terminalManager.resize(input, terminalViewerId(metadata)), @@ -3286,6 +3373,7 @@ export const layerWsRpc = ( ), ) : Effect.succeed(new Set()); + const catalogDigests = input.providerCatalogDigests === true; // Buffer changes through discovery and the initial snapshot's send. const keybindingsChanges = yield* keybindings.subscribeChanges; const settingsChanges = yield* serverSettings.subscribeChanges; @@ -3303,6 +3391,7 @@ export const layerWsRpc = ( workspaceSnapshotsOnDemand, deferNetworkDiscovery: true, catalogsPerCwd: yield* currentCatalogsPerCwd, + catalogDigests, }); const keybindingsUpdates = keybindingsChanges.pipe( Stream.map((event) => ({ @@ -3335,7 +3424,11 @@ export const layerWsRpc = ( workspaceSnapshotsOnDemand ? currentCatalogsPerCwd.pipe( Effect.map((catalogsPerCwd) => - withoutWorkspaceSnapshots(commandProviders, catalogsPerCwd), + withoutWorkspaceSnapshots( + commandProviders, + catalogsPerCwd, + catalogDigests, + ), ), ) : Effect.succeed(commandProviders), @@ -3414,19 +3507,22 @@ export const layerWsRpc = ( remoteResult?._tag === "Success" ? { ...config, remoteOpenTargets: remoteResult.value } : config; + const updates = withLateConfigDiscovery( + initialConfig, + liveUpdates, + externalLauncher, + { + resolveTargets: () => Fiber.join(remoteTargets), + }, + { resolve: () => Fiber.join(directResult) }, + ); return Stream.concat( rpcInitialItems([ { version: 1 as const, type: "snapshot" as const, config: initialConfig }, ]), - withLateConfigDiscovery( - initialConfig, - liveUpdates, - externalLauncher, - { - resolveTargets: () => Fiber.join(remoteTargets), - }, - { resolve: () => Fiber.join(directResult) }, - ), + input.providerStatusDeltas === true + ? updates.pipe(withProviderStatusDeltas(initialConfig.providers)) + : updates, ); }), ), @@ -3453,21 +3549,47 @@ export const layerWsRpc = ( [WS_METHODS.subscribeAuthAccess]: (_input) => Stream.unwrap( Effect.gen(function* () { + const liveBuffer = yield* Queue.unbounded< + PairingGrantStore.BootstrapCredentialChange | SessionStore.SessionCredentialChange + >(); + // Each store subscribes only when consumed, so start both before the snapshot read. + yield* Effect.forkScoped( + Stream.runForEach(bootstrapCredentials.streamChanges, (change) => + Queue.offer(liveBuffer, change), + ), + { startImmediately: true }, + ); + yield* Effect.forkScoped( + Stream.runForEach(sessions.streamChanges, (change) => + Queue.offer(liveBuffer, change), + ), + { startImmediately: true }, + ); const initialSnapshot = yield* loadAuthAccessSnapshot(); const revisionRef = yield* Ref.make(1); - const accessChanges: Stream.Stream< - PairingGrantStore.BootstrapCredentialChange | SessionStore.SessionCredentialChange - > = Stream.merge(bootstrapCredentials.streamChanges, sessions.streamChanges); - const liveEvents: Stream.Stream = accessChanges.pipe( - Stream.mapEffect((change) => - Ref.updateAndGet(revisionRef, (revision) => revision + 1).pipe( - Effect.map((revision) => - toAuthAccessStreamEvent(change, revision, currentSessionId), + const liveEvents: Stream.Stream = + Stream.fromQueue(liveBuffer).pipe( + Stream.mapEffect((change) => + Ref.updateAndGet(revisionRef, (revision) => revision + 1).pipe( + Effect.flatMap((revision) => + // Device trust has no incremental event older clients could decode. + change.type === "connectDevicesChanged" + ? loadAuthAccessSnapshot().pipe( + Effect.map((payload): AuthAccessStreamEvent => ({ + version: 1, + revision, + type: "snapshot", + payload, + })), + ) + : Effect.succeed( + toAuthAccessStreamEvent(change, revision, currentSessionId), + ), + ), ), ), - ), - ); + ); return Stream.concat( rpcInitialItems([ diff --git a/apps/server/vite.config.ts b/apps/server/vite.config.ts index 02fe582953ab..0ec539e5ae16 100644 --- a/apps/server/vite.config.ts +++ b/apps/server/vite.config.ts @@ -5,6 +5,7 @@ import { defineConfig, mergeConfig } from "vite-plus"; import baseConfig from "../../vite.config.ts"; import { loadRepoEnv } from "../../scripts/lib/public-config.ts"; +import { resolveTaggedSourceVersion } from "../../scripts/lib/tagged-source-version.ts"; import rootPackageJson from "../../package.json" with { type: "json" }; import packageJson from "./package.json" with { type: "json" }; import { WeightedShardSequencer } from "./src/testUtils/weightedShardSequencer.ts"; @@ -18,9 +19,12 @@ import { export { shouldBundleCliDependency }; const repoEnv = loadRepoEnv(); -const cliBuildChannel = /^[^-+]+-(?:nightly|preview)\./.test(packageJson.version) - ? "nightly" - : "latest"; +const serverManifestPath = NodePath.join(import.meta.dirname, "package.json"); +// The desktop packager passes APP_VERSION; it can equal the manifest at a commit tagged otherwise. +const buildVersion = + process.env.APP_VERSION?.trim() || + resolveTaggedSourceVersion(NodePath.join(import.meta.dirname, "../.."), packageJson.version); +const cliBuildChannel = /^[^-+]+-(?:nightly|preview)\./.test(buildVersion) ? "nightly" : "latest"; // Official release workflows set the telemetry channel; local builds and forks use "dev". const TELEMETRY_CHANNELS = ["stable", "nightly", "preview", "dev"]; const telemetryChannel = process.env.T3CODE_TELEMETRY_CHANNEL?.trim() || "dev"; @@ -88,6 +92,19 @@ export default mergeConfig( }, }, pack: { + plugins: [ + { + // Every server module reads its version from this manifest import. + name: "t3code:build-version", + load(id) { + if (NodePath.resolve(id) !== serverManifestPath) return; + return { + code: JSON.stringify({ ...packageJson, version: buildVersion }), + moduleType: "json", + }; + }, + }, + ], // SEA embeds one entry; expose the history worker as a hidden subcommand. entry: packBootstrap ? ["src/bin.ts"] @@ -115,6 +132,8 @@ export default mergeConfig( }, } : {}), + // jsonc-parser's UMD main requires its own files at runtime, which the bundle cannot follow. + alias: { "jsonc-parser": "jsonc-parser/lib/esm/main.js" }, deps: { // alwaysBundle=false has no opinion; neverBundle must retain transitive native acceleration. alwaysBundle: shouldBundleCliDependency, diff --git a/apps/web/index.html b/apps/web/index.html index 1a8bc02e19c9..051e3090a5d4 100644 --- a/apps/web/index.html +++ b/apps/web/index.html @@ -15,6 +15,7 @@ const LIGHT_BACKGROUND = "#ffffff"; const DARK_BACKGROUND = "#000000"; const CUSTOM_THEMES_STORAGE_KEY = "t3code:themes:v1"; + const ENVIRONMENT_THEMES_STORAGE_KEY = "t3code:environment-themes:v1"; const THEME_FOLLOW_SYSTEM_STORAGE_KEY = "t3code:theme-follow-system"; const THEME_APPEARANCE_MODE_STORAGE_KEY = "t3code:theme-appearance-mode"; const THEME_HALVES_STORAGE_KEY = "t3code:theme-halves:v1"; @@ -197,12 +198,9 @@ (appearance === value.appearance || isRecord(colors)), ); }; - const findCustomTheme = (themeId) => { - if (RESERVED_THEME_IDS.includes(themeId)) return null; + const findStoredTheme = (storageKey, themeId) => { try { - const parsed = JSON.parse( - window.localStorage.getItem(CUSTOM_THEMES_STORAGE_KEY) ?? "null", - ); + const parsed = JSON.parse(window.localStorage.getItem(storageKey) ?? "null"); if (!Array.isArray(parsed)) return null; return ( parsed.find((value) => isStoredCustomTheme(value) && value.id === themeId) ?? null @@ -211,6 +209,14 @@ return null; } }; + // Same precedence as getThemeDefinition: saved custom themes, then published ones. + const findCustomTheme = (themeId) => { + if (RESERVED_THEME_IDS.includes(themeId)) return null; + return ( + findStoredTheme(CUSTOM_THEMES_STORAGE_KEY, themeId) ?? + findStoredTheme(ENVIRONMENT_THEMES_STORAGE_KEY, themeId) + ); + }; try { const storedTheme = window.localStorage.getItem("t3code:theme"); diff --git a/apps/web/package.json b/apps/web/package.json index 11d73266aac6..ae40b208869d 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -1,6 +1,6 @@ { "name": "@t3tools/web", - "version": "0.0.110", + "version": "0.0.111", "private": true, "type": "module", "scripts": { diff --git a/apps/web/src/appInit.tsx b/apps/web/src/appInit.tsx index fa8e88e6b1a9..faa71137db04 100644 --- a/apps/web/src/appInit.tsx +++ b/apps/web/src/appInit.tsx @@ -14,6 +14,7 @@ import { startElectronBrowserHost } from "./browser/ElectronBrowserHost"; import { startBrowserProfileReporting } from "./browser/BrowserProfileReporter"; import { startPreviewTitleRecording } from "./browserHistoryStore"; import { retainThreadOverlaysForThread } from "./components/chat/threadOverlayRetention"; +import { resolveWindowTitleThreadStatus } from "./components/Sidebar.logic"; import { startBackgroundThreadAnnouncements } from "./components/BackgroundThreadAnnouncer"; import { startSshPasswordPromptQueue } from "./components/desktop/SshPasswordPromptDialog"; import { startHostFreezeToasts } from "./components/HostFreezeToastCoordinator"; @@ -133,7 +134,10 @@ function readOpenThreadRef(router: AppRouter): ScopedThreadRef | null { return resolveActiveThreadRouteRef(target, draft); } -/** Names the open thread and its project in the window title, or the app when none is open. */ +/** + * Names the open thread and its project in the window title, or the app when none is open. + * Screen readers read it on Alt+Tab, so it can lead with the thread and what it's doing. + */ function startDocumentTitle(router: AppRouter): void { const readOnGuestPage = () => router.state.matches.some((match) => match.routeId === "/guest"); let openThreadRef = readOpenThreadRef(router); @@ -163,6 +167,11 @@ function startDocumentTitle(router: AppRouter): void { }), projectTitle: project?.title ?? null, threadTitle: thread?.title ?? null, + ...(getClientSettings().screenReaderWindowTitle && { + screenReader: { + threadStatus: thread === null ? null : resolveWindowTitleThreadStatus(thread), + }, + }), desktop: isElectron, }); // Projects change whenever any project does; most changes leave the title alone. @@ -198,6 +207,7 @@ function startDocumentTitle(router: AppRouter): void { watchOpenThread(); appAtomRegistry.subscribe(primaryServerConfigAtom, write, { immediate: true }); appAtomRegistry.subscribe(environmentProjects.projectsAtom, write); + subscribeClientSettings(write); router.subscribe("onResolved", updateOpenThread); useComposerDraftStore.subscribe(updateOpenThread); } diff --git a/apps/web/src/appearanceFonts.ts b/apps/web/src/appearanceFonts.ts index 19f2d88d3c29..50a664fea336 100644 --- a/apps/web/src/appearanceFonts.ts +++ b/apps/web/src/appearanceFonts.ts @@ -276,6 +276,18 @@ function resolveGenericFamilyLabel(generic: string): string | null { return null; } +// Synthetic Apple labels are not necessarily available as named CSS families. +export function displayFontFamily(family: string): string { + const label = family + .trim() + .replace(/^(['"])(.*)\1$/, "$2") + .trim() + .toLowerCase(); + if (label === "sf pro") return appearanceFontStack(family, DEFAULT_SANS_FONT_STACK); + if (label === "sf mono") return DEFAULT_CODE_FONT_STACK; + return cssFontFamilies(family) ?? ""; +} + export function resolveDefaultFamilyLabel(stack: string): string | null { for (const raw of stack.split(",")) { const family = raw.trim().replace(/^(['"])(.*)\1$/, "$2"); diff --git a/apps/web/src/authBootstrap.test.ts b/apps/web/src/authBootstrap.test.ts index 65c6f8edd342..f444c64cb16a 100644 --- a/apps/web/src/authBootstrap.test.ts +++ b/apps/web/src/authBootstrap.test.ts @@ -8,6 +8,7 @@ import { import { createBrowserHistory } from "@tanstack/react-router"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; +import { HttpClientError, HttpClientRequest } from "effect/http"; import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; import { installEnvironmentHttpTest } from "../test/environmentHttpTest"; @@ -165,6 +166,54 @@ describe("resolveInitialServerAuthGateState", () => { expect(testApi.calls.browserSession).toEqual([{ credential: "desktop-bootstrap-token" }]); }); + it("retries the desktop bootstrap while the backend is still unreachable", async () => { + const nextSession = sequence( + unauthenticatedSession(DESKTOP_AUTH), + authenticatedSession(DESKTOP_AUTH), + ); + const testApi = await installAuthApi({ + session: nextSession, + browserSession: () => Effect.succeed(browserSession(["orchestration:read", "access:write"])), + }); + // What FetchHttpClient yields when nothing listens on the backend port. + const connectionRefused = new HttpClientError.HttpClientError({ + reason: new HttpClientError.TransportError({ + request: HttpClientRequest.get("http://localhost:3773/api/auth/session"), + cause: new TypeError("Failed to fetch"), + }), + }); + let unreachableAttempts = 1; + __setPrimaryHttpRunnerForTests((effect) => { + if (unreachableAttempts > 0) { + unreachableAttempts -= 1; + return Promise.reject(connectionRefused); + } + return testApi.run(effect); + }); + + const testWindow = installTestBrowser("http://localhost/"); + testWindow.desktopBridge = { + getLocalEnvironmentBootstraps: () => [ + { + id: "primary", + label: "Local", + httpBaseUrl: "http://localhost:3773", + wsBaseUrl: "ws://localhost:3773", + bootstrapToken: "desktop-bootstrap-token", + }, + ], + } as unknown as DesktopBridge; + + const { resolveInitialServerAuthGateState } = await import("./environments/primary"); + + vi.useFakeTimers({ toFake: ["setTimeout", "Date"] }); + await Promise.all([ + expect(resolveInitialServerAuthGateState()).resolves.toEqual({ status: "authenticated" }), + vi.advanceTimersByTimeAsync(1_000), + ]); + expect(testApi.calls.browserSession).toEqual([{ credential: "desktop-bootstrap-token" }]); + }); + it("takes a pairing token from the location hash and strips it immediately", async () => { const testWindow = installTestBrowser("http://localhost/#token=pairing-token"); const { takePairingTokenFromUrl } = await import("./environments/primary"); diff --git a/apps/web/src/branding.logic.ts b/apps/web/src/branding.logic.ts index 6490edbb4a05..f53103d904ad 100644 --- a/apps/web/src/branding.logic.ts +++ b/apps/web/src/branding.logic.ts @@ -60,14 +60,23 @@ export function resolveWindowTitle(input: { readonly appDisplayName: string; readonly projectTitle: string | null; readonly threadTitle: string | null; + /** The screen reader title: status, then thread, then project, since a screen reader reads from the start. */ + readonly screenReader?: { readonly threadStatus: string | null }; readonly desktop: boolean; }): string { const threadTitle = input.threadTitle?.trim() ?? ""; + const status = input.screenReader?.threadStatus; if (threadTitle === "") { - return input.appDisplayName; + return status ? `${status}: ${input.appDisplayName}` : input.appDisplayName; } const projectTitle = input.projectTitle?.trim() ?? ""; - const context = projectTitle === "" ? threadTitle : `${projectTitle} / ${threadTitle}`; - return input.desktop ? context : `${context} - ${input.appDisplayName}`; + const context = + projectTitle === "" + ? threadTitle + : input.screenReader + ? `${threadTitle} - ${projectTitle}` + : `${projectTitle} / ${threadTitle}`; + const title = status ? `${status}: ${context}` : context; + return input.desktop ? title : `${title} - ${input.appDisplayName}`; } diff --git a/apps/web/src/browser/HostedBrowserWebview.tsx b/apps/web/src/browser/HostedBrowserWebview.tsx index 44de1196b2b7..023fb03c3aa1 100644 --- a/apps/web/src/browser/HostedBrowserWebview.tsx +++ b/apps/web/src/browser/HostedBrowserWebview.tsx @@ -20,6 +20,7 @@ import { useActiveBrowserRecordingTabIds } from "./browserRecording"; import { browserViewportSettingKey, resolveBrowserViewportLayout, + resolveBrowserViewportTransform, resolveFittedBrowserViewport, } from "./browserViewportLayout"; import { BrowserDeviceToolbar } from "./BrowserDeviceToolbar"; @@ -184,16 +185,24 @@ export function HostedBrowserWebview(props: { if (!disposed) remountCrashedWebview(); }, recovery.delayMs); }; - // Guest clicks only emit focus here; replay pointerdown to dismiss host popups. + // Guest clicks never reach this document, so replay a pointerdown to dismiss host popups. + // The guest preload reports each press, since the guest does not always take focus. const dismissHostPopups = () => { webview.dispatchEvent( new PointerEvent("pointerdown", { bubbles: true, pointerType: "mouse" }), ); }; + const guestPointerDown = (event: Event) => { + // Mirrors POINTER_DOWN_HOST_CHANNEL in apps/desktop/src/preview/GuestProtocol.ts. + if ((event as Event & { channel?: string }).channel === "preview:pointer-down") { + dismissHostPopups(); + } + }; webview.addEventListener("did-attach", register); webview.addEventListener("dom-ready", register); webview.addEventListener("render-process-gone", recoverGuest); webview.addEventListener("focus", dismissHostPopups); + webview.addEventListener("ipc-message", guestPointerDown); register(); return () => { disposed = true; @@ -202,6 +211,7 @@ export function HostedBrowserWebview(props: { webview.removeEventListener("dom-ready", register); webview.removeEventListener("render-process-gone", recoverGuest); webview.removeEventListener("focus", dismissHostPopups); + webview.removeEventListener("ipc-message", guestPointerDown); }; }, [ clientSettingsHydrated, @@ -304,6 +314,23 @@ export function HostedBrowserWebview(props: { return () => withDesktopTab(runtimeTabId, () => bridge.setShown(runtimeTabId, false)); }, [clientSettingsHydrated, runtimeTabId, serverDriven, shown]); + // Captures copy the guest's raster, which a scaled guest only keeps at full detail on request. + const fullDetail = backgroundActivity || pictureInPicture || recordingActive; + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- tells the desktop host once the guest is drawn at full detail, so its screenshot waits for that frame + useEffect(() => { + const bridge = window.desktopBridge?.preview; + if (!clientSettingsHydrated || !bridge || !serverDriven || !fullDetail) return; + let frame = requestAnimationFrame(() => { + frame = requestAnimationFrame(() => + withDesktopTab(runtimeTabId, () => bridge.setCaptureDrawn(runtimeTabId, true)), + ); + }); + return () => { + cancelAnimationFrame(frame); + withDesktopTab(runtimeTabId, () => bridge.setCaptureDrawn(runtimeTabId, false)); + }; + }, [clientSettingsHydrated, fullDetail, runtimeTabId, serverDriven]); + if (!clientSettingsHydrated || !config) return null; // This fixed wrapper sits outside the right panel shell, so it takes thumb presses itself. @@ -313,6 +340,7 @@ export function HostedBrowserWebview(props: { }; const renderingActive = active || backgroundActivity || pictureInPicture || recordingActive; + const viewportTransform = resolveBrowserViewportTransform(layout.viewportScale, fullDetail); const wrapperStyle = resolveHostedBrowserWebviewWrapperStyle({ active, renderingActive, @@ -345,47 +373,56 @@ export function HostedBrowserWebview(props: { onChange={commitViewportChange} /> ) : null} - + > + + {active && effectiveViewport._tag !== "fill" && !fittedSourceViewport ? ( <> void; readonly history: (delta: -1 | 1) => void; readonly reload: (options?: { readonly ignoreCache?: boolean }) => void; + readonly stopLoading: () => void; readonly viewport: (setting: PreviewViewportSetting) => void; readonly canvas: () => HTMLCanvasElement | null; } @@ -311,6 +312,7 @@ export function ServerBrowserSurface(props: { history: (delta) => send({ type: "history", delta }), reload: (options) => send({ type: "reload", ...(options?.ignoreCache ? { ignoreCache: true } : {}) }), + stopLoading: () => send({ type: "stopLoading" }), viewport: (setting) => send({ type: "viewport", setting }), canvas: () => (!pdfUrl && hasFrameRef.current ? canvasRef.current : null), }), diff --git a/apps/web/src/browser/browserDefaults.ts b/apps/web/src/browser/browserDefaults.ts index bf1c2da40db8..5cc74bb1b7f0 100644 --- a/apps/web/src/browser/browserDefaults.ts +++ b/apps/web/src/browser/browserDefaults.ts @@ -5,6 +5,7 @@ import { type DesktopPreviewTabDefaults, type PreviewAppearancePreference, type PreviewViewportSetting, + type PreviewZoomFactor, } from "@t3tools/contracts"; import { @@ -17,7 +18,7 @@ import { resolveResponsiveBrowserViewportSize } from "./browserViewportLayout"; export interface BrowserDefaults { readonly viewport: PreviewViewportSetting; - readonly zoomFactor: number; + readonly zoomFactor: PreviewZoomFactor; readonly appearance: PreviewAppearancePreference; readonly autoShowFloatingPreview: boolean; readonly profiles: ReadonlyArray; @@ -26,7 +27,7 @@ export interface BrowserDefaults { const toBrowserDefaults = (settings: { readonly browserDefaultViewport: PreviewViewportSetting; - readonly browserDefaultZoomFactor: number; + readonly browserDefaultZoomFactor: PreviewZoomFactor; readonly browserDefaultAppearance: PreviewAppearancePreference; readonly browserAutoShowFloatingPreview: boolean; readonly browserProfiles: ReadonlyArray; diff --git a/apps/web/src/browser/browserLinkTarget.ts b/apps/web/src/browser/browserLinkTarget.ts index f1c4e65a9991..74330bf3edbd 100644 --- a/apps/web/src/browser/browserLinkTarget.ts +++ b/apps/web/src/browser/browserLinkTarget.ts @@ -7,6 +7,9 @@ import { import { ensureClientSettingsHydrated, getClientSettings } from "~/hooks/useSettings"; import { readEnvironmentScope } from "~/state/session"; import { isPreviewAvailableFor } from "./previewRuntime"; +import { isWebUrl } from "./webUrl"; + +export { isWebUrl }; export interface ResolveLinkTargetInput { readonly url: string; @@ -27,15 +30,6 @@ export function resolveLinkTarget(input: ResolveLinkTargetInput): BrowserLinkTar return "app"; } -export function isWebUrl(url: string): boolean { - try { - const { protocol } = new URL(url); - return protocol === "http:" || protocol === "https:"; - } catch { - return false; - } -} - /** Before hydration, the default "system" would ignore a saved "app" link target. */ export async function resolveBrowserLinkTargetPreference(): Promise { await ensureClientSettingsHydrated(); diff --git a/apps/web/src/browser/browserViewportLayout.test.ts b/apps/web/src/browser/browserViewportLayout.test.ts index 1932a634fb6c..c72f68e1e20d 100644 --- a/apps/web/src/browser/browserViewportLayout.test.ts +++ b/apps/web/src/browser/browserViewportLayout.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vite-plus/test"; import { resizeBrowserViewportFromRail, + resolveBrowserViewportTransform, resolveFittedBrowserViewport, } from "./browserViewportLayout"; @@ -43,3 +44,19 @@ describe("resolveBrowserViewportLayout", () => { ).toEqual({ width: 460, height: 409 }); }); }); + +describe("resolveBrowserViewportTransform", () => { + it("keeps a full-detail guest at raster scale 1 while it draws at the fitted scale", () => { + expect(resolveBrowserViewportTransform(1, true)).toBeUndefined(); + expect(resolveBrowserViewportTransform(0.6, false)).toBe("scale(0.6)"); + const values = resolveBrowserViewportTransform(0.6, true) + ?.match(/^matrix3d\((.*)\)$/)?.[1] + ?.split(",") + .map(Number); + // Column-major: the x and y columns keep their 0.6 screen terms and reach length 1 through z. + expect(values?.slice(0, 2)).toEqual([0.6, 0]); + expect(values?.slice(4, 6)).toEqual([0, 0.6]); + expect(Math.hypot(...(values?.slice(0, 3) ?? []))).toBeCloseTo(1); + expect(Math.hypot(...(values?.slice(4, 7) ?? []))).toBeCloseTo(1); + }); +}); diff --git a/apps/web/src/browser/browserViewportLayout.ts b/apps/web/src/browser/browserViewportLayout.ts index 2fe4e035100a..7fe6b170c671 100644 --- a/apps/web/src/browser/browserViewportLayout.ts +++ b/apps/web/src/browser/browserViewportLayout.ts @@ -125,6 +125,22 @@ export function resolveBrowserViewportLayout( }; } +/** + * Chromium rasters a CSS-scaled guest at its on-screen scale, so captures of it lose detail. + * Its raster scale also counts x/y-to-z terms, which a `preserve-3d` parent keeps and the + * screen drops, so `fullDetail` draws like `scale()` but keeps full-resolution frames. + */ +export function resolveBrowserViewportTransform( + scale: number, + fullDetail: boolean, +): string | undefined { + if (scale >= 1) return undefined; + if (!fullDetail) return `scale(${scale})`; + // Positive depth keeps the guest in front of its parent for hit testing. + const depth = Math.sqrt(1 - scale * scale); + return `matrix3d(${scale}, 0, ${depth}, 0, 0, ${scale}, ${depth}, 0, 0, 0, 1, 0, 0, 0, 0, 1)`; +} + export function resolveBrowserDeviceViewportLayout( container: { readonly width: number; readonly height: number }, setting: Exclude, diff --git a/apps/web/src/browser/openFileInPreview.ts b/apps/web/src/browser/openFileInPreview.ts index a997ee2b552e..db1a58e98dfb 100644 --- a/apps/web/src/browser/openFileInPreview.ts +++ b/apps/web/src/browser/openFileInPreview.ts @@ -85,7 +85,9 @@ export async function openUrlInPreview(input: { url: input.url, viewport: defaults.viewport, profileId: input.profileId ?? defaults.profileId, - ...(runtime === undefined ? {} : { runtime }), + ...(runtime === undefined + ? {} + : { runtime, zoomFactor: defaults.zoomFactor, colorScheme: defaults.appearance }), }, }); return mapAtomCommandResult(result, (snapshot) => { diff --git a/apps/web/src/browser/webUrl.ts b/apps/web/src/browser/webUrl.ts new file mode 100644 index 000000000000..19e637794b74 --- /dev/null +++ b/apps/web/src/browser/webUrl.ts @@ -0,0 +1,8 @@ +export function isWebUrl(url: string): boolean { + try { + const { protocol } = new URL(url); + return protocol === "http:" || protocol === "https:"; + } catch { + return false; + } +} diff --git a/apps/web/src/clientPersistenceStorage.ts b/apps/web/src/clientPersistenceStorage.ts index e1c1459facb3..5e3c97649fd0 100644 --- a/apps/web/src/clientPersistenceStorage.ts +++ b/apps/web/src/clientPersistenceStorage.ts @@ -23,3 +23,18 @@ export function writeBrowserClientSettings(settings: ClientSettings): void { setLocalStorageItem(CLIENT_SETTINGS_STORAGE_KEY, settings, ClientSettingsSchema); } + +/** Storage events fire only for other tabs' writes; a null key means storage was cleared. */ +export function subscribeBrowserClientSettingsWrites(onChange: () => void): () => void { + if (!hasWindow()) { + return () => undefined; + } + const handleStorage = (event: StorageEvent) => { + if (event.key === CLIENT_SETTINGS_STORAGE_KEY || event.key === null) { + onChange(); + } + }; + const target = window; + target.addEventListener("storage", handleStorage); + return () => target.removeEventListener("storage", handleStorage); +} diff --git a/apps/web/src/cloud/t3ConnectState.ts b/apps/web/src/cloud/t3ConnectState.ts index c468c73d965e..67fef2eaee1a 100644 --- a/apps/web/src/cloud/t3ConnectState.ts +++ b/apps/web/src/cloud/t3ConnectState.ts @@ -3,6 +3,7 @@ import type { RelayManagedEndpoint, RelayManagedEndpointProviderKind, } from "@t3tools/contracts/relay"; +import { parseUrl } from "@t3tools/shared/url"; // Require managed discovery: stale tunnel config after a publish-only switch causes endpoint_provider_not_managed. export function isManagedTunnelActive( @@ -31,9 +32,9 @@ export function t3ConnectMcpUrl(input: { readonly relayEndpoint: RelayManagedEndpoint | undefined; }): string | null { const url = input.thisMachine - ? URL.parse(`https://${input.tunnelHostname ?? ""}`) + ? parseUrl(`https://${input.tunnelHostname ?? ""}`) : input.relayEndpoint?.providerKind === "cloudflare_tunnel" - ? URL.parse(input.relayEndpoint.httpBaseUrl) + ? parseUrl(input.relayEndpoint.httpBaseUrl) : null; return url?.protocol === "https:" && TUNNEL_HOSTNAME.test(url.hostname) ? `${url.origin}/mcp` diff --git a/apps/web/src/components/AddProjectToMachineDialog.tsx b/apps/web/src/components/AddProjectToMachineDialog.tsx index 3d9c064cfcb2..3bb70c8b609e 100644 --- a/apps/web/src/components/AddProjectToMachineDialog.tsx +++ b/apps/web/src/components/AddProjectToMachineDialog.tsx @@ -26,6 +26,7 @@ import { useAddProjectTargets } from "../hooks/useAddProjectTargets"; import { useNewThreadHandler } from "../hooks/useHandleNewThread"; import { useLogicalProjectEnvironments } from "../hooks/useLogicalProjectEnvironments"; import { + COPIED_FILES_MAX_COUNT, COPYABLE_FILE_MAX_BYTES, addProjectToMachine, isDefaultCopiedFile, @@ -35,6 +36,7 @@ import { newProjectId } from "../lib/utils"; import { readProject, subscribeProject, useProject } from "../state/entities"; import { useEnvironments } from "../state/environments"; import { projectEnvironment } from "../state/projects"; +import { useDebouncedValue } from "../state/queries"; import { useEnvironmentQuery } from "../state/query"; import { EnvironmentMachineIcon } from "./EnvironmentMachineIcon"; import { Button } from "./ui/button"; @@ -121,8 +123,10 @@ function AddProjectToMachineDialog({ request }: { request: AddProjectToMachineRe const [pathOverride, setPathOverride] = useState(null); const destinationPath = pathOverride ?? defaultPath; - const canListFiles = source?.serverConfig?.environment.capabilities.ignoredFileListing === true; - const ignoredFiles = useEnvironmentQuery( + const capabilities = source?.serverConfig?.environment.capabilities; + const canListFiles = capabilities?.ignoredFileListing === true; + const canSearchFiles = canListFiles && capabilities?.ignoredFileSearch === true; + const allIgnoredFiles = useEnvironmentQuery( project && canListFiles ? projectEnvironment.listIgnoredFiles({ environmentId: project.environmentId, @@ -130,13 +134,29 @@ function AddProjectToMachineDialog({ request }: { request: AddProjectToMachineRe }) : null, ); + const [fileQuery, setFileQuery] = useState(""); + const sentFileQuery = useDebouncedValue(fileQuery.trim(), 120); + const searching = canSearchFiles && sentFileQuery.length > 0; + const searchedFiles = useEnvironmentQuery( + project && searching + ? projectEnvironment.listIgnoredFiles({ + environmentId: project.environmentId, + input: { cwd: project.workspaceRoot, query: sentFileQuery }, + }) + : null, + ); + const ignoredFiles = searching ? searchedFiles : allIgnoredFiles; const files = ignoredFiles.data?.files ?? []; - const [selected, setSelected] = useState | null>(null); - const selectedFiles = selected ?? defaultSelection(files); + // Keyed by path with the file kept, so picks made under one search survive the next. + const [selected, setSelected] = useState | null>(null); + const selectedFiles = selected ?? defaultSelection(allIgnoredFiles.data?.files ?? []); const copyable = files.filter((file) => file.byteLength <= COPYABLE_FILE_MAX_BYTES); - const selectedBytes = files - .filter((file) => selectedFiles.has(file.path)) - .reduce((total, file) => total + file.byteLength, 0); + const allCopyableSelected = copyable.every((file) => selectedFiles.has(file.path)); + const atSelectionLimit = selectedFiles.size >= COPIED_FILES_MAX_COUNT; + const selectedBytes = [...selectedFiles.values()].reduce( + (total, file) => total + file.byteLength, + 0, + ); const [submitting, setSubmitting] = useState(false); const [error, setError] = useState(null); @@ -146,11 +166,19 @@ function AddProjectToMachineDialog({ request }: { request: AddProjectToMachineRe const unavailable = project === null || target === null || - (canListFiles && ignoredFiles.data === null && ignoredFiles.error === null); - const toggleFile = (path: string, checked: boolean) => { - const next = new Set(selectedFiles); - if (checked) next.add(path); - else next.delete(path); + (canListFiles && allIgnoredFiles.data === null && allIgnoredFiles.error === null); + const toggleFile = (file: ProjectIgnoredFile, checked: boolean) => { + const next = new Map(selectedFiles); + if (!checked) next.delete(file.path); + else if (!atSelectionLimit) next.set(file.path, file); + setSelected(next); + }; + const toggleShownFiles = () => { + const next = new Map(selectedFiles); + for (const file of copyable) { + if (allCopyableSelected) next.delete(file.path); + else if (next.size < COPIED_FILES_MAX_COUNT) next.set(file.path, file); + } setSelected(next); }; @@ -198,7 +226,7 @@ function AddProjectToMachineDialog({ request }: { request: AddProjectToMachineRe title: project.title, remoteUrl, destinationPath: destinationPath.trim(), - files: files.filter((file) => selectedFiles.has(file.path)).map((file) => file.path), + files: [...selectedFiles.keys()], }); close(); openClonedProject(scopeProjectRef(target.environmentId, projectId)); @@ -334,33 +362,41 @@ function AddProjectToMachineDialog({ request }: { request: AddProjectToMachineRe {copyable.length > 0 ? ( <> - {selectedFiles.size} of {files.length} + {selectedFiles.size} of {allIgnoredFiles.data?.files.length ?? 0} {selectedBytes > 0 ? ` · ${formatBytes(selectedBytes)}` : ""} - ) : null} + {canSearchFiles && (allIgnoredFiles.data?.truncated || fileQuery.length > 0) ? ( + setFileQuery(event.target.value)} + // Enter would otherwise submit the clone. + onKeyDown={(event) => { + if (event.key === "Enter") event.preventDefault(); + }} + spellCheck={false} + autoComplete="off" + /> + ) : null} {ignoredFiles.error ? (

    {ignoredFiles.error}

    ) : files.length === 0 ? (

    - {ignoredFiles.isSuccess - ? "No ignored files to copy." - : "Finding ignored files…"} + {!ignoredFiles.isSuccess + ? "Finding ignored files…" + : searching + ? "No ignored files match." + : "No ignored files to copy."}

    ) : (
    @@ -369,7 +405,8 @@ function AddProjectToMachineDialog({ request }: { request: AddProjectToMachineRe key={file.path} file={file} checked={selectedFiles.has(file.path)} - onCheckedChange={(checked) => toggleFile(file.path, checked)} + disabled={atSelectionLimit && !selectedFiles.has(file.path)} + onCheckedChange={(checked) => toggleFile(file, checked)} /> ))}
    @@ -377,6 +414,7 @@ function AddProjectToMachineDialog({ request }: { request: AddProjectToMachineRe {ignoredFiles.data?.truncated ? (

    Showing the first {files.length} files. + {canSearchFiles ? " Search to find the rest." : ""}

    ) : null}

    @@ -415,29 +453,37 @@ function projectSubdirectory(repositoryRoot: string | undefined, workspaceRoot: : ""; } -function defaultSelection(files: readonly ProjectIgnoredFile[]): ReadonlySet { - return new Set( +function defaultSelection( + files: readonly ProjectIgnoredFile[], +): ReadonlyMap { + return new Map( files .filter( (file) => file.byteLength <= COPYABLE_FILE_MAX_BYTES && isDefaultCopiedFile(file.path), ) - .map((file) => file.path), + .map((file) => [file.path, file]), ); } function IgnoredFileRow({ file, checked, + disabled, onCheckedChange, }: { file: ProjectIgnoredFile; checked: boolean; + disabled: boolean; onCheckedChange: (checked: boolean) => void; }) { const tooLarge = file.byteLength > COPYABLE_FILE_MAX_BYTES; return (

    +