diff --git a/.agents/skills/test-t3-mobile/SKILL.md b/.agents/skills/test-t3-mobile/SKILL.md index 306038007e9a0..a11eee4e3f4eb 100644 --- a/.agents/skills/test-t3-mobile/SKILL.md +++ b/.agents/skills/test-t3-mobile/SKILL.md @@ -43,10 +43,15 @@ prints one JSON line; on failure it names the next command. Mobile verification includes that step unless the user prohibits it. For a device you opened in an agent-device session elsewhere (a device pool, -cloud, or remote host), run `vp run mobile:install --session ` -from the directory you opened the session in. It installs the cached build into -that session (`--abi arm64-v8a` for an ARM Android device; `--metro ` also -opens it against Metro) and never builds. +cloud, or remote host), run `vp run mobile:install --session +--config ` from the directory you opened the session in, passing the +`--config` file from `device_open`'s `targetArgs`. Without it, a device pool's +gateway refuses the install, wrongly saying agent-device is too old. It installs +the cached build into that session (`--abi arm64-v8a` for an ARM Android device; +pool Android emulators are x86_64, the default; `--metro ` also opens it +against Metro). Without a cached build for this checkout it builds first, which can +outlast the lease, so run `vp run mobile:install --prepare` (with +the same `--abi`) before you open the device. Start `vp run dev:client` from `apps/mobile`, or reuse a healthy Metro belonging to this checkout. Open its printed development-client URL with AgentDevice diff --git a/.github/VOUCHED.td b/.github/VOUCHED.td index 69c1ea6575a4a..315ff5742fd5c 100644 --- a/.github/VOUCHED.td +++ b/.github/VOUCHED.td @@ -22,6 +22,7 @@ github:btsouth github:cestercian github:chrisdeeming github:chuks-qua +-github:coygeek Filed over 270 automated issues, about 50 a day at peak, far past what maintainers can triage github:cursoragent github:D3OXY github:dbalders diff --git a/.github/scripts/pr-limits.cjs b/.github/scripts/pr-limits.cjs index 77c09785e4104..f3eb528a08042 100644 --- a/.github/scripts/pr-limits.cjs +++ b/.github/scripts/pr-limits.cjs @@ -100,14 +100,22 @@ function parseExemptions(text) { } // VOUCHED.td: one `github:login` per line, `#` comments, and `-github:login reason` to denounce. -function vouchedLogins(text) { - const logins = new Set(); +// A login listed both ways is an error: pr-vouch.yml's action reads its first line and +// issue-vouch.yml its last, so no answer here would agree with both labels. +function parseVouchList(text) { + const vouched = new Set(); + const denounced = new Set(); for (const raw of text.split("\n")) { - const entry = raw.trim().split(/\s+/)[0]; - if (entry.startsWith("github:")) logins.add(entry.slice("github:".length).toLowerCase()); + const match = /^(-?)github:(\S+)/.exec(raw.trim()); + if (!match) continue; + const login = match[2].toLowerCase(); + const [listed, other] = match[1] ? [denounced, vouched] : [vouched, denounced]; + if (other.has(login)) + throw new Error(`${VOUCHED_PATH}: ${login} is both vouched and denounced`); + listed.add(login); } - if (logins.size === 0) throw new Error(`${VOUCHED_PATH}: no entries`); - return logins; + if (vouched.size === 0) throw new Error(`${VOUCHED_PATH}: no entries`); + return { vouched, denounced }; } function labeledAt(pr, name) { @@ -278,7 +286,8 @@ async function enforceOpenLimit({ github, context, core, dryRun }) { // lookups as "not vouched". Every read below throws on failure, and a throw closes nothing. if (pr.user.type === "Bot") return; if (parseExemptions(fs.readFileSync(EXEMPTIONS_PATH, "utf8")).has(author.toLowerCase())) return; - if (vouchedLogins(fs.readFileSync(VOUCHED_PATH, "utf8")).has(author.toLowerCase())) return; + const { vouched, denounced } = parseVouchList(fs.readFileSync(VOUCHED_PATH, "utf8")); + if (vouched.has(author.toLowerCase())) return; const { data: access } = await github.rest.repos.getCollaboratorPermissionLevel({ ...context.repo, username: author, @@ -289,6 +298,21 @@ async function enforceOpenLimit({ github, context, core, dryRun }) { const reopened = context.payload.action === "reopened"; if (reopened && context.payload.sender.login.toLowerCase() !== author.toLowerCase()) return; + if (denounced.has(author.toLowerCase())) { + if (dryRun) { + core.notice(`Dry run. PR #${pr.number} would close: ${author} is denounced.`); + return; + } + await closeWithComment( + github, + context, + pr.number, + `Closing: ${author} is banned from contributing here, so this PR will not be reviewed. See [bans](${guideLink(context, "bans")}).`, + ); + core.info(`PR #${pr.number}: closed, ${author} is denounced.`); + return; + } + const q = `repo:${context.repo.owner}/${context.repo.repo} is:pr is:open author:${author} sort:created-asc`; const { search } = await read(github, AUTHOR_OPEN_PRS_QUERY, { q }); // A new PR counts only the older ones, so two opened together cannot close each other. diff --git a/.github/workflows/agent-context-budget.yml b/.github/workflows/agent-context-budget.yml index 3a9a148194ad2..5935fb4653c32 100644 --- a/.github/workflows/agent-context-budget.yml +++ b/.github/workflows/agent-context-budget.yml @@ -10,10 +10,7 @@ on: permissions: contents: read -concurrency: - group: agent-context-budget-${{ github.event.pull_request.number }} - cancel-in-progress: true - +# No concurrency group: label events arrive in bursts, and a cancelled run reports as a failed check. jobs: budget: name: Agent context budget diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79713fe162597..6bfb648fe368c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -591,7 +591,7 @@ jobs: name: Mobile Native Static Analysis needs: changes if: ${{ !cancelled() && needs.changes.outputs.mobile_native != 'false' }} - runs-on: blacksmith-6vcpu-macos-26 + runs-on: blacksmith-6vcpu-macos-27 timeout-minutes: 10 steps: - name: Checkout diff --git a/.github/workflows/desktop-macos-preview-publish.yml b/.github/workflows/desktop-macos-preview-publish.yml index a9615439929fe..88f96079b8979 100644 --- a/.github/workflows/desktop-macos-preview-publish.yml +++ b/.github/workflows/desktop-macos-preview-publish.yml @@ -316,6 +316,7 @@ jobs: rust_target: aarch64-apple-darwin resource_key: darwin-arm64 cli_archive: false + untrusted_bundle: true # Release assets download without a GitHub account, unlike workflow # artifacts. All preview DMGs live on one rolling prerelease tagged diff --git a/.github/workflows/issue-vouch.yml b/.github/workflows/issue-vouch.yml new file mode 100644 index 0000000000000..d65882593c649 --- /dev/null +++ b/.github/workflows/issue-vouch.yml @@ -0,0 +1,110 @@ +name: Issue Vouch + +on: + issues: + types: [opened, reopened] + push: + branches: + - main + paths: + - .github/VOUCHED.td + - .github/workflows/issue-vouch.yml + workflow_dispatch: + +permissions: + contents: read + issues: write + +concurrency: + group: issue-vouch-${{ github.event.issue.number || 'all' }} + cancel-in-progress: true + +jobs: + label: + name: Label issues by author trust + runs-on: ubuntu-24.04 + steps: + - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + with: + script: | + const { owner, repo } = context.repo; + // Only listed authors get a label; open issues number in the thousands, + // so unvouched is the default and stays unlabelled. + const labelByStatus = { vouched: "vouch:trusted", denounced: "vouch:denounced" }; + const managedLabelNames = new Set(["vouch:trusted", "vouch:unvouched", "vouch:denounced"]); + + const { data: file } = await github.rest.repos.getContent({ + owner, + repo, + path: ".github/VOUCHED.td", + ref: context.payload.repository.default_branch, + }); + const statusByUser = new Map(); + const listedBothWays = new Set(); + for (const raw of Buffer.from(file.content, "base64").toString("utf8").split("\n")) { + const match = /^(-?)github:(\S+)/.exec(raw.trim()); + if (!match) continue; + const login = match[2].toLowerCase(); + const status = match[1] ? "denounced" : "vouched"; + if (statusByUser.has(login) && statusByUser.get(login) !== status) listedBothWays.add(login); + statusByUser.set(login, status); + } + + const issues = + context.eventName === "issues" + ? [context.payload.issue] + : ( + await github.paginate(github.rest.issues.listForRepo, { + owner, + repo, + state: "open", + per_page: 100, + }) + ).filter((issue) => !issue.pull_request); + + for (const issue of issues) { + const status = statusByUser.get(issue.user.login.toLowerCase()); + const next = labelByStatus[status]; + const current = issue.labels.map((label) => label.name).filter((name) => managedLabelNames.has(name)); + + for (const name of current) { + if (name === next) continue; + try { + await github.rest.issues.removeLabel({ owner, repo, issue_number: issue.number, name }); + } catch (error) { + if (error.status !== 404) throw error; + } + } + if (next && !current.includes(next)) { + await github.rest.issues.addLabels({ owner, repo, issue_number: issue.number, labels: [next] }); + core.info(`Issue #${issue.number} (${issue.user.login}): ${status} -> ${next}`); + } + + // Only a newly opened issue is closed. One filed before its author was denounced + // may already have a fix on the way, so the sweep leaves those to maintainers. + // Someone else reopening it is a maintainer keeping it open. + const reopenedByOther = + context.payload.action === "reopened" && + context.payload.sender.login.toLowerCase() !== issue.user.login.toLowerCase(); + // The PR rules refuse a list that names a login both ways, so nothing closes for one here. + const trusted = + issue.user.type === "Bot" || + ["OWNER", "MEMBER", "COLLABORATOR"].includes(issue.author_association) || + listedBothWays.has(issue.user.login.toLowerCase()); + if (status === "denounced" && context.eventName === "issues" && !reopenedByOther && !trusted) { + await github.rest.issues.createComment({ + owner, + repo, + issue_number: issue.number, + body: `Closing: ${issue.user.login} is banned from contributing here, so this issue will not be triaged. See [bans](https://github.com/${owner}/${repo}/blob/main/CONTRIBUTING.md#bans).`, + }); + await github.rest.issues.update({ + owner, + repo, + issue_number: issue.number, + state: "closed", + state_reason: "not_planned", + }); + core.info(`Issue #${issue.number}: closed, ${issue.user.login} is denounced.`); + } + } diff --git a/.github/workflows/mobile-dev-client.yml b/.github/workflows/mobile-dev-client.yml index e204b16f01626..b3e11c1ed6b6b 100644 --- a/.github/workflows/mobile-dev-client.yml +++ b/.github/workflows/mobile-dev-client.yml @@ -219,7 +219,7 @@ jobs: ios: name: iOS simulator if: github.repository == 'pingdotgg/t3code' && (github.event_name != 'workflow_dispatch' || inputs.platform == 'ios') - runs-on: blacksmith-12vcpu-macos-26 + runs-on: blacksmith-12vcpu-macos-27 timeout-minutes: 60 concurrency: group: mobile-dev-client-ios-${{ github.event_name == 'pull_request' && github.event.pull_request.number || inputs.fingerprint || github.sha }} diff --git a/.github/workflows/mobile-eas-production.yml b/.github/workflows/mobile-eas-production.yml index 39f9d01f61348..cdabb1e4b7ca3 100644 --- a/.github/workflows/mobile-eas-production.yml +++ b/.github/workflows/mobile-eas-production.yml @@ -680,8 +680,8 @@ jobs: echo ":rocket: android: submitted and recorded \`$BUILD_TAG\`" >> "$GITHUB_STEP_SUMMARY" # Off unless the MOBILE_IOS_LOCAL_BUILDS repository variable is `true`; the - # production job only sets ios_build then. Store binaries need the final - # Xcode 27.1; MOBILE_IOS_ALLOW_XCODE_RC=true accepts the release candidate. + # production job only sets ios_build then. Xcode 27.1 matches the EAS build + # image in eas.json. # No sticky disk: it mounts ext4 and only works on Linux runners. ios: name: iOS store build @@ -701,7 +701,6 @@ jobs: EXPO_TOKEN: ${{ secrets.EXPO_PRODUCTION_TOKEN }} PROFILE: ${{ needs.production.outputs.ios_profile }} BUILD_TAG: mobile-ios-build/${{ needs.production.outputs.ios_profile }}/${{ needs.production.outputs.ios_version }}/${{ needs.production.outputs.ios_fingerprint }} - ALLOW_XCODE_RC: ${{ vars.MOBILE_IOS_ALLOW_XCODE_RC == 'true' }} steps: - name: Checkout uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 @@ -739,12 +738,9 @@ jobs: if: steps.record.outputs.built != 'true' run: | xcode=/Applications/Xcode_27.1.app - if [ ! -d "$xcode" ] && [ "$ALLOW_XCODE_RC" = "true" ]; then - xcode=/Applications/Xcode_27.1_Release_Candidate.app - fi if [ ! -d "$xcode" ]; then ls -d /Applications/Xcode*.app - echo "::error::Xcode 27.1 (final) is not on this runner. Set the MOBILE_IOS_ALLOW_XCODE_RC repository variable to true to build store binaries with the release candidate." + echo "::error::Xcode 27.1 is not on this runner." exit 1 fi sudo xcode-select -s "$xcode/Contents/Developer" diff --git a/.github/workflows/mobile-showcase-screenshots.yml b/.github/workflows/mobile-showcase-screenshots.yml index 3210ae37d7671..402ff86445378 100644 --- a/.github/workflows/mobile-showcase-screenshots.yml +++ b/.github/workflows/mobile-showcase-screenshots.yml @@ -45,7 +45,7 @@ jobs: ios: name: iPhone, iPhone Duo, and iPad 13 if: inputs.platform == 'all' || inputs.platform == 'ios' - runs-on: blacksmith-12vcpu-macos-26 + runs-on: blacksmith-12vcpu-macos-27 # Capturing every palette multiplies the device matrix by six, and only the # one native build is shared between them. timeout-minutes: ${{ inputs.theme == 'all' && 300 || 60 }} diff --git a/.github/workflows/pr-vouch.yml b/.github/workflows/pr-vouch.yml index 39d368f9a20d6..eb53c07894025 100644 --- a/.github/workflows/pr-vouch.yml +++ b/.github/workflows/pr-vouch.yml @@ -101,17 +101,17 @@ jobs: { name: "vouch:trusted", color: "1f883d", - description: "PR author is trusted by repo permissions or the VOUCHED list.", + description: "Author is trusted by repo permissions or the VOUCHED list.", }, { name: "vouch:unvouched", color: "fbca04", - description: "PR author is not yet trusted in the VOUCHED list.", + description: "Author is not yet trusted in the VOUCHED list.", }, { name: "vouch:denounced", color: "d1242f", - description: "PR author is explicitly blocked by the VOUCHED list.", + description: "Author is explicitly blocked by the VOUCHED list.", }, ]; diff --git a/.github/workflows/release-desktop.yml b/.github/workflows/release-desktop.yml index 48f410f0c920a..4252cb46e7a05 100644 --- a/.github/workflows/release-desktop.yml +++ b/.github/workflows/release-desktop.yml @@ -70,6 +70,12 @@ on: required: false default: false type: boolean + # Whether the JS bundle came from a pull request. This job holds the + # signing secrets, so packaging then skips every step that runs the bundle. + untrusted_bundle: + required: false + default: false + type: boolean # A runner of the target's own platform and architecture, for a `runner` # that cannot execute the target. The build job skips everything that # runs the packaged binaries, and a second job runs it there. @@ -134,6 +140,14 @@ jobs: T3CODE_CLERK_CLI_OAUTH_CLIENT_ID: ${{ inputs.clerk_cli_oauth_client_id }} T3CODE_RELAY_URL: ${{ inputs.relay_url }} steps: + # The launch check and the CLI archive smoke test both run the bundle. + - name: Reject running an untrusted bundle + if: inputs.untrusted_bundle && (inputs.launch_check || inputs.cli_archive) + shell: bash + run: | + echo "::error::untrusted_bundle cannot be combined with launch_check or cli_archive, which run the bundle." + exit 1 + # This repository is public, so Git needs no credentials. checkout's # credential cleanup runs submodule foreach even with submodules disabled, # which fails on the orphaned gitlinks in our vendored .repos tree. @@ -573,6 +587,7 @@ jobs: pnpm_config_prefer_offline: "true" T3CODE_DESKTOP_REUSE_RESOURCE_MONITOR: true T3CODE_DESKTOP_REUSE_LINUX_CAPTURE_HELPERS: ${{ steps.capture_helper_cache.outputs.cache-hit == 'true' }} + T3CODE_DESKTOP_UNTRUSTED_BUNDLE: ${{ inputs.untrusted_bundle }} CSC_LINK: ${{ secrets.CSC_LINK }} CSC_KEY_PASSWORD: ${{ secrets.CSC_KEY_PASSWORD }} APPLE_API_KEY: ${{ secrets.APPLE_API_KEY }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 993968245c988..c1bd673dab60f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -903,34 +903,12 @@ jobs: fi # electron-updater reads one manifest per platform and channel and picks - # the file entry whose name carries the running arch, so the per-arch - # manifests the build jobs wrote are merged back into that one file. - - name: Merge macOS updater manifests + # the file entry whose name carries the running arch. Require every + # platform's two build jobs to have produced matching metadata before + # merging it, so a successful release can never strand one architecture. + - name: Merge desktop updater manifests if: needs.preflight.outputs.release_channel != 'preview' - run: | - shopt -s nullglob - for x64_manifest in release-assets/*-mac-x64.yml; do - arm64_manifest="${x64_manifest%-x64.yml}.yml" - if [[ -f "$arm64_manifest" ]]; then - node scripts/merge-update-manifests.ts --platform mac "$arm64_manifest" "$x64_manifest" - rm -f "$x64_manifest" - fi - done - - - name: Merge Windows updater manifests - if: needs.preflight.outputs.release_channel != 'preview' - run: | - shopt -s nullglob - for x64_manifest in release-assets/*-win-x64.yml; do - arm64_manifest="${x64_manifest%-x64.yml}-arm64.yml" - merged_manifest="${x64_manifest%-win-x64.yml}.yml" - if [[ -f "$arm64_manifest" ]]; then - node scripts/merge-update-manifests.ts --platform win "$x64_manifest" "$arm64_manifest" "$merged_manifest" - rm -f "$x64_manifest" "$arm64_manifest" - else - mv "$x64_manifest" "$merged_manifest" - fi - done + run: node scripts/merge-release-update-manifests.ts release-assets # Updater manifests and blockmaps are what electron-updater consumes. # They are only listed for channels an updater is meant to follow. diff --git a/.gitignore b/.gitignore index 85b7c5b4bf02f..1637f475fbbd0 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,5 @@ node_modules/ # pnpm content-addressable store; never belongs in the repo. .pnpm-store/ + +/packages/ssh/.ssh-launcher-test-*/ diff --git a/AGENTS.md b/AGENTS.md index 9073e293deba2..e5941a175264c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,6 @@ # T3 Code -T3 Code is a minimal GUI for coding agents. A Node WebSocket server wraps provider CLIs and agents (Codex, Claude Code, Cursor, Grok, OpenCode, Antigravity, Kiro) and serves web, desktop, and mobile clients. +T3 Code is a minimal GUI for coding agents. A Node WebSocket server wraps provider CLIs and agents (Codex, Claude Code, Cursor, Grok, OpenCode, Antigravity, Kiro, Pi, Muse Code, fx) and serves web, desktop, and mobile clients. You can think of T3 Code as an open source "bring-your-own-subscription" alternative to apps like Claude Desktop, Codex App, Cursor Glass and Conductor. @@ -68,7 +68,7 @@ The most common defect in this repo is a change that works on the path you teste - **Entry points.** A behavior reachable from the chat view is usually also reachable from Settings, the command palette, and a keybinding. Fixing one is not fixing the feature. - **Clients.** Web, desktop (wraps web, adds Electron shell/IPC), and mobile (React Native, separate navigation). Shared logic lives in `packages/client-runtime` -- **Providers.** Codex, Claude, Cursor, Grok, OpenCode, Antigravity, and Kiro each have an adapter. Provider-shaped features need a decision per adapter, even if the decision is "not supported here". +- **Providers.** Codex, Claude, Cursor, Grok, OpenCode, Antigravity, Kiro, Pi, Muse Code, fx, and ACP Registry each have an adapter. Provider-shaped features need a decision per adapter, even if the decision is "not supported here". - **Agents.** A capability a user can trigger is usually one an agent should reach through MCP tools, and scheduled tasks run the same paths. That only works when it is a service method, not handler code. - **Contracts.** Anything crossing the wire is typed in `packages/contracts`. Change the schema and the server, web, mobile, and desktop all follow. - **Reverse states.** If you added a way in, add the way out and the way to see it. Snooze needs unsnooze. Close needs reopen. A one-way door is a bug. @@ -83,7 +83,8 @@ The most common defect in this repo is a change that works on the path you teste - Ports derive from the worktree path and are stable across restarts, but read the real ones from the `[dev-runner]` line since occupied ports shift. - Sharing over the tailnet is three steps: run `vp run dev --share` in the background, wait for the `pairingUrl:` line in its output, then give that full URL to an unpaired browser. Do not wire up `tailscale serve` by hand, open the URL yourself, or consume the user's pairing link. A browser with the reusable dev cookie can use the bare origin. If a normal one-time token was consumed, mint a fresh one with `node apps/server/src/bin.ts pair`. It carries standard scopes, while the startup URL carries admin scopes needed for Connections settings. - To reuse web dev auth across worktrees, configure one fixed `T3CODE_DEV_AUTH_TOKEN` in the main checkout's gitignored `.env`. The `t3.json` setup links that file into worktrees. Never commit or publish the token or a startup URL. See [Reusable dev credential](docs/operations/development.md#reusable-dev-credential). -- Stop what you started, by the PID you tracked. See rule 1. +- Stop what you started, by the PID you tracked. See rule 1. `vp` does not pass SIGTERM on, so signalling the `vp run dev` PID alone leaves Vite and the server running. Stop the background task with your harness's own stop, which ends the whole tree. +- Run dev servers, Metro, and builds from your own shell so they end with your session. Do not detach them with `systemd-run`, `nohup`, `setsid`, or `tmux` unless the user asks for one that outlives the session. Detached servers ran for weeks and filled RAM and swap. The dev runner stops itself above `--memory-limit-gb` (half of RAM, at most 16 GiB) on Linux. - Need a second checkout? `git worktree add` it next to this worktree, not in `/tmp`. Remove it when done. ## Test data @@ -149,7 +150,7 @@ For mobile verification, a missing or outdated native client is an install step, ## Pull requests - Never make a PR unless the developer explicitly asks you to do so. -- If `origin` is git.cafe, follow [GitCafe mirror](docs/operations/development.md#gitcafe-mirror-optional): push to `origin`, open and track PRs with `cafe` and their git.cafe URLs, and keep `gh` for merging. +- `origin` is not always GitHub. Check `git remote get-url origin` before your first forge command and use that forge's CLI and skill. See [Forge mirror](docs/operations/development.md#forge-mirror-optional). - Conventional commit titles, plain language: `fix(web): new threads no longer spike CPU`. - Body: the problem in a sentence or two, then how you fixed it. End with the model and harness that did the work. - UI changes need before/after images. Motion or timing needs a short video. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 63e71dc1e250c..0c886bd493389 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -115,6 +115,9 @@ the open PR limit. - If you make our lives harder as reviewers and maintainers, you will be banned. Examples are repeated pings, a fight over a closure, and a second account to get around a limit. +A banned account is denounced in [.github/VOUCHED.td](.github/VOUCHED.td). Any issue or PR it opens +is closed when it is opened. + ## When your PR is closed diff --git a/apps/desktop/scripts/ensure-electron-runtime.mjs b/apps/desktop/scripts/ensure-electron-runtime.mjs index 933a763dfa354..741a2d9ee85d8 100644 --- a/apps/desktop/scripts/ensure-electron-runtime.mjs +++ b/apps/desktop/scripts/ensure-electron-runtime.mjs @@ -209,6 +209,10 @@ export function ensureElectronRuntime() { return electronPath; } +export function setup() { + ensureElectronRuntime(); +} + // `file://${argv[1]}` never matches on Windows (drive letters need `file:///C:/`). if (process.argv[1] && NodeURL.pathToFileURL(process.argv[1]).href === import.meta.url) { const electronPath = ensureElectronRuntime(); diff --git a/apps/desktop/src/app/DesktopApp.test.ts b/apps/desktop/src/app/DesktopApp.test.ts index 10e0d1cd16c9b..aca3b4fc7fe66 100644 --- a/apps/desktop/src/app/DesktopApp.test.ts +++ b/apps/desktop/src/app/DesktopApp.test.ts @@ -86,6 +86,7 @@ it.effect.each([false, true])( const historyStarted = yield* Deferred.make(); const releaseHistory = yield* Deferred.make(); const quitting = yield* Ref.make(false); + const tokenRequested = yield* Deferred.make(); const events: string[] = []; const record = (event: string) => Effect.sync(() => { @@ -144,7 +145,7 @@ it.effect.each([false, true])( configure: Effect.void, } as PreviewPasskeys.PreviewPasskeys["Service"]), Effect.provideService(DesktopApplicationMenu.DesktopApplicationMenu, { - configure: Effect.void, + configure: record("menu"), }), Effect.provideService(DesktopUpdates.DesktopUpdates, { configure: Effect.void, @@ -222,7 +223,12 @@ it.effect.each([false, true])( Layer.mock(DesktopClientSettings.DesktopClientSettings, {}), Layer.mock(DesktopConnectionCatalogStore.DesktopConnectionCatalogStore, {}), Layer.mock(DesktopIpc.DesktopIpc, {}), - Layer.mock(DesktopLocalEnvironmentAuth.DesktopLocalEnvironmentAuth, {}), + Layer.mock(DesktopLocalEnvironmentAuth.DesktopLocalEnvironmentAuth, { + getBearerToken: record("token").pipe( + Effect.andThen(Deferred.succeed(tokenRequested, undefined)), + Effect.as("token"), + ), + }), Layer.mock(DesktopManagedPolicy.DesktopManagedPolicy, {}), Layer.succeed(DesktopPrimaryPaste.DesktopPrimaryPaste, { enabled: true }), Layer.mock(DesktopSshEnvironment.DesktopSshEnvironment, {}), @@ -253,7 +259,13 @@ it.effect.each([false, true])( ); yield* Deferred.await(registrationStarted); yield* Deferred.await(windowStarted); - assert.deepEqual(events, ["identity", "ready", "identity", "backend", "window"]); + yield* Deferred.await(tokenRequested); + // The backend spawns before menus, and its token is minted without waiting for the renderer. + assert.deepEqual( + events.filter((event) => event !== "token"), + ["identity", "ready", "identity", "backend", "menu", "window"], + ); + assert.isAbove(events.indexOf("token"), events.indexOf("backend")); assert.isFalse(yield* Deferred.isDone(registrationStopped)); const shutdown = yield* DesktopShutdown.DesktopShutdown; yield* shutdown.request; diff --git a/apps/desktop/src/app/DesktopApp.ts b/apps/desktop/src/app/DesktopApp.ts index 7ebeaa74a021d..0e7f949bc042c 100644 --- a/apps/desktop/src/app/DesktopApp.ts +++ b/apps/desktop/src/app/DesktopApp.ts @@ -22,6 +22,7 @@ import * as PreviewPasskeys from "../preview/Passkeys.ts"; import * as DesktopBackendManager from "../backend/DesktopBackendManager.ts"; import * as DesktopBackendPool from "../backend/DesktopBackendPool.ts"; import * as DesktopHomeOwner from "../backend/DesktopHomeOwner.ts"; +import * as DesktopLocalEnvironmentAuth from "../backend/DesktopLocalEnvironmentAuth.ts"; import * as DesktopEnvironment from "./DesktopEnvironment.ts"; import * as DesktopLegacyLocalStorage from "./DesktopLegacyLocalStorage.ts"; import * as DesktopLifecycle from "./DesktopLifecycle.ts"; @@ -168,13 +169,12 @@ const stopAllPoolInstances = Effect.fn("desktop.app.stopAllPoolInstances")( }, ); -const bootstrap = Effect.gen(function* () { +// Spawns the local backend as soon as the protocol, IPC, and legacy storage that every window needs exist. +const startLocalBackend = Effect.gen(function* () { const state = yield* DesktopState.DesktopState; const environment = yield* DesktopEnvironment.DesktopEnvironment; const desktopSettings = yield* DesktopAppSettings.DesktopAppSettings; const desktopWindow = yield* DesktopWindow.DesktopWindow; - const snapShot = yield* DesktopSnapShot.DesktopSnapShot; - const appActivation = yield* DesktopAppActivation.DesktopAppActivation; yield* logBootstrapInfo("bootstrap start"); const settings = yield* desktopSettings.get; @@ -193,32 +193,13 @@ const bootstrap = Effect.gen(function* () { yield* (yield* DesktopAppIdentity.DesktopAppIdentity).resolveUserDataPath, ); - yield* snapShot.initialize; - // Only the user's own login shell PATH shows every t3 a new terminal runs; a fresh install has - // no cached one. Without it, the command is left to Settings. - const installCli = (yield* DesktopCliCommand.DesktopCliCommand).installOnFirstLaunch; - yield* Effect.forkScoped( - (yield* DesktopShellEnvironment.DesktopShellEnvironment).awaitUserShellDiscovery.pipe( - Effect.flatMap((found) => - found || environment.platform === "win32" - ? installCli - : logBootstrapInfo("skipped installing the t3 command without the shell PATH"), - ), - ), - ); - if (!settings.localEnvironmentEnabled) { yield* logBootstrapInfo("bootstrap skipping local environment (disabled in settings)"); - if (!(yield* Ref.get(state.quitting))) { - yield* desktopWindow.createMainIfBackendReady; - } - return; + return false; } - const pool = yield* DesktopBackendPool.DesktopBackendPool; - const primaryBackend = yield* pool.primary; + const primaryBackend = yield* (yield* DesktopBackendPool.DesktopBackendPool).primary; const serverExposure = yield* DesktopServerExposure.DesktopServerExposure; - const wslBackend = yield* DesktopWslBackend.DesktopWslBackend; if (environment.isDevelopment && Option.isNone(environment.configuredBackendPort)) { return yield* new DesktopDevelopmentBackendPortRequiredError(); @@ -237,7 +218,8 @@ const bootstrap = Effect.gen(function* () { "This T3 home is unavailable", DesktopHomeOwner.homeOwnedMessage(owner), ); - return yield* (yield* ElectronApp.ElectronApp).quit; + yield* (yield* ElectronApp.ElectronApp).quit; + return false; } const backendPortSelection = yield* resolveDesktopBackendPort(environment.configuredBackendPort); @@ -275,29 +257,73 @@ const bootstrap = Effect.gen(function* () { ); } - if (!(yield* Ref.get(state.quitting))) { - if (settings.wslOnly === true && settings.wslBackendEnabled === true) { - yield* desktopWindow.showConnectingSplash; - } - yield* primaryBackend.start; - yield* logBootstrapInfo("bootstrap backend start requested"); - // WSL-only fallback can change the primary endpoint before readiness. - const config = yield* primaryBackend.currentConfig; - if ( - !environment.isDevelopment && - !settings.wslOnly && - Option.isSome(config) && - Option.isNone(config.value.preflightFailure) - ) { - yield* desktopWindow.ensureMain; + if (yield* Ref.get(state.quitting)) { + return false; + } + if (settings.wslOnly === true && settings.wslBackendEnabled === true) { + yield* desktopWindow.showConnectingSplash; + } + yield* primaryBackend.start; + yield* logBootstrapInfo("bootstrap backend start requested"); + // The renderer asks for this token first; minting it at readiness saves a round trip. + yield* Effect.forkScoped( + (yield* DesktopLocalEnvironmentAuth.DesktopLocalEnvironmentAuth).getBearerToken.pipe( + Effect.ignore, + ), + ); + return true; +}).pipe(Effect.withSpan("desktop.startLocalBackend")); + +const bootstrap = Effect.fn("desktop.bootstrap")(function* (backendStarted: boolean) { + const state = yield* DesktopState.DesktopState; + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const desktopSettings = yield* DesktopAppSettings.DesktopAppSettings; + const desktopWindow = yield* DesktopWindow.DesktopWindow; + const snapShot = yield* DesktopSnapShot.DesktopSnapShot; + const appActivation = yield* DesktopAppActivation.DesktopAppActivation; + const settings = yield* desktopSettings.get; + if (settings.localEnvironmentEnabled && (!backendStarted || (yield* Ref.get(state.quitting)))) { + return; + } + + yield* snapShot.initialize; + // Only the user's own login shell PATH shows every t3 a new terminal runs; a fresh install has + // no cached one. Without it, the command is left to Settings. + const installCli = (yield* DesktopCliCommand.DesktopCliCommand).installOnFirstLaunch; + yield* Effect.forkScoped( + (yield* DesktopShellEnvironment.DesktopShellEnvironment).awaitUserShellDiscovery.pipe( + Effect.flatMap((found) => + found || environment.platform === "win32" + ? installCli + : logBootstrapInfo("skipped installing the t3 command without the shell PATH"), + ), + ), + ); + + if (!settings.localEnvironmentEnabled) { + if (!(yield* Ref.get(state.quitting))) { + yield* desktopWindow.createMainIfBackendReady; } - yield* appActivation.start.pipe( - Effect.tap(() => logBootstrapInfo("desktop app control socket ready")), - Effect.catch((error) => logStartupError("desktop app control socket unavailable", { error })), - ); - yield* Effect.forkScoped(wslBackend.reconcile); + return; } -}).pipe(Effect.withSpan("desktop.bootstrap")); + + const primaryBackend = yield* (yield* DesktopBackendPool.DesktopBackendPool).primary; + // WSL-only fallback can change the primary endpoint before readiness. + const config = yield* primaryBackend.currentConfig; + if ( + !environment.isDevelopment && + !settings.wslOnly && + Option.isSome(config) && + Option.isNone(config.value.preflightFailure) + ) { + yield* desktopWindow.ensureMain; + } + yield* appActivation.start.pipe( + Effect.tap(() => logBootstrapInfo("desktop app control socket ready")), + Effect.catch((error) => logStartupError("desktop app control socket unavailable", { error })), + ); + yield* Effect.forkScoped((yield* DesktopWslBackend.DesktopWslBackend).reconcile); +}); const startup = Effect.gen(function* () { const appIdentity = yield* DesktopAppIdentity.DesktopAppIdentity; @@ -369,14 +395,20 @@ const startup = Effect.gen(function* () { }); } yield* appIdentity.configure; - yield* previewPasskeys.configure; - yield* applicationMenu.configure; + // Sets canInstallUpdates, which the backend bootstrap payload reads. yield* updates.configure; yield* DesktopRemoteUpdates.listen; yield* DesktopStartupRecovery.listen; + const backendStarted = yield* startLocalBackend.pipe( + Effect.catchCause((cause) => fatalStartupCause("bootstrap", cause)), + ); + yield* previewPasskeys.configure; + yield* applicationMenu.configure; // Portal identity is prepared pre-ready; MIME helpers must not gate the backend or window. yield* linuxUrlHandler.register.pipe(Effect.forkScoped); - yield* bootstrap.pipe(Effect.catchCause((cause) => fatalStartupCause("bootstrap", cause))); + yield* bootstrap(backendStarted).pipe( + Effect.catchCause((cause) => fatalStartupCause("bootstrap", cause)), + ); }).pipe(Effect.withSpan("desktop.startup")); const scopedProgram = Effect.scoped( diff --git a/apps/desktop/src/app/DesktopClerk.test.ts b/apps/desktop/src/app/DesktopClerk.test.ts index 8a823cde8db79..de687d41a4e68 100644 --- a/apps/desktop/src/app/DesktopClerk.test.ts +++ b/apps/desktop/src/app/DesktopClerk.test.ts @@ -424,7 +424,7 @@ it.effect.each(["startup", "open-url", "early-second-instance"] as const)( ), Effect.provideService(DesktopWebLinks.DesktopWebLinks, ignoreWebLinks), ); - }).pipe(Effect.scoped), + }), ); it.effect("hands web and app links to the renderer and leaves other links alone", () => @@ -481,5 +481,5 @@ it.effect("hands web and app links to the renderer and leaves other links alone" }), ), ); - }).pipe(Effect.scoped), + }), ); diff --git a/apps/desktop/src/app/DesktopCliCommand.test.ts b/apps/desktop/src/app/DesktopCliCommand.test.ts index 2dc59c58899ee..dd61b7984ec1f 100644 --- a/apps/desktop/src/app/DesktopCliCommand.test.ts +++ b/apps/desktop/src/app/DesktopCliCommand.test.ts @@ -160,7 +160,7 @@ it.layer(NodeServices.layer)("DesktopCliCommand", (it) => { expect(yield* fs.exists(upgradedLink)).toBe(false); expect((yield* upgraded.install).installedPath).toBe(upgradedLink); } - }).pipe(Effect.scoped), + }), ); it.effect("never replaces or removes a t3 it did not create", () => @@ -202,7 +202,7 @@ it.layer(NodeServices.layer)("DesktopCliCommand", (it) => { expect((yield* Effect.flip(command.install)).message).toContain("Could not set up"); yield* command.uninstall; expect(yield* fs.readFileString(launcher)).toBe("npm's t3\n"); - }).pipe(Effect.scoped), + }), ); it.effect("finds and removes a link left by a previous T3 home", () => @@ -259,7 +259,7 @@ it.layer(NodeServices.layer)("DesktopCliCommand", (it) => { ); expect(yield* fs.exists(path.join(home, ".local", "bin", "t3"))).toBe(false); expect(yield* fs.readFileString(path.join(home, "bin", "t3.staging"))).toBe("mine"); - }).pipe(Effect.scoped), + }), ); it.effect("refuses to install behind another t3 that runs first", () => @@ -309,7 +309,7 @@ it.layer(NodeServices.layer)("DesktopCliCommand", (it) => { installedPath: path.join(home, ".local", "bin", "t3"), }); expect(installed.shadowedBy).toBeUndefined(); - }).pipe(Effect.scoped), + }), ); it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( @@ -343,7 +343,7 @@ it.layer(NodeServices.layer)("DesktopCliCommand", (it) => { onPath: true, installedPath: path.join(home, ".local", "bin", "t3"), }); - }).pipe(Effect.scoped), + }), ); it.effect("does not treat its own launcher as another t3 when its folder is first on PATH", () => @@ -372,7 +372,7 @@ it.layer(NodeServices.layer)("DesktopCliCommand", (it) => { expect(yield* command.state).toEqual(removed); expect(yield* fs.exists(installed.installedPath!)).toBe(false); expect(yield* fs.exists(launcher)).toBe(true); - }).pipe(Effect.scoped), + }), ); it.effect("leaves the Windows PATH alone when it cannot be read", () => @@ -411,7 +411,7 @@ it.layer(NodeServices.layer)("DesktopCliCommand", (it) => { const reader = yield* commandIn({ home, platform: "win32" }, inherited.spawner); yield* reader.install; expect(inherited.registry).toMatchObject({ path: `${userPath};${launcherDir}`, writes: 1 }); - }).pipe(Effect.scoped), + }), ); it.effect("does not claim or remove a Windows PATH entry the user added", () => @@ -441,6 +441,6 @@ it.layer(NodeServices.layer)("DesktopCliCommand", (it) => { yield* (yield* commandIn({ home: besideHome, platform: "win32" }, beside.spawner)) .installOnFirstLaunch; expect(beside.registry).toMatchObject({ path: "C:\\Tools", writes: 0 }); - }).pipe(Effect.scoped), + }), ); }); diff --git a/apps/desktop/src/app/DesktopCliShim.test.ts b/apps/desktop/src/app/DesktopCliShim.test.ts index e2f97c7d2ff71..3e8b4967dfde5 100644 --- a/apps/desktop/src/app/DesktopCliShim.test.ts +++ b/apps/desktop/src/app/DesktopCliShim.test.ts @@ -39,7 +39,7 @@ const run = (shim: string, args: ReadonlyArray, env: NodeJS.ProcessEnv = }); const REPORTER = - '#!/bin/sh\necho "node=$ELECTRON_RUN_AS_NODE cli=$T3CODE_CLI_PATH home=$T3CODE_HOME"\nprintf "%s\\n" "$@"\nexit 3\n'; + '#!/bin/sh\necho "node=$ELECTRON_RUN_AS_NODE cli=$T3CODE_CLI_PATH home=$T3CODE_HOME appimage=$APPIMAGE"\nprintf "%s\\n" "$@"\nexit 3\n'; const fakeAppImage = (canMount = true, app = REPORTER) => { const root = tempRoot(); @@ -56,7 +56,7 @@ const fakeAppImage = (canMount = true, app = REPORTER) => { `echo $$ >'${root}/mounter.pid'`, canMount ? `echo '${image}'` : 'echo "Cannot mount AppImage"; exit 1', // Like the FUSE runtime, any of these unmounts. fd 3 reports to the test when it is open. - `trap 'echo unmounted >&3 2>/dev/null; touch "${root}/unmounted"; exit 0' TERM HUP INT`, + "trap 'echo unmounted >&3 2>/dev/null; exit 0' TERM HUP INT", "trap 'echo mounted >&3 2>/dev/null' WINCH", "while :; do sleep 0.05; done", "", @@ -150,6 +150,14 @@ describe("renderCliShim", () => { ); }); + // The server skips its compile cache only under APPIMAGE; each mount path is new. + it("tells the CLI it runs from the mounted AppImage", () => { + const { appImage, shim } = fakeAppImage(); + const result = run(shim, ["--version"]); + expect(result.status).toBe(3); + expect(result.stdout).toContain(`appimage=${appImage}`); + }); + it("keeps the install's T3 home under sudo, unless one is set", () => { const root = tempRoot(); writeExecutable(NodePath.join(root, "app"), REPORTER); diff --git a/apps/desktop/src/app/DesktopCliShim.ts b/apps/desktop/src/app/DesktopCliShim.ts index eba61f561f526..217f3b327dc9e 100644 --- a/apps/desktop/src/app/DesktopCliShim.ts +++ b/apps/desktop/src/app/DesktopCliShim.ts @@ -109,6 +109,8 @@ export const renderCliShim = (input: { ' (cd "$work" && "$app" --appimage-extract >/dev/null) || { echo "Could not open $app." >&2; exit 1; }', ' mount="$work/squashfs-root"', "fi", + // The runtime sets this when it launches the app; the CLI skips its path-keyed compile cache on it. + 'export APPIMAGE="$app"', run("$mount"), "", ].join("\n"); diff --git a/apps/desktop/src/app/DesktopConnectionCatalogStore.test.ts b/apps/desktop/src/app/DesktopConnectionCatalogStore.test.ts index 98c75385afc51..b58b583fd6cd5 100644 --- a/apps/desktop/src/app/DesktopConnectionCatalogStore.test.ts +++ b/apps/desktop/src/app/DesktopConnectionCatalogStore.test.ts @@ -99,7 +99,7 @@ const withStore = ( prefix: "t3-desktop-connection-catalog-test-", }); return yield* effect.pipe(Effect.provide(layerFor(baseDir, encryptionAvailable))); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped); + }).pipe(Effect.provide(NodeServices.layer)); describe("DesktopConnectionCatalogStore", () => { it.effect("persists and reads an encrypted connection catalog", () => @@ -298,6 +298,68 @@ describe("DesktopConnectionCatalogStore", () => { assert.notEqual(error.message, decryptError.message); yield* Ref.set(failDecrypt, false); assert.deepStrictEqual(yield* store.get, Option.some('{"schemaVersion":1,"targets":[]}')); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + const emptyCatalog = + '{"schemaVersion":1,"targets":[],"profiles":[],"credentials":[],"remoteDpopTokens":[],"disabledEnvironmentIds":[]}'; + const savedFilesHolding = Effect.fn(function* (stateDir: string, contents: string) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const kept: string[] = []; + for (const name of yield* fileSystem.readDirectory(stateDir)) { + const file = path.join(stateDir, name); + if ((yield* fileSystem.stat(file)).type !== "File") continue; + if ((yield* fileSystem.readFileString(file)) === contents) kept.push(name); + } + return kept; + }); + + it.effect("keeps a catalog this build cannot decode when the next save replaces it", () => + withStore( + Effect.gen(function* () { + const path = yield* Path.Path; + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const fileSystem = yield* FileSystem.FileSystem; + const store = yield* DesktopConnectionCatalogStore.DesktopConnectionCatalogStore; + const catalogPath = path.join(environment.stateDir, "connection-catalog.json"); + assert.isTrue( + yield* store.set( + '{"schemaVersion":1,"targets":[{"_tag":"FutureConnectionTarget","environmentId":"remote"}],"profiles":[],"credentials":[],"remoteDpopTokens":[],"disabledEnvironmentIds":[]}', + ), + ); + const original = yield* fileSystem.readFileString(catalogPath); + + yield* store.get; + assert.isTrue(yield* store.set(emptyCatalog)); + + assert.isNotEmpty(yield* savedFilesHolding(environment.stateDir, original)); + }), + ), + ); + + it.effect("keeps a catalog that no longer decrypts when the next save replaces it", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const fileSystem = yield* FileSystem.FileSystem; + const baseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-desktop-connection-catalog-test-", + }); + const failDecrypt = yield* Ref.make(false); + const store = yield* DesktopConnectionCatalogStore.DesktopConnectionCatalogStore.pipe( + Effect.provide(layerFor(baseDir, true, failDecrypt)), + ); + const stateDir = path.join(baseDir, "userdata"); + assert.isTrue(yield* store.set('{"schemaVersion":1,"targets":[]}')); + const original = yield* fileSystem.readFileString( + path.join(stateDir, "connection-catalog.json"), + ); + + yield* Ref.set(failDecrypt, true); + yield* store.get.pipe(Effect.flip); + assert.isTrue(yield* store.set(emptyCatalog)); + + assert.isNotEmpty(yield* savedFilesHolding(stateDir, original)); }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), ); }); diff --git a/apps/desktop/src/app/DesktopConnectionCatalogStore.ts b/apps/desktop/src/app/DesktopConnectionCatalogStore.ts index 3240df3636e38..5db49e5f9504c 100644 --- a/apps/desktop/src/app/DesktopConnectionCatalogStore.ts +++ b/apps/desktop/src/app/DesktopConnectionCatalogStore.ts @@ -15,6 +15,7 @@ import { fromLenientJson } from "@t3tools/shared/schemaJson"; import * as Context from "effect/Context"; import * as SecureRandom from "effect/crypto/SecureRandom"; import * as Effect from "effect/Effect"; +import * as Clock from "effect/Clock"; import * as Base64 from "effect/encoding/Base64"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; @@ -45,6 +46,9 @@ const RuntimeConnectionCatalogDocumentJson = Schema.fromJsonString( const encodeRuntimeConnectionCatalogDocumentJson = Schema.encodeEffect( RuntimeConnectionCatalogDocumentJson, ); +const decodeRuntimeConnectionCatalogDocumentJson = Schema.decodeEffect( + RuntimeConnectionCatalogDocumentJson, +); const DesktopConnectionCatalogStoreWriteOperation = Schema.Literals([ "create-temporary-file-name", @@ -52,6 +56,7 @@ const DesktopConnectionCatalogStoreWriteOperation = Schema.Literals([ "create-directory", "write-temporary-file", "replace-catalog-file", + "keep-unreadable-catalog", ]); const DesktopConnectionCatalogStoreMigrationOperation = Schema.Literals([ @@ -395,6 +400,57 @@ export const make = Effect.gen(function* () { ), ); + const decryptDocument = (document: EncryptedConnectionCatalogDocument) => + decodeSecretBytes(catalogPath, document.encryptedCatalog).pipe( + Effect.flatMap((encryptedCatalog) => + safeStorage.decryptString(encryptedCatalog).pipe( + Effect.mapError( + (cause) => + new DesktopConnectionCatalogStoreProtectionError({ + operation: "decrypt-catalog", + catalogPath, + cause, + }), + ), + ), + ), + ); + + // A catalog this build cannot decrypt or decode may still hold another build's saved + // environments and credentials, so a save copies it aside instead of destroying it. + const keepUnreadableCatalog = Effect.gen(function* () { + const readable = yield* readDocument(fileSystem, catalogPath).pipe( + Effect.flatMap( + Option.match({ + onNone: () => Effect.succeed(true), + onSome: (document) => + decryptDocument(document).pipe( + Effect.flatMap(decodeRuntimeConnectionCatalogDocumentJson), + Effect.as(true), + ), + }), + ), + Effect.orElseSucceed(() => false), + ); + if (readable) { + return; + } + const keptPath = `${catalogPath}.unreadable-${yield* Clock.currentTimeMillis}`; + yield* fileSystem.copyFile(catalogPath, keptPath).pipe( + Effect.mapError( + (cause) => + new DesktopConnectionCatalogStoreWriteError({ + operation: "keep-unreadable-catalog", + path: keptPath, + cause, + }), + ), + ); + yield* Effect.logWarning("Kept an unreadable connection catalog before replacing it.", { + keptPath, + }); + }); + const writeCatalog = Effect.fn("desktop.connectionCatalogStore.writeCatalog")(function* ( catalog: string, ) { @@ -470,26 +526,13 @@ export const make = Effect.gen(function* () { if (!(yield* encryptionAvailable)) { return Option.none(); } - const decrypted = yield* decodeSecretBytes(catalogPath, document.value.encryptedCatalog).pipe( - Effect.flatMap((encryptedCatalog) => - safeStorage.decryptString(encryptedCatalog).pipe( - Effect.mapError( - (cause) => - new DesktopConnectionCatalogStoreProtectionError({ - operation: "decrypt-catalog", - catalogPath, - cause, - }), - ), - ), - ), - ); - return Option.some(decrypted); + return Option.some(yield* decryptDocument(document.value)); }).pipe(Effect.withSpan("desktop.connectionCatalogStore.get")), set: Effect.fn("desktop.connectionCatalogStore.set")(function* (catalog) { if (!(yield* encryptionAvailable)) { return false; } + yield* keepUnreadableCatalog; yield* writeCatalog(catalog); return true; }), diff --git a/apps/desktop/src/app/DesktopExitChildren.test.ts b/apps/desktop/src/app/DesktopExitChildren.test.ts index 490c6740f0e00..800a0f5bd4c0f 100644 --- a/apps/desktop/src/app/DesktopExitChildren.test.ts +++ b/apps/desktop/src/app/DesktopExitChildren.test.ts @@ -3,7 +3,7 @@ import * as NodeChildProcess from "node:child_process"; import * as NodeFS from "node:fs"; import { assert, describe, it } from "@effect/vitest"; -import { stopChildProcessesBeforeExit } from "./DesktopExitChildren.ts"; +import { stopChildProcessesBeforeExit, stopZygotesBeforeExit } from "./DesktopExitChildren.ts"; const isRunning = (pid: number) => { try { @@ -39,4 +39,24 @@ describe.runIf(NodeFS.existsSync("/proc/self/task"))("stopChildProcessesBeforeEx parent.kill("SIGKILL"); } }); + + it("kills zygote children before returning and leaves other helpers running", async () => { + const spawnTitled = async (title: string) => { + const child = NodeChildProcess.spawn("sleep", ["60"], { argv0: title, stdio: "ignore" }); + await new Promise((resolve) => child.once("spawn", resolve)); + return child; + }; + // Chromium rewrites a zygote's argv into one space-separated title. + const zygote = await spawnTitled("/opt/T3 Code/t3code --type=zygote --no-sandbox"); + const utility = await spawnTitled("/opt/T3 Code/t3code --type=utility"); + try { + stopZygotesBeforeExit(process.pid); + + assert.isFalse(isRunning(zygote.pid!)); + assert.isTrue(isRunning(utility.pid!)); + } finally { + zygote.kill("SIGKILL"); + utility.kill("SIGKILL"); + } + }); }); diff --git a/apps/desktop/src/app/DesktopExitChildren.ts b/apps/desktop/src/app/DesktopExitChildren.ts index 7cc64c43833e7..9426c1b4ff884 100644 --- a/apps/desktop/src/app/DesktopExitChildren.ts +++ b/apps/desktop/src/app/DesktopExitChildren.ts @@ -29,6 +29,13 @@ const childPids = (pid: number): ReadonlyArray => { ); }; +// A zygote rewrites its argv into one space-separated title. +const isZygote = (pid: number): boolean => + (readProc(`/proc/${pid}/cmdline`) ?? "") + .replaceAll("\0", " ") + .split(" ") + .includes("--type=zygote"); + // A zombie has already released its memory mappings. const isRunning = (pid: number): boolean => { const stat = readProc(`/proc/${pid}/stat`); @@ -68,3 +75,13 @@ export const stopChildProcessesBeforeExit = (pid: number): ReadonlyArray waitUntilStopped(survivors, KILL_WAIT_MS); return survivors; }; + +/** + * Chromium's zygotes exit only after main does, so on an AppImage they outlive the mount even on + * an orderly quit. Kills them while main still holds the image mounted. Linux only (/proc). + */ +export const stopZygotesBeforeExit = (pid: number): void => { + const zygotes = childPids(pid).filter(isZygote); + signal(zygotes, "SIGKILL"); + waitUntilStopped(zygotes, KILL_WAIT_MS); +}; diff --git a/apps/desktop/src/app/DesktopLifecycle.test.ts b/apps/desktop/src/app/DesktopLifecycle.test.ts index 30df96c58ad79..e11e946dd41ba 100644 --- a/apps/desktop/src/app/DesktopLifecycle.test.ts +++ b/apps/desktop/src/app/DesktopLifecycle.test.ts @@ -2,6 +2,7 @@ import { assert, describe, it } from "@effect/vitest"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Ref from "effect/Ref"; import * as TestClock from "effect/testing/TestClock"; @@ -117,6 +118,7 @@ describe("DesktopLifecycle", () => { const layerEnvironment = Layer.succeed(DesktopEnvironment.DesktopEnvironment, { platform, isDevelopment: false, + appImagePath: Option.none(), } as DesktopEnvironment.DesktopEnvironment["Service"]); const layer = DesktopLifecycle.layer.pipe( diff --git a/apps/desktop/src/app/DesktopLifecycle.ts b/apps/desktop/src/app/DesktopLifecycle.ts index 56fb25fcfa907..9568e72dde310 100644 --- a/apps/desktop/src/app/DesktopLifecycle.ts +++ b/apps/desktop/src/app/DesktopLifecycle.ts @@ -3,6 +3,7 @@ import * as Context from "effect/Context"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; @@ -12,7 +13,7 @@ import type * as Electron from "electron"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as DesktopEnvironment from "./DesktopEnvironment.ts"; -import { stopChildProcessesBeforeExit } from "./DesktopExitChildren.ts"; +import { stopChildProcessesBeforeExit, stopZygotesBeforeExit } from "./DesktopExitChildren.ts"; import { makeComponentLogger } from "./DesktopObservability.ts"; import * as DesktopShutdown from "./DesktopShutdown.ts"; import * as ElectronApp from "../electron/ElectronApp.ts"; @@ -340,6 +341,13 @@ export const make = DesktopLifecycle.of({ quitFromSignal("SIGTERM", runEffect, quit); }); } + + // Unscoped: the app scope closes before Electron exits, and every exit path must run this. + if (environment.platform === "linux" && Option.isSome(environment.appImagePath)) { + yield* Effect.sync(() => { + process.once("exit", () => stopZygotesBeforeExit(process.pid)); + }); + } }).pipe(Effect.withSpan("desktop.lifecycle.register")), }); diff --git a/apps/desktop/src/app/DesktopLinuxUrlHandler.test.ts b/apps/desktop/src/app/DesktopLinuxUrlHandler.test.ts index 7e685358865c7..8e8a2812ab245 100644 --- a/apps/desktop/src/app/DesktopLinuxUrlHandler.test.ts +++ b/apps/desktop/src/app/DesktopLinuxUrlHandler.test.ts @@ -204,6 +204,35 @@ describe("DesktopLinuxUrlHandler", () => { assert.include(entry, "Icon=/home/al ice/icons/T3\\\\x.png"); }); + it.each([ + { from: "T3-Code-0.1.8-x86_64.AppImage", to: "T3-Code-x86_64.AppImage", restored: true }, + { + from: "T3-Code-0.1.9-nightly.20261110.4449-arm64.AppImage", + to: "T3-Code-Nightly-arm64.AppImage", + restored: true, + }, + { from: "T3-Code-0.1.8-x86_64.AppImage", to: "T3-Code-0.1.9-x86_64.AppImage", restored: false }, + { from: "t3code.AppImage", to: "T3-Code-x86_64.AppImage", restored: false }, + { from: "T3-Code-x86_64.AppImage", to: "T3-Code-x86_64.AppImage", restored: false }, + { from: "../T3-Code-0.1.8-x86_64.AppImage", to: "T3-Code-x86_64.AppImage", restored: false }, + ])("finds the launch path an update from $from to $to removed", ({ from, to, restored }) => { + for (const dir of ["/home/alice/Applications", '/home/al ice/My "100%" $Apps\\x']) { + const previous = dir + "/" + from; + const previousEntry = DesktopLinuxUrlHandler.renderUrlHandlerDesktopEntry({ + displayName: "T3 Code (Beta)", + execTarget: previous, + scheme: "t3code", + }); + assert.equal( + DesktopLinuxUrlHandler.replacedVersionedAppImagePath({ + previousEntry, + appImagePath: dir + "/" + to, + }), + restored ? previous : null, + ); + } + }); + it.effect( "writes the handler entry, refreshes the MIME cache, and claims the scheme default", () => { @@ -602,7 +631,7 @@ it.layer(NodeServices.layer)("DesktopLinuxUrlHandler theme icons", (it) => { } assert.isFalse(yield* fileSystem.exists(fixture.cachePath)); assert.equal(iconCacheRuns(run), 0); - }).pipe(Effect.scoped), + }), ); it.effect("replaces same-length stale icons and refreshes an existing cache once", () => @@ -634,7 +663,7 @@ it.layer(NodeServices.layer)("DesktopLinuxUrlHandler theme icons", (it) => { run.commands.filter(({ command }) => command === "gtk-update-icon-cache"), [{ command: "gtk-update-icon-cache", args: ["-f", "-t", fixture.hicolorDir] }], ); - }).pipe(Effect.scoped), + }), ); it.effect("refreshes when this launch changed an icon even if the cache looks newer", () => @@ -655,7 +684,7 @@ it.layer(NodeServices.layer)("DesktopLinuxUrlHandler theme icons", (it) => { assert.isTrue(yield* fileSystem.exists(path.join(fixture.appsDir, "com.t3tools.T3Code.png"))); assert.equal(iconCacheRuns(run), 1); - }).pipe(Effect.scoped), + }), ); it.effect("retries a failed icon cache refresh on the next launch", () => @@ -678,7 +707,7 @@ it.layer(NodeServices.layer)("DesktopLinuxUrlHandler theme icons", (it) => { yield* registerWithRealFileSystem(recovering); yield* registerWithRealFileSystem(recovering); assert.equal(iconCacheRuns(recovering), 1); - }).pipe(Effect.scoped), + }), ); it.effect("still registers the handler when the theme icon install fails", () => @@ -702,6 +731,6 @@ it.layer(NodeServices.layer)("DesktopLinuxUrlHandler theme icons", (it) => { missingTool.commands.map(({ command }) => command), ["gtk-update-icon-cache", "update-desktop-database", "xdg-mime"], ); - }).pipe(Effect.scoped), + }), ); }); diff --git a/apps/desktop/src/app/DesktopLinuxUrlHandler.ts b/apps/desktop/src/app/DesktopLinuxUrlHandler.ts index 64535c5b3ac11..35fd46dd7ae50 100644 --- a/apps/desktop/src/app/DesktopLinuxUrlHandler.ts +++ b/apps/desktop/src/app/DesktopLinuxUrlHandler.ts @@ -96,6 +96,38 @@ export function renderUrlHandlerDesktopEntry(input: { ].join("\n"); } +function parseUrlHandlerExecTarget(entry: string): string | null { + const exec = /^Exec=(.*) %U$/mu.exec(entry)?.[1]; + if (exec === undefined) return null; + const unescaped = exec.replace(/\\([\\nrt])/gu, (_, char: string) => + char === "n" ? "\n" : char === "r" ? "\r" : char === "t" ? "\t" : char, + ); + const quoted = /^"(.*)"$/su.exec(unescaped)?.[1]; + return ( + quoted === undefined ? unescaped : quoted.replace(/\\([\\`$"])/gu, (_, char: string) => char) + ).replaceAll("%%", "%"); +} + +const APPIMAGE_NAME_VERSION = /\d+\.\d+\.\d+/u; + +// An updater from before version-free AppImage names deletes its own file and installs the update +// beside it under the new name. The previous launch's entry still names the path launchers use. +export function replacedVersionedAppImagePath(input: { + readonly previousEntry: string; + readonly appImagePath: string; +}): string | null { + const previous = parseUrlHandlerExecTarget(input.previousEntry); + if (previous === null) return null; + const previousName = previous.slice(previous.lastIndexOf("/") + 1); + const currentName = input.appImagePath.slice(input.appImagePath.lastIndexOf("/") + 1); + return previous.startsWith("/") && + previous.slice(0, -previousName.length) === input.appImagePath.slice(0, -currentName.length) && + APPIMAGE_NAME_VERSION.test(previousName) && + !APPIMAGE_NAME_VERSION.test(currentName) + ? previous + : null; +} + // Docks may look up the Wayland app id without a desktop entry; icon theme lookups are case-sensitive. const LINUX_THEME_ICON_SIZE = "256x256"; diff --git a/apps/desktop/src/app/DesktopPreReadyPlatform.test.ts b/apps/desktop/src/app/DesktopPreReadyPlatform.test.ts index 34607e08b844f..45887a928abb7 100644 --- a/apps/desktop/src/app/DesktopPreReadyPlatform.test.ts +++ b/apps/desktop/src/app/DesktopPreReadyPlatform.test.ts @@ -16,6 +16,7 @@ const { copyFileSyncMock, execFileSyncMock, readFileSyncMock, + symlinkSyncMock, } = vi.hoisted(() => ({ appendSwitchMock: vi.fn(), getSwitchValueMock: vi.fn(), @@ -27,6 +28,7 @@ const { copyFileSyncMock: vi.fn(), execFileSyncMock: vi.fn(), readFileSyncMock: vi.fn(), + symlinkSyncMock: vi.fn(), })); vi.mock("electron", () => ({ @@ -51,6 +53,7 @@ vi.mock("node:fs", () => ({ mkdirSync: mkdirSyncMock, writeFileSync: writeFileSyncMock, copyFileSync: copyFileSyncMock, + symlinkSync: symlinkSyncMock, })); vi.mock("node:child_process", () => ({ @@ -72,6 +75,7 @@ describe("DesktopPreReadyPlatform", () => { copyFileSyncMock.mockReset(); execFileSyncMock.mockReset(); readFileSyncMock.mockReset().mockReturnValue("{}"); + symlinkSyncMock.mockReset(); }); it.effect("persists Linux scaling synchronously across relaunches and settings rewrites", () => { @@ -181,6 +185,47 @@ describe("DesktopPreReadyPlatform", () => { ).pipe(Effect.ensuring(Effect.sync(() => vi.unstubAllEnvs()))); }); + it.effect.each([ + { relaunchedByUpdater: "true", restored: true }, + { relaunchedByUpdater: "", restored: false }, + ])( + "restores the versioned AppImage path an update removed (updater relaunch: $relaunchedByUpdater)", + ({ relaunchedByUpdater, restored }) => { + vi.stubEnv("VITE_DEV_SERVER_URL", ""); + getSwitchValueMock.mockReturnValue(""); + const entryPath = "/xdg/applications/com.t3tools.T3Code.desktop"; + const files = new Map([[entryPath, "Exec=/Applications/T3-Code-0.1.8-x86_64.AppImage %U\n"]]); + readFileSyncMock.mockImplementation((path: string) => files.get(path) ?? "{}"); + writeFileSyncMock.mockImplementation((path: string, contents: string | Buffer) => { + files.set(path, String(contents)); + }); + + return Effect.scoped( + Effect.gen(function* () { + yield* Layer.build( + DesktopPreReadyPlatform.layer.pipe( + Layer.provide(Layer.succeed(HostProcess.Platform, "linux")), + Layer.provide( + DesktopConfig.layerTest({ + XDG_DATA_HOME: "/xdg", + APPIMAGE: "/Applications/T3-Code-x86_64.AppImage", + APPIMAGE_SILENT_INSTALL: relaunchedByUpdater, + }), + ), + ), + ); + assert.deepEqual( + symlinkSyncMock.mock.calls, + restored + ? [["T3-Code-x86_64.AppImage", "/Applications/T3-Code-0.1.8-x86_64.AppImage"]] + : [], + ); + assert.include(files.get(entryPath), "Exec=/Applications/T3-Code-x86_64.AppImage %U"); + }), + ).pipe(Effect.ensuring(Effect.sync(() => vi.unstubAllEnvs()))); + }, + ); + it.effect("points the early portal entry at a configured distribution launcher", () => { vi.stubEnv("VITE_DEV_SERVER_URL", ""); getSwitchValueMock.mockReturnValue(""); diff --git a/apps/desktop/src/app/DesktopPreReadyPlatform.ts b/apps/desktop/src/app/DesktopPreReadyPlatform.ts index df8059133fc20..8ac4654636ab6 100644 --- a/apps/desktop/src/app/DesktopPreReadyPlatform.ts +++ b/apps/desktop/src/app/DesktopPreReadyPlatform.ts @@ -14,7 +14,10 @@ import * as HostProcess from "@t3tools/shared/HostProcess"; import * as DesktopEarlyElectronStartup from "./DesktopEarlyElectronStartup.ts"; import { trimmedString } from "./DesktopConfig.ts"; import { resolveDesktopAppBranding } from "./DesktopEnvironment.ts"; -import { renderUrlHandlerDesktopEntry } from "./DesktopLinuxUrlHandler.ts"; +import { + renderUrlHandlerDesktopEntry, + replacedVersionedAppImagePath, +} from "./DesktopLinuxUrlHandler.ts"; import * as ElectronProtocol from "../electron/ElectronProtocol.ts"; export interface DesktopPreReadyCommandLineReader { @@ -56,6 +59,8 @@ const LinuxDesktopEntryConfig = Config.all({ xdgDataHome: trimmedString("XDG_DATA_HOME"), launcherPath: trimmedString("T3CODE_DESKTOP_LAUNCHER_PATH"), appImagePath: trimmedString("APPIMAGE"), + // electron-updater sets this on the AppImage it relaunches after installing an update. + relaunchedByUpdater: trimmedString("APPIMAGE_SILENT_INSTALL"), }); /** @public */ @@ -116,8 +121,26 @@ export const make = Effect.gen(function* () { ); } catch {} } + const desktopEntryPath = NodePath.posix.join(applicationsDir, linux.linuxDesktopEntryName); + const { appImagePath } = linuxDesktopEntryConfig; + if ( + Option.isSome(appImagePath) && + Option.isNone(linuxDesktopEntryConfig.launcherPath) && + Option.contains(linuxDesktopEntryConfig.relaunchedByUpdater, "true") + ) { + try { + const replacedPath = replacedVersionedAppImagePath({ + previousEntry: NodeFS.readFileSync(desktopEntryPath, "utf8"), + appImagePath: appImagePath.value, + }); + // Relative, so the pair survives a moved folder; symlink never overwrites a file. + if (replacedPath !== null) { + NodeFS.symlinkSync(NodePath.posix.basename(appImagePath.value), replacedPath); + } + } catch {} + } NodeFS.writeFileSync( - NodePath.posix.join(applicationsDir, linux.linuxDesktopEntryName), + desktopEntryPath, renderUrlHandlerDesktopEntry({ displayName: resolveDesktopAppBranding({ isDevelopment: linux.isDevelopment, diff --git a/apps/desktop/src/app/DesktopStartupRecovery.ts b/apps/desktop/src/app/DesktopStartupRecovery.ts index c8228183aad91..e6c210043cf26 100644 --- a/apps/desktop/src/app/DesktopStartupRecovery.ts +++ b/apps/desktop/src/app/DesktopStartupRecovery.ts @@ -217,7 +217,6 @@ export const listen: Effect.Effect< const download = yield* updates.download; if (!download.completed) return; } - // Waits for a running background check; plain install would refuse. yield* updates.installPrepared(current.version); return; } diff --git a/apps/desktop/src/app/stdioErrors.test.ts b/apps/desktop/src/app/stdioErrors.test.ts new file mode 100644 index 0000000000000..ccfa3ed951178 --- /dev/null +++ b/apps/desktop/src/app/stdioErrors.test.ts @@ -0,0 +1,22 @@ +import * as NodeOS from "node:os"; +import { assert, it } from "vite-plus/test"; + +import { handleStdioError } from "./stdioErrors.ts"; + +const errorWithCode = (code: string, errno?: number) => + Object.assign(new Error(code), { code, errno }); +const EDQUOT_ERRNO = NodeOS.constants.errno.EDQUOT; + +it.each(["EPIPE", "ENOSPC"])("ignores %s on console output", (code) => { + assert.doesNotThrow(() => handleStdioError(errorWithCode(code))); +}); + +// Node reports EDQUOT as UNKNOWN because libuv has no name for it. +it.skipIf(EDQUOT_ERRNO === undefined)("ignores EDQUOT on console output", () => { + assert.doesNotThrow(() => handleStdioError(errorWithCode("UNKNOWN", -(EDQUOT_ERRNO ?? 0)))); +}); + +it("rethrows other console output errors", () => { + assert.throws(() => handleStdioError(errorWithCode("EIO")), "EIO"); + assert.throws(() => handleStdioError(errorWithCode("UNKNOWN", -5)), "UNKNOWN"); +}); diff --git a/apps/desktop/src/app/stdioErrors.ts b/apps/desktop/src/app/stdioErrors.ts new file mode 100644 index 0000000000000..b23c0e89e79a8 --- /dev/null +++ b/apps/desktop/src/app/stdioErrors.ts @@ -0,0 +1,13 @@ +import * as NodeOS from "node:os"; + +// libuv has no name for EDQUOT, so Node reports it as code "UNKNOWN" with the raw errno. +const EDQUOT_ERRNO = NodeOS.constants.errno.EDQUOT; + +// Redirected console output can fail when its log volume is full. +export function handleStdioError(error: unknown): void { + if (typeof error === "object" && error !== null) { + if ("code" in error && (error.code === "EPIPE" || error.code === "ENOSPC")) return; + if (EDQUOT_ERRNO !== undefined && "errno" in error && error.errno === -EDQUOT_ERRNO) return; + } + throw error; +} diff --git a/apps/desktop/src/backend/DesktopBackendConfiguration.test.ts b/apps/desktop/src/backend/DesktopBackendConfiguration.test.ts index 17ba6a674f76c..70f7ed87d2214 100644 --- a/apps/desktop/src/backend/DesktopBackendConfiguration.test.ts +++ b/apps/desktop/src/backend/DesktopBackendConfiguration.test.ts @@ -373,7 +373,7 @@ describe("DesktopBackendConfiguration", () => { ); assert.equal(config.runningDistro, "Ubuntu"); - assert.deepEqual(config.args.slice(0, 2), ["-d", "Ubuntu"]); + assert.deepEqual(config.args.slice(0, 5), ["-d", "Ubuntu", "--cd", "~", "--exec"]); assert.deepEqual(observedDistros, ["Ubuntu", "Ubuntu", "Ubuntu"]); assert.isTrue(Option.isNone(config.preflightFailure)); }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), @@ -436,6 +436,8 @@ describe("DesktopBackendConfiguration", () => { assert.deepEqual(config.args, [ "-d", "Ubuntu", + "--cd", + "~", "--exec", "env", `PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${resolvedPath}`, @@ -651,6 +653,8 @@ describe("DesktopBackendConfiguration", () => { assert.deepEqual(config.args, [ "-d", "Ubuntu", + "--cd", + "~", "--exec", "env", "PATH=/home/test user's/.nvm/versions/node/v22.0.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/test user/bin:/opt/test's tools/bin:/usr/bin:/bin", @@ -683,6 +687,43 @@ describe("DesktopBackendConfiguration", () => { ), ); + it.effect("gives the Windows and WSL backends one host scope that survives a restart", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const baseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-desktop-backend-config-test-", + }); + const launch = Effect.gen(function* () { + const configuration = yield* DesktopBackendConfiguration.DesktopBackendConfiguration; + const primary = yield* configuration.resolvePrimary; + const wsl = yield* configuration.resolveWsl({ port: 5050, distro: null }); + assert.equal(wsl.env.T3CODE_DESKTOP_HOST_SCOPE, primary.env.T3CODE_DESKTOP_HOST_SCOPE); + assert.include((wsl.env.WSLENV ?? "").split(":"), "T3CODE_DESKTOP_HOST_SCOPE"); + return primary.env.T3CODE_DESKTOP_HOST_SCOPE; + }).pipe( + Effect.provide( + layerBackendConfiguration().pipe( + Layer.provideMerge(layerServerExposure), + Layer.provideMerge(DesktopAppSettings.layerTest()), + Layer.provideMerge(DesktopWslServerTree.layerTest()), + Layer.provideMerge( + DesktopWslEnvironment.layerTest({ + isAvailable: true, + windowsToWslPath: () => Option.some("/mnt/c/repo/apps/server/src/index.ts"), + getDistroIp: () => Option.some("172.27.0.99"), + }), + ), + Layer.provideMerge(layerEnvironment(baseDir, { platform: "win32" })), + ), + ), + ); + + const first = yield* launch; + assert.match(first ?? "", /^[0-9a-f]{32}$/); + assert.equal(yield* launch, first); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), + ); + it.effect("resolveWsl carries the kill switch and resource attributes into the distro", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; @@ -748,7 +789,7 @@ describe("DesktopBackendConfiguration", () => { assert.equal(config.env.T3CODE_OTLP_PROTOCOL, "http/protobuf"); assert.equal( config.env.WSLENV, - "GOPATH/p:OPENAI_API_KEY/u:EMPTY::AZURE_DEVOPS_EXT_PAT/u:ANTHROPIC_API_KEY:T3CODE_OTLP_HEADERS:T3CODE_OTLP_PROTOCOL", + "GOPATH/p:OPENAI_API_KEY/u:EMPTY::AZURE_DEVOPS_EXT_PAT/u:T3CODE_DESKTOP_HOST_SCOPE:ANTHROPIC_API_KEY:T3CODE_OTLP_HEADERS:T3CODE_OTLP_PROTOCOL", ); }).pipe( Effect.provide( diff --git a/apps/desktop/src/backend/DesktopBackendConfiguration.ts b/apps/desktop/src/backend/DesktopBackendConfiguration.ts index e7b8405f8124d..5f499e6f4ee37 100644 --- a/apps/desktop/src/backend/DesktopBackendConfiguration.ts +++ b/apps/desktop/src/backend/DesktopBackendConfiguration.ts @@ -243,8 +243,38 @@ interface SharedBootstrapInput { readonly bootstrapToken: string; readonly bootstrapSecret: string; readonly observabilitySettings: BackendObservabilitySettings; + readonly hostScope: string; } +// Matches the server's DESKTOP_HOST_SCOPE_ENV. +const DESKTOP_HOST_SCOPE_ENV = "T3CODE_DESKTOP_HOST_SCOPE"; +const HOST_SCOPE_FILE_NAME = "desktop-host-scope"; +const HOST_SCOPE_PATTERN = /^[0-9a-f]{32}$/; + +/** + * Not a credential: it only tells this install's Windows and WSL servers that + * they run on one machine, so usage both read from one Windows directory counts + * once. Persisted so summaries from earlier launches still match. + */ +const readOrCreateHostScope = Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const random = yield* SecureRandom.SecureRandom; + const scopePath = environment.path.join(environment.stateDir, HOST_SCOPE_FILE_NAME); + const existing = (yield* fileSystem + .readFileString(scopePath) + .pipe(Effect.orElseSucceed(() => ""))).trim(); + if (HOST_SCOPE_PATTERN.test(existing)) return existing; + const scope = Hex.encode(yield* random.randomBytes(16).pipe(Effect.orDie)); + yield* fileSystem.makeDirectory(environment.stateDir, { recursive: true }).pipe( + Effect.andThen(fileSystem.writeFileString(scopePath, scope)), + Effect.catch((cause) => + Effect.logWarning("Could not persist the desktop host scope.", { cause }), + ), + ); + return scope; +}); + type WslPreflightRuntime = | { readonly kind: "executable"; @@ -540,6 +570,7 @@ const resolvePrimaryStartConfig = Effect.fn("desktop.backendConfiguration.resolv ELECTRON_RUN_AS_NODE: "1", T3CODE_DESKTOP_SHELL_DISCOVERY_PENDING: "1", T3CODE_CLI_PATH: Option.getOrUndefined(input.cliPath), + [DESKTOP_HOST_SCOPE_ENV]: input.hostScope, }, extendEnv: true, bootstrap, @@ -636,8 +667,8 @@ const resolveWslStartConfig = Effect.fn("desktop.backendConfiguration.resolveWsl const httpBaseUrl = new URL(`http://${rendererHost}:${input.port}`); const distroArgs = distroForConfig ? ["-d", distroForConfig] : []; - const forwardedEnv: Record = {}; - const forwardedEnvNames: string[] = []; + const forwardedEnv: Record = { [DESKTOP_HOST_SCOPE_ENV]: input.hostScope }; + const forwardedEnvNames: string[] = [DESKTOP_HOST_SCOPE_ENV]; for (const name of WSL_FORWARDED_ENV_NAMES) { const value = env[name]; if (value !== undefined && value.length > 0) { @@ -708,6 +739,9 @@ const resolveWslStartConfig = Effect.fn("desktop.backendConfiguration.resolveWsl ...baseConfig, args: [ ...distroArgs, + // WSL otherwise inherits the Windows-mounted desktop working directory. + "--cd", + "~", "--exec", "env", `PATH=${launchPath}`, @@ -735,6 +769,13 @@ export const make = Effect.gen(function* () { const random = yield* SecureRandom.SecureRandom; // Random generation yields; serialize it so concurrent backends cannot capture distinct bootstrap secrets. const secretRef = yield* SynchronizedRef.make(Option.none()); + const hostScope = yield* Effect.cached( + readOrCreateHostScope.pipe( + Effect.provideService(FileSystem.FileSystem, fileSystem), + Effect.provideService(DesktopEnvironment.DesktopEnvironment, environment), + Effect.provideService(SecureRandom.SecureRandom, random), + ), + ); const getOrCreateBootstrapSecret = SynchronizedRef.modifyEffect(secretRef, (current) => Option.match(current, { onSome: (secret) => Effect.succeed([secret, current] as const), @@ -762,6 +803,7 @@ export const make = Effect.gen(function* () { bootstrapToken, bootstrapSecret, observabilitySettings, + hostScope: yield* hostScope, } satisfies SharedBootstrapInput; }); diff --git a/apps/desktop/src/backend/DesktopBackendManager.test.ts b/apps/desktop/src/backend/DesktopBackendManager.test.ts index 3126d12a2e938..06a1f768f4bd4 100644 --- a/apps/desktop/src/backend/DesktopBackendManager.test.ts +++ b/apps/desktop/src/backend/DesktopBackendManager.test.ts @@ -5,6 +5,11 @@ import { type DesktopBackendBootstrap as DesktopBackendBootstrapValue, DesktopTelemetryControlMessage, } from "@t3tools/contracts"; +import { + DATABASE_MIGRATION_PROGRESS_MESSAGE, + DATABASE_MIGRATIONS_FINISHED_MESSAGE, + DATABASE_MIGRATIONS_STARTED_MESSAGE, +} from "@t3tools/shared/databaseMigrationLog"; import { assert, describe, it } from "@effect/vitest"; import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; @@ -194,129 +199,125 @@ function makeTestInstance(input: MakeInstanceInput) { describe("DesktopBackendManager", () => { it.effect("notifies readiness waiters on HTTP readiness and stop without clock advancement", () => - Effect.scoped( - Effect.gen(function* () { - const probe = yield* Deferred.make(); - const ready = yield* Deferred.make(); - const exit = yield* Deferred.make(); - const instance = yield* makeTestInstance({ - spawnerLayer: Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => - Effect.acquireRelease( - Effect.succeed( - makeProcess({ - exitCode: Deferred.await(exit), - kill: () => - Deferred.succeed(exit, ChildProcessSpawner.ExitCode(0)).pipe(Effect.asVoid), - }), - ), - () => Deferred.succeed(exit, ChildProcessSpawner.ExitCode(0)), + Effect.gen(function* () { + const probe = yield* Deferred.make(); + const ready = yield* Deferred.make(); + const exit = yield* Deferred.make(); + const instance = yield* makeTestInstance({ + spawnerLayer: Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.acquireRelease( + Effect.succeed( + makeProcess({ + exitCode: Deferred.await(exit), + kill: () => + Deferred.succeed(exit, ChildProcessSpawner.ExitCode(0)).pipe(Effect.asVoid), + }), ), + () => Deferred.succeed(exit, ChildProcessSpawner.ExitCode(0)), ), ), - httpClientLayer: Layer.succeed( - HttpClient.HttpClient, - HttpClient.make((request) => - Deferred.succeed(probe, undefined).pipe( - Effect.andThen(Deferred.await(ready)), - Effect.as(HttpClientResponse.fromWeb(request, new Response("{}"))), - ), + ), + httpClientLayer: Layer.succeed( + HttpClient.HttpClient, + HttpClient.make((request) => + Deferred.succeed(probe, undefined).pipe( + Effect.andThen(Deferred.await(ready)), + Effect.as(HttpClientResponse.fromWeb(request, new Response("{}"))), ), ), - }); - yield* instance.start; - yield* Deferred.await(probe); - const waiter = yield* instance.waitForReady(Duration.seconds(15)).pipe(Effect.forkChild); - yield* Deferred.succeed(ready, undefined); - assert.isTrue(yield* Fiber.join(waiter)); - yield* instance.stop(); - assert.isFalse(yield* instance.waitForReady(Duration.seconds(15))); - }), - ), + ), + }); + yield* instance.start; + yield* Deferred.await(probe); + const waiter = yield* instance.waitForReady(Duration.seconds(15)).pipe(Effect.forkChild); + yield* Deferred.succeed(ready, undefined); + assert.isTrue(yield* Fiber.join(waiter)); + yield* instance.stop(); + assert.isFalse(yield* instance.waitForReady(Duration.seconds(15))); + }), ); it.effect("spawns the backend with fd3 bootstrap and fd4 telemetry", () => - Effect.scoped( - Effect.gen(function* () { - let spawnedCommand: ChildProcess.Command | undefined; - let bootstrapJson = ""; - let telemetryJson = ""; - let readyCount = 0; - const ready = yield* Deferred.make(); - const exited = yield* Queue.unbounded(); - - const layerSpawner = Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make((command) => - Effect.gen(function* () { - spawnedCommand = command; - if (command._tag === "StandardCommand") { - const fd3 = command.options.additionalFds?.fd3; - if (fd3?.type === "input" && fd3.stream) { - bootstrapJson = yield* fd3.stream.pipe(Stream.decodeText(), Stream.mkString); - } - const fd4 = command.options.additionalFds?.fd4; - if (fd4?.type === "input" && fd4.stream) { - telemetryJson = yield* fd4.stream.pipe(Stream.decodeText(), Stream.mkString); - } + Effect.gen(function* () { + let spawnedCommand: ChildProcess.Command | undefined; + let bootstrapJson = ""; + let telemetryJson = ""; + let readyCount = 0; + const ready = yield* Deferred.make(); + const exited = yield* Queue.unbounded(); + + const layerSpawner = Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make((command) => + Effect.gen(function* () { + spawnedCommand = command; + if (command._tag === "StandardCommand") { + const fd3 = command.options.additionalFds?.fd3; + if (fd3?.type === "input" && fd3.stream) { + bootstrapJson = yield* fd3.stream.pipe(Stream.decodeText(), Stream.mkString); + } + const fd4 = command.options.additionalFds?.fd4; + if (fd4?.type === "input" && fd4.stream) { + telemetryJson = yield* fd4.stream.pipe(Stream.decodeText(), Stream.mkString); } + } - return makeProcess({ - exitCode: Deferred.await(ready).pipe(Effect.as(ChildProcessSpawner.ExitCode(0))), - }); - }), - ), - ); + return makeProcess({ + exitCode: Deferred.await(ready).pipe(Effect.as(ChildProcessSpawner.ExitCode(0))), + }); + }), + ), + ); - const instance = yield* makeTestInstance({ - config: { - ...baseConfig, - bootstrap: configWithObservability, - }, - spawnerLayer: layerSpawner, - desktopTelemetryStream: Stream.encodeText( - Stream.make('{"version":1,"type":"desktopTelemetryHello","electronPid":123}\n'), - ), - onReady: Effect.sync(() => { - readyCount += 1; - }).pipe(Effect.andThen(Deferred.succeed(ready, void 0)), Effect.asVoid), - backendOutputLog: { - persistFailure: () => Queue.offer(exited, void 0).pipe(Effect.asVoid), - }, - }); + const instance = yield* makeTestInstance({ + config: { + ...baseConfig, + bootstrap: configWithObservability, + }, + spawnerLayer: layerSpawner, + desktopTelemetryStream: Stream.encodeText( + Stream.make('{"version":1,"type":"desktopTelemetryHello","electronPid":123}\n'), + ), + onReady: Effect.sync(() => { + readyCount += 1; + }).pipe(Effect.andThen(Deferred.succeed(ready, void 0)), Effect.asVoid), + backendOutputLog: { + persistFailure: () => Queue.offer(exited, void 0).pipe(Effect.asVoid), + }, + }); - yield* instance.start; - yield* Queue.take(exited); - - assert.equal(readyCount, 1); - assert.isDefined(spawnedCommand); - if (spawnedCommand._tag !== "StandardCommand") { - throw new Error("Expected backend to spawn a standard command."); - } - - assert.equal(spawnedCommand.command, "/electron"); - assert.deepEqual(spawnedCommand.args, ["/server/bin.mjs", "--bootstrap-fd", "3"]); - assert.equal(spawnedCommand.options.cwd, "/server"); - assert.equal(spawnedCommand.options.extendEnv, true); - assert.equal(spawnedCommand.options.stdout, "pipe"); - assert.equal(spawnedCommand.options.stderr, "pipe"); - assert.equal(spawnedCommand.options.killSignal, "SIGTERM"); - assert.isDefined(spawnedCommand.options.forceKillAfter); - assert.equal(spawnedCommand.options.additionalFds?.fd4?.type, "input"); - assert.equal(spawnedCommand.options.additionalFds?.fd5?.type, "output"); - assert.equal( - Duration.toMillis(Duration.fromInputUnsafe(spawnedCommand.options.forceKillAfter)), - 5_000, - ); - - assert.deepEqual(yield* decodeBootstrap(bootstrapJson), configWithObservability); - assert.equal( - telemetryJson, - '{"version":1,"type":"desktopTelemetryHello","electronPid":123}\n', - ); - }), - ), + yield* instance.start; + yield* Queue.take(exited); + + assert.equal(readyCount, 1); + assert.isDefined(spawnedCommand); + if (spawnedCommand._tag !== "StandardCommand") { + throw new Error("Expected backend to spawn a standard command."); + } + + assert.equal(spawnedCommand.command, "/electron"); + assert.deepEqual(spawnedCommand.args, ["/server/bin.mjs", "--bootstrap-fd", "3"]); + assert.equal(spawnedCommand.options.cwd, "/server"); + assert.equal(spawnedCommand.options.extendEnv, true); + assert.equal(spawnedCommand.options.stdout, "pipe"); + assert.equal(spawnedCommand.options.stderr, "pipe"); + assert.equal(spawnedCommand.options.killSignal, "SIGTERM"); + assert.isDefined(spawnedCommand.options.forceKillAfter); + assert.equal(spawnedCommand.options.additionalFds?.fd4?.type, "input"); + assert.equal(spawnedCommand.options.additionalFds?.fd5?.type, "output"); + assert.equal( + Duration.toMillis(Duration.fromInputUnsafe(spawnedCommand.options.forceKillAfter)), + 5_000, + ); + + assert.deepEqual(yield* decodeBootstrap(bootstrapJson), configWithObservability); + assert.equal( + telemetryJson, + '{"version":1,"type":"desktopTelemetryHello","electronPid":123}\n', + ); + }), ); it.effect("reports a SIGKILL exit without an exit code", () => @@ -344,285 +345,308 @@ describe("DesktopBackendManager", () => { ); it.effect("reports child exit before waiting for trailing output to drain", () => - Effect.scoped( - Effect.gen(function* () { - const exitObserved = yield* Deferred.make(); - const finishOutputDrain = yield* Deferred.make(); - const layerSpawner = Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => - Effect.succeed( - makeProcess({ - stdout: Stream.fromEffect( - Deferred.await(finishOutputDrain).pipe( - Effect.as(new TextEncoder().encode("trailing output\n")), - ), + Effect.gen(function* () { + const exitObserved = yield* Deferred.make(); + const finishOutputDrain = yield* Deferred.make(); + const layerSpawner = Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.succeed( + makeProcess({ + stdout: Stream.fromEffect( + Deferred.await(finishOutputDrain).pipe( + Effect.as(new TextEncoder().encode("trailing output\n")), ), - exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(1)), - }), - ), + ), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(1)), + }), ), - ); + ), + ); - const runFiber = yield* DesktopBackendManager.runBackendProcess({ - ...baseConfig, - desktopTelemetryStream: Stream.empty, - onExitObserved: () => Deferred.succeed(exitObserved, void 0).pipe(Effect.asVoid), - }).pipe( - Effect.provide(Layer.merge(layerSpawner, layerHealthyHttpClient)), - Effect.forkChild, - ); - - yield* Deferred.await(exitObserved); - assert.isUndefined(runFiber.pollUnsafe()); - - yield* Deferred.succeed(finishOutputDrain, void 0); - assert.equal((yield* Fiber.join(runFiber)).code.pipe(Option.getOrUndefined), 1); - }), - ), + const runFiber = yield* DesktopBackendManager.runBackendProcess({ + ...baseConfig, + desktopTelemetryStream: Stream.empty, + onExitObserved: () => Deferred.succeed(exitObserved, void 0).pipe(Effect.asVoid), + }).pipe(Effect.provide(Layer.merge(layerSpawner, layerHealthyHttpClient)), Effect.forkChild); + + yield* Deferred.await(exitObserved); + assert.isUndefined(runFiber.pollUnsafe()); + + yield* Deferred.succeed(finishOutputDrain, void 0); + assert.equal((yield* Fiber.join(runFiber)).code.pipe(Option.getOrUndefined), 1); + }), ); - it.effect("continues routing desktop telemetry control messages after an invalid line", () => + it.effect("becomes ready from the server ready line while HTTP probes stall", () => Effect.scoped( Effect.gen(function* () { - const handled = yield* Deferred.make(); - const controlMessage = encodeDesktopTelemetryControl({ - version: 1, - type: "setDiagnosticsDemand", - enabled: true, - }); - const layerSpawner = Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => - Effect.succeed( - makeProcess({ - getOutputFd: (fd) => - fd === 5 - ? Stream.encodeText(Stream.make(`not-json\n${controlMessage}\n`)) - : Stream.empty, - exitCode: Deferred.await(handled).pipe(Effect.as(ChildProcessSpawner.ExitCode(0))), - }), + const probeStarted = yield* Deferred.make(); + const exit = yield* Deferred.make(); + const readyLine = encodeDesktopTelemetryControl({ version: 1, type: "serverReady" }); + const instance = yield* makeTestInstance({ + spawnerLayer: Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.succeed( + makeProcess({ + getOutputFd: (fd) => + fd === 5 + ? Stream.fromEffect(Deferred.await(probeStarted)).pipe( + Stream.flatMap(() => Stream.encodeText(Stream.make(`${readyLine}\n`))), + ) + : Stream.empty, + exitCode: Deferred.await(exit), + kill: () => + Deferred.succeed(exit, ChildProcessSpawner.ExitCode(0)).pipe(Effect.asVoid), + }), + ), ), ), - ); - const instance = yield* makeTestInstance({ - spawnerLayer: layerSpawner, - desktopTelemetryPublisher: { - handleControlForSource: (_sourceId, message) => - message.type === "setDiagnosticsDemand" - ? Deferred.succeed(handled, message.enabled).pipe(Effect.asVoid) - : Effect.void, - }, + httpClientLayer: layerHttpClient(() => + Deferred.succeed(probeStarted, undefined).pipe(Effect.andThen(Effect.never)), + ), }); yield* instance.start; - assert.isTrue(yield* Deferred.await(handled)); + assert.isTrue(yield* instance.waitForReady(Duration.seconds(15))); }), ), ); + it.effect("continues routing desktop telemetry control messages after an invalid line", () => + Effect.gen(function* () { + const handled = yield* Deferred.make(); + const controlMessage = encodeDesktopTelemetryControl({ + version: 1, + type: "setDiagnosticsDemand", + enabled: true, + }); + const layerSpawner = Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.succeed( + makeProcess({ + getOutputFd: (fd) => + fd === 5 + ? Stream.encodeText(Stream.make(`not-json\n${controlMessage}\n`)) + : Stream.empty, + exitCode: Deferred.await(handled).pipe(Effect.as(ChildProcessSpawner.ExitCode(0))), + }), + ), + ), + ); + const instance = yield* makeTestInstance({ + spawnerLayer: layerSpawner, + desktopTelemetryPublisher: { + handleControlForSource: (_sourceId, message) => + message.type === "setDiagnosticsDemand" + ? Deferred.succeed(handled, message.enabled).pipe(Effect.asVoid) + : Effect.void, + }, + }); + + yield* instance.start; + assert.isTrue(yield* Deferred.await(handled)); + }), + ); + it.effect("stops the scoped process with output still pending", () => - Effect.scoped( - Effect.gen(function* () { - let startCount = 0; - let closedCount = 0; - const closed = yield* Deferred.make(); - const teardownStarted = yield* Deferred.make(); - const finishTeardown = yield* Deferred.make(); - const startedPids = yield* Queue.unbounded(); - const ready = yield* Deferred.make(); - const backendReadyFlag = yield* Ref.make(false); - let shutdownCount = 0; - let persistedFailureCount = 0; - let discardedSessionCount = 0; - let removedTelemetrySources = 0; - - const layerSpawner = Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => - Effect.gen(function* () { - const scope = yield* Scope.Scope; - startCount += 1; - yield* Queue.offer(startedPids, 123); - const close = Deferred.succeed(teardownStarted, undefined).pipe( - Effect.andThen(Deferred.await(finishTeardown)), - Effect.andThen( - Effect.sync(() => { - closedCount += 1; - }), - ), - Effect.andThen(Deferred.succeed(closed, void 0)), - Effect.asVoid, - ); + Effect.gen(function* () { + let startCount = 0; + let closedCount = 0; + const closed = yield* Deferred.make(); + const teardownStarted = yield* Deferred.make(); + const finishTeardown = yield* Deferred.make(); + const startedPids = yield* Queue.unbounded(); + const ready = yield* Deferred.make(); + const backendReadyFlag = yield* Ref.make(false); + let shutdownCount = 0; + let persistedFailureCount = 0; + let discardedSessionCount = 0; + let removedTelemetrySources = 0; - yield* Scope.addFinalizer(scope, close); + const layerSpawner = Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.gen(function* () { + const scope = yield* Scope.Scope; + startCount += 1; + yield* Queue.offer(startedPids, 123); + const close = Deferred.succeed(teardownStarted, undefined).pipe( + Effect.andThen(Deferred.await(finishTeardown)), + Effect.andThen( + Effect.sync(() => { + closedCount += 1; + }), + ), + Effect.andThen(Deferred.succeed(closed, void 0)), + Effect.asVoid, + ); + + yield* Scope.addFinalizer(scope, close); + + return makeProcess({ + stdout: Stream.never, + exitCode: Deferred.await(closed).pipe(Effect.as(ChildProcessSpawner.ExitCode(0))), + kill: () => close, + }); + }), + ), + ); - return makeProcess({ - stdout: Stream.never, - exitCode: Deferred.await(closed).pipe(Effect.as(ChildProcessSpawner.ExitCode(0))), - kill: () => close, - }); + const instance = yield* makeTestInstance({ + spawnerLayer: layerSpawner, + onReady: Ref.set(backendReadyFlag, true).pipe( + Effect.andThen(Deferred.succeed(ready, void 0)), + Effect.asVoid, + ), + onShutdown: Ref.set(backendReadyFlag, false).pipe( + Effect.andThen( + Effect.sync(() => { + shutdownCount += 1; }), ), - ); - - const instance = yield* makeTestInstance({ - spawnerLayer: layerSpawner, - onReady: Ref.set(backendReadyFlag, true).pipe( - Effect.andThen(Deferred.succeed(ready, void 0)), - Effect.asVoid, - ), - onShutdown: Ref.set(backendReadyFlag, false).pipe( - Effect.andThen( - Effect.sync(() => { - shutdownCount += 1; - }), - ), - ), - backendOutputLog: { - persistFailure: () => - Effect.sync(() => { - persistedFailureCount += 1; - }), - discardSession: Effect.sync(() => { - discardedSessionCount += 1; + ), + backendOutputLog: { + persistFailure: () => + Effect.sync(() => { + persistedFailureCount += 1; }), - }, - desktopTelemetryPublisher: { - removeControlSource: () => - Effect.sync(() => { - removedTelemetrySources += 1; - }), - }, - }); - assert.isTrue(Option.isNone(yield* instance.currentConfig)); - - yield* instance.start; - assert.equal(yield* Queue.take(startedPids), 123); - yield* Deferred.await(ready); - assert.isTrue(yield* Ref.get(backendReadyFlag)); - assert.deepEqual(yield* instance.currentConfig, Option.some(baseConfig)); - - const runningSnapshot = yield* instance.snapshot; - assert.equal(runningSnapshot.ready, true); - assert.deepEqual(runningSnapshot.activePid, Option.some(123)); - - const stopFiber = yield* instance.stop().pipe(Effect.forkChild); - yield* Deferred.await(teardownStarted).pipe(Effect.timeout("1 second")); - assert.isFalse(yield* Ref.get(backendReadyFlag)); - assert.equal(shutdownCount, 1); - yield* Deferred.succeed(finishTeardown, undefined); - yield* Fiber.join(stopFiber).pipe(Effect.timeout("1 second")); - assert.equal(startCount, 1); - assert.equal(closedCount, 1); - assert.equal(persistedFailureCount, 0); - assert.equal(discardedSessionCount, 1); - assert.equal(removedTelemetrySources, 1); - - const stoppedSnapshot = yield* instance.snapshot; - assert.isFalse(yield* Ref.get(backendReadyFlag)); - assert.equal(shutdownCount, 1); - assert.equal(stoppedSnapshot.desiredRunning, false); - assert.equal(stoppedSnapshot.ready, false); - assert.equal(Option.isNone(stoppedSnapshot.activePid), true); - }), - ), + discardSession: Effect.sync(() => { + discardedSessionCount += 1; + }), + }, + desktopTelemetryPublisher: { + removeControlSource: () => + Effect.sync(() => { + removedTelemetrySources += 1; + }), + }, + }); + assert.isTrue(Option.isNone(yield* instance.currentConfig)); + + yield* instance.start; + assert.equal(yield* Queue.take(startedPids), 123); + yield* Deferred.await(ready); + assert.isTrue(yield* Ref.get(backendReadyFlag)); + assert.deepEqual(yield* instance.currentConfig, Option.some(baseConfig)); + + const runningSnapshot = yield* instance.snapshot; + assert.equal(runningSnapshot.ready, true); + assert.deepEqual(runningSnapshot.activePid, Option.some(123)); + + const stopFiber = yield* instance.stop().pipe(Effect.forkChild); + yield* Deferred.await(teardownStarted).pipe(Effect.timeout("1 second")); + assert.isFalse(yield* Ref.get(backendReadyFlag)); + assert.equal(shutdownCount, 1); + yield* Deferred.succeed(finishTeardown, undefined); + yield* Fiber.join(stopFiber).pipe(Effect.timeout("1 second")); + assert.equal(startCount, 1); + assert.equal(closedCount, 1); + assert.equal(persistedFailureCount, 0); + assert.equal(discardedSessionCount, 1); + assert.equal(removedTelemetrySources, 1); + + const stoppedSnapshot = yield* instance.snapshot; + assert.isFalse(yield* Ref.get(backendReadyFlag)); + assert.equal(shutdownCount, 1); + assert.equal(stoppedSnapshot.desiredRunning, false); + assert.equal(stoppedSnapshot.ready, false); + assert.equal(Option.isNone(stoppedSnapshot.activePid), true); + }), ); it.effect("restarts when start is requested during stop teardown", () => - Effect.scoped( - Effect.gen(function* () { - const starts = yield* Queue.unbounded(); - const teardownStarted = yield* Deferred.make(); - const finishTeardown = yield* Deferred.make(); - let startCount = 0; + Effect.gen(function* () { + const starts = yield* Queue.unbounded(); + const teardownStarted = yield* Deferred.make(); + const finishTeardown = yield* Deferred.make(); + let startCount = 0; - const layerSpawner = Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => - Effect.gen(function* () { - const scope = yield* Scope.Scope; - const closed = yield* Deferred.make(); - startCount += 1; - yield* Queue.offer(starts, startCount); - if (startCount === 1) { - yield* Scope.addFinalizer( - scope, - Deferred.succeed(teardownStarted, undefined).pipe( - Effect.andThen(Deferred.await(finishTeardown)), - Effect.andThen(Deferred.succeed(closed, undefined)), - Effect.asVoid, - ), - ); - } else { - yield* Scope.addFinalizer( - scope, - Deferred.succeed(closed, undefined).pipe(Effect.asVoid), - ); - } - return makeProcess({ - exitCode: Deferred.await(closed).pipe(Effect.as(ChildProcessSpawner.ExitCode(0))), - }); - }), - ), - ); + const layerSpawner = Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.gen(function* () { + const scope = yield* Scope.Scope; + const closed = yield* Deferred.make(); + startCount += 1; + yield* Queue.offer(starts, startCount); + if (startCount === 1) { + yield* Scope.addFinalizer( + scope, + Deferred.succeed(teardownStarted, undefined).pipe( + Effect.andThen(Deferred.await(finishTeardown)), + Effect.andThen(Deferred.succeed(closed, undefined)), + Effect.asVoid, + ), + ); + } else { + yield* Scope.addFinalizer( + scope, + Deferred.succeed(closed, undefined).pipe(Effect.asVoid), + ); + } + return makeProcess({ + exitCode: Deferred.await(closed).pipe(Effect.as(ChildProcessSpawner.ExitCode(0))), + }); + }), + ), + ); - const instance = yield* makeTestInstance({ - spawnerLayer: layerSpawner, - httpClientLayer: layerHttpClient(() => Effect.never), - }); + const instance = yield* makeTestInstance({ + spawnerLayer: layerSpawner, + httpClientLayer: layerHttpClient(() => Effect.never), + }); - yield* instance.start; - assert.equal(yield* Queue.take(starts), 1); + yield* instance.start; + assert.equal(yield* Queue.take(starts), 1); - const stopFiber = yield* instance.stop().pipe(Effect.forkChild); - yield* Deferred.await(teardownStarted).pipe(Effect.timeout("1 second")); - yield* instance.start; - assert.equal((yield* instance.snapshot).desiredRunning, true); + const stopFiber = yield* instance.stop().pipe(Effect.forkChild); + yield* Deferred.await(teardownStarted).pipe(Effect.timeout("1 second")); + yield* instance.start; + assert.equal((yield* instance.snapshot).desiredRunning, true); - yield* Deferred.succeed(finishTeardown, undefined); - yield* Fiber.join(stopFiber).pipe(Effect.timeout("1 second")); - yield* TestClock.adjust(Duration.millis(500)); + yield* Deferred.succeed(finishTeardown, undefined); + yield* Fiber.join(stopFiber).pipe(Effect.timeout("1 second")); + yield* TestClock.adjust(Duration.millis(500)); - assert.equal(yield* Queue.take(starts).pipe(Effect.timeout("1 second")), 2); - const restarted = yield* instance.snapshot; - assert.equal(restarted.desiredRunning, true); - assert.deepEqual(restarted.activePid, Option.some(123)); - }).pipe(Effect.provide(TestClock.layer())), - ), + assert.equal(yield* Queue.take(starts).pipe(Effect.timeout("1 second")), 2); + const restarted = yield* instance.snapshot; + assert.equal(restarted.desiredRunning, true); + assert.deepEqual(restarted.activePid, Option.some(123)); + }).pipe(Effect.provide(TestClock.layer())), ); it.effect("reports state-directory ownership once without scheduling another backend", () => - Effect.scoped( - Effect.gen(function* () { - let startCount = 0; - const refused = yield* Deferred.make(); - const instance = yield* makeTestInstance({ - spawnerLayer: Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => - Effect.sync(() => { - startCount += 1; - return makeProcess({ - exitCode: Effect.succeed( - ChildProcessSpawner.ExitCode(SERVER_EXIT_CODE_STATE_DIR_OWNED), - ), - }); - }), - ), + Effect.gen(function* () { + let startCount = 0; + const refused = yield* Deferred.make(); + const instance = yield* makeTestInstance({ + spawnerLayer: Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.sync(() => { + startCount += 1; + return makeProcess({ + exitCode: Effect.succeed( + ChildProcessSpawner.ExitCode(SERVER_EXIT_CODE_STATE_DIR_OWNED), + ), + }); + }), ), - httpClientLayer: layerHttpClient(() => Effect.never), - onStateDirOwned: Deferred.succeed(refused, undefined).pipe(Effect.asVoid), - }); - yield* instance.start; - yield* Deferred.await(refused); - const snapshot = yield* instance.snapshot; - assert.isFalse(snapshot.desiredRunning); - assert.isFalse(snapshot.restartScheduled); - yield* TestClock.adjust(Duration.minutes(1)); - assert.equal(startCount, 1); - }).pipe(Effect.provide(TestClock.layer())), - ), + ), + httpClientLayer: layerHttpClient(() => Effect.never), + onStateDirOwned: Deferred.succeed(refused, undefined).pipe(Effect.asVoid), + }); + yield* instance.start; + yield* Deferred.await(refused); + const snapshot = yield* instance.snapshot; + assert.isFalse(snapshot.desiredRunning); + assert.isFalse(snapshot.restartScheduled); + yield* TestClock.adjust(Duration.minutes(1)); + assert.equal(startCount, 1); + }).pipe(Effect.provide(TestClock.layer())), ); // A policy refusal is reported on the first exit, since retrying cannot fix it. @@ -632,145 +656,167 @@ describe("DesktopBackendManager", () => { ])( "reports exit $code with the server output after $exits exits", ({ code, exits, managedPolicy }) => - Effect.scoped( - Effect.gen(function* () { - const message = "Error: Cannot find module 'zod/v4' in /Users/René"; - let exitCount = 0; - const firstExit = yield* Deferred.make(); - const reports = yield* Queue.unbounded<{ - readonly exitCount: number; - readonly failure: DesktopBackendManager.BackendStartupFailure; - }>(); - const instance = yield* makeTestInstance({ - spawnerLayer: Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => - Effect.succeed( - makeProcess({ - // "é" split across chunks must not decode as U+FFFD, and the 4,000-char cut - // lands inside the emoji's surrogate pair. - stderr: Stream.make( - new TextEncoder().encode( - `😀${"x".repeat(3_999 - message.length)}${message.slice(0, -1)}`, - ), - new Uint8Array([0xc3]), - new Uint8Array([0xa9]), + Effect.gen(function* () { + const message = "Error: Cannot find module 'zod/v4' in /Users/René"; + let exitCount = 0; + const firstExit = yield* Deferred.make(); + const reports = yield* Queue.unbounded<{ + readonly exitCount: number; + readonly failure: DesktopBackendManager.BackendStartupFailure; + }>(); + const instance = yield* makeTestInstance({ + spawnerLayer: Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.succeed( + makeProcess({ + // "é" split across chunks must not decode as U+FFFD, and the 4,000-char cut + // lands inside the emoji's surrogate pair. + stderr: Stream.make( + new TextEncoder().encode( + `😀${"x".repeat(3_999 - message.length)}${message.slice(0, -1)}`, ), - exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(code)), - }), - ), + new Uint8Array([0xc3]), + new Uint8Array([0xa9]), + ), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(code)), + }), ), ), - httpClientLayer: layerHttpClient(() => Effect.never), - backendOutputLog: { - persistFailure: () => { - exitCount += 1; - return Deferred.succeed(firstExit, undefined).pipe(Effect.asVoid); - }, + ), + httpClientLayer: layerHttpClient(() => Effect.never), + backendOutputLog: { + persistFailure: () => { + exitCount += 1; + return Deferred.succeed(firstExit, undefined).pipe(Effect.asVoid); }, - onStartupFailure: (failure) => - Queue.offer(reports, { exitCount, failure }).pipe(Effect.asVoid), - }); - yield* instance.start; - yield* Deferred.await(firstExit); - yield* TestClock.adjust(Duration.minutes(1)); - const { exitCount: exitsWhenReported, failure } = yield* Queue.take(reports); - assert.equal(exitsWhenReported, exits); - assert.equal(failure.reason, `The server could not start (code=${code}).`); - assert.equal(failure.managedPolicy, managedPolicy); - assert.isTrue(failure.output.endsWith(message)); - assert.isTrue(failure.output.isWellFormed()); - }).pipe(Effect.provide(TestClock.layer())), - ), + }, + onStartupFailure: (failure) => + Queue.offer(reports, { exitCount, failure }).pipe(Effect.asVoid), + }); + yield* instance.start; + yield* Deferred.await(firstExit); + yield* TestClock.adjust(Duration.minutes(1)); + const { exitCount: exitsWhenReported, failure } = yield* Queue.take(reports); + assert.equal(exitsWhenReported, exits); + assert.equal(failure.reason, `The server could not start (code=${code}).`); + assert.equal(failure.managedPolicy, managedPolicy); + assert.isTrue(failure.output.endsWith(message)); + assert.isTrue(failure.output.isWellFormed()); + }).pipe(Effect.provide(TestClock.layer())), ); it.effect("waits out a slow startup and reports one that never becomes ready", () => - Effect.scoped( - Effect.gen(function* () { - const reports = yield* Queue.unbounded(); - const startInstance = (status: Ref.Ref, onReady?: Effect.Effect) => - makeTestInstance({ - spawnerLayer: Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => - Effect.succeed(makeProcess({ exitCode: Effect.never })), + Effect.gen(function* () { + const reports = yield* Queue.unbounded(); + const startInstance = (status: Ref.Ref, onReady?: Effect.Effect, stdout = "") => + makeTestInstance({ + spawnerLayer: Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.succeed( + makeProcess({ + exitCode: Effect.never, + stdout: Stream.make(new TextEncoder().encode(stdout)), + }), ), ), - httpClientLayer: layerHttpClient((request) => - Ref.get(status).pipe(Effect.map((code) => responseForRequest(request, code))), - ), - onStartupFailure: (failure) => Queue.offer(reports, failure).pipe(Effect.asVoid), - ...(onReady ? { onReady } : {}), - }).pipe(Effect.tap((instance) => instance.start)); - - const slowStatus = yield* Ref.make(503); - const ready = yield* Deferred.make(); - yield* startInstance(slowStatus, Deferred.succeed(ready, undefined).pipe(Effect.asVoid)); - yield* TestClock.adjust(Duration.seconds(66)); - yield* Ref.set(slowStatus, 200); - yield* TestClock.adjust(Duration.millis(100)); - yield* Deferred.await(ready); - assert.isTrue(Option.isNone(yield* Queue.poll(reports))); - - yield* startInstance(yield* Ref.make(503)); - yield* TestClock.adjust(Duration.seconds(179)); - assert.isTrue(Option.isNone(yield* Queue.poll(reports))); - yield* TestClock.adjust(Duration.seconds(1)); - const failure = yield* Queue.take(reports); - assert.equal(failure.reason, "The server started but did not become ready in time."); - }).pipe(Effect.provide(TestClock.layer())), - ), + ), + httpClientLayer: layerHttpClient((request) => + Ref.get(status).pipe(Effect.map((code) => responseForRequest(request, code))), + ), + onStartupFailure: (failure) => Queue.offer(reports, failure).pipe(Effect.asVoid), + ...(onReady ? { onReady } : {}), + }).pipe(Effect.tap((instance) => instance.start)); + + const slowStatus = yield* Ref.make(503); + const ready = yield* Deferred.make(); + yield* startInstance(slowStatus, Deferred.succeed(ready, undefined).pipe(Effect.asVoid)); + yield* TestClock.adjust(Duration.seconds(66)); + yield* Ref.set(slowStatus, 200); + yield* TestClock.adjust(Duration.millis(100)); + yield* Deferred.await(ready); + assert.isTrue(Option.isNone(yield* Queue.poll(reports))); + + const migratingStatus = yield* Ref.make(503); + const migrated = yield* Deferred.make(); + yield* startInstance( + migratingStatus, + Deferred.succeed(migrated, undefined).pipe(Effect.asVoid), + // Enough progress lines that the start line scrolls out of the output tail. + `INFO ${DATABASE_MIGRATIONS_STARTED_MESSAGE}\n` + + Array.from( + { length: 80 }, + (_, index) => + `INFO ${DATABASE_MIGRATION_PROGRESS_MESSAGE} migration=${index}_Migration progress="${index + 1} of 80"\n`, + ).join(""), + ); + yield* TestClock.adjust(Duration.minutes(10)); + assert.isTrue(Option.isNone(yield* Queue.poll(reports))); + yield* Ref.set(migratingStatus, 200); + yield* TestClock.adjust(Duration.millis(100)); + yield* Deferred.await(migrated); + + yield* startInstance( + yield* Ref.make(503), + undefined, + `${DATABASE_MIGRATIONS_STARTED_MESSAGE}\n${DATABASE_MIGRATIONS_FINISHED_MESSAGE}\n`, + ); + yield* TestClock.adjust(Duration.seconds(179)); + assert.isTrue(Option.isNone(yield* Queue.poll(reports))); + yield* TestClock.adjust(Duration.seconds(1)); + const failure = yield* Queue.take(reports); + assert.equal(failure.reason, "The server started but did not become ready in time."); + }).pipe(Effect.provide(TestClock.layer())), ); it.effect("the quit stop keeps waiting for a backend that outlasts its deadline", () => - Effect.scoped( - Effect.gen(function* () { - const spawned = yield* Deferred.make(); - const teardownStarted = yield* Deferred.make(); - const teardownFinished = yield* Deferred.make(); - const instance = yield* makeTestInstance({ - spawnerLayer: Layer.succeed( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => - Effect.gen(function* () { - const scope = yield* Scope.Scope; - yield* Deferred.succeed(spawned, undefined); - yield* Scope.addFinalizer( - scope, - Deferred.succeed(teardownStarted, undefined).pipe( - Effect.andThen(Effect.sleep(Duration.seconds(10))), - Effect.andThen(Deferred.succeed(teardownFinished, undefined)), - Effect.asVoid, - ), - ); - return makeProcess({ - exitCode: Deferred.await(teardownFinished).pipe( - Effect.as(ChildProcessSpawner.ExitCode(0)), - ), - }); - }), - ), + Effect.gen(function* () { + const spawned = yield* Deferred.make(); + const teardownStarted = yield* Deferred.make(); + const teardownFinished = yield* Deferred.make(); + const instance = yield* makeTestInstance({ + spawnerLayer: Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.gen(function* () { + const scope = yield* Scope.Scope; + yield* Deferred.succeed(spawned, undefined); + yield* Scope.addFinalizer( + scope, + Deferred.succeed(teardownStarted, undefined).pipe( + Effect.andThen(Effect.sleep(Duration.seconds(10))), + Effect.andThen(Deferred.succeed(teardownFinished, undefined)), + Effect.asVoid, + ), + ); + return makeProcess({ + exitCode: Deferred.await(teardownFinished).pipe( + Effect.as(ChildProcessSpawner.ExitCode(0)), + ), + }); + }), ), - httpClientLayer: layerHttpClient(() => Effect.never), - }); - yield* instance.start; - yield* Deferred.await(spawned); - - // App quit and the updater stop every backend with this deadline. - const quitDone = yield* Deferred.make(); - const quitFiber = yield* instance - .stop({ timeout: DesktopBackendManager.BACKEND_QUIT_STOP_TIMEOUT }) - .pipe(Effect.andThen(Deferred.succeed(quitDone, undefined)), Effect.forkChild); - - yield* Deferred.await(teardownStarted); - // Past the quit deadline, quit must still wait for the backend to exit. - yield* TestClock.adjust(Duration.seconds(9)); - assert.isFalse(yield* Deferred.isDone(quitDone)); - - yield* TestClock.adjust(Duration.seconds(1)); - yield* Fiber.join(quitFiber); - assert.isTrue(yield* Deferred.isDone(teardownFinished)); - }).pipe(Effect.provide(TestClock.layer())), - ), + ), + httpClientLayer: layerHttpClient(() => Effect.never), + }); + yield* instance.start; + yield* Deferred.await(spawned); + + // App quit and the updater stop every backend with this deadline. + const quitDone = yield* Deferred.make(); + const quitFiber = yield* instance + .stop({ timeout: DesktopBackendManager.BACKEND_QUIT_STOP_TIMEOUT }) + .pipe(Effect.andThen(Deferred.succeed(quitDone, undefined)), Effect.forkChild); + + yield* Deferred.await(teardownStarted); + // Past the quit deadline, quit must still wait for the backend to exit. + yield* TestClock.adjust(Duration.seconds(9)); + assert.isFalse(yield* Deferred.isDone(quitDone)); + + yield* TestClock.adjust(Duration.seconds(1)); + yield* Fiber.join(quitFiber); + assert.isTrue(yield* Deferred.isDone(teardownFinished)); + }).pipe(Effect.provide(TestClock.layer())), ); }); diff --git a/apps/desktop/src/backend/DesktopBackendManager.ts b/apps/desktop/src/backend/DesktopBackendManager.ts index d24b2f52f295c..4acaa8a3c18b8 100644 --- a/apps/desktop/src/backend/DesktopBackendManager.ts +++ b/apps/desktop/src/backend/DesktopBackendManager.ts @@ -1,5 +1,6 @@ import * as Brand from "effect/Brand"; import * as Cause from "effect/Cause"; +import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; @@ -25,6 +26,7 @@ import { DesktopTelemetryControlMessage, type DesktopTelemetryControlMessage as DesktopTelemetryControlMessageValue, } from "@t3tools/contracts"; +import { isRunningDatabaseMigrations } from "@t3tools/shared/databaseMigrationLog"; import { waitForHttpReady as waitForHttpReadyShared } from "@t3tools/shared/httpReadiness"; import * as DesktopObservability from "../app/DesktopObservability.ts"; @@ -204,7 +206,7 @@ interface RunBackendProcessOptions extends DesktopBackendStartConfig { readonly outputDrainTimeout?: Duration.Duration; readonly onStarted?: (pid: number) => Effect.Effect; readonly onExitObserved?: () => Effect.Effect; - readonly onReady?: () => Effect.Effect; + readonly onReady?: (pid: number) => Effect.Effect; readonly onReadinessFailure?: (error: BackendReadinessTimeoutError) => Effect.Effect; readonly onOutput?: ( streamName: BackendProcessOutputStream, @@ -251,8 +253,8 @@ export interface BackendInstanceSpec { readonly id: BackendInstanceId; readonly label: Effect.Effect; readonly configResolve: Effect.Effect; - /** Receives the ready run's resolved URL; reading current config could race a subsequent restart. */ - readonly onReady?: (httpBaseUrl: URL) => Effect.Effect; + /** Receives the ready run's URL and pid; reading current config could race a subsequent restart. */ + readonly onReady?: (httpBaseUrl: URL, pid: number) => Effect.Effect; readonly onShutdown?: () => Effect.Effect; readonly onPreflightFailed?: (failure: PreflightFailure) => Effect.Effect; readonly onStateDirOwned?: () => Effect.Effect; @@ -483,21 +485,23 @@ export const runBackendProcess = Effect.fn("runBackendProcess")(function* ( ), ); const outputFibers: Array> = []; + const serverReady = yield* Deferred.make(); yield* options.onStarted?.(handle.pid) ?? Effect.void; - if ( - options.bootstrap.desktopTelemetryControlFd !== undefined && - options.onDesktopTelemetryControl !== undefined - ) { + if (options.bootstrap.desktopTelemetryControlFd !== undefined) { const controlFd = options.bootstrap.desktopTelemetryControlFd; - const handleControl = options.onDesktopTelemetryControl; + const handleControl = options.onDesktopTelemetryControl ?? (() => Effect.void); yield* handle.getOutputFd(controlFd).pipe( Stream.decodeText(), Stream.splitLines, Stream.filter((line) => line.trim().length > 0), Stream.runForEach((line) => decodeDesktopTelemetryControlLine(line).pipe( - Effect.flatMap(handleControl), + Effect.flatMap((message) => + message.type === "serverReady" + ? Deferred.succeed(serverReady, undefined) + : handleControl(message), + ), Effect.catchCause((cause) => logBackendProcessWarning("ignored invalid desktop telemetry control message", { fd: controlFd, @@ -569,15 +573,19 @@ export const runBackendProcess = Effect.fn("runBackendProcess")(function* ( ); } // Cold WSL boots across /mnt/c can exceed one probe budget; retry while the backend is alive. + // HTTP polling stays as the fallback for backends without the control fd, such as WSL. const probeReadiness = Effect.fn("desktop.backendProcess.probeReadiness")(() => - waitForHttpReady({ - executablePath: options.executablePath, - entryPath: options.entryPath, - cwd: options.cwd, - httpBaseUrl: options.httpBaseUrl, - timeout: options.readinessTimeout ?? DEFAULT_BACKEND_READINESS_TIMEOUT, - }).pipe( - Effect.flatMap(() => options.onReady?.() ?? Effect.void), + Effect.raceFirst( + waitForHttpReady({ + executablePath: options.executablePath, + entryPath: options.entryPath, + cwd: options.cwd, + httpBaseUrl: options.httpBaseUrl, + timeout: options.readinessTimeout ?? DEFAULT_BACKEND_READINESS_TIMEOUT, + }), + Deferred.await(serverReady), + ).pipe( + Effect.flatMap(() => options.onReady?.(handle.pid) ?? Effect.void), Effect.as(true), Effect.catchTags({ BackendReadinessTimeoutError: (error) => @@ -938,7 +946,7 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( ...run, exitObserved: true, })), - onReady: Effect.fn("desktop.backendInstance.onReady")(function* () { + onReady: Effect.fn("desktop.backendInstance.onReady")(function* (pid) { const isCurrentRun = yield* SubscriptionRef.modify(state, (latest) => { const activeRun = Option.getOrUndefined(latest.active); if (activeRun?.id !== runId) { @@ -958,7 +966,7 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( return; } - yield* spec.onReady?.(config.value.httpBaseUrl) ?? Effect.void; + yield* spec.onReady?.(config.value.httpBaseUrl, pid) ?? Effect.void; if ( config.value.runningDistro !== undefined && config.value.wslRuntimeId !== undefined @@ -971,6 +979,12 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( }), onReadinessFailure: Effect.fn("desktop.backendInstance.onReadinessFailure")( function* (error) { + if (isRunningDatabaseMigrations(yield* Ref.get(outputTail))) { + yield* logInstanceWarning("backend is still migrating its database; waiting", { + error: error.message, + }); + return; + } yield* logInstanceWarning("backend readiness check failed during bootstrap", { error: error.message, }); diff --git a/apps/desktop/src/backend/DesktopBackendPool.test.ts b/apps/desktop/src/backend/DesktopBackendPool.test.ts index c46e0b3a26078..8e9f4320c08c7 100644 --- a/apps/desktop/src/backend/DesktopBackendPool.test.ts +++ b/apps/desktop/src/backend/DesktopBackendPool.test.ts @@ -1,5 +1,5 @@ -import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, describe, it } from "@effect/vitest"; +import { DESKTOP_UPDATE_IN_PROGRESS_TTL_MS } from "@t3tools/contracts"; import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; @@ -110,27 +110,52 @@ const runningConfig = { function layerPool( labelRef: Ref.Ref, - backend?: Effect.Success>, + options: { + readonly config?: DesktopBackendStartConfig; + readonly fileSystem?: Partial; + readonly spawner?: ChildProcessSpawner.ChildProcessSpawner["Service"]; + readonly httpClient?: HttpClient.HttpClient; + readonly onReady?: Effect.Effect; + readonly backend?: Effect.Success>; + } = {}, ): Layer.Layer { return DesktopBackendPool.layer.pipe( Layer.provideMerge( Layer.mergeAll( - FileSystem.layerNoop({ exists: () => Effect.succeed(true) }), + DesktopEnvironment.layer({ + dirname: "/repo/apps/desktop/src", + homeDirectory: "/test-home", + platform: "linux", + processArch: "x64", + appVersion: "1.2.3", + appPath: "/repo", + isPackaged: true, + resourcesPath: "/missing/resources", + runningUnderArm64Translation: false, + }).pipe( + Layer.provide( + Layer.mergeAll(Path.layer, DesktopConfig.layerTest({ T3CODE_HOME: "/test-home" })), + ), + Layer.orDie, + ), + FileSystem.layerNoop(options.fileSystem ?? { exists: () => Effect.succeed(true) }), Path.layer, - backend?.layer ?? + options.backend?.layer ?? Layer.succeed( ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => Effect.die("unexpected child process spawn")), + options.spawner ?? + ChildProcessSpawner.make(() => Effect.die("unexpected child process spawn")), ), Layer.succeed( HttpClient.HttpClient, - HttpClient.make((request) => - backend === undefined - ? Effect.die("unexpected HTTP request") - : Effect.succeed( - HttpClientResponse.fromWeb(request, new Response(null, { status: 200 })), - ), - ), + options.httpClient ?? + HttpClient.make((request) => + options.backend === undefined + ? Effect.die("unexpected HTTP request") + : Effect.succeed( + HttpClientResponse.fromWeb(request, new Response(null, { status: 200 })), + ), + ), ), Layer.succeed(DesktopObservability.DesktopBackendOutputLogFactory, { forInstance: () => @@ -155,26 +180,14 @@ function layerPool( }), DesktopBrowserHost.layer, Layer.succeed(DesktopBackendConfiguration.DesktopBackendConfiguration, { - resolvePrimary: Effect.die("unexpected primary config resolve"), + resolvePrimary: options.config + ? Effect.succeed(options.config) + : Effect.die("unexpected primary config resolve"), resolvePrimaryLabel: Ref.get(labelRef), resolveWsl: () => Effect.die("unexpected WSL config resolve"), currentBootstrapToken: Effect.die("unexpected bootstrap token read"), } satisfies DesktopBackendConfiguration.DesktopBackendConfiguration["Service"]), DesktopAppSettings.layerTest(), - DesktopEnvironment.layer({ - dirname: "/repo/apps/desktop/src", - homeDirectory: "/home/test", - platform: "linux", - processArch: "x64", - appVersion: "0.0.0", - appPath: "/repo", - isPackaged: true, - resourcesPath: "/missing/resources", - runningUnderArm64Translation: false, - }).pipe( - Layer.provide(Layer.mergeAll(NodeServices.layer, DesktopConfig.layerTest({}))), - Layer.orDie, - ), DesktopState.layer, DesktopWslEnvironment.layerTest(), ElectronDialog.layer, @@ -186,7 +199,7 @@ function layerPool( activate: Effect.die("unexpected window activate"), createMainIfBackendReady: Effect.die("unexpected window create"), showConnectingSplash: Effect.void, - handleBackendReady: () => Effect.void, + handleBackendReady: () => options.onReady ?? Effect.void, handleBackendNotReady: Effect.void, flushMainWindowBounds: Effect.void, prepareCaptureReveal: Effect.void, @@ -201,86 +214,268 @@ function layerPool( ); } +const markerPath = "/test-home/runtime/desktop-update-in-progress"; +const runtimePath = "/test-home/userdata/server-runtime.json"; + +const makeUpdateMarkerFixture = () => + Effect.gen(function* () { + const allowReady = yield* Deferred.make(); + const runtimeChecked = yield* Deferred.make(); + const markerRemoved = yield* Deferred.make(); + const readyRuns = yield* Queue.unbounded(); + let nextPid = 123; + const files = new Map([[markerPath, JSON.stringify({ startedAt: 0, expiresAt: 60_000 })]]); + const removed: string[] = []; + const config: DesktopBackendStartConfig = { + executablePath: "/electron", + args: [], + entryPath: "/server/bin.mjs", + cwd: "/server", + env: {}, + bootstrap: { + mode: "desktop", + noBrowser: true, + port: 3774, + t3Home: "/test-home", + host: "127.0.0.1", + desktopBootstrapToken: "token", + tailscaleServeEnabled: false, + tailscaleServePort: 443, + }, + bootstrapDelivery: "fd3", + extendEnv: true, + httpBaseUrl: new URL("http://127.0.0.1:3774"), + captureOutput: false, + preflightFailure: Option.none(), + }; + const layer = layerPool(Ref.makeUnsafe("Local"), { + config, + fileSystem: { + exists: (path) => Effect.succeed(files.has(path) || path === config.entryPath), + readFileString: (path) => + (path === runtimePath ? Deferred.succeed(runtimeChecked, undefined) : Effect.void).pipe( + Effect.andThen(Effect.sync(() => files.get(path) ?? "")), + ), + remove: (path) => + Effect.sync(() => { + removed.push(path); + files.delete(path); + }).pipe( + Effect.andThen( + path === markerPath ? Deferred.succeed(markerRemoved, undefined) : Effect.void, + ), + Effect.asVoid, + ), + }, + spawner: ChildProcessSpawner.make(() => + Effect.gen(function* () { + const exited = yield* Deferred.make(); + const exit = Deferred.succeed(exited, ChildProcessSpawner.ExitCode(0)).pipe( + Effect.asVoid, + ); + // Like the real spawner, closing the run's scope ends the process. + yield* Effect.addFinalizer(() => exit); + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(nextPid++), + stdout: Stream.empty, + stderr: Stream.empty, + all: Stream.empty, + exitCode: Deferred.await(exited), + isRunning: Effect.succeed(true), + kill: () => exit, + stdin: Sink.drain, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }); + }), + ), + httpClient: HttpClient.make((request) => + Deferred.await(allowReady).pipe( + Effect.as(HttpClientResponse.fromWeb(request, new Response(null, { status: 200 }))), + ), + ), + onReady: Queue.offer(readyRuns, undefined).pipe(Effect.asVoid), + }); + const publishRuntime = (pid: number) => + files.set( + runtimePath, + JSON.stringify({ + version: 1, + pid, + port: 3774, + origin: "http://127.0.0.1:3774", + startedAt: "2026-10-27T00:00:00.000Z", + }), + ); + return { + layer, + files, + removed, + allowReady, + readyRuns, + runtimeChecked, + markerRemoved, + publishRuntime, + }; + }); + describe("DesktopBackendPool", () => { - it.effect("unregister during pool quit does not interrupt sibling stops", () => - Effect.scoped( + it.effect( + "keeps the update marker after HTTP readiness until the backend publishes its runtime", + () => Effect.gen(function* () { - const backend = yield* makeFakeBackendProcess(); + const fixture = yield* makeUpdateMarkerFixture(); yield* Effect.gen(function* () { const pool = yield* DesktopBackendPool.DesktopBackendPool; - const id = DesktopBackendPool.BackendInstanceId("wsl:Ubuntu"); - const ready = yield* Deferred.make(); - const siblingStarted = yield* Deferred.make(); - const finishSibling = yield* Deferred.make(); - let siblingCompleted = false; - const instance = yield* pool.register({ - id, - label: Effect.succeed("WSL (Ubuntu)"), - configResolve: Effect.succeed(runningConfig), - onReady: () => Deferred.succeed(ready, undefined).pipe(Effect.asVoid), - }); - yield* instance.start; - yield* backend.started; - yield* Deferred.await(ready); - const quitFiber = yield* Effect.all( - [ - instance.stop({ timeout: Duration.seconds(5) }), - Deferred.succeed(siblingStarted, undefined).pipe( - Effect.andThen(Deferred.await(finishSibling)), - Effect.andThen( - Effect.sync(() => { - siblingCompleted = true; - }), - ), - ), - ], - { concurrency: "unbounded" }, - ).pipe(Effect.forkChild); - yield* Deferred.await(siblingStarted); - assert.equal(yield* Queue.take(backend.signals), "SIGTERM"); - const unregisterFiber = yield* pool.unregister(id).pipe(Effect.forkChild); + const primary = yield* pool.primary; + yield* primary.start; + yield* Deferred.succeed(fixture.allowReady, undefined); + yield* Queue.take(fixture.readyRuns); + assert.isTrue( + fixture.files.has(markerPath), + "a launcher could neither wait nor adopt between readiness and publication", + ); + yield* Deferred.await(fixture.runtimeChecked); yield* TestClock.adjust(Duration.seconds(5)); - assert.equal(yield* Queue.take(backend.signals), "SIGKILL"); - assert.isUndefined(quitFiber.pollUnsafe()); - assert.isUndefined(unregisterFiber.pollUnsafe()); - assert.isTrue(yield* backend.isRunning); - yield* TestClock.adjust(backend.killWait); - yield* Fiber.join(unregisterFiber); - yield* Deferred.succeed(finishSibling, undefined); - yield* Fiber.join(quitFiber); - - assert.isTrue(siblingCompleted); - assert.isFalse(yield* backend.isRunning); - assert.isTrue(Option.isNone((yield* instance.snapshot).activePid)); - assert.isTrue(Option.isNone(yield* pool.get(id))); - }).pipe(Effect.provide(layerPool(Ref.makeUnsafe("Windows"), backend))); + assert.isTrue(fixture.files.has(markerPath)); + fixture.publishRuntime(456); + yield* TestClock.adjust(Duration.seconds(1)); + assert.isTrue( + fixture.files.has(markerPath), + "another process's runtime is not the replacement", + ); + fixture.publishRuntime(123); + yield* TestClock.adjust(Duration.seconds(1)); + yield* Deferred.await(fixture.markerRemoved); + assert.deepEqual(fixture.removed, [markerPath]); + }).pipe(Effect.provide(fixture.layer)); }), - ), ); - it.effect("unregister stops the registered instance", () => - Effect.scoped( - Effect.gen(function* () { - const id = DesktopBackendPool.BackendInstanceId("wsl:Ubuntu"); + it.effect("leaves a replacement update marker written while an earlier wait is pending", () => + Effect.gen(function* () { + const fixture = yield* makeUpdateMarkerFixture(); + yield* Effect.gen(function* () { + const pool = yield* DesktopBackendPool.DesktopBackendPool; + const primary = yield* pool.primary; + yield* primary.start; + yield* Deferred.succeed(fixture.allowReady, undefined); + yield* Queue.take(fixture.readyRuns); + yield* Deferred.await(fixture.runtimeChecked); + yield* TestClock.adjust(Duration.millis(DESKTOP_UPDATE_IN_PROGRESS_TTL_MS - 1_000)); + const replacement = JSON.stringify({ startedAt: 59_000, expiresAt: 119_000 }); + fixture.files.set(markerPath, replacement); + yield* TestClock.adjust(Duration.millis(DESKTOP_UPDATE_IN_PROGRESS_TTL_MS)); + assert.equal(fixture.files.get(markerPath), replacement); + assert.deepEqual(fixture.removed, []); + }).pipe(Effect.provide(fixture.layer)); + }), + ); + + it.effect("a later backend run's readiness supersedes the earlier run's marker wait", () => + Effect.gen(function* () { + const fixture = yield* makeUpdateMarkerFixture(); + yield* Effect.gen(function* () { const pool = yield* DesktopBackendPool.DesktopBackendPool; + const primary = yield* pool.primary; + yield* primary.start; + yield* Deferred.succeed(fixture.allowReady, undefined); + yield* Queue.take(fixture.readyRuns); + yield* Deferred.await(fixture.runtimeChecked); + yield* TestClock.adjust(Duration.seconds(30)); + yield* primary.stop(); + yield* primary.start; + yield* Queue.take(fixture.readyRuns); + yield* TestClock.adjust(Duration.seconds(31)); + assert.isTrue( + fixture.files.has(markerPath), + "the earlier run's wait must not end the later run's handoff", + ); + assert.deepEqual(fixture.removed, []); + fixture.publishRuntime(124); + yield* TestClock.adjust(Duration.millis(100)); + yield* Deferred.await(fixture.markerRemoved); + assert.deepEqual(fixture.removed, [markerPath]); + }).pipe(Effect.provide(fixture.layer)); + }), + ); + + it.effect("unregister during pool quit does not interrupt sibling stops", () => + Effect.gen(function* () { + const backend = yield* makeFakeBackendProcess(); + yield* Effect.gen(function* () { + const pool = yield* DesktopBackendPool.DesktopBackendPool; + const id = DesktopBackendPool.BackendInstanceId("wsl:Ubuntu"); + const ready = yield* Deferred.make(); + const siblingStarted = yield* Deferred.make(); + const finishSibling = yield* Deferred.make(); + let siblingCompleted = false; const instance = yield* pool.register({ id, label: Effect.succeed("WSL (Ubuntu)"), - configResolve: Effect.succeed({ - ...runningConfig, - preflightFailure: Option.some({ reason: "WSL starting", fatal: false }), - }), + configResolve: Effect.succeed(runningConfig), + onReady: () => Deferred.succeed(ready, undefined).pipe(Effect.asVoid), }); yield* instance.start; - assert.equal((yield* instance.snapshot).restartScheduled, true); + yield* backend.started; + yield* Deferred.await(ready); + const quitFiber = yield* Effect.all( + [ + instance.stop({ timeout: Duration.seconds(5) }), + Deferred.succeed(siblingStarted, undefined).pipe( + Effect.andThen(Deferred.await(finishSibling)), + Effect.andThen( + Effect.sync(() => { + siblingCompleted = true; + }), + ), + ), + ], + { concurrency: "unbounded" }, + ).pipe(Effect.forkChild); + yield* Deferred.await(siblingStarted); + assert.equal(yield* Queue.take(backend.signals), "SIGTERM"); + const unregisterFiber = yield* pool.unregister(id).pipe(Effect.forkChild); + yield* TestClock.adjust(Duration.seconds(5)); + assert.equal(yield* Queue.take(backend.signals), "SIGKILL"); + assert.isUndefined(quitFiber.pollUnsafe()); + assert.isUndefined(unregisterFiber.pollUnsafe()); + assert.isTrue(yield* backend.isRunning); + yield* TestClock.adjust(backend.killWait); + yield* Fiber.join(unregisterFiber); + yield* Deferred.succeed(finishSibling, undefined); + yield* Fiber.join(quitFiber); - yield* pool.unregister(id); + assert.isTrue(siblingCompleted); + assert.isFalse(yield* backend.isRunning); + assert.isTrue(Option.isNone((yield* instance.snapshot).activePid)); + assert.isTrue(Option.isNone(yield* pool.get(id))); + }).pipe(Effect.provide(layerPool(Ref.makeUnsafe("Windows"), { backend }))); + }), + ); - const stopped = yield* instance.snapshot; - assert.equal(stopped.desiredRunning, false); - assert.equal(stopped.restartScheduled, false); - assert.equal(Option.isNone(yield* pool.get(id)), true); - }).pipe(Effect.provide(layerPool(Ref.makeUnsafe("Windows")))), - ), + it.effect("unregister stops the registered instance", () => + Effect.gen(function* () { + const id = DesktopBackendPool.BackendInstanceId("wsl:Ubuntu"); + const pool = yield* DesktopBackendPool.DesktopBackendPool; + const instance = yield* pool.register({ + id, + label: Effect.succeed("WSL (Ubuntu)"), + configResolve: Effect.succeed({ + ...runningConfig, + preflightFailure: Option.some({ reason: "WSL starting", fatal: false }), + }), + }); + yield* instance.start; + assert.equal((yield* instance.snapshot).restartScheduled, true); + + yield* pool.unregister(id); + + const stopped = yield* instance.snapshot; + assert.equal(stopped.desiredRunning, false); + assert.equal(stopped.restartScheduled, false); + assert.equal(Option.isNone(yield* pool.get(id)), true); + }).pipe(Effect.provide(layerPool(Ref.makeUnsafe("Windows")))), ); }); diff --git a/apps/desktop/src/backend/DesktopBackendPool.ts b/apps/desktop/src/backend/DesktopBackendPool.ts index 82d542b322c84..d69e1ce1427fe 100644 --- a/apps/desktop/src/backend/DesktopBackendPool.ts +++ b/apps/desktop/src/backend/DesktopBackendPool.ts @@ -1,10 +1,17 @@ +import { + DESKTOP_UPDATE_IN_PROGRESS_MARKER_FILE, + DESKTOP_UPDATE_IN_PROGRESS_TTL_MS, +} from "@t3tools/contracts"; import * as Context from "effect/Context"; import * as Deferred from "effect/Deferred"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; +import * as FiberHandle from "effect/FiberHandle"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import * as Schedule from "effect/Schedule"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as SubscriptionRef from "effect/SubscriptionRef"; @@ -31,6 +38,10 @@ import * as ElectronApp from "../electron/ElectronApp.ts"; const { logWarning: logBackendPoolWarning } = DesktopObservability.makeComponentLogger("desktop-backend-pool"); +const decodeServerRuntime = Schema.decodeUnknownEffect( + Schema.fromJsonString(Schema.Struct({ pid: Schema.Number, port: Schema.Number })), +); + export type BackendInstanceId = DesktopBackendManager.BackendInstanceId; export const BackendInstanceId = DesktopBackendManager.BackendInstanceId; export const PRIMARY_INSTANCE_ID = DesktopBackendManager.PRIMARY_INSTANCE_ID; @@ -106,6 +117,8 @@ type UnregisterAction = export const layer = Layer.effect( DesktopBackendPool, Effect.gen(function* () { + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const fileSystem = yield* FileSystem.FileSystem; const configuration = yield* DesktopBackendConfiguration.DesktopBackendConfiguration; const desktopWindow = yield* DesktopWindow.DesktopWindow; const electronDialog = yield* ElectronDialog.ElectronDialog; @@ -119,6 +132,41 @@ export const layer = Layer.effect( FileSystem.FileSystem | Path.Path | DesktopEnvironment.DesktopEnvironment >(); + const updateMarkerPath = environment.path.join( + environment.baseDir, + "runtime", + DESKTOP_UPDATE_IN_PROGRESS_MARKER_FILE, + ); + const serverRuntimePath = environment.path.join(environment.stateDir, "server-runtime.json"); + // The server publishes its runtime after HTTP readiness. Until then an SSH launcher + // finds nothing to adopt, so dropping the marker earlier lets it start a competing server. + const releaseUpdateMarker = Effect.fn("desktop.backendPool.releaseUpdateMarker")(function* ( + httpBaseUrl: URL, + pid: number, + ) { + const marker = yield* fileSystem.readFileString(updateMarkerPath).pipe(Effect.option); + if (Option.isNone(marker)) { + return; + } + yield* fileSystem.readFileString(serverRuntimePath).pipe( + Effect.flatMap(decodeServerRuntime), + Effect.map((runtime) => runtime.pid === pid && runtime.port === Number(httpBaseUrl.port)), + Effect.orElseSucceed(() => false), + Effect.repeat({ + until: (published) => published, + schedule: Schedule.spaced(Duration.millis(100)), + }), + Effect.timeoutOption(Duration.millis(DESKTOP_UPDATE_IN_PROGRESS_TTL_MS)), + ); + // A later update may have replaced the marker; that handoff is not ours to end. + const current = yield* fileSystem.readFileString(updateMarkerPath).pipe(Effect.option); + if (Option.isSome(current) && current.value === marker.value) { + yield* fileSystem.remove(updateMarkerPath, { force: true }).pipe(Effect.ignore); + } + }); + // One wait at a time: each run's shutdown or readiness ends the previous run's wait. + const updateMarkerWait = yield* FiberHandle.make(); + // Persist fatal Windows fallbacks; transient transport failures must preserve the user's WSL preference. const handlePrimaryPreflightFailure = Effect.fn("desktop.backendPool.primaryPreflightFailed")( function* (failure: DesktopBackendManager.PreflightFailure) { @@ -165,8 +213,9 @@ export const layer = Layer.effect( label: configuration.resolvePrimaryLabel, configResolve: configuration.resolvePrimary, // Absorb window errors so readiness can reset restartAttempt. - onReady: (httpBaseUrl) => - desktopWindow.handleBackendReady(httpBaseUrl).pipe( + onReady: (httpBaseUrl, pid) => + FiberHandle.run(updateMarkerWait, releaseUpdateMarker(httpBaseUrl, pid)).pipe( + Effect.andThen(desktopWindow.handleBackendReady(httpBaseUrl)), // Only once the main window exists, so the recovery window never closes into nothing. Effect.andThen(SubscriptionRef.set(primaryBackendStartup, { _tag: "Ready" })), Effect.catch((error) => @@ -177,7 +226,10 @@ export const layer = Layer.effect( ), onStartupFailure: (failure) => SubscriptionRef.set(primaryBackendStartup, { _tag: "Failing", ...failure }), - onShutdown: () => desktopWindow.handleBackendNotReady, + onShutdown: () => + FiberHandle.clear(updateMarkerWait).pipe( + Effect.andThen(desktopWindow.handleBackendNotReady), + ), onPreflightFailed: handlePrimaryPreflightFailure, onStateDirOwned: () => DesktopHomeOwner.readLiveHomeOwner().pipe( diff --git a/apps/desktop/src/backend/DesktopLocalEnvironmentAuth.test.ts b/apps/desktop/src/backend/DesktopLocalEnvironmentAuth.test.ts index e82e3afea7303..c15ce3932bb80 100644 --- a/apps/desktop/src/backend/DesktopLocalEnvironmentAuth.test.ts +++ b/apps/desktop/src/backend/DesktopLocalEnvironmentAuth.test.ts @@ -1,11 +1,13 @@ import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import { assert, describe, it } from "@effect/vitest"; +import * as Clock from "effect/Clock"; import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Ref from "effect/Ref"; +import * as Result from "effect/Result"; import * as HttpClient from "effect/http/HttpClient"; import * as HttpClientRequest from "effect/http/HttpClientRequest"; import * as HttpClientResponse from "effect/http/HttpClientResponse"; @@ -238,4 +240,34 @@ describe("DesktopLocalEnvironmentAuth", () => { assert.strictEqual(yield* Ref.get(requestCount), 1); }).pipe(Effect.provide(TestClock.layer())), ); + + it.effect("waits through a backend that stalls for 30 seconds after it is ready", () => + Effect.gen(function* () { + const requestCount = yield* Ref.make(0); + const layerHttpClient = Layer.succeed( + HttpClient.HttpClient, + HttpClient.make((request) => + Effect.gen(function* () { + yield* Ref.update(requestCount, (count) => count + 1); + const now = yield* Clock.currentTimeMillis; + return now < 30_000 ? yield* Effect.never : tokenResponse(request); + }), + ), + ); + const auth = yield* DesktopLocalEnvironmentAuth.DesktopLocalEnvironmentAuth.pipe( + Effect.provide( + DesktopLocalEnvironmentAuth.layer.pipe( + Layer.provide(Layer.mergeAll(layerPool, layerHttpClient, NodeCrypto.layerMac)), + ), + ), + ); + + const fiber = yield* auth.getBearerToken.pipe(Effect.result, Effect.forkChild); + yield* TestClock.adjust(Duration.seconds(45)); + const result = yield* Fiber.join(fiber); + + assert.isTrue(Result.isSuccess(result)); + assert.strictEqual(yield* Ref.get(requestCount), 4); + }).pipe(Effect.provide(TestClock.layer())), + ); }); diff --git a/apps/desktop/src/backend/DesktopLocalEnvironmentAuth.ts b/apps/desktop/src/backend/DesktopLocalEnvironmentAuth.ts index 6c7d24623d40f..89aa3cd2a678c 100644 --- a/apps/desktop/src/backend/DesktopLocalEnvironmentAuth.ts +++ b/apps/desktop/src/backend/DesktopLocalEnvironmentAuth.ts @@ -17,7 +17,8 @@ import * as HttpClient from "effect/http/HttpClient"; import * as DesktopBackendPool from "./DesktopBackendPool.ts"; -const BOOTSTRAP_TRANSIENT_RETRY_TIMEOUT = Duration.seconds(15); +// Each /oauth/token attempt may take 10 s, so a backend busy after ready needs several attempts. +const BOOTSTRAP_TRANSIENT_RETRY_TIMEOUT = Duration.seconds(60); const BOOTSTRAP_TRANSIENT_RETRY_INTERVAL = Duration.millis(500); const TRANSIENT_BOOTSTRAP_STATUS_CODES = new Set([502, 503, 504]); diff --git a/apps/desktop/src/electron/ElectronMenu.ts b/apps/desktop/src/electron/ElectronMenu.ts index 4ff3df6bf10f0..540326346edf8 100644 --- a/apps/desktop/src/electron/ElectronMenu.ts +++ b/apps/desktop/src/electron/ElectronMenu.ts @@ -1,4 +1,4 @@ -import type { ContextMenuItem } from "@t3tools/contracts"; +import type { ContextMenuItem, ContextMenuSource } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; @@ -17,6 +17,7 @@ export interface ElectronMenuContextInput { readonly window: Electron.BrowserWindow; readonly items: readonly ContextMenuItem[]; readonly position: Option.Option; + readonly source: Option.Option; } export interface ElectronMenuTemplateInput { @@ -259,7 +260,11 @@ export const make = Effect.gen(function* () { callback: () => complete(Option.none()), }), }); - menu.popup(popupOptions); + menu.popup( + Option.isSome(input.source) + ? { ...popupOptions, sourceType: input.source.value } + : popupOptions, + ); } catch (cause) { if (completed) { return; diff --git a/apps/desktop/src/electron/ElectronProtocol.test.ts b/apps/desktop/src/electron/ElectronProtocol.test.ts index aec3a2d5b45c0..b0cf4823126e2 100644 --- a/apps/desktop/src/electron/ElectronProtocol.test.ts +++ b/apps/desktop/src/electron/ElectronProtocol.test.ts @@ -63,7 +63,7 @@ describe("ElectronProtocol", () => { assert.equal((yield* request("/%invalid")).status, 400); assert.equal((yield* request("/", { method: "POST" })).status, 405); assert.equal(netFetchMock.mock.calls.length, 0); - }).pipe(Effect.provide(Layer.merge(layerProtocol, NodeServices.layer)), Effect.scoped), + }).pipe(Effect.provide(Layer.merge(layerProtocol, NodeServices.layer))), ); it.effect("proxies the stable renderer origin to the current app server", () => diff --git a/apps/desktop/src/electron/ElectronTheme.test.ts b/apps/desktop/src/electron/ElectronTheme.test.ts index 4239bfe4c5bca..47aa0ce1f8df8 100644 --- a/apps/desktop/src/electron/ElectronTheme.test.ts +++ b/apps/desktop/src/electron/ElectronTheme.test.ts @@ -42,7 +42,7 @@ describe("ElectronTheme", () => { assert.equal(nativeTheme.themeSource, "light"); portal.report(null); assert.equal(nativeTheme.themeSource, "system"); - }).pipe(Effect.scoped), + }), ); it.effect("keeps an explicit appearance over the desktop color scheme", () => @@ -57,7 +57,7 @@ describe("ElectronTheme", () => { yield* theme.setSource("system"); assert.equal(nativeTheme.themeSource, "dark"); - }).pipe(Effect.scoped), + }), ); it.effect("stops following the desktop when the layer is released", () => @@ -76,6 +76,6 @@ describe("ElectronTheme", () => { }); yield* theme.setSource("system"); assert.equal(nativeTheme.themeSource, "system"); - }).pipe(Effect.scoped), + }), ); }); diff --git a/apps/desktop/src/electron/ElectronWindow.test.ts b/apps/desktop/src/electron/ElectronWindow.test.ts index 9eb41ed4e2402..eac944722e73b 100644 --- a/apps/desktop/src/electron/ElectronWindow.test.ts +++ b/apps/desktop/src/electron/ElectronWindow.test.ts @@ -148,6 +148,34 @@ describe("ElectronWindow", () => { }).pipe(Effect.provide(layerTestFor("win32"))), ); + it.effect("reports which windows were created with a title bar overlay", () => + Effect.gen(function* () { + const framed = { isDestroyed: () => false }; + const overlay = { isDestroyed: () => false }; + browserWindowMock + .mockImplementationOnce(function FramedWindowMock() { + return framed; + }) + .mockImplementationOnce(function OverlayWindowMock() { + return overlay; + }); + const electronWindow = yield* ElectronWindow.ElectronWindow; + yield* electronWindow.create({}); + yield* electronWindow.create({ + titleBarStyle: "hidden", + titleBarOverlay: { color: "#01000000", height: 40 }, + }); + getAllWindowsMock.mockReturnValue([framed, overlay, { isDestroyed: () => false }]); + + const seen: Array = []; + yield* electronWindow.syncAllAppearance((_window, { titleBarOverlay }) => + Effect.sync(() => seen.push(titleBarOverlay)), + ); + + assert.deepEqual(seen, [false, true, false]); + }).pipe(Effect.provide(layerTestFor("win32"))), + ); + it.effect("preserves destroy failures and continues with later windows", () => Effect.gen(function* () { const cause = new Error("window destroy failed"); diff --git a/apps/desktop/src/electron/ElectronWindow.ts b/apps/desktop/src/electron/ElectronWindow.ts index 15a117a52b050..b5f89969e66c7 100644 --- a/apps/desktop/src/electron/ElectronWindow.ts +++ b/apps/desktop/src/electron/ElectronWindow.ts @@ -125,7 +125,10 @@ export class ElectronWindow extends Context.Service< readonly sendAll: (channel: string, ...args: readonly unknown[]) => Effect.Effect; readonly destroyAll: Effect.Effect; readonly syncAllAppearance: ( - sync: (window: Electron.BrowserWindow) => Effect.Effect, + sync: ( + window: Electron.BrowserWindow, + appearance: { readonly titleBarOverlay: boolean }, + ) => Effect.Effect, ) => Effect.Effect; } >()("@t3tools/desktop/electron/ElectronWindow") {} @@ -145,6 +148,7 @@ export const make = Effect.gen(function* () { yield* Effect.addFinalizer(() => Effect.sync(() => windowsForegroundFocus?.close())); const mainWindowRef = yield* Ref.make>(Option.none()); const auxiliaryWindows = new WeakSet(); + const titleBarOverlayWindows = new WeakSet(); const listWindows = Effect.try({ try: () => Electron.BrowserWindow.getAllWindows(), @@ -241,6 +245,9 @@ export const make = Effect.gen(function* () { try: () => { const window = new Electron.BrowserWindow(options); if (behavior?.auxiliary === true) auxiliaryWindows.add(window); + if (options.titleBarOverlay !== undefined && options.titleBarOverlay !== false) { + titleBarOverlayWindows.add(window); + } return window; }, catch: (cause) => new ElectronWindowCreateError({ options: diagnosticOptions, cause }), @@ -383,14 +390,17 @@ export const make = Effect.gen(function* () { } }), syncAllAppearance: Effect.fn("desktop.electron.window.syncAllAppearance")(function* ( - sync: (window: Electron.BrowserWindow) => Effect.Effect, + sync: ( + window: Electron.BrowserWindow, + appearance: { readonly titleBarOverlay: boolean }, + ) => Effect.Effect, ) { const windows = yield* listWindows; for (const window of windows) { if (yield* isWindowDestroyed(window)) { continue; } - yield* sync(window); + yield* sync(window, { titleBarOverlay: titleBarOverlayWindows.has(window) }); } }), }); diff --git a/apps/desktop/src/ipc/DesktopIpc.test.ts b/apps/desktop/src/ipc/DesktopIpc.test.ts index 45f63b951f366..1973ab49cefeb 100644 --- a/apps/desktop/src/ipc/DesktopIpc.test.ts +++ b/apps/desktop/src/ipc/DesktopIpc.test.ts @@ -63,5 +63,5 @@ it.effect("answers only the main window's top frame", () => appSubframe: ["rejected", undefined], previewGuest: ["rejected", undefined], }); - }).pipe(Effect.scoped), + }), ); diff --git a/apps/desktop/src/ipc/DesktopIpcHandlers.ts b/apps/desktop/src/ipc/DesktopIpcHandlers.ts index 9f2f68be61520..57d7101cf9ec7 100644 --- a/apps/desktop/src/ipc/DesktopIpcHandlers.ts +++ b/apps/desktop/src/ipc/DesktopIpcHandlers.ts @@ -50,6 +50,7 @@ import { probeRemoteEditors, revealWindow, pickFolder, + toLocalBackendPath, pickProjectFavicon, pickThemeFiles, setTheme, @@ -145,6 +146,7 @@ export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers" yield* ipc.handle(setWslOnly); yield* ipc.handle(pickFolder); + yield* ipc.handle(toLocalBackendPath); yield* ipc.handle(pickProjectFavicon); yield* ipc.handle(pickThemeFiles); yield* ipc.handle(setTheme); diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts index 80053c78caee4..4de0a23af35b3 100644 --- a/apps/desktop/src/ipc/channels.ts +++ b/apps/desktop/src/ipc/channels.ts @@ -1,4 +1,5 @@ export const PICK_FOLDER_CHANNEL = "desktop:pick-folder"; +export const TO_LOCAL_BACKEND_PATH_CHANNEL = "desktop:to-local-backend-path"; export const SET_NOTIFICATION_BADGE_CHANNEL = "desktop:set-notification-badge"; export const PICK_PROJECT_FAVICON_CHANNEL = "desktop:pick-project-favicon"; export const PICK_THEME_FILES_CHANNEL = "desktop:pick-theme-files"; @@ -88,6 +89,7 @@ export const PREVIEW_REGISTER_WEBVIEW_CHANNEL = "desktop:preview-register-webvie export const PREVIEW_NAVIGATE_CHANNEL = "desktop:preview-navigate"; export const PREVIEW_GO_BACK_CHANNEL = "desktop:preview-go-back"; export const PREVIEW_GO_FORWARD_CHANNEL = "desktop:preview-go-forward"; +export const PREVIEW_STOP_LOADING_CHANNEL = "desktop:preview-stop-loading"; export const PREVIEW_REFRESH_CHANNEL = "desktop:preview-refresh"; export const PREVIEW_ZOOM_IN_CHANNEL = "desktop:preview-zoom-in"; export const PREVIEW_ZOOM_OUT_CHANNEL = "desktop:preview-zoom-out"; @@ -99,6 +101,7 @@ export const PREVIEW_SET_AUDIO_MUTED_CHANNEL = "desktop:preview-set-audio-muted" export const PREVIEW_FIND_IN_PAGE_CHANNEL = "desktop:preview-find-in-page"; export const PREVIEW_STOP_FIND_IN_PAGE_CHANNEL = "desktop:preview-stop-find-in-page"; export const PREVIEW_SET_SHOWN_CHANNEL = "desktop:preview-set-shown"; +export const PREVIEW_SET_CAPTURE_DRAWN_CHANNEL = "desktop:preview-set-capture-drawn"; export const PREVIEW_OPEN_DEVTOOLS_CHANNEL = "desktop:preview-open-devtools"; export const PREVIEW_CLEAR_COOKIES_CHANNEL = "desktop:preview-clear-cookies"; export const PREVIEW_CLEAR_CACHE_CHANNEL = "desktop:preview-clear-cache"; diff --git a/apps/desktop/src/ipc/methods/preview.ts b/apps/desktop/src/ipc/methods/preview.ts index ea291b66b3149..757e1e463f538 100644 --- a/apps/desktop/src/ipc/methods/preview.ts +++ b/apps/desktop/src/ipc/methods/preview.ts @@ -13,6 +13,7 @@ import { DesktopPreviewScreenshotArtifactSchema, DesktopPreviewSetAudioMutedInputSchema, DesktopPreviewSetShownInputSchema, + DesktopPreviewSetCaptureDrawnInputSchema, DesktopPreviewSetColorSchemeInputSchema, DesktopPreviewSetZoomFactorInputSchema, BrowserImportResult, @@ -168,6 +169,11 @@ const refresh = tabMethod( "desktop.ipc.preview.refresh", (manager, tabId) => manager.refresh(tabId), ); +const stopLoading = tabMethod( + IpcChannels.PREVIEW_STOP_LOADING_CHANNEL, + "desktop.ipc.preview.stopLoading", + (manager, tabId) => manager.stopLoading(tabId), +); const zoomIn = tabMethod( IpcChannels.PREVIEW_ZOOM_IN_CHANNEL, "desktop.ipc.preview.zoomIn", @@ -229,6 +235,15 @@ const setShown = DesktopIpc.makeIpcMethod({ yield* manager.setShown(tabId, shown); }), }); +const setCaptureDrawn = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.PREVIEW_SET_CAPTURE_DRAWN_CHANNEL, + payload: DesktopPreviewSetCaptureDrawnInputSchema, + result: Schema.Void, + handler: Effect.fn("desktop.ipc.preview.setCaptureDrawn")(function* ({ tabId, drawn }) { + const manager = yield* PreviewManager.PreviewManager; + yield* manager.setCaptureDrawn(tabId, drawn); + }), +}); const openDevTools = tabMethod( IpcChannels.PREVIEW_OPEN_DEVTOOLS_CHANNEL, "desktop.ipc.preview.openDevTools", @@ -472,6 +487,7 @@ export const methods = [ goBack, goForward, refresh, + stopLoading, zoomIn, zoomOut, resetZoom, @@ -482,6 +498,7 @@ export const methods = [ setZoomFactor, setAudioMuted, setShown, + setCaptureDrawn, openDevTools, clearCookies, clearCache, diff --git a/apps/desktop/src/ipc/methods/window.test.ts b/apps/desktop/src/ipc/methods/window.test.ts index 376f85d2baf9d..de202529c4113 100644 --- a/apps/desktop/src/ipc/methods/window.test.ts +++ b/apps/desktop/src/ipc/methods/window.test.ts @@ -1,6 +1,8 @@ import { assert, describe, it } from "@effect/vitest"; import * as NodeServices from "@effect/platform-node/NodeServices"; import type { ManagedPolicyLimits } from "@t3tools/contracts"; +import { EDITORS, REMOTE_CAPABLE_EDITOR_IDS } from "@t3tools/contracts"; +import { resolveEditorCommand } from "@t3tools/shared/editor"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { ManagedPolicyFileError } from "@t3tools/shared/managedPolicyFile"; import * as FileSystem from "effect/FileSystem"; @@ -25,6 +27,7 @@ import * as DesktopBackendManager from "../../backend/DesktopBackendManager.ts"; import * as DesktopBackendPool from "../../backend/DesktopBackendPool.ts"; import * as DesktopAppSettings from "../../settings/DesktopAppSettings.ts"; import * as DesktopManagedPolicy from "../../app/DesktopManagedPolicy.ts"; +import * as DesktopWslEnvironment from "../../wsl/DesktopWslEnvironment.ts"; import * as ElectronDialog from "../../electron/ElectronDialog.ts"; import * as ElectronWindow from "../../electron/ElectronWindow.ts"; import * as IpcChannels from "../channels.ts"; @@ -36,6 +39,7 @@ import { probeRemoteEditors, readMedia, syncHtmlFullscreen, + toLocalBackendPath, } from "./window.ts"; const readyWslConfig: DesktopBackendManager.DesktopBackendStartConfig = { @@ -185,6 +189,47 @@ describe("getLocalEnvironmentBootstraps", () => { }); }); +describe("toLocalBackendPath", () => { + const { runningDistro: _, ...windowsConfig } = readyWslConfig; + it.effect("converts paths through wslpath only for backends running in WSL", () => + Effect.gen(function* () { + const convert = (backendId: string, path = "C:\\Users\\me\\project") => + toLocalBackendPath.handler({ backendId, path }); + assert.strictEqual(yield* convert("primary"), "C:\\Users\\me\\project"); + assert.strictEqual(yield* convert("wsl:default"), "/drives/c/Users/me/project"); + assert.strictEqual(yield* convert("wsl:missing"), null); + assert.strictEqual( + yield* convert("wsl:default", "\\\\wsl.localhost\\ubuntu\\home\\me\\repo"), + "/home/me/repo", + ); + assert.strictEqual(yield* convert("wsl:default", "\\\\wsl$\\Debian\\home\\me\\repo"), null); + assert.strictEqual( + yield* convert("primary", "\\\\wsl$\\Debian\\home\\me\\repo"), + "\\\\wsl$\\Debian\\home\\me\\repo", + ); + }).pipe( + Effect.provide( + Layer.mergeAll( + DesktopBackendPool.layerTest([ + { + ...defaultWslInstance, + id: DesktopBackendManager.BackendInstanceId("primary"), + currentConfig: Effect.succeedSome(windowsConfig), + }, + defaultWslInstance, + ]), + DesktopWslEnvironment.layerTest({ + windowsToWslPath: (distro, windowsPath) => + distro === "Ubuntu" && windowsPath === "C:\\Users\\me\\project" + ? Option.some("/drives/c/Users/me/project") + : Option.none(), + }), + ), + ), + ), + ); +}); + describe("installRelayTelemetryEnabled", () => { // The renderer reads this over sync IPC, so the answer must not wait on the filesystem. const relayTelemetryEnabled = ( @@ -474,3 +519,108 @@ it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( assert.include(editors, "webstorm"); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); + +describe("probeRemoteEditors PATH scan", () => { + const platform = HostProcess.Platform.defaultValue(); + const launcherName = (command: string) => (platform === "win32" ? `${command}.cmd` : command); + + const makeFakePath = Effect.fnUntraced(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-probe-path-" }); + const dirs = ["a", "b", "c", "d"].map((name) => path.join(root, name)); + for (const dir of dirs) yield* fs.makeDirectory(dir, { recursive: true }); + // Launchers in different PATH directories, deliberately not in REMOTE_CAPABLE_EDITOR_IDS order. + for (const [dir, command] of [ + [dirs[2]!, "webstorm"], + [dirs[0]!, "cursor"], + [dirs[3]!, "code"], + [dirs[1]!, "zeditor"], + ] as const) { + const file = path.join(dir, launcherName(command)); + yield* fs.writeFileString(file, "#!/bin/sh\n"); + if (platform !== "win32") yield* fs.chmod(file, 0o755); + } + return { + root, + env: { + HOME: root, + USERPROFILE: root, + PATH: dirs.join(platform === "win32" ? ";" : ":"), + PATHEXT: ".COM;.EXE;.BAT;.CMD", + }, + }; + }); + + const countingFileSystem = (counter: { stat: number; reads: Map }) => + Layer.effect( + FileSystem.FileSystem, + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + return FileSystem.FileSystem.of({ + ...fs, + stat: (target) => { + counter.stat += 1; + return fs.stat(target); + }, + readDirectory: (target, options) => { + counter.reads.set(target, (counter.reads.get(target) ?? 0) + 1); + return fs.readDirectory(target, options); + }, + }); + }), + ).pipe(Layer.provideMerge(NodeServices.layer)); + + const serialReference = (env: Record) => + Effect.gen(function* () { + const available: Array = []; + for (const editorId of REMOTE_CAPABLE_EDITOR_IDS) { + const editor = EDITORS.find((candidate) => candidate.id === editorId); + if (editor && Option.isSome(yield* resolveEditorCommand(editor, env))) { + available.push(editorId); + } + } + return available; + }); + + it.effect("reads each PATH directory once instead of stat-ing every candidate per editor", () => + Effect.gen(function* () { + const { env: baseEnv } = yield* makeFakePath(); + const delimiter = platform === "win32" ? ";" : ":"; + const uncachedCounter = { stat: 0, reads: new Map() }; + const cachedCounter = { stat: 0, reads: new Map() }; + const uncachedEnv = { ...baseEnv, PATH: `${baseEnv.PATH}${delimiter}${baseEnv.HOME}x1` }; + const cachedEnv = { ...baseEnv, PATH: `${baseEnv.PATH}${delimiter}${baseEnv.HOME}x2` }; + const uncached = yield* serialReference(uncachedEnv).pipe( + Effect.provide(countingFileSystem(uncachedCounter)), + ); + const cached = yield* probeRemoteEditors + .handler(undefined) + .pipe( + Effect.provideService(HostProcess.Environment, cachedEnv), + Effect.provideService(HostProcess.Platform, platform), + Effect.provide(countingFileSystem(cachedCounter)), + ); + // The host can add editors from app folders the fixture can't redirect (e.g. /Applications). + const fixtureEditors = ["cursor", "vscode", "zed", "webstorm"]; + assert.deepStrictEqual( + uncached.filter((id) => fixtureEditors.includes(id)), + fixtureEditors, + ); + assert.deepStrictEqual(cached, uncached); + assert.isAbove(uncachedCounter.stat, 0); + // On every platform the probe lists each PATH directory at most once. Only PATH entries count: + // resolveEditorCommand also reads app folders such as /Applications on macOS. + const pathReads = cachedEnv.PATH.split(delimiter) + .filter((entry) => entry.length > 0) + .map((entry) => cachedCounter.reads.get(entry) ?? 0); + assert.isAbove(Math.max(...pathReads), 0); + assert.isAtMost(Math.max(...pathReads), 1); + // Without the listings, Windows stats every PATH entry x PATHEXT candidate for each editor; + // elsewhere there is no PATHEXT, so the stat count is about the same either way. + if (platform === "win32") { + assert.isBelow(cachedCounter.stat, uncachedCounter.stat / 2); + } + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +}); diff --git a/apps/desktop/src/ipc/methods/window.ts b/apps/desktop/src/ipc/methods/window.ts index 53a1138d71a99..5bd23e68c2705 100644 --- a/apps/desktop/src/ipc/methods/window.ts +++ b/apps/desktop/src/ipc/methods/window.ts @@ -1,5 +1,6 @@ import { ContextMenuItemSchema, + ContextMenuSource, DesktopAppBrandingSchema, DesktopEnvironmentBootstrapSchema, DesktopMediaReadSchema, @@ -16,6 +17,7 @@ import { } from "@t3tools/contracts"; import { PROJECT_FAVICON_EXTENSIONS } from "@t3tools/shared/projectFavicon"; import { resolveEditorCommand } from "@t3tools/shared/editor"; +import { withPathDirectoryListings } from "@t3tools/shared/shell"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { isRelayTracingEnabled } from "@t3tools/shared/otelEnvironment"; import * as FileSystem from "effect/FileSystem"; @@ -25,6 +27,7 @@ import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import * as DesktopBackendConfiguration from "../../backend/DesktopBackendConfiguration.ts"; +import * as DesktopBackendManager from "../../backend/DesktopBackendManager.ts"; import * as DesktopBackendPool from "../../backend/DesktopBackendPool.ts"; import * as DesktopLocalEnvironmentAuth from "../../backend/DesktopLocalEnvironmentAuth.ts"; import * as DesktopEnvironment from "../../app/DesktopEnvironment.ts"; @@ -58,6 +61,7 @@ const ContextMenuPosition = Schema.Struct({ const ContextMenuInput = Schema.Struct({ items: Schema.Array(ContextMenuItemSchema), position: Schema.optionalKey(ContextMenuPosition), + source: Schema.optionalKey(ContextMenuSource), }); function toWebSocketBaseUrl(httpBaseUrl: URL): string { @@ -204,6 +208,20 @@ export const getLocalEnvironmentBearerToken = DesktopIpc.makeIpcMethod({ }), }); +// Ask the distro: its automount root is configurable, so /mnt/ is only the default. +const windowsPathToWsl = Effect.fn("desktop.ipc.window.windowsPathToWsl")(function* ( + distro: string | null, + windowsPath: string, +) { + const linuxUncPath = wslUncPathToLinuxPath(windowsPath); + if (linuxUncPath !== null) return Option.some(linuxUncPath); + const wslEnvironment = yield* DesktopWslEnvironment.DesktopWslEnvironment; + return yield* wslEnvironment.windowsToWslPath( + extractDistroFromUncPath(windowsPath) ?? distro, + windowsPath, + ); +}); + export const pickFolder = DesktopIpc.makeIpcMethod({ channel: IpcChannels.PICK_FOLDER_CHANNEL, payload: Schema.UndefinedOr(PickFolderOptionsSchema), @@ -250,20 +268,29 @@ export const pickFolder = DesktopIpc.makeIpcMethod({ if (!useWsl) { return selectedPath.value; } - - const linuxUncPath = wslUncPathToLinuxPath(selectedPath.value); - if (linuxUncPath !== null) { - return linuxUncPath; - } - - const converted = yield* wslEnvironment.windowsToWslPath( - extractDistroFromUncPath(selectedPath.value) ?? wslDistro, - selectedPath.value, - ); + const converted = yield* windowsPathToWsl(wslDistro, selectedPath.value); return Option.getOrElse(converted, () => selectedPath.value); }), }); +export const toLocalBackendPath = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.TO_LOCAL_BACKEND_PATH_CHANNEL, + payload: Schema.Struct({ backendId: Schema.String, path: Schema.String }), + result: Schema.NullOr(Schema.String), + handler: Effect.fn("desktop.ipc.window.toLocalBackendPath")(function* ({ backendId, path }) { + const pool = yield* DesktopBackendPool.DesktopBackendPool; + const instance = yield* pool.get(DesktopBackendManager.BackendInstanceId(backendId)); + if (Option.isNone(instance)) return null; + const config = yield* instance.value.currentConfig; + if (Option.isNone(config) || Option.isSome(config.value.preflightFailure)) return null; + const { runningDistro } = config.value; + if (runningDistro === undefined) return path; + const uncDistro = extractDistroFromUncPath(path); + if (uncDistro !== null && uncDistro.toLowerCase() !== runningDistro.toLowerCase()) return null; + return Option.getOrNull(yield* windowsPathToWsl(runningDistro, path)); + }), +}); + export const pickProjectFavicon = DesktopIpc.makeIpcMethod({ channel: IpcChannels.PICK_PROJECT_FAVICON_CHANNEL, payload: Schema.UndefinedOr(Schema.String), @@ -316,6 +343,7 @@ export const showContextMenu = DesktopIpc.makeIpcMethod({ window: window.value, items: input.items, position: Option.fromNullishOr(input.position), + source: Option.fromNullishOr(input.source), }); return Option.getOrNull(selectedItemId); }), @@ -400,20 +428,26 @@ export const openSystemSettings = DesktopIpc.makeIpcMethod({ }), }); +const REMOTE_EDITOR_PROBE_CONCURRENCY = 8; + export const probeRemoteEditors = DesktopIpc.makeIpcMethod({ channel: IpcChannels.PROBE_REMOTE_EDITORS_CHANNEL, payload: Schema.Undefined, result: Schema.Array(EditorId), handler: Effect.fn("desktop.ipc.window.probeRemoteEditors")(function* () { - const available: Array = []; const env = yield* HostProcess.Environment; - for (const editorId of REMOTE_CAPABLE_EDITOR_IDS) { - const editor = EDITORS.find((editor) => editor.id === editorId); - if (editor && Option.isSome(yield* resolveEditorCommand(editor, env))) { - available.push(editorId); - } - } - return available; + // Shared listings stop each editor re-stating every PATH entry x PATHEXT candidate on Windows. + const resolved = yield* Effect.forEach( + REMOTE_CAPABLE_EDITOR_IDS, + (editorId) => { + const editor = EDITORS.find((editor) => editor.id === editorId); + return editor + ? resolveEditorCommand(editor, env).pipe(Effect.map(Option.isSome)) + : Effect.succeed(false); + }, + { concurrency: REMOTE_EDITOR_PROBE_CONCURRENCY }, + ).pipe(withPathDirectoryListings); + return REMOTE_CAPABLE_EDITOR_IDS.filter((_, index) => resolved[index]); }), }); diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index f241e14587f34..432577917a3ea 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -1,8 +1,8 @@ import * as MacPermissions from "./permissions/MacPermissions.ts"; +import { handleStdioError } from "./app/stdioErrors.ts"; + for (const stream of [process.stdout, process.stderr]) { - stream.on("error", (err: NodeJS.ErrnoException) => { - if (err.code !== "EPIPE") throw err; - }); + stream.on("error", handleStdioError); } import * as NodeHttpClient from "@effect/platform-node/NodeHttpClient"; @@ -12,7 +12,9 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; // @effect-diagnostics-next-line nodeBuiltinImport:off - Version output must flush before Electron exits, without acquiring the runtime. import * as NodeFS from "node:fs"; import * as NodeOS from "node:os"; +import * as SecureRandom from "effect/crypto/SecureRandom"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; @@ -125,6 +127,15 @@ const layerDesktopSshEnvironment = Layer.unwrap( const environment = yield* DesktopEnvironment.DesktopEnvironment; return DesktopSshEnvironment.layer({ resolveCliRunner: Effect.succeed(resolveDesktopSshCliRunner(environment)), + resolveLauncherId: yield* Effect.cached( + DesktopSshEnvironment.readOrCreateLauncherId( + environment.path.join(environment.stateDir, "ssh-launcher-id"), + ).pipe( + Effect.provide( + yield* Effect.context(), + ), + ), + ), }); }), ); diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index a538b821ff77c..f0f9648173dc8 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -238,14 +238,17 @@ contextBridge.exposeInMainWorld("desktopBridge", { setWslDistro: (distro) => ipcRenderer.invoke(IpcChannels.SET_WSL_DISTRO_CHANNEL, distro), setWslOnly: (enabled) => ipcRenderer.invoke(IpcChannels.SET_WSL_ONLY_CHANNEL, enabled), pickFolder: (options) => ipcRenderer.invoke(IpcChannels.PICK_FOLDER_CHANNEL, options), + toLocalBackendPath: (backendId, path) => + ipcRenderer.invoke(IpcChannels.TO_LOCAL_BACKEND_PATH_CHANNEL, { backendId, path }), pickProjectFavicon: (initialPath) => ipcRenderer.invoke(IpcChannels.PICK_PROJECT_FAVICON_CHANNEL, initialPath), pickThemeFiles: () => ipcRenderer.invoke(IpcChannels.PICK_THEME_FILES_CHANNEL, undefined), setTheme: (theme) => ipcRenderer.invoke(IpcChannels.SET_THEME_CHANNEL, theme), - showContextMenu: (items, position) => + showContextMenu: (items, position, source) => ipcRenderer.invoke(IpcChannels.CONTEXT_MENU_CHANNEL, { items, ...(position === undefined ? {} : { position }), + ...(source === undefined ? {} : { source }), }), receiveProviderAuthCallback: (url: string) => ipcRenderer.invoke(IpcChannels.RECEIVE_PROVIDER_AUTH_CALLBACK_CHANNEL, url), @@ -389,6 +392,7 @@ contextBridge.exposeInMainWorld("desktopBridge", { goBack: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_GO_BACK_CHANNEL, { tabId }), goForward: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_GO_FORWARD_CHANNEL, { tabId }), refresh: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_REFRESH_CHANNEL, { tabId }), + stopLoading: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_STOP_LOADING_CHANNEL, { tabId }), zoomIn: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_ZOOM_IN_CHANNEL, { tabId }), zoomOut: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_ZOOM_OUT_CHANNEL, { tabId }), resetZoom: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_RESET_ZOOM_CHANNEL, { tabId }), @@ -410,6 +414,8 @@ contextBridge.exposeInMainWorld("desktopBridge", { ipcRenderer.invoke(IpcChannels.PREVIEW_STOP_FIND_IN_PAGE_CHANNEL, { tabId }), setShown: (tabId, shown) => ipcRenderer.invoke(IpcChannels.PREVIEW_SET_SHOWN_CHANNEL, { tabId, shown }), + setCaptureDrawn: (tabId, drawn) => + ipcRenderer.invoke(IpcChannels.PREVIEW_SET_CAPTURE_DRAWN_CHANNEL, { tabId, drawn }), openDevTools: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_OPEN_DEVTOOLS_CHANNEL, { tabId }), listBrowserImportSources: () => ipcRenderer.invoke(IpcChannels.PREVIEW_IMPORT_SOURCES_CHANNEL), diff --git a/apps/desktop/src/preview/BrowserImport/BrowserImport.test.ts b/apps/desktop/src/preview/BrowserImport/BrowserImport.test.ts index 6309c4ea993f1..484ba6f9358f6 100644 --- a/apps/desktop/src/preview/BrowserImport/BrowserImport.test.ts +++ b/apps/desktop/src/preview/BrowserImport/BrowserImport.test.ts @@ -89,7 +89,7 @@ describe("BrowserImport.importCookies", () => { assert.instanceOf(error, BrowserImport.BrowserImportFailedError); assert.equal(error.reason, "unknownSourceProfile"); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); it.effect.skipIf(!symlinksSupported)( @@ -114,7 +114,7 @@ describe("BrowserImport.importCookies", () => { assert.instanceOf(error, BrowserImport.BrowserImportFailedError); assert.equal(error.reason, "browserRunning"); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); }); diff --git a/apps/desktop/src/preview/BrowserImport/ChromiumCookies.test.ts b/apps/desktop/src/preview/BrowserImport/ChromiumCookies.test.ts index e5aa05f268e43..eb4dd3143eefb 100644 --- a/apps/desktop/src/preview/BrowserImport/ChromiumCookies.test.ts +++ b/apps/desktop/src/preview/BrowserImport/ChromiumCookies.test.ts @@ -134,7 +134,7 @@ describe("readChromiumCookieDatabase", () => { const partitioned = yield* readChromiumCookieDatabase(filename, keys, "linux"); expect(partitioned.cookies).toEqual([]); expect(partitioned.undecryptable).toBe(1); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); it.effect("reads plaintext, encrypted, and genuinely empty cookie values", () => @@ -186,7 +186,7 @@ describe("readChromiumCookieDatabase", () => { { name: "encrypted", value: "stored encrypted" }, { name: "empty", value: "" }, ]); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); it.effect("enforces domain binding only for schema 24 and newer", () => @@ -227,7 +227,7 @@ describe("readChromiumCookieDatabase", () => { ]); expect(result.undecryptable).toBe(2); expect(result.undecryptableHosts).toEqual(["wrong.example", "short.example"]); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); it.effect("decrypts mixed v10 and v11 cookies with their respective keys", () => @@ -273,7 +273,7 @@ describe("readChromiumCookieDatabase", () => { ]); expect(v10Only.undecryptable).toBe(1); expect(v10Only.undecryptableHosts).toEqual(["v11.example"]); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); it.effect("recovers records written with the empty-passphrase key", () => @@ -320,7 +320,7 @@ describe("readChromiumCookieDatabase", () => { const noV11 = yield* readChromiumCookieDatabase(filename, { cbcV10, cbcEmpty }, "linux"); expect(noV11.cookies.map(({ name }) => name)).toEqual(["empty-v10"]); expect(noV11.undecryptableHosts).toEqual(["ev11.example"]); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); it.effect("treats unversioned encrypted values as legacy plaintext on macOS and Linux", () => @@ -355,6 +355,29 @@ describe("readChromiumCookieDatabase", () => { expect(mac.undecryptable).toBe(0); expect(linux.cookies[0]?.value).toBe("legacy cleartext"); expect(linux.undecryptable).toBe(0); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("reads a cookie database the browser has emptied as having no cookies", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const directory = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3code-chromium-cookies-", + }); + const filename = `${directory}/Cookies`; + yield* Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`create table scratch (id integer)`; + yield* sql`drop table scratch`; + }).pipe(Effect.provide(NodeSqliteClient.layer({ filename }))); + + const result = yield* readChromiumCookieDatabase( + filename, + { cbcV10: Buffer.from("0123456789abcdef") }, + "darwin", + ); + + expect(result).toEqual({ cookies: [], undecryptable: 0, undecryptableHosts: [] }); }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), ); @@ -409,6 +432,6 @@ describe("readChromiumCookieDatabase", () => { expect(chips.cookies.map(({ name }) => name)).toEqual(["plain"]); expect(chips.undecryptable).toBe(1); expect(chips.undecryptableHosts).toEqual(["partitioned.example"]); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); }); diff --git a/apps/desktop/src/preview/BrowserImport/ChromiumCookies.ts b/apps/desktop/src/preview/BrowserImport/ChromiumCookies.ts index 63fef9fb3c808..91cbfa03aac7d 100644 --- a/apps/desktop/src/preview/BrowserImport/ChromiumCookies.ts +++ b/apps/desktop/src/preview/BrowserImport/ChromiumCookies.ts @@ -181,6 +181,10 @@ export const readChromiumCookieDatabase = Effect.fn("ChromiumCookies.readChromiu function* (snapshotPath: string, keys: ChromiumKeyMaterial, platform: NodeJS.Platform) { const result = yield* Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; + // Chromium razes a damaged jar to a file with no tables and rebuilds it on its next write. + const tables = + yield* sql`select 1 from sqlite_master where type = 'table' and name = 'cookies'`; + if (tables.length === 0) return { rows: [], schemaVersion: 0 }; const schemaVersion = yield* sql`select value from meta where key = 'version' limit 1`.pipe( Effect.flatMap(decodeSchemaVersion), Effect.map(([row]) => row.value), diff --git a/apps/desktop/src/preview/BrowserImport/CookieDatabase.test.ts b/apps/desktop/src/preview/BrowserImport/CookieDatabase.test.ts index 402397d3fde60..ca0a86cedacd3 100644 --- a/apps/desktop/src/preview/BrowserImport/CookieDatabase.test.ts +++ b/apps/desktop/src/preview/BrowserImport/CookieDatabase.test.ts @@ -11,7 +11,7 @@ import { snapshotCookieDatabase } from "./CookieDatabase.ts"; const runNode = ( effect: Effect.Effect, -) => effect.pipe(Effect.provide(NodeServices.layer), Effect.scoped); +) => effect.pipe(Effect.provide(NodeServices.layer)); describe("snapshotCookieDatabase", () => { it.effect("includes committed WAL data in one consistent database", () => diff --git a/apps/desktop/src/preview/BrowserImport/FirefoxCookies.test.ts b/apps/desktop/src/preview/BrowserImport/FirefoxCookies.test.ts index be41318fa01e6..33e32ec030be3 100644 --- a/apps/desktop/src/preview/BrowserImport/FirefoxCookies.test.ts +++ b/apps/desktop/src/preview/BrowserImport/FirefoxCookies.test.ts @@ -78,7 +78,7 @@ const writeFirefoxCookieDatabase = Effect.fnUntraced(function* ( }); const run = (effect: Effect.Effect) => - effect.pipe(Effect.provide(NodeServices.layer), Effect.scoped); + effect.pipe(Effect.provide(NodeServices.layer)); describe("readFirefoxCookies", () => { it.effect("converts millisecond expiries from schema 16 and newer", () => diff --git a/apps/desktop/src/preview/BrowserImport/LinuxBrowserSecret.test.ts b/apps/desktop/src/preview/BrowserImport/LinuxBrowserSecret.test.ts index 8358088bc1344..8ef376e4887bf 100644 --- a/apps/desktop/src/preview/BrowserImport/LinuxBrowserSecret.test.ts +++ b/apps/desktop/src/preview/BrowserImport/LinuxBrowserSecret.test.ts @@ -64,6 +64,6 @@ it.layer(NodeServices.layer)("Linux browser secret path", (it) => { assert.isUndefined(yield* resolve(false, "win32")); yield* fileSystem.remove(staged); assert.isUndefined(yield* resolve(false)); - }).pipe(Effect.scoped), + }), ); }); diff --git a/apps/desktop/src/preview/BrowserImport/SafariCookies.test.ts b/apps/desktop/src/preview/BrowserImport/SafariCookies.test.ts index 7e92334c957de..f82f2319baeb7 100644 --- a/apps/desktop/src/preview/BrowserImport/SafariCookies.test.ts +++ b/apps/desktop/src/preview/BrowserImport/SafariCookies.test.ts @@ -240,7 +240,7 @@ describe("safariAccessDenied", () => { yield* fileSystem.writeFile(jar, new Uint8Array([0x63, 0x6f, 0x6f, 0x6b])); assert.isFalse(yield* safariAccessDenied(jar)); assert.isFalse(yield* safariAccessDenied(`${directory}/absent.binarycookies`)); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); }); diff --git a/apps/desktop/src/preview/BrowserImport/SafariPermission.test.ts b/apps/desktop/src/preview/BrowserImport/SafariPermission.test.ts index 3b33f872d124d..dc225b04bac72 100644 --- a/apps/desktop/src/preview/BrowserImport/SafariPermission.test.ts +++ b/apps/desktop/src/preview/BrowserImport/SafariPermission.test.ts @@ -39,7 +39,7 @@ it.effect("detects Safari access becoming available without reading or importing assert.isTrue(yield* Effect.promise(check)); yield* fs.remove(jar); assert.isFalse(yield* Effect.promise(check)); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); it.effect("recognizes access when cookies exist only in a named Safari profile", () => @@ -55,5 +55,5 @@ it.effect("recognizes access when cookies exist only in a named Safari profile", yield* fs.makeDirectory(directory, { recursive: true }); yield* fs.writeFileString(`${directory}/Cookies.binarycookies`, "not parsed"); assert.isTrue(yield* Effect.promise(check)); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); diff --git a/apps/desktop/src/preview/BrowserImport/Sources.test.ts b/apps/desktop/src/preview/BrowserImport/Sources.test.ts index 720a57e64a9dc..9b2c3cb26c8f7 100644 --- a/apps/desktop/src/preview/BrowserImport/Sources.test.ts +++ b/apps/desktop/src/preview/BrowserImport/Sources.test.ts @@ -51,7 +51,7 @@ const run = ( E, FileSystem.FileSystem | Path.Path | Scope.Scope | ChildProcessSpawner.ChildProcessSpawner >, -) => effect.pipe(Effect.provide(NodeServices.layer), Effect.scoped); +) => effect.pipe(Effect.provide(NodeServices.layer)); const writeCookieDatabase = (file: string, count: number) => Effect.sync(() => { diff --git a/apps/desktop/src/preview/BrowserImport/Sources.ts b/apps/desktop/src/preview/BrowserImport/Sources.ts index 20a35752ed979..6ecd546a1ad57 100644 --- a/apps/desktop/src/preview/BrowserImport/Sources.ts +++ b/apps/desktop/src/preview/BrowserImport/Sources.ts @@ -345,6 +345,12 @@ const countProfileCookies = Effect.fnUntraced(function* ( if (database === undefined) return undefined; return yield* Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; + // readChromiumCookieDatabase imports a razed jar as empty; Firefox has no such reader guard. + if (definition.engine !== "firefox") { + const tables = + yield* sql`select 1 from sqlite_master where type = 'table' and name = 'cookies'`; + if (tables.length === 0) return 0; + } const rows = definition.engine === "firefox" ? yield* sql`select count(*) as count from moz_cookies where originAttributes = ''` diff --git a/apps/desktop/src/preview/CdpRelay.test.ts b/apps/desktop/src/preview/CdpRelay.test.ts index 6e657ba5fc4d7..564b4dc57d6f4 100644 --- a/apps/desktop/src/preview/CdpRelay.test.ts +++ b/apps/desktop/src/preview/CdpRelay.test.ts @@ -9,6 +9,7 @@ import { const makeTarget = ( send: CdpRelayTarget["send"], setDownloadDirectory: CdpRelayTarget["setDownloadDirectory"] = () => {}, + ackScreencastFrame: CdpRelayTarget["ackScreencastFrame"] = () => {}, ): CdpRelayTarget => ({ send, targetId: async () => "GUEST-TARGET", @@ -16,6 +17,8 @@ const makeTarget = ( title: () => "Fixture", userAgent: () => "Electron", setDownloadDirectory, + receivesScreencast: () => true, + ackScreencastFrame, }); const makeRelay = (target: CdpRelayTarget, write: (raw: string) => void) => @@ -137,9 +140,13 @@ describe("CDP relay", () => { describe("CDP relay extra sessions", () => { it("gives a second page session the page's commands and events", async () => { - const send = vi.fn(async () => ({})); + const send = vi.fn(async () => ({})); const written: Array> = []; - const relay = makeRelay(makeTarget(send), (raw) => written.push(JSON.parse(raw))); + const nativeAcks: Array = []; + const relay = makeRelay( + makeTarget(send, undefined, (frameId) => nativeAcks.push(frameId)), + (raw) => written.push(JSON.parse(raw)), + ); relay.receive(JSON.stringify({ id: 1, method: "Target.setAutoAttach", params: {} })); relay.receive(JSON.stringify({ id: 2, method: "Target.attachToBrowserTarget" })); await settle(); @@ -169,8 +176,63 @@ describe("CDP relay extra sessions", () => { await settle(); expect(send).toHaveBeenLastCalledWith("Page.startScreencast", {}, undefined, extra); written.length = 0; - relay.event("Page.screencastFrame", { data: "x" }, undefined); - expect(written.map((m) => m["sessionId"])).toEqual(["t3-preview-page", extra]); + const frame = (frameId: number) => + relay.event("Page.screencastFrame", { data: "x", sessionId: frameId }, undefined); + const ack = (sessionId: string, frameId: number) => + relay.receive( + JSON.stringify({ + id: 6, + method: "Page.screencastFrameAck", + params: { sessionId: frameId }, + sessionId, + }), + ); + const delivered = () => { + const frames = written + .filter((m) => m["method"]) + .map((m) => [m["sessionId"], (m["params"] as { sessionId?: number }).sessionId]); + written.length = 0; + return frames; + }; + frame(1); + expect(delivered()).toEqual([ + ["t3-preview-page", 1], + [extra, 1], + ]); + // A session that has not acknowledged its frame gets only the newest one, on its ack. + for (let i = 2; i < 100; i++) frame(i); + expect(delivered()).toEqual([]); + // Frames every session is busy for stay unacknowledged, so Chromium pauses at its in-flight limit. + expect(nativeAcks).toEqual([1]); + ack(extra, 1); + await settle(); + expect(delivered()).toEqual([[extra, 99]]); + // Each consumer ack lets Chromium send one more frame. + expect(nativeAcks).toEqual([1, 2]); + ack(extra, 99); + await settle(); + expect(delivered()).toEqual([]); + expect(nativeAcks).toEqual([1, 2, 3]); + frame(100); + expect(delivered()).toEqual([[extra, 100]]); + ack(extra, 99); + await settle(); + frame(101); + expect(delivered()).toEqual([]); + expect(send.mock.calls.some(([method]) => method === "Page.screencastFrameAck")).toBe(false); + // A restart keeps the unacknowledged frame's slot until its own acknowledgement. + relay.receive(JSON.stringify({ id: 7, method: "Page.stopScreencast", sessionId: extra })); + relay.receive(JSON.stringify({ id: 8, method: "Page.startScreencast", sessionId: extra })); + await settle(); + delivered(); + frame(102); + expect(delivered()).toEqual([]); + ack(extra, 100); + await settle(); + expect(delivered()).toEqual([[extra, 102]]); + // Other events still reach a session while its frame is unacknowledged. + relay.event("Page.loadEventFired", { timestamp: 1 }, undefined); + expect(delivered().map(([session]) => session)).toEqual(["t3-preview-page", extra]); }); }); diff --git a/apps/desktop/src/preview/CdpRelay.ts b/apps/desktop/src/preview/CdpRelay.ts index 94a78f47db686..e9f3cf509b446 100644 --- a/apps/desktop/src/preview/CdpRelay.ts +++ b/apps/desktop/src/preview/CdpRelay.ts @@ -9,6 +9,10 @@ export interface CdpRelayTarget { owner?: string, ) => Promise; readonly sessionDetached?: (owner: string) => void; + /** Whether a page session, by owner as `send` names it, gets the page's screencast frames. */ + readonly receivesScreencast: (owner: string | undefined) => boolean; + /** Acknowledges a native screencast frame, letting Chromium send the next one. */ + readonly ackScreencastFrame: (frameId: number) => void; /** The page's own CDP target id; Playwright keys the main frame by it. */ readonly targetId: () => Promise; readonly url: () => string; @@ -101,6 +105,20 @@ export function createCdpRelayConnection( let downloadEventsEnabled = false; /** Relay-created page sessions share one real debugger; child frame sessions retain their own ids. */ const sessions = new Map([[PAGE_SESSION_ID, "page"]]); + // Each page session holds one unacknowledged frame and keeps the newest one skipped meanwhile. + const unackedFrames = new Map(); + const skippedFrames = new Map>(); + // Chromium keeps a few frames in flight and reuses one frame id per run, so pacing it means + // leaving frames every session is busy for unacknowledged and releasing one per consumer ack. + const heldNativeFrames: Array = []; + const releaseHeldFramesWithoutHolders = () => { + if (skippedFrames.size === 0) + for (const frameId of heldNativeFrames.splice(0)) target.ackScreencastFrame(frameId); + }; + const sendFrame = (sessionId: string, params: Record) => { + unackedFrames.set(sessionId, params["sessionId"] as number); + send({ method: "Page.screencastFrame", params, sessionId }); + }; let sessionSequence = 0; const newSession = (kind: "browser" | "page") => { const sessionId = `t3-preview-${kind}-${++sessionSequence}`; @@ -168,6 +186,9 @@ export function createCdpRelayConnection( const sessionId = command.params?.["sessionId"]; if (typeof sessionId === "string" && sessionId !== PAGE_SESSION_ID) { sessions.delete(sessionId); + unackedFrames.delete(sessionId); + skippedFrames.delete(sessionId); + releaseHeldFramesWithoutHolders(); target.sessionDetached?.(sessionId); } return {}; @@ -200,6 +221,27 @@ export function createCdpRelayConnection( } // Playwright asks for this on every page; the host emulates focus only while it is in use. if (command.method === "Emulation.setFocusEmulationEnabled") return Promise.resolve({}); + if (command.method === "Page.screencastFrameAck" && child === undefined) { + // A restart keeps the slot too, or restarting would let frames past an unacknowledged one. + const sessionId = command.sessionId!; + if (unackedFrames.get(sessionId) === command.params?.["sessionId"]) { + unackedFrames.delete(sessionId); + const skipped = skippedFrames.get(sessionId); + skippedFrames.delete(sessionId); + if ( + skipped !== undefined && + target.receivesScreencast(sessionId === PAGE_SESSION_ID ? undefined : sessionId) + ) + sendFrame(sessionId, skipped); + const released = heldNativeFrames.shift(); + if (released !== undefined) target.ackScreencastFrame(released); + } + return Promise.resolve({}); + } + if (command.method === "Page.stopScreencast" && child === undefined) { + skippedFrames.delete(command.sessionId!); + releaseHeldFramesWithoutHolders(); + } // The shared debugger keeps Runtime enabled after an earlier connection, and a repeated // enable reports no existing contexts; Playwright waits for them, so reset once per connection. if (command.method === "Runtime.enable" && child === undefined) { @@ -260,6 +302,27 @@ export function createCdpRelayConnection( send({ method, params, sessionId }); return; } + if (method === "Page.screencastFrame") { + const frame = (params ?? {}) as Record; + const frameId = frame["sessionId"]; + if (typeof frameId !== "number") return; + let held = false; + let delivered = false; + for (const [id, kind] of sessions) { + if (kind !== "page") continue; + if (!target.receivesScreencast(id === PAGE_SESSION_ID ? undefined : id)) continue; + if (unackedFrames.has(id)) { + skippedFrames.set(id, frame); + held = true; + } else { + sendFrame(id, frame); + delivered = true; + } + } + if (held && !delivered) heldNativeFrames.push(frameId); + else target.ackScreencastFrame(frameId); + return; + } for (const [id, kind] of sessions) if (kind === "page") send({ method, params, sessionId: id }); }, diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index 94809f0d10cfa..8e96129293fc8 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -16,7 +16,21 @@ const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); const decodeCdpReply = Schema.decodeUnknownSync( Schema.fromJsonString(Schema.Struct({ id: Schema.Number })), ); +const decodeCdpMessage = Schema.decodeUnknownSync( + Schema.fromJsonString( + Schema.Struct({ + id: Schema.optional(Schema.Number), + method: Schema.optional(Schema.String), + sessionId: Schema.optional(Schema.String), + }), + ), +); const key = { threadId: "thread-1", tabId: "tab-1" }; + +/** A host whose client already draws `key` at full detail, so its screenshots go out at once. */ +const makeDrawnHost = Effect.tap(DesktopBrowserHost.make, (host) => + Effect.sync(() => host.setCaptureDrawn(key, true)), +); /** Sends a page command whose id the fake debugger records as it receives it. */ const command = ( host: DesktopBrowserHost.DesktopBrowserHost["Service"], @@ -43,6 +57,8 @@ const makeDebuggee = ( const pending: Array<() => void> = []; const focusChanges: Array<() => void> = []; const sent: Array = []; + const methods: Array = []; + const rejects: Array<() => void> = []; const sendWaiters: Array<() => void> = []; const focusEmulation: Array = []; const debuggee = Object.assign(emitter, { @@ -67,6 +83,8 @@ const makeDebuggee = ( return; } sent.push(params?.id); + methods.push(method); + rejects.push(() => reject(new Error("Refused"))); pending.push(() => resolve({ method })); sendWaiters.shift()?.(); }), @@ -94,6 +112,10 @@ const makeDebuggee = ( webContents.emit("did-fail-load", {}, code, "Failed", "http://localhost/", true), /** Ids of the commands the debugger received, in order. */ sent, + /** Methods of the same commands, in order. */ + methods, + /** Refuses the pending command the debugger received with this id. */ + refuse: (id: number) => rejects[sent.indexOf(id)]?.(), focusEmulation, nextSend: () => new Promise((resolve) => sendWaiters.push(resolve)), release: () => pending.splice(0).forEach((resolve) => resolve()), @@ -116,16 +138,24 @@ const takeEvents = (host: DesktopBrowserHost.DesktopBrowserHost["Service"], coun /** * Ids of the replies the host sends, in order. A screenshot's reply leaves after its turn * has passed on, so by then the next waiting screenshot, if any, has been sent. Subscribe - * before sending commands: a new subscriber releases the tab's relay. + * before sending commands: a new subscriber releases the tab's relay. Screencast frames + * go to `frames` by the session they were sent to. */ -const watchReplies = (host: DesktopBrowserHost.DesktopBrowserHost["Service"]) => +const watchReplies = ( + host: DesktopBrowserHost.DesktopBrowserHost["Service"], + frames: Array = [], +) => Effect.gen(function* () { const replies = yield* Queue.unbounded(); yield* host.events.pipe( Stream.map((line) => decodeEvent(new TextDecoder().decode(line))), - Stream.runForEach((event) => - event.type === "cdp" ? Queue.offer(replies, decodeCdpReply(event.message).id) : Effect.void, - ), + Stream.runForEach((event) => { + if (event.type !== "cdp") return Effect.void; + const { id, method, sessionId } = decodeCdpMessage(event.message); + if (id !== undefined) return Queue.offer(replies, id); + if (method === "Page.screencastFrame") frames.push(sessionId); + return Effect.void; + }), Effect.forkScoped, ); yield* Effect.yieldNow; @@ -147,7 +177,7 @@ const watchPainting = (host: DesktopBrowserHost.DesktopBrowserHost["Service"]) = describe("DesktopBrowserHost", () => { it.effect("announces tabs already attached to a backend that starts later", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; host.attach(key, makeDebuggee().tab); expect(yield* takeEvents(host, 1)).toEqual([{ type: "attached", ...key }]); @@ -157,7 +187,7 @@ describe("DesktopBrowserHost", () => { it.effect("announces a guest still loading its first page only once that page is up", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const loading = makeDebuggee({ loading: true }); const other = { ...key, tabId: "tab-2" }; const reader = yield* takeEvents(host, 3).pipe(Effect.forkScoped); @@ -180,7 +210,7 @@ describe("DesktopBrowserHost", () => { "keeps a loading guest unannounced across a replaced navigation and a new backend", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const loading = makeDebuggee({ loading: true }); const other = { ...key, tabId: "tab-2" }; host.attach(key, loading.tab); @@ -202,7 +232,7 @@ describe("DesktopBrowserHost", () => { "announces a first page that never comes up, but never a tab closed or destroyed first", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const closed = { ...key, tabId: "tab-2" }; const destroyed = { ...key, tabId: "tab-3" }; const destroyedGuest = makeDebuggee({ loading: true }); @@ -222,7 +252,7 @@ describe("DesktopBrowserHost", () => { it.effect("re-sends shown to a backend that starts later and when a tab attaches again", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; host.attach(key, makeDebuggee().tab); host.setShown(key, true); const reader = yield* takeEvents(host, 5).pipe(Effect.forkScoped); @@ -243,7 +273,7 @@ describe("DesktopBrowserHost", () => { it.effect("replays an explicit hidden report before announcing another tab", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const other = { ...key, tabId: "tab-2" }; host.setShown(key, false); host.attach(key, makeDebuggee().tab); @@ -260,7 +290,7 @@ describe("DesktopBrowserHost", () => { it.effect("drops replies from a relay the server released", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const reader = yield* takeEvents(host, 2).pipe(Effect.forkScoped); @@ -283,12 +313,12 @@ describe("DesktopBrowserHost", () => { expect(reply).toMatchObject({ type: "cdp" }); expect(decodeCdpReply((reply as { message: string }).message).id).toBe(2); - }).pipe(Effect.scoped), + }), ); it.effect("sends a page one screenshot at a time", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -302,12 +332,41 @@ describe("DesktopBrowserHost", () => { debuggee.releaseFirst(); expect(yield* Queue.take(replies)).toBe(1); expect(debuggee.sent).toEqual([1, 3, 2]); - }).pipe(Effect.scoped), + }), + ); + + it.effect( + "sends a screenshot once the client draws the guest at full detail, or soon after", + () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + + yield* command(host, 1, "Page.captureScreenshot"); + yield* settle; + expect(debuggee.sent).toEqual([]); + host.setCaptureDrawn(key, true); + yield* settle; + expect(debuggee.sent).toEqual([1]); + debuggee.releaseFirst(); + expect(yield* Queue.take(replies)).toBe(1); + + // A client that never draws, such as a hidden window, only delays the capture. + host.setCaptureDrawn(key, false); + yield* command(host, 2, "Page.captureScreenshot"); + yield* settle; + expect(debuggee.sent).toEqual([1]); + yield* TestClock.adjust("1 second"); + yield* settle; + expect(debuggee.sent).toEqual([1, 2]); + }).pipe(Effect.scoped), ); it.effect("stops waiting on a screenshot the page navigated away from", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -327,12 +386,12 @@ describe("DesktopBrowserHost", () => { debuggee.releaseFirst(); expect(yield* Queue.take(replies)).toBe(2); expect(debuggee.sent).toEqual([1, 2, 3]); - }).pipe(Effect.scoped), + }), ); it.effect("drops screenshots a released relay left waiting", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); @@ -352,7 +411,7 @@ describe("DesktopBrowserHost", () => { it.effect("emulates focus while the server drives a guest, then lets a hidden one throttle", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); @@ -383,7 +442,7 @@ describe("DesktopBrowserHost", () => { it.effect("keeps a guest focused while a command it is running is still pending", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -399,12 +458,12 @@ describe("DesktopBrowserHost", () => { expect(yield* Queue.take(replies)).toBe(1); yield* TestClock.adjust("1 minute"); expect(debuggee.focusEmulation).toEqual([true, false]); - }).pipe(Effect.scoped), + }), ); it.effect("lets a guest throttle once Chromium can no longer answer its pending command", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); @@ -426,7 +485,7 @@ describe("DesktopBrowserHost", () => { it.effect("keeps a guest focused for the screenshot a navigation lets start", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); @@ -442,7 +501,7 @@ describe("DesktopBrowserHost", () => { it.effect("never sends a screenshot a navigation superseded while focus was changing", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee({ deferFocus: true }); host.attach(key, debuggee.tab); @@ -459,7 +518,7 @@ describe("DesktopBrowserHost", () => { it.effect("keeps a guest focused when its turn starts after the agent's first command", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const turn = (active: boolean) => @@ -481,7 +540,7 @@ describe("DesktopBrowserHost", () => { it.effect("lets a guest throttle through a turn that only sent it housekeeping", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); yield* host.handleCommandLine( @@ -510,7 +569,7 @@ describe("DesktopBrowserHost", () => { it.effect("keeps a guest the agent used focused through its turn's pauses", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const turn = (active: boolean) => @@ -538,7 +597,7 @@ describe("DesktopBrowserHost", () => { "ends a command's hold with the turn that sent it, even if Chromium never answers", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); yield* host.handleCommandLine( @@ -559,7 +618,7 @@ describe("DesktopBrowserHost", () => { it.effect("ends the agent's own session's holds with its turn, but not a viewer's stream", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -587,12 +646,12 @@ describe("DesktopBrowserHost", () => { yield* turn(false); yield* TestClock.adjust("1 minute"); expect(debuggee.focusEmulation).toEqual([true, false, true]); - }).pipe(Effect.scoped), + }), ); it.effect("paints for a screenshot queued behind one its viewer abandoned mid-capture", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -625,12 +684,12 @@ describe("DesktopBrowserHost", () => { expect(yield* Queue.take(replies)).toBe(4); yield* settle; expect(painting).toEqual([true, false]); - }).pipe(Effect.scoped), + }), ); it.effect("stops painting for a queued screenshot once its session detaches", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -664,12 +723,12 @@ describe("DesktopBrowserHost", () => { expect(painting).toEqual([true, false]); yield* TestClock.adjust("1 minute"); expect(debuggee.focusEmulation).toEqual([true, false]); - }).pipe(Effect.scoped), + }), ); it.effect("never sends a screenshot whose session detached while focus was changing", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee({ deferFocus: true }); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -688,12 +747,12 @@ describe("DesktopBrowserHost", () => { debuggee.settleFocus(); yield* settle; expect(debuggee.sent).toEqual([]); - }).pipe(Effect.scoped), + }), ); it.effect("holds no focus for a screenshot still queued when the agent's turn ends", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const painting = yield* watchPainting(host); @@ -718,12 +777,12 @@ describe("DesktopBrowserHost", () => { debuggee.release(); yield* settle; expect(painting).toEqual([true, false]); - }).pipe(Effect.scoped), + }), ); it.effect("lets a guest throttle when Chromium refuses a viewer's screencast", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee({ refuse: (method) => method === "Page.startScreencast" }); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -739,12 +798,12 @@ describe("DesktopBrowserHost", () => { expect(painting).toEqual([true, false]); yield* TestClock.adjust("1 minute"); expect(debuggee.focusEmulation).toEqual([true, false]); - }).pipe(Effect.scoped), + }), ); it.effect("lets a guest throttle after a person's input through a viewer that then closed", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -768,12 +827,12 @@ describe("DesktopBrowserHost", () => { yield* settle; yield* TestClock.adjust("1 minute"); expect(debuggee.focusEmulation).toEqual([true, false]); - }).pipe(Effect.scoped), + }), ); it.effect("keeps a page's screencast when a child session's screencast is refused", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee({ refuse: (method, sessionId) => method === "Page.startScreencast" && sessionId !== undefined, }); @@ -790,12 +849,12 @@ describe("DesktopBrowserHost", () => { expect(yield* Queue.take(replies)).toBe(2); yield* TestClock.adjust("1 minute"); expect(debuggee.focusEmulation).toEqual([true]); - }).pipe(Effect.scoped), + }), ); it.effect("ignores a screencast that arrives on a session after it detached", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -816,12 +875,12 @@ describe("DesktopBrowserHost", () => { yield* settle; yield* TestClock.adjust("1 minute"); expect(debuggee.focusEmulation).toEqual([true, false]); - }).pipe(Effect.scoped), + }), ); it.effect("lets a guest throttle once the session waiting on its command detaches", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -843,12 +902,12 @@ describe("DesktopBrowserHost", () => { }); yield* TestClock.adjust("1 minute"); expect(debuggee.focusEmulation).toEqual([true, false]); - }).pipe(Effect.scoped), + }), ); it.effect("never sends a screenshot queued by a session that has since detached", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -874,12 +933,12 @@ describe("DesktopBrowserHost", () => { yield* TestClock.adjust("1 minute"); expect(debuggee.focusEmulation).toEqual([true, false]); - }).pipe(Effect.scoped), + }), ); it.effect("keeps a streamed guest focused until its screencast ends", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); @@ -908,7 +967,7 @@ describe("DesktopBrowserHost", () => { it.effect("keeps a guest focused while any of its screencasts runs", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -935,12 +994,12 @@ describe("DesktopBrowserHost", () => { }); yield* TestClock.adjust("1 minute"); expect(debuggee.focusEmulation).toEqual([true, false]); - }).pipe(Effect.scoped), + }), ); it.effect("paints a hidden guest only while the screenshot it is taking is pending", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const painting = yield* watchPainting(host); @@ -961,12 +1020,12 @@ describe("DesktopBrowserHost", () => { debuggee.release(); yield* settle; expect(painting).toEqual([true, false]); - }).pipe(Effect.scoped), + }), ); it.effect("stops painting for a screenshot Chromium never answers once its document goes", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const painting = yield* watchPainting(host); @@ -1006,12 +1065,12 @@ describe("DesktopBrowserHost", () => { host.detach(key); yield* settle; expect(painting).toEqual([true, false, true, false, true, false, true, false]); - }).pipe(Effect.scoped), + }), ); it.effect("paints a streamed guest until its last screencast ends", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -1037,14 +1096,190 @@ describe("DesktopBrowserHost", () => { }); yield* settle; expect(painting).toEqual([true, false]); - }).pipe(Effect.scoped), + }), ); - it.effect("saves a server tab's download under its CDP guid where the server asked", () => + it.effect("keeps a guest's one screencast running until its last streaming session stops", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); + const frames: Array = []; + const replies = yield* watchReplies(host, frames); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + const nativeStops = () => + debuggee.methods.filter((method) => method === "Page.stopScreencast").length; + + // A viewer and a recording share Chromium's single screencast for the guest. + yield* raw({ id: 0, method: "Target.setAutoAttach", params: { autoAttach: true } }); + expect(yield* Queue.take(replies)).toBe(0); + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + yield* raw({ id: 2, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect([yield* Queue.take(replies), yield* Queue.take(replies)]).toEqual([1, 2]); + yield* raw({ id: 3, method: "Page.startScreencast", sessionId: "t3-preview-page-1" }); + yield* raw({ id: 4, method: "Page.startScreencast", sessionId: "t3-preview-page-2" }); + yield* settle; + debuggee.release(); + yield* raw({ id: 5, method: "Page.stopScreencast", sessionId: "t3-preview-page-1" }); + yield* settle; + expect(nativeStops(), "stopped the screencast another session still streams").toBe(0); + debuggee.emit("Page.screencastFrame", { data: "frame", sessionId: 1 }); + yield* settle; + expect(frames, "sent a stopped or idle session the screencast's frames").toEqual([ + "t3-preview-page-2", + ]); + + yield* raw({ id: 6, method: "Page.stopScreencast", sessionId: "t3-preview-page-2" }); + yield* settle; + expect(nativeStops()).toBe(1); + }), + ); + + it.effect("keeps a session's screencast when another of its starts is refused", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const frames: Array = []; + const replies = yield* watchReplies(host, frames); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + + yield* raw({ id: 0, method: "Target.setAutoAttach", params: { autoAttach: true } }); + expect(yield* Queue.take(replies)).toBe(0); + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(1); + yield* raw({ + id: 2, + method: "Page.startScreencast", + params: { id: 2 }, + sessionId: "t3-preview-page-1", + }); + yield* raw({ + id: 3, + method: "Page.startScreencast", + params: { id: 3 }, + sessionId: "t3-preview-page-1", + }); + yield* settle; + debuggee.refuse(2); + debuggee.release(); + expect([yield* Queue.take(replies), yield* Queue.take(replies)].toSorted()).toEqual([2, 3]); + yield* settle; + expect(debuggee.methods).not.toContain("Page.stopScreencast"); + debuggee.emit("Page.screencastFrame", { data: "frame", sessionId: 1 }); + yield* settle; + expect(frames).toContain("t3-preview-page-1"); + + // A refused replacement leaves the session's running stream, shared with another session. + yield* raw({ id: 4, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(4); + yield* raw({ + id: 5, + method: "Page.startScreencast", + params: { id: 5 }, + sessionId: "t3-preview-page-2", + }); + yield* raw({ + id: 6, + method: "Page.startScreencast", + params: { id: 6 }, + sessionId: "t3-preview-page-1", + }); + yield* settle; + debuggee.refuse(6); + debuggee.release(); + expect([yield* Queue.take(replies), yield* Queue.take(replies)].toSorted()).toEqual([5, 6]); + yield* raw({ + id: 7, + method: "Page.screencastFrameAck", + params: { sessionId: 1 }, + sessionId: "t3-preview-page-1", + }); + expect(yield* Queue.take(replies)).toBe(7); + yield* settle; + frames.length = 0; + debuggee.emit("Page.screencastFrame", { data: "frame", sessionId: 2 }); + yield* settle; + expect(frames).toEqual(["t3-preview-page-1", "t3-preview-page-2"]); + }), + ); + + it.effect("stops a guest's screencast once its last streaming session detaches", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + const nativeStops = () => + debuggee.methods.filter((method) => method === "Page.stopScreencast").length; + + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + yield* raw({ id: 2, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect([yield* Queue.take(replies), yield* Queue.take(replies)]).toEqual([1, 2]); + yield* raw({ id: 3, method: "Page.startScreencast", sessionId: "t3-preview-page-1" }); + yield* raw({ id: 4, method: "Page.startScreencast", sessionId: "t3-preview-page-2" }); + yield* settle; + debuggee.release(); + // Streamers go without stopping, such as viewers that closed. + yield* raw({ + id: 5, + method: "Target.detachFromTarget", + params: { sessionId: "t3-preview-page-1" }, + }); + yield* settle; + expect(nativeStops(), "stopped the screencast another session still streams").toBe(0); + yield* raw({ + id: 6, + method: "Target.detachFromTarget", + params: { sessionId: "t3-preview-page-2" }, + }); + const answered = []; + for (let reply = 0; reply < 4; reply++) answered.push(yield* Queue.take(replies)); + expect(answered.toSorted()).toEqual([3, 4, 5, 6]); + yield* settle; + expect(nativeStops()).toBe(1); + }), + ); + + it.effect("stops a screencast whose start lands after its session detached", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee({ deferFocus: true }); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(1); + // The start waits on focus emulation while the viewer closes. + yield* raw({ id: 2, method: "Page.startScreencast", sessionId: "t3-preview-page-1" }); + yield* raw({ + id: 3, + method: "Target.detachFromTarget", + params: { sessionId: "t3-preview-page-1" }, + }); + expect(yield* Queue.take(replies)).toBe(3); + debuggee.settleFocus(); + yield* settle; + debuggee.release(); + expect(yield* Queue.take(replies)).toBe(2); + yield* settle; + expect(debuggee.methods.at(-1), "left the late screencast running").toBe( + "Page.stopScreencast", + ); + }), + ); + + it.effect("saves a server tab's download under its CDP guid where the server asked", () => + Effect.gen(function* () { + const host = yield* makeDrawnHost; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); const paths: Array = []; const item = { @@ -1072,12 +1307,12 @@ describe("DesktopBrowserHost", () => { expect(host.placeDownload(debuggee.tab.webContents, item)).toBe(true); expect(paths).toEqual(["/srv/downloads/guid-1"]); expect(host.placeDownload(debuggee.tab.webContents, item)).toBe(false); - }).pipe(Effect.scoped), + }), ); it.effect("tells a download the person clicked from one the agent's input started", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); @@ -1104,9 +1339,44 @@ describe("DesktopBrowserHost", () => { }), ); - it.effect("detaches only the tab whose CDP event is too large to relay", () => + it.effect("stops crediting the agent once a person acts through a viewer", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + const viewer = (id: number, method: string, type: string) => + raw({ id, method, params: { type }, sessionId: "t3-preview-page-1" }); + const led = () => host.agentLedPage(debuggee.tab.webContents); + + yield* command(host, 1, "DOM.getDocument"); + expect(led()).toBe(false); + yield* raw({ id: 2, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(2); + yield* raw({ id: 3, method: "Page.startScreencast", sessionId: "t3-preview-page-1" }); + + for (const [method, type] of [ + ["Input.dispatchKeyEvent", "rawKeyDown"], + ["Input.dispatchMouseEvent", "mousePressed"], + ["Input.dispatchTouchEvent", "touchStart"], + ] as const) { + yield* command(host, 4, "Runtime.evaluate"); + expect(led()).toBe(true); + // The viewer's pointer only passes over the page the agent drives. + yield* viewer(5, "Input.dispatchMouseEvent", "mouseMoved"); + yield* viewer(6, "Input.dispatchMouseEvent", "mouseWheel"); + expect(led()).toBe(true); + yield* viewer(7, method, type); + expect(led()).toBe(false); + } + }).pipe(Effect.scoped), + ); + + it.effect("detaches only the tab whose CDP event is too large to relay", () => + Effect.gen(function* () { + const host = yield* makeDrawnHost; const tabs = [makeDebuggee({ deferFocus: true }), makeDebuggee()]; const send = vi.spyOn(tabs[0]!.tab.debugger, "sendCommand"); const keys = [key, { threadId: "thread-2", tabId: "tab-2" }]; @@ -1170,6 +1440,6 @@ describe("DesktopBrowserHost", () => { expect(JSON.parse((next as { message: string }).message)).toMatchObject({ method: "Page.loadEventFired", }); - }).pipe(Effect.scoped), + }), ); }); diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index a53fc0e3e6ec2..ce5d7074a786c 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -26,8 +26,12 @@ const AGENT_INPUT_COMMAND = /"method":"(?:Input\.|Page\.navigate"|Runtime\.(?:evaluate|callFunctionOn)")/; /** The same commands as above, by method, for the relay's sends. */ const AGENT_INPUT_METHOD = /^(?:Input\.|Page\.navigate$|Runtime\.(?:evaluate|callFunctionOn)$)/; -const AGENT_DOWNLOAD_WINDOW_MS = 5_000; +const AGENT_INPUT_WINDOW_MS = 5_000; +/** A viewer's pointer travelling over the stream activates nothing on the page. */ +const POINTER_TRAVEL = new Set(["mouseMoved", "mouseWheel"]); const FOCUS_IDLE = "5 seconds"; +/** How long a screenshot waits for the client to draw its guest at full detail. */ +const CAPTURE_DRAW_WAIT = "1 second"; /** Well inside the server's 10 s DESKTOP_ATTACH_TIMEOUT, for a first page that never reports it is up. */ const FIRST_PAGE_TIMEOUT = "5 seconds"; const ERR_ABORTED = -3; @@ -60,6 +64,8 @@ interface AttachedTab { downloadDirectory: string | null; pendingDownloadGuid: string | null; agentInputAt: number; + /** When the agent last drove the page with no person's input through a viewer since. */ + agentLedAt: number; focusEmulated: boolean; focusChange: Promise | null; focusUses: number; @@ -75,8 +81,8 @@ interface AttachedTab { */ agentUsed: boolean; focusReleasing: boolean; - /** The sessions streaming the guest; one stopping leaves the others' screencasts running. */ - readonly screencasts: Set; + /** The sessions sharing the guest's one Chromium screencast; it stops with the last of them. */ + readonly screencasts: Map; readonly onMessage: ( event: Electron.Event, method: string, @@ -97,7 +103,11 @@ export class DesktopBrowserHost extends Context.Service< readonly detach: (key: DesktopBrowserTabKey) => void; readonly loadFailed: (key: DesktopBrowserTabKey, failure: PreviewAutomationLoadError) => void; readonly setShown: (key: DesktopBrowserTabKey, shown: boolean | undefined) => void; + /** Whether the client draws the guest at full detail while it is captured. */ + readonly setCaptureDrawn: (key: DesktopBrowserTabKey, drawn: boolean | undefined) => void; readonly humanStartedDownload: (source: Electron.WebContents) => boolean; + /** Whether what the page does now follows the agent's input, not a viewer's. */ + readonly agentLedPage: (source: Electron.WebContents) => boolean; readonly placeDownload: (source: Electron.WebContents, item: Electron.DownloadItem) => boolean; readonly pointers: Stream.Stream<{ readonly key: DesktopBrowserTabKey; @@ -132,6 +142,8 @@ export const make = Effect.gen(function* () { // Kept apart from attachment: the renderer can show a webview before its debugger attaches, // and the server forgets `shown` on detach, for example while DevTools holds the debugger. const shownTabs = new Map(); + const drawnTabs = new Set(); + const drawWaiters = new Map void>>(); const emit = (event: DesktopBrowserEventType) => runFork(PubSub.publish(outbox, event)); const nextScreenshot = (tab: AttachedTab) => { @@ -260,12 +272,32 @@ export const make = Effect.gen(function* () { }); }; + /** + * A scaled guest rasters at its on-screen size until the client draws it at full detail, + * and a screenshot taken before that copies the blurred frame. A client that cannot draw + * (a hidden window, say) only delays the capture. + */ + const whenDrawn = (tab: AttachedTab) => + new Promise((resolve) => { + const id = keyOf(tab.key); + const waiters = drawWaiters.get(id) ?? new Set(); + drawWaiters.set(id, waiters); + const done = () => { + waiters.delete(done); + if (waiters.size === 0 && drawWaiters.get(id) === waiters) drawWaiters.delete(id); + resolve(); + }; + waiters.add(done); + runFork(Effect.sleep(CAPTURE_DRAW_WAIT).pipe(Effect.andThen(Effect.sync(done)))); + }); + /** Chromium answers a capture, and frees its wake lock, only once the guest paints a frame. */ const paintUntilAnswered = (tab: AttachedTab, send: () => Promise) => { const sent = {}; tab.capturing.add(sent); syncPainting(tab); - return send().finally(() => { + // Sent at once when already drawn, so it keeps its place among the commands after it. + return (drawnTabs.has(keyOf(tab.key)) ? send() : whenDrawn(tab).then(send)).finally(() => { tab.capturing.delete(sent); syncPainting(tab); }); @@ -289,9 +321,15 @@ export const make = Effect.gen(function* () { dropFocusHolds(tab); }; + /** An owner that leaves without stopping would otherwise keep the screencast running unseen. */ + const leaveScreencast = (tab: AttachedTab, owner: string | undefined) => { + if (!tab.screencasts.delete(owner) || tab.screencasts.size > 0) return; + void tab.debuggee.debugger.sendCommand("Page.stopScreencast").catch(() => {}); + }; + /** A detached session, such as a closed viewer, waits on nothing it sent. */ const dropSessionHolds = (tab: AttachedTab, owner: string) => { - tab.screencasts.delete(owner); + leaveScreencast(tab, owner); for (const [pending, holder] of tab.focusPending) { if (holder === owner) tab.focusPending.delete(pending); } @@ -330,30 +368,41 @@ export const make = Effect.gen(function* () { { send: (method, params, sessionId, owner) => { // A child session's screencast is not the page's, such as a late one on a detached session. - if (sessionId === undefined && method === "Page.startScreencast") { - tab.screencasts.add(owner); + const stream = + sessionId === undefined && method === "Page.startScreencast" + ? (tab.screencasts.get(owner) ?? { started: false, pending: 0 }) + : undefined; + if (stream) { + stream.pending += 1; + tab.screencasts.set(owner, stream); if (owner !== undefined) streamingOwners.add(owner); syncPainting(tab); } else if (sessionId === undefined && method === "Page.stopScreencast") { tab.screencasts.delete(owner); syncPainting(tab); + if (tab.screencasts.size > 0) return Promise.resolve({}); } // A person's input through a viewer, or Playwright's own housekeeping, is not the agent's. - if ( - AGENT_INPUT_METHOD.test(method) && - (owner === undefined || !streamingOwners.has(owner)) - ) { - tab.agentUsed = true; + if (AGENT_INPUT_METHOD.test(method)) { + if (owner === undefined || !streamingOwners.has(owner)) { + tab.agentUsed = true; + tab.agentLedAt = performance.now(); + } else if (method.startsWith("Input.") && !POINTER_TRAVEL.has(params["type"])) { + tab.agentLedAt = Number.NEGATIVE_INFINITY; + } } // Waits for focus emulation; insertText and other non-CDP input would not queue behind it. const forward = (current?: () => boolean) => { const change = tab.focusChange; - const run = () => - tab.relay !== relay || (current !== undefined && !current()) + const stale = () => tab.relay !== relay || (current !== undefined && !current()); + const sendUnlessStale = () => + stale() ? Promise.reject(new Error("The page this screenshot was for has moved on.")) - : current === undefined - ? send(method, params, sessionId) - : paintUntilAnswered(tab, () => send(method, params, sessionId)); + : send(method, params, sessionId); + const run = () => + current === undefined || stale() + ? sendUnlessStale() + : paintUntilAnswered(tab, sendUnlessStale); return change === null ? run() : change.then(run); }; return method === "Page.captureScreenshot" @@ -365,19 +414,46 @@ export const make = Effect.gen(function* () { held ? holdFocus(tab, owner, () => forward(current)) : forward(current), ) : method === "Page.startScreencast" - ? holdFocus(tab, owner, forward).catch((cause: unknown) => { - // A viewer can outlive a refused stream; nothing is streaming for it. - if (sessionId === undefined && tab.relay === relay) tab.screencasts.delete(owner); - releaseFocusWhenIdle(tab); - syncPainting(tab); - throw cause; - }) + ? holdFocus(tab, owner, forward).then( + (result) => { + if (stream) { + stream.pending -= 1; + stream.started = true; + // A detach's stop can overtake a start still waiting on focus; nobody owns it now. + if (tab.screencasts.get(owner) !== stream && tab.screencasts.size === 0) { + void debuggee.sendCommand("Page.stopScreencast").catch(() => {}); + } + } + return result; + }, + (cause: unknown) => { + if (stream) stream.pending -= 1; + // A viewer can outlive a refused start; its running or pending stream still holds it. + if ( + stream && + !stream.started && + stream.pending === 0 && + tab.relay === relay && + tab.screencasts.get(owner) === stream + ) { + leaveScreencast(tab, owner); + } + releaseFocusWhenIdle(tab); + syncPainting(tab); + throw cause; + }, + ) : holdFocus(tab, owner, forward); }, sessionDetached: (owner) => { detachedOwners.add(owner); dropSessionHolds(tab, owner); }, + receivesScreencast: (owner) => tab.screencasts.has(owner), + ackScreencastFrame: (frameId) => + void debuggee + .sendCommand("Page.screencastFrameAck", { sessionId: frameId }) + .catch(() => {}), targetId: () => debuggee .sendCommand("Target.getTargetInfo") @@ -455,6 +531,7 @@ export const make = Effect.gen(function* () { downloadDirectory: null, pendingDownloadGuid: null, agentInputAt: Number.NEGATIVE_INFINITY, + agentLedAt: Number.NEGATIVE_INFINITY, focusEmulated: false, focusChange: null, focusUses: 0, @@ -463,7 +540,7 @@ export const make = Effect.gen(function* () { painting: false, agentUsed: false, focusReleasing: false, - screencasts: new Set(), + screencasts: new Map(), onMessage: (_event, method, params, sessionId) => { if (method === "Page.downloadWillBegin") { const guid = (params as { guid?: unknown } | undefined)?.guid; @@ -539,6 +616,17 @@ export const make = Effect.gen(function* () { if (tabs.get(id)?.stopWaiting === null) emit({ type: "shown", ...key, shown }); }; + const setCaptureDrawn = (key: DesktopBrowserTabKey, drawn: boolean | undefined) => { + const id = keyOf(key); + if (!drawn) { + drawnTabs.delete(id); + return; + } + drawnTabs.add(id); + for (const done of drawWaiters.get(id) ?? []) done(); + drawWaiters.delete(id); + }; + const loadFailed: DesktopBrowserHost["Service"]["loadFailed"] = (key, failure) => { if (tabs.has(keyOf(key))) emit({ type: "loadFailed", ...key, ...failure }); }; @@ -569,7 +657,12 @@ export const make = Effect.gen(function* () { const humanStartedDownload = (source: Electron.WebContents) => { const tab = [...tabs.values()].find((candidate) => candidate.debuggee.webContents === source); - return tab !== undefined && performance.now() - tab.agentInputAt > AGENT_DOWNLOAD_WINDOW_MS; + return tab !== undefined && performance.now() - tab.agentInputAt > AGENT_INPUT_WINDOW_MS; + }; + + const agentLedPage = (source: Electron.WebContents) => { + const tab = [...tabs.values()].find((candidate) => candidate.debuggee.webContents === source); + return tab !== undefined && performance.now() - tab.agentLedAt <= AGENT_INPUT_WINDOW_MS; }; const announceAll = Effect.suspend(() => { @@ -610,8 +703,10 @@ export const make = Effect.gen(function* () { detach, loadFailed, setShown, + setCaptureDrawn, placeDownload, humanStartedDownload, + agentLedPage, }); }); diff --git a/apps/desktop/src/preview/GuestInput.test.ts b/apps/desktop/src/preview/GuestInput.test.ts index a0889940a6682..48191ce2c4b98 100644 --- a/apps/desktop/src/preview/GuestInput.test.ts +++ b/apps/desktop/src/preview/GuestInput.test.ts @@ -44,6 +44,30 @@ describe("preview guest input", () => { expect(guest.sendCommand).not.toHaveBeenCalledWith("Input.insertText", expect.anything()); }); + it("stops the guest navigation without stopping its embedder", async () => { + let guestLoading = true; + let hostLoading = true; + const guest = { + stop: () => { + guestLoading = false; + }, + }; + const debuggee = { + sendCommand: async () => { + hostLoading = false; + return {}; + }, + }; + await sendPageCommand( + guest as unknown as Electron.WebContents, + debuggee as unknown as Electron.Debugger, + "Page.stopLoading", + {}, + ); + expect(guestLoading).toBe(false); + expect(hostLoading).toBe(true); + }); + it.each([ { sessionId: undefined, diff --git a/apps/desktop/src/preview/GuestInput.ts b/apps/desktop/src/preview/GuestInput.ts index 2af13af8d7daa..4d794beea62bc 100644 --- a/apps/desktop/src/preview/GuestInput.ts +++ b/apps/desktop/src/preview/GuestInput.ts @@ -185,6 +185,10 @@ export const sendPageCommand = async ( else webContents.reload(); return {}; } + if (method === "Page.stopLoading") { + webContents.stop(); + return {}; + } if (!FOCUS_ROUTED_INPUT.has(method) || webContents.focusedFrame) { return debuggee.sendCommand(method, params); } diff --git a/apps/desktop/src/preview/GuestProtocol.ts b/apps/desktop/src/preview/GuestProtocol.ts index 09d4eff6d2013..1d24079d1947f 100644 --- a/apps/desktop/src/preview/GuestProtocol.ts +++ b/apps/desktop/src/preview/GuestProtocol.ts @@ -5,13 +5,16 @@ export const ANNOTATION_CAPTURED_CHANNEL = "preview:annotation-captured"; export const ANNOTATION_THEME_CHANNEL = "preview:annotation-theme"; export const ANNOTATION_SEND_ENABLED_CHANNEL = "preview:annotation-send-enabled"; export const HUMAN_INPUT_CHANNEL = "preview:human-input"; -export const MOUSE_NAVIGATE_CHANNEL = "preview:mouse-navigate"; +// Sent to the hosting element (ipc-message). The web app mirrors this +// literal in HostedBrowserWebview because it cannot import from the desktop app. +export const POINTER_DOWN_HOST_CHANNEL = "preview:pointer-down"; export const PAGE_UNHANDLED_SHORTCUT_CHANNEL = "preview:page-unhandled-shortcut"; export const RECORDING_CURSOR_CHANNEL = "preview:recording-cursor"; export const RECORDING_POINTER_CHANNEL = "preview:recording-pointer"; export const RECORDING_KEY_CHANNEL = "preview:recording-key"; export const RECORDING_INPUT_CHANNEL = "preview:recording-input"; export const RECORDING_CONTROLLER_CHANNEL = "preview:recording-controller"; +export const THUMB_PRESS_CHANNEL = "preview:thumb-press"; export const PASSKEY_CREATE_CHANNEL = "preview:passkey-create"; export const PASSKEY_GET_CHANNEL = "preview:passkey-get"; export const PASSKEY_BRIDGE_ARGUMENT = "--t3code-preview-passkey-bridge"; diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 6db04b389ab6d..2091ff3c7f8f2 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -349,6 +349,47 @@ const makeTestPreviewWebContents = ( } as unknown as TestPreviewWebContents; }; +const makeStillImage = (pixels = "fresh-png") => ({ + toPNG: () => Buffer.from(pixels), + toJPEG: () => Buffer.from(pixels), + getSize: () => ({ width: 100, height: 80 }), +}); + +const makeTestStillImageWebContents = (...args: Parameters) => { + const wc = makeTestPreviewWebContents(...args); + Object.assign(wc, { isDevToolsOpened: () => false }); + holdStillImagePaint(wc, Promise.resolve()); + return wc; +}; + +const holdStillImagePaint = ( + wc: TestPreviewWebContents, + paint: Promise | Array>, +) => { + vi.mocked(wc.debugger.sendCommand).mockImplementation(async (method) => { + if (method === "Page.getFrameTree") return { frameTree: { frame: { id: "main" } } }; + if (method === "Page.createIsolatedWorld") return { executionContextId: 7 }; + if (method === "Runtime.evaluate") { + await (Array.isArray(paint) ? paint.shift() : paint); + return { result: { value: true } }; + } + return undefined; + }); +}; + +const setupStill = (manager: PreviewManager.PreviewManager["Service"], paint = Promise.resolve()) => + Effect.gen(function* () { + const image = makeStillImage(); + const capturePage = vi.fn<() => Promise>().mockResolvedValue(image); + const wc = makeTestStillImageWebContents(capturePage); + holdStillImagePaint(wc, paint); + fromId.mockReturnValue(wc); + yield* manager.createTab("tab_still"); + yield* manager.registerWebview("tab_still", wc.id); + yield* manager.setColorScheme("tab_still", "system"); + return { image, capturePage, wc, capture: manager.captureScreenshot("tab_still") }; + }); + const setupRecordingRaceTabs = (manager: PreviewManager.PreviewManager["Service"]) => Effect.gen(function* () { const capturePage = vi.fn(async () => ({ @@ -576,6 +617,32 @@ describe("PreviewManager", () => { webviewSend.mockClear(); }); + effectIt.effect("preserves recording artifacts saved in the same millisecond", () => + withManager((manager) => + Effect.gen(function* () { + const stored = new Map(); + const save = (path: string, bytes: Uint8Array) => { + stored.set(path, Uint8Array.from(bytes)); + return undefined; + }; + writeFile.mockImplementationOnce(save).mockImplementationOnce(save); + const firstBytes = new Uint8Array([1, 2, 3]); + const secondBytes = new Uint8Array([4, 5, 6]); + const [first, second] = yield* Effect.all( + [ + manager.saveRecording("tab_first", "video/webm", firstBytes), + manager.saveRecording("tab_second", "video/webm", secondBytes), + ], + { concurrency: "unbounded" }, + ); + + expect(first.path).not.toBe(second.path); + expect(stored.get(first.path)).toEqual(firstBytes); + expect(stored.get(second.path)).toEqual(secondBytes); + }), + ), + ); + effectIt.effect.each([false, true])( "preserves simultaneous same-name downloads with an existing destination: %s", (existing) => @@ -617,8 +684,10 @@ describe("PreviewManager", () => { attach: () => {}, detach: () => {}, setShown: () => {}, + setCaptureDrawn: () => {}, loadFailed: () => {}, humanStartedDownload: () => true, + agentLedPage: () => false, placeDownload: () => true, }), Layer.mock(PreviewPasskeys.PreviewPasskeys)({ @@ -671,7 +740,7 @@ describe("PreviewManager", () => { if (existing) { expect(yield* fs.readFileString(`${downloads}/report.txt`)).toBe("existing download"); } - }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), Effect.scoped), + }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), ); effectIt.effect.each([ @@ -745,6 +814,137 @@ describe("PreviewManager", () => { ), ); + effectIt.effect.each([ + "navigate", + "reload", + "goBack", + "goForward", + "refresh", + "hardReload", + ] as const)("%s stops a page script that never yields before it navigates", (operation) => + withManager((manager) => + Effect.gen(function* () { + const preview = makeFaviconWebContents(); + // As in Chromium: the hung renderer answers nothing, and a terminate reaches it only + // while no navigation is pending. + let stopped = false; + const probed = yield* Deferred.make(); + const navigatedAfterStop: Array = []; + const navigated = () => { + navigatedAfterStop.push(stopped); + }; + const sendCommand = async (method: string) => { + if (method === "Runtime.evaluate") { + Deferred.doneUnsafe(probed, Effect.void); + await new Promise(() => {}); + } + if (method === "Runtime.terminateExecution") stopped = true; + }; + Object.assign(preview.webContents, { + getURL: () => "http://localhost:5173/busy", + reload: navigated, + reloadIgnoringCache: navigated, + loadURL: async () => navigated(), + navigationHistory: { + canGoBack: () => true, + canGoForward: () => true, + goBack: navigated, + goForward: navigated, + }, + debugger: { + isAttached: () => false, + attach: vi.fn(), + sendCommand, + on: vi.fn(), + off: vi.fn(), + }, + }); + fromId.mockReturnValue(preview.webContents); + yield* manager.createTab("tab_busy"); + yield* manager.registerWebview("tab_busy", 42); + + const leaving = yield* ( + operation === "navigate" + ? manager.navigate("tab_busy", "http://localhost:5173/next") + : operation === "reload" + ? manager.navigate("tab_busy", "http://localhost:5173/busy") + : manager[operation]("tab_busy") + ).pipe(Effect.forkChild); + const probing = yield* Effect.raceFirst( + Deferred.await(probed).pipe(Effect.as(true)), + Fiber.await(leaving).pipe(Effect.as(false)), + ); + if (probing) yield* TestClock.adjust(1_000); + yield* Fiber.join(leaving); + + expect(navigatedAfterStop).toEqual([true]); + }), + ), + ); + + effectIt.effect("keeps mouse back and forward buttons inside the preview tab", () => + withManager((manager) => + Effect.gen(function* () { + const preview = makeFaviconWebContents(); + const goBack = vi.fn(); + const goForward = vi.fn(); + const send = vi.fn(); + let canGoBack = false; + Object.assign(preview.webContents, { + navigationHistory: { + canGoBack: () => canGoBack, + canGoForward: () => true, + goBack, + goForward, + }, + getZoomFactor: () => 2, + send, + }); + fromId.mockReturnValue(preview.webContents); + yield* manager.createTab("tab_mouse_navigation"); + yield* manager.registerWebview("tab_mouse_navigation", 42); + + // Electron reports thumb buttons as "back"/"forward" for every frame of the guest, iframes included. + const beforeMouse = preview.listeners.get("before-mouse-event"); + expect(beforeMouse, "guest has no before-mouse-event listener").toBeDefined(); + const press = (type: "mouseDown" | "mouseUp", button: string) => { + const preventDefault = vi.fn(); + beforeMouse!({ preventDefault } as never, { type, button, x: 10, y: 10 } as never); + return preventDefault; + }; + + // At the start of tab history the press is still swallowed, or the host app goes back instead. + expect(press("mouseDown", "back")).toHaveBeenCalledOnce(); + expect(press("mouseUp", "back")).toHaveBeenCalledOnce(); + expect(goBack).not.toHaveBeenCalled(); + + canGoBack = true; + expect(press("mouseDown", "back")).toHaveBeenCalledOnce(); + expect(goBack).not.toHaveBeenCalled(); + expect(press("mouseUp", "back")).toHaveBeenCalledOnce(); + expect(goBack).toHaveBeenCalledOnce(); + expect(press("mouseDown", "forward")).toHaveBeenCalledOnce(); + expect(press("mouseUp", "forward")).toHaveBeenCalledOnce(); + expect(goForward).toHaveBeenCalledOnce(); + for (const button of ["left", "right"]) { + expect(press("mouseDown", button)).not.toHaveBeenCalled(); + expect(press("mouseUp", button)).not.toHaveBeenCalled(); + } + expect(goBack).toHaveBeenCalledOnce(); + expect(goForward).toHaveBeenCalledOnce(); + // The guest replays dropped presses for human input and recordings, in CSS px. + expect(send.mock.calls.filter(([channel]) => channel === "preview:thumb-press")).toEqual([ + ["preview:thumb-press", "down", 5, 5, 3], + ["preview:thumb-press", "up", 5, 5, 3], + ["preview:thumb-press", "down", 5, 5, 3], + ["preview:thumb-press", "up", 5, 5, 3], + ["preview:thumb-press", "down", 5, 5, 4], + ["preview:thumb-press", "up", 5, 5, 4], + ]); + }), + ), + ); + effectIt.effect("keeps editing and forwarded shortcuts in whichever tab holds focus", () => withManager((manager) => Effect.gen(function* () { @@ -820,7 +1020,7 @@ describe("PreviewManager", () => { ["linux", false], ["win32", false], ] as const)( - "lets the page and its iframes claim the find shortcut before the host opens find on %s", + "lets the page and its iframes claim the find and zoom shortcuts before the host acts on %s", ([platform, usesMeta]) => withManager( (manager) => @@ -838,8 +1038,23 @@ describe("PreviewManager", () => { yield* manager.registerWebview("tab_page_find", 42); yield* manager.setForwardedShortcuts([ { command: "preview.find", shortcut: parseKeybindingShortcut("mod+f")! }, + { command: "preview.zoomIn", shortcut: parseKeybindingShortcut("mod+=")! }, ]); const beforeInput = preview.listeners.get("before-input-event")!; + const keepsZoom = vi.fn(); + beforeInput( + { preventDefault: keepsZoom } as never, + { + type: "keyDown", + key: "=", + code: "Equal", + meta: usesMeta, + control: !usesMeta, + shift: false, + alt: false, + } as never, + ); + expect(keepsZoom).not.toHaveBeenCalled(); const findKey = { type: "keyDown", key: "f", @@ -1677,6 +1892,30 @@ describe("PreviewManager", () => { ), ); + effectIt.effect("captures final pixels after warmup fails and paint readiness settles", () => + withManager((manager) => + Effect.gen(function* () { + const paint = Promise.withResolvers(); + const { capturePage, wc, capture } = yield* setupStill(manager, paint.promise); + capturePage + .mockRejectedValueOnce(new Error("UnknownVizError")) + .mockRejectedValueOnce(new Error("UnknownVizError")) + .mockRejectedValueOnce(new Error("UnknownVizError")) + .mockResolvedValue(makeStillImage("final-png")); + const fiber = yield* capture.pipe(Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust(300); + expect(capturePage).toHaveBeenCalledTimes(3); + expect(writeFile).not.toHaveBeenCalled(); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false]]); + paint.resolve(); + const artifact = yield* Fiber.join(fiber); + expect(capturePage).toHaveBeenCalledTimes(4); + expect(writeFile.mock.calls).toEqual([[artifact.path, Buffer.from("final-png")]]); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false], [true]]); + }), + ), + ); + effectIt.effect("preserves both screenshots of the same site in the same millisecond", () => withManager((manager) => Effect.gen(function* () { @@ -1690,6 +1929,8 @@ describe("PreviewManager", () => { const capturePage = vi .fn<() => Promise>>() .mockResolvedValueOnce(pngImage(firstPng)) + .mockResolvedValueOnce(pngImage(firstPng)) + .mockResolvedValueOnce(pngImage(secondPng)) .mockResolvedValueOnce(pngImage(secondPng)); fromId.mockReturnValue(makeTestPreviewWebContents(capturePage)); yield* manager.createTab("tab_1"); @@ -1710,6 +1951,115 @@ describe("PreviewManager", () => { ), ); + effectIt.effect( + "waits for paint before capturing and restores rendering after overlapping requests", + () => + withManager((manager) => + Effect.gen(function* () { + const paints = [Promise.withResolvers(), Promise.withResolvers()]; + const pendingPaints = paints.map((paint) => paint.promise); + const { image, capturePage, wc, capture } = yield* setupStill(manager); + holdStillImagePaint(wc, pendingPaints); + const first = Promise.withResolvers(); + const second = Promise.withResolvers(); + capturePage + .mockResolvedValueOnce(makeStillImage("discarded-warmup")) + .mockResolvedValueOnce(makeStillImage("discarded-warmup")) + .mockReturnValueOnce(first.promise) + .mockReturnValueOnce(second.promise); + const hostThrottling = vi.fn(); + yield* manager.setMainWindow({ + isDestroyed: () => false, + once: vi.fn(), + webContents: { setBackgroundThrottling: hostThrottling }, + } as never); + const one = yield* capture.pipe(Effect.forkChild({ startImmediately: true })); + const two = yield* capture.pipe(Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust(0); + expect(capturePage).toHaveBeenCalledTimes(2); + expect(pendingPaints).toHaveLength(0); + expect(writeFile).not.toHaveBeenCalled(); + expect(hostThrottling.mock.calls).toEqual([[false]]); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false]]); + paints.forEach((paint) => paint.resolve()); + yield* TestClock.adjust(0); + expect(capturePage).toHaveBeenCalledTimes(4); + first.resolve(image); + yield* Fiber.join(one); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false]]); + second.resolve(image); + yield* Fiber.join(two); + const saved = writeFile.mock.calls.map(([, bytes]) => bytes); + expect(saved).toEqual([image.toPNG(), image.toPNG()]); + expect(hostThrottling.mock.calls).toEqual([[false], [true]]); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false], [true]]); + }), + ), + ); + + effectIt.effect("keeps picture-in-picture rendering after a still image completes", () => + withManager((manager) => + Effect.gen(function* () { + const { image, capturePage, wc, capture } = yield* setupStill(manager); + const pending = Promise.withResolvers(); + capturePage + .mockResolvedValueOnce(image) + .mockResolvedValueOnce(makeStillImage("discarded-warmup")) + .mockReturnValueOnce(pending.promise); + const { pictureInPictureWindow } = makeTestPictureInPictureWindow(); + browserWindowConstructor.mockImplementation(function () { + return pictureInPictureWindow; + }); + yield* manager.openPictureInPicture("tab_still"); + const still = yield* capture.pipe(Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust(0); + expect(capturePage).toHaveBeenCalledTimes(3); + expect(writeFile).not.toHaveBeenCalled(); + pending.resolve(image); + yield* Fiber.join(still); + expect(writeFile.mock.calls.map((call) => call[1])).toEqual([Buffer.from("fresh-png")]); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false]]); + yield* manager.closePictureInPicture("tab_still"); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false], [true]]); + }), + ), + ); + + effectIt.effect("keeps a pending still image rendering when recording ends", () => + withManager((manager) => + Effect.gen(function* () { + const { image, capturePage, wc, capture } = yield* setupStill(manager); + const pending = Promise.withResolvers(); + capturePage.mockResolvedValueOnce(image).mockReturnValue(pending.promise); + yield* manager.startRecording("tab_still"); + const fiber = yield* capture.pipe(Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust(0); + yield* manager.stopRecording("tab_still"); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false]]); + pending.resolve(image); + yield* Fiber.join(fiber); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false], [true]]); + }), + ), + ); + + effectIt.effect("saves available pixels when paint readiness never settles", () => + withManager((manager) => + Effect.gen(function* () { + const paint = Promise.withResolvers(); + const { capturePage, wc, capture } = yield* setupStill(manager, paint.promise); + const first = yield* Effect.exit(capture).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* TestClock.adjust(1_001); + expect(Exit.isSuccess(yield* Fiber.join(first))).toBe(true); + expect(capturePage).toHaveBeenCalledTimes(2); + expect(writeFile.mock.calls.map((call) => call[1])).toEqual([Buffer.from("fresh-png")]); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false], [true]]); + }), + ), + ); + effectIt.effect("keeps every recorded guest unthrottled until its frame capture stops", () => withManager((manager) => Effect.gen(function* () { @@ -2056,7 +2406,7 @@ describe("PreviewManager", () => { expect(yield* nextType).toBe("attached"); yield* manager.closeTab("tab_1"); expect(yield* nextType).toBe("detached"); - }).pipe(Effect.provide(managerLayer()), Effect.scoped), + }).pipe(Effect.provide(managerLayer())), ); effectIt.effect( @@ -2113,8 +2463,74 @@ describe("PreviewManager", () => { }), ); expect(yield* Queue.take(capturing)).toBe(true); + yield* manager.setCaptureDrawn("tab_1", true); screenshot.resolve({ data: "" }); expect(yield* Queue.take(capturing)).toBe(false); + }).pipe(Effect.provide(managerLayer())), + ); + + effectIt.effect( + "opens a new-tab link in the background only when the agent's input led to it", + () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.DesktopBrowserHost; + const serverTab = { threadId: "thread-1", tabId: "server-tab-1" }; + const contents = Object.assign( + makeTestPreviewWebContents(async () => { + throw new Error("unused"); + }, 42), + { + isDevToolsOpened: () => false, + getUserAgent: () => "Electron", + focusedFrame: null, + }, + ); + fromId.mockReturnValue(contents); + const manager = yield* PreviewManager.PreviewManager; + const openLinks = yield* Queue.unbounded(); + yield* manager.subscribeOpenLinks(({ background }) => + Queue.offer(openLinks, background).pipe(Effect.asVoid), + ); + const attached = yield* Deferred.make(); + yield* host.events.pipe( + Stream.runForEach((line) => + new TextDecoder().decode(line).includes('"type":"attached"') + ? Deferred.succeed(attached, undefined) + : Effect.void, + ), + Effect.forkScoped, + ); + yield* manager.createTab("tab_1", { serverTab }); + yield* manager.registerWebview("tab_1", 42); + yield* Deferred.await(attached); + const openWindow = vi.mocked(contents.setWindowOpenHandler).mock.calls[0]![0]; + const openBlankLink = Effect.suspend(() => { + openWindow({ + url: "https://example.com/next", + disposition: "foreground-tab", + } as Electron.HandlerDetails); + return Queue.take(openLinks); + }); + + expect(yield* openBlankLink).toBe(false); + + yield* host.handleCommandLine( + JSON.stringify({ + type: "cdp", + ...serverTab, + message: JSON.stringify({ + id: 1, + method: "Runtime.evaluate", + params: { expression: "document.querySelector('a').click()" }, + sessionId: "t3-preview-page", + }), + }), + ); + expect(yield* openBlankLink).toBe(true); + + // A person clicking in the guest itself gives it a focused frame. + Object.assign(contents, { focusedFrame: {} }); + expect(yield* openBlankLink).toBe(false); }).pipe(Effect.provide(managerLayer()), Effect.scoped), ); @@ -2206,7 +2622,7 @@ describe("PreviewManager", () => { expect(JSON.parse(yield* Queue.take(lines))).toEqual({ type: "detached", ...serverTab }); expect(JSON.parse(yield* Queue.take(lines))).toEqual({ type: "attached", ...serverTab }); expect(statuses.at(-1)?.kind).toBe("Success"); - }).pipe(Effect.provide(managerLayer()), Effect.scoped), + }).pipe(Effect.provide(managerLayer())), ); effectIt.effect("discards a screenshot that resolves after its guest is replaced", () => @@ -2368,6 +2784,116 @@ describe("PreviewManager", () => { ), ); + effectIt.effect( + "publishes human ownership only on transitions while retaining every recording press", + () => + withManager((manager) => + Effect.gen(function* () { + let humanInput: ((input: unknown) => void) | undefined; + const contents = makeTestPreviewWebContents(async () => ({ + toJPEG: () => Buffer.from("frame"), + getSize: () => ({ width: 800, height: 600 }), + })); + contents.ipc.on = (channel, callback) => { + if (channel === "preview:human-input") + humanInput = (input) => callback({} as Electron.IpcMainEvent, input); + return contents.ipc; + }; + fromId.mockReturnValue(contents); + yield* manager.createTab("tab_human"); + yield* manager.registerWebview("tab_human", 42); + yield* manager.startRecording("tab_human", { + showKeyPresses: true, + showMousePresses: true, + }); + const controllers: string[] = []; + const feedback = yield* Queue.unbounded(); + webviewSend.mockImplementation((channel) => { + if (channel === "preview:recording-controller") Queue.offerUnsafe(feedback, undefined); + }); + yield* manager.subscribeStateChanges((_tabId, state) => + Effect.sync(() => { + controllers.push(state.controller); + }), + ); + for (const press of [ + { kind: "pointer", x: 10, y: 20, button: 0 }, + { kind: "pointer", x: 30, y: 40, button: 0 }, + { kind: "key", key: "a", code: "KeyA" }, + ]) { + humanInput?.(press); + yield* Queue.take(feedback); + } + expect(controllers).toEqual(["human"]); + expect( + webviewSend.mock.calls.filter( + ([channel]) => channel === "preview:recording-controller", + ), + ).toEqual([ + ["preview:recording-controller", "human", { x: 10, y: 20 }], + ["preview:recording-controller", "human", { x: 30, y: 40 }], + ["preview:recording-controller", "human", undefined], + ]); + yield* TestClock.adjust(750); + expect(controllers).toEqual(["human", "none"]); + }), + ), + ); + + effectIt.effect("keeps human ownership until the last input's idle deadline", () => + withManager((manager) => + Effect.gen(function* () { + let humanInput: ((input: unknown) => void) | undefined; + const contents = makeTestPreviewWebContents(async () => ({ + toJPEG: () => Buffer.from("frame"), + getSize: () => ({ width: 800, height: 600 }), + })); + contents.ipc.on = (channel, callback) => { + if (channel === "preview:human-input") + humanInput = (input) => callback({} as Electron.IpcMainEvent, input); + return contents.ipc; + }; + fromId.mockReturnValue(contents); + yield* manager.createTab("tab_idle"); + yield* manager.registerWebview("tab_idle", 42); + const controllers: string[] = []; + yield* manager.subscribeStateChanges((_tabId, state) => + Effect.sync(() => { + controllers.push(state.controller); + }), + ); + const press = () => humanInput?.({ kind: "key", key: "a", code: "KeyA" }); + press(); + yield* TestClock.adjust(500); + press(); + yield* TestClock.adjust(749); + expect(controllers).toEqual(["human"]); + yield* TestClock.adjust(1); + expect(controllers).toEqual(["human", "none"]); + + press(); + yield* TestClock.adjust(500); + const staleInput = humanInput; + const replacement = makeTestPreviewWebContents( + async () => ({ + toJPEG: () => Buffer.from("frame"), + getSize: () => ({ width: 800, height: 600 }), + }), + 43, + ); + replacement.ipc.on = contents.ipc.on; + contents.isDestroyed = () => true; + fromId.mockImplementation((id) => (id === 43 ? replacement : null)); + yield* manager.registerWebview("tab_idle", 43); + expect(controllers.at(-1)).toBe("none"); + const count = controllers.length; + staleInput?.({ kind: "key", key: "b", code: "KeyB" }); + yield* TestClock.adjust(750); + expect(controllers).toHaveLength(count); + }), + ), + ); + effectIt.effect("gates recording decorations and isolates failed subscribers", () => withManager((manager) => Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 189885b9b2178..32ac876996c81 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -81,13 +81,13 @@ import { CANCEL_PICK_CHANNEL, ELEMENT_PICKED_CHANNEL, HUMAN_INPUT_CHANNEL, - MOUSE_NAVIGATE_CHANNEL, PAGE_UNHANDLED_SHORTCUT_CHANNEL, RECORDING_CURSOR_CHANNEL, RECORDING_INPUT_CHANNEL, RECORDING_CONTROLLER_CHANNEL, RECORDING_POINTER_CHANNEL, START_PICK_CHANNEL, + THUMB_PRESS_CHANNEL, } from "./GuestProtocol.ts"; import { isPreviewAnnotationPayload } from "./PickedElementPayload.ts"; import { captureFavicon, safeHttpOrigin, selectFaviconCandidates } from "./FaviconCapture.ts"; @@ -136,6 +136,7 @@ const PICTURE_IN_PICTURE_JPEG_QUALITY = 80; const CAPTURE_PAGE_RETRY_ATTEMPTS = 3; const CAPTURE_PAGE_RETRY_DELAY_MS = 120; const CAPTURE_PAGE_ATTEMPT_TIMEOUT_MS = 1_000; +const BUSY_PAGE_PROBE_MS = 1_000; const PICTURE_IN_PICTURE_INITIAL_WIDTH = 480; const PICTURE_IN_PICTURE_INITIAL_HEIGHT = 320; const PICTURE_IN_PICTURE_MIN_WIDTH = 240; @@ -386,7 +387,8 @@ interface ManagedListeners { readonly webContents: Electron.WebContents; } -type FrameCaptureConsumer = "picture-in-picture" | "recording"; +// Each still-image request owns a separate, short-lived rendering lease. +type FrameCaptureConsumer = "picture-in-picture" | "recording" | symbol; interface FrameCaptureSession { readonly recordingInputOptions?: RecordingInputOptions; @@ -444,6 +446,9 @@ type FindResultListener = (event: DesktopPreviewFindResultEvent) => Effect.Effec const PAGE_FIRST_FORWARDED_COMMANDS: ReadonlySet = new Set([ "preview.find", + "preview.zoomIn", + "preview.zoomOut", + "preview.resetZoom", ]); const decodePageShortcutKeys = Schema.decodeUnknownOption( @@ -884,7 +889,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const update = Effect.fn("PreviewManager.update")(function* ( tabId: string, patch: Partial, - humanPoint?: { readonly x: number; readonly y: number }, ) { const updatedAt = yield* currentIso; const next = yield* SynchronizedRef.modify(tabsRef, (tabs) => { @@ -900,17 +904,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }); // syncTabAudible can commit after this update; emitIfCurrent prevents publishing a stale snapshot. if (Option.isSome(next)) { - if (patch.controller !== undefined && next.value.webContentsId != null) { - const capture = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId); - const webContentsId = next.value.webContentsId; - if (capture?.consumers.has("recording")) { - yield* attempt({ operation: "recording.controller", tabId }, () => { - const contents = webContents.fromId(webContentsId); - if (contents && !contents.isDestroyed()) - contents.send(RECORDING_CONTROLLER_CHANNEL, patch.controller, humanPoint); - }).pipe(Effect.ignore); - } - } yield* emitIfCurrent(tabId, next.value); } }); @@ -1103,28 +1096,13 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ), scope, ); - const handleDebuggerMessage = Effect.fnUntraced(function* ( - method: string, - params: Record, - ) { - if (method === "Page.screencastFrame") { - const sessionId = params["sessionId"]; - if (typeof sessionId === "number") { - yield* attemptPromise( - { - operation: "ackScreencastFrame", - webContentsId: wc.id, - }, - () => wcDebugger.sendCommand("Page.screencastFrameAck", { sessionId }), - ).pipe(Effect.ignore); - } - } + const handleDebuggerMessage = Effect.fnUntraced(function* (method: string) { if (method === "Runtime.consoleAPICalled" || method === "Runtime.exceptionThrown") { yield* Queue.offer(consoleReleases, undefined); } }); - const onMessage: BrowserControlSession["onMessage"] = (_event, method, params) => { - runFork(handleDebuggerMessage(method, params)); + const onMessage: BrowserControlSession["onMessage"] = (_event, method) => { + runFork(handleDebuggerMessage(method)); }; const control: BrowserControlSession = { webContentsId: wc.id, @@ -1439,21 +1417,75 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }), ); }; + let humanInputWorker = false; + let humanInputDeadline = 0; + let listening = true; + const publishHumanController = Effect.fn("PreviewManager.publishHumanController")(function* ( + controller: "human" | "none", + humanPoint?: { readonly x: number; readonly y: number }, + ) { + const now = yield* currentMillis; + const updatedAt = yield* currentIso; + const result = yield* SynchronizedRef.modify(tabsRef, (tabs) => { + const current = tabs.get(tabId); + if ( + !current || + current.webContentsId !== wc.id || + (controller === "human" + ? !listening || webContents.fromId(wc.id) !== wc + : current.controller !== "human" || humanInputDeadline > now) + ) + return [undefined, tabs] as const; + const changed = current.controller !== controller; + const state = changed ? { ...current, controller, updatedAt } : current; + return [ + { state, changed }, + changed + ? replaceMap(tabs, (copy) => { + copy.set(tabId, state); + }) + : tabs, + ] as const; + }); + if (!result) return; + const capture = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId); + if (capture?.consumers.has("recording")) { + yield* attempt({ operation: "recording.controller", tabId }, () => { + if (!wc.isDestroyed()) wc.send(RECORDING_CONTROLLER_CHANNEL, controller, humanPoint); + }).pipe(Effect.ignore); + } + if (result.changed) yield* emitIfCurrent(tabId, result.state); + }); + yield* Scope.addFinalizer( + scope, + Effect.gen(function* () { + listening = false; + humanInputDeadline = 0; + yield* publishHumanController("none"); + }), + ); const handleHumanInput = Effect.fn("PreviewManager.handleHumanInput")(function* ( rawSignal?: unknown, ) { - yield* update( - tabId, - { controller: "human" }, + if (!listening) return; + humanInputDeadline = (yield* currentMillis) + 750; + if (!humanInputWorker) { + humanInputWorker = true; + yield* Effect.gen(function* () { + let remaining: number; + while ((remaining = humanInputDeadline - (yield* currentMillis)) > 0) { + yield* Effect.sleep(remaining); + } + humanInputWorker = false; + yield* publishHumanController("none"); + }).pipe(Effect.forkIn(scope)); + } + yield* publishHumanController( + "human", isPreviewInputSignal(rawSignal) && rawSignal.kind === "pointer" ? { x: rawSignal.x, y: rawSignal.y } : undefined, ); - yield* Effect.sleep(750); - const tabs = yield* SynchronizedRef.get(tabsRef); - if (tabs.get(tabId)?.controller === "human") { - yield* update(tabId, { controller: "none" }); - } }); const recordingInput = (_event: unknown, input: unknown) => { if (!isRecordingInput(input)) return; @@ -1487,15 +1519,27 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const humanInput = (_event: unknown, rawSignal?: unknown): void => { runFork(handleHumanInput(rawSignal)); }; - const mouseNavigate = (_event: unknown, payload?: unknown): void => { - const direction = - typeof payload === "object" && payload !== null && "direction" in payload - ? (payload as { direction?: unknown }).direction - : undefined; - if (direction !== "back" && direction !== "forward") return; + // Chromium navigates the outermost WebContents (the host app) on thumb buttons, even from guest iframes. + const beforeMouse = (event: Electron.Event, mouse: Electron.MouseInputEvent): void => { + // Electron reports thumb buttons as "back" and "forward", which its typings omit. + const button: string | undefined = mouse.button; + if (button !== "back" && button !== "forward") return; + if (mouse.type !== "mouseDown" && mouse.type !== "mouseUp") return; + event.preventDefault(); + // The page never sees the press, so the guest replays it; these coordinates are DIPs, not CSS px. + const zoom = wc.getZoomFactor(); + const phase = mouse.type === "mouseDown" ? "down" : "up"; + wc.send( + THUMB_PRESS_CHANNEL, + phase, + mouse.x / zoom, + mouse.y / zoom, + button === "back" ? 3 : 4, + ); + if (phase !== "up") return; runFork( attempt({ operation: "mouseNavigate", tabId, webContentsId: wc.id }, () => { - if (direction === "back") { + if (button === "back") { if (wc.navigationHistory.canGoBack()) wc.navigationHistory.goBack(); } else if (wc.navigationHistory.canGoForward()) { wc.navigationHistory.goForward(); @@ -1553,7 +1597,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }); if (forwarded && host) { // The page keeps these keys. Only the main-frame preload can report one it - // left unhandled, so a focused iframe (e.g. Google Docs' input) keeps its own find. + // left unhandled, so a focused iframe (e.g. Google Docs' input) keeps its own find or zoom. if (PAGE_FIRST_FORWARDED_COMMANDS.has(forwarded.command)) return; event.preventDefault(); if (!input.isAutoRepeat) { @@ -1605,9 +1649,9 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function wc.off("found-in-page", foundInPage as never); wc.off("did-create-window", windowCreated); wc.off("before-input-event", beforeInput); + wc.off("before-mouse-event", beforeMouse); wc.ipc.off(HUMAN_INPUT_CHANNEL, humanInput); wc.ipc.off(RECORDING_INPUT_CHANNEL, recordingInput); - wc.ipc.off(MOUSE_NAVIGATE_CHANNEL, mouseNavigate); wc.ipc.off(PAGE_UNHANDLED_SHORTCUT_CHANNEL, pageUnhandledShortcut); detachPasskeys(); }).pipe(Effect.ignore), @@ -1628,7 +1672,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function wc.on("found-in-page", foundInPage as never); wc.ipc.on(HUMAN_INPUT_CHANNEL, humanInput); wc.ipc.on(RECORDING_INPUT_CHANNEL, recordingInput); - wc.ipc.on(MOUSE_NAVIGATE_CHANNEL, mouseNavigate); wc.ipc.on(PAGE_UNHANDLED_SHORTCUT_CHANNEL, pageUnhandledShortcut); detachPasskeys = passkeys.attachGuest(wc); wc.setWindowOpenHandler((details) => { @@ -1642,7 +1685,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function emitOpenLink({ tabId, url: details.url, - background: details.disposition === "background-tab", + // A person's own click or key leaves the guest a focused frame; GuestInput hands + // focus back after the agent's press, so a foreground tab would take it again. + background: + details.disposition === "background-tab" || + (wc.focusedFrame === null && browserHost.agentLedPage(wc)), }), ); return { action: "deny" }; @@ -1656,6 +1703,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }); wc.on("did-create-window", windowCreated); wc.on("before-input-event", beforeInput); + wc.on("before-mouse-event", beforeMouse); }); yield* Ref.update(attachedRef, (attached) => replaceMap(attached, (copy) => { @@ -1779,7 +1827,10 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }); if (Option.isNone(tab)) return; const closedTab = tab.value; - if (closedTab.serverTab) browserHost.setShown(closedTab.serverTab, undefined); + if (closedTab.serverTab) { + browserHost.setShown(closedTab.serverTab, undefined); + browserHost.setCaptureDrawn(closedTab.serverTab, undefined); + } if (closedTab.webContentsId != null) { yield* Effect.all( [ @@ -2019,6 +2070,28 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ); }); + /** + * A same-site navigation reuses the renderer, so it never commits while a page script holds it. + * Runs before the navigation starts: Chromium does not deliver a terminate once one is pending. + */ + const leaveBusyPage = Effect.fn("PreviewManager.leaveBusyPage")(function* ( + wc: Electron.WebContents, + ) { + const control = (yield* SynchronizedRef.get(controlSessionsRef)).get(wc.id); + if (!control) return; + const answered = yield* Effect.promise(() => + control.debugger.sendCommand("Runtime.evaluate", { expression: "0" }).then( + () => undefined, + () => undefined, + ), + ).pipe(Effect.timeoutOption(BUSY_PAGE_PROBE_MS)); + if (Option.isSome(answered)) return; + // Not awaited: a renderer blocked on something other than script may never acknowledge it. + yield* Effect.sync(() => { + void control.debugger.sendCommand("Runtime.terminateExecution").catch(() => undefined); + }); + }); + const navigate = Effect.fn("PreviewManager.navigate")(function* (tabId: string, rawUrl: string) { const url = yield* attempt({ operation: "navigate.normalizeUrl", tabId }, () => normalizePreviewUrl(rawUrl), @@ -2101,6 +2174,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ); return; } + yield* leaveBusyPage(wc); if (wc.getURL() === url) { yield* attempt({ operation: "navigate.reload", tabId, webContentsId: wc.id }, () => wc.reload(), @@ -2136,17 +2210,28 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function yield* attempt({ operation, tabId, webContentsId: wc.id }, () => use(wc)); }); + const leavePage = Effect.fn("PreviewManager.leavePage")(function* ( + operation: string, + tabId: string, + use: (wc: Electron.WebContents) => void, + ) { + const wc = yield* requireWebContents(tabId); + yield* leaveBusyPage(wc); + yield* attempt({ operation, tabId, webContentsId: wc.id }, () => use(wc)); + }); + const goBack = (tabId: string) => - withWebContents("goBack", tabId, (wc) => { + leavePage("goBack", tabId, (wc) => { if (wc.navigationHistory.canGoBack()) wc.navigationHistory.goBack(); }); const goForward = (tabId: string) => - withWebContents("goForward", tabId, (wc) => { + leavePage("goForward", tabId, (wc) => { if (wc.navigationHistory.canGoForward()) wc.navigationHistory.goForward(); }); - const refresh = (tabId: string) => withWebContents("refresh", tabId, (wc) => wc.reload()); + const refresh = (tabId: string) => leavePage("refresh", tabId, (wc) => wc.reload()); + const stopLoading = (tabId: string) => withWebContents("stopLoading", tabId, (wc) => wc.stop()); const hardReload = (tabId: string) => - withWebContents("hardReload", tabId, (wc) => wc.reloadIgnoringCache()); + leavePage("hardReload", tabId, (wc) => wc.reloadIgnoringCache()); const findInPage = ( tabId: string, text: string, @@ -2553,12 +2638,21 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function if (serverTab) browserHost.setShown(serverTab, shown); }); + const setCaptureDrawn = Effect.fn("PreviewManager.setCaptureDrawn")(function* ( + tabId: string, + drawn: boolean, + ) { + const serverTab = (yield* SynchronizedRef.get(tabsRef)).get(tabId)?.serverTab; + if (serverTab) browserHost.setCaptureDrawn(serverTab, drawn); + }); + // A window reload or guest crash destroys the guest without the renderer reporting it hidden. // The renderer reports again once a new guest registers. const hideDestroyedGuest = Effect.fnUntraced(function* (webContentsId: number) { for (const tab of (yield* SynchronizedRef.get(tabsRef)).values()) { if (tab.webContentsId === webContentsId && tab.serverTab) { browserHost.setShown(tab.serverTab, false); + browserHost.setCaptureDrawn(tab.serverTab, false); } } }); @@ -2597,7 +2691,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const [createdAt, millis, image] = yield* Effect.all([ currentIso, currentMillis, - capturePageWithRetry( + captureStillImage( { operation: "captureScreenshot.capturePage", tabId, @@ -2645,6 +2739,100 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }; }); + const captureStillImage = Effect.fn("PreviewManager.captureStillImage")(function* ( + errorContext: PreviewOperationContext, + tabId: string, + wc: Electron.WebContents, + ) { + const consumer = Symbol("still-image"); + return yield* Effect.acquireUseRelease( + startFrameCapture(tabId, consumer), + () => + Effect.gen(function* () { + // A DOM mutation can precede its compositor frame even in a shown + // window. Keep the original guest rendering through the next paint; + // a hidden guest otherwise returns old pixels or never completes. + const probe = Effect.fn("PreviewManager.probeStillImagePaint")(function* ( + send: ( + method: string, + params?: Record, + ) => Effect.Effect, + ) { + // CDP executes against the current document without Electron's + // did-stop-loading gate. Isolate scheduling globals from page code. + const tree = (yield* send("Page.getFrameTree")) as + | { frameTree?: { frame?: { id?: string } } } + | undefined; + const frameId = tree?.frameTree?.frame?.id; + if (typeof frameId !== "string") + return yield* new PreviewOperationError({ + ...errorContext, + cause: new Error("The preview paint frame is unavailable."), + }); + const world = (yield* send("Page.createIsolatedWorld", { + frameId, + worldName: "t3-preview-still-image", + })) as { executionContextId?: number } | undefined; + if (typeof world?.executionContextId !== "number") + return yield* new PreviewOperationError({ + ...errorContext, + cause: new Error("The preview paint context is unavailable."), + }); + const response = (yield* send("Runtime.evaluate", { + // Settle and release scheduled work when only timers keep running. + // The outer deadline still bounds a frozen renderer. + expression: `new Promise(resolve => { let frame; const timer = setTimeout(() => { cancelAnimationFrame(frame); resolve(false); }, ${CAPTURE_PAGE_ATTEMPT_TIMEOUT_MS}); frame = requestAnimationFrame(() => { frame = requestAnimationFrame(() => { clearTimeout(timer); resolve(true); }); }); })`, + contextId: world.executionContextId, + awaitPromise: true, + returnByValue: true, + })) as { result?: { value?: unknown }; exceptionDetails?: unknown } | undefined; + if (response?.exceptionDetails || response?.result?.value !== true) + return yield* new PreviewOperationError({ + ...errorContext, + cause: new Error("The preview paint boundary was not confirmed."), + }); + }); + const paintReady = Effect.gen(function* () { + // Only use an existing owned session; never attach or displace + // another debugger to obtain a screenshot. + const control = (yield* SynchronizedRef.get(controlSessionsRef)).get(wc.id); + if (!control) + return yield* new PreviewOperationError({ + ...errorContext, + cause: new Error("The preview paint session is unavailable."), + }); + return yield* probe((method, params) => + attemptPromise(errorContext, () => control.debugger.sendCommand(method, params)), + ); + }).pipe( + Effect.timeout(CAPTURE_PAGE_ATTEMPT_TIMEOUT_MS), + Effect.catch((error) => + Effect.logWarning( + "Preview paint readiness was not confirmed; capturing available pixels with mutation freshness unconfirmed.", + { tabId, webContentsId: wc.id, error }, + ), + ), + ); + // Minimized guests can suspend frames even with throttling disabled. + // A native capture can resume them; discard its pixels and read again + // after the bounded paint probe. + const warmup = capturePageWithRetry(errorContext, tabId, wc).pipe( + Effect.catchTags({ + PreviewOperationError: (error) => + Effect.logWarning("Preview warmup capture failed; continuing with final capture.", { + tabId, + webContentsId: wc.id, + error, + }), + }), + ); + yield* Effect.all([paintReady, warmup], { concurrency: 2, discard: true }); + return yield* capturePageWithRetry(errorContext, tabId, wc); + }), + () => stopFrameCapture(tabId, consumer), + ); + }); + const capturePreviewFrame = Effect.fn("PreviewManager.capturePreviewFrame")(function* ( tabId: string, ) { @@ -3327,7 +3515,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function data: Uint8Array, ) { const [createdAt, millis] = yield* Effect.all([currentIso, currentMillis]); - const id = `browser-recording-${millis.toString(36)}`; + const uuid = yield* random.uuidV4(); + const id = `browser-recording-${millis.toString(36)}-${uuid.slice(0, 8)}`; const extension = recordingFileExtension(mimeType); const artifactPath = path.join(resolvedArtifactDirectory, `${id}.${extension}`); yield* fileSystem.makeDirectory(resolvedArtifactDirectory, { recursive: true }).pipe( @@ -3341,7 +3530,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }), ), ); - yield* fileSystem.writeFile(artifactPath, data).pipe( + yield* fileSystem.writeFile(artifactPath, data, { flag: "wx" }).pipe( Effect.mapError( (cause) => new PreviewOperationError({ @@ -3528,6 +3717,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function prepareWebview, reapplyZoom, refresh, + stopLoading, registerWebview, resetZoom: (tabId: string) => applyZoom(tabId, () => DEFAULT_ZOOM_FACTOR), setZoomFactor: (tabId: string, zoomFactor: number) => @@ -3538,6 +3728,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function setAnnotationSendEnabled, setAudioMuted, setShown, + setCaptureDrawn, setColorScheme, setMainWindow, setForwardedShortcuts: (shortcuts: ReadonlyArray) => @@ -3729,6 +3920,7 @@ export class PreviewManager extends Context.Service< readonly goBack: (tabId: string) => Effect.Effect; readonly goForward: (tabId: string) => Effect.Effect; readonly refresh: (tabId: string) => Effect.Effect; + readonly stopLoading: (tabId: string) => Effect.Effect; readonly zoomIn: (tabId: string) => Effect.Effect; readonly zoomOut: (tabId: string) => Effect.Effect; readonly resetZoom: (tabId: string) => Effect.Effect; @@ -3753,6 +3945,7 @@ export class PreviewManager extends Context.Service< audioMuted: boolean, ) => Effect.Effect; readonly setShown: (tabId: string, shown: boolean) => Effect.Effect; + readonly setCaptureDrawn: (tabId: string, drawn: boolean) => Effect.Effect; readonly openDevTools: (tabId: string) => Effect.Effect; readonly clearCookies: ( partitions?: ReadonlyArray, @@ -3903,6 +4096,7 @@ export const make = Effect.gen(function* PreviewManagerMake() { goBack: operations.goBack, goForward: operations.goForward, refresh: operations.refresh, + stopLoading: operations.stopLoading, zoomIn: operations.zoomIn, zoomOut: operations.zoomOut, resetZoom: operations.resetZoom, @@ -3914,6 +4108,7 @@ export const make = Effect.gen(function* PreviewManagerMake() { setZoomFactor: operations.setZoomFactor, setAudioMuted: operations.setAudioMuted, setShown: operations.setShown, + setCaptureDrawn: operations.setCaptureDrawn, openDevTools: operations.openDevTools, clearCookies: Effect.fn("PreviewManager.clearCookies")(function* (partitions) { yield* browserSession diff --git a/apps/desktop/src/preview/PickPreload.ts b/apps/desktop/src/preview/PickPreload.ts index 88a7fc467e425..42867b9d9cb44 100644 --- a/apps/desktop/src/preview/PickPreload.ts +++ b/apps/desktop/src/preview/PickPreload.ts @@ -39,7 +39,7 @@ import { CANCEL_PICK_CHANNEL, ELEMENT_PICKED_CHANNEL, HUMAN_INPUT_CHANNEL, - MOUSE_NAVIGATE_CHANNEL, + POINTER_DOWN_HOST_CHANNEL, PAGE_UNHANDLED_SHORTCUT_CHANNEL, RECORDING_CURSOR_CHANNEL, RECORDING_POINTER_CHANNEL, @@ -47,6 +47,7 @@ import { RECORDING_INPUT_CHANNEL, RECORDING_CONTROLLER_CHANNEL, START_PICK_CHANNEL, + THUMB_PRESS_CHANNEL, } from "./GuestProtocol.ts"; const OVERLAY_ATTRIBUTE = "data-t3code-annotation-ui"; const INPUT_LAYER_ATTRIBUTE = "data-t3code-annotation-input"; @@ -193,6 +194,9 @@ const applyAnnotationTheme = ( const reportHumanPointerInput = (event: PointerEvent): void => { if (!event.isTrusted) return; + // The guest does not always take focus on a click, so the host's webview + // `focus` event can't be relied on to dismiss open menus and popovers. + ipcRenderer.sendToHost(POINTER_DOWN_HOST_CHANNEL); ipcRenderer.send(HUMAN_INPUT_CHANNEL, { kind: "pointer", x: event.clientX, @@ -245,34 +249,28 @@ const isolateOverlayKey = (event: KeyboardEvent): void => { for (const type of ["keydown", "keypress", "keyup"] as const) window.addEventListener(type, isolateOverlayKey, true); -const MOUSE_BUTTON_BACK = 3; -const MOUSE_BUTTON_FORWARD = 4; - -const navigationDirectionForButton = (button: number): "back" | "forward" | null => { - if (button === MOUSE_BUTTON_BACK) return "back"; - if (button === MOUSE_BUTTON_FORWARD) return "forward"; - return null; -}; - -// Chromium routes thumb-button navigation to the focused WebContents; navigate this hovered guest explicitly. -const suppressNavigationButton = (event: MouseEvent): void => { - if (!event.isTrusted || navigationDirectionForButton(event.button) === null) return; - event.preventDefault(); - event.stopImmediatePropagation(); -}; - -const requestNavigationForButton = (event: MouseEvent): void => { - if (!event.isTrusted) return; - const direction = navigationDirectionForButton(event.button); - if (direction === null) return; - event.preventDefault(); - event.stopImmediatePropagation(); - ipcRenderer.send(MOUSE_NAVIGATE_CHANNEL, { direction }); -}; - -window.addEventListener("mousedown", suppressNavigationButton, true); -window.addEventListener("mouseup", requestNavigationForButton, true); -window.addEventListener("auxclick", suppressNavigationButton, true); +let replayedThumbPress: { readonly x: number; readonly y: number } | null = null; +ipcRenderer.on( + THUMB_PRESS_CHANNEL, + (_event, phase: unknown, x: unknown, y: unknown, button: unknown) => { + if (phase !== "down" && phase !== "up") return; + if (typeof x !== "number" || typeof y !== "number" || typeof button !== "number") return; + // Main reports presses inside a frame in that frame's coordinates. + const inFrame = document.querySelector("iframe:hover, frame:hover, object:hover, embed:hover"); + if (phase === "up") { + const pressed = replayedThumbPress; + replayedThumbPress = null; + // A replayed press is always released, at its own point if the release lands in a frame. + if (pressed) recordingCursor?.press(inFrame ? pressed : { x, y }, "up"); + return; + } + if (inFrame) return; + replayedThumbPress = { x, y }; + ipcRenderer.sendToHost(POINTER_DOWN_HOST_CHANNEL); + ipcRenderer.send(HUMAN_INPUT_CHANNEL, { kind: "pointer", x, y, button }); + recordingCursor?.press({ x, y }, "down"); + }, +); const nextId = createAnnotationIdSource(); diff --git a/apps/desktop/src/preview/RecordingCursor.ts b/apps/desktop/src/preview/RecordingCursor.ts index 29f1865de291f..ab74c361b68e0 100644 --- a/apps/desktop/src/preview/RecordingCursor.ts +++ b/apps/desktop/src/preview/RecordingCursor.ts @@ -156,13 +156,15 @@ export function installRecordingCursor( moveHuman(point); pointer(point, "move"); }; + const press = (point: { readonly x: number; readonly y: number }, phase: "down" | "up") => { + if (phase === "down") moveHuman(point); + pointer(point, phase); + }; const pointerDown = (event: PointerEvent) => { - if (event.pointerType === "touch") return; - moveHuman({ x: event.clientX, y: event.clientY }); - pointer({ x: event.clientX, y: event.clientY }, "down"); + if (event.pointerType !== "touch") press({ x: event.clientX, y: event.clientY }, "down"); }; const pointerUp = (event: PointerEvent) => { - if (event.pointerType !== "touch") pointer({ x: event.clientX, y: event.clientY }, "up"); + if (event.pointerType !== "touch") press({ x: event.clientX, y: event.clientY }, "up"); }; const keyDown = (event: KeyboardEvent) => { if (event.isComposing || event.repeat) return; @@ -196,6 +198,7 @@ export function installRecordingCursor( window.addEventListener("blur", hide); return { move, + press, keyPress, setController, setTheme, diff --git a/apps/desktop/src/settings/DesktopAppSettings.test.ts b/apps/desktop/src/settings/DesktopAppSettings.test.ts index 868e1c139fbdd..04406f63c6b3d 100644 --- a/apps/desktop/src/settings/DesktopAppSettings.test.ts +++ b/apps/desktop/src/settings/DesktopAppSettings.test.ts @@ -4,6 +4,7 @@ import { assert, describe, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as PlatformError from "effect/PlatformError"; import * as Schema from "effect/Schema"; import * as Tracer from "effect/Tracer"; @@ -160,6 +161,49 @@ describe("DesktopSettings", () => { ), ); + it.effect("keeps saved preferences when a read fails before a bounds save", () => + withSettings( + Effect.gen(function* () { + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const fileSystem = yield* FileSystem.FileSystem; + yield* writeSettingsPatch({ serverExposureMode: "network-accessible" }); + const cause = PlatformError.systemError({ + _tag: "PermissionDenied", + module: "FileSystem", + method: "readFileString", + pathOrDescriptor: environment.desktopSettingsPath, + }); + let failRead = true; + const settings = yield* DesktopAppSettings.make.pipe( + Effect.provideService( + FileSystem.FileSystem, + FileSystem.FileSystem.of({ + ...fileSystem, + readFileString: (path) => + Effect.suspend(() => { + if (failRead && path === environment.desktopSettingsPath) { + failRead = false; + return Effect.fail(cause); + } + return fileSystem.readFileString(path); + }), + }), + ), + ); + + yield* Effect.exit(settings.load); + yield* Effect.exit( + settings.setMainWindowBounds({ x: 10, y: 20, width: 900, height: 600 }, false), + ); + + const persisted = yield* decodeDesktopSettingsPatch( + yield* fileSystem.readFileString(environment.desktopSettingsPath), + ); + assert.equal(persisted.serverExposureMode, "network-accessible"); + }), + ), + ); + it.effect("loads lenient persisted desktop settings JSON", () => withSettings( Effect.gen(function* () { @@ -196,6 +240,28 @@ describe("DesktopSettings", () => { ), ); + it.effect("keeps Local environment off when another value in the file is unreadable", () => + withSettings( + Effect.gen(function* () { + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const fileSystem = yield* FileSystem.FileSystem; + const settings = yield* DesktopAppSettings.DesktopAppSettings; + yield* fileSystem.makeDirectory(environment.stateDir, { recursive: true }); + yield* fileSystem.writeFileString( + environment.desktopSettingsPath, + `{ "localEnvironmentEnabled": false, "serverExposureMode": "tailnet" }\n`, + ); + + assert.isFalse((yield* settings.load).localEnvironmentEnabled); + yield* settings.setMainWindowBounds({ x: 10, y: 20, width: 800, height: 600 }, false); + assert.include( + yield* fileSystem.readFileString(environment.desktopSettingsPath), + `"localEnvironmentEnabled":false`, + ); + }), + ), + ); + it.effect("rejects window bounds that do not satisfy the domain schema", () => withSettings( Effect.gen(function* () { diff --git a/apps/desktop/src/settings/DesktopAppSettings.ts b/apps/desktop/src/settings/DesktopAppSettings.ts index 4c2ae8d1d1342..e81da70660648 100644 --- a/apps/desktop/src/settings/DesktopAppSettings.ts +++ b/apps/desktop/src/settings/DesktopAppSettings.ts @@ -13,6 +13,8 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import * as PlatformError from "effect/PlatformError"; +import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import * as SynchronizedRef from "effect/SynchronizedRef"; @@ -88,20 +90,25 @@ const DesktopWindowBoundsDocument = Schema.Struct({ height: Schema.Number, }); +// A value this build cannot read drops only its own key, so the rest of the file +// still loads and is written back instead of every setting resetting to defaults. +const optionalReadableKey = (schema: S) => + Schema.optionalKey(schema.pipe(Schema.catchDecoding(() => Effect.succeedNone))); + const DesktopSettingsDocument = Schema.Struct({ - localEnvironmentEnabled: Schema.optionalKey(Schema.Boolean), - linuxPasswordStore: Schema.optionalKey(Schema.Unknown), - mainWindowBounds: Schema.optionalKey(Schema.NullOr(DesktopWindowBoundsDocument)), - mainWindowMaximized: Schema.optionalKey(Schema.Boolean), - serverExposureMode: Schema.optionalKey(DesktopServerExposureModeSchema), - tailscaleServeEnabled: Schema.optionalKey(Schema.Boolean), - tailscaleServePort: Schema.optionalKey(Schema.Number), - updateChannel: Schema.optionalKey(DesktopUpdateChannelSchema), - updateChannelConfiguredByUser: Schema.optionalKey(Schema.Boolean), - wslBackendEnabled: Schema.optionalKey(Schema.Boolean), - wslMode: Schema.optionalKey(Schema.Literals(["local", "wsl"])), - wslDistro: Schema.optionalKey(Schema.NullOr(Schema.String)), - wslOnly: Schema.optionalKey(Schema.Boolean), + localEnvironmentEnabled: optionalReadableKey(Schema.Boolean), + linuxPasswordStore: optionalReadableKey(Schema.Unknown), + mainWindowBounds: optionalReadableKey(Schema.NullOr(DesktopWindowBoundsDocument)), + mainWindowMaximized: optionalReadableKey(Schema.Boolean), + serverExposureMode: optionalReadableKey(DesktopServerExposureModeSchema), + tailscaleServeEnabled: optionalReadableKey(Schema.Boolean), + tailscaleServePort: optionalReadableKey(Schema.Number), + updateChannel: optionalReadableKey(DesktopUpdateChannelSchema), + updateChannelConfiguredByUser: optionalReadableKey(Schema.Boolean), + wslBackendEnabled: optionalReadableKey(Schema.Boolean), + wslMode: optionalReadableKey(Schema.Literals(["local", "wsl"])), + wslDistro: optionalReadableKey(Schema.NullOr(Schema.String)), + wslOnly: optionalReadableKey(Schema.Boolean), }); type DesktopSettingsDocument = typeof DesktopSettingsDocument.Type; @@ -379,15 +386,20 @@ function applyWslWindowsFallback(settings: DesktopSettings): DesktopSettings { return setWslOnly(setWslBackendEnabled(settings, false), false); } +// Fails only when the file may exist but could not be read; a missing file means defaults. function readSettings( fileSystem: FileSystem.FileSystem, settingsPath: string, appVersion: string, -): Effect.Effect { +): Effect.Effect { const defaultSettings = resolveDefaultDesktopSettings(appVersion); return fileSystem.readFileString(settingsPath).pipe( - Effect.option, + Effect.asSome, + Effect.catchIf( + (cause) => cause.reason._tag === "NotFound", + () => Effect.succeed(Option.none()), + ), Effect.flatMap( Option.match({ onNone: () => Effect.succeed(defaultSettings), @@ -476,6 +488,8 @@ export const make = Effect.gen(function* () { const path = yield* Path.Path; const random = yield* SecureRandom.SecureRandom; const settingsRef = yield* SynchronizedRef.make(environment.defaultDesktopSettings); + // Writing over a settings file we could not read would erase the preferences it holds. + const persistenceEnabled = yield* Ref.make(true); const updateInMemory = (update: (settings: DesktopSettings) => DesktopSettings) => SynchronizedRef.modify(settingsRef, (settings) => { @@ -492,17 +506,23 @@ export const make = Effect.gen(function* () { return Effect.succeed([settingsChange(settings, false), settings] as const); } - return random.uuidV4().pipe( - Effect.map((uuid) => uuid.replace(/-/g, "")), - Effect.flatMap((suffix) => - writeSettings({ - fileSystem, - path, - settingsPath: environment.desktopSettingsPath, - settings: nextSettings, - defaultSettings: environment.defaultDesktopSettings, - suffix, - }), + return Ref.get(persistenceEnabled).pipe( + Effect.flatMap((enabled) => + enabled + ? random.uuidV4().pipe( + Effect.map((uuid) => uuid.replace(/-/g, "")), + Effect.flatMap((suffix) => + writeSettings({ + fileSystem, + path, + settingsPath: environment.desktopSettingsPath, + settings: nextSettings, + defaultSettings: environment.defaultDesktopSettings, + suffix, + }), + ), + ) + : Effect.void, ), Effect.as([settingsChange(nextSettings, true), nextSettings] as const), ); @@ -515,6 +535,17 @@ export const make = Effect.gen(function* () { fileSystem, environment.desktopSettingsPath, environment.appVersion, + ).pipe( + Effect.catch((cause) => + Effect.logWarning( + "Could not read desktop settings; using defaults without saving changes.", + cause, + ).pipe( + Effect.annotateLogs({ settingsPath: environment.desktopSettingsPath }), + Effect.andThen(Ref.set(persistenceEnabled, false)), + Effect.as(resolveDefaultDesktopSettings(environment.appVersion)), + ), + ), ); return yield* SynchronizedRef.setAndGet(settingsRef, settings); }).pipe(Effect.withSpan("desktop.settings.load")), diff --git a/apps/desktop/src/settings/DesktopClientSettings.test.ts b/apps/desktop/src/settings/DesktopClientSettings.test.ts index 1ab9b3159e1f9..90c8740f83725 100644 --- a/apps/desktop/src/settings/DesktopClientSettings.test.ts +++ b/apps/desktop/src/settings/DesktopClientSettings.test.ts @@ -110,7 +110,7 @@ const withClientSettings = ( prefix: "t3-desktop-client-settings-test-", }); return yield* effect.pipe(Effect.provide(layer(baseDir))); - }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), Effect.scoped); + }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))); describe("DesktopClientSettings", () => { it.effect("saves through a symlinked client settings file without replacing the link", () => diff --git a/apps/desktop/src/settings/DesktopSavedEnvironments.test.ts b/apps/desktop/src/settings/DesktopSavedEnvironments.test.ts index 0001548f4534f..7be39dbe1562d 100644 --- a/apps/desktop/src/settings/DesktopSavedEnvironments.test.ts +++ b/apps/desktop/src/settings/DesktopSavedEnvironments.test.ts @@ -150,7 +150,7 @@ const withSavedEnvironments = ( prefix: "t3-desktop-saved-environments-test-", }); return yield* effect.pipe(Effect.provide(layer(baseDir, options))); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped); + }).pipe(Effect.provide(NodeServices.layer)); describe("DesktopSavedEnvironments", () => { it.effect("loads lenient saved environment registry documents", () => diff --git a/apps/desktop/src/shell/DesktopShellEnvironment.test.ts b/apps/desktop/src/shell/DesktopShellEnvironment.test.ts index c00e2ea7c65a2..7808682dfd82b 100644 --- a/apps/desktop/src/shell/DesktopShellEnvironment.test.ts +++ b/apps/desktop/src/shell/DesktopShellEnvironment.test.ts @@ -2,6 +2,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, describe, it } from "@effect/vitest"; import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Deferred from "effect/Deferred"; @@ -62,6 +63,26 @@ function makeProcess(output: string, exitCode = 0): ChildProcessSpawner.ChildPro }); } +function makeSlowProcess( + output: string, + delay: Duration.Input, +): ChildProcessSpawner.ChildProcessHandle { + const stdout = Stream.fromEffect(Effect.as(Effect.sleep(delay), textEncoder.encode(output))); + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(125), + stdout, + stderr: Stream.empty, + all: stdout, + exitCode: Effect.as(Effect.sleep(delay), ChildProcessSpawner.ExitCode(0)), + isRunning: Effect.succeed(true), + kill: () => Effect.void, + stdin: Sink.drain, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }); +} + function makeHungProcess(): ChildProcessSpawner.ChildProcessHandle { return ChildProcessSpawner.makeHandle({ pid: ChildProcessSpawner.ProcessId(124), @@ -486,7 +507,7 @@ describe("DesktopShellEnvironment", () => { Effect.gen(function* () { const env: NodeJS.ProcessEnv = { SHELL: "/bin/zsh", - PATH: "/Users/test/.local/bin:/usr/bin", + PATH: " /Users/test/.local/bin:/usr/bin", }; const commands: ChildProcess.Command[] = []; @@ -496,7 +517,7 @@ describe("DesktopShellEnvironment", () => { handler: (command) => { commands.push(command); return envOutput({ - PATH: "/opt/homebrew/bin:/usr/bin", + PATH: "/opt/homebrew/bin:/usr/bin:/opt/bin ", SSH_AUTH_SOCK: "/tmp/secretive.sock", HOMEBREW_PREFIX: "/opt/homebrew", }); @@ -505,7 +526,7 @@ describe("DesktopShellEnvironment", () => { assert.equal(commands.length, 1); assert.equal(commands[0]?._tag === "StandardCommand" ? commands[0].command : "", "/bin/zsh"); - assert.equal(env.PATH, "/opt/homebrew/bin:/usr/bin:/Users/test/.local/bin"); + assert.equal(env.PATH, "/opt/homebrew/bin:/usr/bin:/opt/bin : /Users/test/.local/bin"); assert.equal(env.SSH_AUTH_SOCK, "/tmp/secretive.sock"); assert.equal(env.HOMEBREW_PREFIX, "/opt/homebrew"); assert.isTrue(prepared); @@ -580,6 +601,39 @@ describe("DesktopShellEnvironment", () => { }), ); + it.effect.each([ + { + name: "hydrates both Android SDK names when neither is inherited", + inherited: {}, + expected: { ANDROID_HOME: "/shell/android-home", ANDROID_SDK_ROOT: "/shell/android-root" }, + }, + { + // The device host reads ANDROID_HOME before ANDROID_SDK_ROOT, so a shell + // ANDROID_HOME next to an inherited ANDROID_SDK_ROOT would change the SDK. + name: "keeps an inherited ANDROID_SDK_ROOT selected over a login-shell ANDROID_HOME", + inherited: { ANDROID_SDK_ROOT: "/inherited/android-sdk" }, + expected: { ANDROID_HOME: undefined, ANDROID_SDK_ROOT: "/inherited/android-sdk" }, + }, + ])("$name", ({ inherited, expected }) => + Effect.gen(function* () { + const env: NodeJS.ProcessEnv = { SHELL: "/bin/zsh", PATH: "/usr/bin", ...inherited }; + + yield* runShellEnvironment({ + env, + platform: "darwin", + handler: () => + envOutput({ + PATH: "/usr/bin", + ANDROID_HOME: "/shell/android-home", + ANDROID_SDK_ROOT: "/shell/android-root", + }), + }); + + assert.equal(env.ANDROID_HOME, expected.ANDROID_HOME); + assert.equal(env.ANDROID_SDK_ROOT, expected.ANDROID_SDK_ROOT); + }), + ); + it.effect("keeps mise's PATH record with the login shell PATH", () => Effect.gen(function* () { const env: NodeJS.ProcessEnv = { @@ -637,6 +691,129 @@ describe("DesktopShellEnvironment", () => { }), ); + it.effect("retries the user's shell when only the fallback shell answered", () => + Effect.gen(function* () { + const env: NodeJS.ProcessEnv = { SHELL: "/usr/bin/fish", PATH: "/usr/bin" }; + const commands: string[] = []; + let fishReady = false; + yield* runShellEnvironment({ + env, + platform: "linux", + handler: (command) => { + if (command._tag !== "StandardCommand") return ""; + commands.push(command.command); + if (command.command !== "/usr/bin/fish") return envOutput({ PATH: "/usr/bin" }); + return fishReady + ? envOutput({ PATH: "/fish/bin:/usr/bin" }) + : { output: "", exitCode: 7 }; + }, + use: (service) => + Effect.gen(function* () { + assert.isTrue(yield* service.awaitDiscovery); + assert.equal(env.PATH, "/usr/bin"); + // The Linux probe checks the D-Bus socket on the real filesystem, which the clock does not await. + const published = yield* service.updates.pipe( + Stream.filter((snapshot) => snapshot.environment.PATH === "/fish/bin:/usr/bin"), + Stream.runHead, + Effect.forkChild({ startImmediately: true }), + ); + + fishReady = true; + yield* TestClock.adjust("30 seconds"); + assert.isTrue(Option.isSome(yield* Fiber.join(published))); + assert.equal(env.PATH, "/fish/bin:/usr/bin"); + return true; + }), + }).pipe(Effect.provide(Logger.layer([]))); + assert.deepEqual(commands, ["/usr/bin/fish", "/bin/bash", "/usr/bin/fish"]); + + // With no user shell to wait for, the fallback's answer is final. + commands.length = 0; + yield* runShellEnvironment({ + env: { PATH: "/usr/bin" }, + platform: "linux", + handler: (command) => { + if (command._tag === "StandardCommand") commands.push(command.command); + return envOutput({ PATH: "/usr/bin" }); + }, + use: (service) => + Effect.gen(function* () { + assert.isTrue(yield* service.awaitDiscovery); + yield* TestClock.adjust("5 minutes"); + return true; + }), + }).pipe(Effect.provide(Logger.layer([]))); + assert.deepEqual(commands, ["/bin/bash"]); + }), + ); + + it.effect("gives background retries time for a locked or slow shell, then publishes", () => + Effect.gen(function* () { + const env: NodeJS.ProcessEnv = { SHELL: "/bin/zsh", PATH: "/usr/bin" }; + // Startup fails fast, the first retry meets a shell blocked on a lock, the second a slow one. + const spawned = yield* Effect.forEach([0, 1, 2], () => Deferred.make()); + const stopped = yield* Deferred.make(); + let attempts = 0; + yield* runShellEnvironment({ + env, + platform: "darwin", + handler: () => "", + processSpawner: ChildProcessSpawner.make((command) => { + if (command._tag !== "StandardCommand" || command.command !== "/bin/zsh") { + return Effect.succeed(makeProcess("")); + } + const attempt = attempts++; + const spawn = Deferred.succeed(spawned[attempt]!, undefined); + if (attempt === 0) return spawn.pipe(Effect.as(makeProcess(""))); + if (attempt === 1) { + return spawn.pipe( + Effect.andThen( + Effect.acquireRelease(Effect.succeed(makeHungProcess()), () => + Deferred.succeed(stopped, command.options), + ), + ), + ); + } + return spawn.pipe( + Effect.as(makeSlowProcess(envOutput({ PATH: "/fresh/bin" }), "20 seconds")), + ); + }), + use: (service) => + Effect.gen(function* () { + assert.isFalse(yield* service.awaitDiscovery); + const published = yield* service.updates.pipe( + Stream.drop(1), + Stream.runHead, + Effect.forkChild({ startImmediately: true }), + ); + + yield* TestClock.adjust("30 seconds"); + yield* Deferred.await(spawned[1]!); + // pyenv waits up to 60 s on its rehash lock before the shell continues. + yield* TestClock.adjust("1 minute"); + assert.isFalse(yield* Deferred.isDone(stopped)); + yield* TestClock.adjust("1 minute"); + const stop = yield* Deferred.await(stopped); + // pyenv releases that lock in an EXIT trap, which SIGTERM runs and SIGKILL skips. + assert.equal(stop.killSignal, "SIGTERM"); + assert.isAtLeast(Duration.toMillis(stop.forceKillAfter ?? 0), 5_000); + + yield* TestClock.adjust("30 seconds"); + yield* Deferred.await(spawned[2]!); + yield* TestClock.adjust("20 seconds"); + const snapshot = yield* Fiber.join(published); + assert.equal(env.PATH, "/fresh/bin:/usr/bin"); + assert.isTrue(Option.isSome(snapshot) && snapshot.value.discovered); + assert.equal( + Option.isSome(snapshot) ? snapshot.value.environment.PATH : undefined, + "/fresh/bin:/usr/bin", + ); + return true; + }), + }); + }), + ); + it.effect("recovers the login-shell PATH when the first probe outlasts the timeout", () => Effect.gen(function* () { const env: NodeJS.ProcessEnv = { SHELL: "/bin/zsh", PATH: "/usr/bin:/bin:/usr/sbin:/sbin" }; diff --git a/apps/desktop/src/shell/DesktopShellEnvironment.ts b/apps/desktop/src/shell/DesktopShellEnvironment.ts index f0983ca622490..351150aa4b987 100644 --- a/apps/desktop/src/shell/DesktopShellEnvironment.ts +++ b/apps/desktop/src/shell/DesktopShellEnvironment.ts @@ -3,6 +3,7 @@ import { applyShellEnvironment, buildEnvironmentCaptureCommand, listLoginShellCandidates, + mergePathValues, readUserLoginShell, setWindowsPath, } from "@t3tools/shared/shell"; @@ -14,6 +15,7 @@ import * as Deferred from "effect/Deferred"; import * as Path from "effect/Path"; import * as PubSub from "effect/PubSub"; import * as Ref from "effect/Ref"; +import * as Schedule from "effect/Schedule"; import * as Scope from "effect/Scope"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; @@ -27,9 +29,17 @@ import * as DesktopEnvironment from "../app/DesktopEnvironment.ts"; type EnvironmentPatch = Record; +interface ProbeBudget { + readonly timeout: Duration.Duration; + // One longer wait for the first login shell that times out. + readonly slowTimeout?: Duration.Duration; + readonly terminateGrace: Duration.Duration; +} + interface ShellEnvironmentConfig { readonly env: NodeJS.ProcessEnv; readonly platform: NodeJS.Platform; + readonly budget: ProbeBudget; } interface WindowsProbeOptions { @@ -88,6 +98,8 @@ export class DesktopShellEnvironment extends Context.Service< const LOGIN_SHELL_ENV_NAMES = [ "PATH", + "ANDROID_HOME", + "ANDROID_SDK_ROOT", "DBUS_SESSION_BUS_ADDRESS", "DISPLAY", "LANG", @@ -114,6 +126,7 @@ const LOGIN_SHELL_ENV_NAMES = [ ] as const; const WINDOWS_PROFILE_ENV_NAMES = ["PATH", "FNM_DIR", "FNM_MULTISHELL_PATH"] as const; const LOCALE_ENV_NAMES = ["LANG", "LC_ALL", "LC_CTYPE"] as const; +const ANDROID_SDK_ENV_NAMES = ["ANDROID_HOME", "ANDROID_SDK_ROOT"] as const; const FALLBACK_LC_CTYPE = "en_US.UTF-8"; const WINDOWS_SHELL_CANDIDATES = ["pwsh.exe", "powershell.exe"] as const; const LOGIN_SHELL_TIMEOUT = Duration.seconds(5); @@ -121,6 +134,20 @@ const LOGIN_SHELL_TIMEOUT = Duration.seconds(5); const SLOW_LOGIN_SHELL_TIMEOUT = Duration.seconds(20); const LAUNCHCTL_TIMEOUT = Duration.seconds(2); const PROCESS_TERMINATE_GRACE = Duration.seconds(1); +const STARTUP_PROBE: ProbeBudget = { + timeout: LOGIN_SHELL_TIMEOUT, + slowTimeout: SLOW_LOGIN_SHELL_TIMEOUT, + terminateGrace: PROCESS_TERMINATE_GRACE, +}; +// Retries block nothing, so they outlast pyenv's 60 s wait on its rehash lock, and SIGTERM gets +// time to run the exit trap that releases it. A SIGKILL leaves the lock for two minutes. +const RETRY_PROBE: ProbeBudget = { + timeout: Duration.minutes(2), + terminateGrace: Duration.seconds(10), +}; +// A Mac asleep at launch fails the first probe on wake, and nothing else would probe again. +const DISCOVERY_RETRY_DELAY = Duration.seconds(30); +const DISCOVERY_RETRIES = 10; const trimNonEmpty = (value: string | null | undefined): Option.Option => Option.fromNullishOr(value).pipe( @@ -128,10 +155,12 @@ const trimNonEmpty = (value: string | null | undefined): Option.Option = Option.filter((entry) => entry.length > 0), ); -const pathDelimiter = (platform: NodeJS.Platform) => (platform === "win32" ? ";" : ":"); +// POSIX directory names may begin or end with whitespace; mergePathValues trims only on Windows. +const nonEmptyPath = (value: string | null | undefined): Option.Option => + Option.fromNullishOr(value).pipe(Option.filter((entry) => entry.length > 0)); const readEnvPath = (env: NodeJS.ProcessEnv): Option.Option => - trimNonEmpty(env.PATH ?? env.Path ?? env.path); + nonEmptyPath(env.PATH ?? env.Path ?? env.path); const normalizeRuntimeDir = (value: string): string => value.replace(/\/+$/u, ""); @@ -150,39 +179,16 @@ const linuxRuntimeDirCandidates = ( return candidates.filter((candidate) => candidate.length > 0); }; -const pathComparisonKey = (entry: string, platform: NodeJS.Platform) => { - const normalized = entry.trim().replace(/^"+|"+$/g, ""); - return platform === "win32" ? normalized.toLowerCase() : normalized; -}; - -const sanitizePathEntry = (entry: string, platform: NodeJS.Platform) => - platform === "win32" ? entry.replaceAll('"', "") : entry; - const mergePaths = ( platform: NodeJS.Platform, values: ReadonlyArray>, -): Option.Option => { - const delimiter = pathDelimiter(platform); - const entries: string[] = []; - const seen = new Set(); - - for (const value of values) { - if (Option.isNone(value)) continue; - - for (const entry of value.value.split(delimiter)) { - const sanitized = sanitizePathEntry(entry.trim(), platform); - if (sanitized.length === 0) continue; - - const key = pathComparisonKey(sanitized, platform); - if (key.length === 0 || seen.has(key)) continue; - - seen.add(key); - entries.push(sanitized); - } - } - - return entries.length > 0 ? Option.some(entries.join(delimiter)) : Option.none(); -}; +): Option.Option => + Option.fromNullishOr( + values.reduce( + (merged, value) => mergePathValues(merged, Option.getOrUndefined(value), platform), + undefined, + ), + ); const knownWindowsCliDirs = (env: NodeJS.ProcessEnv): ReadonlyArray => [ ...trimNonEmpty(env.APPDATA).pipe( @@ -262,6 +268,7 @@ const runCommandOutput = Effect.fn("desktop.shellEnvironment.runCommandOutput")( readonly args: ReadonlyArray; readonly env: NodeJS.ProcessEnv; readonly timeout: Duration.Duration; + readonly terminateGrace?: Duration.Duration; readonly shell?: boolean; }): Effect.fn.Return< string, @@ -280,7 +287,7 @@ const runCommandOutput = Effect.fn("desktop.shellEnvironment.runCommandOutput")( stdout: "pipe", stderr: "ignore", killSignal: "SIGTERM", - forceKillAfter: PROCESS_TERMINATE_GRACE, + forceKillAfter: input.terminateGrace ?? PROCESS_TERMINATE_GRACE, }), ); return yield* Effect.all([Stream.mkString(Stream.decodeText(child.stdout)), child.exitCode], { @@ -346,6 +353,7 @@ const readLoginShellEnvironment = ( names: ReadonlyArray, env: NodeJS.ProcessEnv, timeout: Duration.Duration, + terminateGrace: Duration.Duration, ): Effect.Effect< EnvironmentPatch, DesktopShellEnvironmentCommandTimeoutError, @@ -359,6 +367,7 @@ const readLoginShellEnvironment = ( args: ["-ilc", buildEnvironmentCaptureCommand(shell, names)], env, timeout, + terminateGrace, }).pipe(Effect.map((output) => extractEnvironment(output, names))); const readLaunchctlPath = (env: NodeJS.ProcessEnv) => @@ -375,6 +384,7 @@ const readWindowsEnvironment = Effect.fn("desktop.shellEnvironment.readWindowsEn names: ReadonlyArray, options: WindowsProbeOptions, env: NodeJS.ProcessEnv, + budget: ProbeBudget, ): Effect.fn.Return { if (names.length === 0) return {}; @@ -392,7 +402,8 @@ const readWindowsEnvironment = Effect.fn("desktop.shellEnvironment.readWindowsEn command, args, env, - timeout: LOGIN_SHELL_TIMEOUT, + timeout: budget.timeout, + terminateGrace: budget.terminateGrace, }).pipe(orEmptyOnTimeout); const environment = extractEnvironment(output, names); if (Object.keys(environment).length > 0) { @@ -410,8 +421,13 @@ const installWindowsEnvironment = Effect.fn("desktop.shellEnvironment.installWin ): Effect.fn.Return { const [noProfile, profile] = yield* Effect.all( [ - readWindowsEnvironment(["PATH"], { loadProfile: false }, config.env), - readWindowsEnvironment(WINDOWS_PROFILE_ENV_NAMES, { loadProfile: true }, config.env), + readWindowsEnvironment(["PATH"], { loadProfile: false }, config.env, config.budget), + readWindowsEnvironment( + WINDOWS_PROFILE_ENV_NAMES, + { loadProfile: true }, + config.env, + config.budget, + ), ], { concurrency: 2 }, ); @@ -435,6 +451,9 @@ const installWindowsEnvironment = Effect.fn("desktop.shellEnvironment.installWin }, ); +const listUserShells = (env: NodeJS.ProcessEnv): ReadonlyArray => + [env.SHELL?.trim(), readUserLoginShell()].filter((shell): shell is string => Boolean(shell)); + const installPosixEnvironment = Effect.fn("desktop.shellEnvironment.installPosixEnvironment")( function* ( config: ShellEnvironmentConfig, @@ -445,22 +464,28 @@ const installPosixEnvironment = Effect.fn("desktop.shellEnvironment.installPosix > { const fileSystem = yield* FileSystem.FileSystem; const shellEnvironment: EnvironmentPatch = {}; - const userShells = [config.env.SHELL?.trim(), readUserLoginShell()]; + const userShells = listUserShells(config.env); let answeredBy: string | undefined; let waitedForSlowShell = false; for (const shell of listLoginShellCandidates(config.platform, config.env.SHELL)) { const read = (timeout: Duration.Duration) => - readLoginShellEnvironment(shell, LOGIN_SHELL_ENV_NAMES, config.env, timeout); + readLoginShellEnvironment( + shell, + LOGIN_SHELL_ENV_NAMES, + config.env, + timeout, + config.budget.terminateGrace, + ); Object.assign( shellEnvironment, - yield* read(LOGIN_SHELL_TIMEOUT).pipe( + yield* read(config.budget.timeout).pipe( Effect.catchTags({ DesktopShellEnvironmentCommandTimeoutError: (error) => { - const retry = !waitedForSlowShell; + const slowTimeout = waitedForSlowShell ? undefined : config.budget.slowTimeout; waitedForSlowShell = true; return logShellEnvironmentCommandError(error).pipe( - Effect.andThen(retry ? read(SLOW_LOGIN_SHELL_TIMEOUT) : Effect.succeed({})), + Effect.andThen(slowTimeout ? read(slowTimeout) : Effect.succeed({})), ); }, }), @@ -481,7 +506,7 @@ const installPosixEnvironment = Effect.fn("desktop.shellEnvironment.installPosix ? yield* readLaunchctlPath(config.env) : Option.none(); const mergedPath = mergePaths(config.platform, [ - trimNonEmpty(shellEnvironment.PATH).pipe(Option.orElse(() => launchctlPath)), + nonEmptyPath(shellEnvironment.PATH).pipe(Option.orElse(() => launchctlPath)), readEnvPath(config.env), ]); @@ -528,6 +553,17 @@ const installPosixEnvironment = Effect.fn("desktop.shellEnvironment.installPosix } } + // The device host picks the Android SDK from ANDROID_HOME before ANDROID_SDK_ROOT, + // so a profile-only ANDROID_HOME must not shadow an inherited ANDROID_SDK_ROOT. + if (ANDROID_SDK_ENV_NAMES.every((name) => Option.isNone(trimNonEmpty(config.env[name])))) { + for (const name of ANDROID_SDK_ENV_NAMES) { + const value = trimNonEmpty(shellEnvironment[name]); + if (Option.isSome(value)) { + config.env[name] = value.value; + } + } + } + // Hydrate locale variables together; inherited LC_ALL can override LANG or LC_CTYPE. if ( config.platform === "darwin" && @@ -563,7 +599,10 @@ const installPosixEnvironment = Effect.fn("desktop.shellEnvironment.installPosix } const prepared = Option.isSome(trimNonEmpty(shellEnvironment.PATH)); - return { prepared, fromUserShell: prepared && userShells.includes(answeredBy) }; + return { + prepared, + fromUserShell: prepared && answeredBy !== undefined && userShells.includes(answeredBy), + }; }, ); @@ -647,38 +686,60 @@ export const make = Effect.gen(function* () { yield* Ref.set(latest, Option.some(message)); yield* PubSub.publish(changes, message); }); - const refresh = Effect.gen(function* () { - // Refresh from the inherited baseline so removed cached paths and values do not survive. - const fresh = { ...inherited }; - const discovery = yield* installShellEnvironment({ - env: fresh, - platform: environment.platform, - }); - const { prepared } = discovery; - const patch = Object.fromEntries( - Object.keys(ShellEnvironmentPatch.fields).map((name) => [name, fresh[name] ?? null]), - ); - if (prepared || Option.isNone(yield* Ref.get(latest))) yield* publish(patch, prepared); - if (prepared) { - const cached = Object.fromEntries( - CACHED_ENV_NAMES.flatMap((name) => (fresh[name] ? [[name, fresh[name]]] : [])), - ); - const persist = Effect.gen(function* () { - const encoded = yield* encodeCache({ version: 1, key, environment: cached }); - yield* fileSystem.makeDirectory(environment.stateDir, { recursive: true }); - yield* fileSystem.writeFileString(cachePath + ".tmp", encoded, { mode: 0o600 }); - yield* fileSystem.rename(cachePath + ".tmp", cachePath); + const probe = Effect.fn("desktop.shellEnvironment.refresh")( + function* (budget: ProbeBudget) { + // Refresh from the inherited baseline so removed cached paths and values do not survive. + const fresh = { ...inherited }; + const discovery = yield* installShellEnvironment({ + env: fresh, + platform: environment.platform, + budget, }); - yield* persist.pipe( - Effect.catch(() => Effect.logWarning("Could not cache shell environment")), + const { prepared } = discovery; + const patch = Object.fromEntries( + Object.keys(ShellEnvironmentPatch.fields).map((name) => [name, fresh[name] ?? null]), ); - } - return discovery; - }).pipe( + if (prepared || Option.isNone(yield* Ref.get(latest))) yield* publish(patch, prepared); + if (prepared) { + const cached = Object.fromEntries( + CACHED_ENV_NAMES.flatMap((name) => (fresh[name] ? [[name, fresh[name]]] : [])), + ); + const persist = Effect.gen(function* () { + const encoded = yield* encodeCache({ version: 1, key, environment: cached }); + yield* fileSystem.makeDirectory(environment.stateDir, { recursive: true }); + yield* fileSystem.writeFileString(cachePath + ".tmp", encoded, { mode: 0o600 }); + yield* fileSystem.rename(cachePath + ".tmp", cachePath); + }); + yield* persist.pipe( + Effect.catch(() => Effect.logWarning("Could not cache shell environment")), + ); + } + return discovery; + }, Effect.provideService(FileSystem.FileSystem, fileSystem), Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), - Effect.withSpan("desktop.shellEnvironment.refresh"), + ); + // A fallback shell's PATH can miss folders the user's own shell adds, so keep asking that shell. + const awaitsUserShell = listUserShells(inherited).length > 0; + const settled = ({ prepared, fromUserShell }: ShellDiscovery) => + fromUserShell || (prepared && !awaitsUserShell); + const refresh = probe(STARTUP_PROBE).pipe( Effect.onExit((exit) => Deferred.done(discovered, exit)), + Effect.flatMap((discovery) => + settled(discovery) + ? Effect.void + : Effect.sleep(DISCOVERY_RETRY_DELAY).pipe( + Effect.andThen( + probe(RETRY_PROBE).pipe( + Effect.repeat({ + until: settled, + schedule: Schedule.spaced(DISCOVERY_RETRY_DELAY), + times: DISCOVERY_RETRIES - 1, + }), + ), + ), + ), + ), ); const installIntoProcess = yield* Effect.cached( Effect.gen(function* () { diff --git a/apps/desktop/src/ssh/DesktopSshEnvironment.ts b/apps/desktop/src/ssh/DesktopSshEnvironment.ts index d47984e2bcb59..0693c16cc15fd 100644 --- a/apps/desktop/src/ssh/DesktopSshEnvironment.ts +++ b/apps/desktop/src/ssh/DesktopSshEnvironment.ts @@ -19,6 +19,7 @@ import { import * as SshTunnel from "@t3tools/ssh/tunnel"; import * as Context from "effect/Context"; import type * as Digest from "effect/crypto/Digest"; +import * as SecureRandom from "effect/crypto/SecureRandom"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; @@ -70,8 +71,30 @@ export class DesktopSshEnvironment extends Context.Service< export interface DesktopSshEnvironmentLayerOptions { readonly resolveCliRunner?: Effect.Effect; + readonly resolveLauncherId?: Effect.Effect; } +// Remote servers record this ID as their owner, so it has to outlive restarts, +// updates and edits to a saved connection. Without a stored one there is none. +export const readOrCreateLauncherId = Effect.fn("desktop.ssh.readOrCreateLauncherId")( + function* (filePath: string) { + const fileSystem = yield* FileSystem.FileSystem; + const random = yield* SecureRandom.SecureRandom; + // Any other read failure may hide a stored ID, which a new one would overwrite. + const stored = yield* fileSystem.readFileString(filePath).pipe( + Effect.catchIf( + (cause) => cause.reason._tag === "NotFound", + () => Effect.succeed(""), + ), + ); + if (/^[0-9a-f]{32}$/u.test(stored.trim())) return stored.trim(); + const launcherId = (yield* random.uuidV4()).replace(/-/g, ""); + yield* fileSystem.writeFileString(filePath, `${launcherId}\n`); + return launcherId; + }, + Effect.orElseSucceed(() => null), +); + function discoverDesktopSshHostsEffect(input?: { readonly homeDir?: string }) { return discoverSshHosts(input ?? {}); } @@ -164,10 +187,13 @@ export const make = Effect.gen(function* () { export const layer = (options: DesktopSshEnvironmentLayerOptions = {}) => Layer.effect(DesktopSshEnvironment, make).pipe( Layer.provide( - SshTunnel.SshEnvironmentManager.layer( - options.resolveCliRunner === undefined + SshTunnel.SshEnvironmentManager.layer({ + ...(options.resolveCliRunner === undefined ? {} - : { resolveCliRunner: options.resolveCliRunner }, - ), + : { resolveCliRunner: options.resolveCliRunner }), + ...(options.resolveLauncherId === undefined + ? {} + : { resolveLauncherId: options.resolveLauncherId }), + }), ), ); diff --git a/apps/desktop/src/ssh/DesktopSshPasswordPrompts.test.ts b/apps/desktop/src/ssh/DesktopSshPasswordPrompts.test.ts index 1e5b6dafc6cae..15977b34f398b 100644 --- a/apps/desktop/src/ssh/DesktopSshPasswordPrompts.test.ts +++ b/apps/desktop/src/ssh/DesktopSshPasswordPrompts.test.ts @@ -144,7 +144,7 @@ describe("DesktopSshPasswordPrompts", () => { yield* prompts.resolve({ requestId: request.requestId, password: "secret" }); assert.equal(yield* Fiber.join(fiber), "secret"); - }).pipe(Effect.provide(layer(testWindow.window)), Effect.scoped); + }).pipe(Effect.provide(layer(testWindow.window))); }); it.effect.each(["answered", "cancelled", "timed out", "interrupted"] as const)( @@ -189,7 +189,7 @@ describe("DesktopSshPasswordPrompts", () => { .map((message) => message.args), [[request.requestId]], ); - }).pipe(Effect.provide(layer(testWindow.window)), Effect.scoped); + }).pipe(Effect.provide(layer(testWindow.window))); }, ); @@ -219,6 +219,6 @@ describe("DesktopSshPasswordPrompts", () => { assert.equal(password, "secret"); assert.equal(testWindow.isFocused(), false); assert.equal(testWindow.closedListenerCount(), 0); - }).pipe(Effect.provide(layer(testWindow.window)), Effect.scoped); + }).pipe(Effect.provide(layer(testWindow.window))); }); }); diff --git a/apps/desktop/src/telemetry/DesktopTelemetryPublisher.ts b/apps/desktop/src/telemetry/DesktopTelemetryPublisher.ts index 0a2b10e08c5a8..415901878cafe 100644 --- a/apps/desktop/src/telemetry/DesktopTelemetryPublisher.ts +++ b/apps/desktop/src/telemetry/DesktopTelemetryPublisher.ts @@ -352,6 +352,8 @@ export const make = Effect.fn("desktop.telemetryPublisher.make")(function* () { return Queue.offer(updateCommitQueue, message).pipe(Effect.asVoid); case "cancelDesktopUpdate": return Queue.offer(updateCancellationQueue, message).pipe(Effect.asVoid); + case "serverReady": + return Effect.void; } }; const removeControlSource: DesktopTelemetryPublisher["Service"]["removeControlSource"] = ( diff --git a/apps/desktop/src/updates/DesktopRemoteUpdates.test.ts b/apps/desktop/src/updates/DesktopRemoteUpdates.test.ts index 1f5321148b77b..fdac65f86a595 100644 --- a/apps/desktop/src/updates/DesktopRemoteUpdates.test.ts +++ b/apps/desktop/src/updates/DesktopRemoteUpdates.test.ts @@ -102,6 +102,29 @@ describe("DesktopRemoteUpdates", () => { assert.equal(terminals[0]?.requestId, "req-1"); assert.equal(harness.quitAndInstalls(), 0); + yield* Queue.offer(requests, request("req-retry")); + yield* settle; + assert.deepEqual( + terminalReports(reports).map((report) => report.outcome), + ["ready-to-install", "ready-to-install"], + ); + assert.equal(terminalReports(reports)[1]?.requestId, "req-retry"); + assert.equal(terminalReports(reports)[1]?.preparedRequestId, "req-1"); + assert.equal(harness.quitAndInstalls(), 0); + assert.equal(harness.downloadCount(), 1); + + yield* Queue.offer(requests, { + ...request("req-other-track"), + channel: "nightly", + targetVersion: "1.2.4-nightly.20261027.1", + }); + yield* settle; + assert.equal(terminalReports(reports)[2]?.outcome, "failed"); + assert.equal( + terminalReports(reports)[2]?.reason, + "A prepared desktop update is already in progress.", + ); + yield* Queue.offer(commits, { version: 1, type: "commitDesktopUpdate", @@ -119,6 +142,49 @@ describe("DesktopRemoteUpdates", () => { ); }); + it.effect("does not renew the preparation lease when a retry recovers its token", () => { + const harness = makeHarness(); + + return runRemoteUpdatesTest(harness, ({ reports, requests, commits }) => + Effect.gen(function* () { + harness.emit("update-downloaded", { version: "1.2.4" }); + yield* settle; + yield* Queue.offer(requests, request("req-original")); + yield* settle; + + yield* TestClock.adjust(Duration.minutes(4)); + harness.emit("update-available", { version: "1.2.4" }); + yield* settle; + yield* Queue.offer(requests, request("req-retry")); + yield* settle; + assert.equal(terminalReports(reports)[1]?.preparedRequestId, "req-original"); + + yield* TestClock.adjust(Duration.minutes(1)); + yield* Queue.offer(commits, { + version: 1, + type: "commitDesktopUpdate", + requestId: "req-original", + }); + yield* settle; + assert.equal( + terminalReports(reports)[2]?.reason, + "This desktop update is no longer prepared.", + ); + assert.equal(harness.quitAndInstalls(), 0); + + yield* Queue.offer(requests, request("req-new")); + yield* settle; + yield* Queue.offer(commits, { + version: 1, + type: "commitDesktopUpdate", + requestId: "req-new", + }); + yield* settle; + assert.equal(harness.quitAndInstalls(), 1); + }), + ); + }); + it.effect("switches to the requested track before preparing its release", () => { // In-memory settings keep the channel write off real disk I/O. const harness = makeHarness({ beforeSetUpdateChannel: Effect.void }); @@ -691,7 +757,7 @@ describe("DesktopRemoteUpdates", () => { const backgroundCheck = yield* updates.check("poll").pipe(Effect.forkChild); yield* Deferred.await(backgroundCheckStarted); - assert.equal((yield* updates.getState).status, "checking"); + assert.isTrue(yield* updates.isActionActive); yield* Queue.offer(commits, { version: 1, @@ -701,7 +767,6 @@ describe("DesktopRemoteUpdates", () => { yield* settle; assert.equal(harness.quitAndInstalls(), 0); - harness.emit("update-downloaded", { version: "1.2.4" }); yield* Deferred.succeed(releaseBackgroundCheck, undefined); yield* Fiber.join(backgroundCheck); yield* settle; diff --git a/apps/desktop/src/updates/DesktopRemoteUpdates.ts b/apps/desktop/src/updates/DesktopRemoteUpdates.ts index 560e2e78d4bb1..ce2f23ab3e781 100644 --- a/apps/desktop/src/updates/DesktopRemoteUpdates.ts +++ b/apps/desktop/src/updates/DesktopRemoteUpdates.ts @@ -32,6 +32,8 @@ const PREPARED_UPDATE_TTL = Duration.minutes(5); interface PreparedUpdate { readonly requestId: string; + readonly channel: DesktopTelemetryRequestDesktopUpdate["channel"]; + readonly targetVersion: DesktopTelemetryRequestDesktopUpdate["targetVersion"]; readonly downloadedVersion: string; readonly status: "prepared" | "committing" | "failed"; readonly failureReason?: string; @@ -67,7 +69,11 @@ export const listen: Effect.Effect< const publishReport = ( state: DesktopUpdateState, - terminal?: { readonly outcome: DesktopUpdateRemoteOutcome; readonly reason?: string }, + terminal?: { + readonly outcome: DesktopUpdateRemoteOutcome; + readonly reason?: string; + readonly preparedRequestId?: string; + }, explicitRequestId?: string, ): Effect.Effect => { const reason = normalizeRemoteUpdateReason(terminal?.reason); @@ -82,6 +88,9 @@ export const listen: Effect.Effect< ? { requestId: requestId.value } : {}), ...(terminal === undefined ? {} : { outcome: terminal.outcome }), + ...(terminal?.preparedRequestId === undefined + ? {} + : { preparedRequestId: terminal.preparedRequestId }), ...(reason ? { reason } : {}), state, }), @@ -167,8 +176,22 @@ export const listen: Effect.Effect< : ([current, current] as const), ); if (Option.isSome(prepared) && prepared.value.status !== "failed") { + const state = yield* updates.getState; + if ( + prepared.value.status === "prepared" && + prepared.value.channel === request.channel && + prepared.value.targetVersion === request.targetVersion && + state.downloadedVersion === prepared.value.downloadedVersion + ) { + yield* publishReport( + state, + { outcome: "ready-to-install", preparedRequestId: prepared.value.requestId }, + request.requestId, + ); + return; + } yield* publishReport( - yield* updates.getState, + state, { outcome: "failed", reason: "A prepared desktop update is already in progress.", @@ -301,6 +324,8 @@ export const listen: Effect.Effect< preparedUpdateRef, Option.some({ requestId: request.requestId, + channel: request.channel, + targetVersion: request.targetVersion, downloadedVersion: state.downloadedVersion, status: "prepared", expiresAt: diff --git a/apps/desktop/src/updates/DesktopUpdates.test.ts b/apps/desktop/src/updates/DesktopUpdates.test.ts index b798719949a63..da08e427235aa 100644 --- a/apps/desktop/src/updates/DesktopUpdates.test.ts +++ b/apps/desktop/src/updates/DesktopUpdates.test.ts @@ -125,7 +125,7 @@ describe("DesktopUpdates", () => { ).pipe(Effect.provide(harness.layer)); }); - it.effect("checks for newer releases after an update has been downloaded", () => { + it.effect("re-checks an offered update without announcing it again", () => { const harness = makeHarness(); return Effect.scoped( @@ -138,14 +138,23 @@ describe("DesktopUpdates", () => { releaseNotes: "## What's changed\n- fix: queued update", }); yield* flushCallbacks; + const offeredStates = harness.sentStates.length; + + yield* updates.check("poll"); + harness.emit("update-available", { version: "1.2.4" }); + yield* flushCallbacks; + assert.equal(harness.sentStates.length, offeredStates); + harness.emit("update-downloaded", { version: "1.2.4" }); yield* flushCallbacks; + const downloadedStates = harness.sentStates.length; const result = yield* updates.check("poll"); assert.isTrue(result.checked); harness.emit("update-available", { version: "1.2.4" }); yield* flushCallbacks; + assert.equal(harness.sentStates.length, downloadedStates); const unchangedState = yield* updates.getState; assert.equal(unchangedState.status, "downloaded"); @@ -267,15 +276,26 @@ describe("DesktopUpdates", () => { ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); }); - it.effect( - "rejects install while a refresh check is in progress and releases the reservation", - () => + it.effect.each([ + { action: "download", offer: "update-available", found: null, accepted: true }, + { action: "install", offer: "update-downloaded", found: null, accepted: true }, + { action: "download", offer: "update-available", found: "1.2.5", accepted: true }, + { action: "install", offer: "update-downloaded", found: "1.2.5", accepted: false }, + ] as const)( + "a $action waiting out a refresh check acts on what it found ($found)", + ({ action, offer, found, accepted }) => Effect.gen(function* () { const checkStarted = yield* Deferred.make(); const releaseCheck = yield* Deferred.make(); const harness = makeHarness({ checkForUpdates: Deferred.succeed(checkStarted, undefined).pipe( Effect.andThen(Deferred.await(releaseCheck)), + // electron-updater emits the check's outcome before its promise resolves. + Effect.andThen( + Effect.sync(() => { + if (found !== null) harness.emit("update-available", { version: found }); + }), + ), ), }); @@ -283,22 +303,20 @@ describe("DesktopUpdates", () => { Effect.gen(function* () { const updates = yield* DesktopUpdates.DesktopUpdates; yield* updates.configure; - harness.emit("update-downloaded", { version: "1.2.4" }); + harness.emit(offer, { version: "1.2.4" }); yield* flushCallbacks; - const checkFiber = yield* updates.check("manual").pipe(Effect.forkScoped); + const checkFiber = yield* updates.check("poll").pipe(Effect.forkScoped); yield* Deferred.await(checkStarted); - - const installResult = yield* updates.install; - assert.isFalse(installResult.accepted); + const actionFiber = yield* updates[action].pipe(Effect.forkScoped); + yield* flushCallbacks; yield* Deferred.succeed(releaseCheck, undefined); - const checkResult = yield* Fiber.join(checkFiber); - assert.isTrue(checkResult.checked); - - const followUpCheck = yield* updates.check("manual"); - assert.isTrue(followUpCheck.checked); - assert.equal(harness.checkCount(), 2); + assert.isTrue((yield* Fiber.join(checkFiber)).checked); + assert.equal((yield* Fiber.join(actionFiber)).accepted, accepted); + const state = yield* updates.getState; + assert.equal(state.availableVersion, found ?? "1.2.4"); + if (action === "download") assert.equal(state.status, "downloading"); }), ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); }), @@ -544,14 +562,17 @@ describe("DesktopUpdates", () => { it.effect("marks the backend stop for an install as an update restart", () => { let markersAtStop: ReadonlyArray = []; + let inProgressAtStop: string | undefined; const harness = makeHarness({ stopBackend: Effect.sync(() => { markersAtStop = [...harness.updateRestartMarkers]; + inProgressAtStop = harness.updateInProgressMarker(); }), }); return Effect.scoped( Effect.gen(function* () { + yield* TestClock.setTime(1_000); const environment = yield* DesktopEnvironment.DesktopEnvironment; const updates = yield* DesktopUpdates.DesktopUpdates; yield* updates.configure; @@ -562,6 +583,12 @@ describe("DesktopUpdates", () => { assert.deepEqual(markersAtStop, [ environment.path.join(environment.baseDir, "runtime", DESKTOP_UPDATE_RESTART_MARKER_FILE), ]); + // SSH launchers wait on this until the replacement backend publishes its runtime. + assert.deepEqual(JSON.parse(inProgressAtStop ?? "null"), { + startedAt: 1_000, + expiresAt: 61_000, + }); + assert.isDefined(harness.updateInProgressMarker()); }), ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); }); @@ -644,9 +671,11 @@ describe("DesktopUpdates", () => { const installFiber = yield* updates.install.pipe(Effect.forkScoped); yield* Deferred.await(stopping); assert.equal(harness.updateRestartMarkers.size, 1); + assert.isDefined(harness.updateInProgressMarker()); yield* Fiber.interrupt(installFiber); assert.equal(harness.updateRestartMarkers.size, 0); + assert.isUndefined(harness.updateInProgressMarker()); }), ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); }), @@ -681,6 +710,7 @@ describe("DesktopUpdates", () => { "startBackend", ]); assert.equal(harness.updateRestartMarkers.size, 0); + assert.isUndefined(harness.updateInProgressMarker()); assert.equal(harness.installRecords.size, 0); }), ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); diff --git a/apps/desktop/src/updates/DesktopUpdates.ts b/apps/desktop/src/updates/DesktopUpdates.ts index 044e24537ca37..055c6574391a6 100644 --- a/apps/desktop/src/updates/DesktopUpdates.ts +++ b/apps/desktop/src/updates/DesktopUpdates.ts @@ -1,5 +1,7 @@ import { DESKTOP_UPDATE_RESTART_MARKER_FILE, + DESKTOP_UPDATE_IN_PROGRESS_MARKER_FILE, + DESKTOP_UPDATE_IN_PROGRESS_TTL_MS, DesktopUpdateChannelSchema, type DesktopRuntimeInfo, type DesktopUpdateActionResult, @@ -61,7 +63,7 @@ const CHANNEL_CHANGE_RETRY_DELAY = Duration.seconds(1); const CHANNEL_CHANGE_RETRIES = 30; const AUTO_UPDATE_STARTUP_DELAY = "15 seconds"; const AUTO_UPDATE_POLL_INTERVAL = "4 hours"; -const PREPARED_INSTALL_CHECK_WAIT = Duration.seconds(90); +const UPDATE_CHECK_WAIT = Duration.seconds(90); type UpdateAction = "check" | "download" | "install" | "install-recovery" | "channel"; @@ -316,6 +318,10 @@ function getCanRetryFromState(state: DesktopUpdateState): boolean { return state.availableVersion !== null || state.downloadedVersion !== null; } +function isOffered(state: DesktopUpdateState): boolean { + return state.status === "available" || state.status === "downloaded"; +} + function shouldBroadcastDownloadProgress( currentState: DesktopUpdateState, nextPercent: number, @@ -516,6 +522,23 @@ export const make = Effect.gen(function* () { ), ); + // Re-runs admit after each finished action, so a caller waiting on a check decides on its result. + const admitAfterCheck = Effect.fn("desktop.updates.admitAfterCheck")(function* ( + admit: Effect.Effect<"admitted" | "refused" | "wait-for-check">, + ) { + const actionCompletions = yield* PubSub.subscribe(finishedUpdateActions); + let admission = yield* stateMutex.withPermits(1)(admit); + while (admission === "wait-for-check") { + const finishedAction = yield* PubSub.take(actionCompletions).pipe( + Effect.timeoutOption(UPDATE_CHECK_WAIT), + ); + admission = Option.isNone(finishedAction) + ? "refused" + : yield* stateMutex.withPermits(1)(admit); + } + return admission; + }); + const applyAutoUpdaterChannel = Effect.fn("desktop.updates.applyAutoUpdaterChannel")(function* ( channel: DesktopUpdateChannel, ) { @@ -557,8 +580,10 @@ export const make = Effect.gen(function* () { if (actionReservation === "acquire" && !(yield* tryStartUpdateAction("check"))) return false; const check = Effect.gen(function* () { - const checkedAt = yield* currentIsoTimestamp; - yield* setState(reduceDesktopUpdateStateOnCheckStart(state, checkedAt)); + // An offer stays on screen while re-checking, so finding it again changes nothing. + if (!isOffered(state)) { + yield* setState(reduceDesktopUpdateStateOnCheckStart(state, yield* currentIsoTimestamp)); + } yield* logUpdaterInfo("checking for updates", { reason }); // The flag is shared updater state, so it flips only while this check holds the reservation. @@ -595,15 +620,30 @@ export const make = Effect.gen(function* () { }); const downloadAvailableUpdate = Effect.gen(function* () { - const state = yield* Ref.get(updateStateRef); - if (!(yield* Ref.get(updaterConfiguredRef)) || state.status !== "available") { - return { accepted: false, completed: false }; - } - - if (!(yield* tryStartUpdateAction("download"))) { + const admission = yield* Effect.scoped( + admitAfterCheck( + Effect.gen(function* () { + const state = yield* Ref.get(updateStateRef); + if (!(yield* Ref.get(updaterConfiguredRef)) || state.status !== "available") { + return "refused" as const; + } + const activeAction = yield* Ref.get(activeUpdateActionRef); + if (Option.isSome(activeAction)) { + return activeAction.value === "check" + ? ("wait-for-check" as const) + : ("refused" as const); + } + return (yield* tryStartUpdateAction("download")) + ? ("admitted" as const) + : ("refused" as const); + }), + ), + ); + if (admission === "refused") { return { accepted: false, completed: false }; } + const state = yield* Ref.get(updateStateRef); return yield* Effect.gen(function* () { yield* setState(reduceDesktopUpdateStateOnDownloadStart(state)); yield* electronUpdater.setDisableDifferentialDownload( @@ -669,6 +709,34 @@ export const make = Effect.gen(function* () { ), ); + const updateInProgressMarkerPath = environment.path.join( + updateRestartMarkerDir, + DESKTOP_UPDATE_IN_PROGRESS_MARKER_FILE, + ); + const encodeUpdateMarker = Schema.encodeEffect( + Schema.fromJsonString(Schema.Struct({ startedAt: Schema.Number, expiresAt: Schema.Number })), + ); + const updateInProgressTempPath = `${updateInProgressMarkerPath}.tmp`; + const writeUpdateInProgressMarker = Effect.gen(function* () { + const startedAt = DateTime.toEpochMillis(yield* DateTime.now); + const contents = yield* encodeUpdateMarker({ + startedAt, + expiresAt: startedAt + DESKTOP_UPDATE_IN_PROGRESS_TTL_MS, + }); + yield* fileSystem.writeFileString(updateInProgressTempPath, contents); + yield* fileSystem.rename(updateInProgressTempPath, updateInProgressMarkerPath); + }).pipe( + Effect.ensuring( + fileSystem.remove(updateInProgressTempPath, { force: true }).pipe(Effect.ignore), + ), + Effect.catch((error) => + logUpdaterWarning("Could not write the update-in-progress marker.", { errorTag: error._tag }), + ), + ); + const removeUpdateInProgressMarker = fileSystem + .remove(updateInProgressMarkerPath, { force: true }) + .pipe(Effect.ignore); + // Failed or interrupted installs leave no replacement backend; later quits must release the tunnel. const removeUpdateRestartMarker = fileSystem .remove(updateRestartMarkerPath, { force: true }) @@ -724,6 +792,7 @@ export const make = Effect.gen(function* () { Ref.set(desktopState.quitting, false), Ref.set(installRunningBackendsRef, []), removeUpdateRestartMarker, + removeUpdateInProgressMarker, removeInstallRecord, ], { discard: true }, @@ -741,6 +810,7 @@ export const make = Effect.gen(function* () { yield* Ref.set(desktopState.quitting, false); yield* removeUpdateRestartMarker; + yield* removeUpdateInProgressMarker; yield* removeInstallRecord; yield* Effect.gen(function* () { const runningInstances = yield* Ref.getAndSet(installRunningBackendsRef, []); @@ -766,47 +836,35 @@ export const make = Effect.gen(function* () { const installDownloadedUpdate = (expectedVersion?: string) => Effect.scoped( Effect.gen(function* () { - const actionCompletions = yield* PubSub.subscribe(finishedUpdateActions); - let admission: "admitted" | "refused" | "wait-for-check" = "wait-for-check"; - while (admission === "wait-for-check") { - admission = yield* stateMutex.withPermits(1)( - Effect.gen(function* () { - const state = yield* Ref.get(updateStateRef); - const activeAction = yield* Ref.get(activeUpdateActionRef); - const hasExpectedDownload = - state.downloadedVersion !== null && - (expectedVersion === undefined || state.downloadedVersion === expectedVersion); - if ( - (yield* Ref.get(desktopState.quitting)) || - !(yield* Ref.get(updaterConfiguredRef)) || - !hasExpectedDownload - ) { - return "refused" as const; - } - if (Option.isSome(activeAction)) { - return activeAction.value === "check" && expectedVersion !== undefined - ? ("wait-for-check" as const) - : ("refused" as const); - } - const hasInstallableDownload = - state.status === "downloaded" || - (state.status === "error" && - (state.errorContext === null || state.errorContext === "install")); - if (!hasInstallableDownload) return "refused" as const; - return (yield* tryStartUpdateAction("install")) - ? ("admitted" as const) + const admission = yield* admitAfterCheck( + Effect.gen(function* () { + const state = yield* Ref.get(updateStateRef); + const activeAction = yield* Ref.get(activeUpdateActionRef); + const hasExpectedDownload = + state.downloadedVersion !== null && + (expectedVersion === undefined || state.downloadedVersion === expectedVersion); + if ( + (yield* Ref.get(desktopState.quitting)) || + !(yield* Ref.get(updaterConfiguredRef)) || + !hasExpectedDownload + ) { + return "refused" as const; + } + if (Option.isSome(activeAction)) { + return activeAction.value === "check" + ? ("wait-for-check" as const) : ("refused" as const); - }), - ); - if (admission === "wait-for-check") { - const finishedAction = yield* PubSub.take(actionCompletions).pipe( - Effect.timeoutOption(PREPARED_INSTALL_CHECK_WAIT), - ); - if (Option.isNone(finishedAction)) { - admission = "refused"; } - } - } + const hasInstallableDownload = + state.status === "downloaded" || + (state.status === "error" && + (state.errorContext === null || state.errorContext === "install")); + if (!hasInstallableDownload) return "refused" as const; + return (yield* tryStartUpdateAction("install")) + ? ("admitted" as const) + : ("refused" as const); + }), + ); if (admission === "refused") { return { accepted: false, completed: false, failed: false }; } @@ -816,6 +874,7 @@ export const make = Effect.gen(function* () { return yield* Effect.gen(function* () { yield* writeUpdateRestartMarker; + yield* writeUpdateInProgressMarker; const { downloadedVersion } = yield* Ref.get(updateStateRef); if (downloadedVersion !== null) yield* writeInstallRecord(downloadedVersion); // Stop pooled backends and SSH tunnels before quitAndInstall can outrun their scope finalizers. @@ -949,6 +1008,7 @@ export const make = Effect.gen(function* () { yield* Ref.set(lastLoggedDownloadMilestoneRef, -1); return; } + if (isOffered(state) && state.availableVersion === info.version) return; const checkedAt = yield* currentIsoTimestamp; const { releaseNotes, omittedReleaseCount } = normalizeDesktopUpdateReleaseNotes( @@ -1128,6 +1188,8 @@ export const make = Effect.gen(function* () { ), configure: Effect.gen(function* () { const context = yield* Effect.context(); + // Runs synchronously up to the first await, so a check's outcome is applied before the + // check's promise resolves and wakes the actions waiting on it. const runEffect = (effect: Effect.Effect) => { void Effect.runPromiseWith(context)(effect); }; diff --git a/apps/desktop/src/updates/updatesTestHarness.ts b/apps/desktop/src/updates/updatesTestHarness.ts index 06f9e86a97d5c..b118d15ced674 100644 --- a/apps/desktop/src/updates/updatesTestHarness.ts +++ b/apps/desktop/src/updates/updatesTestHarness.ts @@ -244,6 +244,7 @@ export function makeHarness(options: UpdatesHarnessOptions = {}) { : DesktopAppSettings.layer; const updateRestartMarkers = new Set(); + const updateMarkerContents = new Map(); const installRecords = new Map(); let previousInstallRecord = options.installRecord; const isInstallRecord = (path: string) => @@ -269,11 +270,21 @@ export function makeHarness(options: UpdatesHarnessOptions = {}) { writeFileString: (path, data) => Effect.sync(() => { if (isInstallRecord(path)) installRecords.set(path, data); - else updateRestartMarkers.add(path); + else { + if (path.endsWith("/desktop-update-restart")) updateRestartMarkers.add(path); + updateMarkerContents.set(path, data); + } + }), + rename: (from, to) => + Effect.sync(() => { + const contents = updateMarkerContents.get(from); + if (contents !== undefined) updateMarkerContents.set(to, contents); + updateMarkerContents.delete(from); }), remove: (path) => Effect.sync(() => { updateRestartMarkers.delete(path); + updateMarkerContents.delete(path); installRecords.delete(path); if (isInstallRecord(path)) previousInstallRecord = undefined; }), @@ -306,6 +317,8 @@ export function makeHarness(options: UpdatesHarnessOptions = {}) { quitAndInstalls: () => quitAndInstallCount, installSteps, updateRestartMarkers, + updateInProgressMarker: () => + [...updateMarkerContents].find(([path]) => path.endsWith("/desktop-update-in-progress"))?.[1], installRecords, previousInstallRecord: () => previousInstallRecord, downloadCount: () => downloadCount, diff --git a/apps/desktop/src/window/DesktopApplicationMenu.test.ts b/apps/desktop/src/window/DesktopApplicationMenu.test.ts index 612ec441ba99b..46004ca302295 100644 --- a/apps/desktop/src/window/DesktopApplicationMenu.test.ts +++ b/apps/desktop/src/window/DesktopApplicationMenu.test.ts @@ -3,7 +3,9 @@ import { assert, describe, it } from "@effect/vitest"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import type { DesktopUpdateCheckResult, DesktopUpdateState } from "@t3tools/contracts"; import * as Electron from "electron"; import { vi } from "vite-plus/test"; @@ -18,6 +20,7 @@ import * as DesktopApplicationMenu from "./DesktopApplicationMenu.ts"; import * as DesktopConfig from "../app/DesktopConfig.ts"; import * as DesktopEnvironment from "../app/DesktopEnvironment.ts"; import * as DesktopUpdates from "../updates/DesktopUpdates.ts"; +import { createInitialDesktopUpdateState } from "../updates/updateMachine.ts"; import * as DesktopWindow from "./DesktopWindow.ts"; const environmentInput = { @@ -56,27 +59,37 @@ const layerElectronApp = Layer.succeed(ElectronApp.ElectronApp, { on: () => Effect.void, } satisfies ElectronApp.ElectronApp["Service"]); -const layerElectronDialog = Layer.succeed(ElectronDialog.ElectronDialog, { - pickFolder: () => Effect.succeedNone, - pickFiles: () => Effect.succeed([]), - showMessageBox: () => Effect.succeed({ response: 0, checkboxChecked: false }), - showErrorBox: () => Effect.void, -} satisfies ElectronDialog.ElectronDialog["Service"]); - -const layerDesktopUpdates = Layer.succeed(DesktopUpdates.DesktopUpdates, { - getState: Effect.die("unexpected getState"), - isActionActive: Effect.succeed(false), - isInstallActive: Effect.succeed(false), - subscribe: Effect.die("unexpected subscribe"), - emitState: Effect.void, - disabledReason: Effect.succeedNone, - configure: Effect.void, - setChannel: () => Effect.die("unexpected setChannel"), - check: () => Effect.die("unexpected check"), - download: Effect.die("unexpected download"), - install: Effect.die("unexpected install"), - installPrepared: () => Effect.die("unexpected installPrepared"), -} satisfies DesktopUpdates.DesktopUpdates["Service"]); +interface MenuServices { + readonly checkResult?: DesktopUpdateCheckResult; + readonly shownMessageBox?: Deferred.Deferred; +} + +const layerElectronDialog = ({ shownMessageBox }: MenuServices) => + Layer.succeed(ElectronDialog.ElectronDialog, { + pickFolder: () => Effect.succeedNone, + pickFiles: () => Effect.succeed([]), + showMessageBox: (options) => + (shownMessageBox ? Deferred.succeed(shownMessageBox, options) : Effect.void).pipe( + Effect.as({ response: options.cancelId ?? 0, checkboxChecked: false }), + ), + showErrorBox: () => Effect.void, + } satisfies ElectronDialog.ElectronDialog["Service"]); + +const layerDesktopUpdates = ({ checkResult }: MenuServices) => + Layer.succeed(DesktopUpdates.DesktopUpdates, { + getState: Effect.die("unexpected getState"), + isActionActive: Effect.succeed(false), + isInstallActive: Effect.succeed(false), + subscribe: Effect.die("unexpected subscribe"), + emitState: Effect.void, + disabledReason: Effect.succeedNone, + configure: Effect.void, + setChannel: () => Effect.die("unexpected setChannel"), + check: () => (checkResult ? Effect.succeed(checkResult) : Effect.die("unexpected check")), + download: Effect.die("unexpected download"), + install: Effect.die("unexpected install"), + installPrepared: () => Effect.die("unexpected installPrepared"), + } satisfies DesktopUpdates.DesktopUpdates["Service"]); const layerDesktopWindow = (selectedAction: Deferred.Deferred) => Layer.succeed(DesktopWindow.DesktopWindow, { @@ -113,6 +126,7 @@ const configureMenu = ( selectedAction: Deferred.Deferred, applicationMenuTemplate: Deferred.Deferred, environment: Partial = {}, + services: MenuServices = {}, ) => Effect.gen(function* () { const menu = yield* DesktopApplicationMenu.DesktopApplicationMenu; @@ -122,8 +136,8 @@ const configureMenu = ( DesktopApplicationMenu.layer.pipe( Layer.provideMerge(layerElectronMenu(applicationMenuTemplate)), Layer.provideMerge(layerDesktopWindow(selectedAction)), - Layer.provideMerge(layerDesktopUpdates), - Layer.provideMerge(layerElectronDialog), + Layer.provideMerge(layerDesktopUpdates(services)), + Layer.provideMerge(layerElectronDialog(services)), Layer.provideMerge(layerElectronApp), Layer.provideMerge( DesktopEnvironment.layer({ ...environmentInput, ...environment }).pipe( @@ -134,6 +148,54 @@ const configureMenu = ( ), ); +const checkForUpdatesFromMacMenu = (state: Partial) => + Effect.gen(function* () { + const applicationMenuTemplate = + yield* Deferred.make(); + const shownMessageBox = yield* Deferred.make(); + yield* configureMenu( + yield* Deferred.make(), + applicationMenuTemplate, + { platform: "darwin" }, + { + checkResult: { + checked: true, + state: { + ...createInitialDesktopUpdateState( + "0.1.5", + { hostArch: "arm64", appArch: "arm64", runningUnderArm64Translation: false }, + "latest", + ), + enabled: true, + ...state, + }, + }, + shownMessageBox, + }, + ); + + const appMenu = (yield* Deferred.await(applicationMenuTemplate))[0]; + if (!Array.isArray(appMenu?.submenu)) { + throw new Error("Expected the macOS app menu submenu to be an array."); + } + const checkForUpdates = appMenu.submenu.find((item) => item.label === "Check for Updates..."); + if (typeof checkForUpdates?.click !== "function") { + throw new Error("Expected Check for Updates... menu item to have a click handler."); + } + + // Every stub completes synchronously, so the menu action has finished when click returns. + checkForUpdates.click( + {} as Electron.MenuItem, + {} as Electron.BrowserWindow, + {} as KeyboardEvent, + ); + const dialog = yield* Deferred.poll(shownMessageBox); + if (Option.isNone(dialog)) { + throw new Error("Check for Updates... finished without showing any dialog."); + } + return yield* dialog.value; + }); + describe("DesktopApplicationMenu", () => { it.effect("leaves the accelerator to Chromium instead of injecting a paste", () => Effect.gen(function* () { @@ -285,4 +347,25 @@ describe("DesktopApplicationMenu", () => { assert.equal(yield* Deferred.await(selectedAction), "main-reload"); }), ); + it.effect("tells the user which update the menu check found", () => + Effect.gen(function* () { + const dialog = yield* checkForUpdatesFromMacMenu({ + status: "available", + availableVersion: "0.1.7", + }); + assert.include(`${dialog.title ?? ""} ${dialog.message} ${dialog.detail ?? ""}`, "0.1.7"); + }), + ); + + it.effect("tells the user a downloaded update is ready when the menu check runs", () => + Effect.gen(function* () { + const dialog = yield* checkForUpdatesFromMacMenu({ + status: "downloaded", + availableVersion: "0.1.7", + downloadedVersion: "0.1.7", + downloadPercent: 100, + }); + assert.include(`${dialog.title ?? ""} ${dialog.message} ${dialog.detail ?? ""}`, "0.1.7"); + }), + ); }); diff --git a/apps/desktop/src/window/DesktopApplicationMenu.ts b/apps/desktop/src/window/DesktopApplicationMenu.ts index 44d2c234f8358..b4025ebfde8ef 100644 --- a/apps/desktop/src/window/DesktopApplicationMenu.ts +++ b/apps/desktop/src/window/DesktopApplicationMenu.ts @@ -64,6 +64,9 @@ const runMainContentsCommand = Effect.fn("desktop.menu.runMainContentsCommand")( yield* desktopWindow.runMainContentsCommand(command); }); +const describeUpdate = (version: string | null) => + version === null ? "A new version of T3 Code" : `T3 Code ${version}`; + const checkForUpdatesFromMenu = Effect.gen(function* () { const updates = yield* DesktopUpdates.DesktopUpdates; const electronDialog = yield* ElectronDialog.ElectronDialog; @@ -77,6 +80,35 @@ const checkForUpdatesFromMenu = Effect.gen(function* () { message: `T3 Code ${updateState.currentVersion} is currently the newest version available.`, buttons: ["OK"], }); + } else if (updateState.status === "available") { + const { response } = yield* electronDialog.showMessageBox({ + type: "info", + title: "Update available", + message: `${describeUpdate(updateState.availableVersion)} is available.`, + detail: `You're running T3 Code ${updateState.currentVersion}.`, + buttons: ["Download", "Later"], + defaultId: 0, + cancelId: 1, + }); + if (response === 0) yield* updates.download; + } else if (updateState.status === "downloading") { + yield* electronDialog.showMessageBox({ + type: "info", + title: "Update downloading", + message: `${describeUpdate(updateState.availableVersion)} is already downloading.`, + buttons: ["OK"], + }); + } else if (updateState.status === "downloaded") { + const { response } = yield* electronDialog.showMessageBox({ + type: "info", + title: "Update ready", + message: `${describeUpdate(updateState.downloadedVersion ?? updateState.availableVersion)} is ready to install.`, + detail: "T3 Code will restart to install it. Any running tasks will be interrupted.", + buttons: ["Restart", "Later"], + defaultId: 0, + cancelId: 1, + }); + if (response === 0) yield* updates.install; } else if (updateState.status === "error") { yield* electronDialog.showMessageBox({ type: "warning", diff --git a/apps/desktop/src/window/DesktopWindow.test.ts b/apps/desktop/src/window/DesktopWindow.test.ts index 497b315ef5225..f66586503e24d 100644 --- a/apps/desktop/src/window/DesktopWindow.test.ts +++ b/apps/desktop/src/window/DesktopWindow.test.ts @@ -207,17 +207,20 @@ const layerElectronTheme = Layer.succeed(ElectronTheme.ElectronTheme, { onUpdated: () => Effect.void, } satisfies ElectronTheme.ElectronTheme["Service"]); -const layerDesktopEnvironment = DesktopEnvironment.layer(environmentInput).pipe( - Layer.provide( - Layer.mergeAll( - NodeServices.layer, - DesktopConfig.layerTest({ - T3CODE_PORT: "3773", - VITE_DEV_SERVER_URL: "http://127.0.0.1:5733", - }), +const layerDesktopEnvironmentFor = (platform: NodeJS.Platform) => + DesktopEnvironment.layer({ ...environmentInput, platform }).pipe( + Layer.provide( + Layer.mergeAll( + NodeServices.layer, + DesktopConfig.layerTest({ + T3CODE_PORT: "3773", + VITE_DEV_SERVER_URL: "http://127.0.0.1:5733", + }), + ), ), - ), -); + ); + +const layerDesktopEnvironment = layerDesktopEnvironmentFor(environmentInput.platform); const desktopWindowBoundsEquivalence = Schema.toEquivalence( DesktopAppSettings.DesktopWindowBoundsSchema, @@ -243,6 +246,12 @@ function layerTest(input: { readonly previewZoomReapplies?: number[]; readonly onReveal?: (window: Electron.BrowserWindow) => void; readonly focusedWindow?: Electron.BrowserWindow; + readonly fallbackWindow?: Electron.BrowserWindow; + readonly platform?: NodeJS.Platform; + readonly appearanceWindows?: ReadonlyArray<{ + readonly window: Electron.BrowserWindow; + readonly titleBarOverlay: boolean; + }>; readonly primaryPasteEnabled?: boolean; readonly previewPasskeyBridgeEnabled?: boolean; }) { @@ -291,7 +300,9 @@ function layerTest(input: { Effect.as(input.window), ), main: Ref.get(input.mainWindow), - currentMainOrFirst: Ref.get(input.mainWindow), + currentMainOrFirst: Ref.get(input.mainWindow).pipe( + Effect.map(Option.orElse(() => Option.fromNullishOr(input.fallbackWindow ?? null))), + ), focusedMainOrFirst: input.focusedWindow === undefined ? Ref.get(input.mainWindow) @@ -302,7 +313,14 @@ function layerTest(input: { reveal: (window) => Effect.sync(() => input.onReveal?.(window)), sendAll: () => Effect.void, destroyAll: input.destroyAll ?? Effect.void, - syncAllAppearance: (sync) => sync(input.window), + syncAllAppearance: (sync) => + input.appearanceWindows === undefined + ? sync(input.window, { titleBarOverlay: true }) + : Effect.forEach( + input.appearanceWindows, + ({ window, titleBarOverlay }) => sync(window, { titleBarOverlay }), + { discard: true }, + ), } satisfies ElectronWindow.ElectronWindow["Service"]); return DesktopWindow.layer.pipe( @@ -314,7 +332,9 @@ function layerTest(input: { recordMetrics: () => Effect.void, shutdown: Effect.void, }), - layerDesktopEnvironment, + input.platform === undefined + ? layerDesktopEnvironment + : layerDesktopEnvironmentFor(input.platform), layerDesktopAppSettings, layerDesktopClientSettings, layerDesktopServerExposure, @@ -423,7 +443,8 @@ const makeSplashScenario = (createOutcomes: readonly (Electron.BrowserWindow | n reveal: (window) => Ref.update(revealedWindows, (windows) => [...windows, window]), sendAll: () => Effect.void, destroyAll: Effect.void, - syncAllAppearance: (sync) => (fallbackWindow ? sync(fallbackWindow) : Effect.void), + syncAllAppearance: (sync) => + fallbackWindow ? sync(fallbackWindow, { titleBarOverlay: true }) : Effect.void, } satisfies ElectronWindow.ElectronWindow["Service"]; const layer = DesktopWindow.layer.pipe( @@ -826,7 +847,7 @@ describe("DesktopWindow", () => { }), ); - it.effect("reloads and inspects the main window even when a popup has focus", () => + it.effect("routes menu commands to the main window even when a popup has focus", () => Effect.gen(function* () { const fakeWindow = makeFakeBrowserWindow(); const popup = makeFakeBrowserWindow(); @@ -846,7 +867,10 @@ describe("DesktopWindow", () => { yield* desktopWindow.runMainContentsCommand("reload"); yield* desktopWindow.runMainContentsCommand("forceReload"); yield* desktopWindow.runMainContentsCommand("toggleDevTools"); + yield* desktopWindow.dispatchMenuAction("open-settings"); + assert.deepEqual(fakeWindow.send.mock.calls, [["desktop:menu-action", "open-settings"]]); + assert.equal(popup.send.mock.calls.length, 0); assert.equal(fakeWindow.reload.mock.calls.length, 1); assert.equal(fakeWindow.reloadIgnoringCache.mock.calls.length, 1); assert.equal(fakeWindow.toggleDevTools.mock.calls.length, 1); @@ -857,6 +881,38 @@ describe("DesktopWindow", () => { }), ); + it.effect("recreates main for Settings when only an unregistered popup survives", () => + Effect.gen(function* () { + const main = makeFakeBrowserWindow(); + const auxiliary = makeFakeBrowserWindow(); + const createCount = yield* Ref.make(0); + const mainWindow = yield* Ref.make>(Option.none()); + + yield* Effect.gen(function* () { + const windows = yield* DesktopWindow.DesktopWindow; + yield* windows.dispatchMenuAction("open-settings"); + assert.deepEqual(main.send.mock.calls, [["desktop:menu-action", "open-settings"]]); + assert.equal(auxiliary.send.mock.calls.length, 0); + assert.equal(yield* Ref.get(createCount), 1); + assert.strictEqual(yield* windows.ensureMain, main.window); + assert.equal(yield* Ref.get(createCount), 1); + }).pipe( + Effect.provide( + layerTest({ + window: main.window, + createCount, + mainWindow, + fallbackWindow: auxiliary.window, + desktopSettings: { + ...DesktopAppSettings.DEFAULT_DESKTOP_SETTINGS, + localEnvironmentEnabled: false, + }, + }), + ), + ); + }), + ); + it.effect("does not open main or splash windows once quitting has started", () => Effect.gen(function* () { const fake = makeFakeBrowserWindow(); @@ -1170,6 +1226,36 @@ describe("DesktopWindow", () => { }), ); + it.effect("updates title bar overlays only on windows created with one", () => + Effect.gen(function* () { + const framed = makeFakeBrowserWindow(); + vi.mocked(framed.window.setTitleBarOverlay).mockImplementation(() => { + throw new TypeError("Titlebar overlay is not enabled"); + }); + const overlay = makeFakeBrowserWindow(); + const layer = layerTest({ + window: overlay.window, + createCount: yield* Ref.make(0), + mainWindow: yield* Ref.make>(Option.none()), + platform: "win32", + appearanceWindows: [ + { window: framed.window, titleBarOverlay: false }, + { window: overlay.window, titleBarOverlay: true }, + ], + }); + + yield* Effect.gen(function* () { + const desktopWindow = yield* DesktopWindow.DesktopWindow; + yield* desktopWindow.syncAppearance; + }).pipe(Effect.provide(layer)); + + assert.deepEqual(vi.mocked(framed.window.setBackgroundColor).mock.calls, [["#ffffff"]]); + assert.lengthOf(vi.mocked(framed.window.setTitleBarOverlay).mock.calls, 0); + assert.deepEqual(vi.mocked(overlay.window.setBackgroundColor).mock.calls, [["#ffffff"]]); + assert.lengthOf(vi.mocked(overlay.window.setTitleBarOverlay).mock.calls, 1); + }), + ); + it.effect.each([ { window: "normal", write: "flush" }, { window: "normal", write: "debounce" }, diff --git a/apps/desktop/src/window/DesktopWindow.ts b/apps/desktop/src/window/DesktopWindow.ts index 43404e2d6cfb9..7be14bc720838 100644 --- a/apps/desktop/src/window/DesktopWindow.ts +++ b/apps/desktop/src/window/DesktopWindow.ts @@ -343,6 +343,7 @@ function getWindowTitleBarOptions( function syncWindowAppearance( window: Electron.BrowserWindow, + hasTitleBarOverlay: boolean, shouldUseDarkColors: boolean, platform: NodeJS.Platform, ): Effect.Effect { @@ -353,7 +354,7 @@ function syncWindowAppearance( window.setBackgroundColor(getInitialWindowBackgroundColor(shouldUseDarkColors)); const { titleBarOverlay } = getWindowTitleBarOptions(shouldUseDarkColors, platform); - if (typeof titleBarOverlay === "object") { + if (hasTitleBarOverlay && typeof titleBarOverlay === "object") { window.setTitleBarOverlay(titleBarOverlay); } }); @@ -424,7 +425,6 @@ export const make = Effect.gen(function* () { ); const currentMainWindow = electronWindow.currentMainOrFirst.pipe(Effect.flatMap(withoutSplash)); - const focusedMainWindow = electronWindow.focusedMainOrFirst.pipe(Effect.flatMap(withoutSplash)); const zoomWindow = Effect.fn("desktop.window.zoomWindow")(function* ( window: Electron.BrowserWindow, @@ -1018,7 +1018,7 @@ export const make = Effect.gen(function* () { const ensureMain = Effect.gen(function* () { if (yield* Ref.get(desktopState.quitting)) return Option.none(); - const existingWindow = yield* currentMainWindow; + const existingWindow = yield* electronWindow.main; if (Option.isSome(existingWindow)) { return existingWindow; } @@ -1105,7 +1105,8 @@ export const make = Effect.gen(function* () { { reveal = true }: { readonly reveal?: boolean } = {}, ) { if (yield* Ref.get(desktopState.quitting)) return; - const existingWindow = yield* reveal ? focusedMainWindow : electronWindow.main; + // Only the main renderer loads the app preload that listens on these channels. + const existingWindow = yield* electronWindow.main; if (Option.isNone(existingWindow) && (!reveal || (yield* waitingForBackend))) return; const window = Option.isSome(existingWindow) ? existingWindow : yield* ensureMain; if (Option.isNone(window) || window.value.isDestroyed()) return; @@ -1202,8 +1203,8 @@ export const make = Effect.gen(function* () { }), syncAppearance: Effect.gen(function* () { const shouldUseDarkColors = yield* electronTheme.shouldUseDarkColors; - yield* electronWindow.syncAllAppearance((window) => - syncWindowAppearance(window, shouldUseDarkColors, environment.platform), + yield* electronWindow.syncAllAppearance((window, { titleBarOverlay }) => + syncWindowAppearance(window, titleBarOverlay, shouldUseDarkColors, environment.platform), ); }).pipe(Effect.withSpan("desktop.window.syncAppearance")), }); diff --git a/apps/desktop/src/wsl/DesktopWslBackend.test.ts b/apps/desktop/src/wsl/DesktopWslBackend.test.ts index 8468a9538630c..d9ae6e76c29c2 100644 --- a/apps/desktop/src/wsl/DesktopWslBackend.test.ts +++ b/apps/desktop/src/wsl/DesktopWslBackend.test.ts @@ -154,7 +154,7 @@ describe("DesktopWslBackend", () => { assert.equal(startCount, 1); assert.deepEqual(yield* backend.lastPreflightError, refusal); - yield* clearFailure(new URL("http://127.0.0.1:41773")); + yield* clearFailure(new URL("http://127.0.0.1:41773"), 123); assert.deepEqual(yield* backend.lastPreflightError, Option.none()); yield* recordOwnershipFailure(); const settings = yield* DesktopAppSettings.DesktopAppSettings; diff --git a/apps/desktop/src/wsl/DesktopWslEnvironment.test.ts b/apps/desktop/src/wsl/DesktopWslEnvironment.test.ts index 69962f52e4f18..0787956a72b76 100644 --- a/apps/desktop/src/wsl/DesktopWslEnvironment.test.ts +++ b/apps/desktop/src/wsl/DesktopWslEnvironment.test.ts @@ -1,16 +1,25 @@ // @effect-diagnostics nodeBuiltinImport:off - the executed suite runs the generated install script through a real POSIX shell. +import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, describe, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import * as Redacted from "effect/Redacted"; +import * as Sink from "effect/Sink"; +import * as Stream from "effect/Stream"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; import { afterAll, expect } from "vite-plus/test"; import * as NodeChildProcess from "node:child_process"; import { ManagedPolicyFileError } from "@t3tools/shared/managedPolicyFile"; +import * as DesktopEnvironment from "../app/DesktopEnvironment.ts"; import * as DesktopManagedPolicy from "../app/DesktopManagedPolicy.ts"; import { + DesktopWslEnvironment, + layer as layerDesktopWslEnvironment, buildWslRuntimeInstallScript, makeIsAvailable, buildWslRuntimeInvalidateScript, @@ -18,6 +27,7 @@ import { buildWslRuntimeProbeScript, parseResolvedPath, parseWslRuntimeRoot, + windowsToWslPathImpl, } from "./DesktopWslEnvironment.ts"; // The install script only fails the way this file cares about when a real shell @@ -380,6 +390,93 @@ describe.skipIf(posixShellRunner === null)("WSL runtime install script (executed expect(selected.status, `${selected.stdout}\n${selected.stderr}`).toBe(0); }); + // Runs the distro side on the local bash, so a Windows dev box using the wsl.exe runner skips it. + describe.skipIf(posixShellRunner?.file !== "bash")("prepareRuntime through wsl.exe", () => { + // Emulates `wsl.exe [-d ] [--cd ] [--|--exec] ` in a distro with automount off. + const layerFakeWslExe = (home: string) => + Layer.effect( + ChildProcessSpawner.ChildProcessSpawner, + Effect.map(ChildProcessSpawner.ChildProcessSpawner, (local) => + ChildProcessSpawner.make((command) => { + if (!ChildProcess.isStandardCommand(command) || command.command !== "wsl.exe") { + return local.spawn(command); + } + const args = command.args.map((word) => + Redacted.isRedacted(word) ? Redacted.value(word) : word, + ); + while (args.length > 0) { + const flag = args[0]; + if (flag === "-d" || flag === "--distribution" || flag === "--cd") { + args.splice(0, 2); + } else if (flag === "--" || flag === "--exec" || flag === "-e") { + args.shift(); + break; + } else { + break; + } + } + const [file = "sh", ...rest] = + args[0] === "wslpath" + ? [ + "sh", + "-c", + 'printf "wslpath: %s\\n" "$1" >&2; exit 1', + "wslpath", + args.at(-1) ?? "", + ] + : args; + return local.spawn( + ChildProcess.make(file, rest, { + ...command.options, + env: { ...command.options.env, HOME: home }, + extendEnv: true, + }), + ); + }), + ), + ); + + const layerWslEnvironment = (home: string) => + layerDesktopWslEnvironment.pipe( + Layer.provide( + Layer.mergeAll( + layerFakeWslExe(home), + DesktopManagedPolicy.layerTest(), + Layer.effect( + DesktopEnvironment.DesktopEnvironment, + Effect.map( + Path.Path, + (path) => + ({ platform: "win32", path }) as DesktopEnvironment.DesktopEnvironment["Service"], + ), + ), + ), + ), + Layer.provide(NodeServices.layer), + ); + + it.live("installs the runtime when wslpath cannot translate the archive's Windows path", () => { + const fixture = createFixture(); + return Effect.gen(function* () { + const wsl = yield* DesktopWslEnvironment; + // The archive's Windows path; with no drives mounted, wslpath cannot map it into the distro. + const translated = yield* wsl.windowsToWslPath("Ubuntu", fixture.archivePath); + expect(translated._tag).toBe("None"); + + const prepared = yield* wsl.prepareRuntime("Ubuntu", { + windowsPath: fixture.archivePath, + runtimeId: fixture.runtimeId, + sha256: fixture.archiveSha, + }); + + expect(prepared).toEqual({ ok: true, linuxAppRoot: fixture.runtimeRoot }); + const version = runShell(`${sh(fixture.serverEntry)} --version`); + expect(version.status, version.stderr).toBe(0); + expect(version.stdout).toBe("t3code wsl runtime test server 0.0.0\n"); + }).pipe(Effect.provide(layerWslEnvironment(`${fixture.work}/home`))); + }); + }); + it("prunes old and markerless caches without touching retained, active, locked, or unrelated roots", () => { const result = runShell( [ @@ -459,3 +556,48 @@ describe("makeIsAvailable", () => { }), ); }); + +describe("windowsToWslPathImpl", () => { + it.effect("runs wslpath without a shell so folder names stay one literal argument", () => + Effect.gen(function* () { + const spawned: Array = []; + const stdout = Stream.make(new TextEncoder().encode("/mnt/c/a$b & c;(d)\n")); + const spawner = ChildProcessSpawner.make((command) => { + spawned.push(command); + return Effect.succeed( + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + stdout, + stderr: Stream.empty, + all: stdout, + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(0)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + stdin: Sink.drain, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }), + ); + }); + + const converted = yield* windowsToWslPathImpl("Ubuntu", "C:\\a$b & c;(d)").pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + + assert.deepStrictEqual(converted, Option.some("/mnt/c/a$b & c;(d)")); + assert.lengthOf(spawned, 1); + const [command] = spawned; + assert(command?._tag === "StandardCommand"); + assert.strictEqual(command.command, "wsl.exe"); + assert.deepStrictEqual(command.args, [ + "-d", + "Ubuntu", + "--exec", + "wslpath", + "-u", + "C:/a$b & c;(d)", + ]); + }), + ); +}); diff --git a/apps/desktop/src/wsl/DesktopWslEnvironment.ts b/apps/desktop/src/wsl/DesktopWslEnvironment.ts index 57d6c61f7c43a..4e5e5b423f20e 100644 --- a/apps/desktop/src/wsl/DesktopWslEnvironment.ts +++ b/apps/desktop/src/wsl/DesktopWslEnvironment.ts @@ -5,6 +5,8 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import type * as PlatformError from "effect/PlatformError"; +import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; @@ -173,37 +175,19 @@ const buildWslNodeEnvPreamble = ( ensure_remote_node_path || true `; -// wsl.exe re-escaping mangles quotes inside bash -lc scripts. -const runWslShell = ( - distro: string | null, - bashScript: string, +const runWslProcess = ( + args: ReadonlyArray, + stdin: Stream.Stream, timeout: Duration.Duration, - options: { - readonly nodeEngineRange?: string | null; - readonly resolveNode?: boolean; - } = {}, ): Effect.Effect => { const spawner = ChildProcessSpawner.ChildProcessSpawner; - const resolveNode = options.resolveNode !== false; - const command = ChildProcess.make( - "wsl.exe", - resolveNode - ? [...buildDistroArgs(distro), "--", "bash", "-l", "-s"] - : [...buildDistroArgs(distro), "--exec", "sh", "-s"], - { - stdin: Stream.encodeText( - Stream.make( - resolveNode - ? `${buildWslNodeEnvPreamble(options.nodeEngineRange)}${bashScript}` - : bashScript, - ), - ), - stdout: "pipe", - stderr: "pipe", - killSignal: "SIGTERM", - forceKillAfter: PROCESS_TERMINATE_GRACE, - }, - ); + const command = ChildProcess.make("wsl.exe", args, { + stdin, + stdout: "pipe", + stderr: "pipe", + killSignal: "SIGTERM", + forceKillAfter: PROCESS_TERMINATE_GRACE, + }); return Effect.scoped( Effect.gen(function* () { @@ -249,6 +233,32 @@ const runWslShell = ( ); }; +// wsl.exe re-escaping mangles quotes inside bash -lc scripts. +const runWslShell = ( + distro: string | null, + bashScript: string, + timeout: Duration.Duration, + options: { + readonly nodeEngineRange?: string | null; + readonly resolveNode?: boolean; + } = {}, +): Effect.Effect => { + const resolveNode = options.resolveNode !== false; + return runWslProcess( + resolveNode + ? [...buildDistroArgs(distro), "--", "bash", "-l", "-s"] + : [...buildDistroArgs(distro), "--exec", "sh", "-s"], + Stream.encodeText( + Stream.make( + resolveNode + ? `${buildWslNodeEnvPreamble(options.nodeEngineRange)}${bashScript}` + : bashScript, + ), + ), + timeout, + ); +}; + const shellQuote = (value: string): string => `'${value.replaceAll("'", "'\\''")}'`; const WSL_RUNTIME_READY_MARKER = ".t3code-wsl-runtime-ready"; @@ -257,13 +267,14 @@ const WSL_RUNTIME_SELECTION_GRACE_MINUTES = 5; const sanitizeWslRuntimeId = (value: string): string => value.replace(/[^A-Za-z0-9._-]/g, "_"); +// A null archive path asks for a staging directory to copy the archive into, unless the cache is ready. export const buildWslRuntimeInstallScript = ( - linuxArchivePath: string, + linuxArchivePath: string | null, runtimeId: string, archiveSha256: string, ): string => { const safeRuntimeId = sanitizeWslRuntimeId(runtimeId); - return [ + const readyCheck = [ "set -eu", 'runtime_parent="$HOME/.t3/wsl-runtime"', `runtime_root="$runtime_parent/${safeRuntimeId}"`, @@ -292,6 +303,17 @@ export const buildWslRuntimeInstallScript = ( ` printf 'runtimeRoot:%s\\n' "$runtime_root"`, " exit 0", "fi", + ]; + if (linuxArchivePath === null) { + return [ + ...readyCheck, + // The .tmp. name lets pruning collect a stage orphaned by a crash. + `archive_stage=$(mktemp -d "$runtime_parent/.${safeRuntimeId}.tmp.XXXXXX")`, + `printf 'archiveStage:%s\\n' "$archive_stage"`, + ].join("\n"); + } + return [ + ...readyCheck, `archive_sha=$(sha256sum ${shellQuote(linuxArchivePath)} | cut -d ' ' -f 1)`, `if [ "$archive_sha" != ${shellQuote(archiveSha256)} ]; then`, ` printf 'WSL runtime archive does not match its recorded SHA-256 (expected %s, got %s)\\n' ${shellQuote(archiveSha256)} "$archive_sha" >&2`, @@ -409,14 +431,17 @@ export const buildWslRuntimeInvalidateScript = (runtimeId: string): string => { ].join("\n"); }; -export const parseWslRuntimeRoot = (stdout: string): string | null => { - const prefix = "runtimeRoot:"; +const parseAbsolutePathField = (stdout: string, field: string): string | null => { + const prefix = `${field}:`; const line = stdout.split("\n").find((candidate) => candidate.startsWith(prefix)); if (line === undefined) return null; - const runtimeRoot = line.slice(prefix.length).replace(/\r$/, ""); - return runtimeRoot.startsWith("/") ? runtimeRoot : null; + const value = line.slice(prefix.length).replace(/\r$/, ""); + return value.startsWith("/") ? value : null; }; +export const parseWslRuntimeRoot = (stdout: string): string | null => + parseAbsolutePathField(stdout, "runtimeRoot"); + const NODE_PTY_BINARY_MISSING_EXIT_CODE = 4; const formatNodePtyProbeFailureReason = (exitCode: number): string | null => @@ -775,25 +800,18 @@ const ensureNodePtyImpl = Effect.fn("DesktopWslEnvironment.ensureNodePtyImpl")(f } as const; }); -const prepareWslRuntimeImpl = Effect.fn("desktop.wsl.prepareRuntimeImpl")(function* ( +type WslRuntimeInstallOutcome = + | { readonly ok: true; readonly stdout: string } + | { readonly ok: false; readonly reason: string }; + +const runWslRuntimeInstall = Effect.fn("desktop.wsl.runRuntimeInstall")(function* ( distro: string | null, archive: WslRuntimeArchive, - windowsToWslPath: ( - distro: string | null, - windowsPath: string, - ) => Effect.Effect>, -): Effect.fn.Return { - const linuxArchivePath = yield* windowsToWslPath(distro, archive.windowsPath); - if (Option.isNone(linuxArchivePath)) { - return { - ok: false, - reason: `wslpath conversion failed for ${archive.windowsPath}`, - } as const; - } - + linuxArchivePath: string | null, +): Effect.fn.Return { const install = yield* runWslShell( distro, - buildWslRuntimeInstallScript(linuxArchivePath.value, archive.runtimeId, archive.sha256), + buildWslRuntimeInstallScript(linuxArchivePath, archive.runtimeId, archive.sha256), RUNTIME_INSTALL_TIMEOUT, { resolveNode: false }, ); @@ -813,14 +831,106 @@ const prepareWslRuntimeImpl = Effect.fn("desktop.wsl.prepareRuntimeImpl")(functi reason: `WSL runtime installation failed (exit ${install.exitCode}): ${trimmedTail || "no stderr captured"}`, } as const; } + return { ok: true, stdout: install.stdout } as const; +}); - const linuxAppRoot = parseWslRuntimeRoot(install.stdout); +const toPrepareWslRuntimeResult = (outcome: WslRuntimeInstallOutcome): PrepareWslRuntimeResult => { + if (!outcome.ok) return outcome; + const linuxAppRoot = parseWslRuntimeRoot(outcome.stdout); return linuxAppRoot === null ? { ok: false, reason: "WSL runtime installation completed without reporting its cache path.", } : { ok: true, linuxAppRoot }; +}; + +// Without drive mounts (`[automount] enabled=false`) wslpath cannot reach the archive, so copy it in over stdin. +const installWslRuntimeFromHostCopy = Effect.fn("desktop.wsl.installRuntimeFromHostCopy")( + function* ( + distro: string | null, + archive: WslRuntimeArchive, + archiveBytes: Stream.Stream, + ): Effect.fn.Return { + const staged = yield* runWslRuntimeInstall(distro, archive, null); + if (!staged.ok || parseWslRuntimeRoot(staged.stdout) !== null) { + return toPrepareWslRuntimeResult(staged); + } + const stage = parseAbsolutePathField(staged.stdout, "archiveStage"); + if (stage === null) { + return { + ok: false, + reason: `wslpath conversion failed for ${archive.windowsPath}, and the distro did not provide a directory to copy the runtime archive into.`, + } as const; + } + const linuxArchivePath = `${stage}/runtime.tar.gz`; + + const copyAndInstall = Effect.gen(function* () { + const readError = yield* Ref.make(Option.none()); + // dd reads fd 0 itself; reopening /dev/stdin fails when WSL relays stdin as a socket. + const copy = yield* runWslProcess( + [...buildDistroArgs(distro), "--exec", "dd", `of=${linuxArchivePath}`, "bs=65536"], + archiveBytes.pipe( + // Ending stdin on a read error lets dd exit instead of waiting out the timeout. + Stream.catch((error) => + Stream.drain(Stream.fromEffect(Ref.set(readError, Option.some(error)))), + ), + ), + RUNTIME_INSTALL_TIMEOUT, + ); + const failedRead = yield* Ref.get(readError); + if (Option.isSome(failedRead)) { + return { + ok: false, + reason: `Could not read the WSL runtime archive at ${archive.windowsPath}: ${failedRead.value.message}`, + } as const; + } + if (copy.transportFailure !== null || copy.exitCode !== 0) { + const trimmedTail = copy.stderr.trim().slice(-500); + return { + ok: false, + reason: `Could not copy the WSL runtime archive into the distro (exit ${copy.exitCode})${trimmedTail ? `: ${trimmedTail}` : ""}`, + } as const; + } + return toPrepareWslRuntimeResult( + yield* runWslRuntimeInstall(distro, archive, linuxArchivePath), + ); + }); + + return yield* copyAndInstall.pipe( + Effect.ensuring( + runWslShell(distro, `rm -rf ${shellQuote(stage)}`, RUNTIME_INVALIDATE_TIMEOUT, { + resolveNode: false, + }), + ), + ); + }, +); + +const prepareWslRuntimeImpl = Effect.fn("desktop.wsl.prepareRuntimeImpl")(function* ( + distro: string | null, + archive: WslRuntimeArchive, + windowsToWslPath: ( + distro: string | null, + windowsPath: string, + ) => Effect.Effect>, +): Effect.fn.Return< + PrepareWslRuntimeResult, + never, + ChildProcessSpawner.ChildProcessSpawner | FileSystem.FileSystem +> { + const linuxArchivePath = yield* windowsToWslPath(distro, archive.windowsPath); + if (Option.isNone(linuxArchivePath)) { + const fileSystem = yield* FileSystem.FileSystem; + return yield* installWslRuntimeFromHostCopy( + distro, + archive, + fileSystem.stream(archive.windowsPath), + ); + } + return toPrepareWslRuntimeResult( + yield* runWslRuntimeInstall(distro, archive, linuxArchivePath.value), + ); }); const pruneWslRuntimesImpl = Effect.fn("desktop.wsl.pruneRuntimesImpl")(function* ( @@ -925,7 +1035,7 @@ const preWarmImpl = ( }), ).pipe(Effect.timeoutOption(PRE_WARM_TIMEOUT), Effect.ignore); -const windowsToWslPathImpl = ( +export const windowsToWslPathImpl = ( distro: string | null, windowsPath: string, ): Effect.Effect, never, ChildProcessSpawner.ChildProcessSpawner> => { @@ -936,7 +1046,8 @@ const windowsToWslPathImpl = ( const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const command = ChildProcess.make( "wsl.exe", - [...buildDistroArgs(distro), "--", "wslpath", "-u", normalized], + // --exec skips the distro shell, which would expand `$`, `&` or `;` in folder names. + [...buildDistroArgs(distro), "--exec", "wslpath", "-u", normalized], { stdin: "ignore", stdout: "pipe", @@ -1198,7 +1309,11 @@ export const layer = Layer.effect( getUserHome, getDistroIp, prepareRuntime: Effect.fn("desktop.wsl.prepareRuntime")(function (distro, archive) { - return provideSpawner(prepareWslRuntimeImpl(distro, archive, windowsToWslPath)); + return provideSpawner( + prepareWslRuntimeImpl(distro, archive, windowsToWslPath).pipe( + Effect.provideService(FileSystem.FileSystem, fileSystem), + ), + ); }), pruneRuntimes: Effect.fn("desktop.wsl.pruneRuntimes")(function (distro, runtimeId) { return provideSpawner(pruneWslRuntimesImpl(distro, runtimeId)); diff --git a/apps/desktop/src/wsl/DesktopWslServerTree.test.ts b/apps/desktop/src/wsl/DesktopWslServerTree.test.ts index 7eaf362a326aa..34dcbea08a0ee 100644 --- a/apps/desktop/src/wsl/DesktopWslServerTree.test.ts +++ b/apps/desktop/src/wsl/DesktopWslServerTree.test.ts @@ -6,7 +6,6 @@ import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as PlatformError from "effect/PlatformError"; import * as Ref from "effect/Ref"; -import * as Scope from "effect/Scope"; import * as DesktopConfig from "../app/DesktopConfig.ts"; import * as DesktopEnvironment from "../app/DesktopEnvironment.ts"; @@ -42,20 +41,14 @@ const layerEnvironment = (input: { ), ); -const withTempDir = ( - run: (tempDir: string) => Effect.Effect, -): Effect.Effect< - A, - E | PlatformError.PlatformError, - FileSystem.FileSystem | Exclude -> => +const withTempDir = (run: (tempDir: string) => Effect.Effect) => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; const tempDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-wsl-server-tree-test-", }); return yield* run(tempDir); - }).pipe(Effect.scoped); + }); const ensureWith = (input: { readonly baseDir: string; diff --git a/apps/desktop/src/wsl/wslPathParsing.test.ts b/apps/desktop/src/wsl/wslPathParsing.test.ts index 0a814229665dd..ed491f58381ae 100644 --- a/apps/desktop/src/wsl/wslPathParsing.test.ts +++ b/apps/desktop/src/wsl/wslPathParsing.test.ts @@ -77,8 +77,15 @@ describe("wslUncPathToLinuxPath", () => { expect(wslUncPathToLinuxPath("\\\\wsl.localhost\\Debian\\")).toBe("/"); }); + it("maps UNC paths of distros whose names start or end with a dot or hyphen", () => { + for (const name of ["Ubuntu-", "Ubuntu.", ".Ubuntu", "-Ubuntu"]) { + expect(wslUncPathToLinuxPath(`\\\\wsl.localhost\\${name}\\home\\josh`)).toBe("/home/josh"); + } + }); + it("rejects invalid distro names and non-WSL paths", () => { expect(wslUncPathToLinuxPath("\\\\wsl.localhost\\bad!name\\home")).toBeNull(); + expect(wslUncPathToLinuxPath("\\\\wsl.localhost\\..\\home")).toBeNull(); expect(wslUncPathToLinuxPath("C:\\Users\\Josh\\repo")).toBeNull(); }); }); @@ -107,9 +114,13 @@ describe("resolveWslPickFolderDefaultPath", () => { }); describe("isValidDistroName", () => { - it("rejects names with trailing whitespace, hyphen, or dot", () => { - for (const name of ["Ubuntu ", "Ubuntu-", "Ubuntu."]) { - expect(isValidDistroName(name)).toBe(false); + it("rejects names with trailing whitespace", () => { + expect(isValidDistroName("Ubuntu ")).toBe(false); + }); + + it("accepts names WSL registers with a leading or trailing dot or hyphen", () => { + for (const name of ["Ubuntu-", "Ubuntu.", ".Ubuntu", "-Ubuntu"]) { + expect(isValidDistroName(name)).toBe(true); } }); diff --git a/apps/desktop/src/wsl/wslPathParsing.ts b/apps/desktop/src/wsl/wslPathParsing.ts index a1ce99c129d13..1df64f91dac0e 100644 --- a/apps/desktop/src/wsl/wslPathParsing.ts +++ b/apps/desktop/src/wsl/wslPathParsing.ts @@ -9,7 +9,8 @@ export interface WslConfig { } // Literal spaces only; \s would accept controls and corrupt UNC distro paths. -const DISTRO_NAME_PATTERN = /^\w(?:[\w \-.]*\w)?$/; +// "." and ".." would be path segments in a UNC path, not distro names. +const DISTRO_NAME_PATTERN = /^(?!\.{1,2}$)[\w.-](?:[\w .-]*[\w.-])?$/; export function parseWslDistroList(stdout: Buffer): readonly WslDistro[] { const hasUtf16Bom = stdout.length >= 2 && stdout[0] === 0xff && stdout[1] === 0xfe; diff --git a/apps/desktop/vite.config.ts b/apps/desktop/vite.config.ts index 455901299edc3..3e6fd9531106b 100644 --- a/apps/desktop/vite.config.ts +++ b/apps/desktop/vite.config.ts @@ -164,6 +164,9 @@ export default defineConfig({ // The Windows lane runs workspace suites concurrently; filesystem-heavy // desktop integration tests can exceed Vitest's 5 second default there. testTimeout: 15_000, + // `require("electron")` downloads a missing binary, so concurrent workers race to + // extract it into the same dist/. Install it once before any worker starts. + globalSetup: ["./scripts/ensure-electron-runtime.mjs"], projects: [ { extends: true, diff --git a/apps/marketing/package.json b/apps/marketing/package.json index 04f6830066a1e..8e447d16b2c0e 100644 --- a/apps/marketing/package.json +++ b/apps/marketing/package.json @@ -7,7 +7,8 @@ "stage:install-scripts": "node scripts/stage-install-scripts.mjs", "dev": "node scripts/stage-install-scripts.mjs && astro dev", "build": "node scripts/stage-install-scripts.mjs && astro build", - "preview": "astro preview" + "preview": "astro preview", + "test": "vp test run" }, "dependencies": { "@fontsource-variable/inter": "^5.3.0", diff --git a/apps/marketing/src/i18n/de.ts b/apps/marketing/src/i18n/de.ts index 648b35beebe07..96925adb27069 100644 --- a/apps/marketing/src/i18n/de.ts +++ b/apps/marketing/src/i18n/de.ts @@ -27,6 +27,8 @@ export const de: Messages = { downloadForMac: "Für macOS herunterladen", downloadForWindows: "Für Windows herunterladen", downloadForLinux: "Für Linux herunterladen", + downloadForIos: "Für iOS herunterladen", + downloadForAndroid: "Für Android herunterladen", sourceLink: "Klau unseren Code (legal)", alsoOnPhone: "Auch auf deinem Handy:", screenshotAlt: "T3 Code führt Subagents von Claude, Codex, Muse und Gemini in einem Thread aus", diff --git a/apps/marketing/src/i18n/en.ts b/apps/marketing/src/i18n/en.ts index 1769d81184c5b..1a36424a1bfde 100644 --- a/apps/marketing/src/i18n/en.ts +++ b/apps/marketing/src/i18n/en.ts @@ -27,6 +27,8 @@ export const en = { downloadForMac: "Download for macOS", downloadForWindows: "Download for Windows", downloadForLinux: "Download for Linux", + downloadForIos: "Download for iOS", + downloadForAndroid: "Download for Android", sourceLink: "Steal our code (legally)", alsoOnPhone: "Also on your phone:", screenshotAlt: "T3 Code running Claude, Codex, Muse, and Gemini subagents in one thread", diff --git a/apps/marketing/src/i18n/es.ts b/apps/marketing/src/i18n/es.ts index cb4c28e9871fd..90fcf79bd9429 100644 --- a/apps/marketing/src/i18n/es.ts +++ b/apps/marketing/src/i18n/es.ts @@ -28,6 +28,8 @@ export const es: Messages = { downloadForMac: "Descargar para macOS", downloadForWindows: "Descargar para Windows", downloadForLinux: "Descargar para Linux", + downloadForIos: "Descargar para iOS", + downloadForAndroid: "Descargar para Android", sourceLink: "Roba nuestro código (legalmente)", alsoOnPhone: "También en tu teléfono:", screenshotAlt: "T3 Code ejecutando subagentes de Claude, Codex, Muse y Gemini en un mismo hilo", diff --git a/apps/marketing/src/i18n/fr.ts b/apps/marketing/src/i18n/fr.ts index 8fd918efc815b..3d76747d6c833 100644 --- a/apps/marketing/src/i18n/fr.ts +++ b/apps/marketing/src/i18n/fr.ts @@ -27,6 +27,8 @@ export const fr: Messages = { downloadForMac: "Télécharger pour macOS", downloadForWindows: "Télécharger pour Windows", downloadForLinux: "Télécharger pour Linux", + downloadForIos: "Télécharger pour iOS", + downloadForAndroid: "Télécharger pour Android", sourceLink: "Volez notre code (légalement)", alsoOnPhone: "Aussi sur votre téléphone :", screenshotAlt: diff --git a/apps/marketing/src/i18n/ja.ts b/apps/marketing/src/i18n/ja.ts index 55117b609ac08..9f9fdba51129d 100644 --- a/apps/marketing/src/i18n/ja.ts +++ b/apps/marketing/src/i18n/ja.ts @@ -28,6 +28,8 @@ export const ja: Messages = { downloadForMac: "macOS 版をダウンロード", downloadForWindows: "Windows 版をダウンロード", downloadForLinux: "Linux 版をダウンロード", + downloadForIos: "iOS 版をダウンロード", + downloadForAndroid: "Android 版をダウンロード", sourceLink: "コードを盗もう(合法的に)", alsoOnPhone: "スマホでも使えます:", screenshotAlt: diff --git a/apps/marketing/src/i18n/ko.ts b/apps/marketing/src/i18n/ko.ts index f2c09c0af06ac..e2f4d4e6f7c7a 100644 --- a/apps/marketing/src/i18n/ko.ts +++ b/apps/marketing/src/i18n/ko.ts @@ -27,6 +27,8 @@ export const ko: Messages = { downloadForMac: "macOS용 다운로드", downloadForWindows: "Windows용 다운로드", downloadForLinux: "Linux용 다운로드", + downloadForIos: "iOS용 다운로드", + downloadForAndroid: "Android용 다운로드", sourceLink: "코드 훔쳐 가기(합법적으로)", alsoOnPhone: "휴대폰에서도:", screenshotAlt: "하나의 스레드에서 Claude, Codex, Muse, Gemini 서브에이전트를 실행 중인 T3 Code", diff --git a/apps/marketing/src/i18n/pt-BR.ts b/apps/marketing/src/i18n/pt-BR.ts index 33841e839c527..1947884d33516 100644 --- a/apps/marketing/src/i18n/pt-BR.ts +++ b/apps/marketing/src/i18n/pt-BR.ts @@ -27,6 +27,8 @@ export const ptBR: Messages = { downloadForMac: "Baixar para macOS", downloadForWindows: "Baixar para Windows", downloadForLinux: "Baixar para Linux", + downloadForIos: "Baixar para iOS", + downloadForAndroid: "Baixar para Android", sourceLink: "Roube nosso código (legalmente)", alsoOnPhone: "Também no seu celular:", screenshotAlt: diff --git a/apps/marketing/src/i18n/ru.ts b/apps/marketing/src/i18n/ru.ts index 8fdeefdb9fadd..e0d357813bc6e 100644 --- a/apps/marketing/src/i18n/ru.ts +++ b/apps/marketing/src/i18n/ru.ts @@ -27,6 +27,8 @@ export const ru: Messages = { downloadForMac: "Скачать для macOS", downloadForWindows: "Скачать для Windows", downloadForLinux: "Скачать для Linux", + downloadForIos: "Скачать для iOS", + downloadForAndroid: "Скачать для Android", sourceLink: "Украдите наш код (легально)", alsoOnPhone: "Есть и для телефона:", screenshotAlt: "T3 Code запускает субагентов Claude, Codex, Muse и Gemini в одном треде", diff --git a/apps/marketing/src/i18n/zh.ts b/apps/marketing/src/i18n/zh.ts index bae711530051a..1c554b193c401 100644 --- a/apps/marketing/src/i18n/zh.ts +++ b/apps/marketing/src/i18n/zh.ts @@ -27,6 +27,8 @@ export const zh: Messages = { downloadForMac: "下载 macOS 版", downloadForWindows: "下载 Windows 版", downloadForLinux: "下载 Linux 版", + downloadForIos: "下载 iOS 版", + downloadForAndroid: "下载 Android 版", sourceLink: "来偷我们的代码(合法的)", alsoOnPhone: "手机上也能用:", screenshotAlt: "T3 Code 在同一个线程中运行 Claude、Codex、Muse 和 Gemini 子智能体", diff --git a/apps/marketing/src/layouts/Layout.astro b/apps/marketing/src/layouts/Layout.astro index 0f0a864101743..fa7fa30bc0b18 100644 --- a/apps/marketing/src/layouts/Layout.astro +++ b/apps/marketing/src/layouts/Layout.astro @@ -7,8 +7,8 @@ import { ANDROID_PLAY_STORE_URL, GITHUB_REPOSITORY_URL, IOS_APP_STORE_URL, - MARKETING_STATS, } from "../lib/site"; +import { MARKETING_STATS } from "../lib/stats"; import { type JsonLd, jsonLdGraph, organizationJsonLd, websiteJsonLd } from "../lib/seo"; import { messagesFor } from "../i18n"; import { diff --git a/apps/marketing/src/lib/platform.test.ts b/apps/marketing/src/lib/platform.test.ts new file mode 100644 index 0000000000000..d35abb2ac63aa --- /dev/null +++ b/apps/marketing/src/lib/platform.test.ts @@ -0,0 +1,45 @@ +import { expect, it } from "vite-plus/test"; + +import { detectPlatform, getMobileStoreUrl } from "./platform"; +import { ANDROID_PLAY_STORE_URL, IOS_APP_STORE_URL } from "./site"; + +it.each([ + { + // Android user agents also contain "Linux". + name: "Android", + userAgent: + "Mozilla/5.0 (Linux; Android 15; Pixel 9) AppleWebKit/537.36 Chrome/140.0.0.0 Mobile Safari/537.36", + maxTouchPoints: 5, + os: "android", + storeUrl: ANDROID_PLAY_STORE_URL, + }, + { + name: "iPhone", + userAgent: + "Mozilla/5.0 (iPhone; CPU iPhone OS 18_6 like Mac OS X) AppleWebKit/605.1.15 Version/18.6 Mobile/15E148 Safari/604.1", + maxTouchPoints: 5, + os: "ios", + storeUrl: IOS_APP_STORE_URL, + }, + { + // iPad Safari sends a desktop Mac user agent by default. + name: "iPad with a desktop user agent", + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) AppleWebKit/605.1.15 Version/18.6 Safari/605.1.15", + maxTouchPoints: 5, + os: "ios", + storeUrl: IOS_APP_STORE_URL, + }, + { + name: "macOS", + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 Version/18.6 Safari/605.1.15", + maxTouchPoints: 0, + os: "mac", + storeUrl: null, + }, +])("$name visitors get the $os download", ({ userAgent, maxTouchPoints, os, storeUrl }) => { + const platform = detectPlatform(userAgent, maxTouchPoints); + expect(platform?.os).toBe(os); + expect(platform && getMobileStoreUrl(platform)).toBe(storeUrl); +}); diff --git a/apps/marketing/src/lib/platform.ts b/apps/marketing/src/lib/platform.ts new file mode 100644 index 0000000000000..be17e789a6bb2 --- /dev/null +++ b/apps/marketing/src/lib/platform.ts @@ -0,0 +1,22 @@ +import { ANDROID_PLAY_STORE_URL, IOS_APP_STORE_URL } from "./site"; + +export type Platform = { os: "mac" | "win" | "linux" | "ios" | "android" }; + +export function detectPlatform(userAgent: string, maxTouchPoints: number): Platform | null { + // Mobile checks come first: Android user agents contain "Linux" and iOS + // user agents contain "Mac OS X". + if (/Android/i.test(userAgent)) return { os: "android" }; + if (/iPhone|iPad|iPod/i.test(userAgent)) return { os: "ios" }; + if (/Win/i.test(userAgent)) return { os: "win" }; + // iPadOS Safari sends a Mac user agent by default; only the touch screen + // tells it apart from a Mac. + if (/Mac/i.test(userAgent)) return { os: maxTouchPoints > 1 ? "ios" : "mac" }; + if (/Linux/i.test(userAgent)) return { os: "linux" }; + return null; +} + +export function getMobileStoreUrl(platform: Platform): string | null { + if (platform.os === "ios") return IOS_APP_STORE_URL; + if (platform.os === "android") return ANDROID_PLAY_STORE_URL; + return null; +} diff --git a/apps/marketing/src/lib/releases.test.ts b/apps/marketing/src/lib/releases.test.ts new file mode 100644 index 0000000000000..6527342d674e5 --- /dev/null +++ b/apps/marketing/src/lib/releases.test.ts @@ -0,0 +1,18 @@ +import { afterEach, describe, expect, it, vi } from "vite-plus/test"; + +import { fetchLatestRelease } from "./releases"; + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("fetchLatestRelease", () => { + it("rejects a GitHub API error response instead of returning it as a release", async () => { + vi.stubGlobal("sessionStorage", { getItem: () => null }); + vi.stubGlobal("fetch", async () => + Response.json({ message: "API rate limit exceeded" }, { status: 403 }), + ); + + await expect(fetchLatestRelease("stable")).rejects.toThrow("403"); + }); +}); diff --git a/apps/marketing/src/lib/releases.ts b/apps/marketing/src/lib/releases.ts index 7dc63bdaed859..f50103972cdd8 100644 --- a/apps/marketing/src/lib/releases.ts +++ b/apps/marketing/src/lib/releases.ts @@ -25,12 +25,18 @@ function cacheKey(channel: ReleaseChannel) { return `t3code-${channel}-release`; } +async function fetchJson(url: string): Promise { + const response = await fetch(url); + if (!response.ok) throw new Error(`GitHub API responded ${response.status}`); + return response.json(); +} + async function fetchStable(): Promise { - return fetch(LATEST_API_URL).then((r) => r.json()); + return fetchJson(LATEST_API_URL); } async function fetchNightly(): Promise { - const list: Release[] = await fetch(LIST_API_URL).then((r) => r.json()); + const list = await fetchJson(LIST_API_URL); const nightly = Array.isArray(list) ? list.find((release) => release.tag_name?.includes("-nightly.")) : undefined; diff --git a/apps/marketing/src/lib/site.ts b/apps/marketing/src/lib/site.ts index c4cc8aca59f22..319f6dd800bcb 100644 --- a/apps/marketing/src/lib/site.ts +++ b/apps/marketing/src/lib/site.ts @@ -5,26 +5,3 @@ export const IOS_APP_STORE_URL = export const ANDROID_PLAY_STORE_URL = "https://play.google.com/store/apps/details?id=com.t3tools.t3code"; - -// Runs at build time, so each deploy picks up the current count. -async function fetchGithubStars(): Promise { - const token = process.env.GITHUB_TOKEN; - try { - const response = await fetch("https://api.github.com/repos/pingdotgg/t3code", { - headers: token ? { Authorization: `Bearer ${token}` } : {}, - }); - if (!response.ok) throw new Error(`GitHub API responded ${response.status}`); - const { stargazers_count: stars } = (await response.json()) as { stargazers_count: number }; - return stars < 1000 ? String(stars) : `${Math.floor(stars / 1000)}k+`; - } catch (error) { - // A deploy must not ship a made-up count; offline dev should still render. - if (import.meta.env.PROD) throw new Error("Could not fetch GitHub stars", { cause: error }); - console.warn("Could not fetch GitHub stars", error); - return "?"; - } -} - -export const MARKETING_STATS = { - githubStars: await fetchGithubStars(), - users: "500,000", -} as const; diff --git a/apps/marketing/src/lib/stats.ts b/apps/marketing/src/lib/stats.ts new file mode 100644 index 0000000000000..d291f420b06d1 --- /dev/null +++ b/apps/marketing/src/lib/stats.ts @@ -0,0 +1,22 @@ +// Runs at build time, so each deploy picks up the current count. +async function fetchGithubStars(): Promise { + const token = process.env.GITHUB_TOKEN; + try { + const response = await fetch("https://api.github.com/repos/pingdotgg/t3code", { + headers: token ? { Authorization: `Bearer ${token}` } : {}, + }); + if (!response.ok) throw new Error(`GitHub API responded ${response.status}`); + const { stargazers_count: stars } = (await response.json()) as { stargazers_count: number }; + return stars < 1000 ? String(stars) : `${Math.floor(stars / 1000)}k+`; + } catch (error) { + // A deploy must not ship a made-up count; offline dev should still render. + if (import.meta.env.PROD) throw new Error("Could not fetch GitHub stars", { cause: error }); + console.warn("Could not fetch GitHub stars", error); + return "?"; + } +} + +export const MARKETING_STATS = { + githubStars: await fetchGithubStars(), + users: "500,000", +} as const; diff --git a/apps/marketing/src/pages/95.astro b/apps/marketing/src/pages/95.astro index d6ab25cf867d7..3a9d67f509df9 100644 --- a/apps/marketing/src/pages/95.astro +++ b/apps/marketing/src/pages/95.astro @@ -1,7 +1,8 @@ --- import RetroIcon from "../components/RetroIcon.astro"; import RetroBox from "../components/RetroBox.astro"; -import { GITHUB_REPOSITORY_URL, MARKETING_STATS } from "../lib/site"; +import { GITHUB_REPOSITORY_URL } from "../lib/site"; +import { MARKETING_STATS } from "../lib/stats"; import "../styles/retro.css"; const agents = [ diff --git a/apps/marketing/src/pages/[...locale]/index.astro b/apps/marketing/src/pages/[...locale]/index.astro index 5ae3842180587..e8f4b5f1b5fa5 100644 --- a/apps/marketing/src/pages/[...locale]/index.astro +++ b/apps/marketing/src/pages/[...locale]/index.astro @@ -6,8 +6,8 @@ import { ANDROID_PLAY_STORE_URL, GITHUB_REPOSITORY_URL, IOS_APP_STORE_URL, - MARKETING_STATS, } from "../../lib/site"; +import { MARKETING_STATS } from "../../lib/stats"; import { tweets } from "../../lib/tweets"; import { softwareJsonLd } from "../../lib/seo"; import { messagesFor } from "../../i18n"; @@ -83,6 +83,8 @@ const mobileEndorsementRows = [ data-label-mac={t.downloadForMac} data-label-win={t.downloadForWindows} data-label-linux={t.downloadForLinux} + data-label-ios={t.downloadForIos} + data-label-android={t.downloadForAndroid} > @@ -413,16 +415,7 @@ const mobileEndorsementRows = [ `); + await expect( + ServerBrowserPage.click(page, { selector: "#save", timeoutMs: 2_000 }), + ).rejects.toThrow(); + expect(await page.locator("body").getAttribute("data-clicked")).toBeNull(); + }); + it("does not retarget a removed ref to a replacement node", async () => { await page.setContent(""); const locator = buttonLocator((await takeSnapshot()).accessibilityTree, "original"); @@ -157,6 +375,25 @@ describe("server browser element refs", () => { expect(await page.getByRole("textbox", { name: "parent" }).inputValue()).toBe(""); }); + it("types after a click by coordinates into a shadow-root field, and refuses a field inside an editable frame", async () => { + await page.setContent( + `
`, + ); + const field = page.locator("#host #field"); + const box = (await field.boundingBox())!; + await page.mouse.click(box.x + box.width / 2, box.y + box.height / 2); + await ServerBrowserPage.type(page, { text: "shadow" }); + expect(await field.inputValue()).toBe("shadow"); + + // Refused for now: focus inside a frame cannot be checked from the top document. + const inner = page.frameLocator("iframe").locator("#inner"); + await inner.click(); + await expect(ServerBrowserPage.type(page, { text: "frame" })).rejects.toMatchObject({ + tag: "PreviewAutomationTargetNotEditableError", + }); + expect(await inner.inputValue()).toBe(""); + }); + it("shows dotted fields as dots wherever their value appears, keeping text values and refs", async () => { await page.setContent(`
password:
@@ -391,7 +628,7 @@ describe("server browser snapshot screenshot", () => { const screenshotOf = async (html: string) => { const page = await context.newPage(); await page.setContent(html); - const { screenshot, fullScreenshot } = await ServerBrowserPage.snapshot({ + const input = { page, cdp: await context.newCDPSession(page), renderScale: 2, @@ -399,7 +636,8 @@ describe("server browser snapshot screenshot", () => { consoleEntries: [], networkEntries: [], actionTimeline: [], - }); + }; + const { screenshot, fullScreenshot } = await ServerBrowserPage.snapshot(input); // The server has no DOM types, so the decoder runs as page source. const points = [ [0.15, 0.75], @@ -418,7 +656,7 @@ describe("server browser snapshot screenshot", () => { Array.from(context.getImageData(x * bitmap.width, y * bitmap.height, 1, 1).data), ); })()`)) as Array<[number, number, number, number]>; - return { screenshot, fullScreenshot, pixels }; + return { screenshot, fullScreenshot, pixels, input }; }; const colorOf = ([red, green, blue]: readonly [number, number, number, number]) => blue - red > 100 ? "blue" : red - blue > 100 ? "red" : `rgb(${red}, ${green}, ${blue})`; @@ -433,13 +671,20 @@ describe("server browser snapshot screenshot", () => { }); it("captures a plain page at the shrunk size with its content in place", async () => { - const { screenshot, fullScreenshot, pixels } = await screenshotOf(halves("")); + const { screenshot, fullScreenshot, pixels, input } = await screenshotOf(halves("")); + const metadata = await ServerBrowserPage.snapshotMetadata(input); expect({ size: [screenshot.width, screenshot.height], fullSize: [fullScreenshot?.width, fullScreenshot?.height], + metadataSize: metadata.screenshot, colors: pixels.map(colorOf), - }).toEqual({ size: [1280, 800], fullSize: [2560, 1600], colors: ["blue", "red", "red"] }); + }).toEqual({ + size: [1280, 800], + fullSize: [2560, 1600], + metadataSize: { width: 1280, height: 800 }, + colors: ["blue", "red", "red"], + }); }); it("records the destination of a finite entrance animation without waiting on infinite ones", async () => { @@ -532,6 +777,144 @@ describe("server browser drag", () => { process.off("unhandledRejection", onUnhandled); } }); + + it("moves the mouse after dragstart, so pages that track a drag with mousemove see it", async () => { + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); + await page.setContent(` +
+
source
+
target
+
+ `); + + await ServerBrowserPage.drag(page, { source: "#source", target: "#target" }); + + const events = (await page.evaluate("window.events")) as Array; + expect(events.slice(0, 2)).toEqual(["dragstart", "mousemove"]); + expect(events.at(-1)).toBe("mouseup"); + } finally { + await browser.close(); + } + }); +}); + +describe("server browser action timeout", () => { + type StepOptions = { readonly timeout?: number }; + it.each([ + { + action: "hover", + act: (page: Page) => ServerBrowserPage.hover(page, { selector: "#a", timeoutMs: 9_000 }), + last: ["#a.hover 1"], + }, + { + action: "drag", + act: (page: Page) => + ServerBrowserPage.drag(page, { source: "#a", target: "#b", timeoutMs: 9_000 }), + last: ["#a.dragTo 1", "#b.boundingBox 1"], + }, + ])("gives the $action's steps one shared timeout", async ({ act, last }) => { + let now = 0; + using _now = vi.spyOn(Date, "now").mockImplementation(() => now); + const steps: Array = []; + const box = { x: 0, y: 0, width: 20, height: 20 }; + const page = { + locator: (selector: string) => { + const record = (step: string, options: StepOptions) => + steps.push(`${selector}.${step} ${options.timeout}`); + return { + // Each step that finds the target takes 5 of the 9 seconds. + scrollIntoViewIfNeeded: async (options: StepOptions) => { + record("scrollIntoViewIfNeeded", options); + now += 5_000; + }, + boundingBox: async (options: StepOptions) => { + record("boundingBox", options); + now += 5_000; + return box; + }, + hover: async (options: StepOptions) => record("hover", options), + dragTo: async (_target: unknown, options: StepOptions) => record("dragTo", options), + }; + }, + } as unknown as Page; + + await act(page); + + expect(steps).toEqual(["#a.scrollIntoViewIfNeeded 9000", "#a.boundingBox 4000", ...last]); + }); +}); + +describe("server browser click", () => { + it("clicks a target the page replaced while it was being scrolled into view", async () => { + let scrolls = 0; + const clicks: Array = []; + const page = { + locator: () => ({ + // Playwright resolves the node once; a page re-rendering it detaches that node. + scrollIntoViewIfNeeded: async () => { + if (scrolls++ === 0) { + throw new Error("locator.scrollIntoViewIfNeeded: Element is not attached to the DOM"); + } + }, + boundingBox: async () => ({ x: 0, y: 0, width: 20, height: 20 }), + click: async (options: unknown) => { + clicks.push(options); + }, + count: async () => 1, + }), + once: () => {}, + off: () => {}, + } as unknown as Page; + + await expect(ServerBrowserPage.click(page, { selector: "#save" })).resolves.toEqual({ + x: 10, + y: 10, + }); + expect(clicks).toHaveLength(1); + }); + + it("retries a replaced target only within the action's timeout", async () => { + vi.useFakeTimers(); + try { + const timeouts: Array = []; + const clicks: Array = []; + const detached = new Error( + "locator.scrollIntoViewIfNeeded: Element is not attached to the DOM", + ); + const page = { + locator: () => ({ + scrollIntoViewIfNeeded: async (options?: { timeout?: number }) => { + timeouts.push(options?.timeout); + vi.advanceTimersByTime(60); + throw detached; + }, + click: async (options: unknown) => { + clicks.push(options); + }, + }), + } as unknown as Page; + + await expect( + ServerBrowserPage.click(page, { selector: "#save", timeoutMs: 100 }), + ).rejects.toBe(detached); + expect(timeouts).toEqual([100, 40]); + expect(clicks).toHaveLength(0); + } finally { + vi.useRealTimers(); + } + }); }); describe("server browser evaluate", () => { @@ -568,6 +951,63 @@ describe("server browser evaluate", () => { }); }); +describe("server browser type", () => { + // As on a desktop tab whose Playwright utility world never answers. + const stalledPage = (focusAnswer: () => Promise, typed: string[]) => + ({ + locator: () => ({ last: () => ({ count: () => new Promise(() => {}) }) }), + evaluate: focusAnswer, + keyboard: { insertText: async (text: string) => void typed.push(text) }, + }) as unknown as Page; + + it("types into the focused field when Playwright's utility world never answers", async () => { + vi.useFakeTimers(); + try { + const typed: string[] = []; + let outcome: unknown = "pending"; + const page = stalledPage(async () => true, typed); + void ServerBrowserPage.type(page, { text: "hello", timeoutMs: 300 }).then( + () => (outcome = "typed"), + (error: unknown) => (outcome = error), + ); + await vi.advanceTimersByTimeAsync(300); + expect(outcome).toBe("typed"); + expect(typed).toEqual(["hello"]); + } finally { + vi.useRealTimers(); + } + }); + + it("stops at its timeout without typing when the page never answers", async () => { + vi.useFakeTimers(); + try { + const typed: string[] = []; + let outcome: unknown = "pending"; + const page = stalledPage(() => new Promise(() => {}), typed); + void ServerBrowserPage.type(page, { text: "hello", timeoutMs: 300 }).then( + () => (outcome = "typed"), + (error: unknown) => (outcome = error), + ); + await vi.advanceTimersByTimeAsync(300); + expect(outcome).toMatchObject({ + tag: "PreviewAutomationTimeoutError", + detail: { timeoutMs: 300 }, + }); + expect(typed).toEqual([]); + } finally { + vi.useRealTimers(); + } + }); +}); + +describe("server browser unresponsive script", () => { + it("leaves a page that answers alone", async () => { + const send = vi.fn(async (_method: string) => ({})); + await ServerBrowserPage.stopUnresponsiveScript({ send } as unknown as CDPSession); + expect(send.mock.calls.map(([method]) => method)).toEqual(["Runtime.evaluate"]); + }); +}); + describe("server browser wait", () => { it("ends at its timeout when a check never answers", async () => { // As on a desktop tab whose Playwright utility world never answers. diff --git a/apps/server/src/preview/ServerBrowserPage.ts b/apps/server/src/preview/ServerBrowserPage.ts index 9df6c539014c5..ac37b9914ef76 100644 --- a/apps/server/src/preview/ServerBrowserPage.ts +++ b/apps/server/src/preview/ServerBrowserPage.ts @@ -14,6 +14,8 @@ import { type PreviewAutomationSelectInput, type PreviewAutomationSelectResult, type PreviewAutomationSnapshot, + type PreviewAutomationSnapshotMetadata, + type PreviewAutomationSnapshotStage, type PreviewAutomationTypeInput, type PreviewAutomationUploadInput, type PreviewAutomationWaitForInput, @@ -22,6 +24,7 @@ import { constVoid } from "effect/Function"; import * as Schema from "effect/Schema"; import type { CDPSession, ConsoleMessage, Frame, Locator, Page } from "playwright-core"; import * as NodeCrypto from "node:crypto"; +import * as NodeUtil from "node:util"; import { BrowserControlInterrupted } from "./SessionControl.ts"; const isBrowserControlInterrupted = Schema.is(BrowserControlInterrupted); @@ -29,6 +32,8 @@ const isBrowserControlInterrupted = Schema.is(BrowserControlInterrupted); const MAX_EVALUATION_BYTES = 64_000; const MAX_VISIBLE_TEXT_LENGTH = 20_000; const MAX_SCREENSHOT_WIDTH = 1280; +// Two attempts must fit in a snapshot's 10-second deadline. +const SCREENSHOT_TIMEOUT_MS = 3_000; const WAIT_POLL_MS = 100; export const DIAGNOSTIC_BUFFER_LIMIT = 200; @@ -60,7 +65,7 @@ export const toOperationError = ( if (cause instanceof Error && cause.name === "TimeoutError") { return new ServerBrowserOperationError({ tag: "PreviewAutomationTimeoutError", - message: firstLine, + message: withActionabilityReason(firstLine, cause), detail: timeoutMs === undefined ? undefined : { timeoutMs }, }); } @@ -82,6 +87,17 @@ export const toOperationError = ( }); }; +/** Playwright puts why an element was not actionable in its call log, below the first line. */ +const withActionabilityReason = (message: string, cause: Error) => { + // Playwright dims call-log lines with ANSI escapes when colour is on. + const reason = NodeUtil.stripVTControlCharacters(cause.message) + .split("\n") + .map((line) => line.trim()) + .findLast((line) => line.startsWith("- element is ")) + ?.slice(2); + return reason === undefined ? message : `${message} Playwright last saw: ${reason}.`; +}; + const DEFAULT_TIMEOUT_MS = 15_000; const TARGET_COUNT_MS = 250; const MAX_TARGET_CHARS = 200; @@ -145,15 +161,32 @@ export type PointerReporter = ( ) => Promise; const noPointer: PointerReporter = async () => {}; +// Locator.scrollIntoViewIfNeeded resolves its node once and fails when the page replaces it; the +// input actions after it re-resolve, so the scroll does too, within the same timeout. +const DETACHED = /^locator\.scrollIntoViewIfNeeded: Element is not attached to the DOM(?:\n|$)/; + +const scrollIntoView = async (locator: Locator, timeout: number) => { + const deadline = Date.now() + timeout; + for (;;) { + try { + return await locator.scrollIntoViewIfNeeded({ timeout: Math.max(1, deadline - Date.now()) }); + } catch (cause) { + if (!(cause instanceof Error && DETACHED.test(cause.message)) || Date.now() >= deadline) { + throw cause; + } + } + } +}; + const targetPoint = async ( page: Page, locator: Locator | null, input: { readonly x?: number | undefined; readonly y?: number | undefined }, - timeout: number, + timeout: () => number, ) => { if (locator === null) return { x: input.x ?? 0, y: input.y ?? 0 }; - await locator.scrollIntoViewIfNeeded({ timeout }); - const box = await locator.boundingBox({ timeout }); + await scrollIntoView(locator, timeout()); + const box = await locator.boundingBox({ timeout: timeout() }); if (box) return { x: box.x + box.width / 2, y: box.y + box.height / 2 }; const viewport = page.viewportSize(); return { x: (viewport?.width ?? 0) / 2, y: (viewport?.height ?? 0) / 2 }; @@ -246,13 +279,36 @@ const pageChanged = () => message: "The page changed while capturing its snapshot. Take another snapshot.", }); -const tooBusy = () => +type SnapshotStage = typeof PreviewAutomationSnapshotStage.Type; + +/** + * Running stages are counted, since a retry can overlap the previous attempt's unanswered + * screenshot. A stage whose own timeout fired has stopped running but is still the cause. + */ +interface SnapshotStages { + readonly running: Map; + readonly timedOut: Set; +} + +const snapshotTimedOut = (stages?: SnapshotStages) => + new ServerBrowserOperationError({ + tag: "PreviewAutomationTimeoutError", + message: `Snapshot capture timed out after ${SNAPSHOT_TIMEOUT_MS}ms. Try the snapshot again.`, + detail: { + timeoutMs: SNAPSHOT_TIMEOUT_MS, + pendingStages: stages ? [...new Set([...stages.timedOut, ...stages.running.keys()])] : [], + }, + }); + +const frameUnanswered = () => new ServerBrowserOperationError({ tag: "PreviewAutomationTimeoutError", - message: - "The page was too busy to capture a snapshot in time. Wait for it to settle and take another snapshot.", + message: `A frame on the page didn't answer within ${FRAME_ANSWER_MS}ms, so its content couldn't be checked for hidden values. Try the snapshot again.`, + detail: { timeoutMs: FRAME_ANSWER_MS, pendingStages: ["frames"] }, }); +const isTimeout = (cause: unknown) => cause instanceof Error && cause.name === "TimeoutError"; + const VALUE_TOO_LONG = "A password field holds a value too long to hide, so no snapshot was taken."; /** Reuse one CDP session per out-of-process frame across snapshots. */ @@ -428,6 +484,37 @@ const pngSize = (base64: string) => { return { width: header.readUInt32BE(16), height: header.readUInt32BE(20) }; }; +export const jpegSize = (base64: string) => { + const bytes = Buffer.from(base64, "base64"); + if (bytes.length < 4 || bytes.readUInt16BE(0) !== 0xffd8) + throw new Error("Invalid JPEG screenshot."); + let offset = 2; + while (offset < bytes.length) { + if (bytes[offset++] !== 0xff) throw new Error("Invalid JPEG screenshot."); + while (bytes[offset] === 0xff) offset++; + const marker = bytes[offset++]; + if (marker === undefined || marker === 0xd9 || marker === 0xda) break; + if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) continue; + if (offset + 2 > bytes.length) break; + const length = bytes.readUInt16BE(offset); + if (length < 2 || offset + length > bytes.length) break; + if (marker >= 0xc0 && marker <= 0xcf && ![0xc4, 0xc8, 0xcc].includes(marker)) { + if (length < 7) break; + const height = bytes.readUInt16BE(offset + 3); + const width = bytes.readUInt16BE(offset + 5); + if (width && height) return { width, height }; + break; + } + offset += length; + } + throw new Error("Invalid JPEG screenshot."); +}; + +const scaledSize = (size: { readonly width: number; readonly height: number }, scale: number) => + scale < 1 + ? { width: Math.round(size.width * scale), height: Math.round(size.height * scale) } + : size; + /** * A page's display ratio and CSS viewport, read in an isolated world since page script can * reassign `devicePixelRatio`, `innerWidth`, and `innerHeight` in its own, and its page zoom. @@ -468,10 +555,27 @@ export const captureViewport = async ( cdp: CDPSession, options: { readonly format: "png" | "jpeg"; readonly quality?: number }, ) => { - const { data } = await cdp.send("Page.captureScreenshot", { + const capture = cdp.send("Page.captureScreenshot", { format: options.format, ...(options.quality === undefined ? {} : { quality: options.quality }), }); + let timer: ReturnType | undefined; + const expired = new Promise((_, reject) => { + timer = setTimeout( + () => + reject( + new ServerBrowserOperationError({ + tag: "PreviewAutomationExecutionError", + message: + "The browser is not producing screenshot frames. For a desktop tab, show or restore the T3 Code window and try again.", + }), + ), + SCREENSHOT_TIMEOUT_MS, + ); + }); + // Hidden native guests can leave CDP captures pending forever. This deadline also + // releases a viewer's still-capture lock, which an agent snapshot may be waiting for. + const { data } = await Promise.race([capture, expired]).finally(() => clearTimeout(timer)); return data; }; @@ -561,41 +665,73 @@ const renderingOf = async (input: SnapshotInput) => { const capture = async ( input: SnapshotInput, deadline: number, -): Promise => { - const { renderScale, viewport } = await renderingOf(input); + stages: SnapshotStages, + metadataOnly: boolean, +) => { + const { running } = stages; + const track = async
(stage: SnapshotStage, operation: () => Promise) => { + running.set(stage, (running.get(stage) ?? 0) + 1); + try { + const result = await operation(); + stages.timedOut.delete(stage); + return result; + } catch (cause) { + if (isTimeout(cause)) stages.timedOut.add(stage); + throw cause; + } finally { + const count = (running.get(stage) ?? 1) - 1; + if (count === 0) running.delete(stage); + else running.set(stage, count); + } + }; + const { renderScale, viewport } = await track("metrics", () => renderingOf(input)); // A late metrics reply must not begin a capture after viewers have resumed. - if (Date.now() >= deadline) throw tooBusy(); + if (Date.now() >= deadline) throw snapshotTimedOut(); const scale = Math.min(1, MAX_SCREENSHOT_WIDTH / (viewport.width * renderScale)); const state = refsFor(input.page); invalidateRefs(input.page); const generation = state.generation; - const { readers, read } = await openReaders(input.page, input.cdp); - const before = await read(); + const { readers, read } = await track("frames", () => openReaders(input.page, input.cdp)); + const before = await track("frames", () => read()); // Bracket the tree with value reads to detect changes; await tree completion before retrying failed captures. const pageAndScreenshot = Promise.all([ - input.page.evaluate(SNAPSHOT_SCRIPT) as Promise< - Pick< - PreviewAutomationSnapshot, - "url" | "title" | "loading" | "visibleText" | "interactiveElements" - > - >, - captureViewport(input.cdp, { format: "png" }).then(async (full) => - scale < 1 - ? { - data: await shrinkPng(input.cdp, full, scale), - full: input.fullScreenshot ? full : undefined, - } - : { data: full, full: undefined }, + track( + "page state", + () => + input.page.evaluate(SNAPSHOT_SCRIPT) as Promise< + Pick< + PreviewAutomationSnapshot, + "url" | "title" | "loading" | "visibleText" | "interactiveElements" + > + >, + ), + // A cheap real frame still sizes the screenshot; computing it from metrics drifted from the PNG. + track("screenshot", () => + metadataOnly + ? captureViewport(input.cdp, { format: "jpeg", quality: 10 }).then((data) => ({ + data, + full: undefined, + })) + : captureViewport(input.cdp, { format: "png" }).then(async (full) => + scale < 1 + ? { + data: await shrinkPng(input.cdp, full, scale), + full: input.fullScreenshot ? full : undefined, + } + : { data: full, full: undefined }, + ), ), ]); void pageAndScreenshot.catch(constVoid); // Leaves time for the second read; a frame that never loads holds the tree until then. - const tree = await input.page.ariaSnapshot({ - mode: "ai", - boxes: true, - timeout: Math.max(1, deadline - Date.now() - 2_000), - }); - const after = await read(before); + const tree = await track("accessibility tree", () => + input.page.ariaSnapshot({ + mode: "ai", + boxes: true, + timeout: Math.max(1, deadline - Date.now() - 2_000), + }), + ); + const after = await track("frames", () => read(before)); // A value that changed meanwhile may be in the tree under neither reading. if (after.some((values, i) => values && JSON.stringify(values) !== JSON.stringify(before[i]))) throw pageChanged(); @@ -604,11 +740,13 @@ const capture = async ( const unread = readers.flatMap((reader, i) => (after[i] ? [] : reader.frames)); const filled = /^\s*- iframe (?:\[active\] )?\[ref=(\S+?)\].*:$/gm; for (const [, ref] of unread.length ? tree.matchAll(filled) : []) { - const iframe = await input.page.locator(`aria-ref=${ref}`).elementHandle({ timeout: 2_000 }); + const iframe = await track("frames", () => + input.page.locator(`aria-ref=${ref}`).elementHandle({ timeout: 2_000 }), + ); if (!iframe) throw pageChanged(); - const frame = await iframe.contentFrame(); + const frame = await track("frames", () => iframe.contentFrame()); void iframe.dispose().catch(constVoid); - if (frame && unread.includes(frame)) throw tooBusy(); + if (frame && unread.includes(frame)) throw frameUnanswered(); } const [page, { full, data }] = await pageAndScreenshot; if (state.generation !== generation) throw pageChanged(); @@ -632,11 +770,10 @@ const capture = async ( consoleEntries: [...input.consoleEntries], networkEntries: [...input.networkEntries], actionTimeline: [...input.actionTimeline], - screenshot: { - mimeType: "image/png", - data, - ...pngSize(data), - }, + // Reports the size the PNG path would have shrunk to. + screenshot: metadataOnly + ? scaledSize(jpegSize(data), scale) + : { mimeType: "image/png" as const, data, ...pngSize(data) }, ...(full !== undefined ? { fullScreenshot: { data: full, ...pngSize(full) } } : {}), }; }; @@ -644,41 +781,50 @@ const capture = async ( /** Snapshots begun per page; observers' snapshots aren't queued, so two can overlap. */ const snapshotTurns = new WeakMap(); -export const snapshot = async (input: SnapshotInput): Promise => { +const runSnapshot = async (input: SnapshotInput, metadataOnly: boolean) => { const deadline = Date.now() + SNAPSHOT_TIMEOUT_MS; + const stages: SnapshotStages = { running: new Map(), timedOut: new Set() }; let timer: ReturnType | undefined; const expired = new Promise((_, reject) => { - timer = setTimeout(() => reject(tooBusy()), SNAPSHOT_TIMEOUT_MS); + timer = setTimeout(() => reject(snapshotTimedOut(stages)), SNAPSHOT_TIMEOUT_MS); }); const turn = (snapshotTurns.get(input.page) ?? 0) + 1; snapshotTurns.set(input.page, turn); // Retry only before the deadline and the next snapshot, whose refs a later retry would revoke. - const captured = capture(input, deadline).catch((cause: unknown) => + const captured = capture(input, deadline, stages, metadataOnly).catch((cause: unknown) => Date.now() < deadline && snapshotTurns.get(input.page) === turn && !(cause instanceof Error && cause.message === VALUE_TOO_LONG) - ? capture(input, deadline) + ? capture(input, deadline, stages, metadataOnly) : Promise.reject(cause), ); try { return await Promise.race([captured, expired]); } catch (cause) { - throw cause instanceof Error && cause.name === "TimeoutError" ? tooBusy() : cause; + throw isTimeout(cause) ? snapshotTimedOut(stages) : cause; } finally { clearTimeout(timer); } }; +export const snapshot = (input: SnapshotInput) => + runSnapshot(input, false) as Promise; + +export const snapshotMetadata = (input: SnapshotInput) => + runSnapshot(input, true) as Promise; + /** Return when a dialog opens; waiting for the action to finish would block until dialog resolution. */ export const click = async ( page: Page, input: PreviewAutomationClickInput, pointer: PointerReporter = noPointer, ): Promise<{ readonly x: number; readonly y: number }> => { - const timeout = input.timeoutMs ?? DEFAULT_TIMEOUT_MS; + // One deadline across steps keeps the reply inside the broker's grace; Playwright reads 0 as none. + const deadline = Date.now() + (input.timeoutMs ?? DEFAULT_TIMEOUT_MS); + const remaining = () => Math.max(1, deadline - Date.now()); const locator = targetLocator(page, input); try { - return await clickTarget(page, locator, input, timeout, pointer); + return await clickTarget(page, locator, input, remaining, pointer); } catch (cause) { throw locator === null ? cause : await namedTargetTimeout(locator, input, cause); } @@ -714,10 +860,11 @@ const namedTargetTimeout = async ( : matches === 0 ? `The click may have happened and removed its target or navigated; check the page before retrying. No element matches ${target} after the timeout; take a fresh snapshot and use its locator.` : `The click may have happened; check the page before retrying. ${target} still matches an element after the timeout.`; + const message = withActionabilityReason(reason, cause); return new ServerBrowserOperationError({ tag: "PreviewAutomationTimeoutError", - message: reason, - detail: { reason }, + message, + detail: { reason: message }, cause, }); }; @@ -726,7 +873,7 @@ const clickTarget = async ( page: Page, locator: Locator | null, input: PreviewAutomationClickInput, - timeout: number, + timeout: () => number, pointer: PointerReporter, ) => { const point = await targetPoint(page, locator, input, timeout); @@ -735,7 +882,7 @@ const clickTarget = async ( const clicked = locator === null ? page.mouse.click(point.x, point.y, options) - : locator.click({ ...options, timeout }); + : locator.click({ ...options, timeout: timeout() }); let onDialog = constVoid; const dialogOpened = new Promise<"dialog">((resolve) => { onDialog = () => resolve("dialog"); @@ -749,6 +896,76 @@ const clickTarget = async ( return point; }; +type FocusCandidate = { + readonly tagName: string; + readonly type?: string; + readonly disabled?: boolean; + readonly readOnly?: boolean; + readonly isContentEditable?: boolean; + readonly shadowRoot?: { readonly activeElement: FocusCandidate | null } | null; + readonly focus?: () => void; + getRootNode(): { readonly activeElement?: FocusCandidate | null }; + contains(other: FocusCandidate): boolean; +}; + +// Serialized into the page's main world, so it must not reference module scope. +const focusTextTarget = (target: FocusCandidate | null) => { + let element = + target ?? + (globalThis as unknown as { document: { activeElement: FocusCandidate | null } }).document + .activeElement; + if (target === null) { + while (element?.shadowRoot?.activeElement) element = element.shadowRoot.activeElement; + // Focus inside a frame cannot be checked from here, and an editable frame would pass as contentEditable. + if (element && ["IFRAME", "FRAME", "FENCEDFRAME", "OBJECT", "EMBED"].includes(element.tagName)) + return false; + } + if (element == null) return false; + // Require an enabled, focused text target to avoid losing input or sending it to the previous field. + const nonText = [ + "button", + "checkbox", + "color", + "file", + "hidden", + "image", + "radio", + "range", + "reset", + "submit", + ]; + const textControl = + element.tagName === "TEXTAREA" || + (element.tagName === "INPUT" && !nonText.includes(element.type ?? "text")); + if (!(textControl || element.isContentEditable) || element.disabled || element.readOnly) { + return false; + } + element.focus?.(); + const active = element.getRootNode().activeElement; + return active != null && (active === element || element.contains(active)); +}; + +// A `*:focus` locator needs Playwright's utility world, which some desktop tabs never get. +const focusedTextField = async (page: Page, timeout: number) => { + let timer: ReturnType | undefined; + const expired = new Promise((_, reject) => { + timer = setTimeout( + () => + reject( + new ServerBrowserOperationError({ + tag: "PreviewAutomationTimeoutError", + message: `The page did not report its focused element within ${timeout}ms, so no text was typed.`, + detail: { timeoutMs: timeout }, + }), + ), + timeout, + ); + }); + return await Promise.race([page.evaluate(focusTextTarget, null), expired]).finally(() => + clearTimeout(timer), + ); +}; + export const type = async (page: Page, input: PreviewAutomationTypeInput) => { const timeout = input.timeoutMs ?? DEFAULT_TIMEOUT_MS; const locator = targetLocator(page, input); @@ -756,46 +973,10 @@ export const type = async (page: Page, input: PreviewAutomationTypeInput) => { await locator.fill(input.text, { timeout }); return; } - // A shadow host also matches :focus; use its innermost focused descendant. - const target = locator ?? page.locator("*:focus").last(); const focused = - (locator !== null || (await target.count()) > 0) && - (await target.evaluate( - (element) => { - // Require an enabled, focused text target to avoid losing input or sending it to the previous field. - const control = element as unknown as { - readonly type?: string; - readonly disabled?: boolean; - readonly readOnly?: boolean; - readonly isContentEditable?: boolean; - readonly focus?: () => void; - }; - const nonText = [ - "button", - "checkbox", - "color", - "file", - "hidden", - "image", - "radio", - "range", - "reset", - "submit", - ]; - const textControl = - element.tagName === "TEXTAREA" || - (element.tagName === "INPUT" && !nonText.includes(control.type ?? "text")); - if (!(textControl || control.isContentEditable) || control.disabled || control.readOnly) { - return false; - } - control.focus?.(); - const root = element.getRootNode() as { readonly activeElement?: typeof element | null }; - const active = root.activeElement; - return active != null && (active === element || element.contains(active)); - }, - undefined, - { timeout }, - )); + locator === null + ? await focusedTextField(page, timeout) + : await locator.evaluate(focusTextTarget, undefined, { timeout }); if (focused !== true) { throw new ServerBrowserOperationError({ tag: "PreviewAutomationTargetNotEditableError", @@ -814,14 +995,15 @@ export const hover = async ( input: PreviewAutomationHoverInput, pointer: PointerReporter = noPointer, ) => { - const timeout = input.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const deadline = Date.now() + (input.timeoutMs ?? DEFAULT_TIMEOUT_MS); + const remaining = () => Math.max(1, deadline - Date.now()); const locator = targetLocator(page, input); - await pointer(await targetPoint(page, locator, input, timeout), "move"); + await pointer(await targetPoint(page, locator, input, remaining), "move"); if (locator === null) { await page.mouse.move(input.x ?? 0, input.y ?? 0); return; } - await locator.hover({ timeout }); + await locator.hover({ timeout: remaining() }); }; export const select = async ( @@ -864,16 +1046,18 @@ export const drag = async ( input: PreviewAutomationDragInput, pointer: PointerReporter = noPointer, ) => { - const timeout = input.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const deadline = Date.now() + (input.timeoutMs ?? DEFAULT_TIMEOUT_MS); + const remaining = () => Math.max(1, deadline - Date.now()); const source = targetLocator(page, { locator: input.source })!; const target = targetLocator(page, { locator: input.target })!; - await pointer(await targetPoint(page, source, {}, timeout), "move"); + await pointer(await targetPoint(page, source, {}, remaining), "move"); // Observe drag rejection immediately; an unhandled rejection during cursor movement exits the server. - const dragFailure = source.dragTo(target, { timeout }).then( + // Chromium delivers a single move before dragstart; steps give the page mousemoves during the drag. + const dragFailure = source.dragTo(target, { timeout: remaining(), steps: 5 }).then( () => null, (error: unknown) => ({ error }), ); - const end = await target.boundingBox({ timeout }).catch(() => null); + const end = await target.boundingBox({ timeout: remaining() }).catch(() => null); if (end) await pointer({ x: end.x + end.width / 2, y: end.y + end.height / 2 }, "move"); const failure = await dragFailure; if (failure) throw failure.error; @@ -901,6 +1085,26 @@ export const scroll = async (page: Page, input: PreviewAutomationScrollInput) => await locator.evaluate((element, [x, y]) => element.scrollBy(x, y), delta); }; +const UNRESPONSIVE_SCRIPT_PROBE_MS = 1_000; + +/** + * A same-site navigation reuses the renderer, so it never commits while a page script holds it. + * Call before starting the navigation: Chrome does not deliver a terminate once one is pending. + */ +export const stopUnresponsiveScript = async (cdp: CDPSession, budgetMs = Infinity) => { + let timer: ReturnType | undefined; + const answered = cdp.send("Runtime.evaluate", { expression: "0" }).then( + () => true, + () => true, + ); + const expired = new Promise((resolve) => { + timer = setTimeout(() => resolve(false), Math.min(budgetMs, UNRESPONSIVE_SCRIPT_PROBE_MS)); + }); + const responsive = await Promise.race([answered, expired]).finally(() => clearTimeout(timer)); + // Not awaited: a renderer blocked on something other than script may never acknowledge it. + if (!responsive) void cdp.send("Runtime.terminateExecution").catch(constVoid); +}; + /** Terminate expired page scripts so a hung expression cannot block the tab's control queue. */ export const evaluate = async ( cdp: CDPSession, @@ -1083,7 +1287,8 @@ export const recordingEncoderScript = (frameRate: BrowserRecordingFrameRate) => window.__t3Recorder = { async frame(base64, cssWidth) { if (tooLarge) return false; - const bytes = Uint8Array.from(atob(base64), (char) => char.charCodeAt(0)); + // A per-character atob copy costs ~30 ms on a 3200x1800 frame and throttles recordings. + const bytes = Uint8Array.fromBase64(base64); const bitmap = await createImageBitmap(new Blob([bytes], { type: "image/jpeg" })); if (tooLarge) { bitmap.close(); return false; } if (!recorder) { diff --git a/apps/server/src/preview/ServerBrowserStream.test.ts b/apps/server/src/preview/ServerBrowserStream.test.ts index c4a709ed8acc4..c0a9fbc923ff2 100644 --- a/apps/server/src/preview/ServerBrowserStream.test.ts +++ b/apps/server/src/preview/ServerBrowserStream.test.ts @@ -57,6 +57,7 @@ const mutations = [ { type: "navigate", url: "https://example.com" }, { type: "history", delta: -1 }, { type: "reload" }, + { type: "stopLoading" }, { type: "probe", x: 10, y: 10 }, { type: "takeControl" }, { type: "releaseControl" }, @@ -149,7 +150,7 @@ it.effect.each([ }, ]); expect(auth.requests).toEqual([resource]); - }).pipe(Effect.scoped), + }), ); it.effect("sends replies ahead of stale frames and only the newest frame after an ack", () => @@ -221,7 +222,7 @@ it.effect("sends replies ahead of stale frames and only the newest frame after a yield* Effect.promise(() => next.promise); expect(received).toEqual([1, "probe", 4]); expect(released).toEqual([2, 3, 1]); - }).pipe(Effect.scoped), + }), ); it.effect.each([ @@ -264,7 +265,7 @@ it.effect.each([ ); expect(response.status).toBe(testCase.status); expect(attachments).toBe(0); - }).pipe(Effect.scoped), + }), ); it.effect("serves a tab's download only to an authorized session", () => @@ -369,7 +370,7 @@ it.effect("passes uploaded files to the page's open picker and needs operate sco { chooserId: "chooser-1", files: [{ name: "notes.txt", text: "hello" }] }, ]); expect((yield* upload([AuthPreviewOperateScope], "stale")).status).toBe(409); - }).pipe(Effect.scoped), + }), ); const decodeHostSetup = Schema.decodeUnknownEffect(Schema.fromJsonString(PreviewStreamHostSetup)); @@ -430,5 +431,5 @@ it.effect.each([ need, command: "sudo t3 browser setup", }); - }).pipe(Effect.scoped), + }), ); diff --git a/apps/server/src/process/NonBlockingSpawner.test.ts b/apps/server/src/process/NonBlockingSpawner.test.ts new file mode 100644 index 0000000000000..81563cbab70dc --- /dev/null +++ b/apps/server/src/process/NonBlockingSpawner.test.ts @@ -0,0 +1,200 @@ +// @effect-diagnostics nodeBuiltinImport:off - the spawner's checks are node:fs/promises calls, which the test spies on. +import * as NodeFSP from "node:fs/promises"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { describe, expect, it } from "@effect/vitest"; +import { vi } from "vite-plus/test"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as ChildProcess from "effect/process/ChildProcess"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; + +import * as NonBlockingSpawner from "./NonBlockingSpawner.ts"; + +vi.mock("node:fs/promises", { spy: true }); + +const hostPath = process.env.PATH; + +const wrapped = NonBlockingSpawner.layer.pipe(Layer.provideMerge(NodeServices.layer)); + +/** What a caller sees of a command: its output, or the error of a spawn that failed. */ +const outcome = (command: ChildProcess.Command) => + ChildProcessSpawner.ChildProcessSpawner.use((spawner) => spawner.string(command)).pipe( + Effect.match({ + onSuccess: (output) => ({ output }), + onFailure: (error) => ({ reason: error.reason._tag, message: error.message }), + }), + ); + +describe.skipIf(HostProcess.Platform.defaultValue() !== "linux")("NonBlockingSpawner", () => { + it.live("gives callers what the native spawner gives them", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const root = yield* fileSystem.realPath(yield* fileSystem.makeTempDirectoryScoped()); + for (const [file, mode] of [ + ["stale/tool", 0o644], + ["fresh/tool", 0o755], + ["target/bin/tool", 0o755], + // GNU env reads a lone `-` as -i and `NAME=value` as an assignment, even after `--`. + ["fresh/-", 0o755], + ["fresh/launch=stable", 0o755], + ] as const) { + yield* fileSystem.makeDirectory(`${root}/${file.slice(0, file.lastIndexOf("/"))}`, { + recursive: true, + }); + yield* fileSystem.writeFileString(`${root}/${file}`, `#!/bin/sh\necho ${file} "$@"\n`); + yield* fileSystem.chmod(`${root}/${file}`, mode); + } + yield* fileSystem.makeDirectory(`${root}/target/subdir`); + yield* fileSystem.symlink(`${root}/target/subdir`, `${root}/link`); + const PATH = `${root}/stale:${root}/fresh`; + + const commands = { + "enters the cwd": ChildProcess.make("pwd", ["-P"], { cwd: root }), + "keeps argv": ChildProcess.make( + "printf", + ["%s\\n", "-n", "--", "a b", `it's "quoted"`, "$HOME", "; exit 3", "", "A=b"], + { cwd: root }, + ), + // dash would drop the exported function and add PWD and OLDPWD. + "keeps the environment": ChildProcess.make("env", [], { + cwd: root, + env: { PATH: hostPath, A: "1", "BASH_FUNC_x%%": "() { echo exported\n}" }, + }), + "skips a non-executable PATH entry": ChildProcess.make("tool", [], { env: { PATH } }), + "resolves `..` in PATH after a symlink": ChildProcess.make("tool", [], { + env: { PATH: `${root}/link/../bin` }, + }), + "searches PATH relative to the cwd": ChildProcess.make("tool", [], { + cwd: root, + env: { PATH: "fresh" }, + }), + "a command named -": ChildProcess.make("-", [], { cwd: root, env: { PATH } }), + "a command containing =": ChildProcess.make("launch=stable", [], { env: { PATH } }), + "shell mode": ChildProcess.make('pwd -P; echo "$0" "a b"', [], { cwd: root, shell: true }), + "a pipeline": ChildProcess.make("printf", ["a\\nb\\n"], { cwd: root }).pipe( + ChildProcess.pipeTo(ChildProcess.make("tail", ["-n", "1"])), + ), + "a missing command": ChildProcess.make("t3code-missing-command", ["--version"]), + "a missing command in a cwd": ChildProcess.make("t3code-missing-command", [], { + cwd: root, + }), + "a non-executable command": ChildProcess.make("tool", [], { + env: { PATH: `${root}/stale` }, + }), + "a missing explicit path": ChildProcess.make("./missing", [], { cwd: root }), + "a non-executable explicit path": ChildProcess.make(`${root}/stale/tool`, []), + "a missing cwd": ChildProcess.make("true", [], { cwd: `${root}/missing` }), + // The native check sees `missing`, although the path resolves to an existing directory. + "a cwd through a missing directory": ChildProcess.make("true", [], { + cwd: `${root}/missing/..`, + }), + }; + for (const [name, command] of Object.entries(commands)) { + const native = yield* outcome(command).pipe(Effect.provide(NodeServices.layer)); + expect(yield* outcome(command).pipe(Effect.provide(wrapped)), name).toEqual(native); + } + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.live("hands the native spawner only `env`, never a cwd or a command to look up", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const directory = yield* fileSystem.makeTempDirectoryScoped(); + const received: Array = []; + const record = (command: ChildProcess.Command): void => { + if (command._tag === "StandardCommand") received.push(command); + else [command.left, command.right].forEach(record); + }; + const recording = Layer.effect( + ChildProcessSpawner.ChildProcessSpawner, + Effect.map(ChildProcessSpawner.ChildProcessSpawner, (native) => + ChildProcessSpawner.make((command) => { + record(command); + return native.spawn(command); + }), + ), + ).pipe(Layer.provideMerge(NodeServices.layer)); + const commands = [ + ChildProcess.make("true", [], { cwd: directory }), + ChildProcess.make("true", []), + ChildProcess.make("/bin/true", []), + ChildProcess.make("t3code-missing-command", []), + ChildProcess.make("true", [], { cwd: directory, shell: true }), + ChildProcess.make("printf", ["a"], { cwd: directory }).pipe( + ChildProcess.pipeTo(ChildProcess.make("cat", [], { cwd: directory })), + ), + ]; + yield* Effect.forEach(commands, (command) => + ChildProcessSpawner.ChildProcessSpawner.use((spawner) => spawner.exitCode(command)).pipe( + Effect.exit, + ), + ).pipe(Effect.provide(NonBlockingSpawner.layer.pipe(Layer.provide(recording)))); + + expect(received.map((command) => [command.command, command.options.cwd])).toEqual( + Array.from({ length: 6 }, () => ["/usr/bin/env", undefined]), + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.live("checks at most two paths at once, however many directories hang", () => + Effect.gen(function* () { + const allArrived = yield* Deferred.make(); + let arrivals = 0; + // Each spawn resolves its cwd, then starts its check within microtasks. + const arrivalPath = Layer.effect( + Path.Path, + Effect.map(Path.Path, (path) => ({ + ...path, + resolve: (...segments: ReadonlyArray) => { + if (++arrivals === 3) setImmediate(() => Deferred.doneUnsafe(allArrived, Effect.void)); + return path.resolve(...segments); + }, + })), + ); + const hung: Array<() => void> = []; + let releasing = false; + let running = 0; + let mostRunning = 0; + using _access = vi.spyOn(NodeFSP, "access").mockImplementation( + () => + new Promise((_resolve, reject) => { + mostRunning = Math.max(mostRunning, ++running); + const fail = () => { + running--; + reject(Object.assign(new Error("ENOENT"), { code: "ENOENT" })); + }; + if (releasing) fail(); + else hung.push(fail); + }), + ); + const spawns = yield* Effect.forEach( + ["a", "b", "c"], + (name) => + ChildProcessSpawner.ChildProcessSpawner.use((spawner) => + spawner.exitCode(ChildProcess.make("true", [], { cwd: `/t3code-hung/${name}` })), + ).pipe(Effect.flip), + { concurrency: "unbounded" }, + ).pipe( + Effect.provide( + NonBlockingSpawner.layer.pipe( + Layer.provide(arrivalPath), + Layer.provide(NodeServices.layer), + ), + ), + Effect.forkChild, + ); + yield* Deferred.await(allArrived); + expect(mostRunning).toBe(2); + releasing = true; + for (const fail of hung.splice(0)) fail(); + yield* Fiber.join(spawns); + expect(mostRunning).toBe(2); + }), + ); +}); diff --git a/apps/server/src/process/NonBlockingSpawner.ts b/apps/server/src/process/NonBlockingSpawner.ts new file mode 100644 index 0000000000000..0807b287cb9cc --- /dev/null +++ b/apps/server/src/process/NonBlockingSpawner.ts @@ -0,0 +1,178 @@ +// @effect-diagnostics nodeBuiltinImport:off - execvp's X_OK test has no Effect FileSystem equivalent. +import * as NodeFS from "node:fs"; +import * as NodeFSP from "node:fs/promises"; + +import { envTrampoline, supportsEnvChdir } from "@t3tools/shared/envTrampoline"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as PlatformError from "effect/PlatformError"; +import * as Redacted from "effect/Redacted"; +import * as ChildProcess from "effect/process/ChildProcess"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; +import * as CommandLine from "effect/process/CommandLine"; + +// libuv's _PATH_DEFPATH, searched when the child environment has no PATH. +const DEFAULT_PATH = "/usr/bin:/bin"; + +// A check on a hung mount holds a libuv threadpool thread until the mount answers, and the +// pool's four threads also compress every WebSocket frame. Checks therefore run one at a time +// per directory and two at a time overall; the rest wait without holding a thread. +const CHECK_LIMIT = 2; +const directoryTails = new Map>(); +const waiting: Array<() => void> = []; +let running = 0; + +const access = async (path: string, mode: number) => { + if (running < CHECK_LIMIT) running++; + else await new Promise((resume) => waiting.push(resume)); + try { + await NodeFSP.access(path, mode); + return undefined; + } catch (error) { + return error as NodeJS.ErrnoException; + } finally { + // A waiter takes over the slot. + const next = waiting.shift(); + if (next === undefined) running--; + else next(); + } +}; + +/** The errno error of `access(path, mode)`, or undefined when it passes. */ +const check = (directory: string, path: string, mode: number) => { + const result = (directoryTails.get(directory) ?? Promise.resolve()).then(() => + access(path, mode), + ); + directoryTails.set(directory, result); + void result.then(() => { + if (directoryTails.get(directory) === result) directoryTails.delete(directory); + }); + return Effect.promise(() => result); +}; + +// The PATH the child searches, from the environment Effect's Node spawner gives it. +const childPath = ({ env, extendEnv }: ChildProcess.CommandOptions) => { + if (env === undefined) return process.env.PATH; + if (!Object.hasOwn(env, "PATH")) return extendEnv === true ? process.env.PATH : undefined; + const value = env.PATH; + return value === undefined || !Redacted.isRedacted(value) ? value : Redacted.value(value); +}; + +// The spawn error Effect's Node spawner reports when execvp fails with `code`. +const spawnFailure = (command: ChildProcess.StandardCommand, code: "ENOENT" | "EACCES") => { + const syscall = `spawn ${command.command}`; + return PlatformError.systemError({ + _tag: code === "ENOENT" ? "NotFound" : "PermissionDenied", + module: "ChildProcess", + method: "spawn", + pathOrDescriptor: CommandLine.display([command.command, ...command.args], { format: "posix" }), + syscall, + cause: Object.assign(new Error(`${syscall} ${code}`), { + errno: code === "ENOENT" ? -2 : -13, + code, + syscall, + path: command.command, + spawnargs: command.args.map((arg) => (Redacted.isRedacted(arg) ? "" : arg)), + }), + }); +}; + +// The error of the `FileSystem.access(cwd)` Effect's Node spawner runs before it spawns. +const cwdFailure = (cwd: string, cause: NodeJS.ErrnoException) => + PlatformError.systemError({ + _tag: + cause.code === "ENOENT" + ? "NotFound" + : cause.code === "EACCES" || cause.code === "EPERM" + ? "PermissionDenied" + : cause.code === "ENOTDIR" || cause.code === "ELOOP" + ? "BadResource" + : "Unknown", + module: "FileSystem", + method: "access", + pathOrDescriptor: cwd, + syscall: cause.syscall, + cause, + }); + +/** + * Fails as a native spawn does when the command is missing or not executable: callers read + * that error to tell a CLI that is not installed from one that failed. execvp reports EACCES + * when a candidate exists but cannot be run, and ENOENT otherwise. + */ +const findExecutable = Effect.fnUntraced(function* ( + command: ChildProcess.StandardCommand, + cwd: string | undefined, +) { + // Joined as text: the kernel resolves `..` after a symlink differently from a normalized path. + const fromCwd = (path: string) => + cwd === undefined || path.startsWith("/") ? path : `${cwd}/${path}`; + const candidates = command.command.includes("/") + ? [fromCwd(command.command)] + : (childPath(command.options) ?? DEFAULT_PATH) + .split(":") + .map((entry) => `${fromCwd(entry || ".")}/${command.command}`); + let denied = false; + for (const candidate of candidates) { + const error = yield* check( + candidate.slice(0, candidate.lastIndexOf("/")) || "/", + candidate, + NodeFS.constants.X_OK, + ); + // Any other errno is left for `env` to report through its exit code. + if (error === undefined || (error.code !== "ENOENT" && error.code !== "EACCES")) return; + denied ||= error.code === "EACCES"; + } + return yield* spawnFailure(command, denied ? "EACCES" : "ENOENT"); +}); + +const make = Effect.gen(function* () { + const native = yield* ChildProcessSpawner.ChildProcessSpawner; + if ((yield* HostProcess.Platform) !== "linux") return native; + if (!(yield* Effect.promise(supportsEnvChdir))) return native; + const path = yield* Path.Path; + + // Runs inside the NonBlockingSpawner.spawn span. + const prepareStandard = Effect.fnUntraced(function* (input: ChildProcess.StandardCommand) { + const { cwd: givenCwd, shell, ...options } = input.options; + // Effect rejects a shell command with arguments, and a NUL, itself. + if ((shell && input.args.length > 0) || input.command.includes("\0")) return input; + // Node's shell mode on Unix, spelled out so the trampoline can run it. + const command = shell + ? ChildProcess.make(typeof shell === "string" ? shell : "/bin/sh", ["-c", input.command]) + : input; + // Checked as given and entered resolved, as Effect's Node spawner does. + const cwd = givenCwd === undefined ? undefined : path.resolve(givenCwd); + const trampoline = envTrampoline(command.command, command.args, cwd); + if (trampoline === undefined) return input; + if (givenCwd !== undefined && cwd !== undefined) { + const error = yield* check(cwd, givenCwd, NodeFS.constants.F_OK); + if (error !== undefined) return yield* cwdFailure(givenCwd, error); + } + yield* findExecutable(ChildProcess.make(command.command, command.args, options), cwd); + return ChildProcess.make(trampoline.command, trampoline.args, options); + }); + + const prepare = ( + command: ChildProcess.Command, + ): Effect.Effect => + command._tag === "StandardCommand" + ? prepareStandard(command) + : Effect.map(Effect.all([prepare(command.left), prepare(command.right)]), ([left, right]) => + ChildProcess.pipeTo(left, right, command.options), + ); + + return ChildProcessSpawner.make( + Effect.fn("NonBlockingSpawner.spawn")(function* (command) { + return yield* native.spawn(yield* prepare(command)); + }), + ); +}); + +/** + * On Linux with GNU `env`, replaces the provided `ChildProcessSpawner` with one that never + * hands libuv a cwd or a PATH search; elsewhere it passes the provided one through. + */ +export const layer = Layer.effect(ChildProcessSpawner.ChildProcessSpawner, make); diff --git a/apps/server/src/process/SpawnWorker.test.ts b/apps/server/src/process/SpawnWorker.test.ts new file mode 100644 index 0000000000000..27d31aa69c179 --- /dev/null +++ b/apps/server/src/process/SpawnWorker.test.ts @@ -0,0 +1,172 @@ +// @effect-diagnostics nodeBuiltinImport:off processEnvInEffect:off - fixtures need real paths, the process resource list and the live environment. +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { describe, expect, it } from "@effect/vitest"; +import { vi } from "vite-plus/test"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Result from "effect/Result"; +import * as Stream from "effect/Stream"; +import * as ChildProcess from "effect/process/ChildProcess"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; +import * as HostProcess from "@t3tools/shared/HostProcess"; + +import * as SpawnWorker from "./SpawnWorker.ts"; + +const envChdir = vi.hoisted(() => ({ supported: true })); +vi.mock("@t3tools/shared/envTrampoline", async (importOriginal) => ({ + ...(await importOriginal()), + supportsEnvChdir: () => Promise.resolve(envChdir.supported), +})); + +const layer = SpawnWorker.layer.pipe(Layer.provide(NodeServices.layer)); +const text = (stream: Stream.Stream) => + stream.pipe(Stream.decodeText(), Stream.mkString); +const node = (script: string, args: ReadonlyArray = [], options = {}) => + ChildProcess.make(process.execPath, ["-e", script, ...args], options); + +describe("SpawnWorker", () => { + it.live("runs children on a launcher thread where the server thread would wait on them", () => + Effect.gen(function* () { + for (const [platform, hasEnvChdir, onServerThread] of [ + ["win32", true, false], + ["linux", true, true], + ["linux", false, false], + ] as const) { + envChdir.supported = hasEnvChdir; + const launched = Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* spawner.spawn( + node("process.stdout.write('ready');process.stdin.resume()"), + ); + expect(yield* child.stdout.pipe(Stream.take(1), text)).toBe("ready"); + // The thread that called spawn owns the native process handle. + expect(process.getActiveResourcesInfo().includes("ProcessWrap")).toBe(onServerThread); + expect(yield* child.isRunning).toBe(true); + }); + yield* launched.pipe( + Effect.scoped, + Effect.provide(SpawnWorker.layerServer.pipe(Layer.provide(NodeServices.layer))), + Effect.provideService(HostProcess.Platform, platform), + ); + } + }), + ); + + it.live("preserves stdin, large output, env, cwd, literal args and additional descriptors", () => + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const literal = "quote' \" $HOME; * & %PATH%"; + const cwd = NodeOS.tmpdir(); + const script = ` + const fs = require('node:fs'); + const input = fs.readFileSync(0, 'utf8'); + fs.writeSync(1, JSON.stringify([input, process.cwd(), process.env.SPAWN_WORKER_TEST, process.argv[1]]) + '\\n'); + fs.writeSync(1, 'a'.repeat(1024 * 1024)); + fs.writeSync(2, 'b'.repeat(1024 * 1024)); + `; + for (let i = 0; i < 3; i++) { + const child = yield* spawner.spawn( + node(script, [literal], { + cwd, + env: { SPAWN_WORKER_TEST: "inherited value" }, + extendEnv: true, + stdin: Stream.encodeText(Stream.make("stdin value")), + }), + ); + const [stdout, stderr, code] = yield* Effect.all( + [text(child.stdout), text(child.stderr), child.exitCode], + { concurrency: "unbounded" }, + ); + expect(stdout).toBe( + JSON.stringify(["stdin value", NodePath.resolve(cwd), "inherited value", literal]) + + "\n" + + "a".repeat(1024 * 1024), + ); + expect(stderr).toBe("b".repeat(1024 * 1024)); + expect(code).toBe(0); + } + // The launcher threads already exist, so this needs the server's environment at launch time. + process.env.SPAWN_WORKER_LATE = "set after start"; + const late = yield* spawner.spawn( + node("process.stdout.write(process.env.SPAWN_WORKER_LATE)"), + ); + expect(yield* text(late.stdout)).toBe("set after start"); + delete process.env.SPAWN_WORKER_LATE; + const extra = yield* spawner.spawn( + node("require('node:fs').writeSync(3, 'fd3 value')", [], { + additionalFds: { fd3: { type: "output" } }, + }), + ); + const [fd3, extraCode] = yield* Effect.all([text(extra.getOutputFd(3)), extra.exitCode], { + concurrency: "unbounded", + }); + expect(fd3).toBe("fd3 value"); + expect(extraCode).toBe(0); + const reader = yield* spawner.spawn( + node("process.stdin.on('end', () => process.stdout.write('eof')).resume()"), + ); + yield* Stream.fail("abandoned input").pipe(Stream.run(reader.stdin), Effect.ignore); + expect(yield* text(reader.stdout)).toBe("eof"); + const pipeline = node("process.stdout.write('piped')").pipe( + ChildProcess.pipeTo( + node("process.stdin.on('data', (d) => process.stdout.write(String(d).toUpperCase()))"), + ), + ); + const piped = yield* spawner.spawn(pipeline); + expect(yield* text(piped.stdout)).toBe("PIPED"); + expect(yield* piped.exitCode).toBe(0); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("reports launch errors and exit codes, and keeps launching after either", () => + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const missing = yield* spawner + .spawn(ChildProcess.make("t3-missing-command-25245")) + .pipe(Effect.result); + expect(Result.isFailure(missing)).toBe(true); + if (Result.isFailure(missing)) expect(missing.failure.reason._tag).toBe("NotFound"); + const failed = yield* spawner.spawn(node("process.exit(23)")); + expect(yield* failed.exitCode).toBe(23); + const healthy = yield* spawner.spawn(node("process.stdout.write('ok')")); + expect(yield* text(healthy.stdout)).toBe("ok"); + expect(yield* healthy.exitCode).toBe(0); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("kills running commands, detached or not, and on scope interruption", () => + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + for (const detached of [true, false]) { + const child = yield* spawner.spawn( + node("process.stdout.write(String(process.pid));process.stdin.resume()", [], { + detached, + }), + ); + const pid = yield* child.stdout.pipe(Stream.take(1), text); + expect(Number(pid)).toBe(child.pid); + yield* child.kill(); + expect(yield* child.isRunning).toBe(false); + } + const started = yield* Deferred.make(); + const running = yield* Effect.gen(function* () { + const scoped = yield* spawner.spawn(node("process.stdin.resume()")); + yield* Deferred.succeed(started, scoped); + return yield* Effect.never; + }).pipe(Effect.scoped, Effect.forkChild); + const scoped = yield* Deferred.await(started); + yield* Fiber.interrupt(running); + expect(yield* scoped.isRunning).toBe(false); + // Direct callers such as the Claude adapter pass an AbortSignal, which cannot cross to a worker. + const direct = SpawnWorker.spawnPiped(process.execPath, ["-e", "process.stdin.resume()"], { + signal: AbortSignal.abort(), + }); + const exited = new Promise((resolve) => direct.once("exit", () => resolve(direct.killed))); + expect(yield* Effect.promise(() => exited)).toBe(true); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); +}); diff --git a/apps/server/src/process/SpawnWorker.ts b/apps/server/src/process/SpawnWorker.ts new file mode 100644 index 0000000000000..a43868b6d7b41 --- /dev/null +++ b/apps/server/src/process/SpawnWorker.ts @@ -0,0 +1,541 @@ +// @effect-diagnostics nodeBuiltinImport:off - worker threads and Node streams implement the platform spawner boundary. +import * as NodeEvents from "node:events"; +import type * as NodeChildProcess from "node:child_process"; +import * as NodeStream from "node:stream"; +import * as NodeWorkerThreads from "node:worker_threads"; +import * as NodeChildProcessSpawner from "@effect/platform-node-shared/NodeChildProcessSpawner"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import { supportsEnvChdir } from "@t3tools/shared/envTrampoline"; +import * as HostProcess from "@t3tools/shared/HostProcess"; + +// Windows runs CreateProcess synchronously on the calling thread, and antivirus or a busy disk can +// hold it for seconds; Linux's spawn waits for chdir and the PATH search, which a hung mount +// stalls. Workers own the children, so only the worker waits. +const MAX_WORKERS = 4; +const MAX_IDLE_WORKERS = 1; + +type ToWorker = + | { + readonly type: "spawn"; + readonly id: number; + readonly command: string; + readonly args: ReadonlyArray; + readonly options: NodeChildProcess.SpawnOptions; + } + | { readonly type: "write"; readonly id: number; readonly fd: number; readonly data: Uint8Array } + | { readonly type: "end"; readonly id: number; readonly fd: number } + | { readonly type: "flow"; readonly id: number; readonly fd: number; readonly paused: boolean } + | { readonly type: "destroy"; readonly id: number; readonly fd: number } + | { readonly type: "kill"; readonly id: number; readonly signal: NodeJS.Signals | number }; + +interface SerializedError { + readonly message: string; + readonly [key: string]: unknown; +} + +type FromWorker = + | { readonly type: "spawn"; readonly id: number; readonly pid: number | undefined } + | { readonly type: "error"; readonly id: number; readonly error: SerializedError } + | { readonly type: "data"; readonly id: number; readonly fd: number; readonly data: Uint8Array } + | { + readonly type: "written"; + readonly id: number; + readonly fd: number; + readonly error?: SerializedError; + } + | { readonly type: "eof"; readonly id: number; readonly fd: number } + | { readonly type: "finished"; readonly id: number; readonly fd: number } + | { + readonly type: "stream-error"; + readonly id: number; + readonly fd: number; + readonly error: SerializedError; + } + | { + readonly type: "exit"; + readonly id: number; + readonly code: number | null; + readonly signal: NodeJS.Signals | null; + } + | { readonly type: "close"; readonly id: number }; + +// Inline source runs from bundles, asar archives and single-executable builds without module files. +const workerSource = ` +const { parentPort } = require("node:worker_threads"); +const { spawn } = require("node:child_process"); +const children = new Map(); +const post = (message, transfer) => parentPort.postMessage(message, transfer); +const serialize = (error) => { + const out = { message: String(error && error.message) }; + for (const key of ["code", "errno", "syscall", "path", "spawnargs"]) { + if (error && error[key] !== undefined) out[key] = error[key]; + } + return out; +}; +parentPort.on("message", (request) => { + const { id } = request; + if (request.type === "spawn") { + let child; + try { + child = spawn(request.command, request.args, request.options); + } catch (error) { + post({ type: "error", id, error: serialize(error) }); + return; + } + children.set(id, child); + child.on("spawn", () => post({ type: "spawn", id, pid: child.pid })); + child.on("error", (error) => post({ type: "error", id, error: serialize(error) })); + child.on("exit", (code, signal) => post({ type: "exit", id, code, signal })); + // Output can still arrive after exit; close follows the last of it. + child.on("close", () => { + children.delete(id); + post({ type: "close", id }); + }); + child.stdio?.forEach((stream, fd) => { + if (!stream) return; + stream.on("error", (error) => post({ type: "stream-error", id, fd, error: serialize(error) })); + if (fd === 0) return; + // Cloning a Buffer copies its whole backing store; copy the bytes once and transfer them. + stream.on("data", (chunk) => { + const data = new Uint8Array(chunk); + post({ type: "data", id, fd, data }, [data.buffer]); + }); + stream.on("end", () => post({ type: "eof", id, fd })); + }); + return; + } + const child = children.get(id); + if (child === undefined) { + if (request.type === "write") post({ type: "written", id, fd: request.fd, error: { message: "write after exit", code: "EPIPE" } }); + if (request.type === "end") post({ type: "finished", id, fd: request.fd }); + return; + } + const stream = child.stdio?.[request.fd]; + switch (request.type) { + case "write": + if (!stream || stream.destroyed) { + post({ type: "written", id, fd: request.fd, error: { message: "write after end", code: "EPIPE" } }); + return; + } + try { + stream.write(request.data, (error) => post({ type: "written", id, fd: request.fd, error: error ? serialize(error) : undefined })); + } catch (error) { + post({ type: "written", id, fd: request.fd, error: serialize(error) }); + } + return; + case "end": + if (!stream || stream.destroyed || stream.writableFinished) post({ type: "finished", id, fd: request.fd }); + else stream.end(() => post({ type: "finished", id, fd: request.fd })); + return; + case "flow": + if (request.paused) stream?.pause(); + else stream?.resume(); + return; + case "destroy": + stream?.destroy(); + return; + case "kill": + try { + child.kill(request.signal); + } catch {} + return; + } +}); +`; + +const toError = (serialized: SerializedError) => + Object.assign(new Error(serialized.message), serialized); + +const toBuffer = (data: Uint8Array) => Buffer.from(data.buffer, data.byteOffset, data.byteLength); + +class WorkerChild + extends NodeEvents.EventEmitter + implements NodeChildProcessSpawner.SpawnedChildProcess +{ + pid: number | undefined; + exitCode: number | null = null; + signalCode: NodeJS.Signals | null = null; + stdin: NodeStream.Writable | null = null; + stdout: NodeStream.Readable | null = null; + stderr: NodeStream.Readable | null = null; + readonly stdio: Array = []; + referenced = true; + killed = false; + spawned = false; + exited = false; + readonly #writes = new Map void>>(); + readonly #finals = new Map void>(); + readonly #paused = new Set(); + + readonly worker: SpawnWorker; + readonly id: number; + + constructor(worker: SpawnWorker, id: number, stdio: ReadonlyArray) { + super(); + this.worker = worker; + this.id = id; + stdio.forEach((option, fd) => { + if (option !== "pipe" && option !== "overlapped") return; + const stream = this.#relay(fd); + this.stdio[fd] = stream; + if (fd === 0) this.stdin = stream as NodeStream.Writable; + else if (fd === 1) this.stdout = stream as NodeStream.Readable; + else if (fd === 2) this.stderr = stream as NodeStream.Readable; + }); + } + + #relay(fd: number): NodeStream.Readable | NodeStream.Writable { + const read = () => { + if (this.#paused.delete(fd)) + this.worker.post({ type: "flow", id: this.id, fd, paused: false }); + }; + const write = ( + chunk: Buffer, + _encoding: BufferEncoding, + callback: (error?: Error | null) => void, + ) => { + if (this.worker.closed) { + callback(Object.assign(new Error("write EPIPE"), { code: "EPIPE" })); + return; + } + const pending = this.#writes.get(fd) ?? []; + pending.push(callback); + this.#writes.set(fd, pending); + const data = new Uint8Array(chunk); + this.worker.post({ type: "write", id: this.id, fd, data }, [data.buffer]); + }; + const final = (callback: (error?: Error | null) => void) => { + this.#finals.set(fd, callback); + this.worker.post({ type: "end", id: this.id, fd }); + }; + // A caller that destroys its end closes the child pipe, as it would with a real pipe. + const destroy = (error: Error | null, callback: (error?: Error | null) => void) => { + if (!this.exited) this.worker.post({ type: "destroy", id: this.id, fd }); + callback(error); + }; + if (fd === 0) return new NodeStream.Writable({ write, final, destroy }); + if (fd <= 2) return new NodeStream.Readable({ read, destroy }); + return new NodeStream.Duplex({ read, write, final, destroy, allowHalfOpen: true }); + } + + onData(fd: number, data: Uint8Array): void { + const stream = this.stdio[fd]; + if (!stream || !("push" in stream)) return; + if (!stream.push(toBuffer(data)) && !this.#paused.has(fd)) { + this.#paused.add(fd); + this.worker.post({ type: "flow", id: this.id, fd, paused: true }); + } + } + + onEof(fd: number): void { + const stream = this.stdio[fd]; + if (stream && "push" in stream) stream.push(null); + } + + onFinished(fd: number): void { + const callback = this.#finals.get(fd); + this.#finals.delete(fd); + callback?.(); + } + + onWritten(fd: number, error: SerializedError | undefined): void { + const callback = this.#writes.get(fd)?.shift(); + callback?.(error === undefined ? null : toError(error)); + } + + onStreamError(fd: number, error: SerializedError): void { + this.stdio[fd]?.destroy(toError(error)); + } + + kill(signal: NodeJS.Signals | number = "SIGTERM"): boolean { + if (this.exited) return false; + this.killed = true; + this.worker.post({ type: "kill", id: this.id, signal }); + return true; + } + + ref(): void { + this.referenced = true; + this.worker.syncRef(); + } + + unref(): void { + this.referenced = false; + this.worker.syncRef(); + } + + exit(code: number | null, signal: NodeJS.Signals | null): void { + if (this.exited) return; + this.exited = true; + this.exitCode = code; + this.signalCode = signal; + // Node destroys stdin on exit; the Effect sink may already have removed its error listener. + this.stdin?.on("error", () => {}); + this.stdin?.destroy(); + this.emit("exit", code, signal); + } + + /** The worker is gone: end output so readers finish, and let the exit report the kill. */ + abandon(): void { + // Not destroy(error): Effect leaves an unread stderr's pipe without an error listener. + for (const stream of this.stdio) { + if (stream && "push" in stream && !stream.readableEnded) stream.push(null); + else if (stream && !("push" in stream)) stream.destroy(); + } + } +} + +class SpawnWorker { + readonly worker: NodeWorkerThreads.Worker; + readonly children = new Map(); + readonly pool: SpawnWorkerPool; + /** Launches sent but not yet answered; the worker is inside CreateProcess for one of them. */ + launching = 0; + closed = false; + + constructor(pool: SpawnWorkerPool) { + this.pool = pool; + this.worker = new NodeWorkerThreads.Worker(workerSource, { eval: true }); + this.worker.on("message", (message: FromWorker) => this.onMessage(message)); + this.worker.on("error", (error) => this.fail(error)); + this.worker.on("exit", () => this.fail(new Error("The process launcher thread exited."))); + this.worker.unref(); + } + + post(message: ToWorker, transfer?: ReadonlyArray): void { + if (!this.closed) this.worker.postMessage(message, transfer); + } + + spawn( + id: number, + command: string, + args: ReadonlyArray, + options: NodeChildProcess.SpawnOptions, + ): WorkerChild { + const io = options.stdio; + const stdio = + typeof io === "string" || io === undefined ? [io ?? "pipe", io ?? "pipe", io ?? "pipe"] : io; + const child = new WorkerChild(this, id, stdio); + // An AbortSignal cannot cross to the worker, so this thread kills on abort as Node would. + const { signal, ...rest } = options; + if (signal !== undefined) { + const abort = () => child.kill(); + if (signal.aborted) queueMicrotask(abort); + else { + signal.addEventListener("abort", abort, { once: true }); + child.once("exit", () => signal.removeEventListener("abort", abort)); + } + } + this.children.set(id, child); + this.launching++; + this.syncRef(); + this.post({ + type: "spawn", + id, + command, + args: [...args], + // A worker's process.env is a copy from when it started; children get the server's current one. + options: { ...rest, stdio, env: rest.env ?? { ...process.env } }, + }); + return child; + } + + syncRef(): void { + if (this.closed) return; + const referenced = + this.launching > 0 || [...this.children.values()].some((child) => child.referenced); + if (referenced) this.worker.ref(); + else this.worker.unref(); + } + + onMessage(message: FromWorker): void { + const child = this.children.get(message.id); + if (child === undefined) return; + switch (message.type) { + case "spawn": + this.launching--; + child.spawned = true; + child.pid = message.pid; + child.emit("spawn"); + break; + case "error": + if (!child.spawned) { + this.launching--; + child.exited = true; + this.release(child); + } + child.emit("error", toError(message.error)); + break; + case "data": + child.onData(message.fd, message.data); + break; + case "written": + child.onWritten(message.fd, message.error); + break; + case "eof": + child.onEof(message.fd); + break; + case "finished": + child.onFinished(message.fd); + break; + case "stream-error": + child.onStreamError(message.fd, message.error); + break; + case "exit": + child.exit(message.code, message.signal); + break; + case "close": + child.emit("close", child.exitCode, child.signalCode); + this.release(child); + break; + } + } + + release(child: WorkerChild): void { + this.children.delete(child.id); + this.syncRef(); + this.pool.release(this); + } + + fail(error: Error): void { + if (this.closed) return; + this.close(); + for (const child of this.children.values()) { + if (child.exited) { + child.abandon(); + continue; + } + if (!child.spawned) { + child.exited = true; + child.emit("error", error); + continue; + } + // The worker owned the only handle, so try to end the whole tree before reporting the exit. + const finish = () => { + child.exit(null, "SIGKILL"); + child.abandon(); + }; + if (child.pid === undefined) finish(); + else this.pool.killTree(child.pid, finish); + } + this.children.clear(); + } + + close(): void { + if (this.closed) return; + this.closed = true; + this.pool.workers.delete(this); + void this.worker.terminate(); + } +} + +class SpawnWorkerPool { + readonly workers = new Set(); + readonly platform: NodeJS.Platform; + #nextId = 0; + + constructor(platform: NodeJS.Platform) { + this.platform = platform; + } + + readonly spawn = ( + command: string, + args: ReadonlyArray, + options: NodeChildProcess.SpawnOptions, + ) => { + const workers = [...this.workers]; + // A worker inside CreateProcess cannot start another launch until it returns. + const worker = + workers.find((candidate) => candidate.launching === 0) ?? + (workers.length < MAX_WORKERS + ? this.#add() + : workers.reduce((best, candidate) => + candidate.launching < best.launching ? candidate : best, + )); + return worker.spawn(++this.#nextId, command, args, options); + }; + + killTree(pid: number, done: () => void): void { + const killPid = () => { + try { + process.kill(pid, "SIGKILL"); + } catch {} + }; + if (this.platform !== "win32") { + killPid(); + done(); + return; + } + let settled = false; + const finish = (killed: boolean) => { + if (settled) return; + settled = true; + // As Effect's own kill does when taskkill fails, end at least the direct child. + if (!killed) killPid(); + done(); + }; + this.spawn("taskkill", ["/pid", String(pid), "/T", "/F"], { + stdio: "ignore", + windowsHide: true, + }) + .on("error", () => finish(false)) + .on("exit", (code) => finish(code === 0)); + } + + #add(): SpawnWorker { + const worker = new SpawnWorker(this); + this.workers.add(worker); + return worker; + } + + release(worker: SpawnWorker): void { + if (worker.children.size > 0 || worker.launching > 0) return; + const idle = [...this.workers].filter( + (candidate) => candidate.children.size === 0 && candidate.launching === 0, + ); + if (idle.length > MAX_IDLE_WORKERS) worker.close(); + } + + close(): void { + for (const worker of this.workers) worker.close(); + } +} + +let sharedPool: SpawnWorkerPool | undefined; + +/** A piped `child_process.spawn` for Windows launches outside Effect's spawner. */ +export const spawnPiped = ( + command: string, + args: ReadonlyArray, + options: Omit, +) => + (sharedPool ??= new SpawnWorkerPool("win32")).spawn(command, args, { + ...options, + stdio: ["pipe", "pipe", "pipe"], + }) as WorkerChild & { + readonly stdin: NodeStream.Writable; + readonly stdout: NodeStream.Readable; + readonly stderr: NodeStream.Readable; + }; + +export const layer = Layer.unwrap( + Effect.gen(function* () { + const platform = yield* HostProcess.Platform; + const pool = yield* Effect.acquireRelease( + Effect.sync(() => new SpawnWorkerPool(platform)), + (pool) => Effect.sync(() => pool.close()), + ); + return NodeChildProcessSpawner.layerWithSpawn(pool.spawn); + }), +); + +/** Workers on Windows, and on Linux where `NonBlockingSpawner` has no GNU `env -C` to use. */ +export const layerServer = Layer.unwrap( + Effect.gen(function* () { + const platform = yield* HostProcess.Platform; + if (platform === "win32") return layer; + if (platform === "linux" && !(yield* Effect.promise(supportsEnvChdir))) return layer; + return NodeChildProcessSpawner.layer; + }), +); diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index 0095146a3021f..49674354b2532 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -12,12 +12,21 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Logger from "effect/Logger"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; +import { vi } from "vite-plus/test"; import * as ServerConfig from "../config.ts"; import { makeCodexAppServerSpawnCommand } from "../orchestration-v2/Adapters/CodexAdapterV2.ts"; import * as ProcessRunner from "../processRunner.ts"; +import * as ProviderProcesses from "./providerProcesses.ts"; import { classifyScope, make, ownedAgentSlice, type ScopeState } from "./agentScope.ts"; +const GiB = 1024 ** 3; +const memory = vi.hoisted(() => ({ total: 16 * 1024 ** 3 })); +vi.mock("node:os", async (importOriginal) => ({ + ...(await importOriginal()), + totalmem: () => memory.total, +})); + const running: ScopeState = { loadState: "loaded", activeState: "active", @@ -48,6 +57,7 @@ const withFakeSystemd = ( readonly scope: Effect.Success; readonly units: Map; readonly calls: ReadonlyArray>; + readonly failingVerbs: Set; readonly stateDir: string; readonly cgroupRoot: string; }) => Effect.Effect, @@ -55,7 +65,14 @@ const withFakeSystemd = ( { oomPolicy = true, stopFails = false, - }: { readonly oomPolicy?: boolean; readonly stopFails?: boolean } = {}, + leftovers = false, + liveOwner = false, + }: { + readonly oomPolicy?: boolean; + readonly stopFails?: boolean; + readonly leftovers?: boolean; + readonly liveOwner?: boolean; + } = {}, ) => Effect.gen(function* () { const bin = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-agent-scope-")); @@ -65,6 +82,7 @@ const withFakeSystemd = ( const cgroupRoot = NodePath.join(bin, "cgroup"); const units = new Map(); const calls: Array> = []; + const failingVerbs = new Set(); const output = (stdout: string, code = 0): ProcessRunner.ProcessRunOutput => ({ stdout, stderr: "", @@ -83,6 +101,7 @@ const withFakeSystemd = ( return output("", 1); } const [, verb, unit = ""] = args; + if (failingVerbs.has(verb!)) return output("", 1); if (stopFails && verb === "stop") { return { ...output(""), @@ -102,7 +121,12 @@ const withFakeSystemd = ( }), }); const fileSystem = yield* FileSystem.FileSystem; + const { stateDir } = yield* ServerConfig.ServerConfig; + if (leftovers) { + units.set(ownedAgentSlice(stateDir), { activeState: "active", result: "success" }); + } const scope = yield* make.pipe( + Effect.provideService(ProviderProcesses.ProviderProcesses, { hasOtherLiveOwner: liveOwner }), Effect.provideService(ProcessRunner.ProcessRunner, runner), Effect.provideService(HostProcess.Platform, "linux"), Effect.provideService(HostProcess.Environment, { PATH: bin, XDG_RUNTIME_DIR: "/run/user/1" }), @@ -114,8 +138,7 @@ const withFakeSystemd = ( fileSystem.writeFileString(path.replace("/sys/fs/cgroup", cgroupRoot), data, options), }), ); - const { stateDir } = yield* ServerConfig.ServerConfig; - return yield* body({ scope, units, calls, stateDir, cgroupRoot }).pipe( + return yield* body({ scope, units, calls, failingVerbs, stateDir, cgroupRoot }).pipe( Effect.ensuring(Effect.sync(() => NodeFS.rmSync(bin, { recursive: true, force: true }))), ); }).pipe( @@ -308,27 +331,68 @@ describe("AgentScope service", () => { ), ); - it.effect("stops agents a killed server left before the first launch", () => - withFakeSystemd(({ scope, units, calls, stateDir }) => + it.effect("follows a live memory change at the next launch", () => { + memory.total = 16 * GiB; + return withFakeSystemd(({ scope, calls, failingVerbs }) => Effect.gen(function* () { - units.set(ownedAgentSlice(stateDir), { activeState: "active", result: "success" }); - const agent = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" }); - const stops = calls.filter(([, verb]) => verb === "stop"); - expect(stops).toEqual([["--user", "stop", ownedAgentSlice(stateDir)]]); - expect(sliceOf(agent)).toBe(ownedAgentSlice(stateDir)); - expect(agent.args).toContain("--property=TimeoutStopSec=10s"); - const terminal = yield* scope.wrap({ - command: "agent", - args: [], - name: "terminal", - survivesRestart: true, - }); - expect(sliceOf(terminal)).toBe("app-t3code-agents.slice"); - expect(calls.filter(([, verb]) => verb === "stop")).toHaveLength(1); + const launch = scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" }); + const memoryLimits = () => + calls + .filter( + ([, verb, , slice]) => verb === "set-property" && slice === "app-t3code-agents.slice", + ) + .map((args) => args.slice(4)); + yield* launch; + yield* launch; + expect(memoryLimits()).toEqual([[`MemoryHigh=${10 * GiB}`, `MemoryMax=${12 * GiB}`]]); + + // A VM balloon or `pct set --memory` grows RAM while the server keeps running. + memory.total = 22 * GiB; + failingVerbs.add("set-property"); + yield* launch; + yield* launch; + failingVerbs.delete("set-property"); + yield* launch; + yield* launch; + const grown = [`MemoryHigh=${16 * GiB}`, `MemoryMax=${18 * GiB}`]; + expect(memoryLimits()).toEqual([ + [`MemoryHigh=${10 * GiB}`, `MemoryMax=${12 * GiB}`], + grown, + grown, + grown, + ]); }), + ); + }); + + it.effect("stops agents a killed server left before startup completes", () => + withFakeSystemd( + ({ scope, calls, stateDir }) => + Effect.gen(function* () { + expect(stopsOf(calls)).toEqual([["--user", "stop", ownedAgentSlice(stateDir)]]); + const agent = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" }); + expect(sliceOf(agent)).toBe(ownedAgentSlice(stateDir)); + expect(agent.args).toContain("--property=TimeoutStopSec=10s"); + const terminal = yield* scope.wrap({ + command: "agent", + args: [], + name: "terminal", + survivesRestart: true, + }); + expect(sliceOf(terminal)).toBe("app-t3code-agents.slice"); + expect(stopsOf(calls)).toHaveLength(1); + }), + { leftovers: true }, ), ); + it.effect("leaves the slice to another live server sharing this home", () => + withFakeSystemd(({ calls }) => Effect.sync(() => expect(stopsOf(calls)).toEqual([])), { + leftovers: true, + liveOwner: true, + }), + ); + it.effect("stops leftover processes only in an active agent scope", () => withFakeSystemd(({ scope, units, calls, cgroupRoot }) => Effect.gen(function* () { diff --git a/apps/server/src/process/agentScope.ts b/apps/server/src/process/agentScope.ts index b9b6b56acb140..1e52e680b6f8d 100644 --- a/apps/server/src/process/agentScope.ts +++ b/apps/server/src/process/agentScope.ts @@ -14,9 +14,11 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; +import * as Semaphore from "effect/Semaphore"; import * as ServerConfig from "../config.ts"; import * as ProcessRunner from "../processRunner.ts"; +import * as ProviderProcesses from "./providerProcesses.ts"; const AGENT_SLICE = "app-t3code-agents.slice"; // Owns every agent slice and competes with the server's unit or scope under app.slice. @@ -106,6 +108,7 @@ export const make = Effect.gen(function* () { const path = yield* Path.Path; const hostEnvironment = yield* HostProcess.Environment; const { stateDir } = yield* ServerConfig.ServerConfig; + const { hasOtherLiveOwner } = yield* ProviderProcesses.ProviderProcesses; const ownedSlice = ownedAgentSlice(stateDir); const layerScope = yield* Effect.scope; @@ -130,6 +133,7 @@ export const make = Effect.gen(function* () { const stopLeftoverAgents = Effect.fn("agentScope.stopLeftoverAgents")(function* ( systemctl: string, ) { + if (hasOtherLiveOwner) return; const shown = yield* run(systemctl, [ "--user", "show", @@ -190,20 +194,44 @@ export const make = Effect.gen(function* () { slice, ...properties, ]); - if (result._tag === "None" || result.value.code !== 0) { - yield* Effect.logWarning("Could not set resource limits on an agents slice", { - slice, - properties, - stderr: result._tag === "Some" ? result.value.stderr.trim() : undefined, - }); - } + if (result._tag === "Some" && result.value.code === 0) return true; + yield* Effect.logWarning("Could not set resource limits on an agents slice", { + slice, + properties, + stderr: result._tag === "Some" ? result.value.stderr.trim() : undefined, + }); + return false; }); - const limits = agentSliceMemoryLimits(NodeOS.totalmem()); - yield* setProperties(AGENT_SLICE, [`MemoryHigh=${limits.high}`, `MemoryMax=${limits.max}`]); + // VMs and containers can resize RAM live, so each launch re-checks the total + // the limits came from. A failed update is retried by the next launch. + let appliedTotal: number | undefined; + const memoryLimitsLock = yield* Semaphore.make(1); + const applyMemoryLimits = memoryLimitsLock.withPermits(1)( + Effect.gen(function* () { + const total = NodeOS.totalmem(); + if (total === appliedTotal) return; + const limits = agentSliceMemoryLimits(total); + const applied = yield* setProperties(AGENT_SLICE, [ + `MemoryHigh=${limits.high}`, + `MemoryMax=${limits.max}`, + ]); + if (!applied) return; + if (appliedTotal !== undefined) { + yield* Effect.logInfo("Agent memory limits follow a memory change", { + slice: AGENT_SLICE, + previousTotal: appliedTotal, + total, + ...limits, + }); + } + appliedTotal = total; + }), + ); + yield* applyMemoryLimits; yield* setProperties(AGENTS_PARENT_SLICE, AGENTS_PARENT_PROPERTIES); yield* Effect.logInfo("Agent scopes are on", { slice: AGENT_SLICE, - ...limits, + ...agentSliceMemoryLimits(NodeOS.totalmem()), oomPolicy: supportsOomPolicy, }); return { @@ -211,10 +239,15 @@ export const make = Effect.gen(function* () { systemctl, runtimeDir, supportsOomPolicy, + applyMemoryLimits, }; }), ); + // Layer acquisition is the startup barrier: recovery must not terminalize or + // continue old runs while a previous server's providers can still execute. + yield* systemd; + // `shared` scopes serve other threads too: tracked only for OOM diagnosis on old systemd. const scopeByThread = new Map(); @@ -396,6 +429,7 @@ export const make = Effect.gen(function* () { if (resolved === null) return { command, args }; const scope = yield* systemd; if (scope === undefined) return agentScopeCommand({ command: resolved, args }); + yield* scope.applyMemoryLimits; const unit = `t3code-${name}-${NodeCrypto.randomUUID().slice(0, 8)}.scope`; // A continuing shared scope cannot attribute a tool's OOM to one thread. // Clear the launch thread's previous scope instead. @@ -427,4 +461,7 @@ export const make = Effect.gen(function* () { } satisfies AgentScopeShape; }); -export const layer = Layer.effect(AgentScope, make).pipe(Layer.provide(ProcessRunner.layer)); +export const layer = Layer.effect(AgentScope, make).pipe( + Layer.provide(ProcessRunner.layer), + Layer.provideMerge(ProviderProcesses.layer), +); diff --git a/apps/server/src/process/externalLauncher.test.ts b/apps/server/src/process/externalLauncher.test.ts index 0f73ab4fb9c53..f215fe4ebc6a2 100644 --- a/apps/server/src/process/externalLauncher.test.ts +++ b/apps/server/src/process/externalLauncher.test.ts @@ -21,6 +21,7 @@ import { resolveEditorCommand } from "@t3tools/shared/editor"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { SpawnExecutableResolution } from "@t3tools/shared/shell"; import * as ExternalLauncher from "./externalLauncher.ts"; +import { WindowsVerbatimSpawn } from "./windowsVerbatimSpawn.ts"; // Windows drive letters break mocked POSIX PATH splitting; these tests also require /bin/sh stubs. const windowsHost = HostProcess.Platform.defaultValue() === "win32"; @@ -104,36 +105,47 @@ const layerTest = (input: { }; // macOS command lookup cannot search a Windows drive-letter PATH. -it.effect.skipIf(windowsHost)( - "opens home-relative app bundles using the server home directory", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const bin = yield* fs.makeTempDirectoryScoped({ prefix: "t3-app-open-" }); - yield* fs.writeFileString(path.join(bin, "open"), "#!/bin/sh\n"); - yield* fs.chmod(path.join(bin, "open"), 0o755); - const spawned: ChildProcess.StandardCommand[] = []; - yield* Effect.gen(function* () { - const launcher = yield* ExternalLauncher.ExternalLauncher; - for (const cwd of ["~/Applications/My App.app", "/Applications/My App.app"]) { - yield* launcher.launchEditor({ editor: "file-manager", cwd }); - } - }).pipe( - Effect.provide( - layerTest({ - platform: "darwin", - env: { PATH: bin }, - onSpawn: (command) => spawned.push(command), - }), - ), - Effect.provideService(HostProcess.HomeDirectory, "/Users/server"), - ); - assert.deepEqual( - spawned.map((command) => command.args), - [["/Users/server/Applications/My App.app"], ["/Applications/My App.app"]], - ); - }).pipe(Effect.provide(NodeServices.layer)), +it.effect.skipIf(windowsHost)("opens home-relative targets using the server home directory", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const bin = yield* fs.makeTempDirectoryScoped({ prefix: "t3-app-open-" }); + for (const name of ["open", "code", "idea"]) { + yield* fs.writeFileString(path.join(bin, name), "#!/bin/sh\n"); + yield* fs.chmod(path.join(bin, name), 0o755); + } + const spawned: ChildProcess.StandardCommand[] = []; + yield* Effect.gen(function* () { + const launcher = yield* ExternalLauncher.ExternalLauncher; + for (const cwd of ["~/Applications/My App.app", "/Applications/My App.app"]) { + yield* launcher.launchEditor({ editor: "file-manager", cwd }); + } + yield* launcher.launchEditor({ editor: "vscode", cwd: "~/notes/todo.md:3" }); + yield* launcher.launchEditor({ + editor: "idea", + cwd: "~/notes/todo.md:3", + projectRoot: "~/notes", + }); + }).pipe( + Effect.provide( + layerTest({ + platform: "darwin", + env: { PATH: bin }, + onSpawn: (command) => spawned.push(command), + }), + ), + Effect.provideService(HostProcess.HomeDirectory, "/Users/server"), + ); + assert.deepEqual( + spawned.map((command) => command.args), + [ + ["/Users/server/Applications/My App.app"], + ["/Applications/My App.app"], + ["--goto", "/Users/server/notes/todo.md:3"], + ["/Users/server/notes", "--line", "3", "/Users/server/notes/todo.md"], + ], + ); + }).pipe(Effect.provide(NodeServices.layer)), ); it.effect.skipIf(windowsHost)( @@ -194,6 +206,8 @@ it.effect.skipIf(windowsHost)( const codePath = path.join(binDir, "code"); yield* fileSystem.writeFileString(codePath, "#!/bin/sh\n"); yield* fileSystem.chmod(codePath, 0o755); + const colonProject = path.join(project, "build:2026"); + yield* fileSystem.makeDirectory(colonProject); const file = `${path.join(project, "file.ts")}:3:7`; const spawned: ChildProcess.StandardCommand[] = []; @@ -201,6 +215,7 @@ it.effect.skipIf(windowsHost)( const launcher = yield* ExternalLauncher.ExternalLauncher; yield* launcher.launchEditor({ editor: "vscode", cwd: project }); yield* launcher.launchEditor({ editor: "vscode", cwd: file }); + yield* launcher.launchEditor({ editor: "vscode", cwd: colonProject }); }).pipe( Effect.provide( layerTest({ @@ -217,6 +232,7 @@ it.effect.skipIf(windowsHost)( [ ["--new-window", project], ["--goto", file], + ["--new-window", colonProject], ], ); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), @@ -295,6 +311,69 @@ it.effect.skipIf(windowsHost)( }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); +// Windows drive letters cannot resolve the POSIX xdg stubs. +it.effect.skipIf(windowsHost)( + "a Linux server without a display offers and opens the file manager on the desktop session's display", + () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const binDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-editors-" }); + for (const name of ["xdg-open", "xdg-mime"]) { + const filePath = path.join(binDir, name); + yield* fileSystem.writeFileString(filePath, "#!/bin/sh\n"); + yield* fileSystem.chmod(filePath, 0o755); + } + + const fileManager = (sessionEnv: ReadonlyArray) => + Effect.gen(function* () { + const spawned: ChildProcess.StandardCommand[] = []; + const result = yield* Effect.gen(function* () { + const launcher = yield* ExternalLauncher.ExternalLauncher; + const editors = yield* launcher.resolveAvailableEditors(); + const kind = yield* launcher.resolveFileManagerRevealKind(); + const opened = yield* launcher + .launchEditor({ editor: "file-manager", cwd: "/workspace" }) + .pipe(Effect.exit); + return { offered: editors.includes("file-manager"), kind, opened }; + }).pipe( + Effect.provide( + layerTest({ + platform: "linux", + env: { PATH: binDir }, + onSpawn: (command) => spawned.push(command), + spawnResult: (command) => + command.command === "busctl" + ? { stdout: JSON.stringify({ type: "as", data: sessionEnv }) } + : command.command === "xdg-mime" + ? { stdout: "org.kde.dolphin.desktop\n" } + : undefined, + }), + ), + Effect.provideService(HostProcess.Environment, { PATH: binDir }), + ); + const open = spawned.find((command) => command.command === "xdg-open"); + return { ...result, open: open && { args: open.args, env: open.options.env } }; + }); + + const session = yield* fileManager(["DISPLAY=:0", "HOME=/home/user"]); + assert.equal(session.offered, true); + assert.equal(session.kind, "files"); + assert.equal(Exit.isSuccess(session.opened), true); + assert.deepEqual(session.open, { + args: ["/workspace"], + env: { PATH: binDir, DISPLAY: ":0" }, + }); + + // No one is logged in to a desktop, so there is no screen to open it on. + const headless = yield* fileManager(["HOME=/home/user"]); + assert.equal(headless.offered, false); + assert.equal(headless.kind, undefined); + assert.equal(Exit.isFailure(headless.opened), true); + assert.equal(headless.open, undefined); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + it.effect("launches an installed editor with platform-safe arguments", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; @@ -338,6 +417,144 @@ it.effect("launches an installed editor with platform-safe arguments", () => }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); +const wslpathSpawner = (out: string | undefined) => + Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make((command) => + Effect.sync(() => { + if (!ChildProcess.isStandardCommand(command)) { + throw new Error("Expected a standard command"); + } + assert.equal(command.command, "wslpath"); + return makeMockDetachedHandle( + out === undefined ? { exitCode: 1 } : { exitCode: 0, stdout: `${out}\n` }, + ); + }), + ), + ); + +it.effect.each([ + // wslpath is mount-aware: a drive remapped in /etc/wsl.conf keeps its real letter. + { target: "/mnt/d/project", wslpath: "E:\\project", expected: "E:\\project" }, + // wslpath missing or failing falls back to the hand-built UNC path. + { + target: "/home/user/repo", + wslpath: undefined, + expected: "\\\\wsl.localhost\\Ubuntu-24.04\\home\\user\\repo", + }, + { target: "/", wslpath: undefined, expected: "\\\\wsl.localhost\\Ubuntu-24.04" }, +])("resolveWslFileManagerPath resolves $target to $expected", ({ target, wslpath, expected }) => + Effect.gen(function* () { + const result = yield* ExternalLauncher.resolveWslFileManagerPath(target, "Ubuntu-24.04"); + assert.equal(result, expected); + }).pipe(Effect.provide(wslpathSpawner(wslpath))), +); + +it.live.skipIf(!windowsHost)("opens Windows folders through the default shell handler", () => + Effect.gen(function* () { + let spawned: ChildProcess.StandardCommand | undefined; + yield* Effect.gen(function* () { + const launcher = yield* ExternalLauncher.ExternalLauncher; + yield* launcher.launchEditor({ + editor: "file-manager", + cwd: "C:/workspace [draft]/author's ‘left’ ‚low‛ $media/’; Write-Output injected; #", + }); + }).pipe( + Effect.provide( + layerTest({ + platform: "win32", + env: { PATH: "C:\\Windows", PATHEXT: ".EXE" }, + onSpawn: (command) => { + spawned = command; + }, + }), + ), + ); + assert.ok(spawned); + assert.equal(spawned.options.detached, false); + const source = Buffer.from(String(spawned.args.at(-1) ?? ""), "base64").toString("utf16le"); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const output = yield* spawner.string( + ChildProcess.make(spawned.command, [ + ...spawned.args.slice(0, -1), + Buffer.from( + `[Console]::OutputEncoding = [Text.UTF8Encoding]::new($false); function Invoke-Item { param($LiteralPath) Write-Output $LiteralPath }; ${source}`, + "utf16le", + ).toString("base64"), + ]), + ); + assert.equal( + output.trim(), + "C:\\workspace [draft]\\author's ‘left’ ‚low‛ $media\\’; Write-Output injected; #", + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + +for (const [replacement, verb, handler, target, expected] of [ + ["none", "none", "", "C:/repo/file.txt", 'explorer.exe /select,"C:\\repo\\file.txt"'], + ["Directory", "openindopus", "dopusrt.exe", "C:/repo/file.txt", "shell:C:\\repo"], + ["Folder", "openindopus", "dopusrt.exe", "C:/repo/file.txt", "shell:C:\\repo"], + [ + "Folder", + "open", + '"C:\\Windows\\explorer.exe"', + "C:/repo/file.txt", + 'explorer.exe /select,"C:\\repo\\file.txt"', + ], + [ + "Folder", + "open", + "ExplorerDelegate", + "C:/repo/file.txt", + 'explorer.exe /select,"C:\\repo\\file.txt"', + ], + [ + "Drive", + "openindopus", + "dopusrt.exe", + "C:/repo/file.txt", + 'explorer.exe /select,"C:\\repo\\file.txt"', + ], + ["Drive", "openindopus", "dopusrt.exe", "C:/file.txt", "shell:C:\\"], +] as const) { + it.live.skipIf(!windowsHost)( + `reveals ${target} with ${replacement}/${verb} handled by ${handler}`, + () => + Effect.gen(function* () { + let spawned: ChildProcess.StandardCommand | undefined; + yield* Effect.gen(function* () { + const launcher = yield* ExternalLauncher.ExternalLauncher; + yield* launcher.launchEditor({ + editor: "file-manager", + cwd: target, + reveal: true, + }); + }).pipe( + Effect.provide( + layerTest({ + platform: "win32", + env: { PATH: "C:\\Windows", PATHEXT: ".EXE" }, + onSpawn: (command) => { + spawned = command; + }, + }), + ), + ); + assert.ok(spawned); + const source = Buffer.from(String(spawned.args.at(-1) ?? ""), "base64").toString("utf16le"); + const stubs = `function Get-ItemProperty { param($LiteralPath, $ErrorAction) @{ '(default)' = if ($LiteralPath -eq 'Registry::HKEY_CLASSES_ROOT\\${replacement}\\shell') { '${verb}' } elseif ($LiteralPath -eq 'Registry::HKEY_CLASSES_ROOT\\${replacement}\\shell\\${verb}\\command') { '${handler}' } else { 'none' }; DelegateExecute = if ('${handler}' -eq 'ExplorerDelegate') { '{11dbb47c-a525-400b-9e80-a54615a090c0}' } else { '' } } }; function Invoke-Item { param($LiteralPath) Write-Output "shell:$LiteralPath" }; function Start-Process { param($FilePath, $ArgumentList) Write-Output "$FilePath $ArgumentList" }; `; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const output = yield* spawner.string( + ChildProcess.make(spawned.command, [ + ...spawned.args.slice(0, -1), + Buffer.from(stubs + source, "utf16le").toString("base64"), + ]), + ); + assert.equal(output.trim(), expected); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +} + it.effect.skipIf(windowsHost)( "reveals a WSL file in Windows File Explorer through its UNC path", () => @@ -358,7 +575,7 @@ it.effect.skipIf(windowsHost)( const editors = yield* launcher.resolveAvailableEditors(); yield* launcher.launchEditor({ editor: "file-manager", - cwd: "/home/t3/workspace [draft]/author's $media/clip|draft.mp4", + cwd: "/home/t3/workspace [draft]/author's ‘$media/clip|draft.mp4", reveal: true, }); return { kind, editors }; @@ -385,13 +602,85 @@ it.effect.skipIf(windowsHost)( assert.equal(spawned.options.detached, true); const encodedCommand = String(spawned.args[spawned.args.length - 1] ?? ""); const decodedCommand = Buffer.from(encodedCommand, "base64").toString("utf16le"); - assert.equal( + assert.include(decodedCommand, "'Directory', 'Folder'"); + assert.include( decodedCommand, - "$ProgressPreference = 'SilentlyContinue'; $folderVerb = (Get-ItemProperty -LiteralPath 'Registry::HKEY_CLASSES_ROOT\\Directory\\shell' -ErrorAction SilentlyContinue).'(default)'; if ($folderVerb -and $folderVerb -ne 'none') { Invoke-Item -LiteralPath '\\\\wsl.localhost\\Ubuntu-24.04\\home\\t3\\workspace [draft]\\author''s $media' } else { Start-Process 'explorer.exe' -ArgumentList ('/select,\"' + '\\\\wsl.localhost\\Ubuntu-24.04\\home\\t3\\workspace [draft]\\author''s $media\\clip|draft.mp4' + '\"') }", + "Invoke-Item -LiteralPath '\\\\wsl.localhost\\Ubuntu-24.04\\home\\t3\\workspace [draft]\\author''s ‘‘$media'", + ); + assert.include( + decodedCommand, + "'\\\\wsl.localhost\\Ubuntu-24.04\\home\\t3\\workspace [draft]\\author''s ‘‘$media\\clip|draft.mp4'", ); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); +const STOCK_FOLDER_VERB = + "\r\nHKEY_CLASSES_ROOT\\Directory\\shell\r\n (Default) REG_SZ none\r\n\r\n"; + +it.effect.each([ + ["selects the file with Explorer on stock Windows", { stdout: STOCK_FOLDER_VERB }, true], + [ + "keeps the PowerShell reveal for a replacement file manager", + { stdout: " (Default) REG_SZ open\r\n" }, + false, + ], + ["keeps the PowerShell reveal when the registry lookup fails", { exitCode: 1 }, false], +] as const)("%s", ([, registry, native]) => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const binDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-editors-" }); + yield* fileSystem.writeFileString(path.join(binDir, "explorer.EXE"), ""); + const spawned: ChildProcess.StandardCommand[] = []; + const verbatim: Array]> = []; + const exit = yield* Effect.gen(function* () { + const launcher = yield* ExternalLauncher.ExternalLauncher; + return yield* launcher + .launchEditor({ + editor: "file-manager", + cwd: "C:/Users/t3/my project/a,b.ts", + reveal: true, + }) + .pipe(Effect.exit); + }).pipe( + Effect.provideService(WindowsVerbatimSpawn, (command, args) => + Effect.sync(() => verbatim.push([command, args])), + ), + Effect.provide( + layerTest({ + platform: "win32", + env: { PATH: binDir, PATHEXT: ".COM;.EXE;.BAT;.CMD", SYSTEMROOT: "C:\\Windows" }, + onSpawn: (command) => spawned.push(command), + spawnResult: (command) => (command.command.endsWith("reg.exe") ? registry : undefined), + }), + ), + ); + + assert.deepEqual( + spawned.map((command) => command.command), + ["C:\\Windows\\System32\\reg.exe"], + ); + if (native) { + assert.isTrue(Exit.isSuccess(exit)); + assert.deepEqual(verbatim, [ + ["C:\\Windows\\explorer.exe", ['/select,"C:\\Users\\t3\\my project\\a,b.ts"']], + ]); + } else { + assert.deepEqual(verbatim, []); + // The PowerShell launch is reached; this host has no powershell.exe to run it. + assert.isTrue(Exit.isFailure(exit)); + if (Exit.isFailure(exit)) { + const error = Cause.squash(exit.cause); + assert.instanceOf(error, ExternalLauncher.ExternalLauncherCommandNotFoundError); + assert.equal( + (error as ExternalLauncher.ExternalLauncherCommandNotFoundError).command, + "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe", + ); + } + } + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + for (const wsl of [false, true]) { it.effect.skipIf(windowsHost)( "selects a file over D-Bus on " + (wsl ? "WSLg without interop PowerShell" : "Linux"), @@ -614,17 +903,32 @@ it.effect.skipIf(windowsHost)("does not offer the OpenZFS zed daemon as Zed on L ); yield* fs.chmod(zfsZed, 0o755); const editor = EDITORS.find((editor) => editor.id === "zed")!; - const resolve = resolveEditorCommand(editor, { HOME: home, PATH: binDir }).pipe( - Effect.provideService(HostProcess.Platform, "linux"), - Effect.provideService(Path.Path, withSystemBin(path, systemBin)), - ); - assert.deepEqual(yield* resolve, Option.none()); + const laterBin = path.join(home, "later-bin"); + const resolveOn = (platform: NodeJS.Platform) => + resolveEditorCommand(editor, { HOME: home, PATH: `${binDir}:${laterBin}` }).pipe( + Effect.provideService(HostProcess.Platform, platform), + Effect.provideService(Path.Path, withSystemBin(path, systemBin)), + ); + assert.deepEqual(yield* resolveOn("linux"), Option.none()); + assert.deepEqual(yield* resolveOn("darwin"), Option.none()); const zedEditor = path.join(home, ".local/bin/zed"); yield* fs.makeDirectory(path.dirname(zedEditor), { recursive: true }); yield* fs.writeFileString(zedEditor, "#!/bin/sh\necho 'Zed 0.200.0'\n"); yield* fs.chmod(zedEditor, 0o755); - assert.deepEqual(yield* resolve, Option.some({ command: zedEditor, baseArgs: [] })); + assert.deepEqual(yield* resolveOn("linux"), Option.some({ command: zedEditor, baseArgs: [] })); + + yield* fs.remove(zedEditor); + const laterZed = path.join(laterBin, "zed"); + yield* fs.makeDirectory(laterBin, { recursive: true }); + yield* fs.writeFileString(laterZed, "#!/bin/sh\necho 'Zed 0.200.0'\n"); + yield* fs.chmod(laterZed, 0o755); + for (const platform of ["linux", "darwin"] as const) { + assert.deepEqual( + yield* resolveOn(platform), + Option.some({ command: laterZed, baseArgs: [] }), + ); + } }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); @@ -647,7 +951,7 @@ it.effect.skipIf(windowsHost)("does not open Zed with the OpenZFS zed daemon on Effect.provide( layerTest({ platform: "linux", - env: { HOME: home, PATH: binDir }, + env: { HOME: home, PATH: binDir, DISPLAY: ":0" }, path: withSystemBin(path, systemBin), onSpawn: (command) => spawned.push(command), spawnResult: (command) => (command.args[0] === "--version" ? { exitCode: 1 } : undefined), @@ -792,6 +1096,62 @@ it.effect("keeps the macOS VS Code Insiders bundle out of stable VS Code", () => }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); +it.effect.each([ + { + platform: "darwin", + scripts: "Library/Application Support/JetBrains/Toolbox/scripts", + suffix: "", + }, + { platform: "linux", scripts: ".local/share/JetBrains/Toolbox/scripts", suffix: "" }, + { platform: "win32", scripts: "JetBrains/Toolbox/scripts", suffix: ".cmd" }, + { + platform: "win32", + scripts: "JetBrains/Toolbox/scripts", + suffix: ".cmd", + install: "Programs/IntelliJ IDEA/bin/idea64.exe", + }, +] as const)( + "discovers a numbered JetBrains Toolbox script on $platform", + ({ platform, scripts, suffix, ...row }) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "t3-toolbox-numbered-" }); + const dir = path.join(home, scripts); + yield* fs.makeDirectory(dir, { recursive: true }); + for (const name of ["idea10", "idea2"]) { + yield* fs.writeFileString(path.join(dir, `${name}${suffix}`), "#!/bin/sh\n"); + yield* fs.chmod(path.join(dir, `${name}${suffix}`), 0o755); + } + const install = "install" in row ? path.join(home, row.install) : undefined; + if (install) { + yield* fs.makeDirectory(path.dirname(install), { recursive: true }); + yield* fs.writeFileString(install, ""); + } + + const editor = EDITORS.find((editor) => editor.id === "idea")!; + const resolved = yield* resolveEditorCommand(editor, { + HOME: home, + LOCALAPPDATA: home, + PATH: path.join(home, "empty"), + PATHEXT: ".COM;.EXE;.BAT;.CMD", + }).pipe( + Effect.provideService(HostProcess.Platform, platform), + // Keep IDEs installed on the host out of system locations such as /Applications. + Effect.provideService(FileSystem.FileSystem, { + ...fs, + stat: (file) => + fs.stat(file.startsWith(`${home}${path.sep}`) ? file : path.join(home, "missing")), + }), + ); + + assert.deepEqual( + resolved, + Option.some({ command: install ?? path.join(dir, `idea2${suffix}`), baseArgs: [] }), + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + for (const { platform, installPath, onPath } of [ { platform: "darwin", installPath: ".local/bin/agy", onPath: true }, { platform: "linux", installPath: ".local/bin/agy", onPath: false }, @@ -949,6 +1309,57 @@ it.effect( }, ); +it.effect.each(["Zed Nightly", "Zed Preview"])( + "launches stable Zed on Windows when %s comes first on PATH", + (channel) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-zed-channels-" }); + // A user folder naming Zed and a channel does not mark either install. + const programs = path.join(root, "Users", "zedpreview", "Programs"); + const prereleaseBin = path.join(programs, channel, "bin"); + const stableBin = path.join(programs, "Zed", "bin"); + for (const dir of [prereleaseBin, stableBin]) { + yield* fs.makeDirectory(dir, { recursive: true }); + yield* fs.writeFileString(path.join(dir, "zed.EXE"), ""); + } + const env = { PATH: `${prereleaseBin};${stableBin}`, PATHEXT: ".COM;.EXE;.BAT;.CMD" }; + + const launchZed = Effect.gen(function* () { + let spawned: ChildProcess.StandardCommand | undefined; + yield* Effect.gen(function* () { + const launcher = yield* ExternalLauncher.ExternalLauncher; + yield* launcher.launchEditor({ editor: "zed", cwd: "C:\\workspace" }); + }).pipe( + Effect.provide( + layerTest({ + platform: "win32", + env, + onSpawn: (command) => { + spawned = command; + }, + }), + ), + ); + assert.ok(spawned); + return spawned; + }); + + const stable = yield* launchZed; + assert.equal(stable.command, path.join(stableBin, "zed.EXE")); + assert.deepEqual(stable.args, ["C:\\workspace"]); + + env.PATH = `${stableBin};${prereleaseBin}`; + assert.equal((yield* launchZed).command, "zed"); + + env.PATH = `${prereleaseBin};${stableBin}`; + // With only the prerelease install left, it still opens by name. + yield* fs.remove(path.join(stableBin, "zed.EXE")); + assert.equal((yield* launchZed).command, "zed"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + const launchWindowsShimEditor = (cwd: string) => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; diff --git a/apps/server/src/process/externalLauncher.ts b/apps/server/src/process/externalLauncher.ts index aea3e2d1472f9..2bdb7bd3db4db 100644 --- a/apps/server/src/process/externalLauncher.ts +++ b/apps/server/src/process/externalLauncher.ts @@ -34,6 +34,7 @@ import * as ChildProcess from "effect/process/ChildProcess"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import { makeStaleWhileRevalidate } from "../environment/staleWhileRevalidate.ts"; +import { WindowsVerbatimSpawn } from "./windowsVerbatimSpawn.ts"; export { ExternalLauncherError, ExternalLauncherBrowserSpawnError, @@ -148,9 +149,10 @@ function resolveCommandEditorArgs( case "direct-path": return [target]; case "goto": - if (Option.isSome(parsedTarget)) return ["--goto", target]; + // An existing folder wins over a position: `build:2026` is a legal folder name. // Without a window flag, VS Code's window settings may replace the user's last active window. - return isDirectory ? ["--new-window", target] : [target]; + if (isDirectory) return ["--new-window", target]; + return Option.isSome(parsedTarget) ? ["--goto", target] : [target]; case "line-column": return Option.match(parsedTarget, { onNone: () => [target], @@ -181,11 +183,16 @@ function encodeUtf16LeBase64(input: string): string { } function escapePowerShellStringLiteral(input: string): string { - return `'${input.replaceAll("'", "''")}'`; + // PowerShell also recognises typographic apostrophes as single-quote delimiters. + return `'${input.replace(/['\u2018-\u201b]/g, "$&$&")}'`; +} + +function resolveWindowsRoot(env: NodeJS.ProcessEnv): string { + return env.SYSTEMROOT || env.windir || String.raw`C:\Windows`; } function resolvePowerShellPath(env: NodeJS.ProcessEnv = {}): string { - return `${env.SYSTEMROOT || env.windir || String.raw`C:\Windows`}\\System32\\WindowsPowerShell\\v1.0\\powershell.exe`; + return `${resolveWindowsRoot(env)}\\System32\\WindowsPowerShell\\v1.0\\powershell.exe`; } function resolveWslPowerShellPath(): string { @@ -364,11 +371,41 @@ const fileManagerRevealKindForPlatform = Effect.fn( return hasGraphicalLinuxSession(env) ? "files" : undefined; }); -function resolveWslFileManagerPath(target: string, distroName: string): string { +function wslUncFallbackPath(target: string, distroName: string): string { const relativePath = target.replace(/^\/+/, "").replaceAll("/", "\\"); return `\\\\wsl.localhost\\${distroName}${relativePath.length > 0 ? `\\${relativePath}` : ""}`; } +// wslpath -w is mount-aware, so a drive remapped in /etc/wsl.conf resolves to the +// real letter and a Linux path resolves to the \\wsl.localhost UNC path. Fall back +// to the UNC form by hand when wslpath is missing or fails. +export const resolveWslFileManagerPath = Effect.fn("externalLauncher.resolveWslFileManagerPath")( + function* ( + target: string, + distroName: string, + ): Effect.fn.Return { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + return yield* spawner + .spawn(ChildProcess.make("wslpath", ["-w", target], { stdin: "ignore", stderr: "ignore" })) + .pipe( + Effect.flatMap((handle) => + Effect.all([handle.stdout.pipe(Stream.decodeText(), Stream.mkString), handle.exitCode], { + concurrency: "unbounded", + }), + ), + Effect.map(([stdout, exitCode]) => { + const translated = stdout.trim(); + return exitCode === 0 && translated.length > 0 + ? translated + : wslUncFallbackPath(target, distroName); + }), + Effect.scoped, + Effect.timeout("2 seconds"), + Effect.orElseSucceed(() => wslUncFallbackPath(target, distroName)), + ); + }, +); + function buildBrowserLaunch( target: string, platform: NodeJS.Platform, @@ -436,16 +473,23 @@ const resolveBrowserLaunch = Effect.fn("externalLauncher.resolveBrowserLaunch")( return buildBrowserLaunch(target, platform, env); }); +// Editors and the file manager open on the desktop session, so a boot-started +// server discovers and launches them with that session's display. +const readEditorEnv = Effect.fn("externalLauncher.readEditorEnv")(function* () { + const env = { ...(yield* readBrowserLaunchEnv), ...(yield* readCommandLookupEnv) }; + return hasGraphicalLinuxSession(env) ? env : { ...env, ...(yield* readGraphicalSessionEnv()) }; +}); + const resolveAvailableEditors = Effect.fn("externalLauncher.resolveAvailableEditors")(function* () { const platform = yield* HostProcess.Platform; - const env = { ...(yield* readBrowserLaunchEnv), ...(yield* readCommandLookupEnv) }; + const env = yield* readEditorEnv(); return yield* buildAvailableEditors(platform, env).pipe(withPathDirectoryListings); }); const resolveFileManagerRevealKind = Effect.fn("externalLauncher.resolveFileManagerRevealKind")( function* () { const platform = yield* HostProcess.Platform; - const env = { ...(yield* readBrowserLaunchEnv), ...(yield* readCommandLookupEnv) }; + const env = yield* readEditorEnv(); return yield* fileManagerRevealKindForPlatform(platform, env); }, ); @@ -471,7 +515,7 @@ const resolveEditorLaunch = Effect.fn("resolveEditorLaunch")(function* ( FileSystem.FileSystem | Path.Path | ChildProcessSpawner.ChildProcessSpawner > { const platform = yield* HostProcess.Platform; - const env = { ...(yield* readBrowserLaunchEnv), ...(yield* readCommandLookupEnv) }; + const env = yield* readEditorEnv(); yield* Effect.annotateCurrentSpan({ "externalLauncher.editor": input.editor, "externalLauncher.cwd": input.cwd, @@ -517,31 +561,42 @@ const resolveEditorLaunch = Effect.fn("resolveEditorLaunch")(function* ( return yield* new ExternalLauncherUnsupportedEditorError({ editor: input.editor }); } - const target = expandHomePath(input.cwd, yield* HostProcess.HomeDirectory); + const target = input.cwd; if (input.reveal === true) { return yield* resolveFileManagerRevealLaunch(target, platform, env, command); } + if (platform === "win32") { + // ShellExecute honours replacement folder handlers that explorer.exe bypasses. + return { + ...powerShellFileManagerLaunch( + target, + `Invoke-Item -LiteralPath ${escapePowerShellStringLiteral(normalizeWindowsFileManagerPath(target))}`, + resolvePowerShellPath(env), + ), + detached: WINDOWS_POWERSHELL_DETACHED, + }; + } + return { editor: editorDef.id, target, command, args: command === "explorer.exe" && env.WSL_DISTRO_NAME !== undefined - ? [resolveWslFileManagerPath(target, env.WSL_DISTRO_NAME)] - : platform === "win32" - ? // Explorer interprets / as a switch and silently opens a fallback folder. - [normalizeWindowsFileManagerPath(target)] - : [target], + ? [yield* resolveWslFileManagerPath(target, env.WSL_DISTRO_NAME)] + : [target], }; }); -// A replacement manager registers a default Directory shell verb; stock Windows -// uses "none". Explorer needs its /select argument passed verbatim through -// PowerShell because Node's argument quoting changes how Explorer parses it. +// Replacement managers can register under Directory, Drive, or Folder. Explorer +// needs its /select argument passed verbatim through PowerShell because Node's +// argument quoting changes how Explorer parses it. function buildFileManagerRevealPowerShellSource(target: string, parent: string): string { const literal = escapePowerShellStringLiteral(target); - return `$ProgressPreference = 'SilentlyContinue'; $folderVerb = (Get-ItemProperty -LiteralPath 'Registry::HKEY_CLASSES_ROOT\\Directory\\shell' -ErrorAction SilentlyContinue).'(default)'; if ($folderVerb -and $folderVerb -ne 'none') { Invoke-Item -LiteralPath ${escapePowerShellStringLiteral(parent)} } else { Start-Process 'explorer.exe' -ArgumentList ('/select,"' + ${literal} + '"') }`; + const folderClass = /^[a-z]:\\$/i.test(parent) ? "Drive" : "Directory"; + // Explorer's ordinary open/explore handlers may also be explicit defaults. + return String.raw`$ProgressPreference = 'SilentlyContinue'; $folderVerb = '${folderClass}', 'Folder' | ForEach-Object { $key = "Registry::HKEY_CLASSES_ROOT\$_\shell"; $verb = (Get-ItemProperty -LiteralPath $key -ErrorAction SilentlyContinue).'(default)'; if ($verb -and $verb -ne 'none') { $handler = Get-ItemProperty -LiteralPath "$key\$verb\command" -ErrorAction SilentlyContinue; if ($handler.DelegateExecute -ne '{11dbb47c-a525-400b-9e80-a54615a090c0}' -and $handler.'(default)' -notmatch '^\s*"?(?:[^"]*[\\/])?explorer\.exe(?:["\s]|$)') { $verb } } }; if ($folderVerb) { Invoke-Item -LiteralPath ${escapePowerShellStringLiteral(parent)} } else { Start-Process 'explorer.exe' -ArgumentList ('/select,"' + ${literal} + '"') }`; } function powerShellFileManagerLaunch( @@ -593,13 +648,18 @@ const resolveFileManagerRevealLaunch = Effect.fn("resolveFileManagerRevealLaunch shouldUseWindowsHostFromWsl(platform, env) && env.WSL_DISTRO_NAME !== undefined ) { - const explorerTarget = resolveWslFileManagerPath(target, env.WSL_DISTRO_NAME); if (yield* isCommandAvailable(WSL_POWERSHELL_COMMAND, { env })) { // WSL names may contain quotes that Explorer's /select cannot encode. // Open the parent through the shell so replacement managers still apply. const path = yield* Path.Path; // .NET path parsing rejects characters that are legal in WSL filenames. - const parent = resolveWslFileManagerPath(path.dirname(target), env.WSL_DISTRO_NAME); + const [explorerTarget, parent] = yield* Effect.all( + [ + resolveWslFileManagerPath(target, env.WSL_DISTRO_NAME), + resolveWslFileManagerPath(path.dirname(target), env.WSL_DISTRO_NAME), + ], + { concurrency: "unbounded" }, + ); const source = explorerTarget.includes('"') ? `Invoke-Item -LiteralPath ${escapePowerShellStringLiteral(parent)}` : buildFileManagerRevealPowerShellSource(explorerTarget, parent); @@ -614,7 +674,7 @@ const resolveFileManagerRevealLaunch = Effect.fn("resolveFileManagerRevealLaunch editor: "file-manager", target, command: "explorer.exe", - args: [resolveWslFileManagerPath(path.dirname(target), env.WSL_DISTRO_NAME)], + args: [yield* resolveWslFileManagerPath(path.dirname(target), env.WSL_DISTRO_NAME)], }; } @@ -665,18 +725,72 @@ const tryRevealLinuxItem = Effect.fn("externalLauncher.tryRevealLinuxItem")( }), ); +// Stock Windows answers "none"; any other verb may be a replacement file manager, which only +// the PowerShell reveal honors. Reading stdout is about 600 ms faster than spawner.string here. +const hasStockWindowsFolderVerb = Effect.fn("externalLauncher.hasStockWindowsFolderVerb")( + function* () { + const env = yield* readBrowserLaunchEnv; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const handle = yield* spawner.spawn( + ChildProcess.make( + `${resolveWindowsRoot(env)}\\System32\\reg.exe`, + ["query", "HKCR\\Directory\\shell", "/ve"], + { stdin: "ignore", stderr: "ignore" }, + ), + ); + const [stdout, exitCode] = yield* Effect.all( + [handle.stdout.pipe(Stream.decodeText(), Stream.mkString), handle.exitCode], + { concurrency: "unbounded" }, + ); + return exitCode === 0 && /REG_SZ\s+none\s*$/im.test(stdout); + }, + Effect.scoped, + Effect.timeout("2 seconds"), + Effect.orElseSucceed(() => false), +); + +// Explorer parses /select itself, so it needs the argument verbatim rather than Node's quoting. +const revealWindowsItem = Effect.fn("externalLauncher.revealWindowsItem")(function* ( + target: string, +) { + const command = `${resolveWindowsRoot(yield* readBrowserLaunchEnv)}\\explorer.exe`; + const args = [`/select,"${normalizeWindowsFileManagerPath(target)}"`]; + yield* (yield* WindowsVerbatimSpawn)(command, args).pipe( + Effect.mapError( + (cause) => + new ExternalLauncherEditorSpawnError({ + editor: "file-manager", + target, + command, + args, + cause, + }), + ), + ); +}); + const launchEditor = Effect.fn("externalLauncher.launchEditor")(function* ( input: LaunchEditorInput, + hasStockFolderVerb: Effect.Effect, ) { - const launch = yield* resolveEditorLaunch(input); - if ( - input.editor === "file-manager" && - input.reveal === true && - launch.command === "xdg-open" && - (yield* HostProcess.Platform) === "linux" && - (yield* tryRevealLinuxItem(launch.target)) - ) { - return; + // Clients send `~/` host paths as written, and launches run without a shell to expand them. + const home = yield* HostProcess.HomeDirectory; + const cwd = expandHomePath(input.cwd, home); + const projectRoot = + input.projectRoot === undefined ? undefined : expandHomePath(input.projectRoot, home); + const launch = yield* resolveEditorLaunch({ ...input, cwd, projectRoot }); + if (input.editor === "file-manager" && input.reveal === true) { + const platform = yield* HostProcess.Platform; + if ( + platform === "linux" && + launch.command === "xdg-open" && + (yield* tryRevealLinuxItem(launch.target)) + ) { + return; + } + if (platform === "win32" && (yield* hasStockFolderVerb)) { + return yield* revealWindowsItem(launch.target); + } } yield* launchEditorProcess(launch); }); @@ -837,6 +951,12 @@ export const make = Effect.gen(function* () { ), EDITOR_DISCOVERY_CACHE_TTL, ); + const cachedStockWindowsFolderVerb = yield* makeStaleWhileRevalidate( + hasStockWindowsFolderVerb().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ), + EDITOR_DISCOVERY_CACHE_TTL, + ); // Forwarders retain the underlying launcher and discovery spans. return ExternalLauncher.of({ @@ -854,7 +974,9 @@ export const make = Effect.gen(function* () { ), launchEditor: Effect.fnUntraced( function* (input) { - return yield* provideCommandResolutionServices(launchEditor(input)); + return yield* provideCommandResolutionServices( + launchEditor(input, cachedStockWindowsFolderVerb), + ); }, Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), ), diff --git a/apps/server/src/process/providerProcesses.test.ts b/apps/server/src/process/providerProcesses.test.ts new file mode 100644 index 0000000000000..26bad1cec9acc --- /dev/null +++ b/apps/server/src/process/providerProcesses.test.ts @@ -0,0 +1,127 @@ +// @effect-diagnostics nodeBuiltinImport:off -- real processes stand in for servers and CLIs. +import * as NodeChildProcess from "node:child_process"; +import * as NodeCrypto from "node:crypto"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, describe, expect, it } from "@effect/vitest"; +import * as AcpSessionRuntime from "@t3tools/provider-acp/server/AcpSessionRuntime"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Context from "effect/Context"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; + +import * as ServerConfig from "../config.ts"; +import * as ProviderProcesses from "./providerProcesses.ts"; + +// Real child processes stand in for servers and CLIs; only PIDs captured at spawn are signalled. +const ownedProcess = Effect.gen(function* () { + const ready = yield* Deferred.make(); + const exited = yield* Deferred.make(); + const child = yield* Effect.acquireRelease( + Effect.sync(() => { + const child = NodeChildProcess.spawn( + process.execPath, + ["-e", 'process.stdin.resume(); process.send("ready");'], + { stdio: ["pipe", "ignore", "ignore", "ipc"] }, + ); + child.once("message", () => Deferred.doneUnsafe(ready, Effect.void)); + child.once("error", (error) => Deferred.doneUnsafe(ready, Effect.die(error))); + child.once("exit", () => Deferred.doneUnsafe(exited, Effect.void)); + return child; + }), + (child) => + Effect.sync(() => child.kill("SIGKILL")).pipe(Effect.andThen(Deferred.await(exited))), + ); + yield* Deferred.await(ready); + const pid = child.pid; + assert(pid !== undefined); + const controller = AcpSessionRuntime.makePosixProcessTreeController("linux"); + const identity = controller.identity(pid); + assert(identity !== undefined); + const boot = NodeFS.readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim(); + return { + pid, + startTime: `${boot}:${identity.startTime}`, + pgid: 0, + running: () => { + const observed = controller.identity(pid); + return observed !== undefined && !AcpSessionRuntime.posixProcessIsZombie(observed); + }, + stop: Effect.sync(() => child.kill("SIGKILL")).pipe(Effect.andThen(Deferred.await(exited))), + }; +}); + +describe.runIf(HostProcess.Platform.defaultValue() === "linux")("ProviderProcesses", () => { + it.live("reaps only recorded CLIs of dead servers before startup completes", () => + Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + const directory = NodePath.join(config.stateDir, "provider-processes"); + NodeFS.mkdirSync(directory, { recursive: true }); + type Identity = { readonly pid: number; readonly startTime: string }; + const record = (owner: Identity, processes: ReadonlyArray) => { + const instanceId = NodeCrypto.randomUUID(); + NodeFS.writeFileSync( + NodePath.join(directory, `${instanceId}.json`), + JSON.stringify({ + version: 1, + instanceId, + hostname: NodeOS.hostname(), + stateDir: config.stateDir, + owner: { pid: owner.pid, startTime: owner.startTime }, + processes: processes.map(({ pid, startTime }) => ({ pid, startTime, pgid: 0 })), + }), + ); + }; + const reused = (identity: Identity) => ({ + ...identity, + startTime: identity.startTime.replace(/:\d+$/, ":1"), + }); + const deadOwner = yield* ownedProcess; + const liveOwner = yield* ownedProcess; + const reusedOwner = yield* ownedProcess; + const liveProvider = yield* ownedProcess; + const reusedProvider = yield* ownedProcess; + const orphan = yield* ownedProcess; + const orphanOfReusedOwner = yield* ownedProcess; + const unrecorded = yield* ownedProcess; + record(liveOwner, [liveProvider]); + record(deadOwner, [reused(reusedProvider)]); + record(deadOwner, [orphan]); + record(reused(reusedOwner), [orphanOfReusedOwner]); + yield* deadOwner.stop; + + const context = yield* Layer.build(ProviderProcesses.layer); + expect(Context.get(context, ProviderProcesses.ProviderProcesses).hasOtherLiveOwner).toBe( + true, + ); + + expect({ + orphan: orphan.running(), + orphanOfReusedOwner: orphanOfReusedOwner.running(), + liveProvider: liveProvider.running(), + reusedProvider: reusedProvider.running(), + reusedOwner: reusedOwner.running(), + unrecorded: unrecorded.running(), + }).toEqual({ + orphan: false, + orphanOfReusedOwner: false, + liveProvider: true, + reusedProvider: true, + reusedOwner: true, + unrecorded: true, + }); + }).pipe( + Effect.scoped, + Effect.provideService(HostProcess.Platform, "linux"), + Effect.provide( + ServerConfig.layerTest(process.cwd(), { prefix: "t3-restart-ownership-" }).pipe( + Layer.provideMerge(NodeServices.layer), + ), + ), + ), + ); +}); diff --git a/apps/server/src/process/providerProcesses.ts b/apps/server/src/process/providerProcesses.ts new file mode 100644 index 0000000000000..4fe9a47637a04 --- /dev/null +++ b/apps/server/src/process/providerProcesses.ts @@ -0,0 +1,140 @@ +// @effect-diagnostics nodeBuiltinImport:off -- the SDK's synchronous spawn callback must record ownership before releasing the child. +import * as NodeCrypto from "node:crypto"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import * as AcpSessionRuntime from "@t3tools/provider-acp/server/AcpSessionRuntime"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; + +import * as ServerConfig from "../config.ts"; + +const ProcessIdentity = Schema.Struct({ + pid: Schema.Int.check(Schema.isBetween({ minimum: 2, maximum: Number.MAX_SAFE_INTEGER })), + startTime: Schema.String, +}); +const OwnershipRecord = Schema.Struct({ + version: Schema.Literal(1), + instanceId: Schema.String, + hostname: Schema.String, + stateDir: Schema.String, + owner: ProcessIdentity, + processes: Schema.Array(Schema.Struct({ ...ProcessIdentity.fields, pgid: Schema.Int })), +}); +const decodeRecord = Schema.decodeUnknownSync(Schema.fromJsonString(OwnershipRecord)); + +// A concurrent exit can remove the record. Incomplete/unknown records never authorize a signal. +const readRecord = (recordPath: string) => { + try { + return decodeRecord(NodeFS.readFileSync(recordPath, "utf8")); + } catch { + return undefined; + } +}; + +export const ProviderProcesses = Context.Reference<{ + readonly register?: (pid: number) => () => void; + readonly hasOtherLiveOwner: boolean; +}>("t3/process/ProviderProcesses", { defaultValue: () => ({ hasOtherLiveOwner: false }) }); + +const make = Effect.gen(function* () { + const platform = yield* HostProcess.Platform; + if (platform !== "linux") return { hasOtherLiveOwner: false }; + const { stateDir } = yield* ServerConfig.ServerConfig; + const directory = NodePath.join(stateDir, "provider-processes"); + const hostname = NodeOS.hostname(); + const boot = NodeFS.readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim(); + const controller = AcpSessionRuntime.makePosixProcessTreeController(platform); + const identity = (pid: number) => { + const observed = controller.identity(pid); + if (observed === undefined || AcpSessionRuntime.posixProcessIsZombie(observed)) + return undefined; + return { ...observed, startTime: `${boot}:${observed.startTime}` }; + }; + const matches = (expected: typeof ProcessIdentity.Type) => + identity(expected.pid)?.startTime === expected.startTime; + const kill = (survivor: typeof ProcessIdentity.Type) => { + try { + controller.signalProcess(survivor.pid, "SIGKILL"); + } catch {} + }; + const owner = identity(process.pid); + if (owner === undefined) return yield* Effect.die(new Error("Cannot identify provider owner")); + NodeFS.mkdirSync(directory, { recursive: true }); + + const writeRecord = (processes: (typeof OwnershipRecord.Type)["processes"]) => { + const instanceId = NodeCrypto.randomUUID(); + const recordPath = NodePath.join(directory, `${instanceId}.json`); + const pendingPath = `${recordPath}.pending`; + NodeFS.writeFileSync( + pendingPath, + JSON.stringify({ + version: 1, + instanceId, + hostname, + stateDir, + owner: { pid: owner.pid, startTime: owner.startTime }, + processes, + } satisfies typeof OwnershipRecord.Type), + ); + NodeFS.renameSync(pendingPath, recordPath); + return () => NodeFS.rmSync(recordPath, { force: true }); + }; + // Publish the server identity even before its first provider. Another server + // using this home must not stop the shared systemd slice while we are alive. + const forgetOwner = writeRecord([]); + yield* Effect.addFinalizer(() => Effect.sync(forgetOwner)); + let hasOtherLiveOwner = false; + + // Only immutable per-launch records are shared between server instances. A + // live owner's records belong to that server, even when it uses this home. + for (const file of NodeFS.readdirSync(directory)) { + if (!file.endsWith(".json")) continue; + const recordPath = NodePath.join(directory, file); + const record = readRecord(recordPath); + if (record === undefined) continue; + if (record.hostname !== hostname || record.stateDir !== stateDir) continue; + if (matches(record.owner)) { + if (record.owner.pid !== process.pid) hasOtherLiveOwner = true; + continue; + } + let survived = false; + for (const survivor of record.processes) { + if (survivor.pid === process.pid || !matches(survivor)) continue; + // Never infer ownership from argv, executable names, or group membership. + // As in the ACP controller, starttime blocks reuse between observations. + // Node has no pidfd_send_signal, so the final read-to-signal race remains. + kill(survivor); + for (let attempt = 0; matches(survivor) && !survived; attempt++) { + survived = attempt === 100; + if (!survived) yield* Effect.sleep("10 millis"); + } + } + // Like a failed slice stop, an unkillable CLI must not keep the server down; + // its record stays so the next start retries. + if (survived) { + yield* Effect.logWarning("Could not stop a provider left by a previous server", { file }); + continue; + } + NodeFS.rmSync(recordPath, { force: true }); + } + + return { + hasOtherLiveOwner, + register: (pid: number) => { + const provider = identity(pid); + if (provider === undefined || provider.ppid !== process.pid) { + throw new Error("Cannot identify spawned provider"); + } + // A rename keeps startup from seeing a partial write. The child is gated + // until this returns, so an abrupt death cannot leave an unrecorded CLI. + return writeRecord([{ pid, startTime: provider.startTime, pgid: provider.pgid }]); + }, + }; +}); + +export const layer = Layer.effect(ProviderProcesses, make); diff --git a/apps/server/src/process/windowsVerbatimSpawn.ts b/apps/server/src/process/windowsVerbatimSpawn.ts new file mode 100644 index 0000000000000..15075c22455b9 --- /dev/null +++ b/apps/server/src/process/windowsVerbatimSpawn.ts @@ -0,0 +1,29 @@ +// @effect-diagnostics nodeBuiltinImport:off - Effect's spawner has no windowsVerbatimArguments option. +import * as NodeChildProcess from "node:child_process"; + +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; + +export type WindowsVerbatimSpawner = ( + command: string, + args: ReadonlyArray, +) => Effect.Effect; + +const spawnDetachedVerbatim: WindowsVerbatimSpawner = (command, args) => + Effect.callback((resume) => { + const child = NodeChildProcess.spawn(command, args, { + detached: true, + stdio: "ignore", + windowsVerbatimArguments: true, + }); + child.once("error", (error) => resume(Effect.fail(error))); + child.once("spawn", () => { + child.unref(); + resume(Effect.void); + }); + }); + +export const WindowsVerbatimSpawn = Context.Reference( + "t3/process/WindowsVerbatimSpawn", + { defaultValue: () => spawnDetachedVerbatim }, +); diff --git a/apps/server/src/processRunner.test.ts b/apps/server/src/processRunner.test.ts index 3754f25a4c775..6768fa23c763b 100644 --- a/apps/server/src/processRunner.test.ts +++ b/apps/server/src/processRunner.test.ts @@ -2,6 +2,8 @@ import { isWindowsCommandNotFound } from "@t3tools/provider-core/server/snapshot import { describe, expect, it } from "@effect/vitest"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Cause from "effect/Cause"; import * as Layer from "effect/Layer"; import * as PlatformError from "effect/PlatformError"; import * as Sink from "effect/Sink"; @@ -143,6 +145,29 @@ describe("runProcess", () => { }), ); + it.effect("reports child startup before delivering cancellation during acquisition", () => + Effect.gen(function* () { + let started = false; + const spawner = makeSpawner(() => + Effect.withFiber((fiber) => + Effect.sync(() => { + fiber.interruptUnsafe(); + return makeHandle({}); + }), + ), + ); + const result = yield* runWith(spawner)({ + command: "git", + args: ["restore"], + onSpawn: () => { + started = true; + }, + }).pipe(Effect.exit); + expect(Exit.isFailure(result) && Cause.hasInterruptsOnly(result.cause)).toBe(true); + expect(started).toBe(true); + }).pipe(Effect.scoped), + ); + it.effect("fails when output exceeds max buffer in default mode", () => Effect.gen(function* () { const spawner = makeSpawner(() => Effect.succeed(makeHandle({ stdout: "x".repeat(2048) }))); diff --git a/apps/server/src/processRunner.ts b/apps/server/src/processRunner.ts index a345c67eca881..4c7efd3cc9942 100644 --- a/apps/server/src/processRunner.ts +++ b/apps/server/src/processRunner.ts @@ -26,6 +26,8 @@ export interface ProcessRunInput { readonly stdin?: string | undefined; /** Receives every stdout chunk, including bytes beyond the buffered output limit. */ readonly onStdoutChunk?: ((chunk: Uint8Array) => void) | undefined; + /** Runs after successful spawn, before cancellation can interrupt acquisition. */ + readonly onSpawn?: (() => void) | undefined; readonly maxOutputBytes?: number | undefined; readonly outputMode?: "error" | "truncate" | undefined; readonly truncatedMarker?: string | undefined; @@ -290,7 +292,7 @@ const runProcessCore = Effect.fn("processRunner.runProcessCore")(function* ( cause, }); - const child = yield* spawner + const spawn = spawner .spawn( ChildProcess.make(spawnCommand.command, spawnCommand.args, { ...((input.spawnCwd ?? input.cwd) ? { cwd: input.spawnCwd ?? input.cwd } : {}), @@ -309,6 +311,12 @@ const runProcessCore = Effect.fn("processRunner.runProcessCore")(function* ( // Synchronous spawn throws arrive as defects; recover them too so callers see typed spawn failures. Effect.catchDefect((cause) => Effect.fail(toSpawnError(cause))), ); + const child = yield* input.onSpawn === undefined + ? spawn + : spawn.pipe( + Effect.tap(() => Effect.sync(() => input.onSpawn?.())), + Effect.uninterruptible, + ); const stdin = input.stdin; const onStdoutChunk = input.onStdoutChunk; diff --git a/apps/server/src/project/AgentSessionImporter.test.ts b/apps/server/src/project/AgentSessionImporter.test.ts index b05a0a0504cdd..89659aa9979e7 100644 --- a/apps/server/src/project/AgentSessionImporter.test.ts +++ b/apps/server/src/project/AgentSessionImporter.test.ts @@ -1,3 +1,4 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; import { expect, it } from "@effect/vitest"; import { type AgentSessionSource, @@ -22,17 +23,21 @@ import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as Path from "effect/Path"; import * as Stream from "effect/Stream"; import * as SqlClient from "effect/sql/SqlClient"; import * as TestClock from "effect/testing/TestClock"; +import * as ServerConfig from "../config.ts"; import * as EventSink from "../orchestration-v2/EventSink.ts"; import * as EventStore from "../orchestration-v2/EventStore.ts"; import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; +import * as ProjectStore from "../orchestration-v2/ProjectStore.ts"; import * as ProviderAdapterRegistry from "../orchestration-v2/ProviderAdapterRegistry.ts"; import * as ThreadCommandExecutor from "../orchestration-v2/ThreadCommandExecutor.ts"; import { @@ -92,6 +97,7 @@ function candidate( { role: "assistant" as const, text: "Fixed", createdAt: timestamp }, ], }, + worktree: null as { path: string; branch: string | null } | null, } satisfies AgentSessionScanner.AgentSessionRecentThread; } @@ -179,7 +185,7 @@ const seedOwner = Effect.fn(function* ( }); const makeImporter = Effect.fn(function* ( - candidates: ReadonlyArray>, + candidates: ReadonlyArray, beforeTransaction: Effect.Effect = Effect.void, getAdapter?: ProviderAdapterRegistry.ProviderAdapterRegistryV2Shape["get"], threadCommands?: KeyedLock.KeyedLock, @@ -187,6 +193,10 @@ const makeImporter = Effect.fn(function* ( ServerSettings.ServerSettingsService, ServerSettingsError > = ServerSettings.layerTest(), + realScan?: { + readonly workspaceRoot: string; + readonly scanner: AgentSessionScanner.AgentSessionScanner["Service"]; + }, ) { const sql = yield* SqlClient.SqlClient; const events = yield* EventStore.EventStoreV2; @@ -228,15 +238,17 @@ const makeImporter = Effect.fn(function* ( ProviderSessionRuntime.ProviderSessionRuntimeRepository, yield* ProviderSessionRuntime.ProviderSessionRuntimeRepository, ), - Layer.mock(AgentSessionScanner.AgentSessionScanner)({ - recentThreads: () => Stream.fromIterable(candidates), - }), + realScan === undefined + ? Layer.mock(AgentSessionScanner.AgentSessionScanner)({ + recentThreads: () => Stream.fromIterable(candidates), + }) + : Layer.succeed(AgentSessionScanner.AgentSessionScanner, realScan.scanner), Layer.mock(ProjectService.ProjectService)({ getById: () => Effect.succeedSome({ id: projectId, title: "Project", - workspaceRoot: "/workspace/project", + workspaceRoot: realScan?.workspaceRoot ?? "/workspace/project", defaultModelSelection: null, scripts: [], createdAt: timestamp, @@ -304,6 +316,7 @@ it.effect( ); const unowned = candidate(CODEX_MODEL_SELECTION, "unowned"); unowned.thread.omittedMessageCount = 1; + unowned.worktree = { path: "/workspace/project/.claude/worktrees/feat", branch: "feat" }; const unownedPi = candidate(PI_MODEL_SELECTION, "unowned-pi", "pi"); const importer = yield* makeImporter([ active, @@ -356,6 +369,8 @@ it.effect( expect(notices).toEqual([]); } expect(records.thread.historyOrigin).toBe("v1_import"); + expect(records.thread.worktreePath).toBe(source.worktree?.path ?? null); + expect(records.thread.branch).toBe(source.worktree?.branch ?? null); expect(records.thread.activeProviderThreadId).toBe(records.providerThreads[0]?.id); expect(records.providerThreads[0]?.nativeThreadRef?.nativeId).toBe(nativeId(source)); expect(Option.getOrThrow(yield* runtimes.getByThreadId({ threadId }))).toMatchObject({ @@ -364,12 +379,52 @@ it.effect( source.source.provider === "claudeAgent" ? { threadId, resume: source.source.providerSessionId } : { threadId: nativeId(source) }, - runtimePayload: { importedTranscripts: [source.source] }, + runtimePayload: { + cwd: source.worktree?.path ?? "/workspace/project", + importedTranscripts: [source.source], + }, }); } }).pipe(Effect.scoped, Effect.provide(persistence)), ); +it.effect("tolerates a missing transcript owner but propagates failed ownership reads", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const runtimes = yield* ProviderSessionRuntime.ProviderSessionRuntimeRepository; + const source = candidate(CODEX_MODEL_SELECTION, "ownership-read"); + const threadId = importId(source); + yield* runtimes.upsert({ + threadId: ThreadId.make("missing-owner"), + providerName: "codex", + providerInstanceId: source.source.providerInstanceId, + adapterKey: "codex", + runtimeMode: "full-access", + status: "stopped", + lastSeenAt: timestamp, + resumeCursor: null, + runtimePayload: { importedTranscripts: [source.source] }, + }); + yield* runtimes.recordImportedTranscript({ + threadId: ThreadId.make("missing-owner"), + source: source.source, + }); + const importer = yield* makeImporter([source]); + expect(yield* importer.run).toEqual({ importedCount: 1, skippedCount: 0 }); + + yield* sql`UPDATE orchestration_v2_projection_threads SET payload_json = '{}' WHERE thread_id = ${threadId}`; + expect(yield* Effect.flip(importer.run)).toMatchObject({ + _tag: "AgentSessionScanError", + operation: "read-projects", + cause: { + _tag: "OrchestratorProjectionError", + threadId, + cause: { _tag: "ProjectionStoreReadError" }, + }, + }); + }).pipe(Effect.scoped, Effect.provide(persistence)), +); + it.effect("rechecks ownership after the import reaches the commit transaction", () => Effect.gen(function* () { const enteringTransaction = yield* Deferred.make(); @@ -429,8 +484,8 @@ it.effect( expect(yield* runtimes.list()).toEqual([]); yield* sql`DROP TRIGGER fail_runtime`; - expect(yield* importer.run).toEqual({ importedCount: 1, skippedCount: 0 }); - expect(yield* importer.run).toEqual({ importedCount: 1, skippedCount: 0 }); + expect(yield* importer.run).toEqual({ importedCount: 0, skippedCount: 0 }); + expect(yield* importer.run).toEqual({ importedCount: 0, skippedCount: 0 }); expect(yield* events.read({ threadId }).pipe(Stream.runCollect)).toEqual(committed); expect(importer.published).toHaveLength(6); expect( @@ -446,6 +501,82 @@ it.effect( }).pipe(Effect.scoped, Effect.provide(persistence)), ); +it.effect("counts only new threads when re-importing cached and recovered transcripts", () => + Effect.gen(function* () { + const events = yield* EventStore.EventStoreV2; + const runtimes = yield* ProviderSessionRuntime.ProviderSessionRuntimeRepository; + const cached = candidate( + CLAUDE_MODEL_SELECTION, + "020e7f00-dce6-406b-8ec6-ed90498945d6", + "claudeAgent", + ); + const recovered = candidate(CODEX_MODEL_SELECTION, "recovered"); + const initial = yield* makeImporter([cached, recovered]); + expect(yield* initial.run).toEqual({ importedCount: 2, skippedCount: 0 }); + const before = yield* events.read({ threadId: importId(recovered) }).pipe(Stream.runCollect); + // The file changed on disk but still holds the imported messages. + recovered.source.size += 100; + recovered.source.mtimeMs += 1; + const duplicate = { ...recovered.source, filePath: "/tmp/duplicate.jsonl" }; + const fresh = candidate(CODEX_MODEL_SELECTION, "fresh"); + const repeated = yield* makeImporter([ + { _tag: "AlreadyImported", source: cached.source }, + recovered, + { _tag: "Duplicate", source: duplicate }, + fresh, + { _tag: "Skipped" }, + ]); + + expect(yield* repeated.run).toEqual({ importedCount: 1, skippedCount: 1 }); + expect(yield* events.read({ threadId: importId(recovered) }).pipe(Stream.runCollect)).toEqual( + before, + ); + expect( + new Set( + repeated.published.flatMap((stored) => ("event" in stored ? [stored.event.threadId] : [])), + ), + ).toEqual(new Set([importId(fresh)])); + expect( + Option.getOrThrow(yield* runtimes.getByThreadId({ threadId: importId(recovered) })), + ).toMatchObject({ + runtimePayload: { importedTranscripts: [recovered.source, duplicate] }, + }); + }).pipe(Effect.scoped, Effect.provide(persistence)), +); + +it.effect("does not certify new native history after a partial import retry", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const runtimes = yield* ProviderSessionRuntime.ProviderSessionRuntimeRepository; + const original = candidate(CODEX_MODEL_SELECTION, "partial-retry"); + const threadId = importId(original); + const importer = yield* makeImporter([original]); + yield* sql`CREATE TEMP TRIGGER fail_runtime BEFORE INSERT ON provider_session_runtime BEGIN SELECT RAISE(ABORT, 'test runtime failure'); END`; + expect(yield* importer.run).toEqual({ importedCount: 0, skippedCount: 1 }); + yield* sql`DROP TRIGGER fail_runtime`; + expect(yield* runtimes.list()).toEqual([]); + const continued = candidate(CODEX_MODEL_SELECTION, original.source.providerSessionId); + continued.source.size = 200; + continued.source.mtimeMs = 3; + continued.thread.messages.push( + { role: "user", text: "Continue outside T3", createdAt: timestamp }, + { role: "assistant", text: "Native continuation", createdAt: timestamp }, + ); + // The thread holds only the original messages, so the continued transcript stays unrecorded. + expect(yield* (yield* makeImporter([continued])).run).toEqual({ + importedCount: 0, + skippedCount: 1, + }); + expect(yield* runtimes.list()).toEqual([]); + expect( + (yield* projections.getThreadRecords(threadId, ["messages"])).messages.map( + (message) => message.text, + ), + ).toEqual(["Fix it", "Fixed"]); + }).pipe(Effect.scoped, Effect.provide(persistence)), +); + it.effect("does not record a changed transcript as imported without its new messages", () => Effect.gen(function* () { const projections = yield* ProjectionStore.ProjectionStoreV2; @@ -829,3 +960,95 @@ it.effect("imports under a policy ceiling in the project's default, kept under t ); }).pipe(Effect.scoped, Effect.provide(persistence)), ); + +it.effect("repeated imports reach linked-worktree sessions behind the scan budget", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-import-worktree-budget-" }); + const repo = path.join(root, "repo"); + const worktree = path.join(root, "feature"); + const worktreeGitDir = path.join(repo, ".git", "worktrees", "feature"); + yield* fs.makeDirectory(worktreeGitDir, { recursive: true }); + yield* fs.makeDirectory(worktree); + yield* fs.writeFileString(path.join(repo, ".git", "config"), "[core]\n"); + yield* fs.writeFileString(path.join(worktreeGitDir, "commondir"), "../..\n"); + yield* fs.writeFileString(path.join(worktreeGitDir, "HEAD"), "ref: refs/heads/feature\n"); + yield* fs.writeFileString(path.join(worktree, ".git"), `gitdir: ${worktreeGitDir}\n`); + + const claudeHome = path.join(root, "claude"); + const codexHome = path.join(root, "codex"); + const sessions = path.join(codexHome, "sessions", "2026", "09", "01"); + yield* fs.makeDirectory(sessions, { recursive: true }); + const nowMs = DateTime.toEpochMillis(now); + yield* TestClock.setTime(nowMs); + // One more session than the 100-transcript scan budget, all run in the linked worktree. + for (let index = 0; index < 101; index++) { + const file = path.join(sessions, `rollout-${String(index).padStart(3, "0")}.jsonl`); + const records = [ + { type: "session_meta", payload: { id: `session-${index}`, cwd: worktree } }, + { type: "event_msg", payload: { type: "user_message", message: `Request ${index}` } }, + { + type: "response_item", + payload: { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "Done" }], + }, + }, + ]; + yield* fs.writeFileString( + file, + records.map((record) => JSON.stringify(record) + "\n").join(""), + ); + const modifiedAtSeconds = nowMs / 1000 - (index + 1); + yield* fs.utimes(file, modifiedAtSeconds, modifiedAtSeconds); + } + + const scannerContext = yield* Layer.build( + AgentSessionScanner.layer.pipe( + Layer.provide( + Layer.mergeAll( + ServerConfig.layerTest(claudeHome, { prefix: "t3-import-budget-config-" }), + ServerSettings.layerTest({ + providerInstances: { + [ProviderInstanceId.make("claudeAgent")]: { + driver: ProviderDriverKind.make("claudeAgent"), + config: { homePath: claudeHome }, + }, + [ProviderInstanceId.make("codex")]: { + driver: ProviderDriverKind.make("codex"), + config: { homePath: codexHome }, + }, + // Keeps the default Pi instance off the developer's real ~/.pi. + [ProviderInstanceId.make("pi")]: { + driver: ProviderDriverKind.make("pi"), + enabled: false, + config: {}, + }, + }, + }), + Layer.mock(ProjectStore.ProjectStoreV2)({ listShells: () => Effect.succeed([]) }), + Layer.succeed( + ProviderSessionRuntime.ProviderSessionRuntimeRepository, + yield* ProviderSessionRuntime.ProviderSessionRuntimeRepository, + ), + ), + ), + ), + ); + const importer = yield* makeImporter([], Effect.void, undefined, undefined, undefined, { + workspaceRoot: repo, + scanner: Context.get(scannerContext, AgentSessionScanner.AgentSessionScanner), + }); + + expect(yield* importer.run).toEqual({ importedCount: 100, skippedCount: 1 }); + expect(yield* importer.run).toEqual({ importedCount: 1, skippedCount: 0 }); + const oldest = yield* ProjectionStore.ProjectionStoreV2.pipe( + Effect.flatMap((projections) => + projections.getThread(ThreadId.make("import:codex:session-100")), + ), + ); + expect(oldest.worktreePath).toBe(worktree); + }).pipe(Effect.scoped, Effect.provide(Layer.merge(persistence, NodeServices.layer))), +); diff --git a/apps/server/src/project/AgentSessionImporter.ts b/apps/server/src/project/AgentSessionImporter.ts index 9c80ebf1049bf..0f8736986a2df 100644 --- a/apps/server/src/project/AgentSessionImporter.ts +++ b/apps/server/src/project/AgentSessionImporter.ts @@ -35,6 +35,7 @@ import * as Stream from "effect/Stream"; import * as EventSink from "../orchestration-v2/EventSink.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; +import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; import * as ProviderAdapterRegistry from "../orchestration-v2/ProviderAdapterRegistry.ts"; import * as ThreadCommandExecutor from "../orchestration-v2/ThreadCommandExecutor.ts"; import * as ProviderSessionRuntime from "../persistence/ProviderSessionRuntime.ts"; @@ -42,6 +43,10 @@ import * as ServerSettings from "../serverSettings.ts"; import * as AgentSessionScanner from "./AgentSessionScanner.ts"; import * as ProjectService from "./ProjectService.ts"; +const isProjectionStoreThreadNotFoundError = Schema.is( + ProjectionStore.ProjectionStoreThreadNotFoundError, +); + const IMPORT_EVENT_PREFIX = "agent-session-import:v2"; const CLAUDE_SESSION_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; @@ -242,18 +247,21 @@ const make = Effect.gen(function* () { (cause) => new AgentSessionScanError({ operation: "read-projects", cause }), ), ); - const completedSources = runtimeRows.flatMap((runtime) => { - const payload = AgentSessionScanner.decodeImportedTranscriptPayload(runtime.runtimePayload); - if ( - Option.isNone(payload) || - payload.value.cwd === undefined || - normalizeProjectPathForComparison(payload.value.cwd) !== - normalizeProjectPathForComparison(project.workspaceRoot) - ) { - return []; - } - return payload.value.importedTranscripts ?? []; - }); + // The thread's project owns its transcripts: a linked worktree import records the worktree as cwd. + const completedSources = (yield* Effect.forEach(runtimeRows, (runtime) => { + const imported = AgentSessionScanner.decodeImportedTranscriptPayload( + runtime.runtimePayload, + ).pipe(Option.flatMapNullishOr((payload) => payload.importedTranscripts)); + if (Option.isNone(imported) || imported.value.length === 0) return Effect.succeed([]); + return orchestrator.getThreadRecords(runtime.threadId, []).pipe( + Effect.map(({ thread }) => (thread.projectId === input.projectId ? imported.value : [])), + Effect.catch((cause) => + isProjectionStoreThreadNotFoundError(cause.cause) + ? Effect.succeed([]) + : Effect.fail(new AgentSessionScanError({ operation: "read-projects", cause })), + ), + ); + })).flat(); const outcomes = scanner.recentThreads(project.workspaceRoot, completedSources); const importedThreadIds = new Set(); let importedCount = 0; @@ -271,7 +279,6 @@ const make = Effect.gen(function* () { ); if (outcome._tag === "AlreadyImported") { importedThreadIds.add(threadId); - importedCount += 1; return; } if (outcome._tag === "Duplicate") { @@ -309,11 +316,13 @@ const make = Effect.gen(function* () { thread.source === "claudeAgent" ? { threadId, resume: thread.providerSessionId } : { threadId: nativeThreadId }, - runtimePayload: { cwd: project.workspaceRoot }, + runtimePayload: { cwd: outcome.worktree?.path ?? project.workspaceRoot }, }, { onConflict: "ignore" }, ); - const existing = yield* Effect.option(orchestrator.getThreadRecords(threadId, [])); + const existing = yield* Effect.option( + orchestrator.getThreadRecords(threadId, ["messages"]), + ); if (Option.isSome(existing)) { if (existing.value.thread.projectId !== input.projectId) { return yield* new AgentSessionThreadProjectConflictError({ @@ -325,23 +334,23 @@ const make = Effect.gen(function* () { if (existing.value.thread.historyOrigin !== "v1_import") { return yield* new AgentSessionThreadModifiedError({ threadId }); } - // Other recorded transcripts mean this one changed after the import, and the - // thread does not hold its new messages, so it must stay unrecorded. - const recorded = Option.flatMap(yield* runtimes.getByThreadId({ threadId }), (row) => - AgentSessionScanner.decodeImportedTranscriptPayload(row.runtimePayload), - ).pipe(Option.flatMapNullishOr((payload) => payload.importedTranscripts)); + // A transcript can change after the import, so record it only when the + // thread already holds exactly its messages. + const stored = existing.value.messages; if ( - Option.isSome(recorded) && - recorded.value.length > 0 && - !recorded.value.some((imported) => - AgentSessionScanner.sameTranscriptIdentity(imported, source), + stored.length !== thread.messages.length || + stored.some( + (message, index) => + message.role !== thread.messages[index]?.role || + message.text !== thread.messages[index]?.text, ) ) { return false; } yield* persistRuntime; yield* runtimes.recordImportedTranscript({ threadId, source }); - return true; + importedThreadIds.add(threadId); + return null; } const model = thread.model ?? DEFAULT_MODEL_BY_PROVIDER[driver] ?? DEFAULT_MODEL; @@ -362,8 +371,8 @@ const make = Effect.gen(function* () { modelSelection: { instanceId: thread.providerInstanceId, model }, runtimeMode, interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, - branch: null, - worktreePath: null, + branch: outcome.worktree?.branch ?? null, + worktreePath: outcome.worktree?.path ?? null, linkedPullRequest: null, branchPullRequest: null, activeProviderThreadId: providerThreadId, diff --git a/apps/server/src/project/AgentSessionScanner.test.ts b/apps/server/src/project/AgentSessionScanner.test.ts index 82890ca9c8353..ce75e456158a8 100644 --- a/apps/server/src/project/AgentSessionScanner.test.ts +++ b/apps/server/src/project/AgentSessionScanner.test.ts @@ -5,6 +5,7 @@ import * as NodeChildProcess from "node:child_process"; import * as NodeFS from "node:fs"; import * as NodeOS from "node:os"; import * as NodeSqlite from "node:sqlite"; +import * as NodeZlib from "node:zlib"; import { describe, expect, it } from "@effect/vitest"; import { AgentSessionImportSource, @@ -174,6 +175,21 @@ const writeTranscript = Effect.fn("AgentSessionScanner.test.writeTranscript")(fu yield* fileSystem.utimes(input.filePath, seconds, seconds); }); +const writeCompressedTranscript = Effect.fn("AgentSessionScanner.test.writeCompressedTranscript")( + function* (input: { + readonly filePath: string; + readonly contents: string; + readonly mtimeMs: number; + }) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* fileSystem.makeDirectory(path.dirname(input.filePath), { recursive: true }); + yield* fileSystem.writeFile(input.filePath, NodeZlib.zstdCompressSync(input.contents)); + const seconds = input.mtimeMs / 1000; + yield* fileSystem.utimes(input.filePath, seconds, seconds); + }, +); + const claudeSessionLine = (cwd: string) => `${JSON.stringify({ type: "user", cwd, sessionId: "s1" })}\n${JSON.stringify({ type: "assistant" })}\n`; @@ -326,6 +342,30 @@ const awaitGateReader = (gatePath: string) => (handle) => Effect.promise(() => handle.close()), ); +/** + * Offers each of `filePaths` to `finished` once its `cat` child has exited. A + * deadline fires for every read still in flight, so a test must wait for the + * unblocked reads before it moves the clock. + */ +const layerReportingProcessRunner = ( + filePaths: ReadonlySet, + finished: Queue.Queue, +) => + Layer.effect( + ProcessRunner.ProcessRunner, + Effect.gen(function* () { + const runner = yield* ProcessRunner.ProcessRunner; + return ProcessRunner.ProcessRunner.of({ + run: (input) => { + const filePath = input.args.at(-1) ?? ""; + return filePaths.has(filePath) + ? runner.run(input).pipe(Effect.ensuring(Queue.offer(finished, filePath))) + : runner.run(input); + }, + }); + }), + ).pipe(Layer.provide(ProcessRunner.layer), Layer.provide(NodeServices.layer)); + it.layer(NodeServices.layer)("AgentSessionScanner", (it) => { describe("scan", () => { it.effect("reads Claude project cwds from transcripts, newest first", () => @@ -1064,6 +1104,70 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => { }), ); + it.effect("imports a checkout's sessions from the linked worktrees the scan counts", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const fileSystem = yield* FileSystem.FileSystem; + const nowMs = Date.parse("2026-08-24T12:00:00.000Z"); + yield* TestClock.setTime(nowMs); + const claudeHomePath = yield* makeTempDir("t3code-claude-home-"); + const codexHomePath = yield* makeTempDir("t3code-codex-home-"); + const repo = yield* makeTempDir("t3code-workspace-repo-"); + yield* fileSystem.makeDirectory(path.join(repo, ".git")); + yield* fileSystem.writeFileString(path.join(repo, ".git", "config"), "[core]\n"); + const worktreeGitDir = path.join(repo, ".git", "worktrees", "feature"); + yield* fileSystem.makeDirectory(worktreeGitDir, { recursive: true }); + yield* fileSystem.writeFileString(path.join(worktreeGitDir, "commondir"), "../..\n"); + yield* fileSystem.writeFileString( + path.join(worktreeGitDir, "HEAD"), + "ref: refs/heads/feature\n", + ); + const worktree = path.join(repo, ".claude", "worktrees", "feature"); + yield* fileSystem.makeDirectory(worktree, { recursive: true }); + yield* fileSystem.writeFileString( + path.join(worktree, ".git"), + `gitdir: ${worktreeGitDir}\n`, + ); + + for (const [index, [cwd, sessionId]] of [ + [repo, "at-root"], + [worktree, "in-worktree"], + ].entries()) { + yield* writeTranscript({ + filePath: path.join(claudeHomePath, "projects", `-slug-${index}`, `${sessionId}.jsonl`), + contents: `${JSON.stringify({ + type: "user", + cwd, + sessionId, + message: { role: "user", content: "Fix the project" }, + })}\n${JSON.stringify({ + type: "assistant", + sessionId, + message: { role: "assistant", content: [{ type: "text", text: "Done" }] }, + })}\n`, + mtimeMs: nowMs - (index + 1) * 60 * 60 * 1000, + }); + } + + const input = { claudeHomePath, codexHomePath }; + const { candidates } = yield* runScan(input); + expect(candidates.map(({ path, threadCount }) => ({ path, threadCount }))).toEqual([ + { path: repo, threadCount: 2 }, + ]); + const outcomes = yield* runRecentThreadOutcomes({ ...input, workspaceRoot: repo }); + expect( + outcomes.flatMap((outcome) => + outcome._tag === "Importable" + ? [{ sessionId: outcome.thread.providerSessionId, worktree: outcome.worktree }] + : [], + ), + ).toEqual([ + { sessionId: "at-root", worktree: null }, + { sessionId: "in-worktree", worktree: { path: worktree, branch: "feature" } }, + ]); + }), + ); + it.effect("excludes sandboxes under the configured worktrees dir without .t3 in the path", () => Effect.gen(function* () { const path = yield* Path.Path; @@ -1220,6 +1324,32 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => { const result = yield* runScan({ claudeHomePath, codexHomePath }); + expect(result.candidates.map(({ path, threadCount }) => ({ path, threadCount }))).toEqual([ + { path: workspace, threadCount: 1 }, + ]); + }), + ); + it.effect("discovers projects whose Codex rollouts are zstd-compressed", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const claudeHomePath = yield* makeTempDir("t3code-claude-home-"); + const codexHomePath = yield* makeTempDir("t3code-codex-home-"); + const workspace = yield* makeTempDir("t3code-workspace-"); + yield* writeCompressedTranscript({ + filePath: path.join( + codexHomePath, + "sessions", + "2026", + "05", + "03", + "rollout-2026-05-03T00-00-00-019a5f00-0000-7000-8000-00000000000b.jsonl.zst", + ), + contents: codexRolloutLine(workspace), + mtimeMs: Date.parse("2026-05-03T00:00:00.000Z"), + }); + + const result = yield* runScan({ claudeHomePath, codexHomePath }); + expect(result.candidates.map(({ path, threadCount }) => ({ path, threadCount }))).toEqual([ { path: workspace, threadCount: 1 }, ]); @@ -1228,6 +1358,66 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => { }); describe("recentThreads", () => { + it.effect("imports Codex conversations stored only as zstd-compressed rollouts", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const nowMs = Date.parse("2026-08-24T12:00:00.000Z"); + yield* TestClock.setTime(nowMs); + const claudeHomePath = yield* makeTempDir("t3code-claude-home-"); + const codexHomePath = yield* makeTempDir("t3code-codex-home-"); + const workspace = yield* makeTempDir("t3code-workspace-"); + yield* writeCompressedTranscript({ + filePath: path.join( + codexHomePath, + "sessions", + "2026", + "08", + "20", + "rollout-2026-08-20T10-00-00-019a5f00-0000-7000-8000-00000000000c.jsonl.zst", + ), + contents: [ + encodeTranscriptRecord({ + type: "session_meta", + payload: { id: "019a5f00-0000-7000-8000-00000000000c", cwd: workspace }, + }), + encodeTranscriptRecord({ + type: "event_msg", + timestamp: "2026-08-20T10:00:00.000Z", + payload: { type: "user_message", message: "Compressed prompt" }, + }), + encodeTranscriptRecord({ + type: "response_item", + timestamp: "2026-08-20T10:01:00.000Z", + payload: { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "Compressed answer" }], + }, + }), + ].join("\n"), + mtimeMs: nowMs - 4 * 24 * 60 * 60 * 1000, + }); + + const threads = yield* runRecentThreads({ + claudeHomePath, + codexHomePath, + workspaceRoot: workspace, + }); + + expect( + threads.map((thread) => ({ + id: thread.providerSessionId, + messages: thread.messages.map((message) => message.text), + })), + ).toEqual([ + { + id: "019a5f00-0000-7000-8000-00000000000c", + messages: ["Compressed prompt", "Compressed answer"], + }, + ]); + }), + ); + it.effect("imports recent Claude, Codex, and Pi sessions for the selected project only", () => Effect.gen(function* () { const path = yield* Path.Path; @@ -1834,6 +2024,144 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => { }), ); + it.effect("ignores sessions without visible messages without spending the import budget", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const nowMs = Date.parse("2026-08-24T12:00:00.000Z"); + yield* TestClock.setTime(nowMs); + const claudeHomePath = yield* makeTempDir("t3code-claude-home-"); + const codexHomePath = yield* makeTempDir("t3code-codex-home-"); + const workspace = yield* makeTempDir("t3code-workspace-"); + const codexRollout = (name: string) => + path.join(codexHomePath, "sessions", "2026", "08", "24", `rollout-${name}.jsonl`); + // Newer empty sessions would exhaust the 100-transcript import budget before any real one. + const emptySessions = Array.from({ length: 100 }, (_, index) => + index % 2 === 0 + ? { + filePath: codexRollout(`empty-${index}`), + records: [ + { + type: "session_meta", + payload: { id: `codex-empty-${index}`, cwd: workspace, source: "vscode" }, + }, + { + type: "event_msg", + payload: { + type: "thread_settings_applied", + thread_settings: { cwd: workspace }, + }, + }, + ], + } + : { + filePath: path.join( + claudeHomePath, + "projects", + "-slug", + `claude-empty-${index}.jsonl`, + ), + records: [ + { type: "system", subtype: "informational", content: "Status", cwd: workspace }, + { + type: "user", + isMeta: true, + message: { + role: "user", + content: "Caveat", + }, + cwd: workspace, + }, + ], + }, + ); + const transcripts = [ + ...emptySessions, + { + // Without session_meta there is no resumable session ID. + filePath: codexRollout("unusable"), + records: [ + { type: "turn_context", payload: { cwd: workspace } }, + { + type: "event_msg", + payload: { type: "user_message", message: "Import this prompt" }, + }, + ], + }, + { + filePath: codexRollout("real"), + records: [ + { type: "session_meta", payload: { id: "real-session", cwd: workspace } }, + { + type: "event_msg", + payload: { type: "user_message", message: "Import this prompt" }, + }, + ], + }, + ]; + for (const [index, { filePath, records }] of transcripts.entries()) { + yield* writeTranscript({ + filePath, + contents: records.map((record) => encodeTranscriptRecord(record)).join("\n"), + mtimeMs: nowMs - index * 60_000, + }); + } + + const outcomes = yield* runRecentThreadOutcomes({ + claudeHomePath, + codexHomePath, + workspaceRoot: workspace, + }); + + expect( + outcomes.map((outcome) => + outcome._tag === "Importable" ? outcome.thread.providerSessionId : outcome._tag, + ), + ).toEqual(["Skipped", "real-session"]); + }), + ); + + it.effect("skips a rewound Codex rollout whose inherited history is unavailable", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const nowMs = Date.parse("2026-08-24T12:00:00.000Z"); + yield* TestClock.setTime(nowMs); + const claudeHomePath = yield* makeTempDir("t3code-claude-home-"); + const codexHomePath = yield* makeTempDir("t3code-codex-home-"); + const workspace = yield* makeTempDir("t3code-workspace-"); + const threadId = "019a0000-0000-7000-8000-000000000001"; + const rolloutId = "019a0000-0000-7000-8000-000000000002"; + // The ancestor rollout is missing, and this rollout added no messages of its own. + yield* writeTranscript({ + filePath: path.join( + codexHomePath, + "sessions", + "2026", + "08", + "24", + `rollout-2026-08-24T11-00-00-${threadId}_${rolloutId}.jsonl`, + ), + contents: encodeTranscriptRecord({ + type: "session_meta", + payload: { + id: threadId, + cwd: workspace, + history_mode: "paginated", + history_base: { thread_id: threadId, end_ordinal_exclusive: 3, end_byte_offset: 512 }, + }, + }), + mtimeMs: nowMs, + }); + + const outcomes = yield* runRecentThreadOutcomes({ + claudeHomePath, + codexHomePath, + workspaceRoot: workspace, + }); + + expect(outcomes).toEqual([{ _tag: "Skipped" }]); + }), + ); + it.effect("skips excessive records without blocking an older valid transcript", () => Effect.gen(function* () { const path = yield* Path.Path; @@ -2557,28 +2885,11 @@ describe.skipIf(HostProcess.Platform.defaultValue() === "win32")( }); } - // The deadline fires for every read still in flight, so on a slow runner the - // neighbours' `cat` children must have exited before the clock moves. - const neighbourConfigs = new Set( - [repos[0], repos[2]].map((repo) => path.join(repo, ".git", "config")), - ); const finishedNeighbourReads = yield* Queue.unbounded(); - const processRunner = Layer.effect( - ProcessRunner.ProcessRunner, - Effect.gen(function* () { - const runner = yield* ProcessRunner.ProcessRunner; - return ProcessRunner.ProcessRunner.of({ - run: (input) => { - const filePath = input.args.at(-1) ?? ""; - return neighbourConfigs.has(filePath) - ? runner - .run(input) - .pipe(Effect.ensuring(Queue.offer(finishedNeighbourReads, filePath))) - : runner.run(input); - }, - }); - }), - ).pipe(Layer.provide(ProcessRunner.layer), Layer.provide(NodeServices.layer)); + const processRunner = layerReportingProcessRunner( + new Set([repos[0], repos[2]].map((repo) => path.join(repo, ".git", "config"))), + finishedNeighbourReads, + ); for (let attempt = 1; attempt <= 2; attempt++) { const scan = yield* Effect.forkChild( @@ -2687,6 +2998,12 @@ describe.skipIf(HostProcess.Platform.defaultValue() === "win32")( }); } + const finishedPointerReads = yield* Queue.unbounded(); + const processRunner = layerReportingProcessRunner( + new Set([path.join(worktree, ".git")]), + finishedPointerReads, + ); + // One scanner across scans: later scans skip the files that blocked. yield* Effect.gen(function* () { const scanner = yield* AgentSessionScanner.AgentSessionScanner; @@ -2695,22 +3012,24 @@ describe.skipIf(HostProcess.Platform.defaultValue() === "win32")( ); const first = yield* Effect.forkChild(scanPaths); // Pointer files are read first, then the commondir files they lead to. - for (const gatePath of [ - path.join(pointerCheckout, ".git"), - path.join(worktreeGitDir, "commondir"), - ]) { - yield* Effect.scoped( - Effect.gen(function* () { - yield* awaitGateReader(gatePath); - yield* TestClock.adjust("2 seconds"); - }), - ); - } + yield* Effect.scoped( + Effect.gen(function* () { + yield* awaitGateReader(path.join(pointerCheckout, ".git")); + yield* Queue.take(finishedPointerReads); + yield* TestClock.adjust("2 seconds"); + }), + ); + yield* Effect.scoped( + Effect.gen(function* () { + yield* awaitGateReader(path.join(worktreeGitDir, "commondir")); + yield* TestClock.adjust("2 seconds"); + }), + ); expect(yield* Fiber.join(first)).toEqual([pointerCheckout]); for (let attempt = 2; attempt <= LIBUV_POOL_SIZE + 1; attempt++) { expect(yield* scanPaths, `scan ${attempt}`).toEqual([pointerCheckout]); } - }).pipe(Effect.provide(layerScannerTest({ claudeHomePath, codexHomePath }))); + }).pipe(Effect.provide(layerScannerTest({ claudeHomePath, codexHomePath, processRunner }))); // Hangs if the scans left libuv pool threads stuck in open(). yield* fileSystem.readFileString(path.join(worktree, ".git")); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), @@ -3294,6 +3613,7 @@ describe("parseAgentSessionTranscript", () => { "\nGitHub\n", "# AGENTS.md instructions for /tmp/project\n\n\nRules\n", '\nTab\n', + '{"page_id":null}', "Fix the import titles", ].join("\n"); const thread = AgentSessionScanner.parseAgentSessionTranscript({ @@ -3314,6 +3634,27 @@ describe("parseAgentSessionTranscript", () => { expect(thread?.messages.map((message) => message.text)).toEqual([prompt]); }); + it.each([ + "Explain in this transcript", + "Keep this incomplete example", + ])("keeps Codex page markup that is not a complete leading block: %s", (prompt) => { + const thread = AgentSessionScanner.parseAgentSessionTranscript({ + contents: [ + encodeTranscriptRecord({ type: "session_meta", payload: { id: "codex-session" } }), + encodeTranscriptRecord({ + type: "event_msg", + payload: { type: "user_message", message: prompt }, + }), + ].join("\n"), + source: "codex", + providerInstanceId: ProviderInstanceId.make("codex"), + fallbackSessionId: "fallback", + lastActiveAtMs: Date.parse("2026-08-25T08:00:00.000Z"), + }); + + expect(thread?.title).toBe(prompt); + }); + it("skips sessions without a visible user message", () => { const thread = AgentSessionScanner.parseAgentSessionTranscript({ contents: JSON.stringify({ diff --git a/apps/server/src/project/AgentSessionScanner.ts b/apps/server/src/project/AgentSessionScanner.ts index d9ca41b8f26b1..7a89f9c687de0 100644 --- a/apps/server/src/project/AgentSessionScanner.ts +++ b/apps/server/src/project/AgentSessionScanner.ts @@ -1,5 +1,6 @@ import * as NodeOS from "node:os"; import * as NodeSqlite from "node:sqlite"; +import * as NodeZlib from "node:zlib"; import { AgentSessionImportSource, @@ -17,11 +18,13 @@ import { import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; +import * as NodeStream from "@effect/platform-node/NodeStream"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import * as Pull from "effect/Pull"; import * as Schema from "effect/Schema"; import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; @@ -177,7 +180,11 @@ const decodeCodexHistoryBase = Schema.decodeUnknownOption( ); // Original rollouts end in their thread ID, rewound ones in `_`. const CODEX_ROLLOUT_ID_PATTERN = - /-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:_([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}))?\.jsonl$/i; + /-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:_([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}))?\.jsonl(?:\.zst)?$/i; +// Codex zstd-compresses older rollouts; their byte limits and offsets count decompressed bytes. +const COMPRESSED_TRANSCRIPT_SUFFIX = ".jsonl.zst"; +const isCompressedTranscript = (filePath: string) => + filePath.endsWith(COMPRESSED_TRANSCRIPT_SUFFIX); // Codex records a chat's project only here; moving a chat leaves its rollout cwd unchanged. const CODEX_STATE_DATABASE = "state_5.sqlite"; const decodeCodexSessionIndexEntry = Schema.decodeUnknownOption( @@ -216,6 +223,8 @@ export type AgentSessionRecentThread = readonly _tag: "Importable"; readonly thread: AgentSessionThread; readonly source: AgentSessionImportSource; + /** The root's linked worktree the session ran in, or null when it ran in the root. */ + readonly worktree: { readonly path: string; readonly branch: string | null } | null; } | { readonly _tag: "AlreadyImported"; readonly source: AgentSessionImportSource } | { readonly _tag: "Duplicate"; readonly source: AgentSessionImportSource } @@ -360,7 +369,7 @@ function foldClaudeFields(fields: ClaudeFields, record: DecodedTranscriptRecord) // Codex records setup context as user text ahead of the real request. Skip // these leading blocks when deriving a title; imported messages keep them. const CODEX_TITLE_PREAMBLE = - /^(?:\s*(?:<(recommended_plugins|environment_context|user_instructions|INSTRUCTIONS|in-app-browser-context)(?:\s[^>]*)?>[\s\S]*?<\/\1>|# AGENTS\.md instructions\b[^\n]*|## My request for Codex:))+/; + /^(?:\s*(?:<(recommended_plugins|environment_context|user_instructions|INSTRUCTIONS|in-app-browser-context|external_codex_apps_open_page)(?:\s[^>]*)?>[\s\S]*?<\/\1>|# AGENTS\.md instructions\b[^\n]*|## My request for Codex:))+/; // Codex appends an entry per rename; like Codex, the newest nonempty name wins. function parseCodexThreadNames(contents: string): ReadonlyMap { @@ -438,7 +447,8 @@ export function parseAgentSessionTranscript( input.source === "claudeAgent" ? records.reduce(foldClaudeFields, emptyClaudeFields) : emptyClaudeFields; - return parseAgentSessionRecords(input, records, claudeFields, 0); + const thread = parseAgentSessionRecords(input, records, claudeFields, 0); + return thread === "empty" ? null : thread; } function parseAgentSessionRecords( @@ -446,7 +456,7 @@ function parseAgentSessionRecords( records: ReadonlyArray, claudeFields: ClaudeFields, droppedMessageCount: number, -): AgentSessionThread | null { +): AgentSessionThread | "empty" | null { const fallbackTimestamp = DateTime.formatIso(DateTime.makeUnsafe(input.lastActiveAtMs)); // Claude filenames are session IDs. Codex rollout filenames include extra // timestamp text, so only transcript metadata can provide a resumable ID. @@ -639,6 +649,7 @@ function parseAgentSessionRecords( }); } + if (messages.length === 0 && droppedMessageCount === 0) return "empty"; const visibleMessages = messages.map( ({ codexResponseUser: _codexResponseUser, ...message }) => message, ); @@ -793,10 +804,7 @@ type TranscriptIdentity = Pick< const fileIdKey = (inode: TranscriptIdentity["inode"]) => (inode === null ? null : String(inode)); -export function sameTranscriptIdentity( - left: TranscriptIdentity, - right: TranscriptIdentity, -): boolean { +function sameTranscriptIdentity(left: TranscriptIdentity, right: TranscriptIdentity): boolean { // Unsafe legacy numbers may already have lost file ID precision. if ( (typeof left.inode === "number" && !Number.isSafeInteger(left.inode)) || @@ -861,6 +869,33 @@ export const make = Effect.gen(function* () { const statOption = (target: string) => fileSystem.stat(target).pipe(Effect.asSome, Effect.orElseSucceed(Option.none)); + const transcriptChunkReader = Effect.fn("AgentSessionScanner.transcriptChunkReader")(function* ( + file: FileSystem.File, + filePath: string, + ) { + if (!isCompressedTranscript(filePath)) return (size: number) => file.readAlloc(size); + const pull = yield* Stream.toPull( + fileSystem + .stream(filePath) + .pipe(NodeStream.pipeThroughSimple(() => NodeZlib.createZstdDecompress())), + ); + let pending: Array = []; + let done = false; + // Untraced: called once per chunk inside the caller's traced read. + return Effect.fnUntraced(function* (size: number) { + while (pending.length === 0) { + if (done) return Option.none(); + const next = yield* pull.pipe(Pull.catchDone(() => Effect.undefined)); + if (next === undefined) done = true; + else pending = [...next]; + } + const chunk = pending.shift(); + if (chunk === undefined) return Option.none(); + if (chunk.byteLength > size) pending.unshift(chunk.subarray(size)); + return Option.some(chunk.subarray(0, size)); + }); + }); + const directoryIdentity = Effect.fn("AgentSessionScanner.directoryIdentity")(function* ( target: string, knownStats?: FileSystem.File.Info, @@ -945,19 +980,24 @@ export const make = Effect.gen(function* () { | { readonly _tag: "Repository"; readonly git: AgentSessionProjectGit | null } | { readonly _tag: "Worktree"; + readonly gitDir: string; readonly mainCheckout: string | null; readonly git: AgentSessionProjectGit | null; } | { readonly _tag: "NotGit" }; const identities = new Map(); - const repositories: Array<{ directory: string; gitDir: string; linkedWorktree: boolean }> = []; + const repositories: Array<{ + directory: string; + gitDir: string; + worktreeGitDir: string | null; + }> = []; const pointers: Array<{ directory: string; gitPath: string }> = []; for (const directory of directories) { const gitPath = path.join(directory, ".git"); const gitStats = yield* statOption(gitPath); if (Option.isNone(gitStats)) identities.set(directory, { _tag: "NotGit" }); else if (gitStats.value.type === "Directory") { - repositories.push({ directory, gitDir: gitPath, linkedWorktree: false }); + repositories.push({ directory, gitDir: gitPath, worktreeGitDir: null }); } else pointers.push({ directory, gitPath }); } @@ -972,7 +1012,7 @@ export const make = Effect.gen(function* () { const gitDir = path.resolve(directory, target); if (/[\\/]worktrees[\\/][^\\/]+[\\/]?$/.test(gitDir)) { linkedWorktrees.push({ directory, gitDir }); - } else repositories.push({ directory, gitDir, linkedWorktree: false }); + } else repositories.push({ directory, gitDir, worktreeGitDir: null }); } const commonDirTexts = yield* readGitFiles( @@ -981,19 +1021,19 @@ export const make = Effect.gen(function* () { for (const { directory, gitDir } of linkedWorktrees) { const commonDir = (commonDirTexts.get(path.join(gitDir, "commondir")) ?? "").trim(); if (commonDir.length === 0) { - identities.set(directory, { _tag: "Worktree", mainCheckout: null, git: null }); + identities.set(directory, { _tag: "Worktree", gitDir, mainCheckout: null, git: null }); } else repositories.push({ directory, gitDir: path.resolve(gitDir, commonDir), - linkedWorktree: true, + worktreeGitDir: gitDir, }); } const configTexts = yield* readGitFiles( repositories.map(({ gitDir }) => path.join(gitDir, "config")), ); - for (const { directory, gitDir, linkedWorktree } of repositories) { + for (const { directory, gitDir, worktreeGitDir } of repositories) { const configText = configTexts.get(path.join(gitDir, "config")) ?? ""; const originUrl = parseOriginUrlFromGitConfig(configText); const git: AgentSessionProjectGit = { @@ -1002,9 +1042,10 @@ export const make = Effect.gen(function* () { repository: repositoryPathFromRemoteUrl(originUrl), }; // Bare repositories have no main checkout, so offer their linked worktrees independently. - if (linkedWorktree && !isBareGitConfig(configText)) { + if (worktreeGitDir !== null && !isBareGitConfig(configText)) { identities.set(directory, { _tag: "Worktree", + gitDir: worktreeGitDir, // Only a common dir named .git sits inside its main checkout, and an // unreadable config can't rule out a bare repository. mainCheckout: @@ -1018,6 +1059,61 @@ export const make = Effect.gen(function* () { return identities; }); + /** Maps each directory to the project it counts toward: a linked worktree counts toward its main checkout. */ + const resolveDirectoryProjects = Effect.fn("AgentSessionScanner.resolveDirectoryProjects")( + function* (resolvedDirectories: ReadonlyArray) { + // Stat every directory first so one batch reads all git identities. + const directories = new Map>(); + for (const resolved of resolvedDirectories) { + if (directories.has(resolved)) continue; + directories.set(resolved, yield* projectDirectoryStats(resolved)); + } + const directoryGitIdentities = yield* readGitIdentities( + Array.from(directories).flatMap(([directory, stats]) => + Option.isSome(stats) ? [directory] : [], + ), + ); + const projects = new Map< + string, + { + readonly key: string; + readonly path: string; + readonly git: AgentSessionProjectGit | null; + readonly worktreeGitDir: string | null; + } | null + >(); + for (const [resolved, stats] of directories) { + const gitIdentity = directoryGitIdentities.get(resolved); + if (Option.isNone(stats) || gitIdentity === undefined) { + projects.set(resolved, null); + } else if (gitIdentity._tag === "Worktree") { + const { mainCheckout } = gitIdentity; + const mainStats = + mainCheckout === null ? Option.none() : yield* projectDirectoryStats(mainCheckout); + projects.set( + resolved, + mainCheckout !== null && Option.isSome(mainStats) + ? { + key: yield* directoryIdentity(mainCheckout, mainStats.value), + path: mainCheckout, + git: gitIdentity.git, + worktreeGitDir: gitIdentity.gitDir, + } + : null, + ); + } else { + projects.set(resolved, { + key: yield* directoryIdentity(resolved, stats.value), + path: resolved, + git: gitIdentity._tag === "Repository" ? gitIdentity.git : null, + worktreeGitDir: null, + }); + } + } + return projects; + }, + ); + const readCwd = Effect.fn("AgentSessionScanner.readCwd")(function* ( source: AgentSessionSource, transcript: TranscriptCandidate, @@ -1037,11 +1133,15 @@ export const make = Effect.gen(function* () { fileSystem.open(transcript.filePath, { flag: "r" }).pipe( Effect.flatMap((file) => Effect.gen(function* () { + const readChunk = yield* transcriptChunkReader(file, transcript.filePath); + const size = isCompressedTranscript(transcript.filePath) + ? Number.POSITIVE_INFINITY + : transcript.identity.size; const decoder = new TextDecoder(); let remaining = ""; let bytesRead = 0; let recordsRead = 0; - const maxBytes = Math.min(MAX_TRANSCRIPT_SCAN_BYTES, transcript.identity.size); + const maxBytes = Math.min(MAX_TRANSCRIPT_SCAN_BYTES, size); const reserveRecord = () => { if ( recordsRead === MAX_METADATA_RECORDS_PER_TRANSCRIPT || @@ -1073,7 +1173,7 @@ export const make = Effect.gen(function* () { ); budget.operationsRemaining -= 1; budget.bytesRemaining -= readSize; - const next = yield* file.readAlloc(readSize); + const next = yield* readChunk(readSize); if (Option.isNone(next)) { return readLastRecord(); } @@ -1090,7 +1190,7 @@ export const make = Effect.gen(function* () { } } - if (bytesRead < transcript.identity.size) { + if (bytesRead < size) { budget.truncated = true; return null; } @@ -1107,8 +1207,12 @@ export const make = Effect.gen(function* () { source: AgentSessionSource, prefix?: { readonly endByteOffset: number; readonly historyBytes: number }, ) { - const readBytes = prefix?.endByteOffset ?? expected.size; - if (readBytes > expected.size || readBytes > MAX_IMPORTED_TRANSCRIPT_BYTES) return null; + const compressed = isCompressedTranscript(filePath); + const readBytes = + prefix?.endByteOffset ?? (compressed ? MAX_IMPORTED_TRANSCRIPT_BYTES : expected.size); + if ((!compressed && readBytes > expected.size) || readBytes > MAX_IMPORTED_TRANSCRIPT_BYTES) { + return null; + } return yield* Effect.scoped( fileSystem.open(filePath, { flag: "r" }).pipe( @@ -1117,6 +1221,7 @@ export const make = Effect.gen(function* () { if (!sameTranscriptIdentity(expected, transcriptIdentity(filePath, yield* file.stat))) { return null; } + const readChunk = yield* transcriptChunkReader(file, filePath); const records: Array = []; const claudeHistory = createClaudeHistoryWindow(); let cwd = Option.none(); @@ -1177,10 +1282,12 @@ export const make = Effect.gen(function* () { }; while (bytesRead < readBytes) { - const next = yield* file.readAlloc( + const next = yield* readChunk( Math.min(TRANSCRIPT_PREFIX_BYTES, readBytes - bytesRead), ); if (Option.isNone(next)) { + // A compressed rollout's decompressed length is only known at its end. + if (compressed && prefix === undefined) break; return null; } @@ -1398,7 +1505,12 @@ export const make = Effect.gen(function* () { } const directory = path.join(sessionsDir, year, month, day); for (const entry of (yield* readDirectory(directory)).toSorted().toReversed()) { - if (!entry.startsWith("rollout-") || !entry.endsWith(".jsonl")) continue; + if ( + !entry.startsWith("rollout-") || + !(entry.endsWith(".jsonl") || isCompressedTranscript(entry)) + ) { + continue; + } if (operationsRemaining <= 0) { truncated = true; break; @@ -1719,64 +1831,18 @@ export const make = Effect.gen(function* () { git: AgentSessionProjectGit | null; } >(); - // Resolve every candidate directory first so one batch reads all git identities. - const directories = new Map>(); - for (const candidate of raw) { - const expanded = expandHomePath(candidate.cwd.trim(), homeDir); - if (!path.isAbsolute(expanded)) continue; - const resolved = path.resolve(expanded); - if (directories.has(resolved)) continue; - directories.set(resolved, yield* projectDirectoryStats(resolved)); - } - const directoryGitIdentities = yield* readGitIdentities( - Array.from(directories).flatMap(([directory, stats]) => - Option.isSome(stats) ? [directory] : [], - ), + const directoryProjects = yield* resolveDirectoryProjects( + raw.flatMap((candidate) => { + const expanded = expandHomePath(candidate.cwd.trim(), homeDir); + return path.isAbsolute(expanded) ? [path.resolve(expanded)] : []; + }), ); - const directoryProjects = new Map< - string, - { - readonly key: string; - readonly path: string; - readonly git: AgentSessionProjectGit | null; - } | null - >(); - for (const candidate of raw) { const expanded = expandHomePath(candidate.cwd.trim(), homeDir); if (!path.isAbsolute(expanded)) continue; - const resolved = path.resolve(expanded); - const stats = directories.get(resolved); - if (stats === undefined || Option.isNone(stats)) continue; - let project = directoryProjects.get(resolved); - if (project === undefined) { - const gitIdentity = directoryGitIdentities.get(resolved); - if (gitIdentity === undefined) { - project = null; - } else if (gitIdentity._tag === "Worktree") { - // A linked worktree's sessions count toward its main checkout's project. - const { mainCheckout } = gitIdentity; - const mainStats = - mainCheckout === null ? Option.none() : yield* projectDirectoryStats(mainCheckout); - project = - mainCheckout !== null && Option.isSome(mainStats) - ? { - key: yield* directoryIdentity(mainCheckout, mainStats.value), - path: mainCheckout, - git: gitIdentity.git, - } - : null; - } else { - project = { - key: yield* directoryIdentity(resolved, stats.value), - path: resolved, - git: gitIdentity._tag === "Repository" ? gitIdentity.git : null, - }; - } - directoryProjects.set(resolved, project); - } - if (project === null) continue; + const project = directoryProjects.get(path.resolve(expanded)); + if (project === undefined || project === null) continue; const existing = merged.get(project.key); if (!existing) { @@ -1866,6 +1932,45 @@ export const make = Effect.gen(function* () { } const { candidates } = cachedCandidates; + // The scan counts a linked worktree's sessions toward its main checkout, so import them with it. + const candidateDirectories = candidates.flatMap((candidate) => { + const expanded = expandHomePath(candidate.cwd.trim(), homeDir); + return path.isAbsolute(expanded) ? [path.resolve(expanded)] : []; + }); + const directoryProjects = yield* resolveDirectoryProjects(candidateDirectories); + const belongsToRoot = Effect.fn("AgentSessionScanner.belongsToRoot")(function* ( + resolved: string, + ) { + if ((yield* directoryIdentity(resolved)) === rootIdentity) return true; + if (!directoryProjects.has(resolved)) { + const project = (yield* resolveDirectoryProjects([resolved])).get(resolved) ?? null; + directoryProjects.set(resolved, project); + } + return directoryProjects.get(resolved)?.key === rootIdentity; + }); + const worktreesByDirectory = new Map(); + const rootWorktree = Effect.fn("AgentSessionScanner.rootWorktree")(function* ( + resolved: string, + ) { + const known = worktreesByDirectory.get(resolved); + if (known !== undefined) return known; + const gitDir = + (yield* directoryIdentity(resolved)) === rootIdentity + ? null + : (directoryProjects.get(resolved)?.worktreeGitDir ?? null); + let worktree: { path: string; branch: string | null } | null = null; + if (gitDir !== null) { + const headPath = path.join(gitDir, "HEAD"); + const head = (yield* readGitFiles([headPath])).get(headPath) ?? ""; + worktree = { + path: resolved, + branch: /^ref: refs\/heads\/(.+)$/.exec(head.trim())?.[1] ?? null, + }; + } + worktreesByDirectory.set(resolved, worktree); + return worktree; + }); + const eligibleTranscripts: Array<{ readonly candidate: RawCandidate; readonly transcript: RawCandidate["transcripts"][number] & { readonly mtimeMs: number }; @@ -1874,7 +1979,7 @@ export const make = Effect.gen(function* () { const expanded = expandHomePath(candidate.cwd.trim(), homeDir); if (!path.isAbsolute(expanded)) continue; const resolved = path.resolve(expanded); - if ((yield* directoryIdentity(resolved)) !== rootIdentity) continue; + if (!(yield* belongsToRoot(resolved))) continue; for (const transcript of candidate.transcripts) { if ( @@ -2005,12 +2110,10 @@ export const make = Effect.gen(function* () { projectCwd(snapshot.cwd.value, transcript.projectRoots).trim(), homeDir, ); - if ( - !path.isAbsolute(expandedCwd) || - (yield* directoryIdentity(path.resolve(expandedCwd))) !== rootIdentity - ) { + if (!path.isAbsolute(expandedCwd) || !(yield* belongsToRoot(path.resolve(expandedCwd)))) { return Option.some({ _tag: "Skipped" }); } + const worktree = yield* rootWorktree(path.resolve(expandedCwd)); const inheritedRecords = candidate.source === "codex" @@ -2024,7 +2127,12 @@ export const make = Effect.gen(function* () { { source: candidate.source, providerInstanceId: candidate.providerInstanceId, - fallbackSessionId: path.basename(transcript.filePath, ".jsonl"), + fallbackSessionId: path.basename( + transcript.filePath, + isCompressedTranscript(transcript.filePath) + ? COMPRESSED_TRANSCRIPT_SUFFIX + : ".jsonl", + ), lastActiveAtMs: transcript.mtimeMs, }, inheritedRecords === null @@ -2033,6 +2141,15 @@ export const make = Effect.gen(function* () { snapshot.claudeFields, snapshot.droppedMessageCount, ); + if (parsedThread === "empty") { + // null means an unreadable history base, which may hold the conversation. + if (candidate.source === "codex" && inheritedRecords === null) { + return Option.some({ _tag: "Skipped" }); + } + // Never recorded as imported, so charging them would starve real sessions every run. + transcriptsRemaining += 1; + return Option.none(); + } if (parsedThread === null) { return Option.some({ _tag: "Skipped" }); } @@ -2061,6 +2178,7 @@ export const make = Effect.gen(function* () { thread: savedTitle === undefined ? parsedThread : { ...parsedThread, title: savedTitle }, source, + worktree, }); }).pipe(importReadLock.withPermits(1)), ), diff --git a/apps/server/src/project/ManagedProjectFolders.test.ts b/apps/server/src/project/ManagedProjectFolders.test.ts index 011b54a6851b3..07da49c732539 100644 --- a/apps/server/src/project/ManagedProjectFolders.test.ts +++ b/apps/server/src/project/ManagedProjectFolders.test.ts @@ -1,7 +1,13 @@ import { assert, it } from "@effect/vitest"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; -import { CommandId, ProjectId, ThreadId, VcsProcessSpawnError } from "@t3tools/contracts"; +import { + CommandId, + ProjectId, + ThreadId, + VcsProcessSpawnError, + VcsProcessTimeoutError, +} from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; @@ -174,6 +180,44 @@ it.effect("offers a Scratch folder when Git is not installed", () => ), ); +it.effect("offers the Scratch folder once a Git check that timed out is asked again", () => + withScratch( + ({ baseDir }) => + Effect.gen(function* () { + const path = yield* Path.Path; + const scratch = yield* ManagedProjectFolders.ManagedProjectFolders; + assert.isTrue(Option.isNone(yield* scratch.scratchRoot)); + assert.equal(yield* requireRoot, path.resolve(baseDir, "scratch")); + }), + { + vcsProcess: Layer.effect( + VcsProcess.VcsProcess, + Effect.map(VcsProcess.make, (real) => { + let timedOut = false; + return VcsProcess.VcsProcess.of({ + run: (input) => + Effect.suspend(() => { + if (timedOut) return real.run(input); + timedOut = true; + return Effect.fail( + new VcsProcessTimeoutError({ + operation: input.operation, + command: input.command, + cwd: input.cwd, + timeoutMs: 5_000, + }), + ); + }), + }); + }), + ).pipe( + Layer.provide(ProcessRunner.layer), + Layer.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), + ), + }, + ), +); + it.effect("creates one Scratch project, even for concurrent first requests", () => withScratch(() => Effect.gen(function* () { @@ -274,6 +318,15 @@ it.effect("gives each Scratch thread its own folder, named from its message", () ); assert.equal(path.dirname(pasted), root); assert.isAtMost(path.basename(pasted).length, 80); + + const accented = Option.getOrThrow( + yield* scratch.folderForThread({ + projectId, + threadId: ThreadId.make("thread-diacritics"), + text: "tìm kiếm skill", + }), + ); + assert.match(path.basename(accented), /^\d{4}-\d{2}-\d{2}-tim-kiem-skill-/); }), ), ); diff --git a/apps/server/src/project/ManagedProjectFolders.ts b/apps/server/src/project/ManagedProjectFolders.ts index 27316997077b1..d526c381fb404 100644 --- a/apps/server/src/project/ManagedProjectFolders.ts +++ b/apps/server/src/project/ManagedProjectFolders.ts @@ -6,7 +6,7 @@ import { type ThreadId, VcsProcessSpawnError, } from "@t3tools/contracts"; -import { newProjectFolderName } from "@t3tools/shared/path"; +import { foldLatinDiacritics, newProjectFolderName } from "@t3tools/shared/path"; import * as Cause from "effect/Cause"; import * as Context from "effect/Context"; import * as SecureRandom from "effect/crypto/SecureRandom"; @@ -66,6 +66,10 @@ export class ScratchProjectError extends Schema.TaggedError } } +/** Where No project threads keep their folders, whether or not this T3 home allows them. */ +export const scratchFolderRoot = (baseDir: string, path: Path.Path) => + path.resolve(baseDir, "scratch"); + export type ScratchError = ScratchUnavailableError | ScratchFolderError | ScratchProjectError; // The spawner checks cwd before spawning, so NotFound from spawn itself means a missing executable. @@ -129,7 +133,7 @@ export class ManagedProjectFolders extends Context.Service< >()("t3/project/ManagedProjectFolders") {} const folderWords = (text: string) => - text + foldLatinDiacritics(text) .toLowerCase() .split(/[^a-z0-9]+/) .filter(Boolean) @@ -255,7 +259,7 @@ const make = Effect.gen(function* () { Effect.flatMap( (repositoryRoot): Effect.Effect> => { if (repositoryRoot === null) { - return Effect.succeed(Result.succeed(path.resolve(config.baseDir, "scratch"))); + return Effect.succeed(Result.succeed(scratchFolderRoot(config.baseDir, path))); } const unavailable = new ScratchUnavailableError({ baseDir: config.baseDir, @@ -264,15 +268,18 @@ const make = Effect.gen(function* () { return Effect.logWarning(unavailable.message).pipe(Effect.as(Result.fail(unavailable))); }, ), - Effect.catchCause((cause) => - Cause.hasInterrupts(cause) - ? Effect.interrupt - : Effect.succeed(Result.fail(new ScratchUnavailableError({ baseDir: config.baseDir }))), - ), ), - Duration.infinity, + // A failed git check (say, a timeout on a busy machine) is not an answer; ask again next time. + Exit.match({ onSuccess: () => Duration.infinity, onFailure: () => Duration.zero }), + ); + const scratchProbe = probe.pipe( + Effect.catchCause((cause) => + Cause.hasInterrupts(cause) + ? Effect.interrupt + : Effect.succeed(Result.fail(new ScratchUnavailableError({ baseDir: config.baseDir }))), + ), + Effect.onInterrupt(() => invalidate), ); - const scratchProbe = probe.pipe(Effect.onInterrupt(() => invalidate)); // A symlinked scratch folder lets the No project live at either the link or its target. const scratchProjectRoot = (root: string) => fileSystem.realPath(root).pipe( diff --git a/apps/server/src/project/ProjectService.deletion.test.ts b/apps/server/src/project/ProjectService.deletion.test.ts index 28b1c40541ddf..25e5ee54a2737 100644 --- a/apps/server/src/project/ProjectService.deletion.test.ts +++ b/apps/server/src/project/ProjectService.deletion.test.ts @@ -28,6 +28,7 @@ import * as ProjectStore from "../orchestration-v2/ProjectStore.ts"; import * as ThreadCommandExecutor from "../orchestration-v2/ThreadCommandExecutor.ts"; import { planThreadDeletion } from "../orchestration-v2/ThreadDeletion.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as ProjectSettingsOverrideSweep from "../provider/ProjectSettingsOverrideSweep.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; import * as ProjectEnrichmentService from "./ProjectEnrichmentService.ts"; @@ -501,9 +502,13 @@ it.effect("deletes a project without force once its imported threads were delete }).pipe(Effect.provide(layerDatabase)), ); -it.effect( - "clears a deleted project's settings overrides so a machine-disabled instance cannot stay alive", - () => +it.effect.each([ + { policy: "no", worktreeCleanup: undefined }, + { policy: "its off", worktreeCleanup: { mode: "off" } as const }, + { policy: "its legacy", worktreeCleanup: null }, +])( + "clears a deleted project's settings overrides, keeping $policy worktree cleanup policy, so a machine-disabled instance cannot stay alive", + ({ worktreeCleanup }) => Effect.gen(function* () { const projectId = ProjectId.make("project:overrides-cleanup"); const instanceId = ProviderInstanceId.make("codex_work"); @@ -514,7 +519,11 @@ it.effect( const serverSettings = yield* ServerSettings.ServerSettingsService; yield* serverSettings.updateSettings({ projectSettingsOverrides: { - [projectId]: { providerInstances: { [instanceId]: { enabled: true } } }, + [projectId]: { + providerInstances: { [instanceId]: { enabled: true } }, + defaultAutoPull: true, + ...(worktreeCleanup === undefined ? {} : { worktreeCleanup }), + }, }, }); @@ -523,8 +532,35 @@ it.effect( projectId, }); - const settingsAfterDelete = yield* serverSettings.getSettings; - assert.isUndefined(settingsAfterDelete.projectSettingsOverrides[projectId]); + const entry = (yield* serverSettings.getSettings).projectSettingsOverrides[projectId]; + if (worktreeCleanup === undefined) { + assert.isUndefined(entry); + } else { + assert.deepStrictEqual(entry, { worktreeCleanup, providerInstances: {} }); + } + + const liveProjectId = ProjectId.make("project:still-live"); + const missingProjectId = ProjectId.make("project:missing"); + yield* seedProject(liveProjectId); + // Older deletions or interrupted best-effort cleanup can leave the original overrides on restart. + yield* serverSettings.updateSettings({ + projectSettingsOverrides: { + [projectId]: { + providerInstances: { [instanceId]: { enabled: true } }, + defaultAutoPull: true, + ...(worktreeCleanup === undefined ? {} : { worktreeCleanup }), + }, + [liveProjectId]: { defaultAutoPull: true }, + [missingProjectId]: { worktreeCleanup: { mode: "off" } }, + }, + }); + yield* ProjectSettingsOverrideSweep.sweep; + assert.deepStrictEqual((yield* serverSettings.getSettings).projectSettingsOverrides, { + [liveProjectId]: { defaultAutoPull: true }, + ...(worktreeCleanup === undefined + ? {} + : { [projectId]: { worktreeCleanup, providerInstances: {} } }), + }); }).pipe(Effect.provide(layerServices)); }).pipe(Effect.provide(layerDatabase)), ); diff --git a/apps/server/src/project/ProjectService.test.ts b/apps/server/src/project/ProjectService.test.ts index 32ce8b5f27026..492379beecc26 100644 --- a/apps/server/src/project/ProjectService.test.ts +++ b/apps/server/src/project/ProjectService.test.ts @@ -31,6 +31,7 @@ import * as ProjectStore from "../orchestration-v2/ProjectStore.ts"; import * as RuntimeLayer from "../orchestration-v2/runtimeLayer.ts"; import * as ThreadCommandExecutor from "../orchestration-v2/ThreadCommandExecutor.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as ProcessRunner from "../processRunner.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; import * as ProjectEnrichmentService from "./ProjectEnrichmentService.ts"; @@ -552,6 +553,12 @@ it.layer(layerTest)("ProjectService", (it) => { title: "Delete retry", workspaceRoot: "/work/delete-retry", }); + // Enrichment landing between the two deletes would change the project the retry reads. + yield* waitForProject( + service, + projectId, + (project) => project.faviconPath !== null && project.repositoryIdentity !== null, + ); const input = { commandId: CommandId.make("command:delete-retry:delete"), projectId }; const deleted = yield* service.delete(input); assert.deepEqual(yield* service.delete(input), deleted); @@ -777,6 +784,69 @@ it.effect("resolves a restored icon immediately after reselecting its saved path }).pipe(Effect.provide(NodeServices.layer)), ); +it.effect("refuses a worktree container as a new project and as a new workspace root", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const processRunner = yield* ProcessRunner.ProcessRunner; + const fixture = yield* fileSystem.makeTempDirectoryScoped({ prefix: "project-container-" }); + const git = (cwd: string, ...args: ReadonlyArray) => + processRunner.run({ + command: "git", + args: ["-C", cwd, "-c", "user.email=t@t", "-c", "user.name=t", ...args], + }); + const source = `${fixture}/source`; + const container = `${fixture}/container`; + yield* fileSystem.makeDirectory(source); + yield* fileSystem.makeDirectory(container); + yield* git(source, "init", "-b", "develop"); + yield* fileSystem.writeFileString(`${source}/README.md`, "readme\n"); + yield* git(source, "add", "README.md"); + yield* git(source, "commit", "-m", "init"); + yield* git(fixture, "clone", "--bare", source, `${container}/.bare`); + yield* fileSystem.writeFileString(`${container}/.git`, "gitdir: ./.bare\n"); + yield* git(container, "worktree", "add", "develop", "develop"); + + yield* Effect.gen(function* () { + const service = yield* ProjectService.ProjectService; + // As cloned the container is bare; Initialize Git or a config edit clears core.bare. + for (const coreBare of ["true", "false"]) { + yield* git(container, "config", "core.bare", coreBare); + const rejected = yield* service + .create({ + commandId: CommandId.make(`command:container:create:${coreBare}`), + projectId: ProjectId.make(`project:container:${coreBare}`), + title: "Container", + workspaceRoot: container, + }) + .pipe(Effect.flip); + assert.equal(rejected._tag, "ProjectInvalidError", `core.bare=${coreBare}`); + assert.include(rejected.message, "Add one of its worktrees"); + assert.isTrue(Option.isNone(yield* service.getByWorkspaceRoot(container))); + } + + const projectId = ProjectId.make("project:container:worktree"); + const worktree = yield* service.create({ + commandId: CommandId.make("command:container:worktree"), + projectId, + title: "Worktree", + workspaceRoot: `${container}/develop`, + }); + const moved = yield* service + .update({ + commandId: CommandId.make("command:container:move"), + projectId, + workspaceRoot: container, + }) + .pipe(Effect.flip); + assert.equal(moved._tag, "ProjectInvalidError"); + assert.equal( + Option.getOrThrow(yield* service.getById(projectId)).workspaceRoot, + worktree.workspaceRoot, + ); + }).pipe(Effect.provide(layerTest)); + }).pipe(Effect.provide(ProcessRunner.layer.pipe(Layer.provideMerge(NodeServices.layer)))), +); + it.effect("serializes two projects claiming the same workspace root", () => Effect.gen(function* () { const eventSink = yield* EventSink.EventSinkV2; diff --git a/apps/server/src/project/ProjectService.ts b/apps/server/src/project/ProjectService.ts index 464454ebba763..4bdc8386438ca 100644 --- a/apps/server/src/project/ProjectService.ts +++ b/apps/server/src/project/ProjectService.ts @@ -33,8 +33,11 @@ import * as ProjectStore from "../orchestration-v2/ProjectStore.ts"; import * as ThreadCommandExecutor from "../orchestration-v2/ThreadCommandExecutor.ts"; import { planThreadDeletion } from "../orchestration-v2/ThreadDeletion.ts"; import * as ServerSettings from "../serverSettings.ts"; +import { deletedProjectOverridePatch } from "../provider/ProjectSettingsOverrideSweep.ts"; import { makeIsManagedWorktreePath } from "../worktreesDirectory.ts"; +import * as ProcessRunner from "../processRunner.ts"; import * as ProjectEnrichmentService from "./ProjectEnrichmentService.ts"; +import { makeIsWorktreeContainer } from "./worktreeContainer.ts"; import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; export interface ProjectCreateInput extends ProjectCreatePayload { @@ -182,6 +185,9 @@ export const make = Effect.gen(function* () { const serverSettings = yield* ServerSettings.ServerSettingsService; const fileSystem = yield* FileSystem.FileSystem; const isManagedWorktreePath = yield* makeIsManagedWorktreePath; + const isWorktreeContainer = yield* makeIsWorktreeContainer.pipe( + Effect.provide(ProcessRunner.layer), + ); const projectLocks = yield* KeyedLock.make(); const workspaceLocks = yield* KeyedLock.make(); @@ -244,6 +250,17 @@ export const make = Effect.gen(function* () { ), ); + const rejectWorktreeContainer = Effect.fn("ProjectService.rejectWorktreeContainer")(function* ( + projectId: ProjectId, + workspaceRoot: string, + ) { + if (!(yield* isWorktreeContainer(workspaceRoot))) return; + return yield* new ProjectInvalidError({ + projectId, + detail: `${workspaceRoot} holds a repository's worktrees but is not a checkout itself. Add one of its worktrees as the project instead.`, + }); + }); + const commit = Effect.fn("ProjectService.commit")(function* (command: ProjectCommand) { const { projectId } = command; const dispatchError = (cause: unknown) => @@ -383,6 +400,7 @@ export const make = Effect.gen(function* () { workspaceRoot: input.workspaceRoot, createIfMissing: input.createWorkspaceRootIfMissing ?? false, }); + yield* rejectWorktreeContainer(input.projectId, workspaceRoot); yield* commit({ type: "project.create", commandId: input.commandId, @@ -410,6 +428,9 @@ export const make = Effect.gen(function* () { projectId: input.projectId, workspaceRoot: input.workspaceRoot, }); + if (workspaceRoot !== previousRoot) { + yield* rejectWorktreeContainer(input.projectId, workspaceRoot); + } yield* commit({ type: "project.meta.update", commandId: input.commandId, @@ -530,7 +551,14 @@ export const make = Effect.gen(function* () { // Best-effort cleanup releases disabled instances kept alive by overrides; crashes have no replay to revisit deletion. const clearProjectSettingsOverrides = (projectId: ProjectId) => - serverSettings.updateSettings({ projectSettingsOverrides: { [projectId]: null } }).pipe( + serverSettings.getSettings.pipe( + Effect.flatMap(({ projectSettingsOverrides }) => + serverSettings.updateSettings({ + projectSettingsOverrides: { + [projectId]: deletedProjectOverridePatch(projectSettingsOverrides[projectId]), + }, + }), + ), Effect.asVoid, Effect.catchCause((cause) => Effect.logWarning( diff --git a/apps/server/src/project/ProjectSetupScriptRunner.test.ts b/apps/server/src/project/ProjectSetupScriptRunner.test.ts index 034b168e52c79..fd40f24d5a908 100644 --- a/apps/server/src/project/ProjectSetupScriptRunner.test.ts +++ b/apps/server/src/project/ProjectSetupScriptRunner.test.ts @@ -1,16 +1,20 @@ import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; -import { describe, expect, it } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, describe, expect, it } from "@effect/vitest"; import { ProjectId, TerminalNotRunningError, type TerminalEvent, type TerminalSessionSnapshot, } from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Queue from "effect/Queue"; +import * as Stream from "effect/Stream"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; import { TestClock } from "effect/testing"; import * as ServerSettings from "../serverSettings.ts"; @@ -164,7 +168,19 @@ describe("ProjectSetupScriptRunner", () => { yield* emit({ ...event, type: "activity", hasRunningSubprocess: false, label: "zsh" }); yield* emit({ ...event, type: "output", data: "\r\n% \x1b[?2004h" }); - expect(yield* Queue.take(writes)).toMatch(/^ \( vp i\r\);/); + expect(yield* Queue.take(writes)).toContain("vp i"); + }), + ); + + it.effect("drops whole escape sequences from output lines", () => + Effect.gen(function* () { + const { outputLines, emit } = yield* setup(); + yield* emit({ + ...event, + type: "output", + data: "\x1b7\x1b[24;0fProgress: [ 40%]\x1b8\r\n\x1b%G\x1b#8\x1b[4:3mdone\x1bM\r\n", + }); + expect(yield* Queue.takeAll(outputLines)).toEqual(["Progress: [ 40%]", "done"]); }), ); @@ -172,7 +188,7 @@ describe("ProjectSetupScriptRunner", () => { Effect.gen(function* () { const { writes } = yield* setup(); yield* TestClock.adjust("3 seconds"); - expect(yield* Queue.take(writes)).toMatch(/^ \( vp i\r\);/); + expect(yield* Queue.take(writes)).toContain("vp i"); }), ); @@ -194,7 +210,7 @@ describe("ProjectSetupScriptRunner", () => { yield* emit({ ...event, type: "restarted", snapshot: openSnapshot("") }); // The restart lands inside a quiet window, so the next full window decides. yield* TestClock.adjust("6 seconds"); - expect(yield* Queue.take(writes)).toMatch(/^ \( vp i\r\);/); + expect(yield* Queue.take(writes)).toContain("vp i"); }), ); @@ -227,3 +243,110 @@ describe("ProjectSetupScriptRunner", () => { }), ); }); + +const runObservedSetup = Effect.fnUntraced(function* (command: string) { + const { writes, outputLines, emit, completion } = yield* setup({ command }); + yield* emit({ ...event, type: "output", data: "\x1b[?2004h" }); + const submitted = yield* Queue.take(writes); + const cwd = yield* HostProcess.WorkingDirectory; + const env = yield* HostProcess.Environment; + + // Enter becomes a newline at the terminal boundary; transport assertions below + // cover line-editor submissions and canonical TTY limits that bash -c bypasses. + const child = yield* Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const handle = yield* spawner.spawn( + ChildProcess.make( + "/bin/bash", + ["--noprofile", "--norc", "-c", submitted.replaceAll("\r", "\n")], + { + cwd, + env: { PATH: env.PATH }, + }, + ), + ); + return yield* Effect.all( + { + stdout: handle.stdout.pipe(Stream.decodeText(), Stream.mkString), + stderr: handle.stderr.pipe(Stream.decodeText(), Stream.mkString), + exitCode: handle.exitCode, + }, + { concurrency: "unbounded" }, + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)); + yield* emit({ ...event, type: "output", data: submitted + "\n" }); + yield* emit({ ...event, type: "output", data: child.stdout + child.stderr }); + yield* emit({ ...event, type: "closed" }); + return { + submitted, + lines: yield* Queue.clear(outputLines), + exitCode: (yield* completion).exitCode, + }; +}); + +describe.skipIf(HostProcess.Platform.defaultValue() === "win32")("POSIX setup completion", () => { + it.live("submits one line while preserving multiline shell syntax and literal text", () => + Effect.gen(function* () { + const result = yield* runObservedSetup( + [ + "printf '%s\\n' \"it's ready\" 'literal \\n $(printf wrong) `printf wrong` café'", + "cat <<'END'", + "$literal \\path 雪", + "END", + "printf '%s\\n' done # trailing comment", + ].join("\r\n"), + ); + assert.equal(result.submitted.at(-1), "\r"); + assert.notMatch(result.submitted.slice(0, -1), /[\r\n]/); + assert.deepEqual(result.lines, [ + "it's ready", + "literal \\n $(printf wrong) `printf wrong` café", + "$literal \\path 雪", + "done", + ]); + assert.equal(result.exitCode, 0); + }), + ); + + it.live("completes long scripts without exceeding canonical terminal line limits", () => + Effect.gen(function* () { + const expected = Array.from( + { length: 120 }, + (_, index) => `line-${index}: café 雪 'quoted' \\path`, + ); + const command = expected + .map((line) => `printf '%s\\n' "${line.replaceAll("\\", "\\\\")}"`) + .join("\n"); + const utf8 = new TextEncoder(); + assert.isAbove(utf8.encode(command).byteLength, 4_096); + + const result = yield* runObservedSetup(command); + assert.deepEqual(result.lines, expected); + assert.equal(result.exitCode, 0); + assert.equal(result.submitted.at(-1), "\r"); + assert.notInclude(result.submitted, "\n"); + // The runner can write before the shell switches the terminal out of canonical mode. + for (const line of result.submitted.slice(0, -1).split("\r")) { + assert.isBelow(utf8.encode(line).byteLength, 1_024); + } + }), + ); + + it.live("preserves a trailing backslash-newline even when losing it would still exit zero", () => + Effect.gen(function* () { + const result = yield* runObservedSetup("printf '%s\\n' direct\\\n"); + assert.deepEqual(result.lines, ["direct"]); + assert.equal(result.exitCode, 0); + }), + ); + + it.live.each([ + { name: "an explicit exit", command: "exit 7", exitCode: 7 }, + { name: "a syntax error", command: "if then", exitCode: 2 }, + ])("reports a nonzero completion for $name", ({ command, exitCode }) => + Effect.gen(function* () { + const result = yield* runObservedSetup(command); + assert.equal(result.exitCode, exitCode); + }), + ); +}); diff --git a/apps/server/src/project/ProjectSetupScriptRunner.ts b/apps/server/src/project/ProjectSetupScriptRunner.ts index a049bc89b01a1..60084f648a182 100644 --- a/apps/server/src/project/ProjectSetupScriptRunner.ts +++ b/apps/server/src/project/ProjectSetupScriptRunner.ts @@ -145,12 +145,10 @@ function completionSentinelPattern(token: string): RegExp { return new RegExp(`${COMPLETION_SENTINEL_PREFIX}_${token}:(-?\\d+)`); } +// ECMA-48: CSI, OSC, then any other escape as intermediates (0x20-0x2F) and one final byte. function stripTerminalControl(text: string): string { return text - .replace( - /\x1b\[[0-9;?]*[ -/]*[@-~]|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)|\x1b[()][A-Za-z0-9]|\x1b[=>]/g, - "", - ) + .replace(/\x1b\[[0-?]*[ -/]*[@-~]|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)|\x1b[ -/]*[0-~]/g, "") .replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, ""); } @@ -177,7 +175,57 @@ function resolveCompletionShell(shell: string): CompletionShell | null { return "posix"; } -// A block with its own closing line protects the sentinel from trailing comments, heredocs and scripts reading stdin. +const utf8 = new TextEncoder(); +// Canonical TTY input (before a line editor starts reading) caps a line below 1 KiB on Linux. +const POSIX_MAX_LINE_BYTES = 1_023; +const POSIX_APPEND = '__t3_command="$__t3_command"'; + +function encodePosixScript(text: string): string { + let encoded = ""; + for (const character of text) { + const code = character.charCodeAt(0); + if (character === "'") { + encoded += "'\\''"; + } else if (character === "\\" || code <= 0x1f || code === 0x7f) { + encoded += `\\0${code.toString(8).padStart(3, "0")}`; + } else { + encoded += character; + } + } + return encoded; +} + +// Line editors run each submitted line on its own, so every line must be a complete command. +function wrapPosixCommand(command: string, sentinel: string): string { + const normalized = command.replace(/\r\n/g, "\n"); + // The `_` suffix keeps trailing newlines; the subshell contains exits and syntax errors. + const evaluate = (payload: string) => + `( __t3_command=$(printf '%b_' ${payload}); eval "\${__t3_command%_}" ); printf '\\n${sentinel}%s\\n' "$?"`; + const inline = evaluate(`'${encodePosixScript(normalized)}'`); + if (utf8.encode(` ${inline}\r`).byteLength <= POSIX_MAX_LINE_BYTES) return inline; + + const maxChunkBytes = POSIX_MAX_LINE_BYTES - utf8.encode(` ${POSIX_APPEND}''\r`).byteLength; + const lines: string[] = []; + let prefix = "__t3_command="; + let chunk = ""; + let chunkBytes = 0; + for (const character of normalized) { + const encoded = encodePosixScript(character); + const bytes = utf8.encode(encoded).byteLength; + if (chunkBytes + bytes > maxChunkBytes) { + lines.push(`${prefix}'${chunk}'`); + prefix = POSIX_APPEND; + chunk = ""; + chunkBytes = 0; + } + chunk += encoded; + chunkBytes += bytes; + } + lines.push(`${prefix}'${chunk}'`, `${evaluate('"$__t3_command"')}; unset __t3_command`); + // The leading space gives every line the history suppression a short command gets. + return lines.join("\r "); +} + function wrapCommandForCompletion( command: string, shell: CompletionShell, @@ -204,7 +252,7 @@ function wrapCommandForCompletion( ].join("; "); } case "posix": - return `( ${body}\r); printf '\\n${sentinel}%s\\n' "$?"`; + return wrapPosixCommand(command, sentinel); } } diff --git a/apps/server/src/project/RepositoryIdentityResolver.test.ts b/apps/server/src/project/RepositoryIdentityResolver.test.ts index 2953d0dcbf096..50d749c3fb1b0 100644 --- a/apps/server/src/project/RepositoryIdentityResolver.test.ts +++ b/apps/server/src/project/RepositoryIdentityResolver.test.ts @@ -5,8 +5,10 @@ import { expect, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; +import { vi } from "vite-plus/test"; import * as ProcessRunner from "../processRunner.ts"; +import { sourceControlDefinitions } from "../sourceControl/sourceControlDefinitions.ts"; import * as RepositoryIdentityResolver from "./RepositoryIdentityResolver.ts"; const normalizePathSeparators = (value: string) => value.replaceAll("\\", "/"); @@ -70,6 +72,30 @@ it.layer(NodeServices.layer)("RepositoryIdentityResolverLive", (it) => { }).pipe(Effect.provide(RepositoryIdentityResolver.layer)), ); + it.effect("keeps a literal .git in an Azure DevOps repository name", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-repository-identity-azure-git-suffix-test-", + }); + + yield* git(cwd, ["init"]); + yield* git(cwd, [ + "remote", + "add", + "origin", + "https://dev.azure.com/Acme/Platform/_git/Tool.git", + ]); + + const resolver = yield* RepositoryIdentityResolver.RepositoryIdentityResolver; + const identity = yield* resolver.resolve(cwd); + + expect(identity?.displayName).toBe("Acme/Platform/_git/Tool.git"); + expect(identity?.name).toBe("Tool.git"); + expect(sourceControlDefinitions.repositorySelector(identity)).toBe("Tool.git"); + }).pipe(Effect.provide(RepositoryIdentityResolver.layer)), + ); + it.effect("resolves the fetch remote of a partial clone", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; @@ -111,31 +137,34 @@ it.layer(NodeServices.layer)("RepositoryIdentityResolverLive", (it) => { }).pipe(Effect.provide(RepositoryIdentityResolver.layer)), ); - it.effect("returns the git top-level root path when resolving from a nested workspace", () => - Effect.gen(function* () { - const fileSystem = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const repoRoot = yield* fileSystem.makeTempDirectoryScoped({ - prefix: "t3-repository-identity-nested-root-test-", - }); - const nestedWorkspace = path.join(repoRoot, "packages", "web"); + it.effect.each(["repo", "repo "] as const)( + "returns the git top-level root path when resolving from a nested workspace in %j", + (rootName) => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const tempDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-repository-identity-nested-root-test-", + }); + const repoRoot = path.join(tempDir, rootName); + const nestedWorkspace = path.join(repoRoot, "packages", "web"); - yield* fileSystem.makeDirectory(nestedWorkspace, { recursive: true }); - yield* git(repoRoot, ["init"]); - yield* git(repoRoot, ["remote", "add", "origin", "git@github.com:T3Tools/t3code.git"]); + yield* fileSystem.makeDirectory(nestedWorkspace, { recursive: true }); + yield* git(repoRoot, ["init"]); + yield* git(repoRoot, ["remote", "add", "origin", "git@github.com:T3Tools/t3code.git"]); - const resolver = yield* RepositoryIdentityResolver.RepositoryIdentityResolver; - const identity = yield* resolver.resolve(nestedWorkspace); - const resolvedIdentityRoot = - identity?.rootPath === undefined ? "" : NodeFS.realpathSync.native(identity.rootPath); - const resolvedRepoRoot = NodeFS.realpathSync.native(repoRoot); + const resolver = yield* RepositoryIdentityResolver.RepositoryIdentityResolver; + const identity = yield* resolver.resolve(nestedWorkspace); + const resolvedIdentityRoot = + identity?.rootPath === undefined ? "" : NodeFS.realpathSync.native(identity.rootPath); + const resolvedRepoRoot = NodeFS.realpathSync.native(repoRoot); - expect(identity).not.toBeNull(); - expect(identity?.canonicalKey).toBe("github.com/t3tools/t3code"); - expect(normalizeResolvedPath(resolvedIdentityRoot)).toBe( - normalizeResolvedPath(resolvedRepoRoot), - ); - }).pipe(Effect.provide(RepositoryIdentityResolver.layer)), + expect(identity).not.toBeNull(); + expect(identity?.canonicalKey).toBe("github.com/t3tools/t3code"); + expect(normalizeResolvedPath(resolvedIdentityRoot)).toBe( + normalizeResolvedPath(resolvedRepoRoot), + ); + }).pipe(Effect.provide(RepositoryIdentityResolver.layer)), ); it.effect.each(["bare clone", "dot-bare"] as const)( @@ -340,6 +369,75 @@ it.layer(NodeServices.layer)("RepositoryIdentityResolverLive", (it) => { }).pipe(Effect.provide(RepositoryIdentityResolver.layer)), ); + it.effect("follows an OWNER/REPO mark only to a remote that fetches that repository", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-repository-identity-gh-parent-test-", + }); + const resolver = yield* RepositoryIdentityResolver.RepositoryIdentityResolver; + const resolveFresh = () => resolver.resolve(cwd, { refresh: true }); + + yield* git(cwd, ["init"]); + yield* git(cwd, ["remote", "add", "fork", "https://github.com/julius/t3code-fork.git"]); + yield* git(cwd, ["remote", "add", "main", "https://github.com/T3Tools/t3code.git"]); + yield* git(cwd, ["remote", "add", "upstream", "https://github.com/other/t3code.git"]); + yield* git(cwd, ["config", "remote.fork.gh-resolved", "t3tools/t3code"]); + const parent = yield* resolveFresh(); + expect(parent?.canonicalKey).toBe("github.com/t3tools/t3code"); + expect(parent?.locator).toEqual({ + source: "git-remote", + remoteName: "main", + remoteUrl: "https://github.com/T3Tools/t3code.git", + }); + + // No remote fetches the marked repository, and several marks leave gh's choice unknown. + yield* git(cwd, ["config", "remote.fork.gh-resolved", "someone/else"]); + expect((yield* resolveFresh())?.locator.remoteName).toBe("upstream"); + yield* git(cwd, ["config", "remote.fork.gh-resolved", "base"]); + yield* git(cwd, ["config", "remote.main.gh-resolved", "base"]); + expect((yield* resolveFresh())?.locator.remoteName).toBe("upstream"); + + // The same path on another HTTPS port is a different server. + yield* git(cwd, ["config", "--unset", "remote.fork.gh-resolved"]); + yield* git(cwd, ["config", "--unset", "remote.main.gh-resolved"]); + yield* git(cwd, ["remote", "add", "backup", "https://ghe.example.com:9443/acme/web.git"]); + yield* git(cwd, ["remote", "add", "ghe-fork", "https://ghe.example.com:8443/me/web.git"]); + yield* git(cwd, ["remote", "add", "ghe-parent", "https://ghe.example.com:8443/acme/web.git"]); + yield* git(cwd, ["config", "remote.ghe-fork.gh-resolved", "acme/web"]); + expect((yield* resolveFresh())?.locator.remoteName).toBe("ghe-parent"); + }).pipe(Effect.provide(RepositoryIdentityResolver.layer)), + ); + + it.effect("reads the project's gh default when the server inherited GIT_DIR", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-repository-identity-gh-default-git-dir-", + }); + const other = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-repository-identity-inherited-git-dir-", + }); + + yield* git(cwd, ["init"]); + yield* git(cwd, ["remote", "add", "origin", "git@github.com:julius/t3code-fork.git"]); + yield* git(cwd, ["remote", "add", "upstream", "git@github.com:T3Tools/t3code.git"]); + yield* git(cwd, ["config", "remote.origin.gh-resolved", "base"]); + yield* git(other, ["init"]); + yield* git(other, ["config", "remote.upstream.gh-resolved", "base"]); + + vi.stubEnv("GIT_DIR", path.join(other, ".git")); + yield* Effect.addFinalizer(() => Effect.sync(() => vi.unstubAllEnvs())); + + const resolver = yield* RepositoryIdentityResolver.RepositoryIdentityResolver; + const identity = yield* resolver.resolve(cwd); + + expect(identity?.canonicalKey).toBe("github.com/julius/t3code-fork"); + expect(identity?.locator.remoteName).toBe("origin"); + }).pipe(Effect.provide(RepositoryIdentityResolver.layer)), + ); + it.effect("uses the last remote path segment as the repository name for nested groups", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; diff --git a/apps/server/src/project/RepositoryIdentityResolver.ts b/apps/server/src/project/RepositoryIdentityResolver.ts index a1cf97d4a8398..826529e704b23 100644 --- a/apps/server/src/project/RepositoryIdentityResolver.ts +++ b/apps/server/src/project/RepositoryIdentityResolver.ts @@ -1,5 +1,7 @@ import type { RepositoryIdentity, SourceControlProviderError } from "@t3tools/contracts"; +import { parseGitRemoteHost } from "@t3tools/source-control-core/shared/definition"; import { repositoryPathFromRemoteUrl } from "@t3tools/source-control-core/server/SourceControlProvider"; +import { canonicalRepositoryPath, normalizeGitRemoteUrl } from "@t3tools/shared/git"; import * as Cache from "effect/Cache"; import * as Context from "effect/Context"; import * as Data from "effect/Data"; @@ -69,21 +71,44 @@ function parseRemoteFetchUrls(stdout: string): Map { return remotes; } -// `gh repo set-default` marks the chosen remote with `remote..gh-resolved base`. -function parseGhDefaultRemoteName(stdout: string): string | null { - const marked = stdout - .split("\n") - .map((line) => /^remote\.(.+)\.gh-resolved\s+base$/.exec(line.trim())?.[1]) - .filter((name): name is string => name !== undefined); - return marked.length === 1 ? marked[0]! : null; +// `gh repo set-default` marks one remote with `base`, or with OWNER/REPO when it picked a +// parent repository. gh reads PRs from that repository, so the identity follows the mark when +// a remote fetches it. The locator must name a real remote: review checkouts fetch through it. +function pickMarkedRemote( + remotes: ReadonlyMap, + resolved: string, +): { readonly remoteName: string; readonly remoteUrl: string } | null { + const marks = resolved.split("\n").flatMap((line) => { + const match = /^remote\.(.+)\.gh-resolved\s+(\S+)$/u.exec(line.trim()); + const remoteUrl = match?.[1] === undefined ? undefined : remotes.get(match[1]); + return match?.[1] && match[2] && remoteUrl + ? [{ remoteName: match[1], remoteUrl, value: match[2] }] + : []; + }); + // gh's choice is ambiguous with several marks; GitHubSourceControlProvider ignores them too. + const [mark, ...others] = marks; + if (!mark || others.length > 0) return null; + if (mark.value === "base") return { remoteName: mark.remoteName, remoteUrl: mark.remoteUrl }; + // Compare hosts with HTTP ports kept: one hostname can serve separate forges on two ports. + const host = parseGitRemoteHost(mark.remoteUrl)?.host; + const markedPath = mark.value.toLowerCase(); + const [remoteName, remoteUrl] = + [...remotes.entries()].find( + ([, url]) => + parseGitRemoteHost(url)?.host === host && + normalizeGitRemoteUrl(url).split("/").slice(1).join("/") === markedPath, + ) ?? []; + return remoteName && remoteUrl ? { remoteName, remoteUrl } : null; } function pickPrimaryRemote( remotes: ReadonlyMap, - ghDefaultRemoteName: string | null, + resolved: string, ): { readonly remoteName: string; readonly remoteUrl: string } | null { - for (const preferredRemoteName of [ghDefaultRemoteName, "upstream", "origin"]) { - if (preferredRemoteName === null) continue; + const marked = pickMarkedRemote(remotes, resolved); + if (marked) return marked; + + for (const preferredRemoteName of ["upstream", "origin"] as const) { const remoteUrl = remotes.get(preferredRemoteName); if (remoteUrl) { return { remoteName: preferredRemoteName, remoteUrl }; @@ -98,7 +123,13 @@ function pickPrimaryRemote( function repositoryPathOf(canonicalKey: string, remoteUrl: string): string { const canonicalPath = canonicalKey.split("/").slice(1).join("/"); const displayPath = repositoryPathFromRemoteUrl(remoteUrl); - return displayPath?.toLowerCase() === canonicalPath ? displayPath : canonicalPath; + if (displayPath === null) return canonicalPath; + // A host that names repositories literally keeps the `.git` the display path drops. + return ( + [displayPath, `${displayPath}.git`].find( + (path) => canonicalRepositoryPath(path) === canonicalRepositoryPath(canonicalPath), + ) ?? canonicalPath + ); } function buildRepositoryOrigin( @@ -180,7 +211,8 @@ const resolveRepositoryIdentityCacheKey = Effect.fn("RepositoryIdentityResolver. return null; } if (topLevelResult.value.code === 0) { - const rootPath = topLevelResult.value.stdout.trim(); + // Paths may end in spaces, so strip only git's line terminator. + const rootPath = topLevelResult.value.stdout.replace(/\r?\n$/, ""); return rootPath.length > 0 ? new RepositoryIdentityCacheKey({ rootPath, isBare: false }) : null; @@ -224,9 +256,7 @@ const resolveRepositoryIdentityCacheKey = Effect.fn("RepositoryIdentityResolver. if (bareResult._tag === "None" || bareResult.value.timedOut) return null; if (bareResult.value.code !== 0) continue; - const [isBare, rootPath = ""] = bareResult.value.stdout - .split("\n") - .map((line) => line.trim()); + const [isBare, rootPath = ""] = bareResult.value.stdout.split(/\r?\n/); // A checkout's .git directory contains remotes but is not a project root. return isBare === "true" && rootPath.length > 0 ? new RepositoryIdentityCacheKey({ rootPath, isBare: true }) @@ -242,40 +272,31 @@ const resolveRepositoryIdentityFromCacheKey = Effect.fn( cacheKey: RepositoryIdentityCacheKey, ): Effect.fn.Return { const processRunner = yield* ProcessRunner.ProcessRunner; - const remoteResult = yield* processRunner - .run({ - command: "git", - args: ["-C", cacheKey.rootPath, ...(cacheKey.isBare ? ["--git-dir=."] : []), "remote", "-v"], - env: gitEnv(yield* HostProcess.Platform), - timeoutBehavior: "timedOutResult", - }) - .pipe(Effect.option); + const env = gitEnv(yield* HostProcess.Platform); + const runGit = (args: ReadonlyArray) => + processRunner + .run({ + command: "git", + args: ["-C", cacheKey.rootPath, ...(cacheKey.isBare ? ["--git-dir=."] : []), ...args], + env, + timeoutBehavior: "timedOutResult", + }) + .pipe(Effect.option); + const [remoteResult, resolvedResult] = yield* Effect.all( + [runGit(["remote", "-v"]), runGit(["config", "--get-regexp", "^remote\\..*\\.gh-resolved$"])], + { concurrency: "unbounded" }, + ); if (remoteResult._tag === "None" || remoteResult.value.code !== 0) { return null; } - const ghResolvedResult = yield* processRunner - .run({ - command: "git", - args: [ - "-C", - cacheKey.rootPath, - ...(cacheKey.isBare ? ["--git-dir=."] : []), - "config", - "--get-regexp", - "^remote\\..*\\.gh-resolved$", - ], - env: explicitBareRepositoryEnv(process.env, yield* HostProcess.Platform), - timeoutBehavior: "timedOutResult", - }) - .pipe(Effect.option); - const ghDefaultRemoteName = - ghResolvedResult._tag === "Some" && ghResolvedResult.value.code === 0 - ? parseGhDefaultRemoteName(ghResolvedResult.value.stdout) - : null; - const remotes = parseRemoteFetchUrls(remoteResult.value.stdout); - const remote = pickPrimaryRemote(remotes, ghDefaultRemoteName); + // Exits 1 when no remote has a mark. + const resolved = + resolvedResult._tag === "Some" && resolvedResult.value.code === 0 + ? resolvedResult.value.stdout + : ""; + const remote = pickPrimaryRemote(remotes, resolved); return remote ? buildRepositoryIdentity({ ...remote, diff --git a/apps/server/src/project/worktreeContainer.test.ts b/apps/server/src/project/worktreeContainer.test.ts new file mode 100644 index 0000000000000..b10f8db93dab3 --- /dev/null +++ b/apps/server/src/project/worktreeContainer.test.ts @@ -0,0 +1,94 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; + +import * as ProcessRunner from "../processRunner.ts"; +import { makeIsWorktreeContainer } from "./worktreeContainer.ts"; + +const git = Effect.fn("worktreeContainerTest.git")(function* ( + cwd: string, + ...args: ReadonlyArray +) { + const processRunner = yield* ProcessRunner.ProcessRunner; + return yield* processRunner.run({ + command: "git", + args: ["-C", cwd, "-c", "user.email=t@t", "-c", "user.name=t", ...args], + }); +}); + +it.layer(ProcessRunner.layer.pipe(Layer.provideMerge(NodeServices.layer)))( + "isWorktreeContainer", + (it) => { + it.effect("is true only for a folder of worktrees around metadata that is no checkout", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const isWorktreeContainer = yield* makeIsWorktreeContainer; + const fixture = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-container-test-" }); + + const source = path.join(fixture, "source"); + yield* fileSystem.makeDirectory(source); + yield* git(source, "init", "-b", "develop"); + yield* fileSystem.writeFileString(path.join(source, "README.md"), "readme\n"); + yield* git(source, "add", "README.md"); + yield* git(source, "commit", "-m", "init"); + + const makeContainer = Effect.fn(function* (name: string, coreBare: boolean) { + const root = path.join(fixture, name); + yield* fileSystem.makeDirectory(root); + yield* git(fixture, "clone", "--bare", source, path.join(root, ".bare")); + yield* fileSystem.writeFileString(path.join(root, ".git"), "gitdir: ./.bare\n"); + yield* git(root, "worktree", "add", "develop", "develop"); + yield* git(root, "config", "core.bare", String(coreBare)); + return root; + }); + const makeSeparateGitDirCheckout = Effect.fn(function* (name: string) { + const root = path.join(fixture, name); + yield* fileSystem.makeDirectory(root); + yield* git(root, "init", "-b", "develop", "--separate-git-dir", ".metadata", "."); + return root; + }); + + const container = yield* makeContainer("container", false); + assert.isTrue(yield* isWorktreeContainer(container), "container"); + assert.isFalse(yield* isWorktreeContainer(path.join(container, "develop")), "its worktree"); + + const linked = path.join(fixture, "link"); + yield* fileSystem.symlink(container, linked); + assert.isTrue(yield* isWorktreeContainer(linked), "container through a symlink"); + + // The issue's steps leave core.bare set; Initialize Git would clear it later. + const bare = yield* makeContainer("bare", true); + assert.isTrue(yield* isWorktreeContainer(bare), "core.bare container"); + + // Git allows an orphan worktree before the first commit: no index, yet a valid checkout. + const fresh = yield* makeSeparateGitDirCheckout("fresh"); + yield* git(fresh, "worktree", "add", "--orphan", "-b", "feature", "../fresh-feature"); + assert.isFalse(yield* isWorktreeContainer(fresh), "fresh checkout with an orphan worktree"); + + const checkout = yield* makeSeparateGitDirCheckout("checkout"); + yield* fileSystem.writeFileString(path.join(checkout, "README.md"), "readme\n"); + yield* git(checkout, "add", "README.md"); + yield* git(checkout, "commit", "-m", "init"); + yield* git(checkout, "worktree", "add", "worktrees/feature", "-b", "feature"); + assert.isFalse(yield* isWorktreeContainer(checkout), "checkout with a nested worktree"); + + const outside = path.join(fixture, "outside"); + yield* git( + fixture, + "clone", + "--no-checkout", + "--separate-git-dir", + path.join(outside, ".metadata"), + source, + outside, + ); + yield* git(outside, "worktree", "add", "../outside-feature", "-b", "feature"); + assert.isFalse(yield* isWorktreeContainer(outside), "worktrees outside the folder"); + }), + ); + }, +); diff --git a/apps/server/src/project/worktreeContainer.ts b/apps/server/src/project/worktreeContainer.ts new file mode 100644 index 0000000000000..f20e9c4375cae --- /dev/null +++ b/apps/server/src/project/worktreeContainer.ts @@ -0,0 +1,95 @@ +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; + +import * as ProcessRunner from "../processRunner.ts"; +import { explicitBareRepositoryEnv, unsetInheritedRepositoryEnv } from "../vcs/VcsProcess.ts"; + +const parseGitfileTarget = (contents: string): string | undefined => { + const firstLine = contents.split(/\r?\n/u, 1)[0] ?? ""; + if (!firstLine.startsWith("gitdir:")) return undefined; + const target = firstLine.slice("gitdir:".length).trim(); + return target === "" ? undefined : target; +}; + +const splitNul = (stdout: string) => stdout.split("\0").filter((entry) => entry !== ""); + +/** + * Whether a folder only holds a repository's metadata and worktrees: its `.git` file points at + * metadata inside it, a linked worktree lives inside it, and either the repository is bare or it + * has commits but no index. A checkout whose tracked files are all deleted keeps its index, and a + * fresh one has no commit. Anything this cannot establish answers false. + */ +export const makeIsWorktreeContainer = Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const processRunner = yield* ProcessRunner.ProcessRunner; + const platform = yield* HostProcess.Platform; + const env = { + ...unsetInheritedRepositoryEnv(process.env, platform), + ...explicitBareRepositoryEnv(process.env, platform), + }; + + const realPath = (target: string) => fileSystem.realPath(target).pipe(Effect.option); + const isInside = (parent: string, child: string) => { + const relative = path.relative(parent, child); + return ( + relative !== "" && + relative !== ".." && + !relative.startsWith(`..${path.sep}`) && + !path.isAbsolute(relative) + ); + }; + + return Effect.fn("ProjectService.isWorktreeContainer")(function* (workspaceRoot: string) { + // A `.git` directory is an ordinary checkout, so only a gitfile costs any Git process. + const gitfileTarget = yield* fileSystem.readFileString(path.join(workspaceRoot, ".git")).pipe( + Effect.map(parseGitfileTarget), + Effect.orElseSucceed(() => undefined), + ); + if (gitfileTarget === undefined) return false; + const root = yield* realPath(workspaceRoot); + const gitDir = yield* realPath(path.resolve(workspaceRoot, gitfileTarget)); + if (root._tag === "None" || gitDir._tag === "None") return false; + if (!isInside(root.value, gitDir.value)) return false; + + const git = (args: ReadonlyArray) => + processRunner + .run({ + command: "git", + args: ["-C", workspaceRoot, ...args], + env, + timeoutBehavior: "timedOutResult", + }) + .pipe( + Effect.map((result) => (result.code === 0 && !result.timedOut ? result.stdout : null)), + Effect.orElseSucceed(() => null), + ); + + // Initializing Git in a bare container clears core.bare, so it is refused while still bare. + const isBare = yield* git(["rev-parse", "--is-bare-repository"]); + if (isBare === null) return false; + if (isBare.trim() !== "true") { + const hasCommit = yield* git(["rev-parse", "--quiet", "--verify", "HEAD^{commit}"]); + if (hasCommit === null) return false; + const hasIndex = yield* fileSystem + .exists(path.join(gitDir.value, "index")) + .pipe(Effect.orElseSucceed(() => true)); + if (hasIndex) return false; + } + + const worktrees = yield* git(["worktree", "list", "--porcelain", "-z"]); + if (worktrees === null) return false; + // The first entry is the main worktree, which Git reports as the metadata directory here. + const linkedWorktreePaths = splitNul(worktrees) + .filter((field) => field.startsWith("worktree ")) + .slice(1) + .map((field) => field.slice("worktree ".length)); + for (const worktreePath of linkedWorktreePaths) { + const worktree = yield* realPath(worktreePath); + if (worktree._tag === "Some" && isInside(root.value, worktree.value)) return true; + } + return false; + }); +}); diff --git a/apps/server/src/provider/AcpRegistryProbe.test.ts b/apps/server/src/provider/AcpRegistryProbe.test.ts index 3b7b2b42f2b74..30915b31d474f 100644 --- a/apps/server/src/provider/AcpRegistryProbe.test.ts +++ b/apps/server/src/provider/AcpRegistryProbe.test.ts @@ -410,7 +410,6 @@ describe("ACP Registry probe", () => { }).pipe( Effect.provideService(AcpRegistrySupport.AcpRegistryCatalog, catalog), Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), - Effect.scoped, ); }); diff --git a/apps/server/src/provider/AgentContextBudget.test.ts b/apps/server/src/provider/AgentContextBudget.test.ts index 8cdd52d202f12..2a7d02fc28173 100644 --- a/apps/server/src/provider/AgentContextBudget.test.ts +++ b/apps/server/src/provider/AgentContextBudget.test.ts @@ -35,6 +35,7 @@ import { ProjectCloneTracker } from "../project/ProjectCloneTracker.ts"; import { ProjectService } from "../project/ProjectService.ts"; import { ProjectSetupScriptRunner } from "../project/ProjectSetupScriptRunner.ts"; import { PullRequestReviewGuideService } from "../pullRequest/PullRequestReviewGuideService.ts"; +import { PullRequestService } from "../pullRequest/PullRequestService.ts"; import { ScheduledTaskService } from "../scheduledTasks/ScheduledTaskService.ts"; import { SecretRequests } from "../secrets/SecretRequests.ts"; import { ServerSettingsService } from "../serverSettings.ts"; @@ -43,6 +44,7 @@ import { GitVcsDriver } from "../vcs/GitVcsDriver.ts"; import { VcsStatusBroadcaster } from "../vcs/VcsStatusBroadcaster.ts"; import { buildCodexAdditionalContext } from "./CodexDeveloperInstructions.ts"; import { DocumentText } from "./DocumentText.ts"; +import { EnvironmentThemeService } from "../environmentTheme.ts"; import { ProviderRegistry } from "./ProviderRegistry.ts"; import budget from "./agentContextBudget.json" with { type: "json" }; @@ -51,6 +53,7 @@ const toolHandlerServices = Layer.mergeAll( Layer.mock(AssetSigner)({}), Layer.mock(DeviceService)({}), Layer.mock(DocumentText)({}), + Layer.mock(EnvironmentThemeService)({}), Layer.mock(GitVcsDriver)({}), Layer.mock(GitWorkflowService)({}), Layer.mock(ManagedProjectFolders)({ namedProjectsRoot: "" }), @@ -68,6 +71,7 @@ const toolHandlerServices = Layer.mergeAll( Layer.mock(ProviderAdapterRegistryV2)({}), Layer.mock(ProviderRegistry)({}), Layer.mock(PullRequestReviewGuideService)({}), + Layer.mock(PullRequestService)({}), Layer.mock(RemoteDelegation)({}), Layer.mock(ScheduledTaskService)({}), Layer.mock(SecretRequests)({}), @@ -114,13 +118,16 @@ const layerAdvertisedTools = McpHttpServer.layerTools.pipe( ); // Claude Code loads these tools up front instead of behind ToolSearch, outside this budget. -it.effect("only request_secret asks Claude Code to load it up front", () => +it.effect("only request_secret and send_to_user ask Claude Code to load them up front", () => Effect.gen(function* () { const mcp = yield* McpServer.McpServer; const alwaysLoad = mcp.tools .filter(({ tool }) => tool._meta?.["anthropic/alwaysLoad"] !== undefined) .map(({ tool }) => [tool.name, tool._meta]); - expect(alwaysLoad).toEqual([["request_secret", { "anthropic/alwaysLoad": true }]]); + expect(alwaysLoad).toEqual([ + ["request_secret", { "anthropic/alwaysLoad": true }], + ["send_to_user", { "anthropic/alwaysLoad": true }], + ]); }).pipe(Effect.provide(layerAdvertisedTools)), ); diff --git a/apps/server/src/provider/AntigravityProvider.test.ts b/apps/server/src/provider/AntigravityProvider.test.ts index 791ed748de2ef..a676273827859 100644 --- a/apps/server/src/provider/AntigravityProvider.test.ts +++ b/apps/server/src/provider/AntigravityProvider.test.ts @@ -291,561 +291,519 @@ describe("Antigravity model catalog", () => { it.layer(layerTest)("Antigravity provider snapshots", (it) => { it.effect("does not probe or run helper safety checks while disabled", () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeHarness({ enabled: false }); - const snapshot = yield* harness.provider.snapshot.refresh; - expect(snapshot).toMatchObject({ - enabled: false, + Effect.gen(function* () { + const harness = yield* makeHarness({ enabled: false }); + const snapshot = yield* harness.provider.snapshot.refresh; + expect(snapshot).toMatchObject({ + enabled: false, + installed: false, + status: "disabled", + auth: { status: "unknown" }, + models: [], + setup: { canAuthenticate: true, canInstall: true }, + showInteractionModeToggle: false, + supportsConversationRollback: false, + supportsTextGeneration: false, + }); + expect(yield* Ref.get(harness.probeCalls)).toBe(0); + expect(yield* Ref.get(harness.safetyCalls)).toBe(0); + }), + ); + + it.effect("treats initialize as installation proof, not account or model discovery", () => + Effect.gen(function* () { + for (const auth of [ + undefined, + { type: "oauth-personal", label: "Google account" }, + { type: "oauth-business", label: "Gemini Enterprise" }, + { type: "gemini-api-key", label: "Gemini API key" }, + { type: "agent-platform", label: "Agent Platform" }, + ]) { + const harness = yield* makeHarness(auth ? { auth } : {}); + expect(yield* harness.provider.snapshot.getSnapshot).toMatchObject({ installed: false, - status: "disabled", + status: "warning", auth: { status: "unknown" }, models: [], - setup: { canAuthenticate: true, canInstall: true }, - showInteractionModeToggle: false, - supportsConversationRollback: false, - supportsTextGeneration: false, }); - expect(yield* Ref.get(harness.probeCalls)).toBe(0); - expect(yield* Ref.get(harness.safetyCalls)).toBe(0); - }), - ), - ); - - it.effect("treats initialize as installation proof, not account or model discovery", () => - Effect.scoped( - Effect.gen(function* () { - for (const auth of [ - undefined, - { type: "oauth-personal", label: "Google account" }, - { type: "oauth-business", label: "Gemini Enterprise" }, - { type: "gemini-api-key", label: "Gemini API key" }, - { type: "agent-platform", label: "Agent Platform" }, - ]) { - const harness = yield* makeHarness(auth ? { auth } : {}); - expect(yield* harness.provider.snapshot.getSnapshot).toMatchObject({ - installed: false, - status: "warning", - auth: { status: "unknown" }, - models: [], - }); - yield* harness.initialize; - const snapshot = yield* harness.provider.snapshot.getSnapshot; - expect(snapshot).toMatchObject({ - installed: true, - status: "warning", - version: "agy_acp_server_20260818_01_RC01", - auth: { status: "unknown", ...(auth ? { type: auth.type } : {}) }, - message: - auth?.type === "gemini-api-key" || auth?.type === "agent-platform" - ? `Antigravity is installed. Connect with ${auth.label} to load models.` - : "Antigravity is installed. Google account access is not checked yet.", - models: [], - }); - expect(yield* Ref.get(harness.probeCalls)).toBe(1); - } - }), - ), + yield* harness.initialize; + const snapshot = yield* harness.provider.snapshot.getSnapshot; + expect(snapshot).toMatchObject({ + installed: true, + status: "warning", + version: "agy_acp_server_20260818_01_RC01", + auth: { status: "unknown", ...(auth ? { type: auth.type } : {}) }, + message: + auth?.type === "gemini-api-key" || auth?.type === "agent-platform" + ? `Antigravity is installed. Connect with ${auth.label} to load models.` + : "Antigravity is installed. Google account access is not checked yet.", + models: [], + }); + expect(yield* Ref.get(harness.probeCalls)).toBe(1); + } + }), ); it.effect("does not offer Auto, so Auto threads fall back to Supervised", () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeHarness(); - yield* harness.initialize; - // Antigravity has no automatic review; the shared ACP policy would approve every request. - expect((yield* harness.provider.snapshot.getSnapshot).supportedRuntimeModes).toEqual([ - "approval-required", - "auto-accept-edits", - "full-access", - ]); - }), - ), + Effect.gen(function* () { + const harness = yield* makeHarness(); + yield* harness.initialize; + // Antigravity has no automatic review; the shared ACP policy would approve every request. + expect((yield* harness.provider.snapshot.getSnapshot).supportedRuntimeModes).toEqual([ + "approval-required", + "auto-accept-edits", + "full-access", + ]); + }), ); it.effect("publishes session metadata and native commands without another health probe", () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeHarness(); - yield* harness.initialize; - const nextReady = yield* Stream.toPull( - harness.provider.snapshot.streamChanges.pipe( - Stream.filter( - (snapshot) => - snapshot.auth.status === "authenticated" && snapshot.slashCommands.length > 0, - ), + Effect.gen(function* () { + const harness = yield* makeHarness(); + yield* harness.initialize; + const nextReady = yield* Stream.toPull( + harness.provider.snapshot.streamChanges.pipe( + Stream.filter( + (snapshot) => + snapshot.auth.status === "authenticated" && snapshot.slashCommands.length > 0, ), - ); - yield* harness.provider.onSessionStarted(started, "/workspace"); - yield* harness.provider.onAvailableCommands(commands, "/workspace"); - const [snapshot] = yield* nextReady; - expect(snapshot).toMatchObject({ - status: "ready", - auth: { status: "authenticated", type: "oauth-personal" }, - supportsTextGeneration: true, - }); - expect(snapshot.models).toEqual(buildAntigravityModelsFromSession(sessionSetupResult)); - expect(snapshot.slashCommands).toEqual(expectedCommands); - expect((yield* harness.provider.snapshotForCwd("/workspace")).slashCommands).toEqual( - expectedCommands, - ); - expect(yield* Ref.get(harness.probeCalls)).toBe(1); - }), - ), + ), + ); + yield* harness.provider.onSessionStarted(started, "/workspace"); + yield* harness.provider.onAvailableCommands(commands, "/workspace"); + const [snapshot] = yield* nextReady; + expect(snapshot).toMatchObject({ + status: "ready", + auth: { status: "authenticated", type: "oauth-personal" }, + supportsTextGeneration: true, + }); + expect(snapshot.models).toEqual(buildAntigravityModelsFromSession(sessionSetupResult)); + expect(snapshot.slashCommands).toEqual(expectedCommands); + expect((yield* harness.provider.snapshotForCwd("/workspace")).slashCommands).toEqual( + expectedCommands, + ); + expect(yield* Ref.get(harness.probeCalls)).toBe(1); + }), ); it.effect("clears all account metadata on sign-out and authentication failure", () => - Effect.scoped( - Effect.gen(function* () { - const auth = { type: "gemini-api-key", label: "Gemini API key" }; - const harness = yield* makeHarness({ auth }); - yield* harness.initialize; - for (const clear of [harness.provider.onSignedOut, harness.provider.onAuthRequired]) { - yield* harness.provider.onSessionStarted(started, "/workspace"); - yield* harness.provider.onAvailableCommands(commands, "/workspace"); - yield* clear; - expect(yield* harness.provider.snapshot.getSnapshot).toMatchObject({ - installed: true, - status: "warning", - auth: { status: "unauthenticated", type: auth.type, label: auth.label }, - message: "Connect with Gemini API key to use Antigravity and load models.", - models: [], - slashCommands: [], - skills: [], - workspaceSnapshots: [], - supportsTextGeneration: false, - }); - yield* harness.provider.onAvailableCommands(commands, "/workspace"); - yield* harness.provider.onConfigOptionsUpdated([modelConfig]); - expect((yield* harness.provider.snapshotForCwd("/workspace")).slashCommands).toEqual([]); - expect((yield* harness.provider.snapshot.getSnapshot).models).toEqual([]); - } - const refreshed = yield* harness.provider.snapshot.refresh; - expect(refreshed.auth).toEqual({ - status: "unauthenticated", - type: auth.type, - label: auth.label, - }); - expect(refreshed.message).toBe( - "Connect with Gemini API key to use Antigravity and load models.", - ); - expect(refreshed.supportsTextGeneration).toBe(false); - }), - ), - ); - - it.effect("replaces live model choices and accepts an empty catalog", () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeHarness(); - yield* harness.initialize; + Effect.gen(function* () { + const auth = { type: "gemini-api-key", label: "Gemini API key" }; + const harness = yield* makeHarness({ auth }); + yield* harness.initialize; + for (const clear of [harness.provider.onSignedOut, harness.provider.onAuthRequired]) { yield* harness.provider.onSessionStarted(started, "/workspace"); yield* harness.provider.onAvailableCommands(commands, "/workspace"); - const before = yield* harness.provider.snapshot.getSnapshot; - const configOptions = [ - { - ...modelConfig, - currentValue: "gemini-3.8-flash-high", - options: modelOptions.slice(0, 3), - }, - ]; - const nextSnapshot = yield* Stream.toPull( - harness.provider.snapshot.streamChanges.pipe( - Stream.filter((snapshot) => snapshot.models.length === 3), - ), - ); - yield* harness.provider.onConfigOptionsUpdated(configOptions); - expect((yield* nextSnapshot)[0]).toMatchObject({ - models: buildAntigravityModelsFromSession({ configOptions }), - auth: before.auth, - workspaceSnapshots: before.workspaceSnapshots, - slashCommands: expectedCommands, + yield* clear; + expect(yield* harness.provider.snapshot.getSnapshot).toMatchObject({ + installed: true, + status: "warning", + auth: { status: "unauthenticated", type: auth.type, label: auth.label }, + message: "Connect with Gemini API key to use Antigravity and load models.", + models: [], + slashCommands: [], + skills: [], + workspaceSnapshots: [], + supportsTextGeneration: false, }); - yield* harness.provider.onConfigOptionsUpdated([]); + yield* harness.provider.onAvailableCommands(commands, "/workspace"); + yield* harness.provider.onConfigOptionsUpdated([modelConfig]); + expect((yield* harness.provider.snapshotForCwd("/workspace")).slashCommands).toEqual([]); expect((yield* harness.provider.snapshot.getSnapshot).models).toEqual([]); - }), - ), + } + const refreshed = yield* harness.provider.snapshot.refresh; + expect(refreshed.auth).toEqual({ + status: "unauthenticated", + type: auth.type, + label: auth.label, + }); + expect(refreshed.message).toBe( + "Connect with Gemini API key to use Antigravity and load models.", + ); + expect(refreshed.supportsTextGeneration).toBe(false); + }), + ); + + it.effect("replaces live model choices and accepts an empty catalog", () => + Effect.gen(function* () { + const harness = yield* makeHarness(); + yield* harness.initialize; + yield* harness.provider.onSessionStarted(started, "/workspace"); + yield* harness.provider.onAvailableCommands(commands, "/workspace"); + const before = yield* harness.provider.snapshot.getSnapshot; + const configOptions = [ + { + ...modelConfig, + currentValue: "gemini-3.8-flash-high", + options: modelOptions.slice(0, 3), + }, + ]; + const nextSnapshot = yield* Stream.toPull( + harness.provider.snapshot.streamChanges.pipe( + Stream.filter((snapshot) => snapshot.models.length === 3), + ), + ); + yield* harness.provider.onConfigOptionsUpdated(configOptions); + expect((yield* nextSnapshot)[0]).toMatchObject({ + models: buildAntigravityModelsFromSession({ configOptions }), + auth: before.auth, + workspaceSnapshots: before.workspaceSnapshots, + slashCommands: expectedCommands, + }); + yield* harness.provider.onConfigOptionsUpdated([]); + expect((yield* harness.provider.snapshot.getSnapshot).models).toEqual([]); + }), ); it.effect("does not guess a version before a custom launcher's first handshake", () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeHarness(); - yield* Deferred.succeed(harness.initialProbe, { - ...initializeResult, - agentInfo: { ...initializeResult.agentInfo, version: "unknown" }, - }); - yield* harness.initialUpdate; - expect((yield* harness.provider.snapshot.getSnapshot).version).toBeNull(); - }), - ), + Effect.gen(function* () { + const harness = yield* makeHarness(); + yield* Deferred.succeed(harness.initialProbe, { + ...initializeResult, + agentInfo: { ...initializeResult.agentInfo, version: "unknown" }, + }); + yield* harness.initialUpdate; + expect((yield* harness.provider.snapshot.getSnapshot).version).toBeNull(); + }), ); it.effect("does not replace a handshake version with an unknown custom-launcher probe", () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeHarness(); - yield* harness.initialize; - yield* harness.provider.onSessionStarted({ - ...started, - initializeResult: { - ...initializeResult, - agentInfo: { ...initializeResult.agentInfo, version: "1.4.0" }, - }, - }); - yield* Ref.set( - harness.probe, - Effect.succeed({ - ...initializeResult, - agentInfo: { ...initializeResult.agentInfo, version: "unknown" }, - }), - ); - expect((yield* harness.provider.snapshot.refresh).version).toBe("1.4.0"); - }), - ), + Effect.gen(function* () { + const harness = yield* makeHarness(); + yield* harness.initialize; + yield* harness.provider.onSessionStarted({ + ...started, + initializeResult: { + ...initializeResult, + agentInfo: { ...initializeResult.agentInfo, version: "1.4.0" }, + }, + }); + yield* Ref.set( + harness.probe, + Effect.succeed({ + ...initializeResult, + agentInfo: { ...initializeResult.agentInfo, version: "unknown" }, + }), + ); + expect((yield* harness.provider.snapshot.refresh).version).toBe("1.4.0"); + }), ); it.effect("replaces one account's catalog instead of combining accounts", () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeHarness(); - yield* harness.initialize; - yield* harness.provider.onSessionStarted(started); - yield* harness.provider.onSignedOut; - yield* harness.provider.onSessionStarted({ - ...started, - sessionSetupResult: { - sessionId: "new-account-session", - configOptions: [ - { ...modelConfig, currentValue: "gemini-pro-agent", options: [modelOptions[9]!] }, - ], - }, - }); - expect( - (yield* harness.provider.snapshot.getSnapshot).models.map((model) => model.slug), - ).toEqual(["gemini-pro-agent"]); - }), - ), + Effect.gen(function* () { + const harness = yield* makeHarness(); + yield* harness.initialize; + yield* harness.provider.onSessionStarted(started); + yield* harness.provider.onSignedOut; + yield* harness.provider.onSessionStarted({ + ...started, + sessionSetupResult: { + sessionId: "new-account-session", + configOptions: [ + { ...modelConfig, currentValue: "gemini-pro-agent", options: [modelOptions[9]!] }, + ], + }, + }); + expect( + (yield* harness.provider.snapshot.getSnapshot).models.map((model) => model.slug), + ).toEqual(["gemini-pro-agent"]); + }), ); it.effect("retains known account metadata when a local health check fails", () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeHarness(); - yield* harness.initialize; + Effect.gen(function* () { + const harness = yield* makeHarness(); + yield* harness.initialize; + yield* harness.provider.onSessionStarted(started, "/workspace"); + yield* harness.provider.onAvailableCommands(commands, "/workspace"); + yield* Ref.set( + harness.probe, + Effect.fail(EffectAcpErrors.AcpRequestError.internalError("probe failed")), + ); + const snapshot = yield* harness.provider.snapshot.refresh; + expect(snapshot).toMatchObject({ + installed: true, + status: "error", + auth: { status: "authenticated" }, + }); + expect(snapshot.models).toEqual(buildAntigravityModelsFromSession(sessionSetupResult)); + expect(snapshot.slashCommands).toEqual(expectedCommands); + expect(snapshot.workspaceSnapshots?.[0]?.cwd).toBe("/workspace"); + }), + ); + + it.effect("distinguishes missing executables from a failed installed executable", () => + Effect.gen(function* () { + const harness = yield* makeHarness(); + yield* harness.initialize; + const failures = [ + { + error: new EffectAcpErrors.AcpSpawnError({ cause: { code: "ENOENT" } }), + installed: false, + }, + { + error: new EffectAcpErrors.AcpSpawnError({ cause: { code: "EACCES" } }), + installed: true, + }, + { + error: new ProviderSetupError({ + instanceId, + operation: "resolve", + detail: "Antigravity is not installed.", + }), + installed: false, + }, + ]; + for (const { error, installed } of failures) { yield* harness.provider.onSessionStarted(started, "/workspace"); yield* harness.provider.onAvailableCommands(commands, "/workspace"); - yield* Ref.set( - harness.probe, - Effect.fail(EffectAcpErrors.AcpRequestError.internalError("probe failed")), - ); + yield* Ref.set(harness.probe, Effect.fail(error)); const snapshot = yield* harness.provider.snapshot.refresh; expect(snapshot).toMatchObject({ - installed: true, + installed, status: "error", auth: { status: "authenticated" }, }); - expect(snapshot.models).toEqual(buildAntigravityModelsFromSession(sessionSetupResult)); - expect(snapshot.slashCommands).toEqual(expectedCommands); - expect(snapshot.workspaceSnapshots?.[0]?.cwd).toBe("/workspace"); - }), - ), - ); - - it.effect("distinguishes missing executables from a failed installed executable", () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeHarness(); - yield* harness.initialize; - const failures = [ - { - error: new EffectAcpErrors.AcpSpawnError({ cause: { code: "ENOENT" } }), - installed: false, - }, - { - error: new EffectAcpErrors.AcpSpawnError({ cause: { code: "EACCES" } }), - installed: true, - }, - { - error: new ProviderSetupError({ - instanceId, - operation: "resolve", - detail: "Antigravity is not installed.", - }), - installed: false, - }, - ]; - for (const { error, installed } of failures) { - yield* harness.provider.onSessionStarted(started, "/workspace"); - yield* harness.provider.onAvailableCommands(commands, "/workspace"); - yield* Ref.set(harness.probe, Effect.fail(error)); - const snapshot = yield* harness.provider.snapshot.refresh; - expect(snapshot).toMatchObject({ - installed, - status: "error", - auth: { status: "authenticated" }, - }); - expect(snapshot.models).toHaveLength(installed ? 11 : 0); - expect(snapshot.slashCommands).toHaveLength(installed ? 2 : 0); - expect(snapshot.workspaceSnapshots).toHaveLength(installed ? 1 : 0); - expect(snapshot.supportsTextGeneration).toBe(installed); - } - }), - ), + expect(snapshot.models).toHaveLength(installed ? 11 : 0); + expect(snapshot.slashCommands).toHaveLength(installed ? 2 : 0); + expect(snapshot.workspaceSnapshots).toHaveLength(installed ? 1 : 0); + expect(snapshot.supportsTextGeneration).toBe(installed); + } + }), ); it.effect("does not let an old health result restore a signed-out account", () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeHarness(); - yield* harness.initialize; - yield* harness.provider.onSessionStarted(started); - const releaseProbe = yield* Deferred.make(); - const probeEntered = yield* Deferred.make(); - yield* Ref.set( - harness.probe, - Deferred.succeed(probeEntered, undefined).pipe( - Effect.andThen(Deferred.await(releaseProbe)), - ), - ); - const refresh = yield* harness.provider.snapshot.refresh.pipe(Effect.forkChild); - yield* Deferred.await(probeEntered); - yield* harness.provider.onSignedOut; - yield* Deferred.succeed(releaseProbe, initializeResult); - const snapshot = yield* Fiber.join(refresh); - expect(snapshot).toMatchObject({ - auth: { status: "unauthenticated" }, - models: [], - supportsTextGeneration: false, - }); - }), - ), + Effect.gen(function* () { + const harness = yield* makeHarness(); + yield* harness.initialize; + yield* harness.provider.onSessionStarted(started); + const releaseProbe = yield* Deferred.make(); + const probeEntered = yield* Deferred.make(); + yield* Ref.set( + harness.probe, + Deferred.succeed(probeEntered, undefined).pipe( + Effect.andThen(Deferred.await(releaseProbe)), + ), + ); + const refresh = yield* harness.provider.snapshot.refresh.pipe(Effect.forkChild); + yield* Deferred.await(probeEntered); + yield* harness.provider.onSignedOut; + yield* Deferred.succeed(releaseProbe, initializeResult); + const snapshot = yield* Fiber.join(refresh); + expect(snapshot).toMatchObject({ + auth: { status: "unauthenticated" }, + models: [], + supportsTextGeneration: false, + }); + }), ); it.effect("exposes helper support only when the supplied safety check allows it", () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeHarness({ safe: false }); - yield* harness.initialize; - yield* harness.provider.onSessionStarted(started); - expect((yield* harness.provider.snapshot.getSnapshot).supportsTextGeneration).toBe(false); - yield* Ref.set(harness.safety, Effect.succeed(true)); - yield* harness.provider.snapshot.refresh; - expect((yield* harness.provider.snapshot.getSnapshot).supportsTextGeneration).toBe(true); - }), - ), + Effect.gen(function* () { + const harness = yield* makeHarness({ safe: false }); + yield* harness.initialize; + yield* harness.provider.onSessionStarted(started); + expect((yield* harness.provider.snapshot.getSnapshot).supportsTextGeneration).toBe(false); + yield* Ref.set(harness.safety, Effect.succeed(true)); + yield* harness.provider.snapshot.refresh; + expect((yield* harness.provider.snapshot.getSnapshot).supportsTextGeneration).toBe(true); + }), ); it.effect("keeps discovered workspace skills through session and command updates", () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeHarness(); - yield* harness.initialize; - const skills = [ - { - name: "deploy", - description: "Ship it", - path: "/workspace/.agent/skills/deploy", - enabled: true, - }, - ]; - const discovered = yield* harness.provider.snapshotForCwd("/workspace", skills); - expect(discovered.skills).toEqual(skills); - yield* harness.provider.onSessionStarted(started, "/workspace"); - yield* harness.provider.onAvailableCommands(commands, "/workspace"); - const after = yield* harness.provider.snapshot.getSnapshot; - expect( - after.workspaceSnapshots?.find((entry) => entry.cwd === "/workspace")?.skills, - ).toEqual(skills); - expect((yield* harness.provider.snapshotForCwd("/workspace")).skills).toEqual(skills); - - const rescanned = [ - ...skills, - { name: "review", path: "/workspace/.agent/skills/review", enabled: true }, - ]; - yield* harness.provider.snapshotForCwd("/workspace", rescanned); - yield* harness.provider.onSessionStarted(started, "/workspace"); - expect( - (yield* harness.provider.snapshot.getSnapshot).workspaceSnapshots?.find( - (entry) => entry.cwd === "/workspace", - )?.skills, - ).toEqual(rescanned); - }), - ), + Effect.gen(function* () { + const harness = yield* makeHarness(); + yield* harness.initialize; + const skills = [ + { + name: "deploy", + description: "Ship it", + path: "/workspace/.agent/skills/deploy", + enabled: true, + }, + ]; + const discovered = yield* harness.provider.snapshotForCwd("/workspace", skills); + expect(discovered.skills).toEqual(skills); + yield* harness.provider.onSessionStarted(started, "/workspace"); + yield* harness.provider.onAvailableCommands(commands, "/workspace"); + const after = yield* harness.provider.snapshot.getSnapshot; + expect(after.workspaceSnapshots?.find((entry) => entry.cwd === "/workspace")?.skills).toEqual( + skills, + ); + expect((yield* harness.provider.snapshotForCwd("/workspace")).skills).toEqual(skills); + + const rescanned = [ + ...skills, + { name: "review", path: "/workspace/.agent/skills/review", enabled: true }, + ]; + yield* harness.provider.snapshotForCwd("/workspace", rescanned); + yield* harness.provider.onSessionStarted(started, "/workspace"); + expect( + (yield* harness.provider.snapshot.getSnapshot).workspaceSnapshots?.find( + (entry) => entry.cwd === "/workspace", + )?.skills, + ).toEqual(rescanned); + }), ); it.effect("bounds workspace metadata without starting sessions for workspace lookup", () => - Effect.scoped( - Effect.gen(function* () { - const harness = yield* makeHarness(); - yield* harness.initialize; - yield* harness.provider.onSessionStarted(started); - for (let index = 0; index < 35; index++) { - yield* harness.provider.onAvailableCommands(commands, `/workspace-${index}`); - } - const snapshot = yield* harness.provider.snapshotForCwd("/workspace-34"); - expect(snapshot.workspaceSnapshots).toHaveLength(32); - expect(snapshot.workspaceSnapshots?.[0]?.cwd).toBe("/workspace-3"); - expect(snapshot.slashCommands).toEqual(expectedCommands); - expect(yield* Ref.get(harness.probeCalls)).toBe(1); - }), - ), + Effect.gen(function* () { + const harness = yield* makeHarness(); + yield* harness.initialize; + yield* harness.provider.onSessionStarted(started); + for (let index = 0; index < 35; index++) { + yield* harness.provider.onAvailableCommands(commands, `/workspace-${index}`); + } + const snapshot = yield* harness.provider.snapshotForCwd("/workspace-34"); + expect(snapshot.workspaceSnapshots).toHaveLength(32); + expect(snapshot.workspaceSnapshots?.[0]?.cwd).toBe("/workspace-3"); + expect(snapshot.slashCommands).toEqual(expectedCommands); + expect(yield* Ref.get(harness.probeCalls)).toBe(1); + }), ); it.effect( "publishes limits, retains a successful read on failure, and clears them on sign-out", () => - Effect.scoped( - Effect.gen(function* () { - const limits: ServerProviderUsageLimits = { - checkedAt: "2026-10-09T12:00:00.000Z", - windows: [ - { - id: "3p-weekly", - kind: "weekly", - label: "Claude & GPT (shared) · Weekly", - usedPercent: 75, - }, - ], - }; - const usage = yield* Ref.make(limits); - const harness = yield* makeHarness({ probeUsage: Ref.get(usage) }); - const published = yield* harness.nextPublish((snapshot) => !!snapshot.usageLimits); - yield* harness.initialize; - expect((yield* published)?.usageLimits).toEqual(limits); - // The second health check is the one the failed background read triggers. - const checks = yield* Ref.make(0); - const failedReadApplied = yield* Deferred.make(); - yield* Ref.set( - harness.probe, - Ref.updateAndGet(checks, (count) => count + 1).pipe( - Effect.tap((count) => - count === 2 ? Deferred.succeed(failedReadApplied, undefined) : Effect.void, - ), - Effect.as(initializeResult), - ), - ); - yield* Ref.set(usage, { - checkedAt: "2026-10-09T12:01:00.000Z", - windows: [], - unavailable: { reason: "probeFailed" }, - }); - yield* harness.provider.snapshot.refresh; - yield* Deferred.await(failedReadApplied); - expect((yield* harness.provider.snapshot.getSnapshot).usageLimits).toEqual(limits); - yield* harness.provider.onSignedOut; - expect((yield* harness.provider.snapshot.getSnapshot).usageLimits).toBeUndefined(); - yield* Ref.set(usage, { - checkedAt: "2026-10-09T12:02:00.000Z", - windows: [], - unavailable: { reason: "unsupported" }, - }); - const unsupported = yield* harness.nextPublish( - (snapshot) => snapshot.usageLimits?.unavailable?.reason === "unsupported", - ); - yield* harness.provider.snapshot.refresh; - expect((yield* unsupported)?.usageLimits?.windows).toEqual([]); - }), - ), - ); - - it.effect("a quota read finishing after sign-out cannot restore account limits", () => - Effect.scoped( Effect.gen(function* () { - const readStarted = yield* Deferred.make(); - const readResult = yield* Deferred.make(); - const usage = yield* Ref.make>( - Effect.succeed({ checkedAt: "2026-10-09T12:00:00.000Z", windows: [] }), - ); - const harness = yield* makeHarness({ probeUsage: Ref.get(usage).pipe(Effect.flatten) }); + const limits: ServerProviderUsageLimits = { + checkedAt: "2026-10-09T12:00:00.000Z", + windows: [ + { + id: "3p-weekly", + kind: "weekly", + label: "Claude & GPT (shared) · Weekly", + usedPercent: 75, + }, + ], + }; + const usage = yield* Ref.make(limits); + const harness = yield* makeHarness({ probeUsage: Ref.get(usage) }); + const published = yield* harness.nextPublish((snapshot) => !!snapshot.usageLimits); yield* harness.initialize; - yield* Ref.set(harness.probe, Effect.succeed(initializeResult)); + expect((yield* published)?.usageLimits).toEqual(limits); + // The second health check is the one the failed background read triggers. + const checks = yield* Ref.make(0); + const failedReadApplied = yield* Deferred.make(); yield* Ref.set( - usage, - Deferred.succeed(readStarted, undefined).pipe(Effect.andThen(Deferred.await(readResult))), + harness.probe, + Ref.updateAndGet(checks, (count) => count + 1).pipe( + Effect.tap((count) => + count === 2 ? Deferred.succeed(failedReadApplied, undefined) : Effect.void, + ), + Effect.as(initializeResult), + ), ); - const refresh = yield* harness.provider.snapshot.refresh.pipe(Effect.forkChild); - yield* Deferred.await(readStarted); - yield* harness.provider.onSignedOut; - yield* Deferred.succeed(readResult, { + yield* Ref.set(usage, { checkedAt: "2026-10-09T12:01:00.000Z", - windows: [ - { id: "gemini-5h", kind: "session", label: "Gemini · 5-hour", usedPercent: 10 }, - ], + windows: [], + unavailable: { reason: "probeFailed" }, }); - yield* Fiber.join(refresh); + yield* harness.provider.snapshot.refresh; + yield* Deferred.await(failedReadApplied); + expect((yield* harness.provider.snapshot.getSnapshot).usageLimits).toEqual(limits); + yield* harness.provider.onSignedOut; expect((yield* harness.provider.snapshot.getSnapshot).usageLimits).toBeUndefined(); + yield* Ref.set(usage, { + checkedAt: "2026-10-09T12:02:00.000Z", + windows: [], + unavailable: { reason: "unsupported" }, + }); + const unsupported = yield* harness.nextPublish( + (snapshot) => snapshot.usageLimits?.unavailable?.reason === "unsupported", + ); + yield* harness.provider.snapshot.refresh; + expect((yield* unsupported)?.usageLimits?.windows).toEqual([]); }), - ), + ); + + it.effect("a quota read finishing after sign-out cannot restore account limits", () => + Effect.gen(function* () { + const readStarted = yield* Deferred.make(); + const readResult = yield* Deferred.make(); + const usage = yield* Ref.make>( + Effect.succeed({ checkedAt: "2026-10-09T12:00:00.000Z", windows: [] }), + ); + const harness = yield* makeHarness({ probeUsage: Ref.get(usage).pipe(Effect.flatten) }); + yield* harness.initialize; + yield* Ref.set(harness.probe, Effect.succeed(initializeResult)); + yield* Ref.set( + usage, + Deferred.succeed(readStarted, undefined).pipe(Effect.andThen(Deferred.await(readResult))), + ); + const refresh = yield* harness.provider.snapshot.refresh.pipe(Effect.forkChild); + yield* Deferred.await(readStarted); + yield* harness.provider.onSignedOut; + yield* Deferred.succeed(readResult, { + checkedAt: "2026-10-09T12:01:00.000Z", + windows: [{ id: "gemini-5h", kind: "session", label: "Gemini · 5-hour", usedPercent: 10 }], + }); + yield* Fiber.join(refresh); + expect((yield* harness.provider.snapshot.getSnapshot).usageLimits).toBeUndefined(); + }), ); it.effect("reports ready and starts sessions while one shared quota read is pending", () => - Effect.scoped( - Effect.gen(function* () { - const reads = yield* Ref.make(0); - const readStarted = yield* Deferred.make(); - const readResult = yield* Deferred.make(); - const harness = yield* makeHarness({ - probeUsage: Ref.update(reads, (count) => count + 1).pipe( - Effect.andThen(Deferred.succeed(readStarted, undefined)), - Effect.andThen(Deferred.await(readResult)), - ), - }); - const published = yield* harness.nextPublish((snapshot) => !!snapshot.usageLimits); - yield* harness.initialize; - yield* Deferred.await(readStarted); - yield* harness.provider.onSessionStarted(started, "/workspace"); - yield* harness.provider.onSessionStarted(started, "/other"); - const ready = yield* harness.provider.snapshot.getSnapshot; - expect(ready).toMatchObject({ status: "ready", auth: { status: "authenticated" } }); - expect(ready.usageLimits).toBeUndefined(); - expect(yield* Ref.get(reads)).toBe(1); - const limits = { - checkedAt: "2026-10-09T12:00:00.000Z", - windows: [ - { id: "gemini-5h", kind: "session", label: "Gemini · 5-hour", usedPercent: 10 }, - ], - } satisfies ServerProviderUsageLimits; - yield* Deferred.succeed(readResult, limits); - expect((yield* published)?.usageLimits).toEqual(limits); - }), - ), + Effect.gen(function* () { + const reads = yield* Ref.make(0); + const readStarted = yield* Deferred.make(); + const readResult = yield* Deferred.make(); + const harness = yield* makeHarness({ + probeUsage: Ref.update(reads, (count) => count + 1).pipe( + Effect.andThen(Deferred.succeed(readStarted, undefined)), + Effect.andThen(Deferred.await(readResult)), + ), + }); + const published = yield* harness.nextPublish((snapshot) => !!snapshot.usageLimits); + yield* harness.initialize; + yield* Deferred.await(readStarted); + yield* harness.provider.onSessionStarted(started, "/workspace"); + yield* harness.provider.onSessionStarted(started, "/other"); + const ready = yield* harness.provider.snapshot.getSnapshot; + expect(ready).toMatchObject({ status: "ready", auth: { status: "authenticated" } }); + expect(ready.usageLimits).toBeUndefined(); + expect(yield* Ref.get(reads)).toBe(1); + const limits = { + checkedAt: "2026-10-09T12:00:00.000Z", + windows: [{ id: "gemini-5h", kind: "session", label: "Gemini · 5-hour", usedPercent: 10 }], + } satisfies ServerProviderUsageLimits; + yield* Deferred.succeed(readResult, limits); + expect((yield* published)?.usageLimits).toEqual(limits); + }), ); it.effect("a quota read for the previous account is dropped after a new sign-in", () => - Effect.scoped( - Effect.gen(function* () { - const oldReadStarted = yield* Deferred.make(); - const newLimits = { - checkedAt: "2026-10-09T12:01:00.000Z", - windows: [{ id: "3p-weekly", kind: "weekly", label: "Weekly", usedPercent: 40 }], - } satisfies ServerProviderUsageLimits; - const usage = yield* Ref.make>( - Deferred.succeed(oldReadStarted, undefined).pipe(Effect.andThen(Effect.never)), - ); - const harness = yield* makeHarness({ probeUsage: Ref.get(usage).pipe(Effect.flatten) }); - const published = yield* harness.nextPublish((snapshot) => !!snapshot.usageLimits); - yield* harness.initialize; - yield* Deferred.await(oldReadStarted); - yield* Ref.set(usage, Effect.succeed(newLimits)); - yield* harness.provider.onSignedIn; - yield* harness.provider.onSessionStarted(started); - expect((yield* published)?.usageLimits).toEqual(newLimits); - // Switching again must not republish the previous account's limits, even if the read fails. - const failed = { - checkedAt: "2026-10-09T12:02:00.000Z", - windows: [], - unavailable: { reason: "probeFailed" }, - } satisfies ServerProviderUsageLimits; - yield* Ref.set(usage, Effect.succeed(failed)); - const republished = yield* harness.nextPublish( - (snapshot) => snapshot.usageLimits?.unavailable?.reason === "probeFailed", - ); - yield* harness.provider.onSignedIn; - expect((yield* harness.provider.snapshot.getSnapshot).usageLimits).toBeUndefined(); - yield* harness.provider.onSessionStarted(started); - expect((yield* republished)?.usageLimits).toEqual(failed); - }), - ), + Effect.gen(function* () { + const oldReadStarted = yield* Deferred.make(); + const newLimits = { + checkedAt: "2026-10-09T12:01:00.000Z", + windows: [{ id: "3p-weekly", kind: "weekly", label: "Weekly", usedPercent: 40 }], + } satisfies ServerProviderUsageLimits; + const usage = yield* Ref.make>( + Deferred.succeed(oldReadStarted, undefined).pipe(Effect.andThen(Effect.never)), + ); + const harness = yield* makeHarness({ probeUsage: Ref.get(usage).pipe(Effect.flatten) }); + const published = yield* harness.nextPublish((snapshot) => !!snapshot.usageLimits); + yield* harness.initialize; + yield* Deferred.await(oldReadStarted); + yield* Ref.set(usage, Effect.succeed(newLimits)); + yield* harness.provider.onSignedIn; + yield* harness.provider.onSessionStarted(started); + expect((yield* published)?.usageLimits).toEqual(newLimits); + // Switching again must not republish the previous account's limits, even if the read fails. + const failed = { + checkedAt: "2026-10-09T12:02:00.000Z", + windows: [], + unavailable: { reason: "probeFailed" }, + } satisfies ServerProviderUsageLimits; + yield* Ref.set(usage, Effect.succeed(failed)); + const republished = yield* harness.nextPublish( + (snapshot) => snapshot.usageLimits?.unavailable?.reason === "probeFailed", + ); + yield* harness.provider.onSignedIn; + expect((yield* harness.provider.snapshot.getSnapshot).usageLimits).toBeUndefined(); + yield* harness.provider.onSessionStarted(started); + expect((yield* republished)?.usageLimits).toEqual(failed); + }), ); }); diff --git a/apps/server/src/provider/ClaudeCapabilitiesProbe.test.ts b/apps/server/src/provider/ClaudeCapabilitiesProbe.test.ts index bf3c9318877bb..b89cffa423874 100644 --- a/apps/server/src/provider/ClaudeCapabilitiesProbe.test.ts +++ b/apps/server/src/provider/ClaudeCapabilitiesProbe.test.ts @@ -14,10 +14,13 @@ import { buildClaudeCapabilitiesProbeQueryOptions, probeClaudeCapabilities, probeClaudeWorkspaceSnapshot, + spawnClaudeProbeThroughEnv, } from "./ClaudeProvider.ts"; vi.mock("@anthropic-ai/claude-agent-sdk", { spy: true }); +const hostPath = process.env.PATH; + const decodeClaudeSettings = Schema.decodeEffect(ClaudeSettings); it("drops strict MCP mode, and only that, next to an administrator's managed-mcp.json", () => { @@ -27,6 +30,7 @@ it("drops strict MCP mode, and only that, next to an administrator's managed-mcp environment: { HOME: "/home/user" }, cwd: undefined, managedMcpConfig: true, + spawnVia: undefined, }); // Claude exits before initializing when handed --strict-mcp-config here. @@ -35,6 +39,29 @@ it("drops strict MCP mode, and only that, next to an administrator's managed-mcp assert.equal(options.env?.ENABLE_CLAUDEAI_MCP_SERVERS, "false"); }); +// A shell trampoline would not do: dash drops the exported function, and the launcher exits 127. +it.live.skipIf(HostProcess.Platform.defaultValue() !== "linux")( + "starts a Bash launcher that calls an exported function in the probe's workspace", + () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const workspace = yield* fileSystem.realPath(yield* fileSystem.makeTempDirectoryScoped()); + yield* fileSystem.writeFileString(`${workspace}/claude`, '#!/bin/bash\nt3_launch "$@"\n'); + yield* fileSystem.chmod(`${workspace}/claude`, 0o755); + const child = spawnClaudeProbeThroughEnv({ + command: "claude", + args: ["--probe"], + cwd: workspace, + env: { PATH: `.:${hostPath}`, "BASH_FUNC_t3_launch%%": '() { echo "$PWD $@"; }' }, + // @effect-diagnostics-next-line abortControllerInEffect:off - The launcher exits on its own. + signal: new AbortController().signal, + }); + assert.deepEqual(child.spawnargs.slice(0, 3), ["/usr/bin/env", "-C", workspace]); + const output = yield* Effect.promise(() => child.stdout.setEncoding("utf8").toArray()); + assert.equal(output.join(""), `${workspace} --probe\n`); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + it.effect("reports a deleted worktree's scan as finished, not pending", () => Effect.gen(function* () { const snapshot = yield* probeClaudeWorkspaceSnapshot( diff --git a/apps/server/src/provider/ClaudeCodeModelRoutes.test.ts b/apps/server/src/provider/ClaudeCodeModelRoutes.test.ts new file mode 100644 index 0000000000000..2c06148cac680 --- /dev/null +++ b/apps/server/src/provider/ClaudeCodeModelRoutes.test.ts @@ -0,0 +1,67 @@ +import { assert, describe, it } from "@effect/vitest"; +import { type ModelSelection, ProviderInstanceId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Option from "effect/Option"; +import * as TestClock from "effect/testing/TestClock"; + +import { makeClaudeCodeModelRoutes } from "./ClaudeCodeModelRoutes.ts"; +import { + BUNDLED_CLAUDE_MODEL_CATALOG, + resolveClaudeCatalogApiModelId, +} from "./ClaudeModelCatalog.ts"; + +describe("Claude Code model routes", () => { + it.effect( + "a turn that starts before the first probe settles waits and then gets its routes", + () => + Effect.gen(function* () { + const haiku = { + instanceId: ProviderInstanceId.make("claudeAgent"), + model: "claude-haiku-4-5", + } satisfies ModelSelection; + const routes = yield* makeClaudeCodeModelRoutes( + () => Effect.succeed("same settings"), + "/state", + ); + const early = yield* routes + .route(BUNDLED_CLAUDE_MODEL_CATALOG, "/project") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* routes.record([ + { value: "haiku", resolvedModel: "us.anthropic.claude-haiku-4-5-20251001-v1:0" }, + ]); + yield* routes.settle; + assert.strictEqual( + resolveClaudeCatalogApiModelId(yield* Fiber.join(early), haiku), + "haiku", + ); + + // A first check that never reaches the probe still releases waiting turns. + const unprobed = yield* makeClaudeCodeModelRoutes( + () => Effect.succeed("same settings"), + "/state", + ); + const waiting = yield* unprobed + .route(BUNDLED_CLAUDE_MODEL_CATALOG, "/project") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* unprobed.settle; + assert.strictEqual(yield* Fiber.join(waiting), BUNDLED_CLAUDE_MODEL_CATALOG); + + const stuck = yield* makeClaudeCodeModelRoutes( + () => Effect.succeed("same settings"), + "/state", + ); + const fallback = yield* stuck + .route(BUNDLED_CLAUDE_MODEL_CATALOG, "/project") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust("2 minutes"); + assert.strictEqual(yield* Fiber.join(fallback), BUNDLED_CLAUDE_MODEL_CATALOG); + // Later turns, steers and effort changes do not wait for the stuck probe again. + const later = yield* stuck + .route(BUNDLED_CLAUDE_MODEL_CATALOG, "/project") + .pipe(Effect.timeoutOption("1 second"), Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust("1 second"); + assert.deepStrictEqual(yield* Fiber.join(later), Option.some(BUNDLED_CLAUDE_MODEL_CATALOG)); + }), + ); +}); diff --git a/apps/server/src/provider/ClaudeCodeModelRoutes.ts b/apps/server/src/provider/ClaudeCodeModelRoutes.ts new file mode 100644 index 0000000000000..6a9e7a48e64c5 --- /dev/null +++ b/apps/server/src/provider/ClaudeCodeModelRoutes.ts @@ -0,0 +1,116 @@ +import { + type ModelInfo as ClaudeModelInfo, + resolveSettings as resolveClaudeSettings, + type SettingSource, + type Settings as ClaudeSdkSettings, +} from "@anthropic-ai/claude-agent-sdk"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; + +import { claudeLocalSettingsDirectory } from "./Drivers/ClaudeHome.ts"; +import { type ClaudeModelCatalog, withClaudeCodeModelRoutes } from "./ClaudeModelCatalog.ts"; + +// Env names that pick the backend, its region, or the model a family alias resolves to. +const MODEL_SETTINGS_ENV_PREFIXES = [ + "ANTHROPIC_", + "CLAUDE_CODE_USE_", + "AWS_", + "CLOUD_ML_", + "VERTEX_", +]; + +/** Equal keys mean Claude Code resolves model ids the same way under both settings. */ +function claudeModelSettingsKey(settings: ClaudeSdkSettings): string { + const sortedEntries = (record: Readonly> | undefined) => + Object.entries(record ?? {}).toSorted(([a], [b]) => a.localeCompare(b)); + return JSON.stringify([ + settings.availableModels ?? null, + sortedEntries(settings.modelOverrides), + sortedEntries(settings.env).filter(([name]) => + MODEL_SETTINGS_ENV_PREFIXES.some((prefix) => name.startsWith(prefix)), + ), + ]); +} + +const NO_MODEL_SETTINGS_KEY = claudeModelSettingsKey({}); + +class ClaudeModelSettingsReadError extends Schema.TaggedError()( + "ClaudeModelSettingsReadError", + { cause: Schema.Defect() }, +) {} + +export const resolveClaudeModelSettingsKey = Effect.fn("resolveClaudeModelSettingsKey")(function* ( + settingSources: ReadonlyArray, + cwd: string | null, +) { + // The SDK reads the local file from cwd, Claude Code from the main checkout's root. No key, so + // no routes, when that root file touches model settings. + const localDirectory = cwd === null ? null : yield* claudeLocalSettingsDirectory(cwd); + if (localDirectory !== null && localDirectory !== cwd) { + const rootLocal = yield* Effect.tryPromise({ + try: () => resolveClaudeSettings({ cwd: localDirectory, settingSources: ["local"] }), + catch: (cause) => new ClaudeModelSettingsReadError({ cause }), + }).pipe( + Effect.map(({ sources }) => sources.find(({ source }) => source === "local")?.settings), + Effect.orElseSucceed(() => undefined), + ); + if (claudeModelSettingsKey(rootLocal ?? {}) !== NO_MODEL_SETTINGS_KEY) return undefined; + } + return yield* Effect.tryPromise({ + try: () => + resolveClaudeSettings({ + ...(cwd === null ? {} : { cwd }), + settingSources: [...settingSources], + }), + catch: (cause) => new ClaudeModelSettingsReadError({ cause }), + }).pipe( + Effect.map(({ effective }) => claudeModelSettingsKey(effective)), + Effect.orElseSucceed(() => undefined), + ); +}); + +type ReportedModel = Pick; + +/** + * Claude Code's reported model ids, applied to a turn only where its cwd resolves model settings + * as the probe's cwd did. `settle` releases turns waiting for the first probe, whatever its outcome. + */ +export const makeClaudeCodeModelRoutes = Effect.fn("makeClaudeCodeModelRoutes")(function* ( + settingsKey: (cwd: string | null) => Effect.Effect, + probeCwd: string, +) { + const firstProbeSettled = yield* Deferred.make(); + const reported = yield* Ref.make<{ + readonly models: ReadonlyArray; + readonly settingsKey: string | undefined; + }>({ models: [], settingsKey: undefined }); + return { + record: Effect.fn("ClaudeCodeModelRoutes.record")(function* ( + models: ReadonlyArray, + ) { + yield* Ref.set(reported, { models, settingsKey: yield* settingsKey(probeCwd) }); + }), + settle: Deferred.succeed(firstProbeSettled, undefined).pipe(Effect.asVoid), + route: Effect.fn("ClaudeCodeModelRoutes.route")(function* ( + base: ClaudeModelCatalog, + cwd: string | null, + launchSettings?: ClaudeSdkSettings, + ) { + // Leave room for the sequential version and capability probes before falling back once; + // an expired wait counts as settled so later calls do not wait again. + yield* Deferred.await(firstProbeSettled).pipe( + Effect.timeoutOption("2 minutes"), + Effect.andThen(Deferred.succeed(firstProbeSettled, undefined)), + ); + if (claudeModelSettingsKey(launchSettings ?? {}) !== NO_MODEL_SETTINGS_KEY) return base; + const { models, settingsKey: probeKey } = yield* Ref.get(reported); + const routed = withClaudeCodeModelRoutes(base, models); + // Project settings can point an alias at another model, so routes hold only where the + // cwd resolves the same model settings as the probe's. + if (routed === base || probeKey === undefined) return base; + return (yield* settingsKey(cwd)) === probeKey ? routed : base; + }), + }; +}); diff --git a/apps/server/src/provider/ClaudeModelCatalog.test.ts b/apps/server/src/provider/ClaudeModelCatalog.test.ts index c8c8abdfcf2b0..bbcc02f901329 100644 --- a/apps/server/src/provider/ClaudeModelCatalog.test.ts +++ b/apps/server/src/provider/ClaudeModelCatalog.test.ts @@ -1,8 +1,9 @@ import { assert, describe, it } from "@effect/vitest"; -import { ProviderDriverKind, ProviderInstanceId } from "@t3tools/contracts"; +import { type ModelSelection, ProviderDriverKind, ProviderInstanceId } from "@t3tools/contracts"; import { resolveProviderCatalog, type ModelManifestData } from "./ModelManifest.ts"; import { + BUNDLED_CLAUDE_MODEL_CATALOG, capClaudeCatalogEffort, type ClaudeModelCatalog, formatClaudeVersionUpgradeMessage, @@ -16,6 +17,7 @@ import { resolveClaudeModelSlug, resolveClaudeUpdateRequiredModels, scopeClaudeModelCatalog, + withClaudeCodeModelRoutes, } from "./ClaudeModelCatalog.ts"; const manifest = (): ModelManifestData => ({ @@ -135,6 +137,13 @@ describe("Claude model catalog", () => { formatClaudeVersionUpgradeMessage(catalog, "3.1.9"), "Claude Code v3.1.9 is too old for Claude Synthetic Next. Upgrade to v3.2.0 or newer to access it.", ); + // Claude Code before 2.1.321 does not know claude-fable-5-2 and runs it with a 200K window. + assert.notInclude( + resolveClaudeModelsForVersion(BUNDLED_CLAUDE_MODEL_CATALOG, "2.1.320").map( + (model) => model.slug, + ), + "claude-fable-5-2", + ); }); it("resolves aliases and declarative adapter mappings", () => { @@ -233,6 +242,65 @@ describe("Claude model catalog", () => { ); }); + // Rows Claude Code 2.1.310 reports from initialization for each configuration. + it("sends the model id Claude Code reports when it serves a catalog model under another id", () => { + const apiModel = ( + catalog: ClaudeModelCatalog, + model: string, + options?: ModelSelection["options"], + ) => + resolveClaudeCatalogApiModelId(catalog, { + instanceId: ProviderInstanceId.make("claudeAgent"), + model, + ...(options ? { options } : {}), + }); + + // Bedrock with ANTHROPIC_DEFAULT_*_MODEL and an availableModels allowlist of their ids. + const configuredBedrock = withClaudeCodeModelRoutes(BUNDLED_CLAUDE_MODEL_CATALOG, [ + { value: "default", resolvedModel: "global.anthropic.claude-opus-5-5" }, + { value: "sonnet", resolvedModel: "global.anthropic.claude-sonnet-5-5[1m]" }, + { value: "opus", resolvedModel: "global.anthropic.claude-opus-5-5" }, + { value: "haiku", resolvedModel: "global.anthropic.claude-haiku-5-5" }, + ]); + assert.strictEqual(apiModel(configuredBedrock, "claude-haiku-5-5"), "haiku"); + assert.strictEqual(apiModel(configuredBedrock, "haiku-5.5"), "haiku"); + assert.strictEqual(apiModel(configuredBedrock, "claude-sonnet-5-5"), "sonnet"); + assert.strictEqual(apiModel(configuredBedrock, "claude-opus-5-5"), "opus"); + assert.strictEqual(apiModel(configuredBedrock, "claude-opus-5"), "claude-opus-5"); + + // Bedrock with no model settings: Claude Code's `haiku` is Haiku 4.5 there. + const defaultBedrock = withClaudeCodeModelRoutes(BUNDLED_CLAUDE_MODEL_CATALOG, [ + { + value: "us.anthropic.claude-opus-4-6-v1", + resolvedModel: "us.anthropic.claude-opus-4-6-v1", + }, + { + value: "us.anthropic.claude-opus-4-6-v1[1m]", + resolvedModel: "us.anthropic.claude-opus-4-6-v1[1m]", + }, + { value: "haiku", resolvedModel: "us.anthropic.claude-haiku-4-5-20251001-v1:0" }, + ]); + assert.strictEqual( + apiModel(defaultBedrock, "claude-opus-4-6", [{ id: "contextWindow", value: "1m" }]), + "us.anthropic.claude-opus-4-6-v1[1m]", + ); + assert.strictEqual( + apiModel(defaultBedrock, "claude-opus-4-6", [{ id: "contextWindow", value: "200k" }]), + "us.anthropic.claude-opus-4-6-v1", + ); + assert.strictEqual(apiModel(defaultBedrock, "claude-haiku-4-5"), "haiku"); + assert.strictEqual(apiModel(defaultBedrock, "claude-haiku-5-5"), "claude-haiku-5-5"); + + // First-party ids already are what Claude Code runs, even when an alias points elsewhere. + const firstParty = withClaudeCodeModelRoutes(BUNDLED_CLAUDE_MODEL_CATALOG, [ + { value: "default", resolvedModel: "claude-opus-5-5" }, + { value: "opus", resolvedModel: "claude-opus-5-5" }, + { value: "claude-fable-5-1[1m]", resolvedModel: "claude-fable-5-1[1m]" }, + { value: "haiku", resolvedModel: "claude-haiku-4-5-20251001" }, + ]); + assert.strictEqual(firstParty, BUNDLED_CLAUDE_MODEL_CATALOG); + }); + it("marks effort options that would run above the cap without changing ids or defaults", () => { const effortModel = (slug: string) => ({ slug, diff --git a/apps/server/src/provider/ClaudeModelCatalog.ts b/apps/server/src/provider/ClaudeModelCatalog.ts index ceeb94a355e86..e3d4ea4397c0e 100644 --- a/apps/server/src/provider/ClaudeModelCatalog.ts +++ b/apps/server/src/provider/ClaudeModelCatalog.ts @@ -32,6 +32,8 @@ export interface ClaudeCatalogModel { readonly model: ServerProviderModel; readonly runtime: ClaudeCodeProfile; readonly compatibility: ClaudeCodeCompatibility; + /** What to send Claude Code instead of the slug, from its own model list. */ + readonly apiModel?: string; } export interface ClaudeModelCatalog { @@ -221,7 +223,7 @@ export function claudeReportedModels( return models; } -function resolveClaudeCatalogModel( +export function resolveClaudeCatalogModel( catalog: ClaudeModelCatalog, slugOrAlias: string | null | undefined, ): ClaudeCatalogModel | undefined { @@ -235,6 +237,44 @@ function resolveClaudeCatalogModel( ); } +// Cloud backends decorate first-party ids: Bedrock adds a region prefix and a +// version ("us.anthropic.claude-opus-4-6-v1"), Vertex adds "@". +function stripClaudeBackendDecoration(wireModel: string): string { + return wireModel + .replace(/^(?:[a-z-]+\.)?anthropic\./, "") + .replace(/-v\d+(?::\d+)?$/, "") + .replace(/@[^@]*$/, ""); +} + +/** + * On Bedrock, Vertex and similar backends Claude Code does not accept the first-party slug: it + * either sends it verbatim or, when `availableModels` excludes it, silently runs the default + * model instead. Its model list names the id that does work, often a family alias that + * `ANTHROPIC_DEFAULT_*_MODEL` points at a backend id. + */ +export function withClaudeCodeModelRoutes( + catalog: ClaudeModelCatalog, + reportedModels: ReadonlyArray>, +): ClaudeModelCatalog { + const apiModels = new Map(); + for (const reported of reportedModels) { + if (reported.value === "default") continue; + const wireModel = stripClaudeContextWindowSuffix(reported.resolvedModel ?? reported.value); + const firstPartyModel = stripClaudeBackendDecoration(wireModel); + if (firstPartyModel === wireModel) continue; + const entry = resolveClaudeCatalogModel(catalog, firstPartyModel); + if (!entry || apiModels.has(entry.model.slug)) continue; + apiModels.set(entry.model.slug, stripClaudeContextWindowSuffix(reported.value)); + } + if (apiModels.size === 0) return catalog; + return { + models: catalog.models.map((entry) => { + const apiModel = apiModels.get(entry.model.slug); + return apiModel ? { ...entry, apiModel } : entry; + }), + }; +} + export function resolveClaudeModelSlug(catalog: ClaudeModelCatalog, slugOrAlias: string): string { return resolveClaudeCatalogModel(catalog, slugOrAlias)?.model.slug ?? slugOrAlias; } @@ -506,7 +546,7 @@ export function resolveClaudeCatalogApiModelId( modelSelection: ModelSelection, ): string { const entry = resolveClaudeCatalogModel(catalog, modelSelection.model); - const slug = entry?.model.slug ?? modelSelection.model; + const slug = entry?.apiModel ?? entry?.model.slug ?? modelSelection.model; const descriptors = getProviderOptionDescriptors({ caps: entry?.model.capabilities ?? EMPTY_CAPABILITIES, selections: modelSelection.options, diff --git a/apps/server/src/provider/ClaudeProvider.ts b/apps/server/src/provider/ClaudeProvider.ts index f969544fad064..b657a2b813feb 100644 --- a/apps/server/src/provider/ClaudeProvider.ts +++ b/apps/server/src/provider/ClaudeProvider.ts @@ -1,3 +1,6 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off -- the SDK's spawn hook returns a Node child process. +import * as NodeChildProcess from "node:child_process"; + import { resolveProviderProbeTimeoutMs } from "@t3tools/provider-core/server/snapshotProbe"; import { type ClaudeSettings, @@ -18,6 +21,9 @@ import * as Result from "effect/Result"; import * as Schema from "effect/Schema"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; import { createModelCapabilities } from "@t3tools/shared/model"; +import { envTrampoline, supportsEnvChdir } from "@t3tools/shared/envTrampoline"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as SpawnWorker from "../process/SpawnWorker.ts"; import { resolveSpawnCommand } from "@t3tools/shared/shell"; import { query as claudeQuery, @@ -27,6 +33,7 @@ import { type SDKControlGetUsageResponse, type SDKUserMessage, type SettingSource, + type SpawnOptions as ClaudeSpawnOptions, } from "@anthropic-ai/claude-agent-sdk"; import { @@ -171,11 +178,16 @@ function formatClaudeSubscriptionAuthLabel(subscriptionType: string): string { return `Claude ${subscriptionLabel} Subscription`; } +function hasApiKey(apiKeySource: string | undefined): boolean { + return Boolean(apiKeySource) && apiKeySource !== "none"; +} + function claudeAuthMetadata(input: { readonly subscriptionType: string | undefined; readonly authMethod: string | undefined; + readonly apiKeySource: string | undefined; }): { readonly type: string; readonly label: string } | undefined { - if (normalizeClaudeAuthMethod(input.authMethod) === "apiKey") { + if (normalizeClaudeAuthMethod(input.authMethod) === "apiKey" || hasApiKey(input.apiKeySource)) { return { type: "apiKey", label: "Claude API Key", @@ -210,8 +222,7 @@ function claudeAuthStatus( } if (capabilities.tokenSource !== "none") return "authenticated"; // API keys can authenticate with tokenSource: none; also check key and account fields. - const hasApiKey = Boolean(capabilities.apiKeySource) && capabilities.apiKeySource !== "none"; - return hasApiKey || capabilities.email || capabilities.subscriptionType + return hasApiKey(capabilities.apiKeySource) || capabilities.email || capabilities.subscriptionType ? "authenticated" : "unauthenticated"; } @@ -227,14 +238,49 @@ const CLAUDE_CAPABILITIES_PROBE_SETTING_SOURCES = [ "local", ] as const satisfies ReadonlyArray; +/** + * The SDK's own spawn stats the executable and hands libuv the workspace as cwd, so a mount + * that stops answering freezes the server. `env` enters the cwd and finds the CLI in the child. + * A shell would not do: dash drops the exported Bash functions a launcher may call. + */ +export function spawnClaudeProbeThroughEnv({ + command, + args, + cwd, + env, + signal, +}: ClaudeSpawnOptions) { + const launch = envTrampoline(command, args, cwd) ?? { command, args, cwd }; + return NodeChildProcess.spawn(launch.command, launch.args, { + ...("cwd" in launch ? { cwd: launch.cwd } : {}), + env, + signal, + // The probe ignores stderr. + stdio: ["pipe", "pipe", "ignore"], + }); +} + +/** Windows runs CreateProcess on the calling thread, so the probe launches from a launcher thread. */ +function spawnClaudeProbeOffThread({ command, args, cwd, env, signal }: ClaudeSpawnOptions) { + const child = SpawnWorker.spawnPiped(command, args, { cwd, env, signal, windowsHide: true }); + // The probe ignores stderr; drain it so a full pipe cannot stall the CLI. + child.stderr.resume(); + return child; +} + export function buildClaudeCapabilitiesProbeQueryOptions(input: { readonly executablePath: string; readonly abortController: AbortController; readonly environment: NodeJS.ProcessEnv; readonly cwd: string | undefined; readonly managedMcpConfig: boolean; + readonly spawnVia: "env" | "launcher-thread" | undefined; }): ClaudeQueryOptions { return { + ...(input.spawnVia === "env" ? { spawnClaudeCodeProcess: spawnClaudeProbeThroughEnv } : {}), + ...(input.spawnVia === "launcher-thread" + ? { spawnClaudeCodeProcess: spawnClaudeProbeOffThread } + : {}), persistSession: false, pathToClaudeCodeExecutable: input.executablePath, abortController: input.abortController, @@ -464,6 +510,13 @@ const probeClaudeCapabilities = ( ...(yield* readClaudeSettingsEnv(claudeSettings, cwd, claudeEnvironment)), }) : 0; + const platform = yield* HostProcess.Platform; + const spawnVia = + platform === "win32" + ? "launcher-thread" + : platform === "linux" && (yield* Effect.promise(supportsEnvChdir)) + ? "env" + : undefined; return yield* Effect.tryPromise({ try: async () => { const q = claudeQuery({ @@ -477,6 +530,7 @@ const probeClaudeCapabilities = ( environment: claudeEnvironment, cwd, managedMcpConfig, + spawnVia, }), }); const init = await q.initializationResult(); @@ -776,6 +830,7 @@ export const checkClaudeProviderStatus = Effect.fn("checkClaudeProviderStatus")( (yield* readClaudeSettingsEnv(claudeSettings, cwd, resolvedEnvironment)).ANTHROPIC_BASE_URL ?? resolvedEnvironment.ANTHROPIC_BASE_URL; const customApiEndpoint = customClaudeApiEndpointHost(baseUrl) !== undefined; + const endpoint = customApiEndpoint ? URL.parse(baseUrl?.trim() ?? "")?.origin : undefined; if (!customApiEndpoint && claudeAuthStatus(capabilities) === "unauthenticated") { const path = yield* Path.Path; const configDir = (yield* makeClaudeEnvironment(claudeSettings, resolvedEnvironment)) @@ -804,10 +859,13 @@ export const checkClaudeProviderStatus = Effect.fn("checkClaudeProviderStatus")( // A proxy's ANTHROPIC_AUTH_TOKEN reads as an API key; the endpoint is what the token is for. const authMetadata = apiProviderAuthMetadata(capabilities.apiProvider) ?? - (customApiEndpoint ? { type: "customEndpoint", label: "Custom API endpoint" } : undefined) ?? + (customApiEndpoint + ? { type: "customEndpoint", label: "Custom API endpoint", ...(endpoint ? { endpoint } : {}) } + : undefined) ?? claudeAuthMetadata({ subscriptionType: capabilities.subscriptionType, authMethod: capabilities.tokenSource, + apiKeySource: capabilities.apiKeySource, }); const usageCheckedAt = capabilities.usageCheckedAt ?? checkedAt; const usageLimits = !capabilities.usage diff --git a/apps/server/src/provider/CodexProvider.test.ts b/apps/server/src/provider/CodexProvider.test.ts index 7eadf550eab28..ad15bf0896eb2 100644 --- a/apps/server/src/provider/CodexProvider.test.ts +++ b/apps/server/src/provider/CodexProvider.test.ts @@ -2,6 +2,7 @@ import { assert, it } from "@effect/vitest"; import { applyPreferredCodexDefaultModel, + codexCustomEndpoint, codexRoutesSpendNativeQuota, mapCodexModelCapabilities, } from "./CodexProvider.ts"; @@ -192,4 +193,12 @@ it("reports native quota only for an OpenAI login on a native route", () => { name, ); } + // The custom endpoint is what a usage-limit hub is matched against. + assert.strictEqual(codexCustomEndpoint(hub), "https://hub.example.com"); + assert.strictEqual( + codexCustomEndpoint({ openai_base_url: "http://127.0.0.1:8317/v1" }), + "http://127.0.0.1:8317", + ); + assert.strictEqual(codexCustomEndpoint({ openai_base_url: "https://chatgpt.com/x" }), undefined); + assert.strictEqual(codexCustomEndpoint({}), undefined); }); diff --git a/apps/server/src/provider/CodexProvider.ts b/apps/server/src/provider/CodexProvider.ts index ab2cda61e53ee..df105d3ba20fe 100644 --- a/apps/server/src/provider/CodexProvider.ts +++ b/apps/server/src/provider/CodexProvider.ts @@ -57,7 +57,6 @@ import { import * as HostProcess from "@t3tools/shared/HostProcess"; import packageJson from "../../package.json" with { type: "json" }; const isCodexAppServerSpawnError = Schema.is(CodexErrors.CodexAppServerSpawnError); -const RATE_LIMITS_PROBE_TIMEOUT_MS = 3_000; type CodexRateLimitsProbe = | { @@ -95,17 +94,21 @@ const CodexCyberAccessConfig = Schema.Struct({ }); const decodeCodexCyberAccessConfig = Schema.decodeUnknownOption(CodexCyberAccessConfig); -function routesToNativeOpenai(config: unknown): boolean { +// Null is Codex's auth-dependent OpenAI default; undefined is a route that cannot be read. +function codexRouteBaseUrl(config: unknown): string | null | undefined { const decoded = decodeCodexRoutingConfig(config); - if (Option.isNone(decoded)) return false; + if (Option.isNone(decoded)) return undefined; const routing = decoded.value; const providerId = routing.model_provider ?? "openai"; + if (providerId === "openai") return routing.openai_base_url || null; const provider = routing.model_providers?.[providerId]; - if (providerId !== "openai" && !provider) return false; - const baseUrl = - providerId === "openai" ? routing.openai_base_url || undefined : provider?.base_url; - // A provider without an override uses Codex's auth-dependent OpenAI default. - if (baseUrl == null) return true; + return provider ? (provider.base_url ?? null) : undefined; +} + +function routesToNativeOpenai(config: unknown): boolean { + const baseUrl = codexRouteBaseUrl(config); + if (baseUrl === undefined) return false; + if (baseUrl === null) return true; if (!URL.canParse(baseUrl)) return false; const url = new URL(baseUrl); return ( @@ -114,6 +117,15 @@ function routesToNativeOpenai(config: unknown): boolean { ); } +/** The origin of the custom endpoint Codex sends model requests to, if it uses one. */ +export function codexCustomEndpoint(config: unknown): string | undefined { + const baseUrl = codexRouteBaseUrl(config); + const url = baseUrl ? URL.parse(baseUrl) : null; + return url && /^https?:$/.test(url.protocol) && !routesToNativeOpenai(config) + ? url.origin + : undefined; +} + // A stored login does not mean this provider's traffic spends that login's quota. export const codexRoutesSpendNativeQuota = ( account: CodexSchema.V2GetAccountResponse, @@ -141,6 +153,7 @@ const CODEX_PRESENTATION = { export interface CodexAppServerProviderSnapshot { readonly account: CodexSchema.V2GetAccountResponse; + readonly endpoint?: string | undefined; readonly nativeUsage?: boolean; readonly rateLimits?: CodexRateLimitsProbe; readonly version: string | undefined; @@ -670,11 +683,7 @@ const readCodexRateLimits = ( client: CodexClient.CodexAppServerClient["Service"], deadlineMs: number, ) => - withCodexDiscoveryDeadline( - client.request("account/rateLimits/read", null), - deadlineMs, - RATE_LIMITS_PROBE_TIMEOUT_MS, - ).pipe( + withCodexDiscoveryDeadline(client.request("account/rateLimits/read", null), deadlineMs).pipe( Effect.map((response): CodexRateLimitsProbe => response ? { @@ -762,6 +771,7 @@ const probeCodexAppServerProvider = Effect.fn("probeCodexAppServerProvider")(fun const cyberAccess = Option.getOrUndefined(decodeCodexCyberAccessConfig(config)); return { account: accountResponse, + endpoint: config ? codexCustomEndpoint(config) : undefined, nativeUsage, ...(rateLimits ? { rateLimits } : {}), version, @@ -1080,7 +1090,9 @@ export const checkCodexProviderStatus = Effect.fn("checkCodexProviderStatus")(fu installed: true, version: snapshot.version ?? null, status: snapshot.models === undefined ? "warning" : accountStatus.status, - auth: accountStatus.auth, + auth: snapshot.endpoint + ? { ...accountStatus.auth, endpoint: snapshot.endpoint } + : accountStatus.auth, ...(accountStatus.message ? { message: accountStatus.message } : snapshot.models === undefined diff --git a/apps/server/src/provider/CodexThreadRevert.test.ts b/apps/server/src/provider/CodexThreadRevert.test.ts index e48caf094027c..f268719154722 100644 --- a/apps/server/src/provider/CodexThreadRevert.test.ts +++ b/apps/server/src/provider/CodexThreadRevert.test.ts @@ -68,7 +68,6 @@ it.effect("finds the revert boundary across pages of newest-first turns", () => ], }), ), - Effect.scoped, ), ); @@ -87,6 +86,5 @@ it.effect("rejects repeated cursors instead of looping or reverting incomplete h entries: [...page(1, null, 3, ["newest"], "again"), ...page(2, "again", 2, [], "again")], }), ), - Effect.scoped, ), ); diff --git a/apps/server/src/provider/Drivers/AntigravityDriver.test.ts b/apps/server/src/provider/Drivers/AntigravityDriver.test.ts index b43f30a419472..447745fc673a0 100644 --- a/apps/server/src/provider/Drivers/AntigravityDriver.test.ts +++ b/apps/server/src/provider/Drivers/AntigravityDriver.test.ts @@ -383,7 +383,7 @@ it.layer(layerTest)("AntigravityDriver", (it) => { const helper = { fs: { readTextFile: false, writeTextFile: false }, terminal: false, - session: { notices: {} }, + session: { notices: {}, compaction: {} }, }; expect( (yield* h.readRequests) @@ -397,7 +397,7 @@ it.layer(layerTest)("AntigravityDriver", (it) => { { fs: { readTextFile: true, writeTextFile: false }, terminal: false, - session: { notices: {} }, + session: { notices: {}, compaction: {} }, elicitation: { form: {} }, }, ]); @@ -444,7 +444,7 @@ it.layer(layerTest)("AntigravityDriver", (it) => { .map((request) => request.params?.value), ).toEqual(["default"]); } - }).pipe(Effect.scoped), + }), ); it.effect.skipIf(windowsHost)( @@ -464,7 +464,7 @@ it.layer(layerTest)("AntigravityDriver", (it) => { expect(h.acquisitions).toHaveLength(2); expect(h.releases).toEqual([h.first.version, h.signedOut.version]); yield* h.assertClosed; - }).pipe(Effect.scoped), + }), ); it.effect.skipIf(windowsHost)("keeps the previous catalog when executable resolution fails", () => @@ -481,7 +481,7 @@ it.layer(layerTest)("AntigravityDriver", (it) => { expect(h.acquisitions).toHaveLength(2); expect(h.releases).toEqual([h.first.version]); yield* h.assertClosed; - }).pipe(Effect.scoped), + }), ); it.effect.skipIf(windowsHost)( @@ -502,7 +502,7 @@ it.layer(layerTest)("AntigravityDriver", (it) => { expect(after.models).toEqual(before.models); expect(after.auth).toEqual(before.auth); yield* h.assertClosed; - }).pipe(Effect.scoped), + }), ); it.effect.skipIf(windowsHost)( @@ -539,7 +539,7 @@ it.layer(layerTest)("AntigravityDriver", (it) => { expect(after.auth).toEqual(before.auth); expect(JSON.stringify(after)).not.toContain("sk-secret-123"); yield* h.assertClosed; - }).pipe(Effect.scoped), + }), ); it.effect.skipIf(windowsHost)( @@ -577,6 +577,6 @@ it.layer(layerTest)("AntigravityDriver", (it) => { ); expect(yield* fs.exists(directories.runtimeTemp)).toBe(false); expect(yield* fs.exists(legacyRoot)).toBe(false); - }).pipe(Effect.scoped), + }), ); }); diff --git a/apps/server/src/provider/Drivers/AntigravitySkills.test.ts b/apps/server/src/provider/Drivers/AntigravitySkills.test.ts index 5060d8a24e5ed..c81b38a00dbcb 100644 --- a/apps/server/src/provider/Drivers/AntigravitySkills.test.ts +++ b/apps/server/src/provider/Drivers/AntigravitySkills.test.ts @@ -273,15 +273,43 @@ it.layer(NodeServices.layer)("discoverAntigravitySkills", (it) => { }), ); - it.effect("rejects an oversized skill instead of returning an incomplete catalog", () => + it.effect("does not spend the scan budget on skill bodies", () => Effect.gen(function* () { const path = yield* Path.Path; const input = yield* makeWorkspace(); - const skillPath = yield* writeSkill( - path.join(input.cwd, ".agents", "skills", "oversized"), - `---\nname: oversized\ndescription: Read a large skill.\n---\n${"x".repeat(1_000_000)}`, + const root = path.join(input.userHome, ".gemini", "config", "skills"); + const expected = []; + for (let index = 0; index < 9; index += 1) { + const name = `large-${index}`; + const description = `Large skill ${index}. ${"y".repeat(2_000)}`; + const skillPath = yield* writeSkill( + path.join(root, name), + `---\nname: ${name}\ndescription: ${description}\n---\n${"x".repeat(950_000)}`, + ); + expected.push({ name, description, path: skillPath, scope: "user", enabled: true }); + } + + assert.deepEqual(yield* discoverAntigravitySkills(input), expected); + }), + ); + + it.effect("rejects frontmatter that does not close within the per-skill limit", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const input = yield* makeWorkspace(); + const root = path.join(input.cwd, ".agents", "skills"); + const largeBodyPath = yield* writeSkill( + path.join(root, "large-body"), + `---\nname: large-body\n---\n${"x".repeat(2_000_000)}`, ); + assert.deepEqual(yield* discoverAntigravitySkills(input), [ + { name: "large-body", path: largeBodyPath, scope: "project", enabled: true }, + ]); + const skillPath = yield* writeSkill( + path.join(root, "oversized"), + `---\nname: oversized\ndescription: ${"x".repeat(1_000_000)}\n---\n`, + ); const result = yield* discoverAntigravitySkills(input).pipe(Effect.result); assert.equal(result._tag, "Failure"); if (result._tag === "Failure") { @@ -291,7 +319,7 @@ it.layer(NodeServices.layer)("discoverAntigravitySkills", (it) => { }), ); - it.effect("bounds the total read size across skills", () => + it.effect("bounds the total frontmatter read size across skills", () => Effect.gen(function* () { const path = yield* Path.Path; const input = yield* makeWorkspace(); @@ -299,7 +327,7 @@ it.layer(NodeServices.layer)("discoverAntigravitySkills", (it) => { for (let index = 0; index < 9; index += 1) { yield* writeSkill( path.join(root, `large-${index}`), - `---\nname: large-${index}\n---\n${"x".repeat(900_000)}`, + `---\nname: large-${index}\ndescription: ${"x".repeat(900_000)}\n---\n`, ); } diff --git a/apps/server/src/provider/Drivers/AntigravitySkills.ts b/apps/server/src/provider/Drivers/AntigravitySkills.ts index 19e9d8d293f48..96e1fe61109f4 100644 --- a/apps/server/src/provider/Drivers/AntigravitySkills.ts +++ b/apps/server/src/provider/Drivers/AntigravitySkills.ts @@ -37,6 +37,8 @@ export function antigravityUserSkillDirectories( const MAX_SKILL_BYTES = 1_000_000; const MAX_SCAN_BYTES = 8_000_000; const MAX_SCAN_ENTRIES = 10_000; +// Typical frontmatter fits in one chunk, so most skills cost about this much of the scan budget. +const SKILL_READ_CHUNK_BYTES = 1024; const SkillFrontmatter = Schema.Struct({ name: Schema.optional(Schema.NullOr(Schema.String)), @@ -106,7 +108,29 @@ function skillPathSortKey(entry: string) { ); } -const readSkill = Effect.fn("readAntigravitySkill")(function* ( +const DASH = 0x2d; + +/** Finds the byte offset after the second `---`, as parseSkillFrontmatter's indexOf calls do. */ +function makeFrontmatterEndFinder() { + let bytesSeen = 0; + let dashes = 0; + let markers = 0; + return (chunk: Uint8Array) => { + for (const byte of chunk) { + bytesSeen += 1; + dashes = byte === DASH ? dashes + 1 : 0; + if (dashes === 3) { + dashes = 0; + markers += 1; + if (markers === 2) return bytesSeen; + } + } + return undefined; + }; +} + +/** Reads through the frontmatter only, so skill bodies do not spend the scan budget. */ +const readSkillFrontmatter = Effect.fn("readAntigravitySkill")(function* ( skillPath: string, budget: ScanBudget, ) { @@ -115,26 +139,27 @@ const readSkill = Effect.fn("readAntigravitySkill")(function* ( if (info?.type !== "File") return undefined; const byteLimit = Math.min(MAX_SKILL_BYTES, budget.remainingBytes); - if (info.size > BigInt(byteLimit)) { - return yield* new AntigravitySkillsProbeError({ - reason: "scan-budget-exhausted", - path: skillPath, - }); - } + const findFrontmatterEnd = makeFrontmatterEndFinder(); + let frontmatterEnd: number | undefined; const chunks = yield* readIfPresent( - fileSystem.stream(skillPath, { bytesToRead: byteLimit + 1 }).pipe(Stream.runCollect), + fileSystem + .stream(skillPath, { bytesToRead: byteLimit + 1, chunkSize: SKILL_READ_CHUNK_BYTES }) + .pipe( + Stream.takeUntil((chunk) => (frontmatterEnd = findFrontmatterEnd(chunk)) !== undefined), + Stream.runCollect, + ), skillPath, ); if (chunks === undefined) return undefined; const bytes = Buffer.concat(chunks); - if (bytes.byteLength > byteLimit) { + budget.remainingBytes -= bytes.byteLength; + if ((frontmatterEnd ?? bytes.byteLength) > byteLimit) { return yield* new AntigravitySkillsProbeError({ reason: "scan-budget-exhausted", path: skillPath, }); } - budget.remainingBytes -= bytes.byteLength; - return bytes.toString("utf8"); + return bytes.toString("utf8", 0, frontmatterEnd); }); /** Keep read errors typed so workspace snapshots cannot cache partial results. */ @@ -187,7 +212,7 @@ export const discoverAntigravitySkills = Effect.fn("discoverAntigravitySkills")( if (skillFileName !== undefined) { if (!skillFileName.endsWith(".md")) return; const skillPath = path.join(directory, skillFileName); - const contents = yield* readSkill(skillPath, budget); + const contents = yield* readSkillFrontmatter(skillPath, budget); if (contents === undefined) return; const skill = parseSkillFrontmatter(contents, skillFileName); if (!skill || skillsByName.has(skill.name)) return; diff --git a/apps/server/src/provider/Drivers/AntigravityUsage.ts b/apps/server/src/provider/Drivers/AntigravityUsage.ts index 5cb55ed77ac09..2eb9744a3e42a 100644 --- a/apps/server/src/provider/Drivers/AntigravityUsage.ts +++ b/apps/server/src/provider/Drivers/AntigravityUsage.ts @@ -15,7 +15,11 @@ import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import { resolveAntigravityInstanceDirectories } from "../antigravityAuthSupport.ts"; -import { makeAntigravityUsageCache, readAntigravityUsage } from "./antigravityUsageReader.ts"; +import { + antigravityRateModel, + makeAntigravityUsageCache, + readAntigravityUsage, +} from "./antigravityUsageReader.ts"; export class AntigravityUsage extends Context.Service< AntigravityUsage, @@ -123,6 +127,7 @@ export const layer = Layer.effect(AntigravityUsage, make); export const antigravityUsageReader: ProviderUsageReader = { kind: "scan", provider: "antigravity", + rateModel: antigravityRateModel, scan: ({ instances, windowStartMs }) => AntigravityUsage.pipe(Effect.flatMap((usage) => usage.scan({ instances, windowStartMs }))), }; diff --git a/apps/server/src/provider/Drivers/ClaudeDriver.ts b/apps/server/src/provider/Drivers/ClaudeDriver.ts index af20b76754943..d75b1cad86402 100644 --- a/apps/server/src/provider/Drivers/ClaudeDriver.ts +++ b/apps/server/src/provider/Drivers/ClaudeDriver.ts @@ -36,6 +36,10 @@ import { import { makeManagedServerProvider } from "@t3tools/provider-core/server/managedProvider"; import * as ModelCatalog from "@t3tools/provider-core/server/ModelCatalog"; import { resolveClaudeModelCatalog } from "../ClaudeModelCatalog.ts"; +import { + makeClaudeCodeModelRoutes, + resolveClaudeModelSettingsKey, +} from "../ClaudeCodeModelRoutes.ts"; import { defaultProviderContinuationIdentity, type ProviderDriver, @@ -146,6 +150,23 @@ export const ClaudeDriver: ProviderDriver 0 || + processEnv.CLAUDE_CONFIG_DIR !== (yield* HostProcess.Environment).CLAUDE_CONFIG_DIR; + // Only a successful probe records routes, so a failed one never sends turns back to slugs + // the backend rejects. + const modelRoutes = yield* makeClaudeCodeModelRoutes( + (cwd) => + resolveClaudeModelSettingsKey( + customHome ? ["project", "local"] : ["user", "project", "local"], + cwd, + ).pipe( + Effect.provideService(FileSystem.FileSystem, fileSystem), + Effect.provideService(Path.Path, path), + ), + stateDir, + ); const orchestrationAdapter = yield* createClaudeAdapterV2( { instanceId, @@ -159,6 +180,7 @@ export const ClaudeDriver: ProviderDriver snapshot.applyUsageLimits(update), modelCatalog, + routeModelCatalog: modelRoutes.route, }, ).pipe( Effect.mapError( @@ -175,6 +197,7 @@ export const ClaudeDriver: ProviderDriver Ref.set(lastGoodCapabilities, capabilities)), + Effect.tap(({ capabilities }) => + Ref.set(lastGoodCapabilities, capabilities).pipe( + Effect.andThen(modelRoutes.record(capabilities.models ?? [])), + ), + ), + Effect.ensuring(modelRoutes.settle), Effect.catch((error) => Ref.getAndSet(lastGoodCapabilities, undefined).pipe( Effect.flatMap((capabilities) => @@ -262,6 +290,8 @@ export const ClaudeDriver: ProviderDriver { }), ); + it.effect("follows the WinGet launcher to the package's native binary", () => + Effect.gen(function* () { + expect( + yield* resolveClaudeSdkExecutablePath("claude", {}).pipe( + withWindowsResolution({ resolvedCommand: WINGET_SHIM, existingFiles: [WINGET_EXE] }), + ), + ).toBe(WINGET_EXE); + }), + ); + it.effect("follows .bat and .ps1 launcher shims the same way", () => Effect.gen(function* () { for (const shim of [`${NPM_DIR}\\claude.bat`, `${NPM_DIR}\\claude.ps1`]) { diff --git a/apps/server/src/provider/Drivers/ClaudeExecutable.ts b/apps/server/src/provider/Drivers/ClaudeExecutable.ts index 5e4cbb4db6efe..5797066446b99 100644 --- a/apps/server/src/provider/Drivers/ClaudeExecutable.ts +++ b/apps/server/src/provider/Drivers/ClaudeExecutable.ts @@ -13,10 +13,14 @@ const WINDOWS_SHIM_EXTENSIONS: ReadonlySet = new Set([".cmd", ".bat", ". export const isWindowsClaudeLauncherShimPath = (filePath: string): boolean => WINDOWS_SHIM_EXTENSIONS.has(NodePath.win32.extname(filePath).toLowerCase()); -/** New packages provide bin/claude.exe; older packages need a JavaScript runtime for cli.js. */ -const NPM_PACKAGE_ENTRY_CANDIDATES = [ +/** + * npm: new packages provide bin/claude.exe; older ones need a JavaScript runtime for cli.js. + * WinGet: Links\claude.cmd forwards to the package's native executable under ..\Packages. + */ +const PACKAGE_ENTRY_CANDIDATES = [ ["node_modules", "@anthropic-ai", "claude-code", "bin", "claude.exe"], ["node_modules", "@anthropic-ai", "claude-code", "cli.js"], + ["..", "Packages", "Anthropic.ClaudeCode_Microsoft.Winget.Source_8wekyb3d8bbwe", "claude.exe"], ] as const; export type ExecutableFileCheck = (filePath: string) => boolean; @@ -53,7 +57,7 @@ export const resolveClaudeSdkExecutablePath = Effect.fn("resolveClaudeSdkExecuta } const shimDirectory = NodePath.win32.dirname(resolved); - for (const entrySegments of NPM_PACKAGE_ENTRY_CANDIDATES) { + for (const entrySegments of PACKAGE_ENTRY_CANDIDATES) { const candidate = NodePath.win32.join(shimDirectory, ...entrySegments); if (isFile(candidate)) { return candidate; diff --git a/apps/server/src/provider/Drivers/ClaudeHome.ts b/apps/server/src/provider/Drivers/ClaudeHome.ts index 8042a7f16e491..1dfb94bb91a1e 100644 --- a/apps/server/src/provider/Drivers/ClaudeHome.ts +++ b/apps/server/src/provider/Drivers/ClaudeHome.ts @@ -100,6 +100,49 @@ export const hasClaudeManagedMcpConfig = Effect.fn("hasClaudeManagedMcpConfig")( return yield* fileSystem.exists(managedMcpConfigPath).pipe(Effect.orElseSucceed(() => false)); }); +// A worktree's .git file points at
/.git/worktrees/, whose commondir leads back to +//
/.git. A bare repository has no main checkout, so its worktree keeps its own file. +// Untraced because claudeLocalSettingsDirectory already owns the span. +const mainCheckoutRoot = Effect.fnUntraced(function* (checkout: string) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + return yield* fileSystem.readFileString(path.join(checkout, ".git")).pipe( + Effect.flatMap((pointer) => { + const gitDir = /^gitdir:\s*(.+)$/m.exec(pointer)?.[1]?.trim(); + if (gitDir === undefined) return Effect.succeed(checkout); + const worktreeGitDir = path.resolve(checkout, gitDir); + return fileSystem.readFileString(path.join(worktreeGitDir, "commondir")).pipe( + Effect.map((commonDir) => { + const common = path.resolve(worktreeGitDir, commonDir.trim()); + return path.basename(common) === ".git" ? path.dirname(common) : checkout; + }), + ); + }), + Effect.orElseSucceed(() => checkout), + ); +}); + +// Claude Code keeps .claude/settings.local.json at the repository root, or in cwd outside a +// repository, on Windows, or when the root is home. Its owner check is skipped, so callers must +// stay safe when Claude Code ignores the root file. +export const claudeLocalSettingsDirectory = Effect.fn("claudeLocalSettingsDirectory")(function* ( + cwd: string, +) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + if ((yield* HostProcess.Platform) === "win32") return cwd; + for (let directory = cwd; ; directory = path.dirname(directory)) { + const isCheckout = yield* fileSystem + .exists(path.join(directory, ".git")) + .pipe(Effect.orElseSucceed(() => false)); + if (isCheckout) { + const root = yield* mainCheckoutRoot(directory); + return root === (yield* HostProcess.HomeDirectory) ? cwd : root; + } + if (path.dirname(directory) === directory) return cwd; + } +}); + export const makeClaudeContinuationGroupKey = Effect.fn("makeClaudeContinuationGroupKey")( function* ( config: Pick, diff --git a/apps/server/src/provider/Drivers/ClaudeMaintenance.test.ts b/apps/server/src/provider/Drivers/ClaudeMaintenance.test.ts index 6fdf4c52e2a98..33f9f901c8731 100644 --- a/apps/server/src/provider/Drivers/ClaudeMaintenance.test.ts +++ b/apps/server/src/provider/Drivers/ClaudeMaintenance.test.ts @@ -24,6 +24,8 @@ import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; import * as HttpClient from "effect/http/HttpClient"; import * as HttpClientResponse from "effect/http/HttpClientResponse"; +import * as Sink from "effect/Sink"; +import * as ChildProcess from "effect/process/ChildProcess"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import { makeClaudeMaintenanceResolution } from "./ClaudeMaintenance.ts"; @@ -77,6 +79,13 @@ const fixture = Effect.fn("ClaudeMaintenance.test.fixture")(function* ( : platform === "darwin" ? "/Library/Application Support/ClaudeCode" : "/etc/claude-code"; + const npmPath = path.join(directory, "bin/npm"); + if (install === "npm") { + yield* fs.makeDirectory(path.dirname(npmPath), { recursive: true }); + yield* fs.writeFileString(npmPath, "#!/bin/sh\n"); + yield* fs.chmod(npmPath, 0o755); + } + const npmProbes: Array> = []; let installationReads = 0; const testFs = FileSystem.makeNoop({ ...fs, @@ -105,11 +114,37 @@ const fixture = Effect.fn("ClaudeMaintenance.test.fixture")(function* ( managed, managedDirectory, settingsPath, + npmProbes, installationReads: () => installationReads, - makeResolution: makeClaudeMaintenanceResolution({ binaryPath, env: { PATH: "" } }).pipe( + makeResolution: makeClaudeMaintenanceResolution({ + binaryPath, + env: { PATH: install === "npm" ? path.dirname(npmPath) : "" }, + }).pipe( Effect.provideService( ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => Effect.die("version checks must not run a subprocess")), + ChildProcessSpawner.make((command) => { + if (!ChildProcess.isStandardCommand(command) || command.command !== npmPath) { + return Effect.die("version checks may only ask npm"); + } + npmProbes.push(command.args.map(String)); + return Effect.succeed( + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(1)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + stdin: Sink.drain, + stdout: Stream.encodeText( + Stream.make(JSON.stringify({ "@anthropic-ai/claude-code": { latest: "2.1.316" } })), + ), + stderr: Stream.empty, + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }), + ); + }), ), Effect.provideService(FileSystem.FileSystem, testFs), Effect.provideService(HostProcess.Platform, platform), @@ -233,13 +268,18 @@ it.layer(NodeServices.layer)("ClaudeMaintenance", (it) => { status: row.advisory, latestVersion: row.expected, }); - expect(requests).toHaveLength(1); - expect(requests[0]).toMatch( - row.npm ? /registry\.npmjs\.org/ : new RegExp(`claude-code-releases/${row.channel}$`), - ); + // npm installs ask `npm outdated`, which applies npm's release-age settings. + const lookups = row.npm ? f.npmProbes : requests; + expect(lookups).toHaveLength(1); + if (row.npm) { + expect(requests).toEqual([]); + expect(f.npmProbes[0]?.[0]).toBe("outdated"); + } else { + expect(requests[0]).toMatch(new RegExp(`claude-code-releases/${row.channel}$`)); + } yield* enrich; - expect(requests).toHaveLength(1); - }).pipe(Effect.scoped), + expect(lookups).toHaveLength(1); + }), ); it.effect.each([ @@ -297,7 +337,7 @@ it.layer(NodeServices.layer)("ClaudeMaintenance", (it) => { ); yield* check; expect(requests).toHaveLength(3); - }).pipe(Effect.scoped), + }), ); it.effect.each([ @@ -317,7 +357,7 @@ it.layer(NodeServices.layer)("ClaudeMaintenance", (it) => { ), ); expect((yield* resolve()).update?.lockKey).toBe(row.lockKey); - }).pipe(Effect.scoped), + }), ); it.effect("automatic refresh rereads the native channel while preserving both caches", () => @@ -363,6 +403,7 @@ it.layer(NodeServices.layer)("ClaudeMaintenance", (it) => { }).pipe( Effect.provide( Layer.mock(ProviderHost.ProviderHost)({ + awaitServerReady: Effect.void, paths: { cwd: f.directory, baseDir: f.directory, @@ -403,7 +444,7 @@ it.layer(NodeServices.layer)("ClaudeMaintenance", (it) => { ]); expect(f.installationReads()).toBe(1); } - }).pipe(Effect.scoped), + }), ); it.effect("bounds stalled feed bodies and skips disabled update checks", () => @@ -433,6 +474,6 @@ it.layer(NodeServices.layer)("ClaudeMaintenance", (it) => { yield* TestClock.adjust("4 seconds"); expect((yield* Fiber.join(pending)).versionAdvisory?.status).toBe("unknown"); expect(requests).toBe(1); - }).pipe(Effect.scoped), + }), ); }); diff --git a/apps/server/src/provider/Drivers/ClaudeSkillDispatch.ts b/apps/server/src/provider/Drivers/ClaudeSkillDispatch.ts index 6ce141eec733e..bdf2ed1cae61a 100644 --- a/apps/server/src/provider/Drivers/ClaudeSkillDispatch.ts +++ b/apps/server/src/provider/Drivers/ClaudeSkillDispatch.ts @@ -22,12 +22,14 @@ export interface ClaudeSkillDispatch { export function planClaudeSkillDispatch( prompt: string, skills: ReadonlyMap, + /** Mentions before this offset, such as inline handoff history, stay plain text. */ + mentionsFrom = 0, ): ClaudeSkillDispatch | undefined { const mentions = [...prompt.matchAll(SKILL_MENTION_PATTERN)].flatMap((match) => { const name = match[2] ?? ""; const skill = skills.get(name); - if (!skill) return []; const start = (match.index ?? 0) + (match[1]?.length ?? 0); + if (!skill || start < mentionsFrom) return []; return [{ name, skill, start, end: (match.index ?? 0) + match[0].length }]; }); const last = mentions.at(-1); diff --git a/apps/server/src/provider/Drivers/ClaudeSkills.test.ts b/apps/server/src/provider/Drivers/ClaudeSkills.test.ts index 8b9661a779ae1..349294baa8c88 100644 --- a/apps/server/src/provider/Drivers/ClaudeSkills.test.ts +++ b/apps/server/src/provider/Drivers/ClaudeSkills.test.ts @@ -3,10 +3,19 @@ import { assert, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; +import * as HostProcess from "@t3tools/shared/HostProcess"; -import { discoverClaudeSkills } from "./ClaudeSkills.ts"; +import { discoverClaudeSkills as discoverSkills } from "./ClaudeSkills.ts"; import { planClaudeSkillDispatch } from "./ClaudeSkillDispatch.ts"; +// Windows makes the managed directory fixture-controlled without reading host policy or skills. +const discoverClaudeSkills = ( + ...[config, cwd, environment = {}]: Parameters +) => + discoverSkills(config, cwd, environment).pipe( + Effect.provideService(HostProcess.Platform, "win32"), + ); + const writeSkill = Effect.fn(function* ( skillsDir: string, directoryName: string, @@ -578,6 +587,96 @@ it.layer(NodeServices.layer)("discoverClaudeSkills", (it) => { }), ); + it.effect("discovers inline plugins from CLAUDE_CODE_PLUGIN_DIRS over installed namesakes", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const workspace = yield* fs.makeTempDirectoryScoped({ prefix: "t3-claude-skills-" }); + const configDir = path.join(workspace, "config"); + const named = path.join(workspace, "named"); + const unnamed = path.join(workspace, "unnamed"); + const inherited = path.join(workspace, "inherited"); + const muted = path.join(workspace, "muted"); + const installed = path.join(workspace, "installed-acme"); + yield* fs.makeDirectory(path.join(named, ".claude-plugin"), { recursive: true }); + yield* fs.writeFileString( + path.join(named, ".claude-plugin", "plugin.json"), + '{"name":"acme"}', + ); + yield* writeSkill(path.join(named, "skills"), "qa", "---\ndescription: QA a ticket\n---\n"); + yield* writeSkill(path.join(installed, "skills"), "old", "# Old"); + yield* writeSkill(path.join(unnamed, "skills"), "lint", "# Lint"); + yield* writeSkill(path.join(muted, "skills"), "hidden", "# Hidden"); + const quiet = path.join(workspace, "quiet"); + yield* fs.makeDirectory(path.join(quiet, ".claude-plugin"), { recursive: true }); + yield* fs.writeFileString( + path.join(quiet, ".claude-plugin", "plugin.json"), + '{"name":"quiet","defaultEnabled":false}', + ); + yield* writeSkill(path.join(quiet, "skills"), "hush", "# Hush"); + const bundle = path.join(workspace, "bundle"); + yield* fs.makeDirectory(path.join(bundle, "tools", ".claude-plugin"), { recursive: true }); + yield* fs.writeFileString( + path.join(bundle, "tools", ".claude-plugin", "plugin.json"), + '{"name":"tools"}', + ); + yield* writeSkill(path.join(bundle, "tools", "skills"), "fmt", "# Fmt"); + yield* writeSkill(path.join(bundle, "loose", "skills"), "ignored", "# Ignored"); + yield* writeSkill(path.join(inherited, "skills"), "stale", "# Stale"); + yield* writeSkill(path.join(workspace, "relative", "skills"), "skipped", "# Skipped"); + yield* fs.makeDirectory(path.join(configDir, "plugins"), { recursive: true }); + yield* fs.writeFileString( + path.join(configDir, "plugins", "installed_plugins.json"), + JSON.stringify({ plugins: { "acme@shop": [{ scope: "user", installPath: installed }] } }), + ); + yield* fs.writeFileString( + path.join(configDir, "settings.json"), + JSON.stringify({ + enabledPlugins: { "acme@shop": true, "muted@inline": false }, + env: { + CLAUDE_CODE_PLUGIN_DIRS: [named, "relative", ` ${unnamed} `, muted, quiet, bundle].join( + ";", + ), + }, + }), + ); + + const skills = yield* discoverClaudeSkills({ homePath: configDir }, workspace, { + CLAUDE_CODE_PLUGIN_DIRS: inherited, + }); + assert.deepEqual( + skills.map(({ name, scope, description }) => ({ name, scope, description })), + [ + { name: "acme:qa", scope: "user", description: "QA a ticket" }, + { name: "tools:fmt", scope: "user", description: undefined }, + { name: "unnamed:lint", scope: "user", description: undefined }, + ], + ); + + // Managed enabledPlugins entries lock their plugin name against inline copies. + const programFiles = path.join(workspace, "program-files"); + yield* fs.makeDirectory(path.join(programFiles, "ClaudeCode"), { recursive: true }); + yield* fs.writeFileString( + path.join(programFiles, "ClaudeCode", "managed-settings.json"), + '{"enabledPlugins":{"acme@shop":true}}', + ); + assert.deepEqual( + (yield* discoverClaudeSkills({ homePath: path.join(workspace, "empty") }, workspace, { + CLAUDE_CODE_PLUGIN_DIRS: [named, unnamed].join(";"), + ProgramFiles: programFiles, + })).map((skill) => skill.name), + ["unnamed:lint"], + ); + + assert.deepEqual( + (yield* discoverClaudeSkills({ homePath: path.join(workspace, "empty") }, workspace, { + CLAUDE_CODE_PLUGIN_DIRS: inherited, + })).map((skill) => skill.name), + ["inherited:stale"], + ); + }), + ); + it.effect("discovers user and project skills with frontmatter metadata", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -688,7 +787,7 @@ it.layer(NodeServices.layer)("discoverClaudeSkills", (it) => { }), ); - it.effect("prefers user skills on name collisions even with a stray .agents copy", () => + it.effect("prefers managed then user skills on collisions, even with a stray .agents copy", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -723,6 +822,62 @@ it.layer(NodeServices.layer)("discoverClaudeSkills", (it) => { description: "User deploy.", }, ]); + + const programFiles = path.join(tempDir, "Program Files"); + const programData = path.join(tempDir, "ProgramData"); + yield* writeSkill( + path.join(programData, "ClaudeCode", ".claude", "skills"), + "legacy-only", + "# Legacy skills must not be discovered", + ); + yield* fs.writeFileString( + path.join(programData, "ClaudeCode", "managed-settings.json"), + '{"skillOverrides":{"deploy":"off"}}', + ); + for (const ProgramFiles of [undefined, "", programFiles]) { + const legacySkills = yield* discoverClaudeSkills({ homePath: configDir }, workspace, { + PROGRAMDATA: programData, + ...(ProgramFiles !== undefined ? { ProgramFiles } : {}), + }); + assert.deepEqual( + legacySkills, + skills.map((skill) => ({ ...skill, enabled: false })), + ); + } + yield* writeSkill( + path.join(programFiles, "ClaudeCode", ".claude", "skills"), + "deploy", + ["---", "name: deploy", "description: Managed deploy.", "---"].join("\n"), + ); + yield* fs.writeFileString( + path.join(configDir, "settings.json"), + '{"skillOverrides":{"deploy":"off"}}', + ); + yield* fs.writeFileString( + path.join(programFiles, "ClaudeCode", "managed-settings.json"), + '{"skillOverrides":{"deploy":"on"}}', + ); + const managedSkills = yield* discoverClaudeSkills({ homePath: configDir }, workspace, { + ProgramFiles: programFiles, + PROGRAMDATA: programData, + }); + + assert.deepEqual(managedSkills, [ + { + name: "deploy", + path: path.join(programFiles, "ClaudeCode", ".claude", "skills", "deploy", "SKILL.md"), + enabled: true, + scope: "user", + description: "Managed deploy.", + }, + ]); + assert.deepEqual( + yield* discoverClaudeSkills({ homePath: configDir }, workspace, { + PROGRAMFILES: ` ${programFiles} `, + PROGRAMDATA: programData, + }), + managedSkills, + ); }), ); diff --git a/apps/server/src/provider/Drivers/ClaudeSkills.ts b/apps/server/src/provider/Drivers/ClaudeSkills.ts index c24c4afca9e7d..4fdd3565d11b6 100644 --- a/apps/server/src/provider/Drivers/ClaudeSkills.ts +++ b/apps/server/src/provider/Drivers/ClaudeSkills.ts @@ -106,19 +106,41 @@ function parseSkillFrontmatter(contents: string): SkillFrontmatter { }; } -function claudeManagedSettingsPath( +// A caller-supplied environment is a plain object, without Windows' case-insensitive keys. +function windowsEnvironmentValue(environment: NodeJS.ProcessEnv, name: string) { + const key = name.toLowerCase(); + return Object.entries(environment) + .find(([candidate]) => candidate.toLowerCase() === key)?.[1] + ?.trim(); +} + +function claudeManagedDirectory( path: Path.Path, platform: NodeJS.Platform, environment: NodeJS.ProcessEnv, ): string | undefined { if (platform === "darwin") { - return "/Library/Application Support/ClaudeCode/managed-settings.json"; + return "/Library/Application Support/ClaudeCode"; } if (platform === "win32") { - const programData = environment.PROGRAMDATA?.trim(); - return programData ? path.join(programData, "ClaudeCode", "managed-settings.json") : undefined; + const programFiles = windowsEnvironmentValue(environment, "ProgramFiles"); + return programFiles ? path.join(programFiles, "ClaudeCode") : undefined; } - return "/etc/claude-code/managed-settings.json"; + return "/etc/claude-code"; +} + +function claudeManagedSettingsPaths( + path: Path.Path, + platform: NodeJS.Platform, + environment: NodeJS.ProcessEnv, +): ReadonlyArray { + const managedDirectory = claudeManagedDirectory(path, platform, environment); + const programData = + platform === "win32" ? windowsEnvironmentValue(environment, "ProgramData") : undefined; + return [ + ...(programData ? [path.join(programData, "ClaudeCode", "managed-settings.json")] : []), + ...(managedDirectory ? [path.join(managedDirectory, "managed-settings.json")] : []), + ]; } /** Claude Code settings files, lowest precedence first. */ @@ -130,7 +152,6 @@ function claudeSettingsPaths( environment: NodeJS.ProcessEnv, repositoryRoot?: string, ): ReadonlyArray { - const managedPath = claudeManagedSettingsPath(path, platform, environment); const root = repositoryRoot !== undefined && repositoryRoot !== cwd ? repositoryRoot : undefined; return [ path.join(configDirPath, "settings.json"), @@ -141,7 +162,7 @@ function claudeSettingsPaths( ] : []), ...(root ? [path.join(root, ".claude", "settings.local.json")] : []), - ...(managedPath ? [managedPath] : []), + ...claudeManagedSettingsPaths(path, platform, environment), ]; } @@ -250,6 +271,8 @@ const readSkillSettings = Effect.fn("readSkillSettings")(function* ( const platform = yield* HostProcess.Platform; const overridesByName = new Map(); const enabledPlugins = new Map(); + const managedPluginNames = new Set(); + const managedPaths = new Set(claudeManagedSettingsPaths(path, platform, environment)); let syncedPluginsHidden = false; const repositoryRoot = cwd === undefined ? undefined : yield* findRepositoryRoot(cwd); const projectSettingsPath = @@ -285,6 +308,9 @@ const readSkillSettings = Effect.fn("readSkillSettings")(function* ( for (const [key, enabled] of Object.entries(parsed?.enabledPlugins ?? {})) { if (typeof enabled === "boolean") { enabledPlugins.set(key, enabled); + if (managedPaths.has(settingsPath) && key.includes("@")) { + managedPluginNames.add(pluginNameKey(key.slice(0, key.indexOf("@")))); + } } else { // Extended formats replace lower-precedence values without enabling the plugin here. enabledPlugins.delete(key); @@ -296,7 +322,12 @@ const readSkillSettings = Effect.fn("readSkillSettings")(function* ( } } - return { skillOverrides: overridesByName, enabledPlugins, syncedPluginsHidden }; + return { + skillOverrides: overridesByName, + enabledPlugins, + managedPluginNames, + syncedPluginsHidden, + }; }); const InstalledPlugins = Schema.fromJsonString( @@ -313,6 +344,7 @@ const PluginManifest = Schema.fromJsonString( Schema.Struct({ name: Schema.optional(Schema.NonEmptyString), skills: Schema.optional(Schema.Union([Schema.String, Schema.Array(Schema.String)])), + defaultEnabled: Schema.optional(Schema.Unknown), }), ); @@ -433,10 +465,66 @@ const resolveSyncedPluginBucket = Effect.fn("resolveSyncedPluginBucket")(functio return `${organizationId.toLowerCase()}_${accountId.toLowerCase()}`; }); +/** A plugin folder itself, or else the manifest-bearing plugins directly inside it. */ +const expandInlinePluginDirectory = Effect.fn("expandInlinePluginDirectory")(function* ( + directory: string, +) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const exists = (candidate: string) => + fileSystem.exists(candidate).pipe(Effect.orElseSucceed(() => false)); + for (const component of [path.join(".claude-plugin", "plugin.json"), "skills", "SKILL.md"]) { + if (yield* exists(path.join(directory, component))) return [directory]; + } + const children = yield* fileSystem + .readDirectory(directory) + .pipe(Effect.orElseSucceed((): ReadonlyArray => [])); + return yield* Effect.filter( + [...children].sort().map((child) => path.join(directory, child)), + (child) => exists(path.join(child, ".claude-plugin", "plugin.json")), + ); +}); + +/** Plugin folders Claude Code loads inline from CLAUDE_CODE_PLUGIN_DIRS, and their plugin names. */ +const discoverInlinePluginSkillRoots = Effect.fn("discoverInlinePluginSkillRoots")(function* ( + pluginDirectories: string | undefined, + enabledPlugins: ReadonlyMap, + managedPluginNames: ReadonlySet, +) { + const path = yield* Path.Path; + const home = yield* HostProcess.HomeDirectory; + const platform = yield* HostProcess.Platform; + const roots: Array = []; + const names = new Set(); + const seen = new Set(); + for (const entry of (pluginDirectories ?? "").split(platform === "win32" ? ";" : ":")) { + const directory = expandHomePath(entry.trim(), home); + // Claude skips relative entries rather than resolving them against its cwd. + if (!directory || !path.isAbsolute(directory)) continue; + for (const installPath of yield* expandInlinePluginDirectory(path.resolve(directory))) { + if (seen.has(installPath)) continue; + seen.add(installPath); + const manifest = yield* readPluginManifest(installPath); + const prefix = manifest?.name ?? path.basename(installPath); + // A `@inline` setting wins; otherwise the manifest's `defaultEnabled` decides. + const setting = enabledPlugins.get(`${prefix}@inline`); + if (setting === false || (setting === undefined && manifest?.defaultEnabled === false)) { + continue; + } + // Managed enabledPlugins entries, true or false, lock the name against inline copies. + if (managedPluginNames.has(pluginNameKey(prefix))) continue; + names.add(pluginNameKey(prefix)); + roots.push(...(yield* pluginSkillRoots(installPath, "user", prefix, manifest))); + } + } + return { roots, names }; +}); + const discoverPluginSkillRoots = Effect.fn("discoverPluginSkillRoots")(function* ( configDirPath: string, cwd: string | undefined, enabledPlugins: ReadonlyMap, + inlinePluginNames: ReadonlySet, syncedBucket: string | undefined, environment: NodeJS.ProcessEnv, ) { @@ -454,7 +542,8 @@ const discoverPluginSkillRoots = Effect.fn("discoverPluginSkillRoots")(function* Effect.orElseSucceed(() => undefined), ); const roots: Array = []; - const claimedPluginNames = new Set(); + // An enabled inline plugin replaces installed and synced plugins of the same name. + const claimedPluginNames = new Set(inlinePluginNames); // The CLI's process.cwd() resolves symlinks; registry projectPath values remain lexical. const workspace = cwd === undefined @@ -494,7 +583,7 @@ const discoverPluginSkillRoots = Effect.fn("discoverPluginSkillRoots")(function* const installPath = path.resolve(cwd ?? ".", install.installPath); const manifest = yield* readPluginManifest(installPath); const prefix = manifest?.name ?? key.split("@")[0]; - if (!prefix) continue; + if (!prefix || inlinePluginNames.has(pluginNameKey(prefix))) continue; // A registry record whose install is gone loads nothing and claims no namespace. if (Option.isSome(existing)) claimedPluginNames.add(pluginNameKey(prefix)); roots.push(...(yield* pluginSkillRoots(installPath, scope, prefix, manifest))); @@ -560,6 +649,7 @@ const resolveClaudeConfigDirPath = Effect.fn("resolveClaudeConfigDirPath")(funct const SettingsEnv = Schema.Struct({ ANTHROPIC_BASE_URL: Schema.optional(Schema.String), CLAUDE_CODE_ENTRYPOINT: Schema.optional(Schema.String), + CLAUDE_CODE_PLUGIN_DIRS: Schema.optional(Schema.String), // Unquoted JSON values must not fail the decode and drop ANTHROPIC_BASE_URL with them. CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY: Schema.optional(Schema.Unknown), CLAUDE_CODE_GATEWAY_MODEL_DISCOVERY_TIMEOUT_MS: Schema.optional(Schema.Unknown), @@ -627,7 +717,7 @@ export function claudeLaunchArgDirectories(launchArgs?: string): ReadonlyArray & Partial>, cwd?: string, @@ -637,11 +727,9 @@ export const discoverClaudeSkills = Effect.fn("discoverClaudeSkills")(function* const path = yield* Path.Path; const env = environment ?? process.env; const configDirPath = yield* resolveClaudeConfigDirPath(config, env, cwd); - const { skillOverrides, enabledPlugins, syncedPluginsHidden } = yield* readSkillSettings( - configDirPath, - cwd, - env, - ); + const { skillOverrides, enabledPlugins, managedPluginNames, syncedPluginsHidden } = + yield* readSkillSettings(configDirPath, cwd, env); + const managedDirectory = claudeManagedDirectory(path, yield* HostProcess.Platform, env); // Most installs have no synced folder; skip reading `.claude.json` for them. const hasSyncedPlugins = !syncedPluginsHidden && @@ -651,8 +739,20 @@ export const discoverClaudeSkills = Effect.fn("discoverClaudeSkills")(function* const syncedBucket = hasSyncedPlugins ? yield* resolveSyncedPluginBucket(config, configDirPath, env) : undefined; + // Claude applies its settings `env` over the inherited environment. + const inlinePluginDirectories = + (yield* readClaudeSettingsEnv(config, cwd, env)).CLAUDE_CODE_PLUGIN_DIRS ?? + env.CLAUDE_CODE_PLUGIN_DIRS; + const inlinePlugins = yield* discoverInlinePluginSkillRoots( + inlinePluginDirectories, + enabledPlugins, + managedPluginNames, + ); const roots: ReadonlyArray = [ + ...(managedDirectory + ? [{ directory: path.join(managedDirectory, ".claude", "skills"), scope: "user" as const }] + : []), { directory: path.join(configDirPath, "skills"), scope: "user" }, ...(cwd ? [{ directory: path.join(cwd, ".claude", "skills"), scope: "project" as const }] : []), ...claudeLaunchArgDirectories(config.launchArgs) @@ -661,7 +761,15 @@ export const discoverClaudeSkills = Effect.fn("discoverClaudeSkills")(function* directory: path.join(path.resolve(cwd ?? ".", directory), ".claude", "skills"), scope: "project" as const, })), - ...(yield* discoverPluginSkillRoots(configDirPath, cwd, enabledPlugins, syncedBucket, env)), + ...inlinePlugins.roots, + ...(yield* discoverPluginSkillRoots( + configDirPath, + cwd, + enabledPlugins, + inlinePlugins.names, + syncedBucket, + env, + )), ]; const skillsByName = new Map(); diff --git a/apps/server/src/provider/Drivers/CodexDriver.test.ts b/apps/server/src/provider/Drivers/CodexDriver.test.ts index 597be85623115..1462d051346f1 100644 --- a/apps/server/src/provider/Drivers/CodexDriver.test.ts +++ b/apps/server/src/provider/Drivers/CodexDriver.test.ts @@ -42,6 +42,7 @@ import { CodexDriver } from "./CodexDriver.ts"; import * as CodexAdapterV2 from "../../orchestration-v2/Adapters/CodexAdapterV2.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import * as PortScanner from "../../preview/PortScanner.ts"; import * as ProviderCredentialStore from "../ProviderCredentialStore.ts"; import * as ProviderHostLive from "../ProviderHostLive.ts"; @@ -51,6 +52,7 @@ const layerDeps = ServerConfig.layerTest(process.cwd(), { Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), Layer.provideMerge(IdAllocator.layer), Layer.provideMerge(McpProviderSessions.layer), + Layer.provideMerge(PortScanner.layerTest), Layer.provideMerge( Layer.mock(CodexAdapterV2.CodexAppServerClientFactory)({ open: () => Effect.die("Maintenance resolution must not open a Codex session"), @@ -374,10 +376,7 @@ it.layer(layerTest)("CodexDriver", (it) => { lockKey: "codex-native", env: { CODEX_HOME: sharedHome }, }); - }).pipe( - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn), - Effect.scoped, - ), + }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn)), ); it.effect.skipIf(windowsHost)( @@ -431,10 +430,7 @@ it.layer(layerTest)("CodexDriver", (it) => { env: { CODEX_HOME: installHome }, }); } - }).pipe( - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn), - Effect.scoped, - ), + }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn)), ); it.effect.skipIf(windowsHost)("derives the standalone update home from either command path", () => @@ -478,7 +474,7 @@ it.layer(layerTest)("CodexDriver", (it) => { CODEX_HOME: fixture.home, }); } - }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn), Effect.scoped), + }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn)), ); // regression: #23937 `codex update` installs the native channel's release, which can lag npm. @@ -556,10 +552,7 @@ it.layer(layerTest)("CodexDriver", (it) => { maintenanceCapabilities: capabilities, }).status, ).not.toBe("behind_latest"); - }).pipe( - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn), - Effect.scoped, - ), + }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn)), ); it.effect("uses each instance's environment CODEX_HOME for continuation", () => @@ -577,7 +570,36 @@ it.layer(layerTest)("CodexDriver", (it) => { }); expect(instance.continuationIdentity.continuationKey).toBe(`codex:home:${codexHome}`); } - }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn), Effect.scoped), + }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn)), + ); + + it.effect.skipIf(windowsHost)( + "shares state from the environment CODEX_HOME in a shadow home", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const tempDir = yield* fs + .makeTempDirectoryScoped({ prefix: "t3-codex-driver-" }) + .pipe(Effect.flatMap((directory) => fs.realPath(directory))); + const codexHome = NodePath.join(tempDir, "custom-codex"); + const shadowHome = NodePath.join(tempDir, "shadow"); + yield* fs.makeDirectory(codexHome, { recursive: true }); + yield* fs.writeFileString(NodePath.join(codexHome, "config.toml"), 'model = "custom"\n'); + const instance = yield* CodexDriver.create({ + instanceId: ProviderInstanceId.make("env-shadow"), + displayName: "env-shadow", + enabled: false, + environment: [{ name: "CODEX_HOME", value: codexHome, sensitive: false }], + config: { ...CodexDriver.defaultConfig(), shadowHomePath: shadowHome }, + }).pipe(Effect.provideService(HostProcess.HomeDirectory, NodePath.join(tempDir, "user"))); + expect(instance.continuationIdentity.continuationKey).toBe(`codex:home:${codexHome}`); + expect(yield* fs.readLink(NodePath.join(shadowHome, "config.toml"))).toBe( + NodePath.join(codexHome, "config.toml"), + ); + expect(yield* fs.readLink(NodePath.join(shadowHome, "sessions"))).toBe( + NodePath.join(codexHome, "sessions"), + ); + }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn)), ); for (const fixture of [ @@ -653,10 +675,7 @@ it.layer(layerTest)("CodexDriver", (it) => { } else { expect(update).toBeNull(); } - }).pipe( - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn), - Effect.scoped, - ), + }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, noSpawn)), ); } @@ -710,6 +729,7 @@ it.layer(layerTest)("CodexDriver", (it) => { const shimDir = NodePath.join(tempDir, fixture.dataRoot, "shims"); const npmPrefix = NodePath.join(tempDir, "mise", "installs", "node", "24.13.0"); const npmBin = NodePath.join(npmPrefix, "bin"); + const npmPath = NodePath.join(npmBin, "npm"); const npmEntry = NodePath.join( npmPrefix, "lib", @@ -719,7 +739,7 @@ it.layer(layerTest)("CodexDriver", (it) => { "bin", "codex.js", ); - for (const file of [brewPath, misePath, npmEntry]) { + for (const file of [brewPath, misePath, npmPath, npmEntry]) { yield* fs.makeDirectory(NodePath.dirname(file), { recursive: true }); yield* fs.writeFileString(file, "#!/bin/sh\n"); yield* fs.chmod(file, 0o755); @@ -734,12 +754,20 @@ it.layer(layerTest)("CodexDriver", (it) => { ].join(NodePath.delimiter); const probes: Array> = []; const metadataSpawner = ChildProcessSpawner.make((command) => { - if (!ChildProcess.isStandardCommand(command) || command.command !== brewPath) { + const isNpmOutdated = + ChildProcess.isStandardCommand(command) && + command.command === npmPath && + command.args[0] === "outdated"; + if ( + !ChildProcess.isStandardCommand(command) || + (command.command !== brewPath && !isNpmOutdated) + ) { return Effect.die("Provider resolution must not execute a provider or updater"); } - probes.push(command.args.map(String)); - const stdout = - command.args[0] === "--prefix" + if (!isNpmOutdated) probes.push(command.args.map(String)); + const stdout = isNpmOutdated + ? JSON.stringify({ "@openai/codex": { latest: "0.153.4" } }) + : command.args[0] === "--prefix" ? brewPrefix : JSON.stringify({ formulae: [{ versions: { stable: "2026.9.1" } }] }); return Effect.succeed( @@ -805,7 +833,7 @@ it.layer(layerTest)("CodexDriver", (it) => { } else { expect(capabilities.update).toBeNull(); } - }).pipe(Effect.scoped), + }), { skip: windowsHost }, ); @@ -856,6 +884,6 @@ it.layer(layerTest)("CodexDriver", (it) => { yield* TestClock.adjust("250 millis"); yield* Queue.take(probes); - }).pipe(Effect.scoped), + }), ); }); diff --git a/apps/server/src/provider/Drivers/CodexDriver.ts b/apps/server/src/provider/Drivers/CodexDriver.ts index f2bfa4082dcbf..f8a753169b54a 100644 --- a/apps/server/src/provider/Drivers/CodexDriver.ts +++ b/apps/server/src/provider/Drivers/CodexDriver.ts @@ -190,11 +190,11 @@ export const CodexDriver: ProviderDriver< yield* HostProcess.Environment, ); const environmentHome = processEnv.CODEX_HOME; - const homeLayout = yield* resolveCodexHomeLayout( - !config.homePath.trim() && !config.shadowHomePath.trim() && environmentHome?.trim() + const homeConfig = + !config.homePath.trim() && environmentHome?.trim() ? { ...config, homePath: environmentHome } - : config, - ); + : config; + const homeLayout = yield* resolveCodexHomeLayout(homeConfig); const continuationIdentity = codexContinuationIdentity(homeLayout); const stampIdentity = withInstanceIdentity({ instanceId, @@ -245,7 +245,7 @@ export const CodexDriver: ProviderDriver< accentColor, environment, enabled, - config, + config: homeConfig, }, { onUsageLimits: (update) => snapshot.applyUsageLimits(update), diff --git a/apps/server/src/provider/Drivers/antigravityUsageReader.ts b/apps/server/src/provider/Drivers/antigravityUsageReader.ts index 4d473f50c9d67..532233f7facce 100644 --- a/apps/server/src/provider/Drivers/antigravityUsageReader.ts +++ b/apps/server/src/provider/Drivers/antigravityUsageReader.ts @@ -129,18 +129,19 @@ const MODEL_IDS: Record = { }; // Effort and routing variants share the base model's price. -function withRateModel(record: UsageRecord): UsageRecord { - const base = record.model.replace( - /-(?:extra-low|low|medium|high|tiered|control|thinking|agent)$/, - "", - ); - const rateModel = /^gemini-3-flash-(?:agent(?:-[ab])?|[ab])$/.test(record.model) +export const antigravityRateModel = (model: string) => { + const base = model.replace(/-(?:extra-low|low|medium|high|tiered|control|thinking|agent)$/, ""); + return /^gemini-3-flash-(?:agent(?:-[ab])?|[ab])$/.test(model) ? "gemini-3.5-flash" - : record.model === "gemini-3.7-flash-safety-le" + : model === "gemini-3.7-flash-safety-le" ? "gemini-3.7-flash" : base === "gemini-3-flash" ? "gemini-3-flash-preview" : base; +}; + +function withRateModel(record: UsageRecord): UsageRecord { + const rateModel = antigravityRateModel(record.model); return rateModel === record.model ? record : { ...record, rateModel }; } diff --git a/apps/server/src/provider/Drivers/codexUsage.test.ts b/apps/server/src/provider/Drivers/codexUsage.test.ts index cacfc6c06211a..1cdbe05d07669 100644 --- a/apps/server/src/provider/Drivers/codexUsage.test.ts +++ b/apps/server/src/provider/Drivers/codexUsage.test.ts @@ -589,6 +589,98 @@ describe("parseCodexLine", () => { }); }); + const at = (time: string, line: string) => line.replace("05:17:49.919Z", `${time}.000Z`); + const anonymousRequest = (time: string, usage: unknown) => + JSON.stringify({ + type: "token_usage_record", + timestamp: `2026-08-01T${time}.000Z`, + payload: { usage }, + }); + it.each([ + { + name: "does not bill an older snapshot re-emitted after requests without cumulative counters", + lines: [ + at("10:00:00", counted(counters(100, 20), counters(100, 20))), + at("10:00:05", counted(counters(20, 4))), + at("10:00:10", counted(counters(30, 6))), + at("10:00:05", counted(counters(20, 4), counters(120, 24))), + ], + processed: [120, 24, 36, 0], + skipped: 1, + }, + { + name: "bills a rewind that follows a request without cumulative counters", + lines: [ + at("10:00:00", counted(counters(100, 20))), + at("10:00:10", counted(counters(20, 4), counters(60, 12))), + ], + processed: [120, 24], + skipped: 0, + }, + { + name: "keeps the baseline when it rejects an older snapshot, so the replay that follows is not billed", + lines: [ + at("10:00:00", counted(counters(100, 20), counters(100, 20))), + at("10:00:05", counted(counters(20, 4), counters(120, 24))), + at("10:00:10", counted(counters(30, 6), counters(150, 30))), + at("10:00:05", counted(counters(20, 4), counters(120, 24))), + at("10:00:10", counted(counters(30, 6), counters(150, 30))), + ], + processed: [120, 24, 36, 0, 0], + skipped: 1, + }, + { + name: "keeps the baseline when it rejects a lower snapshot from the baseline's own time", + lines: [ + at("10:00:00", counted(counters(100, 20), counters(100, 20))), + at("10:00:00", counted(counters(20, 4), counters(120, 24))), + at("10:00:00", counted(counters(30, 6), counters(150, 30))), + at("10:00:00", counted(counters(20, 4), counters(120, 24))), + at("10:00:00", counted(counters(30, 6), counters(150, 30))), + ], + processed: [120, 24, 36, 0, 0], + skipped: 1, + }, + { + name: "does not bill replayed snapshots against a cached baseline of unknown age", + lines: [ + at("10:00:00", counted(counters(100, 20), counters(100, 20))), + at("10:00:05", counted(counters(20, 4), counters(120, 24))), + at("10:00:10", counted(counters(30, 6), counters(150, 30))), + at("10:00:00", counted(counters(100, 20), counters(100, 20))), + at("10:00:05", counted(counters(20, 4), counters(120, 24))), + ], + undatedAfter: 3, + processed: [120, 24, 36, 0, 0], + skipped: 1, + }, + { + name: "warns about the anonymous request a rewound snapshot does not report", + lines: [ + at("10:00:00", counted(counters(100, 20), counters(100, 20))), + anonymousRequest("10:00:05", counters(40, 8)), + anonymousRequest("10:00:06", counters(20, 4)), + at("10:00:07", counted(counters(20, 4), counters(60, 12))), + at("10:00:08", counted(counters(10, 2), counters(70, 14))), + ], + processed: [120, 0, 0, 24, 12], + skipped: 1, + }, + ])("$name", ({ lines, undatedAfter, processed, skipped }) => { + const state = initialCodexScanState(); + parseCodexLine(turnContext, state); + const billed = lines.map((line, index) => { + if (index === undatedAfter) delete state.latestCumulativeAtMs; + const record = parseCodexLine(line, state); + return record === null ? 0 : totalTokens(record.totals); + }); + + expect({ processed: billed, skipped: codexUsageFormat.readMalformedRecords?.(state) }).toEqual({ + processed, + skipped, + }); + }); + it("ignores a replay of the current sequence's first request without rewinding the baseline", () => { const state = initialCodexScanState(); parseCodexLine(turnContext, state); diff --git a/apps/server/src/provider/Drivers/codexUsage.ts b/apps/server/src/provider/Drivers/codexUsage.ts index c8c9fc9b4cae3..a2977d3080fee 100644 --- a/apps/server/src/provider/Drivers/codexUsage.ts +++ b/apps/server/src/provider/Drivers/codexUsage.ts @@ -54,10 +54,15 @@ function readCodexTokenUsage(value: unknown): CodexTokenUsage | null { return isCodexTokenUsage(usage) ? usage : null; } +// Every move of the baseline dates it, so an older snapshot re-emitted later is never a rewind. +function dateCodexBaseline(state: CodexScanState, timestampMs: number) { + state.latestCumulativeAtMs = Math.max(state.latestCumulativeAtMs ?? timestampMs, timestampMs); +} + // A zeroed baseline rebases the counters, so the sequence start measured against it is forgotten. function rebaseCodexUsage(state: CodexScanState, cumulative: CodexTokenUsage, timestampMs: number) { state.lastCumulativeUsage = cumulative; - state.latestCumulativeAtMs = Math.max(state.latestCumulativeAtMs ?? timestampMs, timestampMs); + dateCodexBaseline(state, timestampMs); delete state.unreportedRequests; if (cumulative.input_tokens + cumulative.output_tokens === 0) delete state.sequenceStartSignature; } @@ -112,6 +117,18 @@ function unreportedRequests( ); } +// A rewound snapshot vouches only for the request it reports, so the anonymous requests read before +// it can be neither billed nor ruled out as that request's prefix. +function discardedRequests( + pending: readonly CodexUnreportedRequest[] | undefined, + last: CodexTokenUsage, +) { + const final = pending?.at(-1); + if (pending === undefined || final === undefined) return 0; + const reported = CODEX_USAGE_FIELDS.every((key) => final.usage[key] === last[key]); + return pending.length - (reported ? 1 : 0); +} + /** * Codex `token_count` events carry no model or service tier, so both are carried forward: the model * from the most recent `turn_context`, the tier from the most recent `thread_settings_applied`. @@ -251,6 +268,7 @@ function parseCodexRecord(parsed: unknown, state: CodexScanState): readonly Usag state.lastCumulativeUsage = advanceCodexUsage(state.lastCumulativeUsage, request.usage); } state.lastCumulativeUsage = advanceCodexUsage(state.lastCumulativeUsage, usage); + dateCodexBaseline(state, timestampMs); delete state.unreportedRequests; return [ ...pending.map((request) => codexRequestRecord(state, request.usage, request)), @@ -295,6 +313,7 @@ function parseCodexRecord(parsed: unknown, state: CodexScanState): readonly Usag if (cumulative !== null) rebaseCodexUsage(state, cumulative, timestampMs); else if (!hasCumulative && last !== null && signature !== state.lastUsageSignature) { state.lastCumulativeUsage = advanceCodexUsage(state.lastCumulativeUsage, last); + dateCodexBaseline(state, timestampMs); } state.lastUsageSignature = signature; return []; @@ -345,15 +364,18 @@ function parseCodexRecord(parsed: unknown, state: CodexScanState): readonly Usag // Only the same event is a replay; a later request can restart with identical counters. if (restarted && sequenceStart === state.sequenceStartSignature) { state.lastCumulativeUsage = previous; + // The retained baseline keeps its own age; one of unknown age must not take the replay's. + if (latestCumulativeAtMs === undefined) delete state.latestCumulativeAtMs; return []; } // Forks and paginated continuations can inherit totals without copying a counter snapshot, and // a later request can continue from an earlier snapshot; a re-emitted old snapshot is not later. - // Only bill the latest request, and require the unseen prefix to be valid usage too. + // Only bill the latest request, and require the unseen prefix to be valid usage too. A baseline + // of unknown age (a scan state cached before every move dated it) cannot order the snapshot. + const lowered = + previous !== null && CODEX_USAGE_FIELDS.some((key) => cumulative[key] < previous[key]); const rewound = - previous !== null && - (latestCumulativeAtMs === undefined || timestampMs > latestCumulativeAtMs) && - CODEX_USAGE_FIELDS.some((key) => cumulative[key] < previous[key]); + lowered && latestCumulativeAtMs !== undefined && timestampMs > latestCumulativeAtMs; const reconciled = previous === null || rewound ? readCodexTokenUsage( @@ -363,10 +385,24 @@ function parseCodexRecord(parsed: unknown, state: CodexScanState): readonly Usag if (!restarted && !reconciled) { const bridged = unreportedRequests(previous, cumulative, last, pending); if (bridged === null) { + // A rejected snapshot from before the baseline's time is a replay, not the next baseline. + // At the baseline's own time only lower counters are one; zeroed counters still rebase. + const replayed = + latestCumulativeAtMs !== undefined && + (timestampMs < latestCumulativeAtMs || + (timestampMs === latestCumulativeAtMs && + lowered && + cumulative.input_tokens + cumulative.output_tokens > 0)); + if (replayed) { + state.lastCumulativeUsage = previous; + if (pending !== undefined) state.unreportedRequests = pending; + } state.malformedRecords += 1; return []; } unreported = bridged; + } else if (!restarted && rewound) { + state.malformedRecords += discardedRequests(pending, last); } if (restarted) state.sequenceStartSignature = sequenceStart; } else { @@ -378,6 +414,7 @@ function parseCodexRecord(parsed: unknown, state: CodexScanState): readonly Usag return []; } state.lastCumulativeUsage = advanced; + dateCodexBaseline(state, timestampMs); } state.lastUsageSignature = signature; @@ -516,10 +553,7 @@ export const codexUsageReader: ProviderUsageReader< const path = yield* Path.Path; const environmentHome = environment.CODEX_HOME?.trim(); const layout = yield* resolveCodexHomeLayout( - config.setupMode !== "managed" && - !config.homePath.trim() && - !config.shadowHomePath.trim() && - environmentHome + config.setupMode !== "managed" && !config.homePath.trim() && environmentHome ? { ...config, homePath: environmentHome } : config, ); diff --git a/apps/server/src/provider/EventNdjsonLogger.test.ts b/apps/server/src/provider/EventNdjsonLogger.test.ts index eed20381b46cb..0bb55b9bcb39b 100644 --- a/apps/server/src/provider/EventNdjsonLogger.test.ts +++ b/apps/server/src/provider/EventNdjsonLogger.test.ts @@ -255,6 +255,65 @@ it.layer(NodeServices.layer)("EventNdjsonLogger", (it) => { }), ); + it.effect("drops Cursor streaming deltas while keeping its lifecycle updates", () => + Effect.gen(function* () { + const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-")); + const basePath = NodePath.join(tempDir, "events.log"); + const threadId = ThreadId.make("cursor-deltas"); + const cursorUpdate = (update: object) => ({ + provider: "cursor", + kind: "protocol", + event: { + direction: "incoming", + stage: "decoded", + payload: { type: "interaction.update", runId: "run-1", update }, + }, + }); + const toolCall = { type: "shell", args: { command: "ls" } }; + const transient = [ + { type: "text-delta", text: "Hel" }, + { type: "thinking-delta", text: "Hmm" }, + { type: "token-delta", tokens: 3 }, + { type: "shell-output-delta", event: { stdout: "a.ts\n" } }, + { type: "partial-tool-call", callId: "call-1", modelCallId: "m-1", toolCall }, + { + type: "tool-call-delta", + callId: "call-1", + modelCallId: "m-1", + taskUpdate: { type: "text-delta", text: "Sub" }, + }, + ].map(cursorUpdate); + const retained = [ + { type: "tool-call-started", callId: "call-1", modelCallId: "m-1", toolCall }, + { + type: "tool-call-delta", + callId: "call-1", + modelCallId: "m-1", + taskUpdate: { type: "tool-call-completed", callId: "call-2", toolCall }, + }, + { type: "tool-call-completed", callId: "call-1", modelCallId: "m-1", toolCall }, + { type: "turn-ended" }, + ].map(cursorUpdate); + + try { + const store = yield* makeEventNdjsonLogStore(basePath, { batchWindowMs: 0 }); + const native = store.logger("native"); + for (const event of [...transient, ...retained]) { + yield* native.write(event, threadId); + } + yield* store.close(); + + const records = NodeFS.readFileSync(ownedLogPath(basePath, "cursor-deltas"), "utf8") + .trim() + .split("\n") + .map((line) => decodeUnknownJson(parseLogLine(line).payload)); + assert.deepEqual(records, retained); + } finally { + NodeFS.rmSync(tempDir, { recursive: true, force: true }); + } + }), + ); + it.effect("bounds oversized records while retaining failure details", () => Effect.gen(function* () { const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-")); diff --git a/apps/server/src/provider/EventNdjsonLogger.ts b/apps/server/src/provider/EventNdjsonLogger.ts index 6a1d3844b2373..5a7b3578578f1 100644 --- a/apps/server/src/provider/EventNdjsonLogger.ts +++ b/apps/server/src/provider/EventNdjsonLogger.ts @@ -63,6 +63,13 @@ const transientNativeMethods = new Set([ "turn/diff/updated", ]); const transientAcpUpdates = new Set(["agent_message_chunk", "agent_thought_chunk"]); +const transientCursorUpdates = new Set([ + "text-delta", + "thinking-delta", + "token-delta", + "shell-output-delta", + "partial-tool-call", +]); export type EventNdjsonStream = "native" | "canonical" | "orchestration"; @@ -219,6 +226,18 @@ export function shouldPersistProviderEvent(stream: EventNdjsonStream, event: unk } } + if (nativeType === "interaction.update") { + let update = Reflect.get(nativeEvent, "update"); + if (typeof update !== "object" || update === null) return true; + // Subagent updates arrive wrapped; keep the wrapper when it carries a lifecycle record. + if (Reflect.get(update, "type") === "tool-call-delta") { + update = Reflect.get(update, "taskUpdate"); + if (typeof update !== "object" || update === null) return true; + } + const updateType = Reflect.get(update, "type"); + return typeof updateType !== "string" || !transientCursorUpdates.has(updateType); + } + const payload = Reflect.get(nativeEvent, "payload"); if (typeof payload !== "object" || payload === null) return true; diff --git a/apps/server/src/provider/GrokProvider.test.ts b/apps/server/src/provider/GrokProvider.test.ts index be899ae540037..3f28649f498b8 100644 --- a/apps/server/src/provider/GrokProvider.test.ts +++ b/apps/server/src/provider/GrokProvider.test.ts @@ -639,7 +639,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("readGrokAccount" ), ); expect(limits.windows[0]?.usedPercent).toBe(37); - }).pipe(Effect.scoped), + }), ); it.effect( @@ -749,7 +749,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("readGrokAccount" message: "Grok could not read usage limits.", }); } - }).pipe(Effect.scoped), + }), ); it.effect("sanitizes HTTP failures and malformed billing responses, keeping the account", () => diff --git a/apps/server/src/provider/ModelManifest.test.ts b/apps/server/src/provider/ModelManifest.test.ts index e94ce85bf533c..0b4c4c062dca5 100644 --- a/apps/server/src/provider/ModelManifest.test.ts +++ b/apps/server/src/provider/ModelManifest.test.ts @@ -266,7 +266,7 @@ const layerService = (input: { ); describe("ModelManifest service", () => { - it.live("prefers a fetched manifest over the bundle and caches it to disk", () => + it.effect("prefers a fetched manifest over the bundle and caches it to disk", () => Effect.gen(function* () { const service = yield* ModelManifest.make; const refreshed = yield* service.refresh; @@ -285,29 +285,32 @@ describe("ModelManifest service", () => { ), ); - it.live("ignores older remote edits without replacing the current manifest or disk cache", () => { - let remote = { ...REMOTE_MANIFEST, updatedAt: "2000-01-01T00:00:00Z" }; - return Effect.gen(function* () { - const service = yield* ModelManifest.make; - assert.deepStrictEqual(yield* service.refresh, ModelManifest.BUNDLED_MODEL_MANIFEST); - assert.deepStrictEqual(yield* service.current, ModelManifest.BUNDLED_MODEL_MANIFEST); + it.effect( + "ignores older remote edits without replacing the current manifest or disk cache", + () => { + let remote = { ...REMOTE_MANIFEST, updatedAt: "2000-01-01T00:00:00Z" }; + return Effect.gen(function* () { + const service = yield* ModelManifest.make; + assert.deepStrictEqual(yield* service.refresh, ModelManifest.BUNDLED_MODEL_MANIFEST); + assert.deepStrictEqual(yield* service.current, ModelManifest.BUNDLED_MODEL_MANIFEST); - remote = { ...REMOTE_MANIFEST, updatedAt: REMOTE_UPDATED_AT }; - assert.deepStrictEqual(yield* service.forceRefresh, REMOTE_MANIFEST); - remote = { ...REMOTE_MANIFEST, updatedAt: ModelManifest.BUNDLED_MODEL_MANIFEST.updatedAt! }; - assert.deepStrictEqual(yield* service.forceRefresh, REMOTE_MANIFEST); - const rebooted = yield* ModelManifest.make; - assert.deepStrictEqual(yield* rebooted.current, REMOTE_MANIFEST); - }).pipe( - Effect.scoped, - Effect.provide( - layerService({ - prefix: "model-manifest-stale-fetch-test", - response: () => Response.json(remote), - }), - ), - ); - }); + remote = { ...REMOTE_MANIFEST, updatedAt: REMOTE_UPDATED_AT }; + assert.deepStrictEqual(yield* service.forceRefresh, REMOTE_MANIFEST); + remote = { ...REMOTE_MANIFEST, updatedAt: ModelManifest.BUNDLED_MODEL_MANIFEST.updatedAt! }; + assert.deepStrictEqual(yield* service.forceRefresh, REMOTE_MANIFEST); + const rebooted = yield* ModelManifest.make; + assert.deepStrictEqual(yield* rebooted.current, REMOTE_MANIFEST); + }).pipe( + Effect.scoped, + Effect.provide( + layerService({ + prefix: "model-manifest-stale-fetch-test", + response: () => Response.json(remote), + }), + ), + ); + }, + ); it.effect("preserves the last-good remote cache when later payloads are invalid", () => { let responseIndex = 0; @@ -364,7 +367,7 @@ describe("ModelManifest service", () => { ); }); - it.live("drops a disk cache of a manifest older than the bundled one", () => + it.effect("drops a disk cache of a manifest older than the bundled one", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -400,7 +403,7 @@ describe("ModelManifest service", () => { ), ); - it.live("does not fetch when provider update checks are disabled", () => + it.effect("does not fetch when provider update checks are disabled", () => Effect.gen(function* () { let fetchCount = 0; const service = yield* ModelManifest.make.pipe( diff --git a/apps/server/src/provider/ProjectSettingsOverrideSweep.ts b/apps/server/src/provider/ProjectSettingsOverrideSweep.ts index aa7d205271494..06f61f9ad8bfe 100644 --- a/apps/server/src/provider/ProjectSettingsOverrideSweep.ts +++ b/apps/server/src/provider/ProjectSettingsOverrideSweep.ts @@ -1,32 +1,41 @@ -import { type ProjectId, type ServerSettings } from "@t3tools/contracts"; +import * as NodeUtil from "node:util"; + +import { type ProjectSettingsOverrides } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as ProjectStore from "../orchestration-v2/ProjectStore.ts"; import * as Settings from "../serverSettings.ts"; -const findOrphanedProjectIds = ( - settings: Pick, - liveProjectIds: ReadonlySet, -): ReadonlyArray => - Object.keys(settings.projectSettingsOverrides) - .map((projectId) => projectId as ProjectId) - .filter((projectId) => !liveProjectIds.has(projectId)); +export function deletedProjectOverridePatch(entry: ProjectSettingsOverrides | undefined) { + // Keep cleanup policy for deleted threads; omitting providerInstances would preserve obsolete references in the merge. + return entry?.worktreeCleanup !== undefined + ? { worktreeCleanup: entry.worktreeCleanup, providerInstances: {} } + : null; +} export const sweep = Effect.gen(function* () { const serverSettings = yield* Settings.ServerSettingsService; const projectStore = yield* ProjectStore.ProjectStoreV2; const settings = yield* serverSettings.getSettings; - const liveShells = yield* projectStore.listShells(); - const liveProjectIds = new Set(liveShells.map((shell) => shell.id)); + const projects = yield* projectStore.list({ includeDeleted: true }); + const projectsById = new Map( + projects.map((project) => [project.projectId, project]), + ); - const orphanedProjectIds = findOrphanedProjectIds(settings, liveProjectIds); - if (orphanedProjectIds.length === 0) return; + const overrides = Object.entries(settings.projectSettingsOverrides).flatMap( + ([projectId, entry]) => { + const project = projectsById.get(projectId); + if (project?.deletedAt === null) return []; + const patch = project ? deletedProjectOverridePatch(entry) : null; + if (NodeUtil.isDeepStrictEqual(entry, patch)) return []; + return [[projectId, patch] as const]; + }, + ); + if (overrides.length === 0) return; yield* serverSettings.updateSettings({ - projectSettingsOverrides: Object.fromEntries( - orphanedProjectIds.map((projectId) => [projectId, null]), - ), + projectSettingsOverrides: Object.fromEntries(overrides), }); }).pipe( Effect.asVoid, diff --git a/apps/server/src/provider/ProviderAuthService.test.ts b/apps/server/src/provider/ProviderAuthService.test.ts index 5023dc5bf47ef..603df5e7e4bcd 100644 --- a/apps/server/src/provider/ProviderAuthService.test.ts +++ b/apps/server/src/provider/ProviderAuthService.test.ts @@ -743,7 +743,7 @@ it.effect("queued logout prompts resolve the current controller after provider r assert.isTrue(yield* Fiber.join(queued)); assert.isTrue(replacementLoggedOut); assert.strictEqual(harness.actions.filter((action) => action === "native-logout").length, 1); - }).pipe(Effect.scoped), + }), ); it.effect.each(["start", "logout", "prompt"] as const)( @@ -814,7 +814,7 @@ it.effect.each(["start", "logout", "prompt"] as const)( const blocked = yield* Effect.flip(harness.service.logout({ instanceId })); assert.include(blocked.detail, "shared sign-in"); assert.equal(replacementMutations, 0); - }).pipe(Effect.scoped), + }), ); it.effect.each(["selection", "drain"] as const)( @@ -855,5 +855,5 @@ it.effect.each(["selection", "drain"] as const)( assert.notInclude(harness.actions, "stop:peer-persisted"); assert.isFalse(harness.sessions.has(ThreadId.make("target-draining"))); assert.include(harness.actions, "native-logout"); - }).pipe(Effect.scoped), + }), ); diff --git a/apps/server/src/provider/ProviderHostLive.ts b/apps/server/src/provider/ProviderHostLive.ts index 6398210599b61..fed07efc035ee 100644 --- a/apps/server/src/provider/ProviderHostLive.ts +++ b/apps/server/src/provider/ProviderHostLive.ts @@ -4,11 +4,13 @@ import * as Digest from "effect/crypto/Digest"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import { resolveAttachmentPath } from "../attachmentStore.ts"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; import * as ServerConfig from "../config.ts"; +import * as ServerLifecycleEvents from "../serverLifecycleEvents.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as DocumentText from "./DocumentText.ts"; import * as ProviderCredentialStore from "./ProviderCredentialStore.ts"; @@ -22,6 +24,10 @@ export const layer = Layer.effect( const secrets = yield* ServerSecretStore.ServerSecretStore; const digest = yield* Digest.Digest; const fileSystem = yield* FileSystem.FileSystem; + // Optional so standalone provider layers in tests probe at once. + const lifecycleEvents = yield* Effect.serviceOption( + ServerLifecycleEvents.ServerLifecycleEvents, + ); return ProviderHost.ProviderHost.of({ paths: { cwd: config.cwd, @@ -37,6 +43,10 @@ export const layer = Layer.effect( subscribe: serverSettings.subscribeChanges, }, shouldRunBackgroundWork: backgroundPolicy.shouldRunScopeWork, + ...Option.match(lifecycleEvents, { + onNone: () => ({}), + onSome: (events) => ({ awaitServerReady: events.awaitReady }), + }), resolveAttachmentPath: (attachment) => resolveAttachmentPath({ attachmentsDir: config.attachmentsDir, attachment }), // DocumentText owns the path-resolution span. diff --git a/apps/server/src/provider/ProviderInstanceRegistry.test.ts b/apps/server/src/provider/ProviderInstanceRegistry.test.ts index ab35fe4d9ae59..ce6416332bae0 100644 --- a/apps/server/src/provider/ProviderInstanceRegistry.test.ts +++ b/apps/server/src/provider/ProviderInstanceRegistry.test.ts @@ -212,7 +212,8 @@ describe("ProviderInstanceRegistry — multi-instance codex slice", () => { ); const layerBase = ProviderHostLive.layer.pipe(Layer.provideMerge(layerBaseDeps)); const layerTest = ProviderOrchestrationAdapterInfrastructure.layer.pipe( - Layer.provide(Layer.merge(PortScanner.layerTest, SqlitePersistence.layerMemory)), + Layer.provide(SqlitePersistence.layerMemory), + Layer.provideMerge(PortScanner.layerTest), Layer.provideMerge(layerBase), ); @@ -346,7 +347,7 @@ describe("ProviderInstanceRegistry — multi-instance codex slice", () => { }).pipe(Effect.provide(layerTest)), ); - it.live("treats an explicit in-config enabled:false as disabling despite the envelope", () => + it.effect("treats an explicit in-config enabled:false as disabling despite the envelope", () => Effect.gen(function* () { const staleId = ProviderInstanceId.make("codex_stale"); const configMap: ProviderInstanceConfigMap = { @@ -370,7 +371,7 @@ describe("ProviderInstanceRegistry — multi-instance codex slice", () => { }).pipe(Effect.provide(layerTest)), ); - it.live("lists a blocked driver's instances as unavailable without running them", () => + it.effect("lists a blocked driver's instances as unavailable without running them", () => Effect.gen(function* () { const instanceId = ProviderInstanceId.make("codex"); const reason = "Example Corp's policy does not allow the Codex provider."; @@ -862,7 +863,7 @@ describe("ProviderInstanceRegistry — multi-instance codex slice", () => { ); } - it.live( + it.effect( "shadows instances whose driver is not registered in this build without failing boot", () => Effect.gen(function* () { @@ -990,7 +991,7 @@ describe("ProviderInstanceRegistry secret environment", () => { { name: "BASE_URL", value: "https://example.test", sensitive: false }, ], ]); - }).pipe(Effect.scoped); + }); }); // An empty secret must not let a driver fall back to a credential the server inherited. @@ -1013,7 +1014,7 @@ describe("ProviderInstanceRegistry secret environment", () => { yield* mutator.reconcile({ ...fixture.configMap }); expect(fixture.reads).toHaveLength(readsAtStartup); expect(yield* registry.listUnavailable).toHaveLength(1); - }).pipe(Effect.scoped); + }); }); it.effect("does not read the secrets of an instance its config decodes as disabled", () => { @@ -1025,7 +1026,7 @@ describe("ProviderInstanceRegistry secret environment", () => { expect(fixture.reads).toEqual([]); expect(fixture.invalidated).toEqual([]); expect(fixture.created).toHaveLength(1); - }).pipe(Effect.scoped); + }); }); it.effect("re-reads secrets on reload and rebuilds only when they changed", () => { @@ -1048,7 +1049,7 @@ describe("ProviderInstanceRegistry secret environment", () => { "sk-live", "sk-rotated", ]); - }).pipe(Effect.scoped); + }); }); it.effect("reads a config field typed SettingsSecretValue and rebuilds when it rotates", () => { @@ -1068,7 +1069,7 @@ describe("ProviderInstanceRegistry secret environment", () => { { apiKey: "gw-live", model: "m" }, { apiKey: "gw-rotated", model: "m" }, ]); - }).pipe(Effect.scoped); + }); }); it.effect.each([ @@ -1084,7 +1085,7 @@ describe("ProviderInstanceRegistry secret environment", () => { return Effect.gen(function* () { yield* fixture.make; expect(fixture.configs).toEqual([expect.objectContaining({ [field]: "provider-live" })]); - }).pipe(Effect.scoped); + }); }); it.effect("keeps a working instance live when a reload cannot read its secret", () => { @@ -1098,7 +1099,7 @@ describe("ProviderInstanceRegistry secret environment", () => { expect(yield* registry.getInstance(instanceId)).toBeDefined(); expect(yield* registry.listUnavailable).toEqual([]); expect(fixture.created).toHaveLength(1); - }).pipe(Effect.scoped); + }); }); it.effect("starts without waiting for secrets to be read", () => { @@ -1115,6 +1116,6 @@ describe("ProviderInstanceRegistry secret environment", () => { yield* PubSub.take(changes); expect(yield* registry.getInstance(instanceId)).toBeDefined(); expect(yield* registry.listUnavailable).toEqual([]); - }).pipe(Effect.scoped); + }); }); }); diff --git a/apps/server/src/provider/ProviderRegistry.test.ts b/apps/server/src/provider/ProviderRegistry.test.ts index 0601714dd618e..c42d78b237d42 100644 --- a/apps/server/src/provider/ProviderRegistry.test.ts +++ b/apps/server/src/provider/ProviderRegistry.test.ts @@ -17,6 +17,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as PubSub from "effect/PubSub"; +import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; @@ -788,6 +789,86 @@ it.layer( }), ); + it.effect("shows Codex usage that answers after three seconds inside the probe budget", () => + Effect.gen(function* () { + const usageRequested = yield* Deferred.make(); + const spawner = ChildProcessSpawner.make(() => + Effect.gen(function* () { + const scope = yield* Effect.scope; + const output = yield* Queue.unbounded(); + const send = (message: unknown) => + Queue.offer(output, encoder.encode(`${JSON.stringify(message)}\n`)); + const results: Record = { + initialize: { + userAgent: "codex_cli_rs/0.167.0", + codexHome: "/nonexistent-codex-home", + platformFamily: "unix", + platformOs: "linux", + }, + "account/read": { + account: { type: "chatgpt", email: "test@example.com", planType: "pro" }, + requiresOpenaiAuth: true, + }, + "config/read": { config: {}, origins: {} }, + "skills/list": { data: [] }, + "model/list": { data: [] }, + }; + const answer = (line: string) => { + const { id, method } = JSON.parse(line) as { id?: number; method?: string }; + if (id === undefined || method === undefined) return Effect.void; + if (method !== "account/rateLimits/read") { + return send({ id, result: results[method] ?? {} }); + } + return Deferred.succeed(usageRequested, undefined).pipe( + Effect.andThen(Effect.sleep("4 seconds")), + Effect.andThen( + send({ + id, + result: { + rateLimits: { + limitId: "codex", + planType: "pro", + primary: { usedPercent: 42, windowDurationMins: 300, resetsAt: 2e9 }, + }, + rateLimitsByLimitId: null, + rateLimitResetCredits: null, + accountId: null, + }, + }), + ), + Effect.forkIn(scope), + ); + }; + const decoder = new TextDecoder(); + let buffered = ""; + return ChildProcessSpawner.makeHandle({ + ...mockHandle({ stdout: "", stderr: "", code: 0 }), + exitCode: Effect.never, + isRunning: Effect.succeed(true), + stdin: Sink.forEach((chunk: Uint8Array) => { + buffered += decoder.decode(chunk, { stream: true }); + const lines = buffered.split("\n"); + buffered = lines.pop() ?? ""; + return Effect.forEach(lines.filter(Boolean), answer, { discard: true }); + }), + stdout: Stream.fromQueue(output), + }); + }), + ); + const fiber = yield* checkCodexProviderStatus(defaultCodexSettings).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.forkChild, + ); + yield* Deferred.await(usageRequested); + yield* TestClock.adjust("25 seconds"); + const status = yield* Fiber.join(fiber); + + assert.strictEqual(status.auth.status, "authenticated"); + assert.isUndefined(status.usageLimits?.unavailable); + assert.strictEqual(status.usageLimits?.windows[0]?.usedPercent, 42); + }), + ); + it.effect("does not run --version when the Codex executable cannot start", () => Effect.gen(function* () { if ((yield* HostProcess.Platform) === "win32") return; @@ -1228,7 +1309,7 @@ it.layer( ); }); - it("retains stale OpenCode models when a refresh fails", () => { + it("retains stale OpenCode models when a refresh fails or lists none", () => { const previousProvider = { instanceId: ProviderInstanceId.make("opencode"), driver: ProviderDriverKind.make("opencode"), @@ -1278,6 +1359,21 @@ it.layer( ProviderRegistry.mergeProviderSnapshot(previousProvider, refreshedProvider).skills, previousProvider.skills, ); + + const emptyCatalogProvider = { + ...refreshedProvider, + status: "warning", + message: "OpenCode 2 is running, but it did not list any models yet.", + slashCommands: [], + skills: [], + } satisfies ServerProvider; + const mergedEmptyCatalog = ProviderRegistry.mergeProviderSnapshot( + previousProvider, + emptyCatalogProvider, + ); + assert.deepStrictEqual(mergedEmptyCatalog.models, [...previousProvider.models]); + assert.deepStrictEqual(mergedEmptyCatalog.slashCommands, previousProvider.slashCommands); + assert.deepStrictEqual(mergedEmptyCatalog.skills, previousProvider.skills); }); describe("Pi model inventories", () => { @@ -4180,6 +4276,7 @@ it.layer( ); assert.strictEqual(status.status, "ready"); assert.strictEqual(status.auth.status, "authenticated"); + assert.strictEqual(status.auth.label, "Claude API Key"); }).pipe( Effect.provide( layerMockSpawner((args) => { diff --git a/apps/server/src/provider/ProviderRegistry.ts b/apps/server/src/provider/ProviderRegistry.ts index ba9ce9b2b6af7..8a7ff0ac7fd37 100644 --- a/apps/server/src/provider/ProviderRegistry.ts +++ b/apps/server/src/provider/ProviderRegistry.ts @@ -220,7 +220,8 @@ const shouldRetainMissingProviderModels = (provider: ServerProvider): boolean => const isAntigravity = provider.driver === ProviderDriverKind.make("antigravity"); const isCodex = provider.driver === ProviderDriverKind.make("codex"); - if (!isAntigravity && !isCodex && provider.driver !== ProviderDriverKind.make("opencode")) { + const isOpenCode = provider.driver === ProviderDriverKind.make("opencode"); + if (!isAntigravity && !isCodex && !isOpenCode) { return true; } @@ -236,9 +237,10 @@ const shouldRetainMissingProviderModels = (provider: ServerProvider): boolean => isAntigravity && provider.status === "warning" && provider.auth.status === "unknown"; const isPendingInitialProbe = provider.enabled && !provider.installed && provider.status === "warning"; + // A cold OpenCode server can finish its probe as a warning before it lists any models. const isIncompleteInstalledProbe = provider.installed && - (provider.status === "error" || (isCodex && provider.status === "warning")); + (provider.status === "error" || ((isCodex || isOpenCode) && provider.status === "warning")); return isPendingAntigravityAuthentication || isPendingInitialProbe || isIncompleteInstalledProbe; }; diff --git a/apps/server/src/provider/acp/AcpJsonRpcConnection.test.ts b/apps/server/src/provider/acp/AcpJsonRpcConnection.test.ts index f06c4f47570dd..4c21e400f3c8e 100644 --- a/apps/server/src/provider/acp/AcpJsonRpcConnection.test.ts +++ b/apps/server/src/provider/acp/AcpJsonRpcConnection.test.ts @@ -15,13 +15,16 @@ import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as Scope from "effect/Scope"; import * as Result from "effect/Result"; +import * as Scheduler from "effect/Scheduler"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; import { ChildProcessSpawner } from "effect/process"; import { describe, expect } from "vite-plus/test"; import * as AcpSessionRuntime from "@t3tools/provider-acp/server/AcpSessionRuntime"; import { AgentScope } from "@t3tools/shared/AgentScope"; import * as EffectAcpErrors from "effect-acp/errors"; +import type * as EffectAcpProtocol from "effect-acp/protocol"; const __dirname = NodePath.dirname(NodeURL.fileURLToPath(import.meta.url)); const mockAgentPath = NodePath.join(__dirname, "../../../scripts/acp-mock-agent.ts"); @@ -44,6 +47,136 @@ const setProcessEnv = (name: string, value: string) => { }; }; +const isPromptStarted = (event: AcpSessionRuntime.AcpSessionRequestLogEvent) => + event.method === "session/prompt" && event.status === "started"; + +const isOutgoing = (tag: string) => (event: EffectAcpProtocol.AcpProtocolLogEvent) => + event.direction === "outgoing" && + event.stage === "decoded" && + typeof event.payload === "object" && + event.payload !== null && + Reflect.get(event.payload, "tag") === tag; + +// The methods the mock agent read, in order, keeping only the given ones. +const readReceived = (requestLogPath: string, methods: ReadonlyArray) => + NodeFS.readFileSync(requestLogPath, "utf8") + .split("\n") + .filter((line) => line.length > 0) + .map((line) => (JSON.parse(line) as { readonly method?: string }).method) + .filter((method) => method !== undefined && methods.includes(method)); + +// Prompts, then cancels while the prompt is held on its way out: in its started request record, +// in its outgoing protocol log record, or by its fiber yielding every few operations. The cancel +// comes on dispatch or from inside the held record. Returns the prompt's exit +// and the order in which the agent read the prompt, the cancel and the request that ends it. +const promptThenCancel = (input: { + readonly cancelBehavior: "interrupt" | "wait-for-prompt"; + readonly holdPrompt: { + readonly in: "request-log" | "protocol-log" | "scheduler"; + readonly cancelOn: "dispatch" | "log"; + }; +}) => + Effect.gen(function* () { + const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "acp-runtime-")); + yield* Effect.addFinalizer(() => + Effect.sync(() => NodeFS.rmSync(tempDir, { recursive: true, force: true })), + ); + const requestLogPath = NodePath.join(tempDir, "requests.ndjson"); + const { holdPrompt } = input; + const promptHeld = yield* Deferred.make(); + const promptReleased = yield* Deferred.make(); + yield* Effect.addFinalizer(() => Deferred.succeed(promptReleased, undefined)); + const testScope = yield* Scope.Scope; + const runtimeMade = yield* Deferred.make(); + const cancelledFromLog = yield* Deferred.make>(); + const hold = ( + holdPrompt.cancelOn === "log" + ? Deferred.await(runtimeMade).pipe( + Effect.flatMap((made) => + made.cancel.pipe(Effect.forkIn(testScope, { startImmediately: true })), + ), + Effect.flatMap((fiber) => Deferred.succeed(cancelledFromLog, fiber)), + ) + : Effect.void + ).pipe( + Effect.andThen(Deferred.succeed(promptHeld, undefined)), + Effect.andThen(Deferred.await(promptReleased)), + ); + const runtime = yield* AcpSessionRuntime.make({ + ...mockRuntimeOptions, + spawn: { + ...mockRuntimeOptions.spawn, + env: { + T3_ACP_COMPLETE_FIRST_PROMPT_ON_CANCEL: "1", + T3_ACP_REQUEST_LOG_PATH: requestLogPath, + }, + }, + cancelBehavior: input.cancelBehavior, + interruptPromptOnCancel: false, + protocolLogging: { + logOutgoing: holdPrompt.in === "protocol-log", + logger: (event) => (isOutgoing("session/prompt")(event) ? hold : Effect.void), + }, + requestLogger: (event) => + holdPrompt.in === "request-log" && isPromptStarted(event) ? hold : Effect.void, + }); + yield* Deferred.succeed(runtimeMade, runtime); + const cancelReceived = yield* Deferred.make(); + yield* runtime.getEvents().pipe( + Stream.runForEach((event) => { + if (event._tag === "EventStreamBarrier") { + return Deferred.succeed(event.acknowledge, undefined); + } + if (event._tag === "ThoughtDelta" && event.text === "native-cancel-received") { + return Deferred.succeed(cancelReceived, undefined); + } + return Effect.void; + }), + Effect.forkChild, + ); + yield* runtime.start(); + const dispatched = yield* Deferred.make(); + const prompt = yield* runtime + .prompt({ prompt: [{ type: "text", text: "first" }] }, { dispatched }) + .pipe( + holdPrompt.in === "scheduler" + ? Effect.provideService(Scheduler.MaxOpsBeforeYield, 3) + : (effect) => effect, + Effect.forkChild, + ); + // A round trip: whatever was queued before it reaches the agent first. + const roundTrip = runtime.request("_test/environment", {}); + let cancel: Fiber.Fiber; + if (holdPrompt.in === "scheduler") { + yield* Deferred.await(dispatched); + cancel = yield* runtime.cancel.pipe(Effect.forkChild({ startImmediately: true })); + } else { + yield* Deferred.await(promptHeld); + if (holdPrompt.cancelOn === "dispatch") { + cancel = yield* Deferred.await(dispatched).pipe( + Effect.andThen(runtime.cancel), + Effect.forkChild({ startImmediately: true }), + ); + } else { + cancel = yield* Deferred.await(cancelledFromLog); + } + yield* roundTrip; + yield* Deferred.succeed(promptReleased, undefined); + } + yield* Deferred.await(cancelReceived); + yield* runtime.request("_test/finish-cancel", {}); + // The agent answers it only after reading every earlier line. + yield* roundTrip; + const received = readReceived(requestLogPath, [ + "session/prompt", + "session/cancel", + "_test/finish-cancel", + ]); + const promptExit = yield* Fiber.await(prompt); + yield* Fiber.join(cancel); + return { received, promptExit }; + }).pipe(Effect.scoped); + describe("AcpSessionRuntime", () => { it.effect.each(["session/new", "session/resume"] as const)( "buffers root metadata while %s startup is still pending", @@ -1239,4 +1372,104 @@ describe("AcpSessionRuntime", () => { Effect.ensuring(Effect.sync(() => NodeFS.rmSync(tempDir, { recursive: true, force: true }))), ); }); + + it.effect.each( + (["interrupt", "wait-for-prompt"] as const).flatMap((cancelBehavior) => + ( + [ + { in: "request-log", cancelOn: "dispatch" }, + { in: "protocol-log", cancelOn: "log" }, + { in: "scheduler", cancelOn: "dispatch" }, + ] as const + ).map((holdPrompt) => ({ cancelBehavior, holdPrompt })), + ), + )( + "sends session/prompt before a cancel issued while the prompt is held in the $holdPrompt.in ($cancelBehavior, cancel on $holdPrompt.cancelOn)", + ({ cancelBehavior, holdPrompt }) => + Effect.gen(function* () { + const run = yield* promptThenCancel({ cancelBehavior, holdPrompt }); + expect(run.received).toEqual(["session/prompt", "session/cancel", "_test/finish-cancel"]); + expect(run.promptExit).toEqual( + Exit.succeed({ stopReason: "cancelled", _meta: { nativeCancel: true } }), + ); + }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), + ); + + it.effect("sends a cancel after the cancel timeout when its prompt never goes out", () => + Effect.gen(function* () { + const runtime = yield* AcpSessionRuntime.make({ + ...mockRuntimeOptions, + interruptPromptOnCancel: false, + cancelTimeout: "5 seconds", + requestLogger: (event) => (isPromptStarted(event) ? Effect.never : Effect.void), + }); + yield* runtime.start(); + const dispatched = yield* Deferred.make(); + yield* runtime + .prompt({ prompt: [{ type: "text", text: "first" }] }, { dispatched }) + .pipe(Effect.forkChild); + yield* Deferred.await(dispatched); + const cancel = yield* runtime.cancel.pipe(Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust("5 seconds"); + yield* Fiber.join(cancel); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), + ); + + it.effect( + "sends a cancel before the next prompt when the prompt it waited for never went out", + () => + Effect.gen(function* () { + const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "acp-runtime-")); + yield* Effect.addFinalizer(() => + Effect.sync(() => NodeFS.rmSync(tempDir, { recursive: true, force: true })), + ); + const requestLogPath = NodePath.join(tempDir, "requests.ndjson"); + let promptsStarted = 0; + const runtime = yield* AcpSessionRuntime.make({ + ...mockRuntimeOptions, + spawn: { ...mockRuntimeOptions.spawn, env: { T3_ACP_REQUEST_LOG_PATH: requestLogPath } }, + interruptPromptOnCancel: false, + // The cancel spends a test-clock second in its outgoing record, the window in which the + // next prompt could overtake it. + protocolLogging: { + logOutgoing: true, + logger: (event) => + isOutgoing("session/cancel")(event) ? Effect.sleep("1 second") : Effect.void, + }, + requestLogger: (event) => + isPromptStarted(event) && promptsStarted++ === 0 ? Effect.never : Effect.void, + }); + yield* runtime.getEvents().pipe( + Stream.runForEach((event) => + event._tag === "EventStreamBarrier" + ? Deferred.succeed(event.acknowledge, undefined) + : Effect.void, + ), + Effect.forkChild, + ); + yield* runtime.start(); + const dispatched = yield* Deferred.make(); + const first = yield* runtime + .prompt({ prompt: [{ type: "text", text: "first" }] }, { dispatched }) + .pipe(Effect.forkChild); + yield* Deferred.await(dispatched); + const cancel = yield* runtime.cancel.pipe(Effect.forkChild({ startImmediately: true })); + const next = yield* runtime + .prompt({ prompt: [{ type: "text", text: "next" }] }) + .pipe(Effect.forkChild); + // The first prompt ends before it is queued, which releases the cancel's wait. + yield* Effect.sleep("1 second").pipe( + Effect.andThen(Fiber.interrupt(first)), + Effect.forkChild, + ); + yield* TestClock.adjust("1 second"); + yield* TestClock.adjust("1 second"); + yield* Fiber.join(cancel); + yield* Fiber.join(next); + expect(readReceived(requestLogPath, ["session/prompt", "session/cancel"])).toEqual([ + "session/cancel", + "session/prompt", + ]); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), + ); }); diff --git a/apps/server/src/provider/acp/AcpSessionRuntime.processTree.test.ts b/apps/server/src/provider/acp/AcpSessionRuntime.processTree.test.ts index 0c6bddbbf8567..6b9afa7f30c50 100644 --- a/apps/server/src/provider/acp/AcpSessionRuntime.processTree.test.ts +++ b/apps/server/src/provider/acp/AcpSessionRuntime.processTree.test.ts @@ -1,6 +1,7 @@ // @effect-diagnostics nodeBuiltinImport:off import * as NodeChildProcess from "node:child_process"; import * as NodeFS from "node:fs"; +import * as NodeModule from "node:module"; import * as NodePath from "node:path"; import { describe, expect, it, vi } from "@effect/vitest"; @@ -813,6 +814,33 @@ describe("terminatePosixOwnedProcessTree", () => { }), ); + it.skipIf(HostProcess.Platform.defaultValue() !== "linux")( + "snapshots /proc when a process exits while its entry is being typed", + () => { + const fs = NodeModule.createRequire(import.meta.url)("node:fs") as typeof NodeFS; + const realReaddirSync = fs.readdirSync; + // Node lstat()s a dirent of unknown type; a process that exits meanwhile makes it throw. + fs.readdirSync = ((path: NodeFS.PathLike, options?: unknown) => { + if (path === "/proc" && typeof options === "object" && options !== null) { + throw Object.assign(new Error("ENOENT: no such file or directory, lstat '/proc/4563'"), { + code: "ENOENT", + syscall: "lstat", + path: "/proc/4563", + }); + } + return Reflect.apply(realReaddirSync, fs, [path, options]); + }) as typeof NodeFS.readdirSync; + NodeModule.syncBuiltinESMExports(); + try { + const snapshot = AcpSessionRuntime.makePosixProcessTreeController("linux").snapshot(); + expect(snapshot.some((entry) => entry.pid === process.pid)).toBe(true); + } finally { + fs.readdirSync = realReaddirSync; + NodeModule.syncBuiltinESMExports(); + } + }, + ); + it("fails closed where a stable POSIX identity provider is unavailable", () => { try { AcpSessionRuntime.makePosixProcessTreeController("darwin"); diff --git a/apps/server/src/provider/acp/AntigravitySessionFiles.test.ts b/apps/server/src/provider/acp/AntigravitySessionFiles.test.ts index 795e80b861e84..81885dc0c89b2 100644 --- a/apps/server/src/provider/acp/AntigravitySessionFiles.test.ts +++ b/apps/server/src/provider/acp/AntigravitySessionFiles.test.ts @@ -31,7 +31,7 @@ it.layer(NodeServices.layer)("retireAntigravityRuntimeTempDirs", (it) => { yield* removeRetired; expect((yield* fs.readDirectory(parent)).toSorted()).toEqual(["other-instance"]); - }).pipe(Effect.scoped), + }), ); it.effect( diff --git a/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts b/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts index 09b8730bd6001..52e96caa08aea 100644 --- a/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts +++ b/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts @@ -158,7 +158,7 @@ describe("watchAntigravityPromptOutcome", () => { code: "QUOTA_EXHAUSTED", retryable: true, }); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); it.effect("reads only this session's own unlinked log, and keeps a typed refusal", () => @@ -206,6 +206,6 @@ describe("watchAntigravityPromptOutcome", () => { code: "QUOTA_EXHAUSTED", retryable: true, }); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); }); diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index 815f96e843eeb..2528d3ab1c744 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { - "claudeSystemAppendBytes": 5597, - "codexAdditionalContextBytes": 7327, - "mcpToolListJsonBytes": 48120 + "claudeSystemAppendBytes": 5930, + "codexAdditionalContextBytes": 7324, + "mcpToolListJsonBytes": 49732 } diff --git a/apps/server/src/provider/claudeUsageLimits.test.ts b/apps/server/src/provider/claudeUsageLimits.test.ts index f20d8b0543636..b683c80f4f5e8 100644 --- a/apps/server/src/provider/claudeUsageLimits.test.ts +++ b/apps/server/src/provider/claudeUsageLimits.test.ts @@ -199,6 +199,22 @@ describe("claudeUsageResponseToLimits", () => { ).toEqual([monthlyWindow]); }); + it("flags a $0 cap instead of reporting a monthly window", () => { + const response = { + rate_limits_available: true, + rate_limits: { + five_hour: null, + seven_day: null, + extra_usage: { is_enabled: true, used_credits: 0, monthly_limit: 0, utilization: null }, + }, + }; + expect(claudeUsageResponseToLimits({ checkedAt, response }).limits).toEqual({ + checkedAt, + windows: [], + zeroSpendCap: true, + }); + }); + it("omits disabled budgets and refuses invalid money without an enabled fallback", () => { const rateLimits = enterpriseResponse.rate_limits; const response = { diff --git a/apps/server/src/provider/claudeUsageLimits.ts b/apps/server/src/provider/claudeUsageLimits.ts index d0acf43bb4968..ba1263ef704bb 100644 --- a/apps/server/src/provider/claudeUsageLimits.ts +++ b/apps/server/src/provider/claudeUsageLimits.ts @@ -47,8 +47,20 @@ export const makeClaudeScopedLimitNames = Ref.make({ overageIncluded: undefined, }); +const SCOPED_WINDOW_PREFIX = "seven_day_"; +const scopeToken = (name: string) => name.toLowerCase().replace(/[^a-z0-9]+/g, "_"); + function scopedWindowId(displayName: string): string { - return `seven_day_${displayName.toLowerCase().replace(/[^a-z0-9]+/g, "_")}`; + return `${SCOPED_WINDOW_PREFIX}${scopeToken(displayName)}`; +} + +/** A model-scoped weekly allowance blocks only its own model; an unknown model is not blocked. */ +export function windowLimitsModel( + window: Pick, + model: string | null, +): boolean { + if (!window.id.startsWith(SCOPED_WINDOW_PREFIX)) return true; + return model !== null && scopeToken(model).includes(window.id.slice(SCOPED_WINDOW_PREFIX.length)); } function scopedWindow( @@ -202,6 +214,7 @@ export function claudeUsageResponseToLimits(input: { overageIncluded ??= entry.display_name; } const spendPercent = readSpendPercent(response.rate_limits); + const extra = response.rate_limits.extra_usage; if (spendPercent !== undefined) { windows.push({ id: "monthly_spend", @@ -211,7 +224,10 @@ export function claudeUsageResponseToLimits(input: { }); } return { - limits: makeUsageLimits({ checkedAt, windows }), + limits: { + ...makeUsageLimits({ checkedAt, windows }), + ...(extra?.is_enabled && extra.monthly_limit === 0 ? { zeroSpendCap: true } : {}), + }, names: { overageIncluded }, }; } diff --git a/apps/server/src/provider/codexLaunchArgs.test.ts b/apps/server/src/provider/codexLaunchArgs.test.ts index 8f4e1b7ca6d6a..e74b29d44818c 100644 --- a/apps/server/src/provider/codexLaunchArgs.test.ts +++ b/apps/server/src/provider/codexLaunchArgs.test.ts @@ -38,6 +38,15 @@ describe("codexExecLaunchArgs", () => { ); }); + it("drops launch args that enable user hooks", () => { + NodeAssert.deepStrictEqual( + codexExecLaunchArgs( + "--enable hooks --enable=hooks --enable codex_hooks --enable=codex_hooks --enable foo --disable hooks", + ), + ["--enable", "foo", "--disable", "hooks"], + ); + }); + it("does not pair value-taking flags with adjacent flags", () => { NodeAssert.deepStrictEqual(codexExecLaunchArgs("--config --strict-config --enable --disable"), [ "--strict-config", diff --git a/apps/server/src/provider/codexLaunchArgs.ts b/apps/server/src/provider/codexLaunchArgs.ts index 31ff37e31deb5..e49314ee83e0d 100644 --- a/apps/server/src/provider/codexLaunchArgs.ts +++ b/apps/server/src/provider/codexLaunchArgs.ts @@ -62,8 +62,17 @@ const filterLaunchArgs = (launchArgs: string | undefined, allowed: LaunchArgFilt return kept; }; -export const codexExecLaunchArgs = (launchArgs?: string) => - filterLaunchArgs(launchArgs, EXEC_FILTER); +const HOOK_FEATURES = new Set(["hooks", "codex_hooks"]); + +// Feature toggles beat config overrides, so a launch arg's hook enable would undo ours. +export const codexExecLaunchArgs = (launchArgs?: string) => { + const args = filterLaunchArgs(launchArgs, EXEC_FILTER); + return args.filter((arg, index) => { + if (arg.startsWith("--enable=")) return !HOOK_FEATURES.has(arg.slice("--enable=".length)); + if (arg === "--enable") return !HOOK_FEATURES.has(args[index + 1] ?? ""); + return !(args[index - 1] === "--enable" && HOOK_FEATURES.has(arg)); + }); +}; export const codexMcpListLaunchArgs = (launchArgs?: string) => filterLaunchArgs(launchArgs, MCP_LIST_FILTER); diff --git a/apps/server/src/provider/codexUsageLimits.test.ts b/apps/server/src/provider/codexUsageLimits.test.ts index b560aba5d3c28..3a5cae4a9d2c1 100644 --- a/apps/server/src/provider/codexUsageLimits.test.ts +++ b/apps/server/src/provider/codexUsageLimits.test.ts @@ -1,12 +1,15 @@ +import * as DateTime from "effect/DateTime"; import { ProviderDriverKind, ProviderInstanceId } from "@t3tools/contracts"; import { exhaustedUsageWindow } from "@t3tools/shared/usageLimits"; import * as CodexErrors from "effect-codex-app-server/errors"; import { describe, expect, it } from "vite-plus/test"; import { + codexMessageResetAt, codexRateLimitsFailureMessage, codexRateLimitsToLimits, codexRateLimitsToUpdate, + codexTimeZone, mergeCodexRateLimits, } from "./codexUsageLimits.ts"; @@ -313,3 +316,61 @@ describe("codexUsageLimitResetAt", () => { ).toBeNull(); }); }); + +describe("codexMessageResetAt", () => { + const dhaka = DateTime.zoneMakeNamedUnsafe("Asia/Dhaka"); + const at = (iso: string) => DateTime.makeUnsafe(iso); + const failedAt = at("2026-11-07T03:00:00Z"); // 09:00 in Dhaka + + // Codex prints local time without seconds, so the parsed reset is the minute after. + it("reads Codex's same-day and dated reset text in Codex's zone", () => { + expect( + codexMessageResetAt("You’ve hit your usage limit. Try again at 3:37 PM.", failedAt, dhaka), + ).toBe("2026-11-07T09:38:00.000Z"); + expect( + codexMessageResetAt("Upgrade, or try again at Dec 1st, 2026 12:05 AM.", failedAt, dhaka), + ).toBe("2026-11-30T18:06:00.000Z"); + expect(codexMessageResetAt("Try again at 12:30 PM.", failedAt, dhaka)).toBe( + "2026-11-07T06:31:00.000Z", + ); + }); + + it("dates today's form by when Codex failed, across local midnight", () => { + expect(codexMessageResetAt("Try again at 11:59 PM.", at("2026-11-07T17:59:59Z"), dhaka)).toBe( + "2026-11-07T18:00:00.000Z", + ); + }); + + it("takes the later instant of a repeated DST hour", () => { + // 1:30 AM occurs twice in New York on Nov 1st, 2026: 05:30Z (EDT) and 06:30Z (EST). + const newYork = DateTime.zoneMakeNamedUnsafe("America/New_York"); + for (const iso of ["2026-11-01T05:15:00Z", "2026-11-01T06:15:00Z"]) { + expect(codexMessageResetAt("Try again at 1:30 AM.", at(iso), newYork)).toBe( + "2026-11-01T06:31:00.000Z", + ); + } + }); + + it("leaves the reset unknown without reset text or once it has passed", () => { + expect( + codexMessageResetAt("You’ve hit your usage limit. Try again later.", failedAt, dhaka), + ).toBeNull(); + expect( + codexMessageResetAt( + "exceeded retry limit, last status: 429 Too Many Requests", + failedAt, + dhaka, + ), + ).toBeNull(); + expect(codexMessageResetAt("Try again at 8:15 AM.", failedAt, dhaka)).toBeNull(); + }); +}); + +describe("codexTimeZone", () => { + it("uses Codex's TZ and falls back to the server's zone", () => { + const local = DateTime.zoneToString(DateTime.zoneMakeLocal()); + expect(DateTime.zoneToString(codexTimeZone({ TZ: ":Asia/Dhaka" }))).toBe("Asia/Dhaka"); + expect(DateTime.zoneToString(codexTimeZone({ TZ: "not/a-zone" }))).toBe(local); + expect(DateTime.zoneToString(codexTimeZone({}))).toBe(local); + }); +}); diff --git a/apps/server/src/provider/codexUsageLimits.ts b/apps/server/src/provider/codexUsageLimits.ts index d4f9c621119df..526852332033d 100644 --- a/apps/server/src/provider/codexUsageLimits.ts +++ b/apps/server/src/provider/codexUsageLimits.ts @@ -201,3 +201,51 @@ export function codexUsageLimitResetAt( null, ); } + +const CODEX_RESET_TEXT = + /[Tt]ry again at (?:([A-Z][a-z]{2}) (\d{1,2})(?:st|nd|rd|th), (\d{4}) )?(\d{1,2}):(\d{2}) ([AP]M)\./; +const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; + +/** The zone Codex prints reset times in: its own `TZ`, else the server's. */ +export function codexTimeZone( + environment: Readonly>, +): DateTime.TimeZone { + return Option.getOrElse( + DateTime.zoneMakeNamed(environment.TZ?.replace(/^:/, "") ?? ""), + DateTime.zoneMakeLocal, + ); +} + +/** + * Codex writes a usage limit's reset into its error text in its own local time: + * "Try again at 3:37 PM." for today, else "try again at Nov 11th, 2026 3:37 PM.". + * It is the only reset left when a proxy such as CLIProxyAPI strips the rate-limit snapshot. + * `failedAt` is when Codex reported the failure, which fixes the date of today's form. + */ +export function codexMessageResetAt( + message: string, + failedAt: DateTime.DateTime, + zone: DateTime.TimeZone, +): string | null { + const match = CODEX_RESET_TEXT.exec(message); + if (!match) return null; + const [, month, day, year, hour, minute, meridiem] = match; + const monthIndex = month === undefined ? undefined : MONTHS.indexOf(month); + if (monthIndex === -1) return null; + const today = DateTime.toParts(DateTime.setZone(failedAt, zone)); + const reset = DateTime.makeZoned( + { + year: monthIndex === undefined ? today.year : Number(year), + month: monthIndex === undefined ? today.month : monthIndex + 1, + day: monthIndex === undefined ? today.day : Number(day), + hour: (Number(hour) % 12) + (meridiem === "PM" ? 12 : 0), + minute: Number(minute), + }, + // The repeated hour at a DST fall-back takes its later instant so a resume never fires early. + { timeZone: zone, adjustForTimeZone: true, disambiguation: "later" }, + ); + if (Option.isNone(reset)) return null; + // Codex drops the seconds, so the next minute is the first one past the reset. + const resetAt = DateTime.add(reset.value, { minutes: 1 }); + return DateTime.isGreaterThan(resetAt, failedAt) ? DateTime.formatIso(resetAt) : null; +} diff --git a/apps/server/src/provider/documentTextWorker.ts b/apps/server/src/provider/documentTextWorker.ts index 81008be757dac..b031547be6160 100644 --- a/apps/server/src/provider/documentTextWorker.ts +++ b/apps/server/src/provider/documentTextWorker.ts @@ -41,12 +41,43 @@ interface XmlNode { hasTextChoice?: boolean; } -function xml(source: string, odfSpaces = false, htmlEntities = false): XmlNode { +// ODF 1.3 Part 3 §6.1.2: paragraph character data turns TAB, CR and LF into spaces, collapses +// space runs and trims the ends; only , and keep literal ones. +function collapseOdfWhitespace(paragraph: XmlNode) { + let afterSpace = true; + let tail: { node: XmlNode; index: number; value: string } | undefined; + const visit = (node: XmlNode) => { + for (const [index, child] of node.children.entries()) { + if (typeof child !== "string") { + if (HIDDEN.has(child.name)) continue; + if (child.spaceCount === undefined && !["tab", "line-break", "p", "h"].includes(child.name)) + visit(child); + else { + afterSpace = false; + tail = undefined; + } + continue; + } + let value = child.replaceAll(/[ \t\r\n]+/g, " "); + if (afterSpace && value.startsWith(" ")) value = value.slice(1); + if (value) { + afterSpace = value.endsWith(" "); + tail = { node, index, value }; + } + node.children[index] = value; + } + }; + visit(paragraph); + if (tail && afterSpace) tail.node.children[tail.index] = tail.value.slice(0, -1); +} + +function xml(source: string, extension: string): XmlNode { + const odf = [".odt", ".ods", ".odp"].includes(extension); const root: XmlNode = { name: "root", attrs: {}, children: [] }; const stack = [root]; let nodes = 0; const parser = new SaxesParser(); - if (htmlEntities) Object.assign(parser.ENTITIES, characterEntities); + if (extension === ".epub") Object.assign(parser.ENTITIES, characterEntities); parser.on("opentag", (tag) => { if (++nodes > 200_000 || stack.length > 128) throw new Truncation("XML structure limit"); const node: XmlNode = { @@ -59,20 +90,25 @@ function xml(source: string, odfSpaces = false, htmlEntities = false): XmlNode { ), children: [], }; - if (odfSpaces && node.name === "s") node.spaceCount = repeatCount(node.attrs.c); + if (odf && node.name === "s") node.spaceCount = repeatCount(node.attrs.c); stack.at(-1)!.children.push(node); stack.push(node); }); parser.on("closetag", () => { const node = stack.pop()!; const parent = stack.at(-1)!; + if (odf && (node.name === "p" || node.name === "h")) collapseOdfWhitespace(node); + if (extension === ".docx" && node.name === "del") { + parent.children.pop(); + return; + } // Markup-compatibility fallbacks repeat their mc:Choice content, e.g. Word text boxes. // Keep one when no Choice carries text, such as a chart with a VML caption. if (node.name === "Fallback" && parent.hasTextChoice) { parent.children.pop(); return; } - if (!node.hasText) return; + if (!node.hasText && node.name !== "noBreakHyphen") return; parent.hasText = true; if (node.name === "Choice") parent.hasTextChoice = true; }); @@ -99,6 +135,7 @@ function* text(node: XmlNode): Generator { if (HIDDEN.has(node.name)) return; if (["tab", "cell"].includes(node.name)) yield "\t"; else if (["br", "line-break", "cr"].includes(node.name)) yield "\n"; + else if (node.name === "noBreakHyphen") yield "\u2011"; else if (node.spaceCount !== undefined) { yield " ".repeat(Math.min(MAX_OUTPUT, node.spaceCount)); if (node.spaceCount > MAX_OUTPUT) throw new Truncation("Space repeat limit"); @@ -108,6 +145,23 @@ function* text(node: XmlNode): Generator { else yield* text(child); } } +// SpreadsheetML `t` and `v` values are ST_Xstring: `_xHHHH_` is one UTF-16 code unit, and an +// escaped lead underscore (`_x005F_xHHHH_`) keeps the literal `_xHHHH_`. Decoded in one pass. +function* xstringText(node: XmlNode): Generator { + if (HIDDEN.has(node.name)) return; + if (node.name !== "t" && node.name !== "v") { + for (const child of children(node)) yield* xstringText(child); + return; + } + yield node.children + .filter((child) => typeof child === "string") + .join("") + .replaceAll( + /_x005[Ff]_(x[\dA-Fa-f]{4}_)|_x([\dA-Fa-f]{4})_/g, + (_, literal?: string, hex?: string) => + literal ? `_${literal}` : String.fromCharCode(Number.parseInt(hex!, 16)), + ); +} function* paragraphs(node: XmlNode): Generator { if (HIDDEN.has(node.name)) return; for (const child of node.children) { @@ -164,7 +218,7 @@ function odfValue(cell: XmlNode): string { } } function hasCellText(cell: XmlNode): boolean { - if (odfValue(cell).trim()) return true; + if (odfValue(cell).trim() || cell.name === "noBreakHyphen") return true; // Space markers can have enormous repeat counts; emptiness never requires expanding them. if (HIDDEN.has(cell.name) || cell.spaceCount !== undefined) return false; return cell.children.some((child) => @@ -342,6 +396,10 @@ function* rtf(source: string): Generator { ["ldblquote", "“"], ["rdblquote", "”"], ["bullet", "•"], + ["zwj", "\u200d"], + ["zwnj", "\u200c"], + ["ltrmark", "\u200e"], + ["rtlmark", "\u200f"], ["~", "\u00a0"], ["_", "\u2011"], ["-", ""], @@ -437,7 +495,7 @@ async function* document(path: string, extension: string): AsyncGenerator typeof child === "string") ?? ""; if (cell.attrs.t === "s") { const shared = strings[Number(raw)]; - if (shared) yield* inCell(text(shared)); - } else if (cell.attrs.t === "inlineStr") yield* inCell(text(cell)); + if (shared) yield* inCell(xstringText(shared)); + } else if (cell.attrs.t === "inlineStr") yield* inCell(xstringText(cell)); + else if (cell.attrs.t === "b" && (raw.trim() === "0" || raw.trim() === "1")) + yield* inCell([raw.trim() === "1" ? "true" : "false"]); else { - if (value && hasCellText(value)) yield* inCell(text(value)); + if (value && hasCellText(value)) yield* inCell(xstringText(value)); else for (const formula of all(cell, "f")) yield* inCell(text(formula)); const formatId = formats[Number(cell.attrs.s ?? 0)]?.attrs.numFmtId; const format = formatId ? customFormats.get(formatId) : undefined; diff --git a/apps/server/src/provider/model-manifest.json b/apps/server/src/provider/model-manifest.json index 4906358bab70b..01727cc2680b0 100644 --- a/apps/server/src/provider/model-manifest.json +++ b/apps/server/src/provider/model-manifest.json @@ -1,6 +1,6 @@ { "version": 1, - "updatedAt": "2026-11-12T18:25:36Z", + "updatedAt": "2026-11-14T15:58:47Z", "compatibility": [ { "driver": "codex", @@ -95,8 +95,8 @@ "t3CodeRange": ">=0.0.46", "recommendedRange": ">=1.0.0", "ranges": [ - { "range": ">=0.80.5", "status": "supported" }, - { "range": "<0.80.5", "status": "unsupported" } + { "range": ">=0.84.3", "status": "supported" }, + { "range": "<0.84.3", "status": "unsupported" } ] }, { @@ -771,7 +771,7 @@ "aliases": ["fable", "fable-5.2", "claude-fable-5.2"], "status": "current", "profile": "fable-5", - "adapter": { "claudeCode": { "minVersion": "2.1.290" } } + "adapter": { "claudeCode": { "minVersion": "2.1.321" } } }, { "slug": "claude-fable-5-1", diff --git a/apps/server/src/provider/providerCompatibility.test.ts b/apps/server/src/provider/providerCompatibility.test.ts index 1c599113a2091..ed722b7c53515 100644 --- a/apps/server/src/provider/providerCompatibility.test.ts +++ b/apps/server/src/provider/providerCompatibility.test.ts @@ -325,5 +325,5 @@ it.effect("a remote policy refresh preserves a newer health result on the regist ), ), ); - }).pipe(Effect.scoped), + }), ); diff --git a/apps/server/src/provider/testFixtures/grok-text-mock-agent.mjs b/apps/server/src/provider/testFixtures/grok-text-mock-agent.mjs index 357bdb7fe564d..b8cb949a5fb71 100644 --- a/apps/server/src/provider/testFixtures/grok-text-mock-agent.mjs +++ b/apps/server/src/provider/testFixtures/grok-text-mock-agent.mjs @@ -5,12 +5,35 @@ const sessionId = "grok-text-session"; let currentModelId = "grok-mock-default"; const models = ["grok-mock-default", "grok-mock-alt"]; const write = (message) => process.stdout.write(`${JSON.stringify(message)}\n`); +const permissionRequestId = 9001; +let pendingPromptId; +const completePrompt = (id) => { + write({ + jsonrpc: "2.0", + method: "session/update", + params: { + sessionId, + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: process.env.T3_ACP_PROMPT_RESPONSE_TEXT ?? "" }, + }, + }, + }); + write({ jsonrpc: "2.0", id, result: { stopReason: "end_turn" } }); +}; for await (const line of NodeReadline.createInterface({ input: process.stdin })) { const request = JSON.parse(line); if (process.env.T3_ACP_REQUEST_LOG_PATH) { NodeFS.appendFileSync(process.env.T3_ACP_REQUEST_LOG_PATH, `${line}\n`); } + if (request.method === undefined) { + if (request.id === permissionRequestId && pendingPromptId !== undefined) { + completePrompt(pendingPromptId); + pendingPromptId = undefined; + } + continue; + } if (request.id === undefined) continue; let result; switch (request.method) { @@ -43,19 +66,25 @@ for await (const line of NodeReadline.createInterface({ input: process.stdin })) result = {}; break; case "session/prompt": - write({ - jsonrpc: "2.0", - method: "session/update", - params: { - sessionId, - update: { - sessionUpdate: "agent_message_chunk", - content: { type: "text", text: process.env.T3_ACP_PROMPT_RESPONSE_TEXT ?? "" }, + if (process.env.T3_ACP_REQUEST_PERMISSION) { + pendingPromptId = request.id; + write({ + jsonrpc: "2.0", + id: permissionRequestId, + method: "session/request_permission", + params: { + sessionId, + toolCall: { toolCallId: "tool-1", kind: "execute", title: "Run `git status`" }, + options: [ + { optionId: "allow-once", name: "Yes", kind: "allow_once" }, + { optionId: "reject-once", name: "No", kind: "reject_once" }, + ], }, - }, - }); - result = { stopReason: "end_turn" }; - break; + }); + continue; + } + completePrompt(request.id); + continue; default: write({ jsonrpc: "2.0", diff --git a/apps/server/src/provider/userInputAttachments.test.ts b/apps/server/src/provider/userInputAttachments.test.ts index 056f2b7bd97f5..0911f94a6a6aa 100644 --- a/apps/server/src/provider/userInputAttachments.test.ts +++ b/apps/server/src/provider/userInputAttachments.test.ts @@ -285,6 +285,11 @@ describe("question answer paths", () => { String.raw`{\rtf1\ansi A\enspace B\emspace C\qmspace D\~E\_F\-G}`, "A\u2002B\u2003C\u2005D\u00a0E\u2011FG", ], + [ + "joiner and direction marks", + String.raw`{\rtf1\ansi A\zwj B\zwnj C\ltrmark D\rtlmark E}`, + "A\u200dB\u200cC\u200eD\u200fE", + ], [ "text symbol fallback", String.raw`{\rtf1\ansi\uc10\u945\endash\emdash\enspace\emspace\qmspace\lquote\rquote\ldblquote\rdblquote\bullet Z}`, @@ -690,28 +695,32 @@ describe("question answer paths", () => { bounded: true, parts: { "word/document.xml": - 'Report & totalsNorth1234BeforeInnerAfterNeighbor', + 'Report & totalsNorth1234BeforeInnerAfterNeighbor', }, - expected: ["## Report & totals", "North\t12 34", "Before Inner After\tNeighbor"], + expected: [ + "## Report & totals", + "North\t12 34", + "Before Inner After\tNeighbor\t\u2011\n\u2011", + ], }, { ext: "docx", bounded: true, parts: { "word/document.xml": - 'FirstSecondThird', + 'FirstSecondThirdparty‑End', }, - expected: ["First\nSecond\nThird"], + expected: ["First\nSecond\nThird\u2011party\u2011End"], }, { ext: "docx", bounded: true, parts: { "word/document.xml": - 'Intro PAGEREF _Toc1 \\h 3Fee $500$700Moveddue', + 'Intro PAGEREF _Toc1 \\h 3Fee $500$700Moveddue', }, expected: ["Intro 3\nFee $700\tdue"], - absent: ["PAGEREF", "$500", "Moved"], + absent: ["PAGEREF", "$500", "Moved", "\u2011"], }, { ext: "odt", @@ -748,6 +757,18 @@ describe("question answer paths", () => { ], absent: ["SUM(0,0)", "[Extracted text truncated"], }, + { + ext: "xlsx", + bounded: true, + parts: { + "xl/workbook.xml": '', + "xl/_rels/workbook.xml.rels": + '', + "xl/worksheets/a.xml": + '1012', + }, + expected: ["true\tfalse\t1\t2"], + }, { ext: "xlsx", parts: { @@ -788,6 +809,22 @@ describe("question answer paths", () => { expected: ["## Sheet: Phonetics", "東京\t東京"], absent: ["とうきょう"], }, + { + ext: "xlsx", + bounded: true, + parts: { + "xl/workbook.xml": '', + "xl/_rels/workbook.xml.rels": + '', + "xl/sharedStrings.xml": + "first_x000D_\nsecond_x005F_x0041__x005F_x005F_x0041_Bold_x000D__x005F_x000D_&_xnothex_ cap_X0041_ _x000d_ tail", + "xl/worksheets/a.xml": + '01234_x005F_x005F_first_x000D_secondA1Total_x000D_due', + }, + expected: [ + "## Sheet: Escapes\nfirst second\t_x0041_\t_x005F_x0041_\tBold _x000D_\t&_xnothex_ cap_X0041_ tail\n_x005F_\tfirst second\tTotal due", + ], + }, { ext: "xlsx", bounded: true, @@ -891,9 +928,11 @@ describe("question answer paths", () => { ext: "odt", parts: { "content.xml": - 'Orchard19violetcrates', + '\n Orchard\n19violetcrates \n\t shipped todayDockGate\u00a0B SecondAconsole.log(1);\n B', }, - expected: ["## Orchard", "19 violet crates"], + expected: [ + "## Orchard\n19 violet crates shipped today\tDock\nGate\u00a0B\nSecond\nA B", + ], }, { ext: "ods", @@ -940,9 +979,9 @@ describe("question answer paths", () => { ext: "odp", parts: { "content.xml": - "FirstSecond", + "\n First\n\t slide Second", }, - expected: ["## Slide 1", "First", "## Slide 2", "Second"], + expected: ["## Slide 1\nFirst slide\n\n## Slide 2\nSecond"], }, { ext: "ods", @@ -1037,7 +1076,7 @@ describe("question answer paths", () => { "OEBPS/book.opf": '', "OEBPS/a.xhtml": - '

Last chapter

Price 10 8

', + '

Last chapter

Price 10 8 old

', "OEBPS/b.xhtml": "

First chapter

Larkspur

Direct div
Direct section
  • Direct item emphasis trailing
  • Account: A
    Balance: 731Currency: USD
    HeadingTotal
    Important value731
    Before
    Inner
    After
    Neighbor
    ", }, @@ -1054,7 +1093,7 @@ describe("question answer paths", () => { "Before Inner After\tNeighbor", "## Chapter 2", "Last chapter", - "Price 10 8", + "Price 10 8 old", ], }, ]; diff --git a/apps/server/src/pullRequest/PullRequestReadCache.test.ts b/apps/server/src/pullRequest/PullRequestReadCache.test.ts index 071f1d4978a7d..99b3cdba22ff4 100644 --- a/apps/server/src/pullRequest/PullRequestReadCache.test.ts +++ b/apps/server/src/pullRequest/PullRequestReadCache.test.ts @@ -1,5 +1,4 @@ import * as NodeV8 from "node:v8"; -import * as NodeVM from "node:vm"; import { assert, it } from "@effect/vitest"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; @@ -16,9 +15,6 @@ import * as KeyValueStore from "effect/persistence/KeyValueStore"; import * as TestClock from "effect/testing/TestClock"; import * as PullRequestReadCache from "./PullRequestReadCache.ts"; -NodeV8.setFlagsFromString("--expose-gc"); -const collectGarbage = NodeVM.runInNewContext("gc") as () => void; - const cacheLayer = (directory: string) => PullRequestReadCache.make.pipe(Effect.provide(KeyValueStore.layerFileSystem(directory))); @@ -203,13 +199,10 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("PR filesystem ca it.effect("does not keep a caller's data alive after its reads and invalidations", () => Effect.gen(function* () { const cache = yield* PullRequestReadCache.make; - const sweeps: Array> = []; + class SweepData extends Map {} // Like a sync sweep: an Effect.fn made per run that closes over that run's data. const sweep = Effect.fn("sweep")(function* (index: number) { - const groups = new Map( - Array.from({ length: 1_000 }, (_, id) => [`group-${id}`, id]), - ); - sweeps.push(new WeakRef(groups)); + const groups = new SweepData(Array.from({ length: 1_000 }, (_, id) => [`group-${id}`, id])); const read = Effect.fn("read")(function* (pr: string) { const size = groups.size; yield* cache.invalidate(pr); @@ -219,10 +212,8 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("PR filesystem ca yield* Effect.forEach(["pr-1", "pr-2"], read, { discard: true }); }); for (let index = 0; index < 5; index++) yield* sweep(index); - // WeakRef targets stay alive until the current job ends, so yield before collecting. - yield* Effect.yieldNow; - collectGarbage(); - assert.strictEqual(sweeps.filter((sweepData) => sweepData.deref() !== undefined).length, 0); + // Counts after a full GC. A WeakRef would keep its target alive until the job ends. + assert.strictEqual(NodeV8.queryObjects(SweepData, { format: "count" }), 0); }).pipe(Effect.provide(KeyValueStore.layerMemory)), ); diff --git a/apps/server/src/pullRequest/PullRequestReviewGuideService.test.ts b/apps/server/src/pullRequest/PullRequestReviewGuideService.test.ts index 3ee929f923a57..200ea36b7d64d 100644 --- a/apps/server/src/pullRequest/PullRequestReviewGuideService.test.ts +++ b/apps/server/src/pullRequest/PullRequestReviewGuideService.test.ts @@ -390,208 +390,198 @@ it.layer(layerBase, { excludeTestServices: true })("PullRequestReviewGuideServic expected: ["local objects:", "the commit:", "branch feature:", "refs/pull/7/head:"], }, ])("refuses to write a guide for $name", ({ reportedHead, shadow, expected }) => - Effect.scoped( - Effect.gen(function* () { - const run = yield* startSingleBranchRun({ - repositoryName: "refused-head", - reportedHead, - refuseCommitFetch: false, - prepare: (repositories, workspace) => - git(workspace, ["fetch", "--quiet", "origin", "feature"]).pipe( - Effect.andThen( - shadow === "none" - ? Effect.void - : git(workspace, [shadow, repositories.headSha.slice(0, 12), "HEAD"]), - ), - Effect.asVoid, + Effect.gen(function* () { + const run = yield* startSingleBranchRun({ + repositoryName: "refused-head", + reportedHead, + refuseCommitFetch: false, + prepare: (repositories, workspace) => + git(workspace, ["fetch", "--quiet", "origin", "feature"]).pipe( + Effect.andThen( + shadow === "none" + ? Effect.void + : git(workspace, [shadow, repositories.headSha.slice(0, 12), "HEAD"]), ), - }); - const result = yield* run.settled; - assert.isFalse(yield* Deferred.isDone(run.stored), "a guide was written"); - assert.strictEqual(result.run?._tag, "Failed"); - const detail = result.run?._tag === "Failed" ? result.run.detail : ""; - for (const part of expected) assert.include(detail, part); - // Only the host's own head ref is tried. - assert.notInclude(detail, "refs/merge-requests/"); - assert.deepStrictEqual(run.observed, []); - assert.isFalse(run.fetchedRefs.some((ref) => ref.startsWith("-"))); - const fs = yield* FileSystem.FileSystem; - assert.isFalse(yield* fs.exists(`${run.repositories.root}/injected`)); - }), - ), + Effect.asVoid, + ), + }); + const result = yield* run.settled; + assert.isFalse(yield* Deferred.isDone(run.stored), "a guide was written"); + assert.strictEqual(result.run?._tag, "Failed"); + const detail = result.run?._tag === "Failed" ? result.run.detail : ""; + for (const part of expected) assert.include(detail, part); + // Only the host's own head ref is tried. + assert.notInclude(detail, "refs/merge-requests/"); + assert.deepStrictEqual(run.observed, []); + assert.isFalse(run.fetchedRefs.some((ref) => ref.startsWith("-"))); + const fs = yield* FileSystem.FileSystem; + assert.isFalse(yield* fs.exists(`${run.repositories.root}/injected`)); + }), ); it.effect("measures only the PR change using the renamed verified remote's fetch URL", () => - Effect.scoped( - Effect.gen(function* () { - const repositories = yield* makeRepositories; - const fork = `${repositories.root}/fork`; - yield* git(repositories.root, ["clone", "--quiet", repositories.remote, fork]); - yield* git(fork, ["branch", "stack-base", "main~1"]); - // The cached identity still names origin, which now belongs to the stale fork. - yield* git(repositories.workspace, ["remote", "rename", "origin", "upstream"]); - yield* git(repositories.workspace, ["remote", "add", "origin", fork]); - yield* git(repositories.workspace, [ - "config", - `url.${repositories.remote}.insteadOf`, - "review-upstream:", - ]); - yield* git(repositories.workspace, ["remote", "set-url", "upstream", "review-upstream:"]); - const changed = yield* Deferred.make(); - const driver = yield* GitVcsDriver.GitVcsDriver; - const stored = yield* Deferred.make(); - const context = yield* Layer.build( - makeLayer({ - ...repositories, - stored, - generate: (input) => - git(input.cwd, ["diff", "--name-only", `${input.baseCommit}...HEAD`]).pipe( - Effect.provideService(GitVcsDriver.GitVcsDriver, driver), - Effect.orDie, - Effect.flatMap((names) => Deferred.succeed(changed, names)), - Effect.as({ stops: [] }), - ), - }), - ); - const service = Context.get( - context, - PullRequestReviewGuideService.PullRequestReviewGuideService, - ); - yield* service.generate(reference); - assert.strictEqual(yield* Deferred.await(changed), "feature.txt"); - yield* Deferred.await(stored); - const result = yield* service.get(reference); - assert.strictEqual(result.guide?.headSha, repositories.headSha); - assert.deepStrictEqual(result.guide?.stops[0]?.files, [{ path: "feature.txt" }]); - }), - ), + Effect.gen(function* () { + const repositories = yield* makeRepositories; + const fork = `${repositories.root}/fork`; + yield* git(repositories.root, ["clone", "--quiet", repositories.remote, fork]); + yield* git(fork, ["branch", "stack-base", "main~1"]); + // The cached identity still names origin, which now belongs to the stale fork. + yield* git(repositories.workspace, ["remote", "rename", "origin", "upstream"]); + yield* git(repositories.workspace, ["remote", "add", "origin", fork]); + yield* git(repositories.workspace, [ + "config", + `url.${repositories.remote}.insteadOf`, + "review-upstream:", + ]); + yield* git(repositories.workspace, ["remote", "set-url", "upstream", "review-upstream:"]); + const changed = yield* Deferred.make(); + const driver = yield* GitVcsDriver.GitVcsDriver; + const stored = yield* Deferred.make(); + const context = yield* Layer.build( + makeLayer({ + ...repositories, + stored, + generate: (input) => + git(input.cwd, ["diff", "--name-only", `${input.baseCommit}...HEAD`]).pipe( + Effect.provideService(GitVcsDriver.GitVcsDriver, driver), + Effect.orDie, + Effect.flatMap((names) => Deferred.succeed(changed, names)), + Effect.as({ stops: [] }), + ), + }), + ); + const service = Context.get( + context, + PullRequestReviewGuideService.PullRequestReviewGuideService, + ); + yield* service.generate(reference); + assert.strictEqual(yield* Deferred.await(changed), "feature.txt"); + yield* Deferred.await(stored); + const result = yield* service.get(reference); + assert.strictEqual(result.guide?.headSha, repositories.headSha); + assert.deepStrictEqual(result.guide?.stops[0]?.files, [{ path: "feature.txt" }]); + }), ); it.effect("keeps excerpts despite the checkout's diff driver and a failing hook", () => - Effect.scoped( - Effect.gen(function* () { - const repositories = yield* makeRepositories; - const fs = yield* FileSystem.FileSystem; - yield* fs.writeFileString( - `${repositories.workspace}/.git/info/attributes`, - "feature.txt diff=blank\n", - ); - yield* git(repositories.workspace, ["config", "diff.blank.textconv", "true"]); - const hook = `${repositories.workspace}/.git/hooks/post-checkout`; - yield* fs.writeFileString(hook, "#!/bin/sh\nexit 1\n"); - yield* fs.chmod(hook, 0o755); - const stored = yield* Deferred.make(); - const context = yield* Layer.build( - makeLayer({ - ...repositories, - stored, - generate: () => - Effect.succeed({ - stops: [ - { - title: "The feature", - summary: "The change.", - attention: null, - excerpts: [{ path: "feature.txt", side: "new", start: 1, end: 3, caption: "" }], - files: [], - notes: [], - questions: [], - }, - ], - }), - }), - ); - const service = Context.get( - context, - PullRequestReviewGuideService.PullRequestReviewGuideService, - ); - const runs = yield* watchRuns; - yield* runs.generate(service, reference); - const result = yield* runs.settled(service, reference); - assert.notStrictEqual(result.run?._tag, "Failed", JSON.stringify(result.run)); - assert.deepStrictEqual(result.guide?.stops[0]?.excerpts, [ - { path: "feature.txt", side: "new", start: 1, end: 3, caption: "" }, - ]); - }), - ), + Effect.gen(function* () { + const repositories = yield* makeRepositories; + const fs = yield* FileSystem.FileSystem; + yield* fs.writeFileString( + `${repositories.workspace}/.git/info/attributes`, + "feature.txt diff=blank\n", + ); + yield* git(repositories.workspace, ["config", "diff.blank.textconv", "true"]); + const hook = `${repositories.workspace}/.git/hooks/post-checkout`; + yield* fs.writeFileString(hook, "#!/bin/sh\nexit 1\n"); + yield* fs.chmod(hook, 0o755); + const stored = yield* Deferred.make(); + const context = yield* Layer.build( + makeLayer({ + ...repositories, + stored, + generate: () => + Effect.succeed({ + stops: [ + { + title: "The feature", + summary: "The change.", + attention: null, + excerpts: [{ path: "feature.txt", side: "new", start: 1, end: 3, caption: "" }], + files: [], + notes: [], + questions: [], + }, + ], + }), + }), + ); + const service = Context.get( + context, + PullRequestReviewGuideService.PullRequestReviewGuideService, + ); + const runs = yield* watchRuns; + yield* runs.generate(service, reference); + const result = yield* runs.settled(service, reference); + assert.notStrictEqual(result.run?._tag, "Failed", JSON.stringify(result.run)); + assert.deepStrictEqual(result.guide?.stops[0]?.excerpts, [ + { path: "feature.txt", side: "new", start: 1, end: 3, caption: "" }, + ]); + }), ); it.effect("refuses generation when the verified remote disappeared despite a usable origin", () => - Effect.scoped( - Effect.gen(function* () { - const repositories = yield* makeRepositories; - const fork = `${repositories.root}/fork`; - yield* git(repositories.root, ["clone", "--quiet", repositories.remote, fork]); - yield* git(fork, ["branch", "stack-base", repositories.lowerSha]); - yield* git(repositories.workspace, ["remote", "set-url", "origin", fork]); - const started = yield* Deferred.make>(); - const stored = yield* Deferred.make(); - const generated = yield* Ref.make(false); - const driver = yield* GitVcsDriver.GitVcsDriver; - const context = yield* Layer.build( - makeLayer({ - ...repositories, - stored, - generate: () => Ref.set(generated, true).pipe(Effect.as({ stops: [] })), - }).pipe( - Layer.provide( - Layer.succeed(GitVcsDriver.GitVcsDriver, { - ...driver, - execute: (call) => - Effect.withFiber((fiber) => - Deferred.succeed(started, fiber).pipe(Effect.andThen(driver.execute(call))), - ), - }), - ), + Effect.gen(function* () { + const repositories = yield* makeRepositories; + const fork = `${repositories.root}/fork`; + yield* git(repositories.root, ["clone", "--quiet", repositories.remote, fork]); + yield* git(fork, ["branch", "stack-base", repositories.lowerSha]); + yield* git(repositories.workspace, ["remote", "set-url", "origin", fork]); + const started = yield* Deferred.make>(); + const stored = yield* Deferred.make(); + const generated = yield* Ref.make(false); + const driver = yield* GitVcsDriver.GitVcsDriver; + const context = yield* Layer.build( + makeLayer({ + ...repositories, + stored, + generate: () => Ref.set(generated, true).pipe(Effect.as({ stops: [] })), + }).pipe( + Layer.provide( + Layer.succeed(GitVcsDriver.GitVcsDriver, { + ...driver, + execute: (call) => + Effect.withFiber((fiber) => + Deferred.succeed(started, fiber).pipe(Effect.andThen(driver.execute(call))), + ), + }), ), - ); - const service = Context.get( - context, - PullRequestReviewGuideService.PullRequestReviewGuideService, - ); - const before = yield* service.get(reference); - yield* service.generate(reference); - // Join the background run so its failure is visible without polling service state. - yield* Fiber.await(yield* Deferred.await(started)); - const result = yield* service.get(reference); - assert.strictEqual(result.run?._tag, "Failed"); - assert.include( - result.run?._tag === "Failed" ? result.run.detail : "", - "no longer has a remote for the pull request's repository", - ); - assert.deepStrictEqual(result.guide, before.guide); - assert.isFalse(yield* Deferred.isDone(stored)); - assert.isFalse(yield* Ref.get(generated)); - }), - ), + ), + ); + const service = Context.get( + context, + PullRequestReviewGuideService.PullRequestReviewGuideService, + ); + const before = yield* service.get(reference); + yield* service.generate(reference); + // Join the background run so its failure is visible without polling service state. + yield* Fiber.await(yield* Deferred.await(started)); + const result = yield* service.get(reference); + assert.strictEqual(result.run?._tag, "Failed"); + assert.include( + result.run?._tag === "Failed" ? result.run.detail : "", + "no longer has a remote for the pull request's repository", + ); + assert.deepStrictEqual(result.guide, before.guide); + assert.isFalse(yield* Deferred.isDone(stored)); + assert.isFalse(yield* Ref.get(generated)); + }), ); it.effect("refuses a pull request from a repository the project is not a checkout of", () => - Effect.scoped( - Effect.gen(function* () { - const repositories = yield* makeRepositories; - const stored = yield* Deferred.make(); - const context = yield* Layer.build( - makeLayer({ - ...repositories, - stored, - identity: { - ...ACME_APP, - canonicalKey: "github.com/acme/other", - displayName: "acme/other", - name: "other", - }, - generate: () => - Effect.die("a guide must not be written from another repository's checkout"), - }), - ); - const service = Context.get( - context, - PullRequestReviewGuideService.PullRequestReviewGuideService, - ); - const result = yield* Effect.flip(service.generate(reference)); - assert.include(result.message, "in another repository"); - }), - ), + Effect.gen(function* () { + const repositories = yield* makeRepositories; + const stored = yield* Deferred.make(); + const context = yield* Layer.build( + makeLayer({ + ...repositories, + stored, + identity: { + ...ACME_APP, + canonicalKey: "github.com/acme/other", + displayName: "acme/other", + name: "other", + }, + generate: () => + Effect.die("a guide must not be written from another repository's checkout"), + }), + ); + const service = Context.get( + context, + PullRequestReviewGuideService.PullRequestReviewGuideService, + ); + const result = yield* Effect.flip(service.generate(reference)); + assert.include(result.message, "in another repository"); + }), ); }); diff --git a/apps/server/src/pullRequest/PullRequestService.test.ts b/apps/server/src/pullRequest/PullRequestService.test.ts index 89a3ab7cd56c4..74106b14cd403 100644 --- a/apps/server/src/pullRequest/PullRequestService.test.ts +++ b/apps/server/src/pullRequest/PullRequestService.test.ts @@ -1429,174 +1429,160 @@ it.effect( ); it.effect("publishes a merge for immediate settlement only after host confirmation", () => - Effect.scoped( - Effect.gen(function* () { - const mergedAt = "2026-09-03T02:00:00.000Z"; - let state: "open" | "merged" = "open"; - let confirmationFails = false; - const reference = { projectId: "p1" as ProjectId, repository: "acme/web", number: 1 }; - const service = yield* makeService({ - projects: [ - project({ id: "p1", title: "web", workspaceRoot: "/a", repository: "acme/web" }), - ], - providers: [ - fakePullRequestProvider(SourceControlProviderKind.make("github"), { - getChangeRequestSummary: () => - confirmationFails - ? Effect.fail( - new PullRequestProviderError({ - provider: SourceControlProviderKind.make("github"), - operation: "getChangeRequestSummary", - reason: "failed", - detail: "HTTP 504", - }), - ) - : Effect.succeed({ ...changeRequest(1, mergedAt), state, mergedAt }), - }), - ], - }); - const merges = yield* service.subscribeMerges; - const observedMerge = yield* Stream.runHead(merges).pipe( - Effect.forkChild({ startImmediately: true }), - ); + Effect.gen(function* () { + const mergedAt = "2026-09-03T02:00:00.000Z"; + let state: "open" | "merged" = "open"; + let confirmationFails = false; + const reference = { projectId: "p1" as ProjectId, repository: "acme/web", number: 1 }; + const service = yield* makeService({ + projects: [project({ id: "p1", title: "web", workspaceRoot: "/a", repository: "acme/web" })], + providers: [ + fakePullRequestProvider(SourceControlProviderKind.make("github"), { + getChangeRequestSummary: () => + confirmationFails + ? Effect.fail( + new PullRequestProviderError({ + provider: SourceControlProviderKind.make("github"), + operation: "getChangeRequestSummary", + reason: "failed", + detail: "HTTP 504", + }), + ) + : Effect.succeed({ ...changeRequest(1, mergedAt), state, mergedAt }), + }), + ], + }); + const merges = yield* service.subscribeMerges; + const observedMerge = yield* Stream.runHead(merges).pipe( + Effect.forkChild({ startImmediately: true }), + ); - assert.isUndefined(yield* service.runAction({ ...reference, action: "merge" })); - const queuedRefresh = Option.getOrThrow(yield* Stream.runHead(service.subscribeRefreshes)); - confirmationFails = true; - yield* service.runAction({ ...reference, action: "merge" }); - assert.isAbove( - Option.getOrThrow(yield* Stream.runHead(service.subscribeRefreshes)), - queuedRefresh, - ); - confirmationFails = false; - state = "merged"; - // The event carries the host's merge time, not the moment the confirmation ran. - yield* TestClock.setTime(Date.parse(mergedAt) + 60_000); - yield* service.runAction({ - ...reference, - repository: " ACME/WEB ", - action: "merge", - mergeMethod: "merge", - }); + assert.isUndefined(yield* service.runAction({ ...reference, action: "merge" })); + const queuedRefresh = Option.getOrThrow(yield* Stream.runHead(service.subscribeRefreshes)); + confirmationFails = true; + yield* service.runAction({ ...reference, action: "merge" }); + assert.isAbove( + Option.getOrThrow(yield* Stream.runHead(service.subscribeRefreshes)), + queuedRefresh, + ); + confirmationFails = false; + state = "merged"; + // The event carries the host's merge time, not the moment the confirmation ran. + yield* TestClock.setTime(Date.parse(mergedAt) + 60_000); + yield* service.runAction({ + ...reference, + repository: " ACME/WEB ", + action: "merge", + mergeMethod: "merge", + }); - assert.deepStrictEqual(Option.getOrThrow(yield* Fiber.join(observedMerge)), { - ...reference, - mergedAt, - }); - }), - ), + assert.deepStrictEqual(Option.getOrThrow(yield* Fiber.join(observedMerge)), { + ...reference, + mergedAt, + }); + }), ); it.effect("returns the host's merged reply as a summary and trusts it without another read", () => - Effect.scoped( - Effect.gen(function* () { - const mergedAt = "2026-09-03T02:00:00.000Z"; - const merged = { - number: 1, - title: "Change request 1", - url: "https://host/pull/1", - headBranch: "feat/1", - baseBranch: "main", - state: "merged" as const, - isDraft: false, - updatedAt: mergedAt, - mergedAt, - headSha: "abc123", - autoMergeEnabled: false, - }; - let summaryReads = 0; - const reference = { projectId: "p1" as ProjectId, repository: "acme/web", number: 1 }; - const service = yield* makeService({ - projects: [ - project({ id: "p1", title: "web", workspaceRoot: "/a", repository: "acme/web" }), - ], - providers: [ - fakePullRequestProvider(SourceControlProviderKind.make("github"), { - runAction: () => Effect.succeed(merged), - getChangeRequestSummary: () => - Effect.sync(() => { - summaryReads++; - return { ...changeRequest(1, mergedAt), state: "merged" as const, mergedAt }; - }), - }), - ], - }); - const merges = yield* service.subscribeMerges; - const observedMerge = yield* Stream.runHead(merges).pipe( - Effect.forkChild({ startImmediately: true }), - ); - const observedAt = Date.parse("2026-09-03T02:00:01.000Z"); - yield* TestClock.setTime(observedAt); - - // Clients that did not ask decode the success as void, so they must get nothing back. - assert.isUndefined(yield* service.runAction({ ...reference, action: "merge" })); - const result = yield* service.runAction({ - ...reference, - action: "merge", - replyWithSummary: true, - }); + Effect.gen(function* () { + const mergedAt = "2026-09-03T02:00:00.000Z"; + const merged = { + number: 1, + title: "Change request 1", + url: "https://host/pull/1", + headBranch: "feat/1", + baseBranch: "main", + state: "merged" as const, + isDraft: false, + updatedAt: mergedAt, + mergedAt, + headSha: "abc123", + autoMergeEnabled: false, + }; + let summaryReads = 0; + const reference = { projectId: "p1" as ProjectId, repository: "acme/web", number: 1 }; + const service = yield* makeService({ + projects: [project({ id: "p1", title: "web", workspaceRoot: "/a", repository: "acme/web" })], + providers: [ + fakePullRequestProvider(SourceControlProviderKind.make("github"), { + runAction: () => Effect.succeed(merged), + getChangeRequestSummary: () => + Effect.sync(() => { + summaryReads++; + return { ...changeRequest(1, mergedAt), state: "merged" as const, mergedAt }; + }), + }), + ], + }); + const merges = yield* service.subscribeMerges; + const observedMerge = yield* Stream.runHead(merges).pipe( + Effect.forkChild({ startImmediately: true }), + ); + const observedAt = Date.parse("2026-09-03T02:00:01.000Z"); + yield* TestClock.setTime(observedAt); - assert.deepStrictEqual(result, { - ...merged, - provider: SourceControlProviderKind.make("github"), - projectId: reference.projectId, - repository: reference.repository, - observedAt, - }); - assert.strictEqual(summaryReads, 0); - assert.deepStrictEqual(Option.getOrThrow(yield* Fiber.join(observedMerge)), { - ...reference, - mergedAt, - }); - }), - ), + // Clients that did not ask decode the success as void, so they must get nothing back. + assert.isUndefined(yield* service.runAction({ ...reference, action: "merge" })); + const result = yield* service.runAction({ + ...reference, + action: "merge", + replyWithSummary: true, + }); + + assert.deepStrictEqual(result, { + ...merged, + provider: SourceControlProviderKind.make("github"), + projectId: reference.projectId, + repository: reference.repository, + observedAt, + }); + assert.strictEqual(summaryReads, 0); + assert.deepStrictEqual(Option.getOrThrow(yield* Fiber.join(observedMerge)), { + ...reference, + mergedAt, + }); + }), ); it.effect("refreshes every reader before a queued merge confirmation finishes", () => - Effect.scoped( - Effect.gen(function* () { - const confirmationStarted = yield* Deferred.make(); - const confirm = yield* Deferred.make(); - const reference = { projectId: "p1" as ProjectId, repository: "acme/web", number: 1 }; - const service = yield* makeService({ - projects: [ - project({ id: "p1", title: "web", workspaceRoot: "/a", repository: "acme/web" }), - ], - providers: [ - fakePullRequestProvider(SourceControlProviderKind.make("github"), { - getChangeRequestSummary: () => - Effect.gen(function* () { - yield* Deferred.succeed(confirmationStarted, undefined); - yield* Deferred.await(confirm); - return changeRequest(1, "2026-09-16T00:00:00.000Z"); - }), - }), - ], - }); - const merges = yield* service.subscribeMerges; - const observedMerge = yield* Stream.runHead(merges).pipe( - Effect.forkChild({ startImmediately: true }), - ); - const readers = yield* Effect.forEach([0, 1], () => - Stream.runHead(service.subscribeRefreshes).pipe( - Effect.forkChild({ startImmediately: true }), - ), - ); - const action = yield* service - .runAction({ ...reference, action: "merge" }) - .pipe(Effect.forkChild({ startImmediately: true })); - yield* Deferred.await(confirmationStarted); - const revisions = yield* Effect.forEach(readers, (reader) => - Fiber.join(reader).pipe(Effect.map(Option.getOrThrow)), - ); - assert.isAbove(revisions[0]!, 0); - assert.strictEqual(revisions[0], revisions[1]); - assert.isUndefined(action.pollUnsafe()); - yield* Deferred.succeed(confirm, undefined); - yield* Fiber.join(action); - assert.isUndefined(observedMerge.pollUnsafe()); - }), - ), + Effect.gen(function* () { + const confirmationStarted = yield* Deferred.make(); + const confirm = yield* Deferred.make(); + const reference = { projectId: "p1" as ProjectId, repository: "acme/web", number: 1 }; + const service = yield* makeService({ + projects: [project({ id: "p1", title: "web", workspaceRoot: "/a", repository: "acme/web" })], + providers: [ + fakePullRequestProvider(SourceControlProviderKind.make("github"), { + getChangeRequestSummary: () => + Effect.gen(function* () { + yield* Deferred.succeed(confirmationStarted, undefined); + yield* Deferred.await(confirm); + return changeRequest(1, "2026-09-16T00:00:00.000Z"); + }), + }), + ], + }); + const merges = yield* service.subscribeMerges; + const observedMerge = yield* Stream.runHead(merges).pipe( + Effect.forkChild({ startImmediately: true }), + ); + const readers = yield* Effect.forEach([0, 1], () => + Stream.runHead(service.subscribeRefreshes).pipe(Effect.forkChild({ startImmediately: true })), + ); + const action = yield* service + .runAction({ ...reference, action: "merge" }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(confirmationStarted); + const revisions = yield* Effect.forEach(readers, (reader) => + Fiber.join(reader).pipe(Effect.map(Option.getOrThrow)), + ); + assert.isAbove(revisions[0]!, 0); + assert.strictEqual(revisions[0], revisions[1]); + assert.isUndefined(action.pollUnsafe()); + yield* Deferred.succeed(confirm, undefined); + yield* Fiber.join(action); + assert.isUndefined(observedMerge.pollUnsafe()); + }), ); it.effect("runs branch deletion only when the host and viewer support it, never for a stack", () => @@ -2215,6 +2201,72 @@ it.effect("routes a hosted reference to another repository through a project on }), ); +it.effect("an uncached summary reaches the host despite a cached read in another project", () => + Effect.gen(function* () { + const caller = project({ + id: "caller", + title: "caller", + workspaceRoot: "/caller", + repository: "acme/other", + }); + const target = project({ + id: "gitlab", + title: "gitlab", + workspaceRoot: "/gitlab", + repository: "team/project", + provider: "gitlab", + }); + let missing = false; + let reads = 0; + const service = yield* makeService({ + projects: [caller, target], + providers: [ + fakePullRequestProvider(SourceControlProviderKind.make("github")), + fakePullRequestProvider(SourceControlProviderKind.make("gitlab"), { + getChangeRequestSummary: (input) => + Effect.suspend(() => { + reads++; + assert.strictEqual(input.cwd, target.workspaceRoot); + assert.strictEqual(input.repository, "team/project"); + assert.strictEqual(input.host, "gitlab.com"); + return missing + ? Effect.fail( + new PullRequestProviderError({ + provider: SourceControlProviderKind.make("gitlab"), + operation: "summary", + reason: "missing-tool", + detail: "The hosting CLI is unavailable.", + }), + ) + : Effect.succeed(changeRequest(42, "2026-07-02T00:00:00Z")); + }), + }), + ], + }); + const reference = { + projectId: caller.id, + host: "gitlab.com", + repository: "team/project", + number: 42, + allowStale: false, + }; + yield* service.summary(reference, { recoverTransientFailure: false }); + missing = true; + yield* service.summary(reference, { recoverTransientFailure: false }); + assert.strictEqual(reads, 1); + + const error = yield* service + .summary(reference, { recoverTransientFailure: false, bypassCache: true }) + .pipe(Effect.flip); + expect(error).toMatchObject({ + _tag: "PullRequestUnavailableError", + reason: "cli-missing", + provider: "gitlab", + }); + assert.strictEqual(reads, 2); + }), +); + it.effect("routes Azure reads and writes through the requested organization's checkout", () => Effect.gen(function* () { const seen: string[] = []; @@ -2798,56 +2850,52 @@ it.effect("a read during a refresh cannot keep the provider's old change request ); it.effect("close and reopen notify subscribed readers after invalidating their cached state", () => - Effect.scoped( - Effect.gen(function* () { - let hostCalls = 0; - let state: "open" | "closed" = "open"; - const reference = { projectId: "p1" as ProjectId, repository: "acme/web", number: 1 }; - const service = yield* makeService({ - projects: [ - project({ id: "p1", title: "web", workspaceRoot: "/a", repository: "acme/web" }), - ], - providers: [ - fakePullRequestProvider(SourceControlProviderKind.make("github"), { - getChangeRequestSummary: () => - Effect.succeed({ ...changeRequest(1, "2026-09-16T00:00:00.000Z"), state }), - runAction: (input) => - Effect.sync(() => { - state = input.action === "close" ? "closed" : "open"; - }), - listChangeRequests: () => { - hostCalls += 1; - return Effect.succeed({ items: [], truncated: false, continues: false }); - }, - }), - ], - }); - - yield* service.refreshAfterTurn(reference.projectId); - yield* service.list({ state: "open" }); - assert.strictEqual((yield* service.summary(reference)).state, "open"); - for (const action of ["close", "reopen"] as const) { - const refreshed = yield* service.subscribeRefreshes.pipe( - Stream.drop(1), - Stream.take(1), - Stream.mapEffect(() => - Effect.gen(function* () { - yield* service.list({ state: "open" }); - return yield* service.summary(reference); + Effect.gen(function* () { + let hostCalls = 0; + let state: "open" | "closed" = "open"; + const reference = { projectId: "p1" as ProjectId, repository: "acme/web", number: 1 }; + const service = yield* makeService({ + projects: [project({ id: "p1", title: "web", workspaceRoot: "/a", repository: "acme/web" })], + providers: [ + fakePullRequestProvider(SourceControlProviderKind.make("github"), { + getChangeRequestSummary: () => + Effect.succeed({ ...changeRequest(1, "2026-09-16T00:00:00.000Z"), state }), + runAction: (input) => + Effect.sync(() => { + state = input.action === "close" ? "closed" : "open"; }), - ), - Stream.runHead, - Effect.forkChild({ startImmediately: true }), - ); - yield* service.runAction({ ...reference, action }); - assert.strictEqual( - Option.getOrThrow(yield* Fiber.join(refreshed)).state, - action === "close" ? "closed" : "open", - ); - } - assert.strictEqual(hostCalls, 3); - }), - ), + listChangeRequests: () => { + hostCalls += 1; + return Effect.succeed({ items: [], truncated: false, continues: false }); + }, + }), + ], + }); + + yield* service.refreshAfterTurn(reference.projectId); + yield* service.list({ state: "open" }); + assert.strictEqual((yield* service.summary(reference)).state, "open"); + for (const action of ["close", "reopen"] as const) { + const refreshed = yield* service.subscribeRefreshes.pipe( + Stream.drop(1), + Stream.take(1), + Stream.mapEffect(() => + Effect.gen(function* () { + yield* service.list({ state: "open" }); + return yield* service.summary(reference); + }), + ), + Stream.runHead, + Effect.forkChild({ startImmediately: true }), + ); + yield* service.runAction({ ...reference, action }); + assert.strictEqual( + Option.getOrThrow(yield* Fiber.join(refreshed)).state, + action === "close" ? "closed" : "open", + ); + } + assert.strictEqual(hostCalls, 3); + }), ); it.effect("reads a host's repositories in one search, and files the rows back under each", () => @@ -2969,7 +3017,13 @@ it.effect("carries a slice on without sending the rows it already sent", () => cursors: { "github.com acme/web": "2026-07-02T00:00:00Z|1|3" }, }); - assert.deepStrictEqual(cursors, [{ updatedBefore: "2026-07-02T00:00:00Z", delivered: 1 }]); + assert.deepStrictEqual(cursors, [ + { + updatedBefore: "2026-07-02T00:00:00Z", + delivered: 1, + seenAt: [{ repository: "acme/web", number: 3 }], + }, + ]); assert.deepStrictEqual( result.entries.map((entry) => entry.number), [4], @@ -3404,6 +3458,7 @@ it.effect( Effect.succeed({ ...changeRequest(1, "2026-07-02T00:00:00Z"), isDraft: true, + rebaseRequired: false, reviewDecision: "approved", checksState: "passing", }), @@ -3413,6 +3468,7 @@ it.effect( deletions: 3, changedFiles: 5, mergeability: "conflicting", + rebaseRequired: true, }), }), ], @@ -3420,6 +3476,8 @@ it.effect( yield* service.summary(reference); const detail = yield* service.detail(reference); const summary = yield* service.summary(reference); + assert.strictEqual(detail.rebaseRequired, true); + assert.strictEqual(summary.rebaseRequired, true); assert.strictEqual(summary.isDraft, false); assert.deepStrictEqual(summary.author, detail.author); assert.strictEqual(summary.additions, 14); @@ -3520,19 +3578,31 @@ it.effect("deduplicates routed PR reports without dropping draft changes", () => }), ); -it.effect( - "reuses fresh detail checks without extending their refresh interval on detail cache hits", - () => +it.effect.each(["github", "azure-devops"])( + "reuses fresh %s detail checks without extending their refresh interval on detail cache hits", + (provider) => Effect.gen(function* () { let reads = 0; - const reference = { projectId: "p1" as ProjectId, repository: "acme/web", number: 1 }; + const repository = provider === "azure-devops" ? "web" : "acme/web"; + const reference = { projectId: "p1" as ProjectId, repository, number: 1 }; const service = yield* makeService({ projects: [ - project({ id: "p1", title: "web", workspaceRoot: "/a", repository: "acme/web" }), + project({ + id: "p1", + title: "web", + workspaceRoot: "/a", + repository, + provider, + ...(provider === "azure-devops" ? { host: "dev.azure.com" } : {}), + }), ], providers: [ - fakePullRequestProvider(SourceControlProviderKind.make("github"), { - getChangeRequest: () => Effect.succeed(hostedChangeRequest("Description")), + fakePullRequestProvider(SourceControlProviderKind.make(provider), { + getChangeRequest: () => + Effect.succeed({ + ...hostedChangeRequest("Description"), + checks: [{ name: "build", status: "success", description: null, url: null }], + }), getChangeRequestChecks: () => Effect.sync(() => { reads += 1; @@ -3562,6 +3632,37 @@ it.effect( }), ); +it.effect("asks Azure for checks when its detail came back with none", () => + Effect.gen(function* () { + let reads = 0; + const reference = { projectId: "p1" as ProjectId, repository: "web", number: 1 }; + const service = yield* makeService({ + projects: [ + project({ + id: "p1", + title: "web", + workspaceRoot: "/a", + repository: "web", + provider: "azure-devops", + }), + ], + providers: [ + fakePullRequestProvider(SourceControlProviderKind.make("azure-devops"), { + getChangeRequest: () => Effect.succeed(hostedChangeRequest("Description")), + getChangeRequestChecks: () => + Effect.sync(() => { + reads += 1; + return { state: "open" as const, checks: [] }; + }), + }), + ], + }); + yield* service.detail(reference); + yield* service.checks(reference); + expect(reads).toBe(1); + }), +); + it.effect("preserves check observation time on cache hits and advances it on refresh", () => Effect.gen(function* () { let reads = 0; @@ -4328,6 +4429,43 @@ it.effect("keeps viewed files itself for a host that keeps none of its own", () }), ); +it.effect("records shown revisions until the reader ticks the pushed version", () => + Effect.gen(function* () { + const asked: Array> = []; + const revisions = new Map([ + ["src/a.ts", "pushed-a"], + ["src/b.ts", "current-b"], + ]); + const service = yield* environmentViewedService(revisions, asked); + + yield* service.setFilesViewed({ + ...GITLAB_REFERENCE, + files: [ + { path: "src/a.ts", viewed: true, revision: "shown-a" }, + { path: "src/b.ts", viewed: true }, + ], + }); + assert.deepStrictEqual(asked, [["src/b.ts"]]); + yield* service.invalidate({ reference: GITLAB_REFERENCE }); + assert.deepStrictEqual((yield* service.filesViewed(GITLAB_REFERENCE)).files, [ + { path: "src/a.ts", state: "dismissed" }, + { path: "src/b.ts", state: "viewed" }, + ]); + + const reads = asked.length; + revisions.set("src/a.ts", "pushed-a-again"); + yield* service.setFilesViewed({ + ...GITLAB_REFERENCE, + files: [{ path: "src/a.ts", viewed: true, revision: "pushed-a-again" }], + }); + assert.strictEqual(asked.length, reads); + assert.deepStrictEqual((yield* service.filesViewed(GITLAB_REFERENCE)).files, [ + { path: "src/a.ts", state: "viewed" }, + { path: "src/b.ts", state: "viewed" }, + ]); + }), +); + it.effect("keeps a deleted file cleared, which the head has no version of at all", () => Effect.gen(function* () { const service = yield* environmentViewedService(new Map(), []); diff --git a/apps/server/src/pullRequest/PullRequestService.ts b/apps/server/src/pullRequest/PullRequestService.ts index 6cd890eb7e0fb..85a9349b7af82 100644 --- a/apps/server/src/pullRequest/PullRequestService.ts +++ b/apps/server/src/pullRequest/PullRequestService.ts @@ -198,7 +198,11 @@ export class PullRequestService extends Context.Service< ) => Effect.Effect; readonly summary: ( input: PullRequestRef, - options?: { readonly recoverTransientFailure?: boolean }, + options?: { + readonly recoverTransientFailure?: boolean; + /** Registration preflights must reach the provider despite a successful cached read. */ + readonly bypassCache?: boolean; + }, ) => Effect.Effect; /** Whether the reference's host batches summary reads; false when it cannot be resolved. */ readonly batchesReads: (input: PullRequestRef) => Effect.Effect; @@ -220,6 +224,9 @@ export class PullRequestService extends Context.Service< input: PullRequestReportStateInput, ) => Effect.Effect; readonly subscribeRefreshes: Stream.Stream; + readonly subscribeLinkedThreadRefreshes: Stream.Stream; + /** Wakes linked-thread readers only; other pull request reads don't depend on thread links. */ + readonly notifyLinkedThreadsChanged: Effect.Effect; readonly refreshAfterTurn: ( projectId: ProjectId, options?: { readonly pushed?: boolean }, @@ -734,6 +741,7 @@ export const make = Effect.gen(function* () { const mergedPullRequests = yield* PubSub.sliding(64); const stateChanges = yield* PubSub.sliding(64); const pullRequestRefreshes = yield* SubscriptionRef.make(0); + const linkedThreadRefreshes = yield* SubscriptionRef.make(0); const registry = yield* PullRequestProviderRegistry.PullRequestProviderRegistry; const requirementOf: UnavailableRequirementOf = (kind) => registry.get(kind)?.unavailableRequirement; @@ -1301,6 +1309,9 @@ export const make = Effect.gen(function* () { state: input.item.state, isDraft: input.item.isDraft, mergeability: input.item.mergeability, + ...(input.item.rebaseRequired === undefined + ? {} + : { rebaseRequired: input.item.rebaseRequired }), additions: input.item.additions, deletions: input.item.deletions, createdAt: input.item.createdAt, @@ -1542,7 +1553,18 @@ export const make = Effect.gen(function* () { filters: input.filters, ...(cursor === undefined ? {} - : { cursor: { updatedBefore: cursor.updatedBefore, delivered: cursor.delivered } }), + : { + cursor: { + updatedBefore: cursor.updatedBefore, + delivered: cursor.delivered, + seenAt: chunk.flatMap((project) => + (cursorOf(project)?.seenAt ?? []).map((number) => ({ + repository: project.repository, + number, + })), + ), + }, + }), }).pipe( observeRead, Effect.flatMap(({ value: page, observedAt }) => { @@ -1773,6 +1795,9 @@ export const make = Effect.gen(function* () { ...(changeRequest.mergeability === undefined ? {} : { mergeability: changeRequest.mergeability }), + ...(changeRequest.rebaseRequired === undefined + ? {} + : { rebaseRequired: changeRequest.rebaseRequired }), ...(changeRequest.stack === undefined ? {} : { stack: changeRequest.stack }), ...(changeRequest.headSha?.trim() ? { headSha: changeRequest.headSha.trim() } : {}), ...(changeRequest.autoMergeEnabled === undefined @@ -1872,6 +1897,9 @@ export const make = Effect.gen(function* () { state: changeRequest.state, isDraft: changeRequest.isDraft, mergeability: changeRequest.mergeability, + ...(changeRequest.rebaseRequired === undefined + ? {} + : { rebaseRequired: changeRequest.rebaseRequired }), additions: changeRequest.additions, deletions: changeRequest.deletions, changedFiles: changeRequest.changedFiles, @@ -2077,7 +2105,14 @@ export const make = Effect.gen(function* () { // Deliberately not `invalidatedByMutation`: ticking a file off says nothing about the // change request, and dropping a 300-file diff on every checkbox is the whole cost of // the feature. - Effect.ensuring(Effect.sync(() => bumpFilesViewedEpoch(ref))), + Effect.ensuring( + Effect.sync(() => { + bumpFilesViewedEpoch(ref); + if (input.files.some((file) => file.viewed && file.revision !== undefined)) { + bumpEpoch(fileRevisionEpochs, ref); + } + }), + ), ), ), ); @@ -2993,6 +3028,7 @@ export const make = Effect.gen(function* () { const filesViewedEpoch = (ref: PullRequestRef) => filesViewedEpochs.get(refScope(ref)) ?? 0; const bumpFilesViewedEpoch = (ref: PullRequestRef) => bumpEpoch(filesViewedEpochs, ref); + const fileRevisionEpochs = new Map(); let everyFileRevisionEpoch = 0; const viewedFiles = ViewedFiles.make({ filesViewedStore, @@ -3001,7 +3037,8 @@ export const make = Effect.gen(function* () { toPullRequestError, runFork, refEpoch, - fileRevisionsEpoch: () => everyFileRevisionEpoch, + fileRevisionsEpoch: (ref) => + `${everyFileRevisionEpoch}:${fileRevisionEpochs.get(refScope(ref)) ?? 0}`, }); const filtersOfKey = ( @@ -3067,13 +3104,17 @@ export const make = Effect.gen(function* () { const summary: PullRequestService["Service"]["summary"] = (input, options) => { const key = refCacheKey(input); - const cached = persistedRead(input, "summary", summaryCodec, summaryUncached(input)); + const read = + options?.bypassCache === true + ? summaryUncached(input) + : persistedRead(input, "summary", summaryCodec, summaryUncached(input)); const held = lastGoodSummary.peek(key); - return held !== undefined && + return options?.bypassCache !== true && + held !== undefined && input.allowStale !== false && (options?.recoverTransientFailure !== false || held.state === "merged") ? Effect.succeed(held) - : cached.pipe( + : read.pipe( Effect.tap((value) => shouldReplaceHeldSummary(key, value) ? lastGoodSummary.record(key, value) : Effect.void, ), @@ -3244,9 +3285,11 @@ export const make = Effect.gen(function* () { return detailUncached(refOfCacheKey(key)).pipe( Effect.tap( Effect.fn("PullRequestService.recordDetailStats")(function* (value: PullRequestDetail) { - // GitHub detail includes the same checks; the checks TTL counts from their observation. + // GitHub and Azure detail include the same checks; the checks TTL counts from their observation. + // Azure detail answers a failed policy read with no checks, which must not be cached. if ( - value.provider === "github" && + (value.provider === "github" || + (value.provider === "azure-devops" && value.checks.length > 0)) && value.observedAt !== undefined && !(yield* Cache.has(checksCache, key)) ) { @@ -3296,6 +3339,7 @@ export const make = Effect.gen(function* () { deletions: detail.deletions, changedFiles: detail.changedFiles, mergeability: detail.mergeability, + ...(detail.rebaseRequired === undefined ? {} : { rebaseRequired: detail.rebaseRequired }), headBranch: detail.headBranch, baseBranch: detail.baseBranch, closedAt: detail.closedAt, @@ -3679,6 +3723,11 @@ export const make = Effect.gen(function* () { yield* SubscriptionRef.set(pullRequestRefreshes, epoch); }); + const notifyLinkedThreadsChanged = SubscriptionRef.update( + linkedThreadRefreshes, + (revision) => revision + 1, + ); + const invalidatedByMutation = ( method: (input: I) => Effect.Effect, @@ -3878,6 +3927,10 @@ export const make = Effect.gen(function* () { subscribeRefreshes: SubscriptionRef.changes(pullRequestRefreshes).pipe( Stream.filter((revision) => revision > 0), ), + subscribeLinkedThreadRefreshes: SubscriptionRef.changes(linkedThreadRefreshes).pipe( + Stream.filter((revision) => revision > 0), + ), + notifyLinkedThreadsChanged, refreshAfterTurn, detail: credentialCached(detail), checks: credentialCached((input) => Cache.get(checksCache, refCacheKey(input))), diff --git a/apps/server/src/pullRequest/PullRequestWatchService.test.ts b/apps/server/src/pullRequest/PullRequestWatchService.test.ts new file mode 100644 index 0000000000000..2c4f4d92e4693 --- /dev/null +++ b/apps/server/src/pullRequest/PullRequestWatchService.test.ts @@ -0,0 +1,294 @@ +import { + ProjectId, + ProviderInstanceId, + ThreadId, + PullRequestOperationError, + PullRequestUnavailableError, + type PullRequestSummary, + type OrchestrationV2ServerCommand as OrchestrationCommand, + type OrchestrationProjectShell, + type ThreadPullRequestLink, + SourceControlProviderKind, +} from "@t3tools/contracts"; +import { describe, expect, it } from "@effect/vitest"; +import * as SecureRandom from "effect/crypto/SecureRandom"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Ref from "effect/Ref"; + +import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; +import { + type PullRequestTestThread, + v2PullRequestThread, +} from "../orchestration-v2/testkit/pullRequestFixtures.ts"; +import * as ProjectService from "../project/ProjectService.ts"; +import * as PullRequestService from "./PullRequestService.ts"; +import * as PullRequestWatchService from "./PullRequestWatchService.ts"; +import type { PullRequestTarget } from "./pullRequestTarget.ts"; + +const PROJECT_ID = ProjectId.make("project-1"); +const THREAD_ID = ThreadId.make("thread-1"); +const testCrypto = SecureRandom.make((size) => new Uint8Array(size).fill(7)); + +function makeProject( + repositoryIdentity: OrchestrationProjectShell["repositoryIdentity"] = { + canonicalKey: "github.com/t3tools/t3code", + locator: { + source: "git-remote", + remoteName: "origin", + remoteUrl: "git@github.com:T3Tools/T3Code.git", + }, + provider: SourceControlProviderKind.make("github"), + displayName: "T3Tools/T3Code", + owner: "T3Tools", + name: "T3Code", + }, +): OrchestrationProjectShell { + return { + id: PROJECT_ID, + title: "Project", + workspaceRoot: "/workspace/project", + defaultModelSelection: null, + scripts: [], + repositoryIdentity, + createdAt: "2026-08-01T00:00:00.000Z", + updatedAt: "2026-08-01T00:00:00.000Z", + }; +} + +function makeThread(pullRequests: ReadonlyArray): PullRequestTestThread { + return { + id: THREAD_ID, + projectId: PROJECT_ID, + title: "Thread", + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5" }, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + pullRequests, + createdAt: "2026-08-01T00:00:00.000Z", + updatedAt: "2026-08-20T00:00:00.000Z", + archivedAt: null, + settledOverride: null, + settledAt: null, + latestUserMessageAt: "2026-08-20T00:00:00.000Z", + }; +} + +function makeLink( + number: number, + overrides: Partial & { + readonly headBranch?: string; + readonly baseBranch?: string; + } = {}, +): ThreadPullRequestLink { + const { headBranch, baseBranch, ...rest } = overrides; + return { + host: "github.com", + repository: "t3tools/t3code", + number, + url: `https://github.com/t3tools/t3code/pull/${number}`, + source: "manual", + linkedAt: "2026-08-10T00:00:00.000Z", + snapshot: + headBranch === undefined + ? null + : { + state: "open", + title: `PR ${number}`, + headBranch, + baseBranch: baseBranch ?? "main", + isDraft: false, + updatedAt: null, + syncedAt: "2026-08-27T00:00:00.000Z", + }, + stack: null, + ...rest, + }; +} + +interface HarnessOptions { + readonly thread?: PullRequestTestThread | null; + readonly project?: OrchestrationProjectShell | null; + readonly readFailure?: PullRequestService.PullRequestError; + readonly readState?: PullRequestSummary["state"]; +} + +const makeHarness = Effect.fn("makePullRequestWatchHarness")(function* ( + options: HarnessOptions = {}, +) { + const commands = yield* Ref.make>([]); + const thread = options.thread === undefined ? makeThread([]) : options.thread; + const project = options.project === undefined ? makeProject() : options.project; + const dependencies = Layer.mergeAll( + Layer.mock(ProjectService.ProjectService)({ + getShell: () => Effect.succeed(Option.fromNullishOr(project)), + }), + Layer.mock(Orchestrator.OrchestratorV2)({ + getThreadShell: (id) => + Effect.succeed(id === thread?.id ? v2PullRequestThread(thread) : null), + dispatch: (command) => + Ref.update(commands, (recorded) => [...recorded, command]).pipe( + Effect.as({ sequence: 1, storedEvents: [] }), + ), + }), + Layer.mock(PullRequestService.PullRequestService)({ + summary: (input) => + options.readFailure !== undefined + ? Effect.fail(options.readFailure) + : Effect.succeed({ + ...input, + provider: SourceControlProviderKind.make("github"), + title: "Pull request", + url: `https://github.com/${input.repository}/pull/${input.number}`, + state: options.readState ?? "open", + headBranch: "feature", + baseBranch: "main", + updatedAt: "2026-08-20T00:00:00.000Z", + }), + }), + Layer.succeed(SecureRandom.SecureRandom, testCrypto), + ); + const service = yield* PullRequestWatchService.PullRequestWatchService.pipe( + Effect.provide(PullRequestWatchService.layer.pipe(Layer.provide(dependencies))), + ); + return { + commands, + setWatching: (watching: boolean, target: PullRequestTarget) => + service.setWatching({ threadId: THREAD_ID, target, watching }), + }; +}); + +describe("PullRequestWatchService", () => { + it.effect("links an unlinked watch even while the host has a transient read failure", () => + Effect.gen(function* () { + for (const options of [ + {}, + { + readFailure: new PullRequestOperationError({ + operation: "summary", + detail: "The host is temporarily unavailable.", + }), + }, + ]) { + const harness = yield* makeHarness(options); + const result = yield* harness.setWatching(true, { + url: "https://github.com/t3tools/t3code/pull/9", + }); + expect(result).toMatchObject({ number: 9, wasWatching: false }); + expect(yield* Ref.get(harness.commands)).toMatchObject([ + { + type: "thread.pull-request.watch", + number: 9, + watching: true, + link: { url: "https://github.com/t3tools/t3code/pull/9", source: "agent" }, + }, + ]); + } + }), + ); + + it.effect("reports a missing hosting CLI before saving a watch and still allows unwatch", () => + Effect.gen(function* () { + const missing = new PullRequestUnavailableError({ + reason: "cli-missing", + provider: SourceControlProviderKind.make("gitlab"), + requirement: "Install glab on the T3 server's PATH.", + }); + const harness = yield* makeHarness({ readFailure: missing }); + const target = { url: "https://gitlab.example.com/team/project/-/merge_requests/42" }; + const error = yield* harness.setWatching(true, target).pipe(Effect.flip); + expect(error).toEqual(missing); + expect(yield* Ref.get(harness.commands)).toEqual([]); + + yield* harness.setWatching(false, target); + expect(yield* Ref.get(harness.commands)).toMatchObject([ + { type: "thread.pull-request.watch", watching: false }, + ]); + }), + ); + + it.effect("rejects merged PRs and keeps an active watch usable without hosting access", () => + Effect.gen(function* () { + const freshMerged = yield* makeHarness({ readState: "merged" }); + const freshError = yield* freshMerged + .setWatching(true, { repository: "t3tools/t3code", number: 1 }) + .pipe(Effect.flip); + expect(freshError).toMatchObject({ _tag: "PullRequestNotOpenError", state: "merged" }); + expect(yield* Ref.get(freshMerged.commands)).toEqual([]); + + const watch = { + startedAt: "2026-08-20T00:00:00.000Z", + headSha: null, + failedChecks: [], + passed: false, + passedChecks: [], + remarksThrough: "2026-08-20T00:00:00.000Z", + remarkIds: [], + conflicting: false, + wakes: 0, + }; + const merged = makeLink(1, { headBranch: "done" }); + const harness = yield* makeHarness({ + readFailure: new PullRequestUnavailableError({ reason: "cli-missing" }), + thread: makeThread([ + { ...merged, snapshot: merged.snapshot && { ...merged.snapshot, state: "merged" } }, + makeLink(2, { headBranch: "idle" }), + makeLink(3, { headBranch: "watched", watch }), + ]), + }); + const error = yield* harness + .setWatching(true, { repository: "t3tools/t3code", number: 1 }) + .pipe(Effect.flip); + expect(error).toMatchObject({ _tag: "PullRequestNotOpenError", state: "merged" }); + expect( + yield* harness.setWatching(true, { repository: "t3tools/t3code", number: 3 }), + ).toMatchObject({ watching: true, wasWatching: true }); + expect( + yield* harness.setWatching(false, { repository: "t3tools/t3code", number: 3 }), + ).toMatchObject({ wasWatching: true }); + expect(yield* Ref.get(harness.commands)).toMatchObject([ + { type: "thread.pull-request.watch", number: 3, watching: true }, + { type: "thread.pull-request.watch", number: 3, watching: false }, + ]); + }), + ); + + it.effect("watches a pull request saved as closed, since it may have reopened", () => + Effect.gen(function* () { + const closed = makeLink(1, { headBranch: "closed" }); + const harness = yield* makeHarness({ + readState: "closed", + thread: makeThread([ + { ...closed, snapshot: closed.snapshot && { ...closed.snapshot, state: "closed" } }, + ]), + }); + yield* harness.setWatching(true, { repository: "t3tools/t3code", number: 1 }); + expect(yield* Ref.get(harness.commands)).toMatchObject([ + { type: "thread.pull-request.watch", number: 1, watching: true }, + ]); + }), + ); + + it.effect("refuses a watch from a subagent thread, whose parent owns the pull request", () => + Effect.gen(function* () { + const harness = yield* makeHarness({ + thread: { + ...makeThread([makeLink(1, { headBranch: "feature" })]), + lineage: { + rootThreadId: ThreadId.make("parent"), + parentThreadId: ThreadId.make("parent"), + relationshipToParent: "subagent", + }, + }, + }); + const error = yield* harness + .setWatching(true, { repository: "t3tools/t3code", number: 1 }) + .pipe(Effect.flip); + expect(error).toMatchObject({ _tag: "PullRequestWatchFromSubagentError" }); + expect(yield* Ref.get(harness.commands)).toEqual([]); + }), + ); +}); diff --git a/apps/server/src/pullRequest/PullRequestWatchService.ts b/apps/server/src/pullRequest/PullRequestWatchService.ts new file mode 100644 index 0000000000000..6ed736075c018 --- /dev/null +++ b/apps/server/src/pullRequest/PullRequestWatchService.ts @@ -0,0 +1,180 @@ +import { + CommandId, + ThreadId, + type OrchestrationV2ThreadShell, + type PullRequestUnavailableError, +} from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; +import * as Context from "effect/Context"; +import * as SecureRandom from "effect/crypto/SecureRandom"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; + +import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; +import * as ProjectService from "../project/ProjectService.ts"; +import * as PullRequestService from "./PullRequestService.ts"; +import { + resolveTarget, + type PullRequestTarget, + type PullRequestTargetError, + type ResolvedTarget, +} from "./pullRequestTarget.ts"; +import { + threadPullRequestKeysEqual, + threadPullRequestsOf, +} from "../sourceControl/threadPullRequests.ts"; + +export class PullRequestWatchFailedError extends Schema.TaggedError()( + "PullRequestWatchFailedError", + { cause: Schema.Defect() }, +) { + override get message(): string { + return "Could not change whether the pull request is watched."; + } +} + +export class PullRequestNotOpenError extends Schema.TaggedError()( + "PullRequestNotOpenError", + { state: Schema.String }, +) { + override get message(): string { + return `The pull request is ${this.state}, so there is nothing to watch.`; + } +} + +export class PullRequestWatchFromSubagentError extends Schema.TaggedError()( + "PullRequestWatchFromSubagentError", + {}, +) { + override get message(): string { + return "This thread is a subagent, so it cannot watch pull requests. Its parent thread owns the pull request: finish your task and report back instead."; + } +} + +export class PullRequestWatchThreadNotFoundError extends Schema.TaggedError()( + "PullRequestWatchThreadNotFoundError", + { threadId: ThreadId }, +) { + override get message(): string { + return `Thread ${this.threadId} was not found.`; + } +} + +export class PullRequestWatchService extends Context.Service< + PullRequestWatchService, + { + readonly setWatching: (input: { + readonly threadId: ThreadId; + readonly target: PullRequestTarget; + readonly watching: boolean; + }) => Effect.Effect< + ResolvedTarget & { readonly watching: boolean; readonly wasWatching: boolean }, + | PullRequestWatchFailedError + | PullRequestNotOpenError + | PullRequestWatchFromSubagentError + | PullRequestWatchThreadNotFoundError + | PullRequestTargetError + | PullRequestUnavailableError + >; + } +>()("t3/pullRequest/PullRequestWatchService") {} + +const make = Effect.gen(function* () { + const engine = yield* Orchestrator.OrchestratorV2; + const projects = yield* ProjectService.ProjectService; + const pullRequests = yield* PullRequestService.PullRequestService; + const random = yield* SecureRandom.SecureRandom; + + const requireThread = Effect.fn("PullRequestWatchService.requireThread")(function* ( + threadId: ThreadId, + ) { + const thread = yield* engine + .getThreadShell(threadId) + .pipe(Effect.mapError((cause) => new PullRequestWatchFailedError({ cause }))); + if (thread === null || thread.deletedAt !== null) { + return yield* new PullRequestWatchThreadNotFoundError({ threadId }); + } + return thread; + }); + + // Keep the registration span stable across the service extraction. + const setWatching = Effect.fn("PullRequestsToolkit.setWatching")(function* (input: { + readonly threadId: ThreadId; + readonly target: PullRequestTarget; + readonly watching: boolean; + }) { + const { watching } = input; + const thread = yield* requireThread(input.threadId); + const project = yield* projects.getShell(thread.projectId).pipe( + Effect.map(Option.getOrUndefined), + Effect.mapError((cause) => new PullRequestWatchFailedError({ cause })), + ); + const target = yield* resolveTarget(input.target, project); + const watchedLink = (shell: OrchestrationV2ThreadShell) => + threadPullRequestsOf(shell).find( + (link) => link.source !== "stack-dismissed" && threadPullRequestKeysEqual(link, target), + ); + if (watching && thread.lineage.relationshipToParent === "subagent") { + return yield* new PullRequestWatchFromSubagentError(); + } + const before = watchedLink(thread); + if (watching && before?.snapshot?.state === "merged") { + return yield* new PullRequestNotOpenError({ state: "merged" }); + } + if (watching && before?.watch === undefined) { + yield* pullRequests + .summary( + { + projectId: thread.projectId, + host: target.host, + repository: target.repository, + number: target.number, + allowStale: false, + }, + { recoverTransientFailure: false, bypassCache: true }, + ) + .pipe( + Effect.flatMap((summary) => + summary.state === "merged" + ? Effect.fail(new PullRequestNotOpenError({ state: "merged" })) + : Effect.void, + ), + Effect.catchTags({ + // Host and transport errors can recover while the watch is active. + PullRequestOperationError: () => Effect.void, + }), + ); + } + const uuid = yield* random.uuidV4().pipe(Effect.orDie); + yield* engine + .dispatch({ + type: "thread.pull-request.watch", + commandId: CommandId.make(`server:mcp-pr-watch:${thread.id}:${uuid}`), + threadId: thread.id, + host: target.host, + repository: target.repository, + number: target.number, + watching, + ...(watching ? { link: { url: target.url, source: "agent" as const } } : {}), + }) + .pipe( + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.failCause(cause as Cause.Cause) + : Effect.fail(new PullRequestWatchFailedError({ cause })), + ), + ); + const after = yield* requireThread(thread.id); + return { + ...target, + watching: watchedLink(after)?.watch !== undefined, + wasWatching: before?.watch !== undefined, + }; + }); + + return PullRequestWatchService.of({ setWatching }); +}); + +export const layer = Layer.effect(PullRequestWatchService, make); diff --git a/apps/server/src/pullRequest/linkedThreads.test.ts b/apps/server/src/pullRequest/linkedThreads.test.ts index 8992a32ea10ca..2dac788641586 100644 --- a/apps/server/src/pullRequest/linkedThreads.test.ts +++ b/apps/server/src/pullRequest/linkedThreads.test.ts @@ -1,10 +1,18 @@ +import { + ProjectId, + ThreadId, + type OrchestrationV2AppThread, + type OrchestrationV2DomainEvent, + type ThreadPullRequestLink, +} from "@t3tools/contracts"; import { assert, expect, it } from "@effect/vitest"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; -import { listLinkedPullRequestThreads } from "./linkedThreads.ts"; +import { listLinkedPullRequestThreads, makeLinkedThreadsChangeDetector } from "./linkedThreads.ts"; const encodePayload = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); @@ -209,3 +217,132 @@ it.effect( ); }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); + +it("signals linked-thread changes only when a linked answer could change", () => { + const now = DateTime.makeUnsafe("2026-11-12T00:00:00.000Z"); + const threadId = ThreadId.make("thread-a"); + const link: ThreadPullRequestLink = { + host: "github.com", + repository: "acme/web", + number: 7, + url: "https://github.com/acme/web/pull/7", + source: "agent", + linkedAt: "2026-11-12T00:00:00.000Z", + snapshot: null, + stack: null, + }; + const base = { + id: threadId, + projectId: ProjectId.make("project-a"), + title: "Thread A", + lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId }, + archivedAt: null, + settledOverride: null, + settledAt: null, + deletedAt: null, + updatedAt: now, + } as unknown as OrchestrationV2AppThread; + const event = ( + type: OrchestrationV2DomainEvent["type"], + payload: Partial, + ) => ({ type, threadId, payload: { ...base, ...payload } }) as OrchestrationV2DomainEvent; + const changed = makeLinkedThreadsChangeDetector(); + + expect( + [ + event("thread.metadata-updated", { title: "Unlinked rename" }), + event("thread.pull-request-synced", { pullRequests: [link] }), + event("thread.pull-request-synced", { + pullRequests: [{ ...link, watch: { since: "2026-11-12T00:00:00.000Z" } as never }], + }), + event("thread.visited", { pullRequests: [link] }), + event("thread.pinned", { pullRequests: [link], settledAt: now }), + { type: "thread.pull-request-link-updated", threadId, payload: link } as never, + event("thread.metadata-updated", { pullRequests: [link], title: "Renamed" }), + event("thread.archived", { pullRequests: [link], title: "Renamed", archivedAt: now }), + event("thread.pull-request-synced", { + pullRequests: [{ ...link, source: "stack-dismissed" }], + title: "Renamed", + archivedAt: now, + }), + event("thread.deleted", { pullRequests: [], deletedAt: now }), + ].map(changed), + ).toEqual([ + "changed", + "changed", + null, + null, + "changed", + null, + "changed", + "changed", + "changed", + null, + ]); + + const afterRestart = makeLinkedThreadsChangeDetector(); + expect(afterRestart(event("thread.created", { title: "New" }))).toBe(null); + expect( + makeLinkedThreadsChangeDetector()( + event("thread.deleted", { pullRequests: [], deletedAt: now }), + ), + ).toBe("changed"); +}); + +it("signals when activity reorders the threads linked to a pull request", () => { + const at = (second: number) => DateTime.makeUnsafe(`2026-11-12T00:00:0${second}.000Z`); + const link = { + host: "github.com", + repository: "acme/web", + number: 7, + url: "https://github.com/acme/web/pull/7", + source: "agent", + }; + const thread = (id: string, second: number, pullRequests: ReadonlyArray = [link]) => + ({ + type: "thread.pull-request-synced", + threadId: ThreadId.make(id), + payload: { + id: ThreadId.make(id), + projectId: ProjectId.make("project-a"), + title: id, + lineage: { rootThreadId: ThreadId.make(id) }, + pullRequests, + archivedAt: null, + settledOverride: null, + settledAt: null, + deletedAt: null, + updatedAt: at(second), + }, + }) as unknown as OrchestrationV2DomainEvent; + const activity = (id: string, second: number) => + ({ + type: "message.updated", + threadId: ThreadId.make(id), + occurredAt: at(second), + }) as unknown as OrchestrationV2DomainEvent; + const changed = makeLinkedThreadsChangeDetector(); + changed(thread("a", 1)); + changed(thread("b", 2)); + + expect( + [ + { ...thread("a", 3), type: "thread.pinned" } as OrchestrationV2DomainEvent, + activity("a", 4), + activity("b", 5), + activity("b", 5), + // Seen since startup without links, so its activity can't reorder an answer. + thread("unlinked", 5, []), + activity("unlinked", 6), + // Not seen since startup, so it may be linked from before the restart. + activity("persisted", 6), + ].map(changed), + ).toEqual(["reordered", "reordered", "reordered", null, "changed", null, "reordered"]); + + // Other threads linked to the pull request may have had no event since the restart. + const afterRestart = makeLinkedThreadsChangeDetector(); + afterRestart(thread("a", 2)); + expect( + afterRestart({ ...thread("a", 6), type: "thread.pinned" } as OrchestrationV2DomainEvent), + ).toBe("reordered"); +}); diff --git a/apps/server/src/pullRequest/linkedThreads.ts b/apps/server/src/pullRequest/linkedThreads.ts index 87ffae525eb9c..1fb6b5f5dfebf 100644 --- a/apps/server/src/pullRequest/linkedThreads.ts +++ b/apps/server/src/pullRequest/linkedThreads.ts @@ -1,12 +1,18 @@ import { normalizeThreadPullRequestKey, + threadPullRequestKeyOf, threadPullRequestKeysEqual, + visibleThreadPullRequests, } from "../sourceControl/threadPullRequests.ts"; import { PullRequestLinkedThreadsResult, PullRequestOperationError, + type OrchestrationV2AppThread, + type OrchestrationV2DomainEvent, + type ThreadId, type ThreadPullRequestKey, } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; import * as Schema from "effect/Schema"; @@ -66,3 +72,67 @@ export const listLinkedPullRequestThreads = Effect.fn("listLinkedPullRequestThre }), ), ); + +// Mirrors the rows and filters of `listLinkedPullRequestThreads`. +function linkedKeys(thread: OrchestrationV2AppThread): ReadonlyArray { + if (thread.deletedAt !== null || thread.lineage.sideChat === true) return []; + return visibleThreadPullRequests(thread.pullRequests ?? []) + .map(threadPullRequestKeyOf) + .toSorted(); +} + +function linkedThreadsView( + thread: OrchestrationV2AppThread, + keys: ReadonlyArray, +): string | null { + if (keys.length === 0) return null; + const settled = thread.settledAt !== null || thread.settledOverride === "settled"; + return JSON.stringify([ + keys, + thread.title, + thread.projectId, + thread.archivedAt !== null, + settled, + ]); +} + +/** + * Tells how a domain event changed `pullRequests.linkedThreads` answers, so readers can refresh on + * change instead of polling. `reordered` means activity on a linked thread, or on one whose links + * are unknown since a restart, which can reorder answers. A thread's first full event after a + * restart counts as a change, since its earlier links are unknown. + */ +export function makeLinkedThreadsChangeDetector() { + const views = new Map(); + // Last activity of each thread with visible links. + const activeAt = new Map(); + const active = (threadId: ThreadId, at: DateTime.Utc): boolean => { + const previous = activeAt.get(threadId); + if (previous === undefined) return false; + const millis = DateTime.toEpochMillis(at); + if (millis <= previous) return false; + activeAt.set(threadId, millis); + return true; + }; + return (event: OrchestrationV2DomainEvent): "changed" | "reordered" | null => { + // The projection stamps every non-thread event as the thread's activity. A thread not seen + // since startup may be linked already, since history is not replayed. + if (!event.type.startsWith("thread.")) { + if (!views.has(event.threadId)) return "reordered"; + return active(event.threadId, event.occurredAt) ? "reordered" : null; + } + if (!("lineage" in event.payload)) return null; + const thread = event.payload; + const view = linkedThreadsView(thread, linkedKeys(thread)); + const known = views.has(event.threadId) || event.type === "thread.created"; + const previous = views.get(event.threadId) ?? null; + if (event.type === "thread.deleted") views.delete(event.threadId); + else views.set(event.threadId, view); + if (view === null) activeAt.delete(event.threadId); + if (!known || view !== previous) { + if (view !== null) activeAt.set(event.threadId, DateTime.toEpochMillis(thread.updatedAt)); + return "changed"; + } + return active(event.threadId, thread.updatedAt) ? "reordered" : null; + }; +} diff --git a/apps/server/src/pullRequest/pullRequestTarget.ts b/apps/server/src/pullRequest/pullRequestTarget.ts new file mode 100644 index 0000000000000..4520b0523a369 --- /dev/null +++ b/apps/server/src/pullRequest/pullRequestTarget.ts @@ -0,0 +1,123 @@ +import type { OrchestrationProjectShell, SourceControlProviderKind } from "@t3tools/contracts"; +import { canonicalizeGitCafePullRequestUrl } from "@t3tools/source-control-definitions/registry"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +import { sourceControlDefinitions } from "../sourceControl/sourceControlDefinitions.ts"; +import { normalizeThreadPullRequestKey } from "../sourceControl/threadPullRequests.ts"; + +export interface PullRequestTarget { + readonly url?: string | undefined; + readonly repository?: string | undefined; + readonly number?: number | undefined; + readonly host?: string | undefined; +} + +export class PullRequestUrlInvalidError extends Schema.TaggedError()( + "PullRequestUrlInvalidError", + {}, +) { + override get message(): string { + return "This is not a recognised pull request URL. Pass repository and number instead."; + } +} + +export class PullRequestTargetIncompleteError extends Schema.TaggedError()( + "PullRequestTargetIncompleteError", + {}, +) { + override get message(): string { + return "Pass either url, or both repository and number."; + } +} + +export class PullRequestHostRequiredError extends Schema.TaggedError()( + "PullRequestHostRequiredError", + {}, +) { + override get message(): string { + return "This thread's project has no recognised remote. Pass host or url."; + } +} + +export class PullRequestUrlUnavailableError extends Schema.TaggedError()( + "PullRequestUrlUnavailableError", + {}, +) { + override get message(): string { + return "This thread's project is on a host with no known pull request URL shape. Pass url."; + } +} + +export type PullRequestTargetError = + | PullRequestUrlInvalidError + | PullRequestTargetIncompleteError + | PullRequestHostRequiredError + | PullRequestUrlUnavailableError; + +export interface ResolvedTarget { + readonly host: string; + readonly repository: string; + readonly number: number; + readonly url: string; +} + +function projectHostAndProvider(project: OrchestrationProjectShell | undefined): { + readonly host: string | null; + readonly kind: SourceControlProviderKind | null; +} { + const identity = project?.repositoryIdentity; + const kind = identity?.provider ?? null; + if (!identity || kind === null) return { host: null, kind: null }; + return { + host: sourceControlDefinitions.changeRequestHost(identity, kind), + kind, + }; +} + +export const resolveTarget = Effect.fn("PullRequestsToolkit.resolveTarget")(function* ( + input: PullRequestTarget, + project: OrchestrationProjectShell | undefined, +) { + if (input.url !== undefined) { + const parsed = sourceControlDefinitions.parseChangeRequestUrl(input.url)?.link; + if (parsed === undefined) { + return yield* new PullRequestUrlInvalidError({}); + } + return { + ...normalizeThreadPullRequestKey(parsed), + url: canonicalizeGitCafePullRequestUrl(input.url), + } satisfies ResolvedTarget; + } + if (input.repository === undefined || input.number === undefined) { + return yield* new PullRequestTargetIncompleteError({}); + } + const projectHost = projectHostAndProvider(project); + const host = (input.host ?? projectHost.host)?.toLowerCase(); + if (host === undefined) { + return yield* new PullRequestHostRequiredError({}); + } + const repository = input.repository.toLowerCase(); + // Without a URL, a public forge host says what shape its change request URLs take; + // any other host is taken to run the project's own forge. + const recognizedKind = sourceControlDefinitions.recognizeRemote(`https://${host}`)?.kind; + const kind = + sourceControlDefinitions.findByPublicHost(host)?.kind ?? + (recognizedKind === "gitcafe" ? recognizedKind : projectHost.kind); + const url = + kind === null + ? null + : sourceControlDefinitions.get(kind).changeRequestUrl({ + host, + repository, + number: input.number, + remoteUrl: project?.repositoryIdentity?.locator.remoteUrl, + }); + if (url === null) { + return yield* new PullRequestUrlUnavailableError({}); + } + return { + ...normalizeThreadPullRequestKey({ host, repository, number: input.number, url }), + url: canonicalizeGitCafePullRequestUrl(url), + } satisfies ResolvedTarget; +}); diff --git a/apps/server/src/pullRequest/pullRequestViewedFiles.ts b/apps/server/src/pullRequest/pullRequestViewedFiles.ts index ee956dfa0fe9c..7558e8b6a166e 100644 --- a/apps/server/src/pullRequest/pullRequestViewedFiles.ts +++ b/apps/server/src/pullRequest/pullRequestViewedFiles.ts @@ -30,7 +30,7 @@ const MAX_FILE_REVISION_PATHS = 1_000; interface FileRevisionsDependencies { readonly runFork: (effect: Effect.Effect) => unknown; readonly refEpoch: (ref: PullRequestRef) => number; - readonly fileRevisionsEpoch: () => number; + readonly fileRevisionsEpoch: (ref: PullRequestRef) => string; readonly toPullRequestError: ( operation: string, ) => (error: PullRequestProviderError) => PullRequestError; @@ -50,7 +50,7 @@ const makeFileRevisions = (dependencies: FileRevisionsDependencies) => { const fileRevisionsKey = (project: SupportedProject, ref: PullRequestRef) => [ refEpoch({ ...ref, host: project.host, repository: project.repository }), - fileRevisionsEpoch(), + fileRevisionsEpoch({ ...ref, host: project.host, repository: project.repository }), ref.projectId, project.repository.trim().toLowerCase(), ref.number, diff --git a/apps/server/src/relay/AgentAwarenessRelay.test.ts b/apps/server/src/relay/AgentAwarenessRelay.test.ts index dd8d608deeb83..10388e99fe7c5 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.test.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.test.ts @@ -238,6 +238,7 @@ const makeTestRelay = Effect.fnUntraced(function* ( getThreadHistoryPage: () => Effect.die("unused"), getTimelinePage: () => Effect.die("Unused timeline read"), getMessageCount: () => Effect.die("unused message count"), + getMessagesById: () => Effect.die("unused messages by id"), getTurnItem: () => Effect.die("unused turn item read"), getThreadRecords: () => Effect.die("unused record read"), getThreadProjection: unused, @@ -434,7 +435,7 @@ describe("AgentAwarenessRelay", () => { assert.equal(publications.length, 1); assert.isTrue(publications[0]?.url.includes(`/threads/${THREAD_ID}/`) ?? false); assert.equal(publications[0]?.state?.phase, "waiting_for_input"); - }).pipe(Effect.scoped), + }), ); it.effect( @@ -528,7 +529,7 @@ describe("AgentAwarenessRelay", () => { yield* relay.drain; assert.equal(publications.length, 2); assert.equal(publications[1]?.state?.phase, "completed"); - }).pipe(Effect.scoped), + }), ); it.effect("republishes a completed parent for its waiting child after a relink", () => @@ -554,7 +555,7 @@ describe("AgentAwarenessRelay", () => { yield* relay.drain; assert.equal(publications.length, 2); assert.equal(publications[1]?.state?.phase, "waiting_for_approval"); - }).pipe(Effect.scoped), + }), ); it.effect("keeps a child's alert time when the parent's own request comes and goes", () => @@ -722,7 +723,7 @@ describe("AgentAwarenessRelay", () => { yield* relay.drain; assert.equal(publications.length, 3); assert.equal(publications[2]?.state, null); - }).pipe(Effect.scoped), + }), ); it.effect("holds a completion through a continuation gap, then publishes it once", () => @@ -752,6 +753,37 @@ describe("AgentAwarenessRelay", () => { }), ); + it.effect( + "clears a completion that landed while snoozed, even when processed after the wake", + () => + Effect.gen(function* () { + yield* TestClock.setTime(DateTime.toEpochMillis(DateTime.makeUnsafe(NOW))); + const { relay, currentShell, publications } = yield* makeTestRelay(); + const snooze = { + snoozedAt: DateTime.makeUnsafe(NOW), + snoozedUntil: DateTime.makeUnsafe("2026-09-04T13:00:00.000Z"), + }; + yield* Ref.set(currentShell, shell(snooze)); + yield* relay.publishThread(THREAD_ID); + yield* TestClock.adjust("2 hours"); + yield* Ref.set( + currentShell, + shell({ + ...snooze, + status: "completed", + latestRunCompletedAt: DateTime.makeUnsafe("2026-09-04T12:30:00.000Z"), + }), + ); + yield* relay.publishThread(THREAD_ID); + yield* TestClock.adjust("5 seconds"); + yield* relay.drain; + assert.deepEqual( + publications.map((publication) => publication.state?.phase ?? null), + ["running", null], + ); + }), + ); + it.effect("still publishes a completion when the thread settles during the hold", () => Effect.gen(function* () { const { relay, currentShell, publications } = yield* makeTestRelay(); @@ -1170,5 +1202,5 @@ it.effect("catches up after startup without polling after success", () => assert.equal(catchUp.activitySnapshotReads, 3); assert.equal(publications.length, 2); assert.equal(publications[1]?.state?.threadId, THREAD_ID); - }).pipe(Effect.scoped), + }), ); diff --git a/apps/server/src/relay/AgentAwarenessRelay.ts b/apps/server/src/relay/AgentAwarenessRelay.ts index 73b52571e7416..ef2e34799742c 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.ts @@ -51,6 +51,7 @@ import { import { EnvironmentSigningKey } from "../cloud/environmentKeys.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import * as ThreadManagement from "../orchestration-v2/ThreadManagementService.ts"; +import { isSnoozed } from "../orchestration-v2/ThreadSettlementService.ts"; import * as ProjectService from "../project/ProjectService.ts"; import { forkParked } from "../serverActivation.ts"; @@ -117,6 +118,7 @@ function shouldPublishAgentAwarenessEvent( case "thread.active-reordered": case "thread.tags-updated": case "thread.visited": + case "thread.visit-recorded": case "thread.marked-unread": case "thread.runtime-mode-updated": case "thread.interaction-mode-updated": @@ -359,12 +361,27 @@ function resolveAgentAwarenessRelayPublishSnapshot(input: { state?.phase !== projectThreadAwarenessV2(awarenessInput)?.phase; return { projectId: input.thread.value.projectId, - state: sanitizeRelayAgentActivityState(state), + state: snoozedCompletion(input.thread.value, state) + ? null + : sanitizeRelayAgentActivityState(state), subagentWait: childDrivesWait ? input.pendingSubagentRequest : null, reason: "snapshot", }; } +// Checked at completion time, so a completion processed after the wake stays silent too. +function snoozedCompletion( + thread: AgentAwarenessThread, + state: { readonly phase: string } | null, +): boolean { + if (state?.phase !== "completed" || thread.latestRunCompletedAt == null) return false; + const completedAt = DateTime.toEpochMillis(thread.latestRunCompletedAt); + if (thread.snoozedAt != null && completedAt < DateTime.toEpochMillis(thread.snoozedAt)) { + return false; + } + return isSnoozed(thread, completedAt); +} + function terminalWorkSinceStart(thread: AgentAwarenessThread, startedAt: number): boolean { return ( thread.latestRunCompletedAt != null && diff --git a/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.ts b/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.ts index 7d9de87ecfd98..ca77c9e26220c 100644 --- a/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.ts +++ b/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.ts @@ -187,6 +187,8 @@ export class DesktopTelemetryReceiver extends Context.Service< readonly setDiagnosticsDemand: ( enabled: boolean, ) => Effect.Effect; + /** Tells the desktop parent that the server accepts requests, so it can skip readiness polling. */ + readonly announceServerReady: Effect.Effect; readonly requestDesktopUpdate: ( requestId: string, trackSwitch?: DesktopTrackSwitch, @@ -717,6 +719,9 @@ export const make = Effect.fn("resourceTelemetry.desktopTelemetryReceiver.make") health: Ref.get(health), subscribeHealth: subscribeBeforeSnapshotWithoutMutex(healthChanges, Ref.get(health)), setDiagnosticsDemand, + announceServerReady: sendControlMessage({ version: 1, type: "serverReady" }).pipe( + Effect.withSpan("DesktopTelemetryReceiver.announceServerReady"), + ), requestDesktopUpdate: Effect.fn("DesktopTelemetryReceiver.requestDesktopUpdate")( function (requestId, trackSwitch) { return sendControlMessage({ @@ -787,6 +792,7 @@ export const layerTest = ( setDiagnosticsDemand: Effect.fn("DesktopTelemetryReceiver.setDiagnosticsDemand")(function () { return Effect.void; }), + announceServerReady: Effect.void, requestDesktopUpdate: Effect.fn("DesktopTelemetryReceiver.requestDesktopUpdate")(function () { return Effect.void; }), diff --git a/apps/server/src/resourceTelemetry/ResourceTelemetry.test.ts b/apps/server/src/resourceTelemetry/ResourceTelemetry.test.ts index c12e2e44ed7eb..8a3640977ddf2 100644 --- a/apps/server/src/resourceTelemetry/ResourceTelemetry.test.ts +++ b/apps/server/src/resourceTelemetry/ResourceTelemetry.test.ts @@ -272,131 +272,127 @@ describe("ResourceTelemetry", () => { ); it.effect("rejects buffered snapshots from an earlier sidecar generation", () => - Effect.scoped( - Effect.gen(function* () { - const startedAt = DateTime.toEpochMillis(yield* DateTime.now); - const stale = nativeSnapshot({ - sequence: 100, - sampledAtUnixMs: startedAt + 1_000, - childCpuTimeMs: 100, - childWriteBytes: 1_000, - }); - const current = nativeSnapshot({ - sequence: 1, - sampledAtUnixMs: startedAt + 2_000, - childCpuTimeMs: 200, - childWriteBytes: 2_000, - }); - const nativeSnapshots = - yield* PubSub.unbounded(); - const layerNative = NativeTelemetryClient.layerTest({ - snapshots: Stream.fromPubSub(nativeSnapshots), - sampleNow: Effect.never, - health: Effect.succeed({ - status: "healthy", - hello: Option.none(), - lastSampleAt: Option.none(), - lastError: Option.none(), - restartCount: 1, - sampleIntervalMs: 1_000, - }), - }); - const layerTelemetry = ResourceTelemetry.layer.pipe( - Layer.provide( - Layer.mergeAll( - layerNative, - DesktopTelemetryReceiver.layerTest(), - ResourceAttribution.layer, - ), + Effect.gen(function* () { + const startedAt = DateTime.toEpochMillis(yield* DateTime.now); + const stale = nativeSnapshot({ + sequence: 100, + sampledAtUnixMs: startedAt + 1_000, + childCpuTimeMs: 100, + childWriteBytes: 1_000, + }); + const current = nativeSnapshot({ + sequence: 1, + sampledAtUnixMs: startedAt + 2_000, + childCpuTimeMs: 200, + childWriteBytes: 2_000, + }); + const nativeSnapshots = + yield* PubSub.unbounded(); + const layerNative = NativeTelemetryClient.layerTest({ + snapshots: Stream.fromPubSub(nativeSnapshots), + sampleNow: Effect.never, + health: Effect.succeed({ + status: "healthy", + hello: Option.none(), + lastSampleAt: Option.none(), + lastError: Option.none(), + restartCount: 1, + sampleIntervalMs: 1_000, + }), + }); + const layerTelemetry = ResourceTelemetry.layer.pipe( + Layer.provide( + Layer.mergeAll( + layerNative, + DesktopTelemetryReceiver.layerTest(), + ResourceAttribution.layer, ), - ); + ), + ); - yield* Effect.gen(function* () { - const telemetry = yield* ResourceTelemetry.ResourceTelemetry; - const subscription = yield* telemetry.subscribe; - const nextSnapshot = yield* Stream.runHead(subscription.changes).pipe(Effect.forkChild); + yield* Effect.gen(function* () { + const telemetry = yield* ResourceTelemetry.ResourceTelemetry; + const subscription = yield* telemetry.subscribe; + const nextSnapshot = yield* Stream.runHead(subscription.changes).pipe(Effect.forkChild); - yield* PubSub.publish(nativeSnapshots, nativeGeneration(stale, 0)); - yield* Effect.yieldNow; - expect((yield* telemetry.latest).health.scannedProcessCount).toBe(0); + yield* PubSub.publish(nativeSnapshots, nativeGeneration(stale, 0)); + yield* Effect.yieldNow; + expect((yield* telemetry.latest).health.scannedProcessCount).toBe(0); - yield* PubSub.publish(nativeSnapshots, nativeGeneration(current, 1)); - const received = yield* Fiber.join(nextSnapshot); + yield* PubSub.publish(nativeSnapshots, nativeGeneration(current, 1)); + const received = yield* Fiber.join(nextSnapshot); - expect(Option.isSome(received)).toBe(true); - expect(DateTime.toEpochMillis(Option.getOrThrow(received).readAt)).toBe( - current.sampledAtUnixMs, - ); - }).pipe(Effect.provide(layerTelemetry)); - }), - ), + expect(Option.isSome(received)).toBe(true); + expect(DateTime.toEpochMillis(Option.getOrThrow(received).readAt)).toBe( + current.sampledAtUnixMs, + ); + }).pipe(Effect.provide(layerTelemetry)); + }), ); it.effect("atomically subscribes while a desktop update is being rebuilt", () => - Effect.scoped( - Effect.gen(function* () { - const startedAt = DateTime.toEpochMillis(yield* DateTime.now); - const nextDesktopSnapshot = desktopSnapshot(startedAt + 1_000); - const desktopChanges = yield* PubSub.unbounded(); - const rebuildStarted = yield* Deferred.make(); - const finishRebuild = yield* Deferred.make(); - const attributionReads = yield* Ref.make(0); - const layerAttribution = Layer.succeed( - ResourceAttribution.ResourceAttribution, - ResourceAttribution.ResourceAttribution.of({ - record: () => Effect.void, - snapshot: Ref.updateAndGet(attributionReads, (count) => count + 1).pipe( - Effect.flatMap((count) => - count === 1 - ? Effect.void - : Deferred.succeed(rebuildStarted, undefined).pipe( - Effect.andThen(Deferred.await(finishRebuild)), - ), - ), - Effect.andThen( - Effect.succeed({ - readAt: DateTime.makeUnsafe(startedAt), - entries: [], - }), - ), + Effect.gen(function* () { + const startedAt = DateTime.toEpochMillis(yield* DateTime.now); + const nextDesktopSnapshot = desktopSnapshot(startedAt + 1_000); + const desktopChanges = yield* PubSub.unbounded(); + const rebuildStarted = yield* Deferred.make(); + const finishRebuild = yield* Deferred.make(); + const attributionReads = yield* Ref.make(0); + const layerAttribution = Layer.succeed( + ResourceAttribution.ResourceAttribution, + ResourceAttribution.ResourceAttribution.of({ + record: () => Effect.void, + snapshot: Ref.updateAndGet(attributionReads, (count) => count + 1).pipe( + Effect.flatMap((count) => + count === 1 + ? Effect.void + : Deferred.succeed(rebuildStarted, undefined).pipe( + Effect.andThen(Deferred.await(finishRebuild)), + ), ), - }), - ); - const layerNative = NativeTelemetryClient.layerTest({ - sampleNow: Effect.never, - health: Effect.succeed({ - status: "healthy", - hello: Option.none(), - lastSampleAt: Option.none(), - lastError: Option.none(), - restartCount: 0, - sampleIntervalMs: 1_000, - }), - }); - const layerDesktop = DesktopTelemetryReceiver.layerTest({ - changes: Stream.fromPubSub(desktopChanges), - }); - const layerTelemetry = ResourceTelemetry.layer.pipe( - Layer.provide(Layer.mergeAll(layerNative, layerDesktop, layerAttribution)), - ); + Effect.andThen( + Effect.succeed({ + readAt: DateTime.makeUnsafe(startedAt), + entries: [], + }), + ), + ), + }), + ); + const layerNative = NativeTelemetryClient.layerTest({ + sampleNow: Effect.never, + health: Effect.succeed({ + status: "healthy", + hello: Option.none(), + lastSampleAt: Option.none(), + lastError: Option.none(), + restartCount: 0, + sampleIntervalMs: 1_000, + }), + }); + const layerDesktop = DesktopTelemetryReceiver.layerTest({ + changes: Stream.fromPubSub(desktopChanges), + }); + const layerTelemetry = ResourceTelemetry.layer.pipe( + Layer.provide(Layer.mergeAll(layerNative, layerDesktop, layerAttribution)), + ); - yield* Effect.gen(function* () { - const telemetry = yield* ResourceTelemetry.ResourceTelemetry; - yield* Effect.yieldNow; - yield* PubSub.publish(desktopChanges, nextDesktopSnapshot); - yield* Deferred.await(rebuildStarted); + yield* Effect.gen(function* () { + const telemetry = yield* ResourceTelemetry.ResourceTelemetry; + yield* Effect.yieldNow; + yield* PubSub.publish(desktopChanges, nextDesktopSnapshot); + yield* Deferred.await(rebuildStarted); - const subscriptionFiber = yield* telemetry.subscribe.pipe(Effect.forkChild); - yield* Effect.yieldNow; - yield* Deferred.succeed(finishRebuild, undefined); - const subscription = yield* Fiber.join(subscriptionFiber); + const subscriptionFiber = yield* telemetry.subscribe.pipe(Effect.forkChild); + yield* Effect.yieldNow; + yield* Deferred.succeed(finishRebuild, undefined); + const subscription = yield* Fiber.join(subscriptionFiber); - expect(DateTime.toEpochMillis(subscription.latest.readAt)).toBe( - nextDesktopSnapshot.sampledAtUnixMs, - ); - expect(subscription.latest.power).toEqual(nextDesktopSnapshot.power); - }).pipe(Effect.provide(layerTelemetry)); - }), - ), + expect(DateTime.toEpochMillis(subscription.latest.readAt)).toBe( + nextDesktopSnapshot.sampledAtUnixMs, + ); + expect(subscription.latest.power).toEqual(nextDesktopSnapshot.power); + }).pipe(Effect.provide(layerTelemetry)); + }), ); }); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.interval.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.interval.test.ts index ecc761d1d0b47..86d6f26e06c44 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.interval.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.interval.test.ts @@ -26,6 +26,7 @@ const dependencies = Layer.mergeAll( ManagedPolicy.layerTest(), Layer.mock(ThreadLaunchService.ThreadLaunchService)({ launch: () => Effect.die("synthetic dispatch failure"), + isNonGitProject: () => Effect.succeed(false), }), Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ namedProjectsRoot: "/projects", diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index ee4385ff2e0af..1335e369484a5 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -4,6 +4,7 @@ import * as NodeUtil from "node:util"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import { assert, it, vi } from "@effect/vitest"; import { + CommandId, ProjectId, ProviderInstanceId, ScheduledTaskError, @@ -29,6 +30,7 @@ import * as TestClock from "effect/testing/TestClock"; import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts"; import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts"; +import { OrchestratorDispatchError } from "../orchestration-v2/Orchestrator.ts"; import * as SecretRequests from "../secrets/SecretRequests.ts"; import * as ManagedProjectFolders from "../project/ManagedProjectFolders.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; @@ -663,6 +665,67 @@ it.effect("a fresh-session task launches one thread, then starts each later run }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); +it.effect("a refused dispatch records its refusal, and opaque causes stay generic", () => + Effect.gen(function* () { + const refusal = "Thread thread:refused was stopped."; + const refused = (cause: unknown) => + new OrchestratorDispatchError({ + commandId: CommandId.make("command:refused"), + commandType: "message.dispatch", + cause, + }); + const generic = refused(undefined).message; + const cases = [ + [refusal, refusal], + [" ", generic], + [new Error("private-nested-secret"), generic], + ] as const; + const failure = yield* Ref.make(undefined); + const layer = ScheduledTaskService.layer.pipe( + Layer.provide( + Layer.mergeAll( + ManagedPolicy.layerTest(), + Layer.mock(ThreadLaunchService.ThreadLaunchService)({}), + Layer.mock(ThreadManagementService.ThreadManagementService)({ + getThreadShell: () => Effect.succeed(threadShell(ProjectId.make("project:test"))), + sendToThread: () => + Ref.get(failure).pipe(Effect.flatMap((cause) => Effect.fail(refused(cause)))), + }), + Layer.mock(SecretRequests.SecretRequests)({}), + Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ + namedProjectsRoot: "/projects", + isScratchProject: () => Effect.succeed(false), + }), + NodeCrypto.layer, + Layer.succeed(Scheduler.Scheduler, { register: () => Effect.void }), + ), + ), + ); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* service.upsert({ + id: ScheduledTaskId.make("task:refused"), + title: "Refused dispatch", + prompt: "Check progress", + enabled: true, + schedule: { type: "interval", everyMs: 60_000 }, + projectId: ProjectId.make("project:test"), + threadId: ThreadId.make("thread:refused"), + workspaceStrategy: { type: "root" }, + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" }, + runtimeMode: "full-access", + interactionMode: "default", + }); + for (const [cause, expected] of cases) { + yield* Ref.set(failure, cause); + const result = yield* service.runNow({ id: task.id }); + assert.equal(result.task.lastRunStatus, "failed"); + assert.equal(result.task.lastRunError, expected); + } + }).pipe(Effect.provide(layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + it.effect("a first run whose thread was deleted before it bound removes the task", () => Effect.gen(function* () { const layer = ScheduledTaskService.layer.pipe( diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 5af0aba39e6ed..9ed134be2da5a 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -53,6 +53,8 @@ import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts import * as ManagedProjectFolders from "../project/ManagedProjectFolders.ts"; import * as Metrics from "../observability/Metrics.ts"; import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts"; +import { OrchestratorDispatchError } from "../orchestration-v2/Orchestrator.ts"; +import { userFacingDispatchErrorMessage } from "../orchestration-v2/UserFacingErrors.ts"; import * as SecretRequests from "../secrets/SecretRequests.ts"; import * as ManagedPolicy from "../policy/ManagedPolicy.ts"; import * as DueHint from "../scheduling/DueHint.ts"; @@ -79,6 +81,7 @@ import { constantTimeEquals, verifyWebhookSignature } from "./webhookVerificatio export const WEBHOOK_ROUTE_PREFIX = "/api/hooks"; const isScheduledTaskError = Schema.is(ScheduledTaskError); +const isOrchestratorDispatchError = Schema.is(OrchestratorDispatchError); const PROMPT_TOO_LONG = "The filled-in prompt is too long."; const WEBHOOK_DELIVERY_RETENTION = 50; const WEBHOOK_DEDUPE_DELIVERY_RETENTION = 20; @@ -364,6 +367,14 @@ function errorMessage(error: unknown): string { return failure instanceof Error ? failure.message : String(failure); } +function dispatchErrorMessage(cause: Cause.Cause): string { + const failure = Cause.squash(cause); + // Direct orchestration refusals are user-facing; nested defects can contain private data. + return isOrchestratorDispatchError(failure) && typeof failure.cause === "string" + ? (userFacingDispatchErrorMessage(failure.cause) ?? errorMessage(failure)) + : errorMessage(failure); +} + function webhookPath(taskId: string, token: string): string { return `${WEBHOOK_ROUTE_PREFIX}/${encodeURIComponent(taskId)}/${token}`; } @@ -1098,7 +1109,7 @@ export const layer = Layer.effect( if (webhook !== undefined && runSucceeded) yield* Ref.set(webhook.dispatched, true); const completedAt = yield* localNow; const lastRunStatus = runSucceeded ? ("succeeded" as const) : ("failed" as const); - const lastRunError = runSucceeded ? null : errorMessage(result.cause); + const lastRunError = runSucceeded ? null : dispatchErrorMessage(result.cause); const threadId = runSucceeded ? result.value : null; // One transaction, so an edit cannot land between reading the schedule and // writing the next run computed from it. The origin is read first: it reads @@ -1511,9 +1522,11 @@ export const layer = Layer.effect( return { token, secret, secretChanged }; }) : { token: null, secret: null, secretChanged: true }; + // A folder that is not a Git repository, Scratch included, cannot hold a worktree. const workspaceStrategy: ScheduledTask["workspaceStrategy"] = input.workspaceStrategy.type === "worktree" && - (yield* managedFolders.isScratchProject(input.projectId)) + ((yield* managedFolders.isScratchProject(input.projectId)) || + (yield* threadLaunch.isNonGitProject(input.projectId))) ? { type: "root" } : input.workspaceStrategy; const scheduleUnchanged = diff --git a/apps/server/src/secrets/SecretCheck.test.ts b/apps/server/src/secrets/SecretCheck.test.ts index 9c0bad4cd3111..473cab1dfc65c 100644 --- a/apps/server/src/secrets/SecretCheck.test.ts +++ b/apps/server/src/secrets/SecretCheck.test.ts @@ -228,7 +228,7 @@ it.layer(layerTest)("SecretCheck", (it) => { ]); assert.deepStrictEqual(server.refreshed, [dependent]); assert.deepStrictEqual(yield* server.registry.listUnavailable, []); - }).pipe(Effect.scoped), + }), ); it.effect("an unreadable secret fails naming its provider and leaves instances alone", () => @@ -250,7 +250,7 @@ it.layer(layerTest)("SecretCheck", (it) => { (yield* server.registry.listUnavailable).map((entry) => entry.instanceId), [dependent], ); - }).pipe(Effect.scoped), + }), ); it.effect("a failed check keeps the value the server already holds", () => @@ -267,7 +267,7 @@ it.layer(layerTest)("SecretCheck", (it) => { assert.deepStrictEqual(yield* server.resolver.read([locked]), ["locked-key"]); assert.deepStrictEqual(server.reads.slice(readsBefore), [locked.reference]); - }).pipe(Effect.scoped), + }), ); it.effect("a check applies to an instance whose other secret failed a later refresh", () => @@ -289,7 +289,7 @@ it.layer(layerTest)("SecretCheck", (it) => { secrets: ["wanted-new", "locked-key"], }); assert.deepStrictEqual(yield* server.registry.listUnavailable, []); - }).pipe(Effect.scoped), + }), ); it.effect("a check applies the value it read when a refresh re-reads that secret first", () => @@ -358,6 +358,6 @@ it.layer(layerTest)("SecretCheck", (it) => { assert.deepStrictEqual(server.createdConfigs.at(-1), { keys: { a: "wanted-new", b: "locked-key" }, }); - }).pipe(Effect.scoped), + }), ); }); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 2510c86fd0f8b..47c4a4c9a20ec 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -29,6 +29,8 @@ import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder"; import * as BackgroundPolicy from "./background/BackgroundPolicy.ts"; import * as HostPowerMonitor from "./background/HostPowerMonitor.ts"; +import * as NonBlockingSpawner from "./process/NonBlockingSpawner.ts"; +import * as KeepAwake from "./background/KeepAwake.ts"; import * as ServerConfig from "./config.ts"; import { withUntracedRequests } from "./http.ts"; import * as ServerHttp from "./http.ts"; @@ -42,6 +44,7 @@ import { fixPath } from "./os-jank.ts"; import * as Ws from "./ws.ts"; import * as AgentScopeLive from "./process/agentScope.ts"; import * as ExternalLauncher from "./process/externalLauncher.ts"; +import * as SpawnWorker from "./process/SpawnWorker.ts"; import * as NodePtyAdapter from "./terminal/NodePtyAdapter.ts"; import * as PullRequestHttp from "./pullRequest/http.ts"; import * as SpeechModelStore from "./speech/SpeechModelStore.ts"; @@ -128,6 +131,7 @@ import * as KnownWorkspaces from "./workspace/KnownWorkspaces.ts"; import * as WorkspaceFileSystem from "./workspace/WorkspaceFileSystem.ts"; import * as WorkspacePaths from "./workspace/WorkspacePaths.ts"; import * as GitVcsDriver from "./vcs/GitVcsDriver.ts"; +import * as VcsAutomaticFetchInterval from "./vcs/VcsAutomaticFetchInterval.ts"; import * as VcsDriverRegistry from "./vcs/VcsDriverRegistry.ts"; import * as VcsProjectConfig from "./vcs/VcsProjectConfig.ts"; import * as VcsProcess from "./vcs/VcsProcess.ts"; @@ -174,12 +178,14 @@ import * as ConnectReachability from "./cloud/ConnectReachability.ts"; import { pendingServiceUpdateExists } from "./cloud/updateHandoff.ts"; import * as RelayTracing from "./cloud/relayTracing.ts"; import * as CloudManagedEndpointRuntime from "./cloud/ManagedEndpointRuntime.ts"; +import * as RelayClientLedger from "./cloud/RelayClientLedger.ts"; import { MANAGED_TUNNEL_FIRST_REGISTRATION_JITTER, makeManagedTunnelRecovery, managedTunnelStartupAction, retryDesiredLinkReconcile, retryManagedTunnelRegistration, + watchManagedTunnelOrigin, } from "./cloud/managedTunnelStartup.ts"; import * as CloudCliTokenManager from "./cloud/CliTokenManager.ts"; import * as CloudCliState from "./cloud/CliState.ts"; @@ -343,6 +349,9 @@ const layerRelayClient = Layer.unwrap( }), ).pipe(Layer.provide(ManagedPolicy.layer)); +// NodeServices and NodeHttpServer each output Node's own spawner, so each is followed by this one. +const layerSpawner = SpawnWorker.layerServer.pipe(Layer.provide(NodeServices.layer)); + const layerHttpServer = Layer.unwrap( Effect.gen(function* () { const config = yield* ServerConfig.ServerConfig; @@ -379,12 +388,17 @@ const layerHttpServer = Layer.unwrap( // `threshold` when context takeover is disabled). websocket: { perMessageDeflate: true }, }, - ); + ).pipe(Layer.merge(layerSpawner)); }), -).pipe(Layer.provide(ManagedPolicy.layer)); +).pipe( + Layer.provide(ManagedPolicy.layer), + // NodeHttpServer.layer also outputs NodeServices, whose spawner would shadow the wrapped one. + (layer) => NonBlockingSpawner.layer.pipe(Layer.provideMerge(layer)), +); const layerPlatformServices = DocumentText.layer.pipe( - Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), + Layer.provideMerge(NonBlockingSpawner.layer), + Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer, layerSpawner)), ); const layerPersistence = Layer.empty.pipe(Layer.provideMerge(SqlitePersistence.layerConfig)); @@ -480,7 +494,6 @@ const layerVcs = Layer.empty.pipe( Layer.provideMerge( VcsStatusBroadcaster.layer.pipe( Layer.provide(layerGitWorkflow), - Layer.provide(VcsStatusBroadcaster.layerAutomaticFetchInterval), // The orchestration runtime also consumes the broadcaster (run // finalization), so the policy cannot read the store from the runtime's // output. @@ -493,7 +506,10 @@ const layerVcs = Layer.empty.pipe( const layerCheckpointStore = CheckpointStore.layer.pipe(Layer.provide(layerVcsDriverRegistry)); -const layerPortScanner = PortScanner.layer.pipe(Layer.provide(ProcessRunner.layer)); +const layerPortScanner = PortScanner.layer.pipe( + Layer.provide(ProcessRunner.layer), + Layer.provide(layerBackground), +); const layerTerminal = TerminalManager.layer( ProviderInstanceRegistryHydration.deriveProviderInstanceConfigMap, @@ -546,6 +562,14 @@ const layerCloudManagedEndpointRuntime = Layer.mergeAll( CloudManagedEndpointRuntime.layer.pipe( Layer.provide(ServerSecretStore.layer), Layer.provide(layerRelayClient), + Layer.provide( + Layer.unwrap( + Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + return RelayClientLedger.layer({ stateDir: config.stateDir }); + }), + ), + ), ), ); @@ -573,6 +597,7 @@ const layerScheduledTaskWebhookOrigin = Layer.effect( ); const layerOrchestrationV2Runtime = RuntimeLayer.layerProduction.pipe( + Layer.provide(UsageLimitSources.layerResetFallback), Layer.provide(layerScheduledTaskWebhookOrigin), Layer.provide(ProviderUsageAnalytics.layer), Layer.provide(layerCheckpointStore), @@ -660,6 +685,7 @@ const layerRuntimeCoreDependenciesBase = Layer.mergeAll( Layer.provide(layerServerRestart), ), layerThreadPullRequestWorker, + KeepAwake.layer, DelegatedTaskAnalytics.layer, Layer.effectDiscard( Effect.gen(function* () { @@ -698,15 +724,15 @@ const layerRuntimeCoreDependenciesBase = Layer.mergeAll( Layer.provideMerge(Layer.merge(ProjectStore.layer, ThreadSearch.layer)), Layer.provideMerge(SecretCheck.layer), Layer.provideMerge(SecretResolver.layer), + // Above the settings it writes when a theme is applied. + Layer.provideMerge(EnvironmentTheme.layer), Layer.provideMerge(layerServerSettings), Layer.provideMerge(layerSourceControlProviderRegistry), Layer.provideMerge(layerGit), Layer.provideMerge(layerVcs), Layer.provideMerge(Layer.mergeAll(layerTerminal, layerPreview, layerDevice)), Layer.provideMerge(layerPersistence), - Layer.provideMerge( - Layer.mergeAll(Keybindings.layer, EnvironmentTheme.layer, UsageLimitSources.layer), - ), + Layer.provideMerge(Layer.mergeAll(Keybindings.layer, UsageLimitSources.layer)), Layer.provideMerge(ProviderRegistry.layer), Layer.provideMerge( ProviderInstanceRegistryHydration.layer.pipe(Layer.provide(SqlitePersistence.layerConfig)), @@ -783,6 +809,9 @@ const layerRuntimeDependencies = layerRuntimeCoreDependencies.pipe( Layer.provideMerge(RemoteOpenTargets.layer), Layer.provideMerge(DirectEndpoints.layer.pipe(Layer.provide(layerServerEnvironment))), Layer.provideMerge(ServerLifecycleEvents.layer), + // GitVcsDriver.layer is memoized and some of its build sites live outside this file, so the + // fetch interval goes in context here, before the first build captures it. + Layer.provideMerge(VcsAutomaticFetchInterval.layer.pipe(Layer.provide(layerServerSettings))), Layer.provide(NetService.layer), ); @@ -890,6 +919,17 @@ const layerMakeServer = Layer.unwrap( yield* startup.markHttpListening; }), ); + const layerDesktopReadySignal = Layer.effectDiscard( + Effect.gen(function* () { + const startup = yield* ServerRuntimeStartup.ServerRuntimeStartup; + const receiver = yield* DesktopTelemetryReceiver.DesktopTelemetryReceiver; + yield* startup.awaitCommandReady.pipe( + Effect.andThen(receiver.announceServerReady), + Effect.ignore, + Effect.forkScoped, + ); + }), + ); const layerRuntimeState = Layer.effectDiscard( Effect.gen(function* () { yield* Deferred.succeed(runtimeStateParked, undefined).pipe(Effect.orDie); @@ -1123,6 +1163,28 @@ const layerMakeServer = Layer.unwrap( Stream.runForEach(recoverManagedTunnel), Effect.forkScoped, ); + yield* watchManagedTunnelOrigin({ + localOrigin, + readIngressOrigin: endpointRuntime.inspectConnector.pipe( + Effect.map((connector) => connector?.origin ?? null), + ), + // Like startup, a publish-only link never routes the tunnel to this host. + register: CloudCliState.readCliDesiredLinkMode.pipe( + Effect.flatMap((mode) => + mode === "publish_only" + ? Effect.void + : cloudLink + .registerManagedTunnelRecovery(localOrigin) + .pipe( + Effect.flatMap((registration) => + registration.status === "recovery_required" + ? endpointRuntime.requestRecovery(registration.config) + : Effect.void, + ), + ), + ), + ), + }).pipe(Effect.forkScoped); const wantsCliLink = hasCloudPublicConfig ? yield* CloudCliState.readCliDesiredCloudLink.pipe( Effect.catch((cause) => @@ -1270,6 +1332,7 @@ const layerMakeServer = Layer.unwrap( const layerServerApplication = Layer.mergeAll( layerRoutes, layerHttpListening, + layerDesktopReadySignal, layerRuntimeState.pipe(Layer.provide(layerLauncher)), layerTailscaleServe, layerCloudDesiredLinkReconcile, @@ -1299,4 +1362,7 @@ const layerMakeServer = Layer.unwrap( }), ).pipe(Layer.provide(ProcessRunner.layer)); -export const runServer = Layer.launch(layerMakeServer); +export const runServer = Layer.launch( + // For the layers above whose spawner requirement is not met by one of them. + layerMakeServer.pipe(Layer.provide(NonBlockingSpawner.layer.pipe(Layer.provide(layerSpawner)))), +); diff --git a/apps/server/src/serverLifecycleEvents.ts b/apps/server/src/serverLifecycleEvents.ts index 7d394e4dc9134..d90b7fa57bc35 100644 --- a/apps/server/src/serverLifecycleEvents.ts +++ b/apps/server/src/serverLifecycleEvents.ts @@ -1,5 +1,6 @@ import type { ServerLifecycleStreamEvent } from "@t3tools/contracts"; import * as Context from "effect/Context"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as PubSub from "effect/PubSub"; @@ -22,11 +23,14 @@ export class ServerLifecycleEvents extends Context.Service< readonly publish: (event: LifecycleEventInput) => Effect.Effect; readonly snapshot: Effect.Effect; readonly stream: Stream.Stream; + /** Completes once the first ready event is published. */ + readonly awaitReady: Effect.Effect; } >()("t3/serverLifecycleEvents") {} const make = Effect.gen(function* () { const pubsub = yield* PubSub.unbounded(); + const ready = yield* Deferred.make(); const state = yield* Ref.make({ sequence: 0, events: [], @@ -45,8 +49,14 @@ const make = Effect.gen(function* () { ...current.events.filter((entry) => entry.type !== nextEvent.type), ]; return [nextEvent, { sequence: nextSequence, events: nextEvents }] as const; - }).pipe(Effect.tap((event) => PubSub.publish(pubsub, event))), + }).pipe( + Effect.tap((event) => PubSub.publish(pubsub, event)), + Effect.tap((event) => + event.type === "ready" ? Deferred.succeed(ready, undefined) : Effect.void, + ), + ), snapshot: Ref.get(state), + awaitReady: Deferred.await(ready), get stream() { return Stream.fromPubSub(pubsub); }, diff --git a/apps/server/src/serverRuntimeStartup.autoPull.test.ts b/apps/server/src/serverRuntimeStartup.autoPull.test.ts index 2bc8f31c8f8b3..563bf47f8ae6f 100644 --- a/apps/server/src/serverRuntimeStartup.autoPull.test.ts +++ b/apps/server/src/serverRuntimeStartup.autoPull.test.ts @@ -37,166 +37,162 @@ import * as AnalyticsService from "./telemetry/AnalyticsService.ts"; import * as GitVcsDriver from "./vcs/GitVcsDriver.ts"; it.effect("parks automatic pull until activation without delaying command readiness", () => - Effect.scoped( - Effect.gen(function* () { - const activation = yield* Deferred.make(); - const prepared = yield* Deferred.make(); - const commitTrial = yield* Deferred.make(); - const statusCalled = yield* Deferred.make(); - const statusInterrupted = yield* Deferred.make(); - const cwd = "/auto-pull-project"; - const updatedAt = "2026-01-01T00:00:00.000Z"; - const snapshot = { - projects: [ - { - id: ProjectId.make("auto-pull-project"), - title: "Auto pull project", - workspaceRoot: cwd, - repositoryIdentity: null, - defaultModelSelection: null, - scripts: [], - createdAt: updatedAt, - updatedAt, - deletedAt: null, - }, - ], - updatedAt, - }; - const recovery = { - terminalizedRuns: 0, - stoppedSessions: 0, - closedRequests: 0, - retiredEffects: 0, - requeuedEffects: 0, - }; - const importSummary = { importedThreadCount: 0, importedMessageCount: 0 }; - const dependencies: Layer.Layer< - Layer.Services> - > = Layer.mergeAll( - Layer.mock(ServerConfig.ServerConfig)({ - ...(yield* ServerConfig.deriveServerPaths(cwd, undefined).pipe( - Effect.provide(Path.layer), - )), - baseDir: cwd, - logLevel: "Error", - traceMinLevel: "Info", - traceTimingEnabled: false, - traceBatchWindowMs: 200, - traceMaxBytes: 1024, - traceMaxFiles: 1, - otlpTracesUrl: undefined, - otlpMetricsUrl: undefined, - otlpLogsUrl: undefined, - otlpTracesExport: DEFAULT_SIGNAL_EXPORT, - otlpMetricsExport: DEFAULT_SIGNAL_EXPORT, - otlpLogsExport: DEFAULT_SIGNAL_EXPORT, - otelEnvironment: OtelEnvironment.none, - staticDir: undefined, - devAllowedOrigins: [], - desktopBootstrapToken: undefined, - logWebSocketEvents: false, - tailscaleServeEnabled: false, - tailscaleServePort: 443, - mode: "desktop", - cwd, - host: "localhost", - port: 3773, - devUrl: undefined, - noBrowser: true, - startupPresentation: "browser", - autoBootstrapProjectFromCwd: false, + Effect.gen(function* () { + const activation = yield* Deferred.make(); + const prepared = yield* Deferred.make(); + const commitTrial = yield* Deferred.make(); + const statusCalled = yield* Deferred.make(); + const statusInterrupted = yield* Deferred.make(); + const cwd = "/auto-pull-project"; + const updatedAt = "2026-01-01T00:00:00.000Z"; + const snapshot = { + projects: [ + { + id: ProjectId.make("auto-pull-project"), + title: "Auto pull project", + workspaceRoot: cwd, + repositoryIdentity: null, + defaultModelSelection: null, + scripts: [], + createdAt: updatedAt, + updatedAt, + deletedAt: null, + }, + ], + updatedAt, + }; + const recovery = { + terminalizedRuns: 0, + stoppedSessions: 0, + closedRequests: 0, + retiredEffects: 0, + requeuedEffects: 0, + }; + const importSummary = { importedThreadCount: 0, importedMessageCount: 0 }; + const dependencies: Layer.Layer< + Layer.Services> + > = Layer.mergeAll( + Layer.mock(ServerConfig.ServerConfig)({ + ...(yield* ServerConfig.deriveServerPaths(cwd, undefined).pipe(Effect.provide(Path.layer))), + baseDir: cwd, + logLevel: "Error", + traceMinLevel: "Info", + traceTimingEnabled: false, + traceBatchWindowMs: 200, + traceMaxBytes: 1024, + traceMaxFiles: 1, + otlpTracesUrl: undefined, + otlpMetricsUrl: undefined, + otlpLogsUrl: undefined, + otlpTracesExport: DEFAULT_SIGNAL_EXPORT, + otlpMetricsExport: DEFAULT_SIGNAL_EXPORT, + otlpLogsExport: DEFAULT_SIGNAL_EXPORT, + otelEnvironment: OtelEnvironment.none, + staticDir: undefined, + devAllowedOrigins: [], + desktopBootstrapToken: undefined, + logWebSocketEvents: false, + tailscaleServeEnabled: false, + tailscaleServePort: 443, + mode: "desktop", + cwd, + host: "localhost", + port: 3773, + devUrl: undefined, + noBrowser: true, + startupPresentation: "browser", + autoBootstrapProjectFromCwd: false, + }), + Layer.mock(Keybindings.Keybindings)({ start: Effect.void }), + Layer.mock(LegacyV1ThreadImporter.LegacyV1ThreadImporter)({ + pendingThreadCount: Effect.succeed(0), + reconcileShells: Effect.succeed(importSummary), + importPendingTranscripts: Effect.succeed(importSummary), + }), + Layer.mock(ProviderRuntimeRecovery.ProviderRuntimeRecoveryService)({ + recover: Effect.succeed(recovery), + prepareForShutdown: Effect.void, + reconcile: () => Effect.succeed(recovery), + }), + Layer.mock(Orchestrator.OrchestratorV2)({ recoverDelegatedTasks: Effect.void }), + Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ shutdown: Effect.void }), + Layer.mock(AgentAwarenessRelay.AgentAwarenessRelay)({ start: () => Effect.void }), + Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ runOnce: () => Effect.never }), + Layer.mock(ServerLifecycleEvents.ServerLifecycleEvents)({ + publish: (event) => Effect.succeed({ ...event, sequence: 1 }), + }), + Layer.mock(ServerSettings.ServerSettingsService)({ + start: Effect.void, + getSettings: Effect.succeed({ ...DEFAULT_SERVER_SETTINGS, defaultAutoPull: true }), + }), + Layer.mock(ServerEnvironment.ServerEnvironment)({ + getDescriptor: Effect.succeed({ + environmentId: EnvironmentId.make("auto-pull-environment"), + label: "Test environment", + platform: { os: "darwin", arch: "arm64" }, + serverVersion: "0.0.0-test", + capabilities: { repositoryIdentity: true }, }), - Layer.mock(Keybindings.Keybindings)({ start: Effect.void }), - Layer.mock(LegacyV1ThreadImporter.LegacyV1ThreadImporter)({ - pendingThreadCount: Effect.succeed(0), - reconcileShells: Effect.succeed(importSummary), - importPendingTranscripts: Effect.succeed(importSummary), - }), - Layer.mock(ProviderRuntimeRecovery.ProviderRuntimeRecoveryService)({ - recover: Effect.succeed(recovery), - prepareForShutdown: Effect.void, - reconcile: () => Effect.succeed(recovery), - }), - Layer.mock(Orchestrator.OrchestratorV2)({ recoverDelegatedTasks: Effect.void }), - Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ shutdown: Effect.void }), - Layer.mock(AgentAwarenessRelay.AgentAwarenessRelay)({ start: () => Effect.void }), - Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ runOnce: () => Effect.never }), - Layer.mock(ServerLifecycleEvents.ServerLifecycleEvents)({ - publish: (event) => Effect.succeed({ ...event, sequence: 1 }), - }), - Layer.mock(ServerSettings.ServerSettingsService)({ - start: Effect.void, - getSettings: Effect.succeed({ ...DEFAULT_SERVER_SETTINGS, defaultAutoPull: true }), - }), - Layer.mock(ServerEnvironment.ServerEnvironment)({ - getDescriptor: Effect.succeed({ - environmentId: EnvironmentId.make("auto-pull-environment"), - label: "Test environment", - platform: { os: "darwin", arch: "arm64" }, - serverVersion: "0.0.0-test", - capabilities: { repositoryIdentity: true }, - }), - }), - Layer.mock(ProjectStore.ProjectStoreV2)({ - listShells: () => Effect.succeed(snapshot.projects), - }), - Layer.mock(ProjectService.ProjectService)({ snapshot: Effect.succeed(snapshot) }), - Layer.mock(ThreadManagement.ThreadManagementService)({ - countThreads: () => Effect.succeed(0), - }), - Layer.mock(ThreadLaunch.ThreadLaunchService)({}), - Layer.mock(ServiceLauncherClient.ServiceLauncherClient)({ - managed: true, - protocol: SERVICE_LAUNCHER_PROTOCOL, - prepareTrial: () => - Deferred.succeed(prepared, undefined).pipe( - Effect.andThen(Deferred.await(commitTrial)), - Effect.as(undefined), - ), - }), - Layer.mock(GitVcsDriver.GitVcsDriver)({ - statusDetails: (root) => - Deferred.succeed(statusCalled, root).pipe( - Effect.andThen(Effect.never), - Effect.onInterrupt(() => Deferred.succeed(statusInterrupted, undefined)), - ), - }), - Layer.mock(AnalyticsService.AnalyticsService)({ record: () => Effect.void }), - // No provider logins or Connect link, so startup records no owners. - FileSystem.layerNoop({}), - Layer.mock(ServerSecretStore.ServerSecretStore)({ get: () => Effect.succeedNone }), - NodeCrypto.layer, - Layer.mock(EnvironmentAuth.EnvironmentAuth)({}), - Layer.mock(ExternalLauncher.ExternalLauncher)({}), - Layer.mock(HttpServer.HttpServer)({ - address: NetAddress.inetAddressFromIpStringUnsafe("127.0.0.1", 3773), - }), - Path.layer, - ); + }), + Layer.mock(ProjectStore.ProjectStoreV2)({ + listShells: () => Effect.succeed(snapshot.projects), + }), + Layer.mock(ProjectService.ProjectService)({ snapshot: Effect.succeed(snapshot) }), + Layer.mock(ThreadManagement.ThreadManagementService)({ + countThreads: () => Effect.succeed(0), + }), + Layer.mock(ThreadLaunch.ThreadLaunchService)({}), + Layer.mock(ServiceLauncherClient.ServiceLauncherClient)({ + managed: true, + protocol: SERVICE_LAUNCHER_PROTOCOL, + prepareTrial: () => + Deferred.succeed(prepared, undefined).pipe( + Effect.andThen(Deferred.await(commitTrial)), + Effect.as(undefined), + ), + }), + Layer.mock(GitVcsDriver.GitVcsDriver)({ + statusDetails: (root) => + Deferred.succeed(statusCalled, root).pipe( + Effect.andThen(Effect.never), + Effect.onInterrupt(() => Deferred.succeed(statusInterrupted, undefined)), + ), + }), + Layer.mock(AnalyticsService.AnalyticsService)({ record: () => Effect.void }), + // No provider logins or Connect link, so startup records no owners. + FileSystem.layerNoop({}), + Layer.mock(ServerSecretStore.ServerSecretStore)({ get: () => Effect.succeedNone }), + NodeCrypto.layer, + Layer.mock(EnvironmentAuth.EnvironmentAuth)({}), + Layer.mock(ExternalLauncher.ExternalLauncher)({}), + Layer.mock(HttpServer.HttpServer)({ + address: NetAddress.inetAddressFromIpStringUnsafe("127.0.0.1", 3773), + }), + Path.layer, + ); - yield* Effect.gen(function* () { - const startup = yield* ServerRuntimeStartup.ServerRuntimeStartup; - yield* startup.markHttpListening; + yield* Effect.gen(function* () { + const startup = yield* ServerRuntimeStartup.ServerRuntimeStartup; + yield* startup.markHttpListening; - yield* Effect.raceFirst(Deferred.await(prepared), Deferred.await(statusCalled)); - expect(yield* Deferred.isDone(activation)).toBe(false); - expect(yield* Deferred.isDone(statusCalled)).toBe(false); - expect(yield* Deferred.isDone(prepared)).toBe(true); + yield* Effect.raceFirst(Deferred.await(prepared), Deferred.await(statusCalled)); + expect(yield* Deferred.isDone(activation)).toBe(false); + expect(yield* Deferred.isDone(statusCalled)).toBe(false); + expect(yield* Deferred.isDone(prepared)).toBe(true); - yield* Deferred.succeed(commitTrial, undefined); - yield* Deferred.await(activation); - expect(yield* Deferred.await(statusCalled)).toBe(cwd); - yield* startup.awaitCommandReady; - }).pipe( - Effect.provide( - ServerRuntimeStartup.layerWithOptions({ - activate: Deferred.succeed(activation, undefined).pipe(Effect.asVoid), - awaitAuxiliaryParked: Effect.void, - }).pipe(Layer.provide(dependencies)), - ), - Effect.provideService(ServerActivation, Deferred.await(activation)), - ); - expect(yield* Deferred.isDone(statusInterrupted)).toBe(true); - }), - ), + yield* Deferred.succeed(commitTrial, undefined); + yield* Deferred.await(activation); + expect(yield* Deferred.await(statusCalled)).toBe(cwd); + yield* startup.awaitCommandReady; + }).pipe( + Effect.provide( + ServerRuntimeStartup.layerWithOptions({ + activate: Deferred.succeed(activation, undefined).pipe(Effect.asVoid), + awaitAuxiliaryParked: Effect.void, + }).pipe(Layer.provide(dependencies)), + ), + Effect.provideService(ServerActivation, Deferred.await(activation)), + ); + expect(yield* Deferred.isDone(statusInterrupted)).toBe(true); + }), ); diff --git a/apps/server/src/serverRuntimeStartup.test.ts b/apps/server/src/serverRuntimeStartup.test.ts index 180defacbfcc6..2b8d9874a8fe1 100644 --- a/apps/server/src/serverRuntimeStartup.test.ts +++ b/apps/server/src/serverRuntimeStartup.test.ts @@ -52,110 +52,114 @@ it.effect.each([ "disabled-default", "launch-failure", "project-failure", + "project-refused", ] as const)("keeps bootstrap launch failures separate from project failures: %s", (scenario) => - Effect.scoped( - Effect.gen(function* () { - const projectId = ProjectId.make("bootstrap-project"); - const instanceId = ProviderInstanceId.make( - scenario === "disabled-default" ? "custom" : "codex", - ); - const events: ServerLifecycleStreamEvent[] = []; - const recovery = { - terminalizedRuns: 0, - stoppedSessions: 0, - closedRequests: 0, - retiredEffects: 0, - requeuedEffects: 0, - }; - const imports = { importedThreadCount: 0, importedMessageCount: 0 }; - const dependencies = Layer.mergeAll( - NodeCrypto.layer, - Path.layer, - Layer.mock(ServerConfig.ServerConfig)({ - ...(yield* ServerConfig.deriveServerPaths("/bootstrap-project", undefined).pipe( - Effect.provide(Path.layer), - )), - baseDir: "/bootstrap-project", - logLevel: "Error", - traceMinLevel: "Info", - traceTimingEnabled: false, - traceBatchWindowMs: 200, - traceMaxBytes: 1024, - traceMaxFiles: 1, - otlpTracesUrl: undefined, - otlpMetricsUrl: undefined, - otlpLogsUrl: undefined, - otlpTracesExport: DEFAULT_SIGNAL_EXPORT, - otlpMetricsExport: DEFAULT_SIGNAL_EXPORT, - otlpLogsExport: DEFAULT_SIGNAL_EXPORT, - otelEnvironment: OtelEnvironment.none, - staticDir: undefined, - devAllowedOrigins: [], - desktopBootstrapToken: undefined, - logWebSocketEvents: false, - tailscaleServeEnabled: false, - tailscaleServePort: 443, - mode: "web", - cwd: "/bootstrap-project", - host: "localhost", - port: 3773, - devUrl: undefined, - noBrowser: true, - startupPresentation: "browser", - autoBootstrapProjectFromCwd: true, - }), - Layer.mock(Keybindings.Keybindings)({ start: Effect.void }), - Layer.mock(LegacyV1ThreadImporter.LegacyV1ThreadImporter)({ - pendingThreadCount: Effect.succeed(0), - reconcileShells: Effect.succeed(imports), - importPendingTranscripts: Effect.succeed(imports), - }), - Layer.mock(ProviderRuntimeRecovery.ProviderRuntimeRecoveryService)({ - recover: Effect.succeed(recovery), - prepareForShutdown: Effect.void, - reconcile: () => Effect.succeed(recovery), - }), - Layer.mock(Orchestrator.OrchestratorV2)({ recoverDelegatedTasks: Effect.void }), - Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ shutdown: Effect.void }), - Layer.mock(AgentAwarenessRelay.AgentAwarenessRelay)({ start: () => Effect.void }), - Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ runOnce: () => Effect.never }), - Layer.mock(ServerLifecycleEvents.ServerLifecycleEvents)({ - publish: (event) => - Effect.sync(() => { - const published = { ...event, sequence: events.length + 1 }; - events.push(published); - return published; - }), - }), - Layer.mock(ServerSettings.ServerSettingsService)({ - start: Effect.void, - getSettings: Effect.succeed({ - ...DEFAULT_SERVER_SETTINGS, - defaultModelSelection: - scenario === "disabled-default" ? { instanceId, model: "custom-model" } : null, - providerInstances: { - [instanceId]: { - driver: ProviderDriverKind.make("codex"), - enabled: false, - config: {}, - }, - }, + Effect.gen(function* () { + const projectId = ProjectId.make("bootstrap-project"); + const instanceId = ProviderInstanceId.make( + scenario === "disabled-default" ? "custom" : "codex", + ); + const events: ServerLifecycleStreamEvent[] = []; + const recovery = { + terminalizedRuns: 0, + stoppedSessions: 0, + closedRequests: 0, + retiredEffects: 0, + requeuedEffects: 0, + }; + const imports = { importedThreadCount: 0, importedMessageCount: 0 }; + const dependencies = Layer.mergeAll( + NodeCrypto.layer, + Path.layer, + Layer.mock(ServerConfig.ServerConfig)({ + ...(yield* ServerConfig.deriveServerPaths("/bootstrap-project", undefined).pipe( + Effect.provide(Path.layer), + )), + baseDir: "/bootstrap-project", + logLevel: "Error", + traceMinLevel: "Info", + traceTimingEnabled: false, + traceBatchWindowMs: 200, + traceMaxBytes: 1024, + traceMaxFiles: 1, + otlpTracesUrl: undefined, + otlpMetricsUrl: undefined, + otlpLogsUrl: undefined, + otlpTracesExport: DEFAULT_SIGNAL_EXPORT, + otlpMetricsExport: DEFAULT_SIGNAL_EXPORT, + otlpLogsExport: DEFAULT_SIGNAL_EXPORT, + otelEnvironment: OtelEnvironment.none, + staticDir: undefined, + devAllowedOrigins: [], + desktopBootstrapToken: undefined, + logWebSocketEvents: false, + tailscaleServeEnabled: false, + tailscaleServePort: 443, + mode: "web", + cwd: "/bootstrap-project", + host: "localhost", + port: 3773, + devUrl: undefined, + noBrowser: true, + startupPresentation: "browser", + autoBootstrapProjectFromCwd: true, + }), + Layer.mock(Keybindings.Keybindings)({ start: Effect.void }), + Layer.mock(LegacyV1ThreadImporter.LegacyV1ThreadImporter)({ + pendingThreadCount: Effect.succeed(0), + reconcileShells: Effect.succeed(imports), + importPendingTranscripts: Effect.succeed(imports), + }), + Layer.mock(ProviderRuntimeRecovery.ProviderRuntimeRecoveryService)({ + recover: Effect.succeed(recovery), + prepareForShutdown: Effect.void, + reconcile: () => Effect.succeed(recovery), + }), + Layer.mock(Orchestrator.OrchestratorV2)({ recoverDelegatedTasks: Effect.void }), + Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ shutdown: Effect.void }), + Layer.mock(AgentAwarenessRelay.AgentAwarenessRelay)({ start: () => Effect.void }), + Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ runOnce: () => Effect.never }), + Layer.mock(ServerLifecycleEvents.ServerLifecycleEvents)({ + publish: (event) => + Effect.sync(() => { + const published = { ...event, sequence: events.length + 1 }; + events.push(published); + return published; }), + }), + Layer.mock(ServerSettings.ServerSettingsService)({ + start: Effect.void, + getSettings: Effect.succeed({ + ...DEFAULT_SERVER_SETTINGS, + defaultModelSelection: + scenario === "disabled-default" ? { instanceId, model: "custom-model" } : null, + providerInstances: { + [instanceId]: { + driver: ProviderDriverKind.make("codex"), + enabled: false, + config: {}, + }, + }, }), - Layer.mock(ServerEnvironment.ServerEnvironment)({ - getDescriptor: Effect.succeed({ - environmentId: EnvironmentId.make("bootstrap-environment"), - label: "Test environment", - platform: { os: "linux", arch: "x64" }, - serverVersion: "0.0.0-test", - capabilities: { repositoryIdentity: true }, - }), + }), + Layer.mock(ServerEnvironment.ServerEnvironment)({ + getDescriptor: Effect.succeed({ + environmentId: EnvironmentId.make("bootstrap-environment"), + label: "Test environment", + platform: { os: "linux", arch: "x64" }, + serverVersion: "0.0.0-test", + capabilities: { repositoryIdentity: true }, }), - Layer.mock(ProjectStore.ProjectStoreV2)({}), - Layer.mock(ProjectService.ProjectService)({ - bootstrap: () => - scenario === "project-failure" - ? Effect.die("project bootstrap failed") + }), + Layer.mock(ProjectStore.ProjectStoreV2)({}), + Layer.mock(ProjectService.ProjectService)({ + bootstrap: () => + scenario === "project-failure" + ? Effect.die("project bootstrap failed") + : scenario === "project-refused" + ? Effect.fail( + new ProjectService.ProjectInvalidError({ projectId, detail: "refused folder" }), + ) : Effect.succeed({ created: true, project: { @@ -174,131 +178,127 @@ it.effect.each([ : null, }, }), - }), - Layer.mock(ThreadManagement.ThreadManagementService)({ - countThreads: () => Effect.succeed(0), - listActiveShellIndex: () => Effect.succeed([]), - }), - Layer.mock(ThreadLaunch.ThreadLaunchService)({ - launch: (input) => { - assert.equal(input.modelSelection.instanceId, instanceId); - return scenario === "launch-failure" - ? Effect.fail( - new ThreadLaunch.ThreadLaunchError({ - operation: "create-thread", - commandId: CommandId.make("bootstrap-command"), - projectId, - cause: "test launch failure", - }), - ) - : Effect.fail( - new ThreadManagement.ThreadManagementProviderInstanceDisabledError({ - projectId, - instanceId, - }), - ); - }, - }), - Layer.mock(ServiceLauncherClient.ServiceLauncherClient)({ - managed: false, - protocol: undefined, - prepareTrial: () => Effect.undefined, - }), - Layer.mock(GitVcsDriver.GitVcsDriver)({}), - Layer.mock(AnalyticsService.AnalyticsService)({}), - // No provider logins or Connect link, so startup records no owners. - FileSystem.layerNoop({}), - Layer.mock(ServerSecretStore.ServerSecretStore)({ get: () => Effect.succeedNone }), - Layer.mock(EnvironmentAuth.EnvironmentAuth)({}), - Layer.mock(ExternalLauncher.ExternalLauncher)({}), - Layer.mock(HttpServer.HttpServer)({ - address: NetAddress.inetAddressFromIpStringUnsafe("127.0.0.1", 3773), - }), - ); - yield* Effect.gen(function* () { - const startup = yield* ServerRuntimeStartup.ServerRuntimeStartup; - yield* startup.markHttpListening; - const ready = yield* Effect.exit(startup.awaitCommandReady); - assert.equal(Exit.isSuccess(ready), scenario !== "project-failure"); - const welcome = events.find((event) => event.type === "welcome"); - if (scenario === "project-failure") { - assert.isUndefined(welcome); - } else { - assert.isDefined(welcome); - if (welcome?.type === "welcome") { - assert.equal(welcome.payload.bootstrapProjectId, projectId); - assert.isUndefined(welcome.payload.bootstrapThreadId); - } + }), + Layer.mock(ThreadManagement.ThreadManagementService)({ + countThreads: () => Effect.succeed(0), + listActiveShellIndex: () => Effect.succeed([]), + }), + Layer.mock(ThreadLaunch.ThreadLaunchService)({ + launch: (input) => { + assert.equal(input.modelSelection.instanceId, instanceId); + return scenario === "launch-failure" + ? Effect.fail( + new ThreadLaunch.ThreadLaunchError({ + operation: "create-thread", + commandId: CommandId.make("bootstrap-command"), + projectId, + cause: "test launch failure", + }), + ) + : Effect.fail( + new ThreadManagement.ThreadManagementProviderInstanceDisabledError({ + projectId, + instanceId, + }), + ); + }, + }), + Layer.mock(ServiceLauncherClient.ServiceLauncherClient)({ + managed: false, + protocol: undefined, + prepareTrial: () => Effect.undefined, + }), + Layer.mock(GitVcsDriver.GitVcsDriver)({}), + Layer.mock(AnalyticsService.AnalyticsService)({}), + // No provider logins or Connect link, so startup records no owners. + FileSystem.layerNoop({}), + Layer.mock(ServerSecretStore.ServerSecretStore)({ get: () => Effect.succeedNone }), + Layer.mock(EnvironmentAuth.EnvironmentAuth)({}), + Layer.mock(ExternalLauncher.ExternalLauncher)({}), + Layer.mock(HttpServer.HttpServer)({ + address: NetAddress.inetAddressFromIpStringUnsafe("127.0.0.1", 3773), + }), + ); + yield* Effect.gen(function* () { + const startup = yield* ServerRuntimeStartup.ServerRuntimeStartup; + yield* startup.markHttpListening; + const ready = yield* Effect.exit(startup.awaitCommandReady); + assert.equal(Exit.isSuccess(ready), scenario !== "project-failure"); + const welcome = events.find((event) => event.type === "welcome"); + if (scenario === "project-failure") { + assert.isUndefined(welcome); + } else { + assert.isDefined(welcome); + if (welcome?.type === "welcome") { + assert.equal( + welcome.payload.bootstrapProjectId, + scenario === "project-refused" ? undefined : projectId, + ); + assert.isUndefined(welcome.payload.bootstrapThreadId); } - }).pipe( - Effect.provide(ServerRuntimeStartup.layerWithOptions().pipe(Layer.provide(dependencies))), - Effect.provideService(ServerActivation, Effect.never), - ); - }), - ), + } + }).pipe( + Effect.provide(ServerRuntimeStartup.layerWithOptions().pipe(Layer.provide(dependencies))), + Effect.provideService(ServerActivation, Effect.never), + ); + }), ); it.effect("interrupts the effect worker when awareness relay startup fails", () => - Effect.scoped( - Effect.gen(function* () { - const workerInterrupted = yield* Ref.make(false); - const workerFiberRef = yield* Ref.make | null>(null); + Effect.gen(function* () { + const workerInterrupted = yield* Ref.make(false); + const workerFiberRef = yield* Ref.make | null>(null); - const exit = yield* ServerRuntimeStartup.startEffectWorkerWithRelay({ - runWorker: Effect.never.pipe(Effect.ensuring(Ref.set(workerInterrupted, true))), - startRelay: Effect.yieldNow.pipe( - Effect.andThen(Effect.die("awareness relay startup failed")), - ), - workerFiberRef, - }).pipe(Effect.exit); + const exit = yield* ServerRuntimeStartup.startEffectWorkerWithRelay({ + runWorker: Effect.never.pipe(Effect.ensuring(Ref.set(workerInterrupted, true))), + startRelay: Effect.yieldNow.pipe( + Effect.andThen(Effect.die("awareness relay startup failed")), + ), + workerFiberRef, + }).pipe(Effect.exit); - assert.isTrue(Exit.isFailure(exit)); - assert.isTrue(yield* Ref.get(workerInterrupted)); - assert.isNull(yield* Ref.get(workerFiberRef)); - }), - ), + assert.isTrue(Exit.isFailure(exit)); + assert.isTrue(yield* Ref.get(workerInterrupted)); + assert.isNull(yield* Ref.get(workerFiberRef)); + }), ); it.effect("queues commands until startup signals readiness", () => - Effect.scoped( - Effect.gen(function* () { - const gate = yield* ServerRuntimeStartup.makeCommandGate; - const count = yield* Ref.make(0); - const queued = yield* gate - .enqueueCommand(Ref.updateAndGet(count, (value) => value + 1)) - .pipe(Effect.forkScoped); + Effect.gen(function* () { + const gate = yield* ServerRuntimeStartup.makeCommandGate; + const count = yield* Ref.make(0); + const queued = yield* gate + .enqueueCommand(Ref.updateAndGet(count, (value) => value + 1)) + .pipe(Effect.forkScoped); - yield* Effect.yieldNow; - assert.equal(yield* Ref.get(count), 0); - yield* gate.signalCommandReady; - assert.equal(yield* Fiber.join(queued), 1); - }), - ), + yield* Effect.yieldNow; + assert.equal(yield* Ref.get(count), 0); + yield* gate.signalCommandReady; + assert.equal(yield* Fiber.join(queued), 1); + }), ); it.effect("enqueueCommand fails queued work when readiness fails", () => - Effect.scoped( - Effect.gen(function* () { - const commandGate = yield* ServerRuntimeStartup.makeCommandGate; - const failure = yield* Deferred.make(); + Effect.gen(function* () { + const commandGate = yield* ServerRuntimeStartup.makeCommandGate; + const failure = yield* Deferred.make(); - const queuedCommandFiber = yield* commandGate - .enqueueCommand(Deferred.await(failure).pipe(Effect.as("should-not-run"))) - .pipe(Effect.forkScoped); + const queuedCommandFiber = yield* commandGate + .enqueueCommand(Deferred.await(failure).pipe(Effect.as("should-not-run"))) + .pipe(Effect.forkScoped); - yield* commandGate.failCommandReady( - new ServerRuntimeStartup.ServerRuntimeStartupError({ - mode: "web", - host: "127.0.0.1", - port: 3773, - cause: new Error("test startup failure"), - }), - ); + yield* commandGate.failCommandReady( + new ServerRuntimeStartup.ServerRuntimeStartupError({ + mode: "web", + host: "127.0.0.1", + port: 3773, + cause: new Error("test startup failure"), + }), + ); - const error = yield* Effect.flip(Fiber.join(queuedCommandFiber)); - assert.equal(error.message, "Server runtime startup failed before command readiness."); - }), - ), + const error = yield* Effect.flip(Fiber.join(queuedCommandFiber)); + assert.equal(error.message, "Server runtime startup failed before command readiness."); + }), ); it.effect("automatic pull only updates enabled, behind, clean default-branch checkouts", () => diff --git a/apps/server/src/serverRuntimeStartup.ts b/apps/server/src/serverRuntimeStartup.ts index 689659ab23f59..0d84e99169e1b 100644 --- a/apps/server/src/serverRuntimeStartup.ts +++ b/apps/server/src/serverRuntimeStartup.ts @@ -270,12 +270,24 @@ const resolveAutoBootstrapWelcomeTargets = Effect.gen(function* () { if (serverConfig.autoBootstrapProjectFromCwd) { const threadModelSelection = getAutoBootstrapThreadModelSelection(); - const { project } = yield* projects.bootstrap({ - commandId: CommandId.make(yield* randomUUID), - projectId: ProjectId.make(yield* randomUUID), - title: path.basename(serverConfig.cwd) || "project", - workspaceRoot: serverConfig.cwd, - }); + const bootstrapped = yield* projects + .bootstrap({ + commandId: CommandId.make(yield* randomUUID), + projectId: ProjectId.make(yield* randomUUID), + title: path.basename(serverConfig.cwd) || "project", + workspaceRoot: serverConfig.cwd, + }) + .pipe( + // A folder the project rules refuse must not stop the server from starting there. + Effect.catchTags({ + ProjectInvalidError: (error) => + Effect.logWarning("Skipped bootstrap project", { detail: error.detail }).pipe( + Effect.as(undefined), + ), + }), + ); + if (bootstrapped === undefined) return {} satisfies AutoBootstrapWelcomeTargets; + const { project } = bootstrapped; const existingThread = (yield* threads.listActiveShellIndex()).find( (thread) => thread.projectId === project.id && isListedThreadLineage(thread.lineage), ); diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index 3ca2d29da7c81..873649cb168f5 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -1469,7 +1469,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" { kind: "eas", id: "eas", - label: "EAS Simulator", + label: "Expo Simulators", accessToken: "expo-secret", maxSessionMinutes: 30, idleMinutes: 10, @@ -1786,7 +1786,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" { kind: "eas", id: "eas", - label: "EAS Simulator", + label: "Expo Simulators", accessToken: easReference, maxSessionMinutes: 30, idleMinutes: 10, @@ -1906,7 +1906,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" { kind: "eas", id: "eas", - label: "EAS Simulator", + label: "Expo Simulators", accessToken: personal("Expo/token"), maxSessionMinutes: 30, idleMinutes: 10, diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index bd0da06157db6..4bfe776a3aba8 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -29,6 +29,7 @@ import { readServiceState, registerServiceTask, serviceLauncherIsRunning, + serviceLauncherIsServing, serviceLauncherLockPath, stateDirOwned, writeServiceState, @@ -201,6 +202,212 @@ const fakeServiceUnit = ( }); it.layer(NodeServices.layer)("service state persistence", (it) => { + it.effect.each([ + "startup", + "log write fails", + "home probe error", + "database probe error", + "pending update", + "update handoff", + "trial exit", + "failed trial start", + "stop", + ] as const)("waits for disk recovery during %s without losing the update lifecycle", (scenario) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-disk-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const databasePath = path.join(root, "userdata", "statev2.sqlite"); + let backupPath = path.join(root, "runtime", "db-backup", "disk-update"); + const pending = { + id: "disk-update", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + dbPath: databasePath, + status: "pending" as const, + }; + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); + yield* fs.writeFileString(databasePath, "before trial"); + const childSource = ` +const fs = require("node:fs"); +const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); +if (context.update?.status === "pending") { + if (context.childVersion !== context.update.targetVersion) throw new Error("wrong trial version"); + if (fs.readFileSync(${JSON.stringify(path.dirname(backupPath))} + "/" + context.update.id + "/database", "utf8") !== "before trial") { + throw new Error("missing original backup"); + } + if (${JSON.stringify(scenario)} === "trial exit") { + fs.writeFileSync(context.update.dbPath, "migrated"); + process.on("message", () => process.exit(1)); + process.send("migrated"); + } else { + process.send({ type: "prepared", updateId: context.update.id }); + process.on("message", (message) => { + if (message.type === "committed") process.send("ready"); + }); + } +} else if (${JSON.stringify(scenario)} === "update handoff" && context.update === undefined) { + process.on("message", (message) => { + if (message.type === "update-accepted") process.send("exiting"); + if (message === "exit") process.exit(0); + }); + process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${JSON.stringify(databasePath)} }); +} else { + process.send("ready"); +} +setInterval(() => {}, 1_000); +`; + for (const version of ["1.0.0", "1.1.0"]) { + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", version), + childSource, + ); + } + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + ...([ + "startup", + "log write fails", + "home probe error", + "database probe error", + "update handoff", + "stop", + ].includes(scenario) + ? {} + : { update: pending }), + }), + ); + let low = [ + "startup", + "log write fails", + "home probe error", + "database probe error", + "pending update", + "stop", + ].includes(scenario); + let recovered = false; + const blocked = Promise.withResolvers(); + const ready = Promise.withResolvers(); + const probing = Promise.withResolvers(); + const capacityResult = Promise.withResolvers(); + const actualStatfs = NodeFSP.statfs; + const stats = yield* Effect.promise(() => actualStatfs(root)); + yield* Effect.acquireRelease( + Effect.sync(() => { + vi.useFakeTimers({ + toFake: ["setInterval", "clearInterval", "setTimeout", "clearTimeout"], + }); + const capacity = vi.spyOn(NodeFSP, "statfs").mockImplementation(async (directory) => { + if (scenario === "stop" && directory === root) { + probing.resolve(); + await capacityResult.promise; + } + if ( + (scenario === "home probe error" && directory === root) || + (scenario === "database probe error" && directory === path.dirname(databasePath)) + ) { + throw new Error("statfs unavailable"); + } + if (scenario === "failed trial start" && !recovered && NodeFS.existsSync(backupPath)) + low = true; + return { ...stats, bavail: low ? 0 : 1024 ** 4, bsize: 1 }; + }); + const log = vi.spyOn(process.stderr, "write").mockImplementation((chunk) => { + if (String(chunk).includes("Waiting for free space")) { + blocked.resolve(); + if (scenario === "log write fails") { + throw Object.assign(new Error("no space left"), { code: "ENOSPC" }); + } + } + return true; + }); + spawnHook.current = (child) => + child.on("message", (message) => { + if (message === "ready") ready.resolve(); + if (message === "migrated" || message === "exiting") { + low = true; + child.send("exit"); + } + }); + return { capacity, log }; + }), + ({ capacity, log }) => + Effect.sync(() => { + capacity.mockRestore(); + log.mockRestore(); + spawnHook.current = undefined; + vi.useRealTimers(); + }), + ); + const beforeSpawns = spawnedChildren.length; + const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath))); + const running = launcher.run(); + yield* Effect.addFinalizer(() => + Effect.promise(async () => { + await launcher.stop("SIGTERM"); + await running; + }), + ); + if (scenario === "stop") { + yield* Effect.promise(() => probing.promise); + const stopping = launcher.stop("SIGTERM"); + capacityResult.resolve(); + yield* Effect.promise(() => stopping); + yield* Effect.promise(() => running); + assert.equal(spawnedChildren.length, beforeSpawns); + assert.equal(vi.getTimerCount(), 0); + return; + } + yield* Effect.promise(() => Promise.race([blocked.promise, ready.promise, running])); + assert.equal( + spawnedChildren.length - beforeSpawns, + ["trial exit", "update handoff"].includes(scenario) ? 1 : 0, + ); + if (scenario === "update handoff") { + const state = yield* Effect.promise(() => readServiceState(statePath)); + backupPath = path.join(root, "runtime", "db-backup", state.update!.id); + } + assert.equal( + yield* fs.exists(backupPath), + scenario === "trial exit" || scenario === "failed trial start", + ); + if (scenario === "trial exit") + assert.equal(yield* fs.readFileString(databasePath), "migrated"); + if (scenario === "failed trial start") { + yield* fs.remove(path.join(root, "runtime", "versions", "1.1.0"), { recursive: true }); + } + low = false; + recovered = true; + yield* Effect.promise(() => vi.advanceTimersByTimeAsync(30_000)); + yield* Effect.promise(() => Promise.race([ready.promise, running])); + const state = yield* Effect.promise(() => readServiceState(statePath)); + assert.equal( + state.activeVersion, + ["pending update", "update handoff"].includes(scenario) ? "1.1.0" : "1.0.0", + ); + assert.equal( + state.update?.status, + { + startup: undefined, + "log write fails": undefined, + "home probe error": undefined, + "database probe error": undefined, + "pending update": "committed", + "update handoff": "committed", + "trial exit": "rolled-back", + "failed trial start": "failed", + }[scenario], + ); + assert.equal(yield* fs.readFileString(databasePath), "before trial"); + assert.isFalse(yield* fs.exists(backupPath)); + }), + ); + it.effect( "loads optional service.env literally without changing inherited or managed variables", () => @@ -318,6 +525,70 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { }), ); + it.effect( + "repoints the systemd unit the manager loads when environment.d moves XDG_CONFIG_HOME", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-xdg-" }); + const home = path.join(root, "home"); + const managerUnitDir = path.join(home, ".config", "systemd", "user"); + const unitPath = path.join(managerUnitDir, "t3code.service"); + const render = (version: string) => + renderBootServiceUnit({ ...unitPlan(root, version), unitPath }); + yield* fs.makeDirectory(managerUnitDir, { recursive: true }); + yield* fs.writeFileString(unitPath, render("1.0.0")); + // Fake manager: busctl reports the unit search path, systemctl reloads. + const bin = path.join(root, "bin"); + yield* fs.makeDirectory(bin, { recursive: true }); + const unitPathJson = JSON.stringify({ + type: "as", + data: [`${managerUnitDir}.control`, managerUnitDir, "/usr/lib/systemd/user"], + }); + yield* fs.writeFileString( + path.join(bin, "busctl"), + `#!/bin/sh\nprintf '%s\\n' '${unitPathJson}'\n`, + ); + yield* fs.writeFileString( + path.join(bin, "systemctl"), + `#!/bin/sh\necho reload >> '${path.join(root, "reloads")}'\n`, + ); + yield* fs.chmod(path.join(bin, "busctl"), 0o755); + yield* fs.chmod(path.join(bin, "systemctl"), 0o755); + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", "1.1.0"), + "process.exit(0);\n", + ); + yield* Effect.promise(() => + writeServiceState(path.join(root, "runtime", "service-state.json"), { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.1.0", + update: { + id: "update-1", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + status: "committed", + }, + }), + ); + vi.stubEnv("PATH", bin); + vi.stubEnv("HOME", home); + // environment.d changes what services inherit, not where the manager reads units. + vi.stubEnv("XDG_CONFIG_HOME", path.join(home, "service-config")); + vi.stubEnv("T3_BOOT_SERVICE_UNIT", "t3code.service"); + + yield* Effect.promise(() => + expect(runLauncher(root)).rejects.toThrow("Active child exited unexpectedly (0)."), + ); + + assert.equal(yield* fs.readFileString(unitPath), render("1.1.0")); + assert.equal(yield* fs.readFileString(path.join(root, "reloads")), "reload\n"); + }), + ); + it.effect("keeps the service manager's append log bounded when it starts", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -392,6 +663,36 @@ process.exit(0); }), ); + it.effect("starts the runtime in a T3 home whose name ends in a space", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const parent = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-space-" }); + const root = path.join(parent, "T3 data "); + const versionDir = path.join(root, "runtime", "versions", "1.0.0"); + yield* writeFakeRuntime( + fs, + path, + versionDir, + ` +import { writeFileSync } from "node:fs"; +writeFileSync(new URL("./started", import.meta.url), ""); +process.exit(0); +`, + ); + yield* fs.writeFileString( + path.join(root, "runtime", "service-state.json"), + JSON.stringify({ protocol: 4, activeVersion: "1.0.0" }), + ); + + yield* Effect.promise(() => + expect(runLauncher(root)).rejects.toThrow("Active child exited unexpectedly (0)."), + ); + + assert.isTrue(yield* fs.exists(path.join(versionDir, "started"))); + }), + ); + it.effect("refuses non-install legacy documents without rewriting or spawning", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -654,8 +955,11 @@ process.exit(0); }); yield* Effect.promise(() => refused.promise); assert.isFalse(completed); + // An idle launcher must not make an SSH launch wait for a server it is not starting. + assert.isFalse(yield* Effect.promise(() => serviceLauncherIsServing(root))); otherOwner.close(); yield* Effect.promise(() => owned.promise); + assert.isTrue(yield* Effect.promise(() => serviceLauncherIsServing(root))); yield* Effect.promise(() => launcher.stop("SIGTERM")); yield* Effect.promise(() => running); assert.strictEqual(spawnedChildren.length - spawnsBefore, 2); @@ -1492,6 +1796,56 @@ process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: paths.dat }), ); + it.effect("starts no trial when stopped during the pre-update backup", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-backup-stop-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const childSource = "setInterval(() => {}, 1000);\n"; + for (const version of ["1.0.0", "1.1.0"]) { + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", version), + childSource, + ); + } + const dbPath = path.join(root, "userdata", "state.sqlite"); + yield* fs.makeDirectory(path.dirname(dbPath), { recursive: true }); + yield* fs.writeFileString(dbPath, "database"); + const pending = { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + update: { + id: "update-1", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + dbPath, + status: "pending", + }, + } as const; + yield* Effect.promise(() => writeServiceState(statePath, pending)); + const childCount = spawnedChildren.length; + const launcher = new Launcher(root, pending); + + // Recovery is already queued, so this stop lands while the backup runs. + const running = launcher.run(); + yield* Effect.promise(() => launcher.stop("SIGTERM")); + yield* Effect.promise(() => running); + + // A trial started here arms its 30-minute prepared timer, which keeps the stopped launcher alive. + assert.deepEqual( + spawnedChildren.slice(childCount).map((child) => child.spawnargs.join(" ")), + [], + ); + assert.deepEqual(yield* Effect.promise(() => readServiceState(statePath)), pending); + const backupDir = path.join(root, "runtime", "db-backup", "update-1"); + assert.isFalse(yield* fs.exists(backupDir)); + assert.isFalse(yield* fs.exists(`${backupDir}.staging`)); + }), + ); + it.effect("records no update when the server exits during the backup check", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index db245e16485ce..fad391896d307 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -8,6 +8,7 @@ import * as NodeFSP from "node:fs/promises"; import * as NodePath from "node:path"; import * as NodeUtil from "node:util"; import { acquireServiceSelectionLock } from "./cloud/serviceSelectionLock.ts"; +import { isNoCopyOnWrite, markNoCopyOnWrite } from "./noCopyOnWrite.ts"; import type { PendingServiceUpdate, @@ -29,12 +30,14 @@ import { serviceUnitLogPathOf, SERVICE_ENV_FILE, SERVICE_LAUNCHER_LOCK_FILE, + SERVICE_SERVING_LOCK_FILE, SERVICE_LAUNCHER_CONTEXT_ENV, SERVICE_LAUNCHER_PROTOCOL, SERVICE_STATE_FILE, SERVICE_RESTART_PENDING_FILE, SERVICE_STOP_MARKER_FILE, SERVICE_UNIT_ENV, + systemdUnitDirectoryOf, } from "./cloud/serviceProtocol.ts"; import { legacyOwnerStartedBeforeRecord, @@ -48,6 +51,9 @@ const HANDOFF_DELAY_MS = 2_000; */ const PREPARED_TIMEOUT_MS = 30 * 60_000; const TERMINATE_GRACE_MS = 5_000; +// Keep SQLite startup and journaling from turning a full filesystem into a restart loop. +const MIN_SERVICE_START_FREE_BYTES = 64 * 1024 ** 2; +const DISK_SPACE_RETRY_MS = 30_000; // A replacement started by a handoff waits up to two minutes for the stopping launcher. const LOCK_RETRY_MS = 500; const LOCK_ATTEMPTS = 240; @@ -123,7 +129,34 @@ export async function registerServiceTask( } } -function serviceUnitFromEnvironment(env: NodeJS.ProcessEnv = process.env): ServiceUnit | undefined { +// The manager's environment can differ from the one it gives services (environment.d). +async function readSystemdUnitDirectory(): Promise { + try { + const { stdout } = await NodeUtil.promisify(NodeChildProcess.execFile)( + "busctl", + [ + "--user", + "--json=short", + "get-property", + "org.freedesktop.systemd1", + "/org/freedesktop/systemd1", + "org.freedesktop.systemd1.Manager", + "UnitPath", + ], + { timeout: 5_000 }, + ); + const { data } = JSON.parse(stdout) as { readonly data?: unknown }; + return Array.isArray(data) && data.every((entry) => typeof entry === "string") + ? systemdUnitDirectoryOf(data) + : undefined; + } catch { + return undefined; + } +} + +async function serviceUnitFromEnvironment( + env: NodeJS.ProcessEnv = process.env, +): Promise { const unitName = env[SERVICE_UNIT_ENV]?.trim(); if (!unitName) return undefined; if (unitName.endsWith(".xml")) { @@ -141,14 +174,18 @@ function serviceUnitFromEnvironment(env: NodeJS.ProcessEnv = process.env): Servi reload: () => Promise.resolve(), }; } - // The manager's own environment, which decides where it reads user units. const xdgConfigHome = env.XDG_CONFIG_HOME?.trim(); - const configHome = - xdgConfigHome && NodePath.isAbsolute(xdgConfigHome) - ? xdgConfigHome - : NodePath.join(home, ".config"); + const unitDir = + (await readSystemdUnitDirectory()) ?? + NodePath.join( + xdgConfigHome && NodePath.isAbsolute(xdgConfigHome) + ? xdgConfigHome + : NodePath.join(home, ".config"), + "systemd", + "user", + ); return { - path: NodePath.join(configHome, "systemd", "user", unitName), + path: NodePath.join(unitDir, unitName), reload: () => new Promise((resolve, reject) => { NodeChildProcess.spawn("systemctl", ["--user", "daemon-reload"], { @@ -294,14 +331,18 @@ async function syncDirectory(directory: string): Promise { } } -function runCp(args: ReadonlyArray): Promise { +function runCp(args: ReadonlyArray, signal?: AbortSignal): Promise { return new Promise((resolve) => { - NodeChildProcess.execFile("cp", args, (error) => resolve(error === null)); + NodeChildProcess.execFile("cp", args, { signal }, (error) => resolve(error === null)); }); } // macOS fs.copyFile clone flags fall back to full copies. -async function copyDatabaseFile(source: string, destination: string): Promise { +async function copyDatabaseFile( + source: string, + destination: string, + signal?: AbortSignal, +): Promise { // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime. const platform = process.platform; // cp -c replaces files and breaks hard links; copy into hard-linked destinations instead. @@ -314,7 +355,9 @@ async function copyDatabaseFile(source: string, destination: string): Promise total + file.size, 0); @@ -493,7 +537,11 @@ async function startOnlineDatabaseBackup( } // Preserve completed backups; restarted trials may already have changed the live database. -async function backupDatabaseOnce(baseDir: string, pending: PendingServiceUpdate): Promise { +async function backupDatabaseOnce( + baseDir: string, + pending: PendingServiceUpdate, + signal: AbortSignal, +): Promise { const backupDir = databaseBackupDir(baseDir, pending.id); if (await pathExists(backupDir)) return; @@ -502,10 +550,11 @@ async function backupDatabaseOnce(baseDir: string, pending: PendingServiceUpdate await NodeFSP.mkdir(stagingDir, { recursive: true, mode: 0o700 }); try { for (const suffix of DB_FILE_SUFFIXES) { + signal.throwIfAborted(); const source = `${pending.dbPath}${suffix}`; if (suffix !== "" && !(await pathExists(source))) continue; const destination = databaseBackupFile(stagingDir, suffix); - await copyDatabaseFile(source, destination); + await copyDatabaseFile(source, destination, signal); await syncFile(destination); } await NodeFSP.rename(stagingDir, backupDir); @@ -710,11 +759,12 @@ export const serviceLauncherLockPath = (baseDir: string) => // An exclusive SQLite transaction, like the server's owner lock: the OS drops it when the holder dies. async function tryAcquireSqliteLock( lockPath: string, + busyTimeoutMs = 0, ): Promise<{ readonly close: () => void } | undefined> { const { DatabaseSync } = await import("node:sqlite"); const db = new DatabaseSync(lockPath); try { - db.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;"); + db.exec(`PRAGMA busy_timeout = ${busyTimeoutMs}; BEGIN EXCLUSIVE;`); return db; } catch (cause) { db.close(); @@ -730,6 +780,13 @@ export async function serviceLauncherIsRunning(lockPath: string): Promise + NodePath.join(baseDir, "runtime", SERVICE_SERVING_LOCK_FILE); + +/** Whether a launcher has, or is about to start, a server for this home. An idle one does not. */ +export const serviceLauncherIsServing = (baseDir: string): Promise => + serviceLauncherIsRunning(servingLockPath(baseDir)).catch(() => false); + /** Resolves once no launcher holds the lock; the caller bounds the wait through `signal`. */ export async function waitForServiceLauncherExit( lockPath: string, @@ -798,6 +855,7 @@ export class Launcher { #timer: NodeJS.Timeout | undefined; #transitions: Promise = Promise.resolve(); #stopRequested = false; + readonly #stopSignal = new AbortController(); #stopping = false; #done = false; readonly #completion = Promise.withResolvers(); @@ -808,6 +866,7 @@ export class Launcher { #stateDirFreeProbes = 0; readonly #output: number | undefined; readonly #lockPath: string | undefined; + #servingLock: { readonly close: () => void } | undefined; readonly #stopChildOverIpc: boolean; constructor( @@ -861,6 +920,7 @@ export class Launcher { } } } finally { + this.#releaseServing(); process.off("SIGTERM", onSigterm); process.off("SIGINT", onSigint); } @@ -878,13 +938,29 @@ export class Launcher { return undefined; } + // Advisory: without it an SSH launch may start its own server, as it does beside no service. + async #claimServing(): Promise { + // An SSH launch probes this lock for a moment; wait out that probe. + this.#servingLock ??= await tryAcquireSqliteLock(servingLockPath(this.#baseDir), 250).catch( + () => undefined, + ); + } + + #releaseServing(): void { + this.#servingLock?.close(); + this.#servingLock = undefined; + } + #terminate(child: NodeChildProcess.ChildProcess, signal: NodeJS.Signals = "SIGTERM") { return terminateChild(child, signal, this.#stopChildOverIpc); } + // Diagnostics are best-effort: a full log filesystem must not stop the launcher waiting for space. #log(line: string): void { - if (this.#output === undefined) process.stderr.write(line); - else NodeFS.writeSync(this.#output, line); + try { + if (this.#output === undefined) process.stderr.write(line); + else NodeFS.writeSync(this.#output, line); + } catch {} } #enqueue(transition: () => Promise): void { @@ -916,6 +992,7 @@ export class Launcher { return; } this.#stopRequested = true; + this.#stopSignal.abort(); this.#clearTimer(); this.#enqueue(async () => { // Let in-progress updates start replacements first; replacements observe the marker and own pre-activation tunnel cleanup. @@ -936,6 +1013,55 @@ export class Launcher { this.#timer = undefined; } + async #serviceStartDiskSpaceBlocker(): Promise { + const databasePath = + this.#state.update?.status === "pending" + ? this.#state.update.dbPath + : NodePath.join(this.#baseDir, "userdata", "statev2.sqlite"); + const resolvedDatabasePath = await restoreTarget(databasePath).catch(() => databasePath); + const freeBytes = async (directory: string): Promise => { + try { + return await filesystemBackupSpaceProbe.readFreeBytes(directory); + } catch (cause) { + const parent = NodePath.dirname(directory); + if ((cause as NodeJS.ErrnoException).code === "ENOENT" && parent !== directory) { + return freeBytes(parent); + } + return Number.POSITIVE_INFINITY; + } + }; + const available = await Promise.all([ + freeBytes(this.#baseDir), + freeBytes(NodePath.dirname(resolvedDatabasePath)), + ]); + const free = Math.min(...available); + return free < MIN_SERVICE_START_FREE_BYTES + ? `T3 Code needs at least ${MIN_SERVICE_START_FREE_BYTES} bytes free before starting the server; ${free} bytes available.` + : undefined; + } + + async #deferForDiskSpace(retry: () => Promise): Promise { + const reason = await this.#serviceStartDiskSpaceBlocker(); + if (reason === undefined) return false; + // A stop during the probe must not arm a retry that outlives it. + if (this.#stopRequested) return true; + this.#clearTimer(); + this.#releaseServing(); + this.#log(`[service-launcher] ${reason} Waiting for free space before retrying.\n`); + this.#timer = setInterval( + () => + this.#enqueue(async () => { + if (this.#child !== null || this.#stopping) return; + if ((await this.#serviceStartDiskSpaceBlocker()) !== undefined) return; + this.#clearTimer(); + await this.#claimServing(); + await retry(); + }), + DISK_SPACE_RETRY_MS, + ); + return true; + } + async #selectState(next: ServiceState): Promise { const release = await acquireServiceSelectionLock(this.#statePath); try { @@ -957,6 +1083,7 @@ export class Launcher { } async #recover(): Promise { + await this.#claimServing(); // Clear old stop markers to preserve future tunnels; clear restart-pending only for the awaited version to avoid CLI races. await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined); const restartPending = restartPendingPath(this.#baseDir); @@ -1010,12 +1137,16 @@ export class Launcher { } async #startTrial(pending: PendingServiceUpdate): Promise { + if (await this.#deferForDiskSpace(() => this.#startTrial(pending))) return; try { - await backupDatabaseOnce(this.#baseDir, pending); + await backupDatabaseOnce(this.#baseDir, pending, this.#stopSignal.signal); } catch { + // A stop aborts the copy and leaves the update pending for the next start. + if (this.#stopRequested) return; await this.#returnToPrevious(pending, "failed", "db-backup-failed"); return; } + if (this.#stopRequested) return; try { await this.#startChild(pending.targetVersion, "trial", pending); } catch { @@ -1025,6 +1156,14 @@ export class Launcher { async #startChild(version: string, role: ChildRole, update?: ServiceUpdateRecord): Promise { if (this.#stopping) return; + if ( + await this.#deferForDiskSpace(() => + role === "trial" && update?.status === "pending" + ? this.#startTrial(update) + : this.#startChild(version, role, update), + ) + ) + return; if (!(await runtimeExists(this.#baseDir, version))) { throw new Error(`Selected t3@${version} runtime is missing or incomplete.`); } @@ -1170,8 +1309,6 @@ export class Launcher { this.#child = null; await this.#terminate(child.process); await finishOnlineBackup?.(); - // A stop during a minutes-long copy should not start a trial; restart resumes the pending update. - if (this.#stopRequested) return; await this.#startTrial(pending); } @@ -1285,6 +1422,12 @@ export class Launcher { await this.#startTrial(pending); return; } + if ( + await this.#deferForDiskSpace(() => + this.#startChild(child.version, "active", this.#state.update), + ) + ) + return; throw new Error(`Active child exited unexpectedly (${String(code ?? signal ?? "unknown")}).`); } @@ -1310,6 +1453,7 @@ export class Launcher { await this.#repointServiceUnit(next.activeVersion); } } + this.#releaseServing(); this.#log( "[service-launcher] Another T3 Code server owns this T3 home. This service starts again once it stops.\n", ); @@ -1334,6 +1478,7 @@ export class Launcher { this.#stateDirFreeProbes = owned ? 0 : this.#stateDirFreeProbes + 1; if (this.#stateDirFreeProbes < 2) return; this.#clearTimer(); + await this.#claimServing(); this.#log("[service-launcher] The T3 home is free again. Starting the server.\n"); const update = this.#state.update; if (update?.status === "pending") await this.#startTrial(update); @@ -1350,6 +1495,8 @@ export class Launcher { this.#child = null; await this.#terminate(child.process); } + if (await this.#deferForDiskSpace(() => this.#returnToPrevious(pending, status, reason))) + return; await restoreDatabaseBackup(this.#baseDir, pending); const outcome = terminalUpdate({ pending, status, reason }); const next: ServiceState = { @@ -1375,8 +1522,8 @@ export interface LauncherOptions { } export async function main(options: LauncherOptions = {}): Promise { - const baseDir = (options.baseDir ?? process.env.T3CODE_HOME)?.trim(); - if (baseDir === undefined || baseDir === "") { + const baseDir = options.baseDir ?? process.env.T3CODE_HOME; + if (baseDir === undefined || baseDir.trim() === "") { throw new Error("T3CODE_HOME is required by the T3 Code service launcher."); } const output = @@ -1384,6 +1531,8 @@ export async function main(options: LauncherOptions = {}): Promise { // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime. const windows = process.platform === "win32"; let logTimer: NodeJS.Timeout | undefined; + // A failed stderr write (ENOSPC on a redirected log) is emitted asynchronously; without a listener it is fatal. + process.stderr.on("error", () => undefined); try { // What the service manager started the launcher with. It locates the unit; service.env // changes only what the server gets. @@ -1403,7 +1552,7 @@ export async function main(options: LauncherOptions = {}): Promise { throw new Error(`Selected t3@${state.activeVersion} runtime is missing or incomplete.`); } } - const serviceUnit = serviceUnitFromEnvironment(managerEnvironment); + const serviceUnit = await serviceUnitFromEnvironment(managerEnvironment); const logPath = await serviceLogPath(options, serviceUnit); if (logPath !== undefined) { await boundServiceLog(logPath); diff --git a/apps/server/src/settingsSecrets.ts b/apps/server/src/settingsSecrets.ts index 22ea8ae882583..a38072d19813f 100644 --- a/apps/server/src/settingsSecrets.ts +++ b/apps/server/src/settingsSecrets.ts @@ -235,7 +235,7 @@ function mapSecretFieldGroups( ) as EasDeviceHostConfig, ); - // Its own prefix: an EAS and a Simlock host may share an id. + // Its own prefix: an Expo and a Simlock host may share an id. const simlockDeviceHosts = settings.simlockDeviceHosts.map( (host) => mapFields( diff --git a/apps/server/src/sourceControl/SourceControlProviderRegistry.test.ts b/apps/server/src/sourceControl/SourceControlProviderRegistry.test.ts index c532d93e81ad9..9d472fe6e77c9 100644 --- a/apps/server/src/sourceControl/SourceControlProviderRegistry.test.ts +++ b/apps/server/src/sourceControl/SourceControlProviderRegistry.test.ts @@ -283,7 +283,7 @@ it.effect( url, ); } - }).pipe(Effect.scoped), + }), ); it.effect("does not refine a context whose server address is empty", () => diff --git a/apps/server/src/sourceControl/SourceControlRepositoryService.test.ts b/apps/server/src/sourceControl/SourceControlRepositoryService.test.ts index 4a0533da0b86e..bebdeef9ece33 100644 --- a/apps/server/src/sourceControl/SourceControlRepositoryService.test.ts +++ b/apps/server/src/sourceControl/SourceControlRepositoryService.test.ts @@ -78,6 +78,7 @@ function layer(input: { Layer.provide( Layer.mock(GitVcsDriver.GitVcsDriver)({ execute: () => Effect.succeed(processOutput()), + invalidateRefs: () => Effect.void, ensureRemote: () => Effect.succeed("origin"), pushCurrentBranch: () => Effect.succeed({ @@ -220,7 +221,7 @@ it.effect.each(["--bare", "source.git"])( }); assert.strictEqual(result.cwd, destinationPath); assert.isTrue(yield* fs.exists(path.join(destinationPath, ".git"))); - }).pipe(Effect.provide(layer({ git: { execute: git.execute } }))); + }).pipe(Effect.provide(layer({ git }))); }).pipe( Effect.provide( GitVcsDriver.layer.pipe( @@ -235,6 +236,72 @@ it.effect.each(["--bare", "source.git"])( ), ); +it.effect.each([ + { readsWhen: "before git creates the repository", readBeforeInit: true }, + { readsWhen: "while HEAD is unborn", readBeforeInit: false }, +])("lists the cloned branches after a ref list was read $readsWhen", ({ readBeforeInit }) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const git = yield* GitVcsDriver.GitVcsDriver; + const parent = yield* fs.makeTempDirectoryScoped({ prefix: "t3-clone-refs-" }); + const source = path.join(parent, "source"); + const destinationPath = path.join(parent, "checkout"); + const run = (cwd: string, args: ReadonlyArray) => + git.execute({ operation: "test.git", cwd, args }); + yield* run(parent, ["init", "--initial-branch=staging", source]); + yield* run(source, [ + "-c", + "user.name=Test", + "-c", + "user.email=test@example.com", + "commit", + "--allow-empty", + "-m", + "init", + ]); + const refNames = git + .listRefs({ cwd: destinationPath }) + .pipe(Effect.map((result) => result.refs.map((ref) => ref.name))); + let duringClone: ReadonlyArray | null = null; + // Stands in for git clone so the read lands at a chosen point inside it. + const cloneInTwoSteps = Effect.gen(function* () { + yield* fs.makeDirectory(destinationPath).pipe(Effect.orDie); + if (readBeforeInit) duringClone = yield* refNames; + yield* run(parent, ["init", "--initial-branch=staging", destinationPath]); + if (!readBeforeInit) duringClone = yield* refNames; + return yield* run(destinationPath, ["pull", source, "staging"]); + }); + yield* Effect.gen(function* () { + const service = yield* SourceControlRepositoryService.SourceControlRepositoryService; + yield* service.cloneRepository({ remoteUrl: source, destinationPath }); + }).pipe( + Effect.provide( + layer({ + git: { + ...git, + execute: (input) => (input.args[0] === "clone" ? cloneInTwoSteps : git.execute(input)), + }, + }), + ), + ); + + assert.deepStrictEqual(duringClone, []); + assert.deepStrictEqual(yield* refNames, ["staging"]); + }).pipe( + Effect.provide( + GitVcsDriver.layer.pipe( + Layer.provide( + ServerConfig.layerTest(process.cwd(), { + prefix: "t3-clone-refs-config-", + }), + ), + Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), + ), + ), + ), +); + it.effect("reports clone progress from git's stderr and keeps its error text on failure", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/server/src/sourceControl/SourceControlRepositoryService.ts b/apps/server/src/sourceControl/SourceControlRepositoryService.ts index 3db65b0bd5f95..b4b2dc0e92fe5 100644 --- a/apps/server/src/sourceControl/SourceControlRepositoryService.ts +++ b/apps/server/src/sourceControl/SourceControlRepositoryService.ts @@ -429,6 +429,8 @@ export const make = Effect.gen(function* () { ), ); + // A ref list read while the clone ran saw an unborn HEAD and no refs. + yield* git.invalidateRefs(prepared.destinationPath); return { cwd: prepared.destinationPath, remoteUrl, diff --git a/apps/server/src/speech/SpeechService.test.ts b/apps/server/src/speech/SpeechService.test.ts index c47174e46f1cb..3f580b0743b40 100644 --- a/apps/server/src/speech/SpeechService.test.ts +++ b/apps/server/src/speech/SpeechService.test.ts @@ -449,14 +449,19 @@ it.effect("honors exact aliases and phrase boundaries in batch and streamed text { term: "GitHub", aliases: [] }, { term: "McDonald’s", aliases: [] }, { term: "King’s Cross", aliases: [] }, + { term: "C#", aliases: [] }, + { term: "React", aliases: ["瑞安特"] }, + { term: "Kubernetes", aliases: [] }, ], }; const spoken = 'a test with a i and ay eye. Robert, Roberts. "the publisher" and the publisher. (git hub), git (hub). ' + - "git hub's API, git hub’s API, git’s hub, mac donald's menu, king's cross."; + "git hub's API, git hub’s API, git’s hub, mac donald's menu, king's cross. vitamin c and plan c. " + + "我用github和瑞安特,部署到kubernetis上。Robert用git hub。githubで管理する。"; const expected = "a test with AI and AI. Bob, Roberts. O'Reilly and the publisher. (GitHub), git (hub). " + - "GitHub's API, GitHub’s API, git’s hub, McDonald’s menu, King’s Cross."; + "GitHub's API, GitHub’s API, git’s hub, McDonald’s menu, King’s Cross. vitamin c and plan c. " + + "我用GitHub和React,部署到Kubernetes上。Bob用GitHub。GitHubで管理する。"; native.transcribe.mockResolvedValueOnce({ text: spoken }); native.finish.mockResolvedValueOnce(spoken); expect(yield* speech.transcribe(pcm(), options)).toBe(expected); diff --git a/apps/server/src/speech/customWords.ts b/apps/server/src/speech/customWords.ts index e788c9188a092..981632c46172c 100644 --- a/apps/server/src/speech/customWords.ts +++ b/apps/server/src/speech/customWords.ts @@ -1,5 +1,11 @@ import type { SpeechCustomWords } from "@t3tools/contracts"; +// These scripts are written without spaces, so a neighbor in one of them still ends a word. +const UNSPACED_SCRIPTS = "\\p{scx=Han}\\p{scx=Hiragana}\\p{scx=Katakana}"; +const LETTER = `[[\\p{L}\\p{M}\\p{N}]--[${UNSPACED_SCRIPTS}]]`; +const WORD_CHARACTER = `[[\\p{L}\\p{M}\\p{N}\\p{Pc}]--[${UNSPACED_SCRIPTS}]]`; +const TOKEN = new RegExp(`${LETTER}(?:[^\\s${UNSPACED_SCRIPTS}]*${LETTER})?`, "gv"); + /** Fuzzy-matches preferred terms and rewrites aliases to their preferred spelling. */ export function makeSpeechTextCorrector( words: readonly string[], @@ -71,8 +77,14 @@ export function makeSpeechTextCorrector( const customWordKeys = (words: readonly string[]): CustomWordKey[] => words.flatMap((word) => { - const keys = [matchKey(word)]; - if (word.includes("&")) keys.push(matchKey(word.replaceAll("&", " and "))); + const sources = [word]; + if (word.includes("&")) sources.push(word.replaceAll("&", " and ")); + const keys = sources.flatMap((source) => { + const key = matchKey(source); + // A term that loses its symbols to one letter (C# -> c) would claim every standalone letter. + const lostSymbols = key !== source.replace(/\s+/gu, "").toLocaleLowerCase(); + return key.length < 2 && lostSymbols ? [] : [key]; + }); return [...new Set(keys)].filter(isAsciiKey).map((key) => ({ word, key })); }); @@ -106,7 +118,7 @@ export function makeSpeechTextCorrector( const apply = (text: string): string => { if (keys.length === 0) return text; // Leave surrounding punctuation in the source; only horizontal whitespace can join words. - const tokens = [...text.matchAll(/[\p{L}\p{M}\p{N}](?:\S*[\p{L}\p{M}\p{N}])?/gu)] + const tokens = [...text.matchAll(TOKEN)] .filter(([word]) => /^[\p{L}\p{M}\p{N}'’-]+$/u.test(word)) .map((match) => ({ value: match[0], @@ -162,10 +174,9 @@ export function makeSpeechTextCorrector( const lookup = new Map( spellings.map(({ spelling, term }) => [spelling.toLocaleLowerCase(), term]), ); - const word = "[\\p{L}\\p{M}\\p{N}\\p{Pc}]"; const pattern = new RegExp( - `(? escapePattern(spelling)).join("|")})(?!${word})`, - "giu", + `(? escapePattern(spelling)).join("|")})(?!${WORD_CHARACTER})`, + "giv", ); return (text: string) => { let output = ""; diff --git a/apps/server/src/speech/http.test.ts b/apps/server/src/speech/http.test.ts index d86f77fbdbd83..07f2fe6b99d5d 100644 --- a/apps/server/src/speech/http.test.ts +++ b/apps/server/src/speech/http.test.ts @@ -269,7 +269,7 @@ effectIt.effect( handler(new Request("http://localhost/api/voice/models"), context), ); expect(yield* Effect.promise(() => listed.text())).not.toContain("/home/someone"); - }).pipe(Effect.scoped), + }), ); effectIt.effect.each([ @@ -361,5 +361,5 @@ effectIt.effect.each([ code: "invalid_request", reason, }); - }).pipe(Effect.scoped), + }), ); diff --git a/apps/server/src/speech/native.test.ts b/apps/server/src/speech/native.test.ts index 8712c63ed347d..a41f917b2a682 100644 --- a/apps/server/src/speech/native.test.ts +++ b/apps/server/src/speech/native.test.ts @@ -1,4 +1,8 @@ // @effect-diagnostics nodeBuiltinImport:off - verifies isolated native worker processes and their exit events. +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + import { expect, it } from "vite-plus/test"; import { loadNativeSpeechModel } from "./native.ts"; @@ -155,6 +159,37 @@ it("kills a hung streaming feed without waiting for native cleanup", async () => await model.dispose(); await rejected; }); + +it("skips process exit teardown when the backend dies during native inference", async () => { + const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3code-speech-orphan-")); + const marker = NodePath.join(directory, "exit-teardown"); + // The exit hook stands in for native atexit teardown; the interval for the threadpool call. + const model = await loadNativeSpeechModel( + "unused.gguf", + new AbortController().signal, + streamingFixture(`async () => { + const { writeFileSync } = process.getBuiltinModule("node:fs"); + process.once("exit", () => writeFileSync(${JSON.stringify(marker)}, "ran")); + setInterval(() => {}, 1 << 30); + process.disconnect(); + return await new Promise(() => {}); + }`), + ); + try { + await model.begin({ timestamps: "none" }, false); + await expect(model.feed(new Float32Array(512).fill(0.25))).rejects.toThrow( + "Speech process exited", + ); + expect( + NodeFS.existsSync(marker), + "orphaned worker ran exit teardown while native inference was in flight", + ).toBe(false); + } finally { + await model.dispose(); + NodeFS.rmSync(directory, { recursive: true, force: true }); + } +}); + const vadFixture = ({ streaming = false, failVad = false, diff --git a/apps/server/src/speech/native.ts b/apps/server/src/speech/native.ts index 8331e1b5e0ec1..70c5d21c6038d 100644 --- a/apps/server/src/speech/native.ts +++ b/apps/server/src/speech/native.ts @@ -182,7 +182,8 @@ process.on("message", async (message) => { error: error instanceof Error ? error.stack ?? error.message : String(error) }); } }); -process.on("disconnect", () => process.exit(0)); +// Exit teardown under in-flight native inference segfaults, so terminate without it. +process.on("disconnect", () => process.kill(process.pid, "SIGKILL")); `; type Reply = { diff --git a/apps/server/src/speech/postProcessing.test.ts b/apps/server/src/speech/postProcessing.test.ts new file mode 100644 index 0000000000000..3dd64c9b1bcc7 --- /dev/null +++ b/apps/server/src/speech/postProcessing.test.ts @@ -0,0 +1,81 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import { ProviderInstanceId, type ModelSelection } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Ref from "effect/Ref"; + +import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import * as TextGeneration from "../textGeneration/TextGeneration.ts"; +import * as SpeechPostProcessing from "./postProcessing.ts"; + +const textGenerationSelection = { + instanceId: ProviderInstanceId.make("fakeapi"), + model: "fake-model", +}; +const acpSelection = { instanceId: ProviderInstanceId.make("acp_local"), model: "default" }; + +it.effect("cleanup pinned to a provider that cannot generate text uses the text model", () => + Effect.gen(function* () { + const used = yield* Ref.make(null); + const layer = SpeechPostProcessing.layer.pipe( + Layer.provide( + Layer.succeed(TextGeneration.TextGeneration, { + generateTranscriptionPostProcessing: (input: { modelSelection: ModelSelection }) => + Ref.set(used, input.modelSelection).pipe(Effect.as({ transcription: "Hello, world." })), + } as unknown as TextGeneration.TextGeneration["Service"]), + ), + Layer.provide( + Layer.succeed(ProviderRegistry.ProviderRegistry, { + getProviders: Effect.succeed([ + { + instanceId: acpSelection.instanceId, + driver: "acpRegistry", + enabled: true, + installed: true, + supportsTextGeneration: false, + }, + ]), + } as unknown as ProviderRegistry.ProviderRegistry["Service"]), + ), + Layer.provide( + ServerSettings.layerTest({ + providerInstances: { + [acpSelection.instanceId]: { + driver: "acpRegistry", + enabled: true, + config: { source: "local", commandPath: "acp-agent" }, + }, + }, + textGenerators: { + [textGenerationSelection.instanceId]: { + kind: "openaiCompatible", + baseUrl: "http://127.0.0.1:18999/v1", + enabled: true, + }, + }, + textGenerationModelSelection: textGenerationSelection, + speechPostProcessingModelSelection: acpSelection, + }), + ), + Layer.provide(NodeServices.layer), + ); + yield* Effect.gen(function* () { + const postProcessing = yield* SpeechPostProcessing.SpeechPostProcessing; + yield* postProcessing.process({ + options: { + speechCustomWords: [], + speechPostProcessingPrompt: { mode: "default", customInstructions: "" }, + }, + transcript: "um hello world", + draft: { text: "", selection: { start: 0, end: 0 } }, + } as never); + }).pipe(Effect.provide(layer)); + const selection = yield* Ref.get(used); + assert.deepStrictEqual( + { instanceId: selection?.instanceId, model: selection?.model }, + textGenerationSelection, + ); + }), +); diff --git a/apps/server/src/speech/postProcessing.ts b/apps/server/src/speech/postProcessing.ts index 3208afd057287..92f4af17696c3 100644 --- a/apps/server/src/speech/postProcessing.ts +++ b/apps/server/src/speech/postProcessing.ts @@ -8,6 +8,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import { resolveSpeechPostProcessingModelSelection } from "@t3tools/shared/serverSettings"; +import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as TextGeneration from "../textGeneration/TextGeneration.ts"; import { buildTranscriptionPostProcessingPrompt } from "../textGeneration/TranscriptionPostProcessing.ts"; @@ -27,11 +28,13 @@ const make = Effect.gen(function* () { const settingsService = yield* ServerSettings.ServerSettingsService; const fileSystem = yield* FileSystem.FileSystem; const textGeneration = yield* TextGeneration.TextGeneration; + const providerRegistry = yield* ProviderRegistry.ProviderRegistry; return SpeechPostProcessing.of({ process: Effect.fn("SpeechPostProcessing.process")( function* (input) { if (input.transcript.trim().length === 0) return { text: input.transcript }; const settings = yield* settingsService.getSettings; + const providers = yield* providerRegistry.getProviders; const { mode, customInstructions } = input.options.speechPostProcessingPrompt; const instructions = (mode === "custom" ? customInstructions.trim() : "") || @@ -46,7 +49,7 @@ const make = Effect.gen(function* () { input.draft, input.options.speechCorrectionWord, ), - modelSelection: resolveSpeechPostProcessingModelSelection(settings), + modelSelection: resolveSpeechPostProcessingModelSelection(settings, providers), }); const text = generated.transcription.trim(); return { diff --git a/apps/server/src/startupAccess.test.ts b/apps/server/src/startupAccess.test.ts index a48bb5263f8ec..159778b3c7e03 100644 --- a/apps/server/src/startupAccess.test.ts +++ b/apps/server/src/startupAccess.test.ts @@ -21,6 +21,7 @@ const ipv4 = (address: string, internal = false): NodeOS.NetworkInterfaceInfo => it("resolves wildcard hosts to a concrete external interface when one is available", () => { const connectionString = resolveHeadlessConnectionString("0.0.0.0", 3773, { interfaces: { en0: [ipv4("192.168.1.42")], lo0: [ipv4("127.0.0.1", true)] }, + platform: "linux", }); expect(connectionString).toBe("http://192.168.1.42:3773"); @@ -34,16 +35,21 @@ it("skips public and container addresses for a private-network listener", () => eth1: [ipv4("10.0.0.5")], }; - expect(resolveHeadlessConnectionString("0.0.0.0", 3773, { interfaces })).toBe( + expect(resolveHeadlessConnectionString("0.0.0.0", 3773, { interfaces, platform: "linux" })).toBe( "http://203.0.113.7:3773", ); expect( - resolveHeadlessConnectionString("0.0.0.0", 3773, { interfaces, privateNetworkOnly: true }), + resolveHeadlessConnectionString("0.0.0.0", 3773, { + interfaces, + privateNetworkOnly: true, + platform: "linux", + }), ).toBe("http://10.0.0.5:3773"); expect( resolveHeadlessConnectionString("0.0.0.0", 3773, { interfaces: { eth0: [ipv4("203.0.113.7")] }, privateNetworkOnly: true, + platform: "linux", }), ).toBe("http://localhost:3773"); }); @@ -53,10 +59,21 @@ it("prefers a LAN address over a tailnet one for a private-network pairing URL", resolveHeadlessConnectionString("0.0.0.0", 3773, { interfaces: { tailscale0: [ipv4("100.101.102.103")], en0: [ipv4("192.168.1.42")] }, privateNetworkOnly: true, + platform: "linux", }), ).toBe("http://192.168.1.42:3773"); }); +it("skips NixOS container and systemd-nspawn veths for a private-network pairing URL", () => { + expect( + resolveHeadlessConnectionString("0.0.0.0", 3773, { + interfaces: { "ve-jellyfin": [ipv4("192.168.53.193")], eth0: [ipv4("192.168.0.18")] }, + privateNetworkOnly: true, + platform: "linux", + }), + ).toBe("http://192.168.0.18:3773"); +}); + it("targets the bound interface when the listener binds a specific address", () => { const hostFor = (ip: string) => loopbackUrlHostForAddress(NetAddress.inetAddressFromIpStringUnsafe(ip, 3773)); diff --git a/apps/server/src/startupAccess.ts b/apps/server/src/startupAccess.ts index f393faecebf91..1720300bd7beb 100644 --- a/apps/server/src/startupAccess.ts +++ b/apps/server/src/startupAccess.ts @@ -7,6 +7,7 @@ import { isSharedAddressSpaceHost, isWildcardHost, } from "@t3tools/shared/hostClassification"; +import * as HostProcess from "@t3tools/shared/HostProcess"; import { QrCode } from "@t3tools/shared/qrCode"; import { buildPairingAppLink } from "@t3tools/shared/remote"; import * as Effect from "effect/Effect"; @@ -44,9 +45,19 @@ export const isLoopbackHost = (host: string | undefined): boolean => { * host bridge such as `br0` or Proxmox's `vmbr0` carries the LAN address, so * it stays listed. */ -export const VIRTUAL_INTERFACE = +const VIRTUAL_INTERFACE = /^(docker|br-|veth|virbr|vmnet|vboxnet|vEthernet|podman|cni|flannel|cali|lxcbr|lxdbr|bridge[1-9]\d\d)/; +/** + * systemd-nspawn's `ve-*`/`vz-*` and NixOS containers' `vb-*` name host-side + * veths after the machine. Windows aliases are user-renameable, so these + * prefixes only apply to Linux interface discovery. + */ +const LINUX_CONTAINER_INTERFACE = /^(ve-|vb-|vz-)/; + +export const isVirtualInterface = (name: string, platform: NodeJS.Platform): boolean => + VIRTUAL_INTERFACE.test(name) || (platform === "linux" && LINUX_CONTAINER_INTERFACE.test(name)); + /** * Loopback or private-range IPv4 (LAN, tailnet, CGNAT), not link-local, in * plain or IPv4-mapped IPv6 form. @@ -86,6 +97,7 @@ const resolveHeadlessConnectionHost = ( host: string | undefined, interfaces: NetworkInterfacesMap, privateNetworkOnly: boolean, + platform: NodeJS.Platform, ): string => { if (!host) { return "localhost"; @@ -96,7 +108,7 @@ const resolveHeadlessConnectionHost = ( } const interfaceEntries = Object.entries(interfaces).flatMap(([name, entries]) => - privateNetworkOnly && VIRTUAL_INTERFACE.test(name) ? [] : (entries ?? []), + privateNetworkOnly && isVirtualInterface(name, platform) ? [] : (entries ?? []), ); const externalIpv4Entries = interfaceEntries.filter( (entry) => @@ -128,12 +140,14 @@ export const resolveHeadlessConnectionString = ( options: { readonly interfaces?: NetworkInterfacesMap; readonly privateNetworkOnly?: boolean | undefined; - } = {}, + readonly platform: NodeJS.Platform; + }, ): string => { const connectionHost = resolveHeadlessConnectionHost( host, options.interfaces ?? NodeOS.networkInterfaces(), options.privateNetworkOnly === true, + options.platform, ); return `http://${formatHostForUrl(connectionHost)}:${port}`; }; @@ -196,10 +210,11 @@ export const issueHeadlessServeAccessInfo = Effect.fn("issueHeadlessServeAccessI const serverConfig = yield* ServerConfig.ServerConfig; const httpServer = yield* HttpServer.HttpServer; const serverAuth = yield* EnvironmentAuth.EnvironmentAuth; + const platform = yield* HostProcess.Platform; const connectionString = resolveHeadlessConnectionString( serverConfig.host, resolveListeningPort(httpServer.address, serverConfig.port), - { privateNetworkOnly: serverConfig.privateNetworkOnly }, + { privateNetworkOnly: serverConfig.privateNetworkOnly, platform }, ); const issued = yield* serverAuth.issueStartupPairingCredential(); diff --git a/apps/server/src/storageCleanup.test.ts b/apps/server/src/storageCleanup.test.ts index c37a5f387b28c..cf56bde07f214 100644 --- a/apps/server/src/storageCleanup.test.ts +++ b/apps/server/src/storageCleanup.test.ts @@ -2,6 +2,8 @@ import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Effect from "effect/Effect"; import * as Context from "effect/Context"; +import type * as Digest from "effect/crypto/Digest"; +import * as Base64Url from "effect/encoding/Base64Url"; import * as Deferred from "effect/Deferred"; import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; @@ -14,6 +16,8 @@ import * as Stream from "effect/Stream"; import { SCAN_CACHE_FILE_NAME } from "./usage/usageScanCache.ts"; import { TestClock } from "effect/testing"; import * as SqlClient from "effect/sql/SqlClient"; +import * as Sqlite from "./persistence/Sqlite.ts"; +import { checkpointRefForScopeOrdinal } from "./orchestration-v2/CheckpointService.ts"; import { DEFAULT_SERVER_SETTINGS, type WorktreeKeepWhen, @@ -26,17 +30,20 @@ import * as ServerConfig from "./config.ts"; import * as StorageCleanup from "./storageCleanup.ts"; import * as GitVcsDriver from "./vcs/GitVcsDriver.ts"; import * as GitManager from "./git/GitManager.ts"; +import * as GitWorkflowService from "./git/GitWorkflowService.ts"; import * as Settings from "./serverSettings.ts"; import * as ProjectionStore from "./orchestration-v2/ProjectionStore.ts"; import * as ProviderSessionManager from "./orchestration-v2/ProviderSessionManager.ts"; import * as ProjectStore from "./orchestration-v2/ProjectStore.ts"; import * as Orchestrator from "./orchestration-v2/Orchestrator.ts"; +import * as ThreadManagement from "./orchestration-v2/ThreadManagementService.ts"; import * as TerminalManager from "./terminal/Manager.ts"; import { CodexProviderCapabilitiesV2 } from "./orchestration-v2/Adapters/CodexAdapterV2.ts"; import { ServerActivation } from "./serverActivation.ts"; import { withWorkspaceLease } from "./workspace/workspaceLease.ts"; import { describe, expect, it } from "@effect/vitest"; import { + CheckpointScopeId, EventId, ProjectId, ProviderDriverKind, @@ -46,6 +53,7 @@ import { RuntimeRequestId, ThreadId, type OrchestrationV2DomainEvent, + type OrchestrationV2ServerCommand, type OrchestrationV2ThreadShell, } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; @@ -325,6 +333,7 @@ const cleanupFixture = ( worktreeAfterDays: 8, worktreeOnDelete: true, }, + sqlOverride?: SqlClient.SqlClient, ) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -357,6 +366,7 @@ const cleanupFixture = ( let sessionLiveChecks = 0; let onLiveCheck: Effect.Effect = Effect.void; let outboxStatus: string | null = null; + let scheduledThreadIds: string[] = []; let fullStatusCalls = 0; let gitInspections = 0; let snapshotReads = 0; @@ -426,6 +436,31 @@ const cleanupFixture = ( getThreadShell: (threadId: ThreadId) => Effect.sync(() => threads.find((thread) => thread.id === threadId) ?? null), } as unknown as ProjectionStore.ProjectionStoreV2["Service"]), + Effect.provideService(ThreadManagement.ThreadManagementService, { + getShellSnapshot: (input?: { threadIds?: ReadonlyArray }) => + Effect.sync(() => ({ + threads: threads.filter((thread) => input?.threadIds?.includes(thread.id) ?? true), + archivedThreads: [], + })), + dispatch: (command: OrchestrationV2ServerCommand) => + Effect.sync(() => { + if (command.type === "thread.metadata.update" && command.branch != null) { + const branch = command.branch; + threads = threads.map((thread) => + thread.id === command.threadId && thread.branch === command.expectedBranch + ? { ...thread, branch } + : thread, + ); + } + return { sequence: 0, storedEvents: [] }; + }), + } as unknown as ThreadManagement.ThreadManagementService["Service"]), + Effect.provideService(GitWorkflowService.GitWorkflowService, { + currentBranch: (cwd: string) => + command(cwd, ["branch", "--show-current"]).pipe( + Effect.map((result) => result.stdout.trim() || null), + ), + } as unknown as GitWorkflowService.GitWorkflowService["Service"]), Effect.provideService(ProviderSessionManager.ProviderSessionManagerV2, { isLive: () => Effect.sync(() => { @@ -436,39 +471,49 @@ const cleanupFixture = ( Effect.provideService(Orchestrator.OrchestratorV2, { streamDomainEvents: Stream.suspend(() => events), } as unknown as Orchestrator.OrchestratorV2["Service"]), - Effect.provideService(SqlClient.SqlClient, ((strings: TemplateStringsArray) => - Effect.sync(() => - openSessionCwd !== null && strings.join("").includes("projection_provider_sessions") - ? [ - { - payload_json: JSON.stringify({ - id: "session-1", - driver: "codex", - providerInstanceId: "codex", - status: "ready", - cwd: openSessionCwd, - model: null, - capabilities: CodexProviderCapabilitiesV2, - createdAt: "2026-06-01T12:00:00.000Z", - updatedAt: "2026-06-01T12:00:00.000Z", - lastError: null, - }), - }, - ] - : outboxStatus !== null && strings.join("").includes("orchestration_v2_effect_outbox") - ? [{ status: outboxStatus }] - : strings.join("").includes("deleted_at IS NOT NULL") - ? deletedThreads.map((thread) => ({ - payload_json: JSON.stringify(thread), - workspaceRoot: repo, - })) - : strings.join("").includes('AS "worktreePath"') + Effect.provideService( + SqlClient.SqlClient, + sqlOverride ?? + (((strings: TemplateStringsArray) => + Effect.sync(() => + strings.join("").includes("retained_checkpoint_threads") + ? [] + : openSessionCwd !== null && + strings.join("").includes("projection_provider_sessions") ? [ - ...threads.flatMap((thread) => thread.worktreePath ?? []), - ...storedWorktreePaths, - ].map((worktreePath) => ({ worktreePath })) - : [], - )) as unknown as SqlClient.SqlClient), + { + payload_json: JSON.stringify({ + id: "session-1", + driver: "codex", + providerInstanceId: "codex", + status: "ready", + cwd: openSessionCwd, + model: null, + capabilities: CodexProviderCapabilitiesV2, + createdAt: "2026-06-01T12:00:00.000Z", + updatedAt: "2026-06-01T12:00:00.000Z", + lastError: null, + }), + }, + ] + : outboxStatus !== null && + strings.join("").includes("orchestration_v2_effect_outbox") + ? [{ status: outboxStatus }] + : strings.join("").includes("deleted_at IS NOT NULL") + ? deletedThreads.map((thread) => ({ + payload_json: JSON.stringify(thread), + workspaceRoot: repo, + })) + : strings.join("").includes('AS "worktreePath"') + ? [ + ...threads.flatMap((thread) => thread.worktreePath ?? []), + ...storedWorktreePaths, + ].map((worktreePath) => ({ worktreePath })) + : strings.join("").includes("scheduled_tasks") + ? scheduledThreadIds.map((threadId) => ({ threadId })) + : [], + )) as unknown as SqlClient.SqlClient), + ), Effect.provideService(GitManager.GitManager, { invalidateStatus: () => Effect.void, } as unknown as GitManager.GitManager["Service"]), @@ -527,12 +572,16 @@ const cleanupFixture = ( setThreads: (next: typeof threads) => { threads = next; }, + threadBranches: () => threads.map((thread) => thread.branch), setOnLiveCheck: (hook: Effect.Effect) => { onLiveCheck = hook; }, setOutboxStatus: (status: string | null) => { outboxStatus = status; }, + setScheduledThreadIds: (next: string[]) => { + scheduledThreadIds = next; + }, setDeletedThreads: (next: OrchestrationV2ThreadShell[]) => { deletedThreads = next; }, @@ -600,17 +649,203 @@ const cleanupTestLayer = GitVcsDriver.layer.pipe( Layer.provideMerge(ServerConfig.layerTest(process.cwd(), { prefix: "storage-cleanup-" })), Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), ); +const annexWorktree = Effect.fnUntraced(function* ( + fixture: Effect.Success>, + worktree: string, +) { + const adminDir = (yield* fixture.command(worktree, ["rev-parse", "--absolute-git-dir"])).stdout; + yield* fixture.fs.remove(`${worktree}/.git`); + yield* fixture.fs.symlink(adminDir.trim(), `${worktree}/.git`); +}); + const runCleanupTest = ( effect: Effect.Effect< A, E, | FileSystem.FileSystem + | Digest.Digest | ServerConfig.ServerConfig | GitVcsDriver.GitVcsDriver | Path.Path | Scope.Scope >, -) => effect.pipe(Effect.provide(cleanupTestLayer), Effect.scoped); +) => effect.pipe(Effect.provide(cleanupTestLayer)); + +const legacyCheckpointRef = (threadId: string, turn: number) => + `refs/t3/checkpoints/${Base64Url.encode(threadId)}/turn/${turn}`; + +const checkpointCleanupFixture = Effect.fn("test.checkpointCleanupFixture")(function* () { + const sql = yield* SqlClient.SqlClient; + const git = yield* GitVcsDriver.GitVcsDriver; + const fixture = yield* cleanupFixture(DEFAULT_SERVER_SETTINGS.storageCleanup, sql); + const { repo, worktree, command } = fixture; + const timestamp = DateTime.formatIso(at(0)); + yield* sql`INSERT INTO projection_projects ${sql.insert({ + project_id: "project-1", + title: "Checkpoint cleanup", + workspace_root: repo, + scripts_json: "[]", + created_at: timestamp, + updated_at: timestamp, + deleted_at: timestamp, + })}`; + const addThread = Effect.fn("test.addCheckpointThread")(function* ( + id: string, + options: { deleted?: boolean; archived?: boolean; source?: string; legacyOnly?: boolean } = {}, + ) { + const scopeId = CheckpointScopeId.make(`scope:${id}`); + yield* sql`INSERT INTO orchestration_v2_projection_threads ${sql.insert({ + thread_id: id, + project_id: "project-1", + title: id, + default_provider: "codex", + runtime_mode: "full-access", + interaction_mode: "default", + created_at: timestamp, + updated_at: timestamp, + archived_at: options.archived ? timestamp : null, + deleted_at: options.deleted ? timestamp : null, + payload_json: JSON.stringify({ + worktreePath: worktree, + forkedFrom: options.source ? { type: "run", threadId: options.source } : null, + }), + })}`; + if (!options.legacyOnly) + yield* sql`INSERT INTO orchestration_v2_projection_checkpoint_scopes ${sql.insert({ + scope_id: scopeId, + thread_id: id, + node_id: `node:${id}`, + kind: "root_run", + ordinal_within_parent: 0, + advances_app_run_count: 1, + created_at: timestamp, + payload_json: JSON.stringify({ cwd: worktree }), + })}`; + const refs: string[] = options.legacyOnly + ? [legacyCheckpointRef(id, 0)] + : [ + yield* checkpointRefForScopeOrdinal({ scopeId, ordinalWithinScope: 0 }), + yield* checkpointRefForScopeOrdinal({ scopeId, ordinalWithinScope: 1 }), + legacyCheckpointRef(id, 0), + ]; + for (const ref of refs) yield* command(repo, ["update-ref", ref, "HEAD"]); + return refs; + }); + const listRefs = () => + git + .execute({ + operation: "cleanup-test.list-checkpoints", + cwd: repo, + args: ["for-each-ref", "--format=%(refname)", "refs/t3/"], + maxOutputBytes: 16 * 1024 * 1024, + }) + .pipe(Effect.map(({ stdout }) => stdout.trim().split("\n").filter(Boolean).sort())); + return { ...fixture, sql, addThread, listRefs }; +}); + +describe("checkpoint ref cleanup", () => { + it.live("prunes refs owned by deleted threads after their worktree is removed", () => + runCleanupTest( + Effect.gen(function* () { + const { service, repo, worktree, command, addThread, listRefs } = + yield* checkpointCleanupFixture(); + yield* addThread("deleted", { deleted: true }); + yield* addThread("imported-deleted", { deleted: true, legacyOnly: true }); + const retained = [ + ...(yield* addThread("active")), + ...(yield* addThread("archived", { archived: true })), + legacyCheckpointRef("unknown-owner", 0), + ]; + yield* command(repo, ["update-ref", retained.at(-1)!, "HEAD"]); + yield* command(repo, ["pack-refs", "--all"]); + yield* command(repo, ["worktree", "remove", worktree]); + yield* service.runNow; + expect(yield* listRefs()).toEqual(retained.sort()); + }).pipe(Effect.provide(Sqlite.layerMemory)), + ), + ); + + it.live("keeps deleted fork ancestry while a live or archived descendant needs it", () => + runCleanupTest( + Effect.gen(function* () { + const { service, sql, addThread, listRefs } = yield* checkpointCleanupFixture(); + const root = yield* addThread("root", { deleted: true }); + const intermediate = yield* addThread("intermediate", { deleted: true, source: "root" }); + const child = yield* addThread("child", { archived: true, source: "intermediate" }); + yield* service.runNow; + expect(yield* listRefs()).toEqual([...root, ...intermediate, ...child].sort()); + yield* sql`UPDATE orchestration_v2_projection_threads SET deleted_at = 'deleted' WHERE thread_id = 'child'`; + yield* service.runNow; + expect(yield* listRefs()).toEqual([]); + }).pipe(Effect.provide(Sqlite.layerMemory)), + ), + ); + + it.live("waits for a deleted thread's pending checkpoint effects", () => + runCleanupTest( + Effect.gen(function* () { + const { service, sql, addThread, listRefs } = yield* checkpointCleanupFixture(); + const refs = yield* addThread("capturing", { deleted: true }); + yield* sql`INSERT INTO orchestration_v2_effect_outbox ${sql.insert({ + effect_id: "capture", + command_id: "capture", + thread_id: "capturing", + effect_type: "checkpoint.capture", + payload_json: "{}", + status: "running", + available_at: "2026-01-01T00:00:00Z", + created_at: "2026-01-01T00:00:00Z", + updated_at: "2026-01-01T00:00:00Z", + })}`; + yield* service.runNow; + expect(yield* listRefs()).toEqual(refs.toSorted()); + yield* sql`UPDATE orchestration_v2_effect_outbox SET status = 'succeeded'`; + yield* service.runNow; + expect(yield* listRefs()).toEqual([]); + }).pipe(Effect.provide(Sqlite.layerMemory)), + ), + ); + + it.live("deletes a symbolic checkpoint ref without its target or neighbouring refs", () => + runCleanupTest( + Effect.gen(function* () { + const { service, repo, command, addThread, listRefs } = yield* checkpointCleanupFixture(); + const refs = yield* addThread("deleted", { deleted: true }); + yield* command(repo, ["symbolic-ref", refs[0]!, "refs/heads/main"]); + const retained = [ + "refs/t3/unrelated/0", + `${refs[1]}-other`, + refs[1]!.replace(/1$/, "2/nested"), + ]; + for (const ref of retained) yield* command(repo, ["update-ref", ref, "HEAD"]); + yield* service.runNow; + expect(yield* listRefs()).toEqual(retained.sort()); + yield* command(repo, ["show-ref", "--verify", "refs/heads/main"]); + }).pipe(Effect.provide(Sqlite.layerMemory)), + ), + ); + + it.live("cleans more checkpoint refs than default git output holds", () => + runCleanupTest( + Effect.gen(function* () { + const { service, repo, addThread, listRefs } = yield* checkpointCleanupFixture(); + const git = yield* GitVcsDriver.GitVcsDriver; + const prefix = (yield* addThread("many-checkpoints", { deleted: true }))[0]!.slice(0, -1); + yield* git.execute({ + operation: "cleanup-test.seed-checkpoints", + cwd: repo, + args: ["update-ref", "--stdin"], + stdin: Array.from( + { length: 12_283 }, + (_, ordinal) => `update ${prefix}${ordinal} HEAD\n`, + ).join(""), + }); + yield* service.runNow; + expect(yield* listRefs()).toEqual([]); + }).pipe(Effect.provide(Sqlite.layerMemory)), + ), + ); +}); it.live.each(["repository", "uppercase", "bare"] as const)( "keeps a worktree holding an ignored nested %s", @@ -727,8 +962,26 @@ it.live("keeps an ignored repository inside a registered submodule", () => it.live("removes clean submodules recursively, including deinitialized repositories", () => runCleanupTest( Effect.gen(function* () { - const { service, fs, repo, worktree, submodule, command, recordSubmodules } = - yield* submoduleCleanupFixture(); + const fs = yield* FileSystem.FileSystem; + const git = yield* GitVcsDriver.GitVcsDriver; + let editedDuringClean = false; + const { service, repo, worktree, submodule, command, recordSubmodules, setPolicy } = + yield* submoduleCleanupFixture().pipe( + Effect.provideService(GitVcsDriver.GitVcsDriver, { + ...git, + execute: (input) => + git.execute(input).pipe( + Effect.tap(() => { + if (input.args[0] !== "clean") return Effect.void; + editedDuringClean = true; + return fs.writeFileString(`${input.cwd}/tracked.txt`, "late edit\n"); + }), + ), + }), + ); + setPolicy("tracked-changes"); + yield* fs.writeFileString(`${worktree}/notes.txt`, "untracked\n"); + yield* fs.writeFileString(`${worktree}/.env`, "ignored\n"); yield* command(submodule, [ "-c", "protocol.file.allow=always", @@ -770,8 +1023,22 @@ it.live("removes clean submodules recursively, including deinitialized repositor yield* command(legacy, ["submodule", "deinit", "stored"]); yield* recordSubmodules(); yield* command(submodule, ["submodule", "deinit", "stored"]); + const checkpoint = (yield* command(submodule, [ + "commit-tree", + "HEAD^{tree}", + "-p", + "HEAD", + "-m", + "checkpoint", + ])).stdout.trim(); + yield* command(submodule, [ + "update-ref", + "refs/t3/orchestration-v2/checkpoints/test", + checkpoint, + ]); const admin = (yield* command(worktree, ["rev-parse", "--absolute-git-dir"])).stdout.trim(); const report = yield* service.runNow; + expect(editedDuringClean).toBe(false); expect(report.entries.find((entry) => entry.path === worktree)).toMatchObject({ outcome: "removed", }); @@ -877,15 +1144,21 @@ it.live("keeps unpushed submodule history after checkout and deinitialization", it.live("keeps submodules when stored repository metadata is malformed", () => runCleanupTest( Effect.gen(function* () { - const { service, fs, worktree, submodule, child, command } = yield* submoduleCleanupFixture(); + const { service, fs, worktree, submodule, child, command, setPolicy } = + yield* submoduleCleanupFixture(); + setPolicy("tracked-changes"); const childGit = (yield* command(child, ["rev-parse", "--absolute-git-dir"])).stdout.trim(); yield* command(submodule, ["submodule", "deinit", "child"]); yield* fs.writeFileString(`${childGit}/HEAD`, "not a ref\n"); + yield* fs.writeFileString(`${worktree}/notes.txt`, "keep while the worktree stays\n"); expect( (yield* service.runNow).entries.find((entry) => entry.path === worktree), ).toMatchObject({ outcome: "kept", reason: expect.stringContaining("inspected") }); expect(yield* fs.exists(worktree)).toBe(true); expect(yield* fs.readFileString(`${childGit}/HEAD`)).toBe("not a ref\n"); + expect(yield* fs.readFileString(`${worktree}/notes.txt`)).toBe( + "keep while the worktree stays\n", + ); }), ), ); @@ -1005,10 +1278,10 @@ describe("subagent worktree ownership", () => { ? "Thread is running or has pending work" : "Not settled", }); - }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), + }).pipe(Effect.provide(cleanupTestLayer)), ); - it.live.each([ + it.effect.each([ "parent-unsettled", "parent-resettled", "child-unsettled", @@ -1017,6 +1290,7 @@ describe("subagent worktree ownership", () => { ] as const)("cancels inherited settlement during measurement (%s)", (change) => runCleanupTest( Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); const fixture = yield* pausedMeasurementFixture({ ...DEFAULT_SERVER_SETTINGS.storageCleanup, worktreeSettledAfterDays: 0, @@ -1073,6 +1347,8 @@ describe("subagent worktree ownership", () => { it.effect.each([ "idle", "grandchild", + "branchless", + "branchless-running", "null-branch", "stale-branch", "isolated-owner", @@ -1104,7 +1380,7 @@ describe("subagent worktree ownership", () => { }, }); const root = shell({ - branch: "feature", + branch: ["branchless", "branchless-running"].includes(scenario) ? null : "feature", worktreePath: fixture.worktree, settledAt: scenario === "child-settled-only" ? null : at(-20 * DAY_MS), ...(scenario === "isolated-owner" || scenario === "fork-owner" @@ -1146,7 +1422,7 @@ describe("subagent worktree ownership", () => { : "subagent", ...(scenario === "side-chat" ? { sideChat: true } : {}), }, - status: scenario === "running" ? "running" : "completed", + status: ["running", "branchless-running"].includes(scenario) ? "running" : "completed", pendingBackgroundTasks: scenario === "pending" ? [{ taskId: "task", kind: "command" }] : [], latestRunCompletedAt: scenario === "recent-activity" || scenario === "recent-inactive" @@ -1195,7 +1471,7 @@ describe("subagent worktree ownership", () => { outcome: removed ? "removed" : "kept", reason: removed ? "Settled for 20 days" - : scenario === "running" || scenario === "pending" + : ["running", "pending", "branchless-running"].includes(scenario) ? "Thread is running or has pending work" : shared ? "Shared by 2 threads" @@ -1207,12 +1483,60 @@ describe("subagent worktree ownership", () => { ? "Inactive for 2 of 8 days" : ["recent-grace", "recent-settle", "child-settled-now"].includes(scenario) ? "Settled moments ago" - : "Not settled", + : scenario === "branchless" + ? "Thread has no branch" + : "Not settled", }); - }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), + }).pipe(Effect.provide(cleanupTestLayer)), ); - it.live.each([ + it.effect("keeps a checkout two top-level threads store through a symlinked parent", () => + Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); + const fixture = yield* cleanupFixture({ + ...DEFAULT_SERVER_SETTINGS.storageCleanup, + worktreeSettledAfterDays: 8, + }); + const linkedParent = `${yield* fixture.fs.makeTempDirectoryScoped({ prefix: "cleanup-link-" })}/worktrees`; + yield* fixture.fs.symlink(fixture.config.worktreesDir, linkedParent); + const settled = { branch: "feature", settledAt: at(-20 * DAY_MS) }; + fixture.setThreads([ + shell({ ...settled, worktreePath: fixture.worktree }), + shell({ ...settled, id: ThreadId.make("other"), worktreePath: `${linkedParent}/feature` }), + ]); + const report = yield* fixture.service.runNow; + expect(report.entries).toEqual([ + expect.objectContaining({ outcome: "kept", reason: "Shared by 2 threads" }), + ]); + expect(yield* fixture.fs.exists(fixture.worktree)).toBe(true); + }).pipe(Effect.provide(cleanupTestLayer)), + ); + + it.effect("keeps a checkout two top-level threads store in different case on Windows", () => + Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); + const fixture = yield* cleanupFixture({ + ...DEFAULT_SERVER_SETTINGS.storageCleanup, + worktreeSettledAfterDays: 8, + }).pipe(Effect.provideService(HostProcess.Platform, "win32")); + const settled = { branch: "feature", settledAt: at(-20 * DAY_MS) }; + fixture.setThreads([ + shell({ ...settled, worktreePath: fixture.worktree }), + shell({ + ...settled, + id: ThreadId.make("other"), + worktreePath: fixture.worktree.toUpperCase(), + }), + ]); + const report = yield* fixture.service.runNow; + expect(report.entries).toEqual([ + expect.objectContaining({ outcome: "kept", reason: "Shared by 2 threads" }), + ]); + expect(yield* fixture.fs.exists(fixture.worktree)).toBe(true); + }).pipe(Effect.provide(cleanupTestLayer)), + ); + + it.effect.each([ "running", "pending", "activity", @@ -1224,6 +1548,7 @@ describe("subagent worktree ownership", () => { ] as const)("cancels removal when a child changes during measurement (%s)", (change) => runCleanupTest( Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); const fixture = yield* pausedMeasurementFixture(); const root = shell({ branch: "feature", worktreePath: fixture.worktree }); const child = { @@ -1313,7 +1638,29 @@ describe("deleted thread worktrees", () => { reason: "Thread was deleted", }); expect(yield* fixture.fs.exists(fixture.worktree)).toBe(false); - }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), + }).pipe(Effect.provide(cleanupTestLayer)), + ); + + it.effect("leaves a deleted owner's subagent on the owner's saved branch", () => + Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); + const fixture = yield* cleanupFixture({ + ...DEFAULT_SERVER_SETTINGS.storageCleanup, + worktreeAfterDays: null, + worktreeOnDelete: true, + }); + const owner = parent(fixture.worktree); + fixture.setDeletedThreads([owner]); + fixture.setThreads([subagent(owner)]); + yield* fixture.command(fixture.worktree, ["switch", "-c", "renamed"]); + expect((yield* fixture.service.runNow).entries[0]).toMatchObject({ + threadId: owner.id, + outcome: "kept", + reason: "Repository or branch changed", + }); + // The deleted owner cannot follow, so moving its subagent would split their records. + expect(fixture.threadBranches()).toEqual(["feature"]); + }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), ); it.effect("applies the remaining subagent's rules when deletion cleanup is off", () => @@ -1335,7 +1682,7 @@ describe("deleted thread worktrees", () => { reason: "Inactive for 30 days", }); expect(yield* fixture.fs.exists(fixture.worktree)).toBe(false); - }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), + }).pipe(Effect.provide(cleanupTestLayer)), ); // Releases one stopped-session event after the startup sweep and returns the snapshot reads it caused. @@ -1394,7 +1741,7 @@ describe("deleted thread worktrees", () => { const fixture = yield* cleanupFixture(); const reads = yield* stopSessionAfterStartupSweep(fixture, ThreadId.make("thread-1")); expect(reads, "a live thread's stopped session re-read every thread").toBe(0); - }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), + }).pipe(Effect.provide(cleanupTestLayer)), ); it.effect("removes a deleted thread's worktree once its session stops", () => @@ -1416,7 +1763,7 @@ describe("deleted thread worktrees", () => { ); expect(reads).toBeGreaterThan(0); expect(yield* fixture.fs.exists(fixture.worktree)).toBe(false); - }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), + }).pipe(Effect.provide(cleanupTestLayer)), ); it.effect("removes a deleted thread's worktree once a live thread's session in it stops", () => @@ -1446,7 +1793,7 @@ describe("deleted thread worktrees", () => { ); expect(reads).toBeGreaterThan(0); expect(yield* fixture.fs.exists(fixture.worktree)).toBe(false); - }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), + }).pipe(Effect.provide(cleanupTestLayer)), ); it.effect( @@ -1492,7 +1839,7 @@ describe("deleted thread worktrees", () => { ); expect(reads).toBeGreaterThan(0); expect(yield* fixture.fs.exists(fixture.worktree)).toBe(false); - }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), + }).pipe(Effect.provide(cleanupTestLayer)), ); it.effect( @@ -1525,7 +1872,7 @@ describe("deleted thread worktrees", () => { ); expect(reads).toBeGreaterThan(0); expect(yield* fixture.fs.exists(fixture.worktree)).toBe(false); - }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), + }).pipe(Effect.provide(cleanupTestLayer)), ); it.effect("does not sweep for a kept deleted worktree when a session elsewhere stops", () => @@ -1547,7 +1894,7 @@ describe("deleted thread worktrees", () => { ); expect(reads).toBe(0); expect(yield* fixture.fs.exists(fixture.worktree)).toBe(true); - }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), + }).pipe(Effect.provide(cleanupTestLayer)), ); it.effect( @@ -1577,7 +1924,7 @@ describe("deleted thread worktrees", () => { fixture.repo, ); expect(reads).toBe(0); - }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), + }).pipe(Effect.provide(cleanupTestLayer)), ); it.effect( @@ -1651,7 +1998,7 @@ describe("deleted thread worktrees", () => { yield* Deferred.await(delivered); yield* fixture.service.drain; expect(yield* fixture.fs.exists(fixture.worktree)).toBe(false); - }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), + }).pipe(Effect.provide(cleanupTestLayer)), ); it.effect("waits out the settle grace of a subagent that just settled", () => @@ -1672,7 +2019,7 @@ describe("deleted thread worktrees", () => { reason: "Settled moments ago", }); expect(yield* fixture.fs.exists(fixture.worktree)).toBe(true); - }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped), + }).pipe(Effect.provide(cleanupTestLayer)), ); }); @@ -1761,7 +2108,7 @@ describe("orphan worktree reports", () => { ), ); - it.live.each(["nested", "direct"] as const)( + it.live.each(["nested", "direct", "annexed"] as const)( "reports a linked orphan's size without removing its files (%s)", (location) => runCleanupTest( @@ -1770,8 +2117,13 @@ describe("orphan worktree reports", () => { fixture.setThreads([]); const orphan = `${fixture.config.worktreesDir}/${location === "nested" ? "repository/" : ""}orphan`; yield* fixture.command(fixture.repo, ["worktree", "add", "-b", "orphan", orphan]); - const sizes = yield* Effect.forEach(yield* fixture.fs.readDirectory(orphan), (name) => - allocatedBytes(fixture.fs, `${orphan}/${name}`), + if (location === "annexed") yield* annexWorktree(fixture, orphan); + const sizes = yield* Effect.forEach( + // Links are not measured, and the annexed `.git` is one. + (yield* fixture.fs.readDirectory(orphan)).filter( + (name) => location !== "annexed" || name !== ".git", + ), + (name) => allocatedBytes(fixture.fs, `${orphan}/${name}`), ); const outside = yield* fixture.fs.makeTempDirectoryScoped(); yield* fixture.fs.writeFileString(`${outside}/external`, "outside data"); @@ -2244,7 +2596,8 @@ describe("storage cleanup reports and local file policies", () => { occurredAt: at(0), payload: { ...shell({ deletedAt: at(0) }), lastVisitedAt: null }, }); - expect(fixture.snapshotReads()).toBe(startupReads + 3 * Number(worktreeOnDelete)); + // Deletion always sweeps so the deleted thread's checkpoint refs are pruned. + expect(fixture.snapshotReads()).toBe(startupReads + 2 * Number(worktreeOnDelete) + 1); expect(fixture.sessionLiveChecks()).toBe(worktreeOnDelete ? 6 : 0); }), ), @@ -2284,6 +2637,114 @@ describe("storage cleanup reports and local file policies", () => { }), ), ); + it.effect("removes an idle No project thread's scratch folder past the retention", () => + runCleanupTest( + Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); + const fixture = yield* cleanupFixture(); + const scratchRoot = `${fixture.config.baseDir}/scratch`; + const folder = `${scratchRoot}/quiet-otter`; + yield* fixture.fs.makeDirectory(folder, { recursive: true }); + yield* fixture.fs.writeFileString(`${folder}/notes.md`, "scratch\n"); + fixture.setProjectRoots([scratchRoot]); + fixture.setStoredWorktreePaths([]); + // Only this rule lets cleanup delete untracked files, which is all a scratch folder has. + fixture.setPolicy("tracked-changes"); + fixture.setThreads([ + shell({ worktreePath: folder, settledAt: at(-20 * DAY_MS), status: "completed" }), + ]); + const report = yield* fixture.service.runNow; + expect(report.entries).toEqual([ + expect.objectContaining({ path: folder, outcome: "removed" }), + ]); + expect(yield* fixture.fs.exists(folder)).toBe(false); + }), + ), + ); + it.effect.each([ + { + name: "a Git repository in a subfolder", + policy: "tracked-changes", + repository: "clone/.git", + reason: "Contains a nested Git repository", + }, + { + name: "a Git repository of its own", + policy: "tracked-changes", + repository: ".git", + reason: "Contains a nested Git repository", + }, + { name: "files under the default keep rule", reason: "Has local files (notes.md)" }, + { + name: "a scheduled task", + policy: "tracked-changes", + scheduled: true, + reason: "Thread has a scheduled task", + }, + ] as const)( + "keeps an idle No project thread's scratch folder with $name", + ({ + policy, + repository, + scheduled, + reason, + }: { + readonly policy?: WorktreeKeepWhen; + readonly repository?: string; + readonly scheduled?: boolean; + readonly reason: string; + }) => + runCleanupTest( + Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); + const fixture = yield* cleanupFixture(); + const scratchRoot = `${fixture.config.baseDir}/scratch`; + const folder = `${scratchRoot}/quiet-otter`; + yield* fixture.fs.makeDirectory( + repository === undefined ? folder : `${folder}/${repository}`, + { recursive: true }, + ); + yield* fixture.fs.writeFileString(`${folder}/notes.md`, "scratch\n"); + // No thread owns this one, so it is listed and never removed. + const leftover = `${scratchRoot}/leftover`; + yield* fixture.fs.makeDirectory(leftover); + yield* fixture.fs.utimes(leftover, 1, 1); + fixture.setProjectRoots([scratchRoot]); + fixture.setStoredWorktreePaths([]); + if (policy !== undefined) fixture.setPolicy(policy); + if (scheduled) fixture.setScheduledThreadIds(["thread-1"]); + fixture.setThreads([ + shell({ worktreePath: folder, settledAt: at(-20 * DAY_MS), status: "completed" }), + ]); + const report = yield* fixture.service.runNow; + expect( + report.entries.map(({ path, outcome, reason }) => ({ path, outcome, reason })), + ).toEqual([ + { path: leftover, outcome: "kept", reason: "No thread uses this folder" }, + { path: folder, outcome: "kept", reason }, + ]); + expect(yield* fixture.fs.exists(`${folder}/notes.md`)).toBe(true); + }), + ), + ); + it.effect("keeps a thread's worktree folder that lost its .git link", () => + runCleanupTest( + Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); + const fixture = yield* cleanupFixture(); + fixture.setThreads([ + shell({ branch: "feature", worktreePath: fixture.worktree, archivedAt: at(-DAY_MS) }), + ]); + yield* fixture.fs.remove(`${fixture.worktree}/.git`); + yield* fixture.fs.makeDirectory(`${fixture.worktree}/node_modules`); + const report = yield* fixture.service.runNow; + expect(report.entries).toEqual([ + expect.objectContaining({ path: fixture.worktree, outcome: "kept" }), + ]); + expect(yield* fixture.fs.exists(`${fixture.worktree}/node_modules`)).toBe(true); + }), + ), + ); it.live("skips a stored worktree path whose parent is a regular file", () => runCleanupTest( Effect.gen(function* () { @@ -2299,18 +2760,30 @@ describe("storage cleanup reports and local file policies", () => { }), ), ); - it.live.each(["before-check", "while-measuring"] as const)( - "keeps a worktree protected by a missing project root (%s)", - (timing) => + it.live.each([ + ["before-check", "stored"], + ["while-measuring", "stored"], + ["before-check", "symlinked parent"], + ["while-measuring", "symlinked parent"], + ] as const)( + "keeps a worktree protected by a missing project root (%s, %s spelling)", + ([timing, spelling]) => runCleanupTest( Effect.gen(function* () { const fixture = yield* pausedMeasurementFixture(); + let worktree = fixture.worktree; + if (spelling === "symlinked parent") { + const linkedParent = `${yield* fixture.fs.makeTempDirectoryScoped({ prefix: "cleanup-link-" })}/worktrees`; + yield* fixture.fs.symlink(fixture.config.worktreesDir, linkedParent); + worktree = `${linkedParent}/feature`; + fixture.setThreads([shell({ branch: "feature", worktreePath: worktree })]); + } if (timing === "before-check") - fixture.setProjectRoots([fixture.repo, `${fixture.worktree}/removed-checkout`]); + fixture.setProjectRoots([fixture.repo, `${worktree}/removed-checkout`]); const cleanup = yield* fixture.service.runNow.pipe(Effect.forkChild); if (timing === "while-measuring") { yield* Deferred.await(fixture.measuring); - fixture.setProjectRoots([fixture.repo, `${fixture.worktree}/removed-checkout`]); + fixture.setProjectRoots([fixture.repo, `${worktree}/removed-checkout`]); } yield* Deferred.succeed(fixture.resume, undefined); const report = yield* Fiber.join(cleanup); @@ -2444,12 +2917,13 @@ describe("storage cleanup reports and local file policies", () => { }), ), ); - it.live.each([ + it.effect.each([ { days: 8, reason: "Inactive for 1 of 8 days" }, { days: null, reason: "No rules apply" }, ])("keeps an ineligible age/delete-only worktree without Git ($days)", ({ days, reason }) => runCleanupTest( Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); const fixture = yield* cleanupFixture({ ...DEFAULT_SERVER_SETTINGS.storageCleanup, worktreeAfterDays: days, @@ -2459,9 +2933,7 @@ describe("storage cleanup reports and local file policies", () => { shell({ branch: "feature", worktreePath: fixture.worktree, - latestUserMessageAt: DateTime.makeUnsafe( - DateTime.toEpochMillis(yield* DateTime.now) - DAY_MS, - ), + latestUserMessageAt: at(-DAY_MS), }), ]); yield* fixture.fs.writeFileString(`${fixture.worktree}/tracked.txt`, "local edit\n"); @@ -2505,6 +2977,17 @@ describe("storage cleanup reports and local file policies", () => { }), ), ); + it.live("removes a worktree whose .git file git-annex replaced with a symlink", () => + runCleanupTest( + Effect.gen(function* () { + const fixture = yield* cleanupFixture(); + const { service, worktree, fs } = fixture; + yield* annexWorktree(fixture, worktree); + expect((yield* service.runNow).entries[0]).toMatchObject({ outcome: "removed" }); + expect(yield* fs.exists(worktree)).toBe(false); + }), + ), + ); it.live("keeps a worktree when the inspection cannot read its directories", () => runCleanupTest( Effect.gen(function* () { @@ -2527,7 +3010,37 @@ describe("storage cleanup reports and local file policies", () => { }), ), ); - it.live.each([ + it.effect("removes a worktree owned by a repository other than the project root", () => + runCleanupTest( + Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); + const fixture = yield* cleanupFixture(); + const { fs, command, config } = fixture; + const oldRepo = yield* fs.realPath( + yield* fs.makeTempDirectoryScoped({ prefix: "cleanup-old-repo-" }), + ); + yield* command(oldRepo, ["init", "-b", "main"]); + yield* command(oldRepo, ["config", "user.email", "cleanup@example.test"]); + yield* command(oldRepo, ["config", "user.name", "Cleanup test"]); + yield* fs.writeFileString(`${oldRepo}/tracked.txt`, "original\n"); + yield* command(oldRepo, ["add", "."]); + yield* command(oldRepo, ["commit", "-m", "initial"]); + yield* fs.makeDirectory(`${config.worktreesDir}/old-repo`, { recursive: true }); + const oldWorktree = `${config.worktreesDir}/old-repo/feature`; + yield* command(oldRepo, ["worktree", "add", "-b", "feature", oldWorktree]); + fixture.setThreads([shell({ branch: "feature", worktreePath: oldWorktree })]); + const report = yield* fixture.service.runNow; + expect(report.entries.find((entry) => entry.path === oldWorktree)).toMatchObject({ + outcome: "removed", + }); + expect(yield* fs.exists(oldWorktree)).toBe(false); + expect((yield* command(oldRepo, ["worktree", "list", "--porcelain"])).stdout).not.toContain( + oldWorktree, + ); + }), + ), + ); + it.effect.each([ [ "its owner's branch is cleared", (worktree: string) => [shell({ branch: null, worktreePath: worktree, updatedAt: at(0) })], @@ -2564,6 +3077,7 @@ describe("storage cleanup reports and local file policies", () => { ] as const)("keeps a worktree when %s during size measurement", ([, threadsWhileMeasuring]) => runCleanupTest( Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); const fixture = yield* pausedMeasurementFixture(); const { fs, worktree } = fixture; fixture.setPolicy("tracked-changes"); @@ -2584,6 +3098,20 @@ describe("storage cleanup reports and local file policies", () => { }), ), ); + it.live("follows a branch switched while nothing watched the worktree", () => + runCleanupTest( + Effect.gen(function* () { + const { service, worktree, command, threadBranches } = yield* cleanupFixture(); + yield* command(worktree, ["switch", "-c", "renamed"]); + expect((yield* service.runNow).entries[0]).toMatchObject({ + outcome: "kept", + reason: "Repository or branch changed", + }); + expect(threadBranches()).toEqual(["renamed"]); + expect((yield* service.runNow).entries[0]).toMatchObject({ outcome: "removed" }); + }), + ), + ); it.live("does not measure a worktree kept for an open provider session", () => runCleanupTest( Effect.gen(function* () { @@ -2615,24 +3143,39 @@ describe("storage cleanup reports and local file policies", () => { }), ), ); + it.live("keeps a worktree for a live session in a missing folder under its stored alias", () => + runCleanupTest( + Effect.gen(function* () { + const fixture = yield* cleanupFixture(); + const linkedParent = `${yield* fixture.fs.makeTempDirectoryScoped({ prefix: "cleanup-link-" })}/worktrees`; + yield* fixture.fs.symlink(fixture.config.worktreesDir, linkedParent); + fixture.setThreads([shell({ branch: "feature", worktreePath: `${linkedParent}/feature` })]); + fixture.setOpenSessionCwd(`${linkedParent}/feature/removed-folder`); + const report = yield* fixture.service.runNow; + expect(report.entries[0]).toMatchObject({ + outcome: "kept", + reason: "Provider session is still open", + }); + expect(yield* fixture.fs.exists(fixture.worktree)).toBe(true); + }), + ), + ); it.live("compares the checked-out branch without a full status", () => runCleanupTest( Effect.gen(function* () { - const { service, fs, repo, worktree, command, setThreads, fullStatusCalls } = + const { service, fs, repo, worktree, command, fullStatusCalls, threadBranches } = yield* cleanupFixture(); const changed = { outcome: "kept", reason: "Repository or branch changed" }; yield* command(worktree, ["checkout", "--detach"]); expect((yield* service.runNow).entries[0]).toMatchObject(changed); yield* command(worktree, ["switch", "--orphan", "other"]); expect((yield* service.runNow).entries[0]).toMatchObject(changed); - yield* command(worktree, ["switch", "-c", "(feature)", "feature"]); - setThreads([shell({ branch: "(feature)", worktreePath: worktree })]); - expect((yield* service.runNow).entries[0]).toMatchObject(changed); - setThreads([shell({ branch: "feature", worktreePath: worktree })]); yield* command(worktree, ["tag", "feature", "feature"]); // Newer Git refuses to create a tag named HEAD, but still reads one. yield* command(worktree, ["update-ref", "refs/tags/HEAD", "refs/heads/feature"]); yield* command(worktree, ["switch", "feature"]); + expect((yield* service.runNow).entries[0]).toMatchObject(changed); + expect(threadBranches()).toEqual(["feature"]); const index = (yield* command(worktree, ["rev-parse", "--git-path", "index"])).stdout; yield* fs.writeFileString(`${index.trim()}.lock`, ""); const locked = (yield* service.runNow).entries[0]; @@ -2699,7 +3242,7 @@ describe("storage cleanup reports and local file policies", () => { }), ), ); - it.live.each([ + it.effect.each([ ["unchanged", "removed", "No commits beyond the project branch"], ["unmerged", "kept", "Has commits beyond the project branch"], ["detached", "kept", "Project branch is unavailable"], @@ -2707,6 +3250,7 @@ describe("storage cleanup reports and local file policies", () => { ] as const)("cleans archived local-only worktrees: %s", ([state, outcome, reason]) => runCleanupTest( Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); const { service, fs, repo, worktree, command, setThreads } = yield* cleanupFixture({ ...DEFAULT_SERVER_SETTINGS.storageCleanup, worktreeUnchanged: state !== "merged-only", @@ -2729,15 +3273,55 @@ describe("storage cleanup reports and local file policies", () => { }), ), ); + it.live("keeps an unsettled new thread's unchanged worktree after its first turn", () => + runCleanupTest( + Effect.gen(function* () { + const { service, fs, worktree, setThreads } = yield* cleanupFixture({ + ...DEFAULT_SERVER_SETTINGS.storageCleanup, + worktreeSettledAfterDays: 11, + worktreeOnMerge: true, + worktreeUnchanged: true, + }); + const now = yield* DateTime.now; + const minutesAgo = (minutes: number) => DateTime.subtract(now, { minutes }); + const thread = shell({ + branch: "feature", + worktreePath: worktree, + status: "completed", + createdAt: minutesAgo(40), + latestUserMessageAt: minutesAgo(40), + latestRunRequestedAt: minutesAgo(40), + latestRunStartedAt: minutesAgo(40), + latestRunCompletedAt: minutesAgo(20), + updatedAt: minutesAgo(20), + }); + setThreads([thread]); + + const report = yield* service.runNow; + expect(report.entries[0]).toMatchObject({ outcome: "kept" }); + expect(yield* fs.exists(worktree)).toBe(true); + + setThreads([{ ...thread, settledAt: minutesAgo(10), updatedAt: minutesAgo(10) }]); + const settled = yield* service.runNow; + expect(settled.entries[0]).toMatchObject({ + outcome: "removed", + reason: "No commits beyond the project branch", + }); + }), + ), + ); it.live( "discovers trunk without a remote HEAD or tracking ref and removes an integrated worktree", () => runCleanupTest( Effect.gen(function* () { - const { service, fs, repo, worktree, command } = yield* cleanupFixture({ + const { service, fs, repo, worktree, command, setThreads } = yield* cleanupFixture({ ...DEFAULT_SERVER_SETTINGS.storageCleanup, worktreeUnchanged: true, }); + setThreads([ + shell({ branch: "feature", worktreePath: worktree, archivedAt: at(-DAY_MS) }), + ]); const remote = yield* fs.makeTempDirectoryScoped({ prefix: "cleanup-remote-" }); yield* command(remote, ["init", "--bare", "-b", "trunk"]); yield* command(repo, ["remote", "add", "origin", remote]); @@ -2759,10 +3343,13 @@ describe("storage cleanup reports and local file policies", () => { () => runCleanupTest( Effect.gen(function* () { - const { service, fs, repo, worktree, command } = yield* cleanupFixture({ + const { service, fs, repo, worktree, command, setThreads } = yield* cleanupFixture({ ...DEFAULT_SERVER_SETTINGS.storageCleanup, worktreeUnchanged: true, }); + setThreads([ + shell({ branch: "feature", worktreePath: worktree, archivedAt: at(-DAY_MS) }), + ]); const remote = yield* fs.makeTempDirectoryScoped({ prefix: "cleanup-remote-" }); yield* command(remote, ["init", "--bare", "-b", "trunk"]); yield* command(repo, ["remote", "add", "origin", remote]); @@ -2787,10 +3374,11 @@ describe("storage cleanup reports and local file policies", () => { it.live("keeps the worktree when the remote default is unavailable even if main exists", () => runCleanupTest( Effect.gen(function* () { - const { service, fs, repo, worktree, command } = yield* cleanupFixture({ + const { service, fs, repo, worktree, command, setThreads } = yield* cleanupFixture({ ...DEFAULT_SERVER_SETTINGS.storageCleanup, worktreeUnchanged: true, }); + setThreads([shell({ branch: "feature", worktreePath: worktree, archivedAt: at(-DAY_MS) })]); const remote = yield* fs.makeTempDirectoryScoped({ prefix: "cleanup-remote-" }); yield* command(remote, ["init", "--bare", "-b", "trunk"]); yield* command(repo, ["remote", "add", "origin", remote]); @@ -3096,11 +3684,12 @@ describe("storage cleanup reports and local file policies", () => { }); describe("settled worktree rechecks", () => { - it.live.each(["autoResumeLimitedThreads", "snoozeLimitedThreads"] as const)( + it.effect.each(["autoResumeLimitedThreads", "snoozeLimitedThreads"] as const)( "keeps a usage-limited checkout when %s is enabled during measurement", (preference) => runCleanupTest( Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); const fixture = yield* pausedMeasurementFixture(); fixture.setSettings({ autoResumeLimitedThreads: false, snoozeLimitedThreads: false }); fixture.setThreads([ @@ -3118,16 +3707,20 @@ describe("settled worktree rechecks", () => { fixture.setSettings({ [preference]: true }); yield* Deferred.succeed(fixture.resume, undefined); const report = yield* Fiber.join(run); - expect(report.entries[0]?.outcome).toBe("kept"); + expect(report.entries[0]).toMatchObject({ + outcome: "kept", + reason: "Settings changed since check", + }); expect(yield* fixture.fs.exists(fixture.worktree)).toBe(true); }), ), ); - it.live.each([{ settledAt: null }, { settledAt: at(DAY_MS) }])( + it.effect.each([{ settledAt: null }, { settledAt: at(DAY_MS) }])( "keeps a checkout changed during measurement (%j)", (change) => runCleanupTest( Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); const fixture = yield* pausedMeasurementFixture(); const thread = shell({ branch: "feature", @@ -3155,9 +3748,10 @@ describe("settled worktree rechecks", () => { ); }); -it.live("keeps saved usage data when removing a settled checkout", () => +it.effect("keeps saved usage data when removing a settled checkout", () => runCleanupTest( Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); const fixture = yield* cleanupFixture(); fixture.setSettings({ storageCleanup: { ...DEFAULT_SERVER_SETTINGS.storageCleanup, worktreeSettledAfterDays: 0 }, diff --git a/apps/server/src/storageCleanup.ts b/apps/server/src/storageCleanup.ts index aa65f5ffa3915..43ff8254daa6d 100644 --- a/apps/server/src/storageCleanup.ts +++ b/apps/server/src/storageCleanup.ts @@ -1,10 +1,12 @@ // @effect-diagnostics nodeBuiltinImport:off import * as NodeFSP from "node:fs/promises"; import { + CheckpointScopeId, GitCommandError, OrchestrationV2AppThreadJson, OrchestrationV2ProviderSessionJson, } from "@t3tools/contracts"; +import * as Base64Url from "effect/encoding/Base64Url"; import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; import type { @@ -40,19 +42,23 @@ import { parse as parseYaml } from "yaml"; import * as ServerConfig from "./config.ts"; import * as GitManager from "./git/GitManager.ts"; +import * as GitWorkflowService from "./git/GitWorkflowService.ts"; import * as ProjectStore from "./orchestration-v2/ProjectStore.ts"; +import { scratchFolderRoot } from "./project/ManagedProjectFolders.ts"; import * as Orchestrator from "./orchestration-v2/Orchestrator.ts"; import * as ProjectionStore from "./orchestration-v2/ProjectionStore.ts"; import * as ProviderSessionManager from "./orchestration-v2/ProviderSessionManager.ts"; +import { checkpointRefForScopeOrdinal } from "./orchestration-v2/CheckpointService.ts"; import { threadHasQueuedTurnStart } from "./orchestration-v2/ThreadSettlementService.ts"; import { MAX_OVERDUE_ARM_MS } from "./orchestration-v2/UsageLimitRecoveryWorker.ts"; +import * as WorktreeBranchSync from "./orchestration-v2/WorktreeBranchSync.ts"; import { forkParked } from "./serverActivation.ts"; import * as Settings from "./serverSettings.ts"; import * as TerminalManager from "./terminal/Manager.ts"; import { isFilesystemRoot, managedWorktreesDirectories } from "./worktreesDirectory.ts"; import * as GitVcsDriver from "./vcs/GitVcsDriver.ts"; import { isNonRepositoryGitStderr } from "./vcs/GitVcsDriverCore.ts"; -import { removeLinkedWorktree } from "./vcs/worktreeRemoval.ts"; +import { isLinkedWorktree, removeLinkedWorktree } from "./vcs/worktreeRemoval.ts"; import { withWorkspaceLease } from "./workspace/workspaceLease.ts"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; @@ -320,6 +326,8 @@ const make = Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; const git = yield* GitVcsDriver.GitVcsDriver; const gitManager = yield* GitManager.GitManager; + const gitWorkflow = yield* GitWorkflowService.GitWorkflowService; + const reconcileBranch = yield* WorktreeBranchSync.makeReconcile; const terminals = yield* TerminalManager.TerminalManager; const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -346,20 +354,6 @@ const make = Effect.gen(function* () { !path.isAbsolute(relative) ); }; - const hasTerminal = (worktreePath: string) => - [...liveTerminals.values()] - .flatMap((entries) => [...entries.values()]) - .some((terminal) => { - if (terminal.status !== "starting" && terminal.status !== "running") return false; - const cwd = path.resolve(terminal.cwd); - return ( - (terminal.worktreePath !== null && - pathKey(terminal.worktreePath) === pathKey(worktreePath)) || - pathKey(cwd) === pathKey(worktreePath) || - inside(worktreePath, cwd) - ); - }); - // pnpm imports packages by clone or reflink when its store shares a device // with the worktree. Those files have one link but share blocks with the // store, so removing them frees almost nothing. A store on another device @@ -390,13 +384,14 @@ const make = Effect.gen(function* () { ); const measureWorktree = Effect.fn("storageCleanup.measureWorktree")( - function* (worktreePath: string, inspectRepositories = false) { + function* (worktreePath: string, inspectRepositories: boolean | "plain-folder" = false) { const root = yield* fs.realPath(worktreePath); const skipped = yield* sharedPnpmVirtualStore(root); const incomplete = { bytes: null, reason: "worktree could not be fully inspected" }; - const repositories = new Set([pathKey(root)]); + // A plain folder is not a checkout, so a repository at its root counts like one below it. + const repositories = new Set(inspectRepositories === "plain-folder" ? [] : [pathKey(root)]); const moduleStores: string[] = []; - if (inspectRepositories && (yield* fs.exists(path.join(root, ".gitmodules")))) { + if (inspectRepositories === true && (yield* fs.exists(path.join(root, ".gitmodules")))) { const submodules = yield* git.execute({ operation: "StorageCleanup.nestedRepositories", cwd: root, @@ -500,18 +495,21 @@ const make = Effect.gen(function* () { }); const containsProjectRoot = Effect.fn("StorageCleanup.containsProjectRoot")(function* ( - worktreePath: string, + worktreePaths: ReadonlyArray, projects: ReadonlyArray<{ readonly workspaceRoot: string }>, ) { + const contains = (projectPath: string) => + worktreePaths.some( + (worktreePath) => + pathKey(projectPath) === pathKey(worktreePath) || inside(worktreePath, projectPath), + ); for (const project of projects) { const projectPath = path.resolve(project.workspaceRoot); - if (pathKey(projectPath) === pathKey(worktreePath) || inside(worktreePath, projectPath)) - return true; + if (contains(projectPath)) return true; const realPath = yield* fs .realPath(projectPath) .pipe(Effect.orElseSucceed(() => projectPath)); - if (pathKey(realPath) === pathKey(worktreePath) || inside(worktreePath, realPath)) - return true; + if (contains(realPath)) return true; } return false; }); @@ -520,6 +518,7 @@ const make = Effect.gen(function* () { roots: ReadonlyArray, projects: ReadonlyArray<{ readonly workspaceRoot: string }>, entries: StorageCleanupReportEntry[], + scratchRoot: string | null, ) { const rows = yield* sql<{ worktreePath: string }>` SELECT DISTINCT json_extract(payload_json, '$.worktreePath') AS "worktreePath" @@ -558,12 +557,14 @@ const make = Effect.gen(function* () { (root) => !inside(root, managedRoot) && (pathKey(root) === pathKey(target) || inside(root, target)), ); - const directories = Effect.fnUntraced(function* (parent: string, managedRoot: string) { - if (inProject(parent, managedRoot)) return []; + // The scratch root is itself a project, so its folders skip the project check. + const directories = Effect.fnUntraced(function* (parent: string, managedRoot: string | null) { + const skipped = (target: string) => managedRoot !== null && inProject(target, managedRoot); + if (skipped(parent)) return []; const result: string[] = []; for (const name of yield* fs.readDirectory(parent).pipe(Effect.orElseSucceed(() => []))) { const target = path.join(parent, name); - if (!inside(parent, target) || inProject(target, managedRoot)) continue; + if (!inside(parent, target) || skipped(target)) continue; const directory = yield* Effect.gen(function* () { if (pathKey(yield* fs.realPath(target)) !== pathKey(target)) return false; return (yield* fs.stat(target)).type === "Directory"; @@ -578,7 +579,7 @@ const make = Effect.gen(function* () { let firstMeasurement = true; const reported = new Set(); const report = Effect.fnUntraced( - function* (worktreePath: string) { + function* (worktreePath: string, scratch = false) { const key = pathKey(worktreePath); if ( reported.has(key) || @@ -590,7 +591,7 @@ const make = Effect.gen(function* () { const modified = Option.getOrNull((yield* fs.stat(worktreePath)).mtime); if (modified !== null && modified.getTime() > now - ORPHAN_WORKTREE_GRACE_MS) return; const gitPath = path.join(worktreePath, ".git"); - const linked = (yield* fs.exists(gitPath)) && (yield* fs.stat(gitPath)).type === "File"; + const linked = (yield* fs.exists(gitPath)) && (yield* isLinkedWorktree(worktreePath)); reported.add(key); const remaining = measurementDeadline - (yield* Clock.currentTimeMillis); if (!orphanSizes.has(key) && (firstMeasurement || remaining > 0)) { @@ -606,7 +607,9 @@ const make = Effect.gen(function* () { outcome: "kept", reason: linked ? "No thread uses this worktree" - : "No thread uses this folder, and it isn't a git worktree", + : scratch + ? "No thread uses this folder" + : "No thread uses this folder, and it isn't a git worktree", path: worktreePath, projectPath: null, threadId: null, @@ -633,7 +636,7 @@ const make = Effect.gen(function* () { (worktree) => Effect.gen(function* () { if (pathKey(yield* fs.realPath(worktree)) !== pathKey(worktree)) return false; - return (yield* fs.stat(path.join(worktree, ".git"))).type === "File"; + return yield* isLinkedWorktree(worktree); }).pipe(Effect.orElseSucceed(() => false)), ); }); @@ -644,7 +647,7 @@ const make = Effect.gen(function* () { if (referenced.has(pathKey(projectDirectory))) return; const gitPath = path.join(projectDirectory, ".git"); if (yield* fs.exists(gitPath)) { - if ((yield* fs.stat(gitPath)).type === "File") yield* report(projectDirectory); + if (yield* isLinkedWorktree(projectDirectory)) yield* report(projectDirectory); return; } for (const worktreePath of yield* directories(projectDirectory, root)) { @@ -654,6 +657,8 @@ const make = Effect.gen(function* () { }).pipe(Effect.orElseSucceed(() => undefined)); } } + if (scratchRoot !== null) + for (const folder of yield* directories(scratchRoot, null)) yield* report(folder, true); for (const key of orphanSizes.keys()) { if (!reported.has(key)) orphanSizes.delete(key); } @@ -706,9 +711,24 @@ const make = Effect.gen(function* () { if (head === null) return true; const ref = head.exitCode === 0 ? head.stdout.trim() : ""; const name = ref.startsWith("refs/heads/") ? ref.slice("refs/heads/".length).trim() : null; - return (name?.startsWith("(") ? null : name) !== branch; + return name !== branch; }); + // Nothing watches an idle worktree, so its threads still record the branch it left. + // Record the checked-out one so the next sweep can apply the rules to it. + const followBranch = Effect.fn("StorageCleanup.followBranch")( + function* (worktreePath: string) { + const cwd = yield* fs.realPath(worktreePath); + yield* reconcileBranch({ cwd, refName: yield* gitWorkflow.currentBranch(cwd) }); + }, + (effect, worktreePath) => + effect.pipe( + Effect.catchCause((cause) => + Effect.logWarning("Failed to reconcile worktree branch", { cwd: worktreePath, cause }), + ), + ), + ); + // Git status and non-forced worktree removal both skip assume-unchanged and skip-worktree files. const hiddenTrackedChange = Effect.fn("StorageCleanup.hiddenTrackedChange")(function* ( cwd: string, @@ -802,6 +822,21 @@ const make = Effect.gen(function* () { return changed === undefined ? null : reason(changed.file); }); + // A worktree can outlive its project moving to another clone. Git removes a + // worktree through the repository that registered it, not through the + // project's current checkout, so resolve the owning repository first. + const worktreeRepository = Effect.fn("StorageCleanup.worktreeRepository")(function* ( + worktreePath: string, + ) { + const commonDir = yield* git.execute({ + operation: "StorageCleanup.worktreeRepository", + cwd: worktreePath, + args: ["rev-parse", "--git-common-dir"], + }); + const resolved = path.resolve(worktreePath, commonDir.stdout.trim()); + return path.basename(resolved) === ".git" ? path.dirname(resolved) : resolved; + }); + const localChangesReason = Effect.fn("StorageCleanup.localChangesReason")(function* ( cwd: string, rules: WorktreeCleanupRules, @@ -899,6 +934,8 @@ const make = Effect.gen(function* () { "rev-list", "--max-count=1", "HEAD", + // Checkpoint refs hold workspace snapshots, not commits a user could lose. + "--exclude=refs/t3/*", "--all", "--reflog", "--not", @@ -1011,6 +1048,32 @@ const make = Effect.gen(function* () { ); if (root !== null && !isFilesystemRoot(root, path)) roots.push(root); } + const scratchRoot = yield* fs + .realPath(scratchFolderRoot(config.baseDir, path)) + .pipe(Effect.orElseSucceed(() => null)); + // Nothing can bring a scratch folder's files back, so these hold whatever the retention says. + const scratchKeepReason = Effect.fn("StorageCleanup.scratchKeepReason")(function* ( + folder: string, + threadIds: ReadonlyArray, + rules: WorktreeCleanupRules, + ) { + const realParent = yield* fs.realPath(path.dirname(folder)); + if ((yield* fs.realPath(folder)) !== path.join(realParent, path.basename(folder))) + return "folder is a symbolic link"; + // Every file here is untracked, and only "tracked-changes" lets cleanup delete those. + if (rules.worktreeKeepWhen !== "tracked-changes") { + const file = (yield* fs.readDirectory(folder)).find((name) => name !== "node_modules"); + if (file !== undefined) return `has local files (${file})`; + } + // Each run of a scheduled task works on what its earlier runs left in the folder. + const scheduled = yield* sql<{ threadId: string }>` + SELECT thread_id AS "threadId" FROM scheduled_tasks + WHERE enabled = 1 AND thread_id IS NOT NULL + `.pipe(NodeSqliteClient.readOnly); + if (scheduled.some((task) => threadIds.includes(task.threadId))) + return "thread has a scheduled task"; + return null; + }); const worktreePathReason = Effect.fn("StorageCleanup.worktreePathReason")(function* ( worktreePath: string, ) { @@ -1021,8 +1084,10 @@ const make = Effect.gen(function* () { return "worktree is a symbolic link"; if (!roots.some((root) => inside(root, realPath))) return "outside the managed worktree folder"; - if ((yield* fs.stat(path.join(worktreePath, ".git"))).type !== "File") - return "not a linked worktree"; + const linked = yield* isLinkedWorktree(worktreePath).pipe( + Effect.catchReason("PlatformError", "NotFound", () => Effect.succeed(false)), + ); + if (!linked) return "not a linked worktree"; return null; }); const deletedRows = yield* sql<{ payload_json: string; workspaceRoot: string }>` @@ -1040,20 +1105,58 @@ const make = Effect.gen(function* () { const threadsById = new Map(snapshot.threads.map((thread) => [thread.id, thread])); const refreshedDefaultRefs = new Map>(); const defaultBranches = new Map(); - const groups = Map.groupBy( - snapshot.threads.filter((thread) => thread.worktreePath !== null), - (thread) => path.resolve(thread.worktreePath!), + // Threads may store one checkout through different symlinked spellings; compare real paths. + // Keys are only compared, never opened, so they take pathKey's Windows case folding. + const canonicalPaths = new Map(); + const canonical = Effect.fnUntraced(function* (target: string) { + const resolved = path.resolve(target); + const cached = canonicalPaths.get(resolved); + if (cached !== undefined) return cached; + const realPath = yield* fs.realPath(resolved).pipe(Effect.orElseSucceed(() => resolved)); + const key = pathKey(realPath); + canonicalPaths.set(resolved, key); + return key; + }); + // A cwd in a deleted folder cannot be resolved, so stored spellings are compared too. + const occupies = Effect.fnUntraced(function* ( + worktreePaths: ReadonlyArray, + target: string, + ) { + const targets = [path.resolve(target), yield* canonical(target)]; + return worktreePaths.some((worktree) => + targets.some((cwd) => pathKey(cwd) === pathKey(worktree) || inside(worktree, cwd)), + ); + }); + const hasTerminal = Effect.fnUntraced(function* (worktreePaths: ReadonlyArray) { + for (const terminal of [...liveTerminals.values()].flatMap((entries) => [ + ...entries.values(), + ])) { + if (terminal.status !== "starting" && terminal.status !== "running") continue; + if ( + (terminal.worktreePath !== null && + (yield* occupies(worktreePaths, terminal.worktreePath))) || + (yield* occupies(worktreePaths, terminal.cwd)) + ) + return true; + } + return false; + }); + const liveThreads = snapshot.threads.filter((thread) => thread.worktreePath !== null); + const liveKeys = yield* Effect.forEach(liveThreads, (thread) => + canonical(thread.worktreePath!), ); - const deletedByPath = Map.groupBy(deletedThreads, (thread) => - path.resolve(thread.worktreePath!), + const groups = Map.groupBy(liveThreads, (_thread, index) => liveKeys[index]!); + const deletedKeys = yield* Effect.forEach(deletedThreads, (thread) => + canonical(thread.worktreePath!), ); + const deletedByPath = Map.groupBy(deletedThreads, (_thread, index) => deletedKeys[index]!); const candidates = [ ...[...groups].map( - ([worktreePath, group]) => + ([worktreeKey, group]) => // A deleted thread still owns the worktree its remaining subagents share, // but only deletion cleanup acts on it; otherwise the live rules apply. deletedByPath - .get(worktreePath) + .get(worktreeKey) ?.find( (deleted) => resolveWorktreeCleanup(serverSettings, deleted.projectId).worktreeOnDelete && @@ -1062,13 +1165,20 @@ const make = Effect.gen(function* () { worktreeOwner(group) ?? group[0]!, ), - ...deletedThreads.filter((thread) => !groups.has(path.resolve(thread.worktreePath!))), + ...deletedThreads.filter((_thread, index) => !groups.has(deletedKeys[index]!)), ]; for (const thread of candidates) { const settings = resolveWorktreeCleanup(serverSettings, thread.projectId); if (!worktreeCleanupEnabled(settings)) continue; const worktreePath = path.resolve(thread.worktreePath!); - const group = groups.get(worktreePath) ?? []; + const worktreeKey = yield* canonical(worktreePath); + const worktreePaths = [worktreePath, worktreeKey]; + const group = groups.get(worktreeKey) ?? []; + // No project threads have no branch; their folders sit directly in the scratch root. + const scratch = + thread.branch === null && + scratchRoot !== null && + pathKey(path.dirname(worktreeKey)) === pathKey(scratchRoot); const activityAt = Math.max(...group.map(storageCleanupActivityAt)); const deleted = "workspaceRoot" in thread; const settlementChain = deleted @@ -1115,14 +1225,12 @@ const make = Effect.gen(function* () { if (members.length > 1 && worktreeOwner(members) === undefined) return keep(`shared by ${members.length} threads`); if (project === undefined) return keep("project is unavailable"); - if ( - thread.branch === null || - !group.every((member) => storageCleanupThreadIdle(member, now, serverSettings)) - ) + if (!group.every((member) => storageCleanupThreadIdle(member, now, serverSettings))) return keep(waitForStop("thread is running or has pending work")); + if (thread.branch === null && !scratch) return keep("thread has no branch"); if (group.some((member) => member.branch !== thread.branch)) return keep("a subagent thread's branch differs from this worktree's branch"); - if (hasTerminal(worktreePath)) return keep(waitForStop("open terminal")); + if (yield* hasTerminal(worktreePaths)) return keep(waitForStop("open terminal")); if ( [...group, ...settlementChain.slice(1)].some( (member) => @@ -1145,6 +1253,9 @@ const make = Effect.gen(function* () { settledFor !== null && settings.worktreeSettledAfterDays !== null && settledFor >= settings.worktreeSettledAfterDays * DAY_MS; + // A worktree is unchanged before its first commit, so only finished threads qualify. + const unchangedApplies = + settings.worktreeUnchanged && (settledThread !== undefined || thread.archivedAt !== null); let eligible = deleted || old || settled; let removalReason = deleted ? "thread was deleted" @@ -1160,18 +1271,167 @@ const make = Effect.gen(function* () { : `settled for ${settledDays} of ${settings.worktreeSettledAfterDays} days` : settings.worktreeOnMerge ? "not merged" - : settings.worktreeUnchanged + : unchangedApplies ? "has commits beyond the default branch" - : "no rules apply"; + : settings.worktreeUnchanged + ? "not settled" + : "no rules apply"; + // The merge and unchanged rules read Git, which a scratch folder does not have. + if (scratch && !eligible) + return keep( + settings.worktreeAfterDays !== null || settings.worktreeSettledAfterDays !== null + ? notDueReason + : "no rules apply", + ); // Only Git-based rules can make a not-yet-due worktree eligible, so skip Git otherwise. - if (!eligible && !settings.worktreeUnchanged && !settings.worktreeOnMerge) - return keep(notDueReason); + if (!eligible && !unchangedApplies && !settings.worktreeOnMerge) return keep(notDueReason); + // Re-read this path's threads and sessions so new activity cancels removal; full per-worktree snapshots take seconds. + const blockedSinceCheck = Effect.fnUntraced(function* () { + const latestProjects = yield* projectStore.listShells(); + if (yield* containsProjectRoot(worktreePaths, [project, ...latestProjects])) + return "contains a project checkout"; + const bindings = yield* projections.getThreadWorktreePaths({ includeArchived: true }); + const bindingKeys = yield* Effect.forEach(bindings, (binding) => + canonical(binding.worktreePath), + ); + const latest = bindings.filter((_binding, index) => bindingKeys[index] === worktreeKey); + if (yield* hasTerminal(worktreePaths)) return waitForStop("open terminal"); + if (deleted) { + if ( + !resolveWorktreeCleanup(yield* settingsService.getSettings, thread.projectId) + .worktreeOnDelete + ) + return "thread or cleanup settings changed since check"; + // Durable deletion cleanup must succeed or be explicitly cancelled before removing its checkout. + const pendingCleanup = yield* sql<{ status: string }>` + SELECT status FROM orchestration_v2_effect_outbox + WHERE thread_id = ${thread.id} AND status NOT IN ('succeeded', 'cancelled') + `.pipe(NodeSqliteClient.readOnly); + // Failed effects never retry, so only running work can finish and free the path. + if (pendingCleanup.some((row) => row.status !== "failed")) + return waitForStop("thread deletion is still pending"); + if (pendingCleanup.length > 0) return "thread deletion is still pending"; + } + const latestShells = yield* Effect.forEach(latest, (entry) => + projections.getThreadShell(entry.threadId), + ); + const latestGroup = latestShells.filter((shell) => shell !== null); + const latestKeys = yield* Effect.forEach(latestGroup, (shell) => + shell.worktreePath === null ? Effect.succeed(null) : canonical(shell.worktreePath), + ); + const latestOwner = worktreeOwner(deleted ? [thread, ...latestGroup] : latestGroup); + if ( + latestGroup.length !== group.length || + latestOwner?.id !== thread.id || + latestOwner?.branch !== thread.branch || + latestGroup.some((shell, index) => { + const previous = group.find((member) => member.id === shell.id); + return ( + previous === undefined || + shell.branch !== thread.branch || + !Equal.equals(shell.settledAt, previous.settledAt) || + (shell.settledAt !== null && !Equal.equals(shell.updatedAt, previous.updatedAt)) || + !storageCleanupThreadIdle(shell, now, serverSettings) || + latestKeys[index] !== worktreeKey || + storageCleanupActivityAt(shell) !== storageCleanupActivityAt(previous) + ); + }) + ) + return waitForStop("thread activity or shared worktree changed since check"); + if (settlementChain.length > 1) { + const ancestors = yield* Effect.forEach(settlementChain.slice(1), (ancestor) => + projections.getThreadShell(ancestor.id), + ); + const latestSettlement = worktreeSettlementChain( + latestGroup.find((shell) => shell.id === thread.id)!, + new Map( + [...latestGroup, ...ancestors.filter((ancestor) => ancestor !== null)].map( + (ancestor) => [ancestor.id, ancestor], + ), + ), + now, + serverSettings, + ).at(-1); + if ( + latestSettlement?.id !== settledThread?.id || + !Equal.equals(latestSettlement?.settledAt, settledThread?.settledAt) || + !Equal.equals(latestSettlement?.updatedAt, settledThread?.updatedAt) + ) + return "parent settlement changed since check"; + } + // Sessions can outlive their run and can be shared across app threads. + const sessionRows = yield* sql<{ payload_json: string }>` + SELECT payload_json FROM orchestration_v2_projection_provider_sessions + WHERE status != 'stopped' + `.pipe(NodeSqliteClient.readOnly); + const sessions = yield* Effect.forEach(sessionRows, (row) => + decodeCleanupSession(row.payload_json), + ); + // Ready persisted sessions may have released their runtime; only live sessions protect the path. + for (const session of sessions) { + if ( + (yield* occupies(worktreePaths, session.cwd)) && + (yield* providerSessions.isLive(session.id)) + ) + return waitForStop("provider session is still open"); + } + return null; + }); + const settingsChangedSinceCheck = Effect.gen(function* () { + const currentSettings = yield* settingsService.getSettings; + const current = resolveWorktreeCleanup(currentSettings, thread.projectId); + return ( + currentSettings.autoResumeLimitedThreads !== serverSettings.autoResumeLimitedThreads || + currentSettings.snoozeLimitedThreads !== serverSettings.snoozeLimitedThreads || + Object.keys(settings).some( + (key) => + current[key as keyof typeof settings] !== settings[key as keyof typeof settings], + ) + ); + }); + if (scratch) { + const threadIds = group.map((member) => member.id); + const blocked = + (yield* scratchKeepReason(worktreePath, threadIds, settings)) ?? + (yield* blockedSinceCheck()); + if (blocked !== null) return keep(blocked); + const inspection = yield* measureWorktree(worktreePath, "plain-folder"); + if (inspection.reason !== null) return keep(inspection.reason); + return yield* withWorkspaceLease( + worktreePath, + Effect.gen(function* () { + // Walked again under the lease: a repository deleted with the folder is gone for good. + const finalInspection = yield* measureWorktree(worktreePath, "plain-folder"); + if (finalInspection.reason !== null) return keep(finalInspection.reason); + // The walk takes time, so the cheap checks run last, right before removal. + const blockedWhileMeasuring = + (yield* scratchKeepReason(worktreePath, threadIds, settings)) ?? + (yield* blockedSinceCheck()); + if (blockedWhileMeasuring !== null) return keep(blockedWhileMeasuring); + if (yield* settingsChangedSinceCheck) return keep("settings changed since check"); + yield* fs.remove(worktreePath, { recursive: true }); + entries.push({ + ...entry, + outcome: "removed", + reason: sentence(removalReason), + bytes: inspection.bytes, + }); + // The stored path stays so the next turn recreates the folder, empty. + yield* Effect.logInfo("storage cleanup removed scratch folder", { + threadId: thread.id, + }); + }), + ); + } const pathReason = yield* worktreePathReason(worktreePath); if (pathReason !== null) return keep(pathReason); - if (yield* containsProjectRoot(worktreePath, [project, ...snapshot.projects])) + if (yield* containsProjectRoot(worktreePaths, [project, ...snapshot.projects])) return keep("contains a project checkout"); - if (yield* branchChanged(worktreePath, thread.branch)) + if (yield* branchChanged(worktreePath, thread.branch)) { + // A deleted owner cannot follow, so its remaining subagents stay on its branch. + if (!deleted) yield* followBranch(worktreePath); return keep("repository or branch changed"); + } const changes = yield* localChangesReason(worktreePath, settings); if (changes !== null) return keep(changes); const head = yield* git.resolveCommit({ cwd: worktreePath, revision: "HEAD" }); @@ -1189,8 +1449,12 @@ const make = Effect.gen(function* () { let branch: string | null | undefined; let defaultRef: string; if (remote === null) { - if (!settings.worktreeUnchanged) - return keep("no remote is configured for merged-worktree cleanup"); + if (!unchangedApplies) + return keep( + settings.worktreeUnchanged + ? notDueReason + : "no remote is configured for merged-worktree cleanup", + ); const projectHead = yield* git.execute({ operation: "StorageCleanup.resolveProjectBranch", cwd: repositoryCwd, @@ -1249,7 +1513,7 @@ const make = Effect.gen(function* () { allowNonZeroExit: true, }); const integrated = ancestor.exitCode === 0; - eligible = integrated && settings.worktreeUnchanged; + eligible = integrated && unchangedApplies; if (eligible) removalReason = remote === null @@ -1268,97 +1532,10 @@ const make = Effect.gen(function* () { }); if (eligible) removalReason = "pull request was merged"; } - if (!eligible && remote === null) return keep("has commits beyond the project branch"); + if (!eligible && remote === null) + return keep(integrated ? notDueReason : "has commits beyond the project branch"); } if (!eligible) return keep(notDueReason); - // Re-read this path's threads and sessions so new activity cancels removal; full per-worktree snapshots take seconds. - const blockedSinceCheck = Effect.fnUntraced(function* () { - const latestProjects = yield* projectStore.listShells(); - if (yield* containsProjectRoot(worktreePath, [project, ...latestProjects])) - return "contains a project checkout"; - const latest = (yield* projections.getThreadWorktreePaths({ - includeArchived: true, - })).filter((entry) => path.resolve(entry.worktreePath) === worktreePath); - if (hasTerminal(worktreePath)) return waitForStop("open terminal"); - if (deleted) { - if ( - !resolveWorktreeCleanup(yield* settingsService.getSettings, thread.projectId) - .worktreeOnDelete - ) - return "thread or cleanup settings changed since check"; - // Durable deletion cleanup must succeed or be explicitly cancelled before removing its checkout. - const pendingCleanup = yield* sql<{ status: string }>` - SELECT status FROM orchestration_v2_effect_outbox - WHERE thread_id = ${thread.id} AND status NOT IN ('succeeded', 'cancelled') - `.pipe(NodeSqliteClient.readOnly); - // Failed effects never retry, so only running work can finish and free the path. - if (pendingCleanup.some((row) => row.status !== "failed")) - return waitForStop("thread deletion is still pending"); - if (pendingCleanup.length > 0) return "thread deletion is still pending"; - } - const latestShells = yield* Effect.forEach(latest, (entry) => - projections.getThreadShell(entry.threadId), - ); - const latestGroup = latestShells.filter((shell) => shell !== null); - const latestOwner = worktreeOwner(deleted ? [thread, ...latestGroup] : latestGroup); - if ( - latestGroup.length !== group.length || - latestOwner?.id !== thread.id || - latestOwner?.branch !== thread.branch || - latestGroup.some((shell) => { - const previous = group.find((member) => member.id === shell.id); - return ( - previous === undefined || - shell.branch !== thread.branch || - !Equal.equals(shell.settledAt, previous.settledAt) || - (shell.settledAt !== null && !Equal.equals(shell.updatedAt, previous.updatedAt)) || - !storageCleanupThreadIdle(shell, now, serverSettings) || - path.resolve(shell.worktreePath!) !== worktreePath || - storageCleanupActivityAt(shell) !== storageCleanupActivityAt(previous) - ); - }) - ) - return waitForStop("thread activity or shared worktree changed since check"); - if (settlementChain.length > 1) { - const ancestors = yield* Effect.forEach(settlementChain.slice(1), (ancestor) => - projections.getThreadShell(ancestor.id), - ); - const latestSettlement = worktreeSettlementChain( - latestGroup.find((shell) => shell.id === thread.id)!, - new Map( - [...latestGroup, ...ancestors.filter((ancestor) => ancestor !== null)].map( - (ancestor) => [ancestor.id, ancestor], - ), - ), - now, - serverSettings, - ).at(-1); - if ( - latestSettlement?.id !== settledThread?.id || - !Equal.equals(latestSettlement?.settledAt, settledThread?.settledAt) || - !Equal.equals(latestSettlement?.updatedAt, settledThread?.updatedAt) - ) - return "parent settlement changed since check"; - } - // Sessions can outlive their run and can be shared across app threads. - const sessionRows = yield* sql<{ payload_json: string }>` - SELECT payload_json FROM orchestration_v2_projection_provider_sessions - WHERE status != 'stopped' - `.pipe(NodeSqliteClient.readOnly); - const sessions = yield* Effect.forEach(sessionRows, (row) => - decodeCleanupSession(row.payload_json), - ); - // Ready persisted sessions may have released their runtime; only live sessions protect the path. - for (const session of sessions) { - const cwd = path.resolve(session.cwd); - if ( - (pathKey(cwd) === pathKey(worktreePath) || inside(worktreePath, cwd)) && - (yield* providerSessions.isLive(session.id)) - ) - return waitForStop("provider session is still open"); - } - return null; - }); const blocked = yield* blockedSinceCheck(); if (blocked !== null) return keep(blocked); // Inspect before the lease; provider startup must not wait for directory traversal. @@ -1381,35 +1558,25 @@ const make = Effect.gen(function* () { return keep("commit changed since check"); const finalChanges = yield* localChangesReason(worktreePath, settings); if (finalChanges !== null) return keep(finalChanges); - const currentSettings = yield* settingsService.getSettings; - const current = resolveWorktreeCleanup(currentSettings, thread.projectId); - if ( - currentSettings.autoResumeLimitedThreads !== - serverSettings.autoResumeLimitedThreads || - currentSettings.snoozeLimitedThreads !== serverSettings.snoozeLimitedThreads || - Object.keys(settings).some( - (key) => - current[key as keyof typeof settings] !== settings[key as keyof typeof settings], - ) - ) - return keep("settings changed since check"); - // Clean only untracked files. Ordinary worktrees retain Git's final tracked-edit check. - if (settings.worktreeKeepWhen === "tracked-changes") - yield* cleanupGit(worktreePath, ["clean", "-ffdx"], "git clean"); + if (yield* settingsChangedSinceCheck) return keep("settings changed since check"); const submodules = yield* submoduleRemoval(worktreePath, settings); if (submodules.reason !== null) return keep(submodules.reason); if (submodules.force) { const changes = yield* localChangesReason(worktreePath, settings); if (changes !== null) return keep(changes); } + const repository = yield* worktreeRepository(worktreePath); + // Forced removal already discards local files; do not widen its final-check window. + if (settings.worktreeKeepWhen === "tracked-changes" && !submodules.force) + yield* cleanupGit(worktreePath, ["clean", "-ffdx"], "git clean"); yield* removeLinkedWorktree({ operation: "StorageCleanup.cleanupGit", - cwd: project.workspaceRoot, + cwd: repository, worktreePath, // Git requires force for submodules; recursive checks above cannot make removal atomic. force: submodules.force, execute: git.execute, - run: (args) => cleanupGit(project.workspaceRoot, [...args], "git worktree remove"), + run: (args) => cleanupGit(repository, [...args], "git worktree remove"), }); entries.push({ ...entry, outcome: "removed", reason: sentence(removalReason), bytes }); yield* gitManager.invalidateStatus(project.workspaceRoot); @@ -1438,6 +1605,7 @@ const make = Effect.gen(function* () { roots, yield* projectStore.list({ includeDeleted: true }), entries, + scratchRoot, ); return { waiting, kept }; }); @@ -1481,6 +1649,115 @@ const make = Effect.gen(function* () { }); const reportRef = yield* SubscriptionRef.make(null); + const cleanCheckpointRefs = Effect.fn("StorageCleanup.cleanCheckpointRefs")(function* () { + // Forks display their source history even after its thread is deleted. Keep + // that ancestry, including deleted intermediate forks and archived children. + const rows = yield* sql<{ + threadId: string; + worktreePath: string | null; + workspaceRoot: string | null; + scopeId: string | null; + cwd: string | null; + }>` + WITH RECURSIVE retained_checkpoint_threads(thread_id) AS ( + SELECT thread_id FROM orchestration_v2_projection_threads WHERE deleted_at IS NULL + UNION + SELECT json_extract(t.payload_json, '$.forkedFrom.threadId') + FROM orchestration_v2_projection_threads t + JOIN retained_checkpoint_threads retained ON retained.thread_id = t.thread_id + WHERE json_extract(t.payload_json, '$.forkedFrom.type') = 'run' + ) + SELECT t.thread_id AS "threadId", + json_extract(t.payload_json, '$.worktreePath') AS "worktreePath", + p.workspace_root AS "workspaceRoot", s.scope_id AS "scopeId", + json_extract(s.payload_json, '$.cwd') AS cwd + FROM orchestration_v2_projection_threads t + LEFT JOIN projection_projects p ON p.project_id = t.project_id + LEFT JOIN orchestration_v2_projection_checkpoint_scopes s ON s.thread_id = t.thread_id + WHERE t.deleted_at IS NOT NULL + AND NOT EXISTS (SELECT 1 FROM retained_checkpoint_threads WHERE thread_id = t.thread_id) + AND NOT EXISTS ( + SELECT 1 FROM orchestration_v2_effect_outbox e + WHERE e.thread_id = t.thread_id AND e.status IN ('pending', 'running') + ) + `.pipe(NodeSqliteClient.readOnly); + const targets = new Map>(); + for (const row of rows) { + // Derive ownership from tombstones, never from absence in this database: + // another T3 home can own refs in the same Git repository. + const prefixes = [`refs/t3/checkpoints/${Base64Url.encode(row.threadId)}/turn/`]; + if (row.scopeId !== null) { + prefixes.push( + (yield* checkpointRefForScopeOrdinal({ + scopeId: CheckpointScopeId.make(row.scopeId), + ordinalWithinScope: 0, + })).slice(0, -1), + ); + } + // A worktree may already be gone; its refs live in the shared repository. + for (const cwd of [row.cwd, row.worktreePath, row.workspaceRoot]) { + if (cwd === null) continue; + const key = path.resolve(cwd); + const owned = targets.get(key) ?? new Set(); + for (const prefix of prefixes) owned.add(prefix); + targets.set(key, owned); + } + } + const repositories = new Map }>(); + for (const [cwd, prefixes] of targets) { + yield* Effect.gen(function* () { + if (!(yield* fs.exists(cwd))) return; + const result = yield* git.execute({ + operation: "StorageCleanup.checkpointRepository", + cwd, + args: ["rev-parse", "--path-format=absolute", "--git-common-dir"], + allowNonZeroExit: true, + }); + if (result.exitCode !== 0) return; + const commonDir = yield* fs.realPath(result.stdout.trim()); + const repository = repositories.get(commonDir) ?? { cwd, prefixes: new Set() }; + for (const prefix of prefixes) repository.prefixes.add(prefix); + repositories.set(commonDir, repository); + }).pipe( + Effect.catch((cause) => + Effect.logWarning("checkpoint repository inspection failed", { cause }), + ), + ); + } + for (const { cwd, prefixes } of repositories.values()) { + yield* Effect.gen(function* () { + const result = yield* git.execute({ + operation: "StorageCleanup.listCheckpointRefs", + cwd, + args: [ + "for-each-ref", + "--format=%(refname)", + "refs/t3/checkpoints/", + "refs/t3/orchestration-v2/checkpoints/", + ], + maxOutputBytes: 16 * 1024 * 1024, + }); + const refs = result.stdout.split("\n").filter((ref) => { + const ordinalStart = ref.lastIndexOf("/") + 1; + return ( + prefixes.has(ref.slice(0, ordinalStart)) && + /^(0|[1-9]\d*)$/.test(ref.slice(ordinalStart)) + ); + }); + if (refs.length === 0) return; + yield* git.execute({ + operation: "StorageCleanup.deleteCheckpointRefs", + cwd, + // Remove a symbolic checkpoint itself, never its branch/tag target. + args: ["update-ref", "--no-deref", "-z", "--stdin"], + stdin: refs.map((ref) => `delete ${ref}\0\0`).join(""), + }); + yield* Effect.logInfo("storage cleanup removed checkpoint refs", { refCount: refs.length }); + }).pipe( + Effect.catch((cause) => Effect.logWarning("checkpoint ref cleanup failed", { cause })), + ); + } + }); // Only deleted threads' worktrees wait on session stops: any stop while one waits, or a stop inside one kept for another reason. // A deletion stays pending until a sweep that started after it finishes. let deletedWorktreeWaits = true; @@ -1495,6 +1772,9 @@ const make = Effect.gen(function* () { const serverSettings = yield* settingsService.getSettings; const settings = serverSettings.storageCleanup; const entries: StorageCleanupReportEntry[] = []; + yield* cleanCheckpointRefs().pipe( + Effect.catch((cause) => Effect.logWarning("checkpoint ref cleanup failed", { cause })), + ); const deletedWorktrees = yield* cleanWorktrees(serverSettings, now, entries).pipe( Effect.catch((error) => Effect.gen(function* () { @@ -1673,6 +1953,7 @@ const make = Effect.gen(function* () { ) return Effect.void; } + if (event.type === "thread.deleted") return requestSweep; if (!anyWorktreePolicy(lastSettings, (rules) => rules.worktreeOnDelete)) return Effect.void; return event.type === "provider-session.updated" && event.payload.status === "error" ? providerSessions diff --git a/apps/server/src/telemetry/ProviderDimensions.test.ts b/apps/server/src/telemetry/ProviderDimensions.test.ts index d88010f600e35..00e59c11228cb 100644 --- a/apps/server/src/telemetry/ProviderDimensions.test.ts +++ b/apps/server/src/telemetry/ProviderDimensions.test.ts @@ -76,3 +76,16 @@ it("drops input strings outside the schema's literals, since rejected calls are handoffSettings("t3_thread_launch", { workspaceStrategy: { type: "/home/me/secret" } }, {}), ).toEqual({ targetChosen: false, workspace: "root" }); }); + +it("records the binding the saved scheduled task resolved to, and none for failures", () => { + expect(handoffSettings("schedule_task", {}, { boundThreadId: null })).toEqual({ + bindToCurrentThread: false, + }); + expect( + handoffSettings( + "schedule_task", + { bindToCurrentThread: true }, + { _tag: "OrchestratorMcpFailure", code: "invalid_request", message: "Different project." }, + ), + ).toEqual({}); +}); diff --git a/apps/server/src/telemetry/ProviderDimensions.ts b/apps/server/src/telemetry/ProviderDimensions.ts index f81a22e844aef..8352ed0d15cc0 100644 --- a/apps/server/src/telemetry/ProviderDimensions.ts +++ b/apps/server/src/telemetry/ProviderDimensions.ts @@ -128,9 +128,9 @@ function callerOrigin(input: { } /** - * Settings the agent chose for a handoff, read only from closed enums and - * booleans in the tool's arguments and result. Prompts, titles, and ids are - * never read. + * Settings for a handoff, recorded only as closed enums and booleans derived + * from the tool's arguments and result. Prompts, titles, and ids are never + * recorded. */ export function handoffSettings(tool: string, args: unknown, result: unknown): Fields { const input = asFields(args); @@ -166,8 +166,13 @@ export function handoffSettings(tool: string, args: unknown, result: unknown): F return defined({ delivery: stringField(output, "delivery") }); case "t3_thread_send_attachments": return { delivery: "auto" }; - case "schedule_task": - return { bindToCurrentThread: input.bindToCurrentThread !== false }; + case "schedule_task": { + // Omitting the input lets the server decide, so read the saved task. + const bound = output.boundThreadId; + return bound === null || typeof bound === "string" + ? { bindToCurrentThread: bound !== null } + : {}; + } default: return {}; } diff --git a/apps/server/src/terminal/Manager.test.ts b/apps/server/src/terminal/Manager.test.ts index e937f07cd193d..331c99726ad00 100644 --- a/apps/server/src/terminal/Manager.test.ts +++ b/apps/server/src/terminal/Manager.test.ts @@ -13,6 +13,8 @@ import { ProviderDriverKind, ProviderInstanceId, retainedTerminalHistory, + terminalSequenceEnd, + terminalSequenceStateAfter, TerminalProviderInstanceNotFoundError, } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; @@ -337,7 +339,6 @@ function retainedHistory(text: string, maxLines: number, maxBytes = Infinity): s if (terminated) lines.pop(); const retained = lines.slice(Math.max(0, lines.length - maxLines)).join("\n"); const capped = terminated ? `${retained}\n` : retained; - if (Buffer.byteLength(capped) <= maxBytes) return capped; const points = Array.from(capped); let start = points.length; let bytes = 0; @@ -347,7 +348,11 @@ function retainedHistory(text: string, maxLines: number, maxBytes = Infinity): s bytes += next; start -= 1; } - return points.slice(start).join(""); + const kept = points.slice(start).join(""); + if (maxLines <= 0 || maxBytes <= 0) return kept; + // A cut inside a control sequence drops the rest of that sequence too. + const state = terminalSequenceStateAfter(0, text, 0, text.length - kept.length); + return kept.slice(terminalSequenceEnd(state, kept)); } it("preserves line and byte limits across arbitrary chunks, Unicode, ANSI sequences, and clear", () => { @@ -365,6 +370,10 @@ it("preserves line and byte limits across arbitrary chunks, Unicode, ANSI sequen "\u001b[31m", "\u001b[0m", "\u001b]8;;url\u0007", + "\u001b]0;", + "\u001bP", + "\u0007", + "\u001b\\", "\ud83d", "\ude80", ]; @@ -375,7 +384,8 @@ it("preserves line and byte limits across arbitrary chunks, Unicode, ANSI sequen for (const maxBytes of [0, 3, 8, 64, Infinity]) { for (const maxLines of [0, 1, 3, 5, 5_000]) { - let expected = retainedHistory("before\ninitial\n", maxLines, maxBytes); + let full = "before\ninitial\n"; + let expected = retainedHistory(full, maxLines, maxBytes); const history = new TerminalManager.BoundedTerminalHistory( maxLines, "before\ninitial\n", @@ -386,18 +396,33 @@ it("preserves line and byte limits across arbitrary chunks, Unicode, ANSI sequen for (let step = 0; step < 300; step += 1) { if (step % 73 === 0) { history.clear(); - expected = ""; - expect(history.value()).toBe(expected); + full = ""; + expect(history.value()).toBe(""); } const chunk = nextFragment() + nextFragment(); history.append(chunk); - expected = retainedHistory(expected + chunk, maxLines, maxBytes); + full += chunk; + expected = retainedHistory(full, maxLines, maxBytes); expect(history.value()).toBe(expected); } } } }); +it("does not retain the tail of a control sequence whose introducer was trimmed", () => { + for (const [text, maxBytes, retained] of [ + ["prefix\u001b[31mOK", 6, "OK"], + ["prefix\u001b]0;HIDDEN\u0007OK", 10, "OK"], + ["prefix\u001b]0;🙂HIDDEN\u0007OK", 9, "OK"], + // libghostty leaves every string but OSC at the emoji's second byte, so HELLO was visible. + ["prefix\u001bP🙂HELLO\u001b\\OK", 9, "HELLO\u001b\\OK"], + ] as const) { + expect(new TerminalManager.BoundedTerminalHistory(5_000, text, maxBytes).value()).toBe( + retained, + ); + } +}); + it("bounds long partial lines and joins surrogate pairs across chunk boundaries", () => { const maxBytes = 65_539; let expected = ""; @@ -1172,6 +1197,55 @@ it.layer( }), ); + it.effect("clears a terminal that keeps printing while its history is saved", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + let printWhileSaving = Effect.void; + const printing = FileSystem.FileSystem.of({ + ...fs, + writeFileString: (filePath, data, options) => + printWhileSaving.pipe(Effect.andThen(fs.writeFileString(filePath, data, options))), + }); + const { manager, ptyAdapter, getEvents } = yield* createManager().pipe( + Effect.provideService(FileSystem.FileSystem, printing), + ); + yield* manager.open(openInput()); + const process = ptyAdapter.processes[0]!; + // Each save sees fresh output, so the history worker never goes idle. + printWhileSaving = Effect.gen(function* () { + const printed = yield* Deferred.make(); + const unsubscribe = yield* manager.subscribe((event) => + event.type === "output" ? Deferred.succeed(printed, undefined) : Effect.void, + ); + process.emitData("tick\n"); + yield* Deferred.await(printed); + unsubscribe(); + }); + + yield* manager.clear({ threadId: "thread-1", terminalId: DEFAULT_TERMINAL_ID }); + + expect((yield* getEvents).map((event) => event.type)).toContain("cleared"); + }), + ); + + it.effect("restores a log tail that starts inside a control sequence without its remainder", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + for (const [persisted, restored] of [ + ["\u001b]0;x\u0007ABCD", "ABCD"], + ["名\u001b]0;名\u0007ABCD", "ABCD"], + ["\u001b]0;x\u0007\u001b[31mABCD", "ABCD"], + ] as const) { + const { manager, logsDir } = yield* createManager(5, { historyByteLimit: 8 }); + const path = yield* historyLogPath(logsDir); + yield* fs.makeDirectory(logsDir, { recursive: true }); + yield* fs.writeFileString(path, persisted); + expect((yield* manager.open(openInput())).history).toBe(restored); + expect(yield* fs.readFileString(path)).toBe(restored); + } + }), + ); + it.effect("persists small terminal updates without rewriting the retained history", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -1399,9 +1473,11 @@ it.layer( process.emitData("prompt "); process.emitData("\u001b[?2026$p\u001b[?2026;2$y\u001b[>q\u001b[?u\u001b[?31u"); - process.emitData("\u001b[=c\u001b[=0c"); + process.emitData("\u001b[=c\u001b[=0c\u001bZ"); process.emitData("\u001bP$q m\u001b\\\u001bP1$r0m\u001b\\"); process.emitData("\u001bP+q544e\u001b\\\u001bP1+r544e=1b\u001b\\"); + // BEL is DCS payload, not a terminator: only ST ends the string. + process.emitData("\u001bP$qm\u0007hidden\u001b\\\u001bP+q544e\u0007hidden\u001b\\"); process.emitData('\u001b[!p\u001b["p\u001b[4 q\u001b[u'); process.emitData("\u001b]0;x\u001b\u001b[?2026$p"); process.emitData("done\n"); @@ -1417,6 +1493,50 @@ it.layer( }), ); + // Ghostty executes 0x00-0x0F inside a CSI and ignores DEL and 0x10-0x1F; none change the query. + it.effect.each([ + { name: "NUL in DA", data: "A\u001b[0\u0000cB", history: "AB" }, + { name: "DEL in DA", data: "A\u001b[0\u007fcB", history: "AB" }, + { name: "NUL in DECRQM", data: "A\u001b[?2026\u0000$pB", history: "AB" }, + { name: "CR in DSR", data: "AB\u001b[5\rnC", history: "AB\rC" }, + { name: "DLE in DSR", data: "AB\u001b[5\u0010nC", history: "ABC" }, + // Ghostty reads non-ASCII there as UTF-8 bytes: it drops é and runs 0x80-0x9F as C1 controls. + { name: "é in DA", data: "A\u001b[0écB", history: "AB" }, + { name: "DECID byte in a CSI", data: "A\u001b[嚠B", history: "A\ufffdB" }, + { + name: "C1 DCS then C1 CSI in DSR", + data: "A\u001b[\u0090\u009b6nB", + history: "A\u001bP\u009cB", + }, + { name: "ESC in DSR", data: "A\u001b[5\u001b[6nB", history: "AB" }, + ])( + "strips a query with an embedded control but keeps what the terminal ran ($name)", + ({ data, history }) => + Effect.gen(function* () { + const { manager, ptyAdapter } = yield* createManager(); + yield* manager.open(openInput()); + ptyAdapter.processes[0]!.emitData(data); + yield* manager.close({ threadId: "thread-1" }); + + const reopened = yield* manager.open(openInput()); + const restored = reopened.history.replaceAll("\u0000", ""); + expect(restored.split("\u001b[?1049h")[0]).toBe(history); + }), + ); + + // Ghostty aborts a CSI at a non-ASCII code point and prints it and what follows. + it.effect("keeps a CSI interrupted by non-ASCII text in history", () => + Effect.gen(function* () { + const { manager, ptyAdapter } = yield* createManager(); + yield* manager.open(openInput()); + ptyAdapter.processes[0]!.emitData("A\u001b[界nB"); + yield* manager.close({ threadId: "thread-1" }); + + const reopened = yield* manager.open(openInput()); + expect(reopened.history.split("\u001b[?1049h")[0]).toBe("A\u001b[界nB"); + }), + ); + it.effect.each([ { name: "OSC", sequence: "\u001b]0;unfinished" }, { name: "DCS", sequence: "\u001bPunfinished" }, @@ -1468,6 +1588,21 @@ it.layer( }), ); + it.effect("keeps a decoded U+009C inside a stripped OSC query out of history", () => + Effect.gen(function* () { + const { manager, ptyAdapter } = yield* createManager(); + yield* manager.open(openInput()); + const process = ptyAdapter.processes[0]!; + // libghostty receives UTF-8 (c2 9c), which is OSC payload; only ESC \ ends this string. + process.emitData("A\u001b]11;?\u009cHIDDEN\u001b\\B\n"); + // A DCS string does end at U+009C there, so the text after it stays visible. + process.emitData("C\u001bP$qm\u009cSHOWN\u001b\\D\n"); + yield* manager.close({ threadId: "thread-1" }); + + expect((yield* manager.open(openInput())).history).toMatch(/^AB\nCSHOWN\u001b\\D\n/); + }), + ); + it.effect("preserves large chunked OSC 52 and DCS strings across reopen", () => Effect.gen(function* () { const { manager, ptyAdapter, logsDir } = yield* createManager(); @@ -1572,6 +1707,28 @@ it.layer( }), ); + // Case-insensitive filesystems resolve "Terminal_YQ.log" to thread "a"'s "terminal_YQ.log", + // so the mixed-case file stands in for that alias on case-sensitive hosts. + it.effect.each(["terminal_", "Terminal_"])( + "never resolves a %s legacy history name to another thread's history file", + (prefix) => + Effect.gen(function* () { + const { manager, logsDir } = yield* createManager(); + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const threadId = `${prefix}${Base64Url.encode("a")}`; + const otherThreadPath = path.join(logsDir, `${threadId}.log`); + yield* fs.makeDirectory(logsDir, { recursive: true }); + yield* fs.writeFileString(otherThreadPath, "thread a\n"); + + expect((yield* manager.open(openInput({ threadId }))).history).toBe(""); + yield* manager.close({ threadId, terminalId: DEFAULT_TERMINAL_ID, deleteHistory: true }); + yield* manager.close({ threadId, deleteHistory: true }); + + expect(yield* fs.readFileString(otherThreadPath)).toBe("thread a\n"); + }), + ); + it.effect("keeps reading and deleting history at an encoded path that already fit", () => Effect.gen(function* () { const threadId = "x".repeat(48); @@ -1820,6 +1977,32 @@ it.layer( }), ); + it.effect("starts a $SHELL whose path contains a space", () => + Effect.gen(function* () { + const { manager, ptyAdapter } = yield* createManager(5, { + env: { SHELL: "/Users/Jane Doe/.local/bin/fish" }, + }).pipe(Effect.provide(layerWithHostPlatform("darwin"))); + + yield* manager.open(openInput()); + + expect(ptyAdapter.spawnInputs.map((input) => input.shell)).toEqual([ + "/Users/Jane Doe/.local/bin/fish", + ]); + }), + ); + + it.effect("starts a quoted $SHELL whose path contains a space and a dash", () => + Effect.gen(function* () { + const { manager, ptyAdapter } = yield* createManager(5, { + env: { SHELL: '"/opt/my -shell/fish" -l' }, + }).pipe(Effect.provide(layerWithHostPlatform("darwin"))); + + yield* manager.open(openInput()); + + expect(ptyAdapter.spawnInputs.map((input) => input.shell)).toEqual(["/opt/my -shell/fish"]); + }), + ); + it.effect( "writes completion syntax for the fallback shell instead of the configured fish shell", () => @@ -2442,7 +2625,7 @@ it.layer( { type: "snapshot", snapshot: { status: "running", pid: ptyAdapter.processes[0]?.pid } }, ]); expect(ptyAdapter.processes).toHaveLength(1); - }).pipe(Effect.scoped), + }), ); it.effect("resyncs bounded history and follows close/reopen across a stalled stream", () => @@ -2516,7 +2699,7 @@ it.layer( ptyAdapter.processes.at(-1)!.emitData("new session only\n"); expect(yield* pull).toMatchObject([{ type: "output", data: "new session only\n" }]); expect(ptyAdapter.spawnInputs).toHaveLength(4); - }).pipe(Effect.scoped), + }), ); it.effect.each([ @@ -2563,7 +2746,7 @@ it.layer( yield* Deferred.await(restored); expect(process.resizeCalls.at(-1)).toEqual({ cols: 130, rows: 15 }); - }).pipe(Effect.scoped), + }), ); it.effect("sends bounded attach streams only the history the client retains", () => @@ -2614,7 +2797,7 @@ it.layer( expect(yield* full).toEqual(yield* bounded); yield* manager.attachStream({ ...openInput(), restartIfNotRunning: true }, () => Effect.void); yield* expectRetained(); - }).pipe(Effect.scoped), + }), ); it.effect("does not replay in-flight output already included in an observer snapshot", () => @@ -2651,7 +2834,7 @@ it.layer( expect(observed.filter((event) => event.type === "output")).toMatchObject([ { data: "after\n" }, ]); - }).pipe(Effect.scoped), + }), ); it.effect("observes terminal history and live output without changing the process", () => diff --git a/apps/server/src/terminal/Manager.ts b/apps/server/src/terminal/Manager.ts index 193857f570b54..d658083a13a01 100644 --- a/apps/server/src/terminal/Manager.ts +++ b/apps/server/src/terminal/Manager.ts @@ -6,6 +6,9 @@ import { DEFAULT_TERMINAL_ID, DEFAULT_MAX_TERMINAL_BUFFER_BYTES, retainedTerminalHistory, + TERMINAL_SEQUENCE_GROUND, + terminalSequenceEnd, + terminalSequenceStateAfter, TerminalCwdError, TerminalCwdNotDirectoryError, TerminalCwdNotFoundError, @@ -536,9 +539,11 @@ function normalizeShellCommand( return trimmed; } - const firstToken = trimmed.split(/\s+/g)[0]?.trim(); - if (!firstToken) return null; - return firstToken.replace(/^['"]|['"]$/g, ""); + // A shell path can contain spaces, so only text from the first flag (`bash -l`) is dropped, + // and a quoted path is kept whole even when it contains " -". + const quoted = /^(['"])(.*?)\1(?:\s|$)/.exec(trimmed); + const command = quoted ? quoted[2] : trimmed.split(/\s+(?=-)/)[0]?.replace(/^['"]|['"]$/g, ""); + return command || null; } function basenameForPlatform(command: string, platform: NodeJS.Platform): string { @@ -888,6 +893,14 @@ interface TerminalHistoryPersistenceCursor { readonly fileBytes: number; } +function countLineBreaks(data: string): number { + let lineBreaks = 0; + for (let index = data.indexOf("\n"); index !== -1; index = data.indexOf("\n", index + 1)) { + lineBreaks += 1; + } + return lineBreaks; +} + export class BoundedTerminalHistory { private readonly maxLines: number; private readonly maxBytes: number; @@ -897,6 +910,8 @@ export class BoundedTerminalHistory { private lineBreaks = 0; // Reading the old string's tail on each append can force chunk concatenation. private lastCodeUnit: number | undefined; + // The terminal parser's state at the first retained code unit, which is where a replay starts. + private sequenceState = TERMINAL_SEQUENCE_GROUND; private cachedValue: string | null = ""; private generation = 0; private appendedLength = 0; @@ -961,10 +976,7 @@ export class BoundedTerminalHistory { private appendChunk(data: string): void { const byteLength = Buffer.byteLength(data); - let lineBreaks = 0; - for (let index = data.indexOf("\n"); index !== -1; index = data.indexOf("\n", index + 1)) { - lineBreaks += 1; - } + const lineBreaks = countLineBreaks(data); const previous = this.chunks.at(-1); if (previous && previous.data.length + data.length <= MAX_HISTORY_CHUNK_LENGTH) { previous.data += data; @@ -981,6 +993,7 @@ export class BoundedTerminalHistory { private discardChunk(): void { const first = this.chunks[this.start]!; + this.sequenceState = terminalSequenceStateAfter(this.sequenceState, first.data); this.byteLength -= first.byteLength; this.lineBreaks -= first.lineBreaks; this.chunks[this.start++] = undefined; @@ -992,6 +1005,7 @@ export class BoundedTerminalHistory { this.discardChunk(); return; } + this.sequenceState = terminalSequenceStateAfter(this.sequenceState, first.data, 0, offset); first.data = first.data.slice(offset); first.byteLength -= byteLength; first.lineBreaks -= lineBreaks; @@ -1039,6 +1053,12 @@ export class BoundedTerminalHistory { } this.trimChunk(offset, bytes, lineBreaks); } + // A cut inside a control sequence would replay the rest of it as text. + while (this.sequenceState !== TERMINAL_SEQUENCE_GROUND && this.start < this.chunks.length) { + const first = this.chunks[this.start]!; + const dropped = first.data.slice(0, terminalSequenceEnd(this.sequenceState, first.data)); + this.trimChunk(dropped.length, Buffer.byteLength(dropped), countLineBreaks(dropped)); + } if ( this.start === this.chunks.length || (this.start > 2_048 && this.start * 2 >= this.chunks.length) @@ -1057,6 +1077,7 @@ export class BoundedTerminalHistory { this.byteLength = 0; this.lineBreaks = 0; this.lastCodeUnit = undefined; + this.sequenceState = TERMINAL_SEQUENCE_GROUND; this.cachedValue = ""; } @@ -1194,16 +1215,70 @@ function stripStringTerminator(value: string): string { return value; } -function findStringTerminatorIndex(input: string, start: number): number | null { +interface C1Effect { + /** ESC form of the sequence the code point starts, or "" when it returns to ground. */ + readonly next: string; + /** Its 0x9A byte ran DECID, which replies with device attributes. */ + readonly query: boolean; + /** What replays its visible effect from ground. */ + readonly safe: string; +} + +// Outside ground and OSC, libghostty parses a code point's UTF-8 bytes one at a time: 0x80-0x9F +// act as C1 controls, the rest are dropped. Bytes left over after a return to ground print U+FFFD. +function c1Effect(codePoint: number): C1Effect | undefined { + // A lone surrogate is encoded as U+FFFD, whose bytes are all dropped. + if (codePoint >= 0xd800 && codePoint <= 0xdfff) return undefined; + let next: string | undefined; + let query = false; + let safe = ""; + for (let shift = codePoint > 0xffff ? 12 : codePoint > 0x7ff ? 6 : 0; shift >= 0; shift -= 6) { + const byte = 0x80 | ((codePoint >> shift) & 0x3f); + if (next === "") { + safe += "\ufffd"; + } else if (next?.startsWith("\u001b]")) { + next += "\ufffd"; + } else if (byte === 0x9b) { + next = "\u001b["; + } else if (byte === 0x9d) { + next = "\u001b]"; + } else if (byte === 0x90) { + next = "\u001bP"; + } else if (byte === 0x98 || byte === 0x9e || byte === 0x9f) { + next = "\u001b_"; + } else if (byte < 0xa0) { + next = ""; + query = byte === 0x9a; + // Any other C1 control but ST runs as ESC plus the byte less 0x40. + safe = query || byte === 0x9c ? "" : `\u001b${String.fromCharCode(byte - 0x40)}`; + } + } + return next === undefined ? undefined : { next, query, safe }; +} + +// BEL ends only OSC; inside DCS, PM and APC it is payload. Non-ASCII is the reverse: libghostty +// keeps it inside OSC, while a C1 byte in it ends the other strings. +function findStringEnd( + input: string, + start: number, + isOsc: boolean, +): { end: number; c1?: { effect: C1Effect; index: number } } | null { for (let index = start; index < input.length; index += 1) { const codePoint = input.charCodeAt(index); - if (codePoint === 0x07 || codePoint === 0x9c || isControlSequenceCancel(codePoint)) { - return index + 1; + if ((isOsc && codePoint === 0x07) || isControlSequenceCancel(codePoint)) { + return { end: index + 1 }; } // Like xterm.js, an ESC that does not start ST ends the string and begins the next sequence. if (codePoint === 0x1b) { if (index + 1 >= input.length) return null; - return input.charCodeAt(index + 1) === 0x5c ? index + 2 : index; + return { end: input.charCodeAt(index + 1) === 0x5c ? index + 2 : index }; + } + if (codePoint >= 0x80 && !isOsc) { + const fullCodePoint = input.codePointAt(index) ?? codePoint; + const end = index + (fullCodePoint > 0xffff ? 2 : 1); + const effect = c1Effect(fullCodePoint); + if (effect) return { end, c1: { effect, index } }; + index = end - 1; } } return null; @@ -1249,6 +1324,8 @@ function sanitizeTerminalHistoryChunk( const pendingScanStart = Math.max(0, pendingControlSequence.length - 1); let visibleText = ""; let index = 0; + // Set when a C1 code point began the sequence at `index`: the ESC form that stands in for it. + let c1Introducer = ""; const append = (value: string) => { visibleText += value; @@ -1257,29 +1334,68 @@ function sanitizeTerminalHistoryChunk( while (index < input.length) { const codePoint = input.charCodeAt(index); - if (codePoint === 0x1b) { - const nextCodePoint = input.charCodeAt(index + 1); + if (c1Introducer.length > 0 || codePoint === 0x1b) { + // Empty for a sequence the input introduces itself, which starts with ESC at `index`. + const prefix = c1Introducer; + c1Introducer = ""; + const contentStart = prefix.length > 0 ? index : index + 2; + const nextCodePoint = (prefix || input).charCodeAt(prefix.length > 0 ? 1 : index + 1); if (Number.isNaN(nextCodePoint)) { return { visibleText, pendingControlSequence: input.slice(index) }; } if (nextCodePoint === 0x5b) { - let cursor = index + 2; + let cursor = contentStart; + // Inside a CSI Ghostty executes 0x00-0x0F and ignores DEL and 0x10-0x1F, which it would + // print in ground state. None join the parameters; only executed ones survive a strip. + let body = ""; + let executedControls = ""; while (cursor < input.length) { - const cancelled = isControlSequenceCancel(input.charCodeAt(cursor)); - if (cancelled || isCsiFinalByte(input.charCodeAt(cursor))) { - const sequence = input.slice(index, cursor + 1); - const body = input.slice(index + 2, cursor); + const cursorCodePoint = input.charCodeAt(cursor); + const cancelled = isControlSequenceCancel(cursorCodePoint); + if (cancelled || isCsiFinalByte(cursorCodePoint)) { if (cancelled || !shouldStripCsiSequence(body, input[cursor] ?? "")) { - append(sequence); + append(prefix + input.slice(index, cursor + 1)); + } else { + append(executedControls); } index = cursor + 1; break; } + if (cursorCodePoint >= 0x80) { + const fullCodePoint = input.codePointAt(cursor) ?? cursorCodePoint; + const end = cursor + (fullCodePoint > 0xffff ? 2 : 1); + const c1 = c1Effect(fullCodePoint); + if (c1) { + // The CSI ends here undispatched, so it is kept only when the terminal is back in + // ground with nothing to answer. + if (c1.next.length === 0 && !c1.query) { + append(prefix + input.slice(index, end)); + } else { + append(executedControls + c1.safe); + } + c1Introducer = c1.next; + index = end; + break; + } + cursor = end; + continue; + } + // An ESC abandons the CSI and starts the next sequence, which may be stripped. + if (cursorCodePoint === 0x1b) { + append(executedControls); + index = cursor; + break; + } + if (cursorCodePoint < 0x10) { + executedControls += input[cursor]; + } else if (cursorCodePoint >= 0x20 && cursorCodePoint !== 0x7f) { + body += input[cursor]; + } cursor += 1; } if (cursor >= input.length) { - return { visibleText, pendingControlSequence: input.slice(index) }; + return { visibleText, pendingControlSequence: prefix + input.slice(index) }; } continue; } @@ -1290,22 +1406,32 @@ function sanitizeTerminalHistoryChunk( nextCodePoint === 0x5e || nextCodePoint === 0x5f ) { - const terminatorIndex = findStringTerminatorIndex( + const stringEnd = findStringEnd( input, - Math.max(index + 2, pendingScanStart), + Math.max(contentStart, pendingScanStart), + nextCodePoint === 0x5d, ); - if (terminatorIndex === null) { - return { visibleText, pendingControlSequence: input.slice(index) }; + if (stringEnd === null) { + return { visibleText, pendingControlSequence: prefix + input.slice(index) }; } - const sequence = input.slice(index, terminatorIndex); - const content = stripStringTerminator(input.slice(index + 2, terminatorIndex)); + const { end, c1 } = stringEnd; + const content = stripStringTerminator( + prefix.slice(2) + input.slice(contentStart, c1?.index ?? end), + ); const strip = (nextCodePoint === 0x5d && shouldStripOscSequence(content)) || (nextCodePoint === 0x50 && shouldStripDcsSequence(content)); - if (!strip) { - append(sequence); + if (c1 === undefined || (!strip && c1.effect.next.length === 0 && !c1.effect.query)) { + if (!strip) { + append(prefix + input.slice(index, end)); + } + } else { + // ST stands in for a code point that cannot be replayed as it came. + const closed = strip ? "" : `${prefix}${input.slice(index, c1.index)}\u009c`; + append(closed + c1.effect.safe); + c1Introducer = c1.effect.next; } - index = terminatorIndex; + index = end; continue; } @@ -1313,7 +1439,10 @@ function sanitizeTerminalHistoryChunk( if (escapeSequenceEndIndex === null) { return { visibleText, pendingControlSequence: input.slice(index) }; } - append(input.slice(index, escapeSequenceEndIndex)); + // ESC Z (DECID) is the legacy primary device attributes query. + if (nextCodePoint !== 0x5a) { + append(input.slice(index, escapeSequenceEndIndex)); + } index = escapeSequenceEndIndex; continue; } @@ -1325,7 +1454,7 @@ function sanitizeTerminalHistoryChunk( index = nextSequenceStart; } - return { visibleText, pendingControlSequence: "" }; + return { visibleText, pendingControlSequence: c1Introducer }; } function legacySafeThreadId(threadId: string): string { @@ -1734,14 +1863,19 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func ); }; - const legacyHistoryPath = (threadId: string) => - path.join(logsDir, `${legacySafeThreadId(threadId)}.log`); + const legacyHistoryName = (threadId: string) => `${legacySafeThreadId(threadId)}.log`; + // Legacy names in the current "terminal_" namespace would alias another thread's history, + // case-insensitively on some filesystems. + const hasLegacyHistoryName = (threadId: string) => + !legacySafeThreadId(threadId).toLowerCase().startsWith("terminal_"); const historyPaths = (threadId: string, terminalId: string) => [ ...new Set([ historyPath(threadId, terminalId), encodedHistoryPath(threadId, terminalId), - ...(terminalId === DEFAULT_TERMINAL_ID ? [legacyHistoryPath(threadId)] : []), + ...(terminalId === DEFAULT_TERMINAL_ID && hasLegacyHistoryName(threadId) + ? [path.join(logsDir, legacyHistoryName(threadId))] + : []), ]), ]; @@ -2068,6 +2202,19 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func const info = yield* file.stat; const limit = BigInt(historyByteLimit); const offset = info.size > limit ? info.size - limit : 0n; + // The tail can start inside a control sequence; only the bytes before it say which. + const decoder = new TextDecoder("utf-8", { ignoreBOM: true }); + let sequenceState = TERMINAL_SEQUENCE_GROUND; + const skipped = new Uint8Array(Math.min(Number(offset), 64 * 1024)); + for (let remaining = Number(offset); remaining > 0;) { + const read = Number(yield* file.read(skipped.subarray(0, remaining))); + if (read === 0) break; + remaining -= read; + sequenceState = terminalSequenceStateAfter( + sequenceState, + decoder.decode(skipped.subarray(0, read), { stream: true }), + ); + } yield* file.seek(offset, "start"); const bytes = new Uint8Array(Number(info.size - offset)); let length = 0; @@ -2081,8 +2228,15 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func // A tail read can start inside a UTF-8 code point. Skip its remaining bytes. while (start < length && ((bytes[start] ?? 0) & 0xc0) === 0x80) start += 1; } + sequenceState = terminalSequenceStateAfter( + sequenceState, + decoder.decode(bytes.subarray(0, start)), + ); + const history = new TextDecoder("utf-8", { ignoreBOM: true }).decode( + bytes.subarray(start, length), + ); return { - history: new TextDecoder("utf-8", { ignoreBOM: true }).decode(bytes.subarray(start, length)), + history: history.slice(terminalSequenceEnd(sequenceState, history)), truncated: offset > 0n, }; }); @@ -2193,7 +2347,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func yield* Effect.forEach( entries.filter( (name) => - name === `${legacySafeThreadId(threadId)}.log` || + (hasLegacyHistoryName(threadId) && name === legacyHistoryName(threadId)) || threadParts.some((part) => name === `${part}.log` || name.startsWith(`${part}_`)), ), (name) => @@ -3503,7 +3657,12 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func session.pendingProcessEventIndex = 0; session.processEventDrainRunning = false; const eventStamp = advanceEventSequence(session); - yield* persistHistory(input.threadId, terminalId, session.history); + // A drain waits for the key to go idle, which never happens while the shell keeps + // printing; every reader of the file flushes it first. + yield* persistWorker.enqueue(toSessionKey(input.threadId, terminalId), { + history: session.history, + immediate: true, + }); yield* publishEvent({ type: "cleared", threadId: input.threadId, diff --git a/apps/server/src/terminal/OutputProtocol.test.ts b/apps/server/src/terminal/OutputProtocol.test.ts index 48cb40e09d978..6d851b93d374d 100644 --- a/apps/server/src/terminal/OutputProtocol.test.ts +++ b/apps/server/src/terminal/OutputProtocol.test.ts @@ -101,7 +101,7 @@ describe("terminal output window", () => { assert.equal(yield* Queue.size(responses), 0); yield* write(0, { _tag: "Interrupt", requestId: "1" }); assert.equal((yield* Queue.take(responses))._tag, "Exit"); - }).pipe(Effect.provide(RpcSerialization.layerJson), Effect.scoped), + }).pipe(Effect.provide(RpcSerialization.layerJson)), ); }); @@ -261,7 +261,7 @@ describe("terminal output stream", () => { assert.deepEqual(yield* takeChunk(healthy), [after]); assert.deepEqual(yield* takeChunk(stalled), [after]); assert.equal(yield* Queue.size(stalled), 0); - }).pipe(Effect.provide(RpcSerialization.layerJson), Effect.scoped), + }).pipe(Effect.provide(RpcSerialization.layerJson)), ); it.effect("interrupts recovery while its subscription is still waiting", () => @@ -290,7 +290,7 @@ describe("terminal output stream", () => { yield* Fiber.interrupt(recovery); yield* Deferred.await(interrupted); assert.equal(publisher.listeners.size, 0); - }).pipe(Effect.scoped), + }), ); it.effect("cleans listeners when cancelled during a successful acquisition handoff", () => @@ -324,7 +324,7 @@ describe("terminal output stream", () => { yield* Fiber.join(cancellation); assert.equal(publisher.listeners.size, 0); } - }).pipe(Effect.scoped), + }), ); it.effect("caps each delivered batch at the output window in UTF-8 bytes", () => @@ -349,7 +349,7 @@ describe("terminal output stream", () => { delivered += batch.length; } assert.equal(delivered, 1024); - }).pipe(Effect.scoped), + }), ); it.effect("charges snapshot history but never overflows on one snapshot", () => @@ -367,6 +367,6 @@ describe("terminal output stream", () => { assert.equal(publisher.listeners.size, 1); yield* publisher.publish(snapshot(history)); assert.equal(publisher.listeners.size, 0); - }).pipe(Effect.scoped), + }), ); }); diff --git a/apps/server/src/terminal/nodePtyFlushDataAndCleanUp.test.ts b/apps/server/src/terminal/nodePtyFlushDataAndCleanUp.test.ts new file mode 100644 index 0000000000000..7dc0861c7f53c --- /dev/null +++ b/apps/server/src/terminal/nodePtyFlushDataAndCleanUp.test.ts @@ -0,0 +1,131 @@ +import * as NodeModule from "node:module"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; + +const requireCjs = NodeModule.createRequire(import.meta.url); +const { + WindowsPtyAgent, +}: { + WindowsPtyAgent: { + prototype: { + _flushDataAndCleanUp(this: FakeAgent): void; + _cleanUpProcess(this: FakeAgent): void; + }; + }; +} = requireCjs("node-pty/lib/windowsPtyAgent.js"); + +interface FakeAgent { + _useConptyDll: boolean; + _closeTimeout?: unknown; + _inSocket: { readable: boolean }; + _outSocket: { readable: boolean; destroy: () => void }; + _cleanUpProcess: () => void; +} + +// The sockets are stand-ins; the timer logic and _cleanUpProcess are node-pty's own (as patched by T3). +const makeAgent = (): FakeAgent => { + const agent: FakeAgent = { + _useConptyDll: false, + _inSocket: { readable: true }, + _outSocket: { readable: true, destroy: vi.fn() }, + _cleanUpProcess: () => WindowsPtyAgent.prototype._cleanUpProcess.call(agent), + }; + return agent; +}; + +const flush = (agent: FakeAgent) => WindowsPtyAgent.prototype._flushDataAndCleanUp.call(agent); + +// A minimal event loop that mirrors Node's phases: expired timers run first, then immediates (the check phase). +// node-pty calls the global timer functions at call time, so replacing them lets the test place a "poll phase" +// (delivery of already-buffered pipe data) exactly between the two. +interface Timer { + readonly id: number; + readonly dueAt: number; + readonly run: () => void; +} +let now = 0; +let nextId = 0; +let timers: Timer[] = []; +let immediates: Array<() => void> = []; + +const runExpiredTimers = () => { + for (const timer of timers.filter((t) => t.dueAt <= now).toSorted((a, b) => a.dueAt - b.dueAt)) { + timers = timers.filter((t) => t.id !== timer.id); + timer.run(); + } +}; +const runImmediates = () => { + const batch = immediates; + immediates = []; + for (const run of batch) run(); +}; +/** One loop iteration after `ms` of wall time; `poll` runs between the timers and check phases. */ +const loopIteration = (ms: number, poll?: () => void) => { + now += ms; + runExpiredTimers(); + poll?.(); + runImmediates(); +}; + +// ConPTY never closes the output pipe after the shell exits, so node-pty ends the session on a timer. +// T3 shortens it (patches/node-pty@1.2.0-beta.15.patch) so the terminal reports "exited" about 0.9 s sooner. +describe("node-pty Windows exit flush", () => { + beforeEach(() => { + now = 0; + nextId = 0; + timers = []; + immediates = []; + vi.stubGlobal("setTimeout", (run: () => void, ms: number) => { + const id = ++nextId; + timers.push({ id, dueAt: now + ms, run }); + return id; + }); + vi.stubGlobal("clearTimeout", (id: number) => { + timers = timers.filter((t) => t.id !== id); + }); + vi.stubGlobal("setImmediate", (run: () => void) => { + immediates.push(run); + return ++nextId; + }); + }); + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("closes the output socket 100 ms after the exit, not a second later", () => { + const agent = makeAgent(); + flush(agent); + + loopIteration(99); + expect(agent._outSocket.destroy).not.toHaveBeenCalled(); + + loopIteration(1); + expect(agent._outSocket.destroy).toHaveBeenCalledTimes(1); + expect(agent._outSocket.readable).toBe(false); + expect(agent._inSocket.readable).toBe(false); + }); + + // Timers run before the poll phase. After a main-thread stall longer than the window, output that is already + // buffered in the pipe is read only after the timer has expired, so the cleanup has to wait one loop turn for it. + it("keeps the socket open when buffered output is read right after a stalled timer expires", () => { + const agent = makeAgent(); + flush(agent); + + loopIteration(150, () => flush(agent)); + expect(agent._outSocket.destroy).not.toHaveBeenCalled(); + + loopIteration(99); + expect(agent._outSocket.destroy).not.toHaveBeenCalled(); + + loopIteration(1); + expect(agent._outSocket.destroy).toHaveBeenCalledTimes(1); + }); + + it("closes after a stall when no output was buffered", () => { + const agent = makeAgent(); + flush(agent); + + loopIteration(150); + expect(agent._outSocket.destroy).toHaveBeenCalledTimes(1); + }); +}); diff --git a/apps/server/src/testUtils/fakeWsl.ts b/apps/server/src/testUtils/fakeWsl.ts new file mode 100644 index 0000000000000..d8f88fcde8db8 --- /dev/null +++ b/apps/server/src/testUtils/fakeWsl.ts @@ -0,0 +1,37 @@ +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; + +// `wslpath` honours the distro's configurable automount root, so the fake maps drives under a temp dir. +export const makeFakeWslHost = Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const mountRoot = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-wsl-mnt-" }); + const binDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-wsl-bin-" }); + const wslpath = path.join(binDir, "wslpath"); + yield* fileSystem + .writeFileString( + wslpath, + [ + "#!/bin/sh", + '[ "$1" = "-u" ] || exit 1', + "p=$(printf '%s' \"$2\" | tr '\\\\' '/')", + "drive=$(printf '%s' \"$p\" | cut -c1 | tr 'A-Z' 'a-z')", + `printf '%s/%s%s\\n' '${mountRoot}' "$drive" "\${p#?:}"`, + "", + ].join("\n"), + ) + .pipe(Effect.orDie); + yield* fileSystem.chmod(wslpath, 0o755).pipe(Effect.orDie); + const environment = { + ...HostProcess.Environment.defaultValue(), + PATH: `${binDir}:${HostProcess.Environment.defaultValue().PATH ?? ""}`, + }; + const underWsl = (effect: Effect.Effect) => + effect.pipe( + Effect.provideService(HostProcess.Platform, "linux"), + Effect.provideService(HostProcess.Environment, environment), + ); + return { mountRoot, underWsl }; +}); diff --git a/apps/server/src/textGeneration/AntigravityTextGeneration.test.ts b/apps/server/src/textGeneration/AntigravityTextGeneration.test.ts index 5cb6a1f8d4d8b..783417693c722 100644 --- a/apps/server/src/textGeneration/AntigravityTextGeneration.test.ts +++ b/apps/server/src/textGeneration/AntigravityTextGeneration.test.ts @@ -343,7 +343,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { { type: "text", text: expect.stringContaining("+handleRemoteCallback()") }, ]); yield* fixture.assertCleaned; - }).pipe(Effect.scoped), + }), ); it.effect.each(["tool_call", "tool_call_update"] as const)( @@ -362,7 +362,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { expect(error.detail).toContain("tool work"); expect(fixture.state.cancellations).toBe(1); yield* fixture.assertCleaned; - }).pipe(Effect.scoped), + }), ); it.effect("rejects native permission and choice requests instead of answering them", () => @@ -387,7 +387,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { expect(reply).toEqual({ outcome: { outcome: "cancelled" } }); expect(error.detail).toContain("permission or user input"); yield* fixture.assertCleaned; - }).pipe(Effect.scoped), + }), ); it.effect.each(["file", "terminal", "extension"] as const)( @@ -416,7 +416,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { const error = yield* Fiber.join(child).pipe(Effect.flip); expect(error.detail).toContain("tool or user input"); yield* fixture.assertCleaned; - }).pipe(Effect.scoped), + }), ); it.effect.each([ @@ -432,7 +432,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { .pipe(Effect.flip); expect(error.detail).toContain(detail); yield* fixture.assertCleaned; - }).pipe(Effect.scoped), + }), ); it.effect("does not return partial JSON after native cancellation", () => @@ -449,7 +449,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { .pipe(Effect.flip); expect(error.detail).toContain("cancelled"); yield* fixture.assertCleaned; - }).pipe(Effect.scoped), + }), ); it.effect.each([ @@ -470,7 +470,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { expect(error.detail).toBe(detail); expect(error.detail).not.toContain("Raw provider failure text."); yield* fixture.assertCleaned; - }).pipe(Effect.scoped), + }), ); it.effect.each([ @@ -517,7 +517,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { expect(error.detail).toContain("wrote files"); if (target !== undefined) expect(yield* fs.exists(target)).toBe(true); yield* fixture.assertCleaned; - }).pipe(Effect.scoped), + }), ); it.effect("uses the native default without sending T3's default selection as a model ID", () => @@ -530,7 +530,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { expect(result).toEqual({ title: "Repair login" }); expect(fixture.state.selectedModels).toEqual([]); yield* fixture.assertCleaned; - }).pipe(Effect.scoped), + }), ); it.effect.each([ @@ -562,7 +562,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { expect(Exit.isSuccess(result)).toBe(available); expect(fixture.state.workspaces.length).toBe(available ? 1 : 0); yield* fixture.assertCleaned; - }).pipe(Effect.scoped), + }), ); it.effect("does not launch during sign-out or before sign-in", () => @@ -575,7 +575,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { expect(error.detail).toBe("Sign in before starting Antigravity."); expect(fixture.state.workspaces).toEqual([]); yield* fixture.assertCleaned; - }).pipe(Effect.scoped), + }), ); it.effect.each(["caller", "sign-out"] as const)( @@ -596,7 +596,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { } expect(fixture.state.cancellations).toBe(1); yield* fixture.assertCleaned; - }).pipe(Effect.scoped), + }), ); it.effect("refuses unsafe native session IDs without deleting other files", () => @@ -609,7 +609,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { expect(fixture.state.prompts).toEqual([]); expect(yield* fixture.fs.readFileString(`${fixture.sessionBase}.db`)).toBe("helper session"); expect(fixture.state.closed).toEqual(fixture.state.workspaces); - }).pipe(Effect.scoped), + }), ); it.effect("keeps native sessions whose metadata belongs to a different workspace", () => @@ -627,7 +627,7 @@ it.layer(NodeServices.layer)("AntigravityTextGeneration", (it) => { "keep this session", ); expect(yield* fixture.fs.readFileString(`${fixture.sessionBase}.meta`)).toBe(metadata); - }).pipe(Effect.scoped), + }), ); }); @@ -640,5 +640,5 @@ it.effect("retains filesystem services supplied only during construction", () => title: "Repair login", }); yield* fixture.assertCleaned; - }).pipe(Effect.scoped), + }), ); diff --git a/apps/server/src/textGeneration/ClaudeTextGeneration.test.ts b/apps/server/src/textGeneration/ClaudeTextGeneration.test.ts index a7041d5d249e5..d2e165ea3b2b5 100644 --- a/apps/server/src/textGeneration/ClaudeTextGeneration.test.ts +++ b/apps/server/src/textGeneration/ClaudeTextGeneration.test.ts @@ -234,6 +234,35 @@ it.layer(layerClaudeTextGenerationTest)("ClaudeTextGeneration", (it) => { }), ); + it.effect("passes --settings from the instance launch args to Claude text generation", () => + withFakeClaudeEnv( + { + output: JSON.stringify({ + structured_output: { + title: "Use instance settings", + }, + }), + argsMustContain: `"env":{"ANTHROPIC_VERTEX_PROJECT_ID":"vertex-project-a"}`, + claudeConfig: { + launchArgs: `--settings '{"env":{"ANTHROPIC_VERTEX_PROJECT_ID":"vertex-project-a"}}'`, + }, + }, + (textGeneration) => + Effect.gen(function* () { + const generated = yield* textGeneration.generateThreadTitle({ + cwd: process.cwd(), + message: "thread title", + modelSelection: { + instanceId: ProviderInstanceId.make("claudeAgent"), + model: SYNTHETIC_CLAUDE_STANDARD_MODEL, + }, + }); + + expect(generated.title).toBe(sanitizeThreadTitle("Use instance settings")); + }), + ), + ); + it.effect("retries without --no-session-persistence when the Claude CLI rejects it", () => withFakeClaudeEnv( { diff --git a/apps/server/src/textGeneration/ClaudeTextGeneration.ts b/apps/server/src/textGeneration/ClaudeTextGeneration.ts index 34e7415b3c9f4..a523a3873a2c5 100644 --- a/apps/server/src/textGeneration/ClaudeTextGeneration.ts +++ b/apps/server/src/textGeneration/ClaudeTextGeneration.ts @@ -1,7 +1,9 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Option from "effect/Option"; +import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; @@ -32,6 +34,7 @@ import { hasClaudeManagedMcpConfig, makeClaudeEnvironment, } from "../provider/Drivers/ClaudeHome.ts"; +import { readClaudeLaunchArgSettings } from "../orchestration-v2/Adapters/ClaudeAdapterV2.ts"; const CLAUDE_TIMEOUT_MS = 180_000; /** `--tools` takes bare names; patterns go in the allow list. */ @@ -97,9 +100,14 @@ export const makeClaudeTextGeneration = Effect.fn("makeClaudeTextGeneration")(fu claudeSettings: ClaudeSettings, environment?: NodeJS.ProcessEnv, modelCatalog: Effect.Effect = Effect.succeed(BUNDLED_CLAUDE_MODEL_CATALOG), + routeModelCatalog: ( + catalog: ClaudeModelCatalog, + cwd: string, + ) => Effect.Effect = Effect.succeed, ) { const commandSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; const claudeEnvironment = yield* makeClaudeEnvironment(claudeSettings, environment); const scopedModelCatalog = modelCatalog.pipe( Effect.map((catalog) => scopeClaudeModelCatalog(catalog, claudeSettings.customModels)), @@ -146,9 +154,19 @@ export const makeClaudeTextGeneration = Effect.fn("makeClaudeTextGeneration")(fu }: TextGenerationOperations.Request): Effect.fn.Return< S["Type"], TextGenerationError, - S["DecodingServices"] + S["DecodingServices"] | Scope.Scope > { - const catalog = yield* scopedModelCatalog; + const workingDirectory = + operation === "generateThreadTitle" + ? yield* fileSystem + .makeTempDirectoryScoped({ prefix: "t3code-claude-title-" }) + .pipe( + Effect.mapError((cause) => + normalizeCliError("claude", operation, cause, "Failed to create title directory"), + ), + ) + : cwd; + const catalog = yield* routeModelCatalog(yield* scopedModelCatalog, workingDirectory); const resolvedModelSelection = { ...modelSelection, model: resolveClaudeModelSlug(catalog, modelSelection.model), @@ -185,7 +203,18 @@ export const makeClaudeTextGeneration = Effect.fn("makeClaudeTextGeneration")(fu : undefined; const fastMode = fastModeDescriptor?.type === "boolean" ? fastModeDescriptor.currentValue : undefined; + const launchArgSettings = yield* readClaudeLaunchArgSettings({ + settings: claudeSettings, + cwd, + fileSystem, + path, + }).pipe( + Effect.mapError( + (cause) => new TextGenerationError({ operation, detail: cause.message, cause }), + ), + ); const settings = { + ...launchArgSettings, disableAllHooks: true, ...(typeof thinking === "boolean" ? { alwaysThinkingEnabled: thinking } : {}), ...(fastMode ? { fastMode: true } : {}), @@ -200,16 +229,6 @@ export const makeClaudeTextGeneration = Effect.fn("makeClaudeTextGeneration")(fu const runClaudeCommand = Effect.fn("runClaudeJson.runClaudeCommand")(function* ( noSessionPersistence: boolean, ) { - const workingDirectory = - operation === "generateThreadTitle" - ? yield* fileSystem - .makeTempDirectoryScoped({ prefix: "t3code-claude-title-" }) - .pipe( - Effect.mapError((cause) => - normalizeCliError("claude", operation, cause, "Failed to create title directory"), - ), - ) - : cwd; const managedMcpConfig = yield* hasClaudeManagedMcpConfig(); const spawnCommand = yield* resolveSpawnCommand( claudeSettings.binaryPath || "claude", @@ -344,5 +363,9 @@ export const makeClaudeTextGeneration = Effect.fn("makeClaudeTextGeneration")(fu ); }); - return TextGenerationOperations.fromRunner("ClaudeTextGeneration", runClaudeJson); + return TextGenerationOperations.fromRunner( + "ClaudeTextGeneration", + (request: TextGenerationOperations.Request) => + Effect.scoped(runClaudeJson(request)), + ); }); diff --git a/apps/server/src/textGeneration/CodexTextGeneration.test.ts b/apps/server/src/textGeneration/CodexTextGeneration.test.ts index ae5c32bf57432..1a1a5c43b0a0d 100644 --- a/apps/server/src/textGeneration/CodexTextGeneration.test.ts +++ b/apps/server/src/textGeneration/CodexTextGeneration.test.ts @@ -58,6 +58,7 @@ interface FakeCodexInput { mcpServers?: ReadonlyArray<{ name: string; enabled: boolean }> | number; requiredCwd?: string; requireInstructions?: string; + requireMcpServersOff?: ReadonlyArray; } function makeFakeCodexBinary(dir: string, input: FakeCodexInput) { @@ -73,6 +74,7 @@ function makeFakeCodexBinary(dir: string, input: FakeCodexInput) { mcpServers: input.mcpServers ?? [], requiredCwd: input.requiredCwd ?? null, requireInstructions: input.requireInstructions ?? null, + requireMcpServersOff: input.requireMcpServersOff ?? [], stderr: input.stderr ?? null, output: input.output, exitCode: input.exitCode ?? 0, @@ -129,6 +131,33 @@ function makeFakeCodexBinary(dir: string, input: FakeCodexInput) { ' process.stderr.write(message + "\\n");', " process.exit(code);", "}", + "// Like Codex: a later mcp_servers={...} override replaces servers earlier -c flags defined.", + "const servers = new Map();", + "const offServers = new Set();", + "for (let index = 0; index < args.length; index += 1) {", + ' if (args[index] !== "-c" && args[index] !== "--config") continue;', + ' const value = args[index + 1] ?? "";', + ' if (value.startsWith("mcp_servers={")) {', + ' const named = [...value.matchAll(/"([^"]+)"=\\{enabled=false\\}/g)].map((match) => match[1]);', + " for (const name of named) offServers.add(name);", + " for (const name of [...servers.keys()]) {", + " if (named.includes(name)) servers.set(name, false);", + " else servers.delete(name);", + " }", + ' } else if (value.startsWith("mcp_servers.")) {', + ' const [name, key] = value.slice("mcp_servers.".length).split(/[.=]/);', + " if (value === `mcp_servers.${name}.enabled=false`) offServers.add(name);", + " // An enabled flag alone toggles a server another layer (config.toml) defines.", + ' if (key === "enabled" && !servers.has(name)) continue;', + ' servers.set(name, servers.get(name) === true || key === "url" || key === "command");', + " }", + "}", + "for (const [name, hasTransport] of servers) {", + ' if (!hasTransport) fail("Error loading config.toml: invalid transport in `mcp_servers." + name + "`", 12);', + "}", + "for (const name of check.requireMcpServersOff) {", + ' if (!offServers.has(name)) fail("MCP server left on: " + name, 13);', + "}", 'if (check.requiredCwd !== null && process.cwd() !== check.requiredCwd) fail("cannot inspect checkout", 10);', 'if (check.requireInstructions !== null && !instructions.includes(check.requireInstructions)) fail("missing writer task", 11);', "if (check.requireArg !== null && !originalArgs.includes(` ${check.requireArg} `)) {", @@ -365,7 +394,7 @@ it.layer(layerCodexTextGenerationTest)("CodexTextGeneration", (it) => { { name: "already-off", enabled: false }, ], requireArg: - '--config features.plugins=false --config features.apps=false --config mcp_servers={"augmenter"={enabled=false},"my.server"={enabled=false}}', + 'exec --config mcp_servers={"augmenter"={enabled=false},"my.server"={enabled=false}} --config developer_instructions="Use', }, (textGeneration) => Effect.gen(function* () { @@ -384,6 +413,52 @@ it.layer(layerCodexTextGenerationTest)("CodexTextGeneration", (it) => { ), ); + it.effect("turns off an MCP server that launch args define without dropping its transport", () => + withFakeCodexEnv( + { + output: JSON.stringify({ title: "Remove default shapes" }), + launchArgs: `-c 'mcp_servers.t3-code.url="http://127.0.0.1:1/mcp"' -c 'mcp_servers.t3-code.enabled=true'`, + mcpServers: [{ name: "t3-code", enabled: true }], + requireMcpServersOff: ["t3-code"], + requireArg: + "--config features.apps=false --config mcp_servers.t3-code.enabled=false --ephemeral", + }, + (textGeneration) => + Effect.gen(function* () { + const result = yield* textGeneration.generateThreadTitle({ + cwd: process.cwd(), + message: "Remove default shapes from characters", + modelSelection: DEFAULT_TEST_MODEL_SELECTION, + }); + expect(result.title).toBe("Remove default shapes"); + }), + ), + ); + + it.effect("keeps MCP servers off when launch args set their own mcp_servers table", () => + withFakeCodexEnv( + { + output: JSON.stringify({ title: "Remove default shapes" }), + launchArgs: `-c 'mcp_servers={"my.server"={command="echo",enabled=true}}'`, + mcpServers: [ + { name: "my.server", enabled: true }, + { name: "my server", enabled: true }, + ], + requireArg: + '--config features.apps=false --config mcp_servers={"my.server"={enabled=false},"my server"={enabled=false}} --config mcp_servers.my server.enabled=false --ephemeral', + }, + (textGeneration) => + Effect.gen(function* () { + const result = yield* textGeneration.generateThreadTitle({ + cwd: process.cwd(), + message: "Remove default shapes from characters", + modelSelection: DEFAULT_TEST_MODEL_SELECTION, + }); + expect(result.title).toBe("Remove default shapes"); + }), + ), + ); + it.effect("still generates when Codex cannot list its MCP servers", () => withFakeCodexEnv( { @@ -424,6 +499,26 @@ it.layer(layerCodexTextGenerationTest)("CodexTextGeneration", (it) => { ), ); + it.effect("keeps user hooks off over launch args that turn them on", () => + withFakeCodexEnv( + { + output: JSON.stringify({ title: "Hookless title" }), + launchArgs: "--config features.hooks=true --enable hooks", + requireArg: "--config features.codex_hooks=false --config features.hooks=false", + forbidArg: "--enable", + }, + (textGeneration) => + Effect.gen(function* () { + const result = yield* textGeneration.generateThreadTitle({ + cwd: process.cwd(), + message: "Describe this change", + modelSelection: DEFAULT_TEST_MODEL_SELECTION, + }); + expect(result.title).toBe("Hookless title"); + }), + ), + ); + it.effect("passes exec-safe launch args into codex exec", () => withFakeCodexEnv( { diff --git a/apps/server/src/textGeneration/CodexTextGeneration.ts b/apps/server/src/textGeneration/CodexTextGeneration.ts index 4de5392502c8c..fc741aaaeb262 100644 --- a/apps/server/src/textGeneration/CodexTextGeneration.ts +++ b/apps/server/src/textGeneration/CodexTextGeneration.ts @@ -55,6 +55,11 @@ const codexMcpServersOff = (names: ReadonlyArray): string | null => names.length === 0 ? null : `mcp_servers={${names.map((name) => `${JSON.stringify(name)}={enabled=false}`).join(",")}}`; +// Codex splits -c keys at the first "=" and on every dot, so only those names take a dotted flag. +const codexMcpServerOffFlags = (names: ReadonlyArray) => + names + .filter((name) => !name.includes(".") && !name.includes("=")) + .flatMap((name) => ["--config", `mcp_servers.${name}.enabled=false`]); const encodeJsonString = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); export const makeCodexTextGeneration = Effect.fn("makeCodexTextGeneration")(function* ( codexConfig: CodexSettings, @@ -291,20 +296,28 @@ export const makeCodexTextGeneration = Effect.fn("makeCodexTextGeneration")(func } : {}), }; - const mcpServersOff = yield* listEnabledMcpServers({ + const enabledMcpServers = yield* listEnabledMcpServers({ binary, cwd, env, launchArgs: codexMcpListLaunchArgs(launchArgs), - }).pipe(Effect.map(codexMcpServersOff)); + }); + const mcpServersOff = codexMcpServersOff(enabledMcpServers); + const execLaunchArgs = codexExecLaunchArgs(launchArgs); + // A launch-arg mcp_servers={...} table replaces ours and can re-enable dotted names. + const launchArgsSetMcpTable = execLaunchArgs.some((arg) => /\bmcp_servers\s*=/.test(arg)); const spawnCommand = yield* resolveSpawnCommand( binary, [ "exec", - ...codexExecLaunchArgs(launchArgs), + // Before launch args: a later table would replace servers their -c flags define. + ...(mcpServersOff ? ["--config", mcpServersOff] : []), + ...execLaunchArgs, // A user's MCP server could act on the text-generation prompt. ...CODEX_NO_PLUGIN_ARGS, - ...(mcpServersOff ? ["--config", mcpServersOff] : []), + ...(mcpServersOff && launchArgsSetMcpTable ? ["--config", mcpServersOff] : []), + // After launch args, so a launch arg's enabled=true cannot turn a server back on. + ...codexMcpServerOffFlags(enabledMcpServers), "--ephemeral", "--skip-git-repo-check", "-s", @@ -313,6 +326,11 @@ export const makeCodexTextGeneration = Effect.fn("makeCodexTextGeneration")(func model, "--config", `model_reasoning_effort="${reasoningEffort}"`, + // Current Codex gives hooks precedence over the historical codex_hooks alias. + "--config", + "features.codex_hooks=false", + "--config", + "features.hooks=false", ...(serviceTier ? ["--config", `service_tier="${serviceTier}"`] : []), "--output-schema", schemaPath, diff --git a/apps/server/src/textGeneration/GrokTextGeneration.test.ts b/apps/server/src/textGeneration/GrokTextGeneration.test.ts index 26ab9e79c7103..adb911623f800 100644 --- a/apps/server/src/textGeneration/GrokTextGeneration.test.ts +++ b/apps/server/src/textGeneration/GrokTextGeneration.test.ts @@ -35,7 +35,7 @@ function makeAcpGrokWrapper(dir: string, env: Record): string { env, source: execScriptSource({ scriptPath: mockAgentPath, - expectedArgs: ["agent", "stdio"], + expectedArgs: ["--permission-mode", "default", "agent", "stdio"], }), }); } @@ -58,14 +58,19 @@ function withFakeAcpGrok( }).pipe(Effect.scoped); } -function readJsonRpcRequests( - filePath: string, -): ReadonlyArray<{ readonly method?: string; readonly params?: Record }> { +function readJsonRpcRequests(filePath: string): ReadonlyArray<{ + readonly id?: unknown; + readonly method?: string; + readonly params?: Record; +}> { return NodeFS.readFileSync(filePath, "utf8") .trim() .split("\n") .filter((line) => line.length > 0) - .map((line) => JSON.parse(line) as { method?: string; params?: Record }); + .map( + (line) => + JSON.parse(line) as { id?: unknown; method?: string; params?: Record }, + ); } it.layer(layerGrokTextGenerationTest)("GrokTextGeneration", (it) => { @@ -78,6 +83,7 @@ it.layer(layerGrokTextGenerationTest)("GrokTextGeneration", (it) => { return withFakeAcpGrok( { T3_ACP_REQUEST_LOG_PATH: requestLogPath, + T3_ACP_REQUEST_PERMISSION: "1", T3_ACP_PROMPT_RESPONSE_TEXT: JSON.stringify({ subject: "Add Grok provider", body: "Wire up the ACP runtime and headless text generation path.", @@ -110,6 +116,9 @@ it.layer(layerGrokTextGenerationTest)("GrokTextGeneration", (it) => { request.params?.modelId === "grok-mock-alt", ), ).toBe(true); + expect(requests.find((request) => request.id === 9001)).toMatchObject({ + result: { outcome: { outcome: "cancelled" } }, + }); }), ); }); diff --git a/apps/server/src/textGeneration/OpenCode2TextGeneration.test.ts b/apps/server/src/textGeneration/OpenCode2TextGeneration.test.ts index 869cd1b4fb5e7..6cb1bfad71be7 100644 --- a/apps/server/src/textGeneration/OpenCode2TextGeneration.test.ts +++ b/apps/server/src/textGeneration/OpenCode2TextGeneration.test.ts @@ -70,7 +70,7 @@ it.layer(layer)("OpenCode2TextGeneration", (it) => { files: [{ uri: NodeURL.pathToFileURL(screenshotPath).href, name: "screen.png" }], }); } - }).pipe(Effect.scoped), + }), ); it.effect("keeps a provider's own error message out of the caller-visible detail", () => @@ -153,7 +153,7 @@ it.layer(layer)("OpenCode2TextGeneration", (it) => { .pipe(Effect.flip); assert.equal(failure.detail, "OpenCode could not generate the text."); assert.deepEqual(failure.cause, refusal); - }).pipe(Effect.scoped), + }), ); it.effect("times out when the prompt request and its interrupt never answer", () => @@ -219,6 +219,6 @@ it.layer(layer)("OpenCode2TextGeneration", (it) => { yield* TestClock.adjust("5 seconds"); const failure = yield* Fiber.join(generation); assert.equal(failure.detail, "OpenCode did not finish generating in time."); - }).pipe(Effect.scoped), + }), ); }); diff --git a/apps/server/src/textGeneration/TextGenerationPresets.ts b/apps/server/src/textGeneration/TextGenerationPresets.ts index b53ecc543597d..2dd3b95321290 100644 --- a/apps/server/src/textGeneration/TextGenerationPresets.ts +++ b/apps/server/src/textGeneration/TextGenerationPresets.ts @@ -3,7 +3,7 @@ import type { TextGenerationPolicy } from "@t3tools/provider-core/server/textGen export const conventionalCommitsTextGenerationPolicy: TextGenerationPolicy = { kind: "conventional_commits", commitInstructions: - "Use Conventional Commits when generating commit subjects. Prefer the narrowest accurate type and include a scope only when it is obvious from the diff.", + "Use Conventional Commits when generating commit subjects. Prefer the narrowest accurate type and include a scope only when it is obvious from the diff. Start the description after the colon with a lowercase word, never a proper noun or acronym, because commitlint rejects a capitalized description.", changeRequestInstructions: "Keep the change request title concise. Do not force Conventional Commit syntax into the title unless the repository already uses it.", inferRepositoryConventions: false, diff --git a/apps/server/src/usage/UsageLimitSources.test.ts b/apps/server/src/usage/UsageLimitSources.test.ts index 274e63b46c906..fc8a93f9e1bc6 100644 --- a/apps/server/src/usage/UsageLimitSources.test.ts +++ b/apps/server/src/usage/UsageLimitSources.test.ts @@ -1,11 +1,21 @@ import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import { expect, it } from "@effect/vitest"; -import { SecretProviderId, ServerConfigStreamEvent, UsageLimitSourceId } from "@t3tools/contracts"; +import { + ProviderDriverKind, + SecretProviderId, + ServerConfigStreamEvent, + UsageLimitSourceId, + type ServerProviderUsageWindow, + type UsageLimitSourceSnapshot, +} from "@t3tools/contracts"; import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; import { HttpClient, HttpClientResponse } from "effect/http"; import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; @@ -26,6 +36,15 @@ it.effect("keeps malformed source URLs encodable for config subscribers", () => "hub.test:8317", "Usage limit source", ]); + // Only an http(s) hub URL names an endpoint an agent can be sent to. + expect(snapshots.map((source) => source.origin)).toEqual([ + undefined, + undefined, + undefined, + "https://hub.test:8317", + undefined, + ]); + expect(snapshots[3]?.endpoint).toBe("https://hub.test:8317/team-a"); expect(() => encodeConfigEvent({ version: 1, @@ -66,7 +85,7 @@ it.effect("keeps malformed source URLs encodable for config subscribers", () => }, [UsageLimitSourceId.make("valid-hub")]: { kind: "cliproxy", - url: "https://hub.test:8317", + url: "https://hub.test:8317/team-a/management.html", managementKey: "", enabled: true, }, @@ -87,7 +106,6 @@ it.effect("keeps malformed source URLs encodable for config subscribers", () => ), NodeCrypto.layer, ]), - Effect.scoped, ), ); @@ -154,6 +172,137 @@ it.effect("reads a management key from its secret provider", () => { ), NodeCrypto.layer, ]), - Effect.scoped, ); }); + +const NOW = Date.parse("2026-11-13T12:00:00.000Z"); +const at = (hours: number) => DateTime.formatIso(DateTime.makeUnsafe(NOW + hours * 3_600_000)); +const spent = (id: string, hours: number): ServerProviderUsageWindow => ({ + id, + kind: "weekly", + label: id, + usedPercent: 100, + resetsAt: at(hours), +}); +const hub = ( + accounts: ReadonlyArray<{ + driver?: string; + windows?: ReadonlyArray; + unavailable?: true; + }>, +): UsageLimitSourceSnapshot => ({ + id: UsageLimitSourceId.make("hub"), + kind: "cliproxy", + label: "hub", + checkedAt: at(0), + accounts: accounts.map((account, index) => ({ + id: `account-${index}`, + driver: ProviderDriverKind.make(account.driver ?? "claudeAgent"), + usageLimits: { + checkedAt: at(0), + windows: account.windows ?? [], + ...(account.unavailable ? { unavailable: { reason: "probeFailed" as const } } : {}), + }, + })), +}); + +it.each([ + { + name: "earliest refill when every account is spent, each at its latest spent window", + accounts: [ + { windows: [spent("five_hour", 1), spent("seven_day", 3)] }, + { windows: [spent("five_hour", 2)] }, + { driver: "codex", windows: [] }, + ], + model: "claude-haiku-5-5", + expected: at(2), + }, + { + name: "unknown while an account still has room", + accounts: [{ windows: [spent("five_hour", 1)] }, { windows: [] }], + model: "claude-haiku-5-5", + expected: null, + }, + { + name: "unknown while an account could not be read", + accounts: [{ windows: [spent("five_hour", 1)] }, { unavailable: true as const }], + model: "claude-haiku-5-5", + expected: null, + }, + { + name: "unknown when a spent window has no reset or already reset", + accounts: [{ windows: [{ ...spent("five_hour", 1), resetsAt: undefined }] }], + model: "claude-haiku-5-5", + expected: null, + }, + { + name: "a model-scoped weekly window blocks only its own model", + accounts: [{ windows: [spent("seven_day_fable", 5)] }], + model: "claude-haiku-5-5", + expected: null, + }, + { + name: "a model-scoped weekly window blocks its own model", + accounts: [{ windows: [spent("seven_day_fable", 5)] }], + model: "claude-fable-5-2", + expected: at(5), + }, + { + name: "no account of the driver", + accounts: [{ driver: "codex", windows: [spent("five_hour", 1)] }], + model: "claude-haiku-5-5", + expected: null, + }, +])("hub pool reset: $name", ({ accounts, model, expected }) => { + expect( + UsageLimitSources.hubPoolResetAt( + [hub(accounts)], + ProviderDriverKind.make("claudeAgent"), + model, + NOW, + ), + ).toBe(expected); +}); + +it.effect("the reset fallback refreshes its hub, and a stalled hub cannot hold the failure", () => + Effect.gen(function* () { + yield* TestClock.setTime(NOW); + const roomLeft = { ...hub([{ windows: [spent("five_hour", 2)] }]), origin: "http://hub.test" }; + const cached = { + ...roomLeft, + accounts: roomLeft.accounts.map((account) => ({ + ...account, + usageLimits: { + ...account.usageLimits, + windows: account.usageLimits.windows.map((window) => ({ ...window, usedPercent: 99 })), + }, + })), + }; + let current: ReadonlyArray = [cached]; + let refresh: Effect.Effect = Effect.sync(() => { + current = [roomLeft]; + }); + const lookup = UsageLimitSources.UsageLimitResetFallback.pipe( + Effect.flatMap((fallback) => + fallback.resetAt(ProviderDriverKind.make("claudeAgent"), "http://hub.test", null), + ), + Effect.provide( + UsageLimitSources.layerResetFallback.pipe( + Layer.provide( + Layer.mock(UsageLimitSources.UsageLimitSources)({ + current: Effect.sync(() => current), + refresh: Effect.suspend(() => refresh), + }), + ), + ), + ), + ); + // The last poll saw room; the account was spent after it. + expect(yield* lookup).toBe(at(2)); + current = [cached]; + refresh = Effect.never; + const stalled = yield* Effect.forkChild(lookup); + yield* TestClock.adjust("5 seconds"); + expect(yield* Fiber.join(stalled)).toBeNull(); + }), +); diff --git a/apps/server/src/usage/UsageLimitSources.ts b/apps/server/src/usage/UsageLimitSources.ts index 7d40b0fb9ecea..d6f867d621d23 100644 --- a/apps/server/src/usage/UsageLimitSources.ts +++ b/apps/server/src/usage/UsageLimitSources.ts @@ -3,6 +3,7 @@ import { UsageLimitSourceError, type UsageLimitSourceConsumeResetCreditInput, type ProviderConsumeResetCreditResult, + type ProviderDriverKind, type ServerSettings, UsageLimitSourceConfig, settingsSecretLiteral, @@ -26,6 +27,7 @@ import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretReso import * as SecretResolverLayer from "../secrets/SecretResolver.ts"; import { resolveSettingsSecrets } from "../secrets/settingsSecretValue.ts"; import * as Settings from "../serverSettings.ts"; +import { windowLimitsModel } from "../provider/claudeUsageLimits.ts"; import { makeCliproxyApi } from "./cliproxyApi.ts"; export class UsageLimitSources extends Context.Service< @@ -79,7 +81,19 @@ export const make = Effect.gen(function* () { config: UsageLimitSourceConfig, ) { const checkedAt = DateTime.formatIso(yield* DateTime.now); - const base = { id, kind: config.kind, label: sourceLabel(id, config), checkedAt } as const; + const url = URL.parse(config.url); + const base = { + id, + kind: config.kind, + label: sourceLabel(id, config), + ...(url && /^https?:$/.test(url.protocol) + ? { + origin: url.origin, + endpoint: `${url.origin}${url.pathname.replace(/(\/management\.html)?\/*$/i, "")}`, + } + : {}), + checkedAt, + } as const; const managementKey = yield* readManagementKey(config); if (managementKey.length === 0) { return { ...base, accounts: [], error: missingManagementKey(config) }; @@ -198,3 +212,64 @@ export const make = Effect.gen(function* () { export const layer = Layer.effect(UsageLimitSources, make).pipe( Layer.provide(SecretResolverLayer.layer), ); + +/** A hub answers with a usage limit only when every account it routes the driver to is spent. */ +export function hubPoolResetAt( + sources: ReadonlyArray, + driver: ProviderDriverKind, + model: string | null, + now: number, +): string | null { + let earliest: number | null = null; + for (const source of sources) { + for (const account of source.accounts) { + if (account.driver !== driver) continue; + // An unread account may be the one still blocked, so its refill is unknown. + if (account.usageLimits.unavailable) return null; + const spent = account.usageLimits.windows.filter( + (window) => window.usedPercent >= 100 && windowLimitsModel(window, model), + ); + if (spent.length === 0) return null; + const resets = spent.map((window) => Date.parse(window.resetsAt ?? "")); + if (resets.some((reset) => !Number.isFinite(reset))) return null; + const recoversAt = Math.max(...resets); + if (recoversAt <= now) return null; + earliest = earliest === null ? recoversAt : Math.min(earliest, recoversAt); + } + } + return earliest === null ? null : DateTime.formatIso(DateTime.makeUnsafe(earliest)); +} + +export class UsageLimitResetFallback extends Context.Reference<{ + readonly resetAt: ( + driver: ProviderDriverKind, + endpoint: string, + model: string | null, + ) => Effect.Effect; +}>("t3/usage/UsageLimitResetFallback", { + defaultValue: () => ({ resetAt: () => Effect.succeed(null) }), +}) {} + +export const layerResetFallback = Layer.effect( + UsageLimitResetFallback, + Effect.gen(function* () { + const sources = yield* UsageLimitSources; + return { + resetAt: Effect.fn("UsageLimitResetFallback.resetAt")(function* (driver, endpoint, model) { + // Only the hub at the instance's endpoint serves its turns. + const atEndpoint = Effect.map(sources.current, (current) => + current.filter((source) => source.origin === endpoint), + ); + if ((yield* atEndpoint).length === 0) return null; + // The pool usually ran out after the last poll, so the cached accounts still show room. + yield* sources.refresh.pipe(Effect.timeout("5 seconds"), Effect.ignore); + return hubPoolResetAt( + yield* atEndpoint, + driver, + model, + DateTime.toEpochMillis(yield* DateTime.now), + ); + }), + }; + }), +); diff --git a/apps/server/src/usage/UsageService.test.ts b/apps/server/src/usage/UsageService.test.ts index e1562070cf28f..01cbf3a9f52ce 100644 --- a/apps/server/src/usage/UsageService.test.ts +++ b/apps/server/src/usage/UsageService.test.ts @@ -51,6 +51,7 @@ import * as AntigravityUsage from "../provider/Drivers/AntigravityUsage.ts"; import * as ProviderHostLive from "../provider/ProviderHostLive.ts"; import { totalTokens, type UsageRecord } from "@t3tools/provider-core/server/usage"; import * as UsageService from "./UsageService.ts"; +import { DESKTOP_HOST_SCOPE_ENV, physicalSourceIdOf } from "./usageTranscriptReader.ts"; import * as MuseSdk from "../../../../packages/provider-muse/src/server/sdk.ts"; import { MuseUsageHome } from "@t3tools/provider-muse/server"; import { makeScanCacheWriter, SCAN_CACHE_FILE_NAME } from "./usageScanCache.ts"; @@ -213,6 +214,7 @@ const layerService = (input: { Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), Layer.provideMerge(Layer.succeed(HostProcess.Platform, input.platform ?? "linux")), Layer.provideMerge(Layer.succeed(HostProcess.Hostname, input.hostname ?? "usage-test-host")), + Layer.provideMerge(Layer.succeed(HostProcess.HomeDirectory, input.home)), Layer.provideMerge(ServerSettings.layerTest(input.settings)), Layer.provideMerge( Layer.succeed( @@ -521,7 +523,74 @@ describe("Codex usage history", () => { }); describe("UsageService", () => { - it.live("counts compressed Codex rollouts across refresh, rewrites, and cache restarts", () => + it.effect("keeps pricing metadata paired with the rates used during a concurrent refresh", () => + Effect.gen(function* () { + const { home, settings } = yield* setup; + const sessions = NodePath.join(home, "codex", "sessions"); + yield* Effect.promise(async () => { + await NodeFSP.mkdir(sessions, { recursive: true }); + await NodeFSP.writeFile( + NodePath.join(sessions, "session.jsonl"), + [ + { type: "session_meta", payload: { id: "session", model_provider: "openai" } }, + { type: "turn_context", payload: { model: "gpt-6-astra", turn_id: "turn" } }, + { + type: "event_msg", + timestamp: "2026-08-01T10:00:00Z", + payload: { + type: "token_count", + info: { last_token_usage: { input_tokens: 0, output_tokens: 1 } }, + }, + }, + ] + .map((record) => encodeUnknownJsonString(record)) + .join("\n") + "\n", + ); + }); + let rate = 0; + yield* Effect.gen(function* () { + const resolving = yield* Deferred.make(); + const release = yield* Deferred.make(); + const service = yield* UsageService.make.pipe( + Effect.provideService(CodexUsageHistory.CodexUsageHistory, { + resolve: () => + Deferred.succeed(resolving, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.as(new Map()), + ), + }), + ); + const initialPricing = yield* service.refreshRates; + yield* TestClock.adjust(61_000); + const pending = yield* service.readSummary(WINDOW).pipe(Effect.forkChild); + yield* Deferred.await(resolving); + yield* service.refreshRates; + yield* Deferred.succeed(release, undefined); + const original = yield* Fiber.join(pending); + assert.strictEqual(original.buckets[0]?.costUsd, 1); + assert.strictEqual(original.pricing.fetchedAt, initialPricing.fetchedAt); + const updated = yield* service.readSummary(WINDOW); + assert.strictEqual(updated.buckets[0]?.costUsd, 2); + assert.notStrictEqual(updated.pricing.fetchedAt, initialPricing.fetchedAt); + }).pipe( + Effect.provide( + layerService({ + prefix: "concurrent-rates", + home, + settings, + ratesResponse: () => + Effect.succeed( + Response.json({ + "gpt-6-astra": { input_cost_per_token: 0, output_cost_per_token: ++rate }, + }), + ), + }), + ), + ); + }).pipe(Effect.scoped), + ); + + it.effect("counts compressed Codex rollouts across refresh, rewrites, and cache restarts", () => Effect.gen(function* () { const { home, settings } = yield* setup; const sessions = NodePath.join(home, "codex", "sessions"); @@ -631,10 +700,10 @@ describe("UsageService", () => { Effect.scoped, Effect.provide(layerService({ prefix: "usage-compressed-codex", home, settings })), ); - }).pipe(Effect.scoped), + }), ); - it.live("keeps the first host identity across recreation and explicit override removal", () => + it.effect("keeps the first host identity across recreation and explicit override removal", () => Effect.gen(function* () { const { transcript, settings, home } = yield* setup; yield* Effect.promise(() => NodeFSP.writeFile(transcript, claudeLine(1, 5))); @@ -662,10 +731,10 @@ describe("UsageService", () => { const source = summary.sources.find((source) => source.fingerprint.provider === "claude"); assert.strictEqual(source?.fingerprint.hostId, expected); } - }).pipe(Effect.scoped), + }), ); - it.live("concurrent initialization keeps one host identity across service recreation", () => + it.effect("concurrent initialization keeps one host identity across service recreation", () => Effect.gen(function* () { const { transcript, settings, home } = yield* setup; yield* Effect.promise(() => NodeFSP.writeFile(transcript, claudeLine(1, 5))); @@ -718,10 +787,10 @@ describe("UsageService", () => { const source = summary.sources.find((source) => source.fingerprint.provider === "claude"); assert.strictEqual(source?.fingerprint.hostId, hostId); } - }).pipe(Effect.scoped), + }), ); - it.live( + it.effect( "repairs initial Codex prices after unchanged-file retries, persists evidence and corrections, and honors later native tiers", () => Effect.gen(function* () { @@ -870,10 +939,10 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped), + }), ); - it.live( + it.effect( "keeps resolved Codex tiers when a grown pre-v15 cache entry re-parses without history", () => Effect.gen(function* () { @@ -966,10 +1035,10 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped), + }), ); - it.live( + it.effect( "reads Pi per-model usage and auxiliary costs once across forks and repeated entries", () => Effect.gen(function* () { @@ -1155,10 +1224,80 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped), + }), ); - it.live( + it.effect("prices Pi usage mapped onto a Pi model at that model's rate", () => + Effect.gen(function* () { + const { home, settings } = yield* setup; + const dir = NodePath.join(home, "pi", "sessions", "project"); + const message = (id: string, model: string) => ({ + type: "message", + id, + timestamp: "2026-08-01T10:00:00Z", + message: { + role: "assistant", + provider: "anthropic", + model, + usage: { input: 10, output: 5 }, + }, + }); + yield* Effect.promise(async () => { + await NodeFSP.mkdir(dir, { recursive: true }); + await NodeFSP.writeFile( + NodePath.join(dir, "session.jsonl"), + [ + { type: "session", id: "session" }, + message("released", "actual-model"), + message("preview", "actual-model-preview"), + ] + .map((line) => encodeUnknownJsonString(line)) + .join("\n") + "\n", + ); + }); + const summary = yield* Effect.gen(function* () { + const service = yield* UsageService.make; + return yield* service.readSummary(WINDOW); + }).pipe( + Effect.scoped, + Effect.provide( + layerService({ + prefix: "usage-pi-model-alias", + home, + ratesDocument: { + "actual-model": { input_cost_per_token: 1, output_cost_per_token: 4 }, + }, + settings: { + ...settings, + usageModelAliases: { "anthropic/actual-model-preview": "anthropic/actual-model" }, + }, + }), + ), + ); + assert.deepStrictEqual( + summary.buckets + .filter((bucket) => bucket.provider === "pi") + .map(({ model, records, unpricedRecords, costUsd, costSource }) => ({ + model, + records, + unpricedRecords, + costUsd, + costSource, + })), + [ + { + model: "anthropic/actual-model", + records: 2, + unpricedRecords: 0, + costUsd: 60, + costSource: "modelPriced", + }, + ], + ); + }), + ); + + it.effect( "reads Muse inference history once, persists appended usage, and reprices unknown models", () => Effect.gen(function* () { @@ -1377,10 +1516,10 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped), + }), ); - it.live("returns other providers' usage while Muse home discovery is still pending", () => + it.effect("returns other providers' usage while Muse home discovery is still pending", () => Effect.gen(function* () { const { transcript, settings, home } = yield* setup; yield* Effect.promise(() => NodeFSP.writeFile(transcript, claudeLine(1, 5))); @@ -1437,73 +1576,108 @@ describe("UsageService", () => { assert.deepStrictEqual(museSources(refreshed), [ [NodePath.join(museHome, "sessions"), "ok", undefined], ]); - }).pipe(Effect.scoped), + }), ); - it.live("reports missing Pi history and skips malformed records without losing valid usage", () => + it.effect("identifies a transcript directory created after the server first read usage", () => Effect.gen(function* () { const { home, settings } = yield* setup; + const scope = "0123456789abcdef0123456789abcdef"; + const dir = NodePath.join(home, "pi", "sessions"); yield* Effect.gen(function* () { const service = yield* UsageService.make; - const missing = yield* service.readSummary(WINDOW); + const piFingerprint = (summary: UsageSummary) => + summary.sources.find((source) => source.fingerprint.provider === "pi")?.fingerprint; + const missing = piFingerprint(yield* service.readSummary(WINDOW)); + assert.strictEqual(missing?.resolvedHomePath, dir); + assert.isUndefined(missing?.physicalSourceId); + yield* Effect.promise(() => NodeFSP.mkdir(dir, { recursive: true })); assert.strictEqual( - missing.sources.find((source) => source.fingerprint.provider === "pi")?.status, - "missing", + piFingerprint(yield* service.readSummary(WINDOW))?.physicalSourceId, + physicalSourceIdOf(scope, yield* Effect.promise(() => NodeFSP.realpath(dir))), ); - const dir = NodePath.join(home, "pi", "sessions"); - yield* Effect.promise(async () => { - await NodeFSP.mkdir(dir, { recursive: true }); - await NodeFSP.writeFile( - NodePath.join(dir, "session.jsonl"), - [ - "{broken", - "null", - encodeUnknownJsonString({ type: "session", id: "valid" }), - encodeUnknownJsonString({ - type: "message", - id: "bad", - timestamp: "invalid", - message: { - role: "assistant", - provider: "openai", - model: "model", - usage: { input: 100 }, - }, - }), - encodeUnknownJsonString({ - type: "message", - id: "valid", - timestamp: "2026-08-01T10:00:00Z", - message: { - role: "assistant", - provider: "openai", - model: "model", - usage: { - input: -1, - cacheRead: "100", - output: 7, - reasoning: 50, - cost: { total: -1 }, - }, - }, - }), - ].join("\n") + "\n", - ); - }); - const summary = yield* service.readSummary(WINDOW); - const pi = summary.buckets.filter((bucket) => bucket.provider === "pi"); - assert.strictEqual(pi.length, 1); - assert.strictEqual(pi[0] && totalTokens(pi[0].totals), 7); - assert.strictEqual(pi[0]?.totals.reasoningTokens, 7); - assert.strictEqual(pi[0]?.costSource, "unpriced"); }).pipe( Effect.scoped, - Effect.provide(layerService({ prefix: "usage-pi-malformed", home, settings })), + Effect.provide( + layerService({ + prefix: "usage-physical-source-created", + home, + settings, + // The native Windows server reads identity from realpath alone. + platform: "win32", + environment: { [DESKTOP_HOST_SCOPE_ENV]: scope }, + }), + ), ); - }).pipe(Effect.scoped), + }), ); - it.live("discovers Pi history in its default home and directory overrides", () => + it.effect( + "reports missing Pi history and skips malformed records without losing valid usage", + () => + Effect.gen(function* () { + const { home, settings } = yield* setup; + yield* Effect.gen(function* () { + const service = yield* UsageService.make; + const missing = yield* service.readSummary(WINDOW); + assert.strictEqual( + missing.sources.find((source) => source.fingerprint.provider === "pi")?.status, + "missing", + ); + const dir = NodePath.join(home, "pi", "sessions"); + yield* Effect.promise(async () => { + await NodeFSP.mkdir(dir, { recursive: true }); + await NodeFSP.writeFile( + NodePath.join(dir, "session.jsonl"), + [ + "{broken", + "null", + encodeUnknownJsonString({ type: "session", id: "valid" }), + encodeUnknownJsonString({ + type: "message", + id: "bad", + timestamp: "invalid", + message: { + role: "assistant", + provider: "openai", + model: "model", + usage: { input: 100 }, + }, + }), + encodeUnknownJsonString({ + type: "message", + id: "valid", + timestamp: "2026-08-01T10:00:00Z", + message: { + role: "assistant", + provider: "openai", + model: "model", + usage: { + input: -1, + cacheRead: "100", + output: 7, + reasoning: 50, + cost: { total: -1 }, + }, + }, + }), + ].join("\n") + "\n", + ); + }); + const summary = yield* service.readSummary(WINDOW); + const pi = summary.buckets.filter((bucket) => bucket.provider === "pi"); + assert.strictEqual(pi.length, 1); + assert.strictEqual(pi[0] && totalTokens(pi[0].totals), 7); + assert.strictEqual(pi[0]?.totals.reasoningTokens, 7); + assert.strictEqual(pi[0]?.costSource, "unpriced"); + }).pipe( + Effect.scoped, + Effect.provide(layerService({ prefix: "usage-pi-malformed", home, settings })), + ); + }), + ); + + it.effect("discovers Pi history in its default home and directory overrides", () => Effect.gen(function* () { const { home, settings } = yield* setup; for (const [agentDir, sessionDir, relativeDir, launchArgs, instanceHome, platform] of [ @@ -1607,10 +1781,10 @@ describe("UsageService", () => { Effect.provideService(HostProcess.HomeDirectory, home), ); } - }).pipe(Effect.scoped), + }), ); - it.live("reads shared managed default and disabled extra account history once", () => + it.effect("reads shared managed default and disabled extra account history once", () => Effect.gen(function* () { const { home, settings } = yield* setup; const summary = yield* Effect.gen(function* () { @@ -1687,10 +1861,10 @@ describe("UsageService", () => { ).length, 1, ); - }).pipe(Effect.scoped), + }), ); - it.live("restores the Cursor account cache after a restart", () => + it.effect("restores the Cursor account cache after a restart", () => Effect.gen(function* () { const { settings, home } = yield* setup; yield* TestClock.setTime(CURSOR_NOW); @@ -1729,10 +1903,10 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped, Effect.provide(TestClock.layer())), + }), ); - it.live("reads the Cursor account of an API key or T3 sign-in instead of a CLI login", () => + it.effect("reads the Cursor account of an API key or T3 sign-in instead of a CLI login", () => Effect.gen(function* () { const { settings, home } = yield* setup; const signIn = new TextEncoder().encode( @@ -1781,10 +1955,10 @@ describe("UsageService", () => { ]); assert.strictEqual(cursorSource(summary)?.status, "ok"); } - }).pipe(Effect.scoped), + }), ); - it.live("does not show a previous Cursor account's history after the API key changes", () => + it.effect("does not show a previous Cursor account's history after the API key changes", () => Effect.gen(function* () { const { settings, home } = yield* setup; yield* TestClock.setTime(CURSOR_NOW); @@ -1823,10 +1997,10 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped, Effect.provide(TestClock.layer())), + }), ); - it.live("keeps a new Cursor key's refresh visible beside a failing key", () => + it.effect("keeps a new Cursor key's refresh visible beside a failing key", () => Effect.gen(function* () { const { settings, home } = yield* setup; const cursor = makeFakeCursor(); @@ -1908,10 +2082,10 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped), + }), ); - it.live("reports one Cursor source when two API keys belong to the same account", () => + it.effect("reports one Cursor source when two API keys belong to the same account", () => Effect.gen(function* () { const { settings, home } = yield* setup; yield* TestClock.setTime(CURSOR_NOW); @@ -1970,10 +2144,10 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped, Effect.provide(TestClock.layer())), + }), ); - it.live( + it.effect( "includes OpenCode history but does not substitute desktop usage for an unavailable Cursor account", () => Effect.gen(function* () { @@ -2027,10 +2201,10 @@ describe("UsageService", () => { ?.distinctSessions, 1, ); - }).pipe(Effect.scoped), + }), ); - it.live("counts unpriced tokens from the unpriced records of a mixed model", () => + it.effect("counts unpriced tokens from the unpriced records of a mixed model", () => Effect.gen(function* () { const { settings, home } = yield* setup; const message = (id: string, cost: number, input: number, output: number, minute: number) => @@ -2064,10 +2238,10 @@ describe("UsageService", () => { assert.strictEqual(model?.costUsd, 1); assert.strictEqual(model?.totalTokens, 10_000); assert.strictEqual(model?.unpricedTokens, 9_900); - }).pipe(Effect.scoped), + }), ); - it.live("includes host and merged OpenCode instance histories once across list aliases", () => + it.effect("includes host and merged OpenCode instance histories once across list aliases", () => Effect.gen(function* () { const { settings, home } = yield* setup; const hostRoot = NodePath.join(home, "opencode"); @@ -2143,10 +2317,10 @@ describe("UsageService", () => { .sort((a, b) => a - b), [5, 7, 11], ); - }).pipe(Effect.scoped), + }), ); - it.live("counts aliased OpenCode and Antigravity directories once", () => + it.effect("counts aliased OpenCode and Antigravity directories once", () => Effect.gen(function* () { const { settings, home } = yield* setup; const opencode = NodePath.join(home, "opencode-store"); @@ -2197,10 +2371,10 @@ describe("UsageService", () => { sourcesFor("antigravity")[0]?.fingerprint.resolvedHomePath, yield* Effect.promise(() => NodeFSP.realpath(conversations)), ); - }).pipe(Effect.scoped), + }), ); - it.live("reads Antigravity history from each instance's data directory", () => + it.effect("reads Antigravity history from each instance's data directory", () => Effect.gen(function* () { const { settings, home } = yield* setup; const first = NodePath.join(home, "antigravity-first"); @@ -2268,10 +2442,10 @@ describe("UsageService", () => { outputTokens, ); } - }).pipe(Effect.scoped), + }), ); - it.live("reads configured and disabled accounts once across shared and aliased homes", () => + it.effect("reads configured and disabled accounts once across shared and aliased homes", () => Effect.gen(function* () { const { transcript, settings, home } = yield* setup; const codexHome = NodePath.join(home, "codex-account"); @@ -2337,6 +2511,7 @@ describe("UsageService", () => { }, [ProviderInstanceId.make("codex-work")]: { driver: ProviderDriverKind.make("codex"), + config: { shadowHomePath: NodePath.join(home, "work-shadow") }, environment: [{ name: "CODEX_HOME", value: codexHome, sensitive: false }], }, [ProviderInstanceId.make("codex-alias")]: { @@ -2386,10 +2561,38 @@ describe("UsageService", () => { sources.filter((source) => source.fingerprint.provider === "codex").length, 1, ); - }).pipe(Effect.scoped), + }), + ); + + it.effect("keeps default-home history after the default account moves to a custom home", () => + Effect.gen(function* () { + const { transcript, settings, home } = yield* setup; + const defaultHome = NodePath.join(home, "default-claude"); + const defaultProjects = NodePath.join(defaultHome, "projects", "proj"); + yield* Effect.promise(async () => { + await NodeFSP.writeFile(transcript, claudeLine(1, 5)); + await NodeFSP.mkdir(defaultProjects, { recursive: true }); + await NodeFSP.writeFile( + NodePath.join(defaultProjects, "earlier.jsonl"), + claudeLine(2, 7, "claude-fable-5", "earlier-session"), + ); + }); + const summary = yield* UsageService.make.pipe( + Effect.flatMap((service) => service.readSummary(WINDOW)), + Effect.provide( + layerService({ + prefix: "usage-service-default-home-test", + home, + environment: { CLAUDE_CONFIG_DIR: defaultHome }, + settings, + }), + ), + ); + assert.strictEqual(totalOutputTokens(summary), 12); + }), ); - it.live( + it.effect( "uses explicit account settings before environment and default homes, then refreshes them", () => Effect.gen(function* () { @@ -2469,10 +2672,10 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped), + }), ); - it.live( + it.effect( "uses inherited home variables when explicit default accounts have no home settings", () => Effect.gen(function* () { @@ -2515,10 +2718,10 @@ describe("UsageService", () => { .resolvedHomePath, NodePath.join(home, "grok", "sessions"), ); - }).pipe(Effect.scoped), + }), ); - it.live.skipIf(HostProcess.Platform.defaultValue() === "win32" || process.getuid?.() === 0)( + it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32" || process.getuid?.() === 0)( "reports unreadable transcripts as partial and recovers once they can be read", () => Effect.gen(function* () { @@ -2543,10 +2746,10 @@ describe("UsageService", () => { }).pipe( Effect.provide(layerService({ prefix: "usage-service-unreadable", home, settings })), ); - }).pipe(Effect.scoped), + }), ); - it.live( + it.effect( "answers a provisional read from the last scan, then the exact read adds what changed", () => Effect.gen(function* () { @@ -2612,10 +2815,10 @@ describe("UsageService", () => { Effect.scoped, Effect.provide(layerService({ prefix: "usage-service-provisional", home, settings })), ); - }).pipe(Effect.scoped), + }), ); - it.live("filters original model IDs before aliases, pricing, buckets and session counts", () => + it.effect("filters original model IDs before aliases, pricing, buckets and session counts", () => Effect.gen(function* () { const { transcript, settings, home } = yield* setup; yield* Effect.promise(() => @@ -2717,10 +2920,10 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped), + }), ); - it.live("reprices unchanged transcripts when custom prices are added, edited, or removed", () => + it.effect("reprices unchanged transcripts when custom prices are added, edited, or removed", () => Effect.gen(function* () { const { transcript, settings, home } = yield* setup; yield* Effect.promise(() => NodeFSP.writeFile(transcript, claudeLine(1, 5, "example-model"))); @@ -2761,10 +2964,10 @@ describe("UsageService", () => { layerService({ prefix: "usage-service-price-overrides-test", home, settings }), ), ); - }).pipe(Effect.scoped), + }), ); - it.live( + it.effect( "retains Codex reconciliation and partial coverage through persisted restart, append and repair", () => Effect.gen(function* () { @@ -2884,10 +3087,10 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped), + }), ); - it.live( + it.effect( "merges partial Claude copies through tail replacement, newline completion and restart", () => Effect.gen(function* () { @@ -2943,10 +3146,10 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped), + }), ); - it.live( + it.effect( "keeps large-record totals and costs exact through append, dedupe, restart and cleanup", () => Effect.gen(function* () { @@ -2998,13 +3201,14 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped), + }), ); - it.live.each([4, 5, 6, 7, 8, 10, 11, 12, 13, 14])( + it.effect.each([4, 5, 6, 7, 8, 10, 11, 12, 13, 14])( "upgrades a v%s cache: reparses live Codex usage, keeps deleted rollouts, removes the old file", (version) => Effect.gen(function* () { + yield* TestClock.setTime(Date.parse("2026-08-02T12:00:00Z")); const { home, settings } = yield* setup; const sessions = NodePath.join(home, "codex", "sessions"); const rollout = (sessionId: string, outputTokens: number) => @@ -3102,7 +3306,7 @@ describe("UsageService", () => { 60 + 20 * (version === 4 ? 1 : 2), ); yield* upgraded.awaitPersisted; - assert.strictEqual(SCAN_CACHE_FILE_NAME, "usage-scan-cache-v15.json"); + assert.strictEqual(SCAN_CACHE_FILE_NAME, "usage-scan-cache-v17.json"); assert.deepStrictEqual( (yield* Effect.promise(() => NodeFSP.readdir(stateDir))) .filter((name) => name.startsWith("usage-scan-cache")) @@ -3128,10 +3332,10 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped), + }), ); - it.live("re-parses a clean v10 Codex rollout that grew after an interrupted request", () => + it.effect("re-parses a clean v10 Codex rollout that grew after an interrupted request", () => Effect.gen(function* () { const { home, settings } = yield* setup; const sessions = NodePath.join(home, "codex", "sessions"); @@ -3228,10 +3432,10 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped), + }), ); - it.live("preserves saved tokens, costs and sessions after transcript cleanup and restart", () => + it.effect("preserves saved tokens, costs and sessions after transcript cleanup and restart", () => Effect.gen(function* () { const { transcript, settings, home } = yield* setup; const alias = NodePath.join(home, "claude-alias"); @@ -3329,10 +3533,10 @@ describe("UsageService", () => { }), ), ); - }).pipe(Effect.scoped), + }), ); - it.live("credits the same copy of a duplicate after its transcripts are deleted", () => + it.effect("credits the same copy of a duplicate after its transcripts are deleted", () => Effect.gen(function* () { const { transcript, settings, home } = yield* setup; const dir = NodePath.dirname(transcript); @@ -3367,10 +3571,10 @@ describe("UsageService", () => { }).pipe( Effect.provide(layerService({ prefix: "usage-service-copy-order-test", home, settings })), ); - }).pipe(Effect.scoped), + }), ); - it.live("credits the same copy of a duplicate from the last scan as an exact read does", () => + it.effect("credits the same copy of a duplicate from the last scan as an exact read does", () => Effect.gen(function* () { yield* TestClock.setTime(Date.parse("2026-08-15T00:00:00Z")); const { transcript, settings, home } = yield* setup; @@ -3425,10 +3629,10 @@ describe("UsageService", () => { }).pipe( Effect.provide(layerService({ prefix: "usage-service-narrow-order-test", home, settings })), ); - }).pipe(Effect.scoped, Effect.provide(TestClock.layer())), + }), ); - it.live.skipIf(HostProcess.Platform.defaultValue() === "win32")( + it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( "keeps a newer cached read when a slower scan of another window finishes later", () => Effect.gen(function* () { @@ -3455,53 +3659,56 @@ describe("UsageService", () => { }).pipe( Effect.provide(layerService({ prefix: "usage-service-stale-read-test", home, settings })), ); - }).pipe(Effect.scoped), + }), ); - it.live("keeps an absent directory missing when its cached files only contain diagnostics", () => - Effect.gen(function* () { - const { transcript, settings, home } = yield* setup; - yield* Effect.promise(() => - NodeFSP.rm(NodePath.join(home, "claude", "projects"), { recursive: true }), - ); - yield* Effect.gen(function* () { - const { stateDir } = yield* ServerConfig.ServerConfig; - const mtimeMs = yield* Clock.currentTimeMillis; - const cache = makeScanCacheWriter()( - new Map([ - [ - transcript, - { - size: 0, - mtimeMs, - provider: "claude" as const, - records: [], - tailRecords: [], - malformedRecords: 2, - position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null }, - }, - ], - ]), - {}, + it.effect( + "keeps an absent directory missing when its cached files only contain diagnostics", + () => + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse("2026-08-02T12:00:00Z")); + const { transcript, settings, home } = yield* setup; + yield* Effect.promise(() => + NodeFSP.rm(NodePath.join(home, "claude", "projects"), { recursive: true }), ); - yield* Effect.promise(async () => { - await NodeFSP.mkdir(stateDir, { recursive: true }); - await NodeFSP.writeFile(NodePath.join(stateDir, SCAN_CACHE_FILE_NAME), cache); - }); - const summary = yield* (yield* UsageService.make).readSummary(WINDOW); - const source = summary.sources.find((source) => source.fingerprint.provider === "claude"); - assert.strictEqual(source?.status, "missing"); - assert.strictEqual(source?.scannedFiles, 0); - assert.strictEqual(source?.malformedRecords, 2); - }).pipe( - Effect.provide( - layerService({ prefix: "usage-service-missing-diagnostics", home, settings }), - ), - ); - }).pipe(Effect.scoped), + yield* Effect.gen(function* () { + const { stateDir } = yield* ServerConfig.ServerConfig; + const mtimeMs = yield* Clock.currentTimeMillis; + const cache = makeScanCacheWriter()( + new Map([ + [ + transcript, + { + size: 0, + mtimeMs, + provider: "claude" as const, + records: [], + tailRecords: [], + malformedRecords: 2, + position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null }, + }, + ], + ]), + {}, + ); + yield* Effect.promise(async () => { + await NodeFSP.mkdir(stateDir, { recursive: true }); + await NodeFSP.writeFile(NodePath.join(stateDir, SCAN_CACHE_FILE_NAME), cache); + }); + const summary = yield* (yield* UsageService.make).readSummary(WINDOW); + const source = summary.sources.find((source) => source.fingerprint.provider === "claude"); + assert.strictEqual(source?.status, "missing"); + assert.strictEqual(source?.scannedFiles, 0); + assert.strictEqual(source?.malformedRecords, 2); + }).pipe( + Effect.provide( + layerService({ prefix: "usage-service-missing-diagnostics", home, settings }), + ), + ); + }), ); - it.live("reports saved usage of a removed directory only for windows it reaches", () => + it.effect("reports saved usage of a removed directory only for windows it reaches", () => Effect.gen(function* () { yield* TestClock.setTime(Date.parse("2026-08-12T00:00:00Z")); const { transcript, settings, home } = yield* setup; @@ -3532,10 +3739,10 @@ describe("UsageService", () => { assert.deepStrictEqual(later.buckets, []); assert.strictEqual(later.sources[0]?.status, "missing"); assert.strictEqual(later.sources[0]?.scannedFiles, 0); - }).pipe(Effect.scoped, Effect.provide(TestClock.layer())), + }), ); - it.live("does not share an in-flight scan after custom prices change", () => + it.effect("does not share an in-flight scan after custom prices change", () => Effect.gen(function* () { const { transcript, settings, home } = yield* setup; yield* Effect.promise(() => NodeFSP.writeFile(transcript, claudeLine(1, 5, "example-model"))); @@ -3601,10 +3808,10 @@ describe("UsageService", () => { }).pipe( Effect.provide(layerService({ prefix: "usage-service-price-race-test", home, settings })), ); - }).pipe(Effect.scoped), + }), ); - it.live.each(["network", "http", "invalidJson", "empty"] as const)( + it.effect.each(["network", "http", "invalidJson", "empty"] as const)( "recovers pricing through the GitHub API after a %s failure", (failure) => Effect.gen(function* () { @@ -3660,10 +3867,60 @@ describe("UsageService", () => { assert.strictEqual(cached.pricing.status, "cached"); assert.deepStrictEqual(cached.buckets, summary.buckets); assert.strictEqual(requests.length, 2); - }).pipe(Effect.scoped), + }), ); - it.live("does not orphan an in-flight scan when its first caller is interrupted", () => + it.effect("refetches cached rates early for a model they do not list", () => + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse("2026-08-02T00:00:00Z")); + const { transcript, settings, home } = yield* setup; + yield* Effect.promise(() => + NodeFSP.writeFile( + transcript, + claudeLine(1, 5, "claude-new-model") + claudeLine(2, 5, "claude-unknown-model"), + ), + ); + const requests: string[] = []; + let ratesDocument: Record = { + "claude-fable-5": { input_cost_per_token: 1e-5, output_cost_per_token: 5e-5 }, + }; + const serviceLayer = layerService({ + prefix: "usage-service-unlisted-model-test", + home, + settings, + onRatesFetch: (request) => requests.push(request.url), + ratesResponse: () => Effect.succeed(Response.json(ratesDocument)), + }); + const newModelCost = (summary: UsageSummary) => + summary.buckets.find((bucket) => bucket.model === "claude-new-model")?.costUsd; + + // Rates fetched for this scan are fresh: no second request for the models they lack. + const first = yield* (yield* UsageService.make.pipe( + Effect.provide(serviceLayer), + )).readSummary(WINDOW); + assert.strictEqual(newModelCost(first), 0); + assert.strictEqual(requests.length, 1); + + // An hour later LiteLLM lists the new model; a restarted server starts from the cached table. + ratesDocument = { + ...ratesDocument, + "claude-new-model": { input_cost_per_token: 2e-5, output_cost_per_token: 1e-4 }, + }; + yield* TestClock.adjust(Duration.hours(1)); + const service = yield* UsageService.make.pipe(Effect.provide(serviceLayer)); + const refreshed = yield* service.readSummary(WINDOW); + assert.strictEqual(refreshed.pricing.status, "fresh"); + assert.closeTo(newModelCost(refreshed) ?? -1, 10 * 2e-5 + 5 * 1e-4, 1e-12); + assert.strictEqual(requests.length, 2); + + // A model the fresh table still lacks does not refetch again before the TTL. + yield* TestClock.adjust(Duration.hours(1)); + yield* service.readSummary(WINDOW); + assert.strictEqual(requests.length, 2); + }), + ); + + it.effect("does not orphan an in-flight scan when its first caller is interrupted", () => Effect.gen(function* () { const { settings, home } = yield* setup; const service = yield* UsageService.make.pipe( @@ -3738,6 +3995,6 @@ describe("UsageService", () => { orphanedAt, `interruption left the next matching request pending at scheduler check ${orphanedAt}`, ); - }).pipe(Effect.scoped), + }), ); }); diff --git a/apps/server/src/usage/UsageService.ts b/apps/server/src/usage/UsageService.ts index a55532913c3c6..a59fbc66a02ca 100644 --- a/apps/server/src/usage/UsageService.ts +++ b/apps/server/src/usage/UsageService.ts @@ -40,10 +40,17 @@ import type { UsageRecord, } from "@t3tools/provider-core/server/usage"; import { canonicalModelFilter, resolveModelAliases, UsageAggregator } from "./usageAggregation.ts"; -import { createOverrideRateTable, parseRateTable, type RateTable } from "./usagePricing.ts"; import { + createOverrideRateTable, + isUnlistedModel, + parseRateTable, + type RateTable, +} from "./usagePricing.ts"; +import { + DESKTOP_HOST_SCOPE_ENV, listTranscriptFiles, readDirectoryVolumeId, + readPhysicalSourceId, readTranscriptRecords, } from "./usageTranscriptReader.ts"; import { @@ -83,6 +90,12 @@ const scanReaders = BUILT_IN_USAGE_DRIVERS.flatMap((driver) => driver.usage?.kind === "scan" ? [{ driver, reader: driver.usage }] : [], ); +const providerRateModels = new Map( + BUILT_IN_USAGE_DRIVERS.flatMap(({ usage }) => + usage?.rateModel === undefined ? [] : [[usage.provider, usage.rateModel] as const], + ), +); + const transcriptFormats = new Map( transcriptReaders.map(({ reader }) => [reader.provider, reader.format] as const), ); @@ -92,6 +105,7 @@ interface TranscriptSource { readonly format: TranscriptUsageFormat; readonly dir: string; readonly volumeId: string; + readonly physicalSourceId?: string; readonly fileName?: string; } @@ -158,6 +172,24 @@ export const make = Effect.gen(function* () { const settingsService = yield* ServerSettings.ServerSettingsService; const httpClient = yield* HttpClient.HttpClient; const hostEnvironment = yield* HostProcess.Environment; + const hostPlatform = yield* HostProcess.Platform; + const hostScope = hostEnvironment[DESKTOP_HOST_SCOPE_ENV]?.trim(); + // A mount or a directory's on-disk name does not change under a running server. + // A miss is not cached: the directory may not exist yet. + const physicalSourceIds = new Map>(); + const physicalSourceId = (dir: string) => + Effect.promise(() => { + const cached = physicalSourceIds.get(dir); + if (cached !== undefined) return cached; + const id = readPhysicalSourceId(dir, hostScope, hostPlatform); + physicalSourceIds.set(dir, id); + void id.then((resolved) => { + if (resolved === undefined && physicalSourceIds.get(dir) === id) { + physicalSourceIds.delete(dir); + } + }); + return id; + }); const hostname = yield* HostProcess.Hostname; const readerContext = yield* Effect.context(); @@ -291,6 +323,24 @@ export const make = Effect.gen(function* () { const ensureRates = (force: boolean) => ratesLock.withPermit(loadRates(force)); + // Rates are kept for RATES_TTL_MS, so a model released after the table was fetched would stay unpriced + // that long. A scan that meets a model the table does not list refetches early: once per model against + // a fresh table, with loadRates keeping RATES_REFRESH_FLOOR_MS between attempts. + const checkedUnlistedModels = new Set(); + const refreshForUnlistedModels = Effect.fn("UsageService.refreshForUnlistedModels")(function* ( + models: ReadonlySet, + ) { + const unlisted = [...models].filter( + (model) => !checkedUnlistedModels.has(model) && isUnlistedModel(rates, model), + ); + if (unlisted.length === 0) return; + yield* ensureRates(true); + const now = yield* Clock.currentTimeMillis; + // A failed fetch leaves the models unchecked, so a later scan tries again. + if (ratesFetchedAtMs === null || now - ratesFetchedAtMs >= RATES_REFRESH_FLOOR_MS) return; + for (const model of unlisted) checkedUnlistedModels.add(model); + }); + const refreshRates = ensureRates(true).pipe( Effect.map(pricing), Effect.withSpan("UsageService.refreshRates"), @@ -316,9 +366,8 @@ export const make = Effect.gen(function* () { > = Object.entries(settings.providerInstances) .filter(([, instance]) => instance.driver === driver.driverKind) .map(([id, instance]) => [ProviderInstanceId.make(id), instance, true] as const); - if (!Object.hasOwn(settings.providerInstances, driver.driverKind)) { - entries.push([ProviderInstanceId.make(driver.driverKind), {}, false]); - } + // History in the driver's default home stays counted after every instance moves elsewhere; readers dedupe shared directories. + entries.push([ProviderInstanceId.make(driver.driverKind), {}, false]); const decodeConfig = Schema.decodeUnknownOption(driver.configSchema); return yield* Effect.forEach( entries, @@ -385,11 +434,13 @@ export const make = Effect.gen(function* () { const key = `${provider}\0${dir}`; if (seen.has(key)) continue; seen.add(key); + const sourceId = yield* physicalSourceId(dir); dirs.push({ provider, format, dir, volumeId, + ...(sourceId === undefined ? {} : { physicalSourceId: sourceId }), ...(fileName === undefined ? {} : { fileName }), }); } @@ -629,6 +680,7 @@ export const make = Effect.gen(function* () { readonly provider: UsageProviderKind; readonly dir: string; readonly volumeId: string; + readonly physicalSourceId?: string; readonly hostId?: string; readonly status?: UsageSource["status"]; readonly message?: string; @@ -649,16 +701,22 @@ export const make = Effect.gen(function* () { windowStartMs: number, fromMemory: boolean, ) { - const { provider, format, dir, volumeId, fileName } = source; + const { provider, format, dir, volumeId, physicalSourceId, fileName } = source; + const identity = { + provider, + dir, + volumeId, + ...(physicalSourceId === undefined ? {} : { physicalSourceId }), + }; const exists = yield* fileSystem .exists(dir) .pipe(Effect.catchCause(() => Effect.succeed(false))); if (!exists) { cover(source, windowStartMs); - return { provider, dir, volumeId, files: null } satisfies ScannedDir; + return { ...identity, files: null } satisfies ScannedDir; } // scanSummary aggregates the saved files, as it does for a deleted directory. - if (fromMemory) return { provider, dir, volumeId, files: [] } satisfies ScannedDir; + if (fromMemory) return { ...identity, files: [] } satisfies ScannedDir; const { files, walked, failedPaths } = yield* Effect.promise(() => listTranscriptFiles(dir, windowStartMs, fileName === undefined ? undefined : { fileName }), ); @@ -704,9 +762,7 @@ export const make = Effect.gen(function* () { "usage.files_parsed": read.filter((file) => file.update !== undefined).length, }); return { - provider, - dir, - volumeId, + ...identity, files: parsedFiles, walked, ...(unread > 0 @@ -736,14 +792,19 @@ export const make = Effect.gen(function* () { Effect.flatMap((sources) => Effect.forEach(sources, ({ volumeId, ...source }) => Effect.map( + // A reader's own volumeId names a remote account, not a local directory. volumeId === undefined - ? Effect.promise(() => readDirectoryVolumeId(source.dir)) - : Effect.succeed(volumeId), - (resolved): ScannedDir => ({ + ? Effect.all([ + Effect.promise(() => readDirectoryVolumeId(source.dir)), + physicalSourceId(source.dir), + ]) + : Effect.succeed<[string, string | undefined]>([volumeId, undefined]), + ([resolved, sourceId]): ScannedDir => ({ ...source, files: source.files?.map((file) => ({ ...file, malformedRecords: 0 })) ?? null, provider: reader.provider, volumeId: resolved, + ...(sourceId === undefined ? {} : { physicalSourceId: sourceId }), }), ), ), @@ -848,18 +909,8 @@ export const make = Effect.gen(function* () { { concurrency: 2 }, ); - const aggregator = new UsageAggregator({ - timeZone: input.timeZone, - sinceDay: input.sinceDay, - untilDay: input.untilDay, - resolution: input.resolution ?? "day", - ...hourlyWindow, - rates, - priceOverrides: createOverrideRateTable(settings.usagePriceOverrides), - modelAliases: resolveModelAliases(settings.usageModelAliases), - ...(input.modelFilter === undefined ? {} : { modelFilter: input.modelFilter }), - }); - + const priceOverrides = createOverrideRateTable(settings.usagePriceOverrides); + const modelAliases = resolveModelAliases(settings.usageModelAliases); const sources: UsageSource[] = []; // Deleted transcripts' saved usage still aggregates and deduplicates normally. @@ -925,6 +976,37 @@ export const make = Effect.gen(function* () { for (const filePath of ordered) fileCache.set(filePath, entries.get(filePath)!); cacheDirty = true; } + + // The models priced below, as UsageAggregator resolves them: an alias replaces rateModel. + const rateModels = new Set(); + for (const files of filesByDir) { + for (const file of files) { + for (const record of file.records) { + const alias = modelAliases.get(record.model); + const model = alias ?? record.model; + if (priceOverrides.has(model.trim())) continue; + rateModels.add(alias === undefined ? (record.rateModel ?? model) : model); + } + } + } + // A saved answer prices with saved rates and the full scan after it refetches. + if (!fromMemory) yield* refreshForUnlistedModels(rateModels); + + const summaryPricing = pricing(); + + const aggregator = new UsageAggregator({ + timeZone: input.timeZone, + sinceDay: input.sinceDay, + untilDay: input.untilDay, + resolution: input.resolution ?? "day", + ...hourlyWindow, + rates, + priceOverrides, + modelAliases, + rateModel: (provider, model) => providerRateModels.get(provider)?.(model), + ...(input.modelFilter === undefined ? {} : { modelFilter: input.modelFilter }), + }); + const windowEndMs = hourlyWindow?.untilTimeMs ?? Date.parse(`${input.untilDay}T00:00:00Z`) + 86400000 + MTIME_SLACK_MS; @@ -972,7 +1054,17 @@ export const make = Effect.gen(function* () { for (const [ index, - { provider, dir, volumeId, files, status, message, refreshing, hostId: sourceHostId }, + { + provider, + dir, + volumeId, + physicalSourceId, + files, + status, + message, + refreshing, + hostId: sourceHostId, + }, ] of scannedDirs.entries()) { let scannedFiles = 0; let skippedFiles = 0; @@ -1019,7 +1111,13 @@ export const make = Effect.gen(function* () { const sourceMessage = message ?? (files === null ? "No transcript directory on this environment." : null); sources.push({ - fingerprint: { hostId: sourceHostId ?? hostId, provider, resolvedHomePath: dir, volumeId }, + fingerprint: { + hostId: sourceHostId ?? hostId, + provider, + resolvedHomePath: dir, + volumeId, + ...(physicalSourceId === undefined ? {} : { physicalSourceId }), + }, status: files === null && scannedFiles === 0 ? "missing" @@ -1067,7 +1165,7 @@ export const make = Effect.gen(function* () { untilDay: input.untilDay, buckets: aggregated.buckets, sources, - pricing: pricing(), + pricing: summaryPricing, scanDurationMs: Math.max(0, finishedAtMs - startedAtMs), ...(input.modelFilter === undefined ? {} : { modelFilter: input.modelFilter }), } satisfies UsageSummary; diff --git a/apps/server/src/usage/cliproxyApi.test.ts b/apps/server/src/usage/cliproxyApi.test.ts index 431af7b5971ff..5002d2223d1ba 100644 --- a/apps/server/src/usage/cliproxyApi.test.ts +++ b/apps/server/src/usage/cliproxyApi.test.ts @@ -437,6 +437,26 @@ describe("CLIProxyAPI built-in management API", () => { }), ); + it.effect("keeps a Claude account whose window reports no utilization", () => + Effect.gen(function* () { + const test = fixture({ + accounts: [{ ...accounts[0]!, provider: "claude" }], + upstream: () => ({ + status: 200, + body: { + five_hour: { utilization: null, resets_at: null }, + seven_day: { utilization: 50, resets_at: "2099-01-01T00:00:00Z" }, + }, + }), + }); + const result = yield* (yield* test.api).readAccounts(config); + expect(result[0]?.usageLimits.unavailable).toBeUndefined(); + expect( + result[0]?.usageLimits.windows.map((window) => [window.id, window.usedPercent]), + ).toEqual([["seven_day", 50]]); + }), + ); + const antigravitySummary = { buckets: [ { diff --git a/apps/server/src/usage/cliproxyApi.ts b/apps/server/src/usage/cliproxyApi.ts index 030983f50e33d..537a2b436f4a4 100644 --- a/apps/server/src/usage/cliproxyApi.ts +++ b/apps/server/src/usage/cliproxyApi.ts @@ -70,8 +70,9 @@ const CodexUsage = Schema.Struct({ }), ), }); +// The usage endpoint can report a window without a utilization, as the CLI's own schema allows. const ClaudeWindow = Schema.Struct({ - utilization: Schema.Number, + utilization: Schema.NullOr(Schema.Number), resets_at: Schema.NullOr(Schema.String), }); const ClaudeUsage = Schema.Struct({ diff --git a/apps/server/src/usage/usageAggregation.test.ts b/apps/server/src/usage/usageAggregation.test.ts index f99615085bd2a..58953fbcfafea 100644 --- a/apps/server/src/usage/usageAggregation.test.ts +++ b/apps/server/src/usage/usageAggregation.test.ts @@ -141,6 +141,32 @@ describe("UsageAggregator", () => { } }); + it("keeps a live Grok turn over a saved pre-v17 maximum in either source order", () => { + const turn = (timestampMs: number, outputTokens: number) => + record({ + provider: "grok", + model: "grok-4.5", + timestampMs, + dedupeKey: `s:p:${timestampMs}:grok-4.5`, + reportedCostUsd: outputTokens / 10, + totals: { ...record().totals, uncachedInputTokens: 0, outputTokens }, + }); + const first = turn(Date.parse("2026-08-08T10:00:00Z"), 10); + const resumed = turn(Date.parse("2026-08-08T11:00:00Z"), 20); + const savedMaximum = { ...first, totals: resumed.totals, reportedCostUsd: 2 }; + + const expected = aggregate([first, resumed]).buckets; + + for (const order of [ + [first, resumed, savedMaximum], + [savedMaximum, first, resumed], + ]) { + const result = aggregate(order); + expect(result.duplicatesDropped).toBe(1); + expect(result.buckets).toEqual(expected); + } + }); + it("does not count a key whose first copy fell outside the window", () => { const result = aggregate([ record({ dedupeKey: "msg_1:", timestampMs: Date.parse("2026-07-01T12:00:00Z") }), @@ -251,6 +277,7 @@ describe("UsageAggregator", () => { untilDay: "2026-08-31", rates, modelAliases: resolveModelAliases({ "example-preview": "claude-fable-5" }), + rateModel: () => "claude-fable", }); aggregator.add(record()); aggregator.add(record({ model: "example-preview", rateModel: "example-preview-high" })); @@ -266,20 +293,24 @@ describe("UsageAggregator", () => { describe("resolveModelAliases", () => { it("follows chains to the final model and drops chains that enter a loop", () => { - expect( - resolveModelAliases({ + expect([ + ...resolveModelAliases({ "preview[1m]": "preview", preview: "example-model", + shared: "preview", + intoLoop: "loop", loop: "back", back: "loop", - intoLoop: "loop", + laterIntoLoop: "back", + deepIntoLoop: "intoLoop", self: "self", + validAfterLoops: "shared", }), - ).toEqual( - new Map([ - ["preview[1m]", "example-model"], - ["preview", "example-model"], - ]), - ); + ]).toEqual([ + ["preview[1m]", "example-model"], + ["preview", "example-model"], + ["shared", "example-model"], + ["validAfterLoops", "example-model"], + ]); }); }); diff --git a/apps/server/src/usage/usageAggregation.ts b/apps/server/src/usage/usageAggregation.ts index 9ff9cb9369462..f4c39753548fd 100644 --- a/apps/server/src/usage/usageAggregation.ts +++ b/apps/server/src/usage/usageAggregation.ts @@ -10,7 +10,7 @@ import type { } from "@t3tools/contracts"; import { EMPTY_TOTALS, type UsageRecord } from "@t3tools/provider-core/server/usage"; -import { cacheSavingsUsd, priceUsage, type RateTable } from "./usagePricing.ts"; +import { cacheSavingsUsd, lookupRate, priceUsage, type RateTable } from "./usagePricing.ts"; // en-CA gives ISO-ordered parts in the requested zone; Date getters are host-local. function makeDayFormatter(timeZone: string): (timestampMs: number) => string { @@ -70,6 +70,7 @@ export interface AggregateOptions { readonly rates: RateTable; readonly priceOverrides?: RateTable; readonly modelAliases?: ReadonlyMap; + readonly rateModel?: (provider: UsageRecord["provider"], model: string) => string | undefined; readonly modelFilter?: UsageModelFilter; readonly resolution?: UsageResolution; readonly sinceTimeMs?: number; @@ -79,15 +80,26 @@ export interface AggregateOptions { export function resolveModelAliases( aliases: Readonly>, ): ReadonlyMap { - const resolved = new Map(); - for (const model of Object.keys(aliases)) { - const seen = new Set([model]); - let target = aliases[model]!; - while (Object.hasOwn(aliases, target) && !seen.has(target)) { - seen.add(target); + const models = Object.keys(aliases); + const memo = new Map(); + for (const model of models) { + if (memo.has(model)) continue; + const path: string[] = []; + let target = model; + while (Object.hasOwn(aliases, target) && !memo.has(target)) { + path.push(target); + // Null marks both the active walk and paths that end in a cycle. + memo.set(target, null); target = aliases[target]!; } - if (!Object.hasOwn(aliases, target)) resolved.set(model, target); + const terminal = Object.hasOwn(aliases, target) ? (memo.get(target) ?? null) : target; + for (const alias of path) memo.set(alias, terminal); + } + + const resolved = new Map(); + for (const model of models) { + const target = memo.get(model); + if (target !== null && target !== undefined) resolved.set(model, target); } return resolved; } @@ -170,6 +182,30 @@ export function mergeUsageCopies(held: UsageRecord, copy: UsageRecord): UsageRec }; } +/** + * Grok reports each turn once, complete, so real copies are equal. A larger one is a deleted + * transcript's pre-v17 maximum over turns that shared a prompt id, whichever source it came from. + */ +function smallestGrokCopy(held: UsageRecord, copy: UsageRecord): UsageRecord { + const a = held.totals; + const b = copy.totals; + const reportedCostUsd = + held.reportedCostUsd === null || copy.reportedCostUsd === null + ? (held.reportedCostUsd ?? copy.reportedCostUsd) + : Math.min(held.reportedCostUsd, copy.reportedCostUsd); + return { + ...held, + totals: { + uncachedInputTokens: Math.min(a.uncachedInputTokens, b.uncachedInputTokens), + cachedInputTokens: Math.min(a.cachedInputTokens, b.cachedInputTokens), + cacheCreationTokens: Math.min(a.cacheCreationTokens, b.cacheCreationTokens), + outputTokens: Math.min(a.outputTokens, b.outputTokens), + reasoningTokens: Math.min(a.reasoningTokens, b.reasoningTokens), + }, + reportedCostUsd, + }; +} + interface Contribution { record: UsageRecord; /** Null when the first copy fell outside the window. */ @@ -217,7 +253,12 @@ export class UsageAggregator { const held = this.#seen.get(record.dedupeKey); if (held !== undefined) { this.#duplicatesDropped += 1; - if (held.bucket !== null) held.record = mergeUsageCopies(held.record, record); + if (held.bucket !== null) { + held.record = + record.provider === "grok" + ? smallestGrokCopy(held.record, record) + : mergeUsageCopies(held.record, record); + } return false; } } @@ -301,12 +342,16 @@ export class UsageAggregator { if (priced.costSource === "providerReported") bucket.providerReportedRecords += 1; } - // Aliases use the target's rate, replacing provider-specific rateModel. + // Aliases use the target's rate. Only an unpriced target goes through the provider's own rate naming. #mapModel(record: UsageRecord): UsageRecord { const model = this.#options.modelAliases?.get(record.model); if (model === undefined) return record; const { rateModel: _rateModel, ...rest } = record; - return { ...rest, model }; + const rateModel = + lookupRate(this.#options.rates, model) === null + ? this.#options.rateModel?.(record.provider, model) + : undefined; + return rateModel === undefined ? { ...rest, model } : { ...rest, model, rateModel }; } // Time-ordered records usually reuse the previous bucket; avoid rebuilding its key. diff --git a/apps/server/src/usage/usagePricing.ts b/apps/server/src/usage/usagePricing.ts index 21a1bd02bb178..a9ffc691643fd 100644 --- a/apps/server/src/usage/usagePricing.ts +++ b/apps/server/src/usage/usagePricing.ts @@ -310,6 +310,14 @@ export function lookupRate(table: RateTable, model: string): ModelRate | null { return rate; } +/** A priceable model the table has no rate for; a newer table may list it. */ +export function isUnlistedModel(table: RateTable, model: string): boolean { + const bareName = bareModelName(stripVariantSuffix(normalizeRateKey(model))); + return ( + bareName.length > 0 && !UNPRICEABLE_MODELS.has(bareName) && lookupRate(table, model) === null + ); +} + function resolveRate(table: RateTable, model: string): ModelRate | null { const key = stripVariantSuffix(normalizeRateKey(model)); const bareName = bareModelName(key); diff --git a/apps/server/src/usage/usageScanCache.test.ts b/apps/server/src/usage/usageScanCache.test.ts index 84291da51a189..36864768eb525 100644 --- a/apps/server/src/usage/usageScanCache.test.ts +++ b/apps/server/src/usage/usageScanCache.test.ts @@ -285,6 +285,46 @@ describe("scan cache round trip", () => { ...codex(0), position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null }, }); + // The migrated entries persist without a state and load again under the current version. + expect(decodeScanCache(parse(makeScanCacheWriter()(restored, {})), TEST_FORMAT_MAP)).toEqual( + restored, + ); + }); + + it("restarts a grown v15 Codex entry, whose baseline time can be older than its totals", () => { + const request = { + input_tokens: 100, + cached_input_tokens: 0, + cache_write_input_tokens: 0, + output_tokens: 20, + reasoning_output_tokens: 0, + total_tokens: 120, + }; + const entry: CachedFile = { + size: 10, + mtimeMs: 100, + provider: "codex", + records: [record({ provider: "codex", model: "gpt-6-astra", dedupeKey: null })], + tailRecords: [], + malformedRecords: 0, + position: position({ + state: { + ...initialCodexScanState(), + model: "gpt-6-astra", + lastCumulativeUsage: request, + latestCumulativeAtMs: 1_786_000_000_000, + }, + }), + }; + const current = parse(makeScanCacheWriter()(new Map([["/codex.jsonl", entry]]), {})); + + expect( + decodeScanCache({ ...current, version: 15 }, TEST_FORMAT_MAP).get("/codex.jsonl"), + ).toEqual({ + ...entry, + position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null }, + }); + expect(decodeScanCache(current, TEST_FORMAT_MAP).get("/codex.jsonl")).toEqual(entry); }); it("re-parses a Claude transcript cached before 1-hour cache writes were counted", () => { @@ -299,6 +339,29 @@ describe("scan cache round trip", () => { }); }); + it.each(["s:p:grok-4.5", "s:p:1786000000000:grok-4.5"])( + "re-parses v16 Grok transcripts with key %s", + (dedupeKey) => { + const original = cacheWith([ + ["/grok/updates.jsonl", 100, [record({ provider: "grok", model: "grok-4.5", dedupeKey })]], + ]); + original.set("/grok/updates.jsonl", { + ...original.get("/grok/updates.jsonl")!, + provider: "grok", + }); + const encoded = parse(makeScanCacheWriter()(original, {})); + const restored = decodeScanCache({ ...encoded, version: 16 }, TEST_FORMAT_MAP); + expect(restored.get("/grok/updates.jsonl")).toEqual({ + ...original.get("/grok/updates.jsonl"), + size: -1, + position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null }, + records: [ + record({ provider: "grok", model: "grok-4.5", dedupeKey: "s:p:1786000000000:grok-4.5" }), + ], + }); + }, + ); + it("rewrites only changed entries and still restores the whole cache", () => { const write = makeScanCacheWriter(); const cache = cacheWith([ diff --git a/apps/server/src/usage/usageScanCache.ts b/apps/server/src/usage/usageScanCache.ts index b571dd3708f63..843b414b97f32 100644 --- a/apps/server/src/usage/usageScanCache.ts +++ b/apps/server/src/usage/usageScanCache.ts @@ -10,7 +10,7 @@ import { CLAUDE_SYNTHETIC_MODEL } from "../provider/Drivers/claudeUsage.ts"; import { mergeUsageCopies } from "./usageAggregation.ts"; import { GUARD_LENGTH, type TranscriptParsePosition } from "./usageTranscriptReader.ts"; -const USAGE_SCAN_CACHE_VERSION = 15 as const; +const USAGE_SCAN_CACHE_VERSION = 17 as const; const SPEED_COMPATIBLE_SINCE_VERSION = 4; /** @@ -18,8 +18,10 @@ const SPEED_COMPATIBLE_SINCE_VERSION = 4; * replace it, dropping saved usage for deleted transcripts. Separate files keep * the newer one; older files are removed once it is saved. */ -export const SCAN_CACHE_FILE_NAME = "usage-scan-cache-v15.json"; +export const SCAN_CACHE_FILE_NAME = "usage-scan-cache-v17.json"; export const LEGACY_SCAN_CACHE_FILE_NAMES = [ + "usage-scan-cache-v16.json", + "usage-scan-cache-v15.json", "usage-scan-cache-v14.json", "usage-scan-cache-v13.json", "usage-scan-cache-v12.json", @@ -263,7 +265,16 @@ export function decodeScanCache( ...(version >= 7 && unresolvedServiceTierTurnId != null ? { unresolvedServiceTierTurnId } : {}), - dedupeKey: typeof dedupeKey === "string" ? dedupeKey : null, + dedupeKey: + typeof dedupeKey !== "string" + ? null + : // Saved copies of moved transcripts must still match the re-parsed v17 key. + provider === "grok" && + version < 17 && + dedupeKey.endsWith(`:${model}`) && + !dedupeKey.endsWith(`:${timestampMs}:${model}`) + ? `${dedupeKey.slice(0, -model.length)}${timestampMs}:${model}` + : dedupeKey, }); } return records; @@ -300,17 +311,20 @@ export function decodeScanCache( // Keep history for deleted rollouts, but re-parse incompatible reducer state. // v13 adds Claude 1-hour cache writes; v14 uses Codex request ledgers. // v15 repairs skipped Codex requests whose counters rewound; clean v14 entries stay served. + // v17 keys Grok turns by time, restoring turns previously dropped as repeats. const legacy = (entry.p === "claude" && version < 13) || + (entry.p === "grok" && version < 17) || (format.state !== undefined && (version < 6 || (entry.p === "codex" && (version < 14 || (version < 15 && malformedRecords > 0))))); // A corrupt state disqualifies the entry: resuming with it would attach // appended usage to the wrong model or replay fork-copied history. if (!legacy && !isValidState.get(entry.p)?.(entry.cs)) continue; - // Before v15 the state lacks the latest snapshot time that keeps a re-emitted older snapshot - // from billing twice, so a grown file re-parses whole instead of resuming. - const resumable = !legacy && !(entry.p === "codex" && version < 15); + // Before v16 a Codex baseline can lack the time of its latest move, or carry an older one, + // and that time keeps a re-emitted older snapshot from billing twice. A grown file re-parses + // whole instead of resuming. + const resumable = !legacy && !(entry.p === "codex" && version < 16); const provider = entry.p; const records = decodeRecords(entry.r, provider); diff --git a/apps/server/src/usage/usageTranscriptReader.test.ts b/apps/server/src/usage/usageTranscriptReader.test.ts index f4cb72ec1e25c..ca994dd3ac5b2 100644 --- a/apps/server/src/usage/usageTranscriptReader.test.ts +++ b/apps/server/src/usage/usageTranscriptReader.test.ts @@ -13,6 +13,8 @@ import * as Schema from "effect/Schema"; import { TEST_FORMATS } from "./usageTestFormats.ts"; import { listTranscriptFiles, + physicalSourceIdOf, + readPhysicalSourceId, readTranscriptRecords, transcriptLineGate, } from "./usageTranscriptReader.ts"; @@ -577,3 +579,49 @@ describe("transcriptLineGate", () => { } }); }); + +describe("readPhysicalSourceId", () => { + const scope = "0123456789abcdef0123456789abcdef"; + const windowsPath = "C:\\Users\\User\\.claude\\projects"; + + // One NTFS directory the Windows server reads natively and the WSL server + // through drvfs. The temp dir stands in for the drive's mount point. + async function drvfsFixture(extraMounts: (escapedMountPoint: string) => string = () => "") { + const mountPoint = NodePath.join(dir, "drive c"); + await NodeFSP.mkdir(NodePath.join(mountPoint, "Users", "User", ".claude", "projects"), { + recursive: true, + }); + const escaped = mountPoint.replaceAll(" ", "\\040"); + const mountInfoPath = NodePath.join(dir, "mountinfo"); + await NodeFSP.writeFile( + mountInfoPath, + [ + "24 1 8:32 / / rw,relatime shared:1 - ext4 /dev/sdc rw,discard", + `81 24 0:61 / ${escaped} rw,noatime shared:2 - 9p C:\\134 rw,dirsync,aname=drvfs;path=C:\\;uid=1000;gid=1000;symlinkroot=/mnt/,mmap,access=client,msize=65536,trans=fd`, + extraMounts(escaped), + ].join("\n"), + ); + return { mountPoint, mountInfoPath }; + } + + it("names a drvfs directory as the Windows server names it, whatever its typed case", async () => { + const { mountPoint, mountInfoPath } = await drvfsFixture(); + const wslId = await readPhysicalSourceId( + `${mountPoint}/users/user/.claude/projects`, + scope, + "linux", + mountInfoPath, + ); + assert.strictEqual(wslId, physicalSourceIdOf(scope, windowsPath)); + }); + + it("gives no identity to native or shadowing Linux mounts, or without a desktop scope", async () => { + const { mountPoint, mountInfoPath } = await drvfsFixture( + (escaped) => `90 81 8:48 / ${escaped}/Users rw,relatime - ext4 /dev/sdd rw`, + ); + const projects = `${mountPoint}/Users/User/.claude/projects`; + assert.isUndefined(await readPhysicalSourceId(projects, scope, "linux", mountInfoPath)); + assert.isUndefined(await readPhysicalSourceId(dir, scope, "linux", mountInfoPath)); + assert.isUndefined(await readPhysicalSourceId(projects, undefined, "linux", mountInfoPath)); + }); +}); diff --git a/apps/server/src/usage/usageTranscriptReader.ts b/apps/server/src/usage/usageTranscriptReader.ts index d80c51964a20c..5b92f48356aa4 100644 --- a/apps/server/src/usage/usageTranscriptReader.ts +++ b/apps/server/src/usage/usageTranscriptReader.ts @@ -1,5 +1,6 @@ // @effect-diagnostics nodeBuiltinImport:off // Cold windows span ~1.4 GB/1,500 files; node:fs buffer streaming is ~10x cheaper than materializing files. +import * as NodeCrypto from "node:crypto"; import * as NodeFS from "node:fs"; import * as NodeFSP from "node:fs/promises"; import * as NodePath from "node:path"; @@ -43,6 +44,7 @@ export interface TranscriptParseResult { export const GUARD_LENGTH = 64; const STREAMING_THRESHOLD_BYTES = 8 * 1024 * 1024; +const READ_CHUNK_BYTES = 256 * 1024; const NEWLINE = 0x0a; const STAT_CONCURRENCY = 32; const WALK_CONCURRENCY = 16; @@ -149,6 +151,137 @@ export async function readDirectoryVolumeId(path: string): Promise { } } +/** + * Set by the desktop app, from a value it persists, on its Windows server and + * the WSL server it manages, so both can name one Windows directory alike + * without merging separate machines that share a hostname and path. + */ +export const DESKTOP_HOST_SCOPE_ENV = "T3CODE_DESKTOP_HOST_SCOPE"; + +const MOUNT_FIELD_ESCAPE = /\\([0-7]{3})/g; +const WINDOWS_ROOT = /^(?:[A-Za-z]:|\\\\[^\\]+\\[^\\]+)\\?$/; + +const unescapeMountField = (field: string) => + field.replace(MOUNT_FIELD_ESCAPE, (_, octal: string) => + String.fromCharCode(Number.parseInt(octal, 8)), + ); + +const normalizeWindowsPath = (path: string) => + (/^[a-z]:/.test(path) ? path[0]!.toUpperCase() + path.slice(1) : path).replace(/\\$/, ""); + +/** + * The Windows directory behind the deepest mount holding `dir`, read from + * `/proc/self/mountinfo` so a custom automount root resolves and an ext4 + * mount under `/mnt` never passes for a drive. + */ +function windowsMountFor( + mountInfo: string, + dir: string, +): { readonly mountPoint: string; readonly windowsRoot: string } | undefined { + let deepest: { mountPoint: string; windowsRoot: string | undefined } | undefined; + for (const line of mountInfo.split("\n")) { + const fields = line.split(" "); + const separator = fields.indexOf("-", 6); + if (separator === -1) continue; + const mountPoint = unescapeMountField(fields[4] ?? ""); + const contains = + mountPoint === "/" || + dir === mountPoint || + dir.startsWith(mountPoint.endsWith("/") ? mountPoint : `${mountPoint}/`); + if (!contains || (deepest !== undefined && deepest.mountPoint.length > mountPoint.length)) { + continue; + } + // WSL2 mounts drives over 9p with `aname=drvfs;path=C:\`; WSL1 as `drvfs`. + const options = (fields[separator + 3] ?? "").split(/[,;]/); + const source = options.includes("aname=drvfs") + ? options.find((option) => option.startsWith("path="))?.slice("path=".length) + : fields[separator + 1] === "drvfs" + ? unescapeMountField(fields[separator + 2] ?? "") + : undefined; + const root = unescapeMountField(fields[3] ?? "/") + .split("/") + .filter(Boolean); + deepest = { + mountPoint, + windowsRoot: + source !== undefined && WINDOWS_ROOT.test(source) + ? [normalizeWindowsPath(source), ...root].join("\\") + : undefined, + }; + } + return deepest?.windowsRoot === undefined + ? undefined + : { mountPoint: deepest.mountPoint, windowsRoot: deepest.windowsRoot }; +} + +/** + * drvfs keeps the casing a path was typed with, while Windows reports the + * on-disk names. Read those names rather than folding case, which would join + * distinct entries of a case-sensitive directory. + */ +async function onDiskNames( + mountPoint: string, + parts: readonly string[], +): Promise { + const names: string[] = []; + let current = mountPoint; + for (const part of parts) { + let entries: string[]; + try { + entries = await NodeFSP.readdir(current); + } catch { + return undefined; + } + const folded = part.toUpperCase(); + const [name, ...others] = entries.includes(part) + ? [part] + : entries.filter((entry) => entry.toUpperCase() === folded); + if (name === undefined || others.length > 0) return undefined; + names.push(name); + current = NodePath.posix.join(current, name); + } + return names; +} + +export function physicalSourceIdOf(scope: string, windowsPath: string): string { + return NodeCrypto.createHash("sha256") + .update(`${scope}\0${windowsPath}`) + .digest("hex") + .slice(0, 32); +} + +/** + * Identifies a Windows directory the same way from the Windows server and from + * a WSL server reading it through drvfs. Directories that are not Windows + * backed, and servers the desktop app did not start, get none. + */ +export async function readPhysicalSourceId( + dir: string, + scope: string | undefined, + platform: NodeJS.Platform, + mountInfoPath = "/proc/self/mountinfo", +): Promise { + if (scope === undefined || scope.length === 0) return undefined; + try { + if (platform === "win32") { + // The native realpath reports on-disk casing; the JS one keeps the input's. + return physicalSourceIdOf(scope, normalizeWindowsPath(await NodeFSP.realpath(dir))); + } + if (platform !== "linux") return undefined; + const mount = windowsMountFor(await NodeFSP.readFile(mountInfoPath, "utf8"), dir); + if (mount === undefined) return undefined; + const names = await onDiskNames( + mount.mountPoint, + dir.slice(mount.mountPoint.length).split("/").filter(Boolean), + ); + return names === undefined + ? undefined + : physicalSourceIdOf(scope, [mount.windowsRoot, ...names].join("\\")); + } catch { + return undefined; + } +} + async function guardMatches( handle: NodeFSP.FileHandle, position: TranscriptParsePosition, @@ -168,6 +301,29 @@ async function guardMatches( } } +/** Plain files reuse one buffer, so a chunk is only valid until the next one is requested. */ +async function* readChunks( + handle: NodeFSP.FileHandle, + start: number, + compressed: boolean, +): AsyncGenerator { + if (compressed) { + // Compressed snapshots cache a full parse; decoded byte offsets cannot + // resume the compressed input. + yield* handle + .createReadStream({ start, autoClose: true, highWaterMark: READ_CHUNK_BYTES }) + .compose(NodeZlib.createZstdDecompress()); + return; + } + const buffer = Buffer.allocUnsafe(READ_CHUNK_BYTES); + for (let position = start; ;) { + const { bytesRead } = await handle.read(buffer, 0, buffer.length, position); + if (bytesRead === 0) return; + position += bytesRead; + yield buffer.subarray(0, bytesRead); + } +} + // Match markers in one latin1 alternation; one character per byte preserves exact ASCII offsets. export function transcriptLineGate(markers: readonly [string, ...string[]]) { if (markers.length === 1) { @@ -244,7 +400,8 @@ export async function readTranscriptRecords( const append = (segment: Buffer) => { if (!streaming && pendingBytes + segment.length <= streamingThresholdBytes) { - if (segment.length > 0) pendingChunks.push(segment); + // Copy: the next read overwrites the chunk this segment points into. + if (segment.length > 0) pendingChunks.push(Buffer.from(segment)); pendingBytes += segment.length; return; } @@ -275,15 +432,7 @@ export async function readTranscriptRecords( streaming = undefined; decoder = undefined; }; - const input = handle.createReadStream({ - start, - autoClose: compressed, - highWaterMark: 256 * 1024, - }); - // Compressed snapshots cache a full parse; decoded byte offsets cannot - // resume the compressed input. - const stream = compressed ? input.compose(NodeZlib.createZstdDecompress()) : input; - for await (const chunk of stream) { + for await (const chunk of readChunks(handle, start, compressed)) { let lineStart = 0; while (lineStart < chunk.length) { const newlineIndex = chunk.indexOf(NEWLINE, lineStart); diff --git a/apps/server/src/usage/usageTranscriptStreaming.test.ts b/apps/server/src/usage/usageTranscriptStreaming.test.ts index 856ba2ac5b9d7..635b5ae4e6bec 100644 --- a/apps/server/src/usage/usageTranscriptStreaming.test.ts +++ b/apps/server/src/usage/usageTranscriptStreaming.test.ts @@ -254,7 +254,7 @@ describe("large usage records", () => { outputTokens: 99, reasoningTokens: 10, }, - dedupeKey: "s1:p1:grok-4.5-build", + dedupeKey: "s1:p1:1785578400123:grok-4.5-build", reportedCostUsd: 0.25, }); }); diff --git a/apps/server/src/utils/subscribeBeforeSnapshot.test.ts b/apps/server/src/utils/subscribeBeforeSnapshot.test.ts index 70f0beda25ce6..e37b969a123dc 100644 --- a/apps/server/src/utils/subscribeBeforeSnapshot.test.ts +++ b/apps/server/src/utils/subscribeBeforeSnapshot.test.ts @@ -15,70 +15,66 @@ import { describe("subscribeBeforeSnapshot", () => { it.effect("atomically reads the initial snapshot before receiving later changes", () => - Effect.scoped( - Effect.gen(function* () { - const changes = yield* PubSub.sliding(1); - const latest = yield* Ref.make(1); - const mutex = yield* Semaphore.make(1); - const snapshotStarted = yield* Deferred.make(); - const finishSnapshot = yield* Deferred.make(); - const subscriptionFiber = yield* subscribeBeforeSnapshot( - changes, - Deferred.succeed(snapshotStarted, undefined).pipe( - Effect.andThen(Deferred.await(finishSnapshot)), - Effect.andThen(Ref.get(latest)), - ), - mutex, - ).pipe(Effect.forkChild); + Effect.gen(function* () { + const changes = yield* PubSub.sliding(1); + const latest = yield* Ref.make(1); + const mutex = yield* Semaphore.make(1); + const snapshotStarted = yield* Deferred.make(); + const finishSnapshot = yield* Deferred.make(); + const subscriptionFiber = yield* subscribeBeforeSnapshot( + changes, + Deferred.succeed(snapshotStarted, undefined).pipe( + Effect.andThen(Deferred.await(finishSnapshot)), + Effect.andThen(Ref.get(latest)), + ), + mutex, + ).pipe(Effect.forkChild); - yield* Deferred.await(snapshotStarted); - const publishFiber = yield* mutex - .withPermits(1)(Ref.set(latest, 2).pipe(Effect.andThen(PubSub.publish(changes, 2)))) - .pipe(Effect.forkChild); - yield* Deferred.succeed(finishSnapshot, undefined); + yield* Deferred.await(snapshotStarted); + const publishFiber = yield* mutex + .withPermits(1)(Ref.set(latest, 2).pipe(Effect.andThen(PubSub.publish(changes, 2)))) + .pipe(Effect.forkChild); + yield* Deferred.succeed(finishSnapshot, undefined); - const subscription = yield* Fiber.join(subscriptionFiber); - yield* Fiber.join(publishFiber); - const firstChange = yield* subscription.changes.pipe( - Stream.runHead, - Effect.timeout("1 second"), - ); + const subscription = yield* Fiber.join(subscriptionFiber); + yield* Fiber.join(publishFiber); + const firstChange = yield* subscription.changes.pipe( + Stream.runHead, + Effect.timeout("1 second"), + ); - expect(subscription.latest).toBe(1); - expect(firstChange).toEqual(Option.some(2)); - }), - ), + expect(subscription.latest).toBe(1); + expect(firstChange).toEqual(Option.some(2)); + }), ); it.effect("subscribes before an uncoordinated snapshot can change", () => - Effect.scoped( - Effect.gen(function* () { - const changes = yield* PubSub.sliding(1); - const latest = yield* Ref.make(1); - const snapshotStarted = yield* Deferred.make(); - const finishSnapshot = yield* Deferred.make(); - const subscriptionFiber = yield* subscribeBeforeSnapshotWithoutMutex( - changes, - Deferred.succeed(snapshotStarted, undefined).pipe( - Effect.andThen(Deferred.await(finishSnapshot)), - Effect.andThen(Ref.get(latest)), - ), - ).pipe(Effect.forkChild); + Effect.gen(function* () { + const changes = yield* PubSub.sliding(1); + const latest = yield* Ref.make(1); + const snapshotStarted = yield* Deferred.make(); + const finishSnapshot = yield* Deferred.make(); + const subscriptionFiber = yield* subscribeBeforeSnapshotWithoutMutex( + changes, + Deferred.succeed(snapshotStarted, undefined).pipe( + Effect.andThen(Deferred.await(finishSnapshot)), + Effect.andThen(Ref.get(latest)), + ), + ).pipe(Effect.forkChild); - yield* Deferred.await(snapshotStarted); - yield* Ref.set(latest, 2); - yield* PubSub.publish(changes, 2); - yield* Deferred.succeed(finishSnapshot, undefined); + yield* Deferred.await(snapshotStarted); + yield* Ref.set(latest, 2); + yield* PubSub.publish(changes, 2); + yield* Deferred.succeed(finishSnapshot, undefined); - const subscription = yield* Fiber.join(subscriptionFiber); - const firstChange = yield* subscription.changes.pipe( - Stream.runHead, - Effect.timeout("1 second"), - ); + const subscription = yield* Fiber.join(subscriptionFiber); + const firstChange = yield* subscription.changes.pipe( + Stream.runHead, + Effect.timeout("1 second"), + ); - expect(subscription.latest).toBe(2); - expect(firstChange).toEqual(Option.some(2)); - }), - ), + expect(subscription.latest).toBe(2); + expect(firstChange).toEqual(Option.some(2)); + }), ); }); diff --git a/apps/server/src/vcs/GitVcsDriver.test.ts b/apps/server/src/vcs/GitVcsDriver.test.ts index 89580e0937af7..1acbf036fb52c 100644 --- a/apps/server/src/vcs/GitVcsDriver.test.ts +++ b/apps/server/src/vcs/GitVcsDriver.test.ts @@ -1,3 +1,5 @@ +import * as NodeNet from "node:net"; +import * as SecureRandom from "effect/crypto/SecureRandom"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Cause from "effect/Cause"; @@ -173,6 +175,392 @@ const makeCheckpointFixture = Effect.fn("makeCheckpointFixture")(function* ( return { git, checkpointRef }; }); +it.effect.each(["interrupt", "timeout", "success"] as const)( + "checkpoint bulk staging cleans operation-owned packs after $0 and can retry", + (stop) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const liveProcess = yield* VcsProcess.VcsProcess; + const sandbox = yield* fs.makeTempDirectoryScoped({ prefix: "t3-bulk-checkpoint-" }); + const cwd = path.join(sandbox, "repo"); + yield* fs.makeDirectory(cwd); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + const originalCheckpoint = (yield* git(["rev-parse", checkpointRef])).stdout; + const originalIndex = yield* fs.readFile(path.join(cwd, ".git/index")); + const originalHead = (yield* git(["rev-parse", "HEAD"])).stdout; + yield* git(["config", "core.bigFileThreshold", "1m"]); + const crypto = yield* SecureRandom.SecureRandom; + const payload = yield* crypto.randomBytes(2 * 1024 * 1024); + yield* fs.writeFile(path.join(cwd, "a-media.bin"), payload); + + // Git keeps bulk-checkin's pack open while the next path's clean filter runs. + // The socket is a barrier, not a race against the speed of compression. + const connected = Promise.withResolvers(); + const sockets = new Set(); + let released = false; + const server = yield* Effect.acquireRelease( + Effect.sync(() => + NodeNet.createServer((socket) => { + sockets.add(socket); + connected.resolve(socket); + // Git can run the filter again while refreshing the staged entry. + if (released) socket.end(); + }), + ), + (server) => + Effect.sync(() => { + for (const socket of sockets) socket.destroy(); + server.close(); + }), + ); + yield* Effect.promise( + () => + new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }), + ); + const address = server.address(); + assert.isTrue(address !== null && typeof address !== "string"); + if (address === null || typeof address === "string") return; + const filter = path.join(sandbox, "filter.cjs"); + yield* fs.writeFileString( + filter, + `const s = require('node:net').connect(${address.port}, '127.0.0.1'); s.resume(); process.stdin.resume(); s.on('end', () => process.exit(0));`, + ); + yield* git(["config", "filter.barrier.clean", `"${process.execPath}" "${filter}"`]); + yield* git(["config", "filter.barrier.required", "true"]); + yield* fs.writeFileString(path.join(cwd, ".gitattributes"), "z-gate filter=barrier\n"); + yield* fs.writeFileString(path.join(cwd, "z-gate"), "gate\n"); + const sharedPack = path.join(cwd, ".git/objects/pack/tmp_pack_other_operation"); + yield* fs.writeFileString(sharedPack, "owned by another Git operation"); + let objectDirectory = path.join(cwd, ".git/objects"); + const captureDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe( + Effect.provideService(VcsProcess.VcsProcess, { + run: (input) => { + if (input.args.includes("add") && input.env?.GIT_OBJECT_DIRECTORY) + objectDirectory = input.env.GIT_OBJECT_DIRECTORY; + return liveProcess.run(input); + }, + }), + ); + const fiber = yield* captureDriver.checkpoints + .captureCheckpoint({ cwd, checkpointRef }) + .pipe(Effect.exit, Effect.forkScoped); + const socket = yield* Effect.promise(() => connected.promise).pipe( + Effect.raceFirst( + Fiber.join(fiber).pipe( + Effect.flatMap((result) => + Exit.isFailure(result) + ? Effect.failCause(result.cause) + : Effect.die("capture ended before the filter connected"), + ), + ), + ), + ); + const packs = (yield* fs.readDirectory(path.join(objectDirectory, "pack"))).filter( + (name) => name.startsWith("tmp_pack_") && name !== "tmp_pack_other_operation", + ); + assert.strictEqual(packs.length, 1); + assert.isAbove( + Number((yield* fs.stat(path.join(objectDirectory, "pack", packs[0]!))).size), + 1024 * 1024, + ); + // An overlapping capture publishes the same large blob while the first + // still owns an unfinished pack. Its objects/ref must survive either exit. + const concurrentRef = CheckpointRef.make("refs/t3/checkpoints/concurrent"); + const concurrentDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe( + Effect.provideService(VcsProcess.VcsProcess, { + run: (input) => + liveProcess.run({ ...input, args: ["-c", "filter.barrier.clean=cat", ...input.args] }), + }), + ); + yield* concurrentDriver.checkpoints.captureCheckpoint({ cwd, checkpointRef: concurrentRef }); + const concurrentOid = (yield* git(["rev-parse", concurrentRef])).stdout; + const sharedPackNames = (yield* fs.readDirectory(path.join(cwd, ".git/objects/pack"))).sort(); + const packName = sharedPackNames.find((name) => name.endsWith(".pack")); + assert.isDefined(packName); + const sharedPackBytes = yield* fs.readFile(path.join(cwd, ".git/objects/pack", packName!)); + if (stop === "interrupt") yield* Fiber.interrupt(fiber); + else if (stop === "timeout") yield* TestClock.adjust("30 seconds"); + released = true; + socket.end(); + const result = yield* Fiber.await(fiber); + if (stop === "interrupt") + assert.isTrue(Exit.isFailure(result) && Cause.hasInterruptsOnly(result.cause)); + else if (stop === "timeout") { + assert.isTrue(Exit.isSuccess(result)); + if (Exit.isSuccess(result)) { + assert.isTrue(Exit.isFailure(result.value)); + if (Exit.isFailure(result.value)) { + const error = Cause.findErrorOption(result.value.cause); + assert.isTrue(Option.isSome(error) && error.value._tag === "VcsProcessTimeoutError"); + } + } + } else assert.isTrue(Exit.isSuccess(result) && Exit.isSuccess(result.value)); + assert.deepEqual( + (yield* fs.readDirectory(path.join(cwd, ".git/objects/pack"))).sort(), + sharedPackNames, + ); + assert.deepEqual( + yield* fs.readFile(path.join(cwd, ".git/objects/pack", packName!)), + sharedPackBytes, + ); + assert.strictEqual((yield* git(["rev-parse", concurrentRef])).stdout, concurrentOid); + assert.isFalse(yield* fs.exists(path.join(objectDirectory, "pack", packs[0]!))); + assert.isFalse(yield* fs.exists(objectDirectory)); + assert.strictEqual(yield* fs.readFileString(sharedPack), "owned by another Git operation"); + if (stop !== "success") + assert.strictEqual((yield* git(["rev-parse", checkpointRef])).stdout, originalCheckpoint); + assert.deepEqual(yield* fs.readFile(path.join(cwd, ".git/index")), originalIndex); + assert.strictEqual((yield* git(["rev-parse", "HEAD"])).stdout, originalHead); + assert.deepEqual(yield* fs.readFile(path.join(cwd, "a-media.bin")), payload); + + yield* git(["config", "filter.barrier.clean", "cat"]); + yield* captureDriver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + assert.isFalse(yield* fs.exists(objectDirectory)); + const blob = (yield* git(["hash-object", "a-media.bin"])).stdout.trim(); + assert.strictEqual( + (yield* git(["rev-parse", `${checkpointRef}:a-media.bin`])).stdout.trim(), + blob, + ); + yield* fs.remove(sharedPack); + yield* git(["fsck", "--full"]); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect.each(["failure", "interrupt"] as const)( + "checkpoint publication $0 preserves existing refs and permits retry", + (stop) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-checkpoint-publish-" }); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + const previous = (yield* git(["show-ref"])).stdout; + yield* fs.writeFileString(path.join(cwd, "new.txt"), "new checkpoint content\n"); + const published = yield* Deferred.make(); + const captureDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe( + Effect.provideService(FileSystem.FileSystem, { + ...fs, + link: (source, destination) => + fs.link(source, destination).pipe( + Effect.andThen(Deferred.succeed(published, undefined)), + Effect.andThen( + stop === "interrupt" + ? Effect.never + : Effect.fail( + PlatformError.systemError({ + _tag: "BadResource", + module: "FileSystem", + method: "link", + }), + ), + ), + ), + }), + ); + const fiber = yield* captureDriver.checkpoints + .captureCheckpoint({ cwd, checkpointRef }) + .pipe(Effect.exit, Effect.forkScoped); + yield* Deferred.await(published); + if (stop === "interrupt") yield* Fiber.interrupt(fiber); + const result = yield* Fiber.await(fiber); + if (stop === "interrupt") + assert.isTrue(Exit.isFailure(result) && Cause.hasInterruptsOnly(result.cause)); + else { + assert.isTrue(Exit.isSuccess(result) && Exit.isFailure(result.value)); + if (Exit.isSuccess(result) && Exit.isFailure(result.value)) { + const error = Cause.findErrorOption(result.value.cause); + assert.isTrue( + Option.isSome(error) && + error.value._tag === "VcsCheckpointStorageError" && + error.value.stage === "publish", + ); + } + } + assert.strictEqual((yield* git(["show-ref"])).stdout, previous); + assert.deepEqual( + (yield* fs.readDirectory(path.join(cwd, ".git/objects"))).filter((name) => + name.startsWith("tmp_objdir-t3-checkpoint-"), + ), + [], + ); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + assert.strictEqual( + (yield* git(["show", `${checkpointRef}:new.txt`])).stdout, + "new checkpoint content\n", + ); + yield* git(["fsck", "--full"]); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect.each([ + { stop: "before git restore spawns", gate: "restore", index: "staged", file: "later\n" }, + { stop: "after git restore ran", gate: "clean", index: "head", file: "unstaged\n" }, +] as const)( + "checkpoint restore interrupted $stop leaves the $index index", + ({ gate, index, file }) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const liveProcess = yield* VcsProcess.VcsProcess; + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-restore-interrupt-" }); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + yield* fs.writeFileString(path.join(cwd, "file.txt"), "later\n"); + yield* git(["add", "file.txt"]); + const stagedTree = (yield* git(["write-tree"])).stdout; + const entered = yield* Deferred.make(); + const gatedDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe( + Effect.provideService(VcsProcess.VcsProcess, { + run: (input) => + input.operation === "GitVcsDriver.checkpoints.restoreCheckpoint" && + input.args.includes(gate) + ? Deferred.succeed(entered, undefined).pipe(Effect.andThen(Effect.never)) + : liveProcess.run(input), + }), + ); + const fiber = yield* gatedDriver.checkpoints + .restoreCheckpoint({ cwd, checkpointRef }) + .pipe(Effect.exit, Effect.forkScoped); + yield* Deferred.await(entered); + yield* Fiber.interrupt(fiber); + const result = yield* Fiber.await(fiber); + assert.isTrue(Exit.isFailure(result) && Cause.hasInterruptsOnly(result.cause)); + + assert.strictEqual( + (yield* git(["write-tree"])).stdout, + index === "head" ? (yield* git(["rev-parse", "HEAD^{tree}"])).stdout : stagedTree, + ); + assert.strictEqual(yield* fs.readFileString(path.join(cwd, "file.txt")), file); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect("checkpoint publication falls back to rename where hardlinks are unavailable", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* SecureRandom.SecureRandom; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-publish-no-links-" }); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + yield* git(["config", "core.bigFileThreshold", "1m"]); + yield* fs.writeFile(path.join(cwd, "media.bin"), yield* crypto.randomBytes(2 * 1024 * 1024)); + const captureDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe( + Effect.provideService(FileSystem.FileSystem, { + ...fs, + link: () => + Effect.fail( + PlatformError.systemError({ + _tag: "PermissionDenied", + module: "FileSystem", + method: "link", + }), + ), + }), + ); + yield* captureDriver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + const files = yield* fs.readDirectory(path.join(cwd, ".git/objects/pack")); + assert.isTrue(files.some((name) => name.endsWith(".pack"))); + assert.isTrue(yield* driver.checkpoints.hasCheckpointRef({ cwd, checkpointRef })); + assert.match((yield* git(["count-objects", "-v"])).stdout, /^garbage: 0$/m); + yield* git(["fsck", "--full"]); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( + "checkpoint publication preserves Git's shared object directory permissions", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-publish-shared-" }); + const git = (args: ReadonlyArray) => + driver.execute({ operation: "test.shared", cwd, args }); + yield* git(["init", "--shared=0660"]); + const checkpointRef = CheckpointRef.make("refs/t3/checkpoints/shared"); + yield* fs.writeFileString(path.join(cwd, "shared.txt"), "new shared object\n"); + const blob = (yield* git(["hash-object", "-w", "shared.txt"])).stdout.trim(); + const directory = path.join(cwd, ".git/objects", blob.slice(0, 2)); + const expectedMode = (yield* fs.stat(directory)).mode & 0o7777; + yield* fs.remove(directory, { recursive: true }); + assert.isFalse(yield* fs.exists(directory)); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + assert.strictEqual((yield* fs.stat(directory)).mode & 0o7777, expectedMode); + yield* git(["fsck", "--full"]); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect("checkpoint capture ignores an inherited object directory", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const sandbox = yield* fs.makeTempDirectoryScoped({ prefix: "t3-inherited-objects-" }); + const cwd = path.join(sandbox, "repo"); + yield* fs.makeDirectory(cwd); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + const unrelated = path.join(sandbox, "unrelated"); + yield* git(["init", "--bare", unrelated]); + const unrelatedObjects = path.join(unrelated, "objects"); + // VcsProcess clears inherited selectors from git children; only the driver sees this one. + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }).pipe( + Effect.provideService(HostProcess.Environment, { + ...(yield* HostProcess.Environment), + GIT_OBJECT_DIRECTORY: unrelatedObjects, + }), + ); + assert.strictEqual((yield* git(["show", `${checkpointRef}:file.txt`])).stdout, "unstaged\n"); + assert.deepEqual( + (yield* fs.readDirectory(unrelatedObjects)).filter( + (name) => name !== "info" && name !== "pack", + ), + [], + ); + yield* git(["fsck", "--full"]); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect.each(["sha1", "sha256"] as const)( + "checkpoint object isolation supports linked worktrees with $0 objects", + (format) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const sandbox = yield* fs.makeTempDirectoryScoped({ prefix: "t3-checkpoint-worktree-" }); + // A separator must remain part of the alternate's path, not split the list. + const platform = yield* HostProcess.Platform; + const cwd = path.join(sandbox, platform === "win32" ? "repo;space" : 'repo:space"\u0001'); + yield* fs.makeDirectory(cwd); + yield* driver.execute({ + operation: "test.init", + cwd, + args: ["init", `--object-format=${format}`], + }); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + const worktree = path.join(sandbox, "linked"); + yield* git(["worktree", "add", "-b", "linked", worktree]); + yield* fs.writeFileString(path.join(worktree, "untracked.txt"), "linked worktree\n"); + const originalIndex = yield* fs.readFile(path.join(cwd, ".git/index")); + yield* driver.checkpoints.captureCheckpoint({ cwd: worktree, checkpointRef }); + assert.strictEqual( + (yield* git(["show", `${checkpointRef}:untracked.txt`])).stdout, + "linked worktree\n", + ); + assert.deepEqual(yield* fs.readFile(path.join(cwd, ".git/index")), originalIndex); + yield* git(["fsck", "--full"]); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + it.effect("checkpoint capture skips untracked nested repositories without a commit", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; @@ -861,6 +1249,63 @@ it.effect.each([1_700_000_000, 1_700_000_000.9999])( }).pipe(Effect.scoped, Effect.provide(layerGitContract)), ); +it.effect("a branch switch landing after the snapshot belongs to the next checkpoint", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const liveProcess = yield* VcsProcess.VcsProcess; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-checkpoint-switch-race-" }); + const git = (args: ReadonlyArray) => + driver.execute({ operation: "checkpoint-test", cwd, args }); + yield* git(["init"]); + yield* git(["config", "user.name", "Test"]); + yield* git(["config", "user.email", "test@test.com"]); + yield* fileSystem.writeFileString(path.join(cwd, "f.txt"), "A\n"); + yield* git(["add", "."]); + yield* git(["commit", "-m", "a"]); + yield* git(["checkout", "-b", "b"]); + yield* fileSystem.writeFileString(path.join(cwd, "f.txt"), "B\n"); + yield* git(["commit", "-am", "b"]); + yield* git(["checkout", "-"]); + const [first, second, third] = ["1", "2", "3"].map((turn) => + CheckpointRef.make(`refs/t3/checkpoints/switch-race/${turn}`), + ) as [CheckpointRef, CheckpointRef, CheckpointRef]; + const racedDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe( + Effect.provideService(VcsProcess.VcsProcess, { + run: Effect.fn(function* (input: VcsProcess.VcsProcessInput) { + const result = yield* liveProcess.run(input); + if (input.args.includes("write-tree")) yield* git(["checkout", "b"]).pipe(Effect.orDie); + return result; + }), + }), + ); + const changedSince = (fromCheckpointRef: CheckpointRef, toCheckpointRef: CheckpointRef) => + Effect.gen(function* () { + const refs = { cwd, fromCheckpointRef, toCheckpointRef }; + return { + patch: yield* driver.checkpoints.diffCheckpoints({ ...refs, ignoreWhitespace: false }), + authored: yield* driver.checkpoints.listAuthoredPaths(refs), + }; + }); + + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef: first }); + yield* racedDriver.checkpoints.captureCheckpoint({ + cwd, + checkpointRef: second, + previousCheckpointRef: first, + }); + yield* driver.checkpoints.captureCheckpoint({ + cwd, + checkpointRef: third, + previousCheckpointRef: second, + }); + + assert.deepEqual(yield* changedSince(first, second), { patch: "", authored: null }); + assert.deepEqual(yield* changedSince(second, third), { patch: "", authored: null }); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + it.effect("checkpoint capture preserves racy edits made after resetting the index", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; @@ -1051,6 +1496,7 @@ it.effect.each(["committed", "unborn"] as const)( assert.deepStrictEqual(commands, [ "rev-parse", + ...(head === "committed" ? ["log"] : []), "config", ...(head === "committed" ? ["read-tree", "ls-files"] : []), "add", @@ -1116,6 +1562,8 @@ it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( "write-tree", "commit-tree", "update-ref", + // Without a parsed top level, capture cannot rule out submodules. + "submodule", ]); assert.strictEqual((yield* git(["show", `${checkpointRef}:file.txt`])).stdout, "unstaged\n"); assert.deepEqual(yield* fileSystem.readFile(path.join(cwd, ".git", "index")), originalIndex); @@ -1273,6 +1721,154 @@ it.effect("restores a checkpoint before the first commit without staging its fil }).pipe(Effect.scoped, Effect.provide(layerGitContract)), ); +it.effect("restores edits to skip-worktree files whose checkpoint content matches HEAD", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-skip-worktree-restore-" }); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + const skippedPath = path.join(cwd, "skipped.txt"); + yield* fileSystem.writeFileString(skippedPath, "original\n"); + yield* git(["add", "skipped.txt"]); + yield* git(["commit", "-m", "add skipped file"]); + yield* git(["update-index", "--skip-worktree", "skipped.txt"]); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + yield* fileSystem.writeFileString(skippedPath, "later\n"); + + assert.isTrue( + yield* driver.checkpoints.restoreCheckpoint({ cwd, checkpointRef, fallbackToHead: false }), + ); + + assert.strictEqual(yield* fileSystem.readFileString(skippedPath), "original\n"); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect("sparse restore rewrites manual skip-worktree files and keeps exclusions absent", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const liveProcess = yield* VcsProcess.VcsProcess; + // One byte per chunk, so the index listing splits inside every record. + const driver = yield* GitVcsDriver.makeVcsDriverShape().pipe( + Effect.provideService(VcsProcess.VcsProcess, { + run: (input) => { + const onStdoutChunk = input.onStdoutChunk; + return liveProcess.run( + onStdoutChunk === undefined + ? input + : { + ...input, + onStdoutChunk: (chunk) => { + for (let index = 0; index < chunk.length; index++) + onStdoutChunk(chunk.subarray(index, index + 1)); + }, + }, + ); + }, + }), + ); + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-sparse-restore-" }); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + for (const name of ["in/normal", "in/skipped", "in/s space", "out/deep/absent", "out/stray"]) { + yield* fs.makeDirectory(path.dirname(path.join(cwd, name)), { recursive: true }); + yield* fs.writeFileString(path.join(cwd, name), "original\n"); + } + yield* git(["add", "."]); + yield* git(["commit", "-m", "sparse fixture"]); + yield* git(["sparse-checkout", "set", "--cone", "--sparse-index", "in"]); + // Keeps Git from clearing the bit on files that are present. + yield* git(["config", "sparse.expectFilesOutsideOfPatterns", "true"]); + yield* git(["update-index", "--skip-worktree", "in/skipped", "in/s space"]); + const captureCwd = path.join(cwd, "in"); + yield* driver.checkpoints.captureCheckpoint({ cwd: captureCwd, checkpointRef }); + yield* fs.writeFileString(path.join(cwd, "in/skipped"), "later\n"); + yield* fs.writeFileString(path.join(cwd, "in/s space"), "later\n"); + yield* fs.writeFileString(path.join(cwd, "in/normal"), "later\n"); + + for (const restoreCwd of [captureCwd, cwd]) { + assert.isTrue( + yield* driver.checkpoints.restoreCheckpoint({ + cwd: restoreCwd, + checkpointRef, + fallbackToHead: false, + }), + ); + assert.strictEqual(yield* fs.readFileString(path.join(cwd, "in/skipped")), "original\n"); + assert.strictEqual(yield* fs.readFileString(path.join(cwd, "in/s space")), "original\n"); + assert.strictEqual(yield* fs.readFileString(path.join(cwd, "in/normal")), "original\n"); + assert.isFalse(yield* fs.exists(path.join(cwd, "out"))); + } + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect("restores and deletes checkpoints inside populated submodules", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const git = (cwd: string, args: ReadonlyArray) => + driver.execute({ operation: "test", cwd, args }); + const makeRepository = Effect.fn("makeRepository")(function* (prefix: string) { + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix }); + yield* git(cwd, ["init"]); + yield* git(cwd, ["config", "user.email", "test@test.com"]); + yield* git(cwd, ["config", "user.name", "Test"]); + yield* fileSystem.writeFileString(path.join(cwd, "file.txt"), "original\n"); + yield* git(cwd, ["add", "."]); + yield* git(cwd, ["commit", "-m", "initial"]); + return cwd; + }); + const nested = yield* makeRepository("t3-submodule-nested-"); + const library = yield* makeRepository("t3-submodule-library-"); + const addSubmodule = (cwd: string, url: string, name: string) => + git(cwd, ["-c", "protocol.file.allow=always", "submodule", "add", url, name]).pipe( + Effect.andThen(git(cwd, ["commit", "-m", `add ${name}`])), + ); + yield* addSubmodule(library, nested, "nested"); + const root = yield* makeRepository("t3-submodule-root-"); + yield* addSubmodule(root, library, "library"); + yield* git(root, [ + "-c", + "protocol.file.allow=always", + "submodule", + "update", + "--init", + "--recursive", + ]); + const files = ["file.txt", "library/file.txt", "library/nested/file.txt"].map((file) => + path.join(root, file), + ); + const checkpointRef = CheckpointRef.make("refs/t3/checkpoints/submodules"); + yield* fileSystem.writeFileString(files[1]!, "captured\n"); + yield* driver.checkpoints.captureCheckpoint({ cwd: root, checkpointRef }); + + for (const file of files) { + yield* fileSystem.writeFileString(file, "changed\n"); + yield* fileSystem.writeFileString(`${file}.added`, "new\n"); + } + assert.isTrue( + yield* driver.checkpoints.restoreCheckpoint({ + cwd: root, + checkpointRef, + fallbackToHead: false, + }), + ); + + assert.deepStrictEqual( + yield* Effect.forEach(files, (file) => fileSystem.readFileString(file)), + ["original\n", "captured\n", "original\n"], + ); + for (const file of files) assert.isFalse(yield* fileSystem.exists(`${file}.added`)); + + yield* driver.checkpoints.deleteCheckpointRefs({ cwd: root, checkpointRefs: [checkpointRef] }); + for (const cwd of [root, path.join(root, "library"), path.join(root, "library", "nested")]) { + const ref = yield* git(cwd, ["for-each-ref", "refs/t3"]); + assert.strictEqual(ref.stdout, ""); + } + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + // The real VCS process, spawning `command` wherever the driver runs git. const vcsProcessSpawning = (command: string) => Layer.effect( diff --git a/apps/server/src/vcs/GitVcsDriver.ts b/apps/server/src/vcs/GitVcsDriver.ts index ff0eac7c00093..d1e9dae66f1fa 100644 --- a/apps/server/src/vcs/GitVcsDriver.ts +++ b/apps/server/src/vcs/GitVcsDriver.ts @@ -4,6 +4,7 @@ import * as Context from "effect/Context"; import * as SecureRandom from "effect/crypto/SecureRandom"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; @@ -12,6 +13,7 @@ import { ChildProcessSpawner } from "effect/process"; import { GitCommandError, + VcsCheckpointStorageError, VcsProcessExitError, VcsUnsupportedOperationError, type VcsSwitchRefInput, @@ -37,6 +39,7 @@ import { } from "@t3tools/contracts"; import { isMissingExecutable } from "@t3tools/source-control-core/server/discovery"; import * as HostProcess from "@t3tools/shared/HostProcess"; +import { quoteGitPatchPath } from "@t3tools/shared/gitPatchPath"; import { makeGitVcsDriverCore, PATCH_RENDER_PREFIX_ARGS, @@ -121,7 +124,10 @@ export interface ExecuteGitProgress { } export interface CreateWorktreeProgress { - /** Git refuses existing paths; claimed paths belong to this call and can be removed on cancel. */ + /** + * Runs uninterruptibly once the worktree exists; from then on the caller owns its cleanup. + * For new-branch calls on Git supporting add --reason, cancellation before the claim removes what it made. + */ readonly onWorktreeClaimed?: (path: string) => Effect.Effect; readonly onCheckoutProgress?: (input: { percent: number; @@ -177,12 +183,16 @@ export interface GitPushResult { status: "pushed" | "skipped_up_to_date"; branch: string; upstreamBranch?: string | undefined; + /** Set when `upstreamBranch` is on a configured push remote instead of the tracked upstream. */ + pushRemote?: string | undefined; setUpstream?: boolean | undefined; } export interface GitPushTarget { remoteName: string; branchName: string; + /** The tracked upstream is current, yet pushes go to this configured push remote. */ + viaPushRemote: boolean; } export interface GitRangeContext { @@ -327,6 +337,8 @@ export class GitVcsDriver extends Context.Service< cwd: string, options?: GitRemoteStatusOptions, ) => Effect.Effect; + /** Refs, HEAD and shallow boundaries. Null when the snapshot cannot be read. */ + readonly readRefSnapshot: (cwd: string) => Effect.Effect; readonly prepareCommitContext: ( cwd: string, filePaths?: readonly string[], @@ -458,6 +470,8 @@ export class GitVcsDriver extends Context.Service< input: VcsSwitchRefInput, ) => Effect.Effect; readonly initRepo: (input: VcsInitInput) => Effect.Effect; + /** Drops cached refs after something outside this driver changed them, such as a clone into `cwd`. */ + readonly invalidateRefs: (cwd: string) => Effect.Effect; readonly listLocalBranchNames: (cwd: string) => Effect.Effect; } >()("t3/vcs/GitVcsDriver") {} @@ -467,6 +481,10 @@ const FLAGGED_INDEX_ENTRIES_MAX_BYTES = 1024 * 1024; const CHECKPOINT_RECOVERY_MAX_CANDIDATES = 64; const CHECKPOINT_RECOVERY_TIMEOUT = "5 seconds"; const GIT_CHECK_IGNORE_MAX_STDIN_BYTES = 256 * 1024; +// Reflog subjects that only point a ref at an existing commit. Anything else counts as creating +// one, so an unknown subject keeps its commit attributed. +const REF_MOVE_REFLOG_SUBJECT = + /^(?:$|(?:checkout|reset|branch|clone|fetch)\b|[^:]*\((?:start|finish|abort|reset)\): |(?:merge|pull|rebase|cherry-pick)\b[^()]*: fast-forward$)/i; const WORKSPACE_GIT_HARDENED_CONFIG_ARGS = [ "-c", "core.fsmonitor=false", @@ -557,6 +575,7 @@ const gitCommand = Effect.fn("GitVcsDriver.gitCommand")( args: ReadonlyArray, options?: { readonly stdin?: string; + readonly onSpawn?: () => void; readonly env?: NodeJS.ProcessEnv; readonly allowNonZeroExit?: boolean; readonly timeoutMs?: number; @@ -572,6 +591,7 @@ const gitCommand = Effect.fn("GitVcsDriver.gitCommand")( cwd, spawnCwd: globalThis.process.cwd(), ...(options?.stdin !== undefined ? { stdin: options.stdin } : {}), + ...(options?.onSpawn !== undefined ? { onSpawn: options.onSpawn } : {}), ...(options?.env !== undefined ? { env: options.env } : {}), ...(options?.allowNonZeroExit !== undefined ? { allowNonZeroExit: options.allowNonZeroExit } @@ -585,6 +605,10 @@ const gitCommand = Effect.fn("GitVcsDriver.gitCommand")( }), ); +const HEAD_MOVES_SCAN_LIMIT = 512; +const HEAD_MARK_TRAILER = "T3-Head-Moved-At"; +const HEAD_MARK_TRAILER_PATTERN = new RegExp(`^${HEAD_MARK_TRAILER}: (\\d+) (\\d+)$`, "m"); + export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* () { const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -639,6 +663,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( const execute: VcsDriver.VcsDriver["Service"]["execute"] = (input) => gitCommand(vcsProcess, input.operation, input.cwd, input.args, { ...(input.stdin !== undefined ? { stdin: input.stdin } : {}), + ...(input.onSpawn !== undefined ? { onSpawn: input.onSpawn } : {}), ...(input.env !== undefined ? { env: input.env } : {}), ...(input.allowNonZeroExit !== undefined ? { allowNonZeroExit: input.allowNonZeroExit } : {}), ...(input.timeoutMs !== undefined ? { timeoutMs: input.timeoutMs } : {}), @@ -912,6 +937,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( "--git-common-dir", "--git-path", "index", + "--show-toplevel", "--show-prefix", "--verify", "--quiet", @@ -922,13 +948,15 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( const headExists = result.exitCode === 0; const headChecked = headExists || result.exitCode === 1; const lines = result.stdout.split("\n"); - const [gitCommonDir = "", indexPath = "", prefix = ""] = lines; - if (headChecked && lines.length === (headExists ? 5 : 4) && lines.at(-1) === "") { + const [gitCommonDir = "", indexPath = "", topLevel = "", prefix = "", head = ""] = lines; + if (headChecked && lines.length === (headExists ? 6 : 5) && lines.at(-1) === "") { return { gitCommonDir: gitCommonDir.trim(), indexPath: indexPath.trim(), + topLevel: topLevel.trim(), prefix, headExists, + head: headExists ? head : undefined, }; } // Git before 2.31 echoes --path-format, and a path containing a newline spans lines. @@ -942,11 +970,26 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( return { gitCommonDir: fallbackCommonDir, indexPath: undefined, + topLevel: undefined, prefix: prefixResult.stdout.replace(/\n$/, ""), headExists: headChecked ? headExists : yield* hasHeadCommit(cwd), + head: undefined, }; }); + // $displaypath is relative to cwd, so "../" marks submodules outside a nested workspace. + const listSubmodules = Effect.fn("GitVcsDriver.listSubmodules")(function* (cwd: string) { + const result = yield* execute({ + operation: "GitVcsDriver.checkpoints.listSubmodules", + cwd, + args: ["submodule", "foreach", "--quiet", 'printf "%s\\0" "$displaypath"'], + }); + return result.stdout + .split("\0") + .filter((entry) => entry !== "" && !entry.startsWith("../")) + .map((entry) => path.join(cwd, entry)); + }); + // Git defaults and macOS writeout-only can leave empty refs after crashes; flush before publication. // Set fsync first: Git 2.36 ignores unknown batch without resetting the method. // Batch fully flushes refs and equals fsync on macOS HFS+/APFS and Windows NTFS/ReFS. @@ -965,6 +1008,64 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( // core.longpaths is disabled in the user's Git configuration. const longPaths = hostPlatform === "win32" ? ["-c", "core.longpaths=true"] : []; + // Newest first: the commit each HEAD move landed on, when, and Git's reason for it. + const readHeadMoves = (cwd: string) => + execute({ + operation: VcsProcess.CHECKPOINT_CAPTURE_OPERATION, + cwd, + args: [ + "-c", + "log.showSignature=false", + "log", + "--walk-reflogs", + "-n", + String(HEAD_MOVES_SCAN_LIMIT), + "--date=unix", + "--format=%H %gd %gs", + "HEAD", + "--", + ], + allowNonZeroExit: true, + }).pipe( + Effect.map((result) => + result.exitCode !== 0 + ? [] + : result.stdout.split("\n").flatMap((line) => { + const [, to, at, reason] = /^([0-9a-f]+) HEAD@\{(\d+)\} (.*)$/.exec(line) ?? []; + return to === undefined || at === undefined || reason === undefined + ? [] + : [{ to, at, reason }]; + }), + ), + Effect.orElseSucceed(() => []), + ); + + // A checkpoint taken after a branch switch has a second parent: the checkpoint before it with + // the switch applied, which is where that turn started. Other ranges start at `from` itself. + const resolveTurnStart = (input: VcsDriver.VcsListAuthoredPathsInput) => + execute({ + operation: "GitVcsDriver.checkpoints.resolveTurnStart", + cwd: input.cwd, + args: [ + "-c", + "log.showSignature=false", + "log", + "--no-walk=unsorted", + "--format=%H %P", + `${input.fromCheckpointRef}^{commit}`, + `${input.toCheckpointRef}^2`, + ], + allowNonZeroExit: true, + }).pipe( + Effect.map((result): string => { + const [fromLine = "", startLine = ""] = result.stdout.trimEnd().split("\n"); + const [start = "", , startedFrom] = startLine.split(" "); + return result.exitCode === 0 && startedFrom === fromLine.split(" ")[0] + ? start + : input.fromCheckpointRef; + }), + ); + const checkpoints: VcsDriver.VcsCheckpointOps = { captureCheckpoint: Effect.fn("GitVcsDriver.checkpoints.captureCheckpoint")(function* (input) { const operation = VcsProcess.CHECKPOINT_CAPTURE_OPERATION; @@ -979,14 +1080,56 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( "-c", "splitIndex.sharedIndexExpire=never", ]; - const { gitCommonDir, indexPath, prefix, headExists } = yield* resolveCaptureRepository( - input.cwd, - ); + const { gitCommonDir, indexPath, topLevel, prefix, headExists, head } = + yield* resolveCaptureRepository(input.cwd); + // Read before the snapshot and pinned to the same HEAD: a switch that lands while files are + // being staged then belongs to the next checkpoint instead of being applied to this one. + const headMoves = head === undefined ? [] : yield* readHeadMoves(input.cwd); + const lastMove = headMoves.at(0); + // Moves share a second, so the mark also counts how many of that second it has seen. + const headMark = + lastMove !== undefined && lastMove.to === head + ? `${lastMove.at} ${headMoves.filter((move) => move.at === lastMove.at).length}` + : undefined; const indexId = yield* random.uuidV4(); const tempIndexPath = path.join(gitCommonDir, `t3-checkpoint-index-${indexId}`); const hostEnv = yield* HostProcess.Environment; + // VcsProcess clears inherited object selectors, so every other command uses this store. + const sharedObjects = path.join(gitCommonDir, "objects"); + const quotedObjects = quoteGitPatchPath(sharedObjects); + // Isolate bulk-checkin packs so cleanup cannot remove another Git writer's objects. + const privateObjects = yield* Effect.acquireRelease( + fileSystem + .makeTempDirectory({ directory: sharedObjects, prefix: "tmp_objdir-t3-checkpoint-" }) + .pipe( + Effect.mapError( + (cause) => + new VcsCheckpointStorageError({ + operation, + cwd: input.cwd, + stage: "isolate", + cause, + }), + ), + ), + (directory) => + fileSystem.remove(directory, { recursive: true, force: true }).pipe( + Effect.catch((cause) => + Effect.logWarning("Could not remove private checkpoint objects", { + reason: cause.reason._tag, + }), + ), + ), + ); const commitEnv: NodeJS.ProcessEnv = { GIT_INDEX_FILE: tempIndexPath, + GIT_OBJECT_DIRECTORY: privateObjects, + // A filter must not publish refs that depend on our disposable store. + GIT_QUARANTINE_PATH: privateObjects, + // Git accepts C-quoted alternate paths, including path-list separators. + GIT_ALTERNATE_OBJECT_DIRECTORIES: quotedObjects.startsWith('"') + ? quotedObjects + : `"${quotedObjects}"`, GIT_AUTHOR_NAME: "T3 Code", GIT_AUTHOR_EMAIL: "t3code@users.noreply.github.com", GIT_COMMITTER_NAME: "T3 Code", @@ -1290,6 +1433,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( GIT_INDEX_FILE: undefined, GIT_OBJECT_DIRECTORY: undefined, GIT_ALTERNATE_OBJECT_DIRECTORIES: undefined, + GIT_QUARANTINE_PATH: undefined, }; const exclusions: Array = []; for (const entry of candidates) { @@ -1331,8 +1475,137 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( }); } - const message = `t3 checkpoint ref=${input.checkpointRef}`; - // HEAD is the parent so listAuthoredPaths can see how HEAD moved between checkpoints. + // Switching branches rewrites files the turn never wrote, and only HEAD's reflog tells a + // switch from a reset. The trailer marks where that reflog stood for the next checkpoint. + const turnStart = + headMark === undefined || input.previousCheckpointRef === undefined + ? null + : yield* Effect.gen(function* () { + const previous = yield* execute({ + operation, + cwd: input.cwd, + args: [ + "-c", + "log.showSignature=false", + "log", + "--no-walk", + "--format=%H %T %ct %P%n%B", + `${input.previousCheckpointRef}^{commit}`, + ], + allowNonZeroExit: true, + }); + if (previous.exitCode !== 0) return null; + const [commit = "", tree = "", capturedAt = "", previousHead = ""] = previous.stdout + .slice(0, previous.stdout.indexOf("\n")) + .split(" "); + const [, markedAt, markedCount] = + HEAD_MARK_TRAILER_PATTERN.exec(previous.stdout) ?? []; + const oldestMarked = headMoves.findLastIndex((move) => move.at === markedAt); + // -1 when the scan ended inside the marked second, so the count cannot be placed. + const since = + oldestMarked === HEAD_MOVES_SCAN_LIMIT - 1 + ? -1 + : oldestMarked - Number(markedCount) + 1; + if (headMoves[since]?.to !== previousHead) return null; + const switches = headMoves + .slice(0, since) + .map((move, index) => ({ ...move, from: headMoves[index + 1]!.to })) + .filter( + (move) => + move.reason.startsWith("checkout: moving from ") && move.from !== move.to, + ) + .toReversed(); + const landedOn = switches.at(-1)?.to; + if (landedOn === undefined) return null; + + yield* execute({ + operation, + cwd: input.cwd, + args: [...indexConfig, "-c", "index.sparse=false", "read-tree", tree], + env: commitEnv, + }); + for (const { from, to } of switches) { + const changes = yield* execute({ + operation, + cwd: input.cwd, + args: [ + "-c", + "core.quotePath=true", + "diff-tree", + "-r", + "--no-renames", + from, + to, + ], + maxOutputBytes: VcsDriver.CHECKPOINT_DIFF_MAX_OUTPUT_BYTES, + outputMode: "error", + }); + // A switch succeeds only where the files it replaces were unmodified, so each + // changed path takes the destination's version. Mode 000000 drops the path. + const entries = changes.stdout + .split("\n") + // A switch leaves a submodule's own checkout where it was. + .filter((line) => line !== "" && !/^:(?:160000 |[0-7]{6} 160000 )/.test(line)) + .map((line) => + line.replace( + /^:[0-7]{6} ([0-7]{6}) [0-9a-f]+ ([0-9a-f]+) [A-Z]\t/, + "$1 $2\t", + ), + ); + if (entries.length === 0) continue; + yield* execute({ + operation, + cwd: input.cwd, + args: [ + ...indexConfig, + "-c", + "index.sparse=false", + "update-index", + "--index-info", + ], + stdin: `${entries.join("\n")}\n`, + env: commitEnv, + }); + } + const startTree = (yield* execute({ + operation, + cwd: input.cwd, + args: [...indexConfig, ...durableWrite, "write-tree"], + env: commitEnv, + })).stdout.trim(); + if (startTree === "" || startTree === tree) return null; + const startCommit = yield* execute({ + operation, + cwd: input.cwd, + args: [ + ...durableWrite, + "commit-tree", + startTree, + "-p", + landedOn, + "-p", + commit, + "-m", + `t3 checkpoint turn start ref=${input.checkpointRef}`, + ], + // listAuthoredPaths reads commits newer than the start as the turn's own. + env: { ...commitEnv, GIT_COMMITTER_DATE: `${capturedAt} +0000` }, + }); + return startCommit.stdout.trim() || null; + }).pipe( + Effect.catch((cause) => + Effect.logWarning("checkpoint branch switch turn start failed", { + cwd: input.cwd, + checkpointRef: input.checkpointRef, + cause: String(cause), + }).pipe(Effect.as(null)), + ), + ); + + const message = `t3 checkpoint ref=${input.checkpointRef}${ + headMark === undefined ? "" : `\n\n${HEAD_MARK_TRAILER}: ${headMark}` + }`; + // HEAD is the first parent so listAuthoredPaths can see how HEAD moved between checkpoints. const commitTreeResult = yield* execute({ operation, cwd: input.cwd, @@ -1340,7 +1613,8 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( ...durableWrite, "commit-tree", treeOid, - ...(headExists ? ["-p", "HEAD"] : []), + ...(headExists ? ["-p", head ?? "HEAD"] : []), + ...(turnStart === null ? [] : ["-p", turnStart]), "-m", message, ], @@ -1357,13 +1631,134 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( }); } + // Git writers have exited; bound publication by fanout directory. + yield* fileSystem.readDirectory(privateObjects).pipe( + Effect.flatMap((directories) => + Effect.forEach( + directories.filter( + (directory) => directory === "pack" || /^[0-9a-f]{2}$/.test(directory), + ), + (directory) => + Effect.gen(function* () { + const source = path.join(privateObjects, directory); + const destination = path.join(sharedObjects, directory); + // Most fanouts already exist; avoid constructing an EEXIST error + // for each of them on every capture. mkdir still handles the race. + if (!(yield* fileSystem.exists(destination))) + yield* fileSystem.makeDirectory(destination).pipe( + Effect.andThen( + Effect.gen(function* () { + // Git has already applied core.sharedRepository to the source. + // chmod bypasses umask, but never changes a pre-existing bucket. + const { mode } = yield* fileSystem.stat(source); + yield* fileSystem.chmod(destination, mode & 0o7777); + }), + ), + Effect.catchIf( + (cause) => cause.reason._tag === "AlreadyExists", + () => Effect.void, + ), + Effect.uninterruptible, + ); + const names = yield* fileSystem.readDirectory(source); + // Per-object spans would swamp the capture span for large workspaces. + const linkObject = Effect.fnUntraced(function* (name: string) { + const from = path.join(source, name); + const to = path.join(destination, name); + yield* fileSystem.link(from, to).pipe( + Effect.catchIf( + (cause) => + cause.reason._tag === "PermissionDenied" || + cause.reason._tag === "Unknown", + () => + Effect.gen(function* () { + if (yield* fileSystem.exists(to)) return; + // Git also falls back to rename without hardlinks; identical object names are interchangeable. + yield* fileSystem.rename(from, to); + }), + ), + Effect.catchIf( + (cause) => cause.reason._tag === "AlreadyExists", + () => Effect.void, + ), + ); + }); + if (directory !== "pack") { + yield* Effect.forEach( + names.filter((name) => /^(?:[0-9a-f]{38}|[0-9a-f]{62})$/.test(name)), + linkObject, + { discard: true }, + ); + return; + } + const files = new Set(names); + for (const name of names) { + if (!/^pack-(?:[0-9a-f]{40}|[0-9a-f]{64})\.pack$/.test(name)) continue; + const base = name.slice(0, -5); + if (!files.has(`${base}.idx`)) continue; + // Link metadata before the pack so an I/O error cannot strand a + // large unindexed pack. Defer cancellation across this short group + // of links. Never roll back shared links: a concurrent writer may + // already depend on them. Ref publication still happens last. + yield* Effect.forEach( + [".keep", ".promisor", ".rev", ".idx", ".pack"] + .map((extension) => `${base}${extension}`) + .filter((file) => files.has(file)), + linkObject, + { discard: true }, + ).pipe(Effect.uninterruptible); + } + }), + { concurrency: 16, discard: true }, + ), + ), + Effect.mapError( + (cause) => + new VcsCheckpointStorageError({ operation, cwd: input.cwd, stage: "publish", cause }), + ), + ); + yield* execute({ operation, cwd: input.cwd, args: [...durableWrite, "update-ref", input.checkpointRef, commitOid], }); }).pipe(Effect.ensuring(cleanupTempIndex)); - }), + + // The tree records only each submodule's HEAD, so submodules keep their own ref. + const hasSubmodules = + topLevel === undefined || + (yield* fileSystem + .exists(path.join(topLevel, ".gitmodules")) + .pipe(Effect.orElseSucceed(() => true))); + if (!hasSubmodules) return; + // A submodule that cannot be captured drops its ref, so restore leaves it alone. + yield* listSubmodules(input.cwd).pipe( + Effect.flatMap((submodules) => + Effect.forEach( + submodules, + (cwd) => + checkpoints + .captureCheckpoint({ cwd, checkpointRef: input.checkpointRef }) + .pipe( + Effect.catch((cause) => + Effect.logWarning("submodule checkpoint capture failed", { cwd, cause }).pipe( + Effect.andThen( + checkpoints + .deleteCheckpointRefs({ cwd, checkpointRefs: [input.checkpointRef] }) + .pipe(Effect.ignore), + ), + ), + ), + ), + { discard: true }, + ), + ), + Effect.catch((cause) => + Effect.logWarning("submodule checkpoint capture failed", { cwd: input.cwd, cause }), + ), + ); + }, Effect.scoped), hasCheckpointRef: (input) => resolveCheckpointCommit(input.cwd, input.checkpointRef).pipe( @@ -1388,23 +1783,16 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( cwd: input.cwd, args: ["ls-files", "--cached", `--with-tree=${commitOid}`, "-z", "--", "."], }); - // An empty index and checkpoint have nothing for git restore's pathspec to match. - if (tracked.stdout.length > 0) { - yield* execute({ - operation, - cwd: input.cwd, - args: [ - ...longPaths, - "restore", - "--source", - commitOid, - "--worktree", - "--staged", - "--", - ".", - ], - }); - } + const sparseConfig = + tracked.stdout.length > 0 + ? yield* execute({ + operation, + cwd: input.cwd, + args: ["config", "--bool", "core.sparseCheckout"], + allowNonZeroExit: true, + }) + : null; + const sparseCheckout = sparseConfig?.stdout.trim() === "true"; // Restoring away the last tracked file can remove a nested workspace directory. const recreateWorkspace = fileSystem.makeDirectory(input.cwd, { recursive: true }).pipe( Effect.mapError( @@ -1427,6 +1815,95 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( args: [...longPaths, "reset", "--quiet", "--", "."], }); }); + let mutationStarted = false; + const onSpawn = () => { + mutationStarted = true; + }; + // Queued commands preserve user staging; spawned children stop before this scope resets it. + yield* Effect.addFinalizer((exit) => + mutationStarted && Exit.hasInterrupts(exit) ? resetIndex.pipe(Effect.ignore) : Effect.void, + ); + // An empty index and checkpoint have nothing for git restore's pathspec to match. + if (tracked.stdout.length > 0) { + const restore = [...longPaths, "restore", "--source", commitOid, "--worktree", "--staged"]; + // restore leaves skip-worktree paths alone. Outside a sparse checkout every such bit was + // set by hand and capture recorded the file, so override them all. + yield* execute({ + operation, + cwd: input.cwd, + args: [...restore, ...(sparseCheckout ? [] : ["--ignore-skip-worktree-bits"]), "--", "."], + onSpawn, + }); + if (sparseCheckout) { + // Sparse exclusions stay excluded; only skip-worktree paths the rules select are manual. + const manual = yield* Effect.gen(function* () { + const skipped: string[] = []; + let recordStart = true; + let keep = false; + let record: Uint8Array[] = []; + yield* vcsProcess.run({ + operation, + command: "git", + cwd: input.cwd, + args: ["ls-files", "--full-name", "--sparse", "-v", "-z", "--", "."], + maxOutputBytes: 4_096, + outputMode: "truncate", + // The index can be huge; keep only the "S "/"s " records of the NUL-delimited list. + onStdoutChunk: (chunk) => { + let start = 0; + while (start < chunk.length) { + const end = chunk.indexOf(0, start); + if (recordStart) { + keep = chunk[start] === 83 || chunk[start] === 115; + record = []; + } + if (keep) { + record.push(Buffer.from(chunk.subarray(start, end === -1 ? undefined : end))); + if (end !== -1) { + const entry = Buffer.concat(record).subarray(2); + record = []; + // Sparse directory entries are exclusions; stdin cannot carry non-UTF-8. + if (entry.at(-1) !== 47 && NodeBuffer.isUtf8(entry)) + skipped.push(entry.toString("utf8")); + } + } + recordStart = end !== -1; + if (!recordStart) break; + start = end + 1; + } + }, + }); + if (skipped.length === 0) return skipped; + const stdin = skipped.join("\0") + "\0"; + // check-rules echoes a subset of its input, so the input size bounds its output. + const rules = yield* execute({ + operation, + cwd: input.cwd, + args: ["sparse-checkout", "check-rules", "-z"], + stdin, + maxOutputBytes: Buffer.byteLength(stdin), + outputMode: "error", + }); + return rules.stdout.split("\0").filter((entry) => entry.length > 0); + }).pipe( + // A Git without ls-files --sparse or check-rules keeps the plain restore's result. + Effect.orElseSucceed((): string[] => []), + ); + if (manual.length > 0) { + yield* execute({ + operation, + cwd: input.cwd, + args: [ + ...restore, + "--ignore-skip-worktree-bits", + "--pathspec-from-file=-", + "--pathspec-file-nul", + ], + stdin: manual.map((entry) => `:(top,literal)${entry}\0`).join(""), + }); + } + } + } yield* recreateWorkspace; const clean = execute({ operation, @@ -1434,6 +1911,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( args: [...longPaths, "clean", "-fd", "--", "."], env: { LC_ALL: "C" }, allowNonZeroExit: true, + onSpawn, }); let cleaned = yield* clean; // Removing a new .gitignore exposes the files it hid; clean again until nothing is left. @@ -1486,8 +1964,34 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( } yield* resetIndex; + // A submodule without this ref was not captured, and restoring it to HEAD would lose edits. + // The root is already restored, so a failing submodule must not hide that from the caller. + yield* listSubmodules(input.cwd).pipe( + Effect.flatMap((submodules) => + Effect.forEach( + submodules, + (cwd) => + checkpoints + .restoreCheckpoint({ + cwd, + checkpointRef: input.checkpointRef, + fallbackToHead: false, + }) + .pipe( + Effect.catch((cause) => + Effect.logWarning("submodule checkpoint restore failed", { cwd, cause }), + ), + ), + { discard: true }, + ), + ), + Effect.catch((cause) => + Effect.logWarning("submodule checkpoint restore failed", { cwd: input.cwd, cause }), + ), + ); + return true; - }), + }, Effect.scoped), diffCheckpoints: Effect.fn("GitVcsDriver.checkpoints.diffCheckpoints")(function* (input) { const operation = "GitVcsDriver.checkpoints.diffCheckpoints"; @@ -1501,15 +2005,13 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( "checkpoint.fallback_from_to_head": input.fallbackFromToHead, }); - let fromRevision: string = input.fromCheckpointRef; + let fromRevision = yield* resolveTurnStart(input); if (input.fallbackFromToHead === true) { const resolvedFromCommit = yield* resolveCheckpointCommit( input.cwd, input.fromCheckpointRef, ); - if (resolvedFromCommit) { - fromRevision = resolvedFromCommit; - } else { + if (!resolvedFromCommit) { const headCommit = yield* resolveHeadCommit(input.cwd); if (!headCommit) { return yield* new VcsProcessExitError({ @@ -1568,25 +2070,36 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( listAuthoredPaths: Effect.fn("GitVcsDriver.checkpoints.listAuthoredPaths")(function* (input) { const operation = "GitVcsDriver.checkpoints.listAuthoredPaths"; + const turnStart = yield* resolveTurnStart(input); const heads = yield* execute({ operation, cwd: input.cwd, args: [ "log", "--no-walk=unsorted", - "--format=%P %ct", - `${input.fromCheckpointRef}^{commit}`, + "--format=%ct %P", + `${turnStart}^{commit}`, `${input.toCheckpointRef}^{commit}`, ], }); const [fromLine = "", toLine = ""] = heads.stdout.trimEnd().split("\n"); - const [startHead = "", capturedAt = ""] = fromLine.split(" "); - const [endHead = ""] = toLine.split(" "); + const [capturedAt = "", startHead = ""] = fromLine.split(" "); + const [, endHead = ""] = toLine.split(" "); if (startHead === "" || endHead === "" || startHead === endHead) { return null; } - const listPaths = (args: ReadonlyArray) => + const listLines = (args: ReadonlyArray, stdin?: string) => + execute({ + operation, + cwd: input.cwd, + // log.showSignature prints verification text into the output. + args: ["-c", "log.showSignature=false", ...args], + ...(stdin === undefined ? {} : { stdin }), + maxOutputBytes: VcsDriver.CHECKPOINT_DIFF_MAX_OUTPUT_BYTES, + outputMode: "error", + }).pipe(Effect.map((result) => result.stdout.split("\n").filter((line) => line !== ""))); + const listPaths = (args: ReadonlyArray, stdin?: string) => execute({ operation, cwd: input.cwd, @@ -1602,26 +2115,73 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( "--no-renames", "--no-ext-diff", ], + ...(stdin === undefined ? {} : { stdin }), maxOutputBytes: VcsDriver.CHECKPOINT_DIFF_MAX_OUTPUT_BYTES, outputMode: "error", }).pipe(Effect.map((result) => result.stdout.split("\0"))); + + // A commit is an import when a fetch delivered it since the previous checkpoint and no + // reflog in any worktree shows this repository creating it. --all covers every worktree's + // HEAD, and with --walk-reflogs --since compares each entry's own time. + const listImportedCommits = Effect.gen(function* () { + const entries = (yield* listLines([ + "log", + "--walk-reflogs", + "--all", + `--since=@${capturedAt}`, + "--format=%H %gD %gs", + ])).map((line) => { + const [commit = "", selector = "", ...subject] = line.split(" "); + return { + commit, + fetched: selector.startsWith("refs/remotes/"), + subject: subject.join(" "), + }; + }); + // A push records commits this repository already had. + const fetchedTips = new Set( + entries + .filter((entry) => entry.fetched && entry.subject !== "update by push") + .map((entry) => entry.commit), + ); + if (fetchedTips.size === 0) return fetchedTips; + // Fetching another branch can deliver a commit that was made here and pushed. + const createdHere = new Set( + entries + .filter((entry) => !entry.fetched && !REF_MOVE_REFLOG_SUBJECT.test(entry.subject)) + .map((entry) => entry.commit), + ); + const fetched = yield* listLines( + ["rev-list", `--since=@${capturedAt}`, "--stdin"], + `${[...fetchedTips, `^${startHead}`].join("\n")}\n`, + ); + return new Set(fetched.filter((commit) => !createdHere.has(commit))); + }); + // Remerge diffs retain merge conflict fixes while excluding clean upstream changes. - const pathLists = yield* Effect.all( + const [recentCommits, ...pathLists] = yield* Effect.all( [ - listPaths(["diff", startHead, `${input.fromCheckpointRef}^{commit}`]), + listLines(["rev-list", `--since=@${capturedAt}`, `${startHead}..${endHead}`]), + listPaths(["diff", startHead, `${turnStart}^{commit}`]), listPaths(["diff", endHead, `${input.toCheckpointRef}^{commit}`]), listPaths(["log", "--format=", "--diff-merges=remerge", `${endHead}..${startHead}`]), - listPaths([ - "log", - "--format=", - "--diff-merges=remerge", - `--since=@${capturedAt}`, - `${startHead}..${endHead}`, - ]), ], { concurrency: "unbounded" }, ); - return new Set(pathLists.flat().filter((path) => path.length > 0)); + // Without readable reflogs every recent commit stays attributed, as before. + const importedCommits = + recentCommits.length === 0 + ? new Set() + : yield* listImportedCommits.pipe(Effect.orElseSucceed(() => new Set())); + const ownCommits = recentCommits.filter((commit) => !importedCommits.has(commit)); + const ownCommitPaths = + ownCommits.length === 0 + ? [] + : yield* listPaths( + ["log", "--no-walk=unsorted", "--stdin", "--format=", "--diff-merges=remerge"], + `${ownCommits.join("\n")}\n`, + ); + return new Set([...pathLists.flat(), ...ownCommitPaths].filter((path) => path.length > 0)); }), deleteCheckpointRefs: Effect.fn("GitVcsDriver.checkpoints.deleteCheckpointRefs")( @@ -1637,6 +2197,10 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( }), { discard: true }, ); + const submodules = yield* listSubmodules(input.cwd).pipe(Effect.orElseSucceed(() => [])); + for (const cwd of submodules) { + yield* checkpoints.deleteCheckpointRefs({ cwd, checkpointRefs: input.checkpointRefs }); + } }, ), }; diff --git a/apps/server/src/vcs/GitVcsDriverCore.test.ts b/apps/server/src/vcs/GitVcsDriverCore.test.ts index 8dda59c243dc5..7802a254f127a 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.test.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.test.ts @@ -1,5 +1,7 @@ // @effect-diagnostics nodeBuiltinImport:off - realpathSync.native resolves Windows 8.3 short names, which the Effect realPath does not. +import * as NodeChildProcess from "node:child_process"; import * as NodeFS from "node:fs"; +import * as NodeNet from "node:net"; import * as NodePath from "node:path"; import * as NodeURL from "node:url"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; @@ -10,6 +12,7 @@ import { afterAll, vi } from "vite-plus/test"; import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as Deferred from "effect/Deferred"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; @@ -48,6 +51,7 @@ import { windowsLongPathConfigEnv, } from "./GitVcsDriverCore.ts"; import * as GitVcsDriver from "./GitVcsDriver.ts"; +import { VcsAutomaticFetchInterval } from "./VcsAutomaticFetchInterval.ts"; import { gitConfigEnv } from "./VcsProcess.ts"; const encodeGitCommandError = Schema.encodeEffect(Schema.fromJsonString(GitCommandError)); @@ -342,6 +346,11 @@ const git = ( return result.stdout.trim(); }); +const [, gitMajor, gitMinor] = + /(\d+)\.(\d+)/u.exec(NodeChildProcess.execFileSync("git", ["--version"], { encoding: "utf8" })) ?? + []; +const gitHasReftable = Number(gitMajor) > 2 || (Number(gitMajor) === 2 && Number(gitMinor) >= 45); + const gitWithStdin = ( cwd: string, args: ReadonlyArray, @@ -388,14 +397,15 @@ const initRepoWithCommit = ( return { initialBranch: head.trim().replace(/^ref: refs\/heads\//, "") }; }); -const commitInitialRepo = (cwd: string) => +const commitInitialRepo = (cwd: string, initArgs: ReadonlyArray = []) => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; const pathService = yield* Path.Path; - yield* git(cwd, ["init"]); + yield* git(cwd, ["init", ...initArgs]); + // Newer Git detaches auto-maintenance after commit; its lock file would race the template copy. yield* fileSystem.writeFileString( pathService.join(cwd, ".git", "config"), - "[user]\n\temail = test@test.com\n\tname = Test\n", + "[user]\n\temail = test@test.com\n\tname = Test\n[maintenance]\n\tauto = false\n", { flag: "a" }, ); yield* writeTextFile(cwd, "README.md", "# test\n"); @@ -550,7 +560,10 @@ it.effect.each([ assert.equal(created.failure.exitCode, hookExitCode); } const metadataQueries = spawnedArgs.filter((args) => args.includes("--git-common-dir")); - assert.deepStrictEqual(metadataQueries, [["rev-parse", "--git-common-dir"]]); + assert.deepStrictEqual( + metadataQueries, + Array.from({ length: 2 }, () => ["rev-parse", "--git-common-dir"]), + ); assert.isFalse(spawnedArgs.some((args) => args.includes("symbolic-ref"))); invalidationFailure = "none"; @@ -613,7 +626,7 @@ it.effect.each(["branch", "detached", "nested", "aliased-root", "aliased-nested" newRefName: "feature/cold", }); - assert.equal(spawnedArgs.filter((args) => args.includes("--git-common-dir")).length, 1); + assert.equal(spawnedArgs.filter((args) => args.includes("--git-common-dir")).length, 2); assert.isFalse(spawnedArgs.some((args) => args.includes("--show-toplevel"))); assert.isFalse(spawnedArgs.some((args) => args.includes("symbolic-ref"))); spawnedArgs.length = 0; @@ -675,7 +688,10 @@ it.effect.each(["bare", "unborn"] as const)( assert.equal(result.failure.operation, "GitVcsDriver.createWorktree.createBranch"); assert.equal(result.failure.detail, "git branch failed"); } - assert.equal(spawnedArgs.filter((args) => args.includes("--git-common-dir")).length, 1); + assert.equal( + spawnedArgs.filter((args) => args.includes("--git-common-dir")).length, + state === "bare" ? 2 : 1, + ); assert.isFalse(spawnedArgs.some((args) => args[0] === "symbolic-ref")); spawnedArgs.length = 0; const refs = yield* driver.listRefs({ cwd }); @@ -709,6 +725,46 @@ it.effect("reads the project's own repository when the server inherited GIT_DIR" ).pipe(Effect.provide(layerTest)), ); +it.effect("reads a branch whose name starts with a parenthesis as attached", () => + Effect.scoped( + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + yield* git(cwd, ["switch", "-c", "(topic"]); + const driver = yield* GitVcsDriver.GitVcsDriver; + const status = yield* driver.statusDetailsLocal(cwd, { includeDivergence: false }); + assert.equal(status.branch, "(topic"); + }), + ).pipe(Effect.provide(layerTest)), +); + +it.effect.each(["an existing", "a new"] as const)( + "follows a checkout of %s branch made outside the driver after refs were cached", + (branch) => + Effect.scoped( + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + if (branch === "an existing") yield* git(cwd, ["branch", "feature/outside"]); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* driver.listRefs({ cwd }); + + yield* git( + cwd, + branch === "an existing" + ? ["switch", "feature/outside"] + : ["switch", "-c", "feature/outside"], + ); + + const refs = (yield* driver.listRefs({ cwd })).refs; + const previous = refs.find((ref) => ref.name === initialBranch); + assert.isTrue(refs.find((ref) => ref.name === "feature/outside")?.current); + assert.isFalse(previous?.current); + assert.isNull(previous?.worktreePath ?? null); + }), + ).pipe(Effect.provide(layerTest)), +); + it.effect.each(["on a branch", "detached", "unborn"] as const)( "reads only the repository root for review requests that ignore the branch when %s", (state) => @@ -1068,6 +1124,287 @@ describe("pullCurrentBranch", () => { }; }); + it.effect("resolves a status waiter when Pull claims its queued fetch", () => + Effect.scoped( + Effect.gen(function* () { + const { cwd, remoteHead } = yield* cloneBehindRemote; + const blockers: string[] = []; + for (let index = 0; index < 4; index++) { + const blocker = yield* cloneBehindRemote; + blockers.push(NodeFS.realpathSync.native(blocker.cwd)); + } + const targetPath = NodeFS.realpathSync.native(cwd); + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const release = yield* Deferred.make(); + const blockersStarted = yield* Deferred.make(); + const targetEntered = yield* Deferred.make(); + let running = 0; + let targetFetches = 0; + let waiterSucceeded = false; + const spawner = ChildProcessSpawner.make((command) => + Effect.gen(function* () { + const handle = yield* delegate.spawn(command); + if (!ChildProcess.isStandardCommand(command) || !command.args.includes("fetch")) + return handle; + const commandCwd = String(command.options.cwd); + if (commandCwd === targetPath) targetFetches++; + if (blockers.includes(commandCwd)) { + running++; + if (running === 4) yield* Deferred.succeed(blockersStarted, undefined); + yield* Deferred.await(release); + } + return handle; + }), + ); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options); + if (options.name === "GitVcsDriver.fetchRemoteForStatus.wait") { + const attribute = span.attribute.bind(span); + span.attribute = (key, value) => { + attribute(key, value); + if (key === "statusFetch.joining") Deferred.doneUnsafe(targetEntered, Effect.void); + }; + const end = span.end.bind(span); + span.end = (endTime, exit) => { + waiterSucceeded = Exit.isSuccess(exit); + end(endTime, exit); + }; + } + return span; + }, + }); + yield* Effect.gen(function* () { + for (const blocker of blockers) { + yield* driver.statusDetailsRemote(blocker).pipe(Effect.forkChild); + } + yield* Deferred.await(blockersStarted); + const status = yield* driver + .statusDetailsRemote(cwd) + .pipe(Effect.withTracer(tracer), Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(targetEntered); + const pull = yield* driver.pullCurrentBranch(cwd); + const refreshed = yield* Fiber.join(status); + assert.equal(pull.status, "pulled"); + assert.equal(yield* git(cwd, ["rev-parse", "HEAD"]), remoteHead); + assert.isTrue(waiterSucceeded); + assert.equal(refreshed.behindCount, 0); + assert.equal(targetFetches, 0); + }).pipe(Effect.ensuring(Deferred.succeed(release, undefined))); + }), + ).pipe(Effect.provide(layerTest)), + ); + + const claimedFetchFallbacks = ["diverged", "interrupted", "upstream-only"] as const; + + it.effect.each(claimedFetchFallbacks)( + "falls back to the claimed status fetch when Pull is %s", + (failure) => + Effect.scoped( + Effect.gen(function* () { + const { cwd, updater, remoteHead } = yield* cloneBehindRemote; + if (failure === "upstream-only") { + yield* git(updater, ["push", "origin", "HEAD:refs/heads/other"]); + yield* writeTextFile(cwd, ".git/refs/remotes/origin/other.lock", ""); + } + if (failure === "diverged") { + yield* writeTextFile(cwd, "local.txt", "local\n"); + yield* git(cwd, ["add", "local.txt"]); + yield* git(cwd, ["commit", "-m", "local commit"]); + } + const blockers: string[] = []; + for (let index = 0; index < 4; index++) { + const blocker = yield* cloneBehindRemote; + blockers.push(NodeFS.realpathSync.native(blocker.cwd)); + } + const targetPath = NodeFS.realpathSync.native(cwd); + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const release = yield* Deferred.make(); + const blockersStarted = yield* Deferred.make(); + const targetEntered = yield* Deferred.make(); + const targetFetchStarted = yield* Deferred.make(); + const pullBeforeFetch = yield* Deferred.make(); + const releasePull = yield* Deferred.make(); + let running = 0; + let targetFetches = 0; + let waiterSucceeded = false; + let targetFetchExitCode: number | undefined; + const spawner = ChildProcessSpawner.make((command) => + Effect.gen(function* () { + if (ChildProcess.isStandardCommand(command)) { + const commandCwd = NodeFS.realpathSync.native(String(command.options.cwd)); + if ( + commandCwd === targetPath && + failure === "interrupted" && + command.args[0] === "rev-parse" && + command.args[1] === "HEAD" && + command.args.length === 2 + ) { + yield* Deferred.succeed(pullBeforeFetch, undefined); + yield* Deferred.await(releasePull); + } + } + const handle = yield* delegate.spawn(command); + if (!ChildProcess.isStandardCommand(command) || !command.args.includes("fetch")) + return handle; + const commandCwd = String(command.options.cwd); + if (commandCwd === targetPath) { + targetFetches++; + yield* Deferred.succeed(targetFetchStarted, undefined); + return ChildProcessSpawner.makeHandle({ + ...handle, + exitCode: handle.exitCode.pipe( + Effect.tap((exitCode) => + Effect.sync(() => { + targetFetchExitCode = exitCode; + }), + ), + ), + }); + } + if (blockers.includes(commandCwd)) { + running++; + if (running === 4) yield* Deferred.succeed(blockersStarted, undefined); + yield* Deferred.await(release); + } + return handle; + }), + ); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options); + if (options.name === "GitVcsDriver.fetchRemoteForStatus.wait") { + const attribute = span.attribute.bind(span); + span.attribute = (key, value) => { + attribute(key, value); + if (key === "statusFetch.joining") + Deferred.doneUnsafe(targetEntered, Effect.void); + }; + const end = span.end.bind(span); + span.end = (endTime, exit) => { + waiterSucceeded = Exit.isSuccess(exit); + end(endTime, exit); + }; + } + return span; + }, + }); + yield* Effect.gen(function* () { + for (const blocker of blockers) { + yield* driver.statusDetailsRemote(blocker).pipe(Effect.forkChild); + } + yield* Deferred.await(blockersStarted); + const status = yield* driver + .statusDetailsRemote(cwd) + .pipe(Effect.withTracer(tracer), Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(targetEntered); + if (failure === "interrupted") { + const pull = yield* driver.pullCurrentBranch(cwd).pipe(Effect.forkChild); + yield* Deferred.await(pullBeforeFetch); + yield* Fiber.interrupt(pull); + const exit = yield* Fiber.await(pull); + assert.isTrue(Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)); + } else if (failure === "upstream-only") { + assert.equal((yield* driver.pullCurrentBranch(cwd)).status, "pulled"); + yield* (yield* FileSystem.FileSystem).remove( + `${cwd}/.git/refs/remotes/origin/other.lock`, + ); + } else { + const error = yield* Effect.flip(driver.pullCurrentBranch(cwd)); + assert.equal(error.operation, "GitVcsDriver.pullCurrentBranch.pull"); + } + assert.equal(targetFetches, 0); + yield* Deferred.succeed(release, undefined); + yield* Deferred.await(targetFetchStarted); + assert.isTrue((yield* Fiber.join(status)).isRepo); + assert.isTrue(waiterSucceeded); + assert.equal(targetFetches, 1); + assert.equal(targetFetchExitCode, 0); + if (failure === "upstream-only") { + assert.equal(yield* git(cwd, ["rev-parse", "refs/remotes/origin/other"]), remoteHead); + } + }).pipe( + Effect.ensuring( + Effect.all([ + Deferred.succeed(release, undefined), + Deferred.succeed(releasePull, undefined), + ]), + ), + ); + }), + ).pipe(Effect.provide(layerTest)), + ); + + it.effect("waits for a running status fetch before starting Pull", () => + Effect.scoped( + Effect.gen(function* () { + const { cwd, remoteHead } = yield* cloneBehindRemote; + const targetPath = NodeFS.realpathSync.native(cwd); + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const fetchStarted = yield* Deferred.make(); + const releaseFetch = yield* Deferred.make(); + const pullResolvedCommonDir = yield* Deferred.make(); + const pullStarted = yield* Deferred.make(); + const events: string[] = []; + const spawner = ChildProcessSpawner.make((command) => + Effect.gen(function* () { + if (!ChildProcess.isStandardCommand(command)) return yield* delegate.spawn(command); + const target = NodeFS.realpathSync.native(String(command.options.cwd)) === targetPath; + if (target && command.args.includes("pull")) { + events.push("pull-spawn"); + yield* Deferred.succeed(pullStarted, undefined); + } + const handle = yield* delegate.spawn(command); + if (target && command.args.includes("fetch")) { + yield* Deferred.succeed(fetchStarted, undefined); + yield* Deferred.await(releaseFetch); + yield* handle.exitCode; + events.push("fetch-exit"); + } + return handle; + }), + ); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options); + if (options.name === "resolveGitCommonDir") { + const end = span.end.bind(span); + span.end = (endTime, exit) => { + end(endTime, exit); + Deferred.doneUnsafe(pullResolvedCommonDir, Effect.void); + }; + } + return span; + }, + }); + yield* Effect.gen(function* () { + yield* driver.statusDetailsRemote(cwd).pipe(Effect.forkChild); + yield* Deferred.await(fetchStarted); + const pull = yield* driver + .pullCurrentBranch(cwd) + .pipe(Effect.withTracer(tracer), Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(pullResolvedCommonDir); + assert.isFalse(yield* Deferred.isDone(pullStarted)); + assert.isUndefined(pull.pollUnsafe()); + yield* Deferred.succeed(releaseFetch, undefined); + const result = yield* Fiber.join(pull); + assert.equal(result.status, "pulled"); + assert.equal(yield* git(cwd, ["rev-parse", "HEAD"]), remoteHead); + assert.deepEqual(events, ["fetch-exit", "pull-spawn"]); + }).pipe(Effect.ensuring(Deferred.succeed(releaseFetch, undefined))); + }), + ).pipe(Effect.provide(layerTest)), + ); + it.effect("still fetches other branches on the first pull", () => Effect.gen(function* () { const driver = yield* GitVcsDriver.GitVcsDriver; @@ -1514,6 +1851,53 @@ it.effect("reads worktrees from git that rejects worktree list -z", () => ), ); +it.effect("commits selected files with git that rejects --pathspec-from-file", () => + Effect.scoped( + Effect.gen(function* () { + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + // Git before 2.25 has no `add --pathspec-from-file` and exits with a usage error. + const oldGitSpawner = ChildProcessSpawner.make((command) => { + if (!ChildProcess.isStandardCommand(command)) { + return Effect.die("expected a standard Git command"); + } + if (command.args.at(-1) === "version") { + return Effect.succeed(makeSuccessfulHandle("git version 2.24.4\n")); + } + if (command.args.some((arg) => String(arg).startsWith("--pathspec-from-file"))) { + return Effect.succeed(makeFailedHandle("error: unknown option `pathspec-from-file=-'")); + } + return delegate.spawn(command); + }); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, oldGitSpawner), + ); + yield* Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const selected = ["selected[1].txt", "-dash.txt", "with space.txt"]; + for (const file of selected) yield* writeTextFile(cwd, file, "selected\n"); + yield* writeTextFile(cwd, "left.txt", "left\n"); + + yield* driver.commit(cwd, "Selected", "", { stage: { filePaths: selected } }); + + assert.sameMembers( + (yield* git(cwd, ["show", "--pretty=", "--name-only", "-z", "HEAD"])) + .split("\0") + .filter(Boolean), + selected, + ); + assert.equal(yield* git(cwd, ["status", "--porcelain"]), "?? left.txt"); + }).pipe(Effect.provideService(GitVcsDriver.GitVcsDriver, driver)); + }), + ).pipe( + Effect.provide( + layerServerConfig.pipe( + Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), + ), + ), + ), +); + it.effect("does not report a separate git dir as the worktree of its branch", () => Effect.scoped( Effect.gen(function* () { @@ -1731,6 +2115,37 @@ it.effect("does not fetch for status when the branch tracks a local branch", () ).pipe(Effect.provide(layerTest)), ); +it.effect("does not fetch for status when the automatic fetch interval is zero", () => + Effect.scoped( + Effect.gen(function* () { + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const fetches: Array> = []; + const spawner = ChildProcessSpawner.make((command) => { + if (ChildProcess.isStandardCommand(command) && command.args.includes("fetch")) { + fetches.push(command.args.map(String)); + } + return delegate.spawn(command); + }); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provideService(VcsAutomaticFetchInterval, Effect.succeed(Duration.zero)), + ); + const cwd = yield* makeTmpDir(); + const remote = yield* makeTmpDir("git-vcs-driver-remote-"); + const { initialBranch } = yield* initRepoWithCommit(cwd); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["push", "--set-upstream", "origin", initialBranch]); + + yield* driver.statusDetails(cwd); + const status = yield* driver.statusDetailsRemote(cwd); + + assert.equal(status.upstreamRef, `origin/${initialBranch}`); + assert.deepStrictEqual(fetches, []); + }), + ).pipe(Effect.provide(layerTest)), +); + it.effect("lets a worktree checkout that is still working run past five minutes", () => Effect.gen(function* () { const checkoutStarted = yield* Deferred.make(); @@ -1739,7 +2154,8 @@ it.effect("lets a worktree checkout that is still working run past five minutes" Effect.gen(function* () { if (!ChildProcess.isStandardCommand(command)) return yield* Effect.die("expected Git command"); - if (!command.args.includes("worktree")) return makeSuccessfulHandle(""); + if (!command.args.includes("worktree") || command.args.includes("-h")) + return makeSuccessfulHandle(""); yield* Deferred.succeed(checkoutStarted, undefined); return ChildProcessSpawner.makeHandle({ ...makeSuccessfulHandle(""), @@ -1846,6 +2262,53 @@ it.effect("cancels a slow submodule checkout without reporting best-effort succe ), ); +// regression: #18537 the fetch before creating a worktree outgrew the 30s default +it.effect.each(["healthy", "hung"] as const)("bounds a %s remote fetch at five minutes", (fetch) => + Effect.gen(function* () { + const started = yield* Deferred.make(); + const finished = yield* Deferred.make(); + const spawner = ChildProcessSpawner.make((command) => + Effect.succeed( + ChildProcess.isStandardCommand(command) && command.args[0] === "fetch" + ? ChildProcessSpawner.makeHandle({ + ...makeSuccessfulHandle(""), + exitCode: Deferred.succeed(started, undefined).pipe( + Effect.andThen(Deferred.await(finished)), + Effect.as(ChildProcessSpawner.ExitCode(0)), + ), + }) + : makeNonRepositoryHandle(), + ), + ); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + const fetching = yield* driver + .fetchRemote({ cwd: yield* makeTmpDir(), remoteName: "origin" }) + .pipe(Effect.result, Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(started); + yield* TestClock.adjust("31 seconds"); + if (fetch === "healthy") yield* Deferred.succeed(finished, undefined); + else { + yield* TestClock.adjust("268 seconds"); + assert.isUndefined(fetching.pollUnsafe()); + yield* TestClock.adjust("1 second"); + } + + const result = yield* Fiber.join(fetching); + assert.equal( + Result.isFailure(result) ? result.failure.detail : "fetched", + fetch === "healthy" ? "fetched" : "Git command timed out.", + ); + }).pipe( + Effect.provide( + layerServerConfig.pipe( + Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), + ), + ), + ), +); + describe("rejected git refs", () => { const remoteUrl = "https://user:token-123@git.example.invalid/org/secret-repo.git"; const failedPush = `error: failed to push some refs to '${remoteUrl}'`; @@ -2142,6 +2605,7 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const spawner = ChildProcessSpawner.make((command) => { if ( ChildProcess.isStandardCommand(command) && + !command.args.includes("-h") && (command.args.includes("worktree") || command.args[0] === "branch" || command.args[0] === "checkout") @@ -2176,6 +2640,55 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("explains a non-fast-forward push from a host with a translated Git", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const remote = yield* makeTmpDir("git-remote-"); + const other = yield* makeTmpDir("git-other-"); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["push", "--set-upstream", "origin", initialBranch]); + yield* git(other, ["clone", "--branch", initialBranch, remote, "."]); + yield* git(other, [ + "-c", + "user.email=t@t", + "-c", + "user.name=T", + "commit", + "--allow-empty", + "-m", + "remote", + ]); + yield* git(other, ["push", "origin", initialBranch]); + yield* git(cwd, ["commit", "--allow-empty", "-m", "local"]); + const locales: Array = []; + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const spawner = ChildProcessSpawner.make((command) => { + if (ChildProcess.isStandardCommand(command) && command.args.includes("push")) { + locales.push(command.options.env?.LANGUAGE); + } + return delegate.spawn(command); + }); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provide(layerServerConfig), + ); + const hostEnv = yield* HostProcess.Environment; + + const error = yield* driver + .pushCurrentBranch(cwd, null) + .pipe( + Effect.provideService(HostProcess.Environment, { ...hostEnv, LANGUAGE: "de" }), + Effect.flip, + ); + + assert.equal(error.reason, "push_rejected_non_fast_forward"); + assert.isNotEmpty(locales); + for (const locale of locales) assert.equal(locale, "C"); + }), + ); + it.effect("lets a slow remote branch checkout finish without blocking config writes", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -2741,7 +3254,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }); const source = dirty.sources.find((source) => source.kind === "working-tree")!; assert.deepStrictEqual(source.files, [ - { path: "untracked.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "untracked.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ]); assert.include(source.diff, "+untracked content"); }), @@ -2847,7 +3366,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.isTrue(metadata.sources[0]!.patchesDeferred); assert.isEmpty(metadata.sources[0]!.diff); assert.deepStrictEqual(metadata.sources[0]!.files, [ - { path: "README.md", previousPath: null, additions: 1, deletions: 1 }, + { + path: "README.md", + previousPath: null, + changeType: "change", + additions: 1, + deletions: 1, + }, ]); }), ); @@ -2893,6 +3418,35 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("keeps the review working beside a nested repository with no commits", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* git(cwd, ["init", "scaffold"]); + const committed = `${cwd}/committed`; + yield* (yield* FileSystem.FileSystem).makeDirectory(committed); + yield* initRepoWithCommit(committed); + yield* writeTextFile(cwd, "README.md", "# tracked change\n"); + yield* writeTextFile(cwd, "untracked.txt", "untracked contents\n"); + + const preview = yield* driver.getReviewDiffPreview({ cwd, ignoreWhitespace: false }); + const diff = preview.sources.find((source) => source.kind === "working-tree")?.diff ?? ""; + + assert.include(diff, "+# tracked change"); + assert.include(diff, "+untracked contents"); + assert.notInclude(diff, "scaffold"); + assert.include(diff, "diff --git a/committed b/committed"); + + const status = yield* driver.statusDetailsLocal(cwd, { includeBranchChanges: true }); + assert.deepStrictEqual(status.branchChanges, { + baseRef: null, + insertions: 3, + deletions: 1, + }); + }), + ); + it.effect("keeps untracked filenames with pathspec magic in the review", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -2917,7 +3471,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }); const scopedSource = scoped.sources.find((source) => source.kind === "working-tree")!; assert.deepStrictEqual(scopedSource.files, [ - { path: ":(exclude)after.ts", previousPath: null, additions: 1, deletions: 0 }, + { + path: ":(exclude)after.ts", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ]); assert.include(scopedSource.diff, "+literal pathspec contents"); assert.notInclude(scopedSource.diff, "ordinary.ts"); @@ -2940,7 +3500,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const source = preview.sources.find((candidate) => candidate.kind === "working-tree")!; assert.deepStrictEqual(source.files, [ - { path: "excluded/new.ts", previousPath: null, additions: 1, deletions: 0 }, + { + path: "excluded/new.ts", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ]); assert.include(source.diff, "+outside the cone"); }), @@ -2983,8 +3549,14 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const source = preview.sources.find((candidate) => candidate.kind === "working-tree")!; assert.deepStrictEqual(source.files, [ - { path: "after.ts", previousPath: "before.ts", additions: 1, deletions: 1 }, - ]); + { + path: "after.ts", + previousPath: "before.ts", + changeType: "rename-changed", + additions: 1, + deletions: 1, + }, + ]); const scoped = yield* driver.getReviewDiffPreview({ cwd, file: { path: "after.ts", previousPath: "before.ts", sourceKind: "working-tree" }, @@ -3103,7 +3675,15 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const preview = yield* driver.getReviewDiffPreview({ cwd }); assert.deepStrictEqual( preview.sources.find((source) => source.kind === "working-tree")!.files, - [{ path: "untracked.txt", previousPath: null, additions: 1, deletions: 0 }], + [ + { + path: "untracked.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, + ], ); assert.strictEqual(yield* fs.readFileString(path.join(cwd, ".git/filter-runs")), ""); assert.deepStrictEqual(yield* fs.readFile(indexPath), originalIndex); @@ -3143,8 +3723,20 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const preview = yield* driver.getReviewDiffPreview({ cwd }); const dirty = preview.sources.find((source) => source.kind === "working-tree")!; assert.deepStrictEqual(dirty.files, [ - { path: "tracked.txt", previousPath: null, additions: 1, deletions: 1 }, - { path: "untracked.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "tracked.txt", + previousPath: null, + changeType: "change", + additions: 1, + deletions: 1, + }, + { + path: "untracked.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ]); assert.include(dirty.diff, "-before"); assert.include(dirty.diff, "+after!"); @@ -3208,14 +3800,44 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { } assert.deepStrictEqual(branch.files, [ - { path: "a-large.txt", previousPath: null, additions: 4000, deletions: 0 }, - { path: "untracked.txt", previousPath: null, additions: 4000, deletions: 0 }, - { path: "z-last.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "a-large.txt", + previousPath: null, + changeType: "new", + additions: 4000, + deletions: 0, + }, + { + path: "untracked.txt", + previousPath: null, + changeType: "new", + additions: 4000, + deletions: 0, + }, + { path: "z-last.txt", previousPath: null, changeType: "new", additions: 1, deletions: 0 }, ]); assert.deepStrictEqual(dirty.files, [ - { path: "a-large.txt", previousPath: null, additions: 4000, deletions: 4000 }, - { path: "untracked.txt", previousPath: null, additions: 4000, deletions: 0 }, - { path: "z-last.txt", previousPath: null, additions: 1, deletions: 1 }, + { + path: "a-large.txt", + previousPath: null, + changeType: "change", + additions: 4000, + deletions: 4000, + }, + { + path: "untracked.txt", + previousPath: null, + changeType: "new", + additions: 4000, + deletions: 0, + }, + { + path: "z-last.txt", + previousPath: null, + changeType: "change", + additions: 1, + deletions: 1, + }, ]); }), ); @@ -3226,10 +3848,12 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const { initialBranch } = yield* initRepoWithCommit(cwd); const driver = yield* GitVcsDriver.GitVcsDriver; yield* writeTextFile(cwd, "mode-only.sh", "echo unchanged\n"); - yield* git(cwd, ["add", "mode-only.sh"]); + yield* writeTextFile(cwd, "removed.txt", "removed\n"); + yield* git(cwd, ["add", "mode-only.sh", "removed.txt"]); yield* git(cwd, ["commit", "-m", "add executable candidate"]); yield* git(cwd, ["checkout", "-b", "feature/paths"]); yield* git(cwd, ["mv", "README.md", "renamed.md"]); + yield* git(cwd, ["rm", "removed.txt"]); yield* writeTextFile(cwd, "[literal].txt", "literal\n"); yield* writeTextFile(cwd, " leading.txt", "whitespace path\n"); yield* writeTextFile(cwd, "l.txt", "other\n"); @@ -3269,12 +3893,28 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.deepInclude(branch.files ?? [], { path: "renamed.md", previousPath: "README.md", + changeType: "rename-pure", + additions: 0, + deletions: 0, + }); + assert.deepInclude(branch.files ?? [], { + path: "removed.txt", + previousPath: null, + changeType: "deleted", + additions: 0, + deletions: 1, + }); + assert.deepInclude(branch.files ?? [], { + path: "mode-only.sh", + previousPath: null, + changeType: "change", additions: 0, deletions: 0, }); assert.deepInclude(branch.files ?? [], { path: "binary.dat", previousPath: null, + changeType: "new", additions: 0, deletions: 0, }); @@ -3282,6 +3922,7 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.deepInclude(branch.files ?? [], { path: "tab\tand\nnewline.txt", previousPath: null, + changeType: "new", additions: 1, deletions: 0, }); @@ -3300,8 +3941,14 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const preview = yield* driver.getReviewDiffPreview({ cwd }); const dirty = preview.sources.find((source) => source.kind === "working-tree")!; assert.deepStrictEqual(dirty.files, [ - { path: "staged.txt", previousPath: null, additions: 1, deletions: 0 }, - { path: "untracked.txt", previousPath: null, additions: 1, deletions: 0 }, + { path: "staged.txt", previousPath: null, changeType: "new", additions: 1, deletions: 0 }, + { + path: "untracked.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ]); assert.include(dirty.diff, "b/staged.txt"); assert.include(dirty.diff, "b/untracked.txt"); @@ -3485,6 +4132,77 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("loads working-tree contents as Git diffs them under the ident attribute", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + const fileSystem = yield* FileSystem.FileSystem; + const filePath = (yield* Path.Path).join(cwd, "README.md"); + const committed = "$Id$\nfirst\nsecond\n"; + yield* writeTextFile(cwd, ".gitattributes", "README.md ident\n"); + yield* writeTextFile(cwd, "README.md", committed); + yield* writeTextFile(cwd, "PLAIN.md", "plain\n"); + yield* git(cwd, ["add", "."]); + yield* git(cwd, ["commit", "-m", "add ident keyword"]); + // Checkout smudges the keyword to `$Id: $`. + yield* fileSystem.remove(filePath); + yield* git(cwd, ["checkout-index", "--force", "README.md"]); + const smudged = yield* fileSystem.readFileString(filePath); + assert.match(smudged, /^\$Id: [0-9a-f]{40} \$\n/); + yield* writeTextFile(cwd, "README.md", smudged.replace("first", "edited")); + yield* writeTextFile(cwd, "PLAIN.md", "$Id: literal $\n"); + const disk = yield* fileSystem.readFile(filePath); + + const contents = yield* driver.getReviewDiffFileContents( + makeReviewDiffFileContentsInput(cwd), + ); + const plain = yield* driver.getReviewDiffFileContents( + makeReviewDiffFileContentsInput(cwd, { oldPath: "PLAIN.md", newPath: "PLAIN.md" }), + ); + + assert.deepStrictEqual(contents, { + oldContents: committed, + newContents: "$Id$\nedited\nsecond\n", + }); + assert.strictEqual(plain.newContents, "$Id: literal $\n"); + yield* writeTextFile(cwd, "README.md", "$Id: x $Id: y $\n"); + const sharedDollar = yield* driver.getReviewDiffFileContents( + makeReviewDiffFileContentsInput(cwd), + ); + assert.strictEqual(sharedDollar.newContents, "$Id$Id: y $\n"); + yield* git(cwd, ["config", "filter.unrelated.clean", "touch filter-ran; cat"]); + yield* writeTextFile(cwd, ".gitattributes", "README.md ident text eol=lf\n"); + yield* writeTextFile( + cwd, + "README.md", + smudged.replace("first", "edited").replaceAll("\n", "\r\n"), + ); + const crlf = yield* driver.getReviewDiffFileContents(makeReviewDiffFileContentsInput(cwd)); + assert.strictEqual(crlf.newContents, "$Id$\r\nedited\r\nsecond\r\n"); + yield* fileSystem.writeFile(filePath, disk); + yield* writeTextFile(cwd, ".gitattributes", "README.md ident=set\n"); + const literal = yield* driver.getReviewDiffFileContents( + makeReviewDiffFileContentsInput(cwd), + ); + assert.strictEqual(literal.newContents, smudged.replace("first", "edited")); + yield* writeTextFile(cwd, ".gitattributes", "README.md ident filter=probe\n"); + yield* git(cwd, ["config", "filter.probe.clean", "touch filter-ran; cat"]); + const filtered = yield* driver.getReviewDiffFileContents( + makeReviewDiffFileContentsInput(cwd), + ); + assert.strictEqual(filtered.newContents, literal.newContents); + yield* writeTextFile(cwd, ".gitattributes", "README.md ident filter=unspecified\n"); + yield* git(cwd, ["config", "filter.unspecified.clean", "touch filter-ran; cat"]); + const reservedFilter = yield* driver.getReviewDiffFileContents( + makeReviewDiffFileContentsInput(cwd), + ); + assert.strictEqual(reservedFilter.newContents, literal.newContents); + assert.isFalse(yield* fileSystem.exists((yield* Path.Path).join(cwd, "filter-ran"))); + assert.deepStrictEqual(yield* fileSystem.readFile(filePath), disk); + }), + ); + it.effect("Changes combines commits, uncommitted edits, and untracked files", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -3501,7 +4219,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const clean = yield* driver.getReviewDiffPreview({ cwd, baseRef: initialBranch }); assert.deepStrictEqual(sourceFiles(clean, "working-tree"), []); assert.deepStrictEqual(sourceFiles(clean, "branch-range"), [ - { path: "README.md", previousPath: null, additions: 1, deletions: 0 }, + { + path: "README.md", + previousPath: null, + changeType: "change", + additions: 1, + deletions: 0, + }, ]); yield* writeTextFile(cwd, "README.md", "# test\ncommitted\ndirty\n"); @@ -3509,14 +4233,26 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const dirty = yield* driver.getReviewDiffPreview({ cwd, baseRef: initialBranch }); const changes = dirty.sources.find((source) => source.kind === "branch-range")!; assert.deepStrictEqual(changes.files, [ - { path: "README.md", previousPath: null, additions: 2, deletions: 0 }, - { path: "new.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "README.md", + previousPath: null, + changeType: "change", + additions: 2, + deletions: 0, + }, + { path: "new.txt", previousPath: null, changeType: "new", additions: 1, deletions: 0 }, ]); assert.strictEqual(changes.diff.match(/^diff --git a\/README\.md /gm)?.length, 1); assert.include(changes.diff, "+dirty"); assert.deepStrictEqual(sourceFiles(dirty, "working-tree"), [ - { path: "README.md", previousPath: null, additions: 1, deletions: 0 }, - { path: "new.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "README.md", + previousPath: null, + changeType: "change", + additions: 1, + deletions: 0, + }, + { path: "new.txt", previousPath: null, changeType: "new", additions: 1, deletions: 0 }, ]); }), ); @@ -3545,7 +4281,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const changes = preview.sources.find((source) => source.kind === "branch-range")!; assert.strictEqual(changes.baseRef, "origin/main"); assert.deepStrictEqual(changes.files, [ - { path: "unpushed.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "unpushed.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ]); }), ); @@ -3584,7 +4326,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const changes = preview.sources.find((source) => source.kind === "branch-range")!; assert.strictEqual(changes.baseRef, baseRef); assert.deepStrictEqual(changes.files, [ - { path: "feature.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "feature.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ]); } @@ -3618,8 +4366,20 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const changes = preview.sources.find((source) => source.kind === "branch-range")!; assert.strictEqual(changes.baseRef, "refs/remotes/upstream/main"); assert.deepStrictEqual(changes.files, [ - { path: "feature.txt", previousPath: null, additions: 1, deletions: 0 }, - { path: "upstream.txt", previousPath: null, additions: 1, deletions: 1 }, + { + path: "feature.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, + { + path: "upstream.txt", + previousPath: null, + changeType: "change", + additions: 1, + deletions: 1, + }, ]); const contents = yield* driver.getReviewDiffFileContents( makeReviewDiffFileContentsInput(cwd, { @@ -3674,7 +4434,11 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { // Base config reads start only after status completes; hold every one until the rename lands. const spawner = ChildProcessSpawner.make((command) => Effect.gen(function* () { - if (ChildProcess.isStandardCommand(command) && command.args[0] === "config") { + if ( + ChildProcess.isStandardCommand(command) && + command.args[0] === "config" && + !command.args.includes("status.showUntrackedFiles") + ) { if (!(yield* Ref.getAndSet(renameClaimed, true))) { yield* git(cwd, ["branch", "-m", "t3/temporary", "feature/named"]).pipe( Effect.provideService(GitVcsDriver.GitVcsDriver, outer), @@ -3731,7 +4495,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const changes = preview.sources.find((source) => source.kind === "branch-range")!; assert.strictEqual(changes.baseRef, baseRef); assert.deepStrictEqual(changes.files, [ - { path: "README.md", previousPath: null, additions: 1, deletions: 0 }, + { + path: "README.md", + previousPath: null, + changeType: "change", + additions: 1, + deletions: 0, + }, ]); const scoped = yield* driver.getReviewDiffPreview({ cwd, @@ -3780,7 +4550,15 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const explicit = yield* driver.getReviewDiffPreview({ cwd, baseRef: initialBranch }); assert.deepStrictEqual( explicit.sources.find((source) => source.kind === "branch-range")!.files, - [{ path: "detached.txt", previousPath: null, additions: 1, deletions: 0 }], + [ + { + path: "detached.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, + ], ); const implicit = yield* driver.getReviewDiffPreview({ cwd }); assert.deepStrictEqual( @@ -3810,7 +4588,7 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const changes = preview.sources.find((source) => source.kind === "branch-range")!; assert.isNull(changes.baseRef); assert.deepStrictEqual(changes.files, [ - { path: "new.txt", previousPath: null, additions: 2, deletions: 0 }, + { path: "new.txt", previousPath: null, changeType: "new", additions: 2, deletions: 0 }, ]); const status = yield* driver.statusDetailsLocal(cwd, { includeBranchChanges: true }); assert.deepStrictEqual(status.branchChanges, { @@ -3841,7 +4619,15 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const preview = yield* driver.getReviewDiffPreview({ cwd, baseRef: initialBranch }); assert.deepStrictEqual( preview.sources.find((source) => source.kind === "branch-range")!.files, - [{ path: "feature.txt", previousPath: null, additions: 1, deletions: 0 }], + [ + { + path: "feature.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, + ], ); } }), @@ -3873,6 +4659,25 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("reports a local upstream without a slash and its divergence", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const initialBranch = yield* git(cwd, ["symbolic-ref", "--short", "HEAD"]); + yield* git(cwd, ["branch", "feature"]); + yield* git(cwd, ["commit", "--allow-empty", "-m", "advance base"]); + yield* git(cwd, ["checkout", "feature"]); + yield* git(cwd, ["branch", `--set-upstream-to=${initialBranch}`]); + const driver = yield* GitVcsDriver.GitVcsDriver; + + const remoteStatus = yield* driver.statusDetailsRemote(cwd, { refreshUpstream: false }); + assert.equal(remoteStatus.upstreamRef, initialBranch); + assert.equal(remoteStatus.hasUpstream, true); + assert.equal(remoteStatus.aheadCount, 0); + assert.equal(remoteStatus.behindCount, 1); + }), + ); + it.effect("does not open a project folder laid out as a bare repository", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -3948,6 +4753,7 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { { path: "repeated.txt", previousPath: null, + changeType: "change", additions: Number(additions), deletions: Number(removals), }, @@ -3998,6 +4804,29 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("reads Changes totals without rewriting the real index for stat-dirty files", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* writeTextFile(cwd, "tracked.txt", "same content\n"); + yield* git(cwd, ["add", "tracked.txt"]); + yield* git(cwd, ["commit", "-m", "add tracked file"]); + yield* fileSystem.utimes(path.join(cwd, "tracked.txt"), 1_000_000_000, 1_000_000_000); + const indexPath = path.join(cwd, ".git", "index"); + const originalIndex = yield* fileSystem.readFile(indexPath); + + const status = yield* driver.statusDetailsLocal(cwd, { includeBranchChanges: true }); + + assert.strictEqual(status.branchChanges?.insertions, 0); + assert.strictEqual(status.branchChanges?.deletions, 0); + // Writing the index takes index.lock, which makes concurrent user Git writes fail. + assert.deepStrictEqual(yield* fileSystem.readFile(indexPath), originalIndex); + }), + ); + it.effect("reads Changes totals with untracked files when requested", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -4094,6 +4923,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { yield* git(cwd, ["checkout", "-b", "feature/many-untracked"]); yield* writeTextFile(cwd, ".git/info/exclude", "ignored.txt\n"); yield* writeTextFile(cwd, "bulk[1]/ignored.txt", "ignored\n"); + const clean = yield* driver.statusDetailsLocal(cwd, { includeBranchChanges: true }); + assert.deepStrictEqual(clean.branchChanges, { + baseRef: "main", + insertions: 0, + deletions: 0, + }); + assert.deepStrictEqual(untrackedScanRoots, []); yield* Effect.forEach( Array.from({ length: 1_001 }, (_, index) => `bulk[1]/${index}.txt`), (file) => writeTextFile(cwd, file, "one\ntwo\n"), @@ -4118,13 +4954,15 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { deletions: 0, }); assert.deepStrictEqual(untrackedScanRoots, [ - scenario === "normal" ? [":(top,literal)bulk[1]/"] : [], + scenario === "normal" || scenario === "no" ? [":(top,literal)bulk[1]/"] : [], ]); - if (scenario === "normal") { + if (scenario === "normal" || scenario === "no") { assert.deepStrictEqual( status.workingTree.files.map((file) => file.path), ["bulk[1]/"], ); + } else if (scenario === "all") { + assert.lengthOf(status.workingTree.files, 1_001); } } }), @@ -4219,6 +5057,51 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { insertions: 1, deletions: 1, }); + const preview = yield* driver.getReviewDiffPreview({ cwd, metadataOnly: true }); + assert.deepStrictEqual( + preview.sources.find((source) => source.kind === "working-tree")?.files, + [ + { + path: "asset.data", + previousPath: null, + changeType: "deleted", + additions: 0, + deletions: 1, + }, + { + path: "ordinary.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, + ], + ); + }), + ); + + it.effect("keeps an oversized untracked file out of the review diff", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + const fileSystem = yield* FileSystem.FileSystem; + const pathService = yield* Path.Path; + yield* writeTextFile(cwd, "README.md", "# tracked change\n"); + yield* writeTextFile(cwd, "ordinary.txt", "new\n"); + yield* writeTextFile(cwd, "large.data", ""); + // Sparse: the logical size is over the budget without allocating it. + yield* fileSystem.truncate(pathService.join(cwd, "large.data"), 257 * 1024 * 1024); + + const preview = yield* driver.getReviewDiffPreview({ cwd, metadataOnly: true }); + + const source = preview.sources.find((source) => source.kind === "working-tree")!; + assert.isUndefined(source.files); + assert.isUndefined(source.patchesDeferred); + assert.isTrue(source.truncated); + assert.include(source.diff, "+# tracked change"); + assert.include(source.diff, "b/ordinary.txt"); + assert.notInclude(source.diff, "large.data"); }), ); @@ -4357,7 +5240,7 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); - it.effect("reports remote status on unborn HEAD without failing", () => + it.effect("reports status on unborn HEAD without failing or dropping its upstream", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); const driver = yield* GitVcsDriver.GitVcsDriver; @@ -4371,6 +5254,17 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.equal(status.hasUpstream, false); assert.equal(status.aheadCount, 0); assert.equal(status.behindCount, 0); + + const seed = yield* makeTmpDir(); + const remote = yield* makeTmpDir("git-remote-"); + yield* initRepoWithCommit(seed); + yield* git(remote, ["init", "--bare"]); + yield* git(seed, ["push", remote, `HEAD:refs/heads/${initialBranch}`]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["fetch", "origin"]); + yield* git(cwd, ["config", `branch.${initialBranch}.remote`, "origin"]); + yield* git(cwd, ["config", `branch.${initialBranch}.merge`, `refs/heads/${initialBranch}`]); + assert.equal((yield* driver.statusDetailsLocal(cwd)).hasUpstream, true); }), ); @@ -4441,6 +5335,100 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("a ref snapshot detects branch identity, ref and ancestry changes", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const remote = yield* makeTmpDir("git-vcs-driver-remote-"); + const updater = yield* makeTmpDir("git-vcs-driver-updater-"); + const { initialBranch } = yield* initRepoWithCommit(cwd); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["push", "-u", "origin", initialBranch]); + yield* git(updater, ["clone", remote, "."]); + yield* git(updater, ["config", "user.email", "test@test.com"]); + yield* git(updater, ["config", "user.name", "Test"]); + const driver = yield* GitVcsDriver.GitVcsDriver; + const snapshot = driver.readRefSnapshot(cwd); + + // Newer Git can create origin/HEAD on the first fetch. + yield* git(cwd, ["fetch", "origin"]); + const initial = yield* snapshot; + assert.isNotNull(initial); + yield* git(cwd, ["fetch", "origin"]); + assert.equal(yield* snapshot, initial); + + yield* git(cwd, ["branch", "same-commit"]); + const beforeCheckout = yield* snapshot; + const refsBeforeCheckout = yield* git(cwd, ["show-ref", "--head"]); + yield* git(cwd, ["checkout", "same-commit"]); + assert.equal(yield* git(cwd, ["show-ref", "--head"]), refsBeforeCheckout); + const afterCheckout = yield* snapshot; + assert.isNotNull(afterCheckout); + assert.notEqual(afterCheckout, beforeCheckout); + yield* git(cwd, ["checkout", "--detach"]); + assert.equal(yield* git(cwd, ["show-ref", "--head"]), refsBeforeCheckout); + const detached = yield* snapshot; + assert.isNotNull(detached); + assert.notEqual(detached, beforeCheckout); + assert.notEqual(detached, afterCheckout); + assert.equal(yield* snapshot, detached); + yield* git(cwd, ["checkout", initialBranch]); + assert.equal(yield* snapshot, beforeCheckout); + yield* git(cwd, ["branch", "-d", "same-commit"]); + + yield* writeTextFile(updater, "remote.txt", "remote\n"); + yield* git(updater, ["add", "remote.txt"]); + yield* git(updater, ["commit", "-m", "remote commit"]); + yield* git(updater, ["push", "origin", initialBranch]); + assert.equal(yield* snapshot, initial); + yield* git(cwd, ["fetch", "origin"]); + const fetched = yield* snapshot; + assert.notEqual(fetched, initial); + + const shallow = yield* makeTmpDir("git-vcs-driver-shallow-"); + yield* git(shallow, ["clone", "--depth=1", NodeURL.pathToFileURL(remote).href, "."]); + const shallowRefs = yield* git(shallow, ["show-ref", "--head"]); + const shallowSnapshot = yield* driver.readRefSnapshot(shallow); + assert.isNotNull(shallowSnapshot); + const shallowWorktree = yield* makeTmpDir("git-vcs-driver-shallow-worktree-"); + yield* git(shallow, ["worktree", "add", "--detach", shallowWorktree]); + const worktreeSnapshot = yield* driver.readRefSnapshot(shallowWorktree); + assert.isNotNull(worktreeSnapshot); + yield* git(shallow, ["fetch", "--unshallow", "origin"]); + assert.equal(yield* git(shallow, ["show-ref", "--head"]), shallowRefs); + const unshallowSnapshot = yield* driver.readRefSnapshot(shallow); + const unshallowWorktreeSnapshot = yield* driver.readRefSnapshot(shallowWorktree); + assert.isNotNull(unshallowSnapshot); + assert.isNotNull(unshallowWorktreeSnapshot); + assert.notEqual(unshallowSnapshot, shallowSnapshot); + assert.notEqual(unshallowWorktreeSnapshot, worktreeSnapshot); + + yield* git(cwd, ["checkout", "--detach"]); + yield* git(cwd, ["commit", "--allow-empty", "-m", "detached"]); + assert.notEqual(yield* snapshot, fetched); + + const head = yield* git(cwd, ["rev-parse", "HEAD"]); + yield* driver.execute({ + operation: "GitVcsDriver.test.git", + cwd, + args: ["update-ref", "--stdin"], + stdin: Array.from( + { length: 16_000 }, + (_, index) => `create refs/heads/snapshot-${String(index).padStart(5, "0")} ${head}\n`, + ).join(""), + }); + yield* git(cwd, ["pack-refs", "--all"]); + const large = yield* snapshot; + assert.isNotNull(large); + assert.isAbove(Buffer.byteLength(large ?? "", "utf8"), 1_000_000); + assert.equal(yield* snapshot, large); + yield* git(cwd, ["update-ref", "refs/heads/snapshot-00000", "HEAD~1"]); + assert.notEqual(yield* snapshot, large); + + assert.isNull(yield* driver.readRefSnapshot(yield* makeTmpDir())); + }), + ); + it.effect("does not start Git auto-maintenance from background upstream fetches", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -4533,6 +5521,33 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("does not report a default branch the remote deleted after the clone", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const remote = yield* makeTmpDir("git-vcs-driver-origin-"); + yield* initRepoWithCommit(cwd); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["branch", "-M", "development"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["push", "origin", "development", "development:main"]); + yield* git(cwd, ["remote", "set-head", "origin", "development"]); + yield* git(remote, ["--git-dir=.", "symbolic-ref", "HEAD", "refs/heads/main"]); + yield* git(cwd, ["push", "origin", "--delete", "development"]); + yield* git(cwd, ["fetch", "--prune", "origin"]); + + const driver = yield* GitVcsDriver.GitVcsDriver; + const defaultBranch = yield* driver.resolveDefaultBranchName(cwd, "origin"); + + assert.notEqual(defaultBranch, "development"); + for (const refName of ["origin/HEAD", "HEAD~1", "main^", "@{-1}", "-", ":/fix"]) + assert.isNull(yield* driver.resolveWorktreeBaseRemote({ cwd, refName })); + assert.deepEqual( + yield* driver.resolveWorktreeBaseRemote({ cwd, refName: "feature/a\u00a0b" }), + { remoteName: "origin", branchName: "feature/a\u00a0b" }, + ); + }), + ); + it.effect.each(["checkout", "bare-backed worktree"] as const)( "uses the only remote's HEAD for default-branch detection without origin in a %s", (layout) => @@ -4668,25 +5683,21 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); - it.effect("reports staged file counts on unborn HEAD without failing", () => + it.effect("reports net file counts from the empty tree on unborn HEAD", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); const driver = yield* GitVcsDriver.GitVcsDriver; yield* driver.initRepo({ cwd }); - yield* git(cwd, ["config", "user.email", "test@test.com"]); - yield* git(cwd, ["config", "user.name", "Test"]); yield* writeTextFile(cwd, "initial.ts", "// first file\n"); yield* git(cwd, ["add", "initial.ts"]); + yield* writeTextFile(cwd, "initial.ts", "// edited after staging\n"); - const status = yield* driver.statusDetails(cwd); + const status = yield* driver.statusDetailsLocal(cwd); assert.equal(status.isRepo, true); - assert.equal(status.workingTree.files.length, 1); - const file = status.workingTree.files[0]; - if (file) { - assert.equal(file.path, "initial.ts"); - assert.equal(file.insertions, 1); - } + assert.deepStrictEqual(status.workingTree.files, [ + { path: "initial.ts", insertions: 1, deletions: 0 }, + ]); }), ); }); @@ -4939,6 +5950,58 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect.each(["untracked", "other-remote"] as const)( + "refuses a remote switch that collides with a local branch (%s)", + (tracking) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const remote = yield* makeTmpDir("git-remote-"); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "upstream", remote]); + yield* git(cwd, ["push", "upstream", "HEAD:refs/heads/feature"]); + yield* git(cwd, ["fetch", "upstream"]); + yield* git(cwd, ["checkout", "--no-track", "-b", "feature"]); + yield* git(cwd, ["commit", "--allow-empty", "-m", "Local work"]); + if (tracking === "other-remote") { + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["fetch", "origin"]); + yield* git(cwd, ["branch", "--set-upstream-to=origin/feature", "feature"]); + } + const featureHead = yield* git(cwd, ["rev-parse", "feature"]); + const upstream = yield* git(cwd, [ + "for-each-ref", + "--format=%(upstream)", + "refs/heads/feature", + ]); + yield* git(cwd, ["checkout", initialBranch]); + const head = yield* git(cwd, ["rev-parse", "HEAD"]); + yield* writeTextFile(cwd, "README.md", "Uncommitted work\n"); + const driver = yield* GitVcsDriver.GitVcsDriver; + + const result = yield* driver + .switchRef({ cwd, refName: "upstream/feature" }) + .pipe(Effect.result); + assert.isTrue(Result.isFailure(result)); + assert.equal(yield* git(cwd, ["branch", "--show-current"]), initialBranch); + assert.equal(yield* git(cwd, ["rev-parse", "HEAD"]), head); + assert.equal(yield* git(cwd, ["rev-parse", "feature"]), featureHead); + assert.equal( + yield* git(cwd, ["for-each-ref", "--format=%(upstream)", "refs/heads/feature"]), + upstream, + ); + assert.equal( + yield* (yield* FileSystem.FileSystem).readFileString( + (yield* Path.Path).join(cwd, "README.md"), + ), + "Uncommitted work\n", + ); + // Selecting the existing local branch remains an explicit, attached checkout. + assert.equal((yield* driver.switchRef({ cwd, refName: "feature" })).refName, "feature"); + assert.equal(yield* git(cwd, ["symbolic-ref", "HEAD"]), "refs/heads/feature"); + }), + ); + it.effect("rejects a missing branch without restoring a matching dirty file", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -5042,8 +6105,174 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }); describe("worktree operations", () => { - it.effect("uses parallel checkout without skipping filters or hooks", () => - Effect.gen(function* () { + it.effect.each(["current", "legacy"] as const)( + "preserves another creator's worktree on its new branch at the same path (%s)", + (version) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const path = NodePath.join(yield* makeTmpDir("git-worktrees-"), "contested"); + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const reachedAdd = yield* Deferred.make(); + let paused = false; + const spawner = ChildProcessSpawner.make((command) => + Effect.gen(function* () { + if ( + ChildProcess.isStandardCommand(command) && + command.args.includes("worktree") && + command.args.includes("add") + ) { + if (command.args.includes("-h")) { + return version === "legacy" + ? makeFailedHandle("usage: git worktree add [--lock]") + : yield* delegate.spawn(command); + } + if (!paused) { + paused = true; + yield* Deferred.succeed(reachedAdd, undefined); + return yield* Effect.never; + } + } + return yield* delegate.spawn(command); + }), + ); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provide(layerServerConfig), + ); + const creator = yield* driver + .createWorktree({ + cwd, + path, + refName: initialBranch, + newRefName: "contested", + }) + .pipe(Effect.forkChild); + yield* Deferred.await(reachedAdd); + yield* git(cwd, ["worktree", "add", "--", path, "contested"]); + yield* writeTextFile(path, "owned-by-B.txt", "keep this work\n"); + yield* Fiber.interrupt(creator); + assert.include(yield* git(cwd, ["worktree", "list", "--porcelain"]), path); + assert.equal(yield* git(path, ["branch", "--show-current"]), "contested"); + assert.equal( + yield* (yield* FileSystem.FileSystem).readFileString( + NodePath.join(path, "owned-by-B.txt"), + ), + "keep this work\n", + ); + const created = yield* driver.createWorktree({ + cwd, + path: `${path}-retry`, + refName: initialBranch, + newRefName: "retry", + }); + assert.equal(yield* git(created.worktree.path, ["branch", "--show-current"]), "retry"); + assert.notInclude(yield* git(cwd, ["worktree", "list", "--porcelain"]), "locked"); + }), + ); + + it.effect.each(["main", "linked"] as const)( + "removes its worktree and branch when cancelled before the claim (%s)", + (checkout) => + Effect.gen(function* () { + const repo = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(repo); + const cwd = + checkout === "main" + ? repo + : NodePath.join(yield* makeTmpDir("git-linked-"), "checkout"); + if (checkout === "linked") { + yield* git(repo, ["config", "worktree.useRelativePaths", "true"]); + yield* git(repo, ["worktree", "add", "--detach", cwd]); + } + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.GitVcsDriver; + const worktreePath = path.join(yield* makeTmpDir("git-worktrees-"), "cancelled ü"); + const hookPath = path.join(repo, ".git/hooks/post-checkout"); + const connected = Promise.withResolvers(); + const sockets = new Set(); + const server = yield* Effect.acquireRelease( + Effect.sync(() => + NodeNet.createServer((socket) => { + sockets.add(socket); + connected.resolve(); + }), + ), + (server) => + Effect.sync(() => { + for (const socket of sockets) socket.destroy(); + server.close(); + }), + ); + yield* Effect.promise( + () => + new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }), + ); + const address = server.address(); + assert.isTrue(address !== null && typeof address !== "string"); + if (address === null || typeof address === "string") return; + const gate = path.join(cwd, "hook-gate.cjs"); + yield* fs.writeFileString( + gate, + `const s = require('node:net').connect(${address.port}, '127.0.0.1'); s.on('connect', () => process.stderr.write('Updating files: 50% (1/1000)\\n')); s.resume();`, + ); + yield* writeTextFile( + repo, + ".git/hooks/post-checkout", + `#!/bin/sh\nexec "${process.execPath.replaceAll("\\", "/")}" "${gate.replaceAll("\\", "/")}"\n`, + ); + yield* fs.chmod(hookPath, 0o755); + const hookStarted = yield* Deferred.make(); + let claims = 0; + const input = { + cwd, + path: worktreePath, + refName: initialBranch, + newRefName: "t3/cancel", + }; + const worktrees = git(cwd, ["worktree", "list", "--porcelain"]); + + const cancelled = yield* driver + .createWorktree(input, { + progress: { + onCheckoutProgress: ({ total }) => + total === 1000 ? Deferred.succeed(hookStarted, undefined) : Effect.void, + onWorktreeClaimed: () => Effect.sync(() => void claims++), + }, + }) + .pipe(Effect.forkChild); + yield* Deferred.await(hookStarted); + yield* Effect.promise(() => connected.promise); + assert.include(yield* worktrees, worktreePath); + yield* Fiber.interrupt(cancelled); + + assert.equal(claims, 0); + assert.notInclude(yield* worktrees, worktreePath); + assert.equal(yield* git(cwd, ["branch", "--list", "t3/cancel"]), ""); + assert.isFalse(yield* fs.exists(worktreePath)); + + // A retry gets the same path and branch, and a claimed worktree outlives a later cancel. + yield* fs.remove(hookPath); + const claimed = yield* Deferred.make(); + const retry = yield* driver + .createWorktree(input, { + progress: { onWorktreeClaimed: () => Deferred.succeed(claimed, undefined) }, + }) + .pipe(Effect.forkChild); + yield* Deferred.await(claimed); + yield* Fiber.interrupt(retry); + assert.include(yield* worktrees, worktreePath); + assert.notInclude(yield* worktrees, "locked"); + assert.equal(yield* git(worktreePath, ["branch", "--show-current"]), "t3/cancel"); + }), + ); + + it.effect("uses parallel checkout without skipping filters or hooks", () => + Effect.gen(function* () { const cwd = yield* makeTmpDir(); const { initialBranch } = yield* initRepoWithCommit(cwd); const fs = yield* FileSystem.FileSystem; @@ -5098,6 +6327,19 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { `${expected}\n`, ); } + yield* writeTextFile( + cwd, + ".git/hooks/post-checkout", + '#!/bin/sh\ngit worktree unlock "$PWD" 2>/dev/null || :\ngit worktree lock --reason user-kept "$PWD"\n', + ); + const lockedPath = path.join(yield* makeTmpDir("git-worktrees-"), "hook-locked"); + yield* driver.createWorktree({ + cwd, + path: lockedPath, + refName: initialBranch, + newRefName: "feature/hook-locked", + }); + assert.include(yield* git(cwd, ["worktree", "list", "--porcelain"]), "locked user-kept"); }), ); @@ -5408,6 +6650,44 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("gives parallel renames to one generated name distinct branches", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const path = yield* Path.Path; + const worktrees = yield* makeTmpDir("git-worktrees-"); + const driver = yield* GitVcsDriver.GitVcsDriver; + // A multi-model send launches one worktree per model with the same message, so their + // generated names often match and the renames can land together. + const temporaryBranches = ["t3/aaaa1111", "t3/bbbb2222", "t3/cccc3333"]; + yield* Effect.forEach(temporaryBranches, (branch) => + driver.createWorktree({ + cwd, + path: path.join(worktrees, branch.replace("/", "-")), + refName: initialBranch, + newRefName: branch, + }), + ); + + const renamed = yield* Effect.forEach( + temporaryBranches, + (branch) => + driver.renameBranch({ + cwd: path.join(worktrees, branch.replace("/", "-")), + oldBranch: branch, + newBranch: "feat/add-search", + }), + { concurrency: "unbounded" }, + ); + + assert.deepStrictEqual(renamed.map((result) => result.branch).toSorted(), [ + "feat/add-search", + "feat/add-search-1", + "feat/add-search-2", + ]); + }), + ); + it.effect.each(["release/v2", "refs/heads/release/v2", "deadbeef"])( "records the branch name when creating a worktree from %s", (baseRef) => @@ -5616,12 +6896,16 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const fileSystem = yield* FileSystem.FileSystem; const pathService = yield* Path.Path; + const submoduleRepo = yield* makeTmpDir("git-submodule-"); + yield* initRepoWithCommit(submoduleRepo); + const cwd = yield* makeTmpDir(); const { initialBranch } = yield* initRepoWithCommit(cwd); - yield* writeTextFile( - cwd, - ".gitmodules", - '[submodule "missing"]\n\tpath = missing\n\turl = /nonexistent/repo.git\n', + yield* git(cwd, ["submodule", "add", NodeURL.pathToFileURL(submoduleRepo).href, "shared"]); + yield* fileSystem.writeFileString( + pathService.join(cwd, ".gitmodules"), + (yield* fileSystem.readFileString(pathService.join(cwd, ".gitmodules"))) + + '[submodule "missing"]\n\tpath = missing\n\turl = /nonexistent/repo.git\n', ); yield* git(cwd, ["add", "."]); const gitlink = yield* git(cwd, ["rev-parse", "HEAD"]); @@ -5654,7 +6938,9 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.lengthOf(results, 1); assert.isFalse(results[0]!.ok); assert.include(results[0]!.detail, "GitVcsDriver.createWorktree.updateSubmodules"); - }), + // A submodule that did clone must not be left as a bare gitdir that reads as modified. + assert.equal(yield* git(worktreePath, ["status", "--porcelain"]), ""); + }).pipe(withGitFileProtocol), ); it.effect("creates worktrees under the configured worktrees directory", () => @@ -5976,6 +7262,41 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect.skipIf(!symlinksSupported)( + "removes a worktree whose .git file git-annex replaced with a symlink, keeping the symlink when refused", + () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const pathService = yield* Path.Path; + const worktreePath = pathService.join(yield* makeTmpDir("git-worktrees-"), "annexed"); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* driver.createWorktree({ + cwd, + path: worktreePath, + refName: initialBranch, + newRefName: "feature/annexed", + }); + const gitFile = pathService.join(worktreePath, ".git"); + const adminDir = (yield* git(worktreePath, ["rev-parse", "--absolute-git-dir"])).trim(); + NodeFS.rmSync(gitFile); + NodeFS.symlinkSync(adminDir, gitFile); + yield* writeTextFile(worktreePath, "notes.txt", "draft\n"); + + const refused = yield* Effect.result(driver.removeWorktree({ cwd, path: worktreePath })); + + assert.isTrue(Result.isFailure(refused)); + assert.isTrue(NodeFS.lstatSync(gitFile).isSymbolicLink()); + assert.equal(NodeFS.realpathSync(gitFile), NodeFS.realpathSync(adminDir)); + NodeFS.rmSync(pathService.join(worktreePath, "notes.txt")); + + yield* driver.removeWorktree({ cwd, path: worktreePath }); + + assert.isFalse(NodeFS.existsSync(worktreePath)); + assert.isFalse(NodeFS.existsSync(adminDir)); + }), + ); + it.effect("leaves a folder alone that only shares the worktree's short name", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -6100,6 +7421,112 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }); describe("remote operations", () => { + it.effect.each([ + { + configuration: "ordinary", + fetchRefspecs: ["+refs/heads/*:refs/remotes/origin/*"], + addsMapping: false, + }, + { + configuration: "restricted", + fetchRefspecs: [ + "+refs/heads/main:refs/remotes/origin/main", + "+refs/heads/team/*:refs/remotes/origin/team/*", + "+refs/heads/release/release-*:refs/remotes/origin/release/release-*", + ], + addsMapping: true, + }, + ])( + "sets a fetched branch's upstream with $configuration fetch configuration", + ({ fetchRefspecs, addsMapping }) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const remote = yield* makeTmpDir("git-remote-"); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["config", "--unset-all", "remote.origin.fetch"]); + for (const refspec of fetchRefspecs) { + yield* git(cwd, ["config", "--add", "remote.origin.fetch", refspec]); + } + const remoteBranch = "feature/example-change"; + const branch = "review/example-change"; + const mapping = `+refs/heads/${remoteBranch}*:refs/remotes/origin/${remoteBranch}*`; + yield* git(cwd, [ + "push", + remote, + "HEAD:refs/heads/main", + `HEAD:refs/heads/${remoteBranch}`, + ]); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* driver.fetchRemoteTrackingBranch({ cwd, remoteName: "origin", remoteBranch }); + yield* git(cwd, ["checkout", "--no-track", "-b", branch, `origin/${remoteBranch}`]); + + const input = { cwd, branch, remoteName: "origin", remoteBranch }; + yield* driver.setBranchUpstream(input); + yield* driver.setBranchUpstream(input); + + assert.equal( + yield* git(cwd, ["rev-parse", "--abbrev-ref", `${branch}@{upstream}`]), + `origin/${remoteBranch}`, + ); + assert.equal(yield* git(cwd, ["config", `branch.${branch}.remote`]), "origin"); + assert.equal( + yield* git(cwd, ["config", `branch.${branch}.merge`]), + `refs/heads/${remoteBranch}`, + ); + assert.deepEqual( + (yield* git(cwd, ["config", "--get-all", "remote.origin.fetch"])).split("\n"), + addsMapping ? [...fetchRefspecs, mapping] : fetchRefspecs, + ); + // A merged PR's head is deleted; later fetches of the remote must keep working. + yield* git(remote, ["--git-dir", remote, "branch", "-D", remoteBranch]); + yield* git(cwd, ["fetch", "origin"]); + }), + ); + + it.effect.each(["ordinary", "restricted"] as const)( + "retries upstream configuration with a config lock and %s fetch mappings", + (configuration) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + yield* git(cwd, ["remote", "add", "origin", "https://example.invalid/repo.git"]); + if (configuration === "restricted") { + yield* git(cwd, [ + "config", + "remote.origin.fetch", + `+refs/heads/${initialBranch}:refs/remotes/origin/${initialBranch}`, + ]); + } + yield* git(cwd, ["branch", "review/pr-head"]); + yield* git(cwd, ["update-ref", "refs/remotes/origin/pr-head", "HEAD"]); + const lockPath = (yield* Path.Path).join(cwd, ".git", "config.lock"); + yield* (yield* FileSystem.FileSystem).writeFileString(lockPath, ""); + const { driver, failures } = yield* makeDriverReleasingLock( + configuration === "restricted" + ? (args) => args[0] === "config" && args[1] === "--add" + : (args) => args.includes("--set-upstream-to"), + lockPath, + ); + + yield* TestClock.withLive( + driver.setBranchUpstream({ + cwd, + branch: "review/pr-head", + remoteName: "origin", + remoteBranch: "pr-head", + }), + ); + + assert.equal(yield* Ref.get(failures), 1); + assert.equal( + yield* git(cwd, ["rev-parse", "--abbrev-ref", "review/pr-head@{upstream}"]), + "origin/pr-head", + ); + }), + ); + it.effect("ensureRemote reuses a partial clone remote and avoids its name", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -6657,30 +8084,49 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { ); it.effect.each([ - { location: "root", conflict: false }, - { location: "nested", conflict: false }, - { location: "worktree", conflict: false }, - { location: "root", conflict: true }, - { location: "nested", conflict: true }, - { location: "worktree", conflict: true }, + { location: "root", conflict: false, encoding: "utf8" }, + { location: "nested", conflict: false, encoding: "utf8" }, + { location: "worktree", conflict: false, encoding: "utf8" }, + { location: "root", conflict: true, encoding: "utf8" }, + { location: "nested", conflict: true, encoding: "utf8" }, + { location: "worktree", conflict: true, encoding: "utf8" }, + { location: "root", conflict: true, encoding: "utf16le" }, + { location: "root", conflict: true, encoding: "utf16le", large: true }, ])( - "rejects selected-file merge commits without mutation ($location, conflict: $conflict)", - ({ location, conflict }) => + "rejects selected-file merge commits without mutation ($location, conflict: $conflict, $encoding)", + ({ location, conflict, encoding, large }) => Effect.gen(function* () { let cwd = yield* makeTmpDir(); const { initialBranch } = yield* initRepoWithCommit(cwd); + const writeReadme = (text: string) => + Effect.sync(() => + NodeFS.writeFileSync( + NodePath.join(cwd, "README.md"), + encoding === "utf16le" ? Buffer.from(`\ufeff${text}`, "utf16le") : text, + ), + ); + if (encoding === "utf16le") { + yield* writeTextFile( + cwd, + ".gitattributes", + "README.md text working-tree-encoding=UTF-16\n", + ); + yield* writeReadme("base\n"); + yield* git(cwd, ["add", "."]); + yield* git(cwd, ["commit", "-m", "Declare UTF-16 encoding"]); + } const driver = yield* GitVcsDriver.GitVcsDriver; const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; yield* git(cwd, ["checkout", "-b", "feature"]); yield* writeTextFile(cwd, "feature.txt", "feature\n"); - if (conflict) yield* writeTextFile(cwd, "README.md", "feature version\n"); + if (conflict) yield* writeReadme("feature version\n"); yield* git(cwd, ["add", "."]); yield* git(cwd, ["commit", "-m", "Feature"]); const featureHead = yield* git(cwd, ["rev-parse", "HEAD"]); yield* git(cwd, ["checkout", initialBranch]); yield* writeTextFile(cwd, "main.txt", "main\n"); - if (conflict) yield* writeTextFile(cwd, "README.md", "main version\n"); + if (conflict) yield* writeReadme("main version\n"); yield* git(cwd, ["add", "."]); yield* git(cwd, ["commit", "-m", "Main"]); const mainHead = yield* git(cwd, ["rev-parse", "HEAD"]); @@ -6729,6 +8175,7 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.equal(yield* git(cwd, ["rev-parse", "HEAD"]), mainHead); assert.equal(yield* git(cwd, ["ls-files", "--unmerged"]), unmerged); + const resolvedText = `Heading\n=======\n\nresolved${large ? "x".repeat(17 * 1024 * 1024) : ""}\n`; if (conflict) { const unresolved = yield* driver .commit(commitCwd, "Unresolved merge", "", { stage: {} }) @@ -6737,8 +8184,10 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.equal(yield* git(cwd, ["rev-parse", "HEAD"]), mainHead); assert.equal(yield* git(cwd, ["ls-files", "--unmerged"]), unmerged); // A lone "=======" is a heading underline, not a conflict boundary. - yield* writeTextFile(cwd, "README.md", "Heading\n=======\n\nresolved\n"); - assert.isNotNull(yield* driver.prepareCommitContext(commitCwd)); + yield* writeReadme(resolvedText); + assert.isNotNull( + yield* driver.prepareCommitContext(commitCwd, undefined, { includePatch: false }), + ); } yield* driver.commit(commitCwd, "Finish merge", "", { stage: {} }); assert.equal( @@ -6748,7 +8197,12 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.isFalse(NodeFS.existsSync(path.join(gitDir, "MERGE_HEAD"))); assert.equal(yield* git(cwd, ["show", "HEAD:feature.txt"]), "feature"); assert.equal(yield* git(cwd, ["show", "HEAD:unrelated.txt"]), "unrelated"); - if (conflict) + if (large) { + assert.equal( + yield* git(cwd, ["cat-file", "-s", "HEAD:README.md"]), + String(Buffer.byteLength(resolvedText)), + ); + } else if (conflict) assert.equal( yield* git(cwd, ["show", "HEAD:README.md"]), "Heading\n=======\n\nresolved", @@ -6756,42 +8210,139 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("reports a merge resolved to HEAD's tree as committable", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + // A branch named MERGE_HEAD is not a merge in progress. + yield* git(cwd, ["branch", "MERGE_HEAD"]); + assert.isNull(yield* driver.prepareCommitContext(cwd, undefined, { includePatch: false })); + yield* git(cwd, ["branch", "-D", "MERGE_HEAD"]); + yield* git(cwd, ["checkout", "-b", "feature"]); + yield* writeTextFile(cwd, "README.md", "feature version\n"); + yield* git(cwd, ["commit", "-am", "Feature"]); + yield* git(cwd, ["checkout", initialBranch]); + yield* writeTextFile(cwd, "README.md", "main version\n"); + yield* git(cwd, ["commit", "-am", "Main"]); + const merge = yield* driver.execute({ + operation: "test.merge", + cwd, + args: ["merge", "feature"], + allowNonZeroExit: true, + }); + assert.equal(merge.exitCode, 1); + yield* writeTextFile(cwd, "README.md", "main version\n"); + + assert.isNotNull( + yield* driver.prepareCommitContext(cwd, undefined, { includePatch: false }), + ); + }), + ); + it.effect.each([ - { operation: "cherry-pick", head: "CHERRY_PICK_HEAD" }, - { operation: "revert", head: "REVERT_HEAD" }, - ])("rejects selected-file commits during a $operation", ({ operation, head }) => + { operation: "cherry-pick", head: "CHERRY_PICK_HEAD", refs: "files" }, + { operation: "revert", head: "REVERT_HEAD", refs: "files" }, + // Reftable keeps CHERRY_PICK_HEAD and REVERT_HEAD as refs with no file under .git. + { operation: "cherry-pick", head: "CHERRY_PICK_HEAD", refs: "reftable" }, + { operation: "revert", head: "REVERT_HEAD", refs: "reftable" }, + ])( + "rejects selected-file commits during a $operation ($refs)", + ({ operation, head, refs }, ctx) => + Effect.gen(function* () { + if (refs === "reftable" && !gitHasReftable) + return ctx.skip("Needs Git 2.45 or later for reftable."); + const cwd = yield* makeTmpDir(); + if (refs === "reftable") yield* commitInitialRepo(cwd, ["--ref-format=reftable"]); + else yield* initRepoWithCommit(cwd); + // Reftable leaves .git/HEAD as a stub, so ask Git for the branch. + const initialBranch = yield* git(cwd, ["symbolic-ref", "--short", "HEAD"]); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* git(cwd, ["checkout", "-b", "other"]); + yield* writeTextFile(cwd, "README.md", "other version\n"); + yield* writeTextFile(cwd, "picked.txt", "picked\n"); + yield* git(cwd, ["add", "."]); + yield* git(cwd, ["commit", "-m", "Other"]); + if (operation === "cherry-pick") yield* git(cwd, ["checkout", initialBranch]); + yield* writeTextFile(cwd, "README.md", "main version\n"); + yield* git(cwd, ["commit", "-am", "Main"]); + const originalHead = yield* git(cwd, ["rev-parse", "HEAD"]); + yield* git(cwd, ["branch", head]); + const result = yield* driver.execute({ + operation: `test.${operation}`, + cwd, + args: [operation, operation === "cherry-pick" ? "other" : "HEAD~1"], + allowNonZeroExit: true, + }); + assert.equal(result.exitCode, 1); + yield* writeTextFile(cwd, "README.md", "resolved version\n"); + yield* git(cwd, ["add", "README.md"]); + const selected = operation === "cherry-pick" ? "picked.txt" : "README.md"; + + const commit = yield* driver + .commit(cwd, "Partial", "", { stage: { filePaths: [selected] } }) + .pipe(Effect.result); + + assert.isTrue(Result.isFailure(commit), `Selected-file ${operation} commit succeeded`); + assert.equal(yield* git(cwd, ["rev-parse", "HEAD"]), originalHead); + assert.equal( + yield* git(cwd, [ + "-c", + "core.warnAmbiguousRefs=false", + "rev-parse", + "--verify", + "--symbolic-full-name", + head, + ]), + head, + ); + }), + ); + + it.effect("rejects selected-file commits during a multi-commit no-commit cherry-pick", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); const { initialBranch } = yield* initRepoWithCommit(cwd); const driver = yield* GitVcsDriver.GitVcsDriver; - yield* git(cwd, ["checkout", "-b", "other"]); - yield* writeTextFile(cwd, "README.md", "other version\n"); - yield* writeTextFile(cwd, "picked.txt", "picked\n"); + const path = yield* Path.Path; + yield* writeTextFile(cwd, "a.txt", "base a\n"); + yield* writeTextFile(cwd, "b.txt", "base b\n"); yield* git(cwd, ["add", "."]); - yield* git(cwd, ["commit", "-m", "Other"]); - if (operation === "cherry-pick") yield* git(cwd, ["checkout", initialBranch]); - yield* writeTextFile(cwd, "README.md", "main version\n"); + yield* git(cwd, ["commit", "-m", "Base"]); + yield* git(cwd, ["checkout", "-b", "feature"]); + yield* writeTextFile(cwd, "a.txt", "feature a\n"); + yield* writeTextFile(cwd, "b.txt", "feature b\n"); + yield* git(cwd, ["commit", "-am", "Feature one"]); + yield* writeTextFile(cwd, "later.txt", "later\n"); + yield* git(cwd, ["add", "."]); + yield* git(cwd, ["commit", "-m", "Feature two"]); + yield* git(cwd, ["checkout", initialBranch]); + yield* writeTextFile(cwd, "a.txt", "main a\n"); + yield* writeTextFile(cwd, "b.txt", "main b\n"); yield* git(cwd, ["commit", "-am", "Main"]); - const originalHead = yield* git(cwd, ["rev-parse", "HEAD"]); - const result = yield* driver.execute({ - operation: `test.${operation}`, + const head = yield* git(cwd, ["rev-parse", "HEAD"]); + const started = yield* driver.execute({ + operation: "test.sequencer", cwd, - args: [operation, operation === "cherry-pick" ? "other" : "HEAD~1"], + args: ["cherry-pick", "--no-commit", "feature~1", "feature"], allowNonZeroExit: true, }); - assert.equal(result.exitCode, 1); - yield* writeTextFile(cwd, "README.md", "resolved version\n"); - yield* git(cwd, ["add", "README.md"]); + assert.equal(started.exitCode, 1); + yield* writeTextFile(cwd, "a.txt", "resolved a\n"); const gitDir = yield* git(cwd, ["rev-parse", "--absolute-git-dir"]); - const selected = operation === "cherry-pick" ? "picked.txt" : "README.md"; - - const commit = yield* driver - .commit(cwd, "Partial", "", { stage: { filePaths: [selected] } }) - .pipe(Effect.result); + assert.isFalse(NodeFS.existsSync(path.join(gitDir, "CHERRY_PICK_HEAD"))); + const names = ["index", "sequencer/todo", "sequencer/head"]; + const before = names.map((name) => NodeFS.readFileSync(path.join(gitDir, name))); - assert.isTrue(Result.isFailure(commit), `Selected-file ${operation} commit succeeded`); - assert.equal(yield* git(cwd, ["rev-parse", "HEAD"]), originalHead); - assert.isTrue(NodeFS.existsSync(`${gitDir}/${head}`)); + const error = yield* driver + .commit(cwd, "Partial", "", { stage: { filePaths: ["a.txt"] } }) + .pipe(Effect.flip); + assert.include(error.detail, "Cannot commit selected files during a cherry-pick or revert"); + assert.deepEqual( + names.map((name) => NodeFS.readFileSync(path.join(gitDir, name))), + before, + ); + assert.equal(yield* git(cwd, ["rev-parse", "HEAD"]), head); }), ); @@ -7116,7 +8667,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const preview = yield* driver.getReviewDiffPreview({ cwd: worktreePath }); const changes = preview.sources.find((source) => source.kind === "branch-range")!; assert.deepStrictEqual(changes.files, [ - { path: "feature.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "feature.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ]); assert.include(changes.diff, "+feature work"); assert.notInclude(changes.diff, "upstream-only work"); @@ -7281,6 +8838,92 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("republishes a branch whose upstream was deleted", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const remote = yield* makeTmpDir("git-remote-"); + yield* initRepoWithCommit(cwd); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* (yield* GitVcsDriver.GitVcsDriver).createRef({ + cwd, + refName: "feature/push", + }); + yield* (yield* GitVcsDriver.GitVcsDriver).switchRef({ + cwd, + refName: "feature/push", + }); + yield* writeTextFile(cwd, "feature.txt", "feature\n"); + yield* git(cwd, ["add", "-A"]); + yield* (yield* GitVcsDriver.GitVcsDriver).commit(cwd, "Add feature", ""); + + yield* (yield* GitVcsDriver.GitVcsDriver).pushCurrentBranch(cwd, null); + + yield* git(cwd, ["push", "origin", "--delete", "feature/push"]); + yield* writeTextFile(cwd, "followup.txt", "followup\n"); + yield* git(cwd, ["add", "-A"]); + yield* (yield* GitVcsDriver.GitVcsDriver).commit(cwd, "Add followup", ""); + const republished = yield* (yield* GitVcsDriver.GitVcsDriver).pushCurrentBranch(cwd, null); + assert.deepInclude(republished, { status: "pushed", branch: "feature/push" }); + assert.equal( + yield* git(remote, ["--git-dir=.", "rev-parse", "refs/heads/feature/push"]), + yield* git(cwd, ["rev-parse", "HEAD"]), + ); + + const fork = yield* makeTmpDir("git-fork-"); + yield* git(fork, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "fork", fork]); + yield* git(cwd, ["remote", "add", "feature", fork]); + yield* git(cwd, ["push", "-u", "fork", "feature/push"]); + yield* git(cwd, ["push", "fork", "--delete", "feature/push"]); + yield* writeTextFile(cwd, "fork.txt", "fork\n"); + yield* git(cwd, ["add", "-A"]); + yield* (yield* GitVcsDriver.GitVcsDriver).commit(cwd, "Add fork change", ""); + assert.deepInclude( + yield* (yield* GitVcsDriver.GitVcsDriver).resolvePushTarget(cwd, { + branch: "feature/push", + hasUpstream: false, + }), + { remoteName: "fork", branchName: "feature/push" }, + ); + const forkRepublished = yield* (yield* GitVcsDriver.GitVcsDriver).pushCurrentBranch( + cwd, + null, + ); + assert.deepInclude(forkRepublished, { upstreamBranch: "fork/feature/push" }); + assert.equal( + yield* git(fork, ["--git-dir=.", "rev-parse", "refs/heads/feature/push"]), + yield* git(cwd, ["rev-parse", "HEAD"]), + ); + + yield* git(cwd, ["config", "branch.feature/push.remote", "."]); + yield* git(cwd, ["config", "branch.feature/push.merge", "refs/heads/push"]); + yield* writeTextFile(cwd, "local.txt", "local\n"); + yield* git(cwd, ["add", "-A"]); + yield* (yield* GitVcsDriver.GitVcsDriver).commit(cwd, "Add local change", ""); + const localUpstreamGone = yield* (yield* GitVcsDriver.GitVcsDriver).pushCurrentBranch( + cwd, + null, + ); + assert.deepInclude(localUpstreamGone, { upstreamBranch: "origin/feature/push" }); + + yield* git(cwd, ["push", "fork", "HEAD:refs/heads/release"]); + yield* git(cwd, ["branch", "-u", "fork/release"]); + yield* git(cwd, ["push", "fork", "--delete", "release"]); + yield* writeTextFile(cwd, "release.txt", "release\n"); + yield* git(cwd, ["add", "-A"]); + yield* (yield* GitVcsDriver.GitVcsDriver).commit(cwd, "Add release change", ""); + yield* (yield* GitVcsDriver.GitVcsDriver).pushCurrentBranch(cwd, null); + assert.equal( + yield* (yield* GitVcsDriver.GitVcsDriver).readConfigValue( + cwd, + "branch.feature/push.t3-merge-base", + ), + "release", + ); + }), + ); + it.effect("rebases a pushed branch's pulls when branch.autoSetupRebase asks for it", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -7384,6 +9027,16 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { yield* git(originRemote, ["--git-dir=.", "rev-parse", "refs/heads/feature/push"]), publishedHead, ); + + const pushOnlyRemote = yield* makeTmpDir("git-push-only-remote-"); + yield* git(pushOnlyRemote, ["init", "--bare"]); + yield* git(cwd, ["remote", "set-url", "--push", "origin", pushOnlyRemote]); + const pushedToPushUrl = yield* driver.pushCurrentBranch(cwd, null); + assert.deepInclude(pushedToPushUrl, { status: "pushed", branch }); + assert.equal( + yield* git(pushOnlyRemote, ["--git-dir=.", "rev-parse", "refs/heads/feature/push"]), + head, + ); }), ); @@ -7427,14 +9080,15 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { ); it.effect.each([ - { baseBranch: "dev", local: false }, - { baseBranch: "dev", local: true }, - { baseBranch: "feature/base", local: true }, - { baseBranch: "upstream", local: true }, - { baseBranch: "upstream/feature/x", local: true }, + { baseBranch: "dev", local: false, branch: "feature/x", deleted: false }, + { baseBranch: "dev", local: true, branch: "feature/x", deleted: false }, + { baseBranch: "feature/base", local: true, branch: "feature/x", deleted: false }, + { baseBranch: "upstream", local: true, branch: "feature/x", deleted: false }, + { baseBranch: "upstream/feature/x", local: true, branch: "feature/x", deleted: false }, + { baseBranch: "dev", local: false, branch: "origin/dev", deleted: true }, ])( - "publishes a branch tracking $baseBranch (local=$local) under its own name, not the base", - ({ baseBranch, local }) => + "publishes $branch tracking $baseBranch (local=$local, deleted=$deleted) without updating the base", + ({ baseBranch, local, branch, deleted }) => Effect.gen(function* () { const cwd = yield* makeTmpDir(); const remote = yield* makeTmpDir("git-remote-"); @@ -7459,55 +9113,78 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { "checkout", "--track", "-b", - "feature/x", + branch, local ? `refs/heads/${baseBranch}` : `refs/remotes/upstream/${baseBranch}`, ]); assert.equal( - yield* driver.readConfigValue(cwd, "branch.feature/x.remote"), + yield* driver.readConfigValue(cwd, `branch.${branch}.remote`), local ? "." : "upstream", ); yield* writeTextFile(cwd, "feature.txt", "feature\n"); yield* git(cwd, ["add", "-A"]); yield* driver.commit(cwd, "Add feature", ""); + if (deleted) { + yield* git(cwd, ["push", "upstream", "--delete", baseBranch]); + } + const publishBranch = branch === "origin/dev" ? "dev" : branch; + assert.deepInclude( + yield* driver.resolvePushTarget(cwd, { branch, hasUpstream: !deleted }), + { remoteName: "origin", branchName: publishBranch }, + ); const pushed = yield* driver.pushCurrentBranch(cwd, null); assert.deepInclude(pushed, { status: "pushed", - branch: "feature/x", - upstreamBranch: "origin/feature/x", + branch, + upstreamBranch: `origin/${publishBranch}`, setUpstream: true, }); assert.equal( - yield* git(remote, ["--git-dir=.", "log", "-1", "--pretty=%s", "feature/x"]), + yield* git(remote, ["--git-dir=.", "log", "-1", "--pretty=%s", publishBranch]), "Add feature", ); + if (publishBranch !== baseBranch) { + assert.equal( + yield* git(remote, ["--git-dir=.", "rev-parse", `refs/heads/${baseBranch}`]), + devSha, + ); + } assert.equal( - yield* git(remote, ["--git-dir=.", "rev-parse", `refs/heads/${baseBranch}`]), - devSha, - ); - assert.equal( - yield* git(upstream, ["--git-dir=.", "rev-parse", `refs/heads/${baseBranch}`]), - upstreamDevSha, + yield* git(upstream, [ + "--git-dir=.", + "for-each-ref", + "--format=%(objectname)", + `refs/heads/${baseBranch}`, + ]), + deleted ? "" : upstreamDevSha, ); assert.equal( - yield* git(cwd, ["rev-parse", "--abbrev-ref", "@{upstream}"]), - "origin/feature/x", + yield* git(cwd, ["rev-parse", "--symbolic-full-name", "@{upstream}"]), + `refs/remotes/origin/${publishBranch}`, ); assert.equal( - yield* driver.readConfigValue(cwd, "branch.feature/x.t3-merge-base"), + yield* driver.readConfigValue(cwd, `branch.${branch}.t3-merge-base`), baseBranch, ); assert.equal( - yield* driver.readConfigValue(cwd, "branch.feature/x.t3-merge-base-ref"), + yield* driver.readConfigValue(cwd, `branch.${branch}.t3-merge-base-ref`), local ? `refs/heads/${baseBranch}` : `refs/remotes/upstream/${baseBranch}`, ); assert.equal(yield* git(cwd, ["rev-parse", `refs/heads/${baseBranch}`]), upstreamDevSha); + if (deleted) return; + const preview = yield* driver.getReviewDiffPreview({ cwd }); const changes = preview.sources.find((source) => source.kind === "branch-range")!; assert.deepStrictEqual(changes.files, [ - { path: "feature.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "feature.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ]); assert.equal( changes.baseRef, @@ -7652,6 +9329,32 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { yield* git(remote, ["--git-dir=.", "log", "-1", "--pretty=%s", "effect-atom"]), "Add alias update", ); + + yield* git(cwd, ["push", "my-org/upstream", "--delete", "effect-atom"]); + assert.deepInclude( + yield* driver.resolvePushTarget(cwd, { + branch: "upstream/effect-atom", + hasUpstream: false, + }), + { remoteName: "my-org/upstream", branchName: "effect-atom" }, + ); + assert.deepInclude(yield* driver.pushCurrentBranch(cwd, null), { + upstreamBranch: "my-org/upstream/effect-atom", + setUpstream: true, + }); + assert.equal( + yield* git(remote, ["--git-dir=.", "rev-parse", "refs/heads/effect-atom"]), + yield* git(cwd, ["rev-parse", "HEAD"]), + ); + assert.equal( + yield* git(remote, [ + "--git-dir=.", + "for-each-ref", + "--format=%(refname)", + "refs/heads/upstream/", + ]), + "", + ); }), ); @@ -7690,6 +9393,196 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.notEqual(originMain.exitCode, 0); }), ); + + it.effect.each(["branch.topic.pushRemote", "remote.pushDefault"] as const)( + "pushes a branch tracking its upstream head to the %s remote", + (configKey) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const upstreamRemote = yield* makeTmpDir("git-upstream-remote-"); + const forkRemote = yield* makeTmpDir("git-fork-remote-"); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* git(upstreamRemote, ["init", "--bare"]); + yield* git(forkRemote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "upstream", upstreamRemote]); + yield* git(cwd, ["remote", "add", "fork", forkRemote]); + yield* git(cwd, ["checkout", "-b", "topic"]); + yield* git(cwd, ["push", "upstream", "topic"]); + yield* git(cwd, ["push", "fork", "topic"]); + yield* git(cwd, ["branch", "--set-upstream-to=upstream/topic"]); + yield* git(cwd, ["config", configKey, "fork"]); + const upstreamTopicSha = yield* git(cwd, ["rev-parse", "HEAD"]); + yield* writeTextFile(cwd, "fork.txt", "fork\n"); + yield* git(cwd, ["add", "-A"]); + yield* driver.commit(cwd, "Add fork update", ""); + + const pushed = yield* driver.pushCurrentBranch(cwd, null); + + assert.deepInclude(pushed, { + status: "pushed", + branch: "topic", + upstreamBranch: "fork/topic", + pushRemote: "fork", + setUpstream: false, + }); + assert.equal( + yield* git(forkRemote, ["--git-dir=.", "log", "-1", "--pretty=%s", "topic"]), + "Add fork update", + ); + assert.equal( + yield* git(upstreamRemote, ["--git-dir=.", "rev-parse", "topic"]), + upstreamTopicSha, + ); + assert.equal( + yield* git(cwd, ["rev-parse", "--abbrev-ref", "@{upstream}"]), + "upstream/topic", + ); + }), + ); + + it.effect.each(["deleted", "rewound"] as const)( + "pushes an up-to-date tracked branch to the push remote when its branch there is %s", + (forkState) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const upstreamRemote = yield* makeTmpDir("git-upstream-remote-"); + const forkRemote = yield* makeTmpDir("git-fork-remote-"); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* git(upstreamRemote, ["init", "--bare"]); + yield* git(forkRemote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "upstream", upstreamRemote]); + yield* git(cwd, ["remote", "add", "fork", forkRemote]); + yield* git(cwd, ["checkout", "-b", "topic"]); + yield* writeTextFile(cwd, "upstream.txt", "upstream\n"); + yield* git(cwd, ["add", "-A"]); + yield* driver.commit(cwd, "Add upstream update", ""); + yield* git(cwd, ["push", "-u", "upstream", "topic"]); + yield* git(cwd, ["config", "remote.pushDefault", "fork"]); + // Another checkout changes the fork after this one last fetched it, + // leaving a local fork/topic that still contains HEAD. + yield* git(cwd, ["push", "fork", "topic"]); + yield* git( + forkRemote, + forkState === "deleted" + ? ["--git-dir=.", "update-ref", "-d", "refs/heads/topic"] + : ["--git-dir=.", "update-ref", "refs/heads/topic", "topic~1"], + ); + + const pushed = yield* driver.pushCurrentBranch(cwd, null); + + assert.deepInclude(pushed, { + status: "pushed", + upstreamBranch: "fork/topic", + pushRemote: "fork", + }); + assert.equal( + yield* git(forkRemote, ["--git-dir=.", "log", "-1", "--pretty=%s", "topic"]), + "Add upstream update", + ); + assert.equal( + yield* git(cwd, ["rev-parse", "--abbrev-ref", "@{upstream}"]), + "upstream/topic", + ); + }), + ); + + it.effect.each([ + { pushUrls: [], status: "skipped_up_to_date" }, + { pushUrls: ["mirror"], status: "pushed" }, + { pushUrls: ["fork", "mirror"], status: "pushed" }, + ] as const)( + "skips an up-to-date tracked branch only when every push URL has it (pushurl $pushUrls)", + ({ pushUrls, status }) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const upstreamRemote = yield* makeTmpDir("git-upstream-remote-"); + const remotes = { + fork: yield* makeTmpDir("git-fork-remote-"), + mirror: yield* makeTmpDir("git-mirror-remote-"), + }; + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* git(upstreamRemote, ["init", "--bare"]); + yield* git(remotes.fork, ["init", "--bare"]); + yield* git(remotes.mirror, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "upstream", upstreamRemote]); + yield* git(cwd, ["remote", "add", "fork", remotes.fork]); + yield* git(cwd, ["checkout", "-b", "topic"]); + yield* git(cwd, ["push", "-u", "upstream", "topic"]); + yield* git(cwd, ["push", "fork", "topic"]); + yield* git(cwd, ["config", "remote.pushDefault", "fork"]); + for (const name of pushUrls) { + yield* git(cwd, ["remote", "set-url", "--add", "--push", "fork", remotes[name]]); + } + + const pushed = yield* driver.pushCurrentBranch(cwd, null); + + assert.deepInclude(pushed, { status, upstreamBranch: "fork/topic", pushRemote: "fork" }); + const head = yield* git(cwd, ["rev-parse", "HEAD"]); + for (const name of pushUrls) { + assert.equal( + yield* git(remotes[name], ["--git-dir=.", "rev-parse", "refs/heads/topic"]), + head, + ); + } + assert.equal( + yield* git(cwd, ["rev-parse", "--abbrev-ref", "@{upstream}"]), + "upstream/topic", + ); + }), + ); + + it.effect("pushes an up-to-date branch whose tracked remote has a separate push URL", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const fetchRemote = yield* makeTmpDir("git-fetch-remote-"); + const pushRemote = yield* makeTmpDir("git-push-remote-"); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* git(fetchRemote, ["init", "--bare"]); + yield* git(pushRemote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", fetchRemote]); + yield* git(cwd, ["checkout", "-b", "topic"]); + yield* git(cwd, ["push", "-u", "origin", "topic"]); + yield* git(cwd, ["remote", "set-url", "--push", "origin", pushRemote]); + + const pushed = yield* driver.pushCurrentBranch(cwd, null); + + assert.deepInclude(pushed, { status: "pushed", upstreamBranch: "origin/topic" }); + assert.equal( + yield* git(pushRemote, ["--git-dir=.", "rev-parse", "refs/heads/topic"]), + yield* git(cwd, ["rev-parse", "HEAD"]), + ); + }), + ); + + it.effect("skips an up-to-date branch whose differently named upstream is its base", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const upstreamRemote = yield* makeTmpDir("git-upstream-remote-"); + const forkRemote = yield* makeTmpDir("git-fork-remote-"); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* git(upstreamRemote, ["init", "--bare"]); + yield* git(forkRemote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "upstream", upstreamRemote]); + yield* git(cwd, ["remote", "add", "fork", forkRemote]); + yield* git(cwd, ["push", "upstream", "HEAD:refs/heads/release"]); + yield* git(cwd, ["fetch", "upstream"]); + yield* git(cwd, ["checkout", "-b", "topic", "--track", "upstream/release"]); + yield* git(cwd, ["config", "remote.pushDefault", "fork"]); + + const pushed = yield* driver.pushCurrentBranch(cwd, null); + + assert.deepInclude(pushed, { + status: "skipped_up_to_date", + upstreamBranch: "upstream/release", + }); + assert.equal(yield* git(forkRemote, ["--git-dir=.", "branch", "--list", "topic"]), ""); + }), + ); }); }); diff --git a/apps/server/src/vcs/GitVcsDriverCore.ts b/apps/server/src/vcs/GitVcsDriverCore.ts index e0b372ebc1e87..36bb3fe576cd7 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.ts @@ -1,7 +1,10 @@ import * as Cache from "effect/Cache"; +import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as Data from "effect/Data"; +import * as Deferred from "effect/Deferred"; import * as Digest from "effect/crypto/Digest"; +import * as SecureRandom from "effect/crypto/SecureRandom"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; @@ -26,6 +29,7 @@ import { type GitCommandFailureReason, T3_PROJECT_FILE_NAME, type ReviewBranchCommitsInput, + type ReviewDiffChangeType, type ReviewDiffFileContentsInput, type ReviewDiffPreviewInput, type ReviewDiffFileStat, @@ -42,6 +46,7 @@ import { resolveProjectFileBackedSetting } from "@t3tools/shared/projectSettings import { gitCommandDuration, gitCommandsTotal, withMetrics } from "../observability/Metrics.ts"; import * as GitVcsDriver from "./GitVcsDriver.ts"; import { removeEmptyDirectory, type RemoveEmptyDirectoryError } from "./removeEmptyDirectory.ts"; +import { VcsAutomaticFetchInterval } from "./VcsAutomaticFetchInterval.ts"; import { explicitBareRepositoryEnv, gitConfigEnv, @@ -67,11 +72,19 @@ const branchBaseConfigKey = (branch: string) => `branch.${branch}.t3-merge-base` * CLI's key, which branches cut before then still carry instead. */ const legacyBranchBaseConfigKey = (branch: string) => `branch.${branch}.gh-merge-base`; +// Differently named upstreams are bases, not publish targets; pushing HEAD there would write features to shared branches. +const isAliasOfUpstreamHead = (branch: string, upstreamBranch: string, upstreamRef: string) => + branch === upstreamBranch || + (branch.endsWith(`/${upstreamBranch}`) && upstreamRef.endsWith(`/${branch}`)); const gitProcesses = Semaphore.makeUnsafe(8); const backgroundGitProcesses = Semaphore.makeUnsafe(2); +// The fetch before `git worktree add` downloads everything the remote gained since +// the last fetch; on busy repositories that alone outlasts the 30s default. +const FETCH_REMOTE_TIMEOUT_MS = Duration.toMillis(Duration.minutes(5)); const WORKTREE_INDEX_REFRESH_TIMEOUT_MS = Duration.toMillis(Duration.minutes(5)); const WORKTREE_REMOVE_TIMEOUT_MS = Duration.toMillis(Duration.minutes(5)); const DEFAULT_MAX_OUTPUT_BYTES = 1_000_000; +const REF_SNAPSHOT_MAX_OUTPUT_BYTES = 16 * 1024 * 1024; const OUTPUT_TRUNCATED_MARKER = "\n\n[truncated]"; const PREPARED_COMMIT_PATCH_MAX_OUTPUT_BYTES = 49_000; const RANGE_COMMIT_SUMMARY_MAX_OUTPUT_BYTES = 19_000; @@ -81,13 +94,15 @@ const REVIEW_DIFF_PATCH_MAX_OUTPUT_BYTES = 120_000; const REVIEW_METADATA_MAX_OUTPUT_BYTES = 16 * 1024 * 1024; // Temporary-index git add scales poorly (50k files took ~30 s); report incomplete totals above this cap. const REVIEW_UNTRACKED_MAX_FILES = 5_000; -// git diff --numstat reads every untracked byte (~5 ms/MB), so status totals skip larger untracked input. -const UNTRACKED_TOTALS_MAX_BYTES = 256 * 1024 * 1024; +// git diff reads every untracked byte (~5 ms/MB), so larger untracked input stays out of the review index. +const REVIEW_UNTRACKED_MAX_BYTES = 256 * 1024 * 1024; // The listing walks the whole untracked tree before the cap applies, so a folder whose listing // failed (too many files, output limit, timeout) is not listed again on every status poll. const UNTRACKED_TOTALS_RETRY_MIN_MS = 60_000; const UNTRACKED_TOTALS_RETRY_MAX_MS = 30 * 60_000; const REVIEW_DIFF_FILE_MAX_OUTPUT_BYTES = 1024 * 1024; +// Git's ident clean step (convert.c ident_to_git): `$Id:` up to the next `$` on the same line. +const GIT_IDENT_KEYWORD = /\$Id:[^$\n]*\$/g; const REVIEW_BRANCH_COMMIT_LIMIT = 100; const REVIEW_BRANCH_SUBJECT_LIMIT = 300; const REVIEW_BRANCH_COMMIT_MAX_OUTPUT_BYTES = 128 * 1024; @@ -191,6 +206,7 @@ interface GitRepositoryPaths { } interface GitRefsSnapshot { + readonly defaultBranch: string | null; readonly localBranches: ReadonlyArray; readonly remoteBranches: ReadonlyArray; readonly hasPrimaryRemote: boolean; @@ -218,18 +234,41 @@ function parseBranchAb(value: string): { ahead: number; behind: number } { }; } +function rawChangeType(status: string): ReviewDiffChangeType { + switch (status[0]) { + case "A": + return "new"; + case "D": + return "deleted"; + case "R": + return status === "R100" ? "rename-pure" : "rename-changed"; + default: + return "change"; + } +} + +// Parses `--numstat -z`, optionally preceded by `--raw` records that carry each file's change type. function parseReviewNumstat(stdout: string): ReviewDiffFileStat[] { const fields = stdout.split("\0"); + const changeTypes = new Map(); const files: ReviewDiffFileStat[] = []; for (let index = 0; index < fields.length; index++) { const field = fields[index]!; + if (field.startsWith(":")) { + const status = field.slice(field.lastIndexOf(" ") + 1); + if (/^[RC]/.test(status)) index++; + changeTypes.set(fields[++index]!, rawChangeType(status)); + continue; + } const match = /^(\d+|-)\t(\d+|-)\t([\s\S]*)$/.exec(field); if (!match) continue; const previousPath = match[3] === "" ? fields[++index]! : null; const path = previousPath !== null ? fields[++index]! : match[3]!; + const changeType = changeTypes.get(path); files.push({ path, previousPath, + ...(changeType === undefined ? {} : { changeType }), additions: match[1] === "-" ? 0 : Number(match[1]), deletions: match[2] === "-" ? 0 : Number(match[2]), }); @@ -330,6 +369,14 @@ function splitNullSeparatedPaths(input: string, truncated: boolean): string[] { return parts.filter((value) => value.length > 0); } +// Parses "git version 2.24.3 (Apple Git-128)" or "git version 2.45.1.windows.1". +const gitSupportsPathspecFromFile = (versionOutput: string): boolean => { + const match = /^git version (\d+)\.(\d+)/.exec(versionOutput); + if (match === null) return false; + const major = Number(match[1]); + return major > 2 || (major === 2 && Number(match[2]) >= 25); +}; + export function splitNullSeparatedGitStdoutPaths( result: Pick, ): string[] { @@ -551,7 +598,7 @@ function parseDefaultBranchFromRemoteHeadRef(value: string, remoteName: string): return null; } const refName = trimmed.slice(prefix.length).trim(); - return refName.length > 0 ? refName : null; + return refName.length > 0 && refName !== "HEAD" ? refName : null; } function isMissingGitCwdError(error: GitCommandError): boolean { @@ -585,6 +632,10 @@ export function isNonRepositoryGitStderr(stderr: string): boolean { const normalized = stderr.toLowerCase(); return NON_REPOSITORY_STDERR_PHRASES.some((phrase) => normalized.includes(phrase)); } +const UNTRACKED_FILES_SHOWN_CONFIG_VALUES = new Set(["normal", "all", "true", "yes", "on", "1"]); +function untrackedFilesHiddenByConfig(value: string | null): boolean { + return value !== null && !UNTRACKED_FILES_SHOWN_CONFIG_VALUES.has(value.toLowerCase()); +} function isUnbornHeadStderr(stderr: string): boolean { const normalized = stderr.toLowerCase(); return ( @@ -969,9 +1020,22 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const { worktreesDir, worktreeCheckoutCacheDir } = yield* ServerConfig.ServerConfig; const isServerInternalPath = yield* makeIsServerInternalPath; const digest = yield* Digest.Digest; + const random = yield* SecureRandom.SecureRandom; const hostPlatform = yield* HostProcess.Platform; const serviceScope = yield* Effect.scope; - const statusFetches = new Map>(); + const automaticFetchInterval = yield* VcsAutomaticFetchInterval; + type StatusFetchState = { + phase: "queued" | "running" | "claimed"; + claimed: Deferred.Deferred; + finished: Deferred.Deferred; + }; + const statusFetches = new Map< + string, + { + state: StatusFetchState; + fiber: Fiber.Fiber; + } + >(); const statusFetchProcesses = yield* Semaphore.make(4); const executeRaw: GitVcsDriver.GitVcsDriver["Service"]["execute"] = Effect.fnUntraced( @@ -1398,11 +1462,14 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const fiber = yield* Effect.uninterruptible( Effect.gen(function* () { const pending = statusFetches.get(key); - if (pending) return pending; + if (pending) return pending.fiber; + const claimed = yield* Deferred.make(); + const finished = yield* Deferred.make(); + const state: StatusFetchState = { phase: "queued", claimed, finished }; // Frequent poll deadlines and disconnected clients must not interrupt a // healthy writer. A separate safety limit prevents a hung transport from // occupying its worker slot indefinitely. - const started = yield* executeGit("GitVcsDriver.fetchRemoteForStatus", fetchCwd, args, { + const fetch = executeGit("GitVcsDriver.fetchRemoteForStatus", fetchCwd, args, { env: STATUS_UPSTREAM_REFRESH_ENV, fallbackErrorDetail: "Background Git fetch exited with a non-zero status.", timeoutMs: Duration.toMillis(STATUS_UPSTREAM_FETCH_TIMEOUT), @@ -1415,18 +1482,39 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ), Effect.tapError(() => recordStatusRemoteRefreshFailure(key)), Effect.tapCause((cause) => Effect.logWarning("Background Git fetch failed", cause)), - statusFetchProcesses.withPermits(1), + ); + const started = yield* Effect.raceFirst( + Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + if (state.phase !== "queued") return false; + state.phase = "running"; + yield* restore(fetch); + return true; + }), + ).pipe(statusFetchProcesses.withPermits(1)), + Deferred.await(claimed).pipe(Effect.as(false)), + ).pipe( + Effect.flatMap((completed) => + completed + ? Effect.void + : Deferred.await(finished).pipe( + Effect.flatMap((succeeded) => + succeeded ? Effect.void : fetch.pipe(statusFetchProcesses.withPermits(1)), + ), + ), + ), Effect.onExit(() => Effect.sync(() => { - statusFetches.delete(key); + if (statusFetches.get(key)?.state === state) statusFetches.delete(key); }), ), Effect.forkIn(serviceScope, { startImmediately: false, uninterruptible: false }), ); - statusFetches.set(key, started); + statusFetches.set(key, { state, fiber: started }); return started; }), ); + yield* Effect.annotateCurrentSpan("statusFetch.joining", true); const result = yield* Fiber.join(fiber).pipe( Effect.timeoutOption(STATUS_UPSTREAM_REFRESH_TIMEOUT), ); @@ -1495,23 +1583,23 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* : null; }); + const readCurrentBranch = (operation: string, cwd: string) => + executeGit(operation, cwd, ["symbolic-ref", "--quiet", "HEAD"], { + timeoutMs: 5_000, + allowNonZeroExit: true, + }).pipe( + Effect.map((result) => (result.exitCode === 0 ? branchNameFromHeadRef(result.stdout) : null)), + ); + const resolveRepositoryPathsUncached = Effect.fn("resolveRepositoryPathsUncached")(function* ( cwd: string, ) { const gitCommonDir = yield* readGitCommonDir(cwd); if (gitCommonDir === null) return null; - const [worktreeRoot, currentBranchResult] = yield* Effect.all( + const [worktreeRoot, currentBranch] = yield* Effect.all( [ readWorktreeRoot(cwd), - executeGit( - "GitVcsDriver.resolveRepositoryPaths.currentBranch", - cwd, - ["symbolic-ref", "--quiet", "HEAD"], - { - timeoutMs: 5_000, - allowNonZeroExit: true, - }, - ), + readCurrentBranch("GitVcsDriver.resolveRepositoryPaths.currentBranch", cwd), ], { concurrency: 2 }, ); @@ -1523,8 +1611,6 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* Effect.map((physicalCwd) => path.relative(worktreeRoot, physicalCwd) === ""), Effect.orElseSucceed(() => undefined), ); - const currentBranch = - currentBranchResult.exitCode === 0 ? branchNameFromHeadRef(currentBranchResult.stdout) : null; return { gitCommonDir, @@ -1688,9 +1774,42 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ), }); + const readRefSnapshot: GitVcsDriver.GitVcsDriver["Service"]["readRefSnapshot"] = Effect.fn( + "GitVcsDriver.readRefSnapshot", + )( + function* (cwd) { + const result = yield* executeGit( + "GitVcsDriver.readRefSnapshot", + cwd, + ["show-ref", "--head"], + { + allowNonZeroExit: true, + maxOutputBytes: REF_SNAPSHOT_MAX_OUTPUT_BYTES, + }, + ); + if (result.exitCode !== 0) return null; + const symbolicHead = yield* executeGit( + "GitVcsDriver.readRefSnapshot", + cwd, + ["symbolic-ref", "--quiet", "HEAD"], + { allowNonZeroExit: true }, + ); + // Exit 1 means detached HEAD; other failures require a full status read. + if (symbolicHead.exitCode !== 0 && symbolicHead.exitCode !== 1) return null; + // Deepening a shallow clone changes ancestry without moving refs. + const shallowPath = path.join(yield* resolveGitCommonDir(cwd), "shallow"); + const shallow = (yield* fileSystem.exists(shallowPath)) + ? yield* fileSystem.readFileString(shallowPath) + : ""; + return `${result.stdout}\0${symbolicHead.stdout}\0${shallow}`; + }, + Effect.orElseSucceed(() => null), + ); + const refreshStatusUpstreamIfStale = Effect.fn("refreshStatusUpstreamIfStale")(function* ( cwd: string, ) { + if (Duration.isZero(yield* automaticFetchInterval)) return; const headRef = yield* runGitStdout( "GitVcsDriver.refreshStatusUpstreamIfStale.headRef", cwd, @@ -1721,7 +1840,15 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* executeGit( "GitVcsDriver.resolveDefaultBranchName", cwd, - [...gitDirArgs, "symbolic-ref", `refs/remotes/${remoteName}/HEAD`], + // Unlike `symbolic-ref`, this fails when the remote deleted the branch HEAD points at. + [ + ...gitDirArgs, + "rev-parse", + "--verify", + "--quiet", + "--symbolic-full-name", + `refs/remotes/${remoteName}/HEAD`, + ], { allowNonZeroExit: true }, ).pipe( Effect.map((result) => { @@ -1810,7 +1937,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ), ); - const resolvePushRemoteName = Effect.fn("resolvePushRemoteName")(function* ( + const resolveConfiguredPushRemoteName = Effect.fn("resolveConfiguredPushRemoteName")(function* ( cwd: string, refName: string, ) { @@ -1830,13 +1957,65 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ["config", "--get", "remote.pushDefault"], true, ).pipe(Effect.map((stdout) => stdout.trim())); - if (pushDefaultRemote.length > 0) { - return pushDefaultRemote; + return pushDefaultRemote.length > 0 ? pushDefaultRemote : null; + }); + + const resolvePushRemoteName = Effect.fn("resolvePushRemoteName")(function* ( + cwd: string, + refName: string, + fallbackRemoteName: string | null = null, + ) { + const configuredRemote = yield* resolveConfiguredPushRemoteName(cwd, refName); + if (configuredRemote) { + return configuredRemote; + } + + if (fallbackRemoteName) { + return fallbackRemoteName; } return yield* resolvePrimaryRemoteName(cwd).pipe(Effect.orElseSucceed(() => null)); }); + const remoteRefContainsHead = (cwd: string, remoteRef: string) => + executeGit( + "GitVcsDriver.pushCurrentBranch.publishContainsHead", + cwd, + ["merge-base", "--is-ancestor", "HEAD", `refs/remotes/${remoteRef}`], + { allowNonZeroExit: true }, + ).pipe( + Effect.map((result) => result.exitCode === 0), + Effect.orElseSucceed(() => false), + ); + + // A fetch reads only the remote's first url, while a push writes every push URL (pushurl, else + // each url). A fetched ref is evidence of the push destination only when those are the same one. + const pushesOnlyToFetchUrl = Effect.fn("GitVcsDriver.pushesOnlyToFetchUrl")( + function* (cwd: string, remoteName: string) { + const [fetchUrl, pushUrls] = yield* Effect.all([ + runGitStdout("GitVcsDriver.pushesOnlyToFetchUrl.fetchUrl", cwd, [ + "remote", + "get-url", + "--", + remoteName, + ]), + runGitStdout("GitVcsDriver.pushesOnlyToFetchUrl.pushUrls", cwd, [ + "remote", + "get-url", + "--push", + "--all", + "--", + remoteName, + ]), + ]); + // Preserve whitespace in URLs and reject ambiguous multiline fetch URLs. + return ( + fetchUrl.endsWith("\n") && !fetchUrl.slice(0, -1).includes("\n") && pushUrls === fetchUrl + ); + }, + Effect.orElseSucceed(() => false), + ); + const ensureRemote: GitVcsDriver.GitVcsDriver["Service"]["ensureRemote"] = Effect.fn( "ensureRemote", )(function* (input) { @@ -2098,10 +2277,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* branch = branchNameFromHeadRef(branchResult.stdout); } const currentUpstreamRef = yield* readCurrentUpstreamRef(cwd); - let upstreamRef = - currentUpstreamRef === null - ? null - : (parseUpstreamRefByFirstSeparator(currentUpstreamRef)?.upstreamRef ?? null); + let upstreamRef = currentUpstreamRef; // Trimming a remote that starts with Unicode whitespace can leave a leading slash. if (currentUpstreamRef?.startsWith("/")) { upstreamRef = (yield* resolveCurrentUpstream(cwd, currentUpstreamRef))?.upstreamRef ?? null; @@ -2161,11 +2337,18 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* options?: GitVcsDriver.GitLocalStatusOptions, ) { const includeDivergence = options?.includeDivergence !== false; + // Hidden untracked files could not be committed; `all` stays as configured. + const untrackedFilesHidden = untrackedFilesHiddenByConfig( + yield* readConfigValue(cwd, "status.showUntrackedFiles").pipe( + Effect.orElseSucceed(() => null), + ), + ); const statusArgs = [ "status", "--porcelain=2", "--branch", "-z", + ...(untrackedFilesHidden ? ["--untracked-files=normal"] : []), ...(includeDivergence ? [] : ["--no-ahead-behind"]), ]; const indexResult = yield* executeGitWithStableDiagnostics( @@ -2222,41 +2405,25 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* isMissingGitCwdError(error) ? Effect.succeed(null) : Effect.fail(error), }), ); + // Porcelain diff rewrites indexes despite GIT_OPTIONAL_LOCKS=0; plumbing with -M preserves rename detection. + const readNumstat = (ref: string) => + executeGitWithStableDiagnostics( + "GitVcsDriver.statusDetails.numstat", + cwd, + ["diff-index", "-M", "--numstat", "-z", ref, "--"], + { allowNonZeroExit: true, maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES }, + ); // diff-index does not read status output, so both spawn together; a non-repository discards it. const [statusResult, numstatResult, defaultBranch, hasPrimaryRemote] = yield* Effect.all( [ readStatus, - // Porcelain diff rewrites indexes despite GIT_OPTIONAL_LOCKS=0; plumbing with -M preserves rename detection. - executeGitWithStableDiagnostics( - "GitVcsDriver.statusDetails.numstat", - cwd, - ["diff-index", "-M", "--numstat", "-z", "HEAD", "--"], - { allowNonZeroExit: true, maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES }, - ).pipe( + readNumstat("HEAD").pipe( + Effect.filterOrElse( + (result) => result.exitCode === 0 || !isUnbornHeadStderr(result.stderr), + () => Effect.flatMap(readEmptyTreeHash(cwd), readNumstat), + ), Effect.flatMap((result) => { if (result.exitCode === 0) return Effect.succeed(parseReviewNumstat(result.stdout)); - if (isUnbornHeadStderr(result.stderr)) { - return Effect.map( - Effect.all([ - runGitStdoutWithOptions( - "GitVcsDriver.statusDetails.numstat.unborn", - cwd, - ["diff-files", "--numstat", "-z"], - { maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES }, - ), - runGitStdoutWithOptions( - "GitVcsDriver.statusDetails.numstat.unborn.staged", - cwd, - ["diff", "--cached", "--numstat", "-z"], - { maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES }, - ), - ]), - ([unstagedStdout, stagedStdout]) => [ - ...parseReviewNumstat(stagedStdout), - ...parseReviewNumstat(unstagedStdout), - ], - ); - } return Effect.fail( new GitCommandError({ ...gitCommandContext({ @@ -2313,6 +2480,10 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* let refName: string | null = null; let upstreamRef: string | null = null; + // Git omits branch.ab when the upstream ref is gone, e.g. deleted after its PR merged, + // and also for an unborn HEAD, which can still track a fetched upstream. + let upstreamRefExists = false; + let isUnbornHead = false; let aheadCount = 0; let behindCount = 0; let aheadOfDefaultCount = 0; @@ -2323,9 +2494,13 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const statusRecords = statusStdout.split("\0"); for (let index = 0; index < statusRecords.length; index++) { const line = statusRecords[index]!; + if (line === "# branch.oid (initial)") { + isUnbornHead = true; + continue; + } if (line.startsWith("# branch.head ")) { const value = line.slice("# branch.head ".length).trim(); - refName = value.startsWith("(") ? null : value; + refName = value === "(detached)" ? null : value; continue; } if (line.startsWith("# branch.upstream ")) { @@ -2333,11 +2508,13 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* upstreamRef = value.length > 0 ? value : null; continue; } - if (includeDivergence && line.startsWith("# branch.ab ")) { - const value = line.slice("# branch.ab ".length).trim(); - const parsed = parseBranchAb(value); - aheadCount = parsed.ahead; - behindCount = parsed.behind; + if (line.startsWith("# branch.ab ")) { + upstreamRefExists = true; + if (includeDivergence) { + const parsed = parseBranchAb(line.slice("# branch.ab ".length).trim()); + aheadCount = parsed.ahead; + behindCount = parsed.behind; + } continue; } if (line.length > 0 && !line.startsWith("#")) { @@ -2356,6 +2533,8 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* } } + if (!upstreamRefExists && !isUnbornHead) upstreamRef = null; + const isDefaultBranch = refName !== null && (refName === defaultBranch || @@ -2480,6 +2659,18 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* })), ); + // git add takes --pathspec-from-file from Git 2.25; older Git keeps the argument form. + let pathspecFromFileSupported: boolean | undefined; + const supportsPathspecFromFile = Effect.fn("GitVcsDriver.supportsPathspecFromFile")(function* ( + cwd: string, + ) { + if (pathspecFromFileSupported !== undefined) return pathspecFromFileSupported; + const version = yield* executeGit("GitVcsDriver.version", cwd, ["version"]).pipe(Effect.option); + if (Option.isNone(version)) return false; + pathspecFromFileSupported = gitSupportsPathspecFromFile(version.value.stdout); + return pathspecFromFileSupported; + }); + const stageCommitChanges = Effect.fnUntraced(function* ( cwd: string, filePaths?: readonly string[], @@ -2508,14 +2699,40 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* { head: "MERGE_HEAD", operation: "merge" }, { head: "CHERRY_PICK_HEAD", operation: "cherry-pick" }, { head: "REVERT_HEAD", operation: "revert" }, + // A conflicted multi-commit `cherry-pick --no-commit` writes only the sequencer; reset + // would clear its unmerged entries, so `--continue` would walk past the conflict. + { head: "sequencer", operation: "cherry-pick or revert" }, ] as const; const headPaths = yield* runGitStdout("GitVcsDriver.commit.mergeHead", cwd, [ "rev-parse", ...inProgressHeads.flatMap(({ head }) => ["--git-path", head]), ]); const resolvedHeadPaths = headPaths.trim().split("\n"); - const inProgress = yield* Effect.findFirst(inProgressHeads, (_, index) => - fileSystem.exists(path.resolve(cwd, resolvedHeadPaths[index] ?? "")), + // Reftable keeps CHERRY_PICK_HEAD and REVERT_HEAD as refs with no file under .git. A + // same-named branch resolves to its full name; an ambiguity warning would blank stdout. + const operationRefs = new Set(["CHERRY_PICK_HEAD", "REVERT_HEAD"]); + const isOperationRef = (head: string) => + runGitStdout( + "GitVcsDriver.commit.operationRef", + cwd, + [ + "-c", + "core.warnAmbiguousRefs=false", + "rev-parse", + "--verify", + "--quiet", + "--symbolic-full-name", + head, + ], + true, + ).pipe(Effect.map((ref) => ref.trim() === head)); + const inProgress = yield* Effect.findFirst(inProgressHeads, ({ head }, index) => + fileSystem.exists(path.resolve(cwd, resolvedHeadPaths[index] ?? "")).pipe( + Effect.filterOrElse( + (exists) => exists || !operationRefs.has(head), + () => isOperationRef(head), + ), + ), ).pipe( Effect.mapError( (cause) => @@ -2540,14 +2757,18 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* } yield* runGit("GitVcsDriver.commit.reset", cwd, ["reset", "--quiet"]); } + // Status paths are repository-relative, while cwd can be a subdirectory. + const pathspecs = filePaths.map((file) => `:(top,literal)${file}`); + const fromStdin = yield* supportsPathspecFromFile(cwd); yield* runGit( "GitVcsDriver.commit.addSelected", cwd, - [...config, "add", "-A", "--pathspec-from-file=-", "--pathspec-file-nul"], + fromStdin + ? [...config, "add", "-A", "--pathspec-from-file=-", "--pathspec-file-nul"] + : [...config, "add", "-A", "--", ...pathspecs], { ...(env ? { env } : {}), - // Status paths are repository-relative, while cwd can be a subdirectory. - stdin: filePaths.map((file) => `:(top,literal)${file}\0`).join(""), + ...(fromStdin ? { stdin: pathspecs.map((pathspec) => `${pathspec}\0`).join("") } : {}), }, ); } else { @@ -2582,7 +2803,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ); const reports = check.stdout.split("\n"); const conflicted: string[] = []; - let topLevel: string | undefined; + let emptyTree: string | undefined; for (const file of unmergedPaths) { const markerLines = new Set( reports.flatMap((line) => { @@ -2593,11 +2814,26 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* }), ); if (markerLines.size === 0) continue; - topLevel ??= (yield* runGitStdout("GitVcsDriver.commit.topLevel", cwd, [ - "rev-parse", - "--show-toplevel", - ])).trim(); - const lines = (yield* fileSystem.readFileString(path.join(topLevel, file)).pipe( + emptyTree ??= yield* readEmptyTreeHash(cwd); + // --check numbers lines after working-tree-encoding and clean filters; a diff from the + // empty tree prints that view. Read it in full so large resolved files can still commit. + const fileDiff = yield* runGitStdoutWithOptions( + "GitVcsDriver.commit.conflictMarkers", + cwd, + [ + "-c", + "diff.relative=false", + "diff", + "--no-color", + "--no-ext-diff", + "--no-textconv", + "--text", + emptyTree, + "--", + `:(top,literal)${file}`, + ], + { ...(env ? { env } : {}), maxOutputBytes: Number.POSITIVE_INFINITY }, + ).pipe( Effect.mapError( (cause) => new GitCommandError({ @@ -2608,7 +2844,12 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* cause, }), ), - )).split("\n"); + ); + const diffLines = fileDiff.split("\n"); + const lines = diffLines + .slice(diffLines.findIndex((line) => line.startsWith("@@")) + 1) + .filter((line) => line.startsWith("+")) + .map((line) => line.slice(1)); // Git also reports a lone "=======", which is valid content such as a heading underline. const reported = (marker: string) => lines.some((line, index) => markerLines.has(index) && line.startsWith(marker)); @@ -2714,7 +2955,15 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* { env, maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES }, ).pipe(Effect.map((stdout) => stdout.trim())); if (stagedSummary.length === 0) { - return null; + // Like git commit, a merge resolved to HEAD's tree still records its merge commit. + const mergeHeadPath = yield* runGitStdout( + "GitVcsDriver.prepareCommitContext.mergeHead", + cwd, + ["rev-parse", "--git-path", "MERGE_HEAD"], + ); + if (!(yield* fileSystem.exists(path.resolve(physicalCwd, mergeHeadPath.trim())))) { + return null; + } } // A user-supplied message needs change detection, but no patch for generation. @@ -2829,10 +3078,40 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* hasUpstream: boolean, ) { if (!hasUpstream) { - const remoteName = yield* resolvePushRemoteName(cwd, branch); + // A deleted upstream keeps its config: republish to it when it was this branch's own, + // otherwise it was the base, so keep that before publishing. + let goneUpstream: { remoteName: string; branchName: string } | null = null; + let goneBaseUpstream: { remoteName: string; branchName: string; upstreamRef: string } | null = + null; + const mergeRef = yield* readConfigValue(cwd, `branch.${branch}.merge`); + const mergeBranch = mergeRef?.startsWith("refs/heads/") + ? mergeRef.slice("refs/heads/".length) + : null; + const configuredRemoteName = yield* readConfigValue(cwd, `branch.${branch}.remote`); + if (mergeBranch && configuredRemoteName && configuredRemoteName !== ".") { + if (isAliasOfUpstreamHead(branch, mergeBranch, `${configuredRemoteName}/${mergeBranch}`)) { + goneUpstream = { remoteName: configuredRemoteName, branchName: mergeBranch }; + } else { + goneBaseUpstream = { + remoteName: configuredRemoteName, + branchName: mergeBranch, + upstreamRef: `${configuredRemoteName}/${mergeBranch}`, + }; + } + } + const remoteName = yield* resolvePushRemoteName(cwd, branch, goneUpstream?.remoteName); if (!remoteName) return null; - const branchName = yield* resolvePublishBranchName(cwd, branch); - return { remoteName, branchName, publishes: true, upstream: null }; + // A gone local upstream still gets the full local name, as below. + const branchName = + goneUpstream?.branchName ?? + (configuredRemoteName === "." ? branch : yield* resolvePublishBranchName(cwd, branch)); + return { + remoteName, + branchName, + publishes: true, + upstream: goneBaseUpstream, + pushRemote: null, + }; } if ((yield* readConfigValue(cwd, `branch.${branch}.remote`)) === ".") { @@ -2850,21 +3129,27 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* branchName: branch, publishes: true, upstream: { remoteName: ".", branchName: localUpstream, upstreamRef: localUpstream }, + pushRemote: null, }; } const upstream = yield* resolveCurrentUpstream(cwd).pipe(Effect.orElseSucceed(() => null)); if (!upstream) return null; - // Differently named upstreams are bases, not publish targets; pushing HEAD there would write features to shared branches. - const isAliasOfUpstreamHead = - branch === upstream.branchName || - (branch.endsWith(`/${upstream.branchName}`) && upstream.upstreamRef.endsWith(`/${branch}`)); - if (isAliasOfUpstreamHead) { + if (isAliasOfUpstreamHead(branch, upstream.branchName, upstream.upstreamRef)) { + // A fork workflow pushes to the configured push remote and keeps the upstream for pulls, as plain `git push` does. + const configuredPushRemote = yield* resolveConfiguredPushRemoteName(cwd, branch).pipe( + Effect.orElseSucceed(() => null), + ); + const pushRemote = + configuredPushRemote && configuredPushRemote !== upstream.remoteName + ? configuredPushRemote + : null; return { - remoteName: upstream.remoteName, + remoteName: pushRemote ?? upstream.remoteName, branchName: upstream.branchName, publishes: false, upstream, + pushRemote, }; } const publishRemoteName = yield* resolvePushRemoteName(cwd, branch).pipe( @@ -2876,6 +3161,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* branchName, publishes: true, upstream, + pushRemote: null, }; }); @@ -2884,7 +3170,11 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* )(function* (cwd, current) { const destination = yield* resolvePushDestination(cwd, current.branch, current.hasUpstream); return destination - ? { remoteName: destination.remoteName, branchName: destination.branchName } + ? { + remoteName: destination.remoteName, + branchName: destination.branchName, + viaPushRemote: destination.pushRemote !== null, + } : null; }); @@ -2907,7 +3197,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const requestedRemoteName = options?.remoteName?.trim() || null; if (requestedRemoteName) { const publishBranch = yield* resolvePublishBranchName(cwd, branch); - yield* runGit( + yield* executeGitWithStableDiagnostics( "GitVcsDriver.pushCurrentBranch.pushWithRequestedRemote", cwd, ["push", "--", requestedRemoteName, `HEAD:refs/heads/${publishBranch}`], @@ -2922,16 +3212,41 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* } const hasNoLocalDelta = details.aheadCount === 0 && details.behindCount === 0; - if (hasNoLocalDelta && details.hasUpstream) { - return { - status: "skipped_up_to_date" as const, - branch, - ...(details.upstreamRef ? { upstreamBranch: details.upstreamRef } : {}), - }; - } - const destination = yield* resolvePushDestination(cwd, branch, details.hasUpstream); - if (hasNoLocalDelta) { + if (hasNoLocalDelta && details.hasUpstream) { + if (!destination?.pushRemote) { + // The tracking ref reflects the fetch URL, so it says nothing about a separate push URL. + if (!destination || (yield* pushesOnlyToFetchUrl(cwd, destination.remoteName))) { + return { + status: "skipped_up_to_date" as const, + branch, + ...(details.upstreamRef ? { upstreamBranch: details.upstreamRef } : {}), + }; + } + } else { + const { pushRemote, branchName } = destination; + const pushRemoteRef = `${pushRemote}/${branchName}`; + // The local ref can be stale when the branch was deleted or rewound elsewhere. + const pushRemoteHasHead = + (yield* pushesOnlyToFetchUrl(cwd, pushRemote)) && + (yield* fetchRemoteTrackingBranch({ + cwd, + remoteName: pushRemote, + remoteBranch: branchName, + }).pipe( + Effect.andThen(remoteRefContainsHead(cwd, pushRemoteRef)), + Effect.orElseSucceed(() => false), + )); + if (pushRemoteHasHead) { + return { + status: "skipped_up_to_date" as const, + branch, + upstreamBranch: pushRemoteRef, + pushRemote, + }; + } + } + } else if (hasNoLocalDelta) { const comparableBaseBranch = yield* resolveBaseBranchForNoUpstream(cwd, branch).pipe( Effect.orElseSucceed(() => null), ); @@ -2944,21 +3259,13 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* } // The configured diff base can contain HEAD while the publish branch is still behind. - const publishContainsHead = yield* executeGit( - "GitVcsDriver.pushCurrentBranch.publishContainsHead", - cwd, - [ - "merge-base", - "--is-ancestor", - "HEAD", - `refs/remotes/${destination.remoteName}/${destination.branchName}`, - ], - { allowNonZeroExit: true }, - ).pipe( - Effect.map((result) => result.exitCode === 0), - Effect.orElseSucceed(() => false), - ); - if (publishContainsHead) { + if ( + (yield* remoteRefContainsHead( + cwd, + `${destination.remoteName}/${destination.branchName}`, + )) && + (yield* pushesOnlyToFetchUrl(cwd, destination.remoteName)) + ) { return { status: "skipped_up_to_date" as const, branch, @@ -3001,7 +3308,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* }); } } - yield* runGit( + yield* executeGitWithStableDiagnostics( upstream ? "GitVcsDriver.pushCurrentBranch.pushOwnBranch" : "GitVcsDriver.pushCurrentBranch.pushWithUpstream", @@ -3018,7 +3325,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* } if (destination?.upstream) { - yield* runGit( + yield* executeGitWithStableDiagnostics( "GitVcsDriver.pushCurrentBranch.pushUpstream", cwd, ["push", "--", destination.remoteName, `HEAD:refs/heads/${destination.branchName}`], @@ -3027,12 +3334,19 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* return { status: "pushed" as const, branch, - upstreamBranch: destination.upstream.upstreamRef, + ...(destination.pushRemote + ? { + upstreamBranch: `${destination.remoteName}/${destination.branchName}`, + pushRemote: destination.pushRemote, + } + : { upstreamBranch: destination.upstream.upstreamRef }), setUpstream: false, }; } - yield* runGit("GitVcsDriver.pushCurrentBranch.push", cwd, ["push"], { timeoutMs: null }); + yield* executeGitWithStableDiagnostics("GitVcsDriver.pushCurrentBranch.push", cwd, ["push"], { + timeoutMs: null, + }); return { status: "pushed" as const, branch, @@ -3067,12 +3381,31 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* }); } const remote = yield* readConfigValue(cwd, `branch.${refName}.remote`); + // The upstream-only retry leaves other remote-tracking refs stale, so a claimed status + // fetch must still run after it. + let fetchedUpstreamOnly = false; if (remote) { const gitCommonDir = yield* resolveGitCommonDir(cwd); const pending = statusFetches.get(`${gitCommonDir}\0${remote}`); // The status wait can finish while its fetch still owns reference locks. // Let our existing writer finish before starting the pull's fetch. - if (pending) yield* Fiber.await(pending); + if (pending) { + const claimed = yield* Effect.uninterruptible( + Effect.gen(function* () { + if (pending.state.phase !== "queued") return false; + pending.state.phase = "claimed"; + yield* Effect.addFinalizer((exit) => + Deferred.succeed( + pending.state.finished, + Exit.isSuccess(exit) && !fetchedUpstreamOnly, + ), + ); + yield* Deferred.succeed(pending.state.claimed, undefined); + return true; + }), + ); + if (!claimed) yield* Fiber.await(pending.fiber); + } } const beforeSha = yield* runGitStdout( "GitVcsDriver.pullCurrentBranch.beforeSha", @@ -3126,6 +3459,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* currentSha === beforeSha ) { args = ["pull", "--ff-only", "--", remote, mergeRef]; + fetchedUpstreamOnly = true; result = yield* executeGitWithStableDiagnostics( "GitVcsDriver.pullCurrentBranch.pull", cwd, @@ -3154,11 +3488,14 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const refreshed = yield* readStatusDetailsLocal(cwd); return { - status: beforeSha.length > 0 && beforeSha === afterSha ? "skipped_up_to_date" : "pulled", + status: + beforeSha.length > 0 && beforeSha === afterSha + ? ("skipped_up_to_date" as const) + : ("pulled" as const), refName, upstreamRef: refreshed.upstreamRef, }; - }); + }, Effect.scoped); const readRangeContext: GitVcsDriver.GitVcsDriver["Service"]["readRangeContext"] = Effect.fn( "readRangeContext", @@ -3235,7 +3572,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const prepareReviewIndex = Effect.fn("prepareReviewIndex")(function* ( cwd: string, untrackedPaths: ReadonlyArray, - maxBytes?: number, + allowPartial: boolean, ) { const [stagedDeletionsStdout, indexValue] = yield* Effect.all( [ @@ -3263,21 +3600,42 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* { concurrency: 2 }, ); const stagedDeletions = new Set(stagedDeletionsStdout.split("\0").filter(Boolean)); - const pathsToAdd = untrackedPaths.filter((relativePath) => !stagedDeletions.has(relativePath)); - if (pathsToAdd.length === 0) return undefined; - if (maxBytes !== undefined) { - const sizes = yield* Effect.forEach( - pathsToAdd, - (relativePath) => - fileSystem.lstat(path.resolve(cwd, relativePath)).pipe( - Effect.map((info) => (info.type === "File" ? Number(info.size) : 0)), - Effect.orElseSucceed(() => 0), - ), - { concurrency: 16 }, - ); - // null: over the byte budget, so the caller reports totals as unavailable. - if (sizes.reduce((total, size) => total + size, 0) > maxBytes) return null; + // Staged deletions never enter the index, so their worktree copies must not spend the budget. + // A nested repository (listed with a trailing slash) with no commits cannot be indexed: + // Git before 2.53 fails the add, and later versions accept it and then fail every diff. + const candidates = yield* Effect.filter( + untrackedPaths.filter((relativePath) => !stagedDeletions.has(relativePath)), + (relativePath) => + relativePath.endsWith("/") + ? executeGit( + "GitVcsDriver.readUnifiedWorkingTreeReviewDiff.nestedRepositoryHead", + path.resolve(cwd, relativePath), + ["rev-parse", "--verify", "--quiet", "HEAD"], + { allowNonZeroExit: true }, + ).pipe(Effect.map((result) => result.exitCode !== 1)) + : Effect.succeed(true), + { concurrency: 16 }, + ); + const sizes = yield* Effect.forEach( + candidates, + (relativePath) => + fileSystem.lstat(path.resolve(cwd, relativePath)).pipe( + Effect.map((info) => (info.type === "File" ? Number(info.size) : 0)), + Effect.orElseSucceed(() => 0), + Effect.map((size) => ({ relativePath, size })), + ), + { concurrency: 16 }, + ); + // Smallest first, so which files fit does not depend on how their names sort. + const pathsToAdd: string[] = []; + let bytes = 0; + for (const { relativePath, size } of sizes.toSorted((left, right) => left.size - right.size)) { + bytes += size; + if (bytes > REVIEW_UNTRACKED_MAX_BYTES) break; + pathsToAdd.push(relativePath); } + const omitted = pathsToAdd.length < candidates.length; + if (pathsToAdd.length === 0 || (omitted && !allowPartial)) return { env: undefined, omitted }; const indexPath = path.isAbsolute(indexValue.trim()) ? indexValue.trim() @@ -3330,7 +3688,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ], { env, stdin: `${pathsToAdd.join("\0")}\0` }, ); - return env; + return { env, omitted }; }); const readEmptyTreeHash = Effect.fn("readEmptyTreeHash")(function* (cwd: string) { @@ -3346,8 +3704,8 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const prepareUntrackedReviewIndexOrReason = Effect.fn("prepareUntrackedReviewIndex")(function* ( cwd: string, pathArgs: ReadonlyArray, - onlyPath?: string, - maxBytes?: number, + onlyPath: string | undefined, + allowPartial: boolean, ) { const untracked = yield* executeGit( "GitVcsDriver.review.listUntracked", @@ -3366,10 +3724,10 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* (candidate) => onlyPath === undefined || candidate === onlyPath || candidate === `${onlyPath}/`, ); - if (paths.length === 0) return { env: undefined }; + if (paths.length === 0) return { env: undefined, omitted: false }; if (paths.length > REVIEW_UNTRACKED_MAX_FILES) return { unavailable: "Too many untracked files to count." }; - const env = yield* prepareReviewIndex(cwd, paths, maxBytes).pipe( + const index = yield* prepareReviewIndex(cwd, paths, allowPartial).pipe( Effect.catchTags({ PlatformError: (cause) => Effect.fail( @@ -3383,15 +3741,16 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ), }), ); - if (env === null) return { unavailable: "Untracked files are too large to count." }; - return { env }; + if (index.omitted && !allowPartial) + return { unavailable: "Untracked files are too large to count." }; + return index; }); const prepareUntrackedReviewIndex = ( cwd: string, pathArgs: ReadonlyArray, onlyPath?: string, ) => - prepareUntrackedReviewIndexOrReason(cwd, pathArgs, onlyPath).pipe( + prepareUntrackedReviewIndexOrReason(cwd, pathArgs, onlyPath, true).pipe( Effect.map((result) => ("unavailable" in result ? null : result)), ); @@ -3482,13 +3841,11 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* detail: remembered.detail, }); } - // Empty status can mean untracked files are hidden; retain the full scan. - const untracked = yield* prepareUntrackedReviewIndexOrReason( - cwd, - pathspecs, - undefined, - UNTRACKED_TOTALS_MAX_BYTES, - ); + // Status always lists untracked files, so no roots means there are none to scan. + const untracked = + roots.length === 0 + ? { env: undefined, omitted: false } + : yield* prepareUntrackedReviewIndexOrReason(cwd, pathspecs, undefined, false); if ("unavailable" in untracked) { const count = remembered?.roots === rootsKey ? remembered.count + 1 : 1; untrackedTotalsFailures.delete(cwd); @@ -3516,7 +3873,8 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* executeGit( "GitVcsDriver.readBranchChangeTotals", cwd, - [...REVIEW_DIFF_ARGS, "--numstat", "-z", ref, "--"], + // Plumbing: porcelain diff refreshes and rewrites the real index, taking index.lock. + ["diff-index", "-M", "--numstat", "-z", ref, "--"], { allowNonZeroExit: true, maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES, @@ -3742,7 +4100,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const stat = yield* executeGit( "GitVcsDriver.getReviewDiffPreview.commitStat", cwd, - [...diffArgs, "--numstat", "-z", commit.parent, commit.sha, "--", ...pathArgs], + [...diffArgs, "--raw", "--numstat", "-z", commit.parent, commit.sha, "--", ...pathArgs], { maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES }, ); const files = parseReviewNumstat(stat.stdout); @@ -3792,7 +4150,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ref: string, env?: NodeJS.ProcessEnv, ) { - const args = [...diffArgs, "--numstat", "-z"]; + const args = [...diffArgs, "--raw", "--numstat", "-z"]; const result = yield* executeGit( "GitVcsDriver.getReviewDiffPreview.stat", cwd, @@ -3838,9 +4196,11 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* }); const [dirtyTrackedResult, baseResult] = yield* Effect.gen(function* () { const untracked = input.trackedOnly - ? { env: undefined } + ? { env: undefined, omitted: false } : yield* prepareUntrackedReviewIndex(cwd, pathArgs, input.file?.path); - const metadataOnly = input.metadataOnly === true && !input.file && untracked !== null; + // Untracked files left out make the file list partial, so the bounded patch is sent instead. + const complete = untracked !== null && !untracked.omitted; + const metadataOnly = input.metadataOnly === true && !input.file && complete; const [dirty, base] = review.mergeBase === dirtyRef ? yield* readTrackedDiff(dirtyRef, untracked?.env, metadataOnly).pipe( @@ -3853,7 +4213,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ], { concurrency: 2 }, ); - if (untracked !== null) return [dirty, base] as const; + if (complete) return [dirty, base] as const; const incomplete = (ref: string | null, result: typeof dirty) => ref === null ? result : { ...result, files: undefined, stdoutTruncated: true }; return [incomplete(dirtyRef, dirty), incomplete(review.mergeBase, base)] as const; @@ -4011,7 +4371,43 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* if (bytes.includes(0)) { return yield* fileError("fs.readFile", `Cannot expand binary file '${input.newPath}'.`); } - return new TextDecoder("utf-8", { ignoreBOM: true }).decode(bytes); + const contents = new TextDecoder("utf-8", { ignoreBOM: true }).decode(bytes); + if (!contents.includes("$Id:")) return contents; + const collapsed = contents.replace(GIT_IDENT_KEYWORD, () => "$Id$"); + if (collapsed === contents) return contents; + // check-attr reports both boolean ident and literal ident=set as "set". + const attribute = yield* runGitStdout( + "GitVcsDriver.getReviewDiffFileContents.workingTree.ident", + repositoryRoot, + ["check-attr", "-z", "ident", "filter", "--", input.newPath], + ); + const attributes = attribute.split("\0"); + if (attributes[2] !== "set" || attributes[5] !== "unspecified") return contents; + + // Literal filter=unspecified is indistinguishable from an absent filter in check-attr. + const filters = yield* executeGit( + "GitVcsDriver.getReviewDiffFileContents.workingTree.filters", + repositoryRoot, + ["config", "--name-only", "--get-regexp", "^filter\\.unspecified\\."], + { allowNonZeroExit: true }, + ); + if (filters.exitCode !== 1 || filters.stdout.length > 0) return contents; + // Probe a nonoverlapping keyword so ident conversion runs once on the actual file. + const [canonicalHash, collapsedHash] = yield* Effect.all([ + runGitStdoutWithOptions( + "GitVcsDriver.getReviewDiffFileContents.workingTree.canonicalHash", + repositoryRoot, + ["hash-object", "--stdin", `--path=${input.newPath}`], + { stdin: "$Id: probe $" }, + ), + runGitStdoutWithOptions( + "GitVcsDriver.getReviewDiffFileContents.workingTree.collapsedHash", + repositoryRoot, + ["hash-object", "--stdin", `--path=${input.newPath}`], + { stdin: "$Id$" }, + ), + ]); + return canonicalHash === collapsedHash ? collapsed : contents; }); const getReviewDiffFileContents = Effect.fn("getReviewDiffFileContents")(function* ( @@ -4209,7 +4605,14 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* executeGit( "GitVcsDriver.listRefs.defaultRef", fetchCwd, - [...gitDirArgs, "symbolic-ref", "refs/remotes/origin/HEAD"], + [ + ...gitDirArgs, + "rev-parse", + "--verify", + "--quiet", + "--symbolic-full-name", + "refs/remotes/origin/HEAD", + ], { timeoutMs: 5_000, allowNonZeroExit: true, @@ -4237,7 +4640,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* } const defaultBranch = defaultRefResult.exitCode === 0 - ? defaultRefResult.stdout.trim().replace(/^refs\/remotes\/origin\//, "") + ? parseDefaultBranchFromRemoteHeadRef(defaultRefResult.stdout, "origin") : null; const parsedWorktreeEntries = worktreeListResult.exitCode === 0 @@ -4312,6 +4715,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* : left.ref.name.localeCompare(right.ref.name); return { + defaultBranch, localBranches: localBranches.toSorted(byRecencyThenName).map(({ ref }) => ref), remoteBranches: remoteBranches.toSorted(byRecencyThenName).map(({ ref }) => ref), hasPrimaryRemote: remoteNames.length > 0, @@ -4440,22 +4844,51 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* }; } - const snapshot = yield* resolveListRefsSnapshot( - repositoryPaths.gitCommonDir, - input.refresh === true, + // Both caches outlive a checkout made outside T3 Code, so this checkout's branch is read + // on every call and replaces whatever the cached snapshot claims for it. + const [snapshot, currentBranch] = yield* Effect.all( + [ + resolveListRefsSnapshot(repositoryPaths.gitCommonDir, input.refresh === true), + readCurrentBranch("GitVcsDriver.listRefs.currentBranch", input.cwd).pipe( + Effect.orElseSucceed(() => repositoryPaths.currentBranch), + ), + ], + { concurrency: 2 }, ); - const hasCurrentWorktreeBranch = - repositoryPaths.worktreeRoot !== null && - snapshot.localBranches.some((ref) => ref.worktreePath === repositoryPaths.worktreeRoot); - const localBranches = snapshot.localBranches.map((ref) => ({ - ...ref, - worktreePath: - ref.worktreePath ?? - (ref.name === repositoryPaths.currentBranch ? repositoryPaths.worktreeRoot : null), - current: hasCurrentWorktreeBranch - ? ref.worktreePath === repositoryPaths.worktreeRoot - : ref.name === repositoryPaths.currentBranch, - })); + const { worktreeRoot } = repositoryPaths; + // The snapshot misses a branch created outside T3 Code until it refreshes. An unborn HEAD + // names a branch with no ref yet, so the ref must exist before it is listed. + const currentBranchUnlisted = + currentBranch !== null && !snapshot.localBranches.some((ref) => ref.name === currentBranch); + const unlistedCurrentBranch: ReadonlyArray = + currentBranchUnlisted && + (yield* branchExists(input.cwd, currentBranch).pipe(Effect.orElseSucceed(() => false))) + ? [ + { + name: currentBranch, + current: true, + isRemote: false, + isDefault: currentBranch === snapshot.defaultBranch, + worktreePath: worktreeRoot, + }, + ] + : []; + const listedBranches = snapshot.localBranches.map((ref) => { + const current = ref.name === currentBranch; + return { + ...ref, + worktreePath: + worktreeRoot === null + ? ref.worktreePath + : current + ? worktreeRoot + : ref.worktreePath === worktreeRoot + ? null + : ref.worktreePath, + current, + }; + }); + const localBranches = [...unlistedCurrentBranch, ...listedBranches]; const combinedBranches = input.includeMatchingRemoteRefs ? [...localBranches, ...snapshot.remoteBranches] : dedupeRemoteBranchesWithLocalMatches([...localBranches, ...snapshot.remoteBranches]); @@ -4535,6 +4968,40 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* yield* refresh; }); + // Git writes the lock reason during registration, before checkout or user hooks can run. + const releaseWorktreeRegistration = Effect.fn("releaseWorktreeRegistration")(function* ( + cwd: string, + reason: string, + remove: boolean, + ) { + const commonDir = yield* readGitCommonDir(cwd); + if (commonDir === null) return; + const registrations = path.join(commonDir, "worktrees"); + const entries = yield* fileSystem.readDirectory(registrations).pipe( + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + () => Effect.succeed([]), + ), + ); + for (const entry of entries) { + const adminDir = path.join(registrations, entry); + const locked = yield* fileSystem + .readFileString(path.join(adminDir, "locked")) + .pipe(Effect.orElseSucceed(() => "")); + if (locked.replace(/\r?\n$/, "") !== reason) continue; + // gitdir can be relative to the registration with worktree.useRelativePaths enabled. + const gitdir = yield* fileSystem.readFileString(path.join(adminDir, "gitdir")); + const registered = path.dirname(path.resolve(adminDir, gitdir.replace(/\r?\n$/, ""))); + yield* runGit("GitVcsDriver.createWorktree.unlock", cwd, [ + "worktree", + "unlock", + "--", + registered, + ]); + if (remove) yield* removeWorktree({ cwd, path: registered, force: true }); + } + }); + const createWorktree: GitVcsDriver.GitVcsDriver["Service"]["createWorktree"] = Effect.fn( "createWorktree", )(function* (input, options) { @@ -4579,6 +5046,20 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* } } const newRefName = input.newRefName; + // Older Git still creates worktrees normally, but cannot prove ownership for cancellation. + const addHelp = newRefName + ? yield* executeGit( + "GitVcsDriver.createWorktree.capabilities", + input.cwd, + ["worktree", "add", "-h"], + { allowNonZeroExit: true }, + ) + : null; + const lockReason = + addHelp && `${addHelp.stdout}\n${addHelp.stderr}`.includes("--reason") + ? `t3-create-${yield* random.uuidV4()}` + : undefined; + if (newRefName) { // Kept apart from the checkout so the shared-config lock only covers the tracking write. yield* createBranchAt( @@ -4612,6 +5093,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* `checkout.workers=${checkoutWorkers}`, "worktree", "add", + ...(lockReason ? ["--lock", "--reason", lockReason] : []), "--", worktreePath, checkoutRef, @@ -4639,30 +5121,66 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* : {}), }, ); + // Set inside the claim's uninterruptible region: a pending cancel still surfaces as it ends. + let claimed = false; + const onWorktreeClaimed = progress?.onWorktreeClaimed; yield* ( fastCheckout ? checkoutCache - .create({ cwd: input.cwd, worktreePath, refName: checkoutRef, checkoutWorkers }) + .create({ + cwd: input.cwd, + worktreePath, + refName: checkoutRef, + checkoutWorkers, + lockReason, + }) .pipe(Effect.flatMap((cloned) => (cloned ? Effect.void : stockCheckout))) : stockCheckout ).pipe( - Effect.onError(() => - newRefName - ? deleteLocalBranch({ cwd: input.cwd, refName: newRefName, force: true }).pipe( + Effect.andThen( + fastCheckout ? checkoutCache.schedule(worktreePath, cacheGeneration) : Effect.void, + ), + Effect.andThen( + onWorktreeClaimed + ? Effect.uninterruptible( + onWorktreeClaimed(worktreePath).pipe( + Effect.andThen( + Effect.sync(() => { + claimed = true; + }), + ), + ), + ) + : Effect.void, + ), + Effect.onError((cause) => + newRefName && !claimed + ? // A failed hook keeps its worktree, as Git does; only a cancel abandons it. + (Cause.hasInterrupts(cause) + ? lockReason + ? releaseWorktreeRegistration(input.cwd, lockReason, true) + : Effect.void + : Effect.void + ).pipe( + Effect.andThen( + deleteLocalBranch({ cwd: input.cwd, refName: newRefName, force: true }), + ), Effect.catch((cause) => - Effect.logWarning("could not remove the branch of a failed worktree", { cause }), + Effect.logWarning("could not clean up a failed worktree or its branch", { cause }), + ), + ) + : Effect.void, + ), + Effect.onExit(() => + lockReason + ? releaseWorktreeRegistration(input.cwd, lockReason, false).pipe( + Effect.catch((cause) => + Effect.logWarning("could not unlock a created worktree", { cause }), ), ) : Effect.void, ), ); - if (fastCheckout) { - yield* checkoutCache.schedule(worktreePath, cacheGeneration); - } - - if (progress?.onWorktreeClaimed) { - yield* progress.onWorktreeClaimed(worktreePath); - } // worktree add leaves submodules empty, potentially omitting project skills, tools or source. const hasSubmodules = yield* fileSystem @@ -4697,6 +5215,61 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* yield* progress.onSubmodulesStarted(); } const onSubmoduleLine = progress?.onSubmoduleLine; + const updateSubmodules = (paths: readonly string[]) => + runGit( + "GitVcsDriver.createWorktree.updateSubmodules", + worktreePath, + [ + "submodule", + "update", + "--init", + ...(submoduleMode.value === "recursive" ? ["--recursive"] : []), + // Git reports transfer progress to a pipe only when forced. + ...(onSubmoduleLine ? ["--progress"] : []), + // A clone left by a failed update already has HEAD at the gitlink, so Git would skip it. + ...(paths.length > 0 ? ["--force", "--", ...paths] : []), + ], + { + // Cloning scales with remote size; let it finish or be cancelled, like worktree add. + timeoutMs: null, + ...(onSubmoduleLine + ? { + env: { LANGUAGE: "C" }, + progress: { onStdoutLine: onSubmoduleLine, onStderrLine: onSubmoduleLine }, + // Progress for many submodules can pass the output cap; truncate instead of failing. + appendTruncationMarker: true, + keepLineCallbacksAfterTruncation: true, + } + : {}), + }, + ); + // One failed clone stops Git before it checks out any submodule, leaving the cloned ones + // as empty working trees that read as modified; check those out on their own. + const checkOutClonedSubmodules = Effect.gen(function* () { + // Records are "submodule..path\n". + const submodulePaths = splitNullSeparatedPaths( + yield* runGitStdout("GitVcsDriver.createWorktree.listSubmodules", worktreePath, [ + "config", + "-z", + "--file", + ".gitmodules", + "--get-regexp", + "^submodule\\..*\\.path$", + ]), + false, + ).flatMap((record) => { + const newline = record.indexOf("\n"); + return newline === -1 ? [] : [record.slice(newline + 1)]; + }); + const clonedPaths = yield* Effect.filter(submodulePaths, (submodulePath) => + fileSystem + .exists(path.join(worktreePath, submodulePath, ".git")) + .pipe(Effect.orElseSucceed(() => false)), + ); + if (clonedPaths.length > 0) { + yield* updateSubmodules(clonedPaths); + } + }); // Registering submodule URLs writes the shared config; the clones that follow do not. yield* runGit( "GitVcsDriver.createWorktree.initSubmodules", @@ -4704,39 +5277,23 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ["submodule", "init"], { writesSharedConfig: true }, ).pipe( - Effect.andThen( - runGit( - "GitVcsDriver.createWorktree.updateSubmodules", - worktreePath, - [ - "submodule", - "update", - "--init", - ...(submoduleMode.value === "recursive" ? ["--recursive"] : []), - // Git reports transfer progress to a pipe only when forced. - ...(onSubmoduleLine ? ["--progress"] : []), - ], - { - // Cloning scales with remote size; let it finish or be cancelled, like worktree add. - timeoutMs: null, - ...(onSubmoduleLine - ? { - env: { LANGUAGE: "C" }, - progress: { onStdoutLine: onSubmoduleLine, onStderrLine: onSubmoduleLine }, - // Progress for many submodules can pass the output cap; truncate instead of failing. - appendTruncationMarker: true, - keepLineCallbacksAfterTruncation: true, - } - : {}), - }, - ), - ), + Effect.andThen(updateSubmodules([])), Effect.matchEffect({ onFailure: (cause) => Effect.logWarning("worktree submodule checkout failed; submodules may be incomplete", { worktreePath, cause, }).pipe( + Effect.andThen( + checkOutClonedSubmodules.pipe( + Effect.catch((retryCause) => + Effect.logWarning("could not check out the submodules that did clone", { + worktreePath, + cause: retryCause, + }), + ), + ), + ), Effect.andThen( progress?.onSubmodulesFinished ? progress.onSubmodulesFinished({ ok: false, detail: cause.message }) @@ -4903,6 +5460,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const options = { env: STATUS_UPSTREAM_REFRESH_ENV, fallbackErrorDetail: `git fetch ${input.remoteName} failed`, + timeoutMs: FETCH_REMOTE_TIMEOUT_MS, }; // Another fetch in the same repository, such as the background status // poll, can win the race for a remote-tracking ref. The loser fails, but @@ -5028,7 +5586,14 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* remoteName: "origin", branchName: input.refName, }; - return remoteNames.includes(remoteName) ? { remoteName, branchName } : null; + // `/HEAD` is the remote's default-branch pointer, not a branch it can serve, and + // a revision expression such as `HEAD~1`, `@{-1}` or `:/text` is no branch name at all. + return remoteNames.includes(remoteName) && + branchName !== "HEAD" && + !branchName.startsWith("-") && + !/[\x00-\x20\x7f~^:?*[\\]|\.\.|@\{/.test(branchName) + ? { remoteName, branchName } + : null; }); const fetchWorktreeBaseCommit: GitVcsDriver.GitVcsDriver["Service"]["fetchWorktreeBaseCommit"] = @@ -5091,19 +5656,51 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ]); }); - const setBranchUpstream: GitVcsDriver.GitVcsDriver["Service"]["setBranchUpstream"] = (input) => - executeGitWithStableDiagnostics( - "GitVcsDriver.setBranchUpstream", - input.cwd, - [ + const setBranchUpstream: GitVcsDriver.GitVcsDriver["Service"]["setBranchUpstream"] = Effect.fn( + "setBranchUpstream", + )( + function* (input) { + const args = [ "branch", "--set-upstream-to", - `${input.remoteName}/${input.remoteBranch}`, + `refs/remotes/${input.remoteName}/${input.remoteBranch}`, "--", input.branch, - ], - { writesSharedConfig: true }, - ).pipe(retryWhileConfigLocked, Effect.asVoid); + ]; + const result = yield* executeGitWithStableDiagnostics( + "GitVcsDriver.setBranchUpstream", + input.cwd, + args, + { allowNonZeroExit: true }, + ); + if (result.exitCode === 0) return; + // Git requires a configured fetch mapping even when the tracking ref already exists. The + // mapping is a glob because an exact source fails every later fetch once the branch is deleted. + if ( + /^fatal: cannot set up tracking information; starting point '.+' is not a branch$/m.test( + result.stderr, + ) + ) { + yield* executeGitWithStableDiagnostics( + "GitVcsDriver.setBranchUpstream.addFetchMapping", + input.cwd, + [ + "config", + "--add", + `remote.${input.remoteName}.fetch`, + `+refs/heads/${input.remoteBranch}*:refs/remotes/${input.remoteName}/${input.remoteBranch}*`, + ], + ).pipe(retryWhileConfigLocked); + } + yield* executeGitWithStableDiagnostics( + "GitVcsDriver.setBranchUpstream", + input.cwd, + args, + ).pipe(retryWhileConfigLocked); + }, + // Keep mapping detection and both writes under the same lock across sibling worktrees. + (effect, input) => withSharedConfigLock(input.cwd, effect), + ); /** * Unlinks links under `target` without following them and removes the @@ -5224,22 +5821,28 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* if (input.oldBranch === input.newBranch) { return { branch: input.newBranch }; } - const targetBranch = input.exactName - ? input.newBranch - : yield* resolveAvailableBranchName(input.cwd, input.newBranch); - - yield* executeGit( - "GitVcsDriver.renameBranch", + // Parallel launches often generate the same name; picking it under the lock that serializes + // the renames keeps a second launch from choosing a name the first is about to take. + return yield* withSharedConfigLock( input.cwd, - ["branch", "-m", "--", input.oldBranch, targetBranch], - { - timeoutMs: 10_000, - fallbackErrorDetail: "git branch rename failed", - writesSharedConfig: true, - }, - ); + Effect.gen(function* () { + const targetBranch = input.exactName + ? input.newBranch + : yield* resolveAvailableBranchName(input.cwd, input.newBranch); - return { branch: targetBranch }; + yield* executeGit( + "GitVcsDriver.renameBranch", + input.cwd, + ["branch", "-m", "--", input.oldBranch, targetBranch], + { + timeoutMs: 10_000, + fallbackErrorDetail: "git branch rename failed", + }, + ); + + return { branch: targetBranch }; + }), + ); }); const switchRef: GitVcsDriver.GitVcsDriver["Service"]["switchRef"] = Effect.fn("switchRef")( @@ -5309,15 +5912,13 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const checkoutArgs = localInputExists ? ["checkout", input.refName] - : remoteExists && !localTrackingBranch && localTrackedBranchTargetExists - ? ["checkout", input.refName] - : newTrackingBranch - ? ["checkout", newTrackingBranch] - : remoteExists && !localTrackingBranch - ? ["checkout", "--track", input.refName] - : remoteExists && localTrackingBranch - ? ["checkout", localTrackingBranch] - : ["checkout", input.refName]; + : newTrackingBranch + ? ["checkout", newTrackingBranch] + : remoteExists && !localTrackingBranch + ? ["checkout", "--track", input.refName] + : remoteExists && localTrackingBranch + ? ["checkout", localTrackingBranch] + : ["checkout", input.refName]; yield* refuseOptionLikeRefName("GitVcsDriver.switchRef", input.cwd, checkoutArgs.at(-1)); @@ -5483,6 +6084,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* statusDetails, statusDetailsLocal, statusDetailsRemote, + readRefSnapshot, prepareCommitContext, commit: Effect.fnUntraced(function* (cwd, subject, body, options) { return yield* withListRefsInvalidation(cwd, commit(cwd, subject, body, options)); @@ -5572,6 +6174,13 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* return yield* withListRefsInvalidation(input.cwd, switchRef(input)); }), initRepo: initRepoWithListRefsInvalidation, + invalidateRefs: Effect.fn("GitVcsDriver.invalidateRefs")(function* (cwd) { + // First, so a cached "not a repository" cannot end the ref invalidation early. + const cacheKey = normalizeRepositoryPathsCacheKey(cwd); + yield* Cache.invalidate(repositoryPathsRefreshCache, cacheKey); + yield* Cache.invalidate(repositoryPathsCache, cacheKey); + return yield* withListRefsInvalidation(cwd, Effect.void); + }), listLocalBranchNames, listWorktreePaths, }); diff --git a/apps/server/src/vcs/VcsAutomaticFetchInterval.ts b/apps/server/src/vcs/VcsAutomaticFetchInterval.ts new file mode 100644 index 0000000000000..0f2c900b3ba11 --- /dev/null +++ b/apps/server/src/vcs/VcsAutomaticFetchInterval.ts @@ -0,0 +1,26 @@ +import * as Context from "effect/Context"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import { DEFAULT_AUTOMATIC_GIT_FETCH_INTERVAL } from "@t3tools/contracts"; +import { resolveServerBackgroundActivitySettings } from "@t3tools/shared/backgroundActivitySettings"; + +import * as ServerSettings from "../serverSettings.ts"; + +export class VcsAutomaticFetchInterval extends Context.Reference>( + "t3/vcs/VcsAutomaticFetchInterval", + { defaultValue: () => Effect.succeed(DEFAULT_AUTOMATIC_GIT_FETCH_INTERVAL) }, +) {} + +export const layer = Layer.effect( + VcsAutomaticFetchInterval, + Effect.gen(function* () { + const serverSettings = yield* ServerSettings.ServerSettingsService; + return serverSettings.getSettings.pipe( + Effect.map( + (settings) => resolveServerBackgroundActivitySettings(settings).automaticGitFetchInterval, + ), + Effect.orElseSucceed(() => DEFAULT_AUTOMATIC_GIT_FETCH_INTERVAL), + ); + }), +); diff --git a/apps/server/src/vcs/VcsDriver.ts b/apps/server/src/vcs/VcsDriver.ts index b382922c5ab27..383db4302543f 100644 --- a/apps/server/src/vcs/VcsDriver.ts +++ b/apps/server/src/vcs/VcsDriver.ts @@ -14,6 +14,8 @@ import * as VcsProcess from "./VcsProcess.ts"; export interface VcsCaptureCheckpointInput { readonly cwd: string; readonly checkpointRef: CheckpointRef; + /** The checkpoint this one follows, so a turn that switched branches starts where it landed. */ + readonly previousCheckpointRef?: CheckpointRef; } export interface VcsRestoreCheckpointInput { diff --git a/apps/server/src/vcs/VcsProcess.test.ts b/apps/server/src/vcs/VcsProcess.test.ts index b87dbf5e78250..18eaebe6e0959 100644 --- a/apps/server/src/vcs/VcsProcess.test.ts +++ b/apps/server/src/vcs/VcsProcess.test.ts @@ -92,7 +92,7 @@ describe("VcsProcess.run", () => { yield* fs.remove(lock); yield* git(restore); expect(yield* fs.readFileString(file)).toBe("before\n"); - }).pipe(provideLive, Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(provideLive, Effect.provide(NodeServices.layer)), ); it.effect.each([ diff --git a/apps/server/src/vcs/VcsProcess.ts b/apps/server/src/vcs/VcsProcess.ts index 4bfdcc90037ec..da2f988ce1ed8 100644 --- a/apps/server/src/vcs/VcsProcess.ts +++ b/apps/server/src/vcs/VcsProcess.ts @@ -33,6 +33,7 @@ export interface VcsProcessInput { readonly spawnCwd?: string; readonly stdin?: string; readonly onStdoutChunk?: (chunk: Uint8Array) => void; + readonly onSpawn?: () => void; readonly env?: NodeJS.ProcessEnv; readonly allowNonZeroExit?: boolean; readonly timeoutMs?: number; @@ -232,6 +233,7 @@ export const make = Effect.gen(function* () { ...(input.spawnCwd !== undefined ? { spawnCwd: input.spawnCwd } : {}), ...(input.stdin !== undefined ? { stdin: input.stdin } : {}), ...(input.onStdoutChunk !== undefined ? { onStdoutChunk: input.onStdoutChunk } : {}), + ...(input.onSpawn !== undefined ? { onSpawn: input.onSpawn } : {}), env: { ...(input.command === "git" ? unsetInheritedRepositoryEnv(process.env, hostPlatform) diff --git a/apps/server/src/vcs/VcsStatusBroadcaster.test.ts b/apps/server/src/vcs/VcsStatusBroadcaster.test.ts index 87620e5d4fc4f..852c9f9ba0631 100644 --- a/apps/server/src/vcs/VcsStatusBroadcaster.test.ts +++ b/apps/server/src/vcs/VcsStatusBroadcaster.test.ts @@ -28,6 +28,7 @@ import type { import { GitCommandError, GitManagerError } from "@t3tools/contracts"; import { applyGitStatusStreamEvent } from "@t3tools/shared/git"; +import * as VcsAutomaticFetchInterval from "./VcsAutomaticFetchInterval.ts"; import * as VcsStatusBroadcaster from "./VcsStatusBroadcaster.ts"; import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; import * as GitWorkflowService from "../git/GitWorkflowService.ts"; @@ -71,15 +72,17 @@ const remoteStatusWithPr: VcsStatusRemoteResult = { function layerTestFor(state: { currentLocalStatus: VcsStatusLocalResult; currentRemoteStatus: VcsStatusRemoteResult | null; + currentDivergence?: Pick; localStatusCalls: number; remoteStatusCalls: number; localInvalidationCalls: number; remoteInvalidationCalls: number; remoteStatusRefreshUpstreamValues?: Array; + refSnapshot?: string | null; remoteStatusError?: GitManagerError; backgroundWorkEnabled?: boolean; automaticFetchInterval?: Effect.Effect; - beforeRemoteStatus?: Effect.Effect; + beforeRemoteStatus?: Effect.Effect; beforeLocalStatus?: Effect.Effect; beforeCurrentBranch?: Effect.Effect; beforeInvalidate?: Effect.Effect; @@ -100,7 +103,7 @@ function layerTestFor(state: { Layer.provide(layerBackgroundPolicy(() => state.backgroundWorkEnabled !== false)), Layer.provide( Layer.succeed( - VcsStatusBroadcaster.VcsAutomaticFetchInterval, + VcsAutomaticFetchInterval.VcsAutomaticFetchInterval, state.automaticFetchInterval ?? Effect.succeed(Duration.seconds(30)), ), ), @@ -133,6 +136,18 @@ function layerTestFor(state: { : (state.beforeCurrentBranch ?? Effect.void).pipe( Effect.andThen(Effect.sync(() => state.currentLocalStatus.refName)), ), + readRefSnapshot: () => + Effect.sync(() => (state.refSnapshot === undefined ? "refs" : state.refSnapshot)), + upstreamDivergence: () => + Effect.sync(() => + state.currentRemoteStatus?.hasUpstream && state.currentLocalStatus.refName !== null + ? { + refName: state.currentLocalStatus.refName, + aheadCount: (state.currentDivergence ?? state.currentRemoteStatus).aheadCount, + behindCount: (state.currentDivergence ?? state.currentRemoteStatus).behindCount, + } + : null, + ), remoteStatus: (_input, options) => Effect.suspend(() => { return (state.beforeRemoteStatus ?? Effect.void).pipe( @@ -239,7 +254,7 @@ describe("VcsStatusBroadcaster", () => { yield* broadcaster.refreshLocalStatus("/repo"); assert.strictEqual(state.localStatusCalls, 3); - }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(layerTestFor(state))); }); it.effect( @@ -334,10 +349,84 @@ describe("VcsStatusBroadcaster", () => { ]); assert.deepEqual(yield* branches, [{ refName: null, isRepo: false }]); assert.equal(state.remoteStatusCalls, 0); - }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(layerTestFor(state))); }, ); + it.effect("publishes the new branch before pending detailed status completes", () => { + const state = newState(); + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const branches = yield* Stream.toPull(broadcaster.streamBranchStatus({ cwd: "/repo" })); + yield* branches; + const reading = yield* Deferred.make(); + const resume = yield* Deferred.make(); + state.beforeLocalStatus = Deferred.succeed(reading, undefined).pipe( + Effect.andThen(Deferred.await(resume)), + ); + const details = yield* Stream.runHead( + broadcaster.streamStatus({ cwd: "/repo", includeRemote: false }), + ).pipe(Effect.forkScoped); + yield* Deferred.await(reading); + state.currentLocalStatus = { ...baseLocalStatus, refName: "feature/next" }; + const update = yield* branches.pipe(Effect.forkScoped({ startImmediately: true })); + yield* broadcaster.refreshBranchStatus({ cwd: "/repo" }); + assert.deepEqual( + update.pollUnsafe(), + Exit.succeed([{ refName: "feature/next", isRepo: true }]), + ); + state.beforeLocalStatus = Effect.void; + yield* Deferred.succeed(resume, undefined); + yield* Fiber.join(details); + yield* broadcaster.getStatus({ cwd: "/repo" }); + state.currentLocalStatus = { ...baseLocalStatus, refName: "feature/latest" }; + yield* broadcaster.refreshBranchStatus({ cwd: "/repo" }); + assert.deepEqual(yield* branches, [{ refName: "feature/latest", isRepo: true }]); + }).pipe(Effect.provide(layerTestFor(state))); + }); + + it.effect("invalidates totals for a refresh arriving after detailed demand closes", () => { + const state = newState(); + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const detailScope = yield* Scope.make(); + const details = yield* Stream.toPull( + broadcaster.streamStatus({ cwd: "/repo", includeRemote: false }), + ).pipe(Scope.provide(detailScope)); + yield* details; + const reading = yield* Deferred.make(); + const resume = yield* Deferred.make(); + state.currentLocalStatus = { + ...baseLocalStatus, + refName: "feature/next", + workingTree: { files: [], insertions: 1, deletions: 0 }, + }; + state.beforeLocalStatus = Deferred.succeed(reading, undefined).pipe( + Effect.andThen(Deferred.await(resume)), + ); + yield* broadcaster.refreshBranchStatus({ cwd: "/repo" }); + yield* Deferred.await(reading); + yield* Scope.close(detailScope, Exit.void); + state.currentLocalStatus = { + ...state.currentLocalStatus, + workingTree: { files: [], insertions: 2, deletions: 0 }, + }; + const trailing = yield* broadcaster + .refreshBranchStatus({ cwd: "/repo" }) + .pipe(Effect.forkScoped({ startImmediately: true })); + state.beforeLocalStatus = Effect.void; + yield* Deferred.succeed(resume, undefined); + yield* Fiber.join(trailing); + const reopened = yield* Stream.runHead( + broadcaster.streamStatus({ cwd: "/repo", includeRemote: false }), + ); + assert.deepEqual( + reopened, + Option.some({ _tag: "snapshot" as const, local: state.currentLocalStatus, remote: null }), + ); + }).pipe(Effect.provide(layerTestFor(state))); + }); + it.effect("upgrades an in-flight branch refresh when detailed demand arrives", () => { const state: Parameters[0] = { currentLocalStatus: baseLocalStatus, @@ -371,7 +460,7 @@ describe("VcsStatusBroadcaster", () => { yield* Fiber.join(detailsFiber), Option.some({ _tag: "snapshot" as const, local: state.currentLocalStatus, remote: null }), ); - }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(layerTestFor(state))); }); it.effect("publishes terminal branch switches and validates cached status reads", () => { @@ -528,6 +617,48 @@ describe("VcsStatusBroadcaster", () => { }, ); + it.effect.each([0, 2])( + "refreshes merge divergence while a lookup ends with %s commits behind", + (behindAfterLookup) => { + const state: Parameters[0] = { + currentLocalStatus: baseLocalStatus, + currentRemoteStatus: { ...baseRemoteStatus, aheadCount: 2, behindCount: 3 }, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + backgroundWorkEnabled: false, + }; + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + yield* broadcaster.getStatus({ cwd: "/repo" }); + const started = yield* Deferred.make(); + const release = yield* Deferred.make(); + state.beforeRemoteStatus = Deferred.succeed(started, undefined).pipe( + Effect.andThen(Deferred.await(release)), + ); + const lookup = yield* broadcaster.refreshPullRequestStatus("/repo").pipe(Effect.forkScoped); + yield* Deferred.await(started); + state.currentRemoteStatus = { ...baseRemoteStatus, aheadCount: 4, behindCount: 0 }; + yield* broadcaster.refreshLocalStatus("/repo"); + const status = yield* broadcaster.getStatus({ cwd: "/repo" }); + assert.equal(status.aheadCount, 4); + assert.equal(status.behindCount, 0); + assert.equal(state.remoteStatusCalls, 1); + state.currentDivergence = { aheadCount: 4, behindCount: behindAfterLookup }; + state.currentRemoteStatus = + behindAfterLookup > 0 + ? { ...baseRemoteStatus, ...state.currentDivergence } + : { ...baseRemoteStatus, aheadCount: 2, behindCount: 3 }; + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(lookup); + const afterLookup = yield* broadcaster.getStatus({ cwd: "/repo" }); + assert.equal(afterLookup.aheadCount, 4); + assert.equal(afterLookup.behindCount, behindAfterLookup); + }).pipe(Effect.provide(layerTestFor(state))); + }, + ); + it.effect("a local-only refresh after a branch switch leaves the PR refresh able to read", () => { const state: Parameters[0] = { currentLocalStatus: baseLocalStatus, @@ -620,6 +751,7 @@ describe("VcsStatusBroadcaster", () => { Layer.mock(GitWorkflowService.GitWorkflowService)({ localStatus: () => Effect.succeed(localStatus), currentBranch: () => Effect.succeed(localStatus.refName), + readRefSnapshot: () => Effect.succeed("refs"), remoteStatus: () => Effect.succeed(remoteStatus), invalidateLocalStatus: () => Effect.void, invalidateRemoteStatus: () => Effect.void, @@ -716,6 +848,7 @@ describe("VcsStatusBroadcaster", () => { Layer.mock(GitWorkflowService.GitWorkflowService)({ localStatus: () => Effect.succeed(baseLocalStatus), currentBranch: () => Effect.succeed(baseLocalStatus.refName), + readRefSnapshot: () => Effect.succeed("refs"), remoteStatus: () => Effect.gen(function* () { remoteReads += 1; @@ -747,7 +880,7 @@ describe("VcsStatusBroadcaster", () => { assert.deepStrictEqual(refreshed, remoteStatusWithPr); const final = yield* broadcaster.getStatus({ cwd: "/repo" }); assert.deepStrictEqual(final.pr, remoteStatusWithPr.pr); - }).pipe(Effect.provide(layer), Effect.scoped); + }).pipe(Effect.provide(layer)); }); it.effect("an initial status read cannot overwrite an explicit refresh", () => { @@ -762,6 +895,7 @@ describe("VcsStatusBroadcaster", () => { Layer.mock(GitWorkflowService.GitWorkflowService)({ localStatus: () => Effect.succeed(baseLocalStatus), currentBranch: () => Effect.succeed(baseLocalStatus.refName), + readRefSnapshot: () => Effect.succeed("refs"), remoteStatus: () => Effect.gen(function* () { remoteReads += 1; @@ -791,7 +925,7 @@ describe("VcsStatusBroadcaster", () => { (yield* broadcaster.getStatus({ cwd: "/repo" })).pr, remoteStatusWithPr.pr, ); - }).pipe(Effect.provide(layer), Effect.scoped); + }).pipe(Effect.provide(layer)); }); it.effect("keeps the published local snapshot when a refresh remote read fails", () => { @@ -815,6 +949,7 @@ describe("VcsStatusBroadcaster", () => { return state.currentLocalStatus; }), currentBranch: () => Effect.succeed(state.currentLocalStatus.refName), + readRefSnapshot: () => Effect.succeed("refs"), remoteStatus: () => Effect.suspend(() => { state.remoteStatusCalls += 1; @@ -941,6 +1076,49 @@ describe("VcsStatusBroadcaster", () => { }, ); + it.effect("publishes a checkout's branch before returning, ahead of the remote read", () => { + const state: Parameters[0] = { + currentLocalStatus: baseLocalStatus, + currentRemoteStatus: baseRemoteStatus, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + }; + + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + yield* broadcaster.getStatus({ cwd: "/repo" }); + const subscribed = yield* Deferred.make(); + const localPublished = yield* Deferred.make(); + const remotePublished = yield* Deferred.make(); + yield* broadcaster.streamStatus({ cwd: "/repo", includeRemote: false }).pipe( + Stream.runForEach((event) => { + if (event._tag === "snapshot") return Deferred.succeed(subscribed, undefined); + if (event._tag === "localUpdated") return Deferred.succeed(localPublished, event.local); + return event.remote === null + ? Effect.void + : Deferred.succeed(remotePublished, event.remote); + }), + Effect.forkScoped, + ); + yield* Deferred.await(subscribed); + + const releaseRemote = yield* Deferred.make(); + state.beforeRemoteStatus = Deferred.await(releaseRemote); + state.currentLocalStatus = { ...baseLocalStatus, refName: "main" }; + state.currentRemoteStatus = remoteStatusWithPr; + yield* broadcaster.refreshAfterCheckout("/repo"); + + assert.equal((yield* Deferred.await(localPublished)).refName, "main"); + assert.isFalse(yield* Deferred.isDone(remotePublished)); + + yield* Deferred.succeed(releaseRemote, undefined); + assert.deepStrictEqual(yield* Deferred.await(remotePublished), remoteStatusWithPr); + assert.equal((yield* broadcaster.getStatus({ cwd: "/repo" })).pr?.number, 2978); + }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }); + it.effect("a remote poll refreshes Changes totals when divergence counts stay the same", () => { const state: Parameters[0] = { currentLocalStatus: { @@ -977,7 +1155,7 @@ describe("VcsStatusBroadcaster", () => { yield* Deferred.await(polled); const status = yield* broadcaster.getStatus({ cwd: "/repo" }); assert.deepStrictEqual(status.branchChanges, state.currentLocalStatus.branchChanges); - }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(layerTestFor(state))); }); it.effect("an explicit refresh reads local totals after the fetch", () => { @@ -989,6 +1167,7 @@ describe("VcsStatusBroadcaster", () => { branchChanges: { baseRef: "origin/main", insertions: 745, deletions: 24 }, }; let cachedLocal: VcsStatusLocalResult | undefined; + let refSnapshot = "before"; const readLocalStatus = () => Effect.sync(() => (cachedLocal ??= local)); const layerTest = VcsStatusBroadcaster.layer.pipe( Layer.provideMerge(NodeServices.layer), @@ -1006,6 +1185,7 @@ describe("VcsStatusBroadcaster", () => { }), currentBranch: () => Effect.succeed(baseLocalStatus.refName), invalidateRemoteStatus: () => Effect.void, + readRefSnapshot: () => Effect.sync(() => refSnapshot), remoteStatus: () => Deferred.succeed(fetchStarted, undefined).pipe( Effect.andThen(Deferred.await(finishFetch)), @@ -1024,6 +1204,7 @@ describe("VcsStatusBroadcaster", () => { ...baseLocalStatus, branchChanges: { baseRef: "origin/main", insertions: 0, deletions: 0 }, }; + refSnapshot = "after"; yield* Deferred.succeed(finishFetch, undefined); const status = yield* Fiber.join(refresh); assert.deepStrictEqual(status.branchChanges, local.branchChanges); @@ -1031,6 +1212,58 @@ describe("VcsStatusBroadcaster", () => { }); }); + it.effect("an explicit refresh re-reads local totals only after refs moved", () => { + const totals = (insertions: number): VcsStatusLocalResult => ({ + ...baseLocalStatus, + branchChanges: { baseRef: "origin/main", insertions, deletions: 0 }, + }); + const state: Parameters[0] = { + currentLocalStatus: totals(745), + currentRemoteStatus: remoteStatusWithPr, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + }; + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const subscribed = yield* Deferred.make(); + const events = yield* broadcaster.streamStatus({ cwd: "/repo", includeRemote: false }).pipe( + Stream.tap((event) => + event._tag === "snapshot" ? Deferred.succeed(subscribed, undefined) : Effect.void, + ), + Stream.filter((event) => event._tag !== "snapshot"), + Stream.take(3), + Stream.runCollect, + Effect.forkScoped, + ); + yield* Deferred.await(subscribed); + + state.localStatusCalls = 0; + yield* broadcaster.refreshStatus("/repo"); + assert.equal(state.localStatusCalls, 1); + + state.localStatusCalls = 0; + state.currentRemoteStatus = baseRemoteStatus; + state.beforeRemoteStatus = Effect.sync(() => { + state.currentLocalStatus = totals(0); + state.refSnapshot = "moved"; + }); + const moved = yield* broadcaster.refreshStatus("/repo"); + assert.equal(state.localStatusCalls, 2); + assert.deepStrictEqual(moved.branchChanges, totals(0).branchChanges); + assert.deepStrictEqual( + [...(yield* Fiber.join(events))].map((event) => event._tag), + ["remoteUpdated", "remoteUpdated", "localUpdated"], + ); + + state.localStatusCalls = 0; + state.refSnapshot = null; + yield* broadcaster.refreshStatus("/repo"); + assert.equal(state.localStatusCalls, 2); + }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }); + it.effect("publishes local status while an earlier lookup holds the remote lock", () => { const state: Parameters[0] = { currentLocalStatus: baseLocalStatus, @@ -1077,7 +1310,7 @@ describe("VcsStatusBroadcaster", () => { const status = yield* Fiber.join(refresh); assert.isTrue(status.hasWorkingTreeChanges); assert.deepStrictEqual(status.pr, remoteStatusWithPr.pr); - }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(layerTestFor(state))); }); it.effect("publishes local changes before a remote error and propagates that error", () => { @@ -1139,7 +1372,7 @@ describe("VcsStatusBroadcaster", () => { assert.isTrue(cached.hasWorkingTreeChanges); assert.deepStrictEqual(cached.pr, remoteStatusWithPr.pr); assert.deepStrictEqual(yield* Deferred.await(published), state.currentLocalStatus); - }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(layerTestFor(state))); }); it.effect.each(["getStatus", "refreshStatus", "autoPull", "autoPullFailure"] as const)( @@ -1225,7 +1458,7 @@ describe("VcsStatusBroadcaster", () => { assert.isNull(status.pr); assert.equal(status.aheadCount, 0); assert.equal(status.behindCount, 0); - }).pipe(Effect.provide(layerTest), Effect.scoped); + }).pipe(Effect.provide(layerTest)); }, ); @@ -1275,7 +1508,7 @@ describe("VcsStatusBroadcaster", () => { assert.deepStrictEqual(cached.pr, state.currentRemoteStatus.pr); assert.equal(cached.aheadCount, 2); assert.equal(state.remoteStatusCalls, 2); - }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(layerTestFor(state))); }); it.effect("a terminal checkout publishes the new branch without waiting for a poll", () => @@ -1307,7 +1540,10 @@ describe("VcsStatusBroadcaster", () => { const next = applyGitStatusStreamEvent(clientStates.at(-1) ?? null, event); clientStates.push(next); yield* Deferred.succeed(subscribed, undefined); - if (next.refName === "repro") yield* Deferred.succeed(checkedOut, undefined); + // The refresh ends with the new branch's remote, after its local update. + if (event._tag === "remoteUpdated" && event.remote && next.refName === "repro") { + yield* Deferred.succeed(checkedOut, undefined); + } }), ).pipe(Effect.forkScoped); yield* Deferred.await(subscribed); @@ -1322,12 +1558,12 @@ describe("VcsStatusBroadcaster", () => { assert.deepStrictEqual( clientStates.filter((status) => status.refName === "repro").map((status) => status.pr), - [null], + [null, null], ); // A checkout must not trigger a network fetch. assert.deepStrictEqual(state.remoteStatusRefreshUpstreamValues.slice(1), [false]); }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); it.effect("a terminal checkout updates branch-only subscribers without full status", () => @@ -1360,7 +1596,50 @@ describe("VcsStatusBroadcaster", () => { assert.deepEqual(yield* branches, [{ refName: "repro", isRepo: true }]); assert.equal(state.localStatusCalls, 0); }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("a terminal checkout updates the branch before pending detailed status completes", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const gitDir = yield* fs.makeTempDirectoryScoped({ prefix: "vcs-head-watch-" }); + const headPath = path.join(gitDir, "HEAD"); + yield* fs.writeFileString(headPath, "ref: refs/heads/main\n"); + const state: Parameters[0] = { + currentLocalStatus: { ...baseLocalStatus, refName: "main" }, + currentRemoteStatus: null, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + headPath, + }; + + yield* Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const branches = yield* Stream.toPull(broadcaster.streamBranchStatus({ cwd: "/repo" })); + yield* branches; + const details = yield* Stream.toPull( + broadcaster.streamStatus({ cwd: "/repo", includeRemote: false }), + ); + yield* details; + + const resume = yield* Deferred.make(); + state.beforeLocalStatus = Deferred.await(resume); + state.currentLocalStatus = { ...baseLocalStatus, refName: "repro" }; + const lockPath = path.join(gitDir, "HEAD.lock"); + yield* fs.writeFileString(lockPath, "ref: refs/heads/repro\n"); + yield* fs.rename(lockPath, headPath); + + assert.deepEqual(yield* branches, [{ refName: "repro", isRepo: true }]); + state.currentLocalStatus = { ...baseLocalStatus, refName: "second" }; + yield* fs.writeFileString(lockPath, "ref: refs/heads/second\n"); + yield* fs.rename(lockPath, headPath); + assert.deepEqual(yield* branches, [{ refName: "second", isRepo: true }]); + yield* Deferred.succeed(resume, undefined); + }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(NodeServices.layer)), ); it.effect("a checkout read that fails while index.lock is held retries until it succeeds", () => @@ -1415,7 +1694,7 @@ describe("VcsStatusBroadcaster", () => { yield* Deferred.await(checkedOut); assert.strictEqual(state.localStatusCalls, failedCalls + 1); }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); - }).pipe(Effect.scoped), + }), ); it.effect("subscribers share one HEAD watch that ends with the last of them", () => @@ -1457,7 +1736,7 @@ describe("VcsStatusBroadcaster", () => { yield* subscribe; assert.strictEqual(state.headPathCalls, 2); }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); it.effect("a HEAD watch that ended restarts for the next subscriber and keeps earlier ones", () => @@ -1525,7 +1804,7 @@ describe("VcsStatusBroadcaster", () => { yield* checkout("two"); yield* Deferred.await(first.sawRef("two")); }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); - }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + }).pipe(Effect.provide(NodeServices.layer)), ); it.effect("a HEAD watch ends when its git directory is removed", () => @@ -1668,6 +1947,92 @@ describe("VcsStatusBroadcaster", () => { assert.isTrue(Option.isSome(yield* Deferred.poll(remoteInterrupted))); }).pipe(Effect.provide(layerTest)); }); + + it.effect("refreshes requested during a refresh share one follow-up refresh", () => { + const state: Parameters[0] = { + currentLocalStatus: baseLocalStatus, + currentRemoteStatus: baseRemoteStatus, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + }; + + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const fetchStarted = yield* Deferred.make(); + const finishFetch = yield* Deferred.make(); + state.beforeRemoteStatus = Deferred.succeed(fetchStarted, undefined).pipe( + Effect.andThen(Deferred.await(finishFetch)), + ); + const first = yield* broadcaster.refreshStatus("/repo").pipe(Effect.forkScoped); + yield* Deferred.await(fetchStarted); + // Every read after the first sees a PR, so a caller that reused the + // first refresh would get a status from before its own request. + state.beforeRemoteStatus = Effect.sync(() => { + state.currentRemoteStatus = remoteStatusWithPr; + }); + const later = yield* Effect.all( + [broadcaster.refreshStatus("/repo"), broadcaster.refreshStatus("/repo")], + { concurrency: "unbounded" }, + ).pipe(Effect.forkScoped({ startImmediately: true })); + yield* Deferred.succeed(finishFetch, undefined); + + assert.isNull((yield* Fiber.join(first)).pr); + const [second, third] = yield* Fiber.join(later); + assert.deepStrictEqual(second.pr, remoteStatusWithPr.pr); + assert.deepStrictEqual(third.pr, remoteStatusWithPr.pr); + assert.equal(state.remoteStatusCalls, 2); + + yield* broadcaster.refreshStatus("/repo"); + assert.equal(state.remoteStatusCalls, 3); + }).pipe(Effect.provide(layerTestFor(state))); + }); + + it.effect("a failed refresh releases interrupted waiters and runs the shared follow-up", () => { + const state: Parameters[0] = { + currentLocalStatus: baseLocalStatus, + currentRemoteStatus: remoteStatusWithPr, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + }; + + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const started = yield* Deferred.make(); + const release = yield* Deferred.make(); + state.beforeRemoteStatus = Deferred.succeed(started, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.andThen( + Effect.fail( + new GitManagerError({ + operation: "VcsStatusBroadcaster.test", + cwd: "/repo", + detail: "remote status failed", + }), + ), + ), + ); + const first = yield* broadcaster.refreshStatus("/repo").pipe(Effect.exit, Effect.forkScoped); + yield* Deferred.await(started); + const interrupted = yield* broadcaster + .refreshStatus("/repo") + .pipe(Effect.forkScoped({ startImmediately: true })); + const remaining = yield* broadcaster + .refreshStatus("/repo") + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Fiber.interrupt(interrupted); + state.beforeRemoteStatus = Effect.void; + yield* Deferred.succeed(release, undefined); + assert.isTrue(Exit.isFailure(yield* Fiber.join(first))); + assert.deepStrictEqual((yield* Fiber.join(remaining)).pr, remoteStatusWithPr.pr); + assert.equal(state.remoteStatusCalls, 1); + yield* broadcaster.refreshStatus("/repo"); + assert.equal(state.remoteStatusCalls, 2); + }).pipe(Effect.provide(layerTestFor(state))); + }); }); const newState = (): Parameters[0] => ({ @@ -1709,7 +2074,7 @@ it.effect("detail join during invalidation must get fresh totals", () => { yield* Fiber.join(snapshotFiber), Option.some({ _tag: "snapshot" as const, local: state.currentLocalStatus, remote: null }), ); - }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(layerTestFor(state))); }); it.effect("serializes branch reads with detailed refresh publication", () => { const state = newState(); @@ -1735,7 +2100,7 @@ it.effect("serializes branch reads with detailed refresh publication", () => { const result = yield* b.getStatus({ cwd: "/repo" }); assert.equal(result.refName, "new"); assert.equal(state.localStatusCalls, 1); - }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(layerTestFor(state))); }); it.effect("publishes overlapping branch reads in checkout order", () => { @@ -1766,7 +2131,7 @@ it.effect("publishes overlapping branch reads in checkout order", () => { { refName: "old", isRepo: true }, { refName: "new", isRepo: true }, ]); - }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(layerTestFor(state))); }); it.effect("does not replay older queued branches after a new subscriber snapshot", () => { @@ -1799,5 +2164,5 @@ it.effect("does not replay older queued branches after a new subscriber snapshot yield* b.refreshBranchStatus({ cwd: "/repo" }); assert.deepEqual(yield* Queue.take(values), { refName: "latest", isRepo: true }); yield* Fiber.join(subscriber); - }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(layerTestFor(state))); }); diff --git a/apps/server/src/vcs/VcsStatusBroadcaster.ts b/apps/server/src/vcs/VcsStatusBroadcaster.ts index 99b3a2374162a..8e69679d50638 100644 --- a/apps/server/src/vcs/VcsStatusBroadcaster.ts +++ b/apps/server/src/vcs/VcsStatusBroadcaster.ts @@ -25,7 +25,6 @@ import type { VcsStatusSubscriptionInput, } from "@t3tools/contracts"; import { DEFAULT_AUTOMATIC_GIT_FETCH_INTERVAL } from "@t3tools/contracts"; -import { resolveServerBackgroundActivitySettings } from "@t3tools/shared/backgroundActivitySettings"; import { mergeGitStatusParts } from "@t3tools/shared/git"; import * as KeyedLock from "@t3tools/shared/KeyedLock"; import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; @@ -34,6 +33,7 @@ import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; import * as GitWorkflowService from "../git/GitWorkflowService.ts"; import * as ProjectStore from "../orchestration-v2/ProjectStore.ts"; import * as ServerSettings from "../serverSettings.ts"; +import { VcsAutomaticFetchInterval } from "./VcsAutomaticFetchInterval.ts"; const VCS_STATUS_REFRESH_FAILURE_BASE_DELAY = Duration.seconds(30); const VCS_STATUS_REFRESH_FAILURE_MAX_DELAY = Duration.minutes(15); @@ -160,24 +160,6 @@ interface ActiveRemotePoller { readonly demandCwds: Ref.Ref>; } -export class VcsAutomaticFetchInterval extends Context.Reference>( - "t3/vcs/VcsAutomaticFetchInterval", - { defaultValue: () => Effect.succeed(DEFAULT_AUTOMATIC_GIT_FETCH_INTERVAL) }, -) {} - -export const layerAutomaticFetchInterval = Layer.effect( - VcsAutomaticFetchInterval, - Effect.gen(function* () { - const serverSettings = yield* ServerSettings.ServerSettingsService; - return serverSettings.getSettings.pipe( - Effect.map( - (settings) => resolveServerBackgroundActivitySettings(settings).automaticGitFetchInterval, - ), - Effect.orElseSucceed(() => DEFAULT_AUTOMATIC_GIT_FETCH_INTERVAL), - ); - }), -); - export class VcsAutoPullPolicy extends Context.Reference<{ readonly isEnabled: (cwd: string) => Effect.Effect; }>("t3/vcs/VcsAutoPullPolicy", { @@ -236,6 +218,11 @@ export class VcsStatusBroadcaster extends Context.Service< readonly refreshLocalStatus: ( cwd: string, ) => Effect.Effect; + /** + * For a checkout someone is waiting on. Returns once the new branch is published, so the + * caller's reply does not outrun it, and finishes the slower remote read in the background. + */ + readonly refreshAfterCheckout: (cwd: string) => Effect.Effect; readonly refreshStatus: ( cwd: string, options?: { readonly localOnly?: boolean }, @@ -286,17 +273,29 @@ export const make = Effect.gen(function* () { const detailedSubscribers = new Map(); const branches = new Map< string, - { readonly refName: string | null; readonly isRepo: boolean; readonly revision: number } + { + readonly refName: string | null; + readonly isRepo: boolean; + readonly revision: number; + readonly observation: number; + } >(); let branchRevision = 0; + // A slow detailed read must not replace a newer HEAD-only observation. + let branchObservation = 0; const publishBranch = Effect.fn("VcsStatusBroadcaster.publishBranch")(function* ( cwd: string, { refName, isRepo }: { readonly refName: string | null; readonly isRepo: boolean }, + observation: number, ) { const previous = branches.get(cwd); - if (previous?.refName === refName && previous.isRepo === isRepo) return; + if (previous && previous.observation > observation) return; + if (previous?.refName === refName && previous.isRepo === isRepo) { + branches.set(cwd, { ...previous, observation }); + return; + } const revision = ++branchRevision; - branches.set(cwd, { refName, isRepo, revision }); + branches.set(cwd, { refName, isRepo, revision, observation }); yield* PubSub.publish(branchChangesPubSub, { cwd, refName, isRepo, revision }); }); const localRefreshLocks = yield* KeyedLock.make(); @@ -317,7 +316,12 @@ export const make = Effect.gen(function* () { }); const updateCachedLocalStatus = Effect.fn("VcsStatusBroadcaster.updateCachedLocalStatus")( - function* (cwd: string, local: VcsStatusLocalResult, options?: { publish?: boolean }) { + function* ( + cwd: string, + local: VcsStatusLocalResult, + observation: number, + options?: { publish?: boolean }, + ) { const nextLocal = { fingerprint: fingerprintStatusPart(local), value: local }; const [localChanged, remoteCleared] = yield* Ref.modify(cacheRef, (cache) => { const previous = cache.get(cwd) ?? { local: null, remote: null }; @@ -327,7 +331,7 @@ export const make = Effect.gen(function* () { new Map(cache).set(cwd, { local: nextLocal, remote }), ] as const; }); - yield* publishBranch(cwd, local); + yield* publishBranch(cwd, local, observation); if (options?.publish && remoteCleared) { yield* PubSub.publish(changesPubSub, { cwd, @@ -366,9 +370,11 @@ export const make = Effect.gen(function* () { const loadLocalStatus = Effect.fn("VcsStatusBroadcaster.loadLocalStatus")(function* ( cwd: string, + options?: { publish?: boolean }, ) { + const observation = ++branchObservation; const local = yield* workflow.localStatus({ cwd }); - return yield* updateCachedLocalStatus(cwd, local); + return yield* updateCachedLocalStatus(cwd, local, observation, options); }); const getOrLoadLocalStatusUnlocked = Effect.fn("VcsStatusBroadcaster.getOrLoadLocalStatus")( @@ -378,9 +384,7 @@ export const make = Effect.gen(function* () { const branch = yield* workflow.currentBranch(cwd); if (branch === cached.local.value.refName) return cached.local.value; yield* workflow.invalidateLocalStatus(cwd); - return yield* updateCachedLocalStatus(cwd, yield* workflow.localStatus({ cwd }), { - publish: true, - }); + return yield* loadLocalStatus(cwd, { publish: true }); } return yield* loadLocalStatus(cwd); }, @@ -439,9 +443,10 @@ export const make = Effect.gen(function* () { if (shared !== undefined && shared.startedAt >= requestedAt) return shared.local; const startedAt = localRefreshRequests.get(cwd) ?? requestedAt; yield* workflow.invalidateLocalStatus(cwd); + const observation = ++branchObservation; const local = yield* workflow.localStatus({ cwd }); localRefreshResults.set(cwd, { startedAt, local }); - return yield* updateCachedLocalStatus(cwd, local, { publish: true }); + return yield* updateCachedLocalStatus(cwd, local, observation, { publish: true }); }), ); }, @@ -451,7 +456,30 @@ export const make = Effect.gen(function* () { "VcsStatusBroadcaster.refreshLocalStatus", )(function* (rawCwd) { const cwd = yield* withFileSystem(normalizeCwd(rawCwd)); - return yield* refreshLocalStatusCore(cwd); + const local = yield* refreshLocalStatusCore(cwd); + const cached = yield* getCachedStatus(cwd); + // Divergence is local Git data; a slow forge lookup must not delay turn-end updates. + if (cached?.remote?.value?.hasUpstream) { + const divergence = yield* workflow.upstreamDivergence(cwd); + const latest = yield* getCachedStatus(cwd); + if (divergence && latest?.remote?.value && latest.remote.branch === divergence.refName) { + const value = { + ...latest.remote.value, + aheadCount: divergence.aheadCount, + behindCount: divergence.behindCount, + }; + yield* updateCachedRemoteStatus( + cwd, + { + branch: divergence.refName, + value, + fingerprint: fingerprintStatusPart(value), + }, + { publish: true }, + ); + } + } + return local; }); const readRemoteStatus = Effect.fn("VcsStatusBroadcaster.readRemoteStatus")(function* ( @@ -461,7 +489,18 @@ export const make = Effect.gen(function* () { const branch = yield* workflow.currentBranch(cwd); // GitManager's remote TTL cache is keyed by cwd, not the checked-out branch. yield* workflow.invalidateRemoteStatus(cwd); - const value = yield* workflow.remoteStatus({ cwd }, options); + const previousRemote = (yield* getCachedStatus(cwd))?.remote; + const remote = yield* workflow.remoteStatus({ cwd }, options); + const latest = yield* getCachedStatus(cwd); + // An overlapping turn refresh and fetch can both move divergence during the forge lookup. + const refreshed = + remote?.hasUpstream && latest?.remote !== previousRemote && latest?.remote?.branch === branch + ? yield* workflow.upstreamDivergence(cwd) + : null; + const value = + remote && refreshed?.refName === branch + ? { ...remote, aheadCount: refreshed.aheadCount, behindCount: refreshed.behindCount } + : remote; const currentBranch = yield* workflow.currentBranch(cwd); const cached = yield* getCachedStatus(cwd); if (branch !== currentBranch || cached?.local?.value.refName !== currentBranch) { @@ -551,33 +590,62 @@ export const make = Effect.gen(function* () { ); }); + const refreshRemoteAndLocalTotals = Effect.fn("VcsStatusBroadcaster.refreshRemoteAndLocalTotals")( + function* (cwd: string, policyCwd: string, refsBeforeLocalRead: string | null) { + return yield* withRemoteWriteLock( + cwd, + Effect.gen(function* () { + const refreshUpstream = !Duration.isZero(yield* automaticFetchInterval); + // Bump the PR lookup epoch after earlier remote writers have finished. + yield* workflow.invalidateStatus(cwd); + const remoteRead = yield* readRemoteStatus(cwd, { refreshUpstream }); + // Without a fetch, the cached behind count cannot justify an automatic pull. + const pulled = refreshUpstream + ? yield* maybeAutoPull(cwd, remoteRead, [policyCwd]) + : null; + if (pulled !== null) return mergeGitStatusParts(pulled.local, pulled.remote); + yield* updateCachedRemoteStatus(cwd, remoteRead, { publish: true }); + // A fetch, ours or not, can move the base used for local Changes totals. + if ( + refsBeforeLocalRead === null || + (yield* workflow.readRefSnapshot(cwd)) !== refsBeforeLocalRead + ) { + yield* refreshLocalStatusCore(cwd); + } + const parts = yield* getStatusParts(cwd); + return mergeGitStatusParts(parts.local, parts.remote); + }), + ); + }, + ); + const refreshStatus: VcsStatusBroadcaster["Service"]["refreshStatus"] = Effect.fn( "VcsStatusBroadcaster.refreshStatus", )(function* (rawCwd, options) { const cwd = yield* withFileSystem(normalizeCwd(rawCwd)); + const refsBeforeLocalRead = options?.localOnly ? null : yield* workflow.readRefSnapshot(cwd); yield* refreshLocalStatusCore(cwd); if (options?.localOnly) { // Never cache a remote here: readers take `value: null` as "read and found no PR". const parts = yield* getStatusParts(cwd); return mergeGitStatusParts(parts.local, parts.remote); } - return yield* withRemoteWriteLock( - cwd, - Effect.gen(function* () { - const refreshUpstream = !Duration.isZero(yield* automaticFetchInterval); - // Bump the PR lookup epoch after earlier remote writers have finished. - yield* workflow.invalidateStatus(cwd); - const remoteRead = yield* readRemoteStatus(cwd, { refreshUpstream }); - // Fetch can move the base used for local Changes totals. - yield* refreshLocalStatusCore(cwd); - // Without a fetch, the cached behind count cannot justify an automatic pull. - const pulled = refreshUpstream ? yield* maybeAutoPull(cwd, remoteRead, [rawCwd]) : null; - if (pulled !== null) return mergeGitStatusParts(pulled.local, pulled.remote); - yield* updateCachedRemoteStatus(cwd, remoteRead, { publish: true }); - const parts = yield* getStatusParts(cwd); - return mergeGitStatusParts(parts.local, parts.remote); - }), + return yield* refreshRemoteAndLocalTotals(cwd, rawCwd, refsBeforeLocalRead); + }); + + const refreshAfterCheckout: VcsStatusBroadcaster["Service"]["refreshAfterCheckout"] = Effect.fn( + "VcsStatusBroadcaster.refreshAfterCheckout", + )(function* (rawCwd) { + const cwd = yield* withFileSystem(normalizeCwd(rawCwd)); + const localDone = yield* Deferred.make(); + // Owned by the broadcaster so a client disconnecting mid-switch still leaves status correct. + yield* refreshLocalStatusCore(cwd).pipe( + Effect.ensuring(Deferred.succeed(localDone, undefined)), + Effect.andThen(refreshRemoteAndLocalTotals(cwd, rawCwd, null)), + Effect.ignoreCause({ log: true }), + Effect.forkIn(broadcasterScope), ); + yield* Deferred.await(localDone); }); const refreshPullRequestStatus: VcsStatusBroadcaster["Service"]["refreshPullRequestStatus"] = @@ -765,7 +833,66 @@ export const make = Effect.gen(function* () { } }); - // Serialize branch publication and cache invalidation with detailed reads and refreshes. + // A request made during a refresh needs a read that starts after it, so overlapping requests + // share one follow-up. Keyed by raw cwd because auto-pull policy depends on it. + // Missing means idle; null means running with no follow-up queued. + type SharedRefresh = Deferred.Deferred; + const queuedRefreshesRef = yield* Ref.make(new Map()); + + const runQueuedRefreshes = Effect.fn("VcsStatusBroadcaster.runQueuedRefreshes")(function* ( + rawCwd: string, + first: SharedRefresh, + ) { + let next: SharedRefresh | null = first; + while (next !== null) { + yield* Deferred.into(refreshStatus(rawCwd), next); + next = yield* Ref.modify(queuedRefreshesRef, (queued) => { + const followUp = queued.get(rawCwd) ?? null; + const nextQueued = new Map(queued); + if (followUp === null) { + nextQueued.delete(rawCwd); + } else { + nextQueued.set(rawCwd, null); + } + return [followUp, nextQueued] as const; + }); + } + }); + + const shareQueuedRefreshStatus: VcsStatusBroadcaster["Service"]["refreshStatus"] = Effect.fn( + "VcsStatusBroadcaster.shareQueuedRefreshStatus", + )(function* (rawCwd, options) { + if (options?.localOnly) return yield* refreshStatus(rawCwd, options); + const requested = yield* Deferred.make(); + // Registering and forking together keeps an interrupt from leaving an entry with no worker. + const shared = yield* Effect.uninterruptible( + Effect.gen(function* () { + const followUp = yield* Ref.modify( + queuedRefreshesRef, + (queued): readonly [SharedRefresh | undefined, Map] => { + if (!queued.has(rawCwd)) return [undefined, new Map(queued).set(rawCwd, null)]; + const followUp = queued.get(rawCwd) ?? requested; + return [followUp, new Map(queued).set(rawCwd, followUp)]; + }, + ); + if (followUp !== undefined) return followUp; + // The broadcaster scope owns the work, so one caller's interrupt does not cancel it for the rest. + yield* runQueuedRefreshes(rawCwd, requested).pipe(Effect.forkIn(broadcasterScope)); + return requested; + }), + ); + return yield* Deferred.await(shared); + }); + + const readHeadBranch = Effect.fn("VcsStatusBroadcaster.readHeadBranch")(function* (cwd: string) { + const observation = ++branchObservation; + const refName = yield* workflow.currentBranch(cwd); + // A null branch is a detached HEAD or a checkout that is gone, such as a removed worktree. + const isRepo = refName !== null || (yield* workflow.repositoryRoot(cwd)) !== null; + return { observation, refName, isRepo }; + }); + + // Keep cache invalidation serialized with detailed reads and refreshes. const refreshBranchStatusCore = Effect.fn("VcsStatusBroadcaster.refreshBranchStatusCore")( (cwd: string) => localRefreshLocks.withLock( @@ -773,15 +900,11 @@ export const make = Effect.gen(function* () { Effect.gen(function* () { if (detailedSubscribers.has(cwd)) { const local = yield* getOrLoadLocalStatusUnlocked(cwd); - return { - refName: local.refName, - isRepo: local.isRepo, - revision: branches.get(cwd)?.revision ?? 0, - }; + return ( + branches.get(cwd) ?? { refName: local.refName, isRepo: local.isRepo, revision: 0 } + ); } - const refName = yield* workflow.currentBranch(cwd); - // A null branch is a detached HEAD or a checkout that is gone, such as a removed worktree. - const isRepo = refName !== null || (yield* workflow.repositoryRoot(cwd)) !== null; + const { observation, refName, isRepo } = yield* readHeadBranch(cwd); // HEAD cannot certify working-tree totals, even when it has not moved. localRefreshResults.delete(cwd); yield* workflow.invalidateLocalStatus(cwd); @@ -790,34 +913,49 @@ export const make = Effect.gen(function* () { return previous ? new Map(cache).set(cwd, { ...previous, local: null }) : cache; }); if (detailedSubscribers.has(cwd)) { - const local = yield* workflow.localStatus({ cwd }); - yield* updateCachedLocalStatus(cwd, local, { publish: true }); - return { - refName: local.refName, - isRepo: local.isRepo, - revision: branches.get(cwd)?.revision ?? 0, - }; + const local = yield* loadLocalStatus(cwd, { publish: true }); + return ( + branches.get(cwd) ?? { refName: local.refName, isRepo: local.isRepo, revision: 0 } + ); } - yield* publishBranch(cwd, { refName, isRepo }); - return { refName, isRepo, revision: branches.get(cwd)?.revision ?? 0 }; + yield* publishBranch(cwd, { refName, isRepo }, observation); + return branches.get(cwd) ?? { refName, isRepo, revision: 0 }; }), ), ); // A checkout outside T3 Code, such as `git switch` in the terminal, calls no refresh RPC. // Rereading remote too, without a fetch, sends the new branch with its own PR and counts. - const refreshCheckedOutRef = Effect.fn("VcsStatusBroadcaster.refreshCheckedOutRef")(function* ( + const publishCheckedOutRef = Effect.fn("VcsStatusBroadcaster.publishCheckedOutRef")(function* ( cwd: string, ) { - if (!detailedSubscribers.has(cwd)) { - yield* refreshBranchStatusCore(cwd); - } else if ((yield* getCachedStatus(cwd))?.remote) { - yield* refreshRemoteStatus(cwd, { refreshUpstream: false }); - } else { - yield* refreshLocalStatusCore(cwd); - } + if (!detailedSubscribers.has(cwd)) return yield* refreshBranchStatusCore(cwd); + const { observation, refName, isRepo } = yield* readHeadBranch(cwd); + yield* publishBranch(cwd, { refName, isRepo }, observation); }); + const reconcileCheckedOutRef = Effect.fn("VcsStatusBroadcaster.reconcileCheckedOutRef")( + function* (cwd: string) { + if (!detailedSubscribers.has(cwd)) return; + if ((yield* getCachedStatus(cwd))?.remote) { + yield* refreshRemoteStatus(cwd, { refreshUpstream: false }); + } else { + yield* refreshLocalStatusCore(cwd); + } + }, + ); + + const retryCheckedOutRef = (cwd: string, refresh: Effect.Effect) => + refresh.pipe( + Effect.retry(CHECKOUT_REFRESH_RETRY), + Effect.catchCause((cause) => + Effect.logWarning("VCS checked-out ref refresh failed", { + cwdLength: cwd.length, + ...remoteRefreshFailureDiagnostics(cause), + }), + ), + ); + // Runs for the subscription lifetime, so tracing would leave a span open indefinitely. const watchHead = Effect.fnUntraced(function* (cwd: string, ready: Deferred.Deferred) { const headPath = yield* workflow.headPath(cwd); @@ -838,17 +976,10 @@ export const make = Effect.gen(function* () { Stream.filter((event) => path.basename(event.path) === "HEAD"), // A rebase moves HEAD many times; refresh once for whatever arrived during a refresh. Stream.buffer({ capacity: 1, strategy: "sliding" }), - Stream.runForEach(() => - refreshCheckedOutRef(cwd).pipe( - Effect.retry(CHECKOUT_REFRESH_RETRY), - Effect.catchCause((cause) => - Effect.logWarning("VCS checked-out ref refresh failed", { - cwdLength: cwd.length, - ...remoteRefreshFailureDiagnostics(cause), - }), - ), - ), - ), + Stream.mapEffect(() => retryCheckedOutRef(cwd, publishCheckedOutRef(cwd))), + // Detailed status waits behind its own reads; the branch keeps up on the stage above. + Stream.buffer({ capacity: 1, strategy: "sliding" }), + Stream.runForEach(() => retryCheckedOutRef(cwd, reconcileCheckedOutRef(cwd))), ); }); @@ -959,8 +1090,21 @@ export const make = Effect.gen(function* () { const refreshBranchStatus: VcsStatusBroadcaster["Service"]["refreshBranchStatus"] = Effect.fn( "VcsStatusBroadcaster.refreshBranchStatus", )(function* (input) { - const { refName, isRepo } = yield* refreshBranchStatusCore( - yield* withFileSystem(normalizeCwd(input.cwd)), + const cwd = yield* withFileSystem(normalizeCwd(input.cwd)); + if (!detailedSubscribers.has(cwd)) { + const { refName, isRepo } = yield* refreshBranchStatusCore(cwd); + return { refName, isRepo }; + } + const { observation, refName, isRepo } = yield* readHeadBranch(cwd); + yield* publishBranch(cwd, { refName, isRepo }, observation); + yield* refreshBranchStatusCore(cwd).pipe( + Effect.catchCause((cause) => + Effect.logWarning("VCS branch refresh failed", { + cwdLength: cwd.length, + ...remoteRefreshFailureDiagnostics(cause), + }), + ), + Effect.forkIn(broadcasterScope), ); return { refName, isRepo }; }); @@ -994,7 +1138,8 @@ export const make = Effect.gen(function* () { ), getStatus, refreshLocalStatus, - refreshStatus, + refreshAfterCheckout, + refreshStatus: shareQueuedRefreshStatus, refreshPullRequestStatus, streamStatus, }); diff --git a/apps/server/src/vcs/worktreeCheckoutCache.ts b/apps/server/src/vcs/worktreeCheckoutCache.ts index a72d6d859fc54..8a51189fe6d69 100644 --- a/apps/server/src/vcs/worktreeCheckoutCache.ts +++ b/apps/server/src/vcs/worktreeCheckoutCache.ts @@ -511,6 +511,7 @@ export const makeWorktreeCheckoutCache = Effect.fn("WorktreeCheckoutCache.make") readonly worktreePath: string; readonly refName: string; readonly checkoutWorkers: string; + readonly lockReason?: string | undefined; }) { if (!(yield* gitSupported)) return false; const worktreePath = path.resolve(input.cwd, input.worktreePath); @@ -613,13 +614,14 @@ export const makeWorktreeCheckoutCache = Effect.fn("WorktreeCheckoutCache.make") if (moved === null || (yield* fileSystem.exists(target.indexHook))) return null; return target; }); - const rollback = git("rollback", input.cwd, [ - "worktree", - "remove", - "--force", - "--", - worktreePath, - ]).pipe( + const rollback = ( + input.lockReason + ? git("unlockRollback", input.cwd, ["worktree", "unlock", "--", worktreePath]) + : Effect.void + ).pipe( + Effect.andThen( + git("rollback", input.cwd, ["worktree", "remove", "--force", "--", worktreePath]), + ), Effect.flatMap((removed) => removed === null ? Effect.logWarning("could not remove a partially created worktree", { worktreePath }) @@ -633,6 +635,7 @@ export const makeWorktreeCheckoutCache = Effect.fn("WorktreeCheckoutCache.make") "worktree", "add", "--no-checkout", + ...(input.lockReason ? ["--lock", "--reason", input.lockReason] : []), "--", worktreePath, input.refName, diff --git a/apps/server/src/vcs/worktreeRemoval.ts b/apps/server/src/vcs/worktreeRemoval.ts index f1140cdcc3a48..b48994873b282 100644 --- a/apps/server/src/vcs/worktreeRemoval.ts +++ b/apps/server/src/vcs/worktreeRemoval.ts @@ -9,6 +9,42 @@ import type * as GitVcsDriver from "./GitVcsDriver.ts"; const INDEX_LIST_MAX_BYTES = 64 * 1024 * 1024; +/** + * git-annex replaces a linked worktree's `.git` file with a symlink to its admin + * directory, so its symlinks into `.git/annex` resolve. Git then refuses to remove it. + */ +const annexedAdminDir = Effect.fn("WorktreeRemoval.annexedAdminDir")( + function* (worktreePath: string) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const adminDir = path.resolve( + worktreePath, + yield* fileSystem.readLink(path.join(worktreePath, ".git")), + ); + const pointsBackTo = path.dirname( + path.resolve( + adminDir, + (yield* fileSystem.readFileString(path.join(adminDir, "gitdir"))).trim(), + ), + ); + return (yield* fileSystem.realPath(pointsBackTo)) === (yield* fileSystem.realPath(worktreePath)) + ? Option.some(adminDir) + : Option.none(); + }, + Effect.orElseSucceed(() => Option.none()), +); + +export const isLinkedWorktree = Effect.fn("WorktreeRemoval.isLinkedWorktree")(function* ( + worktreePath: string, +) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + return ( + (yield* fileSystem.stat(path.join(worktreePath, ".git"))).type === "File" || + Option.isSome(yield* annexedAdminDir(worktreePath)) + ); +}); + /** * Git for Windows before 2.54 follows NTFS junctions; newer Git leaves them behind. * On Windows, mirror Git's removal checks, then delete with Node's `rm`, which @@ -33,6 +69,39 @@ export const removeLinkedWorktree = Effect.fn("WorktreeRemoval.removeLinkedWorkt // Git refuses to remove a worktree with untracked files unless forced, but // its check honors `status.showUntrackedFiles=no` and would delete them. const statusConfig = ["-c", "status.showUntrackedFiles=normal"]; + // Git's own refusals, worded the way it prints them. + const refuse = (message: string, cause?: unknown) => + new GitCommandError({ + operation: input.operation, + command: "git worktree remove", + cwd: input.cwd, + detail: `fatal: ${message}`, + ...(cause === undefined ? {} : { cause }), + }); + const annexed = yield* annexedAdminDir(worktreePath); + if (Option.isSome(annexed)) { + const adminDir = annexed.value; + const writeGitFile = fileSystem.writeFileString(gitFile, `gitdir: ${adminDir}\n`); + // A kept worktree's annexed files resolve only through the symlink; a gitdir + // file is the fallback when it cannot be recreated. + yield* Effect.addFinalizer(() => + fileSystem.exists(worktreePath).pipe( + Effect.flatMap((kept) => + kept + ? fileSystem.remove(gitFile, { force: true }).pipe( + Effect.andThen(fileSystem.symlink(adminDir, gitFile)), + Effect.catch(() => writeGitFile), + ) + : Effect.void, + ), + Effect.ignore, + ), + ); + yield* fileSystem.remove(gitFile).pipe( + Effect.andThen(writeGitFile), + Effect.mapError((cause) => refuse(`failed to replace '${gitFile}'`, cause)), + ); + } const linked = (yield* HostProcess.isWindows) && (yield* fileSystem.stat(gitFile).pipe( @@ -50,15 +119,6 @@ export const removeLinkedWorktree = Effect.fn("WorktreeRemoval.removeLinkedWorkt ]); } - // Git's own refusals, worded the way it prints them. - const refuse = (message: string, cause?: unknown) => - new GitCommandError({ - operation: input.operation, - command: "git worktree remove", - cwd: input.cwd, - detail: `fatal: ${message}`, - ...(cause === undefined ? {} : { cause }), - }); const gitOutput = (cwd: string, args: ReadonlyArray, env?: NodeJS.ProcessEnv) => input .execute({ @@ -153,4 +213,4 @@ export const removeLinkedWorktree = Effect.fn("WorktreeRemoval.removeLinkedWorkt ); // The folder is gone, so Git only drops the worktree's metadata. return yield* input.run(["worktree", "remove", "--force", "--force", "--", worktreePath]); -}); +}, Effect.scoped); diff --git a/apps/server/src/workspace/IgnorePatterns.test.ts b/apps/server/src/workspace/IgnorePatterns.test.ts new file mode 100644 index 0000000000000..f10345566eca5 --- /dev/null +++ b/apps/server/src/workspace/IgnorePatterns.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "@effect/vitest"; + +import { parseIgnoreFile } from "./IgnorePatterns.ts"; + +const decide = (contents: string, path: string, isFolder = false) => + parseIgnoreFile(contents).decide(path, isFolder); + +describe("parseIgnoreFile", () => { + it("lets the last matching pattern decide, including negations", () => { + expect(decide("*keep*\n!*keep*\n", "sub/keep-link.ts")).toBe("included"); + expect(decide("!*keep*\n*keep*\n", "sub/keep-link.ts")).toBe("ignored"); + expect(decide("# comment\n\nother\n", "keep.ts")).toBeUndefined(); + }); + + it("anchors patterns that contain a slash and matches bare names at any depth", () => { + expect(decide("link", "a/b/link")).toBe("ignored"); + expect(decide("/link", "a/link")).toBeUndefined(); + expect(decide("/link", "link")).toBe("ignored"); + expect(decide("a/link", "x/a/link")).toBeUndefined(); + expect(decide("a/*.ts", "a/b/c.ts")).toBeUndefined(); + expect(decide("link", "blink")).toBeUndefined(); + expect(decide("link", "a/blink")).toBeUndefined(); + }); + + it("supports globstars, classes, escapes, and folder-only patterns", () => { + expect(decide("**/gen/*.ts", "gen/a.ts")).toBe("ignored"); + expect(decide("a/**/b", "a/x/y/b")).toBe("ignored"); + expect(decide("a/**/b", "a/b")).toBe("ignored"); + expect(decide("foo/**/bar", "foo/xbar")).toBeUndefined(); + expect(decide("foo/**/bar", "foo/x/ybar")).toBeUndefined(); + expect(decide("out/**", "out/x/y")).toBe("ignored"); + expect(decide("link-[0-9]", "link-7")).toBe("ignored"); + expect(decide("link-[!0-9]", "link-7")).toBeUndefined(); + expect(decide("link-?", "link-/")).toBeUndefined(); + expect(decide("\\!bang\n\\#hash\nstar\\*", "!bang")).toBe("ignored"); + expect(decide("\\!bang\n\\#hash\nstar\\*", "#hash")).toBe("ignored"); + expect(decide("\\!bang\n\\#hash\nstar\\*", "starx")).toBeUndefined(); + expect(decide("dist/", "dist")).toBeUndefined(); + expect(decide("dist/", "dist", true)).toBe("ignored"); + }); + + it("matches adversarial patterns in linear time", () => { + // A backtracking regex takes seconds to reject this pair; the vitest timeout catches that. + const pattern = `${"a*".repeat(20)}b\n`; + expect(decide(pattern, `${"a".repeat(200)}c`)).toBeUndefined(); + expect(decide(pattern.repeat(50), `${"a".repeat(1_000)}c`)).toBeUndefined(); + expect(decide(pattern, `${"a".repeat(200)}b`)).toBe("ignored"); + }); + + it("parses unclosed classes in linear time", () => { + // Rescanning the rest of the line for each `[` stalls for over a minute here; linear takes ms. + expect(decide("[".repeat(30_000), "[".repeat(30_000))).toBe("ignored"); + expect(decide(`${"[".repeat(30_000)}]`, "x")).toBeUndefined(); + expect(decide("[a][", "a[")).toBe("ignored"); + }); +}); diff --git a/apps/server/src/workspace/IgnorePatterns.ts b/apps/server/src/workspace/IgnorePatterns.ts new file mode 100644 index 0000000000000..1180b02316ef9 --- /dev/null +++ b/apps/server/src/workspace/IgnorePatterns.ts @@ -0,0 +1,170 @@ +/** + * Gitignore-syntax matching for `.ignore` files, which git cannot evaluate for us. Patterns come + * from the repository, so each runs as an NFA over the path (O(pattern × path)): a backtracking + * regex stalls for seconds on `a*a*a*…b`. POSIX `[:class:]` names are not supported. + */ + +type Token = + | { readonly type: "char"; readonly char: string } + | { readonly type: "one" } + | { readonly type: "class"; readonly negated: boolean; readonly ranges: ReadonlyArray } + | { readonly type: "star" } + | { readonly type: "any" } + /** A leading or inner `**` segment: nothing, or anything ending in `/`. */ + | { readonly type: "folders" }; + +interface Pattern { + readonly tokens: ReadonlyArray; + readonly negated: boolean; + readonly folderOnly: boolean; +} + +export type IgnoreDecision = "ignored" | "included"; + +function parseClass(chars: ReadonlyArray, start: number) { + let index = start + 1; + const negated = chars[index] === "!" || chars[index] === "^"; + if (negated) index += 1; + const ranges: string[] = []; + for (let first = true; index < chars.length; first = false) { + let char = chars[index]!; + if (char === "]" && !first) + return { token: { type: "class", negated, ranges } as const, end: index }; + if (char === "\\" && index + 1 < chars.length) char = chars[++index]!; + if (chars[index + 1] === "-" && index + 2 < chars.length && chars[index + 2] !== "]") { + let last = chars[index + 2]!; + index += 2; + if (last === "\\" && index + 1 < chars.length) last = chars[++index]!; + ranges.push(char, last); + } else { + ranges.push(char, char); + } + index += 1; + } + return undefined; +} + +function parseLine(rawLine: string): Pattern | undefined { + let line = rawLine.endsWith("\r") ? rawLine.slice(0, -1) : rawLine; + // Trailing spaces are dropped unless escaped. + while (line.endsWith(" ") && !line.endsWith("\\ ")) line = line.slice(0, -1); + if (line === "" || line.startsWith("#")) return undefined; + const negated = line.startsWith("!"); + if (negated) line = line.slice(1); + else if (line.startsWith("\\!") || line.startsWith("\\#")) line = line.slice(1); + const folderOnly = line.endsWith("/") && !line.endsWith("\\/"); + if (folderOnly) line = line.slice(0, -1); + if (line === "") return undefined; + const anchored = line.slice(0, -1).includes("/"); + if (line.startsWith("/")) line = line.slice(1); + + const chars = Array.from(line); + const tokens: Token[] = anchored ? [] : [{ type: "folders" }]; + // A `[` that never closes leaves every later `[` unclosed too, so one failed scan settles them + // all; rescanning the suffix for each would make `[[[[…` quadratic. + let classesClose = true; + for (let index = 0; index < chars.length; index += 1) { + const char = chars[index]!; + if (char === "*" && chars[index + 1] === "*") { + const atSegmentStart = index === 0 || chars[index - 1] === "/"; + const next = chars[index + 2]; + if (atSegmentStart && next === "/") { + tokens.push({ type: "folders" }); + index += 2; + continue; + } + if (atSegmentStart && next === undefined) { + tokens.push({ type: "any" }); + index += 1; + continue; + } + } + if (char === "*") tokens.push({ type: "star" }); + else if (char === "?") tokens.push({ type: "one" }); + else if (char === "[") { + const parsed = classesClose ? parseClass(chars, index) : undefined; + if (!parsed) classesClose = false; + if (parsed) { + tokens.push(parsed.token); + index = parsed.end; + } else tokens.push({ type: "char", char }); + } else if (char === "\\" && index + 1 < chars.length) { + tokens.push({ type: "char", char: chars[++index]! }); + } else tokens.push({ type: "char", char }); + } + return { tokens, negated, folderOnly }; +} + +/** + * States are token positions. A `folders` token that has consumed part of a segment is `~position` + * instead: it may only finish after the next `/`, so an inner globstar never matches `xbar` as `bar`. + */ +function matches(tokens: ReadonlyArray, path: ReadonlyArray): boolean { + let states = new Set(); + const add = (set: Set, position: number) => { + // Wildcards also match nothing, so reaching one reaches the token after it. + for (; position <= tokens.length && !set.has(position); position += 1) { + set.add(position); + const type = tokens[position]?.type; + if (type !== "star" && type !== "any" && type !== "folders") return; + } + }; + add(states, 0); + for (const char of path) { + const next = new Set(); + for (const state of states) { + const position = state < 0 ? ~state : state; + const token = tokens[position]; + if (!token) continue; + switch (token.type) { + case "char": + if (token.char === char) add(next, position + 1); + break; + case "one": + if (char !== "/") add(next, position + 1); + break; + case "class": { + let inRange = false; + for (let index = 0; index < token.ranges.length && !inRange; index += 2) { + inRange = token.ranges[index]! <= char && char <= token.ranges[index + 1]!; + } + if (char !== "/" && inRange !== token.negated) add(next, position + 1); + break; + } + case "star": + if (char !== "/") add(next, position); + break; + case "any": + add(next, position); + break; + case "folders": + if (char === "/") add(next, position); + else next.add(~position); + break; + } + } + if (next.size === 0) return false; + states = next; + } + return states.has(tokens.length); +} + +export interface IgnoreFile { + /** The last pattern that matches decides; `undefined` when none does. */ + readonly decide: (relativePath: string, isFolder: boolean) => IgnoreDecision | undefined; +} + +export function parseIgnoreFile(contents: string): IgnoreFile { + const patterns = contents.split("\n").flatMap((line) => parseLine(line) ?? []); + return { + decide: (relativePath, isFolder) => { + const path = Array.from(relativePath); + for (let index = patterns.length - 1; index >= 0; index -= 1) { + const pattern = patterns[index]!; + if (pattern.folderOnly && !isFolder) continue; + if (matches(pattern.tokens, path)) return pattern.negated ? "included" : "ignored"; + } + return undefined; + }, + }; +} diff --git a/apps/server/src/workspace/WorkspaceEntries.test.ts b/apps/server/src/workspace/WorkspaceEntries.test.ts index 348b5f877ac24..1c50b94a7cbcc 100644 --- a/apps/server/src/workspace/WorkspaceEntries.test.ts +++ b/apps/server/src/workspace/WorkspaceEntries.test.ts @@ -433,6 +433,20 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { }); }), ); + + it.effect("finds ignored files that sort past the listing limit", () => + Effect.gen(function* () { + const cwd = yield* makeTempDir({ git: true }); + yield* writeTextFile(cwd, ".gitignore", "*.secret\n"); + for (let index = 1; index <= 205; index += 1) { + yield* writeTextFile(cwd, `file-${String(index).padStart(3, "0")}.secret`, "s"); + } + + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; + const listed = yield* workspaceEntries.listIgnoredFiles({ cwd, query: "file-205" }); + expect(listed.files).toContainEqual({ path: "file-205.secret", byteLength: 1 }); + }), + ); }); describe("search", () => { @@ -532,6 +546,133 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { }), ); + it.effect("indexes symlinked files and directories inside the workspace", () => + Effect.gen(function* () { + if ((yield* HostProcess.Platform) === "win32") return; + const cwd = yield* makeTempDir({ prefix: "t3code-workspace-symlinks-" }); + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* writeTextFile(cwd, "src/index.ts", "export {};"); + yield* fileSystem.symlink("src", path.join(cwd, "src-link")); + yield* fileSystem.symlink("src/index.ts", path.join(cwd, "index-link.ts")); + const symlinkEntries = [ + { kind: "directory", path: "src-link" }, + { kind: "file", path: "index-link.ts" }, + ]; + + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; + const searched = yield* searchWorkspaceEntries({ cwd, query: "link", limit: 100 }); + const listed = yield* workspaceEntries.list({ cwd }); + + for (const entry of symlinkEntries) { + expect(searched.entries).toContainEqual(expect.objectContaining(entry)); + expect(listed.entries).toContainEqual(expect.objectContaining(entry)); + } + }), + ); + it.effect("skips symlinks that ignore rules hide or that leave the workspace", () => + Effect.gen(function* () { + if ((yield* HostProcess.Platform) === "win32") return; + const cwd = yield* makeTempDir({ prefix: "t3code-workspace-symlinks-", git: true }); + const outside = yield* makeTempDir({ prefix: "t3code-workspace-symlinks-outside-" }); + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* writeTextFile(cwd, "src/index.ts", "export {};"); + yield* writeTextFile(cwd, ".ignore", "custom-link\n"); + yield* fileSystem.symlink("src", path.join(cwd, "custom-link")); + yield* fileSystem.symlink("src/index.ts", path.join(cwd, "tracked-link.ts")); + yield* fileSystem.symlink("src", path.join(cwd, "ignored-link")); + yield* fileSystem.symlink("src", path.join(cwd, "kept-link")); + yield* fileSystem.symlink(outside, path.join(cwd, "outside-link")); + yield* git(cwd, ["add", "tracked-link.ts"]); + yield* writeTextFile(cwd, ".gitignore", "tracked-link.ts\nignored-link\n"); + // A submodule path makes `git check-ignore` refuse the whole batch without --no-index. + const submodule = path.join(cwd, "vendor/sub"); + yield* writeTextFile(submodule, "a.ts", "export {};"); + yield* git(submodule, ["init"]); + yield* git(submodule, ["add", "a.ts"]); + yield* git(submodule, [ + "-c", + "user.name=t", + "-c", + "user.email=t@t", + "commit", + "-m", + "init", + ]); + yield* git(cwd, ["add", "vendor/sub"]); + yield* fileSystem.symlink("a.ts", path.join(submodule, "sublink.ts")); + + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; + const searched = yield* searchWorkspaceEntries({ cwd, query: "link", limit: 100 }); + const listed = yield* workspaceEntries.list({ cwd }); + + for (const entries of [searched.entries, listed.entries]) { + const paths = entries.map((entry) => entry.path); + expect(paths).toContain("kept-link"); + expect(paths).toContain("vendor/sub/sublink.ts"); + expect(paths).not.toContain("custom-link"); + expect(paths).not.toContain("tracked-link.ts"); + expect(paths).not.toContain("ignored-link"); + expect(paths).not.toContain("outside-link"); + } + }), + ); + it.effect("lets the nearest .ignore rule decide a symlink, and git's rules still hide it", () => + Effect.gen(function* () { + if ((yield* HostProcess.Platform) === "win32") return; + const cwd = yield* makeTempDir({ prefix: "t3code-workspace-symlinks-", git: true }); + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* writeTextFile(cwd, ".gitignore", "*keep*\n"); + yield* writeTextFile(cwd, ".ignore", "!*keep*\n*hide*\n"); + yield* writeTextFile(cwd, "sub/.ignore", "!*hide*\n"); + yield* writeTextFile(cwd, "keep-real.ts", "export {};"); + yield* writeTextFile(cwd, "sub/hide-real.ts", "export {};"); + yield* fileSystem.symlink("keep-real.ts", path.join(cwd, "keep-link.ts")); + yield* fileSystem.symlink("hide-real.ts", path.join(cwd, "sub/hide-link.ts")); + + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; + const searched = yield* searchWorkspaceEntries({ cwd, query: "link", limit: 100 }); + const listed = yield* workspaceEntries.list({ cwd }); + + for (const entries of [searched.entries, listed.entries]) { + const paths = entries.map((entry) => entry.path); + expect(paths).toContain("sub/hide-link.ts"); + expect(paths).not.toContain("keep-link.ts"); + } + // The real files are shown or hidden by the same rules. + const listedPaths = listed.entries.map((entry) => entry.path); + expect(listedPaths).toContain("sub/hide-real.ts"); + expect(listedPaths).not.toContain("keep-real.ts"); + }), + ); + it.effect("matches .ignore rules against whole names and treats folder links as links", () => + Effect.gen(function* () { + if ((yield* HostProcess.Platform) === "win32") return; + const cwd = yield* makeTempDir({ prefix: "t3code-workspace-symlinks-", git: true }); + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* writeTextFile(cwd, "src/index.ts", "export {};"); + yield* writeTextFile(cwd, ".gitignore", "blink\n"); + // `!link` must not re-include `blink`, and `folder-link/` does not apply to a link. + yield* writeTextFile(cwd, ".ignore", "!link\nfolder-link/\n"); + yield* fileSystem.symlink("src", path.join(cwd, "blink")); + yield* fileSystem.symlink("src", path.join(cwd, "folder-link")); + yield* writeTextFile(cwd, "sub/blink", "regular"); + + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; + const searched = yield* searchWorkspaceEntries({ cwd, query: "link", limit: 100 }); + const listed = yield* workspaceEntries.list({ cwd }); + + for (const entries of [searched.entries, listed.entries]) { + const paths = entries.map((entry) => entry.path); + expect(paths).not.toContain("blink"); + expect(paths).not.toContain("sub/blink"); + expect(paths).toContain("folder-link"); + } + }), + ); it.effect("finds a path inside a gitignored nested git worktree only by its exact path", () => Effect.gen(function* () { const cwd = yield* makeTempDir({ prefix: "t3code-workspace-worktree-", git: true }); @@ -547,6 +688,13 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { yield* git(cwd, ["commit", "-m", "init"], env); yield* git(cwd, ["worktree", "add", "-b", "x", ".worktrees/x"]); yield* writeTextFile(cwd, ".worktrees/x/src/foo.py", "print(1)"); + const expected = [".worktrees/x/src/foo.py"]; + // Windows file names cannot contain a newline. + if ((yield* HostProcess.Platform) !== "win32") { + yield* git(cwd, ["worktree", "add", "--detach", ".worktrees/line\nname"]); + yield* writeTextFile(cwd, ".worktrees/line\nname/src/foo.py", "print(2)"); + expected.unshift(".worktrees/line\nname/src/foo.py"); + } const exact = yield* searchWorkspaceEntries({ cwd, @@ -554,10 +702,14 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { limit: 25, kind: "file", }); - expect(exact.entries).toEqual([{ path: ".worktrees/x/src/foo.py", kind: "file" }]); + expect(exact.entries.toSorted((a, b) => (a.path < b.path ? -1 : 1))).toEqual( + expected.map((path) => ({ path, kind: "file" })), + ); const partial = yield* searchWorkspaceEntries({ cwd, query: "src/fo", limit: 25 }); - expect(partial.entries.map((entry) => entry.path)).not.toContain(".worktrees/x/src/foo.py"); + for (const path of expected) { + expect(partial.entries.map((entry) => entry.path)).not.toContain(path); + } }), ); }); @@ -822,10 +974,13 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { if (created.ok) { const finder = created.value; finders += 1; - // The synchronous rescan waits on fff's picker lock on the event loop. + // The synchronous rescan and grep wait on fff's picker lock on the event loop. finder.scanFiles = () => { throw new Error("refresh must not rescan on the JS thread"); }; + finder.grep = () => { + throw new Error("search must not grep on the JS thread"); + }; const scanFilesAsync = finder.scanFilesAsync.bind(finder); finder.scanFilesAsync = () => { rescans += 1; diff --git a/apps/server/src/workspace/WorkspaceEntries.ts b/apps/server/src/workspace/WorkspaceEntries.ts index cac706f7ca4a2..51b1720126536 100644 --- a/apps/server/src/workspace/WorkspaceEntries.ts +++ b/apps/server/src/workspace/WorkspaceEntries.ts @@ -307,23 +307,30 @@ export const make = Effect.gen(function* () { ) { return []; } - const listed = yield* vcsProcess - .run({ - operation: "WorkspaceEntries.listWorktrees", - command: "git", - args: ["worktree", "list", "--porcelain"], - cwd, - allowNonZeroExit: true, - timeoutMs: 5_000, - }) - .pipe(Effect.orElseSucceed(() => undefined)); - if (!listed || listed.exitCode !== 0) return []; + const listWorktrees = (args: ReadonlyArray) => + vcsProcess + .run({ + operation: "WorkspaceEntries.listWorktrees", + command: "git", + args: ["worktree", "list", "--porcelain", ...args], + cwd, + allowNonZeroExit: true, + timeoutMs: 5_000, + }) + .pipe( + Effect.map((result) => (result.exitCode === 0 ? result.stdout : undefined)), + Effect.orElseSucceed(() => undefined), + ); + // Only NUL framing keeps a newline in a worktree path. Pre-2.36 Git rejects `-z`. + const listed = + (yield* listWorktrees(["-z"])) ?? (yield* listWorktrees([]))?.replaceAll("\n", "\0"); + if (listed === undefined) return []; const realCwd = yield* Effect.promise(() => NodeFSP.realpath(cwd).catch(() => cwd)); - const worktreeRoots = listed.stdout.split(/\n\s*\n/).flatMap((record) => { - const lines = record.split("\n"); - const worktree = lines.find((line) => line.startsWith("worktree ")); + const worktreeRoots = listed.split("\0\0").flatMap((record) => { + const fields = record.split("\0"); + const worktree = fields.find((field) => field.startsWith("worktree ")); // A `prunable` record's directory is gone; another checkout may now sit there. - if (worktree === undefined || lines.some((line) => line.startsWith("prunable"))) return []; + if (worktree === undefined || fields.some((field) => field.startsWith("prunable"))) return []; const root = worktree.slice("worktree ".length); const relativeRoot = path.relative(realCwd, root); return relativeRoot === "" || @@ -576,9 +583,13 @@ export const make = Effect.gen(function* () { }) .pipe(Effect.orElseSucceed(() => undefined)); if (!result || result.exitCode !== 0) return { files: [], truncated: false }; + // Filter before the cap so files sorting past it stay reachable by search. + const query = input.query?.toLowerCase() ?? ""; const paths = result.stdout .split("\0") - .filter((entry) => entry.length > 0 && !entry.endsWith("/")) + .filter( + (entry) => entry.length > 0 && !entry.endsWith("/") && entry.toLowerCase().includes(query), + ) .toSorted(); const files = yield* Effect.promise(() => Promise.all( diff --git a/apps/server/src/workspace/WorkspaceFileSystem.test.ts b/apps/server/src/workspace/WorkspaceFileSystem.test.ts index dffb0701d68aa..bc22117ba74ab 100644 --- a/apps/server/src/workspace/WorkspaceFileSystem.test.ts +++ b/apps/server/src/workspace/WorkspaceFileSystem.test.ts @@ -12,6 +12,7 @@ import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as ServerConfig from "../config.ts"; +import { makeFakeWslHost } from "../testUtils/fakeWsl.ts"; import * as VcsDriverRegistry from "../vcs/VcsDriverRegistry.ts"; import * as VcsProcess from "../vcs/VcsProcess.ts"; import * as KnownWorkspaces from "./KnownWorkspaces.ts"; @@ -96,6 +97,14 @@ const writeTextFile = Effect.fn("writeTextFile")(function* ( yield* fileSystem.writeFileString(absolutePath, contents).pipe(Effect.orDie); }); +const makeWslHost = Effect.gen(function* () { + const { mountRoot, underWsl } = yield* makeFakeWslHost; + const service = yield* underWsl( + WorkspaceFileSystem.make.pipe(Effect.provide(layerKnownWorkspacesTest)), + ); + return { mountRoot, service, underWsl }; +}); + it.layer(layerTest, { excludeTestServices: true })("WorkspaceFileSystemLive", (it) => { describe("getMetadata", () => { it.effect( @@ -181,6 +190,31 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceFileSystemLive", (i expect(statted).toEqual([]); }), ); + + it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( + "reports a Windows drive path under WSL as the file it maps to", + () => + Effect.gen(function* () { + const { mountRoot, service, underWsl } = yield* makeWslHost; + yield* writeTextFile(mountRoot, "c/Users/me/Documents/notes.md", "# Notes\n"); + + const { entries } = yield* underWsl( + service.getMetadata({ + paths: [ + "C:\\Users\\me\\Documents\\notes.md", + "C:/Users/me/Documents/notes.md", + "C:/Users/me/Documents/notes\u0000.md", + ], + }), + ); + + expect(entries).toEqual([ + { kind: "file", byteLength: 8 }, + { kind: "file", byteLength: 8 }, + null, + ]); + }), + ); }); describe("readFile", () => { @@ -202,6 +236,18 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceFileSystemLive", (i truncated: false, }); + const bomContents = "\uFEFFexport const answer = 42;\n"; + yield* writeTextFile(cwd, "bom.ts", bomContents); + const bomRead = yield* workspaceFileSystem.readFile({ cwd, relativePath: "bom.ts" }); + yield* workspaceFileSystem.writeFile({ + cwd, + relativePath: "bom.ts", + contents: bomRead.contents.replace("42", "43"), + }); + const fs = yield* FileSystem.FileSystem; + const saved = yield* fs.readFile(`${cwd}/bom.ts`); + expect([...saved]).toEqual([...new TextEncoder().encode(bomContents.replace("42", "43"))]); + yield* writeTextFile(cwd, "report.txt", "ordinary file"); for (const relativePath of whitespaceFilePaths) { yield* writeTextFile(cwd, relativePath, "whitespace file"); @@ -274,25 +320,27 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceFileSystemLive", (i }), ); - it.effect("reads ~/ paths from the server home outside the workspace", () => - Effect.gen(function* () { - const workspaceFileSystem = yield* WorkspaceFileSystem.WorkspaceFileSystem; - const cwd = yield* makeTempDir; - const home = yield* makeTempDir; - yield* writeTextFile(home, ".claude/CLAUDE.md", "# Notes\n"); - yield* writeTextFile(cwd, "~/.claude/CLAUDE.md", "Workspace decoy\n"); + it.effect.each(["~/.claude/CLAUDE.md", "~\\.claude/CLAUDE.md"])( + "reads %s from the server home outside the workspace", + (relativePath) => + Effect.gen(function* () { + const workspaceFileSystem = yield* WorkspaceFileSystem.WorkspaceFileSystem; + const cwd = yield* makeTempDir; + const home = yield* makeTempDir; + yield* writeTextFile(home, ".claude/CLAUDE.md", "# Notes\n"); + yield* writeTextFile(cwd, "~/.claude/CLAUDE.md", "Workspace decoy\n"); - const result = yield* workspaceFileSystem - .readFile({ cwd, relativePath: "~/.claude/CLAUDE.md" }) - .pipe(Effect.provideService(HostProcess.HomeDirectory, home)); + const result = yield* workspaceFileSystem + .readFile({ cwd, relativePath }) + .pipe(Effect.provideService(HostProcess.HomeDirectory, home)); - expect(result).toEqual({ - relativePath: "~/.claude/CLAUDE.md", - contents: "# Notes\n", - byteLength: 8, - truncated: false, - }); - }), + expect(result).toEqual({ + relativePath, + contents: "# Notes\n", + byteLength: 8, + truncated: false, + }); + }), ); it.effect("does not read server state files", () => @@ -351,6 +399,24 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceFileSystemLive", (i }), ); + it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( + "reads a Windows drive path under WSL from where wslpath mounts it", + () => + Effect.gen(function* () { + const { mountRoot, service, underWsl } = yield* makeWslHost; + const cwd = yield* makeTempDir; + yield* writeTextFile(mountRoot, "c/Users/me/Documents/notes.md", "# Notes\n"); + + for (const relativePath of [ + "C:\\Users\\me\\Documents\\notes.md", + "C:/Users/me/Documents/notes.md", + ]) { + const result = yield* underWsl(service.readFile({ cwd, relativePath })); + expect(result).toMatchObject({ contents: "# Notes\n", truncated: false }); + } + }), + ); + it.effect("refuses a network path outside the workspace", () => Effect.gen(function* () { const workspaceFileSystem = yield* WorkspaceFileSystem.WorkspaceFileSystem; @@ -478,14 +544,17 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceFileSystemLive", (i }), ); - it.effect("rejects binary files without leaking their contents into the error", () => + it.effect.each([ + { name: "a NUL byte", bytes: [0x61, 0, 0x62] }, + { name: "invalid UTF-8", bytes: [0x63, 0x61, 0x66, 0xe9] }, + ])("rejects files with $name without leaking their contents", ({ bytes }) => Effect.gen(function* () { const workspaceFileSystem = yield* WorkspaceFileSystem.WorkspaceFileSystem; const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; const cwd = yield* makeTempDir; const absolutePath = path.join(cwd, "asset.bin"); - yield* fileSystem.writeFile(absolutePath, Uint8Array.from([0x61, 0, 0x62])); + yield* fileSystem.writeFile(absolutePath, Uint8Array.from(bytes)); const error = yield* workspaceFileSystem .readFile({ cwd, relativePath: "asset.bin" }) diff --git a/apps/server/src/workspace/WorkspaceFileSystem.ts b/apps/server/src/workspace/WorkspaceFileSystem.ts index 05409c6616ff0..494472e4ce68d 100644 --- a/apps/server/src/workspace/WorkspaceFileSystem.ts +++ b/apps/server/src/workspace/WorkspaceFileSystem.ts @@ -19,6 +19,7 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import * as Schema from "effect/Schema"; import { makeIsServerInternalPath } from "../auth/serverInternalPaths.ts"; @@ -26,6 +27,7 @@ import * as KnownWorkspaces from "./KnownWorkspaces.ts"; import * as WorkspaceEntries from "./WorkspaceEntries.ts"; import * as WorkspacePaths from "./WorkspacePaths.ts"; import { fileHeaderMimeType } from "./fileHeaderMimeType.ts"; +import { toWslDrivePath } from "./wslDrivePath.ts"; const PROJECT_READ_FILE_MAX_BYTES = 1024 * 1024; @@ -133,6 +135,27 @@ export class WorkspaceFileSystem extends Context.Service< } >()("t3/workspace/WorkspaceFileSystem") {} +const UTF8_BOM = "\uFEFF"; + +async function startsWithUtf8Bom(filePath: string): Promise { + const handle = await NodeFSP.open( + filePath, + // O_NONBLOCK keeps a FIFO with no writer from hanging the save. + NodeFS.constants.O_RDONLY | + (NodeFS.constants.O_NOFOLLOW ?? 0) | + (NodeFS.constants.O_NONBLOCK ?? 0), + ).catch(() => null); + if (!handle) return false; + try { + const { bytesRead, buffer } = await handle.read(Buffer.alloc(3), 0, 3, 0); + return bytesRead === 3 && buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf; + } catch { + return false; + } finally { + await handle.close(); + } +} + /** @public */ export const make = Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; @@ -141,9 +164,18 @@ export const make = Effect.gen(function* () { const knownWorkspaces = yield* KnownWorkspaces.KnownWorkspaces; const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; const isServerInternalPath = yield* makeIsServerInternalPath; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + + const toHostPath = (requestedPath: string) => + toWslDrivePath(requestedPath).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); const metadataForPath = Effect.fnUntraced(function* (requestedPath: string) { - const expandedPath = expandHomePath(requestedPath, yield* HostProcess.HomeDirectory); + const expandedPath = expandHomePath( + yield* toHostPath(requestedPath), + yield* HostProcess.HomeDirectory, + ); // Chips ask as soon as an agent's message renders, and touching a network // path makes Windows sign in to its host with the user's NTLM credentials. if (!path.isAbsolute(expandedPath) || isNetworkPath(expandedPath)) return null; @@ -228,9 +260,10 @@ export const make = Effect.gen(function* () { input: ProjectReadFileInput, ) { const requestedPath = input.relativePath; - const hostPath = requestedPath.startsWith("~/") - ? expandHomePath(requestedPath, yield* HostProcess.HomeDirectory) - : requestedPath; + const hostPath = + requestedPath.startsWith("~/") || requestedPath.startsWith("~\\") + ? expandHomePath(requestedPath, yield* HostProcess.HomeDirectory) + : yield* toHostPath(requestedPath); if ( isNetworkPath(hostPath) && !KnownWorkspaces.isWithinRoot(path, path.resolve(hostPath), path.resolve(input.cwd)) @@ -424,22 +457,22 @@ export const make = Effect.gen(function* () { offset += bytesRead; } const fileBytes = buffer.subarray(0, offset); - if (fileBytes.includes(0)) { - return yield* new WorkspaceBinaryFileError({ + const truncated = stat.size > PROJECT_READ_FILE_MAX_BYTES; + const binaryFileError = () => + new WorkspaceBinaryFileError({ workspaceRoot: input.cwd, relativePath: input.relativePath, resolvedPath: realTargetPath, }); - } - - const truncated = stat.size > PROJECT_READ_FILE_MAX_BYTES; - return { - relativePath: target.relativePath, - // Streaming decode drops a character the byte limit split, instead of ending in U+FFFD. - contents: new TextDecoder("utf-8").decode(fileBytes, { stream: truncated }), - byteLength: stat.size, - truncated, - }; + if (fileBytes.includes(0)) return yield* binaryFileError(); + // Fatal decoding rejects invalid UTF-8, since U+FFFD contents would corrupt the file on save. + // Streaming decode drops a character the byte limit split, instead of rejecting it. + const contents = yield* Effect.try({ + try: () => + new TextDecoder("utf-8", { fatal: true }).decode(fileBytes, { stream: truncated }), + catch: binaryFileError, + }); + return { relativePath: target.relativePath, contents, byteLength: stat.size, truncated }; }), (handle) => handle === null @@ -550,6 +583,12 @@ export const make = Effect.gen(function* () { const replacesFile = yield* fileSystem .exists(realTargetPath) .pipe(Effect.orElseSucceed(() => false)); + // readFile decodes the BOM away, so an editor save would otherwise drop it from the file. + const keepsBom = + replacesFile && + !input.contents.startsWith(UTF8_BOM) && + (yield* Effect.promise(() => startsWithUtf8Bom(realTargetPath))); + const contents = keepsBom ? UTF8_BOM + input.contents : input.contents; yield* fileSystem.makeDirectory(path.dirname(realTargetPath), { recursive: true }).pipe( Effect.mapError( @@ -567,7 +606,7 @@ export const make = Effect.gen(function* () { yield* Effect.tryPromise({ // A symlink swapped in after the check is not followed. Windows lacks O_NOFOLLOW. try: () => - NodeFSP.writeFile(realTargetPath, input.contents, { + NodeFSP.writeFile(realTargetPath, contents, { flag: NodeFS.constants.O_WRONLY | NodeFS.constants.O_CREAT | diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts index 1001d59293c6e..6c4e873f12b47 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts @@ -4,14 +4,19 @@ import { type GrepCursor, type GrepOptions, type GrepResult, + type WatchEvent, } from "@ff-labs/fff-node"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { afterEach, expect, it } from "@effect/vitest"; import * as HostProcess from "@t3tools/shared/HostProcess"; +import { PROJECT_FAVICON_EXTENSIONS, isProjectFaviconPath } from "@t3tools/shared/projectFavicon"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; +import * as TestClock from "effect/testing/TestClock"; import { vi } from "vite-plus/test"; import * as VcsProcess from "../vcs/VcsProcess.ts"; @@ -36,39 +41,98 @@ function fileItem(relativePath: string): FileItem { }; } -it.effect("filters image searches before applying the result limit", () => - Effect.scoped( - Effect.gen(function* () { - const items = [ - ...Array.from({ length: 200 }, (_, index) => fileItem(`src/file-${index}.ts`)), - fileItem("public/icon.svg"), - ]; - const fileSearch = vi.fn(() => ({ - ok: true as const, - value: { - items, - scores: [], - totalMatched: items.length, - totalFiles: items.length, - }, - })); - const finder = { - destroy: vi.fn(), - waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), - fileSearch, - } as unknown as FileFinder; - vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); +it.live("image searches match the image subset of a file search without fetching every file", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const vcsProcess = yield* VcsProcess.VcsProcess; + const writeFiles = Effect.fn(function* (root: string, relativePaths: ReadonlyArray) { + for (const relativePath of relativePaths) { + yield* fileSystem.makeDirectory(path.dirname(path.join(root, relativePath)), { + recursive: true, + }); + yield* fileSystem.writeFileString(path.join(root, relativePath), ""); + yield* fileSystem.utimes(path.join(root, relativePath), 0, 0); + } + }); - const searchIndex = yield* WorkspaceSearchIndex.make("/workspace/project"); - const resultWithoutKind = yield* searchIndex.search("", 200, undefined, true); - const resultWithDirectoryKind = yield* searchIndex.search("", 200, "directory", true); + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-image-search-" }); + yield* vcsProcess.run({ operation: "test.gitInit", command: "git", args: ["init"], cwd }); + yield* fileSystem.writeFileString(path.join(cwd, ".git/info/exclude"), "hidden/\n"); + yield* writeFiles(cwd, [ + ...Array.from({ length: 220 }, (_, index) => `assets/source-${index}.ts`), + ...PROJECT_FAVICON_EXTENSIONS.flatMap((extension) => [ + `assets/logo${extension}`, + `other/logo${extension.toUpperCase()}`, + ]), + "assets/logo.ts", + "assets/logotype.png", + "assetss/logotype.png", + "assets/images/logo.PNG", + "assets/foo{bar}.png", + "assets/[id].png", + "hidden/logo.png", + ]); - expect(resultWithoutKind.entries).toEqual([{ kind: "file", path: "public/icon.svg" }]); - expect(resultWithDirectoryKind.entries).toEqual([{ kind: "file", path: "public/icon.svg" }]); - expect(fileSearch).toHaveBeenCalledTimes(2); - expect(fileSearch).toHaveBeenCalledWith("", { pageSize: 25_002 }); - }), - ).pipe(Effect.provide(testLayer)), + const fileSearch = vi.spyOn(FileFinder.prototype, "fileSearchAsync"); + const searchIndex = yield* WorkspaceSearchIndex.make(cwd); + yield* searchIndex.search("", 200, undefined, true); + const nativeResult = yield* Effect.promise( + (): ReturnType => fileSearch.mock.results[0]?.value, + ); + if (!nativeResult.ok) return yield* Effect.die(nativeResult.error); + expect(nativeResult.value.items.length).toBeLessThanOrEqual(201); + + // The image glob must not change fff's parse of fuzzy paths, globs, negations or locations. + for (const query of [ + "", + "logo", + "lgo", + "*.png", + "!*.png", + "assets/**", + "!assets/", + "assets*/", + "/assets", + "assets/", + "logo:12", + "/assets/logotype.png", + "assets/images/", + "logo *.svg !other", + "foo{bar}.png", + "[id].png", + "*.{png,svg}", + "hidden/", + ]) { + const ordinary = yield* searchIndex.search(query, 1000, "file"); + const expected = ordinary.entries.filter((entry) => isProjectFaviconPath(entry.path)); + for (const limit of [1, 200]) { + const images = yield* searchIndex.search(query, limit, "directory", true); + expect(images, `query=${query} limit=${limit}`).toEqual({ + entries: expected.slice(0, limit), + truncated: expected.length > limit, + }); + } + } + + // Past 15,000 matches fff drops its filename fallback, which the image glob would otherwise + // reorder: "x/logo/logo.png" must keep its place relative to "a/xxlogo.png". + const largeCwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-image-rank-" }); + yield* writeFiles(largeCwd, [ + ...Array.from({ length: 14_998 }, (_, index) => `b/logo-${index}.png`), + "x/logo/source.ts", + "a/xxlogo.png", + "x/logo/logo.png", + ]); + const largeIndex = yield* WorkspaceSearchIndex.make(largeCwd); + const pairPaths = ["a/xxlogo.png", "x/logo/logo.png"]; + const ordinary = yield* largeIndex.search("logo", 20_000, "file"); + const images = yield* largeIndex.search("logo", 20_000, "file", true); + expect(images.entries).toHaveLength(15_000); + expect(images.entries.filter((entry) => pairPaths.includes(entry.path))).toEqual( + ordinary.entries.filter((entry) => pairPaths.includes(entry.path)), + ); + }).pipe(Effect.scoped, Effect.provide(testLayer)), ); it.effect("fetches past a first page of Git-ignored results", () => @@ -89,7 +153,7 @@ it.effect("fetches past a first page of Git-ignored results", () => fileItem("hidden/c.ts"), fileItem("src/a.ts"), ]; - const fileSearch = vi.fn((_query: string, options?: { pageSize?: number }) => ({ + const fileSearch = vi.fn(async (_query: string, options?: { pageSize?: number }) => ({ ok: true as const, value: { items: items.slice(0, options?.pageSize), @@ -101,7 +165,12 @@ it.effect("fetches past a first page of Git-ignored results", () => const finder = { destroy: vi.fn(), waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), - fileSearch, + getScanProgressAsync: async () => ({ + ok: true as const, + value: { isScanning: false }, + }), + watch: () => ({ ok: true as const, value: () => undefined }), + fileSearchAsync: fileSearch, } as unknown as FileFinder; vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); @@ -123,7 +192,12 @@ it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( const finder = { destroy: vi.fn(), waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), - fileSearch: vi.fn(() => ({ + getScanProgressAsync: async () => ({ + ok: true as const, + value: { isScanning: false }, + }), + watch: () => ({ ok: true as const, value: () => undefined }), + fileSearchAsync: vi.fn(async () => ({ ok: true as const, value: { items, scores: [], totalMatched: 1, totalFiles: 1 }, })), @@ -135,7 +209,7 @@ it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( expect(result.entries).toEqual([{ kind: "file", path: "dir\\one/report.txt" }]); }), - ), + ).pipe(Effect.provide(testLayer)), ); it.effect("continues whole-word searches after a filtered grep page", () => @@ -174,7 +248,7 @@ it.effect("continues whole-word searches after a filtered grep page", () => filteredFileCount: 1, nextCursor: cursor, }); - const grep = vi.fn((_query: string, options?: GrepOptions) => + const grep = vi.fn(async (_query: string, options?: GrepOptions) => options?.cursor ? { ok: true as const, value: grepResult("needle", [[0, 6]], null) } : { @@ -185,7 +259,12 @@ it.effect("continues whole-word searches after a filtered grep page", () => const finder = { destroy: vi.fn(), waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), - grep, + getScanProgressAsync: async () => ({ + ok: true as const, + value: { isScanning: false }, + }), + watch: () => ({ ok: true as const, value: () => undefined }), + grepAsync: grep, } as unknown as FileFinder; vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); @@ -215,6 +294,249 @@ it.effect("continues whole-word searches after a filtered grep page", () => ).pipe(Effect.provide(testLayer)), ); +function fakeSymlinkFinder( + directories: ReadonlyArray, + options: { + readonly indexed?: ReadonlyArray<{ + readonly path: string; + readonly kind: "file" | "directory"; + }>; + readonly watchFailures?: number; + } = {}, +) { + const watchers: Array<(events: ReadonlyArray) => void> = []; + let watchFailures = options.watchFailures ?? 0; + // fff's pageIndex is an offset into the results, not a page number. + const directorySearch = vi.fn( + async (_query: string, searchOptions?: { pageIndex?: number; pageSize?: number }) => { + const offset = searchOptions?.pageIndex ?? 0; + const items = directories.slice(offset, offset + (searchOptions?.pageSize ?? 100)); + return { + ok: true as const, + value: { + items: items.map((relativePath) => ({ relativePath })), + totalMatched: directories.length, + }, + }; + }, + ); + const indexedItems = (options.indexed ?? []).map((entry) => ({ + type: entry.kind, + item: { relativePath: entry.path, fileName: entry.path.split("/").at(-1) }, + })); + const finder = { + destroy: vi.fn(), + waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), + // fff's isScanning waits on the picker lock on the JS thread. + isScanning: () => { + throw new Error("isScanning must not run on the JS thread"); + }, + getScanProgressAsync: async () => ({ + ok: true as const, + value: { isScanning: false }, + }), + watch: (callback: (events: ReadonlyArray) => void) => { + if (watchFailures > 0) { + watchFailures -= 1; + throw new Error("File system watcher is not ready"); + } + watchers.push(callback); + return { ok: true as const, value: () => undefined }; + }, + directorySearchAsync: directorySearch, + fileSearchAsync: async () => { + const items = indexedItems.flatMap((entry) => (entry.type === "file" ? [entry.item] : [])); + return { ok: true as const, value: { items, scores: [], totalMatched: items.length } }; + }, + mixedSearchAsync: async () => ({ + ok: true as const, + value: { items: indexedItems, totalMatched: indexedItems.length }, + }), + } as unknown as FileFinder; + vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); + return { + directorySearch, + notifyWatchers: (events: ReadonlyArray) => + watchers.forEach((watcher) => watcher(events)), + }; +} + +const listedPaths = (searchIndex: WorkspaceSearchIndex.WorkspaceSearchIndex["Service"]) => + searchIndex.list().pipe(Effect.map((result) => result.entries.map((entry) => entry.path))); + +it.effect("reads symlinks again after the watcher reports a change", () => + Effect.scoped( + Effect.gen(function* () { + if ((yield* HostProcess.Platform) === "win32") return; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-symlink-watch-" }); + const outside = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-outside-" }); + yield* fileSystem.writeFileString(path.join(cwd, "a.ts"), ""); + yield* fileSystem.symlink("a.ts", path.join(cwd, "old-link.ts")); + const { directorySearch, notifyWatchers } = fakeSymlinkFinder([]); + + const searchIndex = yield* WorkspaceSearchIndex.make(cwd); + const concurrent = yield* Effect.all( + Array.from({ length: 8 }, () => listedPaths(searchIndex)), + { concurrency: "unbounded" }, + ); + yield* fileSystem.remove(path.join(cwd, "old-link.ts")); + yield* fileSystem.symlink("a.ts", path.join(cwd, "new-link.ts")); + const beforeNotice = yield* listedPaths(searchIndex); + // Editing a file cannot move a link, so it does not cost a rescan. + notifyWatchers([{ kind: "modified", path: path.join(cwd, "a.ts") }]); + const afterEdit = yield* listedPaths(searchIndex); + notifyWatchers([{ kind: "created", path: path.join(cwd, "new-link.ts") }]); + const afterNotice = yield* listedPaths(searchIndex); + // fff reports a link replaced in place as modified. + yield* fileSystem.remove(path.join(cwd, "new-link.ts")); + yield* fileSystem.symlink(outside, path.join(cwd, "new-link.ts")); + notifyWatchers([{ kind: "modified", path: path.join(cwd, "new-link.ts") }]); + const afterReplace = yield* listedPaths(searchIndex); + + expect(concurrent).toEqual(Array.from({ length: 8 }, () => ["old-link.ts"])); + expect(beforeNotice).toEqual(["old-link.ts"]); + expect(afterEdit).toEqual(["old-link.ts"]); + expect(afterNotice).toEqual(["new-link.ts"]); + expect(afterReplace).toEqual([]); + expect(directorySearch).toHaveBeenCalledTimes(3); + }), + ).pipe(Effect.provide(testLayer)), +); + +it.effect("answers from the last symlink scan while a slow rescan runs", () => + Effect.scoped( + Effect.gen(function* () { + if ((yield* HostProcess.Platform) === "win32") return; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-symlink-stale-" }); + yield* fileSystem.writeFileString(path.join(cwd, "a.ts"), ""); + yield* fileSystem.symlink("a.ts", path.join(cwd, "old-link.ts")); + const { notifyWatchers } = fakeSymlinkFinder([]); + // Holding git's ignore check open keeps the rescan from finishing. + const gate = yield* Deferred.make(); + let holdIgnoreChecks = false; + const vcsProcess = yield* VcsProcess.VcsProcess; + const searchIndex = yield* WorkspaceSearchIndex.make(cwd).pipe( + Effect.provideService(VcsProcess.VcsProcess, { + ...vcsProcess, + run: (input) => + holdIgnoreChecks && input.operation === "WorkspaceSearchIndex.listGitIgnored" + ? Deferred.await(gate).pipe(Effect.andThen(vcsProcess.run(input))) + : vcsProcess.run(input), + }), + ); + expect(yield* listedPaths(searchIndex)).toEqual(["old-link.ts"]); + + holdIgnoreChecks = true; + yield* fileSystem.symlink("a.ts", path.join(cwd, "new-link.ts")); + notifyWatchers([{ kind: "created", path: path.join(cwd, "new-link.ts") }]); + const duringRescan = yield* Effect.forkChild(listedPaths(searchIndex)); + yield* TestClock.adjust("1 second"); + expect(yield* Fiber.join(duringRescan)).toEqual(["old-link.ts"]); + + yield* Deferred.succeed(gate, undefined); + expect(yield* listedPaths(searchIndex)).toEqual(["new-link.ts", "old-link.ts"]); + }), + ).pipe(Effect.provide(testLayer)), +); + +it.effect("keeps symlinks once the watcher subscribes after starting late", () => + Effect.scoped( + Effect.gen(function* () { + if ((yield* HostProcess.Platform) === "win32") return; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-symlink-watch-" }); + yield* fileSystem.writeFileString(path.join(cwd, "a.ts"), ""); + yield* fileSystem.symlink("a.ts", path.join(cwd, "link.ts")); + const { directorySearch } = fakeSymlinkFinder([], { watchFailures: 2 }); + + const searchIndex = yield* WorkspaceSearchIndex.make(cwd); + const scansAfterRound: number[] = []; + for (let round = 0; round < 5; round += 1) { + expect(yield* listedPaths(searchIndex)).toEqual(["link.ts"]); + expect(yield* listedPaths(searchIndex)).toEqual(["link.ts"]); + scansAfterRound.push(directorySearch.mock.calls.length); + yield* TestClock.adjust("1 minute"); + } + + // An unwatched scan serves reads only briefly; the third has a watcher and is kept. + expect(scansAfterRound).toEqual([1, 2, 3, 3, 3]); + }), + ).pipe(Effect.provide(testLayer)), +); + +it.effect( + "lists a symlink the watcher indexed once, with the link's kind, and drops rejected ones", + () => + Effect.scoped( + Effect.gen(function* () { + if ((yield* HostProcess.Platform) === "win32") return; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-symlink-dupe-" }); + const outside = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-outside-" }); + yield* fileSystem.makeDirectory(path.join(cwd, "src")); + yield* fileSystem.symlink("src", path.join(cwd, "src-link")); + // A link the scan rejects stays out even when the watcher indexed it. + yield* fileSystem.symlink(outside, path.join(cwd, "outside-link")); + fakeSymlinkFinder([], { + indexed: [ + { path: "outside-link", kind: "file" }, + { path: "src-link", kind: "file" }, + { path: "link-file.ts", kind: "file" }, + ], + }); + + const searchIndex = yield* WorkspaceSearchIndex.make(cwd); + const searched = yield* searchIndex.search("link", 10); + const files = yield* searchIndex.search("link", 1, "file"); + const listed = yield* searchIndex.list(); + + expect(searched.entries).toEqual([ + { path: "src-link", kind: "directory" }, + { path: "link-file.ts", kind: "file" }, + ]); + // The dropped link does not take the only slot. + expect(files).toEqual({ + entries: [{ path: "link-file.ts", kind: "file" }], + truncated: false, + }); + expect(listed.entries).toEqual([ + { path: "link-file.ts", kind: "file" }, + { path: "src-link", kind: "directory" }, + ]); + }), + ).pipe(Effect.provide(testLayer)), +); + +it.effect("finds symlinks in folders past the first page of the index", () => + Effect.scoped( + Effect.gen(function* () { + if ((yield* HostProcess.Platform) === "win32") return; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-symlink-pages-" }); + yield* fileSystem.makeDirectory(path.join(cwd, "late")); + yield* fileSystem.writeFileString(path.join(cwd, "late/a.ts"), ""); + yield* fileSystem.symlink("a.ts", path.join(cwd, "late/link.ts")); + // Folders fff lists but that are gone by the time they are read are skipped. + const firstPage = Array.from({ length: 1_000 }, (_, index) => `missing-${index}`); + const { directorySearch } = fakeSymlinkFinder([...firstPage, "late"]); + + const searchIndex = yield* WorkspaceSearchIndex.make(cwd); + + expect(yield* listedPaths(searchIndex)).toEqual(["late", "late/link.ts"]); + expect(directorySearch.mock.calls.map(([, options]) => options?.pageIndex)).toEqual([ + 0, 1_000, + ]); + }), + ).pipe(Effect.provide(testLayer)), +); + // fff reads HOME on Linux and macOS but the user profile on Windows, so a temp HOME cannot stand in there. it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( "refuses to index a workspace rooted at the home directory", diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.ts b/apps/server/src/workspace/WorkspaceSearchIndex.ts index b21f2dde83b08..8d9798baadfd7 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.ts @@ -1,3 +1,5 @@ +// @effect-diagnostics nodeBuiltinImport:off - readdir with file types avoids an lstat per child +import * as NodeFSP from "node:fs/promises"; import * as NodeModule from "node:module"; import type { @@ -7,11 +9,18 @@ import type { GrepCursor, MixedItem, Result, + WatchEvent, + WatchUnsubscribe, } from "@ff-labs/fff-node"; +import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as LayerMap from "effect/LayerMap"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; import type { @@ -23,14 +32,24 @@ import type { ProjectSearchEntriesResult, } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; -import { isProjectFaviconPath } from "@t3tools/shared/projectFavicon"; +import { PROJECT_FAVICON_EXTENSIONS, isProjectFaviconPath } from "@t3tools/shared/projectFavicon"; +import { scoreSubsequenceMatch } from "@t3tools/shared/searchRanking"; import * as VcsProcess from "../vcs/VcsProcess.ts"; +import { type IgnoreDecision, type IgnoreFile, parseIgnoreFile } from "./IgnorePatterns.ts"; // fff-node's native library stays external; single-executables require disk packages instead of importing them. const requireForFff = NodeModule.createRequire(import.meta.url); const { FileFinder } = requireForFff("@ff-labs/fff-node") as typeof import("@ff-labs/fff-node"); +// A glob intersects existing extension constraints; fff ORs bare `*.ext` constraints together. +const IMAGE_SEARCH_CONSTRAINT = `**/*.{${PROJECT_FAVICON_EXTENSIONS.map((extension) => + extension.slice(1).replace(/[a-z]/g, (letter) => `[${letter}${letter.toUpperCase()}]`), +).join(",")}}`; + +// fff 0.11 disables filename fallback above this match count per base/overflow segment. +const FFF_FILENAME_FALLBACK_MAX_MATCHES = 15_000; + const WORKSPACE_INDEX_MAX_ENTRIES = 25_000; const WORKSPACE_INDEX_PAGE_SIZE = WORKSPACE_INDEX_MAX_ENTRIES + 2; const WORKSPACE_INDEX_SCAN_TIMEOUT = "15 seconds"; @@ -38,6 +57,13 @@ const WORKSPACE_INDEX_SCAN_TIMEOUT_MS = 15_000; const WORKSPACE_INDEX_IDLE_TTL = "15 minutes"; const CONTENT_SEARCH_TIME_BUDGET_MS = 250; const CONTENT_SEARCH_MAX_MATCHES_PER_FILE = 100; +const SYMLINK_SCAN_CONCURRENCY = 16; +// A scan no watcher can invalidate is reused this long, so typing does not rescan every folder. +const UNWATCHED_SYMLINK_SCAN_TTL_MS = 2_000; +// A rescan that outlasts this leaves the read on the previous scan. +const SYMLINK_RESCAN_WAIT = "5 millis"; +// Each page is a synchronous native call; small pages let searches run between them. +const SYMLINK_DIRECTORY_PAGE_SIZE = 1_000; export class WorkspaceSearchIndexCreateFailed extends Schema.TaggedError()( "WorkspaceSearchIndexCreateFailed", @@ -157,6 +183,14 @@ function toDirectoryEntry(item: DirItem, hostIsWindows: boolean): ProjectEntry | return normalizedPath ? { path: normalizedPath, kind: "directory" } : null; } +function isLoneFuzzyPath(query: string): boolean { + const trimmed = query.trim(); + return ( + !/\s/u.test(trimmed) && + (trimmed.startsWith("/") || (trimmed.endsWith("/") && !/^[*!\\]/u.test(trimmed))) + ); +} + type SearchPage = { readonly items: ReadonlyArray; readonly totalMatched: number }; function mapFileSearchResult( @@ -290,10 +324,140 @@ function withDirectoryAncestors(entries: ReadonlyArray): ProjectEn return [...entryByPath.values()]; } +interface ScannedDirectory { + readonly directory: string; + readonly links: ReadonlyArray; + /** Every symlink in the folder, including ones dropped for leaving the workspace. */ + readonly linkPaths: ReadonlyArray; + /** A nested repository (or the workspace's own), whose ignore rules own what is below it. */ + readonly isRepository: boolean; + readonly ignoreFile: string | undefined; +} + +const joinEntryPath = (directory: string, name: string) => + directory ? `${directory}/${name}` : name; + +/** + * fff skips symlinks, and `followSymlinks` drops file links, dedupes folder links, and walks out + * of the workspace. So links are read from the folders fff already indexed (ignore rules applied), + * without descending into them, and kept only when they resolve inside the workspace. + */ +const scanDirectory = Effect.fn("WorkspaceSearchIndex.scanDirectory")(function* ( + fileSystem: FileSystem.FileSystem, + path: Path.Path, + cwd: string, + root: string, + directory: string, +) { + // Effect's readDirectory returns bare names; one lstat per child would cost far more. + const children = yield* Effect.promise(() => + NodeFSP.readdir(path.join(cwd, directory), { withFileTypes: true }).catch(() => []), + ); + const links: ProjectEntry[] = []; + const linkPaths: string[] = []; + let isRepository = false; + let ignoreFile: string | undefined; + for (const child of children) { + if (child.name === ".git") isRepository = true; + if (child.name === ".ignore" && child.isFile()) { + ignoreFile = yield* fileSystem + .readFileString(path.join(cwd, directory, child.name)) + .pipe(Effect.orElseSucceed(() => undefined)); + } + if (!child.isSymbolicLink()) continue; + const entryPath = joinEntryPath(directory, child.name); + linkPaths.push(entryPath); + const absolutePath = path.join(cwd, entryPath); + const target = yield* Effect.all([ + fileSystem.realPath(absolutePath), + fileSystem.stat(absolutePath), + ]).pipe(Effect.orElseSucceed(() => undefined)); + if (!target) continue; + const relative = path.relative(root, target[0]); + if ( + relative === ".." || + relative.startsWith(`..${path.sep}`) || + path.isAbsolute(relative) || + relative.split(path.sep).includes(".git") || + (target[1].type !== "Directory" && target[1].type !== "File") + ) { + continue; + } + links.push({ path: entryPath, kind: target[1].type === "Directory" ? "directory" : "file" }); + } + return { directory, links, linkPaths, isRepository, ignoreFile } satisfies ScannedDirectory; +}); + +/** Whether `entryPath` is `directory` itself or below it. */ +const isWithin = (entryPath: string, directory: string) => + directory === "" || entryPath.startsWith(`${directory}/`); + +/** + * fff reads `.ignore` files alongside git's rules, and like ripgrep the nearest one that matches + * decides, ahead of any `.gitignore`. `ignoreFiles` is ordered deepest folder first. + */ +function decideByIgnoreFiles( + entry: ProjectEntry, + ignoreFiles: ReadonlyArray<{ readonly directory: string; readonly file: IgnoreFile }>, +): IgnoreDecision | undefined { + for (const { directory, file } of ignoreFiles) { + if (!isWithin(entry.path, directory) || entry.path === directory) continue; + const relative = directory ? entry.path.slice(directory.length + 1) : entry.path; + // Like git and ripgrep, a link is never a folder to a folder-only rule, whatever it targets. + const decision = file.decide(relative, false); + if (decision) return decision; + } + return undefined; +} + +/** Places each symlink before the first index result that matches the query less closely. */ +function mergeSymlinkMatches( + indexed: ProjectSearchEntriesResult, + linkPaths: ReadonlySet, + symlinks: ReadonlyArray, + query: string, + limit: number, +): ProjectSearchEntriesResult { + if (linkPaths.size === 0) return indexed; + // fff's watcher indexes links created after its scan, under a kind of its own choosing, so a + // path it already returned takes the link's kind (or drops out of a kind-filtered search). + const symlinkByPath = new Map(symlinks.map((entry) => [entry.path, entry])); + const score = (entry: ProjectEntry) => + scoreSubsequenceMatch(entry.path.toLowerCase(), query) ?? Number.POSITIVE_INFINITY; + const indexedPaths = new Set(indexed.entries.map((entry) => entry.path)); + const pending = symlinks + .flatMap((entry) => { + const entryScore = score(entry); + return Number.isFinite(entryScore) && !indexedPaths.has(entry.path) + ? [{ entry, score: entryScore }] + : []; + }) + .toSorted((left, right) => left.score - right.score); + const entries: ProjectEntry[] = []; + let nextPending = 0; + for (const indexedEntry of indexed.entries) { + const entry = linkPaths.has(indexedEntry.path) + ? symlinkByPath.get(indexedEntry.path) + : indexedEntry; + if (!entry) continue; + const entryScore = score(entry); + while (nextPending < pending.length && pending[nextPending]!.score < entryScore) { + entries.push(pending[nextPending++]!.entry); + } + entries.push(entry); + } + for (; nextPending < pending.length; nextPending += 1) entries.push(pending[nextPending]!.entry); + return { + entries: entries.slice(0, limit), + truncated: indexed.truncated || entries.length > limit, + }; +} + // fff applies .gitignore files only; Git also ignores paths from .git/info/exclude and the global excludes file. const listGitIgnoredPaths = Effect.fn("WorkspaceSearchIndex.listGitIgnoredPaths")(function* ( vcsProcess: VcsProcess.VcsProcess["Service"], cwd: string, + hostIsWindows: boolean, ) { const result = yield* vcsProcess .run({ @@ -320,7 +484,7 @@ const listGitIgnoredPaths = Effect.fn("WorkspaceSearchIndex.listGitIgnoredPaths" result.stdout .split("\0") .filter((entry) => entry.length > 0) - .map((entry) => trimDirectorySeparator(toPosixPath(entry))), + .map((entry) => trimDirectorySeparator(toPosixPath(entry, hostIsWindows))), ); }); @@ -328,9 +492,15 @@ function mixedItemPath(item: MixedItem): string { return item.item.relativePath; } -function isIgnoredPath(ignoredPaths: ReadonlySet, relativePath: string): boolean { +function isIgnoredPath( + ignoredPaths: ReadonlySet, + relativePath: string, + hostIsWindows: boolean, +): boolean { if (ignoredPaths.size === 0) return false; - let candidate: string | undefined = trimDirectorySeparator(toPosixPath(relativePath)); + let candidate: string | undefined = trimDirectorySeparator( + toPosixPath(relativePath, hostIsWindows), + ); while (candidate) { if (ignoredPaths.has(candidate)) return true; candidate = parentPathOf(candidate); @@ -394,6 +564,8 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( variant: WorkspaceSearchIndexVariant = "paths", ) { const hostIsWindows = yield* HostProcess.isWindows; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; const finder = yield* Effect.acquireRelease(createFinder(cwd, variant), (finder) => Effect.try({ try: () => finder.destroy(), @@ -402,7 +574,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( ); let initialScanTimedOut = false; const vcsProcess = yield* VcsProcess.VcsProcess; - let ignoredPaths = yield* listGitIgnoredPaths(vcsProcess, cwd); + let ignoredPaths = yield* listGitIgnoredPaths(vcsProcess, cwd, hostIsWindows); yield* waitForIndexReady( cwd, finder, @@ -423,22 +595,32 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( }), ); - const hasIncompleteInitialScan = () => initialScanTimedOut && finder.isScanning(); + // fff's isScanning takes the picker lock too, so progress is read off the JS thread. + const isScanning = Effect.promise(async () => { + try { + const progress = await finder.getScanProgressAsync(); + return !progress.ok || progress.value.isScanning; + } catch { + return true; + } + }); + const hasIncompleteInitialScan = () => (initialScanTimedOut ? isScanning : Effect.succeed(false)); const runSearch = Effect.fn("WorkspaceSearchIndex.runSearch")(function*
    ( query: string, pageSize: number, operation: "directorySearch" | "fileSearch" | "grep" | "mixedSearch", - execute: () => Result, + execute: () => Promise>, ): Effect.fn.Return { - const result = yield* Effect.try({ + // fff's searches wait on its picker lock, which its watcher can hold for minutes. + const result = yield* Effect.tryPromise({ try: execute, catch: (cause) => new WorkspaceSearchIndexSearchFailed({ cwd, queryLength: query.length, pageSize, - reason: `FileFinder.${operation} threw unexpectedly.`, + reason: `FileFinder.${operation} rejected unexpectedly.`, cause, }), }); @@ -453,26 +635,250 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( return result.value; }); + const root = yield* fileSystem.realPath(cwd).pipe(Effect.orElseSucceed(() => cwd)); + + const listIndexedDirectories = Effect.fn("WorkspaceSearchIndex.listIndexedDirectories")( + function* () { + const directories: string[] = []; + // fff's pageIndex is an offset into the results, not a page number. + for (let pageIndex = 0; ; pageIndex += SYMLINK_DIRECTORY_PAGE_SIZE) { + const result = yield* Effect.promise(async () => { + try { + return await finder.directorySearchAsync("", { + pageIndex, + pageSize: SYMLINK_DIRECTORY_PAGE_SIZE, + }); + } catch { + return undefined; + } + }); + if (!result?.ok) return directories; + for (const item of result.value.items) { + const directory = trimDirectorySeparator(toPosixPath(item.relativePath, hostIsWindows)); + if (directory) directories.push(directory); + } + if (result.value.items.length < SYMLINK_DIRECTORY_PAGE_SIZE) return directories; + } + }, + ); + + /** Ignore rules belong to the repository that owns a path, so each one answers for its own. */ + const listGitIgnored = Effect.fn("WorkspaceSearchIndex.listGitIgnored")(function* ( + repository: string, + entries: ReadonlyArray, + ) { + const relativePaths = entries.map((entry) => + repository ? entry.path.slice(repository.length + 1) : entry.path, + ); + const repositoryCwd = path.join(cwd, repository); + const result = yield* vcsProcess + .run({ + operation: "WorkspaceSearchIndex.listGitIgnored", + command: "git", + // fff applies ignore rules to tracked paths too, so the index is not consulted. + args: ["-c", "core.fsmonitor=false", "check-ignore", "--no-index", "-z", "--stdin"], + cwd: repositoryCwd, + stdin: `${relativePaths.join("\0")}\0`, + allowNonZeroExit: true, + timeoutMs: 10_000, + maxOutputBytes: 16 * 1024 * 1024, + }) + .pipe(Effect.orElseSucceed(() => undefined)); + // Exit 1 means nothing matched. + if (result?.exitCode === 1) return []; + if (result?.exitCode !== 0 || result.stdoutTruncated) { + // Outside git no rules apply (git exits before reading stdin, so the cause is lost). + // Any other failure hides the repository's links rather than leaking ones it ignores. + const insideRepository = yield* vcsProcess + .run({ + operation: "WorkspaceSearchIndex.listGitIgnored", + command: "git", + args: ["rev-parse", "--git-dir"], + cwd: repositoryCwd, + timeoutMs: 10_000, + }) + .pipe( + Effect.as(true), + Effect.orElseSucceed(() => false), + ); + return insideRepository ? entries.map((entry) => entry.path) : []; + } + return result.stdout + .split("\0") + .filter(Boolean) + .map((relativePath) => (repository ? `${repository}/${relativePath}` : relativePath)); + }); + + const scanSymlinkEntries = Effect.fn("WorkspaceSearchIndex.scanSymlinkEntries")(function* () { + const indexComplete = !(yield* isScanning); + const directories = yield* listIndexedDirectories(); + const scanned = yield* Effect.forEach( + ["", ...directories], + (directory) => scanDirectory(fileSystem, path, cwd, root, directory), + { concurrency: SYMLINK_SCAN_CONCURRENCY }, + ); + const candidates = scanned.flatMap((directory) => directory.links); + const linkPaths = new Set(scanned.flatMap((directory) => directory.linkPaths)); + if (candidates.length === 0) return { entries: candidates, linkPaths, indexComplete }; + + const ignoreFiles = scanned + .flatMap(({ directory, ignoreFile }) => + ignoreFile === undefined ? [] : [{ directory, file: parseIgnoreFile(ignoreFile) }], + ) + .toSorted((left, right) => right.directory.length - left.directory.length); + const decisions = new Map( + candidates.map((entry) => [entry.path, decideByIgnoreFiles(entry, ignoreFiles)]), + ); + const repositories = [ + "", + ...scanned.flatMap(({ directory, isRepository }) => + isRepository && directory ? [directory] : [], + ), + ].toSorted((left, right) => right.length - left.length); + const entriesByRepository = Map.groupBy( + candidates.filter((entry) => decisions.get(entry.path) === undefined), + (entry) => + repositories.find( + (repository) => isWithin(entry.path, repository) && entry.path !== repository, + ) ?? "", + ); + const ignored = new Set( + (yield* Effect.forEach( + entriesByRepository, + ([repository, entries]) => listGitIgnored(repository, entries), + { concurrency: 4 }, + )).flat(), + ); + return { + entries: candidates.filter((entry) => + decisions.get(entry.path) === undefined + ? !ignored.has(entry.path) + : decisions.get(entry.path) === "included", + ), + linkPaths, + indexComplete, + }; + }); + + // One scan runs at a time and outlives the reads that started it. Only the first read waits for + // it in full: after that a change or expiry marks the result stale, and reads wait briefly for + // the rescan before answering from the last finished one, so file churn cannot stall search. + type SymlinkScan = Effect.Success>; + let lastSymlinkScan: SymlinkScan | undefined; + let lastSymlinkScanExpiresAt: number | undefined; + let pendingSymlinkScan: Deferred.Deferred | undefined; + let symlinkScanStale = false; + let knownLinkPaths: ReadonlySet = new Set(); + const invalidateSymlinkScan = () => { + symlinkScanStale = true; + }; + const isKnownLink = (absolutePath: string) => + [cwd, root].some((base) => + knownLinkPaths.has(toPosixPath(path.relative(base, absolutePath), hostIsWindows)), + ); + const onWatchBatch = (events: ReadonlyArray) => { + // fff reports a link replaced in place as `modified`, so edits to known links count too. + const editsOnly = events.every( + (event) => + event.kind === "modified" && + !/(^|[\\/])\.(git)?ignore$/.test(event.path) && + !isKnownLink(event.path), + ); + if (!editsOnly) invalidateSymlinkScan(); + }; + // fff refuses subscriptions until its watcher is up, which can trail the initial scan, so + // each read retries until one is in place. + let unwatch: WatchUnsubscribe | undefined; + yield* Effect.addFinalizer(() => Effect.sync(() => unwatch?.())); + const ensureWatching = () => { + if (unwatch) return true; + try { + const result = finder.watch(onWatchBatch); + if (result.ok) unwatch = result.value; + } catch { + // Not ready yet; the next read tries again. + } + return unwatch !== undefined; + }; + const indexScope = yield* Effect.scope; + const startSymlinkScan = Effect.fn("WorkspaceSearchIndex.startSymlinkScan")(function* () { + symlinkScanStale = false; + // Subscribing before the scan starts means no change during it goes unseen. + const watched = ensureWatching(); + const done = yield* Deferred.make(); + pendingSymlinkScan = done; + yield* scanSymlinkEntries().pipe( + Effect.tap((result) => + Effect.gen(function* () { + lastSymlinkScan = result; + knownLinkPaths = result.linkPaths; + // Without a watcher, or before fff has seen every folder, the result is kept only briefly. + lastSymlinkScanExpiresAt = + watched && result.indexComplete + ? undefined + : (yield* Clock.currentTimeMillis) + UNWATCHED_SYMLINK_SCAN_TTL_MS; + }), + ), + Effect.onExit((exit) => + Effect.sync(() => { + pendingSymlinkScan = undefined; + }).pipe(Effect.andThen(Deferred.done(done, exit))), + ), + Effect.forkIn(indexScope), + ); + return done; + }); + const listSymlinkEntries = Effect.fn("WorkspaceSearchIndex.listSymlinkEntries")(function* () { + if ( + lastSymlinkScanExpiresAt !== undefined && + (yield* Clock.currentTimeMillis) >= lastSymlinkScanExpiresAt + ) { + symlinkScanStale = true; + } + const pending = + pendingSymlinkScan ?? + (lastSymlinkScan === undefined || symlinkScanStale ? yield* startSymlinkScan() : undefined); + let result = lastSymlinkScan; + if (pending && result === undefined) { + result = yield* Deferred.await(pending); + } else if (pending) { + const fresh = yield* Deferred.await(pending).pipe(Effect.timeoutOption(SYMLINK_RESCAN_WAIT)); + if (Option.isSome(fresh)) result = fresh.value; + } + if (result === undefined) + return yield* Effect.die("The symlink scan finished without a result."); + return { + // Git's ignore rules hide a link the same way searchVisible hides the files it indexes. + entries: result.entries.filter( + (entry) => !isIgnoredPath(ignoredPaths, entry.path, hostIsWindows), + ), + linkPaths: result.linkPaths, + }; + }); + // Grows the first page until hidden entries can no longer crowd visible ones out of it. - const searchVisible = Effect.fn("WorkspaceSearchIndex.searchVisible")(function* ( + const searchVisible = Effect.fn("WorkspaceSearchIndex.searchVisible")(function* ( query: string, pageSize: number, operation: "directorySearch" | "fileSearch" | "mixedSearch", - execute: (pageSize: number) => Result>, + execute: (pageSize: number) => Promise & Metadata>>, pathOf: (item: I) => string, - ): Effect.fn.Return, WorkspaceSearchIndexSearchFailed> { + ): Effect.fn.Return & Metadata, WorkspaceSearchIndexSearchFailed> { let size = pageSize; while (true) { const fetchSize = size; const result = yield* runSearch(query, fetchSize, operation, () => execute(fetchSize)); if (ignoredPaths.size === 0) return result; - const items = result.items.filter((item) => !isIgnoredPath(ignoredPaths, pathOf(item))); + const items = result.items.filter( + (item) => !isIgnoredPath(ignoredPaths, pathOf(item), hostIsWindows), + ); if ( items.length >= pageSize || result.items.length >= result.totalMatched || fetchSize >= WORKSPACE_INDEX_PAGE_SIZE ) { return { + ...result, items, totalMatched: result.totalMatched - (result.items.length - items.length), }; @@ -512,22 +918,26 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( cause, }), ).pipe(Effect.catchTags({ WorkspaceSearchIndexScanTimedOut: () => Effect.void })); - ignoredPaths = yield* listGitIgnoredPaths(vcsProcess, cwd); + ignoredPaths = yield* listGitIgnoredPaths(vcsProcess, cwd, hostIsWindows); + invalidateSymlinkScan(); }); const list: WorkspaceSearchIndex["Service"]["list"] = Effect.fn("WorkspaceSearchIndex.list")( function* () { - const incompleteBeforeQuery = hasIncompleteInitialScan(); + const incompleteBeforeQuery = yield* hasIncompleteInitialScan(); const result = yield* searchVisible( "", WORKSPACE_INDEX_PAGE_SIZE, "mixedSearch", - (pageSize) => finder.mixedSearch("", { pageSize }), + (pageSize) => finder.mixedSearchAsync("", { pageSize }), mixedItemPath, ); const mapped = mapMixedSearchResult(result, WORKSPACE_INDEX_MAX_ENTRIES, hostIsWindows); - const sortedEntries = withDirectoryAncestors(mapped.entries).toSorted((left, right) => - left.path.localeCompare(right.path), + const symlinks = yield* listSymlinkEntries(); + // fff's watcher can index a link the scan rejected; only accepted links are listed. + const indexed = mapped.entries.filter((entry) => !symlinks.linkPaths.has(entry.path)); + const sortedEntries = withDirectoryAncestors([...indexed, ...symlinks.entries]).toSorted( + (left, right) => left.path.localeCompare(right.path), ); const entries = sortedEntries.slice(0, WORKSPACE_INDEX_MAX_ENTRIES); return { @@ -544,35 +954,68 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( kind: ProjectEntryKind | undefined, imageOnly: boolean | undefined, ) { - const pageSize = imageOnly ? WORKSPACE_INDEX_PAGE_SIZE : Math.max(1, limit + 1); + const { entries: allSymlinks, linkPaths } = yield* listSymlinkEntries(); + const symlinks = allSymlinks.filter((entry) => + imageOnly + ? entry.kind === "file" && isProjectFaviconPath(entry.path) + : kind === undefined || entry.kind === kind, + ); + // Index results that are filtered-out links get dropped in the merge, so fetch past them. + const indexLimit = limit + linkPaths.size - symlinks.length; + const pageSize = Math.max(1, indexLimit + 1); + const merge = (indexed: ProjectSearchEntriesResult) => + mergeSymlinkMatches(indexed, linkPaths, symlinks, query, limit); if (kind === "file" || imageOnly) { - const result = yield* searchVisible( - query, + const searchFiles = (fileQuery: string, size: number) => + searchVisible( + query, + size, + "fileSearch", + (fetchSize) => finder.fileSearchAsync(fileQuery, { pageSize: fetchSize }), + (item: FileItem) => item.relativePath, + ); + // Adding a constraint changes fff's lone-path parse; offset pages can skip tied matches. + if (imageOnly && isLoneFuzzyPath(query)) { + const result = yield* searchFiles(query, WORKSPACE_INDEX_PAGE_SIZE); + return merge(mapFileSearchResult(result, indexLimit, hostIsWindows, true)); + } + let result = yield* searchFiles( + imageOnly ? `${query} ${IMAGE_SEARCH_CONSTRAINT}` : query, pageSize, - "fileSearch", - (size) => finder.fileSearch(query, { pageSize: size }), - (item) => item.relativePath, ); - return mapFileSearchResult(result, limit, hostIsWindows, imageOnly); + if ( + imageOnly && + query.trim() && + result.totalFiles > FFF_FILENAME_FALLBACK_MAX_MATCHES && + result.totalMatched > 0 + ) { + const unfiltered = yield* runSearch(query, 1, "fileSearch", () => + finder.fileSearchAsync(query, { pageSize: 1 }), + ); + if (unfiltered.totalMatched > FFF_FILENAME_FALLBACK_MAX_MATCHES) { + result = yield* searchFiles(query, WORKSPACE_INDEX_PAGE_SIZE); + } + } + return merge(mapFileSearchResult(result, indexLimit, hostIsWindows, imageOnly)); } if (kind === "directory") { const result = yield* searchVisible( query, pageSize, "directorySearch", - (size) => finder.directorySearch(query, { pageSize: size }), + (size) => finder.directorySearchAsync(query, { pageSize: size }), (item) => item.relativePath, ); - return mapDirectorySearchResult(result, limit, hostIsWindows); + return merge(mapDirectorySearchResult(result, indexLimit, hostIsWindows)); } const result = yield* searchVisible( query, pageSize, "mixedSearch", - (size) => finder.mixedSearch(query, { pageSize: size }), + (size) => finder.mixedSearchAsync(query, { pageSize: size }), mixedItemPath, ); - return mapMixedSearchResult(result, limit, hostIsWindows); + return merge(mapMixedSearchResult(result, indexLimit, hostIsWindows)); }); const search: WorkspaceSearchIndex["Service"]["search"] = Effect.fn( @@ -582,7 +1025,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( // matching is subsequence-based, so dropping them still matches "[id]" and "a{b}" paths. // "{" is dropped only when the query finds nothing as a glob, so "*.{ts,tsx}" keeps working. const query = rawQuery.replaceAll("[", ""); - const incompleteBeforeQuery = hasIncompleteInitialScan(); + const incompleteBeforeQuery = yield* hasIncompleteInitialScan(); let mapped = yield* searchNames(query, limit, kind, imageOnly); if (mapped.entries.length === 0 && query.includes("{")) { mapped = yield* searchNames(query.replaceAll("{", ""), limit, kind, imageOnly); @@ -607,7 +1050,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( do { const remainingTimeBudgetMs = Math.max(1, Math.ceil(deadline - performance.now())); const result = yield* runSearch(input.query, input.limit, "grep", () => - finder.grep(searchQuery, { + finder.grepAsync(searchQuery, { mode: regexMode ? "regex" : "plain", smartCase: !input.caseSensitive && !regexMode, maxMatchesPerFile: Math.min(CONTENT_SEARCH_MAX_MATCHES_PER_FILE, rawPageSize), @@ -625,7 +1068,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( } for (const match of result.items) { - if (isIgnoredPath(ignoredPaths, match.relativePath)) continue; + if (isIgnoredPath(ignoredPaths, match.relativePath, hostIsWindows)) continue; const matchRanges = mapContentMatchRanges(match.lineContent, match.matchRanges).filter( (range) => !input.wholeWord || isWholeWordRange(match.lineContent, range), ); diff --git a/apps/server/src/workspace/restoreMissingWorktree.ts b/apps/server/src/workspace/restoreMissingWorktree.ts index 7a164b67d3696..dee23025dbf41 100644 --- a/apps/server/src/workspace/restoreMissingWorktree.ts +++ b/apps/server/src/workspace/restoreMissingWorktree.ts @@ -1,17 +1,22 @@ import { ThreadId, type ProjectId, type TerminalCwdNotFoundError } from "@t3tools/contracts"; +import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; +import * as ServerConfig from "../config.ts"; import * as GitWorkflowService from "../git/GitWorkflowService.ts"; +import { scratchFolderRoot } from "../project/ManagedProjectFolders.ts"; import * as ProjectService from "../project/ProjectService.ts"; import { makeIsManagedWorktreePath } from "../worktreesDirectory.ts"; import { isWithinRoot } from "./KnownWorkspaces.ts"; +import { withWorkspaceLease } from "./workspaceLease.ts"; import { TerminalCwdRecovery } from "../terminal/Manager.ts"; -class WorktreeRecreateError extends Schema.TaggedError()( +export class WorktreeRecreateError extends Schema.TaggedError()( "WorktreeRecreateError", { worktreePath: Schema.String, @@ -33,6 +38,9 @@ class WorktreeRecreateNoBaseError extends Schema.TaggedError root)); + if (path.dirname(folder) !== root && path.dirname(folder) !== realRoot) return; + yield* Effect.logWarning("recreating missing scratch folder", { + threadId: thread.id, + folder, + }); + yield* fileSystem.makeDirectory(folder, { recursive: true }); + }, + // Turn start reports a folder that is still missing. + Effect.orElseSucceed(() => undefined), + ); + + // Only folders holding nothing but leftovers are cleared, never user files. + const isLeftoverFolder = (worktreePath: string) => + fileSystem.exists(path.join(worktreePath, ".git")).pipe( + Effect.flatMap((hasGit) => + hasGit + ? Effect.succeed(false) + : fileSystem.readDirectory(worktreePath).pipe( + Effect.flatMap((entries) => + Effect.forEach(entries, (entry) => + LEFTOVER_WORKTREE_ENTRIES.has(entry) + ? fileSystem + .stat(path.join(worktreePath, entry)) + .pipe(Effect.map((info) => info.type === "Directory")) + : Effect.succeed(false), + ), + ), + Effect.map((leftovers) => leftovers.every(Boolean)), + ), + ), + Effect.orElseSucceed(() => false), + ); const resolveRecreateBase = Effect.fn("restoreMissingWorktree.resolveRecreateBase")(function* ( cwd: string, + remoteName: string | null, ) { // Recovery can follow an external checkout, before the status cache expires. yield* gitWorkflow.invalidateLocalStatus(cwd); @@ -56,14 +110,15 @@ export const makeRestoreMissingWorktree = Effect.gen(function* () { const refName = `refs/heads/${currentBranch}`; return { refName, baseRefName: refName }; } - if (yield* gitWorkflow.hasCommit({ cwd, refName: "refs/remotes/origin/HEAD" })) { + const remoteHead = `refs/remotes/${remoteName}/HEAD`; + if (remoteName !== null && (yield* gitWorkflow.hasCommit({ cwd, refName: remoteHead }))) { const { commitSha } = yield* gitWorkflow.resolveRemoteTrackingCommit({ cwd, refName: "HEAD", - fallbackRemoteName: "origin", + fallbackRemoteName: remoteName, }); // Start from the commit so the thread branch does not track the default branch. - return { refName: commitSha, baseRefName: "refs/remotes/origin/HEAD" }; + return { refName: commitSha, baseRefName: remoteHead }; } if (yield* gitWorkflow.hasCommit({ cwd, refName: "HEAD" })) { return { refName: "HEAD", baseRefName: undefined }; @@ -83,9 +138,12 @@ export const makeRestoreMissingWorktree = Effect.gen(function* () { yield* gitWorkflow.createWorktree({ cwd, refName: branch, path: worktreePath }); return; } + const remoteName = yield* gitWorkflow + .resolvePrimaryRemoteName(cwd) + .pipe(Effect.orElseSucceed(() => null)); // Push keeps the tracking ref current, so recovery reads it without fetching. - const pushedRef = `refs/remotes/origin/${branch}`; - if (yield* gitWorkflow.hasCommit({ cwd, refName: pushedRef })) { + const pushedRef = remoteName === null ? null : `refs/remotes/${remoteName}/${branch}`; + if (pushedRef !== null && (yield* gitWorkflow.hasCommit({ cwd, refName: pushedRef }))) { yield* gitWorkflow.createWorktree({ cwd, refName: pushedRef, @@ -94,7 +152,7 @@ export const makeRestoreMissingWorktree = Effect.gen(function* () { }); return; } - const base = yield* resolveRecreateBase(cwd); + const base = yield* resolveRecreateBase(cwd, remoteName); if (base === null) { return yield* new WorktreeRecreateNoBaseError({ worktreePath, branch }); } @@ -109,9 +167,11 @@ export const makeRestoreMissingWorktree = Effect.gen(function* () { return Effect.fn("restoreMissingWorktree")(function* (thread: WorktreeThread) { const { worktreePath, branch } = thread; - if (worktreePath === null || branch === null) return; + if (worktreePath === null) return; + if (branch === null) return yield* recreateScratchFolder(thread, worktreePath); const exists = yield* fileSystem.exists(worktreePath).pipe(Effect.orElseSucceed(() => true)); - if (exists || !(yield* isManagedWorktreePath(worktreePath))) return; + const leftover = exists && (yield* isLeftoverFolder(worktreePath)); + if ((exists && !leftover) || !(yield* isManagedWorktreePath(worktreePath))) return; const project = yield* projects.getById(thread.projectId).pipe( Effect.map(Option.getOrUndefined), Effect.orElseSucceed(() => undefined), @@ -122,7 +182,9 @@ export const makeRestoreMissingWorktree = Effect.gen(function* () { worktreePath, branch, }); - yield* recreateMissingWorktree({ cwd: project.workspaceRoot, worktreePath, branch }).pipe( + // `git worktree add` refuses a non-empty target. + yield* (leftover ? fileSystem.remove(worktreePath, { recursive: true }) : Effect.void).pipe( + Effect.andThen(recreateMissingWorktree({ cwd: project.workspaceRoot, worktreePath, branch })), Effect.mapError((cause) => cause._tag === "WorktreeRecreateNoBaseError" ? cause @@ -132,6 +194,31 @@ export const makeRestoreMissingWorktree = Effect.gen(function* () { }); }); +export class ThreadWorktreeRestore extends Context.Service< + ThreadWorktreeRestore, + { + readonly restore: ( + thread: WorktreeThread, + ) => Effect.Effect; + } +>()("t3/workspace/restoreMissingWorktree/ThreadWorktreeRestore") {} + +// Turn start restores under the same lease, so concurrent restores see each other's worktree. +export const layerThreadWorktreeRestore = Layer.effect( + ThreadWorktreeRestore, + Effect.gen(function* () { + const path = yield* Path.Path; + const restore = yield* makeRestoreMissingWorktree; + return { + restore: Effect.fn("ThreadWorktreeRestore.restore")(function* (thread: WorktreeThread) { + return yield* thread.worktreePath === null + ? restore(thread) + : withWorkspaceLease(path.resolve(thread.worktreePath), restore(thread)); + }), + }; + }), +); + // Worktree setup uses terminals, so recovery must stay outside TerminalManager. export const makePrepareTerminalLaunch = Effect.fn( "restoreMissingWorktree.makePrepareTerminalLaunch", diff --git a/apps/server/src/workspace/workspaceLease.ts b/apps/server/src/workspace/workspaceLease.ts index 8ad6e41b859c8..90e6124bb1865 100644 --- a/apps/server/src/workspace/workspaceLease.ts +++ b/apps/server/src/workspace/workspaceLease.ts @@ -1,4 +1,7 @@ +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import type * as FileSystem from "effect/FileSystem"; +import type * as Path from "effect/Path"; import * as Semaphore from "effect/Semaphore"; const leases = new Map(); @@ -20,3 +23,92 @@ export const withWorkspaceLease = ( ), ); }); + +// A restore rewrites files under its directory, so it must exclude provider +// startup in any overlapping workspace: symlink aliases, parents, and children. +// Kept apart from withWorkspaceLease because that lease is non-reentrant and +// restores call Git work that leases the project root. +interface Reservation { + readonly cwd: string; + readonly kind: "restore" | "turn-start"; +} + +const reservations = new Set(); +let reservationReleased = Deferred.makeUnsafe(); + +// path.relative folds win32 drive-letter case and treats a POSIX backslash as +// part of a name, unlike a raw prefix check. +const contains = (path: Path.Path, parent: string, child: string) => { + const relative = path.relative(parent, child); + return ( + relative === "" || + (!path.isAbsolute(relative) && relative !== ".." && !relative.startsWith(`..${path.sep}`)) + ); +}; + +const conflicts = (path: Path.Path, a: Reservation, b: Reservation) => + (a.kind === "restore" || b.kind === "restore") && + (contains(path, a.cwd, b.cwd) || contains(path, b.cwd, a.cwd)); + +const reserve = (path: Path.Path, reservation: Reservation): Effect.Effect => + Effect.suspend(() => { + for (const held of reservations) { + if (conflicts(path, held, reservation)) { + return Deferred.await(reservationReleased).pipe(Effect.andThen(reserve(path, reservation))); + } + } + reservations.add(reservation); + return Effect.void; + }); + +const release = (reservation: Reservation) => + Effect.sync(() => { + if (!reservations.delete(reservation)) return; + const released = reservationReleased; + reservationReleased = Deferred.makeUnsafe(); + Deferred.doneUnsafe(released, Effect.void); + }); + +// A worktree removed by storage cleanup is recreated inside the reserved +// section, so resolve its nearest existing ancestor: a symlink alias above it +// must still canonicalize the same on both sides. Walks at most the path depth. +const canonicalize = ( + fileSystem: FileSystem.FileSystem, + path: Path.Path, + target: string, +): Effect.Effect => + fileSystem.realPath(target).pipe( + Effect.catchReason("PlatformError", "NotFound", () => { + const parent = path.dirname(target); + return parent === target + ? Effect.succeed(target) + : canonicalize(fileSystem, path, parent).pipe( + Effect.map((resolved) => path.join(resolved, path.basename(target))), + ); + }), + Effect.orElseSucceed(() => target), + ); + +const withReservation = + (kind: Reservation["kind"]) => + ( + fileSystem: FileSystem.FileSystem, + path: Path.Path, + cwd: string, + effect: Effect.Effect, + ): Effect.Effect => + canonicalize(fileSystem, path, cwd).pipe( + Effect.flatMap((canonical) => { + const reservation: Reservation = { cwd: canonical, kind }; + return reserve(path, reservation).pipe( + Effect.andThen(effect), + Effect.ensuring(release(reservation)), + ); + }), + ); + +/** Exclusive over every overlapping workspace while checkpoint files are restored. */ +export const withCheckpointRestoreReservation = withReservation("restore"); + +/** Waits out overlapping restores; turn startups never wait on each other. */ +export const withTurnStartReservation = withReservation("turn-start"); diff --git a/apps/server/src/workspace/wslDrivePath.ts b/apps/server/src/workspace/wslDrivePath.ts new file mode 100644 index 0000000000000..188e1c85f45ec --- /dev/null +++ b/apps/server/src/workspace/wslDrivePath.ts @@ -0,0 +1,29 @@ +import * as HostProcess from "@t3tools/shared/HostProcess"; +import { isWindowsDrivePath } from "@t3tools/shared/path"; +import * as Effect from "effect/Effect"; +import * as ChildProcess from "effect/process/ChildProcess"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; + +// WSL mounts Windows drives under a configurable automount root, so only wslpath knows where. +// No WSL env check: servers started by systemd lack WSL_DISTRO_NAME, and off WSL the spawn just fails. +// Untraced: it runs for every chip path, and only drive paths on Linux do any work. +export const toWslDrivePath = Effect.fnUntraced(function* (requestedPath: string) { + // spawn throws on a NUL byte instead of failing. + if (!isWindowsDrivePath(requestedPath) || requestedPath.includes("\0")) return requestedPath; + if ((yield* HostProcess.Platform) !== "linux") return requestedPath; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const output = yield* spawner + .string( + ChildProcess.make("wslpath", ["-u", requestedPath], { + env: yield* HostProcess.Environment, + stdin: "ignore", + stderr: "ignore", + }), + ) + .pipe( + Effect.timeout("1 second"), + Effect.orElseSucceed(() => ""), + ); + const linuxPath = output.endsWith("\n") ? output.slice(0, -1) : output; + return linuxPath.startsWith("/") ? linuxPath : requestedPath; +}); diff --git a/apps/server/src/ws.test.ts b/apps/server/src/ws.test.ts index c36b935741e7d..660548c1dd68b 100644 --- a/apps/server/src/ws.test.ts +++ b/apps/server/src/ws.test.ts @@ -237,7 +237,7 @@ it.effect("resends a reveal kind and remote targets that outlast the snapshot ti assert.deepEqual(remote.config.remoteOpenTargets, targets); assert.equal(remote.config.shellRevealInFileManagerKind, "file-explorer"); } - }).pipe(Effect.scoped), + }), ); it.effect("runs editor, remote open target and direct endpoint discovery side by side", () => @@ -276,25 +276,23 @@ const ReplayTestLayer = OrchestrationEventStore.layer.pipe( Layer.provideMerge(SqlitePersistence.layerMemory), ); -it.live.each(["thread", "shell"] as const)( - "%s resume skips unknown types, synchronizes, and receives the live tail", - (scope) => - Effect.gen(function* () { - const store = yield* OrchestrationEventStore.OrchestrationEventStore; - const sql = yield* SqlClient.SqlClient; - const now = "2026-01-05T00:00:00.000Z"; - const threadId = ThreadId.make("thread:replay"); - const projectId = ProjectId.make("project:replay"); - const project = { - id: projectId, - title: "Replay", - workspaceRoot: "/tmp/replay", - defaultModelSelection: null, - scripts: [], - createdAt: now, - updatedAt: now, - } satisfies OrchestrationProjectShell; - const insert = (version: number, unknown = false) => sql<{ sequence: number }>` +const resumeSkipsUnknownTypes = (scope: "thread" | "shell") => + Effect.gen(function* () { + const store = yield* OrchestrationEventStore.OrchestrationEventStore; + const sql = yield* SqlClient.SqlClient; + const now = "2026-01-05T00:00:00.000Z"; + const threadId = ThreadId.make("thread:replay"); + const projectId = ProjectId.make("project:replay"); + const project = { + id: projectId, + title: "Replay", + workspaceRoot: "/tmp/replay", + defaultModelSelection: null, + scripts: [], + createdAt: now, + updatedAt: now, + } satisfies OrchestrationProjectShell; + const insert = (version: number, unknown = false) => sql<{ sequence: number }>` INSERT INTO orchestration_events ( event_id, aggregate_kind, stream_id, stream_version, event_type, occurred_at, actor_kind, payload_json, metadata_json, application_event_version @@ -309,85 +307,91 @@ it.live.each(["thread", "shell"] as const)( )}, '{}', 2 ) RETURNING sequence `; - const [before] = yield* insert(0); - yield* insert(1, true); - const [after] = yield* insert(2); - const [highWater] = yield* insert(3, true); - const updates = yield* PubSub.unbounded(); - let liveSequence = 0; - const consume = ( - stream: Stream.Stream, - ) => - stream.pipe( - Stream.tap((item) => - Effect.gen(function* () { - if (item.kind !== "synchronized") return; - const [live] = yield* insert(4); - liveSequence = live!.sequence; - const events = yield* store - .readApplicationEvents({ - afterSequence: highWater!.sequence, - throughSequence: liveSequence, - }) - .pipe(Stream.runCollect); - yield* store.publishCommitted(events); - }), - ), - Stream.takeUntil((item) => item.sequence === liveSequence), - Stream.map((item) => ({ kind: item.kind as string, sequence: item.sequence })), - Stream.runCollect, - ); - const subscription = Effect.gen(function* () { - return scope === "thread" - ? yield* subscribeOrchestrationV2Thread({ - threadId, - afterSequence: before!.sequence, - requestCompletionMarker: true, - }).pipe(Effect.flatMap(consume)) - : yield* subscribeOrchestrationV2Shell({ - afterSequence: before!.sequence, - requestCompletionMarker: true, - }).pipe(Effect.flatMap(consume)); - }); - const items = yield* subscription.pipe( - Effect.provideService(ThreadManagement.ThreadManagementService, { - ensureLegacyTranscript: () => Effect.void, - getShellSnapshot: () => Effect.die("shell resume must not load a full snapshot"), - listActiveSubagentThreadIds: () => Effect.succeed([]), - streamStoredEventsFrom: ( - input: Parameters< - ThreadManagement.ThreadManagementService["Service"]["streamStoredEventsFrom"] - >[0], - ) => { - assert.equal(input?.afterSequence, highWater!.sequence); - return store - .streamApplicationEvents({ afterSequence: input?.afterSequence ?? 0 }) - .pipe( - Stream.filter((event): event is OrchestrationV2StoredEvent => "event" in event), - ); - }, - } as unknown as ThreadManagement.ThreadManagementService["Service"]), - Effect.provideService(ProjectStore.ProjectStoreV2, { - listShells: () => Effect.succeed([]), - } as unknown as ProjectStore.ProjectStoreV2["Service"]), - Effect.provideService(ProjectService.ProjectService, { - getShell: () => Effect.succeedSome(project), - } as unknown as ProjectService.ProjectService["Service"]), - Effect.provideService(ProjectEnrichment.ProjectEnrichmentService, { - subscribeChanges: PubSub.subscribe(updates), - } as ProjectEnrichment.ProjectEnrichmentService["Service"]), - ); - assert.deepEqual( - items.map((item) => item.kind), - scope === "thread" - ? ["event", "synchronized", "event"] - : ["project.updated", "synchronized", "project.updated"], - ); - assert.deepEqual( - items.flatMap((item) => (item.sequence === undefined ? [] : [item.sequence])), - [after!.sequence, liveSequence], + const [before] = yield* insert(0); + yield* insert(1, true); + const [after] = yield* insert(2); + const [highWater] = yield* insert(3, true); + const updates = yield* PubSub.unbounded(); + let liveSequence = 0; + const consume = ( + stream: Stream.Stream, + ) => + stream.pipe( + Stream.tap((item) => + Effect.gen(function* () { + if (item.kind !== "synchronized") return; + const [live] = yield* insert(4); + liveSequence = live!.sequence; + const events = yield* store + .readApplicationEvents({ + afterSequence: highWater!.sequence, + throughSequence: liveSequence, + }) + .pipe(Stream.runCollect); + yield* store.publishCommitted(events); + }), + ), + Stream.takeUntil((item) => item.sequence === liveSequence), + Stream.map((item) => ({ kind: item.kind as string, sequence: item.sequence })), + Stream.runCollect, ); - }).pipe(Effect.scoped, Effect.provide(ReplayTestLayer)), + const subscription = Effect.gen(function* () { + return scope === "thread" + ? yield* subscribeOrchestrationV2Thread({ + threadId, + afterSequence: before!.sequence, + requestCompletionMarker: true, + }).pipe(Effect.flatMap(consume)) + : yield* subscribeOrchestrationV2Shell({ + afterSequence: before!.sequence, + requestCompletionMarker: true, + }).pipe(Effect.flatMap(consume)); + }); + const items = yield* subscription.pipe( + Effect.provideService(ThreadManagement.ThreadManagementService, { + ensureLegacyTranscript: () => Effect.void, + getShellSnapshot: () => Effect.die("shell resume must not load a full snapshot"), + listActiveSubagentThreadIds: () => Effect.succeed([]), + streamStoredEventsFrom: ( + input: Parameters< + ThreadManagement.ThreadManagementService["Service"]["streamStoredEventsFrom"] + >[0], + ) => { + assert.equal(input?.afterSequence, highWater!.sequence); + return store + .streamApplicationEvents({ afterSequence: input?.afterSequence ?? 0 }) + .pipe(Stream.filter((event): event is OrchestrationV2StoredEvent => "event" in event)); + }, + } as unknown as ThreadManagement.ThreadManagementService["Service"]), + Effect.provideService(ProjectStore.ProjectStoreV2, { + listShells: () => Effect.succeed([]), + } as unknown as ProjectStore.ProjectStoreV2["Service"]), + Effect.provideService(ProjectService.ProjectService, { + getShell: () => Effect.succeedSome(project), + } as unknown as ProjectService.ProjectService["Service"]), + Effect.provideService(ProjectEnrichment.ProjectEnrichmentService, { + subscribeChanges: PubSub.subscribe(updates), + } as ProjectEnrichment.ProjectEnrichmentService["Service"]), + ); + assert.deepEqual( + items.map((item) => item.kind), + scope === "thread" + ? ["event", "synchronized", "event"] + : ["project.updated", "synchronized", "project.updated"], + ); + assert.deepEqual( + items.flatMap((item) => (item.sequence === undefined ? [] : [item.sequence])), + [after!.sequence, liveSequence], + ); + }).pipe(Effect.scoped, Effect.provide(ReplayTestLayer)); + +it.effect("thread resume skips unknown types, synchronizes, and receives the live tail", () => + resumeSkipsUnknownTypes("thread"), +); + +// Shell updates batch through `Stream.groupedWithin`, whose window needs real time to close. +it.live("shell resume skips unknown types, synchronizes, and receives the live tail", () => + resumeSkipsUnknownTypes("shell"), ); it.effect("coalesces streaming replay before synchronization and the live tail", () => @@ -1140,7 +1144,6 @@ it.effect.each([ Layer.mock(ServerBrowser.ServerBrowser)({}), ), ), - Effect.scoped, ), ); @@ -1200,6 +1203,7 @@ it.effect("access changes made while the snapshot loads reach the client after i Layer.mock(ServerEnvironment.ServerEnvironment)({}), Layer.mock(EnvironmentAuth.EnvironmentAuth)({ listPairingLinks: () => Effect.succeed([]), + listConnectDevices: () => Effect.succeed([]), // Another client revokes a session after the snapshot read it. listClientSessions: () => PubSub.publish(sessionChanges, { @@ -1243,6 +1247,18 @@ it.effect("access changes made while the snapshot loads reach the client after i type: "clientRemoved", payload: { sessionId: revokedElsewhere }, }); + yield* server.write(0, { _tag: "Ack", requestId }); + assert.equal((yield* Queue.take(responses)).type, "pairingLinkRemoved"); + yield* server.write(0, { _tag: "Ack", requestId }); + + // Device trust has no incremental event, so a change re-sends the whole snapshot. + yield* PubSub.publish(pairingChanges, { type: "connectDevicesChanged" }); + assert.deepEqual(yield* Queue.take(responses), { + version: 1, + revision: 4, + type: "snapshot", + payload: { pairingLinks: [], clientSessions: [], connectDevices: [] }, + }); yield* server.write(0, { _tag: "Interrupt", requestId, interruptors: [] }); }).pipe( Effect.provide( @@ -1252,7 +1268,6 @@ it.effect("access changes made while the snapshot loads reach the client after i Layer.mock(ServerBrowser.ServerBrowser)({}), ), ), - Effect.scoped, ), ); @@ -1607,7 +1622,6 @@ it.effect("on-demand clients get provider config without workspace snapshots", ( Layer.mock(ServerBrowser.ServerBrowser)({}), ), ), - Effect.scoped, ), ); @@ -1779,7 +1793,6 @@ it.effect("a missing thread worktree gets its project's skills in the cwd snapsh Layer.mock(ServerBrowser.ServerBrowser)({}), ), ), - Effect.scoped, ), ); @@ -1853,7 +1866,6 @@ it.effect("a git action that fails partway still refreshes the repository status Layer.mock(ServerBrowser.ServerBrowser)({}), ), ), - Effect.scoped, ), ); @@ -1941,7 +1953,6 @@ it.effect("a publish whose push fails still refreshes the new remote", () => Layer.mock(ServerBrowser.ServerBrowser)({}), ), ), - Effect.scoped, ), ); @@ -2445,6 +2456,5 @@ it.effect("ACP session RPCs list a thread missing its history and repair it on i Layer.mock(ServerBrowser.ServerBrowser)({}), ), ), - Effect.scoped, ), ); diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 9b26fc3e09c31..09c6d86c050fd 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -727,7 +727,10 @@ const serverWsRpcs = Array.from(ServerWsRpcGroup.requests.values()); const SplitWsRpcGroup = RpcGroup.make(...serverWsRpcs.filter(isSplitWsRpc)); const OtherWsRpcGroup = RpcGroup.make(...serverWsRpcs.filter(isOtherWsRpc)); function toAuthAccessStreamEvent( - change: PairingGrantStore.BootstrapCredentialChange | SessionStore.SessionCredentialChange, + change: Exclude< + PairingGrantStore.BootstrapCredentialChange | SessionStore.SessionCredentialChange, + { readonly type: "connectDevicesChanged" } + >, revision: number, currentSessionId: AuthSessionId, ): AuthAccessStreamEvent { @@ -1780,6 +1783,7 @@ export const layerWsRpc = ( Effect.all({ pairingLinks: serverAuth.listPairingLinks(), clientSessions: serverAuth.listClientSessions(currentSessionId), + connectDevices: serverAuth.listConnectDevices(), }).pipe( Effect.mapError( (error) => @@ -1885,6 +1889,7 @@ export const layerWsRpc = ( }), newProjectsRoot: managedFolders.namedProjectsRoot, ...(lastEventLoopStall === undefined ? {} : { lastEventLoopStall }), + serverTime: DateTime.formatIso(yield* DateTime.now), }; }); @@ -2652,6 +2657,10 @@ export const layerWsRpc = ( Effect.annotateCurrentSpan({ "orchestration_v2.thread_id": input.threadId }).pipe( Effect.andThen(peerLinkRequests.answer(input)), ), + [WS_METHODS.serverGetSettings]: (_input) => + Effect.map(serverSettings.getSettings, (settings) => + ServerSettings.redactServerSettingsForClient(settings, settingsClient), + ), [WS_METHODS.serverUpdateSettings]: /* Request instrumentation already owns the span; keep this helper in that context. */ Effect.fnUntraced(function* ({ patch, providerInstanceMutation }) { @@ -2807,6 +2816,8 @@ export const layerWsRpc = ( ), ), [WS_METHODS.pullRequestsSubscribeRefreshes]: () => pullRequests.subscribeRefreshes, + [WS_METHODS.pullRequestsSubscribeLinkedThreadRefreshes]: () => + pullRequests.subscribeLinkedThreadRefreshes, [WS_METHODS.pullRequestsReviewerCandidates]: (input) => withPullRequestViewer(input, pullRequests.reviewerCandidates(input)), [WS_METHODS.pullRequestsRequestReviewers]: (input) => @@ -2977,7 +2988,7 @@ export const layerWsRpc = ( new ProjectMutationError({ commandId: mutation.commandId, message: - cause._tag === "ProjectNotEmptyError" + cause._tag === "ProjectNotEmptyError" || cause._tag === "ProjectInvalidError" ? cause.message : "Failed to mutate project.", cause, @@ -3155,11 +3166,29 @@ export const layerWsRpc = ( [WS_METHODS.vcsCreateWorktree]: (input) => gitWorkflow.createWorktree(input).pipe(Effect.tap(() => refreshGitStatus(input.cwd))), [WS_METHODS.vcsRemoveWorktree]: (input) => - gitWorkflow.removeWorktree(input).pipe(Effect.tap(() => refreshGitStatus(input.cwd))), + gitWorkflow.removeWorktree(input).pipe( + Effect.tap(() => refreshGitStatus(input.cwd)), + // Subscribers of the removed checkout learn it is no longer a repository. + Effect.tap(() => + vcsStatusBroadcaster + .refreshLocalStatus(input.path) + .pipe(Effect.ignoreCause({ log: true }), Effect.forkDetach, Effect.asVoid), + ), + ), [WS_METHODS.vcsCreateRef]: (input) => - gitWorkflow.createRef(input).pipe(Effect.tap(() => refreshGitStatus(input.cwd))), + gitWorkflow + .createRef(input) + .pipe( + Effect.tap(() => + input.switchRef + ? vcsStatusBroadcaster.refreshAfterCheckout(input.cwd) + : refreshGitStatus(input.cwd), + ), + ), [WS_METHODS.vcsSwitchRef]: (input) => - gitWorkflow.switchRef(input).pipe(Effect.tap(() => refreshGitStatus(input.cwd))), + gitWorkflow + .switchRef(input) + .pipe(Effect.tap(() => vcsStatusBroadcaster.refreshAfterCheckout(input.cwd))), [WS_METHODS.vcsInit]: (input) => vcsProvisioning.initRepository(input).pipe(Effect.tap(() => refreshGitStatus(input.cwd))), [WS_METHODS.reviewGetBranchCommits]: (input) => review.getBranchCommits(input), @@ -3462,17 +3491,28 @@ export const layerWsRpc = ( const initialSnapshot = yield* loadAuthAccessSnapshot(); const revisionRef = yield* Ref.make(1); - const liveEvents: Stream.Stream = Stream.fromQueue( - liveBuffer, - ).pipe( - Stream.mapEffect((change) => - Ref.updateAndGet(revisionRef, (revision) => revision + 1).pipe( - Effect.map((revision) => - toAuthAccessStreamEvent(change, revision, currentSessionId), + const liveEvents: Stream.Stream = + Stream.fromQueue(liveBuffer).pipe( + Stream.mapEffect((change) => + Ref.updateAndGet(revisionRef, (revision) => revision + 1).pipe( + Effect.flatMap((revision) => + // Device trust has no incremental event older clients could decode. + change.type === "connectDevicesChanged" + ? loadAuthAccessSnapshot().pipe( + Effect.map((payload): AuthAccessStreamEvent => ({ + version: 1, + revision, + type: "snapshot", + payload, + })), + ) + : Effect.succeed( + toAuthAccessStreamEvent(change, revision, currentSessionId), + ), + ), ), ), - ), - ); + ); return Stream.concat( rpcInitialItems([ diff --git a/apps/server/vite.config.ts b/apps/server/vite.config.ts index 3291aa304d395..0ec539e5ae169 100644 --- a/apps/server/vite.config.ts +++ b/apps/server/vite.config.ts @@ -5,6 +5,7 @@ import { defineConfig, mergeConfig } from "vite-plus"; import baseConfig from "../../vite.config.ts"; import { loadRepoEnv } from "../../scripts/lib/public-config.ts"; +import { resolveTaggedSourceVersion } from "../../scripts/lib/tagged-source-version.ts"; import rootPackageJson from "../../package.json" with { type: "json" }; import packageJson from "./package.json" with { type: "json" }; import { WeightedShardSequencer } from "./src/testUtils/weightedShardSequencer.ts"; @@ -18,9 +19,12 @@ import { export { shouldBundleCliDependency }; const repoEnv = loadRepoEnv(); -const cliBuildChannel = /^[^-+]+-(?:nightly|preview)\./.test(packageJson.version) - ? "nightly" - : "latest"; +const serverManifestPath = NodePath.join(import.meta.dirname, "package.json"); +// The desktop packager passes APP_VERSION; it can equal the manifest at a commit tagged otherwise. +const buildVersion = + process.env.APP_VERSION?.trim() || + resolveTaggedSourceVersion(NodePath.join(import.meta.dirname, "../.."), packageJson.version); +const cliBuildChannel = /^[^-+]+-(?:nightly|preview)\./.test(buildVersion) ? "nightly" : "latest"; // Official release workflows set the telemetry channel; local builds and forks use "dev". const TELEMETRY_CHANNELS = ["stable", "nightly", "preview", "dev"]; const telemetryChannel = process.env.T3CODE_TELEMETRY_CHANNEL?.trim() || "dev"; @@ -88,6 +92,19 @@ export default mergeConfig( }, }, pack: { + plugins: [ + { + // Every server module reads its version from this manifest import. + name: "t3code:build-version", + load(id) { + if (NodePath.resolve(id) !== serverManifestPath) return; + return { + code: JSON.stringify({ ...packageJson, version: buildVersion }), + moduleType: "json", + }; + }, + }, + ], // SEA embeds one entry; expose the history worker as a hidden subcommand. entry: packBootstrap ? ["src/bin.ts"] diff --git a/apps/web/index.html b/apps/web/index.html index 1a8bc02e19c9d..051e3090a5d48 100644 --- a/apps/web/index.html +++ b/apps/web/index.html @@ -15,6 +15,7 @@ const LIGHT_BACKGROUND = "#ffffff"; const DARK_BACKGROUND = "#000000"; const CUSTOM_THEMES_STORAGE_KEY = "t3code:themes:v1"; + const ENVIRONMENT_THEMES_STORAGE_KEY = "t3code:environment-themes:v1"; const THEME_FOLLOW_SYSTEM_STORAGE_KEY = "t3code:theme-follow-system"; const THEME_APPEARANCE_MODE_STORAGE_KEY = "t3code:theme-appearance-mode"; const THEME_HALVES_STORAGE_KEY = "t3code:theme-halves:v1"; @@ -197,12 +198,9 @@ (appearance === value.appearance || isRecord(colors)), ); }; - const findCustomTheme = (themeId) => { - if (RESERVED_THEME_IDS.includes(themeId)) return null; + const findStoredTheme = (storageKey, themeId) => { try { - const parsed = JSON.parse( - window.localStorage.getItem(CUSTOM_THEMES_STORAGE_KEY) ?? "null", - ); + const parsed = JSON.parse(window.localStorage.getItem(storageKey) ?? "null"); if (!Array.isArray(parsed)) return null; return ( parsed.find((value) => isStoredCustomTheme(value) && value.id === themeId) ?? null @@ -211,6 +209,14 @@ return null; } }; + // Same precedence as getThemeDefinition: saved custom themes, then published ones. + const findCustomTheme = (themeId) => { + if (RESERVED_THEME_IDS.includes(themeId)) return null; + return ( + findStoredTheme(CUSTOM_THEMES_STORAGE_KEY, themeId) ?? + findStoredTheme(ENVIRONMENT_THEMES_STORAGE_KEY, themeId) + ); + }; try { const storedTheme = window.localStorage.getItem("t3code:theme"); diff --git a/apps/web/src/authBootstrap.test.ts b/apps/web/src/authBootstrap.test.ts index 65c6f8edd3424..f444c64cb16ad 100644 --- a/apps/web/src/authBootstrap.test.ts +++ b/apps/web/src/authBootstrap.test.ts @@ -8,6 +8,7 @@ import { import { createBrowserHistory } from "@tanstack/react-router"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; +import { HttpClientError, HttpClientRequest } from "effect/http"; import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; import { installEnvironmentHttpTest } from "../test/environmentHttpTest"; @@ -165,6 +166,54 @@ describe("resolveInitialServerAuthGateState", () => { expect(testApi.calls.browserSession).toEqual([{ credential: "desktop-bootstrap-token" }]); }); + it("retries the desktop bootstrap while the backend is still unreachable", async () => { + const nextSession = sequence( + unauthenticatedSession(DESKTOP_AUTH), + authenticatedSession(DESKTOP_AUTH), + ); + const testApi = await installAuthApi({ + session: nextSession, + browserSession: () => Effect.succeed(browserSession(["orchestration:read", "access:write"])), + }); + // What FetchHttpClient yields when nothing listens on the backend port. + const connectionRefused = new HttpClientError.HttpClientError({ + reason: new HttpClientError.TransportError({ + request: HttpClientRequest.get("http://localhost:3773/api/auth/session"), + cause: new TypeError("Failed to fetch"), + }), + }); + let unreachableAttempts = 1; + __setPrimaryHttpRunnerForTests((effect) => { + if (unreachableAttempts > 0) { + unreachableAttempts -= 1; + return Promise.reject(connectionRefused); + } + return testApi.run(effect); + }); + + const testWindow = installTestBrowser("http://localhost/"); + testWindow.desktopBridge = { + getLocalEnvironmentBootstraps: () => [ + { + id: "primary", + label: "Local", + httpBaseUrl: "http://localhost:3773", + wsBaseUrl: "ws://localhost:3773", + bootstrapToken: "desktop-bootstrap-token", + }, + ], + } as unknown as DesktopBridge; + + const { resolveInitialServerAuthGateState } = await import("./environments/primary"); + + vi.useFakeTimers({ toFake: ["setTimeout", "Date"] }); + await Promise.all([ + expect(resolveInitialServerAuthGateState()).resolves.toEqual({ status: "authenticated" }), + vi.advanceTimersByTimeAsync(1_000), + ]); + expect(testApi.calls.browserSession).toEqual([{ credential: "desktop-bootstrap-token" }]); + }); + it("takes a pairing token from the location hash and strips it immediately", async () => { const testWindow = installTestBrowser("http://localhost/#token=pairing-token"); const { takePairingTokenFromUrl } = await import("./environments/primary"); diff --git a/apps/web/src/browser/HostedBrowserWebview.tsx b/apps/web/src/browser/HostedBrowserWebview.tsx index 44de1196b2b7c..023fb03c3aa17 100644 --- a/apps/web/src/browser/HostedBrowserWebview.tsx +++ b/apps/web/src/browser/HostedBrowserWebview.tsx @@ -20,6 +20,7 @@ import { useActiveBrowserRecordingTabIds } from "./browserRecording"; import { browserViewportSettingKey, resolveBrowserViewportLayout, + resolveBrowserViewportTransform, resolveFittedBrowserViewport, } from "./browserViewportLayout"; import { BrowserDeviceToolbar } from "./BrowserDeviceToolbar"; @@ -184,16 +185,24 @@ export function HostedBrowserWebview(props: { if (!disposed) remountCrashedWebview(); }, recovery.delayMs); }; - // Guest clicks only emit focus here; replay pointerdown to dismiss host popups. + // Guest clicks never reach this document, so replay a pointerdown to dismiss host popups. + // The guest preload reports each press, since the guest does not always take focus. const dismissHostPopups = () => { webview.dispatchEvent( new PointerEvent("pointerdown", { bubbles: true, pointerType: "mouse" }), ); }; + const guestPointerDown = (event: Event) => { + // Mirrors POINTER_DOWN_HOST_CHANNEL in apps/desktop/src/preview/GuestProtocol.ts. + if ((event as Event & { channel?: string }).channel === "preview:pointer-down") { + dismissHostPopups(); + } + }; webview.addEventListener("did-attach", register); webview.addEventListener("dom-ready", register); webview.addEventListener("render-process-gone", recoverGuest); webview.addEventListener("focus", dismissHostPopups); + webview.addEventListener("ipc-message", guestPointerDown); register(); return () => { disposed = true; @@ -202,6 +211,7 @@ export function HostedBrowserWebview(props: { webview.removeEventListener("dom-ready", register); webview.removeEventListener("render-process-gone", recoverGuest); webview.removeEventListener("focus", dismissHostPopups); + webview.removeEventListener("ipc-message", guestPointerDown); }; }, [ clientSettingsHydrated, @@ -304,6 +314,23 @@ export function HostedBrowserWebview(props: { return () => withDesktopTab(runtimeTabId, () => bridge.setShown(runtimeTabId, false)); }, [clientSettingsHydrated, runtimeTabId, serverDriven, shown]); + // Captures copy the guest's raster, which a scaled guest only keeps at full detail on request. + const fullDetail = backgroundActivity || pictureInPicture || recordingActive; + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- tells the desktop host once the guest is drawn at full detail, so its screenshot waits for that frame + useEffect(() => { + const bridge = window.desktopBridge?.preview; + if (!clientSettingsHydrated || !bridge || !serverDriven || !fullDetail) return; + let frame = requestAnimationFrame(() => { + frame = requestAnimationFrame(() => + withDesktopTab(runtimeTabId, () => bridge.setCaptureDrawn(runtimeTabId, true)), + ); + }); + return () => { + cancelAnimationFrame(frame); + withDesktopTab(runtimeTabId, () => bridge.setCaptureDrawn(runtimeTabId, false)); + }; + }, [clientSettingsHydrated, fullDetail, runtimeTabId, serverDriven]); + if (!clientSettingsHydrated || !config) return null; // This fixed wrapper sits outside the right panel shell, so it takes thumb presses itself. @@ -313,6 +340,7 @@ export function HostedBrowserWebview(props: { }; const renderingActive = active || backgroundActivity || pictureInPicture || recordingActive; + const viewportTransform = resolveBrowserViewportTransform(layout.viewportScale, fullDetail); const wrapperStyle = resolveHostedBrowserWebviewWrapperStyle({ active, renderingActive, @@ -345,47 +373,56 @@ export function HostedBrowserWebview(props: { onChange={commitViewportChange} /> ) : null} - + > + + {active && effectiveViewport._tag !== "fill" && !fittedSourceViewport ? ( <> void; readonly history: (delta: -1 | 1) => void; readonly reload: (options?: { readonly ignoreCache?: boolean }) => void; + readonly stopLoading: () => void; readonly viewport: (setting: PreviewViewportSetting) => void; readonly canvas: () => HTMLCanvasElement | null; } @@ -311,6 +312,7 @@ export function ServerBrowserSurface(props: { history: (delta) => send({ type: "history", delta }), reload: (options) => send({ type: "reload", ...(options?.ignoreCache ? { ignoreCache: true } : {}) }), + stopLoading: () => send({ type: "stopLoading" }), viewport: (setting) => send({ type: "viewport", setting }), canvas: () => (!pdfUrl && hasFrameRef.current ? canvasRef.current : null), }), diff --git a/apps/web/src/browser/browserDefaults.ts b/apps/web/src/browser/browserDefaults.ts index bf1c2da40db87..5cc74bb1b7f0d 100644 --- a/apps/web/src/browser/browserDefaults.ts +++ b/apps/web/src/browser/browserDefaults.ts @@ -5,6 +5,7 @@ import { type DesktopPreviewTabDefaults, type PreviewAppearancePreference, type PreviewViewportSetting, + type PreviewZoomFactor, } from "@t3tools/contracts"; import { @@ -17,7 +18,7 @@ import { resolveResponsiveBrowserViewportSize } from "./browserViewportLayout"; export interface BrowserDefaults { readonly viewport: PreviewViewportSetting; - readonly zoomFactor: number; + readonly zoomFactor: PreviewZoomFactor; readonly appearance: PreviewAppearancePreference; readonly autoShowFloatingPreview: boolean; readonly profiles: ReadonlyArray; @@ -26,7 +27,7 @@ export interface BrowserDefaults { const toBrowserDefaults = (settings: { readonly browserDefaultViewport: PreviewViewportSetting; - readonly browserDefaultZoomFactor: number; + readonly browserDefaultZoomFactor: PreviewZoomFactor; readonly browserDefaultAppearance: PreviewAppearancePreference; readonly browserAutoShowFloatingPreview: boolean; readonly browserProfiles: ReadonlyArray; diff --git a/apps/web/src/browser/browserLinkTarget.ts b/apps/web/src/browser/browserLinkTarget.ts index f1c4e65a99913..74330bf3edbd4 100644 --- a/apps/web/src/browser/browserLinkTarget.ts +++ b/apps/web/src/browser/browserLinkTarget.ts @@ -7,6 +7,9 @@ import { import { ensureClientSettingsHydrated, getClientSettings } from "~/hooks/useSettings"; import { readEnvironmentScope } from "~/state/session"; import { isPreviewAvailableFor } from "./previewRuntime"; +import { isWebUrl } from "./webUrl"; + +export { isWebUrl }; export interface ResolveLinkTargetInput { readonly url: string; @@ -27,15 +30,6 @@ export function resolveLinkTarget(input: ResolveLinkTargetInput): BrowserLinkTar return "app"; } -export function isWebUrl(url: string): boolean { - try { - const { protocol } = new URL(url); - return protocol === "http:" || protocol === "https:"; - } catch { - return false; - } -} - /** Before hydration, the default "system" would ignore a saved "app" link target. */ export async function resolveBrowserLinkTargetPreference(): Promise { await ensureClientSettingsHydrated(); diff --git a/apps/web/src/browser/browserViewportLayout.test.ts b/apps/web/src/browser/browserViewportLayout.test.ts index 1932a634fb6c1..c72f68e1e20da 100644 --- a/apps/web/src/browser/browserViewportLayout.test.ts +++ b/apps/web/src/browser/browserViewportLayout.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vite-plus/test"; import { resizeBrowserViewportFromRail, + resolveBrowserViewportTransform, resolveFittedBrowserViewport, } from "./browserViewportLayout"; @@ -43,3 +44,19 @@ describe("resolveBrowserViewportLayout", () => { ).toEqual({ width: 460, height: 409 }); }); }); + +describe("resolveBrowserViewportTransform", () => { + it("keeps a full-detail guest at raster scale 1 while it draws at the fitted scale", () => { + expect(resolveBrowserViewportTransform(1, true)).toBeUndefined(); + expect(resolveBrowserViewportTransform(0.6, false)).toBe("scale(0.6)"); + const values = resolveBrowserViewportTransform(0.6, true) + ?.match(/^matrix3d\((.*)\)$/)?.[1] + ?.split(",") + .map(Number); + // Column-major: the x and y columns keep their 0.6 screen terms and reach length 1 through z. + expect(values?.slice(0, 2)).toEqual([0.6, 0]); + expect(values?.slice(4, 6)).toEqual([0, 0.6]); + expect(Math.hypot(...(values?.slice(0, 3) ?? []))).toBeCloseTo(1); + expect(Math.hypot(...(values?.slice(4, 7) ?? []))).toBeCloseTo(1); + }); +}); diff --git a/apps/web/src/browser/browserViewportLayout.ts b/apps/web/src/browser/browserViewportLayout.ts index 2fe4e035100ab..7fe6b170c671d 100644 --- a/apps/web/src/browser/browserViewportLayout.ts +++ b/apps/web/src/browser/browserViewportLayout.ts @@ -125,6 +125,22 @@ export function resolveBrowserViewportLayout( }; } +/** + * Chromium rasters a CSS-scaled guest at its on-screen scale, so captures of it lose detail. + * Its raster scale also counts x/y-to-z terms, which a `preserve-3d` parent keeps and the + * screen drops, so `fullDetail` draws like `scale()` but keeps full-resolution frames. + */ +export function resolveBrowserViewportTransform( + scale: number, + fullDetail: boolean, +): string | undefined { + if (scale >= 1) return undefined; + if (!fullDetail) return `scale(${scale})`; + // Positive depth keeps the guest in front of its parent for hit testing. + const depth = Math.sqrt(1 - scale * scale); + return `matrix3d(${scale}, 0, ${depth}, 0, 0, ${scale}, ${depth}, 0, 0, 0, 1, 0, 0, 0, 0, 1)`; +} + export function resolveBrowserDeviceViewportLayout( container: { readonly width: number; readonly height: number }, setting: Exclude, diff --git a/apps/web/src/browser/openFileInPreview.ts b/apps/web/src/browser/openFileInPreview.ts index a997ee2b552e4..db1a58e98dfb9 100644 --- a/apps/web/src/browser/openFileInPreview.ts +++ b/apps/web/src/browser/openFileInPreview.ts @@ -85,7 +85,9 @@ export async function openUrlInPreview(input: { url: input.url, viewport: defaults.viewport, profileId: input.profileId ?? defaults.profileId, - ...(runtime === undefined ? {} : { runtime }), + ...(runtime === undefined + ? {} + : { runtime, zoomFactor: defaults.zoomFactor, colorScheme: defaults.appearance }), }, }); return mapAtomCommandResult(result, (snapshot) => { diff --git a/apps/web/src/browser/webUrl.ts b/apps/web/src/browser/webUrl.ts new file mode 100644 index 0000000000000..19e637794b74b --- /dev/null +++ b/apps/web/src/browser/webUrl.ts @@ -0,0 +1,8 @@ +export function isWebUrl(url: string): boolean { + try { + const { protocol } = new URL(url); + return protocol === "http:" || protocol === "https:"; + } catch { + return false; + } +} diff --git a/apps/web/src/cloud/t3ConnectState.ts b/apps/web/src/cloud/t3ConnectState.ts index c468c73d965ea..67fef2eaee1a6 100644 --- a/apps/web/src/cloud/t3ConnectState.ts +++ b/apps/web/src/cloud/t3ConnectState.ts @@ -3,6 +3,7 @@ import type { RelayManagedEndpoint, RelayManagedEndpointProviderKind, } from "@t3tools/contracts/relay"; +import { parseUrl } from "@t3tools/shared/url"; // Require managed discovery: stale tunnel config after a publish-only switch causes endpoint_provider_not_managed. export function isManagedTunnelActive( @@ -31,9 +32,9 @@ export function t3ConnectMcpUrl(input: { readonly relayEndpoint: RelayManagedEndpoint | undefined; }): string | null { const url = input.thisMachine - ? URL.parse(`https://${input.tunnelHostname ?? ""}`) + ? parseUrl(`https://${input.tunnelHostname ?? ""}`) : input.relayEndpoint?.providerKind === "cloudflare_tunnel" - ? URL.parse(input.relayEndpoint.httpBaseUrl) + ? parseUrl(input.relayEndpoint.httpBaseUrl) : null; return url?.protocol === "https:" && TUNNEL_HOSTNAME.test(url.hostname) ? `${url.origin}/mcp` diff --git a/apps/web/src/components/AddProjectToMachineDialog.tsx b/apps/web/src/components/AddProjectToMachineDialog.tsx index 3d9c064cfcb23..3bb70c8b609eb 100644 --- a/apps/web/src/components/AddProjectToMachineDialog.tsx +++ b/apps/web/src/components/AddProjectToMachineDialog.tsx @@ -26,6 +26,7 @@ import { useAddProjectTargets } from "../hooks/useAddProjectTargets"; import { useNewThreadHandler } from "../hooks/useHandleNewThread"; import { useLogicalProjectEnvironments } from "../hooks/useLogicalProjectEnvironments"; import { + COPIED_FILES_MAX_COUNT, COPYABLE_FILE_MAX_BYTES, addProjectToMachine, isDefaultCopiedFile, @@ -35,6 +36,7 @@ import { newProjectId } from "../lib/utils"; import { readProject, subscribeProject, useProject } from "../state/entities"; import { useEnvironments } from "../state/environments"; import { projectEnvironment } from "../state/projects"; +import { useDebouncedValue } from "../state/queries"; import { useEnvironmentQuery } from "../state/query"; import { EnvironmentMachineIcon } from "./EnvironmentMachineIcon"; import { Button } from "./ui/button"; @@ -121,8 +123,10 @@ function AddProjectToMachineDialog({ request }: { request: AddProjectToMachineRe const [pathOverride, setPathOverride] = useState(null); const destinationPath = pathOverride ?? defaultPath; - const canListFiles = source?.serverConfig?.environment.capabilities.ignoredFileListing === true; - const ignoredFiles = useEnvironmentQuery( + const capabilities = source?.serverConfig?.environment.capabilities; + const canListFiles = capabilities?.ignoredFileListing === true; + const canSearchFiles = canListFiles && capabilities?.ignoredFileSearch === true; + const allIgnoredFiles = useEnvironmentQuery( project && canListFiles ? projectEnvironment.listIgnoredFiles({ environmentId: project.environmentId, @@ -130,13 +134,29 @@ function AddProjectToMachineDialog({ request }: { request: AddProjectToMachineRe }) : null, ); + const [fileQuery, setFileQuery] = useState(""); + const sentFileQuery = useDebouncedValue(fileQuery.trim(), 120); + const searching = canSearchFiles && sentFileQuery.length > 0; + const searchedFiles = useEnvironmentQuery( + project && searching + ? projectEnvironment.listIgnoredFiles({ + environmentId: project.environmentId, + input: { cwd: project.workspaceRoot, query: sentFileQuery }, + }) + : null, + ); + const ignoredFiles = searching ? searchedFiles : allIgnoredFiles; const files = ignoredFiles.data?.files ?? []; - const [selected, setSelected] = useState | null>(null); - const selectedFiles = selected ?? defaultSelection(files); + // Keyed by path with the file kept, so picks made under one search survive the next. + const [selected, setSelected] = useState | null>(null); + const selectedFiles = selected ?? defaultSelection(allIgnoredFiles.data?.files ?? []); const copyable = files.filter((file) => file.byteLength <= COPYABLE_FILE_MAX_BYTES); - const selectedBytes = files - .filter((file) => selectedFiles.has(file.path)) - .reduce((total, file) => total + file.byteLength, 0); + const allCopyableSelected = copyable.every((file) => selectedFiles.has(file.path)); + const atSelectionLimit = selectedFiles.size >= COPIED_FILES_MAX_COUNT; + const selectedBytes = [...selectedFiles.values()].reduce( + (total, file) => total + file.byteLength, + 0, + ); const [submitting, setSubmitting] = useState(false); const [error, setError] = useState(null); @@ -146,11 +166,19 @@ function AddProjectToMachineDialog({ request }: { request: AddProjectToMachineRe const unavailable = project === null || target === null || - (canListFiles && ignoredFiles.data === null && ignoredFiles.error === null); - const toggleFile = (path: string, checked: boolean) => { - const next = new Set(selectedFiles); - if (checked) next.add(path); - else next.delete(path); + (canListFiles && allIgnoredFiles.data === null && allIgnoredFiles.error === null); + const toggleFile = (file: ProjectIgnoredFile, checked: boolean) => { + const next = new Map(selectedFiles); + if (!checked) next.delete(file.path); + else if (!atSelectionLimit) next.set(file.path, file); + setSelected(next); + }; + const toggleShownFiles = () => { + const next = new Map(selectedFiles); + for (const file of copyable) { + if (allCopyableSelected) next.delete(file.path); + else if (next.size < COPIED_FILES_MAX_COUNT) next.set(file.path, file); + } setSelected(next); }; @@ -198,7 +226,7 @@ function AddProjectToMachineDialog({ request }: { request: AddProjectToMachineRe title: project.title, remoteUrl, destinationPath: destinationPath.trim(), - files: files.filter((file) => selectedFiles.has(file.path)).map((file) => file.path), + files: [...selectedFiles.keys()], }); close(); openClonedProject(scopeProjectRef(target.environmentId, projectId)); @@ -334,33 +362,41 @@ function AddProjectToMachineDialog({ request }: { request: AddProjectToMachineRe {copyable.length > 0 ? ( <> - {selectedFiles.size} of {files.length} + {selectedFiles.size} of {allIgnoredFiles.data?.files.length ?? 0} {selectedBytes > 0 ? ` · ${formatBytes(selectedBytes)}` : ""} - ) : null} + {canSearchFiles && (allIgnoredFiles.data?.truncated || fileQuery.length > 0) ? ( + setFileQuery(event.target.value)} + // Enter would otherwise submit the clone. + onKeyDown={(event) => { + if (event.key === "Enter") event.preventDefault(); + }} + spellCheck={false} + autoComplete="off" + /> + ) : null} {ignoredFiles.error ? (

    {ignoredFiles.error}

    ) : files.length === 0 ? (

    - {ignoredFiles.isSuccess - ? "No ignored files to copy." - : "Finding ignored files…"} + {!ignoredFiles.isSuccess + ? "Finding ignored files…" + : searching + ? "No ignored files match." + : "No ignored files to copy."}

    ) : (
    @@ -369,7 +405,8 @@ function AddProjectToMachineDialog({ request }: { request: AddProjectToMachineRe key={file.path} file={file} checked={selectedFiles.has(file.path)} - onCheckedChange={(checked) => toggleFile(file.path, checked)} + disabled={atSelectionLimit && !selectedFiles.has(file.path)} + onCheckedChange={(checked) => toggleFile(file, checked)} /> ))}
    @@ -377,6 +414,7 @@ function AddProjectToMachineDialog({ request }: { request: AddProjectToMachineRe {ignoredFiles.data?.truncated ? (

    Showing the first {files.length} files. + {canSearchFiles ? " Search to find the rest." : ""}

    ) : null}

    @@ -415,29 +453,37 @@ function projectSubdirectory(repositoryRoot: string | undefined, workspaceRoot: : ""; } -function defaultSelection(files: readonly ProjectIgnoredFile[]): ReadonlySet { - return new Set( +function defaultSelection( + files: readonly ProjectIgnoredFile[], +): ReadonlyMap { + return new Map( files .filter( (file) => file.byteLength <= COPYABLE_FILE_MAX_BYTES && isDefaultCopiedFile(file.path), ) - .map((file) => file.path), + .map((file) => [file.path, file]), ); } function IgnoredFileRow({ file, checked, + disabled, onCheckedChange, }: { file: ProjectIgnoredFile; checked: boolean; + disabled: boolean; onCheckedChange: (checked: boolean) => void; }) { const tooLarge = file.byteLength > COPYABLE_FILE_MAX_BYTES; return (

    +