diff --git a/.agents/skills/test-t3-mobile/SKILL.md b/.agents/skills/test-t3-mobile/SKILL.md index 306038007e9a0..a11eee4e3f4eb 100644 --- a/.agents/skills/test-t3-mobile/SKILL.md +++ b/.agents/skills/test-t3-mobile/SKILL.md @@ -43,10 +43,15 @@ prints one JSON line; on failure it names the next command. Mobile verification includes that step unless the user prohibits it. For a device you opened in an agent-device session elsewhere (a device pool, -cloud, or remote host), run `vp run mobile:install --session ` -from the directory you opened the session in. It installs the cached build into -that session (`--abi arm64-v8a` for an ARM Android device; `--metro ` also -opens it against Metro) and never builds. +cloud, or remote host), run `vp run mobile:install --session +--config ` from the directory you opened the session in, passing the +`--config` file from `device_open`'s `targetArgs`. Without it, a device pool's +gateway refuses the install, wrongly saying agent-device is too old. It installs +the cached build into that session (`--abi arm64-v8a` for an ARM Android device; +pool Android emulators are x86_64, the default; `--metro ` also opens it +against Metro). Without a cached build for this checkout it builds first, which can +outlast the lease, so run `vp run mobile:install --prepare` (with +the same `--abi`) before you open the device. Start `vp run dev:client` from `apps/mobile`, or reuse a healthy Metro belonging to this checkout. Open its printed development-client URL with AgentDevice diff --git a/.github/scripts/pr-limits.cjs b/.github/scripts/pr-limits.cjs index 77c09785e4104..f3eb528a08042 100644 --- a/.github/scripts/pr-limits.cjs +++ b/.github/scripts/pr-limits.cjs @@ -100,14 +100,22 @@ function parseExemptions(text) { } // VOUCHED.td: one `github:login` per line, `#` comments, and `-github:login reason` to denounce. -function vouchedLogins(text) { - const logins = new Set(); +// A login listed both ways is an error: pr-vouch.yml's action reads its first line and +// issue-vouch.yml its last, so no answer here would agree with both labels. +function parseVouchList(text) { + const vouched = new Set(); + const denounced = new Set(); for (const raw of text.split("\n")) { - const entry = raw.trim().split(/\s+/)[0]; - if (entry.startsWith("github:")) logins.add(entry.slice("github:".length).toLowerCase()); + const match = /^(-?)github:(\S+)/.exec(raw.trim()); + if (!match) continue; + const login = match[2].toLowerCase(); + const [listed, other] = match[1] ? [denounced, vouched] : [vouched, denounced]; + if (other.has(login)) + throw new Error(`${VOUCHED_PATH}: ${login} is both vouched and denounced`); + listed.add(login); } - if (logins.size === 0) throw new Error(`${VOUCHED_PATH}: no entries`); - return logins; + if (vouched.size === 0) throw new Error(`${VOUCHED_PATH}: no entries`); + return { vouched, denounced }; } function labeledAt(pr, name) { @@ -278,7 +286,8 @@ async function enforceOpenLimit({ github, context, core, dryRun }) { // lookups as "not vouched". Every read below throws on failure, and a throw closes nothing. if (pr.user.type === "Bot") return; if (parseExemptions(fs.readFileSync(EXEMPTIONS_PATH, "utf8")).has(author.toLowerCase())) return; - if (vouchedLogins(fs.readFileSync(VOUCHED_PATH, "utf8")).has(author.toLowerCase())) return; + const { vouched, denounced } = parseVouchList(fs.readFileSync(VOUCHED_PATH, "utf8")); + if (vouched.has(author.toLowerCase())) return; const { data: access } = await github.rest.repos.getCollaboratorPermissionLevel({ ...context.repo, username: author, @@ -289,6 +298,21 @@ async function enforceOpenLimit({ github, context, core, dryRun }) { const reopened = context.payload.action === "reopened"; if (reopened && context.payload.sender.login.toLowerCase() !== author.toLowerCase()) return; + if (denounced.has(author.toLowerCase())) { + if (dryRun) { + core.notice(`Dry run. PR #${pr.number} would close: ${author} is denounced.`); + return; + } + await closeWithComment( + github, + context, + pr.number, + `Closing: ${author} is banned from contributing here, so this PR will not be reviewed. See [bans](${guideLink(context, "bans")}).`, + ); + core.info(`PR #${pr.number}: closed, ${author} is denounced.`); + return; + } + const q = `repo:${context.repo.owner}/${context.repo.repo} is:pr is:open author:${author} sort:created-asc`; const { search } = await read(github, AUTHOR_OPEN_PRS_QUERY, { q }); // A new PR counts only the older ones, so two opened together cannot close each other. diff --git a/.github/workflows/issue-vouch.yml b/.github/workflows/issue-vouch.yml index cc0abeefd57df..0e04011ec3f9a 100644 --- a/.github/workflows/issue-vouch.yml +++ b/.github/workflows/issue-vouch.yml @@ -40,9 +40,14 @@ jobs: ref: context.payload.repository.default_branch, }); const statusByUser = new Map(); + const listedBothWays = new Set(); for (const raw of Buffer.from(file.content, "base64").toString("utf8").split("\n")) { const match = /^(-?)github:(\S+)/.exec(raw.trim()); - if (match) statusByUser.set(match[2].toLowerCase(), match[1] ? "denounced" : "vouched"); + if (!match) continue; + const login = match[2].toLowerCase(); + const status = match[1] ? "denounced" : "vouched"; + if (statusByUser.has(login) && statusByUser.get(login) !== status) listedBothWays.add(login); + statusByUser.set(login, status); } const issues = @@ -74,4 +79,32 @@ jobs: await github.rest.issues.addLabels({ owner, repo, issue_number: issue.number, labels: [next] }); core.info(`Issue #${issue.number} (${issue.user.login}): ${status} -> ${next}`); } + + // Only a newly opened issue is closed. One filed before its author was denounced + // may already have a fix on the way, so the sweep leaves those to maintainers. + // Someone else reopening it is a maintainer keeping it open. + const reopenedByOther = + context.payload.action === "reopened" && + context.payload.sender.login.toLowerCase() !== issue.user.login.toLowerCase(); + // The PR rules refuse a list that names a login both ways, so nothing closes for one here. + const trusted = + issue.user.type === "Bot" || + ["OWNER", "MEMBER", "COLLABORATOR"].includes(issue.author_association) || + listedBothWays.has(issue.user.login.toLowerCase()); + if (status === "denounced" && context.eventName === "issues" && !reopenedByOther && !trusted) { + await github.rest.issues.createComment({ + owner, + repo, + issue_number: issue.number, + body: `Closing: ${issue.user.login} is banned from contributing here, so this issue will not be triaged. See [bans](https://github.com/${owner}/${repo}/blob/main/CONTRIBUTING.md#bans).`, + }); + await github.rest.issues.update({ + owner, + repo, + issue_number: issue.number, + state: "closed", + state_reason: "not_planned", + }); + core.info(`Issue #${issue.number}: closed, ${issue.user.login} is denounced.`); + } } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index efdc64acd21ac..d36c4401e43ca 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -903,34 +903,12 @@ jobs: fi # electron-updater reads one manifest per platform and channel and picks - # the file entry whose name carries the running arch, so the per-arch - # manifests the build jobs wrote are merged back into that one file. - - name: Merge macOS updater manifests + # the file entry whose name carries the running arch. Require every + # platform's two build jobs to have produced matching metadata before + # merging it, so a successful release can never strand one architecture. + - name: Merge desktop updater manifests if: needs.preflight.outputs.release_channel != 'preview' - run: | - shopt -s nullglob - for x64_manifest in release-assets/*-mac-x64.yml; do - arm64_manifest="${x64_manifest%-x64.yml}.yml" - if [[ -f "$arm64_manifest" ]]; then - node scripts/merge-update-manifests.ts --platform mac "$arm64_manifest" "$x64_manifest" - rm -f "$x64_manifest" - fi - done - - - name: Merge Windows updater manifests - if: needs.preflight.outputs.release_channel != 'preview' - run: | - shopt -s nullglob - for x64_manifest in release-assets/*-win-x64.yml; do - arm64_manifest="${x64_manifest%-x64.yml}-arm64.yml" - merged_manifest="${x64_manifest%-win-x64.yml}.yml" - if [[ -f "$arm64_manifest" ]]; then - node scripts/merge-update-manifests.ts --platform win "$x64_manifest" "$arm64_manifest" "$merged_manifest" - rm -f "$x64_manifest" "$arm64_manifest" - else - mv "$x64_manifest" "$merged_manifest" - fi - done + run: node scripts/merge-release-update-manifests.ts release-assets # Updater manifests and blockmaps are what electron-updater consumes. # They are only listed for channels an updater is meant to follow. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 63e71dc1e250c..0c886bd493389 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -115,6 +115,9 @@ the open PR limit. - If you make our lives harder as reviewers and maintainers, you will be banned. Examples are repeated pings, a fight over a closure, and a second account to get around a limit. +A banned account is denounced in [.github/VOUCHED.td](.github/VOUCHED.td). Any issue or PR it opens +is closed when it is opened. + ## When your PR is closed diff --git a/apps/desktop/scripts/ensure-electron-runtime.mjs b/apps/desktop/scripts/ensure-electron-runtime.mjs index 933a763dfa354..741a2d9ee85d8 100644 --- a/apps/desktop/scripts/ensure-electron-runtime.mjs +++ b/apps/desktop/scripts/ensure-electron-runtime.mjs @@ -209,6 +209,10 @@ export function ensureElectronRuntime() { return electronPath; } +export function setup() { + ensureElectronRuntime(); +} + // `file://${argv[1]}` never matches on Windows (drive letters need `file:///C:/`). if (process.argv[1] && NodeURL.pathToFileURL(process.argv[1]).href === import.meta.url) { const electronPath = ensureElectronRuntime(); diff --git a/apps/desktop/src/app/DesktopApp.test.ts b/apps/desktop/src/app/DesktopApp.test.ts index 10e0d1cd16c9b..aca3b4fc7fe66 100644 --- a/apps/desktop/src/app/DesktopApp.test.ts +++ b/apps/desktop/src/app/DesktopApp.test.ts @@ -86,6 +86,7 @@ it.effect.each([false, true])( const historyStarted = yield* Deferred.make(); const releaseHistory = yield* Deferred.make(); const quitting = yield* Ref.make(false); + const tokenRequested = yield* Deferred.make(); const events: string[] = []; const record = (event: string) => Effect.sync(() => { @@ -144,7 +145,7 @@ it.effect.each([false, true])( configure: Effect.void, } as PreviewPasskeys.PreviewPasskeys["Service"]), Effect.provideService(DesktopApplicationMenu.DesktopApplicationMenu, { - configure: Effect.void, + configure: record("menu"), }), Effect.provideService(DesktopUpdates.DesktopUpdates, { configure: Effect.void, @@ -222,7 +223,12 @@ it.effect.each([false, true])( Layer.mock(DesktopClientSettings.DesktopClientSettings, {}), Layer.mock(DesktopConnectionCatalogStore.DesktopConnectionCatalogStore, {}), Layer.mock(DesktopIpc.DesktopIpc, {}), - Layer.mock(DesktopLocalEnvironmentAuth.DesktopLocalEnvironmentAuth, {}), + Layer.mock(DesktopLocalEnvironmentAuth.DesktopLocalEnvironmentAuth, { + getBearerToken: record("token").pipe( + Effect.andThen(Deferred.succeed(tokenRequested, undefined)), + Effect.as("token"), + ), + }), Layer.mock(DesktopManagedPolicy.DesktopManagedPolicy, {}), Layer.succeed(DesktopPrimaryPaste.DesktopPrimaryPaste, { enabled: true }), Layer.mock(DesktopSshEnvironment.DesktopSshEnvironment, {}), @@ -253,7 +259,13 @@ it.effect.each([false, true])( ); yield* Deferred.await(registrationStarted); yield* Deferred.await(windowStarted); - assert.deepEqual(events, ["identity", "ready", "identity", "backend", "window"]); + yield* Deferred.await(tokenRequested); + // The backend spawns before menus, and its token is minted without waiting for the renderer. + assert.deepEqual( + events.filter((event) => event !== "token"), + ["identity", "ready", "identity", "backend", "menu", "window"], + ); + assert.isAbove(events.indexOf("token"), events.indexOf("backend")); assert.isFalse(yield* Deferred.isDone(registrationStopped)); const shutdown = yield* DesktopShutdown.DesktopShutdown; yield* shutdown.request; diff --git a/apps/desktop/src/app/DesktopApp.ts b/apps/desktop/src/app/DesktopApp.ts index 7ebeaa74a021d..0e7f949bc042c 100644 --- a/apps/desktop/src/app/DesktopApp.ts +++ b/apps/desktop/src/app/DesktopApp.ts @@ -22,6 +22,7 @@ import * as PreviewPasskeys from "../preview/Passkeys.ts"; import * as DesktopBackendManager from "../backend/DesktopBackendManager.ts"; import * as DesktopBackendPool from "../backend/DesktopBackendPool.ts"; import * as DesktopHomeOwner from "../backend/DesktopHomeOwner.ts"; +import * as DesktopLocalEnvironmentAuth from "../backend/DesktopLocalEnvironmentAuth.ts"; import * as DesktopEnvironment from "./DesktopEnvironment.ts"; import * as DesktopLegacyLocalStorage from "./DesktopLegacyLocalStorage.ts"; import * as DesktopLifecycle from "./DesktopLifecycle.ts"; @@ -168,13 +169,12 @@ const stopAllPoolInstances = Effect.fn("desktop.app.stopAllPoolInstances")( }, ); -const bootstrap = Effect.gen(function* () { +// Spawns the local backend as soon as the protocol, IPC, and legacy storage that every window needs exist. +const startLocalBackend = Effect.gen(function* () { const state = yield* DesktopState.DesktopState; const environment = yield* DesktopEnvironment.DesktopEnvironment; const desktopSettings = yield* DesktopAppSettings.DesktopAppSettings; const desktopWindow = yield* DesktopWindow.DesktopWindow; - const snapShot = yield* DesktopSnapShot.DesktopSnapShot; - const appActivation = yield* DesktopAppActivation.DesktopAppActivation; yield* logBootstrapInfo("bootstrap start"); const settings = yield* desktopSettings.get; @@ -193,32 +193,13 @@ const bootstrap = Effect.gen(function* () { yield* (yield* DesktopAppIdentity.DesktopAppIdentity).resolveUserDataPath, ); - yield* snapShot.initialize; - // Only the user's own login shell PATH shows every t3 a new terminal runs; a fresh install has - // no cached one. Without it, the command is left to Settings. - const installCli = (yield* DesktopCliCommand.DesktopCliCommand).installOnFirstLaunch; - yield* Effect.forkScoped( - (yield* DesktopShellEnvironment.DesktopShellEnvironment).awaitUserShellDiscovery.pipe( - Effect.flatMap((found) => - found || environment.platform === "win32" - ? installCli - : logBootstrapInfo("skipped installing the t3 command without the shell PATH"), - ), - ), - ); - if (!settings.localEnvironmentEnabled) { yield* logBootstrapInfo("bootstrap skipping local environment (disabled in settings)"); - if (!(yield* Ref.get(state.quitting))) { - yield* desktopWindow.createMainIfBackendReady; - } - return; + return false; } - const pool = yield* DesktopBackendPool.DesktopBackendPool; - const primaryBackend = yield* pool.primary; + const primaryBackend = yield* (yield* DesktopBackendPool.DesktopBackendPool).primary; const serverExposure = yield* DesktopServerExposure.DesktopServerExposure; - const wslBackend = yield* DesktopWslBackend.DesktopWslBackend; if (environment.isDevelopment && Option.isNone(environment.configuredBackendPort)) { return yield* new DesktopDevelopmentBackendPortRequiredError(); @@ -237,7 +218,8 @@ const bootstrap = Effect.gen(function* () { "This T3 home is unavailable", DesktopHomeOwner.homeOwnedMessage(owner), ); - return yield* (yield* ElectronApp.ElectronApp).quit; + yield* (yield* ElectronApp.ElectronApp).quit; + return false; } const backendPortSelection = yield* resolveDesktopBackendPort(environment.configuredBackendPort); @@ -275,29 +257,73 @@ const bootstrap = Effect.gen(function* () { ); } - if (!(yield* Ref.get(state.quitting))) { - if (settings.wslOnly === true && settings.wslBackendEnabled === true) { - yield* desktopWindow.showConnectingSplash; - } - yield* primaryBackend.start; - yield* logBootstrapInfo("bootstrap backend start requested"); - // WSL-only fallback can change the primary endpoint before readiness. - const config = yield* primaryBackend.currentConfig; - if ( - !environment.isDevelopment && - !settings.wslOnly && - Option.isSome(config) && - Option.isNone(config.value.preflightFailure) - ) { - yield* desktopWindow.ensureMain; + if (yield* Ref.get(state.quitting)) { + return false; + } + if (settings.wslOnly === true && settings.wslBackendEnabled === true) { + yield* desktopWindow.showConnectingSplash; + } + yield* primaryBackend.start; + yield* logBootstrapInfo("bootstrap backend start requested"); + // The renderer asks for this token first; minting it at readiness saves a round trip. + yield* Effect.forkScoped( + (yield* DesktopLocalEnvironmentAuth.DesktopLocalEnvironmentAuth).getBearerToken.pipe( + Effect.ignore, + ), + ); + return true; +}).pipe(Effect.withSpan("desktop.startLocalBackend")); + +const bootstrap = Effect.fn("desktop.bootstrap")(function* (backendStarted: boolean) { + const state = yield* DesktopState.DesktopState; + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const desktopSettings = yield* DesktopAppSettings.DesktopAppSettings; + const desktopWindow = yield* DesktopWindow.DesktopWindow; + const snapShot = yield* DesktopSnapShot.DesktopSnapShot; + const appActivation = yield* DesktopAppActivation.DesktopAppActivation; + const settings = yield* desktopSettings.get; + if (settings.localEnvironmentEnabled && (!backendStarted || (yield* Ref.get(state.quitting)))) { + return; + } + + yield* snapShot.initialize; + // Only the user's own login shell PATH shows every t3 a new terminal runs; a fresh install has + // no cached one. Without it, the command is left to Settings. + const installCli = (yield* DesktopCliCommand.DesktopCliCommand).installOnFirstLaunch; + yield* Effect.forkScoped( + (yield* DesktopShellEnvironment.DesktopShellEnvironment).awaitUserShellDiscovery.pipe( + Effect.flatMap((found) => + found || environment.platform === "win32" + ? installCli + : logBootstrapInfo("skipped installing the t3 command without the shell PATH"), + ), + ), + ); + + if (!settings.localEnvironmentEnabled) { + if (!(yield* Ref.get(state.quitting))) { + yield* desktopWindow.createMainIfBackendReady; } - yield* appActivation.start.pipe( - Effect.tap(() => logBootstrapInfo("desktop app control socket ready")), - Effect.catch((error) => logStartupError("desktop app control socket unavailable", { error })), - ); - yield* Effect.forkScoped(wslBackend.reconcile); + return; } -}).pipe(Effect.withSpan("desktop.bootstrap")); + + const primaryBackend = yield* (yield* DesktopBackendPool.DesktopBackendPool).primary; + // WSL-only fallback can change the primary endpoint before readiness. + const config = yield* primaryBackend.currentConfig; + if ( + !environment.isDevelopment && + !settings.wslOnly && + Option.isSome(config) && + Option.isNone(config.value.preflightFailure) + ) { + yield* desktopWindow.ensureMain; + } + yield* appActivation.start.pipe( + Effect.tap(() => logBootstrapInfo("desktop app control socket ready")), + Effect.catch((error) => logStartupError("desktop app control socket unavailable", { error })), + ); + yield* Effect.forkScoped((yield* DesktopWslBackend.DesktopWslBackend).reconcile); +}); const startup = Effect.gen(function* () { const appIdentity = yield* DesktopAppIdentity.DesktopAppIdentity; @@ -369,14 +395,20 @@ const startup = Effect.gen(function* () { }); } yield* appIdentity.configure; - yield* previewPasskeys.configure; - yield* applicationMenu.configure; + // Sets canInstallUpdates, which the backend bootstrap payload reads. yield* updates.configure; yield* DesktopRemoteUpdates.listen; yield* DesktopStartupRecovery.listen; + const backendStarted = yield* startLocalBackend.pipe( + Effect.catchCause((cause) => fatalStartupCause("bootstrap", cause)), + ); + yield* previewPasskeys.configure; + yield* applicationMenu.configure; // Portal identity is prepared pre-ready; MIME helpers must not gate the backend or window. yield* linuxUrlHandler.register.pipe(Effect.forkScoped); - yield* bootstrap.pipe(Effect.catchCause((cause) => fatalStartupCause("bootstrap", cause))); + yield* bootstrap(backendStarted).pipe( + Effect.catchCause((cause) => fatalStartupCause("bootstrap", cause)), + ); }).pipe(Effect.withSpan("desktop.startup")); const scopedProgram = Effect.scoped( diff --git a/apps/desktop/src/app/DesktopConnectionCatalogStore.test.ts b/apps/desktop/src/app/DesktopConnectionCatalogStore.test.ts index 5dfde9103f152..b58b583fd6cd5 100644 --- a/apps/desktop/src/app/DesktopConnectionCatalogStore.test.ts +++ b/apps/desktop/src/app/DesktopConnectionCatalogStore.test.ts @@ -300,4 +300,66 @@ describe("DesktopConnectionCatalogStore", () => { assert.deepStrictEqual(yield* store.get, Option.some('{"schemaVersion":1,"targets":[]}')); }).pipe(Effect.provide(NodeServices.layer)), ); + + const emptyCatalog = + '{"schemaVersion":1,"targets":[],"profiles":[],"credentials":[],"remoteDpopTokens":[],"disabledEnvironmentIds":[]}'; + const savedFilesHolding = Effect.fn(function* (stateDir: string, contents: string) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const kept: string[] = []; + for (const name of yield* fileSystem.readDirectory(stateDir)) { + const file = path.join(stateDir, name); + if ((yield* fileSystem.stat(file)).type !== "File") continue; + if ((yield* fileSystem.readFileString(file)) === contents) kept.push(name); + } + return kept; + }); + + it.effect("keeps a catalog this build cannot decode when the next save replaces it", () => + withStore( + Effect.gen(function* () { + const path = yield* Path.Path; + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const fileSystem = yield* FileSystem.FileSystem; + const store = yield* DesktopConnectionCatalogStore.DesktopConnectionCatalogStore; + const catalogPath = path.join(environment.stateDir, "connection-catalog.json"); + assert.isTrue( + yield* store.set( + '{"schemaVersion":1,"targets":[{"_tag":"FutureConnectionTarget","environmentId":"remote"}],"profiles":[],"credentials":[],"remoteDpopTokens":[],"disabledEnvironmentIds":[]}', + ), + ); + const original = yield* fileSystem.readFileString(catalogPath); + + yield* store.get; + assert.isTrue(yield* store.set(emptyCatalog)); + + assert.isNotEmpty(yield* savedFilesHolding(environment.stateDir, original)); + }), + ), + ); + + it.effect("keeps a catalog that no longer decrypts when the next save replaces it", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const fileSystem = yield* FileSystem.FileSystem; + const baseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-desktop-connection-catalog-test-", + }); + const failDecrypt = yield* Ref.make(false); + const store = yield* DesktopConnectionCatalogStore.DesktopConnectionCatalogStore.pipe( + Effect.provide(layerFor(baseDir, true, failDecrypt)), + ); + const stateDir = path.join(baseDir, "userdata"); + assert.isTrue(yield* store.set('{"schemaVersion":1,"targets":[]}')); + const original = yield* fileSystem.readFileString( + path.join(stateDir, "connection-catalog.json"), + ); + + yield* Ref.set(failDecrypt, true); + yield* store.get.pipe(Effect.flip); + assert.isTrue(yield* store.set(emptyCatalog)); + + assert.isNotEmpty(yield* savedFilesHolding(stateDir, original)); + }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + ); }); diff --git a/apps/desktop/src/app/DesktopConnectionCatalogStore.ts b/apps/desktop/src/app/DesktopConnectionCatalogStore.ts index 3240df3636e38..5db49e5f9504c 100644 --- a/apps/desktop/src/app/DesktopConnectionCatalogStore.ts +++ b/apps/desktop/src/app/DesktopConnectionCatalogStore.ts @@ -15,6 +15,7 @@ import { fromLenientJson } from "@t3tools/shared/schemaJson"; import * as Context from "effect/Context"; import * as SecureRandom from "effect/crypto/SecureRandom"; import * as Effect from "effect/Effect"; +import * as Clock from "effect/Clock"; import * as Base64 from "effect/encoding/Base64"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; @@ -45,6 +46,9 @@ const RuntimeConnectionCatalogDocumentJson = Schema.fromJsonString( const encodeRuntimeConnectionCatalogDocumentJson = Schema.encodeEffect( RuntimeConnectionCatalogDocumentJson, ); +const decodeRuntimeConnectionCatalogDocumentJson = Schema.decodeEffect( + RuntimeConnectionCatalogDocumentJson, +); const DesktopConnectionCatalogStoreWriteOperation = Schema.Literals([ "create-temporary-file-name", @@ -52,6 +56,7 @@ const DesktopConnectionCatalogStoreWriteOperation = Schema.Literals([ "create-directory", "write-temporary-file", "replace-catalog-file", + "keep-unreadable-catalog", ]); const DesktopConnectionCatalogStoreMigrationOperation = Schema.Literals([ @@ -395,6 +400,57 @@ export const make = Effect.gen(function* () { ), ); + const decryptDocument = (document: EncryptedConnectionCatalogDocument) => + decodeSecretBytes(catalogPath, document.encryptedCatalog).pipe( + Effect.flatMap((encryptedCatalog) => + safeStorage.decryptString(encryptedCatalog).pipe( + Effect.mapError( + (cause) => + new DesktopConnectionCatalogStoreProtectionError({ + operation: "decrypt-catalog", + catalogPath, + cause, + }), + ), + ), + ), + ); + + // A catalog this build cannot decrypt or decode may still hold another build's saved + // environments and credentials, so a save copies it aside instead of destroying it. + const keepUnreadableCatalog = Effect.gen(function* () { + const readable = yield* readDocument(fileSystem, catalogPath).pipe( + Effect.flatMap( + Option.match({ + onNone: () => Effect.succeed(true), + onSome: (document) => + decryptDocument(document).pipe( + Effect.flatMap(decodeRuntimeConnectionCatalogDocumentJson), + Effect.as(true), + ), + }), + ), + Effect.orElseSucceed(() => false), + ); + if (readable) { + return; + } + const keptPath = `${catalogPath}.unreadable-${yield* Clock.currentTimeMillis}`; + yield* fileSystem.copyFile(catalogPath, keptPath).pipe( + Effect.mapError( + (cause) => + new DesktopConnectionCatalogStoreWriteError({ + operation: "keep-unreadable-catalog", + path: keptPath, + cause, + }), + ), + ); + yield* Effect.logWarning("Kept an unreadable connection catalog before replacing it.", { + keptPath, + }); + }); + const writeCatalog = Effect.fn("desktop.connectionCatalogStore.writeCatalog")(function* ( catalog: string, ) { @@ -470,26 +526,13 @@ export const make = Effect.gen(function* () { if (!(yield* encryptionAvailable)) { return Option.none(); } - const decrypted = yield* decodeSecretBytes(catalogPath, document.value.encryptedCatalog).pipe( - Effect.flatMap((encryptedCatalog) => - safeStorage.decryptString(encryptedCatalog).pipe( - Effect.mapError( - (cause) => - new DesktopConnectionCatalogStoreProtectionError({ - operation: "decrypt-catalog", - catalogPath, - cause, - }), - ), - ), - ), - ); - return Option.some(decrypted); + return Option.some(yield* decryptDocument(document.value)); }).pipe(Effect.withSpan("desktop.connectionCatalogStore.get")), set: Effect.fn("desktop.connectionCatalogStore.set")(function* (catalog) { if (!(yield* encryptionAvailable)) { return false; } + yield* keepUnreadableCatalog; yield* writeCatalog(catalog); return true; }), diff --git a/apps/desktop/src/app/DesktopStartupRecovery.ts b/apps/desktop/src/app/DesktopStartupRecovery.ts index c8228183aad91..e6c210043cf26 100644 --- a/apps/desktop/src/app/DesktopStartupRecovery.ts +++ b/apps/desktop/src/app/DesktopStartupRecovery.ts @@ -217,7 +217,6 @@ export const listen: Effect.Effect< const download = yield* updates.download; if (!download.completed) return; } - // Waits for a running background check; plain install would refuse. yield* updates.installPrepared(current.version); return; } diff --git a/apps/desktop/src/app/stdioErrors.test.ts b/apps/desktop/src/app/stdioErrors.test.ts new file mode 100644 index 0000000000000..ccfa3ed951178 --- /dev/null +++ b/apps/desktop/src/app/stdioErrors.test.ts @@ -0,0 +1,22 @@ +import * as NodeOS from "node:os"; +import { assert, it } from "vite-plus/test"; + +import { handleStdioError } from "./stdioErrors.ts"; + +const errorWithCode = (code: string, errno?: number) => + Object.assign(new Error(code), { code, errno }); +const EDQUOT_ERRNO = NodeOS.constants.errno.EDQUOT; + +it.each(["EPIPE", "ENOSPC"])("ignores %s on console output", (code) => { + assert.doesNotThrow(() => handleStdioError(errorWithCode(code))); +}); + +// Node reports EDQUOT as UNKNOWN because libuv has no name for it. +it.skipIf(EDQUOT_ERRNO === undefined)("ignores EDQUOT on console output", () => { + assert.doesNotThrow(() => handleStdioError(errorWithCode("UNKNOWN", -(EDQUOT_ERRNO ?? 0)))); +}); + +it("rethrows other console output errors", () => { + assert.throws(() => handleStdioError(errorWithCode("EIO")), "EIO"); + assert.throws(() => handleStdioError(errorWithCode("UNKNOWN", -5)), "UNKNOWN"); +}); diff --git a/apps/desktop/src/app/stdioErrors.ts b/apps/desktop/src/app/stdioErrors.ts new file mode 100644 index 0000000000000..b23c0e89e79a8 --- /dev/null +++ b/apps/desktop/src/app/stdioErrors.ts @@ -0,0 +1,13 @@ +import * as NodeOS from "node:os"; + +// libuv has no name for EDQUOT, so Node reports it as code "UNKNOWN" with the raw errno. +const EDQUOT_ERRNO = NodeOS.constants.errno.EDQUOT; + +// Redirected console output can fail when its log volume is full. +export function handleStdioError(error: unknown): void { + if (typeof error === "object" && error !== null) { + if ("code" in error && (error.code === "EPIPE" || error.code === "ENOSPC")) return; + if (EDQUOT_ERRNO !== undefined && "errno" in error && error.errno === -EDQUOT_ERRNO) return; + } + throw error; +} diff --git a/apps/desktop/src/backend/DesktopBackendConfiguration.test.ts b/apps/desktop/src/backend/DesktopBackendConfiguration.test.ts index 17ba6a674f76c..70f7ed87d2214 100644 --- a/apps/desktop/src/backend/DesktopBackendConfiguration.test.ts +++ b/apps/desktop/src/backend/DesktopBackendConfiguration.test.ts @@ -373,7 +373,7 @@ describe("DesktopBackendConfiguration", () => { ); assert.equal(config.runningDistro, "Ubuntu"); - assert.deepEqual(config.args.slice(0, 2), ["-d", "Ubuntu"]); + assert.deepEqual(config.args.slice(0, 5), ["-d", "Ubuntu", "--cd", "~", "--exec"]); assert.deepEqual(observedDistros, ["Ubuntu", "Ubuntu", "Ubuntu"]); assert.isTrue(Option.isNone(config.preflightFailure)); }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), @@ -436,6 +436,8 @@ describe("DesktopBackendConfiguration", () => { assert.deepEqual(config.args, [ "-d", "Ubuntu", + "--cd", + "~", "--exec", "env", `PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${resolvedPath}`, @@ -651,6 +653,8 @@ describe("DesktopBackendConfiguration", () => { assert.deepEqual(config.args, [ "-d", "Ubuntu", + "--cd", + "~", "--exec", "env", "PATH=/home/test user's/.nvm/versions/node/v22.0.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/test user/bin:/opt/test's tools/bin:/usr/bin:/bin", @@ -683,6 +687,43 @@ describe("DesktopBackendConfiguration", () => { ), ); + it.effect("gives the Windows and WSL backends one host scope that survives a restart", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const baseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-desktop-backend-config-test-", + }); + const launch = Effect.gen(function* () { + const configuration = yield* DesktopBackendConfiguration.DesktopBackendConfiguration; + const primary = yield* configuration.resolvePrimary; + const wsl = yield* configuration.resolveWsl({ port: 5050, distro: null }); + assert.equal(wsl.env.T3CODE_DESKTOP_HOST_SCOPE, primary.env.T3CODE_DESKTOP_HOST_SCOPE); + assert.include((wsl.env.WSLENV ?? "").split(":"), "T3CODE_DESKTOP_HOST_SCOPE"); + return primary.env.T3CODE_DESKTOP_HOST_SCOPE; + }).pipe( + Effect.provide( + layerBackendConfiguration().pipe( + Layer.provideMerge(layerServerExposure), + Layer.provideMerge(DesktopAppSettings.layerTest()), + Layer.provideMerge(DesktopWslServerTree.layerTest()), + Layer.provideMerge( + DesktopWslEnvironment.layerTest({ + isAvailable: true, + windowsToWslPath: () => Option.some("/mnt/c/repo/apps/server/src/index.ts"), + getDistroIp: () => Option.some("172.27.0.99"), + }), + ), + Layer.provideMerge(layerEnvironment(baseDir, { platform: "win32" })), + ), + ), + ); + + const first = yield* launch; + assert.match(first ?? "", /^[0-9a-f]{32}$/); + assert.equal(yield* launch, first); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), + ); + it.effect("resolveWsl carries the kill switch and resource attributes into the distro", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; @@ -748,7 +789,7 @@ describe("DesktopBackendConfiguration", () => { assert.equal(config.env.T3CODE_OTLP_PROTOCOL, "http/protobuf"); assert.equal( config.env.WSLENV, - "GOPATH/p:OPENAI_API_KEY/u:EMPTY::AZURE_DEVOPS_EXT_PAT/u:ANTHROPIC_API_KEY:T3CODE_OTLP_HEADERS:T3CODE_OTLP_PROTOCOL", + "GOPATH/p:OPENAI_API_KEY/u:EMPTY::AZURE_DEVOPS_EXT_PAT/u:T3CODE_DESKTOP_HOST_SCOPE:ANTHROPIC_API_KEY:T3CODE_OTLP_HEADERS:T3CODE_OTLP_PROTOCOL", ); }).pipe( Effect.provide( diff --git a/apps/desktop/src/backend/DesktopBackendConfiguration.ts b/apps/desktop/src/backend/DesktopBackendConfiguration.ts index e7b8405f8124d..5f499e6f4ee37 100644 --- a/apps/desktop/src/backend/DesktopBackendConfiguration.ts +++ b/apps/desktop/src/backend/DesktopBackendConfiguration.ts @@ -243,8 +243,38 @@ interface SharedBootstrapInput { readonly bootstrapToken: string; readonly bootstrapSecret: string; readonly observabilitySettings: BackendObservabilitySettings; + readonly hostScope: string; } +// Matches the server's DESKTOP_HOST_SCOPE_ENV. +const DESKTOP_HOST_SCOPE_ENV = "T3CODE_DESKTOP_HOST_SCOPE"; +const HOST_SCOPE_FILE_NAME = "desktop-host-scope"; +const HOST_SCOPE_PATTERN = /^[0-9a-f]{32}$/; + +/** + * Not a credential: it only tells this install's Windows and WSL servers that + * they run on one machine, so usage both read from one Windows directory counts + * once. Persisted so summaries from earlier launches still match. + */ +const readOrCreateHostScope = Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const random = yield* SecureRandom.SecureRandom; + const scopePath = environment.path.join(environment.stateDir, HOST_SCOPE_FILE_NAME); + const existing = (yield* fileSystem + .readFileString(scopePath) + .pipe(Effect.orElseSucceed(() => ""))).trim(); + if (HOST_SCOPE_PATTERN.test(existing)) return existing; + const scope = Hex.encode(yield* random.randomBytes(16).pipe(Effect.orDie)); + yield* fileSystem.makeDirectory(environment.stateDir, { recursive: true }).pipe( + Effect.andThen(fileSystem.writeFileString(scopePath, scope)), + Effect.catch((cause) => + Effect.logWarning("Could not persist the desktop host scope.", { cause }), + ), + ); + return scope; +}); + type WslPreflightRuntime = | { readonly kind: "executable"; @@ -540,6 +570,7 @@ const resolvePrimaryStartConfig = Effect.fn("desktop.backendConfiguration.resolv ELECTRON_RUN_AS_NODE: "1", T3CODE_DESKTOP_SHELL_DISCOVERY_PENDING: "1", T3CODE_CLI_PATH: Option.getOrUndefined(input.cliPath), + [DESKTOP_HOST_SCOPE_ENV]: input.hostScope, }, extendEnv: true, bootstrap, @@ -636,8 +667,8 @@ const resolveWslStartConfig = Effect.fn("desktop.backendConfiguration.resolveWsl const httpBaseUrl = new URL(`http://${rendererHost}:${input.port}`); const distroArgs = distroForConfig ? ["-d", distroForConfig] : []; - const forwardedEnv: Record = {}; - const forwardedEnvNames: string[] = []; + const forwardedEnv: Record = { [DESKTOP_HOST_SCOPE_ENV]: input.hostScope }; + const forwardedEnvNames: string[] = [DESKTOP_HOST_SCOPE_ENV]; for (const name of WSL_FORWARDED_ENV_NAMES) { const value = env[name]; if (value !== undefined && value.length > 0) { @@ -708,6 +739,9 @@ const resolveWslStartConfig = Effect.fn("desktop.backendConfiguration.resolveWsl ...baseConfig, args: [ ...distroArgs, + // WSL otherwise inherits the Windows-mounted desktop working directory. + "--cd", + "~", "--exec", "env", `PATH=${launchPath}`, @@ -735,6 +769,13 @@ export const make = Effect.gen(function* () { const random = yield* SecureRandom.SecureRandom; // Random generation yields; serialize it so concurrent backends cannot capture distinct bootstrap secrets. const secretRef = yield* SynchronizedRef.make(Option.none()); + const hostScope = yield* Effect.cached( + readOrCreateHostScope.pipe( + Effect.provideService(FileSystem.FileSystem, fileSystem), + Effect.provideService(DesktopEnvironment.DesktopEnvironment, environment), + Effect.provideService(SecureRandom.SecureRandom, random), + ), + ); const getOrCreateBootstrapSecret = SynchronizedRef.modifyEffect(secretRef, (current) => Option.match(current, { onSome: (secret) => Effect.succeed([secret, current] as const), @@ -762,6 +803,7 @@ export const make = Effect.gen(function* () { bootstrapToken, bootstrapSecret, observabilitySettings, + hostScope: yield* hostScope, } satisfies SharedBootstrapInput; }); diff --git a/apps/desktop/src/backend/DesktopBackendManager.test.ts b/apps/desktop/src/backend/DesktopBackendManager.test.ts index 8a8389082ce89..06a1f768f4bd4 100644 --- a/apps/desktop/src/backend/DesktopBackendManager.test.ts +++ b/apps/desktop/src/backend/DesktopBackendManager.test.ts @@ -378,6 +378,42 @@ describe("DesktopBackendManager", () => { }), ); + it.effect("becomes ready from the server ready line while HTTP probes stall", () => + Effect.scoped( + Effect.gen(function* () { + const probeStarted = yield* Deferred.make(); + const exit = yield* Deferred.make(); + const readyLine = encodeDesktopTelemetryControl({ version: 1, type: "serverReady" }); + const instance = yield* makeTestInstance({ + spawnerLayer: Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.succeed( + makeProcess({ + getOutputFd: (fd) => + fd === 5 + ? Stream.fromEffect(Deferred.await(probeStarted)).pipe( + Stream.flatMap(() => Stream.encodeText(Stream.make(`${readyLine}\n`))), + ) + : Stream.empty, + exitCode: Deferred.await(exit), + kill: () => + Deferred.succeed(exit, ChildProcessSpawner.ExitCode(0)).pipe(Effect.asVoid), + }), + ), + ), + ), + httpClientLayer: layerHttpClient(() => + Deferred.succeed(probeStarted, undefined).pipe(Effect.andThen(Effect.never)), + ), + }); + + yield* instance.start; + assert.isTrue(yield* instance.waitForReady(Duration.seconds(15))); + }), + ), + ); + it.effect("continues routing desktop telemetry control messages after an invalid line", () => Effect.gen(function* () { const handled = yield* Deferred.make(); diff --git a/apps/desktop/src/backend/DesktopBackendManager.ts b/apps/desktop/src/backend/DesktopBackendManager.ts index 89ebc9e862ecf..4acaa8a3c18b8 100644 --- a/apps/desktop/src/backend/DesktopBackendManager.ts +++ b/apps/desktop/src/backend/DesktopBackendManager.ts @@ -1,5 +1,6 @@ import * as Brand from "effect/Brand"; import * as Cause from "effect/Cause"; +import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; @@ -484,21 +485,23 @@ export const runBackendProcess = Effect.fn("runBackendProcess")(function* ( ), ); const outputFibers: Array> = []; + const serverReady = yield* Deferred.make(); yield* options.onStarted?.(handle.pid) ?? Effect.void; - if ( - options.bootstrap.desktopTelemetryControlFd !== undefined && - options.onDesktopTelemetryControl !== undefined - ) { + if (options.bootstrap.desktopTelemetryControlFd !== undefined) { const controlFd = options.bootstrap.desktopTelemetryControlFd; - const handleControl = options.onDesktopTelemetryControl; + const handleControl = options.onDesktopTelemetryControl ?? (() => Effect.void); yield* handle.getOutputFd(controlFd).pipe( Stream.decodeText(), Stream.splitLines, Stream.filter((line) => line.trim().length > 0), Stream.runForEach((line) => decodeDesktopTelemetryControlLine(line).pipe( - Effect.flatMap(handleControl), + Effect.flatMap((message) => + message.type === "serverReady" + ? Deferred.succeed(serverReady, undefined) + : handleControl(message), + ), Effect.catchCause((cause) => logBackendProcessWarning("ignored invalid desktop telemetry control message", { fd: controlFd, @@ -570,14 +573,18 @@ export const runBackendProcess = Effect.fn("runBackendProcess")(function* ( ); } // Cold WSL boots across /mnt/c can exceed one probe budget; retry while the backend is alive. + // HTTP polling stays as the fallback for backends without the control fd, such as WSL. const probeReadiness = Effect.fn("desktop.backendProcess.probeReadiness")(() => - waitForHttpReady({ - executablePath: options.executablePath, - entryPath: options.entryPath, - cwd: options.cwd, - httpBaseUrl: options.httpBaseUrl, - timeout: options.readinessTimeout ?? DEFAULT_BACKEND_READINESS_TIMEOUT, - }).pipe( + Effect.raceFirst( + waitForHttpReady({ + executablePath: options.executablePath, + entryPath: options.entryPath, + cwd: options.cwd, + httpBaseUrl: options.httpBaseUrl, + timeout: options.readinessTimeout ?? DEFAULT_BACKEND_READINESS_TIMEOUT, + }), + Deferred.await(serverReady), + ).pipe( Effect.flatMap(() => options.onReady?.(handle.pid) ?? Effect.void), Effect.as(true), Effect.catchTags({ diff --git a/apps/desktop/src/backend/DesktopLocalEnvironmentAuth.test.ts b/apps/desktop/src/backend/DesktopLocalEnvironmentAuth.test.ts index e82e3afea7303..c15ce3932bb80 100644 --- a/apps/desktop/src/backend/DesktopLocalEnvironmentAuth.test.ts +++ b/apps/desktop/src/backend/DesktopLocalEnvironmentAuth.test.ts @@ -1,11 +1,13 @@ import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import { assert, describe, it } from "@effect/vitest"; +import * as Clock from "effect/Clock"; import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Ref from "effect/Ref"; +import * as Result from "effect/Result"; import * as HttpClient from "effect/http/HttpClient"; import * as HttpClientRequest from "effect/http/HttpClientRequest"; import * as HttpClientResponse from "effect/http/HttpClientResponse"; @@ -238,4 +240,34 @@ describe("DesktopLocalEnvironmentAuth", () => { assert.strictEqual(yield* Ref.get(requestCount), 1); }).pipe(Effect.provide(TestClock.layer())), ); + + it.effect("waits through a backend that stalls for 30 seconds after it is ready", () => + Effect.gen(function* () { + const requestCount = yield* Ref.make(0); + const layerHttpClient = Layer.succeed( + HttpClient.HttpClient, + HttpClient.make((request) => + Effect.gen(function* () { + yield* Ref.update(requestCount, (count) => count + 1); + const now = yield* Clock.currentTimeMillis; + return now < 30_000 ? yield* Effect.never : tokenResponse(request); + }), + ), + ); + const auth = yield* DesktopLocalEnvironmentAuth.DesktopLocalEnvironmentAuth.pipe( + Effect.provide( + DesktopLocalEnvironmentAuth.layer.pipe( + Layer.provide(Layer.mergeAll(layerPool, layerHttpClient, NodeCrypto.layerMac)), + ), + ), + ); + + const fiber = yield* auth.getBearerToken.pipe(Effect.result, Effect.forkChild); + yield* TestClock.adjust(Duration.seconds(45)); + const result = yield* Fiber.join(fiber); + + assert.isTrue(Result.isSuccess(result)); + assert.strictEqual(yield* Ref.get(requestCount), 4); + }).pipe(Effect.provide(TestClock.layer())), + ); }); diff --git a/apps/desktop/src/backend/DesktopLocalEnvironmentAuth.ts b/apps/desktop/src/backend/DesktopLocalEnvironmentAuth.ts index 6c7d24623d40f..89aa3cd2a678c 100644 --- a/apps/desktop/src/backend/DesktopLocalEnvironmentAuth.ts +++ b/apps/desktop/src/backend/DesktopLocalEnvironmentAuth.ts @@ -17,7 +17,8 @@ import * as HttpClient from "effect/http/HttpClient"; import * as DesktopBackendPool from "./DesktopBackendPool.ts"; -const BOOTSTRAP_TRANSIENT_RETRY_TIMEOUT = Duration.seconds(15); +// Each /oauth/token attempt may take 10 s, so a backend busy after ready needs several attempts. +const BOOTSTRAP_TRANSIENT_RETRY_TIMEOUT = Duration.seconds(60); const BOOTSTRAP_TRANSIENT_RETRY_INTERVAL = Duration.millis(500); const TRANSIENT_BOOTSTRAP_STATUS_CODES = new Set([502, 503, 504]); diff --git a/apps/desktop/src/electron/ElectronAudio.test.ts b/apps/desktop/src/electron/ElectronAudio.test.ts new file mode 100644 index 0000000000000..b38d7bbb3ae47 --- /dev/null +++ b/apps/desktop/src/electron/ElectronAudio.test.ts @@ -0,0 +1,60 @@ +import * as Effect from "effect/Effect"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import { app } from "electron"; +import { expect, vi } from "vite-plus/test"; +import { it } from "@effect/vitest"; +import * as Fiber from "effect/Fiber"; + +const { getAppMetrics, kill } = vi.hoisted(() => ({ getAppMetrics: vi.fn(), kill: vi.fn() })); +vi.mock("node:process", () => ({ kill })); +vi.mock("electron", async () => { + const { EventEmitter } = await import("node:events"); + return { app: Object.assign(new EventEmitter(), { getAppMetrics }) }; +}); + +import { restart } from "./ElectronAudio.ts"; + +const audio = { pid: 42, type: "Utility", serviceName: "audio.mojom.AudioService" }; + +it.effect("coalesces recovery and waits for its own audio utility to exit", () => + Effect.gen(function* () { + getAppMetrics.mockReturnValue([ + { pid: 1, type: "Utility", serviceName: "network.mojom.NetworkService" }, + { pid: 2, type: "Browser", serviceName: audio.serviceName }, + audio, + ]); + const started = Promise.withResolvers(); + kill.mockReset().mockImplementation(() => { + started.resolve(); + }); + const pending = yield* Effect.forkChild( + Effect.all([restart(), restart()], { concurrency: "unbounded" }), + ); + yield* Effect.promise(() => started.promise); + app.emit( + "child-process-gone", + {}, + { type: "Utility", serviceName: "network.mojom.NetworkService" }, + ); + expect(app.listenerCount("child-process-gone")).toBe(1); + app.emit("child-process-gone", {}, audio); + expect(yield* Fiber.join(pending)).toEqual([true, true]); + expect(kill.mock.calls).toEqual([[42, "SIGTERM"]]); + expect(app.listenerCount("child-process-gone")).toBe(0); + }).pipe(Effect.provideService(HostProcess.Platform, "linux")), +); + +it.effect.each(["linux", "darwin", "win32"] as const)( + "leaves other processes alone on %s", + (platform) => + Effect.gen(function* () { + getAppMetrics.mockReturnValue([ + { pid: 1, type: "Utility", serviceName: "network.mojom.NetworkService" }, + ]); + kill.mockReset(); + expect(yield* restart().pipe(Effect.provideService(HostProcess.Platform, platform))).toBe( + false, + ); + expect(kill).not.toHaveBeenCalled(); + }), +); diff --git a/apps/desktop/src/electron/ElectronAudio.ts b/apps/desktop/src/electron/ElectronAudio.ts new file mode 100644 index 0000000000000..6a7f290d6fd83 --- /dev/null +++ b/apps/desktop/src/electron/ElectronAudio.ts @@ -0,0 +1,62 @@ +// @effect-diagnostics globalTimers:off -- The shared native restart deadline must outlive individual IPC callers. +// @effect-diagnostics nodeBuiltinImport:off -- Only Electron's main process can restart its Chromium audio utility. +import * as NodeProcess from "node:process"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import { app, type Details, type Event } from "electron"; + +class AudioServiceRestartError extends Schema.TaggedError()( + "AudioServiceRestartError", + { cause: Schema.Defect() }, +) {} + +let pendingRestart: Promise | undefined; + +export const restart = Effect.fn("ElectronAudio.restart")(function* () { + if ((yield* HostProcess.Platform) !== "linux") return false; + return yield* Effect.tryPromise({ + try: () => { + if (pendingRestart) return pendingRestart; + const audio = app + .getAppMetrics() + .find( + (metric) => + metric.type === "Utility" && metric.serviceName === "audio.mojom.AudioService", + ); + if (!audio) return Promise.resolve(false); + // A failed PulseAudio context survives new getUserMedia calls; Chromium reconnects when its utility exits. + pendingRestart = new Promise((resolve, reject) => { + const cleanup = () => { + clearTimeout(timer); + app.removeListener("child-process-gone", onGone); + }; + const onGone = (_event: Event, details: Details) => { + if (details.type === "Utility" && details.serviceName === audio.serviceName) { + cleanup(); + resolve(true); + } + }; + const timer = setTimeout(() => { + cleanup(); + resolve(false); + }, 5_000); + app.on("child-process-gone", onGone); + try { + NodeProcess.kill(audio.pid, "SIGTERM"); + } catch (cause) { + cleanup(); + reject(cause); + } + }).finally(() => { + pendingRestart = undefined; + }); + return pendingRestart; + }, + catch: (cause) => new AudioServiceRestartError({ cause }), + }).pipe( + Effect.catch((cause) => + Effect.logWarning("Could not restart the audio service", cause).pipe(Effect.as(false)), + ), + ); +}); diff --git a/apps/desktop/src/electron/ElectronMenu.ts b/apps/desktop/src/electron/ElectronMenu.ts index 4ff3df6bf10f0..540326346edf8 100644 --- a/apps/desktop/src/electron/ElectronMenu.ts +++ b/apps/desktop/src/electron/ElectronMenu.ts @@ -1,4 +1,4 @@ -import type { ContextMenuItem } from "@t3tools/contracts"; +import type { ContextMenuItem, ContextMenuSource } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; @@ -17,6 +17,7 @@ export interface ElectronMenuContextInput { readonly window: Electron.BrowserWindow; readonly items: readonly ContextMenuItem[]; readonly position: Option.Option; + readonly source: Option.Option; } export interface ElectronMenuTemplateInput { @@ -259,7 +260,11 @@ export const make = Effect.gen(function* () { callback: () => complete(Option.none()), }), }); - menu.popup(popupOptions); + menu.popup( + Option.isSome(input.source) + ? { ...popupOptions, sourceType: input.source.value } + : popupOptions, + ); } catch (cause) { if (completed) { return; diff --git a/apps/desktop/src/electron/ElectronWindow.test.ts b/apps/desktop/src/electron/ElectronWindow.test.ts index 9eb41ed4e2402..eac944722e73b 100644 --- a/apps/desktop/src/electron/ElectronWindow.test.ts +++ b/apps/desktop/src/electron/ElectronWindow.test.ts @@ -148,6 +148,34 @@ describe("ElectronWindow", () => { }).pipe(Effect.provide(layerTestFor("win32"))), ); + it.effect("reports which windows were created with a title bar overlay", () => + Effect.gen(function* () { + const framed = { isDestroyed: () => false }; + const overlay = { isDestroyed: () => false }; + browserWindowMock + .mockImplementationOnce(function FramedWindowMock() { + return framed; + }) + .mockImplementationOnce(function OverlayWindowMock() { + return overlay; + }); + const electronWindow = yield* ElectronWindow.ElectronWindow; + yield* electronWindow.create({}); + yield* electronWindow.create({ + titleBarStyle: "hidden", + titleBarOverlay: { color: "#01000000", height: 40 }, + }); + getAllWindowsMock.mockReturnValue([framed, overlay, { isDestroyed: () => false }]); + + const seen: Array = []; + yield* electronWindow.syncAllAppearance((_window, { titleBarOverlay }) => + Effect.sync(() => seen.push(titleBarOverlay)), + ); + + assert.deepEqual(seen, [false, true, false]); + }).pipe(Effect.provide(layerTestFor("win32"))), + ); + it.effect("preserves destroy failures and continues with later windows", () => Effect.gen(function* () { const cause = new Error("window destroy failed"); diff --git a/apps/desktop/src/electron/ElectronWindow.ts b/apps/desktop/src/electron/ElectronWindow.ts index 15a117a52b050..b5f89969e66c7 100644 --- a/apps/desktop/src/electron/ElectronWindow.ts +++ b/apps/desktop/src/electron/ElectronWindow.ts @@ -125,7 +125,10 @@ export class ElectronWindow extends Context.Service< readonly sendAll: (channel: string, ...args: readonly unknown[]) => Effect.Effect; readonly destroyAll: Effect.Effect; readonly syncAllAppearance: ( - sync: (window: Electron.BrowserWindow) => Effect.Effect, + sync: ( + window: Electron.BrowserWindow, + appearance: { readonly titleBarOverlay: boolean }, + ) => Effect.Effect, ) => Effect.Effect; } >()("@t3tools/desktop/electron/ElectronWindow") {} @@ -145,6 +148,7 @@ export const make = Effect.gen(function* () { yield* Effect.addFinalizer(() => Effect.sync(() => windowsForegroundFocus?.close())); const mainWindowRef = yield* Ref.make>(Option.none()); const auxiliaryWindows = new WeakSet(); + const titleBarOverlayWindows = new WeakSet(); const listWindows = Effect.try({ try: () => Electron.BrowserWindow.getAllWindows(), @@ -241,6 +245,9 @@ export const make = Effect.gen(function* () { try: () => { const window = new Electron.BrowserWindow(options); if (behavior?.auxiliary === true) auxiliaryWindows.add(window); + if (options.titleBarOverlay !== undefined && options.titleBarOverlay !== false) { + titleBarOverlayWindows.add(window); + } return window; }, catch: (cause) => new ElectronWindowCreateError({ options: diagnosticOptions, cause }), @@ -383,14 +390,17 @@ export const make = Effect.gen(function* () { } }), syncAllAppearance: Effect.fn("desktop.electron.window.syncAllAppearance")(function* ( - sync: (window: Electron.BrowserWindow) => Effect.Effect, + sync: ( + window: Electron.BrowserWindow, + appearance: { readonly titleBarOverlay: boolean }, + ) => Effect.Effect, ) { const windows = yield* listWindows; for (const window of windows) { if (yield* isWindowDestroyed(window)) { continue; } - yield* sync(window); + yield* sync(window, { titleBarOverlay: titleBarOverlayWindows.has(window) }); } }), }); diff --git a/apps/desktop/src/ipc/DesktopIpcHandlers.ts b/apps/desktop/src/ipc/DesktopIpcHandlers.ts index 44ad769b3a2af..bd828c623995d 100644 --- a/apps/desktop/src/ipc/DesktopIpcHandlers.ts +++ b/apps/desktop/src/ipc/DesktopIpcHandlers.ts @@ -45,12 +45,14 @@ import { focusWindow, openExternal, readMedia, + restartAudioService, openSystemSettings, checkSystemPermission, pasteAsText, probeRemoteEditors, revealWindow, pickFolder, + toLocalBackendPath, pickProjectFavicon, pickThemeFiles, setTheme, @@ -110,6 +112,7 @@ export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers" yield* ipc.handle(getHostEnvironmentId); yield* ipc.handle(getLocalEnvironmentBearerToken); + yield* ipc.handle(restartAudioService); yield* ipc.handle(getClientSettings); yield* ipc.handle(setClientSettings); yield* ipc.handle(getConnectionCatalog); @@ -147,6 +150,7 @@ export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers" yield* ipc.handle(setWslOnly); yield* ipc.handle(pickFolder); + yield* ipc.handle(toLocalBackendPath); yield* ipc.handle(pickProjectFavicon); yield* ipc.handle(pickThemeFiles); yield* ipc.handle(setTheme); diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts index 926bdcfd27762..1f7e6dcf990bb 100644 --- a/apps/desktop/src/ipc/channels.ts +++ b/apps/desktop/src/ipc/channels.ts @@ -1,4 +1,5 @@ export const PICK_FOLDER_CHANNEL = "desktop:pick-folder"; +export const TO_LOCAL_BACKEND_PATH_CHANNEL = "desktop:to-local-backend-path"; export const SET_NOTIFICATION_BADGE_CHANNEL = "desktop:set-notification-badge"; export const FOCUS_WINDOW_CHANNEL = "desktop:focus-window"; export const PICK_PROJECT_FAVICON_CHANNEL = "desktop:pick-project-favicon"; @@ -101,6 +102,7 @@ export const PREVIEW_SET_AUDIO_MUTED_CHANNEL = "desktop:preview-set-audio-muted" export const PREVIEW_FIND_IN_PAGE_CHANNEL = "desktop:preview-find-in-page"; export const PREVIEW_STOP_FIND_IN_PAGE_CHANNEL = "desktop:preview-stop-find-in-page"; export const PREVIEW_SET_SHOWN_CHANNEL = "desktop:preview-set-shown"; +export const PREVIEW_SET_CAPTURE_DRAWN_CHANNEL = "desktop:preview-set-capture-drawn"; export const PREVIEW_OPEN_DEVTOOLS_CHANNEL = "desktop:preview-open-devtools"; export const PREVIEW_CLEAR_COOKIES_CHANNEL = "desktop:preview-clear-cookies"; export const PREVIEW_CLEAR_CACHE_CHANNEL = "desktop:preview-clear-cache"; @@ -139,3 +141,5 @@ export const RECEIVE_PROVIDER_AUTH_CALLBACK_CHANNEL = "desktop:receive-provider- export const CANCEL_PROVIDER_AUTH_CALLBACK_CHANNEL = "desktop:cancel-provider-auth-callback"; export const TAKE_LEGACY_LOCAL_STORAGE_CHANNEL = "desktop:take-legacy-local-storage"; export const COMPLETE_LEGACY_LOCAL_STORAGE_CHANNEL = "desktop:complete-legacy-local-storage"; + +export const RESTART_AUDIO_SERVICE_CHANNEL = "desktop:restart-audio-service"; diff --git a/apps/desktop/src/ipc/methods/preview.ts b/apps/desktop/src/ipc/methods/preview.ts index cc810fee072ad..757e1e463f538 100644 --- a/apps/desktop/src/ipc/methods/preview.ts +++ b/apps/desktop/src/ipc/methods/preview.ts @@ -13,6 +13,7 @@ import { DesktopPreviewScreenshotArtifactSchema, DesktopPreviewSetAudioMutedInputSchema, DesktopPreviewSetShownInputSchema, + DesktopPreviewSetCaptureDrawnInputSchema, DesktopPreviewSetColorSchemeInputSchema, DesktopPreviewSetZoomFactorInputSchema, BrowserImportResult, @@ -234,6 +235,15 @@ const setShown = DesktopIpc.makeIpcMethod({ yield* manager.setShown(tabId, shown); }), }); +const setCaptureDrawn = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.PREVIEW_SET_CAPTURE_DRAWN_CHANNEL, + payload: DesktopPreviewSetCaptureDrawnInputSchema, + result: Schema.Void, + handler: Effect.fn("desktop.ipc.preview.setCaptureDrawn")(function* ({ tabId, drawn }) { + const manager = yield* PreviewManager.PreviewManager; + yield* manager.setCaptureDrawn(tabId, drawn); + }), +}); const openDevTools = tabMethod( IpcChannels.PREVIEW_OPEN_DEVTOOLS_CHANNEL, "desktop.ipc.preview.openDevTools", @@ -488,6 +498,7 @@ export const methods = [ setZoomFactor, setAudioMuted, setShown, + setCaptureDrawn, openDevTools, clearCookies, clearCache, diff --git a/apps/desktop/src/ipc/methods/window.test.ts b/apps/desktop/src/ipc/methods/window.test.ts index ee1c64595fb65..da7e7da47fabd 100644 --- a/apps/desktop/src/ipc/methods/window.test.ts +++ b/apps/desktop/src/ipc/methods/window.test.ts @@ -1,6 +1,8 @@ import { assert, describe, it } from "@effect/vitest"; import * as NodeServices from "@effect/platform-node/NodeServices"; import type { ManagedPolicyLimits } from "@t3tools/contracts"; +import { EDITORS, REMOTE_CAPABLE_EDITOR_IDS } from "@t3tools/contracts"; +import { resolveEditorCommand } from "@t3tools/shared/editor"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { ManagedPolicyFileError } from "@t3tools/shared/managedPolicyFile"; import * as FileSystem from "effect/FileSystem"; @@ -25,6 +27,7 @@ import * as DesktopBackendManager from "../../backend/DesktopBackendManager.ts"; import * as DesktopBackendPool from "../../backend/DesktopBackendPool.ts"; import * as DesktopAppSettings from "../../settings/DesktopAppSettings.ts"; import * as DesktopManagedPolicy from "../../app/DesktopManagedPolicy.ts"; +import * as DesktopWslEnvironment from "../../wsl/DesktopWslEnvironment.ts"; import * as ElectronDialog from "../../electron/ElectronDialog.ts"; import * as ElectronWindow from "../../electron/ElectronWindow.ts"; import * as IpcChannels from "../channels.ts"; @@ -37,6 +40,7 @@ import { probeRemoteEditors, readMedia, syncHtmlFullscreen, + toLocalBackendPath, } from "./window.ts"; it.effect.each(["minimized", "hidden", "background"])( @@ -219,6 +223,47 @@ describe("getLocalEnvironmentBootstraps", () => { }); }); +describe("toLocalBackendPath", () => { + const { runningDistro: _, ...windowsConfig } = readyWslConfig; + it.effect("converts paths through wslpath only for backends running in WSL", () => + Effect.gen(function* () { + const convert = (backendId: string, path = "C:\\Users\\me\\project") => + toLocalBackendPath.handler({ backendId, path }); + assert.strictEqual(yield* convert("primary"), "C:\\Users\\me\\project"); + assert.strictEqual(yield* convert("wsl:default"), "/drives/c/Users/me/project"); + assert.strictEqual(yield* convert("wsl:missing"), null); + assert.strictEqual( + yield* convert("wsl:default", "\\\\wsl.localhost\\ubuntu\\home\\me\\repo"), + "/home/me/repo", + ); + assert.strictEqual(yield* convert("wsl:default", "\\\\wsl$\\Debian\\home\\me\\repo"), null); + assert.strictEqual( + yield* convert("primary", "\\\\wsl$\\Debian\\home\\me\\repo"), + "\\\\wsl$\\Debian\\home\\me\\repo", + ); + }).pipe( + Effect.provide( + Layer.mergeAll( + DesktopBackendPool.layerTest([ + { + ...defaultWslInstance, + id: DesktopBackendManager.BackendInstanceId("primary"), + currentConfig: Effect.succeedSome(windowsConfig), + }, + defaultWslInstance, + ]), + DesktopWslEnvironment.layerTest({ + windowsToWslPath: (distro, windowsPath) => + distro === "Ubuntu" && windowsPath === "C:\\Users\\me\\project" + ? Option.some("/drives/c/Users/me/project") + : Option.none(), + }), + ), + ), + ), + ); +}); + describe("installRelayTelemetryEnabled", () => { // The renderer reads this over sync IPC, so the answer must not wait on the filesystem. const relayTelemetryEnabled = ( @@ -519,3 +564,108 @@ it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( assert.include(editors, "webstorm"); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); + +describe("probeRemoteEditors PATH scan", () => { + const platform = HostProcess.Platform.defaultValue(); + const launcherName = (command: string) => (platform === "win32" ? `${command}.cmd` : command); + + const makeFakePath = Effect.fnUntraced(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-probe-path-" }); + const dirs = ["a", "b", "c", "d"].map((name) => path.join(root, name)); + for (const dir of dirs) yield* fs.makeDirectory(dir, { recursive: true }); + // Launchers in different PATH directories, deliberately not in REMOTE_CAPABLE_EDITOR_IDS order. + for (const [dir, command] of [ + [dirs[2]!, "webstorm"], + [dirs[0]!, "cursor"], + [dirs[3]!, "code"], + [dirs[1]!, "zeditor"], + ] as const) { + const file = path.join(dir, launcherName(command)); + yield* fs.writeFileString(file, "#!/bin/sh\n"); + if (platform !== "win32") yield* fs.chmod(file, 0o755); + } + return { + root, + env: { + HOME: root, + USERPROFILE: root, + PATH: dirs.join(platform === "win32" ? ";" : ":"), + PATHEXT: ".COM;.EXE;.BAT;.CMD", + }, + }; + }); + + const countingFileSystem = (counter: { stat: number; reads: Map }) => + Layer.effect( + FileSystem.FileSystem, + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + return FileSystem.FileSystem.of({ + ...fs, + stat: (target) => { + counter.stat += 1; + return fs.stat(target); + }, + readDirectory: (target, options) => { + counter.reads.set(target, (counter.reads.get(target) ?? 0) + 1); + return fs.readDirectory(target, options); + }, + }); + }), + ).pipe(Layer.provideMerge(NodeServices.layer)); + + const serialReference = (env: Record) => + Effect.gen(function* () { + const available: Array = []; + for (const editorId of REMOTE_CAPABLE_EDITOR_IDS) { + const editor = EDITORS.find((candidate) => candidate.id === editorId); + if (editor && Option.isSome(yield* resolveEditorCommand(editor, env))) { + available.push(editorId); + } + } + return available; + }); + + it.effect("reads each PATH directory once instead of stat-ing every candidate per editor", () => + Effect.gen(function* () { + const { env: baseEnv } = yield* makeFakePath(); + const delimiter = platform === "win32" ? ";" : ":"; + const uncachedCounter = { stat: 0, reads: new Map() }; + const cachedCounter = { stat: 0, reads: new Map() }; + const uncachedEnv = { ...baseEnv, PATH: `${baseEnv.PATH}${delimiter}${baseEnv.HOME}x1` }; + const cachedEnv = { ...baseEnv, PATH: `${baseEnv.PATH}${delimiter}${baseEnv.HOME}x2` }; + const uncached = yield* serialReference(uncachedEnv).pipe( + Effect.provide(countingFileSystem(uncachedCounter)), + ); + const cached = yield* probeRemoteEditors + .handler(undefined) + .pipe( + Effect.provideService(HostProcess.Environment, cachedEnv), + Effect.provideService(HostProcess.Platform, platform), + Effect.provide(countingFileSystem(cachedCounter)), + ); + // The host can add editors from app folders the fixture can't redirect (e.g. /Applications). + const fixtureEditors = ["cursor", "vscode", "zed", "webstorm"]; + assert.deepStrictEqual( + uncached.filter((id) => fixtureEditors.includes(id)), + fixtureEditors, + ); + assert.deepStrictEqual(cached, uncached); + assert.isAbove(uncachedCounter.stat, 0); + // On every platform the probe lists each PATH directory at most once. Only PATH entries count: + // resolveEditorCommand also reads app folders such as /Applications on macOS. + const pathReads = cachedEnv.PATH.split(delimiter) + .filter((entry) => entry.length > 0) + .map((entry) => cachedCounter.reads.get(entry) ?? 0); + assert.isAbove(Math.max(...pathReads), 0); + assert.isAtMost(Math.max(...pathReads), 1); + // Without the listings, Windows stats every PATH entry x PATHEXT candidate for each editor; + // elsewhere there is no PATHEXT, so the stat count is about the same either way. + if (platform === "win32") { + assert.isBelow(cachedCounter.stat, uncachedCounter.stat / 2); + } + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +}); diff --git a/apps/desktop/src/ipc/methods/window.ts b/apps/desktop/src/ipc/methods/window.ts index 742c39212de0a..ba87df404690e 100644 --- a/apps/desktop/src/ipc/methods/window.ts +++ b/apps/desktop/src/ipc/methods/window.ts @@ -1,5 +1,6 @@ import { ContextMenuItemSchema, + ContextMenuSource, DesktopAppBrandingSchema, DesktopEnvironmentBootstrapSchema, DesktopMediaReadSchema, @@ -16,6 +17,7 @@ import { } from "@t3tools/contracts"; import { PROJECT_FAVICON_EXTENSIONS } from "@t3tools/shared/projectFavicon"; import { resolveEditorCommand } from "@t3tools/shared/editor"; +import { withPathDirectoryListings } from "@t3tools/shared/shell"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { isRelayTracingEnabled } from "@t3tools/shared/otelEnvironment"; import * as FileSystem from "effect/FileSystem"; @@ -25,6 +27,7 @@ import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import * as DesktopBackendConfiguration from "../../backend/DesktopBackendConfiguration.ts"; +import * as DesktopBackendManager from "../../backend/DesktopBackendManager.ts"; import * as DesktopBackendPool from "../../backend/DesktopBackendPool.ts"; import * as DesktopLocalEnvironmentAuth from "../../backend/DesktopLocalEnvironmentAuth.ts"; import * as DesktopEnvironment from "../../app/DesktopEnvironment.ts"; @@ -34,6 +37,7 @@ import * as DesktopAppSettings from "../../settings/DesktopAppSettings.ts"; import * as DesktopWslBackend from "../../wsl/DesktopWslBackend.ts"; import * as DesktopWslEnvironment from "../../wsl/DesktopWslEnvironment.ts"; import * as ElectronApp from "../../electron/ElectronApp.ts"; +import * as ElectronAudio from "../../electron/ElectronAudio.ts"; import * as ElectronDialog from "../../electron/ElectronDialog.ts"; import * as ElectronMenu from "../../electron/ElectronMenu.ts"; import * as ElectronShell from "../../electron/ElectronShell.ts"; @@ -58,6 +62,7 @@ const ContextMenuPosition = Schema.Struct({ const ContextMenuInput = Schema.Struct({ items: Schema.Array(ContextMenuItemSchema), position: Schema.optionalKey(ContextMenuPosition), + source: Schema.optionalKey(ContextMenuSource), }); function toWebSocketBaseUrl(httpBaseUrl: URL): string { @@ -215,6 +220,20 @@ export const getLocalEnvironmentBearerToken = DesktopIpc.makeIpcMethod({ }), }); +// Ask the distro: its automount root is configurable, so /mnt/ is only the default. +const windowsPathToWsl = Effect.fn("desktop.ipc.window.windowsPathToWsl")(function* ( + distro: string | null, + windowsPath: string, +) { + const linuxUncPath = wslUncPathToLinuxPath(windowsPath); + if (linuxUncPath !== null) return Option.some(linuxUncPath); + const wslEnvironment = yield* DesktopWslEnvironment.DesktopWslEnvironment; + return yield* wslEnvironment.windowsToWslPath( + extractDistroFromUncPath(windowsPath) ?? distro, + windowsPath, + ); +}); + export const pickFolder = DesktopIpc.makeIpcMethod({ channel: IpcChannels.PICK_FOLDER_CHANNEL, payload: Schema.UndefinedOr(PickFolderOptionsSchema), @@ -261,20 +280,29 @@ export const pickFolder = DesktopIpc.makeIpcMethod({ if (!useWsl) { return selectedPath.value; } - - const linuxUncPath = wslUncPathToLinuxPath(selectedPath.value); - if (linuxUncPath !== null) { - return linuxUncPath; - } - - const converted = yield* wslEnvironment.windowsToWslPath( - extractDistroFromUncPath(selectedPath.value) ?? wslDistro, - selectedPath.value, - ); + const converted = yield* windowsPathToWsl(wslDistro, selectedPath.value); return Option.getOrElse(converted, () => selectedPath.value); }), }); +export const toLocalBackendPath = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.TO_LOCAL_BACKEND_PATH_CHANNEL, + payload: Schema.Struct({ backendId: Schema.String, path: Schema.String }), + result: Schema.NullOr(Schema.String), + handler: Effect.fn("desktop.ipc.window.toLocalBackendPath")(function* ({ backendId, path }) { + const pool = yield* DesktopBackendPool.DesktopBackendPool; + const instance = yield* pool.get(DesktopBackendManager.BackendInstanceId(backendId)); + if (Option.isNone(instance)) return null; + const config = yield* instance.value.currentConfig; + if (Option.isNone(config) || Option.isSome(config.value.preflightFailure)) return null; + const { runningDistro } = config.value; + if (runningDistro === undefined) return path; + const uncDistro = extractDistroFromUncPath(path); + if (uncDistro !== null && uncDistro.toLowerCase() !== runningDistro.toLowerCase()) return null; + return Option.getOrNull(yield* windowsPathToWsl(runningDistro, path)); + }), +}); + export const pickProjectFavicon = DesktopIpc.makeIpcMethod({ channel: IpcChannels.PICK_PROJECT_FAVICON_CHANNEL, payload: Schema.UndefinedOr(Schema.String), @@ -327,6 +355,7 @@ export const showContextMenu = DesktopIpc.makeIpcMethod({ window: window.value, items: input.items, position: Option.fromNullishOr(input.position), + source: Option.fromNullishOr(input.source), }); return Option.getOrNull(selectedItemId); }), @@ -411,20 +440,26 @@ export const openSystemSettings = DesktopIpc.makeIpcMethod({ }), }); +const REMOTE_EDITOR_PROBE_CONCURRENCY = 8; + export const probeRemoteEditors = DesktopIpc.makeIpcMethod({ channel: IpcChannels.PROBE_REMOTE_EDITORS_CHANNEL, payload: Schema.Undefined, result: Schema.Array(EditorId), handler: Effect.fn("desktop.ipc.window.probeRemoteEditors")(function* () { - const available: Array = []; const env = yield* HostProcess.Environment; - for (const editorId of REMOTE_CAPABLE_EDITOR_IDS) { - const editor = EDITORS.find((editor) => editor.id === editorId); - if (editor && Option.isSome(yield* resolveEditorCommand(editor, env))) { - available.push(editorId); - } - } - return available; + // Shared listings stop each editor re-stating every PATH entry x PATHEXT candidate on Windows. + const resolved = yield* Effect.forEach( + REMOTE_CAPABLE_EDITOR_IDS, + (editorId) => { + const editor = EDITORS.find((editor) => editor.id === editorId); + return editor + ? resolveEditorCommand(editor, env).pipe(Effect.map(Option.isSome)) + : Effect.succeed(false); + }, + { concurrency: REMOTE_EDITOR_PROBE_CONCURRENCY }, + ).pipe(withPathDirectoryListings); + return REMOTE_CAPABLE_EDITOR_IDS.filter((_, index) => resolved[index]); }), }); @@ -572,3 +607,10 @@ export const checkSystemPermission = DesktopIpc.makeIpcMethod({ return yield* Effect.promise(check); }), }); + +export const restartAudioService = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.RESTART_AUDIO_SERVICE_CHANNEL, + payload: Schema.Void, + result: Schema.Boolean, + handler: ElectronAudio.restart, +}); diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index d7a31216e16a9..79d37499ef92f 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -1,8 +1,8 @@ import * as MacPermissions from "./permissions/MacPermissions.ts"; +import { handleStdioError } from "./app/stdioErrors.ts"; + for (const stream of [process.stdout, process.stderr]) { - stream.on("error", (err: NodeJS.ErrnoException) => { - if (err.code !== "EPIPE") throw err; - }); + stream.on("error", handleStdioError); } import * as NodeHttpClient from "@effect/platform-node/NodeHttpClient"; diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index bdbcc2e9c9dc0..97621f48b4086 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -116,6 +116,7 @@ contextBridge.exposeInMainWorld("desktopBridge", { getPathForFile: (file: File) => webUtils.getPathForFile(file), getClientPlatform: () => clientPlatform, focusWindow: () => ipcRenderer.invoke(IpcChannels.FOCUS_WINDOW_CHANNEL), + restartAudioService: () => ipcRenderer.invoke(IpcChannels.RESTART_AUDIO_SERVICE_CHANNEL), setNotificationBadge: (badge) => ipcRenderer.invoke(IpcChannels.SET_NOTIFICATION_BADGE_CHANNEL, badge), onNotificationBadgeClear: (listener) => { @@ -239,14 +240,17 @@ contextBridge.exposeInMainWorld("desktopBridge", { setWslDistro: (distro) => ipcRenderer.invoke(IpcChannels.SET_WSL_DISTRO_CHANNEL, distro), setWslOnly: (enabled) => ipcRenderer.invoke(IpcChannels.SET_WSL_ONLY_CHANNEL, enabled), pickFolder: (options) => ipcRenderer.invoke(IpcChannels.PICK_FOLDER_CHANNEL, options), + toLocalBackendPath: (backendId, path) => + ipcRenderer.invoke(IpcChannels.TO_LOCAL_BACKEND_PATH_CHANNEL, { backendId, path }), pickProjectFavicon: (initialPath) => ipcRenderer.invoke(IpcChannels.PICK_PROJECT_FAVICON_CHANNEL, initialPath), pickThemeFiles: () => ipcRenderer.invoke(IpcChannels.PICK_THEME_FILES_CHANNEL, undefined), setTheme: (theme) => ipcRenderer.invoke(IpcChannels.SET_THEME_CHANNEL, theme), - showContextMenu: (items, position) => + showContextMenu: (items, position, source) => ipcRenderer.invoke(IpcChannels.CONTEXT_MENU_CHANNEL, { items, ...(position === undefined ? {} : { position }), + ...(source === undefined ? {} : { source }), }), receiveProviderAuthCallback: (url: string) => ipcRenderer.invoke(IpcChannels.RECEIVE_PROVIDER_AUTH_CALLBACK_CHANNEL, url), @@ -412,6 +416,8 @@ contextBridge.exposeInMainWorld("desktopBridge", { ipcRenderer.invoke(IpcChannels.PREVIEW_STOP_FIND_IN_PAGE_CHANNEL, { tabId }), setShown: (tabId, shown) => ipcRenderer.invoke(IpcChannels.PREVIEW_SET_SHOWN_CHANNEL, { tabId, shown }), + setCaptureDrawn: (tabId, drawn) => + ipcRenderer.invoke(IpcChannels.PREVIEW_SET_CAPTURE_DRAWN_CHANNEL, { tabId, drawn }), openDevTools: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_OPEN_DEVTOOLS_CHANNEL, { tabId }), listBrowserImportSources: () => ipcRenderer.invoke(IpcChannels.PREVIEW_IMPORT_SOURCES_CHANNEL), diff --git a/apps/desktop/src/preview/BrowserImport/ChromiumCookies.test.ts b/apps/desktop/src/preview/BrowserImport/ChromiumCookies.test.ts index 92ab35320dab7..eb4dd3143eefb 100644 --- a/apps/desktop/src/preview/BrowserImport/ChromiumCookies.test.ts +++ b/apps/desktop/src/preview/BrowserImport/ChromiumCookies.test.ts @@ -358,6 +358,29 @@ describe("readChromiumCookieDatabase", () => { }).pipe(Effect.provide(NodeServices.layer)), ); + it.effect("reads a cookie database the browser has emptied as having no cookies", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const directory = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3code-chromium-cookies-", + }); + const filename = `${directory}/Cookies`; + yield* Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`create table scratch (id integer)`; + yield* sql`drop table scratch`; + }).pipe(Effect.provide(NodeSqliteClient.layer({ filename }))); + + const result = yield* readChromiumCookieDatabase( + filename, + { cbcV10: Buffer.from("0123456789abcdef") }, + "darwin", + ); + + expect(result).toEqual({ cookies: [], undecryptable: 0, undecryptableHosts: [] }); + }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + ); + it.effect("skips partitioned cookies without breaking pre-CHIPS schemas", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; diff --git a/apps/desktop/src/preview/BrowserImport/ChromiumCookies.ts b/apps/desktop/src/preview/BrowserImport/ChromiumCookies.ts index 63fef9fb3c808..91cbfa03aac7d 100644 --- a/apps/desktop/src/preview/BrowserImport/ChromiumCookies.ts +++ b/apps/desktop/src/preview/BrowserImport/ChromiumCookies.ts @@ -181,6 +181,10 @@ export const readChromiumCookieDatabase = Effect.fn("ChromiumCookies.readChromiu function* (snapshotPath: string, keys: ChromiumKeyMaterial, platform: NodeJS.Platform) { const result = yield* Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; + // Chromium razes a damaged jar to a file with no tables and rebuilds it on its next write. + const tables = + yield* sql`select 1 from sqlite_master where type = 'table' and name = 'cookies'`; + if (tables.length === 0) return { rows: [], schemaVersion: 0 }; const schemaVersion = yield* sql`select value from meta where key = 'version' limit 1`.pipe( Effect.flatMap(decodeSchemaVersion), Effect.map(([row]) => row.value), diff --git a/apps/desktop/src/preview/BrowserImport/Sources.ts b/apps/desktop/src/preview/BrowserImport/Sources.ts index 20a35752ed979..6ecd546a1ad57 100644 --- a/apps/desktop/src/preview/BrowserImport/Sources.ts +++ b/apps/desktop/src/preview/BrowserImport/Sources.ts @@ -345,6 +345,12 @@ const countProfileCookies = Effect.fnUntraced(function* ( if (database === undefined) return undefined; return yield* Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; + // readChromiumCookieDatabase imports a razed jar as empty; Firefox has no such reader guard. + if (definition.engine !== "firefox") { + const tables = + yield* sql`select 1 from sqlite_master where type = 'table' and name = 'cookies'`; + if (tables.length === 0) return 0; + } const rows = definition.engine === "firefox" ? yield* sql`select count(*) as count from moz_cookies where originAttributes = ''` diff --git a/apps/desktop/src/preview/CdpRelay.test.ts b/apps/desktop/src/preview/CdpRelay.test.ts index ccc2b868720b4..564b4dc57d6f4 100644 --- a/apps/desktop/src/preview/CdpRelay.test.ts +++ b/apps/desktop/src/preview/CdpRelay.test.ts @@ -9,6 +9,7 @@ import { const makeTarget = ( send: CdpRelayTarget["send"], setDownloadDirectory: CdpRelayTarget["setDownloadDirectory"] = () => {}, + ackScreencastFrame: CdpRelayTarget["ackScreencastFrame"] = () => {}, ): CdpRelayTarget => ({ send, targetId: async () => "GUEST-TARGET", @@ -17,6 +18,7 @@ const makeTarget = ( userAgent: () => "Electron", setDownloadDirectory, receivesScreencast: () => true, + ackScreencastFrame, }); const makeRelay = (target: CdpRelayTarget, write: (raw: string) => void) => @@ -138,9 +140,13 @@ describe("CDP relay", () => { describe("CDP relay extra sessions", () => { it("gives a second page session the page's commands and events", async () => { - const send = vi.fn(async () => ({})); + const send = vi.fn(async () => ({})); const written: Array> = []; - const relay = makeRelay(makeTarget(send), (raw) => written.push(JSON.parse(raw))); + const nativeAcks: Array = []; + const relay = makeRelay( + makeTarget(send, undefined, (frameId) => nativeAcks.push(frameId)), + (raw) => written.push(JSON.parse(raw)), + ); relay.receive(JSON.stringify({ id: 1, method: "Target.setAutoAttach", params: {} })); relay.receive(JSON.stringify({ id: 2, method: "Target.attachToBrowserTarget" })); await settle(); @@ -170,8 +176,63 @@ describe("CDP relay extra sessions", () => { await settle(); expect(send).toHaveBeenLastCalledWith("Page.startScreencast", {}, undefined, extra); written.length = 0; - relay.event("Page.screencastFrame", { data: "x" }, undefined); - expect(written.map((m) => m["sessionId"])).toEqual(["t3-preview-page", extra]); + const frame = (frameId: number) => + relay.event("Page.screencastFrame", { data: "x", sessionId: frameId }, undefined); + const ack = (sessionId: string, frameId: number) => + relay.receive( + JSON.stringify({ + id: 6, + method: "Page.screencastFrameAck", + params: { sessionId: frameId }, + sessionId, + }), + ); + const delivered = () => { + const frames = written + .filter((m) => m["method"]) + .map((m) => [m["sessionId"], (m["params"] as { sessionId?: number }).sessionId]); + written.length = 0; + return frames; + }; + frame(1); + expect(delivered()).toEqual([ + ["t3-preview-page", 1], + [extra, 1], + ]); + // A session that has not acknowledged its frame gets only the newest one, on its ack. + for (let i = 2; i < 100; i++) frame(i); + expect(delivered()).toEqual([]); + // Frames every session is busy for stay unacknowledged, so Chromium pauses at its in-flight limit. + expect(nativeAcks).toEqual([1]); + ack(extra, 1); + await settle(); + expect(delivered()).toEqual([[extra, 99]]); + // Each consumer ack lets Chromium send one more frame. + expect(nativeAcks).toEqual([1, 2]); + ack(extra, 99); + await settle(); + expect(delivered()).toEqual([]); + expect(nativeAcks).toEqual([1, 2, 3]); + frame(100); + expect(delivered()).toEqual([[extra, 100]]); + ack(extra, 99); + await settle(); + frame(101); + expect(delivered()).toEqual([]); + expect(send.mock.calls.some(([method]) => method === "Page.screencastFrameAck")).toBe(false); + // A restart keeps the unacknowledged frame's slot until its own acknowledgement. + relay.receive(JSON.stringify({ id: 7, method: "Page.stopScreencast", sessionId: extra })); + relay.receive(JSON.stringify({ id: 8, method: "Page.startScreencast", sessionId: extra })); + await settle(); + delivered(); + frame(102); + expect(delivered()).toEqual([]); + ack(extra, 100); + await settle(); + expect(delivered()).toEqual([[extra, 102]]); + // Other events still reach a session while its frame is unacknowledged. + relay.event("Page.loadEventFired", { timestamp: 1 }, undefined); + expect(delivered().map(([session]) => session)).toEqual(["t3-preview-page", extra]); }); }); diff --git a/apps/desktop/src/preview/CdpRelay.ts b/apps/desktop/src/preview/CdpRelay.ts index 658e1cf70623d..e9f3cf509b446 100644 --- a/apps/desktop/src/preview/CdpRelay.ts +++ b/apps/desktop/src/preview/CdpRelay.ts @@ -11,6 +11,8 @@ export interface CdpRelayTarget { readonly sessionDetached?: (owner: string) => void; /** Whether a page session, by owner as `send` names it, gets the page's screencast frames. */ readonly receivesScreencast: (owner: string | undefined) => boolean; + /** Acknowledges a native screencast frame, letting Chromium send the next one. */ + readonly ackScreencastFrame: (frameId: number) => void; /** The page's own CDP target id; Playwright keys the main frame by it. */ readonly targetId: () => Promise; readonly url: () => string; @@ -103,6 +105,20 @@ export function createCdpRelayConnection( let downloadEventsEnabled = false; /** Relay-created page sessions share one real debugger; child frame sessions retain their own ids. */ const sessions = new Map([[PAGE_SESSION_ID, "page"]]); + // Each page session holds one unacknowledged frame and keeps the newest one skipped meanwhile. + const unackedFrames = new Map(); + const skippedFrames = new Map>(); + // Chromium keeps a few frames in flight and reuses one frame id per run, so pacing it means + // leaving frames every session is busy for unacknowledged and releasing one per consumer ack. + const heldNativeFrames: Array = []; + const releaseHeldFramesWithoutHolders = () => { + if (skippedFrames.size === 0) + for (const frameId of heldNativeFrames.splice(0)) target.ackScreencastFrame(frameId); + }; + const sendFrame = (sessionId: string, params: Record) => { + unackedFrames.set(sessionId, params["sessionId"] as number); + send({ method: "Page.screencastFrame", params, sessionId }); + }; let sessionSequence = 0; const newSession = (kind: "browser" | "page") => { const sessionId = `t3-preview-${kind}-${++sessionSequence}`; @@ -170,6 +186,9 @@ export function createCdpRelayConnection( const sessionId = command.params?.["sessionId"]; if (typeof sessionId === "string" && sessionId !== PAGE_SESSION_ID) { sessions.delete(sessionId); + unackedFrames.delete(sessionId); + skippedFrames.delete(sessionId); + releaseHeldFramesWithoutHolders(); target.sessionDetached?.(sessionId); } return {}; @@ -202,6 +221,27 @@ export function createCdpRelayConnection( } // Playwright asks for this on every page; the host emulates focus only while it is in use. if (command.method === "Emulation.setFocusEmulationEnabled") return Promise.resolve({}); + if (command.method === "Page.screencastFrameAck" && child === undefined) { + // A restart keeps the slot too, or restarting would let frames past an unacknowledged one. + const sessionId = command.sessionId!; + if (unackedFrames.get(sessionId) === command.params?.["sessionId"]) { + unackedFrames.delete(sessionId); + const skipped = skippedFrames.get(sessionId); + skippedFrames.delete(sessionId); + if ( + skipped !== undefined && + target.receivesScreencast(sessionId === PAGE_SESSION_ID ? undefined : sessionId) + ) + sendFrame(sessionId, skipped); + const released = heldNativeFrames.shift(); + if (released !== undefined) target.ackScreencastFrame(released); + } + return Promise.resolve({}); + } + if (command.method === "Page.stopScreencast" && child === undefined) { + skippedFrames.delete(command.sessionId!); + releaseHeldFramesWithoutHolders(); + } // The shared debugger keeps Runtime enabled after an earlier connection, and a repeated // enable reports no existing contexts; Playwright waits for them, so reset once per connection. if (command.method === "Runtime.enable" && child === undefined) { @@ -262,15 +302,29 @@ export function createCdpRelayConnection( send({ method, params, sessionId }); return; } - for (const [id, kind] of sessions) { - if (kind !== "page") continue; - if ( - method === "Page.screencastFrame" && - !target.receivesScreencast(id === PAGE_SESSION_ID ? undefined : id) - ) - continue; - send({ method, params, sessionId: id }); + if (method === "Page.screencastFrame") { + const frame = (params ?? {}) as Record; + const frameId = frame["sessionId"]; + if (typeof frameId !== "number") return; + let held = false; + let delivered = false; + for (const [id, kind] of sessions) { + if (kind !== "page") continue; + if (!target.receivesScreencast(id === PAGE_SESSION_ID ? undefined : id)) continue; + if (unackedFrames.has(id)) { + skippedFrames.set(id, frame); + held = true; + } else { + sendFrame(id, frame); + delivered = true; + } + } + if (held && !delivered) heldNativeFrames.push(frameId); + else target.ackScreencastFrame(frameId); + return; } + for (const [id, kind] of sessions) + if (kind === "page") send({ method, params, sessionId: id }); }, }; } diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index d6f25d66aa810..8e96129293fc8 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -26,6 +26,11 @@ const decodeCdpMessage = Schema.decodeUnknownSync( ), ); const key = { threadId: "thread-1", tabId: "tab-1" }; + +/** A host whose client already draws `key` at full detail, so its screenshots go out at once. */ +const makeDrawnHost = Effect.tap(DesktopBrowserHost.make, (host) => + Effect.sync(() => host.setCaptureDrawn(key, true)), +); /** Sends a page command whose id the fake debugger records as it receives it. */ const command = ( host: DesktopBrowserHost.DesktopBrowserHost["Service"], @@ -172,7 +177,7 @@ const watchPainting = (host: DesktopBrowserHost.DesktopBrowserHost["Service"]) = describe("DesktopBrowserHost", () => { it.effect("announces tabs already attached to a backend that starts later", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; host.attach(key, makeDebuggee().tab); expect(yield* takeEvents(host, 1)).toEqual([{ type: "attached", ...key }]); @@ -182,7 +187,7 @@ describe("DesktopBrowserHost", () => { it.effect("announces a guest still loading its first page only once that page is up", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const loading = makeDebuggee({ loading: true }); const other = { ...key, tabId: "tab-2" }; const reader = yield* takeEvents(host, 3).pipe(Effect.forkScoped); @@ -205,7 +210,7 @@ describe("DesktopBrowserHost", () => { "keeps a loading guest unannounced across a replaced navigation and a new backend", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const loading = makeDebuggee({ loading: true }); const other = { ...key, tabId: "tab-2" }; host.attach(key, loading.tab); @@ -227,7 +232,7 @@ describe("DesktopBrowserHost", () => { "announces a first page that never comes up, but never a tab closed or destroyed first", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const closed = { ...key, tabId: "tab-2" }; const destroyed = { ...key, tabId: "tab-3" }; const destroyedGuest = makeDebuggee({ loading: true }); @@ -247,7 +252,7 @@ describe("DesktopBrowserHost", () => { it.effect("re-sends shown to a backend that starts later and when a tab attaches again", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; host.attach(key, makeDebuggee().tab); host.setShown(key, true); const reader = yield* takeEvents(host, 5).pipe(Effect.forkScoped); @@ -268,7 +273,7 @@ describe("DesktopBrowserHost", () => { it.effect("replays an explicit hidden report before announcing another tab", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const other = { ...key, tabId: "tab-2" }; host.setShown(key, false); host.attach(key, makeDebuggee().tab); @@ -285,7 +290,7 @@ describe("DesktopBrowserHost", () => { it.effect("drops replies from a relay the server released", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const reader = yield* takeEvents(host, 2).pipe(Effect.forkScoped); @@ -313,7 +318,7 @@ describe("DesktopBrowserHost", () => { it.effect("sends a page one screenshot at a time", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -330,9 +335,38 @@ describe("DesktopBrowserHost", () => { }), ); + it.effect( + "sends a screenshot once the client draws the guest at full detail, or soon after", + () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + + yield* command(host, 1, "Page.captureScreenshot"); + yield* settle; + expect(debuggee.sent).toEqual([]); + host.setCaptureDrawn(key, true); + yield* settle; + expect(debuggee.sent).toEqual([1]); + debuggee.releaseFirst(); + expect(yield* Queue.take(replies)).toBe(1); + + // A client that never draws, such as a hidden window, only delays the capture. + host.setCaptureDrawn(key, false); + yield* command(host, 2, "Page.captureScreenshot"); + yield* settle; + expect(debuggee.sent).toEqual([1]); + yield* TestClock.adjust("1 second"); + yield* settle; + expect(debuggee.sent).toEqual([1, 2]); + }).pipe(Effect.scoped), + ); + it.effect("stops waiting on a screenshot the page navigated away from", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -357,7 +391,7 @@ describe("DesktopBrowserHost", () => { it.effect("drops screenshots a released relay left waiting", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); @@ -377,7 +411,7 @@ describe("DesktopBrowserHost", () => { it.effect("emulates focus while the server drives a guest, then lets a hidden one throttle", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); @@ -408,7 +442,7 @@ describe("DesktopBrowserHost", () => { it.effect("keeps a guest focused while a command it is running is still pending", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -429,7 +463,7 @@ describe("DesktopBrowserHost", () => { it.effect("lets a guest throttle once Chromium can no longer answer its pending command", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); @@ -451,7 +485,7 @@ describe("DesktopBrowserHost", () => { it.effect("keeps a guest focused for the screenshot a navigation lets start", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); @@ -467,7 +501,7 @@ describe("DesktopBrowserHost", () => { it.effect("never sends a screenshot a navigation superseded while focus was changing", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee({ deferFocus: true }); host.attach(key, debuggee.tab); @@ -484,7 +518,7 @@ describe("DesktopBrowserHost", () => { it.effect("keeps a guest focused when its turn starts after the agent's first command", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const turn = (active: boolean) => @@ -506,7 +540,7 @@ describe("DesktopBrowserHost", () => { it.effect("lets a guest throttle through a turn that only sent it housekeeping", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); yield* host.handleCommandLine( @@ -535,7 +569,7 @@ describe("DesktopBrowserHost", () => { it.effect("keeps a guest the agent used focused through its turn's pauses", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const turn = (active: boolean) => @@ -563,7 +597,7 @@ describe("DesktopBrowserHost", () => { "ends a command's hold with the turn that sent it, even if Chromium never answers", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); yield* host.handleCommandLine( @@ -584,7 +618,7 @@ describe("DesktopBrowserHost", () => { it.effect("ends the agent's own session's holds with its turn, but not a viewer's stream", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -617,7 +651,7 @@ describe("DesktopBrowserHost", () => { it.effect("paints for a screenshot queued behind one its viewer abandoned mid-capture", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -655,7 +689,7 @@ describe("DesktopBrowserHost", () => { it.effect("stops painting for a queued screenshot once its session detaches", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -694,7 +728,7 @@ describe("DesktopBrowserHost", () => { it.effect("never sends a screenshot whose session detached while focus was changing", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee({ deferFocus: true }); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -718,7 +752,7 @@ describe("DesktopBrowserHost", () => { it.effect("holds no focus for a screenshot still queued when the agent's turn ends", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const painting = yield* watchPainting(host); @@ -748,7 +782,7 @@ describe("DesktopBrowserHost", () => { it.effect("lets a guest throttle when Chromium refuses a viewer's screencast", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee({ refuse: (method) => method === "Page.startScreencast" }); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -769,7 +803,7 @@ describe("DesktopBrowserHost", () => { it.effect("lets a guest throttle after a person's input through a viewer that then closed", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -798,7 +832,7 @@ describe("DesktopBrowserHost", () => { it.effect("keeps a page's screencast when a child session's screencast is refused", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee({ refuse: (method, sessionId) => method === "Page.startScreencast" && sessionId !== undefined, }); @@ -820,7 +854,7 @@ describe("DesktopBrowserHost", () => { it.effect("ignores a screencast that arrives on a session after it detached", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -846,7 +880,7 @@ describe("DesktopBrowserHost", () => { it.effect("lets a guest throttle once the session waiting on its command detaches", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -873,7 +907,7 @@ describe("DesktopBrowserHost", () => { it.effect("never sends a screenshot queued by a session that has since detached", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -904,7 +938,7 @@ describe("DesktopBrowserHost", () => { it.effect("keeps a streamed guest focused until its screencast ends", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); @@ -933,7 +967,7 @@ describe("DesktopBrowserHost", () => { it.effect("keeps a guest focused while any of its screencasts runs", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -965,7 +999,7 @@ describe("DesktopBrowserHost", () => { it.effect("paints a hidden guest only while the screenshot it is taking is pending", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const painting = yield* watchPainting(host); @@ -991,7 +1025,7 @@ describe("DesktopBrowserHost", () => { it.effect("stops painting for a screenshot Chromium never answers once its document goes", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const painting = yield* watchPainting(host); @@ -1036,7 +1070,7 @@ describe("DesktopBrowserHost", () => { it.effect("paints a streamed guest until its last screencast ends", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -1157,6 +1191,13 @@ describe("DesktopBrowserHost", () => { debuggee.refuse(6); debuggee.release(); expect([yield* Queue.take(replies), yield* Queue.take(replies)].toSorted()).toEqual([5, 6]); + yield* raw({ + id: 7, + method: "Page.screencastFrameAck", + params: { sessionId: 1 }, + sessionId: "t3-preview-page-1", + }); + expect(yield* Queue.take(replies)).toBe(7); yield* settle; frames.length = 0; debuggee.emit("Page.screencastFrame", { data: "frame", sessionId: 2 }); @@ -1236,7 +1277,7 @@ describe("DesktopBrowserHost", () => { it.effect("saves a server tab's download under its CDP guid where the server asked", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); const replies = yield* watchReplies(host); @@ -1271,7 +1312,7 @@ describe("DesktopBrowserHost", () => { it.effect("tells a download the person clicked from one the agent's input started", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); @@ -1335,7 +1376,7 @@ describe("DesktopBrowserHost", () => { it.effect("detaches only the tab whose CDP event is too large to relay", () => Effect.gen(function* () { - const host = yield* DesktopBrowserHost.make; + const host = yield* makeDrawnHost; const tabs = [makeDebuggee({ deferFocus: true }), makeDebuggee()]; const send = vi.spyOn(tabs[0]!.tab.debugger, "sendCommand"); const keys = [key, { threadId: "thread-2", tabId: "tab-2" }]; diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index 296be2fa6709f..ce5d7074a786c 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -30,6 +30,8 @@ const AGENT_INPUT_WINDOW_MS = 5_000; /** A viewer's pointer travelling over the stream activates nothing on the page. */ const POINTER_TRAVEL = new Set(["mouseMoved", "mouseWheel"]); const FOCUS_IDLE = "5 seconds"; +/** How long a screenshot waits for the client to draw its guest at full detail. */ +const CAPTURE_DRAW_WAIT = "1 second"; /** Well inside the server's 10 s DESKTOP_ATTACH_TIMEOUT, for a first page that never reports it is up. */ const FIRST_PAGE_TIMEOUT = "5 seconds"; const ERR_ABORTED = -3; @@ -101,6 +103,8 @@ export class DesktopBrowserHost extends Context.Service< readonly detach: (key: DesktopBrowserTabKey) => void; readonly loadFailed: (key: DesktopBrowserTabKey, failure: PreviewAutomationLoadError) => void; readonly setShown: (key: DesktopBrowserTabKey, shown: boolean | undefined) => void; + /** Whether the client draws the guest at full detail while it is captured. */ + readonly setCaptureDrawn: (key: DesktopBrowserTabKey, drawn: boolean | undefined) => void; readonly humanStartedDownload: (source: Electron.WebContents) => boolean; /** Whether what the page does now follows the agent's input, not a viewer's. */ readonly agentLedPage: (source: Electron.WebContents) => boolean; @@ -138,6 +142,8 @@ export const make = Effect.gen(function* () { // Kept apart from attachment: the renderer can show a webview before its debugger attaches, // and the server forgets `shown` on detach, for example while DevTools holds the debugger. const shownTabs = new Map(); + const drawnTabs = new Set(); + const drawWaiters = new Map void>>(); const emit = (event: DesktopBrowserEventType) => runFork(PubSub.publish(outbox, event)); const nextScreenshot = (tab: AttachedTab) => { @@ -266,12 +272,32 @@ export const make = Effect.gen(function* () { }); }; + /** + * A scaled guest rasters at its on-screen size until the client draws it at full detail, + * and a screenshot taken before that copies the blurred frame. A client that cannot draw + * (a hidden window, say) only delays the capture. + */ + const whenDrawn = (tab: AttachedTab) => + new Promise((resolve) => { + const id = keyOf(tab.key); + const waiters = drawWaiters.get(id) ?? new Set(); + drawWaiters.set(id, waiters); + const done = () => { + waiters.delete(done); + if (waiters.size === 0 && drawWaiters.get(id) === waiters) drawWaiters.delete(id); + resolve(); + }; + waiters.add(done); + runFork(Effect.sleep(CAPTURE_DRAW_WAIT).pipe(Effect.andThen(Effect.sync(done)))); + }); + /** Chromium answers a capture, and frees its wake lock, only once the guest paints a frame. */ const paintUntilAnswered = (tab: AttachedTab, send: () => Promise) => { const sent = {}; tab.capturing.add(sent); syncPainting(tab); - return send().finally(() => { + // Sent at once when already drawn, so it keeps its place among the commands after it. + return (drawnTabs.has(keyOf(tab.key)) ? send() : whenDrawn(tab).then(send)).finally(() => { tab.capturing.delete(sent); syncPainting(tab); }); @@ -368,12 +394,15 @@ export const make = Effect.gen(function* () { // Waits for focus emulation; insertText and other non-CDP input would not queue behind it. const forward = (current?: () => boolean) => { const change = tab.focusChange; - const run = () => - tab.relay !== relay || (current !== undefined && !current()) + const stale = () => tab.relay !== relay || (current !== undefined && !current()); + const sendUnlessStale = () => + stale() ? Promise.reject(new Error("The page this screenshot was for has moved on.")) - : current === undefined - ? send(method, params, sessionId) - : paintUntilAnswered(tab, () => send(method, params, sessionId)); + : send(method, params, sessionId); + const run = () => + current === undefined || stale() + ? sendUnlessStale() + : paintUntilAnswered(tab, sendUnlessStale); return change === null ? run() : change.then(run); }; return method === "Page.captureScreenshot" @@ -421,6 +450,10 @@ export const make = Effect.gen(function* () { dropSessionHolds(tab, owner); }, receivesScreencast: (owner) => tab.screencasts.has(owner), + ackScreencastFrame: (frameId) => + void debuggee + .sendCommand("Page.screencastFrameAck", { sessionId: frameId }) + .catch(() => {}), targetId: () => debuggee .sendCommand("Target.getTargetInfo") @@ -583,6 +616,17 @@ export const make = Effect.gen(function* () { if (tabs.get(id)?.stopWaiting === null) emit({ type: "shown", ...key, shown }); }; + const setCaptureDrawn = (key: DesktopBrowserTabKey, drawn: boolean | undefined) => { + const id = keyOf(key); + if (!drawn) { + drawnTabs.delete(id); + return; + } + drawnTabs.add(id); + for (const done of drawWaiters.get(id) ?? []) done(); + drawWaiters.delete(id); + }; + const loadFailed: DesktopBrowserHost["Service"]["loadFailed"] = (key, failure) => { if (tabs.has(keyOf(key))) emit({ type: "loadFailed", ...key, ...failure }); }; @@ -659,6 +703,7 @@ export const make = Effect.gen(function* () { detach, loadFailed, setShown, + setCaptureDrawn, placeDownload, humanStartedDownload, agentLedPage, diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index e8ce5eaae3696..2091ff3c7f8f2 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -349,6 +349,47 @@ const makeTestPreviewWebContents = ( } as unknown as TestPreviewWebContents; }; +const makeStillImage = (pixels = "fresh-png") => ({ + toPNG: () => Buffer.from(pixels), + toJPEG: () => Buffer.from(pixels), + getSize: () => ({ width: 100, height: 80 }), +}); + +const makeTestStillImageWebContents = (...args: Parameters) => { + const wc = makeTestPreviewWebContents(...args); + Object.assign(wc, { isDevToolsOpened: () => false }); + holdStillImagePaint(wc, Promise.resolve()); + return wc; +}; + +const holdStillImagePaint = ( + wc: TestPreviewWebContents, + paint: Promise | Array>, +) => { + vi.mocked(wc.debugger.sendCommand).mockImplementation(async (method) => { + if (method === "Page.getFrameTree") return { frameTree: { frame: { id: "main" } } }; + if (method === "Page.createIsolatedWorld") return { executionContextId: 7 }; + if (method === "Runtime.evaluate") { + await (Array.isArray(paint) ? paint.shift() : paint); + return { result: { value: true } }; + } + return undefined; + }); +}; + +const setupStill = (manager: PreviewManager.PreviewManager["Service"], paint = Promise.resolve()) => + Effect.gen(function* () { + const image = makeStillImage(); + const capturePage = vi.fn<() => Promise>().mockResolvedValue(image); + const wc = makeTestStillImageWebContents(capturePage); + holdStillImagePaint(wc, paint); + fromId.mockReturnValue(wc); + yield* manager.createTab("tab_still"); + yield* manager.registerWebview("tab_still", wc.id); + yield* manager.setColorScheme("tab_still", "system"); + return { image, capturePage, wc, capture: manager.captureScreenshot("tab_still") }; + }); + const setupRecordingRaceTabs = (manager: PreviewManager.PreviewManager["Service"]) => Effect.gen(function* () { const capturePage = vi.fn(async () => ({ @@ -576,6 +617,32 @@ describe("PreviewManager", () => { webviewSend.mockClear(); }); + effectIt.effect("preserves recording artifacts saved in the same millisecond", () => + withManager((manager) => + Effect.gen(function* () { + const stored = new Map(); + const save = (path: string, bytes: Uint8Array) => { + stored.set(path, Uint8Array.from(bytes)); + return undefined; + }; + writeFile.mockImplementationOnce(save).mockImplementationOnce(save); + const firstBytes = new Uint8Array([1, 2, 3]); + const secondBytes = new Uint8Array([4, 5, 6]); + const [first, second] = yield* Effect.all( + [ + manager.saveRecording("tab_first", "video/webm", firstBytes), + manager.saveRecording("tab_second", "video/webm", secondBytes), + ], + { concurrency: "unbounded" }, + ); + + expect(first.path).not.toBe(second.path); + expect(stored.get(first.path)).toEqual(firstBytes); + expect(stored.get(second.path)).toEqual(secondBytes); + }), + ), + ); + effectIt.effect.each([false, true])( "preserves simultaneous same-name downloads with an existing destination: %s", (existing) => @@ -617,6 +684,7 @@ describe("PreviewManager", () => { attach: () => {}, detach: () => {}, setShown: () => {}, + setCaptureDrawn: () => {}, loadFailed: () => {}, humanStartedDownload: () => true, agentLedPage: () => false, @@ -746,6 +814,74 @@ describe("PreviewManager", () => { ), ); + effectIt.effect.each([ + "navigate", + "reload", + "goBack", + "goForward", + "refresh", + "hardReload", + ] as const)("%s stops a page script that never yields before it navigates", (operation) => + withManager((manager) => + Effect.gen(function* () { + const preview = makeFaviconWebContents(); + // As in Chromium: the hung renderer answers nothing, and a terminate reaches it only + // while no navigation is pending. + let stopped = false; + const probed = yield* Deferred.make(); + const navigatedAfterStop: Array = []; + const navigated = () => { + navigatedAfterStop.push(stopped); + }; + const sendCommand = async (method: string) => { + if (method === "Runtime.evaluate") { + Deferred.doneUnsafe(probed, Effect.void); + await new Promise(() => {}); + } + if (method === "Runtime.terminateExecution") stopped = true; + }; + Object.assign(preview.webContents, { + getURL: () => "http://localhost:5173/busy", + reload: navigated, + reloadIgnoringCache: navigated, + loadURL: async () => navigated(), + navigationHistory: { + canGoBack: () => true, + canGoForward: () => true, + goBack: navigated, + goForward: navigated, + }, + debugger: { + isAttached: () => false, + attach: vi.fn(), + sendCommand, + on: vi.fn(), + off: vi.fn(), + }, + }); + fromId.mockReturnValue(preview.webContents); + yield* manager.createTab("tab_busy"); + yield* manager.registerWebview("tab_busy", 42); + + const leaving = yield* ( + operation === "navigate" + ? manager.navigate("tab_busy", "http://localhost:5173/next") + : operation === "reload" + ? manager.navigate("tab_busy", "http://localhost:5173/busy") + : manager[operation]("tab_busy") + ).pipe(Effect.forkChild); + const probing = yield* Effect.raceFirst( + Deferred.await(probed).pipe(Effect.as(true)), + Fiber.await(leaving).pipe(Effect.as(false)), + ); + if (probing) yield* TestClock.adjust(1_000); + yield* Fiber.join(leaving); + + expect(navigatedAfterStop).toEqual([true]); + }), + ), + ); + effectIt.effect("keeps mouse back and forward buttons inside the preview tab", () => withManager((manager) => Effect.gen(function* () { @@ -1756,6 +1892,30 @@ describe("PreviewManager", () => { ), ); + effectIt.effect("captures final pixels after warmup fails and paint readiness settles", () => + withManager((manager) => + Effect.gen(function* () { + const paint = Promise.withResolvers(); + const { capturePage, wc, capture } = yield* setupStill(manager, paint.promise); + capturePage + .mockRejectedValueOnce(new Error("UnknownVizError")) + .mockRejectedValueOnce(new Error("UnknownVizError")) + .mockRejectedValueOnce(new Error("UnknownVizError")) + .mockResolvedValue(makeStillImage("final-png")); + const fiber = yield* capture.pipe(Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust(300); + expect(capturePage).toHaveBeenCalledTimes(3); + expect(writeFile).not.toHaveBeenCalled(); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false]]); + paint.resolve(); + const artifact = yield* Fiber.join(fiber); + expect(capturePage).toHaveBeenCalledTimes(4); + expect(writeFile.mock.calls).toEqual([[artifact.path, Buffer.from("final-png")]]); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false], [true]]); + }), + ), + ); + effectIt.effect("preserves both screenshots of the same site in the same millisecond", () => withManager((manager) => Effect.gen(function* () { @@ -1769,6 +1929,8 @@ describe("PreviewManager", () => { const capturePage = vi .fn<() => Promise>>() .mockResolvedValueOnce(pngImage(firstPng)) + .mockResolvedValueOnce(pngImage(firstPng)) + .mockResolvedValueOnce(pngImage(secondPng)) .mockResolvedValueOnce(pngImage(secondPng)); fromId.mockReturnValue(makeTestPreviewWebContents(capturePage)); yield* manager.createTab("tab_1"); @@ -1789,6 +1951,115 @@ describe("PreviewManager", () => { ), ); + effectIt.effect( + "waits for paint before capturing and restores rendering after overlapping requests", + () => + withManager((manager) => + Effect.gen(function* () { + const paints = [Promise.withResolvers(), Promise.withResolvers()]; + const pendingPaints = paints.map((paint) => paint.promise); + const { image, capturePage, wc, capture } = yield* setupStill(manager); + holdStillImagePaint(wc, pendingPaints); + const first = Promise.withResolvers(); + const second = Promise.withResolvers(); + capturePage + .mockResolvedValueOnce(makeStillImage("discarded-warmup")) + .mockResolvedValueOnce(makeStillImage("discarded-warmup")) + .mockReturnValueOnce(first.promise) + .mockReturnValueOnce(second.promise); + const hostThrottling = vi.fn(); + yield* manager.setMainWindow({ + isDestroyed: () => false, + once: vi.fn(), + webContents: { setBackgroundThrottling: hostThrottling }, + } as never); + const one = yield* capture.pipe(Effect.forkChild({ startImmediately: true })); + const two = yield* capture.pipe(Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust(0); + expect(capturePage).toHaveBeenCalledTimes(2); + expect(pendingPaints).toHaveLength(0); + expect(writeFile).not.toHaveBeenCalled(); + expect(hostThrottling.mock.calls).toEqual([[false]]); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false]]); + paints.forEach((paint) => paint.resolve()); + yield* TestClock.adjust(0); + expect(capturePage).toHaveBeenCalledTimes(4); + first.resolve(image); + yield* Fiber.join(one); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false]]); + second.resolve(image); + yield* Fiber.join(two); + const saved = writeFile.mock.calls.map(([, bytes]) => bytes); + expect(saved).toEqual([image.toPNG(), image.toPNG()]); + expect(hostThrottling.mock.calls).toEqual([[false], [true]]); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false], [true]]); + }), + ), + ); + + effectIt.effect("keeps picture-in-picture rendering after a still image completes", () => + withManager((manager) => + Effect.gen(function* () { + const { image, capturePage, wc, capture } = yield* setupStill(manager); + const pending = Promise.withResolvers(); + capturePage + .mockResolvedValueOnce(image) + .mockResolvedValueOnce(makeStillImage("discarded-warmup")) + .mockReturnValueOnce(pending.promise); + const { pictureInPictureWindow } = makeTestPictureInPictureWindow(); + browserWindowConstructor.mockImplementation(function () { + return pictureInPictureWindow; + }); + yield* manager.openPictureInPicture("tab_still"); + const still = yield* capture.pipe(Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust(0); + expect(capturePage).toHaveBeenCalledTimes(3); + expect(writeFile).not.toHaveBeenCalled(); + pending.resolve(image); + yield* Fiber.join(still); + expect(writeFile.mock.calls.map((call) => call[1])).toEqual([Buffer.from("fresh-png")]); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false]]); + yield* manager.closePictureInPicture("tab_still"); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false], [true]]); + }), + ), + ); + + effectIt.effect("keeps a pending still image rendering when recording ends", () => + withManager((manager) => + Effect.gen(function* () { + const { image, capturePage, wc, capture } = yield* setupStill(manager); + const pending = Promise.withResolvers(); + capturePage.mockResolvedValueOnce(image).mockReturnValue(pending.promise); + yield* manager.startRecording("tab_still"); + const fiber = yield* capture.pipe(Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust(0); + yield* manager.stopRecording("tab_still"); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false]]); + pending.resolve(image); + yield* Fiber.join(fiber); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false], [true]]); + }), + ), + ); + + effectIt.effect("saves available pixels when paint readiness never settles", () => + withManager((manager) => + Effect.gen(function* () { + const paint = Promise.withResolvers(); + const { capturePage, wc, capture } = yield* setupStill(manager, paint.promise); + const first = yield* Effect.exit(capture).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* TestClock.adjust(1_001); + expect(Exit.isSuccess(yield* Fiber.join(first))).toBe(true); + expect(capturePage).toHaveBeenCalledTimes(2); + expect(writeFile.mock.calls.map((call) => call[1])).toEqual([Buffer.from("fresh-png")]); + expect(wc.setBackgroundThrottling.mock.calls).toEqual([[false], [true]]); + }), + ), + ); + effectIt.effect("keeps every recorded guest unthrottled until its frame capture stops", () => withManager((manager) => Effect.gen(function* () { @@ -2192,6 +2463,7 @@ describe("PreviewManager", () => { }), ); expect(yield* Queue.take(capturing)).toBe(true); + yield* manager.setCaptureDrawn("tab_1", true); screenshot.resolve({ data: "" }); expect(yield* Queue.take(capturing)).toBe(false); }).pipe(Effect.provide(managerLayer())), @@ -2512,6 +2784,116 @@ describe("PreviewManager", () => { ), ); + effectIt.effect( + "publishes human ownership only on transitions while retaining every recording press", + () => + withManager((manager) => + Effect.gen(function* () { + let humanInput: ((input: unknown) => void) | undefined; + const contents = makeTestPreviewWebContents(async () => ({ + toJPEG: () => Buffer.from("frame"), + getSize: () => ({ width: 800, height: 600 }), + })); + contents.ipc.on = (channel, callback) => { + if (channel === "preview:human-input") + humanInput = (input) => callback({} as Electron.IpcMainEvent, input); + return contents.ipc; + }; + fromId.mockReturnValue(contents); + yield* manager.createTab("tab_human"); + yield* manager.registerWebview("tab_human", 42); + yield* manager.startRecording("tab_human", { + showKeyPresses: true, + showMousePresses: true, + }); + const controllers: string[] = []; + const feedback = yield* Queue.unbounded(); + webviewSend.mockImplementation((channel) => { + if (channel === "preview:recording-controller") Queue.offerUnsafe(feedback, undefined); + }); + yield* manager.subscribeStateChanges((_tabId, state) => + Effect.sync(() => { + controllers.push(state.controller); + }), + ); + for (const press of [ + { kind: "pointer", x: 10, y: 20, button: 0 }, + { kind: "pointer", x: 30, y: 40, button: 0 }, + { kind: "key", key: "a", code: "KeyA" }, + ]) { + humanInput?.(press); + yield* Queue.take(feedback); + } + expect(controllers).toEqual(["human"]); + expect( + webviewSend.mock.calls.filter( + ([channel]) => channel === "preview:recording-controller", + ), + ).toEqual([ + ["preview:recording-controller", "human", { x: 10, y: 20 }], + ["preview:recording-controller", "human", { x: 30, y: 40 }], + ["preview:recording-controller", "human", undefined], + ]); + yield* TestClock.adjust(750); + expect(controllers).toEqual(["human", "none"]); + }), + ), + ); + + effectIt.effect("keeps human ownership until the last input's idle deadline", () => + withManager((manager) => + Effect.gen(function* () { + let humanInput: ((input: unknown) => void) | undefined; + const contents = makeTestPreviewWebContents(async () => ({ + toJPEG: () => Buffer.from("frame"), + getSize: () => ({ width: 800, height: 600 }), + })); + contents.ipc.on = (channel, callback) => { + if (channel === "preview:human-input") + humanInput = (input) => callback({} as Electron.IpcMainEvent, input); + return contents.ipc; + }; + fromId.mockReturnValue(contents); + yield* manager.createTab("tab_idle"); + yield* manager.registerWebview("tab_idle", 42); + const controllers: string[] = []; + yield* manager.subscribeStateChanges((_tabId, state) => + Effect.sync(() => { + controllers.push(state.controller); + }), + ); + const press = () => humanInput?.({ kind: "key", key: "a", code: "KeyA" }); + press(); + yield* TestClock.adjust(500); + press(); + yield* TestClock.adjust(749); + expect(controllers).toEqual(["human"]); + yield* TestClock.adjust(1); + expect(controllers).toEqual(["human", "none"]); + + press(); + yield* TestClock.adjust(500); + const staleInput = humanInput; + const replacement = makeTestPreviewWebContents( + async () => ({ + toJPEG: () => Buffer.from("frame"), + getSize: () => ({ width: 800, height: 600 }), + }), + 43, + ); + replacement.ipc.on = contents.ipc.on; + contents.isDestroyed = () => true; + fromId.mockImplementation((id) => (id === 43 ? replacement : null)); + yield* manager.registerWebview("tab_idle", 43); + expect(controllers.at(-1)).toBe("none"); + const count = controllers.length; + staleInput?.({ kind: "key", key: "b", code: "KeyB" }); + yield* TestClock.adjust(750); + expect(controllers).toHaveLength(count); + }), + ), + ); + effectIt.effect("gates recording decorations and isolates failed subscribers", () => withManager((manager) => Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 56957783405b1..32ac876996c81 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -136,6 +136,7 @@ const PICTURE_IN_PICTURE_JPEG_QUALITY = 80; const CAPTURE_PAGE_RETRY_ATTEMPTS = 3; const CAPTURE_PAGE_RETRY_DELAY_MS = 120; const CAPTURE_PAGE_ATTEMPT_TIMEOUT_MS = 1_000; +const BUSY_PAGE_PROBE_MS = 1_000; const PICTURE_IN_PICTURE_INITIAL_WIDTH = 480; const PICTURE_IN_PICTURE_INITIAL_HEIGHT = 320; const PICTURE_IN_PICTURE_MIN_WIDTH = 240; @@ -386,7 +387,8 @@ interface ManagedListeners { readonly webContents: Electron.WebContents; } -type FrameCaptureConsumer = "picture-in-picture" | "recording"; +// Each still-image request owns a separate, short-lived rendering lease. +type FrameCaptureConsumer = "picture-in-picture" | "recording" | symbol; interface FrameCaptureSession { readonly recordingInputOptions?: RecordingInputOptions; @@ -887,7 +889,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const update = Effect.fn("PreviewManager.update")(function* ( tabId: string, patch: Partial, - humanPoint?: { readonly x: number; readonly y: number }, ) { const updatedAt = yield* currentIso; const next = yield* SynchronizedRef.modify(tabsRef, (tabs) => { @@ -903,17 +904,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }); // syncTabAudible can commit after this update; emitIfCurrent prevents publishing a stale snapshot. if (Option.isSome(next)) { - if (patch.controller !== undefined && next.value.webContentsId != null) { - const capture = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId); - const webContentsId = next.value.webContentsId; - if (capture?.consumers.has("recording")) { - yield* attempt({ operation: "recording.controller", tabId }, () => { - const contents = webContents.fromId(webContentsId); - if (contents && !contents.isDestroyed()) - contents.send(RECORDING_CONTROLLER_CHANNEL, patch.controller, humanPoint); - }).pipe(Effect.ignore); - } - } yield* emitIfCurrent(tabId, next.value); } }); @@ -1106,28 +1096,13 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ), scope, ); - const handleDebuggerMessage = Effect.fnUntraced(function* ( - method: string, - params: Record, - ) { - if (method === "Page.screencastFrame") { - const sessionId = params["sessionId"]; - if (typeof sessionId === "number") { - yield* attemptPromise( - { - operation: "ackScreencastFrame", - webContentsId: wc.id, - }, - () => wcDebugger.sendCommand("Page.screencastFrameAck", { sessionId }), - ).pipe(Effect.ignore); - } - } + const handleDebuggerMessage = Effect.fnUntraced(function* (method: string) { if (method === "Runtime.consoleAPICalled" || method === "Runtime.exceptionThrown") { yield* Queue.offer(consoleReleases, undefined); } }); - const onMessage: BrowserControlSession["onMessage"] = (_event, method, params) => { - runFork(handleDebuggerMessage(method, params)); + const onMessage: BrowserControlSession["onMessage"] = (_event, method) => { + runFork(handleDebuggerMessage(method)); }; const control: BrowserControlSession = { webContentsId: wc.id, @@ -1442,21 +1417,75 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }), ); }; + let humanInputWorker = false; + let humanInputDeadline = 0; + let listening = true; + const publishHumanController = Effect.fn("PreviewManager.publishHumanController")(function* ( + controller: "human" | "none", + humanPoint?: { readonly x: number; readonly y: number }, + ) { + const now = yield* currentMillis; + const updatedAt = yield* currentIso; + const result = yield* SynchronizedRef.modify(tabsRef, (tabs) => { + const current = tabs.get(tabId); + if ( + !current || + current.webContentsId !== wc.id || + (controller === "human" + ? !listening || webContents.fromId(wc.id) !== wc + : current.controller !== "human" || humanInputDeadline > now) + ) + return [undefined, tabs] as const; + const changed = current.controller !== controller; + const state = changed ? { ...current, controller, updatedAt } : current; + return [ + { state, changed }, + changed + ? replaceMap(tabs, (copy) => { + copy.set(tabId, state); + }) + : tabs, + ] as const; + }); + if (!result) return; + const capture = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId); + if (capture?.consumers.has("recording")) { + yield* attempt({ operation: "recording.controller", tabId }, () => { + if (!wc.isDestroyed()) wc.send(RECORDING_CONTROLLER_CHANNEL, controller, humanPoint); + }).pipe(Effect.ignore); + } + if (result.changed) yield* emitIfCurrent(tabId, result.state); + }); + yield* Scope.addFinalizer( + scope, + Effect.gen(function* () { + listening = false; + humanInputDeadline = 0; + yield* publishHumanController("none"); + }), + ); const handleHumanInput = Effect.fn("PreviewManager.handleHumanInput")(function* ( rawSignal?: unknown, ) { - yield* update( - tabId, - { controller: "human" }, + if (!listening) return; + humanInputDeadline = (yield* currentMillis) + 750; + if (!humanInputWorker) { + humanInputWorker = true; + yield* Effect.gen(function* () { + let remaining: number; + while ((remaining = humanInputDeadline - (yield* currentMillis)) > 0) { + yield* Effect.sleep(remaining); + } + humanInputWorker = false; + yield* publishHumanController("none"); + }).pipe(Effect.forkIn(scope)); + } + yield* publishHumanController( + "human", isPreviewInputSignal(rawSignal) && rawSignal.kind === "pointer" ? { x: rawSignal.x, y: rawSignal.y } : undefined, ); - yield* Effect.sleep(750); - const tabs = yield* SynchronizedRef.get(tabsRef); - if (tabs.get(tabId)?.controller === "human") { - yield* update(tabId, { controller: "none" }); - } }); const recordingInput = (_event: unknown, input: unknown) => { if (!isRecordingInput(input)) return; @@ -1798,7 +1827,10 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }); if (Option.isNone(tab)) return; const closedTab = tab.value; - if (closedTab.serverTab) browserHost.setShown(closedTab.serverTab, undefined); + if (closedTab.serverTab) { + browserHost.setShown(closedTab.serverTab, undefined); + browserHost.setCaptureDrawn(closedTab.serverTab, undefined); + } if (closedTab.webContentsId != null) { yield* Effect.all( [ @@ -2038,6 +2070,28 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ); }); + /** + * A same-site navigation reuses the renderer, so it never commits while a page script holds it. + * Runs before the navigation starts: Chromium does not deliver a terminate once one is pending. + */ + const leaveBusyPage = Effect.fn("PreviewManager.leaveBusyPage")(function* ( + wc: Electron.WebContents, + ) { + const control = (yield* SynchronizedRef.get(controlSessionsRef)).get(wc.id); + if (!control) return; + const answered = yield* Effect.promise(() => + control.debugger.sendCommand("Runtime.evaluate", { expression: "0" }).then( + () => undefined, + () => undefined, + ), + ).pipe(Effect.timeoutOption(BUSY_PAGE_PROBE_MS)); + if (Option.isSome(answered)) return; + // Not awaited: a renderer blocked on something other than script may never acknowledge it. + yield* Effect.sync(() => { + void control.debugger.sendCommand("Runtime.terminateExecution").catch(() => undefined); + }); + }); + const navigate = Effect.fn("PreviewManager.navigate")(function* (tabId: string, rawUrl: string) { const url = yield* attempt({ operation: "navigate.normalizeUrl", tabId }, () => normalizePreviewUrl(rawUrl), @@ -2120,6 +2174,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ); return; } + yield* leaveBusyPage(wc); if (wc.getURL() === url) { yield* attempt({ operation: "navigate.reload", tabId, webContentsId: wc.id }, () => wc.reload(), @@ -2155,18 +2210,28 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function yield* attempt({ operation, tabId, webContentsId: wc.id }, () => use(wc)); }); + const leavePage = Effect.fn("PreviewManager.leavePage")(function* ( + operation: string, + tabId: string, + use: (wc: Electron.WebContents) => void, + ) { + const wc = yield* requireWebContents(tabId); + yield* leaveBusyPage(wc); + yield* attempt({ operation, tabId, webContentsId: wc.id }, () => use(wc)); + }); + const goBack = (tabId: string) => - withWebContents("goBack", tabId, (wc) => { + leavePage("goBack", tabId, (wc) => { if (wc.navigationHistory.canGoBack()) wc.navigationHistory.goBack(); }); const goForward = (tabId: string) => - withWebContents("goForward", tabId, (wc) => { + leavePage("goForward", tabId, (wc) => { if (wc.navigationHistory.canGoForward()) wc.navigationHistory.goForward(); }); - const refresh = (tabId: string) => withWebContents("refresh", tabId, (wc) => wc.reload()); + const refresh = (tabId: string) => leavePage("refresh", tabId, (wc) => wc.reload()); const stopLoading = (tabId: string) => withWebContents("stopLoading", tabId, (wc) => wc.stop()); const hardReload = (tabId: string) => - withWebContents("hardReload", tabId, (wc) => wc.reloadIgnoringCache()); + leavePage("hardReload", tabId, (wc) => wc.reloadIgnoringCache()); const findInPage = ( tabId: string, text: string, @@ -2573,12 +2638,21 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function if (serverTab) browserHost.setShown(serverTab, shown); }); + const setCaptureDrawn = Effect.fn("PreviewManager.setCaptureDrawn")(function* ( + tabId: string, + drawn: boolean, + ) { + const serverTab = (yield* SynchronizedRef.get(tabsRef)).get(tabId)?.serverTab; + if (serverTab) browserHost.setCaptureDrawn(serverTab, drawn); + }); + // A window reload or guest crash destroys the guest without the renderer reporting it hidden. // The renderer reports again once a new guest registers. const hideDestroyedGuest = Effect.fnUntraced(function* (webContentsId: number) { for (const tab of (yield* SynchronizedRef.get(tabsRef)).values()) { if (tab.webContentsId === webContentsId && tab.serverTab) { browserHost.setShown(tab.serverTab, false); + browserHost.setCaptureDrawn(tab.serverTab, false); } } }); @@ -2617,7 +2691,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const [createdAt, millis, image] = yield* Effect.all([ currentIso, currentMillis, - capturePageWithRetry( + captureStillImage( { operation: "captureScreenshot.capturePage", tabId, @@ -2665,6 +2739,100 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }; }); + const captureStillImage = Effect.fn("PreviewManager.captureStillImage")(function* ( + errorContext: PreviewOperationContext, + tabId: string, + wc: Electron.WebContents, + ) { + const consumer = Symbol("still-image"); + return yield* Effect.acquireUseRelease( + startFrameCapture(tabId, consumer), + () => + Effect.gen(function* () { + // A DOM mutation can precede its compositor frame even in a shown + // window. Keep the original guest rendering through the next paint; + // a hidden guest otherwise returns old pixels or never completes. + const probe = Effect.fn("PreviewManager.probeStillImagePaint")(function* ( + send: ( + method: string, + params?: Record, + ) => Effect.Effect, + ) { + // CDP executes against the current document without Electron's + // did-stop-loading gate. Isolate scheduling globals from page code. + const tree = (yield* send("Page.getFrameTree")) as + | { frameTree?: { frame?: { id?: string } } } + | undefined; + const frameId = tree?.frameTree?.frame?.id; + if (typeof frameId !== "string") + return yield* new PreviewOperationError({ + ...errorContext, + cause: new Error("The preview paint frame is unavailable."), + }); + const world = (yield* send("Page.createIsolatedWorld", { + frameId, + worldName: "t3-preview-still-image", + })) as { executionContextId?: number } | undefined; + if (typeof world?.executionContextId !== "number") + return yield* new PreviewOperationError({ + ...errorContext, + cause: new Error("The preview paint context is unavailable."), + }); + const response = (yield* send("Runtime.evaluate", { + // Settle and release scheduled work when only timers keep running. + // The outer deadline still bounds a frozen renderer. + expression: `new Promise(resolve => { let frame; const timer = setTimeout(() => { cancelAnimationFrame(frame); resolve(false); }, ${CAPTURE_PAGE_ATTEMPT_TIMEOUT_MS}); frame = requestAnimationFrame(() => { frame = requestAnimationFrame(() => { clearTimeout(timer); resolve(true); }); }); })`, + contextId: world.executionContextId, + awaitPromise: true, + returnByValue: true, + })) as { result?: { value?: unknown }; exceptionDetails?: unknown } | undefined; + if (response?.exceptionDetails || response?.result?.value !== true) + return yield* new PreviewOperationError({ + ...errorContext, + cause: new Error("The preview paint boundary was not confirmed."), + }); + }); + const paintReady = Effect.gen(function* () { + // Only use an existing owned session; never attach or displace + // another debugger to obtain a screenshot. + const control = (yield* SynchronizedRef.get(controlSessionsRef)).get(wc.id); + if (!control) + return yield* new PreviewOperationError({ + ...errorContext, + cause: new Error("The preview paint session is unavailable."), + }); + return yield* probe((method, params) => + attemptPromise(errorContext, () => control.debugger.sendCommand(method, params)), + ); + }).pipe( + Effect.timeout(CAPTURE_PAGE_ATTEMPT_TIMEOUT_MS), + Effect.catch((error) => + Effect.logWarning( + "Preview paint readiness was not confirmed; capturing available pixels with mutation freshness unconfirmed.", + { tabId, webContentsId: wc.id, error }, + ), + ), + ); + // Minimized guests can suspend frames even with throttling disabled. + // A native capture can resume them; discard its pixels and read again + // after the bounded paint probe. + const warmup = capturePageWithRetry(errorContext, tabId, wc).pipe( + Effect.catchTags({ + PreviewOperationError: (error) => + Effect.logWarning("Preview warmup capture failed; continuing with final capture.", { + tabId, + webContentsId: wc.id, + error, + }), + }), + ); + yield* Effect.all([paintReady, warmup], { concurrency: 2, discard: true }); + return yield* capturePageWithRetry(errorContext, tabId, wc); + }), + () => stopFrameCapture(tabId, consumer), + ); + }); + const capturePreviewFrame = Effect.fn("PreviewManager.capturePreviewFrame")(function* ( tabId: string, ) { @@ -3347,7 +3515,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function data: Uint8Array, ) { const [createdAt, millis] = yield* Effect.all([currentIso, currentMillis]); - const id = `browser-recording-${millis.toString(36)}`; + const uuid = yield* random.uuidV4(); + const id = `browser-recording-${millis.toString(36)}-${uuid.slice(0, 8)}`; const extension = recordingFileExtension(mimeType); const artifactPath = path.join(resolvedArtifactDirectory, `${id}.${extension}`); yield* fileSystem.makeDirectory(resolvedArtifactDirectory, { recursive: true }).pipe( @@ -3361,7 +3530,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }), ), ); - yield* fileSystem.writeFile(artifactPath, data).pipe( + yield* fileSystem.writeFile(artifactPath, data, { flag: "wx" }).pipe( Effect.mapError( (cause) => new PreviewOperationError({ @@ -3559,6 +3728,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function setAnnotationSendEnabled, setAudioMuted, setShown, + setCaptureDrawn, setColorScheme, setMainWindow, setForwardedShortcuts: (shortcuts: ReadonlyArray) => @@ -3775,6 +3945,7 @@ export class PreviewManager extends Context.Service< audioMuted: boolean, ) => Effect.Effect; readonly setShown: (tabId: string, shown: boolean) => Effect.Effect; + readonly setCaptureDrawn: (tabId: string, drawn: boolean) => Effect.Effect; readonly openDevTools: (tabId: string) => Effect.Effect; readonly clearCookies: ( partitions?: ReadonlyArray, @@ -3937,6 +4108,7 @@ export const make = Effect.gen(function* PreviewManagerMake() { setZoomFactor: operations.setZoomFactor, setAudioMuted: operations.setAudioMuted, setShown: operations.setShown, + setCaptureDrawn: operations.setCaptureDrawn, openDevTools: operations.openDevTools, clearCookies: Effect.fn("PreviewManager.clearCookies")(function* (partitions) { yield* browserSession diff --git a/apps/desktop/src/shell/DesktopShellEnvironment.test.ts b/apps/desktop/src/shell/DesktopShellEnvironment.test.ts index c00e2ea7c65a2..7808682dfd82b 100644 --- a/apps/desktop/src/shell/DesktopShellEnvironment.test.ts +++ b/apps/desktop/src/shell/DesktopShellEnvironment.test.ts @@ -2,6 +2,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, describe, it } from "@effect/vitest"; import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Deferred from "effect/Deferred"; @@ -62,6 +63,26 @@ function makeProcess(output: string, exitCode = 0): ChildProcessSpawner.ChildPro }); } +function makeSlowProcess( + output: string, + delay: Duration.Input, +): ChildProcessSpawner.ChildProcessHandle { + const stdout = Stream.fromEffect(Effect.as(Effect.sleep(delay), textEncoder.encode(output))); + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(125), + stdout, + stderr: Stream.empty, + all: stdout, + exitCode: Effect.as(Effect.sleep(delay), ChildProcessSpawner.ExitCode(0)), + isRunning: Effect.succeed(true), + kill: () => Effect.void, + stdin: Sink.drain, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }); +} + function makeHungProcess(): ChildProcessSpawner.ChildProcessHandle { return ChildProcessSpawner.makeHandle({ pid: ChildProcessSpawner.ProcessId(124), @@ -486,7 +507,7 @@ describe("DesktopShellEnvironment", () => { Effect.gen(function* () { const env: NodeJS.ProcessEnv = { SHELL: "/bin/zsh", - PATH: "/Users/test/.local/bin:/usr/bin", + PATH: " /Users/test/.local/bin:/usr/bin", }; const commands: ChildProcess.Command[] = []; @@ -496,7 +517,7 @@ describe("DesktopShellEnvironment", () => { handler: (command) => { commands.push(command); return envOutput({ - PATH: "/opt/homebrew/bin:/usr/bin", + PATH: "/opt/homebrew/bin:/usr/bin:/opt/bin ", SSH_AUTH_SOCK: "/tmp/secretive.sock", HOMEBREW_PREFIX: "/opt/homebrew", }); @@ -505,7 +526,7 @@ describe("DesktopShellEnvironment", () => { assert.equal(commands.length, 1); assert.equal(commands[0]?._tag === "StandardCommand" ? commands[0].command : "", "/bin/zsh"); - assert.equal(env.PATH, "/opt/homebrew/bin:/usr/bin:/Users/test/.local/bin"); + assert.equal(env.PATH, "/opt/homebrew/bin:/usr/bin:/opt/bin : /Users/test/.local/bin"); assert.equal(env.SSH_AUTH_SOCK, "/tmp/secretive.sock"); assert.equal(env.HOMEBREW_PREFIX, "/opt/homebrew"); assert.isTrue(prepared); @@ -580,6 +601,39 @@ describe("DesktopShellEnvironment", () => { }), ); + it.effect.each([ + { + name: "hydrates both Android SDK names when neither is inherited", + inherited: {}, + expected: { ANDROID_HOME: "/shell/android-home", ANDROID_SDK_ROOT: "/shell/android-root" }, + }, + { + // The device host reads ANDROID_HOME before ANDROID_SDK_ROOT, so a shell + // ANDROID_HOME next to an inherited ANDROID_SDK_ROOT would change the SDK. + name: "keeps an inherited ANDROID_SDK_ROOT selected over a login-shell ANDROID_HOME", + inherited: { ANDROID_SDK_ROOT: "/inherited/android-sdk" }, + expected: { ANDROID_HOME: undefined, ANDROID_SDK_ROOT: "/inherited/android-sdk" }, + }, + ])("$name", ({ inherited, expected }) => + Effect.gen(function* () { + const env: NodeJS.ProcessEnv = { SHELL: "/bin/zsh", PATH: "/usr/bin", ...inherited }; + + yield* runShellEnvironment({ + env, + platform: "darwin", + handler: () => + envOutput({ + PATH: "/usr/bin", + ANDROID_HOME: "/shell/android-home", + ANDROID_SDK_ROOT: "/shell/android-root", + }), + }); + + assert.equal(env.ANDROID_HOME, expected.ANDROID_HOME); + assert.equal(env.ANDROID_SDK_ROOT, expected.ANDROID_SDK_ROOT); + }), + ); + it.effect("keeps mise's PATH record with the login shell PATH", () => Effect.gen(function* () { const env: NodeJS.ProcessEnv = { @@ -637,6 +691,129 @@ describe("DesktopShellEnvironment", () => { }), ); + it.effect("retries the user's shell when only the fallback shell answered", () => + Effect.gen(function* () { + const env: NodeJS.ProcessEnv = { SHELL: "/usr/bin/fish", PATH: "/usr/bin" }; + const commands: string[] = []; + let fishReady = false; + yield* runShellEnvironment({ + env, + platform: "linux", + handler: (command) => { + if (command._tag !== "StandardCommand") return ""; + commands.push(command.command); + if (command.command !== "/usr/bin/fish") return envOutput({ PATH: "/usr/bin" }); + return fishReady + ? envOutput({ PATH: "/fish/bin:/usr/bin" }) + : { output: "", exitCode: 7 }; + }, + use: (service) => + Effect.gen(function* () { + assert.isTrue(yield* service.awaitDiscovery); + assert.equal(env.PATH, "/usr/bin"); + // The Linux probe checks the D-Bus socket on the real filesystem, which the clock does not await. + const published = yield* service.updates.pipe( + Stream.filter((snapshot) => snapshot.environment.PATH === "/fish/bin:/usr/bin"), + Stream.runHead, + Effect.forkChild({ startImmediately: true }), + ); + + fishReady = true; + yield* TestClock.adjust("30 seconds"); + assert.isTrue(Option.isSome(yield* Fiber.join(published))); + assert.equal(env.PATH, "/fish/bin:/usr/bin"); + return true; + }), + }).pipe(Effect.provide(Logger.layer([]))); + assert.deepEqual(commands, ["/usr/bin/fish", "/bin/bash", "/usr/bin/fish"]); + + // With no user shell to wait for, the fallback's answer is final. + commands.length = 0; + yield* runShellEnvironment({ + env: { PATH: "/usr/bin" }, + platform: "linux", + handler: (command) => { + if (command._tag === "StandardCommand") commands.push(command.command); + return envOutput({ PATH: "/usr/bin" }); + }, + use: (service) => + Effect.gen(function* () { + assert.isTrue(yield* service.awaitDiscovery); + yield* TestClock.adjust("5 minutes"); + return true; + }), + }).pipe(Effect.provide(Logger.layer([]))); + assert.deepEqual(commands, ["/bin/bash"]); + }), + ); + + it.effect("gives background retries time for a locked or slow shell, then publishes", () => + Effect.gen(function* () { + const env: NodeJS.ProcessEnv = { SHELL: "/bin/zsh", PATH: "/usr/bin" }; + // Startup fails fast, the first retry meets a shell blocked on a lock, the second a slow one. + const spawned = yield* Effect.forEach([0, 1, 2], () => Deferred.make()); + const stopped = yield* Deferred.make(); + let attempts = 0; + yield* runShellEnvironment({ + env, + platform: "darwin", + handler: () => "", + processSpawner: ChildProcessSpawner.make((command) => { + if (command._tag !== "StandardCommand" || command.command !== "/bin/zsh") { + return Effect.succeed(makeProcess("")); + } + const attempt = attempts++; + const spawn = Deferred.succeed(spawned[attempt]!, undefined); + if (attempt === 0) return spawn.pipe(Effect.as(makeProcess(""))); + if (attempt === 1) { + return spawn.pipe( + Effect.andThen( + Effect.acquireRelease(Effect.succeed(makeHungProcess()), () => + Deferred.succeed(stopped, command.options), + ), + ), + ); + } + return spawn.pipe( + Effect.as(makeSlowProcess(envOutput({ PATH: "/fresh/bin" }), "20 seconds")), + ); + }), + use: (service) => + Effect.gen(function* () { + assert.isFalse(yield* service.awaitDiscovery); + const published = yield* service.updates.pipe( + Stream.drop(1), + Stream.runHead, + Effect.forkChild({ startImmediately: true }), + ); + + yield* TestClock.adjust("30 seconds"); + yield* Deferred.await(spawned[1]!); + // pyenv waits up to 60 s on its rehash lock before the shell continues. + yield* TestClock.adjust("1 minute"); + assert.isFalse(yield* Deferred.isDone(stopped)); + yield* TestClock.adjust("1 minute"); + const stop = yield* Deferred.await(stopped); + // pyenv releases that lock in an EXIT trap, which SIGTERM runs and SIGKILL skips. + assert.equal(stop.killSignal, "SIGTERM"); + assert.isAtLeast(Duration.toMillis(stop.forceKillAfter ?? 0), 5_000); + + yield* TestClock.adjust("30 seconds"); + yield* Deferred.await(spawned[2]!); + yield* TestClock.adjust("20 seconds"); + const snapshot = yield* Fiber.join(published); + assert.equal(env.PATH, "/fresh/bin:/usr/bin"); + assert.isTrue(Option.isSome(snapshot) && snapshot.value.discovered); + assert.equal( + Option.isSome(snapshot) ? snapshot.value.environment.PATH : undefined, + "/fresh/bin:/usr/bin", + ); + return true; + }), + }); + }), + ); + it.effect("recovers the login-shell PATH when the first probe outlasts the timeout", () => Effect.gen(function* () { const env: NodeJS.ProcessEnv = { SHELL: "/bin/zsh", PATH: "/usr/bin:/bin:/usr/sbin:/sbin" }; diff --git a/apps/desktop/src/shell/DesktopShellEnvironment.ts b/apps/desktop/src/shell/DesktopShellEnvironment.ts index f0983ca622490..351150aa4b987 100644 --- a/apps/desktop/src/shell/DesktopShellEnvironment.ts +++ b/apps/desktop/src/shell/DesktopShellEnvironment.ts @@ -3,6 +3,7 @@ import { applyShellEnvironment, buildEnvironmentCaptureCommand, listLoginShellCandidates, + mergePathValues, readUserLoginShell, setWindowsPath, } from "@t3tools/shared/shell"; @@ -14,6 +15,7 @@ import * as Deferred from "effect/Deferred"; import * as Path from "effect/Path"; import * as PubSub from "effect/PubSub"; import * as Ref from "effect/Ref"; +import * as Schedule from "effect/Schedule"; import * as Scope from "effect/Scope"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; @@ -27,9 +29,17 @@ import * as DesktopEnvironment from "../app/DesktopEnvironment.ts"; type EnvironmentPatch = Record; +interface ProbeBudget { + readonly timeout: Duration.Duration; + // One longer wait for the first login shell that times out. + readonly slowTimeout?: Duration.Duration; + readonly terminateGrace: Duration.Duration; +} + interface ShellEnvironmentConfig { readonly env: NodeJS.ProcessEnv; readonly platform: NodeJS.Platform; + readonly budget: ProbeBudget; } interface WindowsProbeOptions { @@ -88,6 +98,8 @@ export class DesktopShellEnvironment extends Context.Service< const LOGIN_SHELL_ENV_NAMES = [ "PATH", + "ANDROID_HOME", + "ANDROID_SDK_ROOT", "DBUS_SESSION_BUS_ADDRESS", "DISPLAY", "LANG", @@ -114,6 +126,7 @@ const LOGIN_SHELL_ENV_NAMES = [ ] as const; const WINDOWS_PROFILE_ENV_NAMES = ["PATH", "FNM_DIR", "FNM_MULTISHELL_PATH"] as const; const LOCALE_ENV_NAMES = ["LANG", "LC_ALL", "LC_CTYPE"] as const; +const ANDROID_SDK_ENV_NAMES = ["ANDROID_HOME", "ANDROID_SDK_ROOT"] as const; const FALLBACK_LC_CTYPE = "en_US.UTF-8"; const WINDOWS_SHELL_CANDIDATES = ["pwsh.exe", "powershell.exe"] as const; const LOGIN_SHELL_TIMEOUT = Duration.seconds(5); @@ -121,6 +134,20 @@ const LOGIN_SHELL_TIMEOUT = Duration.seconds(5); const SLOW_LOGIN_SHELL_TIMEOUT = Duration.seconds(20); const LAUNCHCTL_TIMEOUT = Duration.seconds(2); const PROCESS_TERMINATE_GRACE = Duration.seconds(1); +const STARTUP_PROBE: ProbeBudget = { + timeout: LOGIN_SHELL_TIMEOUT, + slowTimeout: SLOW_LOGIN_SHELL_TIMEOUT, + terminateGrace: PROCESS_TERMINATE_GRACE, +}; +// Retries block nothing, so they outlast pyenv's 60 s wait on its rehash lock, and SIGTERM gets +// time to run the exit trap that releases it. A SIGKILL leaves the lock for two minutes. +const RETRY_PROBE: ProbeBudget = { + timeout: Duration.minutes(2), + terminateGrace: Duration.seconds(10), +}; +// A Mac asleep at launch fails the first probe on wake, and nothing else would probe again. +const DISCOVERY_RETRY_DELAY = Duration.seconds(30); +const DISCOVERY_RETRIES = 10; const trimNonEmpty = (value: string | null | undefined): Option.Option => Option.fromNullishOr(value).pipe( @@ -128,10 +155,12 @@ const trimNonEmpty = (value: string | null | undefined): Option.Option = Option.filter((entry) => entry.length > 0), ); -const pathDelimiter = (platform: NodeJS.Platform) => (platform === "win32" ? ";" : ":"); +// POSIX directory names may begin or end with whitespace; mergePathValues trims only on Windows. +const nonEmptyPath = (value: string | null | undefined): Option.Option => + Option.fromNullishOr(value).pipe(Option.filter((entry) => entry.length > 0)); const readEnvPath = (env: NodeJS.ProcessEnv): Option.Option => - trimNonEmpty(env.PATH ?? env.Path ?? env.path); + nonEmptyPath(env.PATH ?? env.Path ?? env.path); const normalizeRuntimeDir = (value: string): string => value.replace(/\/+$/u, ""); @@ -150,39 +179,16 @@ const linuxRuntimeDirCandidates = ( return candidates.filter((candidate) => candidate.length > 0); }; -const pathComparisonKey = (entry: string, platform: NodeJS.Platform) => { - const normalized = entry.trim().replace(/^"+|"+$/g, ""); - return platform === "win32" ? normalized.toLowerCase() : normalized; -}; - -const sanitizePathEntry = (entry: string, platform: NodeJS.Platform) => - platform === "win32" ? entry.replaceAll('"', "") : entry; - const mergePaths = ( platform: NodeJS.Platform, values: ReadonlyArray>, -): Option.Option => { - const delimiter = pathDelimiter(platform); - const entries: string[] = []; - const seen = new Set(); - - for (const value of values) { - if (Option.isNone(value)) continue; - - for (const entry of value.value.split(delimiter)) { - const sanitized = sanitizePathEntry(entry.trim(), platform); - if (sanitized.length === 0) continue; - - const key = pathComparisonKey(sanitized, platform); - if (key.length === 0 || seen.has(key)) continue; - - seen.add(key); - entries.push(sanitized); - } - } - - return entries.length > 0 ? Option.some(entries.join(delimiter)) : Option.none(); -}; +): Option.Option => + Option.fromNullishOr( + values.reduce( + (merged, value) => mergePathValues(merged, Option.getOrUndefined(value), platform), + undefined, + ), + ); const knownWindowsCliDirs = (env: NodeJS.ProcessEnv): ReadonlyArray => [ ...trimNonEmpty(env.APPDATA).pipe( @@ -262,6 +268,7 @@ const runCommandOutput = Effect.fn("desktop.shellEnvironment.runCommandOutput")( readonly args: ReadonlyArray; readonly env: NodeJS.ProcessEnv; readonly timeout: Duration.Duration; + readonly terminateGrace?: Duration.Duration; readonly shell?: boolean; }): Effect.fn.Return< string, @@ -280,7 +287,7 @@ const runCommandOutput = Effect.fn("desktop.shellEnvironment.runCommandOutput")( stdout: "pipe", stderr: "ignore", killSignal: "SIGTERM", - forceKillAfter: PROCESS_TERMINATE_GRACE, + forceKillAfter: input.terminateGrace ?? PROCESS_TERMINATE_GRACE, }), ); return yield* Effect.all([Stream.mkString(Stream.decodeText(child.stdout)), child.exitCode], { @@ -346,6 +353,7 @@ const readLoginShellEnvironment = ( names: ReadonlyArray, env: NodeJS.ProcessEnv, timeout: Duration.Duration, + terminateGrace: Duration.Duration, ): Effect.Effect< EnvironmentPatch, DesktopShellEnvironmentCommandTimeoutError, @@ -359,6 +367,7 @@ const readLoginShellEnvironment = ( args: ["-ilc", buildEnvironmentCaptureCommand(shell, names)], env, timeout, + terminateGrace, }).pipe(Effect.map((output) => extractEnvironment(output, names))); const readLaunchctlPath = (env: NodeJS.ProcessEnv) => @@ -375,6 +384,7 @@ const readWindowsEnvironment = Effect.fn("desktop.shellEnvironment.readWindowsEn names: ReadonlyArray, options: WindowsProbeOptions, env: NodeJS.ProcessEnv, + budget: ProbeBudget, ): Effect.fn.Return { if (names.length === 0) return {}; @@ -392,7 +402,8 @@ const readWindowsEnvironment = Effect.fn("desktop.shellEnvironment.readWindowsEn command, args, env, - timeout: LOGIN_SHELL_TIMEOUT, + timeout: budget.timeout, + terminateGrace: budget.terminateGrace, }).pipe(orEmptyOnTimeout); const environment = extractEnvironment(output, names); if (Object.keys(environment).length > 0) { @@ -410,8 +421,13 @@ const installWindowsEnvironment = Effect.fn("desktop.shellEnvironment.installWin ): Effect.fn.Return { const [noProfile, profile] = yield* Effect.all( [ - readWindowsEnvironment(["PATH"], { loadProfile: false }, config.env), - readWindowsEnvironment(WINDOWS_PROFILE_ENV_NAMES, { loadProfile: true }, config.env), + readWindowsEnvironment(["PATH"], { loadProfile: false }, config.env, config.budget), + readWindowsEnvironment( + WINDOWS_PROFILE_ENV_NAMES, + { loadProfile: true }, + config.env, + config.budget, + ), ], { concurrency: 2 }, ); @@ -435,6 +451,9 @@ const installWindowsEnvironment = Effect.fn("desktop.shellEnvironment.installWin }, ); +const listUserShells = (env: NodeJS.ProcessEnv): ReadonlyArray => + [env.SHELL?.trim(), readUserLoginShell()].filter((shell): shell is string => Boolean(shell)); + const installPosixEnvironment = Effect.fn("desktop.shellEnvironment.installPosixEnvironment")( function* ( config: ShellEnvironmentConfig, @@ -445,22 +464,28 @@ const installPosixEnvironment = Effect.fn("desktop.shellEnvironment.installPosix > { const fileSystem = yield* FileSystem.FileSystem; const shellEnvironment: EnvironmentPatch = {}; - const userShells = [config.env.SHELL?.trim(), readUserLoginShell()]; + const userShells = listUserShells(config.env); let answeredBy: string | undefined; let waitedForSlowShell = false; for (const shell of listLoginShellCandidates(config.platform, config.env.SHELL)) { const read = (timeout: Duration.Duration) => - readLoginShellEnvironment(shell, LOGIN_SHELL_ENV_NAMES, config.env, timeout); + readLoginShellEnvironment( + shell, + LOGIN_SHELL_ENV_NAMES, + config.env, + timeout, + config.budget.terminateGrace, + ); Object.assign( shellEnvironment, - yield* read(LOGIN_SHELL_TIMEOUT).pipe( + yield* read(config.budget.timeout).pipe( Effect.catchTags({ DesktopShellEnvironmentCommandTimeoutError: (error) => { - const retry = !waitedForSlowShell; + const slowTimeout = waitedForSlowShell ? undefined : config.budget.slowTimeout; waitedForSlowShell = true; return logShellEnvironmentCommandError(error).pipe( - Effect.andThen(retry ? read(SLOW_LOGIN_SHELL_TIMEOUT) : Effect.succeed({})), + Effect.andThen(slowTimeout ? read(slowTimeout) : Effect.succeed({})), ); }, }), @@ -481,7 +506,7 @@ const installPosixEnvironment = Effect.fn("desktop.shellEnvironment.installPosix ? yield* readLaunchctlPath(config.env) : Option.none(); const mergedPath = mergePaths(config.platform, [ - trimNonEmpty(shellEnvironment.PATH).pipe(Option.orElse(() => launchctlPath)), + nonEmptyPath(shellEnvironment.PATH).pipe(Option.orElse(() => launchctlPath)), readEnvPath(config.env), ]); @@ -528,6 +553,17 @@ const installPosixEnvironment = Effect.fn("desktop.shellEnvironment.installPosix } } + // The device host picks the Android SDK from ANDROID_HOME before ANDROID_SDK_ROOT, + // so a profile-only ANDROID_HOME must not shadow an inherited ANDROID_SDK_ROOT. + if (ANDROID_SDK_ENV_NAMES.every((name) => Option.isNone(trimNonEmpty(config.env[name])))) { + for (const name of ANDROID_SDK_ENV_NAMES) { + const value = trimNonEmpty(shellEnvironment[name]); + if (Option.isSome(value)) { + config.env[name] = value.value; + } + } + } + // Hydrate locale variables together; inherited LC_ALL can override LANG or LC_CTYPE. if ( config.platform === "darwin" && @@ -563,7 +599,10 @@ const installPosixEnvironment = Effect.fn("desktop.shellEnvironment.installPosix } const prepared = Option.isSome(trimNonEmpty(shellEnvironment.PATH)); - return { prepared, fromUserShell: prepared && userShells.includes(answeredBy) }; + return { + prepared, + fromUserShell: prepared && answeredBy !== undefined && userShells.includes(answeredBy), + }; }, ); @@ -647,38 +686,60 @@ export const make = Effect.gen(function* () { yield* Ref.set(latest, Option.some(message)); yield* PubSub.publish(changes, message); }); - const refresh = Effect.gen(function* () { - // Refresh from the inherited baseline so removed cached paths and values do not survive. - const fresh = { ...inherited }; - const discovery = yield* installShellEnvironment({ - env: fresh, - platform: environment.platform, - }); - const { prepared } = discovery; - const patch = Object.fromEntries( - Object.keys(ShellEnvironmentPatch.fields).map((name) => [name, fresh[name] ?? null]), - ); - if (prepared || Option.isNone(yield* Ref.get(latest))) yield* publish(patch, prepared); - if (prepared) { - const cached = Object.fromEntries( - CACHED_ENV_NAMES.flatMap((name) => (fresh[name] ? [[name, fresh[name]]] : [])), - ); - const persist = Effect.gen(function* () { - const encoded = yield* encodeCache({ version: 1, key, environment: cached }); - yield* fileSystem.makeDirectory(environment.stateDir, { recursive: true }); - yield* fileSystem.writeFileString(cachePath + ".tmp", encoded, { mode: 0o600 }); - yield* fileSystem.rename(cachePath + ".tmp", cachePath); + const probe = Effect.fn("desktop.shellEnvironment.refresh")( + function* (budget: ProbeBudget) { + // Refresh from the inherited baseline so removed cached paths and values do not survive. + const fresh = { ...inherited }; + const discovery = yield* installShellEnvironment({ + env: fresh, + platform: environment.platform, + budget, }); - yield* persist.pipe( - Effect.catch(() => Effect.logWarning("Could not cache shell environment")), + const { prepared } = discovery; + const patch = Object.fromEntries( + Object.keys(ShellEnvironmentPatch.fields).map((name) => [name, fresh[name] ?? null]), ); - } - return discovery; - }).pipe( + if (prepared || Option.isNone(yield* Ref.get(latest))) yield* publish(patch, prepared); + if (prepared) { + const cached = Object.fromEntries( + CACHED_ENV_NAMES.flatMap((name) => (fresh[name] ? [[name, fresh[name]]] : [])), + ); + const persist = Effect.gen(function* () { + const encoded = yield* encodeCache({ version: 1, key, environment: cached }); + yield* fileSystem.makeDirectory(environment.stateDir, { recursive: true }); + yield* fileSystem.writeFileString(cachePath + ".tmp", encoded, { mode: 0o600 }); + yield* fileSystem.rename(cachePath + ".tmp", cachePath); + }); + yield* persist.pipe( + Effect.catch(() => Effect.logWarning("Could not cache shell environment")), + ); + } + return discovery; + }, Effect.provideService(FileSystem.FileSystem, fileSystem), Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), - Effect.withSpan("desktop.shellEnvironment.refresh"), + ); + // A fallback shell's PATH can miss folders the user's own shell adds, so keep asking that shell. + const awaitsUserShell = listUserShells(inherited).length > 0; + const settled = ({ prepared, fromUserShell }: ShellDiscovery) => + fromUserShell || (prepared && !awaitsUserShell); + const refresh = probe(STARTUP_PROBE).pipe( Effect.onExit((exit) => Deferred.done(discovered, exit)), + Effect.flatMap((discovery) => + settled(discovery) + ? Effect.void + : Effect.sleep(DISCOVERY_RETRY_DELAY).pipe( + Effect.andThen( + probe(RETRY_PROBE).pipe( + Effect.repeat({ + until: settled, + schedule: Schedule.spaced(DISCOVERY_RETRY_DELAY), + times: DISCOVERY_RETRIES - 1, + }), + ), + ), + ), + ), ); const installIntoProcess = yield* Effect.cached( Effect.gen(function* () { diff --git a/apps/desktop/src/telemetry/DesktopTelemetryPublisher.ts b/apps/desktop/src/telemetry/DesktopTelemetryPublisher.ts index 0a2b10e08c5a8..415901878cafe 100644 --- a/apps/desktop/src/telemetry/DesktopTelemetryPublisher.ts +++ b/apps/desktop/src/telemetry/DesktopTelemetryPublisher.ts @@ -352,6 +352,8 @@ export const make = Effect.fn("desktop.telemetryPublisher.make")(function* () { return Queue.offer(updateCommitQueue, message).pipe(Effect.asVoid); case "cancelDesktopUpdate": return Queue.offer(updateCancellationQueue, message).pipe(Effect.asVoid); + case "serverReady": + return Effect.void; } }; const removeControlSource: DesktopTelemetryPublisher["Service"]["removeControlSource"] = ( diff --git a/apps/desktop/src/updates/DesktopRemoteUpdates.test.ts b/apps/desktop/src/updates/DesktopRemoteUpdates.test.ts index 1f5321148b77b..fdac65f86a595 100644 --- a/apps/desktop/src/updates/DesktopRemoteUpdates.test.ts +++ b/apps/desktop/src/updates/DesktopRemoteUpdates.test.ts @@ -102,6 +102,29 @@ describe("DesktopRemoteUpdates", () => { assert.equal(terminals[0]?.requestId, "req-1"); assert.equal(harness.quitAndInstalls(), 0); + yield* Queue.offer(requests, request("req-retry")); + yield* settle; + assert.deepEqual( + terminalReports(reports).map((report) => report.outcome), + ["ready-to-install", "ready-to-install"], + ); + assert.equal(terminalReports(reports)[1]?.requestId, "req-retry"); + assert.equal(terminalReports(reports)[1]?.preparedRequestId, "req-1"); + assert.equal(harness.quitAndInstalls(), 0); + assert.equal(harness.downloadCount(), 1); + + yield* Queue.offer(requests, { + ...request("req-other-track"), + channel: "nightly", + targetVersion: "1.2.4-nightly.20261027.1", + }); + yield* settle; + assert.equal(terminalReports(reports)[2]?.outcome, "failed"); + assert.equal( + terminalReports(reports)[2]?.reason, + "A prepared desktop update is already in progress.", + ); + yield* Queue.offer(commits, { version: 1, type: "commitDesktopUpdate", @@ -119,6 +142,49 @@ describe("DesktopRemoteUpdates", () => { ); }); + it.effect("does not renew the preparation lease when a retry recovers its token", () => { + const harness = makeHarness(); + + return runRemoteUpdatesTest(harness, ({ reports, requests, commits }) => + Effect.gen(function* () { + harness.emit("update-downloaded", { version: "1.2.4" }); + yield* settle; + yield* Queue.offer(requests, request("req-original")); + yield* settle; + + yield* TestClock.adjust(Duration.minutes(4)); + harness.emit("update-available", { version: "1.2.4" }); + yield* settle; + yield* Queue.offer(requests, request("req-retry")); + yield* settle; + assert.equal(terminalReports(reports)[1]?.preparedRequestId, "req-original"); + + yield* TestClock.adjust(Duration.minutes(1)); + yield* Queue.offer(commits, { + version: 1, + type: "commitDesktopUpdate", + requestId: "req-original", + }); + yield* settle; + assert.equal( + terminalReports(reports)[2]?.reason, + "This desktop update is no longer prepared.", + ); + assert.equal(harness.quitAndInstalls(), 0); + + yield* Queue.offer(requests, request("req-new")); + yield* settle; + yield* Queue.offer(commits, { + version: 1, + type: "commitDesktopUpdate", + requestId: "req-new", + }); + yield* settle; + assert.equal(harness.quitAndInstalls(), 1); + }), + ); + }); + it.effect("switches to the requested track before preparing its release", () => { // In-memory settings keep the channel write off real disk I/O. const harness = makeHarness({ beforeSetUpdateChannel: Effect.void }); @@ -691,7 +757,7 @@ describe("DesktopRemoteUpdates", () => { const backgroundCheck = yield* updates.check("poll").pipe(Effect.forkChild); yield* Deferred.await(backgroundCheckStarted); - assert.equal((yield* updates.getState).status, "checking"); + assert.isTrue(yield* updates.isActionActive); yield* Queue.offer(commits, { version: 1, @@ -701,7 +767,6 @@ describe("DesktopRemoteUpdates", () => { yield* settle; assert.equal(harness.quitAndInstalls(), 0); - harness.emit("update-downloaded", { version: "1.2.4" }); yield* Deferred.succeed(releaseBackgroundCheck, undefined); yield* Fiber.join(backgroundCheck); yield* settle; diff --git a/apps/desktop/src/updates/DesktopRemoteUpdates.ts b/apps/desktop/src/updates/DesktopRemoteUpdates.ts index 560e2e78d4bb1..ce2f23ab3e781 100644 --- a/apps/desktop/src/updates/DesktopRemoteUpdates.ts +++ b/apps/desktop/src/updates/DesktopRemoteUpdates.ts @@ -32,6 +32,8 @@ const PREPARED_UPDATE_TTL = Duration.minutes(5); interface PreparedUpdate { readonly requestId: string; + readonly channel: DesktopTelemetryRequestDesktopUpdate["channel"]; + readonly targetVersion: DesktopTelemetryRequestDesktopUpdate["targetVersion"]; readonly downloadedVersion: string; readonly status: "prepared" | "committing" | "failed"; readonly failureReason?: string; @@ -67,7 +69,11 @@ export const listen: Effect.Effect< const publishReport = ( state: DesktopUpdateState, - terminal?: { readonly outcome: DesktopUpdateRemoteOutcome; readonly reason?: string }, + terminal?: { + readonly outcome: DesktopUpdateRemoteOutcome; + readonly reason?: string; + readonly preparedRequestId?: string; + }, explicitRequestId?: string, ): Effect.Effect => { const reason = normalizeRemoteUpdateReason(terminal?.reason); @@ -82,6 +88,9 @@ export const listen: Effect.Effect< ? { requestId: requestId.value } : {}), ...(terminal === undefined ? {} : { outcome: terminal.outcome }), + ...(terminal?.preparedRequestId === undefined + ? {} + : { preparedRequestId: terminal.preparedRequestId }), ...(reason ? { reason } : {}), state, }), @@ -167,8 +176,22 @@ export const listen: Effect.Effect< : ([current, current] as const), ); if (Option.isSome(prepared) && prepared.value.status !== "failed") { + const state = yield* updates.getState; + if ( + prepared.value.status === "prepared" && + prepared.value.channel === request.channel && + prepared.value.targetVersion === request.targetVersion && + state.downloadedVersion === prepared.value.downloadedVersion + ) { + yield* publishReport( + state, + { outcome: "ready-to-install", preparedRequestId: prepared.value.requestId }, + request.requestId, + ); + return; + } yield* publishReport( - yield* updates.getState, + state, { outcome: "failed", reason: "A prepared desktop update is already in progress.", @@ -301,6 +324,8 @@ export const listen: Effect.Effect< preparedUpdateRef, Option.some({ requestId: request.requestId, + channel: request.channel, + targetVersion: request.targetVersion, downloadedVersion: state.downloadedVersion, status: "prepared", expiresAt: diff --git a/apps/desktop/src/updates/DesktopUpdates.test.ts b/apps/desktop/src/updates/DesktopUpdates.test.ts index 7ee2ff5dd51c1..da08e427235aa 100644 --- a/apps/desktop/src/updates/DesktopUpdates.test.ts +++ b/apps/desktop/src/updates/DesktopUpdates.test.ts @@ -125,7 +125,7 @@ describe("DesktopUpdates", () => { ).pipe(Effect.provide(harness.layer)); }); - it.effect("checks for newer releases after an update has been downloaded", () => { + it.effect("re-checks an offered update without announcing it again", () => { const harness = makeHarness(); return Effect.scoped( @@ -138,14 +138,23 @@ describe("DesktopUpdates", () => { releaseNotes: "## What's changed\n- fix: queued update", }); yield* flushCallbacks; + const offeredStates = harness.sentStates.length; + + yield* updates.check("poll"); + harness.emit("update-available", { version: "1.2.4" }); + yield* flushCallbacks; + assert.equal(harness.sentStates.length, offeredStates); + harness.emit("update-downloaded", { version: "1.2.4" }); yield* flushCallbacks; + const downloadedStates = harness.sentStates.length; const result = yield* updates.check("poll"); assert.isTrue(result.checked); harness.emit("update-available", { version: "1.2.4" }); yield* flushCallbacks; + assert.equal(harness.sentStates.length, downloadedStates); const unchangedState = yield* updates.getState; assert.equal(unchangedState.status, "downloaded"); @@ -267,15 +276,26 @@ describe("DesktopUpdates", () => { ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); }); - it.effect( - "rejects install while a refresh check is in progress and releases the reservation", - () => + it.effect.each([ + { action: "download", offer: "update-available", found: null, accepted: true }, + { action: "install", offer: "update-downloaded", found: null, accepted: true }, + { action: "download", offer: "update-available", found: "1.2.5", accepted: true }, + { action: "install", offer: "update-downloaded", found: "1.2.5", accepted: false }, + ] as const)( + "a $action waiting out a refresh check acts on what it found ($found)", + ({ action, offer, found, accepted }) => Effect.gen(function* () { const checkStarted = yield* Deferred.make(); const releaseCheck = yield* Deferred.make(); const harness = makeHarness({ checkForUpdates: Deferred.succeed(checkStarted, undefined).pipe( Effect.andThen(Deferred.await(releaseCheck)), + // electron-updater emits the check's outcome before its promise resolves. + Effect.andThen( + Effect.sync(() => { + if (found !== null) harness.emit("update-available", { version: found }); + }), + ), ), }); @@ -283,22 +303,20 @@ describe("DesktopUpdates", () => { Effect.gen(function* () { const updates = yield* DesktopUpdates.DesktopUpdates; yield* updates.configure; - harness.emit("update-downloaded", { version: "1.2.4" }); + harness.emit(offer, { version: "1.2.4" }); yield* flushCallbacks; - const checkFiber = yield* updates.check("manual").pipe(Effect.forkScoped); + const checkFiber = yield* updates.check("poll").pipe(Effect.forkScoped); yield* Deferred.await(checkStarted); - - const installResult = yield* updates.install; - assert.isFalse(installResult.accepted); + const actionFiber = yield* updates[action].pipe(Effect.forkScoped); + yield* flushCallbacks; yield* Deferred.succeed(releaseCheck, undefined); - const checkResult = yield* Fiber.join(checkFiber); - assert.isTrue(checkResult.checked); - - const followUpCheck = yield* updates.check("manual"); - assert.isTrue(followUpCheck.checked); - assert.equal(harness.checkCount(), 2); + assert.isTrue((yield* Fiber.join(checkFiber)).checked); + assert.equal((yield* Fiber.join(actionFiber)).accepted, accepted); + const state = yield* updates.getState; + assert.equal(state.availableVersion, found ?? "1.2.4"); + if (action === "download") assert.equal(state.status, "downloading"); }), ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); }), diff --git a/apps/desktop/src/updates/DesktopUpdates.ts b/apps/desktop/src/updates/DesktopUpdates.ts index db31e0bc112ba..055c6574391a6 100644 --- a/apps/desktop/src/updates/DesktopUpdates.ts +++ b/apps/desktop/src/updates/DesktopUpdates.ts @@ -63,7 +63,7 @@ const CHANNEL_CHANGE_RETRY_DELAY = Duration.seconds(1); const CHANNEL_CHANGE_RETRIES = 30; const AUTO_UPDATE_STARTUP_DELAY = "15 seconds"; const AUTO_UPDATE_POLL_INTERVAL = "4 hours"; -const PREPARED_INSTALL_CHECK_WAIT = Duration.seconds(90); +const UPDATE_CHECK_WAIT = Duration.seconds(90); type UpdateAction = "check" | "download" | "install" | "install-recovery" | "channel"; @@ -318,6 +318,10 @@ function getCanRetryFromState(state: DesktopUpdateState): boolean { return state.availableVersion !== null || state.downloadedVersion !== null; } +function isOffered(state: DesktopUpdateState): boolean { + return state.status === "available" || state.status === "downloaded"; +} + function shouldBroadcastDownloadProgress( currentState: DesktopUpdateState, nextPercent: number, @@ -518,6 +522,23 @@ export const make = Effect.gen(function* () { ), ); + // Re-runs admit after each finished action, so a caller waiting on a check decides on its result. + const admitAfterCheck = Effect.fn("desktop.updates.admitAfterCheck")(function* ( + admit: Effect.Effect<"admitted" | "refused" | "wait-for-check">, + ) { + const actionCompletions = yield* PubSub.subscribe(finishedUpdateActions); + let admission = yield* stateMutex.withPermits(1)(admit); + while (admission === "wait-for-check") { + const finishedAction = yield* PubSub.take(actionCompletions).pipe( + Effect.timeoutOption(UPDATE_CHECK_WAIT), + ); + admission = Option.isNone(finishedAction) + ? "refused" + : yield* stateMutex.withPermits(1)(admit); + } + return admission; + }); + const applyAutoUpdaterChannel = Effect.fn("desktop.updates.applyAutoUpdaterChannel")(function* ( channel: DesktopUpdateChannel, ) { @@ -559,8 +580,10 @@ export const make = Effect.gen(function* () { if (actionReservation === "acquire" && !(yield* tryStartUpdateAction("check"))) return false; const check = Effect.gen(function* () { - const checkedAt = yield* currentIsoTimestamp; - yield* setState(reduceDesktopUpdateStateOnCheckStart(state, checkedAt)); + // An offer stays on screen while re-checking, so finding it again changes nothing. + if (!isOffered(state)) { + yield* setState(reduceDesktopUpdateStateOnCheckStart(state, yield* currentIsoTimestamp)); + } yield* logUpdaterInfo("checking for updates", { reason }); // The flag is shared updater state, so it flips only while this check holds the reservation. @@ -597,15 +620,30 @@ export const make = Effect.gen(function* () { }); const downloadAvailableUpdate = Effect.gen(function* () { - const state = yield* Ref.get(updateStateRef); - if (!(yield* Ref.get(updaterConfiguredRef)) || state.status !== "available") { - return { accepted: false, completed: false }; - } - - if (!(yield* tryStartUpdateAction("download"))) { + const admission = yield* Effect.scoped( + admitAfterCheck( + Effect.gen(function* () { + const state = yield* Ref.get(updateStateRef); + if (!(yield* Ref.get(updaterConfiguredRef)) || state.status !== "available") { + return "refused" as const; + } + const activeAction = yield* Ref.get(activeUpdateActionRef); + if (Option.isSome(activeAction)) { + return activeAction.value === "check" + ? ("wait-for-check" as const) + : ("refused" as const); + } + return (yield* tryStartUpdateAction("download")) + ? ("admitted" as const) + : ("refused" as const); + }), + ), + ); + if (admission === "refused") { return { accepted: false, completed: false }; } + const state = yield* Ref.get(updateStateRef); return yield* Effect.gen(function* () { yield* setState(reduceDesktopUpdateStateOnDownloadStart(state)); yield* electronUpdater.setDisableDifferentialDownload( @@ -798,47 +836,35 @@ export const make = Effect.gen(function* () { const installDownloadedUpdate = (expectedVersion?: string) => Effect.scoped( Effect.gen(function* () { - const actionCompletions = yield* PubSub.subscribe(finishedUpdateActions); - let admission: "admitted" | "refused" | "wait-for-check" = "wait-for-check"; - while (admission === "wait-for-check") { - admission = yield* stateMutex.withPermits(1)( - Effect.gen(function* () { - const state = yield* Ref.get(updateStateRef); - const activeAction = yield* Ref.get(activeUpdateActionRef); - const hasExpectedDownload = - state.downloadedVersion !== null && - (expectedVersion === undefined || state.downloadedVersion === expectedVersion); - if ( - (yield* Ref.get(desktopState.quitting)) || - !(yield* Ref.get(updaterConfiguredRef)) || - !hasExpectedDownload - ) { - return "refused" as const; - } - if (Option.isSome(activeAction)) { - return activeAction.value === "check" && expectedVersion !== undefined - ? ("wait-for-check" as const) - : ("refused" as const); - } - const hasInstallableDownload = - state.status === "downloaded" || - (state.status === "error" && - (state.errorContext === null || state.errorContext === "install")); - if (!hasInstallableDownload) return "refused" as const; - return (yield* tryStartUpdateAction("install")) - ? ("admitted" as const) + const admission = yield* admitAfterCheck( + Effect.gen(function* () { + const state = yield* Ref.get(updateStateRef); + const activeAction = yield* Ref.get(activeUpdateActionRef); + const hasExpectedDownload = + state.downloadedVersion !== null && + (expectedVersion === undefined || state.downloadedVersion === expectedVersion); + if ( + (yield* Ref.get(desktopState.quitting)) || + !(yield* Ref.get(updaterConfiguredRef)) || + !hasExpectedDownload + ) { + return "refused" as const; + } + if (Option.isSome(activeAction)) { + return activeAction.value === "check" + ? ("wait-for-check" as const) : ("refused" as const); - }), - ); - if (admission === "wait-for-check") { - const finishedAction = yield* PubSub.take(actionCompletions).pipe( - Effect.timeoutOption(PREPARED_INSTALL_CHECK_WAIT), - ); - if (Option.isNone(finishedAction)) { - admission = "refused"; } - } - } + const hasInstallableDownload = + state.status === "downloaded" || + (state.status === "error" && + (state.errorContext === null || state.errorContext === "install")); + if (!hasInstallableDownload) return "refused" as const; + return (yield* tryStartUpdateAction("install")) + ? ("admitted" as const) + : ("refused" as const); + }), + ); if (admission === "refused") { return { accepted: false, completed: false, failed: false }; } @@ -982,6 +1008,7 @@ export const make = Effect.gen(function* () { yield* Ref.set(lastLoggedDownloadMilestoneRef, -1); return; } + if (isOffered(state) && state.availableVersion === info.version) return; const checkedAt = yield* currentIsoTimestamp; const { releaseNotes, omittedReleaseCount } = normalizeDesktopUpdateReleaseNotes( @@ -1161,6 +1188,8 @@ export const make = Effect.gen(function* () { ), configure: Effect.gen(function* () { const context = yield* Effect.context(); + // Runs synchronously up to the first await, so a check's outcome is applied before the + // check's promise resolves and wakes the actions waiting on it. const runEffect = (effect: Effect.Effect) => { void Effect.runPromiseWith(context)(effect); }; diff --git a/apps/desktop/src/window/DesktopApplicationMenu.test.ts b/apps/desktop/src/window/DesktopApplicationMenu.test.ts index 612ec441ba99b..46004ca302295 100644 --- a/apps/desktop/src/window/DesktopApplicationMenu.test.ts +++ b/apps/desktop/src/window/DesktopApplicationMenu.test.ts @@ -3,7 +3,9 @@ import { assert, describe, it } from "@effect/vitest"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import type { DesktopUpdateCheckResult, DesktopUpdateState } from "@t3tools/contracts"; import * as Electron from "electron"; import { vi } from "vite-plus/test"; @@ -18,6 +20,7 @@ import * as DesktopApplicationMenu from "./DesktopApplicationMenu.ts"; import * as DesktopConfig from "../app/DesktopConfig.ts"; import * as DesktopEnvironment from "../app/DesktopEnvironment.ts"; import * as DesktopUpdates from "../updates/DesktopUpdates.ts"; +import { createInitialDesktopUpdateState } from "../updates/updateMachine.ts"; import * as DesktopWindow from "./DesktopWindow.ts"; const environmentInput = { @@ -56,27 +59,37 @@ const layerElectronApp = Layer.succeed(ElectronApp.ElectronApp, { on: () => Effect.void, } satisfies ElectronApp.ElectronApp["Service"]); -const layerElectronDialog = Layer.succeed(ElectronDialog.ElectronDialog, { - pickFolder: () => Effect.succeedNone, - pickFiles: () => Effect.succeed([]), - showMessageBox: () => Effect.succeed({ response: 0, checkboxChecked: false }), - showErrorBox: () => Effect.void, -} satisfies ElectronDialog.ElectronDialog["Service"]); - -const layerDesktopUpdates = Layer.succeed(DesktopUpdates.DesktopUpdates, { - getState: Effect.die("unexpected getState"), - isActionActive: Effect.succeed(false), - isInstallActive: Effect.succeed(false), - subscribe: Effect.die("unexpected subscribe"), - emitState: Effect.void, - disabledReason: Effect.succeedNone, - configure: Effect.void, - setChannel: () => Effect.die("unexpected setChannel"), - check: () => Effect.die("unexpected check"), - download: Effect.die("unexpected download"), - install: Effect.die("unexpected install"), - installPrepared: () => Effect.die("unexpected installPrepared"), -} satisfies DesktopUpdates.DesktopUpdates["Service"]); +interface MenuServices { + readonly checkResult?: DesktopUpdateCheckResult; + readonly shownMessageBox?: Deferred.Deferred; +} + +const layerElectronDialog = ({ shownMessageBox }: MenuServices) => + Layer.succeed(ElectronDialog.ElectronDialog, { + pickFolder: () => Effect.succeedNone, + pickFiles: () => Effect.succeed([]), + showMessageBox: (options) => + (shownMessageBox ? Deferred.succeed(shownMessageBox, options) : Effect.void).pipe( + Effect.as({ response: options.cancelId ?? 0, checkboxChecked: false }), + ), + showErrorBox: () => Effect.void, + } satisfies ElectronDialog.ElectronDialog["Service"]); + +const layerDesktopUpdates = ({ checkResult }: MenuServices) => + Layer.succeed(DesktopUpdates.DesktopUpdates, { + getState: Effect.die("unexpected getState"), + isActionActive: Effect.succeed(false), + isInstallActive: Effect.succeed(false), + subscribe: Effect.die("unexpected subscribe"), + emitState: Effect.void, + disabledReason: Effect.succeedNone, + configure: Effect.void, + setChannel: () => Effect.die("unexpected setChannel"), + check: () => (checkResult ? Effect.succeed(checkResult) : Effect.die("unexpected check")), + download: Effect.die("unexpected download"), + install: Effect.die("unexpected install"), + installPrepared: () => Effect.die("unexpected installPrepared"), + } satisfies DesktopUpdates.DesktopUpdates["Service"]); const layerDesktopWindow = (selectedAction: Deferred.Deferred) => Layer.succeed(DesktopWindow.DesktopWindow, { @@ -113,6 +126,7 @@ const configureMenu = ( selectedAction: Deferred.Deferred, applicationMenuTemplate: Deferred.Deferred, environment: Partial = {}, + services: MenuServices = {}, ) => Effect.gen(function* () { const menu = yield* DesktopApplicationMenu.DesktopApplicationMenu; @@ -122,8 +136,8 @@ const configureMenu = ( DesktopApplicationMenu.layer.pipe( Layer.provideMerge(layerElectronMenu(applicationMenuTemplate)), Layer.provideMerge(layerDesktopWindow(selectedAction)), - Layer.provideMerge(layerDesktopUpdates), - Layer.provideMerge(layerElectronDialog), + Layer.provideMerge(layerDesktopUpdates(services)), + Layer.provideMerge(layerElectronDialog(services)), Layer.provideMerge(layerElectronApp), Layer.provideMerge( DesktopEnvironment.layer({ ...environmentInput, ...environment }).pipe( @@ -134,6 +148,54 @@ const configureMenu = ( ), ); +const checkForUpdatesFromMacMenu = (state: Partial) => + Effect.gen(function* () { + const applicationMenuTemplate = + yield* Deferred.make(); + const shownMessageBox = yield* Deferred.make(); + yield* configureMenu( + yield* Deferred.make(), + applicationMenuTemplate, + { platform: "darwin" }, + { + checkResult: { + checked: true, + state: { + ...createInitialDesktopUpdateState( + "0.1.5", + { hostArch: "arm64", appArch: "arm64", runningUnderArm64Translation: false }, + "latest", + ), + enabled: true, + ...state, + }, + }, + shownMessageBox, + }, + ); + + const appMenu = (yield* Deferred.await(applicationMenuTemplate))[0]; + if (!Array.isArray(appMenu?.submenu)) { + throw new Error("Expected the macOS app menu submenu to be an array."); + } + const checkForUpdates = appMenu.submenu.find((item) => item.label === "Check for Updates..."); + if (typeof checkForUpdates?.click !== "function") { + throw new Error("Expected Check for Updates... menu item to have a click handler."); + } + + // Every stub completes synchronously, so the menu action has finished when click returns. + checkForUpdates.click( + {} as Electron.MenuItem, + {} as Electron.BrowserWindow, + {} as KeyboardEvent, + ); + const dialog = yield* Deferred.poll(shownMessageBox); + if (Option.isNone(dialog)) { + throw new Error("Check for Updates... finished without showing any dialog."); + } + return yield* dialog.value; + }); + describe("DesktopApplicationMenu", () => { it.effect("leaves the accelerator to Chromium instead of injecting a paste", () => Effect.gen(function* () { @@ -285,4 +347,25 @@ describe("DesktopApplicationMenu", () => { assert.equal(yield* Deferred.await(selectedAction), "main-reload"); }), ); + it.effect("tells the user which update the menu check found", () => + Effect.gen(function* () { + const dialog = yield* checkForUpdatesFromMacMenu({ + status: "available", + availableVersion: "0.1.7", + }); + assert.include(`${dialog.title ?? ""} ${dialog.message} ${dialog.detail ?? ""}`, "0.1.7"); + }), + ); + + it.effect("tells the user a downloaded update is ready when the menu check runs", () => + Effect.gen(function* () { + const dialog = yield* checkForUpdatesFromMacMenu({ + status: "downloaded", + availableVersion: "0.1.7", + downloadedVersion: "0.1.7", + downloadPercent: 100, + }); + assert.include(`${dialog.title ?? ""} ${dialog.message} ${dialog.detail ?? ""}`, "0.1.7"); + }), + ); }); diff --git a/apps/desktop/src/window/DesktopApplicationMenu.ts b/apps/desktop/src/window/DesktopApplicationMenu.ts index 44d2c234f8358..b4025ebfde8ef 100644 --- a/apps/desktop/src/window/DesktopApplicationMenu.ts +++ b/apps/desktop/src/window/DesktopApplicationMenu.ts @@ -64,6 +64,9 @@ const runMainContentsCommand = Effect.fn("desktop.menu.runMainContentsCommand")( yield* desktopWindow.runMainContentsCommand(command); }); +const describeUpdate = (version: string | null) => + version === null ? "A new version of T3 Code" : `T3 Code ${version}`; + const checkForUpdatesFromMenu = Effect.gen(function* () { const updates = yield* DesktopUpdates.DesktopUpdates; const electronDialog = yield* ElectronDialog.ElectronDialog; @@ -77,6 +80,35 @@ const checkForUpdatesFromMenu = Effect.gen(function* () { message: `T3 Code ${updateState.currentVersion} is currently the newest version available.`, buttons: ["OK"], }); + } else if (updateState.status === "available") { + const { response } = yield* electronDialog.showMessageBox({ + type: "info", + title: "Update available", + message: `${describeUpdate(updateState.availableVersion)} is available.`, + detail: `You're running T3 Code ${updateState.currentVersion}.`, + buttons: ["Download", "Later"], + defaultId: 0, + cancelId: 1, + }); + if (response === 0) yield* updates.download; + } else if (updateState.status === "downloading") { + yield* electronDialog.showMessageBox({ + type: "info", + title: "Update downloading", + message: `${describeUpdate(updateState.availableVersion)} is already downloading.`, + buttons: ["OK"], + }); + } else if (updateState.status === "downloaded") { + const { response } = yield* electronDialog.showMessageBox({ + type: "info", + title: "Update ready", + message: `${describeUpdate(updateState.downloadedVersion ?? updateState.availableVersion)} is ready to install.`, + detail: "T3 Code will restart to install it. Any running tasks will be interrupted.", + buttons: ["Restart", "Later"], + defaultId: 0, + cancelId: 1, + }); + if (response === 0) yield* updates.install; } else if (updateState.status === "error") { yield* electronDialog.showMessageBox({ type: "warning", diff --git a/apps/desktop/src/window/DesktopWindow.test.ts b/apps/desktop/src/window/DesktopWindow.test.ts index b446b9df5c8cb..93a0a0eec2afa 100644 --- a/apps/desktop/src/window/DesktopWindow.test.ts +++ b/apps/desktop/src/window/DesktopWindow.test.ts @@ -207,17 +207,20 @@ const layerElectronTheme = Layer.succeed(ElectronTheme.ElectronTheme, { onUpdated: () => Effect.void, } satisfies ElectronTheme.ElectronTheme["Service"]); -const layerDesktopEnvironment = DesktopEnvironment.layer(environmentInput).pipe( - Layer.provide( - Layer.mergeAll( - NodeServices.layer, - DesktopConfig.layerTest({ - T3CODE_PORT: "3773", - VITE_DEV_SERVER_URL: "http://127.0.0.1:5733", - }), +const layerDesktopEnvironmentFor = (platform: NodeJS.Platform) => + DesktopEnvironment.layer({ ...environmentInput, platform }).pipe( + Layer.provide( + Layer.mergeAll( + NodeServices.layer, + DesktopConfig.layerTest({ + T3CODE_PORT: "3773", + VITE_DEV_SERVER_URL: "http://127.0.0.1:5733", + }), + ), ), - ), -); + ); + +const layerDesktopEnvironment = layerDesktopEnvironmentFor(environmentInput.platform); const desktopWindowBoundsEquivalence = Schema.toEquivalence( DesktopAppSettings.DesktopWindowBoundsSchema, @@ -244,6 +247,12 @@ function layerTest(input: { readonly previewZoomReapplies?: number[]; readonly onReveal?: (window: Electron.BrowserWindow) => void; readonly focusedWindow?: Electron.BrowserWindow; + readonly fallbackWindow?: Electron.BrowserWindow; + readonly platform?: NodeJS.Platform; + readonly appearanceWindows?: ReadonlyArray<{ + readonly window: Electron.BrowserWindow; + readonly titleBarOverlay: boolean; + }>; readonly primaryPasteEnabled?: boolean; readonly previewPasskeyBridgeEnabled?: boolean; }) { @@ -292,7 +301,9 @@ function layerTest(input: { Effect.as(input.window), ), main: Ref.get(input.mainWindow), - currentMainOrFirst: Ref.get(input.mainWindow), + currentMainOrFirst: Ref.get(input.mainWindow).pipe( + Effect.map(Option.orElse(() => Option.fromNullishOr(input.fallbackWindow ?? null))), + ), focusedMainOrFirst: input.focusedWindow === undefined ? Ref.get(input.mainWindow) @@ -303,7 +314,14 @@ function layerTest(input: { reveal: (window) => Effect.sync(() => input.onReveal?.(window)), sendAll: () => Effect.void, destroyAll: input.destroyAll ?? Effect.void, - syncAllAppearance: (sync) => sync(input.window), + syncAllAppearance: (sync) => + input.appearanceWindows === undefined + ? sync(input.window, { titleBarOverlay: true }) + : Effect.forEach( + input.appearanceWindows, + ({ window, titleBarOverlay }) => sync(window, { titleBarOverlay }), + { discard: true }, + ), } satisfies ElectronWindow.ElectronWindow["Service"]); return DesktopWindow.layer.pipe( @@ -315,7 +333,9 @@ function layerTest(input: { recordMetrics: () => Effect.void, shutdown: Effect.void, }), - layerDesktopEnvironment, + input.platform === undefined + ? layerDesktopEnvironment + : layerDesktopEnvironmentFor(input.platform), layerDesktopAppSettings, layerDesktopClientSettings, layerDesktopServerExposure, @@ -429,7 +449,8 @@ const makeSplashScenario = (createOutcomes: readonly (Electron.BrowserWindow | n reveal: (window) => Ref.update(revealedWindows, (windows) => [...windows, window]), sendAll: () => Effect.void, destroyAll: Effect.void, - syncAllAppearance: (sync) => (fallbackWindow ? sync(fallbackWindow) : Effect.void), + syncAllAppearance: (sync) => + fallbackWindow ? sync(fallbackWindow, { titleBarOverlay: true }) : Effect.void, } satisfies ElectronWindow.ElectronWindow["Service"]; const layer = DesktopWindow.layer.pipe( @@ -893,7 +914,7 @@ describe("DesktopWindow", () => { }), ); - it.effect("reloads and inspects the main window even when a popup has focus", () => + it.effect("routes menu commands to the main window even when a popup has focus", () => Effect.gen(function* () { const fakeWindow = makeFakeBrowserWindow(); const popup = makeFakeBrowserWindow(); @@ -913,7 +934,10 @@ describe("DesktopWindow", () => { yield* desktopWindow.runMainContentsCommand("reload"); yield* desktopWindow.runMainContentsCommand("forceReload"); yield* desktopWindow.runMainContentsCommand("toggleDevTools"); + yield* desktopWindow.dispatchMenuAction("open-settings"); + assert.deepEqual(fakeWindow.send.mock.calls, [["desktop:menu-action", "open-settings"]]); + assert.equal(popup.send.mock.calls.length, 0); assert.equal(fakeWindow.reload.mock.calls.length, 1); assert.equal(fakeWindow.reloadIgnoringCache.mock.calls.length, 1); assert.equal(fakeWindow.toggleDevTools.mock.calls.length, 1); @@ -924,6 +948,38 @@ describe("DesktopWindow", () => { }), ); + it.effect("recreates main for Settings when only an unregistered popup survives", () => + Effect.gen(function* () { + const main = makeFakeBrowserWindow(); + const auxiliary = makeFakeBrowserWindow(); + const createCount = yield* Ref.make(0); + const mainWindow = yield* Ref.make>(Option.none()); + + yield* Effect.gen(function* () { + const windows = yield* DesktopWindow.DesktopWindow; + yield* windows.dispatchMenuAction("open-settings"); + assert.deepEqual(main.send.mock.calls, [["desktop:menu-action", "open-settings"]]); + assert.equal(auxiliary.send.mock.calls.length, 0); + assert.equal(yield* Ref.get(createCount), 1); + assert.strictEqual(yield* windows.ensureMain, main.window); + assert.equal(yield* Ref.get(createCount), 1); + }).pipe( + Effect.provide( + layerTest({ + window: main.window, + createCount, + mainWindow, + fallbackWindow: auxiliary.window, + desktopSettings: { + ...DesktopAppSettings.DEFAULT_DESKTOP_SETTINGS, + localEnvironmentEnabled: false, + }, + }), + ), + ); + }), + ); + it.effect("does not open main or splash windows once quitting has started", () => Effect.gen(function* () { const fake = makeFakeBrowserWindow(); @@ -1237,6 +1293,36 @@ describe("DesktopWindow", () => { }), ); + it.effect("updates title bar overlays only on windows created with one", () => + Effect.gen(function* () { + const framed = makeFakeBrowserWindow(); + vi.mocked(framed.window.setTitleBarOverlay).mockImplementation(() => { + throw new TypeError("Titlebar overlay is not enabled"); + }); + const overlay = makeFakeBrowserWindow(); + const layer = layerTest({ + window: overlay.window, + createCount: yield* Ref.make(0), + mainWindow: yield* Ref.make>(Option.none()), + platform: "win32", + appearanceWindows: [ + { window: framed.window, titleBarOverlay: false }, + { window: overlay.window, titleBarOverlay: true }, + ], + }); + + yield* Effect.gen(function* () { + const desktopWindow = yield* DesktopWindow.DesktopWindow; + yield* desktopWindow.syncAppearance; + }).pipe(Effect.provide(layer)); + + assert.deepEqual(vi.mocked(framed.window.setBackgroundColor).mock.calls, [["#ffffff"]]); + assert.lengthOf(vi.mocked(framed.window.setTitleBarOverlay).mock.calls, 0); + assert.deepEqual(vi.mocked(overlay.window.setBackgroundColor).mock.calls, [["#ffffff"]]); + assert.lengthOf(vi.mocked(overlay.window.setTitleBarOverlay).mock.calls, 1); + }), + ); + it.effect.each([ { window: "normal", write: "flush" }, { window: "normal", write: "debounce" }, diff --git a/apps/desktop/src/window/DesktopWindow.ts b/apps/desktop/src/window/DesktopWindow.ts index eb76831602866..7d5dfcee7ed97 100644 --- a/apps/desktop/src/window/DesktopWindow.ts +++ b/apps/desktop/src/window/DesktopWindow.ts @@ -372,6 +372,7 @@ function getWindowTitleBarOptions( function syncWindowAppearance( window: Electron.BrowserWindow, + hasTitleBarOverlay: boolean, shouldUseDarkColors: boolean, platform: NodeJS.Platform, ): Effect.Effect { @@ -382,7 +383,7 @@ function syncWindowAppearance( window.setBackgroundColor(getInitialWindowBackgroundColor(shouldUseDarkColors)); const { titleBarOverlay } = getWindowTitleBarOptions(shouldUseDarkColors, platform); - if (typeof titleBarOverlay === "object") { + if (hasTitleBarOverlay && typeof titleBarOverlay === "object") { window.setTitleBarOverlay(titleBarOverlay); } }); @@ -454,7 +455,6 @@ export const make = Effect.gen(function* () { ); const currentMainWindow = electronWindow.currentMainOrFirst.pipe(Effect.flatMap(withoutSplash)); - const focusedMainWindow = electronWindow.focusedMainOrFirst.pipe(Effect.flatMap(withoutSplash)); const zoomWindow = Effect.fn("desktop.window.zoomWindow")(function* ( window: Electron.BrowserWindow, @@ -1048,7 +1048,7 @@ export const make = Effect.gen(function* () { const ensureMain = Effect.gen(function* () { if (yield* Ref.get(desktopState.quitting)) return Option.none(); - const existingWindow = yield* currentMainWindow; + const existingWindow = yield* electronWindow.main; if (Option.isSome(existingWindow)) { return existingWindow; } @@ -1138,7 +1138,8 @@ export const make = Effect.gen(function* () { { reveal = true }: { readonly reveal?: boolean } = {}, ) { if (yield* Ref.get(desktopState.quitting)) return; - const existingWindow = yield* reveal ? focusedMainWindow : electronWindow.main; + // Only the main renderer loads the app preload that listens on these channels. + const existingWindow = yield* electronWindow.main; if (Option.isNone(existingWindow) && (!reveal || (yield* waitingForBackend))) return; const window = Option.isSome(existingWindow) ? existingWindow : yield* ensureMain; if (Option.isNone(window) || window.value.isDestroyed()) return; @@ -1236,8 +1237,8 @@ export const make = Effect.gen(function* () { }), syncAppearance: Effect.gen(function* () { const shouldUseDarkColors = yield* electronTheme.shouldUseDarkColors; - yield* electronWindow.syncAllAppearance((window) => - syncWindowAppearance(window, shouldUseDarkColors, environment.platform), + yield* electronWindow.syncAllAppearance((window, { titleBarOverlay }) => + syncWindowAppearance(window, titleBarOverlay, shouldUseDarkColors, environment.platform), ); }).pipe(Effect.withSpan("desktop.window.syncAppearance")), }); diff --git a/apps/desktop/src/wsl/DesktopWslEnvironment.test.ts b/apps/desktop/src/wsl/DesktopWslEnvironment.test.ts index 69962f52e4f18..0787956a72b76 100644 --- a/apps/desktop/src/wsl/DesktopWslEnvironment.test.ts +++ b/apps/desktop/src/wsl/DesktopWslEnvironment.test.ts @@ -1,16 +1,25 @@ // @effect-diagnostics nodeBuiltinImport:off - the executed suite runs the generated install script through a real POSIX shell. +import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, describe, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import * as Redacted from "effect/Redacted"; +import * as Sink from "effect/Sink"; +import * as Stream from "effect/Stream"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; import { afterAll, expect } from "vite-plus/test"; import * as NodeChildProcess from "node:child_process"; import { ManagedPolicyFileError } from "@t3tools/shared/managedPolicyFile"; +import * as DesktopEnvironment from "../app/DesktopEnvironment.ts"; import * as DesktopManagedPolicy from "../app/DesktopManagedPolicy.ts"; import { + DesktopWslEnvironment, + layer as layerDesktopWslEnvironment, buildWslRuntimeInstallScript, makeIsAvailable, buildWslRuntimeInvalidateScript, @@ -18,6 +27,7 @@ import { buildWslRuntimeProbeScript, parseResolvedPath, parseWslRuntimeRoot, + windowsToWslPathImpl, } from "./DesktopWslEnvironment.ts"; // The install script only fails the way this file cares about when a real shell @@ -380,6 +390,93 @@ describe.skipIf(posixShellRunner === null)("WSL runtime install script (executed expect(selected.status, `${selected.stdout}\n${selected.stderr}`).toBe(0); }); + // Runs the distro side on the local bash, so a Windows dev box using the wsl.exe runner skips it. + describe.skipIf(posixShellRunner?.file !== "bash")("prepareRuntime through wsl.exe", () => { + // Emulates `wsl.exe [-d ] [--cd ] [--|--exec] ` in a distro with automount off. + const layerFakeWslExe = (home: string) => + Layer.effect( + ChildProcessSpawner.ChildProcessSpawner, + Effect.map(ChildProcessSpawner.ChildProcessSpawner, (local) => + ChildProcessSpawner.make((command) => { + if (!ChildProcess.isStandardCommand(command) || command.command !== "wsl.exe") { + return local.spawn(command); + } + const args = command.args.map((word) => + Redacted.isRedacted(word) ? Redacted.value(word) : word, + ); + while (args.length > 0) { + const flag = args[0]; + if (flag === "-d" || flag === "--distribution" || flag === "--cd") { + args.splice(0, 2); + } else if (flag === "--" || flag === "--exec" || flag === "-e") { + args.shift(); + break; + } else { + break; + } + } + const [file = "sh", ...rest] = + args[0] === "wslpath" + ? [ + "sh", + "-c", + 'printf "wslpath: %s\\n" "$1" >&2; exit 1', + "wslpath", + args.at(-1) ?? "", + ] + : args; + return local.spawn( + ChildProcess.make(file, rest, { + ...command.options, + env: { ...command.options.env, HOME: home }, + extendEnv: true, + }), + ); + }), + ), + ); + + const layerWslEnvironment = (home: string) => + layerDesktopWslEnvironment.pipe( + Layer.provide( + Layer.mergeAll( + layerFakeWslExe(home), + DesktopManagedPolicy.layerTest(), + Layer.effect( + DesktopEnvironment.DesktopEnvironment, + Effect.map( + Path.Path, + (path) => + ({ platform: "win32", path }) as DesktopEnvironment.DesktopEnvironment["Service"], + ), + ), + ), + ), + Layer.provide(NodeServices.layer), + ); + + it.live("installs the runtime when wslpath cannot translate the archive's Windows path", () => { + const fixture = createFixture(); + return Effect.gen(function* () { + const wsl = yield* DesktopWslEnvironment; + // The archive's Windows path; with no drives mounted, wslpath cannot map it into the distro. + const translated = yield* wsl.windowsToWslPath("Ubuntu", fixture.archivePath); + expect(translated._tag).toBe("None"); + + const prepared = yield* wsl.prepareRuntime("Ubuntu", { + windowsPath: fixture.archivePath, + runtimeId: fixture.runtimeId, + sha256: fixture.archiveSha, + }); + + expect(prepared).toEqual({ ok: true, linuxAppRoot: fixture.runtimeRoot }); + const version = runShell(`${sh(fixture.serverEntry)} --version`); + expect(version.status, version.stderr).toBe(0); + expect(version.stdout).toBe("t3code wsl runtime test server 0.0.0\n"); + }).pipe(Effect.provide(layerWslEnvironment(`${fixture.work}/home`))); + }); + }); + it("prunes old and markerless caches without touching retained, active, locked, or unrelated roots", () => { const result = runShell( [ @@ -459,3 +556,48 @@ describe("makeIsAvailable", () => { }), ); }); + +describe("windowsToWslPathImpl", () => { + it.effect("runs wslpath without a shell so folder names stay one literal argument", () => + Effect.gen(function* () { + const spawned: Array = []; + const stdout = Stream.make(new TextEncoder().encode("/mnt/c/a$b & c;(d)\n")); + const spawner = ChildProcessSpawner.make((command) => { + spawned.push(command); + return Effect.succeed( + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + stdout, + stderr: Stream.empty, + all: stdout, + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(0)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + stdin: Sink.drain, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }), + ); + }); + + const converted = yield* windowsToWslPathImpl("Ubuntu", "C:\\a$b & c;(d)").pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + + assert.deepStrictEqual(converted, Option.some("/mnt/c/a$b & c;(d)")); + assert.lengthOf(spawned, 1); + const [command] = spawned; + assert(command?._tag === "StandardCommand"); + assert.strictEqual(command.command, "wsl.exe"); + assert.deepStrictEqual(command.args, [ + "-d", + "Ubuntu", + "--exec", + "wslpath", + "-u", + "C:/a$b & c;(d)", + ]); + }), + ); +}); diff --git a/apps/desktop/src/wsl/DesktopWslEnvironment.ts b/apps/desktop/src/wsl/DesktopWslEnvironment.ts index 57d6c61f7c43a..4e5e5b423f20e 100644 --- a/apps/desktop/src/wsl/DesktopWslEnvironment.ts +++ b/apps/desktop/src/wsl/DesktopWslEnvironment.ts @@ -5,6 +5,8 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import type * as PlatformError from "effect/PlatformError"; +import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; @@ -173,37 +175,19 @@ const buildWslNodeEnvPreamble = ( ensure_remote_node_path || true `; -// wsl.exe re-escaping mangles quotes inside bash -lc scripts. -const runWslShell = ( - distro: string | null, - bashScript: string, +const runWslProcess = ( + args: ReadonlyArray, + stdin: Stream.Stream, timeout: Duration.Duration, - options: { - readonly nodeEngineRange?: string | null; - readonly resolveNode?: boolean; - } = {}, ): Effect.Effect => { const spawner = ChildProcessSpawner.ChildProcessSpawner; - const resolveNode = options.resolveNode !== false; - const command = ChildProcess.make( - "wsl.exe", - resolveNode - ? [...buildDistroArgs(distro), "--", "bash", "-l", "-s"] - : [...buildDistroArgs(distro), "--exec", "sh", "-s"], - { - stdin: Stream.encodeText( - Stream.make( - resolveNode - ? `${buildWslNodeEnvPreamble(options.nodeEngineRange)}${bashScript}` - : bashScript, - ), - ), - stdout: "pipe", - stderr: "pipe", - killSignal: "SIGTERM", - forceKillAfter: PROCESS_TERMINATE_GRACE, - }, - ); + const command = ChildProcess.make("wsl.exe", args, { + stdin, + stdout: "pipe", + stderr: "pipe", + killSignal: "SIGTERM", + forceKillAfter: PROCESS_TERMINATE_GRACE, + }); return Effect.scoped( Effect.gen(function* () { @@ -249,6 +233,32 @@ const runWslShell = ( ); }; +// wsl.exe re-escaping mangles quotes inside bash -lc scripts. +const runWslShell = ( + distro: string | null, + bashScript: string, + timeout: Duration.Duration, + options: { + readonly nodeEngineRange?: string | null; + readonly resolveNode?: boolean; + } = {}, +): Effect.Effect => { + const resolveNode = options.resolveNode !== false; + return runWslProcess( + resolveNode + ? [...buildDistroArgs(distro), "--", "bash", "-l", "-s"] + : [...buildDistroArgs(distro), "--exec", "sh", "-s"], + Stream.encodeText( + Stream.make( + resolveNode + ? `${buildWslNodeEnvPreamble(options.nodeEngineRange)}${bashScript}` + : bashScript, + ), + ), + timeout, + ); +}; + const shellQuote = (value: string): string => `'${value.replaceAll("'", "'\\''")}'`; const WSL_RUNTIME_READY_MARKER = ".t3code-wsl-runtime-ready"; @@ -257,13 +267,14 @@ const WSL_RUNTIME_SELECTION_GRACE_MINUTES = 5; const sanitizeWslRuntimeId = (value: string): string => value.replace(/[^A-Za-z0-9._-]/g, "_"); +// A null archive path asks for a staging directory to copy the archive into, unless the cache is ready. export const buildWslRuntimeInstallScript = ( - linuxArchivePath: string, + linuxArchivePath: string | null, runtimeId: string, archiveSha256: string, ): string => { const safeRuntimeId = sanitizeWslRuntimeId(runtimeId); - return [ + const readyCheck = [ "set -eu", 'runtime_parent="$HOME/.t3/wsl-runtime"', `runtime_root="$runtime_parent/${safeRuntimeId}"`, @@ -292,6 +303,17 @@ export const buildWslRuntimeInstallScript = ( ` printf 'runtimeRoot:%s\\n' "$runtime_root"`, " exit 0", "fi", + ]; + if (linuxArchivePath === null) { + return [ + ...readyCheck, + // The .tmp. name lets pruning collect a stage orphaned by a crash. + `archive_stage=$(mktemp -d "$runtime_parent/.${safeRuntimeId}.tmp.XXXXXX")`, + `printf 'archiveStage:%s\\n' "$archive_stage"`, + ].join("\n"); + } + return [ + ...readyCheck, `archive_sha=$(sha256sum ${shellQuote(linuxArchivePath)} | cut -d ' ' -f 1)`, `if [ "$archive_sha" != ${shellQuote(archiveSha256)} ]; then`, ` printf 'WSL runtime archive does not match its recorded SHA-256 (expected %s, got %s)\\n' ${shellQuote(archiveSha256)} "$archive_sha" >&2`, @@ -409,14 +431,17 @@ export const buildWslRuntimeInvalidateScript = (runtimeId: string): string => { ].join("\n"); }; -export const parseWslRuntimeRoot = (stdout: string): string | null => { - const prefix = "runtimeRoot:"; +const parseAbsolutePathField = (stdout: string, field: string): string | null => { + const prefix = `${field}:`; const line = stdout.split("\n").find((candidate) => candidate.startsWith(prefix)); if (line === undefined) return null; - const runtimeRoot = line.slice(prefix.length).replace(/\r$/, ""); - return runtimeRoot.startsWith("/") ? runtimeRoot : null; + const value = line.slice(prefix.length).replace(/\r$/, ""); + return value.startsWith("/") ? value : null; }; +export const parseWslRuntimeRoot = (stdout: string): string | null => + parseAbsolutePathField(stdout, "runtimeRoot"); + const NODE_PTY_BINARY_MISSING_EXIT_CODE = 4; const formatNodePtyProbeFailureReason = (exitCode: number): string | null => @@ -775,25 +800,18 @@ const ensureNodePtyImpl = Effect.fn("DesktopWslEnvironment.ensureNodePtyImpl")(f } as const; }); -const prepareWslRuntimeImpl = Effect.fn("desktop.wsl.prepareRuntimeImpl")(function* ( +type WslRuntimeInstallOutcome = + | { readonly ok: true; readonly stdout: string } + | { readonly ok: false; readonly reason: string }; + +const runWslRuntimeInstall = Effect.fn("desktop.wsl.runRuntimeInstall")(function* ( distro: string | null, archive: WslRuntimeArchive, - windowsToWslPath: ( - distro: string | null, - windowsPath: string, - ) => Effect.Effect>, -): Effect.fn.Return { - const linuxArchivePath = yield* windowsToWslPath(distro, archive.windowsPath); - if (Option.isNone(linuxArchivePath)) { - return { - ok: false, - reason: `wslpath conversion failed for ${archive.windowsPath}`, - } as const; - } - + linuxArchivePath: string | null, +): Effect.fn.Return { const install = yield* runWslShell( distro, - buildWslRuntimeInstallScript(linuxArchivePath.value, archive.runtimeId, archive.sha256), + buildWslRuntimeInstallScript(linuxArchivePath, archive.runtimeId, archive.sha256), RUNTIME_INSTALL_TIMEOUT, { resolveNode: false }, ); @@ -813,14 +831,106 @@ const prepareWslRuntimeImpl = Effect.fn("desktop.wsl.prepareRuntimeImpl")(functi reason: `WSL runtime installation failed (exit ${install.exitCode}): ${trimmedTail || "no stderr captured"}`, } as const; } + return { ok: true, stdout: install.stdout } as const; +}); - const linuxAppRoot = parseWslRuntimeRoot(install.stdout); +const toPrepareWslRuntimeResult = (outcome: WslRuntimeInstallOutcome): PrepareWslRuntimeResult => { + if (!outcome.ok) return outcome; + const linuxAppRoot = parseWslRuntimeRoot(outcome.stdout); return linuxAppRoot === null ? { ok: false, reason: "WSL runtime installation completed without reporting its cache path.", } : { ok: true, linuxAppRoot }; +}; + +// Without drive mounts (`[automount] enabled=false`) wslpath cannot reach the archive, so copy it in over stdin. +const installWslRuntimeFromHostCopy = Effect.fn("desktop.wsl.installRuntimeFromHostCopy")( + function* ( + distro: string | null, + archive: WslRuntimeArchive, + archiveBytes: Stream.Stream, + ): Effect.fn.Return { + const staged = yield* runWslRuntimeInstall(distro, archive, null); + if (!staged.ok || parseWslRuntimeRoot(staged.stdout) !== null) { + return toPrepareWslRuntimeResult(staged); + } + const stage = parseAbsolutePathField(staged.stdout, "archiveStage"); + if (stage === null) { + return { + ok: false, + reason: `wslpath conversion failed for ${archive.windowsPath}, and the distro did not provide a directory to copy the runtime archive into.`, + } as const; + } + const linuxArchivePath = `${stage}/runtime.tar.gz`; + + const copyAndInstall = Effect.gen(function* () { + const readError = yield* Ref.make(Option.none()); + // dd reads fd 0 itself; reopening /dev/stdin fails when WSL relays stdin as a socket. + const copy = yield* runWslProcess( + [...buildDistroArgs(distro), "--exec", "dd", `of=${linuxArchivePath}`, "bs=65536"], + archiveBytes.pipe( + // Ending stdin on a read error lets dd exit instead of waiting out the timeout. + Stream.catch((error) => + Stream.drain(Stream.fromEffect(Ref.set(readError, Option.some(error)))), + ), + ), + RUNTIME_INSTALL_TIMEOUT, + ); + const failedRead = yield* Ref.get(readError); + if (Option.isSome(failedRead)) { + return { + ok: false, + reason: `Could not read the WSL runtime archive at ${archive.windowsPath}: ${failedRead.value.message}`, + } as const; + } + if (copy.transportFailure !== null || copy.exitCode !== 0) { + const trimmedTail = copy.stderr.trim().slice(-500); + return { + ok: false, + reason: `Could not copy the WSL runtime archive into the distro (exit ${copy.exitCode})${trimmedTail ? `: ${trimmedTail}` : ""}`, + } as const; + } + return toPrepareWslRuntimeResult( + yield* runWslRuntimeInstall(distro, archive, linuxArchivePath), + ); + }); + + return yield* copyAndInstall.pipe( + Effect.ensuring( + runWslShell(distro, `rm -rf ${shellQuote(stage)}`, RUNTIME_INVALIDATE_TIMEOUT, { + resolveNode: false, + }), + ), + ); + }, +); + +const prepareWslRuntimeImpl = Effect.fn("desktop.wsl.prepareRuntimeImpl")(function* ( + distro: string | null, + archive: WslRuntimeArchive, + windowsToWslPath: ( + distro: string | null, + windowsPath: string, + ) => Effect.Effect>, +): Effect.fn.Return< + PrepareWslRuntimeResult, + never, + ChildProcessSpawner.ChildProcessSpawner | FileSystem.FileSystem +> { + const linuxArchivePath = yield* windowsToWslPath(distro, archive.windowsPath); + if (Option.isNone(linuxArchivePath)) { + const fileSystem = yield* FileSystem.FileSystem; + return yield* installWslRuntimeFromHostCopy( + distro, + archive, + fileSystem.stream(archive.windowsPath), + ); + } + return toPrepareWslRuntimeResult( + yield* runWslRuntimeInstall(distro, archive, linuxArchivePath.value), + ); }); const pruneWslRuntimesImpl = Effect.fn("desktop.wsl.pruneRuntimesImpl")(function* ( @@ -925,7 +1035,7 @@ const preWarmImpl = ( }), ).pipe(Effect.timeoutOption(PRE_WARM_TIMEOUT), Effect.ignore); -const windowsToWslPathImpl = ( +export const windowsToWslPathImpl = ( distro: string | null, windowsPath: string, ): Effect.Effect, never, ChildProcessSpawner.ChildProcessSpawner> => { @@ -936,7 +1046,8 @@ const windowsToWslPathImpl = ( const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const command = ChildProcess.make( "wsl.exe", - [...buildDistroArgs(distro), "--", "wslpath", "-u", normalized], + // --exec skips the distro shell, which would expand `$`, `&` or `;` in folder names. + [...buildDistroArgs(distro), "--exec", "wslpath", "-u", normalized], { stdin: "ignore", stdout: "pipe", @@ -1198,7 +1309,11 @@ export const layer = Layer.effect( getUserHome, getDistroIp, prepareRuntime: Effect.fn("desktop.wsl.prepareRuntime")(function (distro, archive) { - return provideSpawner(prepareWslRuntimeImpl(distro, archive, windowsToWslPath)); + return provideSpawner( + prepareWslRuntimeImpl(distro, archive, windowsToWslPath).pipe( + Effect.provideService(FileSystem.FileSystem, fileSystem), + ), + ); }), pruneRuntimes: Effect.fn("desktop.wsl.pruneRuntimes")(function (distro, runtimeId) { return provideSpawner(pruneWslRuntimesImpl(distro, runtimeId)); diff --git a/apps/desktop/vite.config.ts b/apps/desktop/vite.config.ts index 455901299edc3..3e6fd9531106b 100644 --- a/apps/desktop/vite.config.ts +++ b/apps/desktop/vite.config.ts @@ -164,6 +164,9 @@ export default defineConfig({ // The Windows lane runs workspace suites concurrently; filesystem-heavy // desktop integration tests can exceed Vitest's 5 second default there. testTimeout: 15_000, + // `require("electron")` downloads a missing binary, so concurrent workers race to + // extract it into the same dist/. Install it once before any worker starts. + globalSetup: ["./scripts/ensure-electron-runtime.mjs"], projects: [ { extends: true, diff --git a/apps/marketing/package.json b/apps/marketing/package.json index 04f6830066a1e..8e447d16b2c0e 100644 --- a/apps/marketing/package.json +++ b/apps/marketing/package.json @@ -7,7 +7,8 @@ "stage:install-scripts": "node scripts/stage-install-scripts.mjs", "dev": "node scripts/stage-install-scripts.mjs && astro dev", "build": "node scripts/stage-install-scripts.mjs && astro build", - "preview": "astro preview" + "preview": "astro preview", + "test": "vp test run" }, "dependencies": { "@fontsource-variable/inter": "^5.3.0", diff --git a/apps/marketing/src/i18n/de.ts b/apps/marketing/src/i18n/de.ts index 648b35beebe07..96925adb27069 100644 --- a/apps/marketing/src/i18n/de.ts +++ b/apps/marketing/src/i18n/de.ts @@ -27,6 +27,8 @@ export const de: Messages = { downloadForMac: "Für macOS herunterladen", downloadForWindows: "Für Windows herunterladen", downloadForLinux: "Für Linux herunterladen", + downloadForIos: "Für iOS herunterladen", + downloadForAndroid: "Für Android herunterladen", sourceLink: "Klau unseren Code (legal)", alsoOnPhone: "Auch auf deinem Handy:", screenshotAlt: "T3 Code führt Subagents von Claude, Codex, Muse und Gemini in einem Thread aus", diff --git a/apps/marketing/src/i18n/en.ts b/apps/marketing/src/i18n/en.ts index 1769d81184c5b..1a36424a1bfde 100644 --- a/apps/marketing/src/i18n/en.ts +++ b/apps/marketing/src/i18n/en.ts @@ -27,6 +27,8 @@ export const en = { downloadForMac: "Download for macOS", downloadForWindows: "Download for Windows", downloadForLinux: "Download for Linux", + downloadForIos: "Download for iOS", + downloadForAndroid: "Download for Android", sourceLink: "Steal our code (legally)", alsoOnPhone: "Also on your phone:", screenshotAlt: "T3 Code running Claude, Codex, Muse, and Gemini subagents in one thread", diff --git a/apps/marketing/src/i18n/es.ts b/apps/marketing/src/i18n/es.ts index cb4c28e9871fd..90fcf79bd9429 100644 --- a/apps/marketing/src/i18n/es.ts +++ b/apps/marketing/src/i18n/es.ts @@ -28,6 +28,8 @@ export const es: Messages = { downloadForMac: "Descargar para macOS", downloadForWindows: "Descargar para Windows", downloadForLinux: "Descargar para Linux", + downloadForIos: "Descargar para iOS", + downloadForAndroid: "Descargar para Android", sourceLink: "Roba nuestro código (legalmente)", alsoOnPhone: "También en tu teléfono:", screenshotAlt: "T3 Code ejecutando subagentes de Claude, Codex, Muse y Gemini en un mismo hilo", diff --git a/apps/marketing/src/i18n/fr.ts b/apps/marketing/src/i18n/fr.ts index 8fd918efc815b..3d76747d6c833 100644 --- a/apps/marketing/src/i18n/fr.ts +++ b/apps/marketing/src/i18n/fr.ts @@ -27,6 +27,8 @@ export const fr: Messages = { downloadForMac: "Télécharger pour macOS", downloadForWindows: "Télécharger pour Windows", downloadForLinux: "Télécharger pour Linux", + downloadForIos: "Télécharger pour iOS", + downloadForAndroid: "Télécharger pour Android", sourceLink: "Volez notre code (légalement)", alsoOnPhone: "Aussi sur votre téléphone :", screenshotAlt: diff --git a/apps/marketing/src/i18n/ja.ts b/apps/marketing/src/i18n/ja.ts index 55117b609ac08..9f9fdba51129d 100644 --- a/apps/marketing/src/i18n/ja.ts +++ b/apps/marketing/src/i18n/ja.ts @@ -28,6 +28,8 @@ export const ja: Messages = { downloadForMac: "macOS 版をダウンロード", downloadForWindows: "Windows 版をダウンロード", downloadForLinux: "Linux 版をダウンロード", + downloadForIos: "iOS 版をダウンロード", + downloadForAndroid: "Android 版をダウンロード", sourceLink: "コードを盗もう(合法的に)", alsoOnPhone: "スマホでも使えます:", screenshotAlt: diff --git a/apps/marketing/src/i18n/ko.ts b/apps/marketing/src/i18n/ko.ts index f2c09c0af06ac..e2f4d4e6f7c7a 100644 --- a/apps/marketing/src/i18n/ko.ts +++ b/apps/marketing/src/i18n/ko.ts @@ -27,6 +27,8 @@ export const ko: Messages = { downloadForMac: "macOS용 다운로드", downloadForWindows: "Windows용 다운로드", downloadForLinux: "Linux용 다운로드", + downloadForIos: "iOS용 다운로드", + downloadForAndroid: "Android용 다운로드", sourceLink: "코드 훔쳐 가기(합법적으로)", alsoOnPhone: "휴대폰에서도:", screenshotAlt: "하나의 스레드에서 Claude, Codex, Muse, Gemini 서브에이전트를 실행 중인 T3 Code", diff --git a/apps/marketing/src/i18n/pt-BR.ts b/apps/marketing/src/i18n/pt-BR.ts index 33841e839c527..1947884d33516 100644 --- a/apps/marketing/src/i18n/pt-BR.ts +++ b/apps/marketing/src/i18n/pt-BR.ts @@ -27,6 +27,8 @@ export const ptBR: Messages = { downloadForMac: "Baixar para macOS", downloadForWindows: "Baixar para Windows", downloadForLinux: "Baixar para Linux", + downloadForIos: "Baixar para iOS", + downloadForAndroid: "Baixar para Android", sourceLink: "Roube nosso código (legalmente)", alsoOnPhone: "Também no seu celular:", screenshotAlt: diff --git a/apps/marketing/src/i18n/ru.ts b/apps/marketing/src/i18n/ru.ts index 8fdeefdb9fadd..e0d357813bc6e 100644 --- a/apps/marketing/src/i18n/ru.ts +++ b/apps/marketing/src/i18n/ru.ts @@ -27,6 +27,8 @@ export const ru: Messages = { downloadForMac: "Скачать для macOS", downloadForWindows: "Скачать для Windows", downloadForLinux: "Скачать для Linux", + downloadForIos: "Скачать для iOS", + downloadForAndroid: "Скачать для Android", sourceLink: "Украдите наш код (легально)", alsoOnPhone: "Есть и для телефона:", screenshotAlt: "T3 Code запускает субагентов Claude, Codex, Muse и Gemini в одном треде", diff --git a/apps/marketing/src/i18n/zh.ts b/apps/marketing/src/i18n/zh.ts index bae711530051a..1c554b193c401 100644 --- a/apps/marketing/src/i18n/zh.ts +++ b/apps/marketing/src/i18n/zh.ts @@ -27,6 +27,8 @@ export const zh: Messages = { downloadForMac: "下载 macOS 版", downloadForWindows: "下载 Windows 版", downloadForLinux: "下载 Linux 版", + downloadForIos: "下载 iOS 版", + downloadForAndroid: "下载 Android 版", sourceLink: "来偷我们的代码(合法的)", alsoOnPhone: "手机上也能用:", screenshotAlt: "T3 Code 在同一个线程中运行 Claude、Codex、Muse 和 Gemini 子智能体", diff --git a/apps/marketing/src/layouts/Layout.astro b/apps/marketing/src/layouts/Layout.astro index 0f0a864101743..fa7fa30bc0b18 100644 --- a/apps/marketing/src/layouts/Layout.astro +++ b/apps/marketing/src/layouts/Layout.astro @@ -7,8 +7,8 @@ import { ANDROID_PLAY_STORE_URL, GITHUB_REPOSITORY_URL, IOS_APP_STORE_URL, - MARKETING_STATS, } from "../lib/site"; +import { MARKETING_STATS } from "../lib/stats"; import { type JsonLd, jsonLdGraph, organizationJsonLd, websiteJsonLd } from "../lib/seo"; import { messagesFor } from "../i18n"; import { diff --git a/apps/marketing/src/lib/platform.test.ts b/apps/marketing/src/lib/platform.test.ts new file mode 100644 index 0000000000000..d35abb2ac63aa --- /dev/null +++ b/apps/marketing/src/lib/platform.test.ts @@ -0,0 +1,45 @@ +import { expect, it } from "vite-plus/test"; + +import { detectPlatform, getMobileStoreUrl } from "./platform"; +import { ANDROID_PLAY_STORE_URL, IOS_APP_STORE_URL } from "./site"; + +it.each([ + { + // Android user agents also contain "Linux". + name: "Android", + userAgent: + "Mozilla/5.0 (Linux; Android 15; Pixel 9) AppleWebKit/537.36 Chrome/140.0.0.0 Mobile Safari/537.36", + maxTouchPoints: 5, + os: "android", + storeUrl: ANDROID_PLAY_STORE_URL, + }, + { + name: "iPhone", + userAgent: + "Mozilla/5.0 (iPhone; CPU iPhone OS 18_6 like Mac OS X) AppleWebKit/605.1.15 Version/18.6 Mobile/15E148 Safari/604.1", + maxTouchPoints: 5, + os: "ios", + storeUrl: IOS_APP_STORE_URL, + }, + { + // iPad Safari sends a desktop Mac user agent by default. + name: "iPad with a desktop user agent", + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) AppleWebKit/605.1.15 Version/18.6 Safari/605.1.15", + maxTouchPoints: 5, + os: "ios", + storeUrl: IOS_APP_STORE_URL, + }, + { + name: "macOS", + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 Version/18.6 Safari/605.1.15", + maxTouchPoints: 0, + os: "mac", + storeUrl: null, + }, +])("$name visitors get the $os download", ({ userAgent, maxTouchPoints, os, storeUrl }) => { + const platform = detectPlatform(userAgent, maxTouchPoints); + expect(platform?.os).toBe(os); + expect(platform && getMobileStoreUrl(platform)).toBe(storeUrl); +}); diff --git a/apps/marketing/src/lib/platform.ts b/apps/marketing/src/lib/platform.ts new file mode 100644 index 0000000000000..be17e789a6bb2 --- /dev/null +++ b/apps/marketing/src/lib/platform.ts @@ -0,0 +1,22 @@ +import { ANDROID_PLAY_STORE_URL, IOS_APP_STORE_URL } from "./site"; + +export type Platform = { os: "mac" | "win" | "linux" | "ios" | "android" }; + +export function detectPlatform(userAgent: string, maxTouchPoints: number): Platform | null { + // Mobile checks come first: Android user agents contain "Linux" and iOS + // user agents contain "Mac OS X". + if (/Android/i.test(userAgent)) return { os: "android" }; + if (/iPhone|iPad|iPod/i.test(userAgent)) return { os: "ios" }; + if (/Win/i.test(userAgent)) return { os: "win" }; + // iPadOS Safari sends a Mac user agent by default; only the touch screen + // tells it apart from a Mac. + if (/Mac/i.test(userAgent)) return { os: maxTouchPoints > 1 ? "ios" : "mac" }; + if (/Linux/i.test(userAgent)) return { os: "linux" }; + return null; +} + +export function getMobileStoreUrl(platform: Platform): string | null { + if (platform.os === "ios") return IOS_APP_STORE_URL; + if (platform.os === "android") return ANDROID_PLAY_STORE_URL; + return null; +} diff --git a/apps/marketing/src/lib/releases.test.ts b/apps/marketing/src/lib/releases.test.ts new file mode 100644 index 0000000000000..6527342d674e5 --- /dev/null +++ b/apps/marketing/src/lib/releases.test.ts @@ -0,0 +1,18 @@ +import { afterEach, describe, expect, it, vi } from "vite-plus/test"; + +import { fetchLatestRelease } from "./releases"; + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("fetchLatestRelease", () => { + it("rejects a GitHub API error response instead of returning it as a release", async () => { + vi.stubGlobal("sessionStorage", { getItem: () => null }); + vi.stubGlobal("fetch", async () => + Response.json({ message: "API rate limit exceeded" }, { status: 403 }), + ); + + await expect(fetchLatestRelease("stable")).rejects.toThrow("403"); + }); +}); diff --git a/apps/marketing/src/lib/releases.ts b/apps/marketing/src/lib/releases.ts index 7dc63bdaed859..f50103972cdd8 100644 --- a/apps/marketing/src/lib/releases.ts +++ b/apps/marketing/src/lib/releases.ts @@ -25,12 +25,18 @@ function cacheKey(channel: ReleaseChannel) { return `t3code-${channel}-release`; } +async function fetchJson(url: string): Promise { + const response = await fetch(url); + if (!response.ok) throw new Error(`GitHub API responded ${response.status}`); + return response.json(); +} + async function fetchStable(): Promise { - return fetch(LATEST_API_URL).then((r) => r.json()); + return fetchJson(LATEST_API_URL); } async function fetchNightly(): Promise { - const list: Release[] = await fetch(LIST_API_URL).then((r) => r.json()); + const list = await fetchJson(LIST_API_URL); const nightly = Array.isArray(list) ? list.find((release) => release.tag_name?.includes("-nightly.")) : undefined; diff --git a/apps/marketing/src/lib/site.ts b/apps/marketing/src/lib/site.ts index c4cc8aca59f22..319f6dd800bcb 100644 --- a/apps/marketing/src/lib/site.ts +++ b/apps/marketing/src/lib/site.ts @@ -5,26 +5,3 @@ export const IOS_APP_STORE_URL = export const ANDROID_PLAY_STORE_URL = "https://play.google.com/store/apps/details?id=com.t3tools.t3code"; - -// Runs at build time, so each deploy picks up the current count. -async function fetchGithubStars(): Promise { - const token = process.env.GITHUB_TOKEN; - try { - const response = await fetch("https://api.github.com/repos/pingdotgg/t3code", { - headers: token ? { Authorization: `Bearer ${token}` } : {}, - }); - if (!response.ok) throw new Error(`GitHub API responded ${response.status}`); - const { stargazers_count: stars } = (await response.json()) as { stargazers_count: number }; - return stars < 1000 ? String(stars) : `${Math.floor(stars / 1000)}k+`; - } catch (error) { - // A deploy must not ship a made-up count; offline dev should still render. - if (import.meta.env.PROD) throw new Error("Could not fetch GitHub stars", { cause: error }); - console.warn("Could not fetch GitHub stars", error); - return "?"; - } -} - -export const MARKETING_STATS = { - githubStars: await fetchGithubStars(), - users: "500,000", -} as const; diff --git a/apps/marketing/src/lib/stats.ts b/apps/marketing/src/lib/stats.ts new file mode 100644 index 0000000000000..d291f420b06d1 --- /dev/null +++ b/apps/marketing/src/lib/stats.ts @@ -0,0 +1,22 @@ +// Runs at build time, so each deploy picks up the current count. +async function fetchGithubStars(): Promise { + const token = process.env.GITHUB_TOKEN; + try { + const response = await fetch("https://api.github.com/repos/pingdotgg/t3code", { + headers: token ? { Authorization: `Bearer ${token}` } : {}, + }); + if (!response.ok) throw new Error(`GitHub API responded ${response.status}`); + const { stargazers_count: stars } = (await response.json()) as { stargazers_count: number }; + return stars < 1000 ? String(stars) : `${Math.floor(stars / 1000)}k+`; + } catch (error) { + // A deploy must not ship a made-up count; offline dev should still render. + if (import.meta.env.PROD) throw new Error("Could not fetch GitHub stars", { cause: error }); + console.warn("Could not fetch GitHub stars", error); + return "?"; + } +} + +export const MARKETING_STATS = { + githubStars: await fetchGithubStars(), + users: "500,000", +} as const; diff --git a/apps/marketing/src/pages/95.astro b/apps/marketing/src/pages/95.astro index d6ab25cf867d7..3a9d67f509df9 100644 --- a/apps/marketing/src/pages/95.astro +++ b/apps/marketing/src/pages/95.astro @@ -1,7 +1,8 @@ --- import RetroIcon from "../components/RetroIcon.astro"; import RetroBox from "../components/RetroBox.astro"; -import { GITHUB_REPOSITORY_URL, MARKETING_STATS } from "../lib/site"; +import { GITHUB_REPOSITORY_URL } from "../lib/site"; +import { MARKETING_STATS } from "../lib/stats"; import "../styles/retro.css"; const agents = [ diff --git a/apps/marketing/src/pages/[...locale]/index.astro b/apps/marketing/src/pages/[...locale]/index.astro index 5ae3842180587..e8f4b5f1b5fa5 100644 --- a/apps/marketing/src/pages/[...locale]/index.astro +++ b/apps/marketing/src/pages/[...locale]/index.astro @@ -6,8 +6,8 @@ import { ANDROID_PLAY_STORE_URL, GITHUB_REPOSITORY_URL, IOS_APP_STORE_URL, - MARKETING_STATS, } from "../../lib/site"; +import { MARKETING_STATS } from "../../lib/stats"; import { tweets } from "../../lib/tweets"; import { softwareJsonLd } from "../../lib/seo"; import { messagesFor } from "../../i18n"; @@ -83,6 +83,8 @@ const mobileEndorsementRows = [ data-label-mac={t.downloadForMac} data-label-win={t.downloadForWindows} data-label-linux={t.downloadForLinux} + data-label-ios={t.downloadForIos} + data-label-android={t.downloadForAndroid} > @@ -413,16 +415,7 @@ const mobileEndorsementRows = [ `, + ); + const field = page.locator("#host #field"); + const box = (await field.boundingBox())!; + await page.mouse.click(box.x + box.width / 2, box.y + box.height / 2); + await ServerBrowserPage.type(page, { text: "shadow" }); + expect(await field.inputValue()).toBe("shadow"); + + // Refused for now: focus inside a frame cannot be checked from the top document. + const inner = page.frameLocator("iframe").locator("#inner"); + await inner.click(); + await expect(ServerBrowserPage.type(page, { text: "frame" })).rejects.toMatchObject({ + tag: "PreviewAutomationTargetNotEditableError", + }); + expect(await inner.inputValue()).toBe(""); + }); + it("shows dotted fields as dots wherever their value appears, keeping text values and refs", async () => { await page.setContent(`
password:
@@ -448,7 +628,7 @@ describe("server browser snapshot screenshot", () => { const screenshotOf = async (html: string) => { const page = await context.newPage(); await page.setContent(html); - const { screenshot, fullScreenshot } = await ServerBrowserPage.snapshot({ + const input = { page, cdp: await context.newCDPSession(page), renderScale: 2, @@ -456,7 +636,8 @@ describe("server browser snapshot screenshot", () => { consoleEntries: [], networkEntries: [], actionTimeline: [], - }); + }; + const { screenshot, fullScreenshot } = await ServerBrowserPage.snapshot(input); // The server has no DOM types, so the decoder runs as page source. const points = [ [0.15, 0.75], @@ -475,7 +656,7 @@ describe("server browser snapshot screenshot", () => { Array.from(context.getImageData(x * bitmap.width, y * bitmap.height, 1, 1).data), ); })()`)) as Array<[number, number, number, number]>; - return { screenshot, fullScreenshot, pixels }; + return { screenshot, fullScreenshot, pixels, input }; }; const colorOf = ([red, green, blue]: readonly [number, number, number, number]) => blue - red > 100 ? "blue" : red - blue > 100 ? "red" : `rgb(${red}, ${green}, ${blue})`; @@ -490,13 +671,20 @@ describe("server browser snapshot screenshot", () => { }); it("captures a plain page at the shrunk size with its content in place", async () => { - const { screenshot, fullScreenshot, pixels } = await screenshotOf(halves("")); + const { screenshot, fullScreenshot, pixels, input } = await screenshotOf(halves("")); + const metadata = await ServerBrowserPage.snapshotMetadata(input); expect({ size: [screenshot.width, screenshot.height], fullSize: [fullScreenshot?.width, fullScreenshot?.height], + metadataSize: metadata.screenshot, colors: pixels.map(colorOf), - }).toEqual({ size: [1280, 800], fullSize: [2560, 1600], colors: ["blue", "red", "red"] }); + }).toEqual({ + size: [1280, 800], + fullSize: [2560, 1600], + metadataSize: { width: 1280, height: 800 }, + colors: ["blue", "red", "red"], + }); }); it("records the destination of a finite entrance animation without waiting on infinite ones", async () => { @@ -622,6 +810,52 @@ describe("server browser drag", () => { }); }); +describe("server browser action timeout", () => { + type StepOptions = { readonly timeout?: number }; + it.each([ + { + action: "hover", + act: (page: Page) => ServerBrowserPage.hover(page, { selector: "#a", timeoutMs: 9_000 }), + last: ["#a.hover 1"], + }, + { + action: "drag", + act: (page: Page) => + ServerBrowserPage.drag(page, { source: "#a", target: "#b", timeoutMs: 9_000 }), + last: ["#a.dragTo 1", "#b.boundingBox 1"], + }, + ])("gives the $action's steps one shared timeout", async ({ act, last }) => { + let now = 0; + using _now = vi.spyOn(Date, "now").mockImplementation(() => now); + const steps: Array = []; + const box = { x: 0, y: 0, width: 20, height: 20 }; + const page = { + locator: (selector: string) => { + const record = (step: string, options: StepOptions) => + steps.push(`${selector}.${step} ${options.timeout}`); + return { + // Each step that finds the target takes 5 of the 9 seconds. + scrollIntoViewIfNeeded: async (options: StepOptions) => { + record("scrollIntoViewIfNeeded", options); + now += 5_000; + }, + boundingBox: async (options: StepOptions) => { + record("boundingBox", options); + now += 5_000; + return box; + }, + hover: async (options: StepOptions) => record("hover", options), + dragTo: async (_target: unknown, options: StepOptions) => record("dragTo", options), + }; + }, + } as unknown as Page; + + await act(page); + + expect(steps).toEqual(["#a.scrollIntoViewIfNeeded 9000", "#a.boundingBox 4000", ...last]); + }); +}); + describe("server browser click", () => { it("clicks a target the page replaced while it was being scrolled into view", async () => { let scrolls = 0; @@ -717,6 +951,63 @@ describe("server browser evaluate", () => { }); }); +describe("server browser type", () => { + // As on a desktop tab whose Playwright utility world never answers. + const stalledPage = (focusAnswer: () => Promise, typed: string[]) => + ({ + locator: () => ({ last: () => ({ count: () => new Promise(() => {}) }) }), + evaluate: focusAnswer, + keyboard: { insertText: async (text: string) => void typed.push(text) }, + }) as unknown as Page; + + it("types into the focused field when Playwright's utility world never answers", async () => { + vi.useFakeTimers(); + try { + const typed: string[] = []; + let outcome: unknown = "pending"; + const page = stalledPage(async () => true, typed); + void ServerBrowserPage.type(page, { text: "hello", timeoutMs: 300 }).then( + () => (outcome = "typed"), + (error: unknown) => (outcome = error), + ); + await vi.advanceTimersByTimeAsync(300); + expect(outcome).toBe("typed"); + expect(typed).toEqual(["hello"]); + } finally { + vi.useRealTimers(); + } + }); + + it("stops at its timeout without typing when the page never answers", async () => { + vi.useFakeTimers(); + try { + const typed: string[] = []; + let outcome: unknown = "pending"; + const page = stalledPage(() => new Promise(() => {}), typed); + void ServerBrowserPage.type(page, { text: "hello", timeoutMs: 300 }).then( + () => (outcome = "typed"), + (error: unknown) => (outcome = error), + ); + await vi.advanceTimersByTimeAsync(300); + expect(outcome).toMatchObject({ + tag: "PreviewAutomationTimeoutError", + detail: { timeoutMs: 300 }, + }); + expect(typed).toEqual([]); + } finally { + vi.useRealTimers(); + } + }); +}); + +describe("server browser unresponsive script", () => { + it("leaves a page that answers alone", async () => { + const send = vi.fn(async (_method: string) => ({})); + await ServerBrowserPage.stopUnresponsiveScript({ send } as unknown as CDPSession); + expect(send.mock.calls.map(([method]) => method)).toEqual(["Runtime.evaluate"]); + }); +}); + describe("server browser wait", () => { it("ends at its timeout when a check never answers", async () => { // As on a desktop tab whose Playwright utility world never answers. diff --git a/apps/server/src/preview/ServerBrowserPage.ts b/apps/server/src/preview/ServerBrowserPage.ts index 3354554f1dd90..ac37b9914ef76 100644 --- a/apps/server/src/preview/ServerBrowserPage.ts +++ b/apps/server/src/preview/ServerBrowserPage.ts @@ -14,6 +14,8 @@ import { type PreviewAutomationSelectInput, type PreviewAutomationSelectResult, type PreviewAutomationSnapshot, + type PreviewAutomationSnapshotMetadata, + type PreviewAutomationSnapshotStage, type PreviewAutomationTypeInput, type PreviewAutomationUploadInput, type PreviewAutomationWaitForInput, @@ -277,13 +279,36 @@ const pageChanged = () => message: "The page changed while capturing its snapshot. Take another snapshot.", }); -const tooBusy = () => +type SnapshotStage = typeof PreviewAutomationSnapshotStage.Type; + +/** + * Running stages are counted, since a retry can overlap the previous attempt's unanswered + * screenshot. A stage whose own timeout fired has stopped running but is still the cause. + */ +interface SnapshotStages { + readonly running: Map; + readonly timedOut: Set; +} + +const snapshotTimedOut = (stages?: SnapshotStages) => + new ServerBrowserOperationError({ + tag: "PreviewAutomationTimeoutError", + message: `Snapshot capture timed out after ${SNAPSHOT_TIMEOUT_MS}ms. Try the snapshot again.`, + detail: { + timeoutMs: SNAPSHOT_TIMEOUT_MS, + pendingStages: stages ? [...new Set([...stages.timedOut, ...stages.running.keys()])] : [], + }, + }); + +const frameUnanswered = () => new ServerBrowserOperationError({ tag: "PreviewAutomationTimeoutError", - message: - "The page was too busy to capture a snapshot in time. Wait for it to settle and take another snapshot.", + message: `A frame on the page didn't answer within ${FRAME_ANSWER_MS}ms, so its content couldn't be checked for hidden values. Try the snapshot again.`, + detail: { timeoutMs: FRAME_ANSWER_MS, pendingStages: ["frames"] }, }); +const isTimeout = (cause: unknown) => cause instanceof Error && cause.name === "TimeoutError"; + const VALUE_TOO_LONG = "A password field holds a value too long to hide, so no snapshot was taken."; /** Reuse one CDP session per out-of-process frame across snapshots. */ @@ -459,6 +484,37 @@ const pngSize = (base64: string) => { return { width: header.readUInt32BE(16), height: header.readUInt32BE(20) }; }; +export const jpegSize = (base64: string) => { + const bytes = Buffer.from(base64, "base64"); + if (bytes.length < 4 || bytes.readUInt16BE(0) !== 0xffd8) + throw new Error("Invalid JPEG screenshot."); + let offset = 2; + while (offset < bytes.length) { + if (bytes[offset++] !== 0xff) throw new Error("Invalid JPEG screenshot."); + while (bytes[offset] === 0xff) offset++; + const marker = bytes[offset++]; + if (marker === undefined || marker === 0xd9 || marker === 0xda) break; + if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) continue; + if (offset + 2 > bytes.length) break; + const length = bytes.readUInt16BE(offset); + if (length < 2 || offset + length > bytes.length) break; + if (marker >= 0xc0 && marker <= 0xcf && ![0xc4, 0xc8, 0xcc].includes(marker)) { + if (length < 7) break; + const height = bytes.readUInt16BE(offset + 3); + const width = bytes.readUInt16BE(offset + 5); + if (width && height) return { width, height }; + break; + } + offset += length; + } + throw new Error("Invalid JPEG screenshot."); +}; + +const scaledSize = (size: { readonly width: number; readonly height: number }, scale: number) => + scale < 1 + ? { width: Math.round(size.width * scale), height: Math.round(size.height * scale) } + : size; + /** * A page's display ratio and CSS viewport, read in an isolated world since page script can * reassign `devicePixelRatio`, `innerWidth`, and `innerHeight` in its own, and its page zoom. @@ -609,41 +665,73 @@ const renderingOf = async (input: SnapshotInput) => { const capture = async ( input: SnapshotInput, deadline: number, -): Promise => { - const { renderScale, viewport } = await renderingOf(input); + stages: SnapshotStages, + metadataOnly: boolean, +) => { + const { running } = stages; + const track = async
(stage: SnapshotStage, operation: () => Promise) => { + running.set(stage, (running.get(stage) ?? 0) + 1); + try { + const result = await operation(); + stages.timedOut.delete(stage); + return result; + } catch (cause) { + if (isTimeout(cause)) stages.timedOut.add(stage); + throw cause; + } finally { + const count = (running.get(stage) ?? 1) - 1; + if (count === 0) running.delete(stage); + else running.set(stage, count); + } + }; + const { renderScale, viewport } = await track("metrics", () => renderingOf(input)); // A late metrics reply must not begin a capture after viewers have resumed. - if (Date.now() >= deadline) throw tooBusy(); + if (Date.now() >= deadline) throw snapshotTimedOut(); const scale = Math.min(1, MAX_SCREENSHOT_WIDTH / (viewport.width * renderScale)); const state = refsFor(input.page); invalidateRefs(input.page); const generation = state.generation; - const { readers, read } = await openReaders(input.page, input.cdp); - const before = await read(); + const { readers, read } = await track("frames", () => openReaders(input.page, input.cdp)); + const before = await track("frames", () => read()); // Bracket the tree with value reads to detect changes; await tree completion before retrying failed captures. const pageAndScreenshot = Promise.all([ - input.page.evaluate(SNAPSHOT_SCRIPT) as Promise< - Pick< - PreviewAutomationSnapshot, - "url" | "title" | "loading" | "visibleText" | "interactiveElements" - > - >, - captureViewport(input.cdp, { format: "png" }).then(async (full) => - scale < 1 - ? { - data: await shrinkPng(input.cdp, full, scale), - full: input.fullScreenshot ? full : undefined, - } - : { data: full, full: undefined }, + track( + "page state", + () => + input.page.evaluate(SNAPSHOT_SCRIPT) as Promise< + Pick< + PreviewAutomationSnapshot, + "url" | "title" | "loading" | "visibleText" | "interactiveElements" + > + >, + ), + // A cheap real frame still sizes the screenshot; computing it from metrics drifted from the PNG. + track("screenshot", () => + metadataOnly + ? captureViewport(input.cdp, { format: "jpeg", quality: 10 }).then((data) => ({ + data, + full: undefined, + })) + : captureViewport(input.cdp, { format: "png" }).then(async (full) => + scale < 1 + ? { + data: await shrinkPng(input.cdp, full, scale), + full: input.fullScreenshot ? full : undefined, + } + : { data: full, full: undefined }, + ), ), ]); void pageAndScreenshot.catch(constVoid); // Leaves time for the second read; a frame that never loads holds the tree until then. - const tree = await input.page.ariaSnapshot({ - mode: "ai", - boxes: true, - timeout: Math.max(1, deadline - Date.now() - 2_000), - }); - const after = await read(before); + const tree = await track("accessibility tree", () => + input.page.ariaSnapshot({ + mode: "ai", + boxes: true, + timeout: Math.max(1, deadline - Date.now() - 2_000), + }), + ); + const after = await track("frames", () => read(before)); // A value that changed meanwhile may be in the tree under neither reading. if (after.some((values, i) => values && JSON.stringify(values) !== JSON.stringify(before[i]))) throw pageChanged(); @@ -652,11 +740,13 @@ const capture = async ( const unread = readers.flatMap((reader, i) => (after[i] ? [] : reader.frames)); const filled = /^\s*- iframe (?:\[active\] )?\[ref=(\S+?)\].*:$/gm; for (const [, ref] of unread.length ? tree.matchAll(filled) : []) { - const iframe = await input.page.locator(`aria-ref=${ref}`).elementHandle({ timeout: 2_000 }); + const iframe = await track("frames", () => + input.page.locator(`aria-ref=${ref}`).elementHandle({ timeout: 2_000 }), + ); if (!iframe) throw pageChanged(); - const frame = await iframe.contentFrame(); + const frame = await track("frames", () => iframe.contentFrame()); void iframe.dispose().catch(constVoid); - if (frame && unread.includes(frame)) throw tooBusy(); + if (frame && unread.includes(frame)) throw frameUnanswered(); } const [page, { full, data }] = await pageAndScreenshot; if (state.generation !== generation) throw pageChanged(); @@ -680,11 +770,10 @@ const capture = async ( consoleEntries: [...input.consoleEntries], networkEntries: [...input.networkEntries], actionTimeline: [...input.actionTimeline], - screenshot: { - mimeType: "image/png", - data, - ...pngSize(data), - }, + // Reports the size the PNG path would have shrunk to. + screenshot: metadataOnly + ? scaledSize(jpegSize(data), scale) + : { mimeType: "image/png" as const, data, ...pngSize(data) }, ...(full !== undefined ? { fullScreenshot: { data: full, ...pngSize(full) } } : {}), }; }; @@ -692,31 +781,38 @@ const capture = async ( /** Snapshots begun per page; observers' snapshots aren't queued, so two can overlap. */ const snapshotTurns = new WeakMap(); -export const snapshot = async (input: SnapshotInput): Promise => { +const runSnapshot = async (input: SnapshotInput, metadataOnly: boolean) => { const deadline = Date.now() + SNAPSHOT_TIMEOUT_MS; + const stages: SnapshotStages = { running: new Map(), timedOut: new Set() }; let timer: ReturnType | undefined; const expired = new Promise((_, reject) => { - timer = setTimeout(() => reject(tooBusy()), SNAPSHOT_TIMEOUT_MS); + timer = setTimeout(() => reject(snapshotTimedOut(stages)), SNAPSHOT_TIMEOUT_MS); }); const turn = (snapshotTurns.get(input.page) ?? 0) + 1; snapshotTurns.set(input.page, turn); // Retry only before the deadline and the next snapshot, whose refs a later retry would revoke. - const captured = capture(input, deadline).catch((cause: unknown) => + const captured = capture(input, deadline, stages, metadataOnly).catch((cause: unknown) => Date.now() < deadline && snapshotTurns.get(input.page) === turn && !(cause instanceof Error && cause.message === VALUE_TOO_LONG) - ? capture(input, deadline) + ? capture(input, deadline, stages, metadataOnly) : Promise.reject(cause), ); try { return await Promise.race([captured, expired]); } catch (cause) { - throw cause instanceof Error && cause.name === "TimeoutError" ? tooBusy() : cause; + throw isTimeout(cause) ? snapshotTimedOut(stages) : cause; } finally { clearTimeout(timer); } }; +export const snapshot = (input: SnapshotInput) => + runSnapshot(input, false) as Promise; + +export const snapshotMetadata = (input: SnapshotInput) => + runSnapshot(input, true) as Promise; + /** Return when a dialog opens; waiting for the action to finish would block until dialog resolution. */ export const click = async ( page: Page, @@ -800,6 +896,76 @@ const clickTarget = async ( return point; }; +type FocusCandidate = { + readonly tagName: string; + readonly type?: string; + readonly disabled?: boolean; + readonly readOnly?: boolean; + readonly isContentEditable?: boolean; + readonly shadowRoot?: { readonly activeElement: FocusCandidate | null } | null; + readonly focus?: () => void; + getRootNode(): { readonly activeElement?: FocusCandidate | null }; + contains(other: FocusCandidate): boolean; +}; + +// Serialized into the page's main world, so it must not reference module scope. +const focusTextTarget = (target: FocusCandidate | null) => { + let element = + target ?? + (globalThis as unknown as { document: { activeElement: FocusCandidate | null } }).document + .activeElement; + if (target === null) { + while (element?.shadowRoot?.activeElement) element = element.shadowRoot.activeElement; + // Focus inside a frame cannot be checked from here, and an editable frame would pass as contentEditable. + if (element && ["IFRAME", "FRAME", "FENCEDFRAME", "OBJECT", "EMBED"].includes(element.tagName)) + return false; + } + if (element == null) return false; + // Require an enabled, focused text target to avoid losing input or sending it to the previous field. + const nonText = [ + "button", + "checkbox", + "color", + "file", + "hidden", + "image", + "radio", + "range", + "reset", + "submit", + ]; + const textControl = + element.tagName === "TEXTAREA" || + (element.tagName === "INPUT" && !nonText.includes(element.type ?? "text")); + if (!(textControl || element.isContentEditable) || element.disabled || element.readOnly) { + return false; + } + element.focus?.(); + const active = element.getRootNode().activeElement; + return active != null && (active === element || element.contains(active)); +}; + +// A `*:focus` locator needs Playwright's utility world, which some desktop tabs never get. +const focusedTextField = async (page: Page, timeout: number) => { + let timer: ReturnType | undefined; + const expired = new Promise((_, reject) => { + timer = setTimeout( + () => + reject( + new ServerBrowserOperationError({ + tag: "PreviewAutomationTimeoutError", + message: `The page did not report its focused element within ${timeout}ms, so no text was typed.`, + detail: { timeoutMs: timeout }, + }), + ), + timeout, + ); + }); + return await Promise.race([page.evaluate(focusTextTarget, null), expired]).finally(() => + clearTimeout(timer), + ); +}; + export const type = async (page: Page, input: PreviewAutomationTypeInput) => { const timeout = input.timeoutMs ?? DEFAULT_TIMEOUT_MS; const locator = targetLocator(page, input); @@ -807,46 +973,10 @@ export const type = async (page: Page, input: PreviewAutomationTypeInput) => { await locator.fill(input.text, { timeout }); return; } - // A shadow host also matches :focus; use its innermost focused descendant. - const target = locator ?? page.locator("*:focus").last(); const focused = - (locator !== null || (await target.count()) > 0) && - (await target.evaluate( - (element) => { - // Require an enabled, focused text target to avoid losing input or sending it to the previous field. - const control = element as unknown as { - readonly type?: string; - readonly disabled?: boolean; - readonly readOnly?: boolean; - readonly isContentEditable?: boolean; - readonly focus?: () => void; - }; - const nonText = [ - "button", - "checkbox", - "color", - "file", - "hidden", - "image", - "radio", - "range", - "reset", - "submit", - ]; - const textControl = - element.tagName === "TEXTAREA" || - (element.tagName === "INPUT" && !nonText.includes(control.type ?? "text")); - if (!(textControl || control.isContentEditable) || control.disabled || control.readOnly) { - return false; - } - control.focus?.(); - const root = element.getRootNode() as { readonly activeElement?: typeof element | null }; - const active = root.activeElement; - return active != null && (active === element || element.contains(active)); - }, - undefined, - { timeout }, - )); + locator === null + ? await focusedTextField(page, timeout) + : await locator.evaluate(focusTextTarget, undefined, { timeout }); if (focused !== true) { throw new ServerBrowserOperationError({ tag: "PreviewAutomationTargetNotEditableError", @@ -865,14 +995,15 @@ export const hover = async ( input: PreviewAutomationHoverInput, pointer: PointerReporter = noPointer, ) => { - const timeout = input.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const deadline = Date.now() + (input.timeoutMs ?? DEFAULT_TIMEOUT_MS); + const remaining = () => Math.max(1, deadline - Date.now()); const locator = targetLocator(page, input); - await pointer(await targetPoint(page, locator, input, () => timeout), "move"); + await pointer(await targetPoint(page, locator, input, remaining), "move"); if (locator === null) { await page.mouse.move(input.x ?? 0, input.y ?? 0); return; } - await locator.hover({ timeout }); + await locator.hover({ timeout: remaining() }); }; export const select = async ( @@ -915,17 +1046,18 @@ export const drag = async ( input: PreviewAutomationDragInput, pointer: PointerReporter = noPointer, ) => { - const timeout = input.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const deadline = Date.now() + (input.timeoutMs ?? DEFAULT_TIMEOUT_MS); + const remaining = () => Math.max(1, deadline - Date.now()); const source = targetLocator(page, { locator: input.source })!; const target = targetLocator(page, { locator: input.target })!; - await pointer(await targetPoint(page, source, {}, () => timeout), "move"); + await pointer(await targetPoint(page, source, {}, remaining), "move"); // Observe drag rejection immediately; an unhandled rejection during cursor movement exits the server. // Chromium delivers a single move before dragstart; steps give the page mousemoves during the drag. - const dragFailure = source.dragTo(target, { timeout, steps: 5 }).then( + const dragFailure = source.dragTo(target, { timeout: remaining(), steps: 5 }).then( () => null, (error: unknown) => ({ error }), ); - const end = await target.boundingBox({ timeout }).catch(() => null); + const end = await target.boundingBox({ timeout: remaining() }).catch(() => null); if (end) await pointer({ x: end.x + end.width / 2, y: end.y + end.height / 2 }, "move"); const failure = await dragFailure; if (failure) throw failure.error; @@ -953,6 +1085,26 @@ export const scroll = async (page: Page, input: PreviewAutomationScrollInput) => await locator.evaluate((element, [x, y]) => element.scrollBy(x, y), delta); }; +const UNRESPONSIVE_SCRIPT_PROBE_MS = 1_000; + +/** + * A same-site navigation reuses the renderer, so it never commits while a page script holds it. + * Call before starting the navigation: Chrome does not deliver a terminate once one is pending. + */ +export const stopUnresponsiveScript = async (cdp: CDPSession, budgetMs = Infinity) => { + let timer: ReturnType | undefined; + const answered = cdp.send("Runtime.evaluate", { expression: "0" }).then( + () => true, + () => true, + ); + const expired = new Promise((resolve) => { + timer = setTimeout(() => resolve(false), Math.min(budgetMs, UNRESPONSIVE_SCRIPT_PROBE_MS)); + }); + const responsive = await Promise.race([answered, expired]).finally(() => clearTimeout(timer)); + // Not awaited: a renderer blocked on something other than script may never acknowledge it. + if (!responsive) void cdp.send("Runtime.terminateExecution").catch(constVoid); +}; + /** Terminate expired page scripts so a hung expression cannot block the tab's control queue. */ export const evaluate = async ( cdp: CDPSession, diff --git a/apps/server/src/process/SpawnWorker.test.ts b/apps/server/src/process/SpawnWorker.test.ts new file mode 100644 index 0000000000000..27d31aa69c179 --- /dev/null +++ b/apps/server/src/process/SpawnWorker.test.ts @@ -0,0 +1,172 @@ +// @effect-diagnostics nodeBuiltinImport:off processEnvInEffect:off - fixtures need real paths, the process resource list and the live environment. +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { describe, expect, it } from "@effect/vitest"; +import { vi } from "vite-plus/test"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Result from "effect/Result"; +import * as Stream from "effect/Stream"; +import * as ChildProcess from "effect/process/ChildProcess"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; +import * as HostProcess from "@t3tools/shared/HostProcess"; + +import * as SpawnWorker from "./SpawnWorker.ts"; + +const envChdir = vi.hoisted(() => ({ supported: true })); +vi.mock("@t3tools/shared/envTrampoline", async (importOriginal) => ({ + ...(await importOriginal()), + supportsEnvChdir: () => Promise.resolve(envChdir.supported), +})); + +const layer = SpawnWorker.layer.pipe(Layer.provide(NodeServices.layer)); +const text = (stream: Stream.Stream) => + stream.pipe(Stream.decodeText(), Stream.mkString); +const node = (script: string, args: ReadonlyArray = [], options = {}) => + ChildProcess.make(process.execPath, ["-e", script, ...args], options); + +describe("SpawnWorker", () => { + it.live("runs children on a launcher thread where the server thread would wait on them", () => + Effect.gen(function* () { + for (const [platform, hasEnvChdir, onServerThread] of [ + ["win32", true, false], + ["linux", true, true], + ["linux", false, false], + ] as const) { + envChdir.supported = hasEnvChdir; + const launched = Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* spawner.spawn( + node("process.stdout.write('ready');process.stdin.resume()"), + ); + expect(yield* child.stdout.pipe(Stream.take(1), text)).toBe("ready"); + // The thread that called spawn owns the native process handle. + expect(process.getActiveResourcesInfo().includes("ProcessWrap")).toBe(onServerThread); + expect(yield* child.isRunning).toBe(true); + }); + yield* launched.pipe( + Effect.scoped, + Effect.provide(SpawnWorker.layerServer.pipe(Layer.provide(NodeServices.layer))), + Effect.provideService(HostProcess.Platform, platform), + ); + } + }), + ); + + it.live("preserves stdin, large output, env, cwd, literal args and additional descriptors", () => + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const literal = "quote' \" $HOME; * & %PATH%"; + const cwd = NodeOS.tmpdir(); + const script = ` + const fs = require('node:fs'); + const input = fs.readFileSync(0, 'utf8'); + fs.writeSync(1, JSON.stringify([input, process.cwd(), process.env.SPAWN_WORKER_TEST, process.argv[1]]) + '\\n'); + fs.writeSync(1, 'a'.repeat(1024 * 1024)); + fs.writeSync(2, 'b'.repeat(1024 * 1024)); + `; + for (let i = 0; i < 3; i++) { + const child = yield* spawner.spawn( + node(script, [literal], { + cwd, + env: { SPAWN_WORKER_TEST: "inherited value" }, + extendEnv: true, + stdin: Stream.encodeText(Stream.make("stdin value")), + }), + ); + const [stdout, stderr, code] = yield* Effect.all( + [text(child.stdout), text(child.stderr), child.exitCode], + { concurrency: "unbounded" }, + ); + expect(stdout).toBe( + JSON.stringify(["stdin value", NodePath.resolve(cwd), "inherited value", literal]) + + "\n" + + "a".repeat(1024 * 1024), + ); + expect(stderr).toBe("b".repeat(1024 * 1024)); + expect(code).toBe(0); + } + // The launcher threads already exist, so this needs the server's environment at launch time. + process.env.SPAWN_WORKER_LATE = "set after start"; + const late = yield* spawner.spawn( + node("process.stdout.write(process.env.SPAWN_WORKER_LATE)"), + ); + expect(yield* text(late.stdout)).toBe("set after start"); + delete process.env.SPAWN_WORKER_LATE; + const extra = yield* spawner.spawn( + node("require('node:fs').writeSync(3, 'fd3 value')", [], { + additionalFds: { fd3: { type: "output" } }, + }), + ); + const [fd3, extraCode] = yield* Effect.all([text(extra.getOutputFd(3)), extra.exitCode], { + concurrency: "unbounded", + }); + expect(fd3).toBe("fd3 value"); + expect(extraCode).toBe(0); + const reader = yield* spawner.spawn( + node("process.stdin.on('end', () => process.stdout.write('eof')).resume()"), + ); + yield* Stream.fail("abandoned input").pipe(Stream.run(reader.stdin), Effect.ignore); + expect(yield* text(reader.stdout)).toBe("eof"); + const pipeline = node("process.stdout.write('piped')").pipe( + ChildProcess.pipeTo( + node("process.stdin.on('data', (d) => process.stdout.write(String(d).toUpperCase()))"), + ), + ); + const piped = yield* spawner.spawn(pipeline); + expect(yield* text(piped.stdout)).toBe("PIPED"); + expect(yield* piped.exitCode).toBe(0); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("reports launch errors and exit codes, and keeps launching after either", () => + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const missing = yield* spawner + .spawn(ChildProcess.make("t3-missing-command-25245")) + .pipe(Effect.result); + expect(Result.isFailure(missing)).toBe(true); + if (Result.isFailure(missing)) expect(missing.failure.reason._tag).toBe("NotFound"); + const failed = yield* spawner.spawn(node("process.exit(23)")); + expect(yield* failed.exitCode).toBe(23); + const healthy = yield* spawner.spawn(node("process.stdout.write('ok')")); + expect(yield* text(healthy.stdout)).toBe("ok"); + expect(yield* healthy.exitCode).toBe(0); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("kills running commands, detached or not, and on scope interruption", () => + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + for (const detached of [true, false]) { + const child = yield* spawner.spawn( + node("process.stdout.write(String(process.pid));process.stdin.resume()", [], { + detached, + }), + ); + const pid = yield* child.stdout.pipe(Stream.take(1), text); + expect(Number(pid)).toBe(child.pid); + yield* child.kill(); + expect(yield* child.isRunning).toBe(false); + } + const started = yield* Deferred.make(); + const running = yield* Effect.gen(function* () { + const scoped = yield* spawner.spawn(node("process.stdin.resume()")); + yield* Deferred.succeed(started, scoped); + return yield* Effect.never; + }).pipe(Effect.scoped, Effect.forkChild); + const scoped = yield* Deferred.await(started); + yield* Fiber.interrupt(running); + expect(yield* scoped.isRunning).toBe(false); + // Direct callers such as the Claude adapter pass an AbortSignal, which cannot cross to a worker. + const direct = SpawnWorker.spawnPiped(process.execPath, ["-e", "process.stdin.resume()"], { + signal: AbortSignal.abort(), + }); + const exited = new Promise((resolve) => direct.once("exit", () => resolve(direct.killed))); + expect(yield* Effect.promise(() => exited)).toBe(true); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); +}); diff --git a/apps/server/src/process/SpawnWorker.ts b/apps/server/src/process/SpawnWorker.ts new file mode 100644 index 0000000000000..a43868b6d7b41 --- /dev/null +++ b/apps/server/src/process/SpawnWorker.ts @@ -0,0 +1,541 @@ +// @effect-diagnostics nodeBuiltinImport:off - worker threads and Node streams implement the platform spawner boundary. +import * as NodeEvents from "node:events"; +import type * as NodeChildProcess from "node:child_process"; +import * as NodeStream from "node:stream"; +import * as NodeWorkerThreads from "node:worker_threads"; +import * as NodeChildProcessSpawner from "@effect/platform-node-shared/NodeChildProcessSpawner"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import { supportsEnvChdir } from "@t3tools/shared/envTrampoline"; +import * as HostProcess from "@t3tools/shared/HostProcess"; + +// Windows runs CreateProcess synchronously on the calling thread, and antivirus or a busy disk can +// hold it for seconds; Linux's spawn waits for chdir and the PATH search, which a hung mount +// stalls. Workers own the children, so only the worker waits. +const MAX_WORKERS = 4; +const MAX_IDLE_WORKERS = 1; + +type ToWorker = + | { + readonly type: "spawn"; + readonly id: number; + readonly command: string; + readonly args: ReadonlyArray; + readonly options: NodeChildProcess.SpawnOptions; + } + | { readonly type: "write"; readonly id: number; readonly fd: number; readonly data: Uint8Array } + | { readonly type: "end"; readonly id: number; readonly fd: number } + | { readonly type: "flow"; readonly id: number; readonly fd: number; readonly paused: boolean } + | { readonly type: "destroy"; readonly id: number; readonly fd: number } + | { readonly type: "kill"; readonly id: number; readonly signal: NodeJS.Signals | number }; + +interface SerializedError { + readonly message: string; + readonly [key: string]: unknown; +} + +type FromWorker = + | { readonly type: "spawn"; readonly id: number; readonly pid: number | undefined } + | { readonly type: "error"; readonly id: number; readonly error: SerializedError } + | { readonly type: "data"; readonly id: number; readonly fd: number; readonly data: Uint8Array } + | { + readonly type: "written"; + readonly id: number; + readonly fd: number; + readonly error?: SerializedError; + } + | { readonly type: "eof"; readonly id: number; readonly fd: number } + | { readonly type: "finished"; readonly id: number; readonly fd: number } + | { + readonly type: "stream-error"; + readonly id: number; + readonly fd: number; + readonly error: SerializedError; + } + | { + readonly type: "exit"; + readonly id: number; + readonly code: number | null; + readonly signal: NodeJS.Signals | null; + } + | { readonly type: "close"; readonly id: number }; + +// Inline source runs from bundles, asar archives and single-executable builds without module files. +const workerSource = ` +const { parentPort } = require("node:worker_threads"); +const { spawn } = require("node:child_process"); +const children = new Map(); +const post = (message, transfer) => parentPort.postMessage(message, transfer); +const serialize = (error) => { + const out = { message: String(error && error.message) }; + for (const key of ["code", "errno", "syscall", "path", "spawnargs"]) { + if (error && error[key] !== undefined) out[key] = error[key]; + } + return out; +}; +parentPort.on("message", (request) => { + const { id } = request; + if (request.type === "spawn") { + let child; + try { + child = spawn(request.command, request.args, request.options); + } catch (error) { + post({ type: "error", id, error: serialize(error) }); + return; + } + children.set(id, child); + child.on("spawn", () => post({ type: "spawn", id, pid: child.pid })); + child.on("error", (error) => post({ type: "error", id, error: serialize(error) })); + child.on("exit", (code, signal) => post({ type: "exit", id, code, signal })); + // Output can still arrive after exit; close follows the last of it. + child.on("close", () => { + children.delete(id); + post({ type: "close", id }); + }); + child.stdio?.forEach((stream, fd) => { + if (!stream) return; + stream.on("error", (error) => post({ type: "stream-error", id, fd, error: serialize(error) })); + if (fd === 0) return; + // Cloning a Buffer copies its whole backing store; copy the bytes once and transfer them. + stream.on("data", (chunk) => { + const data = new Uint8Array(chunk); + post({ type: "data", id, fd, data }, [data.buffer]); + }); + stream.on("end", () => post({ type: "eof", id, fd })); + }); + return; + } + const child = children.get(id); + if (child === undefined) { + if (request.type === "write") post({ type: "written", id, fd: request.fd, error: { message: "write after exit", code: "EPIPE" } }); + if (request.type === "end") post({ type: "finished", id, fd: request.fd }); + return; + } + const stream = child.stdio?.[request.fd]; + switch (request.type) { + case "write": + if (!stream || stream.destroyed) { + post({ type: "written", id, fd: request.fd, error: { message: "write after end", code: "EPIPE" } }); + return; + } + try { + stream.write(request.data, (error) => post({ type: "written", id, fd: request.fd, error: error ? serialize(error) : undefined })); + } catch (error) { + post({ type: "written", id, fd: request.fd, error: serialize(error) }); + } + return; + case "end": + if (!stream || stream.destroyed || stream.writableFinished) post({ type: "finished", id, fd: request.fd }); + else stream.end(() => post({ type: "finished", id, fd: request.fd })); + return; + case "flow": + if (request.paused) stream?.pause(); + else stream?.resume(); + return; + case "destroy": + stream?.destroy(); + return; + case "kill": + try { + child.kill(request.signal); + } catch {} + return; + } +}); +`; + +const toError = (serialized: SerializedError) => + Object.assign(new Error(serialized.message), serialized); + +const toBuffer = (data: Uint8Array) => Buffer.from(data.buffer, data.byteOffset, data.byteLength); + +class WorkerChild + extends NodeEvents.EventEmitter + implements NodeChildProcessSpawner.SpawnedChildProcess +{ + pid: number | undefined; + exitCode: number | null = null; + signalCode: NodeJS.Signals | null = null; + stdin: NodeStream.Writable | null = null; + stdout: NodeStream.Readable | null = null; + stderr: NodeStream.Readable | null = null; + readonly stdio: Array = []; + referenced = true; + killed = false; + spawned = false; + exited = false; + readonly #writes = new Map void>>(); + readonly #finals = new Map void>(); + readonly #paused = new Set(); + + readonly worker: SpawnWorker; + readonly id: number; + + constructor(worker: SpawnWorker, id: number, stdio: ReadonlyArray) { + super(); + this.worker = worker; + this.id = id; + stdio.forEach((option, fd) => { + if (option !== "pipe" && option !== "overlapped") return; + const stream = this.#relay(fd); + this.stdio[fd] = stream; + if (fd === 0) this.stdin = stream as NodeStream.Writable; + else if (fd === 1) this.stdout = stream as NodeStream.Readable; + else if (fd === 2) this.stderr = stream as NodeStream.Readable; + }); + } + + #relay(fd: number): NodeStream.Readable | NodeStream.Writable { + const read = () => { + if (this.#paused.delete(fd)) + this.worker.post({ type: "flow", id: this.id, fd, paused: false }); + }; + const write = ( + chunk: Buffer, + _encoding: BufferEncoding, + callback: (error?: Error | null) => void, + ) => { + if (this.worker.closed) { + callback(Object.assign(new Error("write EPIPE"), { code: "EPIPE" })); + return; + } + const pending = this.#writes.get(fd) ?? []; + pending.push(callback); + this.#writes.set(fd, pending); + const data = new Uint8Array(chunk); + this.worker.post({ type: "write", id: this.id, fd, data }, [data.buffer]); + }; + const final = (callback: (error?: Error | null) => void) => { + this.#finals.set(fd, callback); + this.worker.post({ type: "end", id: this.id, fd }); + }; + // A caller that destroys its end closes the child pipe, as it would with a real pipe. + const destroy = (error: Error | null, callback: (error?: Error | null) => void) => { + if (!this.exited) this.worker.post({ type: "destroy", id: this.id, fd }); + callback(error); + }; + if (fd === 0) return new NodeStream.Writable({ write, final, destroy }); + if (fd <= 2) return new NodeStream.Readable({ read, destroy }); + return new NodeStream.Duplex({ read, write, final, destroy, allowHalfOpen: true }); + } + + onData(fd: number, data: Uint8Array): void { + const stream = this.stdio[fd]; + if (!stream || !("push" in stream)) return; + if (!stream.push(toBuffer(data)) && !this.#paused.has(fd)) { + this.#paused.add(fd); + this.worker.post({ type: "flow", id: this.id, fd, paused: true }); + } + } + + onEof(fd: number): void { + const stream = this.stdio[fd]; + if (stream && "push" in stream) stream.push(null); + } + + onFinished(fd: number): void { + const callback = this.#finals.get(fd); + this.#finals.delete(fd); + callback?.(); + } + + onWritten(fd: number, error: SerializedError | undefined): void { + const callback = this.#writes.get(fd)?.shift(); + callback?.(error === undefined ? null : toError(error)); + } + + onStreamError(fd: number, error: SerializedError): void { + this.stdio[fd]?.destroy(toError(error)); + } + + kill(signal: NodeJS.Signals | number = "SIGTERM"): boolean { + if (this.exited) return false; + this.killed = true; + this.worker.post({ type: "kill", id: this.id, signal }); + return true; + } + + ref(): void { + this.referenced = true; + this.worker.syncRef(); + } + + unref(): void { + this.referenced = false; + this.worker.syncRef(); + } + + exit(code: number | null, signal: NodeJS.Signals | null): void { + if (this.exited) return; + this.exited = true; + this.exitCode = code; + this.signalCode = signal; + // Node destroys stdin on exit; the Effect sink may already have removed its error listener. + this.stdin?.on("error", () => {}); + this.stdin?.destroy(); + this.emit("exit", code, signal); + } + + /** The worker is gone: end output so readers finish, and let the exit report the kill. */ + abandon(): void { + // Not destroy(error): Effect leaves an unread stderr's pipe without an error listener. + for (const stream of this.stdio) { + if (stream && "push" in stream && !stream.readableEnded) stream.push(null); + else if (stream && !("push" in stream)) stream.destroy(); + } + } +} + +class SpawnWorker { + readonly worker: NodeWorkerThreads.Worker; + readonly children = new Map(); + readonly pool: SpawnWorkerPool; + /** Launches sent but not yet answered; the worker is inside CreateProcess for one of them. */ + launching = 0; + closed = false; + + constructor(pool: SpawnWorkerPool) { + this.pool = pool; + this.worker = new NodeWorkerThreads.Worker(workerSource, { eval: true }); + this.worker.on("message", (message: FromWorker) => this.onMessage(message)); + this.worker.on("error", (error) => this.fail(error)); + this.worker.on("exit", () => this.fail(new Error("The process launcher thread exited."))); + this.worker.unref(); + } + + post(message: ToWorker, transfer?: ReadonlyArray): void { + if (!this.closed) this.worker.postMessage(message, transfer); + } + + spawn( + id: number, + command: string, + args: ReadonlyArray, + options: NodeChildProcess.SpawnOptions, + ): WorkerChild { + const io = options.stdio; + const stdio = + typeof io === "string" || io === undefined ? [io ?? "pipe", io ?? "pipe", io ?? "pipe"] : io; + const child = new WorkerChild(this, id, stdio); + // An AbortSignal cannot cross to the worker, so this thread kills on abort as Node would. + const { signal, ...rest } = options; + if (signal !== undefined) { + const abort = () => child.kill(); + if (signal.aborted) queueMicrotask(abort); + else { + signal.addEventListener("abort", abort, { once: true }); + child.once("exit", () => signal.removeEventListener("abort", abort)); + } + } + this.children.set(id, child); + this.launching++; + this.syncRef(); + this.post({ + type: "spawn", + id, + command, + args: [...args], + // A worker's process.env is a copy from when it started; children get the server's current one. + options: { ...rest, stdio, env: rest.env ?? { ...process.env } }, + }); + return child; + } + + syncRef(): void { + if (this.closed) return; + const referenced = + this.launching > 0 || [...this.children.values()].some((child) => child.referenced); + if (referenced) this.worker.ref(); + else this.worker.unref(); + } + + onMessage(message: FromWorker): void { + const child = this.children.get(message.id); + if (child === undefined) return; + switch (message.type) { + case "spawn": + this.launching--; + child.spawned = true; + child.pid = message.pid; + child.emit("spawn"); + break; + case "error": + if (!child.spawned) { + this.launching--; + child.exited = true; + this.release(child); + } + child.emit("error", toError(message.error)); + break; + case "data": + child.onData(message.fd, message.data); + break; + case "written": + child.onWritten(message.fd, message.error); + break; + case "eof": + child.onEof(message.fd); + break; + case "finished": + child.onFinished(message.fd); + break; + case "stream-error": + child.onStreamError(message.fd, message.error); + break; + case "exit": + child.exit(message.code, message.signal); + break; + case "close": + child.emit("close", child.exitCode, child.signalCode); + this.release(child); + break; + } + } + + release(child: WorkerChild): void { + this.children.delete(child.id); + this.syncRef(); + this.pool.release(this); + } + + fail(error: Error): void { + if (this.closed) return; + this.close(); + for (const child of this.children.values()) { + if (child.exited) { + child.abandon(); + continue; + } + if (!child.spawned) { + child.exited = true; + child.emit("error", error); + continue; + } + // The worker owned the only handle, so try to end the whole tree before reporting the exit. + const finish = () => { + child.exit(null, "SIGKILL"); + child.abandon(); + }; + if (child.pid === undefined) finish(); + else this.pool.killTree(child.pid, finish); + } + this.children.clear(); + } + + close(): void { + if (this.closed) return; + this.closed = true; + this.pool.workers.delete(this); + void this.worker.terminate(); + } +} + +class SpawnWorkerPool { + readonly workers = new Set(); + readonly platform: NodeJS.Platform; + #nextId = 0; + + constructor(platform: NodeJS.Platform) { + this.platform = platform; + } + + readonly spawn = ( + command: string, + args: ReadonlyArray, + options: NodeChildProcess.SpawnOptions, + ) => { + const workers = [...this.workers]; + // A worker inside CreateProcess cannot start another launch until it returns. + const worker = + workers.find((candidate) => candidate.launching === 0) ?? + (workers.length < MAX_WORKERS + ? this.#add() + : workers.reduce((best, candidate) => + candidate.launching < best.launching ? candidate : best, + )); + return worker.spawn(++this.#nextId, command, args, options); + }; + + killTree(pid: number, done: () => void): void { + const killPid = () => { + try { + process.kill(pid, "SIGKILL"); + } catch {} + }; + if (this.platform !== "win32") { + killPid(); + done(); + return; + } + let settled = false; + const finish = (killed: boolean) => { + if (settled) return; + settled = true; + // As Effect's own kill does when taskkill fails, end at least the direct child. + if (!killed) killPid(); + done(); + }; + this.spawn("taskkill", ["/pid", String(pid), "/T", "/F"], { + stdio: "ignore", + windowsHide: true, + }) + .on("error", () => finish(false)) + .on("exit", (code) => finish(code === 0)); + } + + #add(): SpawnWorker { + const worker = new SpawnWorker(this); + this.workers.add(worker); + return worker; + } + + release(worker: SpawnWorker): void { + if (worker.children.size > 0 || worker.launching > 0) return; + const idle = [...this.workers].filter( + (candidate) => candidate.children.size === 0 && candidate.launching === 0, + ); + if (idle.length > MAX_IDLE_WORKERS) worker.close(); + } + + close(): void { + for (const worker of this.workers) worker.close(); + } +} + +let sharedPool: SpawnWorkerPool | undefined; + +/** A piped `child_process.spawn` for Windows launches outside Effect's spawner. */ +export const spawnPiped = ( + command: string, + args: ReadonlyArray, + options: Omit, +) => + (sharedPool ??= new SpawnWorkerPool("win32")).spawn(command, args, { + ...options, + stdio: ["pipe", "pipe", "pipe"], + }) as WorkerChild & { + readonly stdin: NodeStream.Writable; + readonly stdout: NodeStream.Readable; + readonly stderr: NodeStream.Readable; + }; + +export const layer = Layer.unwrap( + Effect.gen(function* () { + const platform = yield* HostProcess.Platform; + const pool = yield* Effect.acquireRelease( + Effect.sync(() => new SpawnWorkerPool(platform)), + (pool) => Effect.sync(() => pool.close()), + ); + return NodeChildProcessSpawner.layerWithSpawn(pool.spawn); + }), +); + +/** Workers on Windows, and on Linux where `NonBlockingSpawner` has no GNU `env -C` to use. */ +export const layerServer = Layer.unwrap( + Effect.gen(function* () { + const platform = yield* HostProcess.Platform; + if (platform === "win32") return layer; + if (platform === "linux" && !(yield* Effect.promise(supportsEnvChdir))) return layer; + return NodeChildProcessSpawner.layer; + }), +); diff --git a/apps/server/src/process/agentScope.test.ts b/apps/server/src/process/agentScope.test.ts index a44ff15359f35..49674354b2532 100644 --- a/apps/server/src/process/agentScope.test.ts +++ b/apps/server/src/process/agentScope.test.ts @@ -12,6 +12,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Logger from "effect/Logger"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; +import { vi } from "vite-plus/test"; import * as ServerConfig from "../config.ts"; import { makeCodexAppServerSpawnCommand } from "../orchestration-v2/Adapters/CodexAdapterV2.ts"; @@ -19,6 +20,13 @@ import * as ProcessRunner from "../processRunner.ts"; import * as ProviderProcesses from "./providerProcesses.ts"; import { classifyScope, make, ownedAgentSlice, type ScopeState } from "./agentScope.ts"; +const GiB = 1024 ** 3; +const memory = vi.hoisted(() => ({ total: 16 * 1024 ** 3 })); +vi.mock("node:os", async (importOriginal) => ({ + ...(await importOriginal()), + totalmem: () => memory.total, +})); + const running: ScopeState = { loadState: "loaded", activeState: "active", @@ -49,6 +57,7 @@ const withFakeSystemd = ( readonly scope: Effect.Success; readonly units: Map; readonly calls: ReadonlyArray>; + readonly failingVerbs: Set; readonly stateDir: string; readonly cgroupRoot: string; }) => Effect.Effect, @@ -73,6 +82,7 @@ const withFakeSystemd = ( const cgroupRoot = NodePath.join(bin, "cgroup"); const units = new Map(); const calls: Array> = []; + const failingVerbs = new Set(); const output = (stdout: string, code = 0): ProcessRunner.ProcessRunOutput => ({ stdout, stderr: "", @@ -91,6 +101,7 @@ const withFakeSystemd = ( return output("", 1); } const [, verb, unit = ""] = args; + if (failingVerbs.has(verb!)) return output("", 1); if (stopFails && verb === "stop") { return { ...output(""), @@ -127,7 +138,7 @@ const withFakeSystemd = ( fileSystem.writeFileString(path.replace("/sys/fs/cgroup", cgroupRoot), data, options), }), ); - return yield* body({ scope, units, calls, stateDir, cgroupRoot }).pipe( + return yield* body({ scope, units, calls, failingVerbs, stateDir, cgroupRoot }).pipe( Effect.ensuring(Effect.sync(() => NodeFS.rmSync(bin, { recursive: true, force: true }))), ); }).pipe( @@ -320,6 +331,40 @@ describe("AgentScope service", () => { ), ); + it.effect("follows a live memory change at the next launch", () => { + memory.total = 16 * GiB; + return withFakeSystemd(({ scope, calls, failingVerbs }) => + Effect.gen(function* () { + const launch = scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" }); + const memoryLimits = () => + calls + .filter( + ([, verb, , slice]) => verb === "set-property" && slice === "app-t3code-agents.slice", + ) + .map((args) => args.slice(4)); + yield* launch; + yield* launch; + expect(memoryLimits()).toEqual([[`MemoryHigh=${10 * GiB}`, `MemoryMax=${12 * GiB}`]]); + + // A VM balloon or `pct set --memory` grows RAM while the server keeps running. + memory.total = 22 * GiB; + failingVerbs.add("set-property"); + yield* launch; + yield* launch; + failingVerbs.delete("set-property"); + yield* launch; + yield* launch; + const grown = [`MemoryHigh=${16 * GiB}`, `MemoryMax=${18 * GiB}`]; + expect(memoryLimits()).toEqual([ + [`MemoryHigh=${10 * GiB}`, `MemoryMax=${12 * GiB}`], + grown, + grown, + grown, + ]); + }), + ); + }); + it.effect("stops agents a killed server left before startup completes", () => withFakeSystemd( ({ scope, calls, stateDir }) => diff --git a/apps/server/src/process/agentScope.ts b/apps/server/src/process/agentScope.ts index 7adc4c453be08..1e52e680b6f8d 100644 --- a/apps/server/src/process/agentScope.ts +++ b/apps/server/src/process/agentScope.ts @@ -14,6 +14,7 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; +import * as Semaphore from "effect/Semaphore"; import * as ServerConfig from "../config.ts"; import * as ProcessRunner from "../processRunner.ts"; @@ -193,20 +194,44 @@ export const make = Effect.gen(function* () { slice, ...properties, ]); - if (result._tag === "None" || result.value.code !== 0) { - yield* Effect.logWarning("Could not set resource limits on an agents slice", { - slice, - properties, - stderr: result._tag === "Some" ? result.value.stderr.trim() : undefined, - }); - } + if (result._tag === "Some" && result.value.code === 0) return true; + yield* Effect.logWarning("Could not set resource limits on an agents slice", { + slice, + properties, + stderr: result._tag === "Some" ? result.value.stderr.trim() : undefined, + }); + return false; }); - const limits = agentSliceMemoryLimits(NodeOS.totalmem()); - yield* setProperties(AGENT_SLICE, [`MemoryHigh=${limits.high}`, `MemoryMax=${limits.max}`]); + // VMs and containers can resize RAM live, so each launch re-checks the total + // the limits came from. A failed update is retried by the next launch. + let appliedTotal: number | undefined; + const memoryLimitsLock = yield* Semaphore.make(1); + const applyMemoryLimits = memoryLimitsLock.withPermits(1)( + Effect.gen(function* () { + const total = NodeOS.totalmem(); + if (total === appliedTotal) return; + const limits = agentSliceMemoryLimits(total); + const applied = yield* setProperties(AGENT_SLICE, [ + `MemoryHigh=${limits.high}`, + `MemoryMax=${limits.max}`, + ]); + if (!applied) return; + if (appliedTotal !== undefined) { + yield* Effect.logInfo("Agent memory limits follow a memory change", { + slice: AGENT_SLICE, + previousTotal: appliedTotal, + total, + ...limits, + }); + } + appliedTotal = total; + }), + ); + yield* applyMemoryLimits; yield* setProperties(AGENTS_PARENT_SLICE, AGENTS_PARENT_PROPERTIES); yield* Effect.logInfo("Agent scopes are on", { slice: AGENT_SLICE, - ...limits, + ...agentSliceMemoryLimits(NodeOS.totalmem()), oomPolicy: supportsOomPolicy, }); return { @@ -214,6 +239,7 @@ export const make = Effect.gen(function* () { systemctl, runtimeDir, supportsOomPolicy, + applyMemoryLimits, }; }), ); @@ -403,6 +429,7 @@ export const make = Effect.gen(function* () { if (resolved === null) return { command, args }; const scope = yield* systemd; if (scope === undefined) return agentScopeCommand({ command: resolved, args }); + yield* scope.applyMemoryLimits; const unit = `t3code-${name}-${NodeCrypto.randomUUID().slice(0, 8)}.scope`; // A continuing shared scope cannot attribute a tool's OOM to one thread. // Clear the launch thread's previous scope instead. diff --git a/apps/server/src/process/externalLauncher.test.ts b/apps/server/src/process/externalLauncher.test.ts index 0f73ab4fb9c53..f215fe4ebc6a2 100644 --- a/apps/server/src/process/externalLauncher.test.ts +++ b/apps/server/src/process/externalLauncher.test.ts @@ -21,6 +21,7 @@ import { resolveEditorCommand } from "@t3tools/shared/editor"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { SpawnExecutableResolution } from "@t3tools/shared/shell"; import * as ExternalLauncher from "./externalLauncher.ts"; +import { WindowsVerbatimSpawn } from "./windowsVerbatimSpawn.ts"; // Windows drive letters break mocked POSIX PATH splitting; these tests also require /bin/sh stubs. const windowsHost = HostProcess.Platform.defaultValue() === "win32"; @@ -104,36 +105,47 @@ const layerTest = (input: { }; // macOS command lookup cannot search a Windows drive-letter PATH. -it.effect.skipIf(windowsHost)( - "opens home-relative app bundles using the server home directory", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const bin = yield* fs.makeTempDirectoryScoped({ prefix: "t3-app-open-" }); - yield* fs.writeFileString(path.join(bin, "open"), "#!/bin/sh\n"); - yield* fs.chmod(path.join(bin, "open"), 0o755); - const spawned: ChildProcess.StandardCommand[] = []; - yield* Effect.gen(function* () { - const launcher = yield* ExternalLauncher.ExternalLauncher; - for (const cwd of ["~/Applications/My App.app", "/Applications/My App.app"]) { - yield* launcher.launchEditor({ editor: "file-manager", cwd }); - } - }).pipe( - Effect.provide( - layerTest({ - platform: "darwin", - env: { PATH: bin }, - onSpawn: (command) => spawned.push(command), - }), - ), - Effect.provideService(HostProcess.HomeDirectory, "/Users/server"), - ); - assert.deepEqual( - spawned.map((command) => command.args), - [["/Users/server/Applications/My App.app"], ["/Applications/My App.app"]], - ); - }).pipe(Effect.provide(NodeServices.layer)), +it.effect.skipIf(windowsHost)("opens home-relative targets using the server home directory", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const bin = yield* fs.makeTempDirectoryScoped({ prefix: "t3-app-open-" }); + for (const name of ["open", "code", "idea"]) { + yield* fs.writeFileString(path.join(bin, name), "#!/bin/sh\n"); + yield* fs.chmod(path.join(bin, name), 0o755); + } + const spawned: ChildProcess.StandardCommand[] = []; + yield* Effect.gen(function* () { + const launcher = yield* ExternalLauncher.ExternalLauncher; + for (const cwd of ["~/Applications/My App.app", "/Applications/My App.app"]) { + yield* launcher.launchEditor({ editor: "file-manager", cwd }); + } + yield* launcher.launchEditor({ editor: "vscode", cwd: "~/notes/todo.md:3" }); + yield* launcher.launchEditor({ + editor: "idea", + cwd: "~/notes/todo.md:3", + projectRoot: "~/notes", + }); + }).pipe( + Effect.provide( + layerTest({ + platform: "darwin", + env: { PATH: bin }, + onSpawn: (command) => spawned.push(command), + }), + ), + Effect.provideService(HostProcess.HomeDirectory, "/Users/server"), + ); + assert.deepEqual( + spawned.map((command) => command.args), + [ + ["/Users/server/Applications/My App.app"], + ["/Applications/My App.app"], + ["--goto", "/Users/server/notes/todo.md:3"], + ["/Users/server/notes", "--line", "3", "/Users/server/notes/todo.md"], + ], + ); + }).pipe(Effect.provide(NodeServices.layer)), ); it.effect.skipIf(windowsHost)( @@ -194,6 +206,8 @@ it.effect.skipIf(windowsHost)( const codePath = path.join(binDir, "code"); yield* fileSystem.writeFileString(codePath, "#!/bin/sh\n"); yield* fileSystem.chmod(codePath, 0o755); + const colonProject = path.join(project, "build:2026"); + yield* fileSystem.makeDirectory(colonProject); const file = `${path.join(project, "file.ts")}:3:7`; const spawned: ChildProcess.StandardCommand[] = []; @@ -201,6 +215,7 @@ it.effect.skipIf(windowsHost)( const launcher = yield* ExternalLauncher.ExternalLauncher; yield* launcher.launchEditor({ editor: "vscode", cwd: project }); yield* launcher.launchEditor({ editor: "vscode", cwd: file }); + yield* launcher.launchEditor({ editor: "vscode", cwd: colonProject }); }).pipe( Effect.provide( layerTest({ @@ -217,6 +232,7 @@ it.effect.skipIf(windowsHost)( [ ["--new-window", project], ["--goto", file], + ["--new-window", colonProject], ], ); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), @@ -295,6 +311,69 @@ it.effect.skipIf(windowsHost)( }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); +// Windows drive letters cannot resolve the POSIX xdg stubs. +it.effect.skipIf(windowsHost)( + "a Linux server without a display offers and opens the file manager on the desktop session's display", + () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const binDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-editors-" }); + for (const name of ["xdg-open", "xdg-mime"]) { + const filePath = path.join(binDir, name); + yield* fileSystem.writeFileString(filePath, "#!/bin/sh\n"); + yield* fileSystem.chmod(filePath, 0o755); + } + + const fileManager = (sessionEnv: ReadonlyArray) => + Effect.gen(function* () { + const spawned: ChildProcess.StandardCommand[] = []; + const result = yield* Effect.gen(function* () { + const launcher = yield* ExternalLauncher.ExternalLauncher; + const editors = yield* launcher.resolveAvailableEditors(); + const kind = yield* launcher.resolveFileManagerRevealKind(); + const opened = yield* launcher + .launchEditor({ editor: "file-manager", cwd: "/workspace" }) + .pipe(Effect.exit); + return { offered: editors.includes("file-manager"), kind, opened }; + }).pipe( + Effect.provide( + layerTest({ + platform: "linux", + env: { PATH: binDir }, + onSpawn: (command) => spawned.push(command), + spawnResult: (command) => + command.command === "busctl" + ? { stdout: JSON.stringify({ type: "as", data: sessionEnv }) } + : command.command === "xdg-mime" + ? { stdout: "org.kde.dolphin.desktop\n" } + : undefined, + }), + ), + Effect.provideService(HostProcess.Environment, { PATH: binDir }), + ); + const open = spawned.find((command) => command.command === "xdg-open"); + return { ...result, open: open && { args: open.args, env: open.options.env } }; + }); + + const session = yield* fileManager(["DISPLAY=:0", "HOME=/home/user"]); + assert.equal(session.offered, true); + assert.equal(session.kind, "files"); + assert.equal(Exit.isSuccess(session.opened), true); + assert.deepEqual(session.open, { + args: ["/workspace"], + env: { PATH: binDir, DISPLAY: ":0" }, + }); + + // No one is logged in to a desktop, so there is no screen to open it on. + const headless = yield* fileManager(["HOME=/home/user"]); + assert.equal(headless.offered, false); + assert.equal(headless.kind, undefined); + assert.equal(Exit.isFailure(headless.opened), true); + assert.equal(headless.open, undefined); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + it.effect("launches an installed editor with platform-safe arguments", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; @@ -338,6 +417,144 @@ it.effect("launches an installed editor with platform-safe arguments", () => }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); +const wslpathSpawner = (out: string | undefined) => + Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make((command) => + Effect.sync(() => { + if (!ChildProcess.isStandardCommand(command)) { + throw new Error("Expected a standard command"); + } + assert.equal(command.command, "wslpath"); + return makeMockDetachedHandle( + out === undefined ? { exitCode: 1 } : { exitCode: 0, stdout: `${out}\n` }, + ); + }), + ), + ); + +it.effect.each([ + // wslpath is mount-aware: a drive remapped in /etc/wsl.conf keeps its real letter. + { target: "/mnt/d/project", wslpath: "E:\\project", expected: "E:\\project" }, + // wslpath missing or failing falls back to the hand-built UNC path. + { + target: "/home/user/repo", + wslpath: undefined, + expected: "\\\\wsl.localhost\\Ubuntu-24.04\\home\\user\\repo", + }, + { target: "/", wslpath: undefined, expected: "\\\\wsl.localhost\\Ubuntu-24.04" }, +])("resolveWslFileManagerPath resolves $target to $expected", ({ target, wslpath, expected }) => + Effect.gen(function* () { + const result = yield* ExternalLauncher.resolveWslFileManagerPath(target, "Ubuntu-24.04"); + assert.equal(result, expected); + }).pipe(Effect.provide(wslpathSpawner(wslpath))), +); + +it.live.skipIf(!windowsHost)("opens Windows folders through the default shell handler", () => + Effect.gen(function* () { + let spawned: ChildProcess.StandardCommand | undefined; + yield* Effect.gen(function* () { + const launcher = yield* ExternalLauncher.ExternalLauncher; + yield* launcher.launchEditor({ + editor: "file-manager", + cwd: "C:/workspace [draft]/author's ‘left’ ‚low‛ $media/’; Write-Output injected; #", + }); + }).pipe( + Effect.provide( + layerTest({ + platform: "win32", + env: { PATH: "C:\\Windows", PATHEXT: ".EXE" }, + onSpawn: (command) => { + spawned = command; + }, + }), + ), + ); + assert.ok(spawned); + assert.equal(spawned.options.detached, false); + const source = Buffer.from(String(spawned.args.at(-1) ?? ""), "base64").toString("utf16le"); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const output = yield* spawner.string( + ChildProcess.make(spawned.command, [ + ...spawned.args.slice(0, -1), + Buffer.from( + `[Console]::OutputEncoding = [Text.UTF8Encoding]::new($false); function Invoke-Item { param($LiteralPath) Write-Output $LiteralPath }; ${source}`, + "utf16le", + ).toString("base64"), + ]), + ); + assert.equal( + output.trim(), + "C:\\workspace [draft]\\author's ‘left’ ‚low‛ $media\\’; Write-Output injected; #", + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + +for (const [replacement, verb, handler, target, expected] of [ + ["none", "none", "", "C:/repo/file.txt", 'explorer.exe /select,"C:\\repo\\file.txt"'], + ["Directory", "openindopus", "dopusrt.exe", "C:/repo/file.txt", "shell:C:\\repo"], + ["Folder", "openindopus", "dopusrt.exe", "C:/repo/file.txt", "shell:C:\\repo"], + [ + "Folder", + "open", + '"C:\\Windows\\explorer.exe"', + "C:/repo/file.txt", + 'explorer.exe /select,"C:\\repo\\file.txt"', + ], + [ + "Folder", + "open", + "ExplorerDelegate", + "C:/repo/file.txt", + 'explorer.exe /select,"C:\\repo\\file.txt"', + ], + [ + "Drive", + "openindopus", + "dopusrt.exe", + "C:/repo/file.txt", + 'explorer.exe /select,"C:\\repo\\file.txt"', + ], + ["Drive", "openindopus", "dopusrt.exe", "C:/file.txt", "shell:C:\\"], +] as const) { + it.live.skipIf(!windowsHost)( + `reveals ${target} with ${replacement}/${verb} handled by ${handler}`, + () => + Effect.gen(function* () { + let spawned: ChildProcess.StandardCommand | undefined; + yield* Effect.gen(function* () { + const launcher = yield* ExternalLauncher.ExternalLauncher; + yield* launcher.launchEditor({ + editor: "file-manager", + cwd: target, + reveal: true, + }); + }).pipe( + Effect.provide( + layerTest({ + platform: "win32", + env: { PATH: "C:\\Windows", PATHEXT: ".EXE" }, + onSpawn: (command) => { + spawned = command; + }, + }), + ), + ); + assert.ok(spawned); + const source = Buffer.from(String(spawned.args.at(-1) ?? ""), "base64").toString("utf16le"); + const stubs = `function Get-ItemProperty { param($LiteralPath, $ErrorAction) @{ '(default)' = if ($LiteralPath -eq 'Registry::HKEY_CLASSES_ROOT\\${replacement}\\shell') { '${verb}' } elseif ($LiteralPath -eq 'Registry::HKEY_CLASSES_ROOT\\${replacement}\\shell\\${verb}\\command') { '${handler}' } else { 'none' }; DelegateExecute = if ('${handler}' -eq 'ExplorerDelegate') { '{11dbb47c-a525-400b-9e80-a54615a090c0}' } else { '' } } }; function Invoke-Item { param($LiteralPath) Write-Output "shell:$LiteralPath" }; function Start-Process { param($FilePath, $ArgumentList) Write-Output "$FilePath $ArgumentList" }; `; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const output = yield* spawner.string( + ChildProcess.make(spawned.command, [ + ...spawned.args.slice(0, -1), + Buffer.from(stubs + source, "utf16le").toString("base64"), + ]), + ); + assert.equal(output.trim(), expected); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +} + it.effect.skipIf(windowsHost)( "reveals a WSL file in Windows File Explorer through its UNC path", () => @@ -358,7 +575,7 @@ it.effect.skipIf(windowsHost)( const editors = yield* launcher.resolveAvailableEditors(); yield* launcher.launchEditor({ editor: "file-manager", - cwd: "/home/t3/workspace [draft]/author's $media/clip|draft.mp4", + cwd: "/home/t3/workspace [draft]/author's ‘$media/clip|draft.mp4", reveal: true, }); return { kind, editors }; @@ -385,13 +602,85 @@ it.effect.skipIf(windowsHost)( assert.equal(spawned.options.detached, true); const encodedCommand = String(spawned.args[spawned.args.length - 1] ?? ""); const decodedCommand = Buffer.from(encodedCommand, "base64").toString("utf16le"); - assert.equal( + assert.include(decodedCommand, "'Directory', 'Folder'"); + assert.include( decodedCommand, - "$ProgressPreference = 'SilentlyContinue'; $folderVerb = (Get-ItemProperty -LiteralPath 'Registry::HKEY_CLASSES_ROOT\\Directory\\shell' -ErrorAction SilentlyContinue).'(default)'; if ($folderVerb -and $folderVerb -ne 'none') { Invoke-Item -LiteralPath '\\\\wsl.localhost\\Ubuntu-24.04\\home\\t3\\workspace [draft]\\author''s $media' } else { Start-Process 'explorer.exe' -ArgumentList ('/select,\"' + '\\\\wsl.localhost\\Ubuntu-24.04\\home\\t3\\workspace [draft]\\author''s $media\\clip|draft.mp4' + '\"') }", + "Invoke-Item -LiteralPath '\\\\wsl.localhost\\Ubuntu-24.04\\home\\t3\\workspace [draft]\\author''s ‘‘$media'", + ); + assert.include( + decodedCommand, + "'\\\\wsl.localhost\\Ubuntu-24.04\\home\\t3\\workspace [draft]\\author''s ‘‘$media\\clip|draft.mp4'", ); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); +const STOCK_FOLDER_VERB = + "\r\nHKEY_CLASSES_ROOT\\Directory\\shell\r\n (Default) REG_SZ none\r\n\r\n"; + +it.effect.each([ + ["selects the file with Explorer on stock Windows", { stdout: STOCK_FOLDER_VERB }, true], + [ + "keeps the PowerShell reveal for a replacement file manager", + { stdout: " (Default) REG_SZ open\r\n" }, + false, + ], + ["keeps the PowerShell reveal when the registry lookup fails", { exitCode: 1 }, false], +] as const)("%s", ([, registry, native]) => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const binDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-editors-" }); + yield* fileSystem.writeFileString(path.join(binDir, "explorer.EXE"), ""); + const spawned: ChildProcess.StandardCommand[] = []; + const verbatim: Array]> = []; + const exit = yield* Effect.gen(function* () { + const launcher = yield* ExternalLauncher.ExternalLauncher; + return yield* launcher + .launchEditor({ + editor: "file-manager", + cwd: "C:/Users/t3/my project/a,b.ts", + reveal: true, + }) + .pipe(Effect.exit); + }).pipe( + Effect.provideService(WindowsVerbatimSpawn, (command, args) => + Effect.sync(() => verbatim.push([command, args])), + ), + Effect.provide( + layerTest({ + platform: "win32", + env: { PATH: binDir, PATHEXT: ".COM;.EXE;.BAT;.CMD", SYSTEMROOT: "C:\\Windows" }, + onSpawn: (command) => spawned.push(command), + spawnResult: (command) => (command.command.endsWith("reg.exe") ? registry : undefined), + }), + ), + ); + + assert.deepEqual( + spawned.map((command) => command.command), + ["C:\\Windows\\System32\\reg.exe"], + ); + if (native) { + assert.isTrue(Exit.isSuccess(exit)); + assert.deepEqual(verbatim, [ + ["C:\\Windows\\explorer.exe", ['/select,"C:\\Users\\t3\\my project\\a,b.ts"']], + ]); + } else { + assert.deepEqual(verbatim, []); + // The PowerShell launch is reached; this host has no powershell.exe to run it. + assert.isTrue(Exit.isFailure(exit)); + if (Exit.isFailure(exit)) { + const error = Cause.squash(exit.cause); + assert.instanceOf(error, ExternalLauncher.ExternalLauncherCommandNotFoundError); + assert.equal( + (error as ExternalLauncher.ExternalLauncherCommandNotFoundError).command, + "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe", + ); + } + } + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + for (const wsl of [false, true]) { it.effect.skipIf(windowsHost)( "selects a file over D-Bus on " + (wsl ? "WSLg without interop PowerShell" : "Linux"), @@ -614,17 +903,32 @@ it.effect.skipIf(windowsHost)("does not offer the OpenZFS zed daemon as Zed on L ); yield* fs.chmod(zfsZed, 0o755); const editor = EDITORS.find((editor) => editor.id === "zed")!; - const resolve = resolveEditorCommand(editor, { HOME: home, PATH: binDir }).pipe( - Effect.provideService(HostProcess.Platform, "linux"), - Effect.provideService(Path.Path, withSystemBin(path, systemBin)), - ); - assert.deepEqual(yield* resolve, Option.none()); + const laterBin = path.join(home, "later-bin"); + const resolveOn = (platform: NodeJS.Platform) => + resolveEditorCommand(editor, { HOME: home, PATH: `${binDir}:${laterBin}` }).pipe( + Effect.provideService(HostProcess.Platform, platform), + Effect.provideService(Path.Path, withSystemBin(path, systemBin)), + ); + assert.deepEqual(yield* resolveOn("linux"), Option.none()); + assert.deepEqual(yield* resolveOn("darwin"), Option.none()); const zedEditor = path.join(home, ".local/bin/zed"); yield* fs.makeDirectory(path.dirname(zedEditor), { recursive: true }); yield* fs.writeFileString(zedEditor, "#!/bin/sh\necho 'Zed 0.200.0'\n"); yield* fs.chmod(zedEditor, 0o755); - assert.deepEqual(yield* resolve, Option.some({ command: zedEditor, baseArgs: [] })); + assert.deepEqual(yield* resolveOn("linux"), Option.some({ command: zedEditor, baseArgs: [] })); + + yield* fs.remove(zedEditor); + const laterZed = path.join(laterBin, "zed"); + yield* fs.makeDirectory(laterBin, { recursive: true }); + yield* fs.writeFileString(laterZed, "#!/bin/sh\necho 'Zed 0.200.0'\n"); + yield* fs.chmod(laterZed, 0o755); + for (const platform of ["linux", "darwin"] as const) { + assert.deepEqual( + yield* resolveOn(platform), + Option.some({ command: laterZed, baseArgs: [] }), + ); + } }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); @@ -647,7 +951,7 @@ it.effect.skipIf(windowsHost)("does not open Zed with the OpenZFS zed daemon on Effect.provide( layerTest({ platform: "linux", - env: { HOME: home, PATH: binDir }, + env: { HOME: home, PATH: binDir, DISPLAY: ":0" }, path: withSystemBin(path, systemBin), onSpawn: (command) => spawned.push(command), spawnResult: (command) => (command.args[0] === "--version" ? { exitCode: 1 } : undefined), @@ -792,6 +1096,62 @@ it.effect("keeps the macOS VS Code Insiders bundle out of stable VS Code", () => }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); +it.effect.each([ + { + platform: "darwin", + scripts: "Library/Application Support/JetBrains/Toolbox/scripts", + suffix: "", + }, + { platform: "linux", scripts: ".local/share/JetBrains/Toolbox/scripts", suffix: "" }, + { platform: "win32", scripts: "JetBrains/Toolbox/scripts", suffix: ".cmd" }, + { + platform: "win32", + scripts: "JetBrains/Toolbox/scripts", + suffix: ".cmd", + install: "Programs/IntelliJ IDEA/bin/idea64.exe", + }, +] as const)( + "discovers a numbered JetBrains Toolbox script on $platform", + ({ platform, scripts, suffix, ...row }) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "t3-toolbox-numbered-" }); + const dir = path.join(home, scripts); + yield* fs.makeDirectory(dir, { recursive: true }); + for (const name of ["idea10", "idea2"]) { + yield* fs.writeFileString(path.join(dir, `${name}${suffix}`), "#!/bin/sh\n"); + yield* fs.chmod(path.join(dir, `${name}${suffix}`), 0o755); + } + const install = "install" in row ? path.join(home, row.install) : undefined; + if (install) { + yield* fs.makeDirectory(path.dirname(install), { recursive: true }); + yield* fs.writeFileString(install, ""); + } + + const editor = EDITORS.find((editor) => editor.id === "idea")!; + const resolved = yield* resolveEditorCommand(editor, { + HOME: home, + LOCALAPPDATA: home, + PATH: path.join(home, "empty"), + PATHEXT: ".COM;.EXE;.BAT;.CMD", + }).pipe( + Effect.provideService(HostProcess.Platform, platform), + // Keep IDEs installed on the host out of system locations such as /Applications. + Effect.provideService(FileSystem.FileSystem, { + ...fs, + stat: (file) => + fs.stat(file.startsWith(`${home}${path.sep}`) ? file : path.join(home, "missing")), + }), + ); + + assert.deepEqual( + resolved, + Option.some({ command: install ?? path.join(dir, `idea2${suffix}`), baseArgs: [] }), + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + for (const { platform, installPath, onPath } of [ { platform: "darwin", installPath: ".local/bin/agy", onPath: true }, { platform: "linux", installPath: ".local/bin/agy", onPath: false }, @@ -949,6 +1309,57 @@ it.effect( }, ); +it.effect.each(["Zed Nightly", "Zed Preview"])( + "launches stable Zed on Windows when %s comes first on PATH", + (channel) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-zed-channels-" }); + // A user folder naming Zed and a channel does not mark either install. + const programs = path.join(root, "Users", "zedpreview", "Programs"); + const prereleaseBin = path.join(programs, channel, "bin"); + const stableBin = path.join(programs, "Zed", "bin"); + for (const dir of [prereleaseBin, stableBin]) { + yield* fs.makeDirectory(dir, { recursive: true }); + yield* fs.writeFileString(path.join(dir, "zed.EXE"), ""); + } + const env = { PATH: `${prereleaseBin};${stableBin}`, PATHEXT: ".COM;.EXE;.BAT;.CMD" }; + + const launchZed = Effect.gen(function* () { + let spawned: ChildProcess.StandardCommand | undefined; + yield* Effect.gen(function* () { + const launcher = yield* ExternalLauncher.ExternalLauncher; + yield* launcher.launchEditor({ editor: "zed", cwd: "C:\\workspace" }); + }).pipe( + Effect.provide( + layerTest({ + platform: "win32", + env, + onSpawn: (command) => { + spawned = command; + }, + }), + ), + ); + assert.ok(spawned); + return spawned; + }); + + const stable = yield* launchZed; + assert.equal(stable.command, path.join(stableBin, "zed.EXE")); + assert.deepEqual(stable.args, ["C:\\workspace"]); + + env.PATH = `${stableBin};${prereleaseBin}`; + assert.equal((yield* launchZed).command, "zed"); + + env.PATH = `${prereleaseBin};${stableBin}`; + // With only the prerelease install left, it still opens by name. + yield* fs.remove(path.join(stableBin, "zed.EXE")); + assert.equal((yield* launchZed).command, "zed"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + const launchWindowsShimEditor = (cwd: string) => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; diff --git a/apps/server/src/process/externalLauncher.ts b/apps/server/src/process/externalLauncher.ts index aea3e2d1472f9..2bdb7bd3db4db 100644 --- a/apps/server/src/process/externalLauncher.ts +++ b/apps/server/src/process/externalLauncher.ts @@ -34,6 +34,7 @@ import * as ChildProcess from "effect/process/ChildProcess"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import { makeStaleWhileRevalidate } from "../environment/staleWhileRevalidate.ts"; +import { WindowsVerbatimSpawn } from "./windowsVerbatimSpawn.ts"; export { ExternalLauncherError, ExternalLauncherBrowserSpawnError, @@ -148,9 +149,10 @@ function resolveCommandEditorArgs( case "direct-path": return [target]; case "goto": - if (Option.isSome(parsedTarget)) return ["--goto", target]; + // An existing folder wins over a position: `build:2026` is a legal folder name. // Without a window flag, VS Code's window settings may replace the user's last active window. - return isDirectory ? ["--new-window", target] : [target]; + if (isDirectory) return ["--new-window", target]; + return Option.isSome(parsedTarget) ? ["--goto", target] : [target]; case "line-column": return Option.match(parsedTarget, { onNone: () => [target], @@ -181,11 +183,16 @@ function encodeUtf16LeBase64(input: string): string { } function escapePowerShellStringLiteral(input: string): string { - return `'${input.replaceAll("'", "''")}'`; + // PowerShell also recognises typographic apostrophes as single-quote delimiters. + return `'${input.replace(/['\u2018-\u201b]/g, "$&$&")}'`; +} + +function resolveWindowsRoot(env: NodeJS.ProcessEnv): string { + return env.SYSTEMROOT || env.windir || String.raw`C:\Windows`; } function resolvePowerShellPath(env: NodeJS.ProcessEnv = {}): string { - return `${env.SYSTEMROOT || env.windir || String.raw`C:\Windows`}\\System32\\WindowsPowerShell\\v1.0\\powershell.exe`; + return `${resolveWindowsRoot(env)}\\System32\\WindowsPowerShell\\v1.0\\powershell.exe`; } function resolveWslPowerShellPath(): string { @@ -364,11 +371,41 @@ const fileManagerRevealKindForPlatform = Effect.fn( return hasGraphicalLinuxSession(env) ? "files" : undefined; }); -function resolveWslFileManagerPath(target: string, distroName: string): string { +function wslUncFallbackPath(target: string, distroName: string): string { const relativePath = target.replace(/^\/+/, "").replaceAll("/", "\\"); return `\\\\wsl.localhost\\${distroName}${relativePath.length > 0 ? `\\${relativePath}` : ""}`; } +// wslpath -w is mount-aware, so a drive remapped in /etc/wsl.conf resolves to the +// real letter and a Linux path resolves to the \\wsl.localhost UNC path. Fall back +// to the UNC form by hand when wslpath is missing or fails. +export const resolveWslFileManagerPath = Effect.fn("externalLauncher.resolveWslFileManagerPath")( + function* ( + target: string, + distroName: string, + ): Effect.fn.Return { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + return yield* spawner + .spawn(ChildProcess.make("wslpath", ["-w", target], { stdin: "ignore", stderr: "ignore" })) + .pipe( + Effect.flatMap((handle) => + Effect.all([handle.stdout.pipe(Stream.decodeText(), Stream.mkString), handle.exitCode], { + concurrency: "unbounded", + }), + ), + Effect.map(([stdout, exitCode]) => { + const translated = stdout.trim(); + return exitCode === 0 && translated.length > 0 + ? translated + : wslUncFallbackPath(target, distroName); + }), + Effect.scoped, + Effect.timeout("2 seconds"), + Effect.orElseSucceed(() => wslUncFallbackPath(target, distroName)), + ); + }, +); + function buildBrowserLaunch( target: string, platform: NodeJS.Platform, @@ -436,16 +473,23 @@ const resolveBrowserLaunch = Effect.fn("externalLauncher.resolveBrowserLaunch")( return buildBrowserLaunch(target, platform, env); }); +// Editors and the file manager open on the desktop session, so a boot-started +// server discovers and launches them with that session's display. +const readEditorEnv = Effect.fn("externalLauncher.readEditorEnv")(function* () { + const env = { ...(yield* readBrowserLaunchEnv), ...(yield* readCommandLookupEnv) }; + return hasGraphicalLinuxSession(env) ? env : { ...env, ...(yield* readGraphicalSessionEnv()) }; +}); + const resolveAvailableEditors = Effect.fn("externalLauncher.resolveAvailableEditors")(function* () { const platform = yield* HostProcess.Platform; - const env = { ...(yield* readBrowserLaunchEnv), ...(yield* readCommandLookupEnv) }; + const env = yield* readEditorEnv(); return yield* buildAvailableEditors(platform, env).pipe(withPathDirectoryListings); }); const resolveFileManagerRevealKind = Effect.fn("externalLauncher.resolveFileManagerRevealKind")( function* () { const platform = yield* HostProcess.Platform; - const env = { ...(yield* readBrowserLaunchEnv), ...(yield* readCommandLookupEnv) }; + const env = yield* readEditorEnv(); return yield* fileManagerRevealKindForPlatform(platform, env); }, ); @@ -471,7 +515,7 @@ const resolveEditorLaunch = Effect.fn("resolveEditorLaunch")(function* ( FileSystem.FileSystem | Path.Path | ChildProcessSpawner.ChildProcessSpawner > { const platform = yield* HostProcess.Platform; - const env = { ...(yield* readBrowserLaunchEnv), ...(yield* readCommandLookupEnv) }; + const env = yield* readEditorEnv(); yield* Effect.annotateCurrentSpan({ "externalLauncher.editor": input.editor, "externalLauncher.cwd": input.cwd, @@ -517,31 +561,42 @@ const resolveEditorLaunch = Effect.fn("resolveEditorLaunch")(function* ( return yield* new ExternalLauncherUnsupportedEditorError({ editor: input.editor }); } - const target = expandHomePath(input.cwd, yield* HostProcess.HomeDirectory); + const target = input.cwd; if (input.reveal === true) { return yield* resolveFileManagerRevealLaunch(target, platform, env, command); } + if (platform === "win32") { + // ShellExecute honours replacement folder handlers that explorer.exe bypasses. + return { + ...powerShellFileManagerLaunch( + target, + `Invoke-Item -LiteralPath ${escapePowerShellStringLiteral(normalizeWindowsFileManagerPath(target))}`, + resolvePowerShellPath(env), + ), + detached: WINDOWS_POWERSHELL_DETACHED, + }; + } + return { editor: editorDef.id, target, command, args: command === "explorer.exe" && env.WSL_DISTRO_NAME !== undefined - ? [resolveWslFileManagerPath(target, env.WSL_DISTRO_NAME)] - : platform === "win32" - ? // Explorer interprets / as a switch and silently opens a fallback folder. - [normalizeWindowsFileManagerPath(target)] - : [target], + ? [yield* resolveWslFileManagerPath(target, env.WSL_DISTRO_NAME)] + : [target], }; }); -// A replacement manager registers a default Directory shell verb; stock Windows -// uses "none". Explorer needs its /select argument passed verbatim through -// PowerShell because Node's argument quoting changes how Explorer parses it. +// Replacement managers can register under Directory, Drive, or Folder. Explorer +// needs its /select argument passed verbatim through PowerShell because Node's +// argument quoting changes how Explorer parses it. function buildFileManagerRevealPowerShellSource(target: string, parent: string): string { const literal = escapePowerShellStringLiteral(target); - return `$ProgressPreference = 'SilentlyContinue'; $folderVerb = (Get-ItemProperty -LiteralPath 'Registry::HKEY_CLASSES_ROOT\\Directory\\shell' -ErrorAction SilentlyContinue).'(default)'; if ($folderVerb -and $folderVerb -ne 'none') { Invoke-Item -LiteralPath ${escapePowerShellStringLiteral(parent)} } else { Start-Process 'explorer.exe' -ArgumentList ('/select,"' + ${literal} + '"') }`; + const folderClass = /^[a-z]:\\$/i.test(parent) ? "Drive" : "Directory"; + // Explorer's ordinary open/explore handlers may also be explicit defaults. + return String.raw`$ProgressPreference = 'SilentlyContinue'; $folderVerb = '${folderClass}', 'Folder' | ForEach-Object { $key = "Registry::HKEY_CLASSES_ROOT\$_\shell"; $verb = (Get-ItemProperty -LiteralPath $key -ErrorAction SilentlyContinue).'(default)'; if ($verb -and $verb -ne 'none') { $handler = Get-ItemProperty -LiteralPath "$key\$verb\command" -ErrorAction SilentlyContinue; if ($handler.DelegateExecute -ne '{11dbb47c-a525-400b-9e80-a54615a090c0}' -and $handler.'(default)' -notmatch '^\s*"?(?:[^"]*[\\/])?explorer\.exe(?:["\s]|$)') { $verb } } }; if ($folderVerb) { Invoke-Item -LiteralPath ${escapePowerShellStringLiteral(parent)} } else { Start-Process 'explorer.exe' -ArgumentList ('/select,"' + ${literal} + '"') }`; } function powerShellFileManagerLaunch( @@ -593,13 +648,18 @@ const resolveFileManagerRevealLaunch = Effect.fn("resolveFileManagerRevealLaunch shouldUseWindowsHostFromWsl(platform, env) && env.WSL_DISTRO_NAME !== undefined ) { - const explorerTarget = resolveWslFileManagerPath(target, env.WSL_DISTRO_NAME); if (yield* isCommandAvailable(WSL_POWERSHELL_COMMAND, { env })) { // WSL names may contain quotes that Explorer's /select cannot encode. // Open the parent through the shell so replacement managers still apply. const path = yield* Path.Path; // .NET path parsing rejects characters that are legal in WSL filenames. - const parent = resolveWslFileManagerPath(path.dirname(target), env.WSL_DISTRO_NAME); + const [explorerTarget, parent] = yield* Effect.all( + [ + resolveWslFileManagerPath(target, env.WSL_DISTRO_NAME), + resolveWslFileManagerPath(path.dirname(target), env.WSL_DISTRO_NAME), + ], + { concurrency: "unbounded" }, + ); const source = explorerTarget.includes('"') ? `Invoke-Item -LiteralPath ${escapePowerShellStringLiteral(parent)}` : buildFileManagerRevealPowerShellSource(explorerTarget, parent); @@ -614,7 +674,7 @@ const resolveFileManagerRevealLaunch = Effect.fn("resolveFileManagerRevealLaunch editor: "file-manager", target, command: "explorer.exe", - args: [resolveWslFileManagerPath(path.dirname(target), env.WSL_DISTRO_NAME)], + args: [yield* resolveWslFileManagerPath(path.dirname(target), env.WSL_DISTRO_NAME)], }; } @@ -665,18 +725,72 @@ const tryRevealLinuxItem = Effect.fn("externalLauncher.tryRevealLinuxItem")( }), ); +// Stock Windows answers "none"; any other verb may be a replacement file manager, which only +// the PowerShell reveal honors. Reading stdout is about 600 ms faster than spawner.string here. +const hasStockWindowsFolderVerb = Effect.fn("externalLauncher.hasStockWindowsFolderVerb")( + function* () { + const env = yield* readBrowserLaunchEnv; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const handle = yield* spawner.spawn( + ChildProcess.make( + `${resolveWindowsRoot(env)}\\System32\\reg.exe`, + ["query", "HKCR\\Directory\\shell", "/ve"], + { stdin: "ignore", stderr: "ignore" }, + ), + ); + const [stdout, exitCode] = yield* Effect.all( + [handle.stdout.pipe(Stream.decodeText(), Stream.mkString), handle.exitCode], + { concurrency: "unbounded" }, + ); + return exitCode === 0 && /REG_SZ\s+none\s*$/im.test(stdout); + }, + Effect.scoped, + Effect.timeout("2 seconds"), + Effect.orElseSucceed(() => false), +); + +// Explorer parses /select itself, so it needs the argument verbatim rather than Node's quoting. +const revealWindowsItem = Effect.fn("externalLauncher.revealWindowsItem")(function* ( + target: string, +) { + const command = `${resolveWindowsRoot(yield* readBrowserLaunchEnv)}\\explorer.exe`; + const args = [`/select,"${normalizeWindowsFileManagerPath(target)}"`]; + yield* (yield* WindowsVerbatimSpawn)(command, args).pipe( + Effect.mapError( + (cause) => + new ExternalLauncherEditorSpawnError({ + editor: "file-manager", + target, + command, + args, + cause, + }), + ), + ); +}); + const launchEditor = Effect.fn("externalLauncher.launchEditor")(function* ( input: LaunchEditorInput, + hasStockFolderVerb: Effect.Effect, ) { - const launch = yield* resolveEditorLaunch(input); - if ( - input.editor === "file-manager" && - input.reveal === true && - launch.command === "xdg-open" && - (yield* HostProcess.Platform) === "linux" && - (yield* tryRevealLinuxItem(launch.target)) - ) { - return; + // Clients send `~/` host paths as written, and launches run without a shell to expand them. + const home = yield* HostProcess.HomeDirectory; + const cwd = expandHomePath(input.cwd, home); + const projectRoot = + input.projectRoot === undefined ? undefined : expandHomePath(input.projectRoot, home); + const launch = yield* resolveEditorLaunch({ ...input, cwd, projectRoot }); + if (input.editor === "file-manager" && input.reveal === true) { + const platform = yield* HostProcess.Platform; + if ( + platform === "linux" && + launch.command === "xdg-open" && + (yield* tryRevealLinuxItem(launch.target)) + ) { + return; + } + if (platform === "win32" && (yield* hasStockFolderVerb)) { + return yield* revealWindowsItem(launch.target); + } } yield* launchEditorProcess(launch); }); @@ -837,6 +951,12 @@ export const make = Effect.gen(function* () { ), EDITOR_DISCOVERY_CACHE_TTL, ); + const cachedStockWindowsFolderVerb = yield* makeStaleWhileRevalidate( + hasStockWindowsFolderVerb().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ), + EDITOR_DISCOVERY_CACHE_TTL, + ); // Forwarders retain the underlying launcher and discovery spans. return ExternalLauncher.of({ @@ -854,7 +974,9 @@ export const make = Effect.gen(function* () { ), launchEditor: Effect.fnUntraced( function* (input) { - return yield* provideCommandResolutionServices(launchEditor(input)); + return yield* provideCommandResolutionServices( + launchEditor(input, cachedStockWindowsFolderVerb), + ); }, Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), ), diff --git a/apps/server/src/process/windowsVerbatimSpawn.ts b/apps/server/src/process/windowsVerbatimSpawn.ts new file mode 100644 index 0000000000000..15075c22455b9 --- /dev/null +++ b/apps/server/src/process/windowsVerbatimSpawn.ts @@ -0,0 +1,29 @@ +// @effect-diagnostics nodeBuiltinImport:off - Effect's spawner has no windowsVerbatimArguments option. +import * as NodeChildProcess from "node:child_process"; + +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; + +export type WindowsVerbatimSpawner = ( + command: string, + args: ReadonlyArray, +) => Effect.Effect; + +const spawnDetachedVerbatim: WindowsVerbatimSpawner = (command, args) => + Effect.callback((resume) => { + const child = NodeChildProcess.spawn(command, args, { + detached: true, + stdio: "ignore", + windowsVerbatimArguments: true, + }); + child.once("error", (error) => resume(Effect.fail(error))); + child.once("spawn", () => { + child.unref(); + resume(Effect.void); + }); + }); + +export const WindowsVerbatimSpawn = Context.Reference( + "t3/process/WindowsVerbatimSpawn", + { defaultValue: () => spawnDetachedVerbatim }, +); diff --git a/apps/server/src/processRunner.test.ts b/apps/server/src/processRunner.test.ts index 3754f25a4c775..6768fa23c763b 100644 --- a/apps/server/src/processRunner.test.ts +++ b/apps/server/src/processRunner.test.ts @@ -2,6 +2,8 @@ import { isWindowsCommandNotFound } from "@t3tools/provider-core/server/snapshot import { describe, expect, it } from "@effect/vitest"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Cause from "effect/Cause"; import * as Layer from "effect/Layer"; import * as PlatformError from "effect/PlatformError"; import * as Sink from "effect/Sink"; @@ -143,6 +145,29 @@ describe("runProcess", () => { }), ); + it.effect("reports child startup before delivering cancellation during acquisition", () => + Effect.gen(function* () { + let started = false; + const spawner = makeSpawner(() => + Effect.withFiber((fiber) => + Effect.sync(() => { + fiber.interruptUnsafe(); + return makeHandle({}); + }), + ), + ); + const result = yield* runWith(spawner)({ + command: "git", + args: ["restore"], + onSpawn: () => { + started = true; + }, + }).pipe(Effect.exit); + expect(Exit.isFailure(result) && Cause.hasInterruptsOnly(result.cause)).toBe(true); + expect(started).toBe(true); + }).pipe(Effect.scoped), + ); + it.effect("fails when output exceeds max buffer in default mode", () => Effect.gen(function* () { const spawner = makeSpawner(() => Effect.succeed(makeHandle({ stdout: "x".repeat(2048) }))); diff --git a/apps/server/src/processRunner.ts b/apps/server/src/processRunner.ts index a345c67eca881..4c7efd3cc9942 100644 --- a/apps/server/src/processRunner.ts +++ b/apps/server/src/processRunner.ts @@ -26,6 +26,8 @@ export interface ProcessRunInput { readonly stdin?: string | undefined; /** Receives every stdout chunk, including bytes beyond the buffered output limit. */ readonly onStdoutChunk?: ((chunk: Uint8Array) => void) | undefined; + /** Runs after successful spawn, before cancellation can interrupt acquisition. */ + readonly onSpawn?: (() => void) | undefined; readonly maxOutputBytes?: number | undefined; readonly outputMode?: "error" | "truncate" | undefined; readonly truncatedMarker?: string | undefined; @@ -290,7 +292,7 @@ const runProcessCore = Effect.fn("processRunner.runProcessCore")(function* ( cause, }); - const child = yield* spawner + const spawn = spawner .spawn( ChildProcess.make(spawnCommand.command, spawnCommand.args, { ...((input.spawnCwd ?? input.cwd) ? { cwd: input.spawnCwd ?? input.cwd } : {}), @@ -309,6 +311,12 @@ const runProcessCore = Effect.fn("processRunner.runProcessCore")(function* ( // Synchronous spawn throws arrive as defects; recover them too so callers see typed spawn failures. Effect.catchDefect((cause) => Effect.fail(toSpawnError(cause))), ); + const child = yield* input.onSpawn === undefined + ? spawn + : spawn.pipe( + Effect.tap(() => Effect.sync(() => input.onSpawn?.())), + Effect.uninterruptible, + ); const stdin = input.stdin; const onStdoutChunk = input.onStdoutChunk; diff --git a/apps/server/src/project/AgentSessionImporter.test.ts b/apps/server/src/project/AgentSessionImporter.test.ts index 4378a7fde8470..18c0946144d4a 100644 --- a/apps/server/src/project/AgentSessionImporter.test.ts +++ b/apps/server/src/project/AgentSessionImporter.test.ts @@ -1,3 +1,4 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; import { expect, it } from "@effect/vitest"; import { type AgentSessionSource, @@ -15,24 +16,27 @@ import { type OrchestrationV2ProviderThread, } from "@t3tools/contracts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; -import * as KeyedLock from "@t3tools/shared/KeyedLock"; import * as AcpRegistryRuntimeCoordinator from "@t3tools/provider-acp-registry/server/AcpRegistryRuntimeCoordinator"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as Path from "effect/Path"; import * as Stream from "effect/Stream"; import * as SqlClient from "effect/sql/SqlClient"; import * as TestClock from "effect/testing/TestClock"; +import * as ServerConfig from "../config.ts"; import * as EventSink from "../orchestration-v2/EventSink.ts"; import * as EventStore from "../orchestration-v2/EventStore.ts"; import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; +import * as ProjectStore from "../orchestration-v2/ProjectStore.ts"; import * as ProviderAdapterRegistry from "../orchestration-v2/ProviderAdapterRegistry.ts"; import * as ThreadCommandExecutor from "../orchestration-v2/ThreadCommandExecutor.ts"; import { @@ -180,14 +184,18 @@ const seedOwner = Effect.fn(function* ( }); const makeImporter = Effect.fn(function* ( - candidates: ReadonlyArray>, + candidates: ReadonlyArray, beforeTransaction: Effect.Effect = Effect.void, getAdapter?: ProviderAdapterRegistry.ProviderAdapterRegistryV2Shape["get"], - threadCommands?: KeyedLock.KeyedLock, + threadCommands?: ThreadCommandExecutor.ThreadCommandExecutor["Service"], settings: Layer.Layer< ServerSettings.ServerSettingsService, ServerSettingsError > = ServerSettings.layerTest(), + realScan?: { + readonly workspaceRoot: string; + readonly scanner: AgentSessionScanner.AgentSessionScanner["Service"]; + }, ) { const sql = yield* SqlClient.SqlClient; const events = yield* EventStore.EventStoreV2; @@ -229,15 +237,17 @@ const makeImporter = Effect.fn(function* ( ProviderSessionRuntime.ProviderSessionRuntimeRepository, yield* ProviderSessionRuntime.ProviderSessionRuntimeRepository, ), - Layer.mock(AgentSessionScanner.AgentSessionScanner)({ - recentThreads: () => Stream.fromIterable(candidates), - }), + realScan === undefined + ? Layer.mock(AgentSessionScanner.AgentSessionScanner)({ + recentThreads: () => Stream.fromIterable(candidates), + }) + : Layer.succeed(AgentSessionScanner.AgentSessionScanner, realScan.scanner), Layer.mock(ProjectService.ProjectService)({ getById: () => Effect.succeedSome({ id: projectId, title: "Project", - workspaceRoot: "/workspace/project", + workspaceRoot: realScan?.workspaceRoot ?? "/workspace/project", defaultModelSelection: null, scripts: [], createdAt: timestamp, @@ -260,7 +270,7 @@ const makeImporter = Effect.fn(function* ( ), Layer.succeed( ThreadCommandExecutor.ThreadCommandExecutor, - threadCommands ?? (yield* KeyedLock.make()), + threadCommands ?? (yield* ThreadCommandExecutor.make), ), ), ), @@ -377,6 +387,43 @@ it.effect( }).pipe(Effect.scoped, Effect.provide(persistence)), ); +it.effect("tolerates a missing transcript owner but propagates failed ownership reads", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const runtimes = yield* ProviderSessionRuntime.ProviderSessionRuntimeRepository; + const source = candidate(CODEX_MODEL_SELECTION, "ownership-read"); + const threadId = importId(source); + yield* runtimes.upsert({ + threadId: ThreadId.make("missing-owner"), + providerName: "codex", + providerInstanceId: source.source.providerInstanceId, + adapterKey: "codex", + runtimeMode: "full-access", + status: "stopped", + lastSeenAt: timestamp, + resumeCursor: null, + runtimePayload: { importedTranscripts: [source.source] }, + }); + yield* runtimes.recordImportedTranscript({ + threadId: ThreadId.make("missing-owner"), + source: source.source, + }); + const importer = yield* makeImporter([source]); + expect(yield* importer.run).toEqual({ importedCount: 1, skippedCount: 0 }); + + yield* sql`UPDATE orchestration_v2_projection_threads SET payload_json = '{}' WHERE thread_id = ${threadId}`; + expect(yield* Effect.flip(importer.run)).toMatchObject({ + _tag: "AgentSessionScanError", + operation: "read-projects", + cause: { + _tag: "OrchestratorProjectionError", + threadId, + cause: { _tag: "ProjectionStoreReadError" }, + }, + }); + }).pipe(Effect.scoped, Effect.provide(persistence)), +); + it.effect("rechecks ownership after the import reaches the commit transaction", () => Effect.gen(function* () { const enteringTransaction = yield* Deferred.make(); @@ -436,8 +483,8 @@ it.effect( expect(yield* runtimes.list()).toEqual([]); yield* sql`DROP TRIGGER fail_runtime`; - expect(yield* importer.run).toEqual({ importedCount: 1, skippedCount: 0 }); - expect(yield* importer.run).toEqual({ importedCount: 1, skippedCount: 0 }); + expect(yield* importer.run).toEqual({ importedCount: 0, skippedCount: 0 }); + expect(yield* importer.run).toEqual({ importedCount: 0, skippedCount: 0 }); expect(yield* events.read({ threadId }).pipe(Stream.runCollect)).toEqual(committed); expect(importer.published).toHaveLength(6); expect( @@ -453,6 +500,82 @@ it.effect( }).pipe(Effect.scoped, Effect.provide(persistence)), ); +it.effect("counts only new threads when re-importing cached and recovered transcripts", () => + Effect.gen(function* () { + const events = yield* EventStore.EventStoreV2; + const runtimes = yield* ProviderSessionRuntime.ProviderSessionRuntimeRepository; + const cached = candidate( + CLAUDE_MODEL_SELECTION, + "020e7f00-dce6-406b-8ec6-ed90498945d6", + "claudeAgent", + ); + const recovered = candidate(CODEX_MODEL_SELECTION, "recovered"); + const initial = yield* makeImporter([cached, recovered]); + expect(yield* initial.run).toEqual({ importedCount: 2, skippedCount: 0 }); + const before = yield* events.read({ threadId: importId(recovered) }).pipe(Stream.runCollect); + // The file changed on disk but still holds the imported messages. + recovered.source.size += 100; + recovered.source.mtimeMs += 1; + const duplicate = { ...recovered.source, filePath: "/tmp/duplicate.jsonl" }; + const fresh = candidate(CODEX_MODEL_SELECTION, "fresh"); + const repeated = yield* makeImporter([ + { _tag: "AlreadyImported", source: cached.source }, + recovered, + { _tag: "Duplicate", source: duplicate }, + fresh, + { _tag: "Skipped" }, + ]); + + expect(yield* repeated.run).toEqual({ importedCount: 1, skippedCount: 1 }); + expect(yield* events.read({ threadId: importId(recovered) }).pipe(Stream.runCollect)).toEqual( + before, + ); + expect( + new Set( + repeated.published.flatMap((stored) => ("event" in stored ? [stored.event.threadId] : [])), + ), + ).toEqual(new Set([importId(fresh)])); + expect( + Option.getOrThrow(yield* runtimes.getByThreadId({ threadId: importId(recovered) })), + ).toMatchObject({ + runtimePayload: { importedTranscripts: [recovered.source, duplicate] }, + }); + }).pipe(Effect.scoped, Effect.provide(persistence)), +); + +it.effect("does not certify new native history after a partial import retry", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const runtimes = yield* ProviderSessionRuntime.ProviderSessionRuntimeRepository; + const original = candidate(CODEX_MODEL_SELECTION, "partial-retry"); + const threadId = importId(original); + const importer = yield* makeImporter([original]); + yield* sql`CREATE TEMP TRIGGER fail_runtime BEFORE INSERT ON provider_session_runtime BEGIN SELECT RAISE(ABORT, 'test runtime failure'); END`; + expect(yield* importer.run).toEqual({ importedCount: 0, skippedCount: 1 }); + yield* sql`DROP TRIGGER fail_runtime`; + expect(yield* runtimes.list()).toEqual([]); + const continued = candidate(CODEX_MODEL_SELECTION, original.source.providerSessionId); + continued.source.size = 200; + continued.source.mtimeMs = 3; + continued.thread.messages.push( + { role: "user", text: "Continue outside T3", createdAt: timestamp }, + { role: "assistant", text: "Native continuation", createdAt: timestamp }, + ); + // The thread holds only the original messages, so the continued transcript stays unrecorded. + expect(yield* (yield* makeImporter([continued])).run).toEqual({ + importedCount: 0, + skippedCount: 1, + }); + expect(yield* runtimes.list()).toEqual([]); + expect( + (yield* projections.getThreadRecords(threadId, ["messages"])).messages.map( + (message) => message.text, + ), + ).toEqual(["Fix it", "Fixed"]); + }).pipe(Effect.scoped, Effect.provide(persistence)), +); + it.effect("does not record a changed transcript as imported without its new messages", () => Effect.gen(function* () { const projections = yield* ProjectionStore.ProjectionStoreV2; @@ -692,7 +815,7 @@ const importDuringCommand = Effect.fn(function* ( const projections = yield* ProjectionStore.ProjectionStoreV2; const sink = yield* EventSink.EventSinkV2; const threadId = yield* seedOwner(candidate(CODEX_MODEL_SELECTION, name)); - const lock = yield* KeyedLock.make(); + const lock = yield* ThreadCommandExecutor.make; const loaded = yield* Deferred.make(); const importerCommitting = yield* Deferred.make(); const commandWritten = yield* Deferred.make(); @@ -836,3 +959,95 @@ it.effect("imports under a policy ceiling in the project's default, kept under t ); }).pipe(Effect.scoped, Effect.provide(persistence)), ); + +it.effect("repeated imports reach linked-worktree sessions behind the scan budget", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-import-worktree-budget-" }); + const repo = path.join(root, "repo"); + const worktree = path.join(root, "feature"); + const worktreeGitDir = path.join(repo, ".git", "worktrees", "feature"); + yield* fs.makeDirectory(worktreeGitDir, { recursive: true }); + yield* fs.makeDirectory(worktree); + yield* fs.writeFileString(path.join(repo, ".git", "config"), "[core]\n"); + yield* fs.writeFileString(path.join(worktreeGitDir, "commondir"), "../..\n"); + yield* fs.writeFileString(path.join(worktreeGitDir, "HEAD"), "ref: refs/heads/feature\n"); + yield* fs.writeFileString(path.join(worktree, ".git"), `gitdir: ${worktreeGitDir}\n`); + + const claudeHome = path.join(root, "claude"); + const codexHome = path.join(root, "codex"); + const sessions = path.join(codexHome, "sessions", "2026", "09", "01"); + yield* fs.makeDirectory(sessions, { recursive: true }); + const nowMs = DateTime.toEpochMillis(now); + yield* TestClock.setTime(nowMs); + // One more session than the 100-transcript scan budget, all run in the linked worktree. + for (let index = 0; index < 101; index++) { + const file = path.join(sessions, `rollout-${String(index).padStart(3, "0")}.jsonl`); + const records = [ + { type: "session_meta", payload: { id: `session-${index}`, cwd: worktree } }, + { type: "event_msg", payload: { type: "user_message", message: `Request ${index}` } }, + { + type: "response_item", + payload: { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "Done" }], + }, + }, + ]; + yield* fs.writeFileString( + file, + records.map((record) => JSON.stringify(record) + "\n").join(""), + ); + const modifiedAtSeconds = nowMs / 1000 - (index + 1); + yield* fs.utimes(file, modifiedAtSeconds, modifiedAtSeconds); + } + + const scannerContext = yield* Layer.build( + AgentSessionScanner.layer.pipe( + Layer.provide( + Layer.mergeAll( + ServerConfig.layerTest(claudeHome, { prefix: "t3-import-budget-config-" }), + ServerSettings.layerTest({ + providerInstances: { + [ProviderInstanceId.make("claudeAgent")]: { + driver: ProviderDriverKind.make("claudeAgent"), + config: { homePath: claudeHome }, + }, + [ProviderInstanceId.make("codex")]: { + driver: ProviderDriverKind.make("codex"), + config: { homePath: codexHome }, + }, + // Keeps the default Pi instance off the developer's real ~/.pi. + [ProviderInstanceId.make("pi")]: { + driver: ProviderDriverKind.make("pi"), + enabled: false, + config: {}, + }, + }, + }), + Layer.mock(ProjectStore.ProjectStoreV2)({ listShells: () => Effect.succeed([]) }), + Layer.succeed( + ProviderSessionRuntime.ProviderSessionRuntimeRepository, + yield* ProviderSessionRuntime.ProviderSessionRuntimeRepository, + ), + ), + ), + ), + ); + const importer = yield* makeImporter([], Effect.void, undefined, undefined, undefined, { + workspaceRoot: repo, + scanner: Context.get(scannerContext, AgentSessionScanner.AgentSessionScanner), + }); + + expect(yield* importer.run).toEqual({ importedCount: 100, skippedCount: 1 }); + expect(yield* importer.run).toEqual({ importedCount: 1, skippedCount: 0 }); + const oldest = yield* ProjectionStore.ProjectionStoreV2.pipe( + Effect.flatMap((projections) => + projections.getThread(ThreadId.make("import:codex:session-100")), + ), + ); + expect(oldest.worktreePath).toBe(worktree); + }).pipe(Effect.scoped, Effect.provide(Layer.merge(persistence, NodeServices.layer))), +); diff --git a/apps/server/src/project/AgentSessionImporter.ts b/apps/server/src/project/AgentSessionImporter.ts index 6200fc11188fb..0f8736986a2df 100644 --- a/apps/server/src/project/AgentSessionImporter.ts +++ b/apps/server/src/project/AgentSessionImporter.ts @@ -35,6 +35,7 @@ import * as Stream from "effect/Stream"; import * as EventSink from "../orchestration-v2/EventSink.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; +import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; import * as ProviderAdapterRegistry from "../orchestration-v2/ProviderAdapterRegistry.ts"; import * as ThreadCommandExecutor from "../orchestration-v2/ThreadCommandExecutor.ts"; import * as ProviderSessionRuntime from "../persistence/ProviderSessionRuntime.ts"; @@ -42,6 +43,10 @@ import * as ServerSettings from "../serverSettings.ts"; import * as AgentSessionScanner from "./AgentSessionScanner.ts"; import * as ProjectService from "./ProjectService.ts"; +const isProjectionStoreThreadNotFoundError = Schema.is( + ProjectionStore.ProjectionStoreThreadNotFoundError, +); + const IMPORT_EVENT_PREFIX = "agent-session-import:v2"; const CLAUDE_SESSION_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; @@ -242,18 +247,21 @@ const make = Effect.gen(function* () { (cause) => new AgentSessionScanError({ operation: "read-projects", cause }), ), ); - const completedSources = runtimeRows.flatMap((runtime) => { - const payload = AgentSessionScanner.decodeImportedTranscriptPayload(runtime.runtimePayload); - if ( - Option.isNone(payload) || - payload.value.cwd === undefined || - normalizeProjectPathForComparison(payload.value.cwd) !== - normalizeProjectPathForComparison(project.workspaceRoot) - ) { - return []; - } - return payload.value.importedTranscripts ?? []; - }); + // The thread's project owns its transcripts: a linked worktree import records the worktree as cwd. + const completedSources = (yield* Effect.forEach(runtimeRows, (runtime) => { + const imported = AgentSessionScanner.decodeImportedTranscriptPayload( + runtime.runtimePayload, + ).pipe(Option.flatMapNullishOr((payload) => payload.importedTranscripts)); + if (Option.isNone(imported) || imported.value.length === 0) return Effect.succeed([]); + return orchestrator.getThreadRecords(runtime.threadId, []).pipe( + Effect.map(({ thread }) => (thread.projectId === input.projectId ? imported.value : [])), + Effect.catch((cause) => + isProjectionStoreThreadNotFoundError(cause.cause) + ? Effect.succeed([]) + : Effect.fail(new AgentSessionScanError({ operation: "read-projects", cause })), + ), + ); + })).flat(); const outcomes = scanner.recentThreads(project.workspaceRoot, completedSources); const importedThreadIds = new Set(); let importedCount = 0; @@ -271,7 +279,6 @@ const make = Effect.gen(function* () { ); if (outcome._tag === "AlreadyImported") { importedThreadIds.add(threadId); - importedCount += 1; return; } if (outcome._tag === "Duplicate") { @@ -313,7 +320,9 @@ const make = Effect.gen(function* () { }, { onConflict: "ignore" }, ); - const existing = yield* Effect.option(orchestrator.getThreadRecords(threadId, [])); + const existing = yield* Effect.option( + orchestrator.getThreadRecords(threadId, ["messages"]), + ); if (Option.isSome(existing)) { if (existing.value.thread.projectId !== input.projectId) { return yield* new AgentSessionThreadProjectConflictError({ @@ -325,23 +334,23 @@ const make = Effect.gen(function* () { if (existing.value.thread.historyOrigin !== "v1_import") { return yield* new AgentSessionThreadModifiedError({ threadId }); } - // Other recorded transcripts mean this one changed after the import, and the - // thread does not hold its new messages, so it must stay unrecorded. - const recorded = Option.flatMap(yield* runtimes.getByThreadId({ threadId }), (row) => - AgentSessionScanner.decodeImportedTranscriptPayload(row.runtimePayload), - ).pipe(Option.flatMapNullishOr((payload) => payload.importedTranscripts)); + // A transcript can change after the import, so record it only when the + // thread already holds exactly its messages. + const stored = existing.value.messages; if ( - Option.isSome(recorded) && - recorded.value.length > 0 && - !recorded.value.some((imported) => - AgentSessionScanner.sameTranscriptIdentity(imported, source), + stored.length !== thread.messages.length || + stored.some( + (message, index) => + message.role !== thread.messages[index]?.role || + message.text !== thread.messages[index]?.text, ) ) { return false; } yield* persistRuntime; yield* runtimes.recordImportedTranscript({ threadId, source }); - return true; + importedThreadIds.add(threadId); + return null; } const model = thread.model ?? DEFAULT_MODEL_BY_PROVIDER[driver] ?? DEFAULT_MODEL; diff --git a/apps/server/src/project/AgentSessionScanner.test.ts b/apps/server/src/project/AgentSessionScanner.test.ts index 6dac5a1b0b6c9..ce75e456158a8 100644 --- a/apps/server/src/project/AgentSessionScanner.test.ts +++ b/apps/server/src/project/AgentSessionScanner.test.ts @@ -2024,6 +2024,144 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => { }), ); + it.effect("ignores sessions without visible messages without spending the import budget", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const nowMs = Date.parse("2026-08-24T12:00:00.000Z"); + yield* TestClock.setTime(nowMs); + const claudeHomePath = yield* makeTempDir("t3code-claude-home-"); + const codexHomePath = yield* makeTempDir("t3code-codex-home-"); + const workspace = yield* makeTempDir("t3code-workspace-"); + const codexRollout = (name: string) => + path.join(codexHomePath, "sessions", "2026", "08", "24", `rollout-${name}.jsonl`); + // Newer empty sessions would exhaust the 100-transcript import budget before any real one. + const emptySessions = Array.from({ length: 100 }, (_, index) => + index % 2 === 0 + ? { + filePath: codexRollout(`empty-${index}`), + records: [ + { + type: "session_meta", + payload: { id: `codex-empty-${index}`, cwd: workspace, source: "vscode" }, + }, + { + type: "event_msg", + payload: { + type: "thread_settings_applied", + thread_settings: { cwd: workspace }, + }, + }, + ], + } + : { + filePath: path.join( + claudeHomePath, + "projects", + "-slug", + `claude-empty-${index}.jsonl`, + ), + records: [ + { type: "system", subtype: "informational", content: "Status", cwd: workspace }, + { + type: "user", + isMeta: true, + message: { + role: "user", + content: "Caveat", + }, + cwd: workspace, + }, + ], + }, + ); + const transcripts = [ + ...emptySessions, + { + // Without session_meta there is no resumable session ID. + filePath: codexRollout("unusable"), + records: [ + { type: "turn_context", payload: { cwd: workspace } }, + { + type: "event_msg", + payload: { type: "user_message", message: "Import this prompt" }, + }, + ], + }, + { + filePath: codexRollout("real"), + records: [ + { type: "session_meta", payload: { id: "real-session", cwd: workspace } }, + { + type: "event_msg", + payload: { type: "user_message", message: "Import this prompt" }, + }, + ], + }, + ]; + for (const [index, { filePath, records }] of transcripts.entries()) { + yield* writeTranscript({ + filePath, + contents: records.map((record) => encodeTranscriptRecord(record)).join("\n"), + mtimeMs: nowMs - index * 60_000, + }); + } + + const outcomes = yield* runRecentThreadOutcomes({ + claudeHomePath, + codexHomePath, + workspaceRoot: workspace, + }); + + expect( + outcomes.map((outcome) => + outcome._tag === "Importable" ? outcome.thread.providerSessionId : outcome._tag, + ), + ).toEqual(["Skipped", "real-session"]); + }), + ); + + it.effect("skips a rewound Codex rollout whose inherited history is unavailable", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const nowMs = Date.parse("2026-08-24T12:00:00.000Z"); + yield* TestClock.setTime(nowMs); + const claudeHomePath = yield* makeTempDir("t3code-claude-home-"); + const codexHomePath = yield* makeTempDir("t3code-codex-home-"); + const workspace = yield* makeTempDir("t3code-workspace-"); + const threadId = "019a0000-0000-7000-8000-000000000001"; + const rolloutId = "019a0000-0000-7000-8000-000000000002"; + // The ancestor rollout is missing, and this rollout added no messages of its own. + yield* writeTranscript({ + filePath: path.join( + codexHomePath, + "sessions", + "2026", + "08", + "24", + `rollout-2026-08-24T11-00-00-${threadId}_${rolloutId}.jsonl`, + ), + contents: encodeTranscriptRecord({ + type: "session_meta", + payload: { + id: threadId, + cwd: workspace, + history_mode: "paginated", + history_base: { thread_id: threadId, end_ordinal_exclusive: 3, end_byte_offset: 512 }, + }, + }), + mtimeMs: nowMs, + }); + + const outcomes = yield* runRecentThreadOutcomes({ + claudeHomePath, + codexHomePath, + workspaceRoot: workspace, + }); + + expect(outcomes).toEqual([{ _tag: "Skipped" }]); + }), + ); + it.effect("skips excessive records without blocking an older valid transcript", () => Effect.gen(function* () { const path = yield* Path.Path; @@ -3475,6 +3613,7 @@ describe("parseAgentSessionTranscript", () => { "\nGitHub\n", "# AGENTS.md instructions for /tmp/project\n\n\nRules\n", '\nTab\n', + '{"page_id":null}', "Fix the import titles", ].join("\n"); const thread = AgentSessionScanner.parseAgentSessionTranscript({ @@ -3495,6 +3634,27 @@ describe("parseAgentSessionTranscript", () => { expect(thread?.messages.map((message) => message.text)).toEqual([prompt]); }); + it.each([ + "Explain in this transcript", + "Keep this incomplete example", + ])("keeps Codex page markup that is not a complete leading block: %s", (prompt) => { + const thread = AgentSessionScanner.parseAgentSessionTranscript({ + contents: [ + encodeTranscriptRecord({ type: "session_meta", payload: { id: "codex-session" } }), + encodeTranscriptRecord({ + type: "event_msg", + payload: { type: "user_message", message: prompt }, + }), + ].join("\n"), + source: "codex", + providerInstanceId: ProviderInstanceId.make("codex"), + fallbackSessionId: "fallback", + lastActiveAtMs: Date.parse("2026-08-25T08:00:00.000Z"), + }); + + expect(thread?.title).toBe(prompt); + }); + it("skips sessions without a visible user message", () => { const thread = AgentSessionScanner.parseAgentSessionTranscript({ contents: JSON.stringify({ diff --git a/apps/server/src/project/AgentSessionScanner.ts b/apps/server/src/project/AgentSessionScanner.ts index 0de2e84b1f6e3..7a89f9c687de0 100644 --- a/apps/server/src/project/AgentSessionScanner.ts +++ b/apps/server/src/project/AgentSessionScanner.ts @@ -369,7 +369,7 @@ function foldClaudeFields(fields: ClaudeFields, record: DecodedTranscriptRecord) // Codex records setup context as user text ahead of the real request. Skip // these leading blocks when deriving a title; imported messages keep them. const CODEX_TITLE_PREAMBLE = - /^(?:\s*(?:<(recommended_plugins|environment_context|user_instructions|INSTRUCTIONS|in-app-browser-context)(?:\s[^>]*)?>[\s\S]*?<\/\1>|# AGENTS\.md instructions\b[^\n]*|## My request for Codex:))+/; + /^(?:\s*(?:<(recommended_plugins|environment_context|user_instructions|INSTRUCTIONS|in-app-browser-context|external_codex_apps_open_page)(?:\s[^>]*)?>[\s\S]*?<\/\1>|# AGENTS\.md instructions\b[^\n]*|## My request for Codex:))+/; // Codex appends an entry per rename; like Codex, the newest nonempty name wins. function parseCodexThreadNames(contents: string): ReadonlyMap { @@ -447,7 +447,8 @@ export function parseAgentSessionTranscript( input.source === "claudeAgent" ? records.reduce(foldClaudeFields, emptyClaudeFields) : emptyClaudeFields; - return parseAgentSessionRecords(input, records, claudeFields, 0); + const thread = parseAgentSessionRecords(input, records, claudeFields, 0); + return thread === "empty" ? null : thread; } function parseAgentSessionRecords( @@ -455,7 +456,7 @@ function parseAgentSessionRecords( records: ReadonlyArray, claudeFields: ClaudeFields, droppedMessageCount: number, -): AgentSessionThread | null { +): AgentSessionThread | "empty" | null { const fallbackTimestamp = DateTime.formatIso(DateTime.makeUnsafe(input.lastActiveAtMs)); // Claude filenames are session IDs. Codex rollout filenames include extra // timestamp text, so only transcript metadata can provide a resumable ID. @@ -648,6 +649,7 @@ function parseAgentSessionRecords( }); } + if (messages.length === 0 && droppedMessageCount === 0) return "empty"; const visibleMessages = messages.map( ({ codexResponseUser: _codexResponseUser, ...message }) => message, ); @@ -802,10 +804,7 @@ type TranscriptIdentity = Pick< const fileIdKey = (inode: TranscriptIdentity["inode"]) => (inode === null ? null : String(inode)); -export function sameTranscriptIdentity( - left: TranscriptIdentity, - right: TranscriptIdentity, -): boolean { +function sameTranscriptIdentity(left: TranscriptIdentity, right: TranscriptIdentity): boolean { // Unsafe legacy numbers may already have lost file ID precision. if ( (typeof left.inode === "number" && !Number.isSafeInteger(left.inode)) || @@ -2142,6 +2141,15 @@ export const make = Effect.gen(function* () { snapshot.claudeFields, snapshot.droppedMessageCount, ); + if (parsedThread === "empty") { + // null means an unreadable history base, which may hold the conversation. + if (candidate.source === "codex" && inheritedRecords === null) { + return Option.some({ _tag: "Skipped" }); + } + // Never recorded as imported, so charging them would starve real sessions every run. + transcriptsRemaining += 1; + return Option.none(); + } if (parsedThread === null) { return Option.some({ _tag: "Skipped" }); } diff --git a/apps/server/src/project/ManagedProjectFolders.test.ts b/apps/server/src/project/ManagedProjectFolders.test.ts index 011b54a6851b3..07da49c732539 100644 --- a/apps/server/src/project/ManagedProjectFolders.test.ts +++ b/apps/server/src/project/ManagedProjectFolders.test.ts @@ -1,7 +1,13 @@ import { assert, it } from "@effect/vitest"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; -import { CommandId, ProjectId, ThreadId, VcsProcessSpawnError } from "@t3tools/contracts"; +import { + CommandId, + ProjectId, + ThreadId, + VcsProcessSpawnError, + VcsProcessTimeoutError, +} from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; @@ -174,6 +180,44 @@ it.effect("offers a Scratch folder when Git is not installed", () => ), ); +it.effect("offers the Scratch folder once a Git check that timed out is asked again", () => + withScratch( + ({ baseDir }) => + Effect.gen(function* () { + const path = yield* Path.Path; + const scratch = yield* ManagedProjectFolders.ManagedProjectFolders; + assert.isTrue(Option.isNone(yield* scratch.scratchRoot)); + assert.equal(yield* requireRoot, path.resolve(baseDir, "scratch")); + }), + { + vcsProcess: Layer.effect( + VcsProcess.VcsProcess, + Effect.map(VcsProcess.make, (real) => { + let timedOut = false; + return VcsProcess.VcsProcess.of({ + run: (input) => + Effect.suspend(() => { + if (timedOut) return real.run(input); + timedOut = true; + return Effect.fail( + new VcsProcessTimeoutError({ + operation: input.operation, + command: input.command, + cwd: input.cwd, + timeoutMs: 5_000, + }), + ); + }), + }); + }), + ).pipe( + Layer.provide(ProcessRunner.layer), + Layer.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), + ), + }, + ), +); + it.effect("creates one Scratch project, even for concurrent first requests", () => withScratch(() => Effect.gen(function* () { @@ -274,6 +318,15 @@ it.effect("gives each Scratch thread its own folder, named from its message", () ); assert.equal(path.dirname(pasted), root); assert.isAtMost(path.basename(pasted).length, 80); + + const accented = Option.getOrThrow( + yield* scratch.folderForThread({ + projectId, + threadId: ThreadId.make("thread-diacritics"), + text: "tìm kiếm skill", + }), + ); + assert.match(path.basename(accented), /^\d{4}-\d{2}-\d{2}-tim-kiem-skill-/); }), ), ); diff --git a/apps/server/src/project/ManagedProjectFolders.ts b/apps/server/src/project/ManagedProjectFolders.ts index 007f784374c15..d526c381fb404 100644 --- a/apps/server/src/project/ManagedProjectFolders.ts +++ b/apps/server/src/project/ManagedProjectFolders.ts @@ -6,7 +6,7 @@ import { type ThreadId, VcsProcessSpawnError, } from "@t3tools/contracts"; -import { newProjectFolderName } from "@t3tools/shared/path"; +import { foldLatinDiacritics, newProjectFolderName } from "@t3tools/shared/path"; import * as Cause from "effect/Cause"; import * as Context from "effect/Context"; import * as SecureRandom from "effect/crypto/SecureRandom"; @@ -133,7 +133,7 @@ export class ManagedProjectFolders extends Context.Service< >()("t3/project/ManagedProjectFolders") {} const folderWords = (text: string) => - text + foldLatinDiacritics(text) .toLowerCase() .split(/[^a-z0-9]+/) .filter(Boolean) @@ -268,15 +268,18 @@ const make = Effect.gen(function* () { return Effect.logWarning(unavailable.message).pipe(Effect.as(Result.fail(unavailable))); }, ), - Effect.catchCause((cause) => - Cause.hasInterrupts(cause) - ? Effect.interrupt - : Effect.succeed(Result.fail(new ScratchUnavailableError({ baseDir: config.baseDir }))), - ), ), - Duration.infinity, + // A failed git check (say, a timeout on a busy machine) is not an answer; ask again next time. + Exit.match({ onSuccess: () => Duration.infinity, onFailure: () => Duration.zero }), + ); + const scratchProbe = probe.pipe( + Effect.catchCause((cause) => + Cause.hasInterrupts(cause) + ? Effect.interrupt + : Effect.succeed(Result.fail(new ScratchUnavailableError({ baseDir: config.baseDir }))), + ), + Effect.onInterrupt(() => invalidate), ); - const scratchProbe = probe.pipe(Effect.onInterrupt(() => invalidate)); // A symlinked scratch folder lets the No project live at either the link or its target. const scratchProjectRoot = (root: string) => fileSystem.realPath(root).pipe( diff --git a/apps/server/src/project/ProjectService.deletion.test.ts b/apps/server/src/project/ProjectService.deletion.test.ts index d41a9b1419e79..3f61e96ad1311 100644 --- a/apps/server/src/project/ProjectService.deletion.test.ts +++ b/apps/server/src/project/ProjectService.deletion.test.ts @@ -28,6 +28,7 @@ import * as ProjectStore from "../orchestration-v2/ProjectStore.ts"; import * as ThreadCommandExecutor from "../orchestration-v2/ThreadCommandExecutor.ts"; import { planThreadDeletion } from "../orchestration-v2/ThreadDeletion.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as ProjectSettingsOverrideSweep from "../provider/ProjectSettingsOverrideSweep.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; import * as ProjectEnrichmentService from "./ProjectEnrichmentService.ts"; @@ -508,9 +509,13 @@ it.effect("deletes a project without force once its imported threads were delete }).pipe(Effect.provide(layerDatabase)), ); -it.effect( - "clears a deleted project's settings overrides so a machine-disabled instance cannot stay alive", - () => +it.effect.each([ + { policy: "no", worktreeCleanup: undefined }, + { policy: "its off", worktreeCleanup: { mode: "off" } as const }, + { policy: "its legacy", worktreeCleanup: null }, +])( + "clears a deleted project's settings overrides, keeping $policy worktree cleanup policy, so a machine-disabled instance cannot stay alive", + ({ worktreeCleanup }) => Effect.gen(function* () { const projectId = ProjectId.make("project:overrides-cleanup"); const instanceId = ProviderInstanceId.make("codex_work"); @@ -521,7 +526,11 @@ it.effect( const serverSettings = yield* ServerSettings.ServerSettingsService; yield* serverSettings.updateSettings({ projectSettingsOverrides: { - [projectId]: { providerInstances: { [instanceId]: { enabled: true } } }, + [projectId]: { + providerInstances: { [instanceId]: { enabled: true } }, + defaultAutoPull: true, + ...(worktreeCleanup === undefined ? {} : { worktreeCleanup }), + }, }, }); @@ -530,8 +539,35 @@ it.effect( projectId, }); - const settingsAfterDelete = yield* serverSettings.getSettings; - assert.isUndefined(settingsAfterDelete.projectSettingsOverrides[projectId]); + const entry = (yield* serverSettings.getSettings).projectSettingsOverrides[projectId]; + if (worktreeCleanup === undefined) { + assert.isUndefined(entry); + } else { + assert.deepStrictEqual(entry, { worktreeCleanup, providerInstances: {} }); + } + + const liveProjectId = ProjectId.make("project:still-live"); + const missingProjectId = ProjectId.make("project:missing"); + yield* seedProject(liveProjectId); + // Older deletions or interrupted best-effort cleanup can leave the original overrides on restart. + yield* serverSettings.updateSettings({ + projectSettingsOverrides: { + [projectId]: { + providerInstances: { [instanceId]: { enabled: true } }, + defaultAutoPull: true, + ...(worktreeCleanup === undefined ? {} : { worktreeCleanup }), + }, + [liveProjectId]: { defaultAutoPull: true }, + [missingProjectId]: { worktreeCleanup: { mode: "off" } }, + }, + }); + yield* ProjectSettingsOverrideSweep.sweep; + assert.deepStrictEqual((yield* serverSettings.getSettings).projectSettingsOverrides, { + [liveProjectId]: { defaultAutoPull: true }, + ...(worktreeCleanup === undefined + ? {} + : { [projectId]: { worktreeCleanup, providerInstances: {} } }), + }); }).pipe(Effect.provide(layerServices)); }).pipe(Effect.provide(layerDatabase)), ); diff --git a/apps/server/src/project/ProjectService.ts b/apps/server/src/project/ProjectService.ts index d6bc3e8eedf4d..1db031456f2d7 100644 --- a/apps/server/src/project/ProjectService.ts +++ b/apps/server/src/project/ProjectService.ts @@ -33,6 +33,7 @@ import * as ProjectStore from "../orchestration-v2/ProjectStore.ts"; import * as ThreadCommandExecutor from "../orchestration-v2/ThreadCommandExecutor.ts"; import { planThreadDeletion } from "../orchestration-v2/ThreadDeletion.ts"; import * as ServerSettings from "../serverSettings.ts"; +import { deletedProjectOverridePatch } from "../provider/ProjectSettingsOverrideSweep.ts"; import { makeIsManagedWorktreePath } from "../worktreesDirectory.ts"; import * as ProcessRunner from "../processRunner.ts"; import * as ProjectEnrichmentService from "./ProjectEnrichmentService.ts"; @@ -537,7 +538,7 @@ export const make = Effect.gen(function* () { projectThreads, (thread) => threadCommands - .withLock(thread.id, deleteChildThread(input, thread.id)) + .withDeletionLock(threadCommands.withLock(thread.id, deleteChildThread(input, thread.id))) .pipe( Effect.mapError( (cause) => @@ -550,7 +551,14 @@ export const make = Effect.gen(function* () { // Best-effort cleanup releases disabled instances kept alive by overrides; crashes have no replay to revisit deletion. const clearProjectSettingsOverrides = (projectId: ProjectId) => - serverSettings.updateSettings({ projectSettingsOverrides: { [projectId]: null } }).pipe( + serverSettings.getSettings.pipe( + Effect.flatMap(({ projectSettingsOverrides }) => + serverSettings.updateSettings({ + projectSettingsOverrides: { + [projectId]: deletedProjectOverridePatch(projectSettingsOverrides[projectId]), + }, + }), + ), Effect.asVoid, Effect.catchCause((cause) => Effect.logWarning( diff --git a/apps/server/src/project/ProjectSetupScriptRunner.test.ts b/apps/server/src/project/ProjectSetupScriptRunner.test.ts index 034b168e52c79..fd40f24d5a908 100644 --- a/apps/server/src/project/ProjectSetupScriptRunner.test.ts +++ b/apps/server/src/project/ProjectSetupScriptRunner.test.ts @@ -1,16 +1,20 @@ import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; -import { describe, expect, it } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, describe, expect, it } from "@effect/vitest"; import { ProjectId, TerminalNotRunningError, type TerminalEvent, type TerminalSessionSnapshot, } from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Queue from "effect/Queue"; +import * as Stream from "effect/Stream"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; import { TestClock } from "effect/testing"; import * as ServerSettings from "../serverSettings.ts"; @@ -164,7 +168,19 @@ describe("ProjectSetupScriptRunner", () => { yield* emit({ ...event, type: "activity", hasRunningSubprocess: false, label: "zsh" }); yield* emit({ ...event, type: "output", data: "\r\n% \x1b[?2004h" }); - expect(yield* Queue.take(writes)).toMatch(/^ \( vp i\r\);/); + expect(yield* Queue.take(writes)).toContain("vp i"); + }), + ); + + it.effect("drops whole escape sequences from output lines", () => + Effect.gen(function* () { + const { outputLines, emit } = yield* setup(); + yield* emit({ + ...event, + type: "output", + data: "\x1b7\x1b[24;0fProgress: [ 40%]\x1b8\r\n\x1b%G\x1b#8\x1b[4:3mdone\x1bM\r\n", + }); + expect(yield* Queue.takeAll(outputLines)).toEqual(["Progress: [ 40%]", "done"]); }), ); @@ -172,7 +188,7 @@ describe("ProjectSetupScriptRunner", () => { Effect.gen(function* () { const { writes } = yield* setup(); yield* TestClock.adjust("3 seconds"); - expect(yield* Queue.take(writes)).toMatch(/^ \( vp i\r\);/); + expect(yield* Queue.take(writes)).toContain("vp i"); }), ); @@ -194,7 +210,7 @@ describe("ProjectSetupScriptRunner", () => { yield* emit({ ...event, type: "restarted", snapshot: openSnapshot("") }); // The restart lands inside a quiet window, so the next full window decides. yield* TestClock.adjust("6 seconds"); - expect(yield* Queue.take(writes)).toMatch(/^ \( vp i\r\);/); + expect(yield* Queue.take(writes)).toContain("vp i"); }), ); @@ -227,3 +243,110 @@ describe("ProjectSetupScriptRunner", () => { }), ); }); + +const runObservedSetup = Effect.fnUntraced(function* (command: string) { + const { writes, outputLines, emit, completion } = yield* setup({ command }); + yield* emit({ ...event, type: "output", data: "\x1b[?2004h" }); + const submitted = yield* Queue.take(writes); + const cwd = yield* HostProcess.WorkingDirectory; + const env = yield* HostProcess.Environment; + + // Enter becomes a newline at the terminal boundary; transport assertions below + // cover line-editor submissions and canonical TTY limits that bash -c bypasses. + const child = yield* Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const handle = yield* spawner.spawn( + ChildProcess.make( + "/bin/bash", + ["--noprofile", "--norc", "-c", submitted.replaceAll("\r", "\n")], + { + cwd, + env: { PATH: env.PATH }, + }, + ), + ); + return yield* Effect.all( + { + stdout: handle.stdout.pipe(Stream.decodeText(), Stream.mkString), + stderr: handle.stderr.pipe(Stream.decodeText(), Stream.mkString), + exitCode: handle.exitCode, + }, + { concurrency: "unbounded" }, + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)); + yield* emit({ ...event, type: "output", data: submitted + "\n" }); + yield* emit({ ...event, type: "output", data: child.stdout + child.stderr }); + yield* emit({ ...event, type: "closed" }); + return { + submitted, + lines: yield* Queue.clear(outputLines), + exitCode: (yield* completion).exitCode, + }; +}); + +describe.skipIf(HostProcess.Platform.defaultValue() === "win32")("POSIX setup completion", () => { + it.live("submits one line while preserving multiline shell syntax and literal text", () => + Effect.gen(function* () { + const result = yield* runObservedSetup( + [ + "printf '%s\\n' \"it's ready\" 'literal \\n $(printf wrong) `printf wrong` café'", + "cat <<'END'", + "$literal \\path 雪", + "END", + "printf '%s\\n' done # trailing comment", + ].join("\r\n"), + ); + assert.equal(result.submitted.at(-1), "\r"); + assert.notMatch(result.submitted.slice(0, -1), /[\r\n]/); + assert.deepEqual(result.lines, [ + "it's ready", + "literal \\n $(printf wrong) `printf wrong` café", + "$literal \\path 雪", + "done", + ]); + assert.equal(result.exitCode, 0); + }), + ); + + it.live("completes long scripts without exceeding canonical terminal line limits", () => + Effect.gen(function* () { + const expected = Array.from( + { length: 120 }, + (_, index) => `line-${index}: café 雪 'quoted' \\path`, + ); + const command = expected + .map((line) => `printf '%s\\n' "${line.replaceAll("\\", "\\\\")}"`) + .join("\n"); + const utf8 = new TextEncoder(); + assert.isAbove(utf8.encode(command).byteLength, 4_096); + + const result = yield* runObservedSetup(command); + assert.deepEqual(result.lines, expected); + assert.equal(result.exitCode, 0); + assert.equal(result.submitted.at(-1), "\r"); + assert.notInclude(result.submitted, "\n"); + // The runner can write before the shell switches the terminal out of canonical mode. + for (const line of result.submitted.slice(0, -1).split("\r")) { + assert.isBelow(utf8.encode(line).byteLength, 1_024); + } + }), + ); + + it.live("preserves a trailing backslash-newline even when losing it would still exit zero", () => + Effect.gen(function* () { + const result = yield* runObservedSetup("printf '%s\\n' direct\\\n"); + assert.deepEqual(result.lines, ["direct"]); + assert.equal(result.exitCode, 0); + }), + ); + + it.live.each([ + { name: "an explicit exit", command: "exit 7", exitCode: 7 }, + { name: "a syntax error", command: "if then", exitCode: 2 }, + ])("reports a nonzero completion for $name", ({ command, exitCode }) => + Effect.gen(function* () { + const result = yield* runObservedSetup(command); + assert.equal(result.exitCode, exitCode); + }), + ); +}); diff --git a/apps/server/src/project/ProjectSetupScriptRunner.ts b/apps/server/src/project/ProjectSetupScriptRunner.ts index a049bc89b01a1..60084f648a182 100644 --- a/apps/server/src/project/ProjectSetupScriptRunner.ts +++ b/apps/server/src/project/ProjectSetupScriptRunner.ts @@ -145,12 +145,10 @@ function completionSentinelPattern(token: string): RegExp { return new RegExp(`${COMPLETION_SENTINEL_PREFIX}_${token}:(-?\\d+)`); } +// ECMA-48: CSI, OSC, then any other escape as intermediates (0x20-0x2F) and one final byte. function stripTerminalControl(text: string): string { return text - .replace( - /\x1b\[[0-9;?]*[ -/]*[@-~]|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)|\x1b[()][A-Za-z0-9]|\x1b[=>]/g, - "", - ) + .replace(/\x1b\[[0-?]*[ -/]*[@-~]|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)|\x1b[ -/]*[0-~]/g, "") .replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, ""); } @@ -177,7 +175,57 @@ function resolveCompletionShell(shell: string): CompletionShell | null { return "posix"; } -// A block with its own closing line protects the sentinel from trailing comments, heredocs and scripts reading stdin. +const utf8 = new TextEncoder(); +// Canonical TTY input (before a line editor starts reading) caps a line below 1 KiB on Linux. +const POSIX_MAX_LINE_BYTES = 1_023; +const POSIX_APPEND = '__t3_command="$__t3_command"'; + +function encodePosixScript(text: string): string { + let encoded = ""; + for (const character of text) { + const code = character.charCodeAt(0); + if (character === "'") { + encoded += "'\\''"; + } else if (character === "\\" || code <= 0x1f || code === 0x7f) { + encoded += `\\0${code.toString(8).padStart(3, "0")}`; + } else { + encoded += character; + } + } + return encoded; +} + +// Line editors run each submitted line on its own, so every line must be a complete command. +function wrapPosixCommand(command: string, sentinel: string): string { + const normalized = command.replace(/\r\n/g, "\n"); + // The `_` suffix keeps trailing newlines; the subshell contains exits and syntax errors. + const evaluate = (payload: string) => + `( __t3_command=$(printf '%b_' ${payload}); eval "\${__t3_command%_}" ); printf '\\n${sentinel}%s\\n' "$?"`; + const inline = evaluate(`'${encodePosixScript(normalized)}'`); + if (utf8.encode(` ${inline}\r`).byteLength <= POSIX_MAX_LINE_BYTES) return inline; + + const maxChunkBytes = POSIX_MAX_LINE_BYTES - utf8.encode(` ${POSIX_APPEND}''\r`).byteLength; + const lines: string[] = []; + let prefix = "__t3_command="; + let chunk = ""; + let chunkBytes = 0; + for (const character of normalized) { + const encoded = encodePosixScript(character); + const bytes = utf8.encode(encoded).byteLength; + if (chunkBytes + bytes > maxChunkBytes) { + lines.push(`${prefix}'${chunk}'`); + prefix = POSIX_APPEND; + chunk = ""; + chunkBytes = 0; + } + chunk += encoded; + chunkBytes += bytes; + } + lines.push(`${prefix}'${chunk}'`, `${evaluate('"$__t3_command"')}; unset __t3_command`); + // The leading space gives every line the history suppression a short command gets. + return lines.join("\r "); +} + function wrapCommandForCompletion( command: string, shell: CompletionShell, @@ -204,7 +252,7 @@ function wrapCommandForCompletion( ].join("; "); } case "posix": - return `( ${body}\r); printf '\\n${sentinel}%s\\n' "$?"`; + return wrapPosixCommand(command, sentinel); } } diff --git a/apps/server/src/project/RepositoryIdentityResolver.test.ts b/apps/server/src/project/RepositoryIdentityResolver.test.ts index 40a70c0a8f81b..59d31c0dfb704 100644 --- a/apps/server/src/project/RepositoryIdentityResolver.test.ts +++ b/apps/server/src/project/RepositoryIdentityResolver.test.ts @@ -8,6 +8,7 @@ import * as Path from "effect/Path"; import { vi } from "vite-plus/test"; import * as ProcessRunner from "../processRunner.ts"; +import { sourceControlDefinitions } from "../sourceControl/sourceControlDefinitions.ts"; import * as RepositoryIdentityResolver from "./RepositoryIdentityResolver.ts"; const normalizePathSeparators = (value: string) => value.replaceAll("\\", "/"); @@ -71,6 +72,30 @@ it.layer(NodeServices.layer)("RepositoryIdentityResolverLive", (it) => { }).pipe(Effect.provide(RepositoryIdentityResolver.layer)), ); + it.effect("keeps a literal .git in an Azure DevOps repository name", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-repository-identity-azure-git-suffix-test-", + }); + + yield* git(cwd, ["init"]); + yield* git(cwd, [ + "remote", + "add", + "origin", + "https://dev.azure.com/Acme/Platform/_git/Tool.git", + ]); + + const resolver = yield* RepositoryIdentityResolver.RepositoryIdentityResolver; + const identity = yield* resolver.resolve(cwd); + + expect(identity?.displayName).toBe("Acme/Platform/_git/Tool.git"); + expect(identity?.name).toBe("Tool.git"); + expect(sourceControlDefinitions.repositorySelector(identity)).toBe("Tool.git"); + }).pipe(Effect.provide(RepositoryIdentityResolver.layer)), + ); + it.effect("resolves the fetch remote of a partial clone", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; diff --git a/apps/server/src/project/RepositoryIdentityResolver.ts b/apps/server/src/project/RepositoryIdentityResolver.ts index 49c8c29cbadad..0d6841a5abb77 100644 --- a/apps/server/src/project/RepositoryIdentityResolver.ts +++ b/apps/server/src/project/RepositoryIdentityResolver.ts @@ -1,7 +1,7 @@ import type { RepositoryIdentity, SourceControlProviderError } from "@t3tools/contracts"; import { parseGitRemoteHost } from "@t3tools/source-control-core/shared/definition"; import { repositoryPathFromRemoteUrl } from "@t3tools/source-control-core/server/SourceControlProvider"; -import { normalizeGitRemoteUrl } from "@t3tools/shared/git"; +import { canonicalRepositoryPath, normalizeGitRemoteUrl } from "@t3tools/shared/git"; import * as Cache from "effect/Cache"; import * as Context from "effect/Context"; import * as Data from "effect/Data"; @@ -123,7 +123,13 @@ function pickPrimaryRemote( function repositoryPathOf(canonicalKey: string, remoteUrl: string): string { const canonicalPath = canonicalKey.split("/").slice(1).join("/"); const displayPath = repositoryPathFromRemoteUrl(remoteUrl); - return displayPath?.toLowerCase() === canonicalPath ? displayPath : canonicalPath; + if (displayPath === null) return canonicalPath; + // A host that names repositories literally keeps the `.git` the display path drops. + return ( + [displayPath, `${displayPath}.git`].find( + (path) => canonicalRepositoryPath(path) === canonicalRepositoryPath(canonicalPath), + ) ?? canonicalPath + ); } function buildRepositoryOrigin( diff --git a/apps/server/src/provider/AgentContextBudget.test.ts b/apps/server/src/provider/AgentContextBudget.test.ts index 5f4cb5ed4b8ed..2a7d02fc28173 100644 --- a/apps/server/src/provider/AgentContextBudget.test.ts +++ b/apps/server/src/provider/AgentContextBudget.test.ts @@ -44,6 +44,7 @@ import { GitVcsDriver } from "../vcs/GitVcsDriver.ts"; import { VcsStatusBroadcaster } from "../vcs/VcsStatusBroadcaster.ts"; import { buildCodexAdditionalContext } from "./CodexDeveloperInstructions.ts"; import { DocumentText } from "./DocumentText.ts"; +import { EnvironmentThemeService } from "../environmentTheme.ts"; import { ProviderRegistry } from "./ProviderRegistry.ts"; import budget from "./agentContextBudget.json" with { type: "json" }; @@ -52,6 +53,7 @@ const toolHandlerServices = Layer.mergeAll( Layer.mock(AssetSigner)({}), Layer.mock(DeviceService)({}), Layer.mock(DocumentText)({}), + Layer.mock(EnvironmentThemeService)({}), Layer.mock(GitVcsDriver)({}), Layer.mock(GitWorkflowService)({}), Layer.mock(ManagedProjectFolders)({ namedProjectsRoot: "" }), diff --git a/apps/server/src/provider/ClaudeCapabilitiesProbe.test.ts b/apps/server/src/provider/ClaudeCapabilitiesProbe.test.ts index e2622bc9c2a65..b89cffa423874 100644 --- a/apps/server/src/provider/ClaudeCapabilitiesProbe.test.ts +++ b/apps/server/src/provider/ClaudeCapabilitiesProbe.test.ts @@ -30,7 +30,7 @@ it("drops strict MCP mode, and only that, next to an administrator's managed-mcp environment: { HOME: "/home/user" }, cwd: undefined, managedMcpConfig: true, - spawnThroughEnv: false, + spawnVia: undefined, }); // Claude exits before initializing when handed --strict-mcp-config here. diff --git a/apps/server/src/provider/ClaudeCodeModelRoutes.test.ts b/apps/server/src/provider/ClaudeCodeModelRoutes.test.ts new file mode 100644 index 0000000000000..2c06148cac680 --- /dev/null +++ b/apps/server/src/provider/ClaudeCodeModelRoutes.test.ts @@ -0,0 +1,67 @@ +import { assert, describe, it } from "@effect/vitest"; +import { type ModelSelection, ProviderInstanceId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Option from "effect/Option"; +import * as TestClock from "effect/testing/TestClock"; + +import { makeClaudeCodeModelRoutes } from "./ClaudeCodeModelRoutes.ts"; +import { + BUNDLED_CLAUDE_MODEL_CATALOG, + resolveClaudeCatalogApiModelId, +} from "./ClaudeModelCatalog.ts"; + +describe("Claude Code model routes", () => { + it.effect( + "a turn that starts before the first probe settles waits and then gets its routes", + () => + Effect.gen(function* () { + const haiku = { + instanceId: ProviderInstanceId.make("claudeAgent"), + model: "claude-haiku-4-5", + } satisfies ModelSelection; + const routes = yield* makeClaudeCodeModelRoutes( + () => Effect.succeed("same settings"), + "/state", + ); + const early = yield* routes + .route(BUNDLED_CLAUDE_MODEL_CATALOG, "/project") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* routes.record([ + { value: "haiku", resolvedModel: "us.anthropic.claude-haiku-4-5-20251001-v1:0" }, + ]); + yield* routes.settle; + assert.strictEqual( + resolveClaudeCatalogApiModelId(yield* Fiber.join(early), haiku), + "haiku", + ); + + // A first check that never reaches the probe still releases waiting turns. + const unprobed = yield* makeClaudeCodeModelRoutes( + () => Effect.succeed("same settings"), + "/state", + ); + const waiting = yield* unprobed + .route(BUNDLED_CLAUDE_MODEL_CATALOG, "/project") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* unprobed.settle; + assert.strictEqual(yield* Fiber.join(waiting), BUNDLED_CLAUDE_MODEL_CATALOG); + + const stuck = yield* makeClaudeCodeModelRoutes( + () => Effect.succeed("same settings"), + "/state", + ); + const fallback = yield* stuck + .route(BUNDLED_CLAUDE_MODEL_CATALOG, "/project") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust("2 minutes"); + assert.strictEqual(yield* Fiber.join(fallback), BUNDLED_CLAUDE_MODEL_CATALOG); + // Later turns, steers and effort changes do not wait for the stuck probe again. + const later = yield* stuck + .route(BUNDLED_CLAUDE_MODEL_CATALOG, "/project") + .pipe(Effect.timeoutOption("1 second"), Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust("1 second"); + assert.deepStrictEqual(yield* Fiber.join(later), Option.some(BUNDLED_CLAUDE_MODEL_CATALOG)); + }), + ); +}); diff --git a/apps/server/src/provider/ClaudeCodeModelRoutes.ts b/apps/server/src/provider/ClaudeCodeModelRoutes.ts new file mode 100644 index 0000000000000..6a9e7a48e64c5 --- /dev/null +++ b/apps/server/src/provider/ClaudeCodeModelRoutes.ts @@ -0,0 +1,116 @@ +import { + type ModelInfo as ClaudeModelInfo, + resolveSettings as resolveClaudeSettings, + type SettingSource, + type Settings as ClaudeSdkSettings, +} from "@anthropic-ai/claude-agent-sdk"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; + +import { claudeLocalSettingsDirectory } from "./Drivers/ClaudeHome.ts"; +import { type ClaudeModelCatalog, withClaudeCodeModelRoutes } from "./ClaudeModelCatalog.ts"; + +// Env names that pick the backend, its region, or the model a family alias resolves to. +const MODEL_SETTINGS_ENV_PREFIXES = [ + "ANTHROPIC_", + "CLAUDE_CODE_USE_", + "AWS_", + "CLOUD_ML_", + "VERTEX_", +]; + +/** Equal keys mean Claude Code resolves model ids the same way under both settings. */ +function claudeModelSettingsKey(settings: ClaudeSdkSettings): string { + const sortedEntries = (record: Readonly> | undefined) => + Object.entries(record ?? {}).toSorted(([a], [b]) => a.localeCompare(b)); + return JSON.stringify([ + settings.availableModels ?? null, + sortedEntries(settings.modelOverrides), + sortedEntries(settings.env).filter(([name]) => + MODEL_SETTINGS_ENV_PREFIXES.some((prefix) => name.startsWith(prefix)), + ), + ]); +} + +const NO_MODEL_SETTINGS_KEY = claudeModelSettingsKey({}); + +class ClaudeModelSettingsReadError extends Schema.TaggedError()( + "ClaudeModelSettingsReadError", + { cause: Schema.Defect() }, +) {} + +export const resolveClaudeModelSettingsKey = Effect.fn("resolveClaudeModelSettingsKey")(function* ( + settingSources: ReadonlyArray, + cwd: string | null, +) { + // The SDK reads the local file from cwd, Claude Code from the main checkout's root. No key, so + // no routes, when that root file touches model settings. + const localDirectory = cwd === null ? null : yield* claudeLocalSettingsDirectory(cwd); + if (localDirectory !== null && localDirectory !== cwd) { + const rootLocal = yield* Effect.tryPromise({ + try: () => resolveClaudeSettings({ cwd: localDirectory, settingSources: ["local"] }), + catch: (cause) => new ClaudeModelSettingsReadError({ cause }), + }).pipe( + Effect.map(({ sources }) => sources.find(({ source }) => source === "local")?.settings), + Effect.orElseSucceed(() => undefined), + ); + if (claudeModelSettingsKey(rootLocal ?? {}) !== NO_MODEL_SETTINGS_KEY) return undefined; + } + return yield* Effect.tryPromise({ + try: () => + resolveClaudeSettings({ + ...(cwd === null ? {} : { cwd }), + settingSources: [...settingSources], + }), + catch: (cause) => new ClaudeModelSettingsReadError({ cause }), + }).pipe( + Effect.map(({ effective }) => claudeModelSettingsKey(effective)), + Effect.orElseSucceed(() => undefined), + ); +}); + +type ReportedModel = Pick; + +/** + * Claude Code's reported model ids, applied to a turn only where its cwd resolves model settings + * as the probe's cwd did. `settle` releases turns waiting for the first probe, whatever its outcome. + */ +export const makeClaudeCodeModelRoutes = Effect.fn("makeClaudeCodeModelRoutes")(function* ( + settingsKey: (cwd: string | null) => Effect.Effect, + probeCwd: string, +) { + const firstProbeSettled = yield* Deferred.make(); + const reported = yield* Ref.make<{ + readonly models: ReadonlyArray; + readonly settingsKey: string | undefined; + }>({ models: [], settingsKey: undefined }); + return { + record: Effect.fn("ClaudeCodeModelRoutes.record")(function* ( + models: ReadonlyArray, + ) { + yield* Ref.set(reported, { models, settingsKey: yield* settingsKey(probeCwd) }); + }), + settle: Deferred.succeed(firstProbeSettled, undefined).pipe(Effect.asVoid), + route: Effect.fn("ClaudeCodeModelRoutes.route")(function* ( + base: ClaudeModelCatalog, + cwd: string | null, + launchSettings?: ClaudeSdkSettings, + ) { + // Leave room for the sequential version and capability probes before falling back once; + // an expired wait counts as settled so later calls do not wait again. + yield* Deferred.await(firstProbeSettled).pipe( + Effect.timeoutOption("2 minutes"), + Effect.andThen(Deferred.succeed(firstProbeSettled, undefined)), + ); + if (claudeModelSettingsKey(launchSettings ?? {}) !== NO_MODEL_SETTINGS_KEY) return base; + const { models, settingsKey: probeKey } = yield* Ref.get(reported); + const routed = withClaudeCodeModelRoutes(base, models); + // Project settings can point an alias at another model, so routes hold only where the + // cwd resolves the same model settings as the probe's. + if (routed === base || probeKey === undefined) return base; + return (yield* settingsKey(cwd)) === probeKey ? routed : base; + }), + }; +}); diff --git a/apps/server/src/provider/ClaudeModelCatalog.test.ts b/apps/server/src/provider/ClaudeModelCatalog.test.ts index c8c8abdfcf2b0..bbcc02f901329 100644 --- a/apps/server/src/provider/ClaudeModelCatalog.test.ts +++ b/apps/server/src/provider/ClaudeModelCatalog.test.ts @@ -1,8 +1,9 @@ import { assert, describe, it } from "@effect/vitest"; -import { ProviderDriverKind, ProviderInstanceId } from "@t3tools/contracts"; +import { type ModelSelection, ProviderDriverKind, ProviderInstanceId } from "@t3tools/contracts"; import { resolveProviderCatalog, type ModelManifestData } from "./ModelManifest.ts"; import { + BUNDLED_CLAUDE_MODEL_CATALOG, capClaudeCatalogEffort, type ClaudeModelCatalog, formatClaudeVersionUpgradeMessage, @@ -16,6 +17,7 @@ import { resolveClaudeModelSlug, resolveClaudeUpdateRequiredModels, scopeClaudeModelCatalog, + withClaudeCodeModelRoutes, } from "./ClaudeModelCatalog.ts"; const manifest = (): ModelManifestData => ({ @@ -135,6 +137,13 @@ describe("Claude model catalog", () => { formatClaudeVersionUpgradeMessage(catalog, "3.1.9"), "Claude Code v3.1.9 is too old for Claude Synthetic Next. Upgrade to v3.2.0 or newer to access it.", ); + // Claude Code before 2.1.321 does not know claude-fable-5-2 and runs it with a 200K window. + assert.notInclude( + resolveClaudeModelsForVersion(BUNDLED_CLAUDE_MODEL_CATALOG, "2.1.320").map( + (model) => model.slug, + ), + "claude-fable-5-2", + ); }); it("resolves aliases and declarative adapter mappings", () => { @@ -233,6 +242,65 @@ describe("Claude model catalog", () => { ); }); + // Rows Claude Code 2.1.310 reports from initialization for each configuration. + it("sends the model id Claude Code reports when it serves a catalog model under another id", () => { + const apiModel = ( + catalog: ClaudeModelCatalog, + model: string, + options?: ModelSelection["options"], + ) => + resolveClaudeCatalogApiModelId(catalog, { + instanceId: ProviderInstanceId.make("claudeAgent"), + model, + ...(options ? { options } : {}), + }); + + // Bedrock with ANTHROPIC_DEFAULT_*_MODEL and an availableModels allowlist of their ids. + const configuredBedrock = withClaudeCodeModelRoutes(BUNDLED_CLAUDE_MODEL_CATALOG, [ + { value: "default", resolvedModel: "global.anthropic.claude-opus-5-5" }, + { value: "sonnet", resolvedModel: "global.anthropic.claude-sonnet-5-5[1m]" }, + { value: "opus", resolvedModel: "global.anthropic.claude-opus-5-5" }, + { value: "haiku", resolvedModel: "global.anthropic.claude-haiku-5-5" }, + ]); + assert.strictEqual(apiModel(configuredBedrock, "claude-haiku-5-5"), "haiku"); + assert.strictEqual(apiModel(configuredBedrock, "haiku-5.5"), "haiku"); + assert.strictEqual(apiModel(configuredBedrock, "claude-sonnet-5-5"), "sonnet"); + assert.strictEqual(apiModel(configuredBedrock, "claude-opus-5-5"), "opus"); + assert.strictEqual(apiModel(configuredBedrock, "claude-opus-5"), "claude-opus-5"); + + // Bedrock with no model settings: Claude Code's `haiku` is Haiku 4.5 there. + const defaultBedrock = withClaudeCodeModelRoutes(BUNDLED_CLAUDE_MODEL_CATALOG, [ + { + value: "us.anthropic.claude-opus-4-6-v1", + resolvedModel: "us.anthropic.claude-opus-4-6-v1", + }, + { + value: "us.anthropic.claude-opus-4-6-v1[1m]", + resolvedModel: "us.anthropic.claude-opus-4-6-v1[1m]", + }, + { value: "haiku", resolvedModel: "us.anthropic.claude-haiku-4-5-20251001-v1:0" }, + ]); + assert.strictEqual( + apiModel(defaultBedrock, "claude-opus-4-6", [{ id: "contextWindow", value: "1m" }]), + "us.anthropic.claude-opus-4-6-v1[1m]", + ); + assert.strictEqual( + apiModel(defaultBedrock, "claude-opus-4-6", [{ id: "contextWindow", value: "200k" }]), + "us.anthropic.claude-opus-4-6-v1", + ); + assert.strictEqual(apiModel(defaultBedrock, "claude-haiku-4-5"), "haiku"); + assert.strictEqual(apiModel(defaultBedrock, "claude-haiku-5-5"), "claude-haiku-5-5"); + + // First-party ids already are what Claude Code runs, even when an alias points elsewhere. + const firstParty = withClaudeCodeModelRoutes(BUNDLED_CLAUDE_MODEL_CATALOG, [ + { value: "default", resolvedModel: "claude-opus-5-5" }, + { value: "opus", resolvedModel: "claude-opus-5-5" }, + { value: "claude-fable-5-1[1m]", resolvedModel: "claude-fable-5-1[1m]" }, + { value: "haiku", resolvedModel: "claude-haiku-4-5-20251001" }, + ]); + assert.strictEqual(firstParty, BUNDLED_CLAUDE_MODEL_CATALOG); + }); + it("marks effort options that would run above the cap without changing ids or defaults", () => { const effortModel = (slug: string) => ({ slug, diff --git a/apps/server/src/provider/ClaudeModelCatalog.ts b/apps/server/src/provider/ClaudeModelCatalog.ts index ceeb94a355e86..e3d4ea4397c0e 100644 --- a/apps/server/src/provider/ClaudeModelCatalog.ts +++ b/apps/server/src/provider/ClaudeModelCatalog.ts @@ -32,6 +32,8 @@ export interface ClaudeCatalogModel { readonly model: ServerProviderModel; readonly runtime: ClaudeCodeProfile; readonly compatibility: ClaudeCodeCompatibility; + /** What to send Claude Code instead of the slug, from its own model list. */ + readonly apiModel?: string; } export interface ClaudeModelCatalog { @@ -221,7 +223,7 @@ export function claudeReportedModels( return models; } -function resolveClaudeCatalogModel( +export function resolveClaudeCatalogModel( catalog: ClaudeModelCatalog, slugOrAlias: string | null | undefined, ): ClaudeCatalogModel | undefined { @@ -235,6 +237,44 @@ function resolveClaudeCatalogModel( ); } +// Cloud backends decorate first-party ids: Bedrock adds a region prefix and a +// version ("us.anthropic.claude-opus-4-6-v1"), Vertex adds "@". +function stripClaudeBackendDecoration(wireModel: string): string { + return wireModel + .replace(/^(?:[a-z-]+\.)?anthropic\./, "") + .replace(/-v\d+(?::\d+)?$/, "") + .replace(/@[^@]*$/, ""); +} + +/** + * On Bedrock, Vertex and similar backends Claude Code does not accept the first-party slug: it + * either sends it verbatim or, when `availableModels` excludes it, silently runs the default + * model instead. Its model list names the id that does work, often a family alias that + * `ANTHROPIC_DEFAULT_*_MODEL` points at a backend id. + */ +export function withClaudeCodeModelRoutes( + catalog: ClaudeModelCatalog, + reportedModels: ReadonlyArray>, +): ClaudeModelCatalog { + const apiModels = new Map(); + for (const reported of reportedModels) { + if (reported.value === "default") continue; + const wireModel = stripClaudeContextWindowSuffix(reported.resolvedModel ?? reported.value); + const firstPartyModel = stripClaudeBackendDecoration(wireModel); + if (firstPartyModel === wireModel) continue; + const entry = resolveClaudeCatalogModel(catalog, firstPartyModel); + if (!entry || apiModels.has(entry.model.slug)) continue; + apiModels.set(entry.model.slug, stripClaudeContextWindowSuffix(reported.value)); + } + if (apiModels.size === 0) return catalog; + return { + models: catalog.models.map((entry) => { + const apiModel = apiModels.get(entry.model.slug); + return apiModel ? { ...entry, apiModel } : entry; + }), + }; +} + export function resolveClaudeModelSlug(catalog: ClaudeModelCatalog, slugOrAlias: string): string { return resolveClaudeCatalogModel(catalog, slugOrAlias)?.model.slug ?? slugOrAlias; } @@ -506,7 +546,7 @@ export function resolveClaudeCatalogApiModelId( modelSelection: ModelSelection, ): string { const entry = resolveClaudeCatalogModel(catalog, modelSelection.model); - const slug = entry?.model.slug ?? modelSelection.model; + const slug = entry?.apiModel ?? entry?.model.slug ?? modelSelection.model; const descriptors = getProviderOptionDescriptors({ caps: entry?.model.capabilities ?? EMPTY_CAPABILITIES, selections: modelSelection.options, diff --git a/apps/server/src/provider/ClaudeProvider.ts b/apps/server/src/provider/ClaudeProvider.ts index d5ebcc31220c5..ca5219f03ea42 100644 --- a/apps/server/src/provider/ClaudeProvider.ts +++ b/apps/server/src/provider/ClaudeProvider.ts @@ -22,6 +22,7 @@ import { ChildProcess, ChildProcessSpawner } from "effect/process"; import { createModelCapabilities } from "@t3tools/shared/model"; import { envTrampoline, supportsEnvChdir } from "@t3tools/shared/envTrampoline"; import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as SpawnWorker from "../process/SpawnWorker.ts"; import { resolveSpawnCommand } from "@t3tools/shared/shell"; import { query as claudeQuery, @@ -177,11 +178,16 @@ function formatClaudeSubscriptionAuthLabel(subscriptionType: string): string { return `Claude ${subscriptionLabel} Subscription`; } +function hasApiKey(apiKeySource: string | undefined): boolean { + return Boolean(apiKeySource) && apiKeySource !== "none"; +} + function claudeAuthMetadata(input: { readonly subscriptionType: string | undefined; readonly authMethod: string | undefined; + readonly apiKeySource: string | undefined; }): { readonly type: string; readonly label: string } | undefined { - if (normalizeClaudeAuthMethod(input.authMethod) === "apiKey") { + if (normalizeClaudeAuthMethod(input.authMethod) === "apiKey" || hasApiKey(input.apiKeySource)) { return { type: "apiKey", label: "Claude API Key", @@ -216,8 +222,7 @@ function claudeAuthStatus( } if (capabilities.tokenSource !== "none") return "authenticated"; // API keys can authenticate with tokenSource: none; also check key and account fields. - const hasApiKey = Boolean(capabilities.apiKeySource) && capabilities.apiKeySource !== "none"; - return hasApiKey || capabilities.email || capabilities.subscriptionType + return hasApiKey(capabilities.apiKeySource) || capabilities.email || capabilities.subscriptionType ? "authenticated" : "unauthenticated"; } @@ -259,6 +264,14 @@ export function spawnClaudeProbeThroughEnv({ }); } +/** Windows runs CreateProcess on the calling thread, so the probe launches from a launcher thread. */ +function spawnClaudeProbeOffThread({ command, args, cwd, env, signal }: ClaudeSpawnOptions) { + const child = SpawnWorker.spawnPiped(command, args, { cwd, env, signal, windowsHide: true }); + // The probe ignores stderr; drain it so a full pipe cannot stall the CLI. + child.stderr.resume(); + return child; +} + export function buildClaudeCapabilitiesProbeQueryOptions(input: { readonly executablePath: string; readonly abortController: AbortController; @@ -266,10 +279,13 @@ export function buildClaudeCapabilitiesProbeQueryOptions(input: { readonly cwd: string | undefined; /** An administrator's `managed-mcp.json` is installed; see `hasClaudeManagedMcpConfig`. */ readonly managedMcpConfig: boolean; - readonly spawnThroughEnv: boolean; + readonly spawnVia: "env" | "launcher-thread" | undefined; }): ClaudeQueryOptions { return { - ...(input.spawnThroughEnv ? { spawnClaudeCodeProcess: spawnClaudeProbeThroughEnv } : {}), + ...(input.spawnVia === "env" ? { spawnClaudeCodeProcess: spawnClaudeProbeThroughEnv } : {}), + ...(input.spawnVia === "launcher-thread" + ? { spawnClaudeCodeProcess: spawnClaudeProbeOffThread } + : {}), persistSession: false, pathToClaudeCodeExecutable: input.executablePath, abortController: input.abortController, @@ -504,8 +520,13 @@ const probeClaudeCapabilities = ( ...(yield* readClaudeSettingsEnv(claudeSettings, cwd, claudeEnvironment)), }) : 0; - const spawnThroughEnv = - (yield* HostProcess.Platform) === "linux" && (yield* Effect.promise(supportsEnvChdir)); + const platform = yield* HostProcess.Platform; + const spawnVia = + platform === "win32" + ? "launcher-thread" + : platform === "linux" && (yield* Effect.promise(supportsEnvChdir)) + ? "env" + : undefined; return yield* Effect.tryPromise({ try: async () => { const q = claudeQuery({ @@ -519,7 +540,7 @@ const probeClaudeCapabilities = ( environment: claudeEnvironment, cwd, managedMcpConfig, - spawnThroughEnv, + spawnVia, }), }); const init = await q.initializationResult(); @@ -858,9 +879,8 @@ export const checkClaudeProviderStatus = Effect.fn("checkClaudeProviderStatus")( claudeAuthMetadata({ subscriptionType: capabilities.subscriptionType, authMethod: capabilities.tokenSource, - }) ?? - apiProviderAuthMetadata(capabilities.apiProvider) ?? - (customApiEndpoint ? { type: "customEndpoint", label: "Custom API endpoint" } : undefined); + apiKeySource: capabilities.apiKeySource, + }); const subscriptionUsage = resolveSubscriptionUsage && capabilities.subscriptionType && diff --git a/apps/server/src/provider/Drivers/AntigravityUsage.ts b/apps/server/src/provider/Drivers/AntigravityUsage.ts index 5cb55ed77ac09..2eb9744a3e42a 100644 --- a/apps/server/src/provider/Drivers/AntigravityUsage.ts +++ b/apps/server/src/provider/Drivers/AntigravityUsage.ts @@ -15,7 +15,11 @@ import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import { resolveAntigravityInstanceDirectories } from "../antigravityAuthSupport.ts"; -import { makeAntigravityUsageCache, readAntigravityUsage } from "./antigravityUsageReader.ts"; +import { + antigravityRateModel, + makeAntigravityUsageCache, + readAntigravityUsage, +} from "./antigravityUsageReader.ts"; export class AntigravityUsage extends Context.Service< AntigravityUsage, @@ -123,6 +127,7 @@ export const layer = Layer.effect(AntigravityUsage, make); export const antigravityUsageReader: ProviderUsageReader = { kind: "scan", provider: "antigravity", + rateModel: antigravityRateModel, scan: ({ instances, windowStartMs }) => AntigravityUsage.pipe(Effect.flatMap((usage) => usage.scan({ instances, windowStartMs }))), }; diff --git a/apps/server/src/provider/Drivers/ClaudeDriver.ts b/apps/server/src/provider/Drivers/ClaudeDriver.ts index 26bc74e2c0e8d..6c0eda00949e0 100644 --- a/apps/server/src/provider/Drivers/ClaudeDriver.ts +++ b/apps/server/src/provider/Drivers/ClaudeDriver.ts @@ -36,6 +36,10 @@ import { import { makeManagedServerProvider } from "@t3tools/provider-core/server/managedProvider"; import * as ModelCatalog from "@t3tools/provider-core/server/ModelCatalog"; import { resolveClaudeModelCatalog } from "../ClaudeModelCatalog.ts"; +import { + makeClaudeCodeModelRoutes, + resolveClaudeModelSettingsKey, +} from "../ClaudeCodeModelRoutes.ts"; import { defaultProviderContinuationIdentity, type ProviderDriver, @@ -146,6 +150,23 @@ export const ClaudeDriver: ProviderDriver 0 || + processEnv.CLAUDE_CONFIG_DIR !== (yield* HostProcess.Environment).CLAUDE_CONFIG_DIR; + // Only a successful probe records routes, so a failed one never sends turns back to slugs + // the backend rejects. + const modelRoutes = yield* makeClaudeCodeModelRoutes( + (cwd) => + resolveClaudeModelSettingsKey( + customHome ? ["project", "local"] : ["user", "project", "local"], + cwd, + ).pipe( + Effect.provideService(FileSystem.FileSystem, fileSystem), + Effect.provideService(Path.Path, path), + ), + stateDir, + ); const orchestrationAdapter = yield* createClaudeAdapterV2( { instanceId, @@ -159,6 +180,7 @@ export const ClaudeDriver: ProviderDriver snapshot.applyUsageLimits(update), modelCatalog, + routeModelCatalog: modelRoutes.route, }, ).pipe( Effect.mapError( @@ -175,6 +197,7 @@ export const ClaudeDriver: ProviderDriver Ref.set(lastGoodCapabilities, capabilities)), + Effect.tap(({ capabilities }) => + Ref.set(lastGoodCapabilities, capabilities).pipe( + Effect.andThen(modelRoutes.record(capabilities.models ?? [])), + ), + ), + Effect.ensuring(modelRoutes.settle), Effect.catch((error) => Ref.getAndSet(lastGoodCapabilities, undefined).pipe( Effect.flatMap((capabilities) => @@ -262,6 +290,8 @@ export const ClaudeDriver: ProviderDriver { }), ); + it.effect("follows the WinGet launcher to the package's native binary", () => + Effect.gen(function* () { + expect( + yield* resolveClaudeSdkExecutablePath("claude", {}).pipe( + withWindowsResolution({ resolvedCommand: WINGET_SHIM, existingFiles: [WINGET_EXE] }), + ), + ).toBe(WINGET_EXE); + }), + ); + it.effect("follows .bat and .ps1 launcher shims the same way", () => Effect.gen(function* () { for (const shim of [`${NPM_DIR}\\claude.bat`, `${NPM_DIR}\\claude.ps1`]) { diff --git a/apps/server/src/provider/Drivers/ClaudeExecutable.ts b/apps/server/src/provider/Drivers/ClaudeExecutable.ts index 5e4cbb4db6efe..5797066446b99 100644 --- a/apps/server/src/provider/Drivers/ClaudeExecutable.ts +++ b/apps/server/src/provider/Drivers/ClaudeExecutable.ts @@ -13,10 +13,14 @@ const WINDOWS_SHIM_EXTENSIONS: ReadonlySet = new Set([".cmd", ".bat", ". export const isWindowsClaudeLauncherShimPath = (filePath: string): boolean => WINDOWS_SHIM_EXTENSIONS.has(NodePath.win32.extname(filePath).toLowerCase()); -/** New packages provide bin/claude.exe; older packages need a JavaScript runtime for cli.js. */ -const NPM_PACKAGE_ENTRY_CANDIDATES = [ +/** + * npm: new packages provide bin/claude.exe; older ones need a JavaScript runtime for cli.js. + * WinGet: Links\claude.cmd forwards to the package's native executable under ..\Packages. + */ +const PACKAGE_ENTRY_CANDIDATES = [ ["node_modules", "@anthropic-ai", "claude-code", "bin", "claude.exe"], ["node_modules", "@anthropic-ai", "claude-code", "cli.js"], + ["..", "Packages", "Anthropic.ClaudeCode_Microsoft.Winget.Source_8wekyb3d8bbwe", "claude.exe"], ] as const; export type ExecutableFileCheck = (filePath: string) => boolean; @@ -53,7 +57,7 @@ export const resolveClaudeSdkExecutablePath = Effect.fn("resolveClaudeSdkExecuta } const shimDirectory = NodePath.win32.dirname(resolved); - for (const entrySegments of NPM_PACKAGE_ENTRY_CANDIDATES) { + for (const entrySegments of PACKAGE_ENTRY_CANDIDATES) { const candidate = NodePath.win32.join(shimDirectory, ...entrySegments); if (isFile(candidate)) { return candidate; diff --git a/apps/server/src/provider/Drivers/ClaudeHome.ts b/apps/server/src/provider/Drivers/ClaudeHome.ts index 8042a7f16e491..1dfb94bb91a1e 100644 --- a/apps/server/src/provider/Drivers/ClaudeHome.ts +++ b/apps/server/src/provider/Drivers/ClaudeHome.ts @@ -100,6 +100,49 @@ export const hasClaudeManagedMcpConfig = Effect.fn("hasClaudeManagedMcpConfig")( return yield* fileSystem.exists(managedMcpConfigPath).pipe(Effect.orElseSucceed(() => false)); }); +// A worktree's .git file points at
/.git/worktrees/, whose commondir leads back to +//
/.git. A bare repository has no main checkout, so its worktree keeps its own file. +// Untraced because claudeLocalSettingsDirectory already owns the span. +const mainCheckoutRoot = Effect.fnUntraced(function* (checkout: string) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + return yield* fileSystem.readFileString(path.join(checkout, ".git")).pipe( + Effect.flatMap((pointer) => { + const gitDir = /^gitdir:\s*(.+)$/m.exec(pointer)?.[1]?.trim(); + if (gitDir === undefined) return Effect.succeed(checkout); + const worktreeGitDir = path.resolve(checkout, gitDir); + return fileSystem.readFileString(path.join(worktreeGitDir, "commondir")).pipe( + Effect.map((commonDir) => { + const common = path.resolve(worktreeGitDir, commonDir.trim()); + return path.basename(common) === ".git" ? path.dirname(common) : checkout; + }), + ); + }), + Effect.orElseSucceed(() => checkout), + ); +}); + +// Claude Code keeps .claude/settings.local.json at the repository root, or in cwd outside a +// repository, on Windows, or when the root is home. Its owner check is skipped, so callers must +// stay safe when Claude Code ignores the root file. +export const claudeLocalSettingsDirectory = Effect.fn("claudeLocalSettingsDirectory")(function* ( + cwd: string, +) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + if ((yield* HostProcess.Platform) === "win32") return cwd; + for (let directory = cwd; ; directory = path.dirname(directory)) { + const isCheckout = yield* fileSystem + .exists(path.join(directory, ".git")) + .pipe(Effect.orElseSucceed(() => false)); + if (isCheckout) { + const root = yield* mainCheckoutRoot(directory); + return root === (yield* HostProcess.HomeDirectory) ? cwd : root; + } + if (path.dirname(directory) === directory) return cwd; + } +}); + export const makeClaudeContinuationGroupKey = Effect.fn("makeClaudeContinuationGroupKey")( function* ( config: Pick, diff --git a/apps/server/src/provider/Drivers/ClaudeMaintenance.test.ts b/apps/server/src/provider/Drivers/ClaudeMaintenance.test.ts index 7063df5023377..33f9f901c8731 100644 --- a/apps/server/src/provider/Drivers/ClaudeMaintenance.test.ts +++ b/apps/server/src/provider/Drivers/ClaudeMaintenance.test.ts @@ -24,6 +24,8 @@ import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; import * as HttpClient from "effect/http/HttpClient"; import * as HttpClientResponse from "effect/http/HttpClientResponse"; +import * as Sink from "effect/Sink"; +import * as ChildProcess from "effect/process/ChildProcess"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import { makeClaudeMaintenanceResolution } from "./ClaudeMaintenance.ts"; @@ -77,6 +79,13 @@ const fixture = Effect.fn("ClaudeMaintenance.test.fixture")(function* ( : platform === "darwin" ? "/Library/Application Support/ClaudeCode" : "/etc/claude-code"; + const npmPath = path.join(directory, "bin/npm"); + if (install === "npm") { + yield* fs.makeDirectory(path.dirname(npmPath), { recursive: true }); + yield* fs.writeFileString(npmPath, "#!/bin/sh\n"); + yield* fs.chmod(npmPath, 0o755); + } + const npmProbes: Array> = []; let installationReads = 0; const testFs = FileSystem.makeNoop({ ...fs, @@ -105,11 +114,37 @@ const fixture = Effect.fn("ClaudeMaintenance.test.fixture")(function* ( managed, managedDirectory, settingsPath, + npmProbes, installationReads: () => installationReads, - makeResolution: makeClaudeMaintenanceResolution({ binaryPath, env: { PATH: "" } }).pipe( + makeResolution: makeClaudeMaintenanceResolution({ + binaryPath, + env: { PATH: install === "npm" ? path.dirname(npmPath) : "" }, + }).pipe( Effect.provideService( ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make(() => Effect.die("version checks must not run a subprocess")), + ChildProcessSpawner.make((command) => { + if (!ChildProcess.isStandardCommand(command) || command.command !== npmPath) { + return Effect.die("version checks may only ask npm"); + } + npmProbes.push(command.args.map(String)); + return Effect.succeed( + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(1)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + stdin: Sink.drain, + stdout: Stream.encodeText( + Stream.make(JSON.stringify({ "@anthropic-ai/claude-code": { latest: "2.1.316" } })), + ), + stderr: Stream.empty, + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }), + ); + }), ), Effect.provideService(FileSystem.FileSystem, testFs), Effect.provideService(HostProcess.Platform, platform), @@ -233,12 +268,17 @@ it.layer(NodeServices.layer)("ClaudeMaintenance", (it) => { status: row.advisory, latestVersion: row.expected, }); - expect(requests).toHaveLength(1); - expect(requests[0]).toMatch( - row.npm ? /registry\.npmjs\.org/ : new RegExp(`claude-code-releases/${row.channel}$`), - ); + // npm installs ask `npm outdated`, which applies npm's release-age settings. + const lookups = row.npm ? f.npmProbes : requests; + expect(lookups).toHaveLength(1); + if (row.npm) { + expect(requests).toEqual([]); + expect(f.npmProbes[0]?.[0]).toBe("outdated"); + } else { + expect(requests[0]).toMatch(new RegExp(`claude-code-releases/${row.channel}$`)); + } yield* enrich; - expect(requests).toHaveLength(1); + expect(lookups).toHaveLength(1); }), ); @@ -363,6 +403,7 @@ it.layer(NodeServices.layer)("ClaudeMaintenance", (it) => { }).pipe( Effect.provide( Layer.mock(ProviderHost.ProviderHost)({ + awaitServerReady: Effect.void, paths: { cwd: f.directory, baseDir: f.directory, diff --git a/apps/server/src/provider/Drivers/ClaudeSkills.test.ts b/apps/server/src/provider/Drivers/ClaudeSkills.test.ts index 8b9661a779ae1..349294baa8c88 100644 --- a/apps/server/src/provider/Drivers/ClaudeSkills.test.ts +++ b/apps/server/src/provider/Drivers/ClaudeSkills.test.ts @@ -3,10 +3,19 @@ import { assert, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; +import * as HostProcess from "@t3tools/shared/HostProcess"; -import { discoverClaudeSkills } from "./ClaudeSkills.ts"; +import { discoverClaudeSkills as discoverSkills } from "./ClaudeSkills.ts"; import { planClaudeSkillDispatch } from "./ClaudeSkillDispatch.ts"; +// Windows makes the managed directory fixture-controlled without reading host policy or skills. +const discoverClaudeSkills = ( + ...[config, cwd, environment = {}]: Parameters +) => + discoverSkills(config, cwd, environment).pipe( + Effect.provideService(HostProcess.Platform, "win32"), + ); + const writeSkill = Effect.fn(function* ( skillsDir: string, directoryName: string, @@ -578,6 +587,96 @@ it.layer(NodeServices.layer)("discoverClaudeSkills", (it) => { }), ); + it.effect("discovers inline plugins from CLAUDE_CODE_PLUGIN_DIRS over installed namesakes", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const workspace = yield* fs.makeTempDirectoryScoped({ prefix: "t3-claude-skills-" }); + const configDir = path.join(workspace, "config"); + const named = path.join(workspace, "named"); + const unnamed = path.join(workspace, "unnamed"); + const inherited = path.join(workspace, "inherited"); + const muted = path.join(workspace, "muted"); + const installed = path.join(workspace, "installed-acme"); + yield* fs.makeDirectory(path.join(named, ".claude-plugin"), { recursive: true }); + yield* fs.writeFileString( + path.join(named, ".claude-plugin", "plugin.json"), + '{"name":"acme"}', + ); + yield* writeSkill(path.join(named, "skills"), "qa", "---\ndescription: QA a ticket\n---\n"); + yield* writeSkill(path.join(installed, "skills"), "old", "# Old"); + yield* writeSkill(path.join(unnamed, "skills"), "lint", "# Lint"); + yield* writeSkill(path.join(muted, "skills"), "hidden", "# Hidden"); + const quiet = path.join(workspace, "quiet"); + yield* fs.makeDirectory(path.join(quiet, ".claude-plugin"), { recursive: true }); + yield* fs.writeFileString( + path.join(quiet, ".claude-plugin", "plugin.json"), + '{"name":"quiet","defaultEnabled":false}', + ); + yield* writeSkill(path.join(quiet, "skills"), "hush", "# Hush"); + const bundle = path.join(workspace, "bundle"); + yield* fs.makeDirectory(path.join(bundle, "tools", ".claude-plugin"), { recursive: true }); + yield* fs.writeFileString( + path.join(bundle, "tools", ".claude-plugin", "plugin.json"), + '{"name":"tools"}', + ); + yield* writeSkill(path.join(bundle, "tools", "skills"), "fmt", "# Fmt"); + yield* writeSkill(path.join(bundle, "loose", "skills"), "ignored", "# Ignored"); + yield* writeSkill(path.join(inherited, "skills"), "stale", "# Stale"); + yield* writeSkill(path.join(workspace, "relative", "skills"), "skipped", "# Skipped"); + yield* fs.makeDirectory(path.join(configDir, "plugins"), { recursive: true }); + yield* fs.writeFileString( + path.join(configDir, "plugins", "installed_plugins.json"), + JSON.stringify({ plugins: { "acme@shop": [{ scope: "user", installPath: installed }] } }), + ); + yield* fs.writeFileString( + path.join(configDir, "settings.json"), + JSON.stringify({ + enabledPlugins: { "acme@shop": true, "muted@inline": false }, + env: { + CLAUDE_CODE_PLUGIN_DIRS: [named, "relative", ` ${unnamed} `, muted, quiet, bundle].join( + ";", + ), + }, + }), + ); + + const skills = yield* discoverClaudeSkills({ homePath: configDir }, workspace, { + CLAUDE_CODE_PLUGIN_DIRS: inherited, + }); + assert.deepEqual( + skills.map(({ name, scope, description }) => ({ name, scope, description })), + [ + { name: "acme:qa", scope: "user", description: "QA a ticket" }, + { name: "tools:fmt", scope: "user", description: undefined }, + { name: "unnamed:lint", scope: "user", description: undefined }, + ], + ); + + // Managed enabledPlugins entries lock their plugin name against inline copies. + const programFiles = path.join(workspace, "program-files"); + yield* fs.makeDirectory(path.join(programFiles, "ClaudeCode"), { recursive: true }); + yield* fs.writeFileString( + path.join(programFiles, "ClaudeCode", "managed-settings.json"), + '{"enabledPlugins":{"acme@shop":true}}', + ); + assert.deepEqual( + (yield* discoverClaudeSkills({ homePath: path.join(workspace, "empty") }, workspace, { + CLAUDE_CODE_PLUGIN_DIRS: [named, unnamed].join(";"), + ProgramFiles: programFiles, + })).map((skill) => skill.name), + ["unnamed:lint"], + ); + + assert.deepEqual( + (yield* discoverClaudeSkills({ homePath: path.join(workspace, "empty") }, workspace, { + CLAUDE_CODE_PLUGIN_DIRS: inherited, + })).map((skill) => skill.name), + ["inherited:stale"], + ); + }), + ); + it.effect("discovers user and project skills with frontmatter metadata", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -688,7 +787,7 @@ it.layer(NodeServices.layer)("discoverClaudeSkills", (it) => { }), ); - it.effect("prefers user skills on name collisions even with a stray .agents copy", () => + it.effect("prefers managed then user skills on collisions, even with a stray .agents copy", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -723,6 +822,62 @@ it.layer(NodeServices.layer)("discoverClaudeSkills", (it) => { description: "User deploy.", }, ]); + + const programFiles = path.join(tempDir, "Program Files"); + const programData = path.join(tempDir, "ProgramData"); + yield* writeSkill( + path.join(programData, "ClaudeCode", ".claude", "skills"), + "legacy-only", + "# Legacy skills must not be discovered", + ); + yield* fs.writeFileString( + path.join(programData, "ClaudeCode", "managed-settings.json"), + '{"skillOverrides":{"deploy":"off"}}', + ); + for (const ProgramFiles of [undefined, "", programFiles]) { + const legacySkills = yield* discoverClaudeSkills({ homePath: configDir }, workspace, { + PROGRAMDATA: programData, + ...(ProgramFiles !== undefined ? { ProgramFiles } : {}), + }); + assert.deepEqual( + legacySkills, + skills.map((skill) => ({ ...skill, enabled: false })), + ); + } + yield* writeSkill( + path.join(programFiles, "ClaudeCode", ".claude", "skills"), + "deploy", + ["---", "name: deploy", "description: Managed deploy.", "---"].join("\n"), + ); + yield* fs.writeFileString( + path.join(configDir, "settings.json"), + '{"skillOverrides":{"deploy":"off"}}', + ); + yield* fs.writeFileString( + path.join(programFiles, "ClaudeCode", "managed-settings.json"), + '{"skillOverrides":{"deploy":"on"}}', + ); + const managedSkills = yield* discoverClaudeSkills({ homePath: configDir }, workspace, { + ProgramFiles: programFiles, + PROGRAMDATA: programData, + }); + + assert.deepEqual(managedSkills, [ + { + name: "deploy", + path: path.join(programFiles, "ClaudeCode", ".claude", "skills", "deploy", "SKILL.md"), + enabled: true, + scope: "user", + description: "Managed deploy.", + }, + ]); + assert.deepEqual( + yield* discoverClaudeSkills({ homePath: configDir }, workspace, { + PROGRAMFILES: ` ${programFiles} `, + PROGRAMDATA: programData, + }), + managedSkills, + ); }), ); diff --git a/apps/server/src/provider/Drivers/ClaudeSkills.ts b/apps/server/src/provider/Drivers/ClaudeSkills.ts index c24c4afca9e7d..4fdd3565d11b6 100644 --- a/apps/server/src/provider/Drivers/ClaudeSkills.ts +++ b/apps/server/src/provider/Drivers/ClaudeSkills.ts @@ -106,19 +106,41 @@ function parseSkillFrontmatter(contents: string): SkillFrontmatter { }; } -function claudeManagedSettingsPath( +// A caller-supplied environment is a plain object, without Windows' case-insensitive keys. +function windowsEnvironmentValue(environment: NodeJS.ProcessEnv, name: string) { + const key = name.toLowerCase(); + return Object.entries(environment) + .find(([candidate]) => candidate.toLowerCase() === key)?.[1] + ?.trim(); +} + +function claudeManagedDirectory( path: Path.Path, platform: NodeJS.Platform, environment: NodeJS.ProcessEnv, ): string | undefined { if (platform === "darwin") { - return "/Library/Application Support/ClaudeCode/managed-settings.json"; + return "/Library/Application Support/ClaudeCode"; } if (platform === "win32") { - const programData = environment.PROGRAMDATA?.trim(); - return programData ? path.join(programData, "ClaudeCode", "managed-settings.json") : undefined; + const programFiles = windowsEnvironmentValue(environment, "ProgramFiles"); + return programFiles ? path.join(programFiles, "ClaudeCode") : undefined; } - return "/etc/claude-code/managed-settings.json"; + return "/etc/claude-code"; +} + +function claudeManagedSettingsPaths( + path: Path.Path, + platform: NodeJS.Platform, + environment: NodeJS.ProcessEnv, +): ReadonlyArray { + const managedDirectory = claudeManagedDirectory(path, platform, environment); + const programData = + platform === "win32" ? windowsEnvironmentValue(environment, "ProgramData") : undefined; + return [ + ...(programData ? [path.join(programData, "ClaudeCode", "managed-settings.json")] : []), + ...(managedDirectory ? [path.join(managedDirectory, "managed-settings.json")] : []), + ]; } /** Claude Code settings files, lowest precedence first. */ @@ -130,7 +152,6 @@ function claudeSettingsPaths( environment: NodeJS.ProcessEnv, repositoryRoot?: string, ): ReadonlyArray { - const managedPath = claudeManagedSettingsPath(path, platform, environment); const root = repositoryRoot !== undefined && repositoryRoot !== cwd ? repositoryRoot : undefined; return [ path.join(configDirPath, "settings.json"), @@ -141,7 +162,7 @@ function claudeSettingsPaths( ] : []), ...(root ? [path.join(root, ".claude", "settings.local.json")] : []), - ...(managedPath ? [managedPath] : []), + ...claudeManagedSettingsPaths(path, platform, environment), ]; } @@ -250,6 +271,8 @@ const readSkillSettings = Effect.fn("readSkillSettings")(function* ( const platform = yield* HostProcess.Platform; const overridesByName = new Map(); const enabledPlugins = new Map(); + const managedPluginNames = new Set(); + const managedPaths = new Set(claudeManagedSettingsPaths(path, platform, environment)); let syncedPluginsHidden = false; const repositoryRoot = cwd === undefined ? undefined : yield* findRepositoryRoot(cwd); const projectSettingsPath = @@ -285,6 +308,9 @@ const readSkillSettings = Effect.fn("readSkillSettings")(function* ( for (const [key, enabled] of Object.entries(parsed?.enabledPlugins ?? {})) { if (typeof enabled === "boolean") { enabledPlugins.set(key, enabled); + if (managedPaths.has(settingsPath) && key.includes("@")) { + managedPluginNames.add(pluginNameKey(key.slice(0, key.indexOf("@")))); + } } else { // Extended formats replace lower-precedence values without enabling the plugin here. enabledPlugins.delete(key); @@ -296,7 +322,12 @@ const readSkillSettings = Effect.fn("readSkillSettings")(function* ( } } - return { skillOverrides: overridesByName, enabledPlugins, syncedPluginsHidden }; + return { + skillOverrides: overridesByName, + enabledPlugins, + managedPluginNames, + syncedPluginsHidden, + }; }); const InstalledPlugins = Schema.fromJsonString( @@ -313,6 +344,7 @@ const PluginManifest = Schema.fromJsonString( Schema.Struct({ name: Schema.optional(Schema.NonEmptyString), skills: Schema.optional(Schema.Union([Schema.String, Schema.Array(Schema.String)])), + defaultEnabled: Schema.optional(Schema.Unknown), }), ); @@ -433,10 +465,66 @@ const resolveSyncedPluginBucket = Effect.fn("resolveSyncedPluginBucket")(functio return `${organizationId.toLowerCase()}_${accountId.toLowerCase()}`; }); +/** A plugin folder itself, or else the manifest-bearing plugins directly inside it. */ +const expandInlinePluginDirectory = Effect.fn("expandInlinePluginDirectory")(function* ( + directory: string, +) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const exists = (candidate: string) => + fileSystem.exists(candidate).pipe(Effect.orElseSucceed(() => false)); + for (const component of [path.join(".claude-plugin", "plugin.json"), "skills", "SKILL.md"]) { + if (yield* exists(path.join(directory, component))) return [directory]; + } + const children = yield* fileSystem + .readDirectory(directory) + .pipe(Effect.orElseSucceed((): ReadonlyArray => [])); + return yield* Effect.filter( + [...children].sort().map((child) => path.join(directory, child)), + (child) => exists(path.join(child, ".claude-plugin", "plugin.json")), + ); +}); + +/** Plugin folders Claude Code loads inline from CLAUDE_CODE_PLUGIN_DIRS, and their plugin names. */ +const discoverInlinePluginSkillRoots = Effect.fn("discoverInlinePluginSkillRoots")(function* ( + pluginDirectories: string | undefined, + enabledPlugins: ReadonlyMap, + managedPluginNames: ReadonlySet, +) { + const path = yield* Path.Path; + const home = yield* HostProcess.HomeDirectory; + const platform = yield* HostProcess.Platform; + const roots: Array = []; + const names = new Set(); + const seen = new Set(); + for (const entry of (pluginDirectories ?? "").split(platform === "win32" ? ";" : ":")) { + const directory = expandHomePath(entry.trim(), home); + // Claude skips relative entries rather than resolving them against its cwd. + if (!directory || !path.isAbsolute(directory)) continue; + for (const installPath of yield* expandInlinePluginDirectory(path.resolve(directory))) { + if (seen.has(installPath)) continue; + seen.add(installPath); + const manifest = yield* readPluginManifest(installPath); + const prefix = manifest?.name ?? path.basename(installPath); + // A `@inline` setting wins; otherwise the manifest's `defaultEnabled` decides. + const setting = enabledPlugins.get(`${prefix}@inline`); + if (setting === false || (setting === undefined && manifest?.defaultEnabled === false)) { + continue; + } + // Managed enabledPlugins entries, true or false, lock the name against inline copies. + if (managedPluginNames.has(pluginNameKey(prefix))) continue; + names.add(pluginNameKey(prefix)); + roots.push(...(yield* pluginSkillRoots(installPath, "user", prefix, manifest))); + } + } + return { roots, names }; +}); + const discoverPluginSkillRoots = Effect.fn("discoverPluginSkillRoots")(function* ( configDirPath: string, cwd: string | undefined, enabledPlugins: ReadonlyMap, + inlinePluginNames: ReadonlySet, syncedBucket: string | undefined, environment: NodeJS.ProcessEnv, ) { @@ -454,7 +542,8 @@ const discoverPluginSkillRoots = Effect.fn("discoverPluginSkillRoots")(function* Effect.orElseSucceed(() => undefined), ); const roots: Array = []; - const claimedPluginNames = new Set(); + // An enabled inline plugin replaces installed and synced plugins of the same name. + const claimedPluginNames = new Set(inlinePluginNames); // The CLI's process.cwd() resolves symlinks; registry projectPath values remain lexical. const workspace = cwd === undefined @@ -494,7 +583,7 @@ const discoverPluginSkillRoots = Effect.fn("discoverPluginSkillRoots")(function* const installPath = path.resolve(cwd ?? ".", install.installPath); const manifest = yield* readPluginManifest(installPath); const prefix = manifest?.name ?? key.split("@")[0]; - if (!prefix) continue; + if (!prefix || inlinePluginNames.has(pluginNameKey(prefix))) continue; // A registry record whose install is gone loads nothing and claims no namespace. if (Option.isSome(existing)) claimedPluginNames.add(pluginNameKey(prefix)); roots.push(...(yield* pluginSkillRoots(installPath, scope, prefix, manifest))); @@ -560,6 +649,7 @@ const resolveClaudeConfigDirPath = Effect.fn("resolveClaudeConfigDirPath")(funct const SettingsEnv = Schema.Struct({ ANTHROPIC_BASE_URL: Schema.optional(Schema.String), CLAUDE_CODE_ENTRYPOINT: Schema.optional(Schema.String), + CLAUDE_CODE_PLUGIN_DIRS: Schema.optional(Schema.String), // Unquoted JSON values must not fail the decode and drop ANTHROPIC_BASE_URL with them. CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY: Schema.optional(Schema.Unknown), CLAUDE_CODE_GATEWAY_MODEL_DISCOVERY_TIMEOUT_MS: Schema.optional(Schema.Unknown), @@ -627,7 +717,7 @@ export function claudeLaunchArgDirectories(launchArgs?: string): ReadonlyArray & Partial>, cwd?: string, @@ -637,11 +727,9 @@ export const discoverClaudeSkills = Effect.fn("discoverClaudeSkills")(function* const path = yield* Path.Path; const env = environment ?? process.env; const configDirPath = yield* resolveClaudeConfigDirPath(config, env, cwd); - const { skillOverrides, enabledPlugins, syncedPluginsHidden } = yield* readSkillSettings( - configDirPath, - cwd, - env, - ); + const { skillOverrides, enabledPlugins, managedPluginNames, syncedPluginsHidden } = + yield* readSkillSettings(configDirPath, cwd, env); + const managedDirectory = claudeManagedDirectory(path, yield* HostProcess.Platform, env); // Most installs have no synced folder; skip reading `.claude.json` for them. const hasSyncedPlugins = !syncedPluginsHidden && @@ -651,8 +739,20 @@ export const discoverClaudeSkills = Effect.fn("discoverClaudeSkills")(function* const syncedBucket = hasSyncedPlugins ? yield* resolveSyncedPluginBucket(config, configDirPath, env) : undefined; + // Claude applies its settings `env` over the inherited environment. + const inlinePluginDirectories = + (yield* readClaudeSettingsEnv(config, cwd, env)).CLAUDE_CODE_PLUGIN_DIRS ?? + env.CLAUDE_CODE_PLUGIN_DIRS; + const inlinePlugins = yield* discoverInlinePluginSkillRoots( + inlinePluginDirectories, + enabledPlugins, + managedPluginNames, + ); const roots: ReadonlyArray = [ + ...(managedDirectory + ? [{ directory: path.join(managedDirectory, ".claude", "skills"), scope: "user" as const }] + : []), { directory: path.join(configDirPath, "skills"), scope: "user" }, ...(cwd ? [{ directory: path.join(cwd, ".claude", "skills"), scope: "project" as const }] : []), ...claudeLaunchArgDirectories(config.launchArgs) @@ -661,7 +761,15 @@ export const discoverClaudeSkills = Effect.fn("discoverClaudeSkills")(function* directory: path.join(path.resolve(cwd ?? ".", directory), ".claude", "skills"), scope: "project" as const, })), - ...(yield* discoverPluginSkillRoots(configDirPath, cwd, enabledPlugins, syncedBucket, env)), + ...inlinePlugins.roots, + ...(yield* discoverPluginSkillRoots( + configDirPath, + cwd, + enabledPlugins, + inlinePlugins.names, + syncedBucket, + env, + )), ]; const skillsByName = new Map(); diff --git a/apps/server/src/provider/Drivers/CodexDriver.test.ts b/apps/server/src/provider/Drivers/CodexDriver.test.ts index 6ce377a74a1e4..1462d051346f1 100644 --- a/apps/server/src/provider/Drivers/CodexDriver.test.ts +++ b/apps/server/src/provider/Drivers/CodexDriver.test.ts @@ -42,6 +42,7 @@ import { CodexDriver } from "./CodexDriver.ts"; import * as CodexAdapterV2 from "../../orchestration-v2/Adapters/CodexAdapterV2.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import * as PortScanner from "../../preview/PortScanner.ts"; import * as ProviderCredentialStore from "../ProviderCredentialStore.ts"; import * as ProviderHostLive from "../ProviderHostLive.ts"; @@ -51,6 +52,7 @@ const layerDeps = ServerConfig.layerTest(process.cwd(), { Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), Layer.provideMerge(IdAllocator.layer), Layer.provideMerge(McpProviderSessions.layer), + Layer.provideMerge(PortScanner.layerTest), Layer.provideMerge( Layer.mock(CodexAdapterV2.CodexAppServerClientFactory)({ open: () => Effect.die("Maintenance resolution must not open a Codex session"), @@ -727,6 +729,7 @@ it.layer(layerTest)("CodexDriver", (it) => { const shimDir = NodePath.join(tempDir, fixture.dataRoot, "shims"); const npmPrefix = NodePath.join(tempDir, "mise", "installs", "node", "24.13.0"); const npmBin = NodePath.join(npmPrefix, "bin"); + const npmPath = NodePath.join(npmBin, "npm"); const npmEntry = NodePath.join( npmPrefix, "lib", @@ -736,7 +739,7 @@ it.layer(layerTest)("CodexDriver", (it) => { "bin", "codex.js", ); - for (const file of [brewPath, misePath, npmEntry]) { + for (const file of [brewPath, misePath, npmPath, npmEntry]) { yield* fs.makeDirectory(NodePath.dirname(file), { recursive: true }); yield* fs.writeFileString(file, "#!/bin/sh\n"); yield* fs.chmod(file, 0o755); @@ -751,12 +754,20 @@ it.layer(layerTest)("CodexDriver", (it) => { ].join(NodePath.delimiter); const probes: Array> = []; const metadataSpawner = ChildProcessSpawner.make((command) => { - if (!ChildProcess.isStandardCommand(command) || command.command !== brewPath) { + const isNpmOutdated = + ChildProcess.isStandardCommand(command) && + command.command === npmPath && + command.args[0] === "outdated"; + if ( + !ChildProcess.isStandardCommand(command) || + (command.command !== brewPath && !isNpmOutdated) + ) { return Effect.die("Provider resolution must not execute a provider or updater"); } - probes.push(command.args.map(String)); - const stdout = - command.args[0] === "--prefix" + if (!isNpmOutdated) probes.push(command.args.map(String)); + const stdout = isNpmOutdated + ? JSON.stringify({ "@openai/codex": { latest: "0.153.4" } }) + : command.args[0] === "--prefix" ? brewPrefix : JSON.stringify({ formulae: [{ versions: { stable: "2026.9.1" } }] }); return Effect.succeed( diff --git a/apps/server/src/provider/Drivers/antigravityUsageReader.ts b/apps/server/src/provider/Drivers/antigravityUsageReader.ts index 4d473f50c9d67..532233f7facce 100644 --- a/apps/server/src/provider/Drivers/antigravityUsageReader.ts +++ b/apps/server/src/provider/Drivers/antigravityUsageReader.ts @@ -129,18 +129,19 @@ const MODEL_IDS: Record = { }; // Effort and routing variants share the base model's price. -function withRateModel(record: UsageRecord): UsageRecord { - const base = record.model.replace( - /-(?:extra-low|low|medium|high|tiered|control|thinking|agent)$/, - "", - ); - const rateModel = /^gemini-3-flash-(?:agent(?:-[ab])?|[ab])$/.test(record.model) +export const antigravityRateModel = (model: string) => { + const base = model.replace(/-(?:extra-low|low|medium|high|tiered|control|thinking|agent)$/, ""); + return /^gemini-3-flash-(?:agent(?:-[ab])?|[ab])$/.test(model) ? "gemini-3.5-flash" - : record.model === "gemini-3.7-flash-safety-le" + : model === "gemini-3.7-flash-safety-le" ? "gemini-3.7-flash" : base === "gemini-3-flash" ? "gemini-3-flash-preview" : base; +}; + +function withRateModel(record: UsageRecord): UsageRecord { + const rateModel = antigravityRateModel(record.model); return rateModel === record.model ? record : { ...record, rateModel }; } diff --git a/apps/server/src/provider/EventNdjsonLogger.test.ts b/apps/server/src/provider/EventNdjsonLogger.test.ts index eed20381b46cb..0bb55b9bcb39b 100644 --- a/apps/server/src/provider/EventNdjsonLogger.test.ts +++ b/apps/server/src/provider/EventNdjsonLogger.test.ts @@ -255,6 +255,65 @@ it.layer(NodeServices.layer)("EventNdjsonLogger", (it) => { }), ); + it.effect("drops Cursor streaming deltas while keeping its lifecycle updates", () => + Effect.gen(function* () { + const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-")); + const basePath = NodePath.join(tempDir, "events.log"); + const threadId = ThreadId.make("cursor-deltas"); + const cursorUpdate = (update: object) => ({ + provider: "cursor", + kind: "protocol", + event: { + direction: "incoming", + stage: "decoded", + payload: { type: "interaction.update", runId: "run-1", update }, + }, + }); + const toolCall = { type: "shell", args: { command: "ls" } }; + const transient = [ + { type: "text-delta", text: "Hel" }, + { type: "thinking-delta", text: "Hmm" }, + { type: "token-delta", tokens: 3 }, + { type: "shell-output-delta", event: { stdout: "a.ts\n" } }, + { type: "partial-tool-call", callId: "call-1", modelCallId: "m-1", toolCall }, + { + type: "tool-call-delta", + callId: "call-1", + modelCallId: "m-1", + taskUpdate: { type: "text-delta", text: "Sub" }, + }, + ].map(cursorUpdate); + const retained = [ + { type: "tool-call-started", callId: "call-1", modelCallId: "m-1", toolCall }, + { + type: "tool-call-delta", + callId: "call-1", + modelCallId: "m-1", + taskUpdate: { type: "tool-call-completed", callId: "call-2", toolCall }, + }, + { type: "tool-call-completed", callId: "call-1", modelCallId: "m-1", toolCall }, + { type: "turn-ended" }, + ].map(cursorUpdate); + + try { + const store = yield* makeEventNdjsonLogStore(basePath, { batchWindowMs: 0 }); + const native = store.logger("native"); + for (const event of [...transient, ...retained]) { + yield* native.write(event, threadId); + } + yield* store.close(); + + const records = NodeFS.readFileSync(ownedLogPath(basePath, "cursor-deltas"), "utf8") + .trim() + .split("\n") + .map((line) => decodeUnknownJson(parseLogLine(line).payload)); + assert.deepEqual(records, retained); + } finally { + NodeFS.rmSync(tempDir, { recursive: true, force: true }); + } + }), + ); + it.effect("bounds oversized records while retaining failure details", () => Effect.gen(function* () { const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-")); diff --git a/apps/server/src/provider/EventNdjsonLogger.ts b/apps/server/src/provider/EventNdjsonLogger.ts index 6a1d3844b2373..5a7b3578578f1 100644 --- a/apps/server/src/provider/EventNdjsonLogger.ts +++ b/apps/server/src/provider/EventNdjsonLogger.ts @@ -63,6 +63,13 @@ const transientNativeMethods = new Set([ "turn/diff/updated", ]); const transientAcpUpdates = new Set(["agent_message_chunk", "agent_thought_chunk"]); +const transientCursorUpdates = new Set([ + "text-delta", + "thinking-delta", + "token-delta", + "shell-output-delta", + "partial-tool-call", +]); export type EventNdjsonStream = "native" | "canonical" | "orchestration"; @@ -219,6 +226,18 @@ export function shouldPersistProviderEvent(stream: EventNdjsonStream, event: unk } } + if (nativeType === "interaction.update") { + let update = Reflect.get(nativeEvent, "update"); + if (typeof update !== "object" || update === null) return true; + // Subagent updates arrive wrapped; keep the wrapper when it carries a lifecycle record. + if (Reflect.get(update, "type") === "tool-call-delta") { + update = Reflect.get(update, "taskUpdate"); + if (typeof update !== "object" || update === null) return true; + } + const updateType = Reflect.get(update, "type"); + return typeof updateType !== "string" || !transientCursorUpdates.has(updateType); + } + const payload = Reflect.get(nativeEvent, "payload"); if (typeof payload !== "object" || payload === null) return true; diff --git a/apps/server/src/provider/ProjectSettingsOverrideSweep.ts b/apps/server/src/provider/ProjectSettingsOverrideSweep.ts index aa7d205271494..06f61f9ad8bfe 100644 --- a/apps/server/src/provider/ProjectSettingsOverrideSweep.ts +++ b/apps/server/src/provider/ProjectSettingsOverrideSweep.ts @@ -1,32 +1,41 @@ -import { type ProjectId, type ServerSettings } from "@t3tools/contracts"; +import * as NodeUtil from "node:util"; + +import { type ProjectSettingsOverrides } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as ProjectStore from "../orchestration-v2/ProjectStore.ts"; import * as Settings from "../serverSettings.ts"; -const findOrphanedProjectIds = ( - settings: Pick, - liveProjectIds: ReadonlySet, -): ReadonlyArray => - Object.keys(settings.projectSettingsOverrides) - .map((projectId) => projectId as ProjectId) - .filter((projectId) => !liveProjectIds.has(projectId)); +export function deletedProjectOverridePatch(entry: ProjectSettingsOverrides | undefined) { + // Keep cleanup policy for deleted threads; omitting providerInstances would preserve obsolete references in the merge. + return entry?.worktreeCleanup !== undefined + ? { worktreeCleanup: entry.worktreeCleanup, providerInstances: {} } + : null; +} export const sweep = Effect.gen(function* () { const serverSettings = yield* Settings.ServerSettingsService; const projectStore = yield* ProjectStore.ProjectStoreV2; const settings = yield* serverSettings.getSettings; - const liveShells = yield* projectStore.listShells(); - const liveProjectIds = new Set(liveShells.map((shell) => shell.id)); + const projects = yield* projectStore.list({ includeDeleted: true }); + const projectsById = new Map( + projects.map((project) => [project.projectId, project]), + ); - const orphanedProjectIds = findOrphanedProjectIds(settings, liveProjectIds); - if (orphanedProjectIds.length === 0) return; + const overrides = Object.entries(settings.projectSettingsOverrides).flatMap( + ([projectId, entry]) => { + const project = projectsById.get(projectId); + if (project?.deletedAt === null) return []; + const patch = project ? deletedProjectOverridePatch(entry) : null; + if (NodeUtil.isDeepStrictEqual(entry, patch)) return []; + return [[projectId, patch] as const]; + }, + ); + if (overrides.length === 0) return; yield* serverSettings.updateSettings({ - projectSettingsOverrides: Object.fromEntries( - orphanedProjectIds.map((projectId) => [projectId, null]), - ), + projectSettingsOverrides: Object.fromEntries(overrides), }); }).pipe( Effect.asVoid, diff --git a/apps/server/src/provider/ProviderHostLive.ts b/apps/server/src/provider/ProviderHostLive.ts index 6398210599b61..fed07efc035ee 100644 --- a/apps/server/src/provider/ProviderHostLive.ts +++ b/apps/server/src/provider/ProviderHostLive.ts @@ -4,11 +4,13 @@ import * as Digest from "effect/crypto/Digest"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import { resolveAttachmentPath } from "../attachmentStore.ts"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; import * as ServerConfig from "../config.ts"; +import * as ServerLifecycleEvents from "../serverLifecycleEvents.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as DocumentText from "./DocumentText.ts"; import * as ProviderCredentialStore from "./ProviderCredentialStore.ts"; @@ -22,6 +24,10 @@ export const layer = Layer.effect( const secrets = yield* ServerSecretStore.ServerSecretStore; const digest = yield* Digest.Digest; const fileSystem = yield* FileSystem.FileSystem; + // Optional so standalone provider layers in tests probe at once. + const lifecycleEvents = yield* Effect.serviceOption( + ServerLifecycleEvents.ServerLifecycleEvents, + ); return ProviderHost.ProviderHost.of({ paths: { cwd: config.cwd, @@ -37,6 +43,10 @@ export const layer = Layer.effect( subscribe: serverSettings.subscribeChanges, }, shouldRunBackgroundWork: backgroundPolicy.shouldRunScopeWork, + ...Option.match(lifecycleEvents, { + onNone: () => ({}), + onSome: (events) => ({ awaitServerReady: events.awaitReady }), + }), resolveAttachmentPath: (attachment) => resolveAttachmentPath({ attachmentsDir: config.attachmentsDir, attachment }), // DocumentText owns the path-resolution span. diff --git a/apps/server/src/provider/ProviderInstanceRegistry.test.ts b/apps/server/src/provider/ProviderInstanceRegistry.test.ts index 707275afc97d2..ce6416332bae0 100644 --- a/apps/server/src/provider/ProviderInstanceRegistry.test.ts +++ b/apps/server/src/provider/ProviderInstanceRegistry.test.ts @@ -212,7 +212,8 @@ describe("ProviderInstanceRegistry — multi-instance codex slice", () => { ); const layerBase = ProviderHostLive.layer.pipe(Layer.provideMerge(layerBaseDeps)); const layerTest = ProviderOrchestrationAdapterInfrastructure.layer.pipe( - Layer.provide(Layer.merge(PortScanner.layerTest, SqlitePersistence.layerMemory)), + Layer.provide(SqlitePersistence.layerMemory), + Layer.provideMerge(PortScanner.layerTest), Layer.provideMerge(layerBase), ); diff --git a/apps/server/src/provider/ProviderRegistry.test.ts b/apps/server/src/provider/ProviderRegistry.test.ts index 32e272fcd5a79..1bd7609237095 100644 --- a/apps/server/src/provider/ProviderRegistry.test.ts +++ b/apps/server/src/provider/ProviderRegistry.test.ts @@ -1363,7 +1363,7 @@ it.layer( ); }); - it("retains stale OpenCode models when a refresh fails", () => { + it("retains stale OpenCode models when a refresh fails or lists none", () => { const previousProvider = { instanceId: ProviderInstanceId.make("opencode"), driver: ProviderDriverKind.make("opencode"), @@ -1413,6 +1413,21 @@ it.layer( ProviderRegistry.mergeProviderSnapshot(previousProvider, refreshedProvider).skills, previousProvider.skills, ); + + const emptyCatalogProvider = { + ...refreshedProvider, + status: "warning", + message: "OpenCode 2 is running, but it did not list any models yet.", + slashCommands: [], + skills: [], + } satisfies ServerProvider; + const mergedEmptyCatalog = ProviderRegistry.mergeProviderSnapshot( + previousProvider, + emptyCatalogProvider, + ); + assert.deepStrictEqual(mergedEmptyCatalog.models, [...previousProvider.models]); + assert.deepStrictEqual(mergedEmptyCatalog.slashCommands, previousProvider.slashCommands); + assert.deepStrictEqual(mergedEmptyCatalog.skills, previousProvider.skills); }); describe("Pi model inventories", () => { @@ -4563,6 +4578,7 @@ it.layer( ); assert.strictEqual(status.status, "ready"); assert.strictEqual(status.auth.status, "authenticated"); + assert.strictEqual(status.auth.label, "Claude API Key"); }).pipe( Effect.provide( layerMockSpawner((args) => { diff --git a/apps/server/src/provider/ProviderRegistry.ts b/apps/server/src/provider/ProviderRegistry.ts index ba9ce9b2b6af7..8a7ff0ac7fd37 100644 --- a/apps/server/src/provider/ProviderRegistry.ts +++ b/apps/server/src/provider/ProviderRegistry.ts @@ -220,7 +220,8 @@ const shouldRetainMissingProviderModels = (provider: ServerProvider): boolean => const isAntigravity = provider.driver === ProviderDriverKind.make("antigravity"); const isCodex = provider.driver === ProviderDriverKind.make("codex"); - if (!isAntigravity && !isCodex && provider.driver !== ProviderDriverKind.make("opencode")) { + const isOpenCode = provider.driver === ProviderDriverKind.make("opencode"); + if (!isAntigravity && !isCodex && !isOpenCode) { return true; } @@ -236,9 +237,10 @@ const shouldRetainMissingProviderModels = (provider: ServerProvider): boolean => isAntigravity && provider.status === "warning" && provider.auth.status === "unknown"; const isPendingInitialProbe = provider.enabled && !provider.installed && provider.status === "warning"; + // A cold OpenCode server can finish its probe as a warning before it lists any models. const isIncompleteInstalledProbe = provider.installed && - (provider.status === "error" || (isCodex && provider.status === "warning")); + (provider.status === "error" || ((isCodex || isOpenCode) && provider.status === "warning")); return isPendingAntigravityAuthentication || isPendingInitialProbe || isIncompleteInstalledProbe; }; diff --git a/apps/server/src/provider/acp/AcpJsonRpcConnection.test.ts b/apps/server/src/provider/acp/AcpJsonRpcConnection.test.ts index f06c4f47570dd..4c21e400f3c8e 100644 --- a/apps/server/src/provider/acp/AcpJsonRpcConnection.test.ts +++ b/apps/server/src/provider/acp/AcpJsonRpcConnection.test.ts @@ -15,13 +15,16 @@ import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as Scope from "effect/Scope"; import * as Result from "effect/Result"; +import * as Scheduler from "effect/Scheduler"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; import { ChildProcessSpawner } from "effect/process"; import { describe, expect } from "vite-plus/test"; import * as AcpSessionRuntime from "@t3tools/provider-acp/server/AcpSessionRuntime"; import { AgentScope } from "@t3tools/shared/AgentScope"; import * as EffectAcpErrors from "effect-acp/errors"; +import type * as EffectAcpProtocol from "effect-acp/protocol"; const __dirname = NodePath.dirname(NodeURL.fileURLToPath(import.meta.url)); const mockAgentPath = NodePath.join(__dirname, "../../../scripts/acp-mock-agent.ts"); @@ -44,6 +47,136 @@ const setProcessEnv = (name: string, value: string) => { }; }; +const isPromptStarted = (event: AcpSessionRuntime.AcpSessionRequestLogEvent) => + event.method === "session/prompt" && event.status === "started"; + +const isOutgoing = (tag: string) => (event: EffectAcpProtocol.AcpProtocolLogEvent) => + event.direction === "outgoing" && + event.stage === "decoded" && + typeof event.payload === "object" && + event.payload !== null && + Reflect.get(event.payload, "tag") === tag; + +// The methods the mock agent read, in order, keeping only the given ones. +const readReceived = (requestLogPath: string, methods: ReadonlyArray) => + NodeFS.readFileSync(requestLogPath, "utf8") + .split("\n") + .filter((line) => line.length > 0) + .map((line) => (JSON.parse(line) as { readonly method?: string }).method) + .filter((method) => method !== undefined && methods.includes(method)); + +// Prompts, then cancels while the prompt is held on its way out: in its started request record, +// in its outgoing protocol log record, or by its fiber yielding every few operations. The cancel +// comes on dispatch or from inside the held record. Returns the prompt's exit +// and the order in which the agent read the prompt, the cancel and the request that ends it. +const promptThenCancel = (input: { + readonly cancelBehavior: "interrupt" | "wait-for-prompt"; + readonly holdPrompt: { + readonly in: "request-log" | "protocol-log" | "scheduler"; + readonly cancelOn: "dispatch" | "log"; + }; +}) => + Effect.gen(function* () { + const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "acp-runtime-")); + yield* Effect.addFinalizer(() => + Effect.sync(() => NodeFS.rmSync(tempDir, { recursive: true, force: true })), + ); + const requestLogPath = NodePath.join(tempDir, "requests.ndjson"); + const { holdPrompt } = input; + const promptHeld = yield* Deferred.make(); + const promptReleased = yield* Deferred.make(); + yield* Effect.addFinalizer(() => Deferred.succeed(promptReleased, undefined)); + const testScope = yield* Scope.Scope; + const runtimeMade = yield* Deferred.make(); + const cancelledFromLog = yield* Deferred.make>(); + const hold = ( + holdPrompt.cancelOn === "log" + ? Deferred.await(runtimeMade).pipe( + Effect.flatMap((made) => + made.cancel.pipe(Effect.forkIn(testScope, { startImmediately: true })), + ), + Effect.flatMap((fiber) => Deferred.succeed(cancelledFromLog, fiber)), + ) + : Effect.void + ).pipe( + Effect.andThen(Deferred.succeed(promptHeld, undefined)), + Effect.andThen(Deferred.await(promptReleased)), + ); + const runtime = yield* AcpSessionRuntime.make({ + ...mockRuntimeOptions, + spawn: { + ...mockRuntimeOptions.spawn, + env: { + T3_ACP_COMPLETE_FIRST_PROMPT_ON_CANCEL: "1", + T3_ACP_REQUEST_LOG_PATH: requestLogPath, + }, + }, + cancelBehavior: input.cancelBehavior, + interruptPromptOnCancel: false, + protocolLogging: { + logOutgoing: holdPrompt.in === "protocol-log", + logger: (event) => (isOutgoing("session/prompt")(event) ? hold : Effect.void), + }, + requestLogger: (event) => + holdPrompt.in === "request-log" && isPromptStarted(event) ? hold : Effect.void, + }); + yield* Deferred.succeed(runtimeMade, runtime); + const cancelReceived = yield* Deferred.make(); + yield* runtime.getEvents().pipe( + Stream.runForEach((event) => { + if (event._tag === "EventStreamBarrier") { + return Deferred.succeed(event.acknowledge, undefined); + } + if (event._tag === "ThoughtDelta" && event.text === "native-cancel-received") { + return Deferred.succeed(cancelReceived, undefined); + } + return Effect.void; + }), + Effect.forkChild, + ); + yield* runtime.start(); + const dispatched = yield* Deferred.make(); + const prompt = yield* runtime + .prompt({ prompt: [{ type: "text", text: "first" }] }, { dispatched }) + .pipe( + holdPrompt.in === "scheduler" + ? Effect.provideService(Scheduler.MaxOpsBeforeYield, 3) + : (effect) => effect, + Effect.forkChild, + ); + // A round trip: whatever was queued before it reaches the agent first. + const roundTrip = runtime.request("_test/environment", {}); + let cancel: Fiber.Fiber; + if (holdPrompt.in === "scheduler") { + yield* Deferred.await(dispatched); + cancel = yield* runtime.cancel.pipe(Effect.forkChild({ startImmediately: true })); + } else { + yield* Deferred.await(promptHeld); + if (holdPrompt.cancelOn === "dispatch") { + cancel = yield* Deferred.await(dispatched).pipe( + Effect.andThen(runtime.cancel), + Effect.forkChild({ startImmediately: true }), + ); + } else { + cancel = yield* Deferred.await(cancelledFromLog); + } + yield* roundTrip; + yield* Deferred.succeed(promptReleased, undefined); + } + yield* Deferred.await(cancelReceived); + yield* runtime.request("_test/finish-cancel", {}); + // The agent answers it only after reading every earlier line. + yield* roundTrip; + const received = readReceived(requestLogPath, [ + "session/prompt", + "session/cancel", + "_test/finish-cancel", + ]); + const promptExit = yield* Fiber.await(prompt); + yield* Fiber.join(cancel); + return { received, promptExit }; + }).pipe(Effect.scoped); + describe("AcpSessionRuntime", () => { it.effect.each(["session/new", "session/resume"] as const)( "buffers root metadata while %s startup is still pending", @@ -1239,4 +1372,104 @@ describe("AcpSessionRuntime", () => { Effect.ensuring(Effect.sync(() => NodeFS.rmSync(tempDir, { recursive: true, force: true }))), ); }); + + it.effect.each( + (["interrupt", "wait-for-prompt"] as const).flatMap((cancelBehavior) => + ( + [ + { in: "request-log", cancelOn: "dispatch" }, + { in: "protocol-log", cancelOn: "log" }, + { in: "scheduler", cancelOn: "dispatch" }, + ] as const + ).map((holdPrompt) => ({ cancelBehavior, holdPrompt })), + ), + )( + "sends session/prompt before a cancel issued while the prompt is held in the $holdPrompt.in ($cancelBehavior, cancel on $holdPrompt.cancelOn)", + ({ cancelBehavior, holdPrompt }) => + Effect.gen(function* () { + const run = yield* promptThenCancel({ cancelBehavior, holdPrompt }); + expect(run.received).toEqual(["session/prompt", "session/cancel", "_test/finish-cancel"]); + expect(run.promptExit).toEqual( + Exit.succeed({ stopReason: "cancelled", _meta: { nativeCancel: true } }), + ); + }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), + ); + + it.effect("sends a cancel after the cancel timeout when its prompt never goes out", () => + Effect.gen(function* () { + const runtime = yield* AcpSessionRuntime.make({ + ...mockRuntimeOptions, + interruptPromptOnCancel: false, + cancelTimeout: "5 seconds", + requestLogger: (event) => (isPromptStarted(event) ? Effect.never : Effect.void), + }); + yield* runtime.start(); + const dispatched = yield* Deferred.make(); + yield* runtime + .prompt({ prompt: [{ type: "text", text: "first" }] }, { dispatched }) + .pipe(Effect.forkChild); + yield* Deferred.await(dispatched); + const cancel = yield* runtime.cancel.pipe(Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust("5 seconds"); + yield* Fiber.join(cancel); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), + ); + + it.effect( + "sends a cancel before the next prompt when the prompt it waited for never went out", + () => + Effect.gen(function* () { + const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "acp-runtime-")); + yield* Effect.addFinalizer(() => + Effect.sync(() => NodeFS.rmSync(tempDir, { recursive: true, force: true })), + ); + const requestLogPath = NodePath.join(tempDir, "requests.ndjson"); + let promptsStarted = 0; + const runtime = yield* AcpSessionRuntime.make({ + ...mockRuntimeOptions, + spawn: { ...mockRuntimeOptions.spawn, env: { T3_ACP_REQUEST_LOG_PATH: requestLogPath } }, + interruptPromptOnCancel: false, + // The cancel spends a test-clock second in its outgoing record, the window in which the + // next prompt could overtake it. + protocolLogging: { + logOutgoing: true, + logger: (event) => + isOutgoing("session/cancel")(event) ? Effect.sleep("1 second") : Effect.void, + }, + requestLogger: (event) => + isPromptStarted(event) && promptsStarted++ === 0 ? Effect.never : Effect.void, + }); + yield* runtime.getEvents().pipe( + Stream.runForEach((event) => + event._tag === "EventStreamBarrier" + ? Deferred.succeed(event.acknowledge, undefined) + : Effect.void, + ), + Effect.forkChild, + ); + yield* runtime.start(); + const dispatched = yield* Deferred.make(); + const first = yield* runtime + .prompt({ prompt: [{ type: "text", text: "first" }] }, { dispatched }) + .pipe(Effect.forkChild); + yield* Deferred.await(dispatched); + const cancel = yield* runtime.cancel.pipe(Effect.forkChild({ startImmediately: true })); + const next = yield* runtime + .prompt({ prompt: [{ type: "text", text: "next" }] }) + .pipe(Effect.forkChild); + // The first prompt ends before it is queued, which releases the cancel's wait. + yield* Effect.sleep("1 second").pipe( + Effect.andThen(Fiber.interrupt(first)), + Effect.forkChild, + ); + yield* TestClock.adjust("1 second"); + yield* TestClock.adjust("1 second"); + yield* Fiber.join(cancel); + yield* Fiber.join(next); + expect(readReceived(requestLogPath, ["session/prompt", "session/cancel"])).toEqual([ + "session/cancel", + "session/prompt", + ]); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), + ); }); diff --git a/apps/server/src/provider/acp/AcpSessionRuntime.processTree.test.ts b/apps/server/src/provider/acp/AcpSessionRuntime.processTree.test.ts index 0c6bddbbf8567..6b9afa7f30c50 100644 --- a/apps/server/src/provider/acp/AcpSessionRuntime.processTree.test.ts +++ b/apps/server/src/provider/acp/AcpSessionRuntime.processTree.test.ts @@ -1,6 +1,7 @@ // @effect-diagnostics nodeBuiltinImport:off import * as NodeChildProcess from "node:child_process"; import * as NodeFS from "node:fs"; +import * as NodeModule from "node:module"; import * as NodePath from "node:path"; import { describe, expect, it, vi } from "@effect/vitest"; @@ -813,6 +814,33 @@ describe("terminatePosixOwnedProcessTree", () => { }), ); + it.skipIf(HostProcess.Platform.defaultValue() !== "linux")( + "snapshots /proc when a process exits while its entry is being typed", + () => { + const fs = NodeModule.createRequire(import.meta.url)("node:fs") as typeof NodeFS; + const realReaddirSync = fs.readdirSync; + // Node lstat()s a dirent of unknown type; a process that exits meanwhile makes it throw. + fs.readdirSync = ((path: NodeFS.PathLike, options?: unknown) => { + if (path === "/proc" && typeof options === "object" && options !== null) { + throw Object.assign(new Error("ENOENT: no such file or directory, lstat '/proc/4563'"), { + code: "ENOENT", + syscall: "lstat", + path: "/proc/4563", + }); + } + return Reflect.apply(realReaddirSync, fs, [path, options]); + }) as typeof NodeFS.readdirSync; + NodeModule.syncBuiltinESMExports(); + try { + const snapshot = AcpSessionRuntime.makePosixProcessTreeController("linux").snapshot(); + expect(snapshot.some((entry) => entry.pid === process.pid)).toBe(true); + } finally { + fs.readdirSync = realReaddirSync; + NodeModule.syncBuiltinESMExports(); + } + }, + ); + it("fails closed where a stable POSIX identity provider is unavailable", () => { try { AcpSessionRuntime.makePosixProcessTreeController("darwin"); diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index 0b2ec311f2af5..e731dac9d1966 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { "claudeSystemAppendBytes": 6477, "codexAdditionalContextBytes": 7871, - "mcpToolListJsonBytes": 51600 + "mcpToolListJsonBytes": 52772 } diff --git a/apps/server/src/provider/codexLaunchArgs.test.ts b/apps/server/src/provider/codexLaunchArgs.test.ts index 8f4e1b7ca6d6a..e74b29d44818c 100644 --- a/apps/server/src/provider/codexLaunchArgs.test.ts +++ b/apps/server/src/provider/codexLaunchArgs.test.ts @@ -38,6 +38,15 @@ describe("codexExecLaunchArgs", () => { ); }); + it("drops launch args that enable user hooks", () => { + NodeAssert.deepStrictEqual( + codexExecLaunchArgs( + "--enable hooks --enable=hooks --enable codex_hooks --enable=codex_hooks --enable foo --disable hooks", + ), + ["--enable", "foo", "--disable", "hooks"], + ); + }); + it("does not pair value-taking flags with adjacent flags", () => { NodeAssert.deepStrictEqual(codexExecLaunchArgs("--config --strict-config --enable --disable"), [ "--strict-config", diff --git a/apps/server/src/provider/codexLaunchArgs.ts b/apps/server/src/provider/codexLaunchArgs.ts index 31ff37e31deb5..e49314ee83e0d 100644 --- a/apps/server/src/provider/codexLaunchArgs.ts +++ b/apps/server/src/provider/codexLaunchArgs.ts @@ -62,8 +62,17 @@ const filterLaunchArgs = (launchArgs: string | undefined, allowed: LaunchArgFilt return kept; }; -export const codexExecLaunchArgs = (launchArgs?: string) => - filterLaunchArgs(launchArgs, EXEC_FILTER); +const HOOK_FEATURES = new Set(["hooks", "codex_hooks"]); + +// Feature toggles beat config overrides, so a launch arg's hook enable would undo ours. +export const codexExecLaunchArgs = (launchArgs?: string) => { + const args = filterLaunchArgs(launchArgs, EXEC_FILTER); + return args.filter((arg, index) => { + if (arg.startsWith("--enable=")) return !HOOK_FEATURES.has(arg.slice("--enable=".length)); + if (arg === "--enable") return !HOOK_FEATURES.has(args[index + 1] ?? ""); + return !(args[index - 1] === "--enable" && HOOK_FEATURES.has(arg)); + }); +}; export const codexMcpListLaunchArgs = (launchArgs?: string) => filterLaunchArgs(launchArgs, MCP_LIST_FILTER); diff --git a/apps/server/src/provider/codexUsageLimits.test.ts b/apps/server/src/provider/codexUsageLimits.test.ts index d86a9a714f3fc..b3041cbe8547b 100644 --- a/apps/server/src/provider/codexUsageLimits.test.ts +++ b/apps/server/src/provider/codexUsageLimits.test.ts @@ -1,12 +1,15 @@ +import * as DateTime from "effect/DateTime"; import { ProviderDriverKind, ProviderInstanceId } from "@t3tools/contracts"; import { exhaustedUsageWindow } from "@t3tools/shared/usageLimits"; import * as CodexErrors from "effect-codex-app-server/errors"; import { describe, expect, it } from "vite-plus/test"; import { + codexMessageResetAt, codexRateLimitsFailureMessage, codexRateLimitsToLimits, codexRateLimitsToUpdate, + codexTimeZone, mergeCodexRateLimits, } from "./codexUsageLimits.ts"; @@ -325,3 +328,61 @@ describe("codexUsageLimitResetAt", () => { ).toBeNull(); }); }); + +describe("codexMessageResetAt", () => { + const dhaka = DateTime.zoneMakeNamedUnsafe("Asia/Dhaka"); + const at = (iso: string) => DateTime.makeUnsafe(iso); + const failedAt = at("2026-11-07T03:00:00Z"); // 09:00 in Dhaka + + // Codex prints local time without seconds, so the parsed reset is the minute after. + it("reads Codex's same-day and dated reset text in Codex's zone", () => { + expect( + codexMessageResetAt("You’ve hit your usage limit. Try again at 3:37 PM.", failedAt, dhaka), + ).toBe("2026-11-07T09:38:00.000Z"); + expect( + codexMessageResetAt("Upgrade, or try again at Dec 1st, 2026 12:05 AM.", failedAt, dhaka), + ).toBe("2026-11-30T18:06:00.000Z"); + expect(codexMessageResetAt("Try again at 12:30 PM.", failedAt, dhaka)).toBe( + "2026-11-07T06:31:00.000Z", + ); + }); + + it("dates today's form by when Codex failed, across local midnight", () => { + expect(codexMessageResetAt("Try again at 11:59 PM.", at("2026-11-07T17:59:59Z"), dhaka)).toBe( + "2026-11-07T18:00:00.000Z", + ); + }); + + it("takes the later instant of a repeated DST hour", () => { + // 1:30 AM occurs twice in New York on Nov 1st, 2026: 05:30Z (EDT) and 06:30Z (EST). + const newYork = DateTime.zoneMakeNamedUnsafe("America/New_York"); + for (const iso of ["2026-11-01T05:15:00Z", "2026-11-01T06:15:00Z"]) { + expect(codexMessageResetAt("Try again at 1:30 AM.", at(iso), newYork)).toBe( + "2026-11-01T06:31:00.000Z", + ); + } + }); + + it("leaves the reset unknown without reset text or once it has passed", () => { + expect( + codexMessageResetAt("You’ve hit your usage limit. Try again later.", failedAt, dhaka), + ).toBeNull(); + expect( + codexMessageResetAt( + "exceeded retry limit, last status: 429 Too Many Requests", + failedAt, + dhaka, + ), + ).toBeNull(); + expect(codexMessageResetAt("Try again at 8:15 AM.", failedAt, dhaka)).toBeNull(); + }); +}); + +describe("codexTimeZone", () => { + it("uses Codex's TZ and falls back to the server's zone", () => { + const local = DateTime.zoneToString(DateTime.zoneMakeLocal()); + expect(DateTime.zoneToString(codexTimeZone({ TZ: ":Asia/Dhaka" }))).toBe("Asia/Dhaka"); + expect(DateTime.zoneToString(codexTimeZone({ TZ: "not/a-zone" }))).toBe(local); + expect(DateTime.zoneToString(codexTimeZone({}))).toBe(local); + }); +}); diff --git a/apps/server/src/provider/codexUsageLimits.ts b/apps/server/src/provider/codexUsageLimits.ts index bc7af1ed378ac..286153519d800 100644 --- a/apps/server/src/provider/codexUsageLimits.ts +++ b/apps/server/src/provider/codexUsageLimits.ts @@ -235,3 +235,51 @@ export function codexUsageLimitResetAt( null, ); } + +const CODEX_RESET_TEXT = + /[Tt]ry again at (?:([A-Z][a-z]{2}) (\d{1,2})(?:st|nd|rd|th), (\d{4}) )?(\d{1,2}):(\d{2}) ([AP]M)\./; +const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; + +/** The zone Codex prints reset times in: its own `TZ`, else the server's. */ +export function codexTimeZone( + environment: Readonly>, +): DateTime.TimeZone { + return Option.getOrElse( + DateTime.zoneMakeNamed(environment.TZ?.replace(/^:/, "") ?? ""), + DateTime.zoneMakeLocal, + ); +} + +/** + * Codex writes a usage limit's reset into its error text in its own local time: + * "Try again at 3:37 PM." for today, else "try again at Nov 11th, 2026 3:37 PM.". + * It is the only reset left when a proxy such as CLIProxyAPI strips the rate-limit snapshot. + * `failedAt` is when Codex reported the failure, which fixes the date of today's form. + */ +export function codexMessageResetAt( + message: string, + failedAt: DateTime.DateTime, + zone: DateTime.TimeZone, +): string | null { + const match = CODEX_RESET_TEXT.exec(message); + if (!match) return null; + const [, month, day, year, hour, minute, meridiem] = match; + const monthIndex = month === undefined ? undefined : MONTHS.indexOf(month); + if (monthIndex === -1) return null; + const today = DateTime.toParts(DateTime.setZone(failedAt, zone)); + const reset = DateTime.makeZoned( + { + year: monthIndex === undefined ? today.year : Number(year), + month: monthIndex === undefined ? today.month : monthIndex + 1, + day: monthIndex === undefined ? today.day : Number(day), + hour: (Number(hour) % 12) + (meridiem === "PM" ? 12 : 0), + minute: Number(minute), + }, + // The repeated hour at a DST fall-back takes its later instant so a resume never fires early. + { timeZone: zone, adjustForTimeZone: true, disambiguation: "later" }, + ); + if (Option.isNone(reset)) return null; + // Codex drops the seconds, so the next minute is the first one past the reset. + const resetAt = DateTime.add(reset.value, { minutes: 1 }); + return DateTime.isGreaterThan(resetAt, failedAt) ? DateTime.formatIso(resetAt) : null; +} diff --git a/apps/server/src/provider/documentTextWorker.ts b/apps/server/src/provider/documentTextWorker.ts index 81008be757dac..5957fdd3cd892 100644 --- a/apps/server/src/provider/documentTextWorker.ts +++ b/apps/server/src/provider/documentTextWorker.ts @@ -14,9 +14,10 @@ const CELL_END = Symbol("cell-end"); type TextPart = string | typeof CELL_START | typeof CELL_END | { fallback: string }; const TRUNCATED = "\n\n[Extracted text truncated at document extraction limits.]"; -// Parts the document never displays: tracked deletions and moves, field codes, phonetic guides, -// paragraph properties such as tab-stop definitions. +// Parts the document never displays: the XHTML head, tracked deletions and moves, field codes, +// phonetic guides, paragraph properties such as tab-stop definitions. const HIDDEN = new Set([ + "head", "script", "style", "binary-data", @@ -41,12 +42,43 @@ interface XmlNode { hasTextChoice?: boolean; } -function xml(source: string, odfSpaces = false, htmlEntities = false): XmlNode { +// ODF 1.3 Part 3 §6.1.2: paragraph character data turns TAB, CR and LF into spaces, collapses +// space runs and trims the ends; only , and keep literal ones. +function collapseOdfWhitespace(paragraph: XmlNode) { + let afterSpace = true; + let tail: { node: XmlNode; index: number; value: string } | undefined; + const visit = (node: XmlNode) => { + for (const [index, child] of node.children.entries()) { + if (typeof child !== "string") { + if (HIDDEN.has(child.name)) continue; + if (child.spaceCount === undefined && !["tab", "line-break", "p", "h"].includes(child.name)) + visit(child); + else { + afterSpace = false; + tail = undefined; + } + continue; + } + let value = child.replaceAll(/[ \t\r\n]+/g, " "); + if (afterSpace && value.startsWith(" ")) value = value.slice(1); + if (value) { + afterSpace = value.endsWith(" "); + tail = { node, index, value }; + } + node.children[index] = value; + } + }; + visit(paragraph); + if (tail && afterSpace) tail.node.children[tail.index] = tail.value.slice(0, -1); +} + +function xml(source: string, extension: string): XmlNode { + const odf = [".odt", ".ods", ".odp"].includes(extension); const root: XmlNode = { name: "root", attrs: {}, children: [] }; const stack = [root]; let nodes = 0; const parser = new SaxesParser(); - if (htmlEntities) Object.assign(parser.ENTITIES, characterEntities); + if (extension === ".epub") Object.assign(parser.ENTITIES, characterEntities); parser.on("opentag", (tag) => { if (++nodes > 200_000 || stack.length > 128) throw new Truncation("XML structure limit"); const node: XmlNode = { @@ -59,20 +91,25 @@ function xml(source: string, odfSpaces = false, htmlEntities = false): XmlNode { ), children: [], }; - if (odfSpaces && node.name === "s") node.spaceCount = repeatCount(node.attrs.c); + if (odf && node.name === "s") node.spaceCount = repeatCount(node.attrs.c); stack.at(-1)!.children.push(node); stack.push(node); }); parser.on("closetag", () => { const node = stack.pop()!; const parent = stack.at(-1)!; + if (odf && (node.name === "p" || node.name === "h")) collapseOdfWhitespace(node); + if (extension === ".docx" && node.name === "del") { + parent.children.pop(); + return; + } // Markup-compatibility fallbacks repeat their mc:Choice content, e.g. Word text boxes. // Keep one when no Choice carries text, such as a chart with a VML caption. if (node.name === "Fallback" && parent.hasTextChoice) { parent.children.pop(); return; } - if (!node.hasText) return; + if (!node.hasText && node.name !== "noBreakHyphen") return; parent.hasText = true; if (node.name === "Choice") parent.hasTextChoice = true; }); @@ -99,6 +136,7 @@ function* text(node: XmlNode): Generator { if (HIDDEN.has(node.name)) return; if (["tab", "cell"].includes(node.name)) yield "\t"; else if (["br", "line-break", "cr"].includes(node.name)) yield "\n"; + else if (node.name === "noBreakHyphen") yield "\u2011"; else if (node.spaceCount !== undefined) { yield " ".repeat(Math.min(MAX_OUTPUT, node.spaceCount)); if (node.spaceCount > MAX_OUTPUT) throw new Truncation("Space repeat limit"); @@ -108,6 +146,23 @@ function* text(node: XmlNode): Generator { else yield* text(child); } } +// SpreadsheetML `t` and `v` values are ST_Xstring: `_xHHHH_` is one UTF-16 code unit, and an +// escaped lead underscore (`_x005F_xHHHH_`) keeps the literal `_xHHHH_`. Decoded in one pass. +function* xstringText(node: XmlNode): Generator { + if (HIDDEN.has(node.name)) return; + if (node.name !== "t" && node.name !== "v") { + for (const child of children(node)) yield* xstringText(child); + return; + } + yield node.children + .filter((child) => typeof child === "string") + .join("") + .replaceAll( + /_x005[Ff]_(x[\dA-Fa-f]{4}_)|_x([\dA-Fa-f]{4})_/g, + (_, literal?: string, hex?: string) => + literal ? `_${literal}` : String.fromCharCode(Number.parseInt(hex!, 16)), + ); +} function* paragraphs(node: XmlNode): Generator { if (HIDDEN.has(node.name)) return; for (const child of node.children) { @@ -164,7 +219,7 @@ function odfValue(cell: XmlNode): string { } } function hasCellText(cell: XmlNode): boolean { - if (odfValue(cell).trim()) return true; + if (odfValue(cell).trim() || cell.name === "noBreakHyphen") return true; // Space markers can have enormous repeat counts; emptiness never requires expanding them. if (HIDDEN.has(cell.name) || cell.spaceCount !== undefined) return false; return cell.children.some((child) => @@ -342,6 +397,10 @@ function* rtf(source: string): Generator { ["ldblquote", "“"], ["rdblquote", "”"], ["bullet", "•"], + ["zwj", "\u200d"], + ["zwnj", "\u200c"], + ["ltrmark", "\u200e"], + ["rtlmark", "\u200f"], ["~", "\u00a0"], ["_", "\u2011"], ["-", ""], @@ -437,7 +496,7 @@ async function* document(path: string, extension: string): AsyncGenerator typeof child === "string") ?? ""; if (cell.attrs.t === "s") { const shared = strings[Number(raw)]; - if (shared) yield* inCell(text(shared)); - } else if (cell.attrs.t === "inlineStr") yield* inCell(text(cell)); + if (shared) yield* inCell(xstringText(shared)); + } else if (cell.attrs.t === "inlineStr") yield* inCell(xstringText(cell)); + else if (cell.attrs.t === "b" && (raw.trim() === "0" || raw.trim() === "1")) + yield* inCell([raw.trim() === "1" ? "true" : "false"]); else { - if (value && hasCellText(value)) yield* inCell(text(value)); + if (value && hasCellText(value)) yield* inCell(xstringText(value)); else for (const formula of all(cell, "f")) yield* inCell(text(formula)); const formatId = formats[Number(cell.attrs.s ?? 0)]?.attrs.numFmtId; const format = formatId ? customFormats.get(formatId) : undefined; diff --git a/apps/server/src/provider/model-manifest.json b/apps/server/src/provider/model-manifest.json index 4906358bab70b..01727cc2680b0 100644 --- a/apps/server/src/provider/model-manifest.json +++ b/apps/server/src/provider/model-manifest.json @@ -1,6 +1,6 @@ { "version": 1, - "updatedAt": "2026-11-12T18:25:36Z", + "updatedAt": "2026-11-14T15:58:47Z", "compatibility": [ { "driver": "codex", @@ -95,8 +95,8 @@ "t3CodeRange": ">=0.0.46", "recommendedRange": ">=1.0.0", "ranges": [ - { "range": ">=0.80.5", "status": "supported" }, - { "range": "<0.80.5", "status": "unsupported" } + { "range": ">=0.84.3", "status": "supported" }, + { "range": "<0.84.3", "status": "unsupported" } ] }, { @@ -771,7 +771,7 @@ "aliases": ["fable", "fable-5.2", "claude-fable-5.2"], "status": "current", "profile": "fable-5", - "adapter": { "claudeCode": { "minVersion": "2.1.290" } } + "adapter": { "claudeCode": { "minVersion": "2.1.321" } } }, { "slug": "claude-fable-5-1", diff --git a/apps/server/src/provider/testFixtures/grok-text-mock-agent.mjs b/apps/server/src/provider/testFixtures/grok-text-mock-agent.mjs index 357bdb7fe564d..b8cb949a5fb71 100644 --- a/apps/server/src/provider/testFixtures/grok-text-mock-agent.mjs +++ b/apps/server/src/provider/testFixtures/grok-text-mock-agent.mjs @@ -5,12 +5,35 @@ const sessionId = "grok-text-session"; let currentModelId = "grok-mock-default"; const models = ["grok-mock-default", "grok-mock-alt"]; const write = (message) => process.stdout.write(`${JSON.stringify(message)}\n`); +const permissionRequestId = 9001; +let pendingPromptId; +const completePrompt = (id) => { + write({ + jsonrpc: "2.0", + method: "session/update", + params: { + sessionId, + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: process.env.T3_ACP_PROMPT_RESPONSE_TEXT ?? "" }, + }, + }, + }); + write({ jsonrpc: "2.0", id, result: { stopReason: "end_turn" } }); +}; for await (const line of NodeReadline.createInterface({ input: process.stdin })) { const request = JSON.parse(line); if (process.env.T3_ACP_REQUEST_LOG_PATH) { NodeFS.appendFileSync(process.env.T3_ACP_REQUEST_LOG_PATH, `${line}\n`); } + if (request.method === undefined) { + if (request.id === permissionRequestId && pendingPromptId !== undefined) { + completePrompt(pendingPromptId); + pendingPromptId = undefined; + } + continue; + } if (request.id === undefined) continue; let result; switch (request.method) { @@ -43,19 +66,25 @@ for await (const line of NodeReadline.createInterface({ input: process.stdin })) result = {}; break; case "session/prompt": - write({ - jsonrpc: "2.0", - method: "session/update", - params: { - sessionId, - update: { - sessionUpdate: "agent_message_chunk", - content: { type: "text", text: process.env.T3_ACP_PROMPT_RESPONSE_TEXT ?? "" }, + if (process.env.T3_ACP_REQUEST_PERMISSION) { + pendingPromptId = request.id; + write({ + jsonrpc: "2.0", + id: permissionRequestId, + method: "session/request_permission", + params: { + sessionId, + toolCall: { toolCallId: "tool-1", kind: "execute", title: "Run `git status`" }, + options: [ + { optionId: "allow-once", name: "Yes", kind: "allow_once" }, + { optionId: "reject-once", name: "No", kind: "reject_once" }, + ], }, - }, - }); - result = { stopReason: "end_turn" }; - break; + }); + continue; + } + completePrompt(request.id); + continue; default: write({ jsonrpc: "2.0", diff --git a/apps/server/src/provider/userInputAttachments.test.ts b/apps/server/src/provider/userInputAttachments.test.ts index 075886dd7b109..8dfafcbecb44c 100644 --- a/apps/server/src/provider/userInputAttachments.test.ts +++ b/apps/server/src/provider/userInputAttachments.test.ts @@ -285,6 +285,11 @@ describe("question answer paths", () => { String.raw`{\rtf1\ansi A\enspace B\emspace C\qmspace D\~E\_F\-G}`, "A\u2002B\u2003C\u2005D\u00a0E\u2011FG", ], + [ + "joiner and direction marks", + String.raw`{\rtf1\ansi A\zwj B\zwnj C\ltrmark D\rtlmark E}`, + "A\u200dB\u200cC\u200eD\u200fE", + ], [ "text symbol fallback", String.raw`{\rtf1\ansi\uc10\u945\endash\emdash\enspace\emspace\qmspace\lquote\rquote\ldblquote\rdblquote\bullet Z}`, @@ -690,28 +695,32 @@ describe("question answer paths", () => { bounded: true, parts: { "word/document.xml": - 'Report & totalsNorth1234BeforeInnerAfterNeighbor', + 'Report & totalsNorth1234BeforeInnerAfterNeighbor', }, - expected: ["## Report & totals", "North\t12 34", "Before Inner After\tNeighbor"], + expected: [ + "## Report & totals", + "North\t12 34", + "Before Inner After\tNeighbor\t\u2011\n\u2011", + ], }, { ext: "docx", bounded: true, parts: { "word/document.xml": - 'FirstSecondThird', + 'FirstSecondThirdparty‑End', }, - expected: ["First\nSecond\nThird"], + expected: ["First\nSecond\nThird\u2011party\u2011End"], }, { ext: "docx", bounded: true, parts: { "word/document.xml": - 'Intro PAGEREF _Toc1 \\h 3Fee $500$700Moveddue', + 'Intro PAGEREF _Toc1 \\h 3Fee $500$700Moveddue', }, expected: ["Intro 3\nFee $700\tdue"], - absent: ["PAGEREF", "$500", "Moved"], + absent: ["PAGEREF", "$500", "Moved", "\u2011"], }, { ext: "odt", @@ -748,6 +757,18 @@ describe("question answer paths", () => { ], absent: ["SUM(0,0)", "[Extracted text truncated"], }, + { + ext: "xlsx", + bounded: true, + parts: { + "xl/workbook.xml": '', + "xl/_rels/workbook.xml.rels": + '', + "xl/worksheets/a.xml": + '1012', + }, + expected: ["true\tfalse\t1\t2"], + }, { ext: "xlsx", parts: { @@ -788,6 +809,22 @@ describe("question answer paths", () => { expected: ["## Sheet: Phonetics", "東京\t東京"], absent: ["とうきょう"], }, + { + ext: "xlsx", + bounded: true, + parts: { + "xl/workbook.xml": '', + "xl/_rels/workbook.xml.rels": + '', + "xl/sharedStrings.xml": + "first_x000D_\nsecond_x005F_x0041__x005F_x005F_x0041_Bold_x000D__x005F_x000D_&_xnothex_ cap_X0041_ _x000d_ tail", + "xl/worksheets/a.xml": + '01234_x005F_x005F_first_x000D_secondA1Total_x000D_due', + }, + expected: [ + "## Sheet: Escapes\nfirst second\t_x0041_\t_x005F_x0041_\tBold _x000D_\t&_xnothex_ cap_X0041_ tail\n_x005F_\tfirst second\tTotal due", + ], + }, { ext: "xlsx", bounded: true, @@ -891,9 +928,11 @@ describe("question answer paths", () => { ext: "odt", parts: { "content.xml": - 'Orchard19violetcrates', + '\n Orchard\n19violetcrates \n\t shipped todayDockGate\u00a0B SecondAconsole.log(1);\n B', }, - expected: ["## Orchard", "19 violet crates"], + expected: [ + "## Orchard\n19 violet crates shipped today\tDock\nGate\u00a0B\nSecond\nA B", + ], }, { ext: "ods", @@ -940,9 +979,9 @@ describe("question answer paths", () => { ext: "odp", parts: { "content.xml": - "FirstSecond", + "\n First\n\t slide Second", }, - expected: ["## Slide 1", "First", "## Slide 2", "Second"], + expected: ["## Slide 1\nFirst slide\n\n## Slide 2\nSecond"], }, { ext: "ods", @@ -1037,13 +1076,13 @@ describe("question answer paths", () => { "OEBPS/book.opf": '', "OEBPS/a.xhtml": - '

Last chapter

Price 10 8

', + '

Last chapter

Price 10 8 old

', "OEBPS/b.xhtml": - "

First chapter

Larkspur

Direct div
Direct section
  • Direct item emphasis trailing
  • Account: A
    Balance: 731Currency: USD
    HeadingTotal
    Important value731
    Before
    Inner
    After
    Neighbor
    ", + 'My Novel

    First chapter

    Larkspur

    Direct div
    Direct section
  • Direct item emphasis trailing
  • Account: A
    Balance: 731Currency: USD
    HeadingTotal
    Important value731
    Before
    Inner
    After
    Neighbor
    ', }, expected: [ "## Chapter 1", - "# First chapter", + "\n# First chapter", "Larkspur", "Direct div", "Direct section", @@ -1054,8 +1093,9 @@ describe("question answer paths", () => { "Before Inner After\tNeighbor", "## Chapter 2", "Last chapter", - "Price 10 8", + "Price 10 8 old", ], + absent: ["My Novel", "Hidden Author"], }, ]; for (const [caseIndex, item] of cases.entries()) { diff --git a/apps/server/src/pullRequest/PullRequestService.test.ts b/apps/server/src/pullRequest/PullRequestService.test.ts index 773d0c6b6adee..905c96319d52a 100644 --- a/apps/server/src/pullRequest/PullRequestService.test.ts +++ b/apps/server/src/pullRequest/PullRequestService.test.ts @@ -3017,7 +3017,13 @@ it.effect("carries a slice on without sending the rows it already sent", () => cursors: { "github.com acme/web": "2026-07-02T00:00:00Z|1|3" }, }); - assert.deepStrictEqual(cursors, [{ updatedBefore: "2026-07-02T00:00:00Z", delivered: 1 }]); + assert.deepStrictEqual(cursors, [ + { + updatedBefore: "2026-07-02T00:00:00Z", + delivered: 1, + seenAt: [{ repository: "acme/web", number: 3 }], + }, + ]); assert.deepStrictEqual( result.entries.map((entry) => entry.number), [4], @@ -3243,6 +3249,40 @@ it.effect("reads the fresh diff when detail or summary discovers a changed revis }), ); +it.effect("keeps a withheld file's read out of the page's cached diff", () => + Effect.gen(function* () { + const provider = fakePullRequestProvider(SourceControlProviderKind.make("github"), { + getDiff: (input) => + Effect.succeed({ + patch: input.path === undefined ? "page" : `file ${input.path}`, + truncated: input.path === undefined, + nextCursor: null, + }), + }); + const service = yield* makeService({ + projects: [project({ id: "p1", title: "web", workspaceRoot: "/a", repository: "acme/web" })], + providers: [provider], + }); + const ref = { projectId: "p1" as ProjectId, repository: "acme/web", number: 10 }; + assert.strictEqual((yield* service.diff(ref)).patch, "page"); + assert.strictEqual( + (yield* service.diff({ ...ref, path: "sdk.patch" })).patch, + "file sdk.patch", + ); + assert.strictEqual((yield* service.diff(ref)).patch, "page"); + + // A host that would answer with its whole diff must not be cached as one file's. + const wholeDiffOnly = yield* makeService({ + projects: [project({ id: "p1", title: "web", workspaceRoot: "/a", repository: "acme/web" })], + providers: [ + { ...provider, capabilities: { ...provider.capabilities, withheldFileDiff: false } }, + ], + }); + const error = yield* Effect.flip(wholeDiffOnly.diff({ ...ref, path: "sdk.patch" })); + assert.strictEqual(error._tag, "PullRequestOperationError"); + }), +); + it.effect("keeps routed reads separate when the GitHub account changes", () => Effect.gen(function* () { for (const operation of ["summary", "detail", "checks", "diff", "filesViewed"] as const) { @@ -3452,6 +3492,7 @@ it.effect( Effect.succeed({ ...changeRequest(1, "2026-07-02T00:00:00Z"), isDraft: true, + rebaseRequired: false, reviewDecision: "approved", checksState: "passing", }), @@ -3461,6 +3502,7 @@ it.effect( deletions: 3, changedFiles: 5, mergeability: "conflicting", + rebaseRequired: true, }), }), ], @@ -3468,6 +3510,8 @@ it.effect( yield* service.summary(reference); const detail = yield* service.detail(reference); const summary = yield* service.summary(reference); + assert.strictEqual(detail.rebaseRequired, true); + assert.strictEqual(summary.rebaseRequired, true); assert.strictEqual(summary.isDraft, false); assert.deepStrictEqual(summary.author, detail.author); assert.strictEqual(summary.additions, 14); @@ -3568,19 +3612,31 @@ it.effect("deduplicates routed PR reports without dropping draft changes", () => }), ); -it.effect( - "reuses fresh detail checks without extending their refresh interval on detail cache hits", - () => +it.effect.each(["github", "azure-devops"])( + "reuses fresh %s detail checks without extending their refresh interval on detail cache hits", + (provider) => Effect.gen(function* () { let reads = 0; - const reference = { projectId: "p1" as ProjectId, repository: "acme/web", number: 1 }; + const repository = provider === "azure-devops" ? "web" : "acme/web"; + const reference = { projectId: "p1" as ProjectId, repository, number: 1 }; const service = yield* makeService({ projects: [ - project({ id: "p1", title: "web", workspaceRoot: "/a", repository: "acme/web" }), + project({ + id: "p1", + title: "web", + workspaceRoot: "/a", + repository, + provider, + ...(provider === "azure-devops" ? { host: "dev.azure.com" } : {}), + }), ], providers: [ - fakePullRequestProvider(SourceControlProviderKind.make("github"), { - getChangeRequest: () => Effect.succeed(hostedChangeRequest("Description")), + fakePullRequestProvider(SourceControlProviderKind.make(provider), { + getChangeRequest: () => + Effect.succeed({ + ...hostedChangeRequest("Description"), + checks: [{ name: "build", status: "success", description: null, url: null }], + }), getChangeRequestChecks: () => Effect.sync(() => { reads += 1; @@ -3610,6 +3666,37 @@ it.effect( }), ); +it.effect("asks Azure for checks when its detail came back with none", () => + Effect.gen(function* () { + let reads = 0; + const reference = { projectId: "p1" as ProjectId, repository: "web", number: 1 }; + const service = yield* makeService({ + projects: [ + project({ + id: "p1", + title: "web", + workspaceRoot: "/a", + repository: "web", + provider: "azure-devops", + }), + ], + providers: [ + fakePullRequestProvider(SourceControlProviderKind.make("azure-devops"), { + getChangeRequest: () => Effect.succeed(hostedChangeRequest("Description")), + getChangeRequestChecks: () => + Effect.sync(() => { + reads += 1; + return { state: "open" as const, checks: [] }; + }), + }), + ], + }); + yield* service.detail(reference); + yield* service.checks(reference); + expect(reads).toBe(1); + }), +); + it.effect("preserves check observation time on cache hits and advances it on refresh", () => Effect.gen(function* () { let reads = 0; @@ -4376,6 +4463,43 @@ it.effect("keeps viewed files itself for a host that keeps none of its own", () }), ); +it.effect("records shown revisions until the reader ticks the pushed version", () => + Effect.gen(function* () { + const asked: Array> = []; + const revisions = new Map([ + ["src/a.ts", "pushed-a"], + ["src/b.ts", "current-b"], + ]); + const service = yield* environmentViewedService(revisions, asked); + + yield* service.setFilesViewed({ + ...GITLAB_REFERENCE, + files: [ + { path: "src/a.ts", viewed: true, revision: "shown-a" }, + { path: "src/b.ts", viewed: true }, + ], + }); + assert.deepStrictEqual(asked, [["src/b.ts"]]); + yield* service.invalidate({ reference: GITLAB_REFERENCE }); + assert.deepStrictEqual((yield* service.filesViewed(GITLAB_REFERENCE)).files, [ + { path: "src/a.ts", state: "dismissed" }, + { path: "src/b.ts", state: "viewed" }, + ]); + + const reads = asked.length; + revisions.set("src/a.ts", "pushed-a-again"); + yield* service.setFilesViewed({ + ...GITLAB_REFERENCE, + files: [{ path: "src/a.ts", viewed: true, revision: "pushed-a-again" }], + }); + assert.strictEqual(asked.length, reads); + assert.deepStrictEqual((yield* service.filesViewed(GITLAB_REFERENCE)).files, [ + { path: "src/a.ts", state: "viewed" }, + { path: "src/b.ts", state: "viewed" }, + ]); + }), +); + it.effect("keeps a deleted file cleared, which the head has no version of at all", () => Effect.gen(function* () { const service = yield* environmentViewedService(new Map(), []); diff --git a/apps/server/src/pullRequest/PullRequestService.ts b/apps/server/src/pullRequest/PullRequestService.ts index 2f79c4b524eeb..74da9bacee95d 100644 --- a/apps/server/src/pullRequest/PullRequestService.ts +++ b/apps/server/src/pullRequest/PullRequestService.ts @@ -1309,6 +1309,9 @@ export const make = Effect.gen(function* () { state: input.item.state, isDraft: input.item.isDraft, mergeability: input.item.mergeability, + ...(input.item.rebaseRequired === undefined + ? {} + : { rebaseRequired: input.item.rebaseRequired }), additions: input.item.additions, deletions: input.item.deletions, createdAt: input.item.createdAt, @@ -1550,7 +1553,18 @@ export const make = Effect.gen(function* () { filters: input.filters, ...(cursor === undefined ? {} - : { cursor: { updatedBefore: cursor.updatedBefore, delivered: cursor.delivered } }), + : { + cursor: { + updatedBefore: cursor.updatedBefore, + delivered: cursor.delivered, + seenAt: chunk.flatMap((project) => + (cursorOf(project)?.seenAt ?? []).map((number) => ({ + repository: project.repository, + number, + })), + ), + }, + }), }).pipe( observeRead, Effect.flatMap(({ value: page, observedAt }) => { @@ -1781,6 +1795,9 @@ export const make = Effect.gen(function* () { ...(changeRequest.mergeability === undefined ? {} : { mergeability: changeRequest.mergeability }), + ...(changeRequest.rebaseRequired === undefined + ? {} + : { rebaseRequired: changeRequest.rebaseRequired }), ...(changeRequest.stack === undefined ? {} : { stack: changeRequest.stack }), ...(changeRequest.headSha?.trim() ? { headSha: changeRequest.headSha.trim() } : {}), ...(changeRequest.autoMergeEnabled === undefined @@ -1880,6 +1897,9 @@ export const make = Effect.gen(function* () { state: changeRequest.state, isDraft: changeRequest.isDraft, mergeability: changeRequest.mergeability, + ...(changeRequest.rebaseRequired === undefined + ? {} + : { rebaseRequired: changeRequest.rebaseRequired }), additions: changeRequest.additions, deletions: changeRequest.deletions, changedFiles: changeRequest.changedFiles, @@ -2016,23 +2036,31 @@ export const make = Effect.gen(function* () { const diffUncached: PullRequestService["Service"]["diff"] = (input) => requireProject(input).pipe( Effect.flatMap((project) => - project.api.capabilities.diff - ? project.api - .getDiff({ - cwd: project.project.workspaceRoot, - repository: project.repository, - host: project.host, - number: input.number, - ...(input.cursor === undefined ? {} : { cursor: input.cursor }), - ...(input.commit === undefined ? {} : { commit: input.commit }), - }) - .pipe(Effect.mapError(toPullRequestError("diff"))) - : Effect.fail( + input.path !== undefined && project.api.capabilities.withheldFileDiff !== true + ? Effect.fail( new PullRequestOperationError({ operation: "diff", - detail: "This host cannot provide a diff for a change request.", + detail: "This host cannot read one file of a change request's diff.", }), - ), + ) + : project.api.capabilities.diff + ? project.api + .getDiff({ + cwd: project.project.workspaceRoot, + repository: project.repository, + host: project.host, + number: input.number, + ...(input.path === undefined ? {} : { path: input.path }), + ...(input.cursor === undefined ? {} : { cursor: input.cursor }), + ...(input.commit === undefined ? {} : { commit: input.commit }), + }) + .pipe(Effect.mapError(toPullRequestError("diff"))) + : Effect.fail( + new PullRequestOperationError({ + operation: "diff", + detail: "This host cannot provide a diff for a change request.", + }), + ), ), ); @@ -2085,7 +2113,14 @@ export const make = Effect.gen(function* () { // Deliberately not `invalidatedByMutation`: ticking a file off says nothing about the // change request, and dropping a 300-file diff on every checkbox is the whole cost of // the feature. - Effect.ensuring(Effect.sync(() => bumpFilesViewedEpoch(ref))), + Effect.ensuring( + Effect.sync(() => { + bumpFilesViewedEpoch(ref); + if (input.files.some((file) => file.viewed && file.revision !== undefined)) { + bumpEpoch(fileRevisionEpochs, ref); + } + }), + ), ), ), ); @@ -3001,6 +3036,7 @@ export const make = Effect.gen(function* () { const filesViewedEpoch = (ref: PullRequestRef) => filesViewedEpochs.get(refScope(ref)) ?? 0; const bumpFilesViewedEpoch = (ref: PullRequestRef) => bumpEpoch(filesViewedEpochs, ref); + const fileRevisionEpochs = new Map(); let everyFileRevisionEpoch = 0; const viewedFiles = ViewedFiles.make({ filesViewedStore, @@ -3009,7 +3045,8 @@ export const make = Effect.gen(function* () { toPullRequestError, runFork, refEpoch, - fileRevisionsEpoch: () => everyFileRevisionEpoch, + fileRevisionsEpoch: (ref) => + `${everyFileRevisionEpoch}:${fileRevisionEpochs.get(refScope(ref)) ?? 0}`, }); const filtersOfKey = ( @@ -3256,9 +3293,11 @@ export const make = Effect.gen(function* () { return detailUncached(refOfCacheKey(key)).pipe( Effect.tap( Effect.fn("PullRequestService.recordDetailStats")(function* (value: PullRequestDetail) { - // GitHub detail includes the same checks; the checks TTL counts from their observation. + // GitHub and Azure detail include the same checks; the checks TTL counts from their observation. + // Azure detail answers a failed policy read with no checks, which must not be cached. if ( - value.provider === "github" && + (value.provider === "github" || + (value.provider === "azure-devops" && value.checks.length > 0)) && value.observedAt !== undefined && !(yield* Cache.has(checksCache, key)) ) { @@ -3308,6 +3347,7 @@ export const make = Effect.gen(function* () { deletions: detail.deletions, changedFiles: detail.changedFiles, mergeability: detail.mergeability, + ...(detail.rebaseRequired === undefined ? {} : { rebaseRequired: detail.rebaseRequired }), headBranch: detail.headBranch, baseBranch: detail.baseBranch, closedAt: detail.closedAt, @@ -3471,16 +3511,26 @@ export const make = Effect.gen(function* () { const diffCache = yield* Cache.makeWith( (key: string) => { - const [reference, cursor, commit] = JSON.parse(key) as [string, string | null, string | null]; + const [reference, cursor, commit, , path] = JSON.parse(key) as [ + string, + string | null, + string | null, + string | null, + string | null, + ]; return diffUncached({ ...refOfCacheKey(reference), + ...(path === null ? {} : { path }), ...(cursor === null ? {} : { cursor }), ...(commit === null ? {} : { commit }), } as PullRequestDiffInput).pipe( observeRead, Effect.tap(({ value, observedAt }) => { // Keep the loader's epoch and read time if a refresh occurs during the provider read. - diffFiles.record(JSON.stringify([reference, cursor, commit]), value, observedAt); + // One withheld file is not the page's file list. + if (path === null) { + diffFiles.record(JSON.stringify([reference, cursor, commit]), value, observedAt); + } return Effect.void; }), ); @@ -3527,6 +3577,7 @@ export const make = Effect.gen(function* () { input.commit === undefined ? (lastGoodSummary.peek(refCacheKey(input))?.updatedAt ?? null) : null, + input.path ?? null, ]); const evictDiffs = Cache.entries(diffCache).pipe( Effect.flatMap((entries) => diff --git a/apps/server/src/pullRequest/pullRequestViewedFiles.ts b/apps/server/src/pullRequest/pullRequestViewedFiles.ts index ee956dfa0fe9c..7558e8b6a166e 100644 --- a/apps/server/src/pullRequest/pullRequestViewedFiles.ts +++ b/apps/server/src/pullRequest/pullRequestViewedFiles.ts @@ -30,7 +30,7 @@ const MAX_FILE_REVISION_PATHS = 1_000; interface FileRevisionsDependencies { readonly runFork: (effect: Effect.Effect) => unknown; readonly refEpoch: (ref: PullRequestRef) => number; - readonly fileRevisionsEpoch: () => number; + readonly fileRevisionsEpoch: (ref: PullRequestRef) => string; readonly toPullRequestError: ( operation: string, ) => (error: PullRequestProviderError) => PullRequestError; @@ -50,7 +50,7 @@ const makeFileRevisions = (dependencies: FileRevisionsDependencies) => { const fileRevisionsKey = (project: SupportedProject, ref: PullRequestRef) => [ refEpoch({ ...ref, host: project.host, repository: project.repository }), - fileRevisionsEpoch(), + fileRevisionsEpoch({ ...ref, host: project.host, repository: project.repository }), ref.projectId, project.repository.trim().toLowerCase(), ref.number, diff --git a/apps/server/src/relay/AgentAwarenessRelay.test.ts b/apps/server/src/relay/AgentAwarenessRelay.test.ts index 68e2289524c09..10388e99fe7c5 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.test.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.test.ts @@ -753,6 +753,37 @@ describe("AgentAwarenessRelay", () => { }), ); + it.effect( + "clears a completion that landed while snoozed, even when processed after the wake", + () => + Effect.gen(function* () { + yield* TestClock.setTime(DateTime.toEpochMillis(DateTime.makeUnsafe(NOW))); + const { relay, currentShell, publications } = yield* makeTestRelay(); + const snooze = { + snoozedAt: DateTime.makeUnsafe(NOW), + snoozedUntil: DateTime.makeUnsafe("2026-09-04T13:00:00.000Z"), + }; + yield* Ref.set(currentShell, shell(snooze)); + yield* relay.publishThread(THREAD_ID); + yield* TestClock.adjust("2 hours"); + yield* Ref.set( + currentShell, + shell({ + ...snooze, + status: "completed", + latestRunCompletedAt: DateTime.makeUnsafe("2026-09-04T12:30:00.000Z"), + }), + ); + yield* relay.publishThread(THREAD_ID); + yield* TestClock.adjust("5 seconds"); + yield* relay.drain; + assert.deepEqual( + publications.map((publication) => publication.state?.phase ?? null), + ["running", null], + ); + }), + ); + it.effect("still publishes a completion when the thread settles during the hold", () => Effect.gen(function* () { const { relay, currentShell, publications } = yield* makeTestRelay(); diff --git a/apps/server/src/relay/AgentAwarenessRelay.ts b/apps/server/src/relay/AgentAwarenessRelay.ts index 18c1dcf3f72ed..4e539fc904979 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.ts @@ -51,6 +51,7 @@ import { import { EnvironmentSigningKey } from "../cloud/environmentKeys.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import * as ThreadManagement from "../orchestration-v2/ThreadManagementService.ts"; +import { isSnoozed } from "../orchestration-v2/ThreadSettlementService.ts"; import * as ProjectService from "../project/ProjectService.ts"; import { forkParked } from "../serverActivation.ts"; @@ -360,12 +361,27 @@ function resolveAgentAwarenessRelayPublishSnapshot(input: { state?.phase !== projectThreadAwarenessV2(awarenessInput)?.phase; return { projectId: input.thread.value.projectId, - state: sanitizeRelayAgentActivityState(state), + state: snoozedCompletion(input.thread.value, state) + ? null + : sanitizeRelayAgentActivityState(state), subagentWait: childDrivesWait ? input.pendingSubagentRequest : null, reason: "snapshot", }; } +// Checked at completion time, so a completion processed after the wake stays silent too. +function snoozedCompletion( + thread: AgentAwarenessThread, + state: { readonly phase: string } | null, +): boolean { + if (state?.phase !== "completed" || thread.latestRunCompletedAt == null) return false; + const completedAt = DateTime.toEpochMillis(thread.latestRunCompletedAt); + if (thread.snoozedAt != null && completedAt < DateTime.toEpochMillis(thread.snoozedAt)) { + return false; + } + return isSnoozed(thread, completedAt); +} + function terminalWorkSinceStart(thread: AgentAwarenessThread, startedAt: number): boolean { return ( thread.latestRunCompletedAt != null && diff --git a/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.ts b/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.ts index 7d9de87ecfd98..ca77c9e26220c 100644 --- a/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.ts +++ b/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.ts @@ -187,6 +187,8 @@ export class DesktopTelemetryReceiver extends Context.Service< readonly setDiagnosticsDemand: ( enabled: boolean, ) => Effect.Effect; + /** Tells the desktop parent that the server accepts requests, so it can skip readiness polling. */ + readonly announceServerReady: Effect.Effect; readonly requestDesktopUpdate: ( requestId: string, trackSwitch?: DesktopTrackSwitch, @@ -717,6 +719,9 @@ export const make = Effect.fn("resourceTelemetry.desktopTelemetryReceiver.make") health: Ref.get(health), subscribeHealth: subscribeBeforeSnapshotWithoutMutex(healthChanges, Ref.get(health)), setDiagnosticsDemand, + announceServerReady: sendControlMessage({ version: 1, type: "serverReady" }).pipe( + Effect.withSpan("DesktopTelemetryReceiver.announceServerReady"), + ), requestDesktopUpdate: Effect.fn("DesktopTelemetryReceiver.requestDesktopUpdate")( function (requestId, trackSwitch) { return sendControlMessage({ @@ -787,6 +792,7 @@ export const layerTest = ( setDiagnosticsDemand: Effect.fn("DesktopTelemetryReceiver.setDiagnosticsDemand")(function () { return Effect.void; }), + announceServerReady: Effect.void, requestDesktopUpdate: Effect.fn("DesktopTelemetryReceiver.requestDesktopUpdate")(function () { return Effect.void; }), diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.interval.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.interval.test.ts index ecc761d1d0b47..86d6f26e06c44 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.interval.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.interval.test.ts @@ -26,6 +26,7 @@ const dependencies = Layer.mergeAll( ManagedPolicy.layerTest(), Layer.mock(ThreadLaunchService.ThreadLaunchService)({ launch: () => Effect.die("synthetic dispatch failure"), + isNonGitProject: () => Effect.succeed(false), }), Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ namedProjectsRoot: "/projects", diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index ee4385ff2e0af..1335e369484a5 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -4,6 +4,7 @@ import * as NodeUtil from "node:util"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import { assert, it, vi } from "@effect/vitest"; import { + CommandId, ProjectId, ProviderInstanceId, ScheduledTaskError, @@ -29,6 +30,7 @@ import * as TestClock from "effect/testing/TestClock"; import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts"; import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts"; +import { OrchestratorDispatchError } from "../orchestration-v2/Orchestrator.ts"; import * as SecretRequests from "../secrets/SecretRequests.ts"; import * as ManagedProjectFolders from "../project/ManagedProjectFolders.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; @@ -663,6 +665,67 @@ it.effect("a fresh-session task launches one thread, then starts each later run }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); +it.effect("a refused dispatch records its refusal, and opaque causes stay generic", () => + Effect.gen(function* () { + const refusal = "Thread thread:refused was stopped."; + const refused = (cause: unknown) => + new OrchestratorDispatchError({ + commandId: CommandId.make("command:refused"), + commandType: "message.dispatch", + cause, + }); + const generic = refused(undefined).message; + const cases = [ + [refusal, refusal], + [" ", generic], + [new Error("private-nested-secret"), generic], + ] as const; + const failure = yield* Ref.make(undefined); + const layer = ScheduledTaskService.layer.pipe( + Layer.provide( + Layer.mergeAll( + ManagedPolicy.layerTest(), + Layer.mock(ThreadLaunchService.ThreadLaunchService)({}), + Layer.mock(ThreadManagementService.ThreadManagementService)({ + getThreadShell: () => Effect.succeed(threadShell(ProjectId.make("project:test"))), + sendToThread: () => + Ref.get(failure).pipe(Effect.flatMap((cause) => Effect.fail(refused(cause)))), + }), + Layer.mock(SecretRequests.SecretRequests)({}), + Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ + namedProjectsRoot: "/projects", + isScratchProject: () => Effect.succeed(false), + }), + NodeCrypto.layer, + Layer.succeed(Scheduler.Scheduler, { register: () => Effect.void }), + ), + ), + ); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const { task } = yield* service.upsert({ + id: ScheduledTaskId.make("task:refused"), + title: "Refused dispatch", + prompt: "Check progress", + enabled: true, + schedule: { type: "interval", everyMs: 60_000 }, + projectId: ProjectId.make("project:test"), + threadId: ThreadId.make("thread:refused"), + workspaceStrategy: { type: "root" }, + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" }, + runtimeMode: "full-access", + interactionMode: "default", + }); + for (const [cause, expected] of cases) { + yield* Ref.set(failure, cause); + const result = yield* service.runNow({ id: task.id }); + assert.equal(result.task.lastRunStatus, "failed"); + assert.equal(result.task.lastRunError, expected); + } + }).pipe(Effect.provide(layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + it.effect("a first run whose thread was deleted before it bound removes the task", () => Effect.gen(function* () { const layer = ScheduledTaskService.layer.pipe( diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 5af0aba39e6ed..9ed134be2da5a 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -53,6 +53,8 @@ import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts import * as ManagedProjectFolders from "../project/ManagedProjectFolders.ts"; import * as Metrics from "../observability/Metrics.ts"; import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts"; +import { OrchestratorDispatchError } from "../orchestration-v2/Orchestrator.ts"; +import { userFacingDispatchErrorMessage } from "../orchestration-v2/UserFacingErrors.ts"; import * as SecretRequests from "../secrets/SecretRequests.ts"; import * as ManagedPolicy from "../policy/ManagedPolicy.ts"; import * as DueHint from "../scheduling/DueHint.ts"; @@ -79,6 +81,7 @@ import { constantTimeEquals, verifyWebhookSignature } from "./webhookVerificatio export const WEBHOOK_ROUTE_PREFIX = "/api/hooks"; const isScheduledTaskError = Schema.is(ScheduledTaskError); +const isOrchestratorDispatchError = Schema.is(OrchestratorDispatchError); const PROMPT_TOO_LONG = "The filled-in prompt is too long."; const WEBHOOK_DELIVERY_RETENTION = 50; const WEBHOOK_DEDUPE_DELIVERY_RETENTION = 20; @@ -364,6 +367,14 @@ function errorMessage(error: unknown): string { return failure instanceof Error ? failure.message : String(failure); } +function dispatchErrorMessage(cause: Cause.Cause): string { + const failure = Cause.squash(cause); + // Direct orchestration refusals are user-facing; nested defects can contain private data. + return isOrchestratorDispatchError(failure) && typeof failure.cause === "string" + ? (userFacingDispatchErrorMessage(failure.cause) ?? errorMessage(failure)) + : errorMessage(failure); +} + function webhookPath(taskId: string, token: string): string { return `${WEBHOOK_ROUTE_PREFIX}/${encodeURIComponent(taskId)}/${token}`; } @@ -1098,7 +1109,7 @@ export const layer = Layer.effect( if (webhook !== undefined && runSucceeded) yield* Ref.set(webhook.dispatched, true); const completedAt = yield* localNow; const lastRunStatus = runSucceeded ? ("succeeded" as const) : ("failed" as const); - const lastRunError = runSucceeded ? null : errorMessage(result.cause); + const lastRunError = runSucceeded ? null : dispatchErrorMessage(result.cause); const threadId = runSucceeded ? result.value : null; // One transaction, so an edit cannot land between reading the schedule and // writing the next run computed from it. The origin is read first: it reads @@ -1511,9 +1522,11 @@ export const layer = Layer.effect( return { token, secret, secretChanged }; }) : { token: null, secret: null, secretChanged: true }; + // A folder that is not a Git repository, Scratch included, cannot hold a worktree. const workspaceStrategy: ScheduledTask["workspaceStrategy"] = input.workspaceStrategy.type === "worktree" && - (yield* managedFolders.isScratchProject(input.projectId)) + ((yield* managedFolders.isScratchProject(input.projectId)) || + (yield* threadLaunch.isNonGitProject(input.projectId))) ? { type: "root" } : input.workspaceStrategy; const scheduleUnchanged = diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index e4c270627e188..4f38762cf705c 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -44,6 +44,7 @@ import { fixPath } from "./os-jank.ts"; import * as Ws from "./ws.ts"; import * as AgentScopeLive from "./process/agentScope.ts"; import * as ExternalLauncher from "./process/externalLauncher.ts"; +import * as SpawnWorker from "./process/SpawnWorker.ts"; import * as NodePtyAdapter from "./terminal/NodePtyAdapter.ts"; import * as PullRequestHttp from "./pullRequest/http.ts"; import * as SpeechModelStore from "./speech/SpeechModelStore.ts"; @@ -131,6 +132,7 @@ import * as KnownWorkspaces from "./workspace/KnownWorkspaces.ts"; import * as WorkspaceFileSystem from "./workspace/WorkspaceFileSystem.ts"; import * as WorkspacePaths from "./workspace/WorkspacePaths.ts"; import * as GitVcsDriver from "./vcs/GitVcsDriver.ts"; +import * as VcsAutomaticFetchInterval from "./vcs/VcsAutomaticFetchInterval.ts"; import * as VcsDriverRegistry from "./vcs/VcsDriverRegistry.ts"; import * as VcsProjectConfig from "./vcs/VcsProjectConfig.ts"; import * as VcsProcess from "./vcs/VcsProcess.ts"; @@ -177,12 +179,14 @@ import * as ConnectReachability from "./cloud/ConnectReachability.ts"; import { pendingServiceUpdateExists } from "./cloud/updateHandoff.ts"; import * as RelayTracing from "./cloud/relayTracing.ts"; import * as CloudManagedEndpointRuntime from "./cloud/ManagedEndpointRuntime.ts"; +import * as RelayClientLedger from "./cloud/RelayClientLedger.ts"; import { MANAGED_TUNNEL_FIRST_REGISTRATION_JITTER, makeManagedTunnelRecovery, managedTunnelStartupAction, retryDesiredLinkReconcile, retryManagedTunnelRegistration, + watchManagedTunnelOrigin, } from "./cloud/managedTunnelStartup.ts"; import * as CloudCliTokenManager from "./cloud/CliTokenManager.ts"; import * as CloudCliState from "./cloud/CliState.ts"; @@ -346,6 +350,9 @@ const layerRelayClient = Layer.unwrap( }), ).pipe(Layer.provide(ManagedPolicy.layer)); +// NodeServices and NodeHttpServer each output Node's own spawner, so each is followed by this one. +const layerSpawner = SpawnWorker.layerServer.pipe(Layer.provide(NodeServices.layer)); + const layerHttpServer = Layer.unwrap( Effect.gen(function* () { const config = yield* ServerConfig.ServerConfig; @@ -382,7 +389,7 @@ const layerHttpServer = Layer.unwrap( // `threshold` when context takeover is disabled). websocket: { perMessageDeflate: true }, }, - ); + ).pipe(Layer.merge(layerSpawner)); }), ).pipe( Layer.provide(ManagedPolicy.layer), @@ -392,7 +399,7 @@ const layerHttpServer = Layer.unwrap( const layerPlatformServices = DocumentText.layer.pipe( Layer.provideMerge(NonBlockingSpawner.layer), - Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), + Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer, layerSpawner)), ); const layerPersistence = Layer.empty.pipe(Layer.provideMerge(SqlitePersistence.layerConfig)); @@ -488,7 +495,6 @@ const layerVcs = Layer.empty.pipe( Layer.provideMerge( VcsStatusBroadcaster.layer.pipe( Layer.provide(layerGitWorkflow), - Layer.provide(VcsStatusBroadcaster.layerAutomaticFetchInterval), // The orchestration runtime also consumes the broadcaster (run // finalization), so the policy cannot read the store from the runtime's // output. @@ -501,7 +507,10 @@ const layerVcs = Layer.empty.pipe( const layerCheckpointStore = CheckpointStore.layer.pipe(Layer.provide(layerVcsDriverRegistry)); -const layerPortScanner = PortScanner.layer.pipe(Layer.provide(ProcessRunner.layer)); +const layerPortScanner = PortScanner.layer.pipe( + Layer.provide(ProcessRunner.layer), + Layer.provide(layerBackground), +); const layerTerminal = TerminalManager.layer( ProviderInstanceRegistryHydration.deriveProviderInstanceConfigMap, @@ -554,6 +563,14 @@ const layerCloudManagedEndpointRuntime = Layer.mergeAll( CloudManagedEndpointRuntime.layer.pipe( Layer.provide(ServerSecretStore.layer), Layer.provide(layerRelayClient), + Layer.provide( + Layer.unwrap( + Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + return RelayClientLedger.layer({ stateDir: config.stateDir }); + }), + ), + ), ), ); @@ -716,15 +733,15 @@ const layerRuntimeCoreDependenciesBase = Layer.mergeAll( Layer.provideMerge(Layer.merge(ProjectStore.layer, ThreadSearch.layer)), Layer.provideMerge(SecretCheck.layer), Layer.provideMerge(SecretResolver.layer), + // Above the settings it writes when a theme is applied. + Layer.provideMerge(EnvironmentTheme.layer), Layer.provideMerge(layerServerSettings), Layer.provideMerge(layerSourceControlProviderRegistry), Layer.provideMerge(layerGit), Layer.provideMerge(layerVcs), Layer.provideMerge(Layer.mergeAll(layerTerminal, layerPreview, layerDevice)), Layer.provideMerge(layerPersistence), - Layer.provideMerge( - Layer.mergeAll(Keybindings.layer, EnvironmentTheme.layer, UsageLimitSources.layer), - ), + Layer.provideMerge(Layer.mergeAll(Keybindings.layer, UsageLimitSources.layer)), Layer.provideMerge(ProviderRegistry.layer), Layer.provideMerge( ProviderInstanceRegistryHydration.layer.pipe(Layer.provide(SqlitePersistence.layerConfig)), @@ -801,6 +818,9 @@ const layerRuntimeDependencies = layerRuntimeCoreDependencies.pipe( Layer.provideMerge(RemoteOpenTargets.layer), Layer.provideMerge(DirectEndpoints.layer.pipe(Layer.provide(layerServerEnvironment))), Layer.provideMerge(ServerLifecycleEvents.layer), + // GitVcsDriver.layer is memoized and some of its build sites live outside this file, so the + // fetch interval goes in context here, before the first build captures it. + Layer.provideMerge(VcsAutomaticFetchInterval.layer.pipe(Layer.provide(layerServerSettings))), Layer.provide(NetService.layer), ); @@ -908,6 +928,17 @@ const layerMakeServer = Layer.unwrap( yield* startup.markHttpListening; }), ); + const layerDesktopReadySignal = Layer.effectDiscard( + Effect.gen(function* () { + const startup = yield* ServerRuntimeStartup.ServerRuntimeStartup; + const receiver = yield* DesktopTelemetryReceiver.DesktopTelemetryReceiver; + yield* startup.awaitCommandReady.pipe( + Effect.andThen(receiver.announceServerReady), + Effect.ignore, + Effect.forkScoped, + ); + }), + ); const layerRuntimeState = Layer.effectDiscard( Effect.gen(function* () { yield* Deferred.succeed(runtimeStateParked, undefined).pipe(Effect.orDie); @@ -1141,6 +1172,28 @@ const layerMakeServer = Layer.unwrap( Stream.runForEach(recoverManagedTunnel), Effect.forkScoped, ); + yield* watchManagedTunnelOrigin({ + localOrigin, + readIngressOrigin: endpointRuntime.inspectConnector.pipe( + Effect.map((connector) => connector?.origin ?? null), + ), + // Like startup, a publish-only link never routes the tunnel to this host. + register: CloudCliState.readCliDesiredLinkMode.pipe( + Effect.flatMap((mode) => + mode === "publish_only" + ? Effect.void + : cloudLink + .registerManagedTunnelRecovery(localOrigin) + .pipe( + Effect.flatMap((registration) => + registration.status === "recovery_required" + ? endpointRuntime.requestRecovery(registration.config) + : Effect.void, + ), + ), + ), + ), + }).pipe(Effect.forkScoped); const wantsCliLink = hasCloudPublicConfig ? yield* CloudCliState.readCliDesiredCloudLink.pipe( Effect.catch((cause) => @@ -1288,6 +1341,7 @@ const layerMakeServer = Layer.unwrap( const layerServerApplication = Layer.mergeAll( layerRoutes, layerHttpListening, + layerDesktopReadySignal, layerRuntimeState.pipe(Layer.provide(layerLauncher)), layerTailscaleServe, layerCloudDesiredLinkReconcile, @@ -1318,6 +1372,6 @@ const layerMakeServer = Layer.unwrap( ).pipe(Layer.provide(ProcessRunner.layer)); export const runServer = Layer.launch( - // For the layers above whose spawner requirement is met by the caller's runtime. - layerMakeServer.pipe(Layer.provide(NonBlockingSpawner.layer)), + // For the layers above whose spawner requirement is not met by one of them. + layerMakeServer.pipe(Layer.provide(NonBlockingSpawner.layer.pipe(Layer.provide(layerSpawner)))), ); diff --git a/apps/server/src/serverLifecycleEvents.ts b/apps/server/src/serverLifecycleEvents.ts index 7d394e4dc9134..d90b7fa57bc35 100644 --- a/apps/server/src/serverLifecycleEvents.ts +++ b/apps/server/src/serverLifecycleEvents.ts @@ -1,5 +1,6 @@ import type { ServerLifecycleStreamEvent } from "@t3tools/contracts"; import * as Context from "effect/Context"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as PubSub from "effect/PubSub"; @@ -22,11 +23,14 @@ export class ServerLifecycleEvents extends Context.Service< readonly publish: (event: LifecycleEventInput) => Effect.Effect; readonly snapshot: Effect.Effect; readonly stream: Stream.Stream; + /** Completes once the first ready event is published. */ + readonly awaitReady: Effect.Effect; } >()("t3/serverLifecycleEvents") {} const make = Effect.gen(function* () { const pubsub = yield* PubSub.unbounded(); + const ready = yield* Deferred.make(); const state = yield* Ref.make({ sequence: 0, events: [], @@ -45,8 +49,14 @@ const make = Effect.gen(function* () { ...current.events.filter((entry) => entry.type !== nextEvent.type), ]; return [nextEvent, { sequence: nextSequence, events: nextEvents }] as const; - }).pipe(Effect.tap((event) => PubSub.publish(pubsub, event))), + }).pipe( + Effect.tap((event) => PubSub.publish(pubsub, event)), + Effect.tap((event) => + event.type === "ready" ? Deferred.succeed(ready, undefined) : Effect.void, + ), + ), snapshot: Ref.get(state), + awaitReady: Deferred.await(ready), get stream() { return Stream.fromPubSub(pubsub); }, diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index 3ca2d29da7c81..873649cb168f5 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -1469,7 +1469,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" { kind: "eas", id: "eas", - label: "EAS Simulator", + label: "Expo Simulators", accessToken: "expo-secret", maxSessionMinutes: 30, idleMinutes: 10, @@ -1786,7 +1786,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" { kind: "eas", id: "eas", - label: "EAS Simulator", + label: "Expo Simulators", accessToken: easReference, maxSessionMinutes: 30, idleMinutes: 10, @@ -1906,7 +1906,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" { kind: "eas", id: "eas", - label: "EAS Simulator", + label: "Expo Simulators", accessToken: personal("Expo/token"), maxSessionMinutes: 30, idleMinutes: 10, diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index ac699b9c7d6cb..4bfe776a3aba8 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -29,6 +29,7 @@ import { readServiceState, registerServiceTask, serviceLauncherIsRunning, + serviceLauncherIsServing, serviceLauncherLockPath, stateDirOwned, writeServiceState, @@ -201,6 +202,212 @@ const fakeServiceUnit = ( }); it.layer(NodeServices.layer)("service state persistence", (it) => { + it.effect.each([ + "startup", + "log write fails", + "home probe error", + "database probe error", + "pending update", + "update handoff", + "trial exit", + "failed trial start", + "stop", + ] as const)("waits for disk recovery during %s without losing the update lifecycle", (scenario) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-disk-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const databasePath = path.join(root, "userdata", "statev2.sqlite"); + let backupPath = path.join(root, "runtime", "db-backup", "disk-update"); + const pending = { + id: "disk-update", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + dbPath: databasePath, + status: "pending" as const, + }; + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); + yield* fs.writeFileString(databasePath, "before trial"); + const childSource = ` +const fs = require("node:fs"); +const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); +if (context.update?.status === "pending") { + if (context.childVersion !== context.update.targetVersion) throw new Error("wrong trial version"); + if (fs.readFileSync(${JSON.stringify(path.dirname(backupPath))} + "/" + context.update.id + "/database", "utf8") !== "before trial") { + throw new Error("missing original backup"); + } + if (${JSON.stringify(scenario)} === "trial exit") { + fs.writeFileSync(context.update.dbPath, "migrated"); + process.on("message", () => process.exit(1)); + process.send("migrated"); + } else { + process.send({ type: "prepared", updateId: context.update.id }); + process.on("message", (message) => { + if (message.type === "committed") process.send("ready"); + }); + } +} else if (${JSON.stringify(scenario)} === "update handoff" && context.update === undefined) { + process.on("message", (message) => { + if (message.type === "update-accepted") process.send("exiting"); + if (message === "exit") process.exit(0); + }); + process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${JSON.stringify(databasePath)} }); +} else { + process.send("ready"); +} +setInterval(() => {}, 1_000); +`; + for (const version of ["1.0.0", "1.1.0"]) { + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", version), + childSource, + ); + } + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + ...([ + "startup", + "log write fails", + "home probe error", + "database probe error", + "update handoff", + "stop", + ].includes(scenario) + ? {} + : { update: pending }), + }), + ); + let low = [ + "startup", + "log write fails", + "home probe error", + "database probe error", + "pending update", + "stop", + ].includes(scenario); + let recovered = false; + const blocked = Promise.withResolvers(); + const ready = Promise.withResolvers(); + const probing = Promise.withResolvers(); + const capacityResult = Promise.withResolvers(); + const actualStatfs = NodeFSP.statfs; + const stats = yield* Effect.promise(() => actualStatfs(root)); + yield* Effect.acquireRelease( + Effect.sync(() => { + vi.useFakeTimers({ + toFake: ["setInterval", "clearInterval", "setTimeout", "clearTimeout"], + }); + const capacity = vi.spyOn(NodeFSP, "statfs").mockImplementation(async (directory) => { + if (scenario === "stop" && directory === root) { + probing.resolve(); + await capacityResult.promise; + } + if ( + (scenario === "home probe error" && directory === root) || + (scenario === "database probe error" && directory === path.dirname(databasePath)) + ) { + throw new Error("statfs unavailable"); + } + if (scenario === "failed trial start" && !recovered && NodeFS.existsSync(backupPath)) + low = true; + return { ...stats, bavail: low ? 0 : 1024 ** 4, bsize: 1 }; + }); + const log = vi.spyOn(process.stderr, "write").mockImplementation((chunk) => { + if (String(chunk).includes("Waiting for free space")) { + blocked.resolve(); + if (scenario === "log write fails") { + throw Object.assign(new Error("no space left"), { code: "ENOSPC" }); + } + } + return true; + }); + spawnHook.current = (child) => + child.on("message", (message) => { + if (message === "ready") ready.resolve(); + if (message === "migrated" || message === "exiting") { + low = true; + child.send("exit"); + } + }); + return { capacity, log }; + }), + ({ capacity, log }) => + Effect.sync(() => { + capacity.mockRestore(); + log.mockRestore(); + spawnHook.current = undefined; + vi.useRealTimers(); + }), + ); + const beforeSpawns = spawnedChildren.length; + const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath))); + const running = launcher.run(); + yield* Effect.addFinalizer(() => + Effect.promise(async () => { + await launcher.stop("SIGTERM"); + await running; + }), + ); + if (scenario === "stop") { + yield* Effect.promise(() => probing.promise); + const stopping = launcher.stop("SIGTERM"); + capacityResult.resolve(); + yield* Effect.promise(() => stopping); + yield* Effect.promise(() => running); + assert.equal(spawnedChildren.length, beforeSpawns); + assert.equal(vi.getTimerCount(), 0); + return; + } + yield* Effect.promise(() => Promise.race([blocked.promise, ready.promise, running])); + assert.equal( + spawnedChildren.length - beforeSpawns, + ["trial exit", "update handoff"].includes(scenario) ? 1 : 0, + ); + if (scenario === "update handoff") { + const state = yield* Effect.promise(() => readServiceState(statePath)); + backupPath = path.join(root, "runtime", "db-backup", state.update!.id); + } + assert.equal( + yield* fs.exists(backupPath), + scenario === "trial exit" || scenario === "failed trial start", + ); + if (scenario === "trial exit") + assert.equal(yield* fs.readFileString(databasePath), "migrated"); + if (scenario === "failed trial start") { + yield* fs.remove(path.join(root, "runtime", "versions", "1.1.0"), { recursive: true }); + } + low = false; + recovered = true; + yield* Effect.promise(() => vi.advanceTimersByTimeAsync(30_000)); + yield* Effect.promise(() => Promise.race([ready.promise, running])); + const state = yield* Effect.promise(() => readServiceState(statePath)); + assert.equal( + state.activeVersion, + ["pending update", "update handoff"].includes(scenario) ? "1.1.0" : "1.0.0", + ); + assert.equal( + state.update?.status, + { + startup: undefined, + "log write fails": undefined, + "home probe error": undefined, + "database probe error": undefined, + "pending update": "committed", + "update handoff": "committed", + "trial exit": "rolled-back", + "failed trial start": "failed", + }[scenario], + ); + assert.equal(yield* fs.readFileString(databasePath), "before trial"); + assert.isFalse(yield* fs.exists(backupPath)); + }), + ); + it.effect( "loads optional service.env literally without changing inherited or managed variables", () => @@ -748,8 +955,11 @@ process.exit(0); }); yield* Effect.promise(() => refused.promise); assert.isFalse(completed); + // An idle launcher must not make an SSH launch wait for a server it is not starting. + assert.isFalse(yield* Effect.promise(() => serviceLauncherIsServing(root))); otherOwner.close(); yield* Effect.promise(() => owned.promise); + assert.isTrue(yield* Effect.promise(() => serviceLauncherIsServing(root))); yield* Effect.promise(() => launcher.stop("SIGTERM")); yield* Effect.promise(() => running); assert.strictEqual(spawnedChildren.length - spawnsBefore, 2); diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 1a6c07cf43659..fad391896d307 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -8,6 +8,7 @@ import * as NodeFSP from "node:fs/promises"; import * as NodePath from "node:path"; import * as NodeUtil from "node:util"; import { acquireServiceSelectionLock } from "./cloud/serviceSelectionLock.ts"; +import { isNoCopyOnWrite, markNoCopyOnWrite } from "./noCopyOnWrite.ts"; import type { PendingServiceUpdate, @@ -29,6 +30,7 @@ import { serviceUnitLogPathOf, SERVICE_ENV_FILE, SERVICE_LAUNCHER_LOCK_FILE, + SERVICE_SERVING_LOCK_FILE, SERVICE_LAUNCHER_CONTEXT_ENV, SERVICE_LAUNCHER_PROTOCOL, SERVICE_STATE_FILE, @@ -49,6 +51,9 @@ const HANDOFF_DELAY_MS = 2_000; */ const PREPARED_TIMEOUT_MS = 30 * 60_000; const TERMINATE_GRACE_MS = 5_000; +// Keep SQLite startup and journaling from turning a full filesystem into a restart loop. +const MIN_SERVICE_START_FREE_BYTES = 64 * 1024 ** 2; +const DISK_SPACE_RETRY_MS = 30_000; // A replacement started by a handoff waits up to two minutes for the stopping launcher. const LOCK_RETRY_MS = 500; const LOCK_ATTEMPTS = 240; @@ -438,6 +443,7 @@ export async function databaseBackupBlocker( const { readFreeBytes, canClone } = { ...filesystemBackupSpaceProbe, ...probe }; const backupRoot = NodePath.join(baseDir, "runtime", "db-backup"); await NodeFSP.mkdir(backupRoot, { recursive: true, mode: 0o700 }); + if (await isNoCopyOnWrite(dbPath)) await markNoCopyOnWrite(backupRoot); await removeAbandonedBackupCopies(backupRoot); const [files, free] = await Promise.all([statDatabaseFiles(dbPath), readFreeBytes(backupRoot)]); const size = files.reduce((total, file) => total + file.size, 0); @@ -753,11 +759,12 @@ export const serviceLauncherLockPath = (baseDir: string) => // An exclusive SQLite transaction, like the server's owner lock: the OS drops it when the holder dies. async function tryAcquireSqliteLock( lockPath: string, + busyTimeoutMs = 0, ): Promise<{ readonly close: () => void } | undefined> { const { DatabaseSync } = await import("node:sqlite"); const db = new DatabaseSync(lockPath); try { - db.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;"); + db.exec(`PRAGMA busy_timeout = ${busyTimeoutMs}; BEGIN EXCLUSIVE;`); return db; } catch (cause) { db.close(); @@ -773,6 +780,13 @@ export async function serviceLauncherIsRunning(lockPath: string): Promise + NodePath.join(baseDir, "runtime", SERVICE_SERVING_LOCK_FILE); + +/** Whether a launcher has, or is about to start, a server for this home. An idle one does not. */ +export const serviceLauncherIsServing = (baseDir: string): Promise => + serviceLauncherIsRunning(servingLockPath(baseDir)).catch(() => false); + /** Resolves once no launcher holds the lock; the caller bounds the wait through `signal`. */ export async function waitForServiceLauncherExit( lockPath: string, @@ -852,6 +866,7 @@ export class Launcher { #stateDirFreeProbes = 0; readonly #output: number | undefined; readonly #lockPath: string | undefined; + #servingLock: { readonly close: () => void } | undefined; readonly #stopChildOverIpc: boolean; constructor( @@ -905,6 +920,7 @@ export class Launcher { } } } finally { + this.#releaseServing(); process.off("SIGTERM", onSigterm); process.off("SIGINT", onSigint); } @@ -922,13 +938,29 @@ export class Launcher { return undefined; } + // Advisory: without it an SSH launch may start its own server, as it does beside no service. + async #claimServing(): Promise { + // An SSH launch probes this lock for a moment; wait out that probe. + this.#servingLock ??= await tryAcquireSqliteLock(servingLockPath(this.#baseDir), 250).catch( + () => undefined, + ); + } + + #releaseServing(): void { + this.#servingLock?.close(); + this.#servingLock = undefined; + } + #terminate(child: NodeChildProcess.ChildProcess, signal: NodeJS.Signals = "SIGTERM") { return terminateChild(child, signal, this.#stopChildOverIpc); } + // Diagnostics are best-effort: a full log filesystem must not stop the launcher waiting for space. #log(line: string): void { - if (this.#output === undefined) process.stderr.write(line); - else NodeFS.writeSync(this.#output, line); + try { + if (this.#output === undefined) process.stderr.write(line); + else NodeFS.writeSync(this.#output, line); + } catch {} } #enqueue(transition: () => Promise): void { @@ -981,6 +1013,55 @@ export class Launcher { this.#timer = undefined; } + async #serviceStartDiskSpaceBlocker(): Promise { + const databasePath = + this.#state.update?.status === "pending" + ? this.#state.update.dbPath + : NodePath.join(this.#baseDir, "userdata", "statev2.sqlite"); + const resolvedDatabasePath = await restoreTarget(databasePath).catch(() => databasePath); + const freeBytes = async (directory: string): Promise => { + try { + return await filesystemBackupSpaceProbe.readFreeBytes(directory); + } catch (cause) { + const parent = NodePath.dirname(directory); + if ((cause as NodeJS.ErrnoException).code === "ENOENT" && parent !== directory) { + return freeBytes(parent); + } + return Number.POSITIVE_INFINITY; + } + }; + const available = await Promise.all([ + freeBytes(this.#baseDir), + freeBytes(NodePath.dirname(resolvedDatabasePath)), + ]); + const free = Math.min(...available); + return free < MIN_SERVICE_START_FREE_BYTES + ? `T3 Code needs at least ${MIN_SERVICE_START_FREE_BYTES} bytes free before starting the server; ${free} bytes available.` + : undefined; + } + + async #deferForDiskSpace(retry: () => Promise): Promise { + const reason = await this.#serviceStartDiskSpaceBlocker(); + if (reason === undefined) return false; + // A stop during the probe must not arm a retry that outlives it. + if (this.#stopRequested) return true; + this.#clearTimer(); + this.#releaseServing(); + this.#log(`[service-launcher] ${reason} Waiting for free space before retrying.\n`); + this.#timer = setInterval( + () => + this.#enqueue(async () => { + if (this.#child !== null || this.#stopping) return; + if ((await this.#serviceStartDiskSpaceBlocker()) !== undefined) return; + this.#clearTimer(); + await this.#claimServing(); + await retry(); + }), + DISK_SPACE_RETRY_MS, + ); + return true; + } + async #selectState(next: ServiceState): Promise { const release = await acquireServiceSelectionLock(this.#statePath); try { @@ -1002,6 +1083,7 @@ export class Launcher { } async #recover(): Promise { + await this.#claimServing(); // Clear old stop markers to preserve future tunnels; clear restart-pending only for the awaited version to avoid CLI races. await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined); const restartPending = restartPendingPath(this.#baseDir); @@ -1055,6 +1137,7 @@ export class Launcher { } async #startTrial(pending: PendingServiceUpdate): Promise { + if (await this.#deferForDiskSpace(() => this.#startTrial(pending))) return; try { await backupDatabaseOnce(this.#baseDir, pending, this.#stopSignal.signal); } catch { @@ -1073,6 +1156,14 @@ export class Launcher { async #startChild(version: string, role: ChildRole, update?: ServiceUpdateRecord): Promise { if (this.#stopping) return; + if ( + await this.#deferForDiskSpace(() => + role === "trial" && update?.status === "pending" + ? this.#startTrial(update) + : this.#startChild(version, role, update), + ) + ) + return; if (!(await runtimeExists(this.#baseDir, version))) { throw new Error(`Selected t3@${version} runtime is missing or incomplete.`); } @@ -1331,6 +1422,12 @@ export class Launcher { await this.#startTrial(pending); return; } + if ( + await this.#deferForDiskSpace(() => + this.#startChild(child.version, "active", this.#state.update), + ) + ) + return; throw new Error(`Active child exited unexpectedly (${String(code ?? signal ?? "unknown")}).`); } @@ -1356,6 +1453,7 @@ export class Launcher { await this.#repointServiceUnit(next.activeVersion); } } + this.#releaseServing(); this.#log( "[service-launcher] Another T3 Code server owns this T3 home. This service starts again once it stops.\n", ); @@ -1380,6 +1478,7 @@ export class Launcher { this.#stateDirFreeProbes = owned ? 0 : this.#stateDirFreeProbes + 1; if (this.#stateDirFreeProbes < 2) return; this.#clearTimer(); + await this.#claimServing(); this.#log("[service-launcher] The T3 home is free again. Starting the server.\n"); const update = this.#state.update; if (update?.status === "pending") await this.#startTrial(update); @@ -1396,6 +1495,8 @@ export class Launcher { this.#child = null; await this.#terminate(child.process); } + if (await this.#deferForDiskSpace(() => this.#returnToPrevious(pending, status, reason))) + return; await restoreDatabaseBackup(this.#baseDir, pending); const outcome = terminalUpdate({ pending, status, reason }); const next: ServiceState = { @@ -1430,6 +1531,8 @@ export async function main(options: LauncherOptions = {}): Promise { // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime. const windows = process.platform === "win32"; let logTimer: NodeJS.Timeout | undefined; + // A failed stderr write (ENOSPC on a redirected log) is emitted asynchronously; without a listener it is fatal. + process.stderr.on("error", () => undefined); try { // What the service manager started the launcher with. It locates the unit; service.env // changes only what the server gets. diff --git a/apps/server/src/settingsSecrets.ts b/apps/server/src/settingsSecrets.ts index 22ea8ae882583..a38072d19813f 100644 --- a/apps/server/src/settingsSecrets.ts +++ b/apps/server/src/settingsSecrets.ts @@ -235,7 +235,7 @@ function mapSecretFieldGroups( ) as EasDeviceHostConfig, ); - // Its own prefix: an EAS and a Simlock host may share an id. + // Its own prefix: an Expo and a Simlock host may share an id. const simlockDeviceHosts = settings.simlockDeviceHosts.map( (host) => mapFields( diff --git a/apps/server/src/speech/SpeechService.test.ts b/apps/server/src/speech/SpeechService.test.ts index 2e05132cd6eea..3f580b0743b40 100644 --- a/apps/server/src/speech/SpeechService.test.ts +++ b/apps/server/src/speech/SpeechService.test.ts @@ -450,14 +450,18 @@ it.effect("honors exact aliases and phrase boundaries in batch and streamed text { term: "McDonald’s", aliases: [] }, { term: "King’s Cross", aliases: [] }, { term: "C#", aliases: [] }, + { term: "React", aliases: ["瑞安特"] }, + { term: "Kubernetes", aliases: [] }, ], }; const spoken = 'a test with a i and ay eye. Robert, Roberts. "the publisher" and the publisher. (git hub), git (hub). ' + - "git hub's API, git hub’s API, git’s hub, mac donald's menu, king's cross. vitamin c and plan c."; + "git hub's API, git hub’s API, git’s hub, mac donald's menu, king's cross. vitamin c and plan c. " + + "我用github和瑞安特,部署到kubernetis上。Robert用git hub。githubで管理する。"; const expected = "a test with AI and AI. Bob, Roberts. O'Reilly and the publisher. (GitHub), git (hub). " + - "GitHub's API, GitHub’s API, git’s hub, McDonald’s menu, King’s Cross. vitamin c and plan c."; + "GitHub's API, GitHub’s API, git’s hub, McDonald’s menu, King’s Cross. vitamin c and plan c. " + + "我用GitHub和React,部署到Kubernetes上。Bob用GitHub。GitHubで管理する。"; native.transcribe.mockResolvedValueOnce({ text: spoken }); native.finish.mockResolvedValueOnce(spoken); expect(yield* speech.transcribe(pcm(), options)).toBe(expected); diff --git a/apps/server/src/speech/customWords.ts b/apps/server/src/speech/customWords.ts index 98c55e883749a..981632c46172c 100644 --- a/apps/server/src/speech/customWords.ts +++ b/apps/server/src/speech/customWords.ts @@ -1,5 +1,11 @@ import type { SpeechCustomWords } from "@t3tools/contracts"; +// These scripts are written without spaces, so a neighbor in one of them still ends a word. +const UNSPACED_SCRIPTS = "\\p{scx=Han}\\p{scx=Hiragana}\\p{scx=Katakana}"; +const LETTER = `[[\\p{L}\\p{M}\\p{N}]--[${UNSPACED_SCRIPTS}]]`; +const WORD_CHARACTER = `[[\\p{L}\\p{M}\\p{N}\\p{Pc}]--[${UNSPACED_SCRIPTS}]]`; +const TOKEN = new RegExp(`${LETTER}(?:[^\\s${UNSPACED_SCRIPTS}]*${LETTER})?`, "gv"); + /** Fuzzy-matches preferred terms and rewrites aliases to their preferred spelling. */ export function makeSpeechTextCorrector( words: readonly string[], @@ -112,7 +118,7 @@ export function makeSpeechTextCorrector( const apply = (text: string): string => { if (keys.length === 0) return text; // Leave surrounding punctuation in the source; only horizontal whitespace can join words. - const tokens = [...text.matchAll(/[\p{L}\p{M}\p{N}](?:\S*[\p{L}\p{M}\p{N}])?/gu)] + const tokens = [...text.matchAll(TOKEN)] .filter(([word]) => /^[\p{L}\p{M}\p{N}'’-]+$/u.test(word)) .map((match) => ({ value: match[0], @@ -168,10 +174,9 @@ export function makeSpeechTextCorrector( const lookup = new Map( spellings.map(({ spelling, term }) => [spelling.toLocaleLowerCase(), term]), ); - const word = "[\\p{L}\\p{M}\\p{N}\\p{Pc}]"; const pattern = new RegExp( - `(? escapePattern(spelling)).join("|")})(?!${word})`, - "giu", + `(? escapePattern(spelling)).join("|")})(?!${WORD_CHARACTER})`, + "giv", ); return (text: string) => { let output = ""; diff --git a/apps/server/src/speech/postProcessing.test.ts b/apps/server/src/speech/postProcessing.test.ts new file mode 100644 index 0000000000000..3dd64c9b1bcc7 --- /dev/null +++ b/apps/server/src/speech/postProcessing.test.ts @@ -0,0 +1,81 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import { ProviderInstanceId, type ModelSelection } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Ref from "effect/Ref"; + +import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import * as TextGeneration from "../textGeneration/TextGeneration.ts"; +import * as SpeechPostProcessing from "./postProcessing.ts"; + +const textGenerationSelection = { + instanceId: ProviderInstanceId.make("fakeapi"), + model: "fake-model", +}; +const acpSelection = { instanceId: ProviderInstanceId.make("acp_local"), model: "default" }; + +it.effect("cleanup pinned to a provider that cannot generate text uses the text model", () => + Effect.gen(function* () { + const used = yield* Ref.make(null); + const layer = SpeechPostProcessing.layer.pipe( + Layer.provide( + Layer.succeed(TextGeneration.TextGeneration, { + generateTranscriptionPostProcessing: (input: { modelSelection: ModelSelection }) => + Ref.set(used, input.modelSelection).pipe(Effect.as({ transcription: "Hello, world." })), + } as unknown as TextGeneration.TextGeneration["Service"]), + ), + Layer.provide( + Layer.succeed(ProviderRegistry.ProviderRegistry, { + getProviders: Effect.succeed([ + { + instanceId: acpSelection.instanceId, + driver: "acpRegistry", + enabled: true, + installed: true, + supportsTextGeneration: false, + }, + ]), + } as unknown as ProviderRegistry.ProviderRegistry["Service"]), + ), + Layer.provide( + ServerSettings.layerTest({ + providerInstances: { + [acpSelection.instanceId]: { + driver: "acpRegistry", + enabled: true, + config: { source: "local", commandPath: "acp-agent" }, + }, + }, + textGenerators: { + [textGenerationSelection.instanceId]: { + kind: "openaiCompatible", + baseUrl: "http://127.0.0.1:18999/v1", + enabled: true, + }, + }, + textGenerationModelSelection: textGenerationSelection, + speechPostProcessingModelSelection: acpSelection, + }), + ), + Layer.provide(NodeServices.layer), + ); + yield* Effect.gen(function* () { + const postProcessing = yield* SpeechPostProcessing.SpeechPostProcessing; + yield* postProcessing.process({ + options: { + speechCustomWords: [], + speechPostProcessingPrompt: { mode: "default", customInstructions: "" }, + }, + transcript: "um hello world", + draft: { text: "", selection: { start: 0, end: 0 } }, + } as never); + }).pipe(Effect.provide(layer)); + const selection = yield* Ref.get(used); + assert.deepStrictEqual( + { instanceId: selection?.instanceId, model: selection?.model }, + textGenerationSelection, + ); + }), +); diff --git a/apps/server/src/speech/postProcessing.ts b/apps/server/src/speech/postProcessing.ts index 3208afd057287..92f4af17696c3 100644 --- a/apps/server/src/speech/postProcessing.ts +++ b/apps/server/src/speech/postProcessing.ts @@ -8,6 +8,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import { resolveSpeechPostProcessingModelSelection } from "@t3tools/shared/serverSettings"; +import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as TextGeneration from "../textGeneration/TextGeneration.ts"; import { buildTranscriptionPostProcessingPrompt } from "../textGeneration/TranscriptionPostProcessing.ts"; @@ -27,11 +28,13 @@ const make = Effect.gen(function* () { const settingsService = yield* ServerSettings.ServerSettingsService; const fileSystem = yield* FileSystem.FileSystem; const textGeneration = yield* TextGeneration.TextGeneration; + const providerRegistry = yield* ProviderRegistry.ProviderRegistry; return SpeechPostProcessing.of({ process: Effect.fn("SpeechPostProcessing.process")( function* (input) { if (input.transcript.trim().length === 0) return { text: input.transcript }; const settings = yield* settingsService.getSettings; + const providers = yield* providerRegistry.getProviders; const { mode, customInstructions } = input.options.speechPostProcessingPrompt; const instructions = (mode === "custom" ? customInstructions.trim() : "") || @@ -46,7 +49,7 @@ const make = Effect.gen(function* () { input.draft, input.options.speechCorrectionWord, ), - modelSelection: resolveSpeechPostProcessingModelSelection(settings), + modelSelection: resolveSpeechPostProcessingModelSelection(settings, providers), }); const text = generated.transcription.trim(); return { diff --git a/apps/server/src/startupAccess.test.ts b/apps/server/src/startupAccess.test.ts index a48bb5263f8ec..159778b3c7e03 100644 --- a/apps/server/src/startupAccess.test.ts +++ b/apps/server/src/startupAccess.test.ts @@ -21,6 +21,7 @@ const ipv4 = (address: string, internal = false): NodeOS.NetworkInterfaceInfo => it("resolves wildcard hosts to a concrete external interface when one is available", () => { const connectionString = resolveHeadlessConnectionString("0.0.0.0", 3773, { interfaces: { en0: [ipv4("192.168.1.42")], lo0: [ipv4("127.0.0.1", true)] }, + platform: "linux", }); expect(connectionString).toBe("http://192.168.1.42:3773"); @@ -34,16 +35,21 @@ it("skips public and container addresses for a private-network listener", () => eth1: [ipv4("10.0.0.5")], }; - expect(resolveHeadlessConnectionString("0.0.0.0", 3773, { interfaces })).toBe( + expect(resolveHeadlessConnectionString("0.0.0.0", 3773, { interfaces, platform: "linux" })).toBe( "http://203.0.113.7:3773", ); expect( - resolveHeadlessConnectionString("0.0.0.0", 3773, { interfaces, privateNetworkOnly: true }), + resolveHeadlessConnectionString("0.0.0.0", 3773, { + interfaces, + privateNetworkOnly: true, + platform: "linux", + }), ).toBe("http://10.0.0.5:3773"); expect( resolveHeadlessConnectionString("0.0.0.0", 3773, { interfaces: { eth0: [ipv4("203.0.113.7")] }, privateNetworkOnly: true, + platform: "linux", }), ).toBe("http://localhost:3773"); }); @@ -53,10 +59,21 @@ it("prefers a LAN address over a tailnet one for a private-network pairing URL", resolveHeadlessConnectionString("0.0.0.0", 3773, { interfaces: { tailscale0: [ipv4("100.101.102.103")], en0: [ipv4("192.168.1.42")] }, privateNetworkOnly: true, + platform: "linux", }), ).toBe("http://192.168.1.42:3773"); }); +it("skips NixOS container and systemd-nspawn veths for a private-network pairing URL", () => { + expect( + resolveHeadlessConnectionString("0.0.0.0", 3773, { + interfaces: { "ve-jellyfin": [ipv4("192.168.53.193")], eth0: [ipv4("192.168.0.18")] }, + privateNetworkOnly: true, + platform: "linux", + }), + ).toBe("http://192.168.0.18:3773"); +}); + it("targets the bound interface when the listener binds a specific address", () => { const hostFor = (ip: string) => loopbackUrlHostForAddress(NetAddress.inetAddressFromIpStringUnsafe(ip, 3773)); diff --git a/apps/server/src/startupAccess.ts b/apps/server/src/startupAccess.ts index f393faecebf91..1720300bd7beb 100644 --- a/apps/server/src/startupAccess.ts +++ b/apps/server/src/startupAccess.ts @@ -7,6 +7,7 @@ import { isSharedAddressSpaceHost, isWildcardHost, } from "@t3tools/shared/hostClassification"; +import * as HostProcess from "@t3tools/shared/HostProcess"; import { QrCode } from "@t3tools/shared/qrCode"; import { buildPairingAppLink } from "@t3tools/shared/remote"; import * as Effect from "effect/Effect"; @@ -44,9 +45,19 @@ export const isLoopbackHost = (host: string | undefined): boolean => { * host bridge such as `br0` or Proxmox's `vmbr0` carries the LAN address, so * it stays listed. */ -export const VIRTUAL_INTERFACE = +const VIRTUAL_INTERFACE = /^(docker|br-|veth|virbr|vmnet|vboxnet|vEthernet|podman|cni|flannel|cali|lxcbr|lxdbr|bridge[1-9]\d\d)/; +/** + * systemd-nspawn's `ve-*`/`vz-*` and NixOS containers' `vb-*` name host-side + * veths after the machine. Windows aliases are user-renameable, so these + * prefixes only apply to Linux interface discovery. + */ +const LINUX_CONTAINER_INTERFACE = /^(ve-|vb-|vz-)/; + +export const isVirtualInterface = (name: string, platform: NodeJS.Platform): boolean => + VIRTUAL_INTERFACE.test(name) || (platform === "linux" && LINUX_CONTAINER_INTERFACE.test(name)); + /** * Loopback or private-range IPv4 (LAN, tailnet, CGNAT), not link-local, in * plain or IPv4-mapped IPv6 form. @@ -86,6 +97,7 @@ const resolveHeadlessConnectionHost = ( host: string | undefined, interfaces: NetworkInterfacesMap, privateNetworkOnly: boolean, + platform: NodeJS.Platform, ): string => { if (!host) { return "localhost"; @@ -96,7 +108,7 @@ const resolveHeadlessConnectionHost = ( } const interfaceEntries = Object.entries(interfaces).flatMap(([name, entries]) => - privateNetworkOnly && VIRTUAL_INTERFACE.test(name) ? [] : (entries ?? []), + privateNetworkOnly && isVirtualInterface(name, platform) ? [] : (entries ?? []), ); const externalIpv4Entries = interfaceEntries.filter( (entry) => @@ -128,12 +140,14 @@ export const resolveHeadlessConnectionString = ( options: { readonly interfaces?: NetworkInterfacesMap; readonly privateNetworkOnly?: boolean | undefined; - } = {}, + readonly platform: NodeJS.Platform; + }, ): string => { const connectionHost = resolveHeadlessConnectionHost( host, options.interfaces ?? NodeOS.networkInterfaces(), options.privateNetworkOnly === true, + options.platform, ); return `http://${formatHostForUrl(connectionHost)}:${port}`; }; @@ -196,10 +210,11 @@ export const issueHeadlessServeAccessInfo = Effect.fn("issueHeadlessServeAccessI const serverConfig = yield* ServerConfig.ServerConfig; const httpServer = yield* HttpServer.HttpServer; const serverAuth = yield* EnvironmentAuth.EnvironmentAuth; + const platform = yield* HostProcess.Platform; const connectionString = resolveHeadlessConnectionString( serverConfig.host, resolveListeningPort(httpServer.address, serverConfig.port), - { privateNetworkOnly: serverConfig.privateNetworkOnly }, + { privateNetworkOnly: serverConfig.privateNetworkOnly, platform }, ); const issued = yield* serverAuth.issueStartupPairingCredential(); diff --git a/apps/server/src/storageCleanup.test.ts b/apps/server/src/storageCleanup.test.ts index a70cecab59db8..1a5161e26ab25 100644 --- a/apps/server/src/storageCleanup.test.ts +++ b/apps/server/src/storageCleanup.test.ts @@ -2,6 +2,8 @@ import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Effect from "effect/Effect"; import * as Context from "effect/Context"; +import type * as Digest from "effect/crypto/Digest"; +import * as Base64Url from "effect/encoding/Base64Url"; import * as Deferred from "effect/Deferred"; import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; @@ -14,6 +16,8 @@ import * as Stream from "effect/Stream"; import { SCAN_CACHE_FILE_NAME } from "./usage/usageScanCache.ts"; import { TestClock } from "effect/testing"; import * as SqlClient from "effect/sql/SqlClient"; +import * as Sqlite from "./persistence/Sqlite.ts"; +import { checkpointRefForScopeOrdinal } from "./orchestration-v2/CheckpointService.ts"; import { DEFAULT_SERVER_SETTINGS, type WorktreeKeepWhen, @@ -26,11 +30,13 @@ import * as ServerConfig from "./config.ts"; import * as StorageCleanup from "./storageCleanup.ts"; import * as GitVcsDriver from "./vcs/GitVcsDriver.ts"; import * as GitManager from "./git/GitManager.ts"; +import * as GitWorkflowService from "./git/GitWorkflowService.ts"; import * as Settings from "./serverSettings.ts"; import * as ProjectionStore from "./orchestration-v2/ProjectionStore.ts"; import * as ProviderSessionManager from "./orchestration-v2/ProviderSessionManager.ts"; import * as ProjectStore from "./orchestration-v2/ProjectStore.ts"; import * as Orchestrator from "./orchestration-v2/Orchestrator.ts"; +import * as ThreadManagement from "./orchestration-v2/ThreadManagementService.ts"; import * as TerminalManager from "./terminal/Manager.ts"; import * as PullRequestService from "./pullRequest/PullRequestService.ts"; import { CodexProviderCapabilitiesV2 } from "./orchestration-v2/Adapters/CodexAdapterV2.ts"; @@ -38,6 +44,7 @@ import { ServerActivation } from "./serverActivation.ts"; import { withWorkspaceLease } from "./workspace/workspaceLease.ts"; import { describe, expect, it } from "@effect/vitest"; import { + CheckpointScopeId, EventId, ProjectId, ProviderDriverKind, @@ -50,6 +57,7 @@ import { GitCommandError, type PullRequestSummary, type OrchestrationV2DomainEvent, + type OrchestrationV2ServerCommand, type OrchestrationV2ThreadShell, } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; @@ -329,6 +337,7 @@ const cleanupFixture = ( worktreeAfterDays: 8, worktreeOnDelete: true, }, + sqlOverride?: SqlClient.SqlClient, ) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -441,6 +450,31 @@ const cleanupFixture = ( getThreadShell: (threadId: ThreadId) => Effect.sync(() => threads.find((thread) => thread.id === threadId) ?? null), } as unknown as ProjectionStore.ProjectionStoreV2["Service"]), + Effect.provideService(ThreadManagement.ThreadManagementService, { + getShellSnapshot: (input?: { threadIds?: ReadonlyArray }) => + Effect.sync(() => ({ + threads: threads.filter((thread) => input?.threadIds?.includes(thread.id) ?? true), + archivedThreads: [], + })), + dispatch: (command: OrchestrationV2ServerCommand) => + Effect.sync(() => { + if (command.type === "thread.metadata.update" && command.branch != null) { + const branch = command.branch; + threads = threads.map((thread) => + thread.id === command.threadId && thread.branch === command.expectedBranch + ? { ...thread, branch } + : thread, + ); + } + return { sequence: 0, storedEvents: [] }; + }), + } as unknown as ThreadManagement.ThreadManagementService["Service"]), + Effect.provideService(GitWorkflowService.GitWorkflowService, { + currentBranch: (cwd: string) => + command(cwd, ["branch", "--show-current"]).pipe( + Effect.map((result) => result.stdout.trim() || null), + ), + } as unknown as GitWorkflowService.GitWorkflowService["Service"]), Effect.provideService(ProviderSessionManager.ProviderSessionManagerV2, { isLive: () => Effect.sync(() => { @@ -462,41 +496,49 @@ const cleanupFixture = ( return { sequence: 1, storedEvents: [] }; }), } as unknown as Orchestrator.OrchestratorV2["Service"]), - Effect.provideService(SqlClient.SqlClient, ((strings: TemplateStringsArray) => - Effect.sync(() => - openSessionCwd !== null && strings.join("").includes("projection_provider_sessions") - ? [ - { - payload_json: JSON.stringify({ - id: "session-1", - driver: "codex", - providerInstanceId: "codex", - status: "ready", - cwd: openSessionCwd, - model: null, - capabilities: CodexProviderCapabilitiesV2, - createdAt: "2026-06-01T12:00:00.000Z", - updatedAt: "2026-06-01T12:00:00.000Z", - lastError: null, - }), - }, - ] - : outboxStatus !== null && strings.join("").includes("orchestration_v2_effect_outbox") - ? [{ status: outboxStatus }] - : strings.join("").includes("deleted_at IS NOT NULL") - ? deletedThreads.map((thread) => ({ - payload_json: JSON.stringify(thread), - workspaceRoot: repo, - })) - : strings.join("").includes('AS "worktreePath"') + Effect.provideService( + SqlClient.SqlClient, + sqlOverride ?? + (((strings: TemplateStringsArray) => + Effect.sync(() => + strings.join("").includes("retained_checkpoint_threads") + ? [] + : openSessionCwd !== null && + strings.join("").includes("projection_provider_sessions") ? [ - ...threads.flatMap((thread) => thread.worktreePath ?? []), - ...storedWorktreePaths, - ].map((worktreePath) => ({ worktreePath })) - : strings.join("").includes("scheduled_tasks") - ? scheduledThreadIds.map((threadId) => ({ threadId })) - : [], - )) as unknown as SqlClient.SqlClient), + { + payload_json: JSON.stringify({ + id: "session-1", + driver: "codex", + providerInstanceId: "codex", + status: "ready", + cwd: openSessionCwd, + model: null, + capabilities: CodexProviderCapabilitiesV2, + createdAt: "2026-06-01T12:00:00.000Z", + updatedAt: "2026-06-01T12:00:00.000Z", + lastError: null, + }), + }, + ] + : outboxStatus !== null && + strings.join("").includes("orchestration_v2_effect_outbox") + ? [{ status: outboxStatus }] + : strings.join("").includes("deleted_at IS NOT NULL") + ? deletedThreads.map((thread) => ({ + payload_json: JSON.stringify(thread), + workspaceRoot: repo, + })) + : strings.join("").includes('AS "worktreePath"') + ? [ + ...threads.flatMap((thread) => thread.worktreePath ?? []), + ...storedWorktreePaths, + ].map((worktreePath) => ({ worktreePath })) + : strings.join("").includes("scheduled_tasks") + ? scheduledThreadIds.map((threadId) => ({ threadId })) + : [], + )) as unknown as SqlClient.SqlClient), + ), Effect.provideService(GitManager.GitManager, { invalidateStatus: () => Effect.void, branchPullRequest: ( @@ -656,11 +698,21 @@ const cleanupTestLayer = GitVcsDriver.layer.pipe( Layer.provideMerge(ServerConfig.layerTest(process.cwd(), { prefix: "storage-cleanup-" })), Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), ); +const annexWorktree = Effect.fnUntraced(function* ( + fixture: Effect.Success>, + worktree: string, +) { + const adminDir = (yield* fixture.command(worktree, ["rev-parse", "--absolute-git-dir"])).stdout; + yield* fixture.fs.remove(`${worktree}/.git`); + yield* fixture.fs.symlink(adminDir.trim(), `${worktree}/.git`); +}); + const runCleanupTest = ( effect: Effect.Effect< A, E, | FileSystem.FileSystem + | Digest.Digest | ServerConfig.ServerConfig | GitVcsDriver.GitVcsDriver | Path.Path @@ -668,6 +720,182 @@ const runCleanupTest = ( >, ) => effect.pipe(Effect.provide(cleanupTestLayer)); +const legacyCheckpointRef = (threadId: string, turn: number) => + `refs/t3/checkpoints/${Base64Url.encode(threadId)}/turn/${turn}`; + +const checkpointCleanupFixture = Effect.fn("test.checkpointCleanupFixture")(function* () { + const sql = yield* SqlClient.SqlClient; + const git = yield* GitVcsDriver.GitVcsDriver; + const fixture = yield* cleanupFixture(DEFAULT_SERVER_SETTINGS.storageCleanup, sql); + const { repo, worktree, command } = fixture; + const timestamp = DateTime.formatIso(at(0)); + yield* sql`INSERT INTO projection_projects ${sql.insert({ + project_id: "project-1", + title: "Checkpoint cleanup", + workspace_root: repo, + scripts_json: "[]", + created_at: timestamp, + updated_at: timestamp, + deleted_at: timestamp, + })}`; + const addThread = Effect.fn("test.addCheckpointThread")(function* ( + id: string, + options: { deleted?: boolean; archived?: boolean; source?: string; legacyOnly?: boolean } = {}, + ) { + const scopeId = CheckpointScopeId.make(`scope:${id}`); + yield* sql`INSERT INTO orchestration_v2_projection_threads ${sql.insert({ + thread_id: id, + project_id: "project-1", + title: id, + default_provider: "codex", + runtime_mode: "full-access", + interaction_mode: "default", + created_at: timestamp, + updated_at: timestamp, + archived_at: options.archived ? timestamp : null, + deleted_at: options.deleted ? timestamp : null, + payload_json: JSON.stringify({ + worktreePath: worktree, + forkedFrom: options.source ? { type: "run", threadId: options.source } : null, + }), + })}`; + if (!options.legacyOnly) + yield* sql`INSERT INTO orchestration_v2_projection_checkpoint_scopes ${sql.insert({ + scope_id: scopeId, + thread_id: id, + node_id: `node:${id}`, + kind: "root_run", + ordinal_within_parent: 0, + advances_app_run_count: 1, + created_at: timestamp, + payload_json: JSON.stringify({ cwd: worktree }), + })}`; + const refs: string[] = options.legacyOnly + ? [legacyCheckpointRef(id, 0)] + : [ + yield* checkpointRefForScopeOrdinal({ scopeId, ordinalWithinScope: 0 }), + yield* checkpointRefForScopeOrdinal({ scopeId, ordinalWithinScope: 1 }), + legacyCheckpointRef(id, 0), + ]; + for (const ref of refs) yield* command(repo, ["update-ref", ref, "HEAD"]); + return refs; + }); + const listRefs = () => + git + .execute({ + operation: "cleanup-test.list-checkpoints", + cwd: repo, + args: ["for-each-ref", "--format=%(refname)", "refs/t3/"], + maxOutputBytes: 16 * 1024 * 1024, + }) + .pipe(Effect.map(({ stdout }) => stdout.trim().split("\n").filter(Boolean).sort())); + return { ...fixture, sql, addThread, listRefs }; +}); + +describe("checkpoint ref cleanup", () => { + it.live("prunes refs owned by deleted threads after their worktree is removed", () => + runCleanupTest( + Effect.gen(function* () { + const { service, repo, worktree, command, addThread, listRefs } = + yield* checkpointCleanupFixture(); + yield* addThread("deleted", { deleted: true }); + yield* addThread("imported-deleted", { deleted: true, legacyOnly: true }); + const retained = [ + ...(yield* addThread("active")), + ...(yield* addThread("archived", { archived: true })), + legacyCheckpointRef("unknown-owner", 0), + ]; + yield* command(repo, ["update-ref", retained.at(-1)!, "HEAD"]); + yield* command(repo, ["pack-refs", "--all"]); + yield* command(repo, ["worktree", "remove", worktree]); + yield* service.runNow; + expect(yield* listRefs()).toEqual(retained.sort()); + }).pipe(Effect.provide(Sqlite.layerMemory)), + ), + ); + + it.live("keeps deleted fork ancestry while a live or archived descendant needs it", () => + runCleanupTest( + Effect.gen(function* () { + const { service, sql, addThread, listRefs } = yield* checkpointCleanupFixture(); + const root = yield* addThread("root", { deleted: true }); + const intermediate = yield* addThread("intermediate", { deleted: true, source: "root" }); + const child = yield* addThread("child", { archived: true, source: "intermediate" }); + yield* service.runNow; + expect(yield* listRefs()).toEqual([...root, ...intermediate, ...child].sort()); + yield* sql`UPDATE orchestration_v2_projection_threads SET deleted_at = 'deleted' WHERE thread_id = 'child'`; + yield* service.runNow; + expect(yield* listRefs()).toEqual([]); + }).pipe(Effect.provide(Sqlite.layerMemory)), + ), + ); + + it.live("waits for a deleted thread's pending checkpoint effects", () => + runCleanupTest( + Effect.gen(function* () { + const { service, sql, addThread, listRefs } = yield* checkpointCleanupFixture(); + const refs = yield* addThread("capturing", { deleted: true }); + yield* sql`INSERT INTO orchestration_v2_effect_outbox ${sql.insert({ + effect_id: "capture", + command_id: "capture", + thread_id: "capturing", + effect_type: "checkpoint.capture", + payload_json: "{}", + status: "running", + available_at: "2026-01-01T00:00:00Z", + created_at: "2026-01-01T00:00:00Z", + updated_at: "2026-01-01T00:00:00Z", + })}`; + yield* service.runNow; + expect(yield* listRefs()).toEqual(refs.toSorted()); + yield* sql`UPDATE orchestration_v2_effect_outbox SET status = 'succeeded'`; + yield* service.runNow; + expect(yield* listRefs()).toEqual([]); + }).pipe(Effect.provide(Sqlite.layerMemory)), + ), + ); + + it.live("deletes a symbolic checkpoint ref without its target or neighbouring refs", () => + runCleanupTest( + Effect.gen(function* () { + const { service, repo, command, addThread, listRefs } = yield* checkpointCleanupFixture(); + const refs = yield* addThread("deleted", { deleted: true }); + yield* command(repo, ["symbolic-ref", refs[0]!, "refs/heads/main"]); + const retained = [ + "refs/t3/unrelated/0", + `${refs[1]}-other`, + refs[1]!.replace(/1$/, "2/nested"), + ]; + for (const ref of retained) yield* command(repo, ["update-ref", ref, "HEAD"]); + yield* service.runNow; + expect(yield* listRefs()).toEqual(retained.sort()); + yield* command(repo, ["show-ref", "--verify", "refs/heads/main"]); + }).pipe(Effect.provide(Sqlite.layerMemory)), + ), + ); + + it.live("cleans more checkpoint refs than default git output holds", () => + runCleanupTest( + Effect.gen(function* () { + const { service, repo, addThread, listRefs } = yield* checkpointCleanupFixture(); + const git = yield* GitVcsDriver.GitVcsDriver; + const prefix = (yield* addThread("many-checkpoints", { deleted: true }))[0]!.slice(0, -1); + yield* git.execute({ + operation: "cleanup-test.seed-checkpoints", + cwd: repo, + args: ["update-ref", "--stdin"], + stdin: Array.from( + { length: 12_283 }, + (_, ordinal) => `update ${prefix}${ordinal} HEAD\n`, + ).join(""), + }); + yield* service.runNow; + expect(yield* listRefs()).toEqual([]); + }).pipe(Effect.provide(Sqlite.layerMemory)), + ), + ); +}); + it.live.each(["repository", "uppercase", "bare"] as const)( "keeps a worktree holding an ignored nested %s", (kind) => @@ -1032,8 +1260,26 @@ it.live("keeps an ignored repository inside a registered submodule", () => it.live("removes clean submodules recursively, including deinitialized repositories", () => runCleanupTest( Effect.gen(function* () { - const { service, fs, repo, worktree, submodule, command, recordSubmodules } = - yield* submoduleCleanupFixture(); + const fs = yield* FileSystem.FileSystem; + const git = yield* GitVcsDriver.GitVcsDriver; + let editedDuringClean = false; + const { service, repo, worktree, submodule, command, recordSubmodules, setPolicy } = + yield* submoduleCleanupFixture().pipe( + Effect.provideService(GitVcsDriver.GitVcsDriver, { + ...git, + execute: (input) => + git.execute(input).pipe( + Effect.tap(() => { + if (input.args[0] !== "clean") return Effect.void; + editedDuringClean = true; + return fs.writeFileString(`${input.cwd}/tracked.txt`, "late edit\n"); + }), + ), + }), + ); + setPolicy("tracked-changes"); + yield* fs.writeFileString(`${worktree}/notes.txt`, "untracked\n"); + yield* fs.writeFileString(`${worktree}/.env`, "ignored\n"); yield* command(submodule, [ "-c", "protocol.file.allow=always", @@ -1075,8 +1321,22 @@ it.live("removes clean submodules recursively, including deinitialized repositor yield* command(legacy, ["submodule", "deinit", "stored"]); yield* recordSubmodules(); yield* command(submodule, ["submodule", "deinit", "stored"]); + const checkpoint = (yield* command(submodule, [ + "commit-tree", + "HEAD^{tree}", + "-p", + "HEAD", + "-m", + "checkpoint", + ])).stdout.trim(); + yield* command(submodule, [ + "update-ref", + "refs/t3/orchestration-v2/checkpoints/test", + checkpoint, + ]); const admin = (yield* command(worktree, ["rev-parse", "--absolute-git-dir"])).stdout.trim(); const report = yield* service.runNow; + expect(editedDuringClean).toBe(false); expect(report.entries.find((entry) => entry.path === worktree)).toMatchObject({ outcome: "removed", }); @@ -1188,15 +1448,21 @@ it.live("keeps unpushed submodule history after checkout and deinitialization", it.live("keeps submodules when stored repository metadata is malformed", () => runCleanupTest( Effect.gen(function* () { - const { service, fs, worktree, submodule, child, command } = yield* submoduleCleanupFixture(); + const { service, fs, worktree, submodule, child, command, setPolicy } = + yield* submoduleCleanupFixture(); + setPolicy("tracked-changes"); const childGit = (yield* command(child, ["rev-parse", "--absolute-git-dir"])).stdout.trim(); yield* command(submodule, ["submodule", "deinit", "child"]); yield* fs.writeFileString(`${childGit}/HEAD`, "not a ref\n"); + yield* fs.writeFileString(`${worktree}/notes.txt`, "keep while the worktree stays\n"); expect( (yield* service.runNow).entries.find((entry) => entry.path === worktree), ).toMatchObject({ outcome: "kept", reason: expect.stringContaining("inspected") }); expect(yield* fs.exists(worktree)).toBe(true); expect(yield* fs.readFileString(`${childGit}/HEAD`)).toBe("not a ref\n"); + expect(yield* fs.readFileString(`${worktree}/notes.txt`)).toBe( + "keep while the worktree stays\n", + ); }), ), ); @@ -1386,6 +1652,8 @@ describe("subagent worktree ownership", () => { it.effect.each([ "idle", "grandchild", + "branchless", + "branchless-running", "null-branch", "stale-branch", "isolated-owner", @@ -1417,7 +1685,7 @@ describe("subagent worktree ownership", () => { }, }); const root = shell({ - branch: "feature", + branch: ["branchless", "branchless-running"].includes(scenario) ? null : "feature", worktreePath: fixture.worktree, settledAt: scenario === "child-settled-only" ? null : at(-20 * DAY_MS), ...(scenario === "isolated-owner" || scenario === "fork-owner" @@ -1459,7 +1727,7 @@ describe("subagent worktree ownership", () => { : "subagent", ...(scenario === "side-chat" ? { sideChat: true } : {}), }, - status: scenario === "running" ? "running" : "completed", + status: ["running", "branchless-running"].includes(scenario) ? "running" : "completed", pendingBackgroundTasks: scenario === "pending" ? [{ taskId: "task", kind: "command" }] : [], latestRunCompletedAt: scenario === "recent-activity" || scenario === "recent-inactive" @@ -1497,6 +1765,7 @@ describe("subagent worktree ownership", () => { "isolated-owner", "fork-owner", "side-chat", + "branchless", "null-branch", "stale-branch", ].includes(scenario); @@ -1514,7 +1783,7 @@ describe("subagent worktree ownership", () => { outcome: removed ? "removed" : "kept", reason: removed ? "Settled for 20 days" - : scenario === "running" || scenario === "pending" + : ["running", "pending", "branchless-running"].includes(scenario) ? "Thread is running or has pending work" : shared ? "Shared by 2 threads" @@ -2125,7 +2394,7 @@ describe("orphan worktree reports", () => { ), ); - it.live.each(["nested", "direct"] as const)( + it.live.each(["nested", "direct", "annexed"] as const)( "reports a linked orphan's size without removing its files (%s)", (location) => runCleanupTest( @@ -2134,8 +2403,13 @@ describe("orphan worktree reports", () => { fixture.setThreads([]); const orphan = `${fixture.config.worktreesDir}/${location === "nested" ? "repository/" : ""}orphan`; yield* fixture.command(fixture.repo, ["worktree", "add", "-b", "orphan", orphan]); - const sizes = yield* Effect.forEach(yield* fixture.fs.readDirectory(orphan), (name) => - allocatedBytes(fixture.fs, `${orphan}/${name}`), + if (location === "annexed") yield* annexWorktree(fixture, orphan); + const sizes = yield* Effect.forEach( + // Links are not measured, and the annexed `.git` is one. + (yield* fixture.fs.readDirectory(orphan)).filter( + (name) => location !== "annexed" || name !== ".git", + ), + (name) => allocatedBytes(fixture.fs, `${orphan}/${name}`), ); const outside = yield* fixture.fs.makeTempDirectoryScoped(); yield* fixture.fs.writeFileString(`${outside}/external`, "outside data"); @@ -2608,7 +2882,8 @@ describe("storage cleanup reports and local file policies", () => { occurredAt: at(0), payload: { ...shell({ deletedAt: at(0) }), lastVisitedAt: null }, }); - expect(fixture.snapshotReads()).toBe(startupReads + 3 * Number(worktreeOnDelete)); + // Deletion always sweeps so the deleted thread's checkpoint refs are pruned. + expect(fixture.snapshotReads()).toBe(startupReads + 2 * Number(worktreeOnDelete) + 1); expect(fixture.sessionLiveChecks()).toBe(worktreeOnDelete ? 6 : 0); }), ), @@ -2988,6 +3263,17 @@ describe("storage cleanup reports and local file policies", () => { }), ), ); + it.live("removes a worktree whose .git file git-annex replaced with a symlink", () => + runCleanupTest( + Effect.gen(function* () { + const fixture = yield* cleanupFixture(); + const { service, worktree, fs } = fixture; + yield* annexWorktree(fixture, worktree); + expect((yield* service.runNow).entries[0]).toMatchObject({ outcome: "removed" }); + expect(yield* fs.exists(worktree)).toBe(false); + }), + ), + ); it.live("keeps a worktree when the inspection cannot read its directories", () => runCleanupTest( Effect.gen(function* () { @@ -3010,6 +3296,38 @@ describe("storage cleanup reports and local file policies", () => { }), ), ); + it.effect("removes a worktree owned by a repository other than the project root", () => + runCleanupTest( + Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); + const fixture = yield* cleanupFixture(); + const { fs, command, config } = fixture; + const oldRepo = yield* fs.realPath( + yield* fs.makeTempDirectoryScoped({ prefix: "cleanup-old-repo-" }), + ); + yield* command(oldRepo, ["init", "-b", "main"]); + yield* command(oldRepo, ["config", "user.email", "cleanup@example.test"]); + yield* command(oldRepo, ["config", "user.name", "Cleanup test"]); + yield* fs.writeFileString(`${oldRepo}/tracked.txt`, "original\n"); + yield* command(oldRepo, ["add", "."]); + yield* command(oldRepo, ["commit", "-m", "initial"]); + yield* fs.makeDirectory(`${config.worktreesDir}/old-repo`, { recursive: true }); + const oldWorktree = `${config.worktreesDir}/old-repo/feature`; + yield* command(oldRepo, ["worktree", "add", "-b", "feature", oldWorktree]); + // Cleanup keeps commits no remote-tracking ref holds. + yield* command(oldRepo, ["update-ref", "refs/remotes/backup/main", "HEAD"]); + fixture.setThreads([shell({ branch: "feature", worktreePath: oldWorktree })]); + const report = yield* fixture.service.runNow; + expect(report.entries.find((entry) => entry.path === oldWorktree)).toMatchObject({ + outcome: "removed", + }); + expect(yield* fs.exists(oldWorktree)).toBe(false); + expect((yield* command(oldRepo, ["worktree", "list", "--porcelain"])).stdout).not.toContain( + oldWorktree, + ); + }), + ), + ); it.effect.each([ [ "its owner's branch is cleared", diff --git a/apps/server/src/storageCleanup.ts b/apps/server/src/storageCleanup.ts index fa2e6a6428c9b..41542d44b3f8b 100644 --- a/apps/server/src/storageCleanup.ts +++ b/apps/server/src/storageCleanup.ts @@ -2,11 +2,13 @@ import * as NodeFSP from "node:fs/promises"; import { CommandId, + CheckpointScopeId, GitCommandError, OrchestrationV2AppThreadJson, OrchestrationV2ProviderSessionJson, } from "@t3tools/contracts"; import { randomUuidV4 } from "@t3tools/provider-core/server/randomUuid"; +import * as Base64Url from "effect/encoding/Base64Url"; import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; import type { @@ -49,6 +51,7 @@ import { scratchFolderRoot } from "./project/ManagedProjectFolders.ts"; import * as Orchestrator from "./orchestration-v2/Orchestrator.ts"; import * as ProjectionStore from "./orchestration-v2/ProjectionStore.ts"; import * as ProviderSessionManager from "./orchestration-v2/ProviderSessionManager.ts"; +import { checkpointRefForScopeOrdinal } from "./orchestration-v2/CheckpointService.ts"; import { threadHasQueuedTurnStart } from "./orchestration-v2/ThreadSettlementService.ts"; import { MAX_OVERDUE_ARM_MS } from "./orchestration-v2/UsageLimitRecoveryWorker.ts"; import { forkParked } from "./serverActivation.ts"; @@ -57,7 +60,7 @@ import * as TerminalManager from "./terminal/Manager.ts"; import { isFilesystemRoot, managedWorktreesDirectories } from "./worktreesDirectory.ts"; import * as GitVcsDriver from "./vcs/GitVcsDriver.ts"; import { isNonRepositoryGitStderr } from "./vcs/GitVcsDriverCore.ts"; -import { removeLinkedWorktree } from "./vcs/worktreeRemoval.ts"; +import { isLinkedWorktree, removeLinkedWorktree } from "./vcs/worktreeRemoval.ts"; import { withWorkspaceLease } from "./workspace/workspaceLease.ts"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; @@ -607,7 +610,7 @@ const make = Effect.gen(function* () { const modified = Option.getOrNull((yield* fs.stat(worktreePath)).mtime); if (modified !== null && modified.getTime() > now - ORPHAN_WORKTREE_GRACE_MS) return; const gitPath = path.join(worktreePath, ".git"); - const linked = (yield* fs.exists(gitPath)) && (yield* fs.stat(gitPath)).type === "File"; + const linked = (yield* fs.exists(gitPath)) && (yield* isLinkedWorktree(worktreePath)); reported.add(key); const remaining = measurementDeadline - (yield* Clock.currentTimeMillis); if (!orphanSizes.has(key) && (firstMeasurement || remaining > 0)) { @@ -652,7 +655,7 @@ const make = Effect.gen(function* () { (worktree) => Effect.gen(function* () { if (pathKey(yield* fs.realPath(worktree)) !== pathKey(worktree)) return false; - return (yield* fs.stat(path.join(worktree, ".git"))).type === "File"; + return yield* isLinkedWorktree(worktree); }).pipe(Effect.orElseSucceed(() => false)), ); }); @@ -663,7 +666,7 @@ const make = Effect.gen(function* () { if (referenced.has(pathKey(projectDirectory))) return; const gitPath = path.join(projectDirectory, ".git"); if (yield* fs.exists(gitPath)) { - if ((yield* fs.stat(gitPath)).type === "File") yield* report(projectDirectory); + if (yield* isLinkedWorktree(projectDirectory)) yield* report(projectDirectory); return; } for (const worktreePath of yield* directories(projectDirectory, root)) { @@ -820,6 +823,21 @@ const make = Effect.gen(function* () { return changed === undefined ? null : reason(changed.file); }); + // A worktree can outlive its project moving to another clone. Git removes a + // worktree through the repository that registered it, not through the + // project's current checkout, so resolve the owning repository first. + const worktreeRepository = Effect.fn("StorageCleanup.worktreeRepository")(function* ( + worktreePath: string, + ) { + const commonDir = yield* git.execute({ + operation: "StorageCleanup.worktreeRepository", + cwd: worktreePath, + args: ["rev-parse", "--git-common-dir"], + }); + const resolved = path.resolve(worktreePath, commonDir.stdout.trim()); + return path.basename(resolved) === ".git" ? path.dirname(resolved) : resolved; + }); + const localChangesReason = Effect.fn("StorageCleanup.localChangesReason")(function* ( cwd: string, rules: WorktreeCleanupRules, @@ -933,6 +951,8 @@ const make = Effect.gen(function* () { "rev-list", "--max-count=1", "HEAD", + // Checkpoint refs hold workspace snapshots, not commits a user could lose. + "--exclude=refs/t3/*", "--all", "--reflog", "--not", @@ -1069,10 +1089,10 @@ const make = Effect.gen(function* () { return "worktree is a symbolic link"; if (!roots.some((root) => inside(root, realPath))) return "outside the managed worktree folder"; - const gitLink = yield* fs - .stat(path.join(worktreePath, ".git")) - .pipe(Effect.catchReason("PlatformError", "NotFound", () => Effect.undefined)); - if (gitLink?.type !== "File") return "not a linked worktree"; + const linked = yield* isLinkedWorktree(worktreePath).pipe( + Effect.catchReason("PlatformError", "NotFound", () => Effect.succeed(false)), + ); + if (!linked) return "not a linked worktree"; return null; }); const deletedRows = yield* sql<{ payload_json: string; workspaceRoot: string }>` @@ -1612,15 +1632,16 @@ const make = Effect.gen(function* () { const finalChanges = yield* localChangesReason(worktreePath, settings); if (finalChanges !== null) return keep(finalChanges); if (yield* settingsChangedSinceCheck) return keep("settings changed since check"); - // Clean only untracked files. Ordinary worktrees retain Git's final tracked-edit check. - if (settings.worktreeKeepWhen === "tracked-changes") - yield* cleanupGit(worktreePath, ["clean", "-ffdx"], "git clean"); const submodules = yield* submoduleRemoval(worktreePath, settings); if (submodules.reason !== null) return keep(submodules.reason); if (submodules.force) { const changes = yield* localChangesReason(worktreePath, settings); if (changes !== null) return keep(changes); } + const repository = yield* worktreeRepository(worktreePath); + // Forced removal already discards local files; do not widen its final-check window. + if (settings.worktreeKeepWhen === "tracked-changes" && !submodules.force) + yield* cleanupGit(worktreePath, ["clean", "-ffdx"], "git clean"); const changedBranches = group.filter( (member) => checkout.branch === null || member.branch !== checkout.branch, ); @@ -1640,12 +1661,12 @@ const make = Effect.gen(function* () { } yield* removeLinkedWorktree({ operation: "StorageCleanup.cleanupGit", - cwd: project.workspaceRoot, + cwd: repository, worktreePath, // Git requires force for submodules; recursive checks above cannot make removal atomic. force: submodules.force, execute: git.execute, - run: (args) => cleanupGit(project.workspaceRoot, [...args], "git worktree remove"), + run: (args) => cleanupGit(repository, [...args], "git worktree remove"), }); entries.push({ ...entry, outcome: "removed", reason: sentence(removalReason), bytes }); removalFailures.delete(worktreePath); @@ -1732,6 +1753,115 @@ const make = Effect.gen(function* () { }); const reportRef = yield* SubscriptionRef.make(null); + const cleanCheckpointRefs = Effect.fn("StorageCleanup.cleanCheckpointRefs")(function* () { + // Forks display their source history even after its thread is deleted. Keep + // that ancestry, including deleted intermediate forks and archived children. + const rows = yield* sql<{ + threadId: string; + worktreePath: string | null; + workspaceRoot: string | null; + scopeId: string | null; + cwd: string | null; + }>` + WITH RECURSIVE retained_checkpoint_threads(thread_id) AS ( + SELECT thread_id FROM orchestration_v2_projection_threads WHERE deleted_at IS NULL + UNION + SELECT json_extract(t.payload_json, '$.forkedFrom.threadId') + FROM orchestration_v2_projection_threads t + JOIN retained_checkpoint_threads retained ON retained.thread_id = t.thread_id + WHERE json_extract(t.payload_json, '$.forkedFrom.type') = 'run' + ) + SELECT t.thread_id AS "threadId", + json_extract(t.payload_json, '$.worktreePath') AS "worktreePath", + p.workspace_root AS "workspaceRoot", s.scope_id AS "scopeId", + json_extract(s.payload_json, '$.cwd') AS cwd + FROM orchestration_v2_projection_threads t + LEFT JOIN projection_projects p ON p.project_id = t.project_id + LEFT JOIN orchestration_v2_projection_checkpoint_scopes s ON s.thread_id = t.thread_id + WHERE t.deleted_at IS NOT NULL + AND NOT EXISTS (SELECT 1 FROM retained_checkpoint_threads WHERE thread_id = t.thread_id) + AND NOT EXISTS ( + SELECT 1 FROM orchestration_v2_effect_outbox e + WHERE e.thread_id = t.thread_id AND e.status IN ('pending', 'running') + ) + `.pipe(NodeSqliteClient.readOnly); + const targets = new Map>(); + for (const row of rows) { + // Derive ownership from tombstones, never from absence in this database: + // another T3 home can own refs in the same Git repository. + const prefixes = [`refs/t3/checkpoints/${Base64Url.encode(row.threadId)}/turn/`]; + if (row.scopeId !== null) { + prefixes.push( + (yield* checkpointRefForScopeOrdinal({ + scopeId: CheckpointScopeId.make(row.scopeId), + ordinalWithinScope: 0, + })).slice(0, -1), + ); + } + // A worktree may already be gone; its refs live in the shared repository. + for (const cwd of [row.cwd, row.worktreePath, row.workspaceRoot]) { + if (cwd === null) continue; + const key = path.resolve(cwd); + const owned = targets.get(key) ?? new Set(); + for (const prefix of prefixes) owned.add(prefix); + targets.set(key, owned); + } + } + const repositories = new Map }>(); + for (const [cwd, prefixes] of targets) { + yield* Effect.gen(function* () { + if (!(yield* fs.exists(cwd))) return; + const result = yield* git.execute({ + operation: "StorageCleanup.checkpointRepository", + cwd, + args: ["rev-parse", "--path-format=absolute", "--git-common-dir"], + allowNonZeroExit: true, + }); + if (result.exitCode !== 0) return; + const commonDir = yield* fs.realPath(result.stdout.trim()); + const repository = repositories.get(commonDir) ?? { cwd, prefixes: new Set() }; + for (const prefix of prefixes) repository.prefixes.add(prefix); + repositories.set(commonDir, repository); + }).pipe( + Effect.catch((cause) => + Effect.logWarning("checkpoint repository inspection failed", { cause }), + ), + ); + } + for (const { cwd, prefixes } of repositories.values()) { + yield* Effect.gen(function* () { + const result = yield* git.execute({ + operation: "StorageCleanup.listCheckpointRefs", + cwd, + args: [ + "for-each-ref", + "--format=%(refname)", + "refs/t3/checkpoints/", + "refs/t3/orchestration-v2/checkpoints/", + ], + maxOutputBytes: 16 * 1024 * 1024, + }); + const refs = result.stdout.split("\n").filter((ref) => { + const ordinalStart = ref.lastIndexOf("/") + 1; + return ( + prefixes.has(ref.slice(0, ordinalStart)) && + /^(0|[1-9]\d*)$/.test(ref.slice(ordinalStart)) + ); + }); + if (refs.length === 0) return; + yield* git.execute({ + operation: "StorageCleanup.deleteCheckpointRefs", + cwd, + // Remove a symbolic checkpoint itself, never its branch/tag target. + args: ["update-ref", "--no-deref", "-z", "--stdin"], + stdin: refs.map((ref) => `delete ${ref}\0\0`).join(""), + }); + yield* Effect.logInfo("storage cleanup removed checkpoint refs", { refCount: refs.length }); + }).pipe( + Effect.catch((cause) => Effect.logWarning("checkpoint ref cleanup failed", { cause })), + ); + } + }); // Only deleted threads' worktrees wait on session stops: any stop while one waits, or a stop inside one kept for another reason. // A deletion stays pending until a sweep that started after it finishes. let deletedWorktreeWaits = true; @@ -1746,6 +1876,9 @@ const make = Effect.gen(function* () { const serverSettings = yield* settingsService.getSettings; const settings = serverSettings.storageCleanup; const entries: StorageCleanupReportEntry[] = []; + yield* cleanCheckpointRefs().pipe( + Effect.catch((cause) => Effect.logWarning("checkpoint ref cleanup failed", { cause })), + ); const deletedWorktrees = yield* cleanWorktrees(serverSettings, now, entries).pipe( Effect.catch((error) => Effect.gen(function* () { @@ -1924,6 +2057,7 @@ const make = Effect.gen(function* () { ) return Effect.void; } + if (event.type === "thread.deleted") return requestSweep; if (!anyWorktreePolicy(lastSettings, (rules) => rules.worktreeOnDelete)) return Effect.void; return event.type === "provider-session.updated" && event.payload.status === "error" ? providerSessions diff --git a/apps/server/src/telemetry/ProviderDimensions.test.ts b/apps/server/src/telemetry/ProviderDimensions.test.ts index d88010f600e35..00e59c11228cb 100644 --- a/apps/server/src/telemetry/ProviderDimensions.test.ts +++ b/apps/server/src/telemetry/ProviderDimensions.test.ts @@ -76,3 +76,16 @@ it("drops input strings outside the schema's literals, since rejected calls are handoffSettings("t3_thread_launch", { workspaceStrategy: { type: "/home/me/secret" } }, {}), ).toEqual({ targetChosen: false, workspace: "root" }); }); + +it("records the binding the saved scheduled task resolved to, and none for failures", () => { + expect(handoffSettings("schedule_task", {}, { boundThreadId: null })).toEqual({ + bindToCurrentThread: false, + }); + expect( + handoffSettings( + "schedule_task", + { bindToCurrentThread: true }, + { _tag: "OrchestratorMcpFailure", code: "invalid_request", message: "Different project." }, + ), + ).toEqual({}); +}); diff --git a/apps/server/src/telemetry/ProviderDimensions.ts b/apps/server/src/telemetry/ProviderDimensions.ts index f81a22e844aef..8352ed0d15cc0 100644 --- a/apps/server/src/telemetry/ProviderDimensions.ts +++ b/apps/server/src/telemetry/ProviderDimensions.ts @@ -128,9 +128,9 @@ function callerOrigin(input: { } /** - * Settings the agent chose for a handoff, read only from closed enums and - * booleans in the tool's arguments and result. Prompts, titles, and ids are - * never read. + * Settings for a handoff, recorded only as closed enums and booleans derived + * from the tool's arguments and result. Prompts, titles, and ids are never + * recorded. */ export function handoffSettings(tool: string, args: unknown, result: unknown): Fields { const input = asFields(args); @@ -166,8 +166,13 @@ export function handoffSettings(tool: string, args: unknown, result: unknown): F return defined({ delivery: stringField(output, "delivery") }); case "t3_thread_send_attachments": return { delivery: "auto" }; - case "schedule_task": - return { bindToCurrentThread: input.bindToCurrentThread !== false }; + case "schedule_task": { + // Omitting the input lets the server decide, so read the saved task. + const bound = output.boundThreadId; + return bound === null || typeof bound === "string" + ? { bindToCurrentThread: bound !== null } + : {}; + } default: return {}; } diff --git a/apps/server/src/terminal/Manager.test.ts b/apps/server/src/terminal/Manager.test.ts index da37b82144ac0..331c99726ad00 100644 --- a/apps/server/src/terminal/Manager.test.ts +++ b/apps/server/src/terminal/Manager.test.ts @@ -1197,6 +1197,37 @@ it.layer( }), ); + it.effect("clears a terminal that keeps printing while its history is saved", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + let printWhileSaving = Effect.void; + const printing = FileSystem.FileSystem.of({ + ...fs, + writeFileString: (filePath, data, options) => + printWhileSaving.pipe(Effect.andThen(fs.writeFileString(filePath, data, options))), + }); + const { manager, ptyAdapter, getEvents } = yield* createManager().pipe( + Effect.provideService(FileSystem.FileSystem, printing), + ); + yield* manager.open(openInput()); + const process = ptyAdapter.processes[0]!; + // Each save sees fresh output, so the history worker never goes idle. + printWhileSaving = Effect.gen(function* () { + const printed = yield* Deferred.make(); + const unsubscribe = yield* manager.subscribe((event) => + event.type === "output" ? Deferred.succeed(printed, undefined) : Effect.void, + ); + process.emitData("tick\n"); + yield* Deferred.await(printed); + unsubscribe(); + }); + + yield* manager.clear({ threadId: "thread-1", terminalId: DEFAULT_TERMINAL_ID }); + + expect((yield* getEvents).map((event) => event.type)).toContain("cleared"); + }), + ); + it.effect("restores a log tail that starts inside a control sequence without its remainder", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -1676,6 +1707,28 @@ it.layer( }), ); + // Case-insensitive filesystems resolve "Terminal_YQ.log" to thread "a"'s "terminal_YQ.log", + // so the mixed-case file stands in for that alias on case-sensitive hosts. + it.effect.each(["terminal_", "Terminal_"])( + "never resolves a %s legacy history name to another thread's history file", + (prefix) => + Effect.gen(function* () { + const { manager, logsDir } = yield* createManager(); + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const threadId = `${prefix}${Base64Url.encode("a")}`; + const otherThreadPath = path.join(logsDir, `${threadId}.log`); + yield* fs.makeDirectory(logsDir, { recursive: true }); + yield* fs.writeFileString(otherThreadPath, "thread a\n"); + + expect((yield* manager.open(openInput({ threadId }))).history).toBe(""); + yield* manager.close({ threadId, terminalId: DEFAULT_TERMINAL_ID, deleteHistory: true }); + yield* manager.close({ threadId, deleteHistory: true }); + + expect(yield* fs.readFileString(otherThreadPath)).toBe("thread a\n"); + }), + ); + it.effect("keeps reading and deleting history at an encoded path that already fit", () => Effect.gen(function* () { const threadId = "x".repeat(48); @@ -1924,6 +1977,32 @@ it.layer( }), ); + it.effect("starts a $SHELL whose path contains a space", () => + Effect.gen(function* () { + const { manager, ptyAdapter } = yield* createManager(5, { + env: { SHELL: "/Users/Jane Doe/.local/bin/fish" }, + }).pipe(Effect.provide(layerWithHostPlatform("darwin"))); + + yield* manager.open(openInput()); + + expect(ptyAdapter.spawnInputs.map((input) => input.shell)).toEqual([ + "/Users/Jane Doe/.local/bin/fish", + ]); + }), + ); + + it.effect("starts a quoted $SHELL whose path contains a space and a dash", () => + Effect.gen(function* () { + const { manager, ptyAdapter } = yield* createManager(5, { + env: { SHELL: '"/opt/my -shell/fish" -l' }, + }).pipe(Effect.provide(layerWithHostPlatform("darwin"))); + + yield* manager.open(openInput()); + + expect(ptyAdapter.spawnInputs.map((input) => input.shell)).toEqual(["/opt/my -shell/fish"]); + }), + ); + it.effect( "writes completion syntax for the fallback shell instead of the configured fish shell", () => diff --git a/apps/server/src/terminal/Manager.ts b/apps/server/src/terminal/Manager.ts index b9a279666e8fd..d658083a13a01 100644 --- a/apps/server/src/terminal/Manager.ts +++ b/apps/server/src/terminal/Manager.ts @@ -539,9 +539,11 @@ function normalizeShellCommand( return trimmed; } - const firstToken = trimmed.split(/\s+/g)[0]?.trim(); - if (!firstToken) return null; - return firstToken.replace(/^['"]|['"]$/g, ""); + // A shell path can contain spaces, so only text from the first flag (`bash -l`) is dropped, + // and a quoted path is kept whole even when it contains " -". + const quoted = /^(['"])(.*?)\1(?:\s|$)/.exec(trimmed); + const command = quoted ? quoted[2] : trimmed.split(/\s+(?=-)/)[0]?.replace(/^['"]|['"]$/g, ""); + return command || null; } function basenameForPlatform(command: string, platform: NodeJS.Platform): string { @@ -1861,14 +1863,19 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func ); }; - const legacyHistoryPath = (threadId: string) => - path.join(logsDir, `${legacySafeThreadId(threadId)}.log`); + const legacyHistoryName = (threadId: string) => `${legacySafeThreadId(threadId)}.log`; + // Legacy names in the current "terminal_" namespace would alias another thread's history, + // case-insensitively on some filesystems. + const hasLegacyHistoryName = (threadId: string) => + !legacySafeThreadId(threadId).toLowerCase().startsWith("terminal_"); const historyPaths = (threadId: string, terminalId: string) => [ ...new Set([ historyPath(threadId, terminalId), encodedHistoryPath(threadId, terminalId), - ...(terminalId === DEFAULT_TERMINAL_ID ? [legacyHistoryPath(threadId)] : []), + ...(terminalId === DEFAULT_TERMINAL_ID && hasLegacyHistoryName(threadId) + ? [path.join(logsDir, legacyHistoryName(threadId))] + : []), ]), ]; @@ -2340,7 +2347,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func yield* Effect.forEach( entries.filter( (name) => - name === `${legacySafeThreadId(threadId)}.log` || + (hasLegacyHistoryName(threadId) && name === legacyHistoryName(threadId)) || threadParts.some((part) => name === `${part}.log` || name.startsWith(`${part}_`)), ), (name) => @@ -3650,7 +3657,12 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func session.pendingProcessEventIndex = 0; session.processEventDrainRunning = false; const eventStamp = advanceEventSequence(session); - yield* persistHistory(input.threadId, terminalId, session.history); + // A drain waits for the key to go idle, which never happens while the shell keeps + // printing; every reader of the file flushes it first. + yield* persistWorker.enqueue(toSessionKey(input.threadId, terminalId), { + history: session.history, + immediate: true, + }); yield* publishEvent({ type: "cleared", threadId: input.threadId, diff --git a/apps/server/src/terminal/nodePtyFlushDataAndCleanUp.test.ts b/apps/server/src/terminal/nodePtyFlushDataAndCleanUp.test.ts new file mode 100644 index 0000000000000..7dc0861c7f53c --- /dev/null +++ b/apps/server/src/terminal/nodePtyFlushDataAndCleanUp.test.ts @@ -0,0 +1,131 @@ +import * as NodeModule from "node:module"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; + +const requireCjs = NodeModule.createRequire(import.meta.url); +const { + WindowsPtyAgent, +}: { + WindowsPtyAgent: { + prototype: { + _flushDataAndCleanUp(this: FakeAgent): void; + _cleanUpProcess(this: FakeAgent): void; + }; + }; +} = requireCjs("node-pty/lib/windowsPtyAgent.js"); + +interface FakeAgent { + _useConptyDll: boolean; + _closeTimeout?: unknown; + _inSocket: { readable: boolean }; + _outSocket: { readable: boolean; destroy: () => void }; + _cleanUpProcess: () => void; +} + +// The sockets are stand-ins; the timer logic and _cleanUpProcess are node-pty's own (as patched by T3). +const makeAgent = (): FakeAgent => { + const agent: FakeAgent = { + _useConptyDll: false, + _inSocket: { readable: true }, + _outSocket: { readable: true, destroy: vi.fn() }, + _cleanUpProcess: () => WindowsPtyAgent.prototype._cleanUpProcess.call(agent), + }; + return agent; +}; + +const flush = (agent: FakeAgent) => WindowsPtyAgent.prototype._flushDataAndCleanUp.call(agent); + +// A minimal event loop that mirrors Node's phases: expired timers run first, then immediates (the check phase). +// node-pty calls the global timer functions at call time, so replacing them lets the test place a "poll phase" +// (delivery of already-buffered pipe data) exactly between the two. +interface Timer { + readonly id: number; + readonly dueAt: number; + readonly run: () => void; +} +let now = 0; +let nextId = 0; +let timers: Timer[] = []; +let immediates: Array<() => void> = []; + +const runExpiredTimers = () => { + for (const timer of timers.filter((t) => t.dueAt <= now).toSorted((a, b) => a.dueAt - b.dueAt)) { + timers = timers.filter((t) => t.id !== timer.id); + timer.run(); + } +}; +const runImmediates = () => { + const batch = immediates; + immediates = []; + for (const run of batch) run(); +}; +/** One loop iteration after `ms` of wall time; `poll` runs between the timers and check phases. */ +const loopIteration = (ms: number, poll?: () => void) => { + now += ms; + runExpiredTimers(); + poll?.(); + runImmediates(); +}; + +// ConPTY never closes the output pipe after the shell exits, so node-pty ends the session on a timer. +// T3 shortens it (patches/node-pty@1.2.0-beta.15.patch) so the terminal reports "exited" about 0.9 s sooner. +describe("node-pty Windows exit flush", () => { + beforeEach(() => { + now = 0; + nextId = 0; + timers = []; + immediates = []; + vi.stubGlobal("setTimeout", (run: () => void, ms: number) => { + const id = ++nextId; + timers.push({ id, dueAt: now + ms, run }); + return id; + }); + vi.stubGlobal("clearTimeout", (id: number) => { + timers = timers.filter((t) => t.id !== id); + }); + vi.stubGlobal("setImmediate", (run: () => void) => { + immediates.push(run); + return ++nextId; + }); + }); + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("closes the output socket 100 ms after the exit, not a second later", () => { + const agent = makeAgent(); + flush(agent); + + loopIteration(99); + expect(agent._outSocket.destroy).not.toHaveBeenCalled(); + + loopIteration(1); + expect(agent._outSocket.destroy).toHaveBeenCalledTimes(1); + expect(agent._outSocket.readable).toBe(false); + expect(agent._inSocket.readable).toBe(false); + }); + + // Timers run before the poll phase. After a main-thread stall longer than the window, output that is already + // buffered in the pipe is read only after the timer has expired, so the cleanup has to wait one loop turn for it. + it("keeps the socket open when buffered output is read right after a stalled timer expires", () => { + const agent = makeAgent(); + flush(agent); + + loopIteration(150, () => flush(agent)); + expect(agent._outSocket.destroy).not.toHaveBeenCalled(); + + loopIteration(99); + expect(agent._outSocket.destroy).not.toHaveBeenCalled(); + + loopIteration(1); + expect(agent._outSocket.destroy).toHaveBeenCalledTimes(1); + }); + + it("closes after a stall when no output was buffered", () => { + const agent = makeAgent(); + flush(agent); + + loopIteration(150); + expect(agent._outSocket.destroy).toHaveBeenCalledTimes(1); + }); +}); diff --git a/apps/server/src/testUtils/fakeWsl.ts b/apps/server/src/testUtils/fakeWsl.ts new file mode 100644 index 0000000000000..d8f88fcde8db8 --- /dev/null +++ b/apps/server/src/testUtils/fakeWsl.ts @@ -0,0 +1,37 @@ +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; + +// `wslpath` honours the distro's configurable automount root, so the fake maps drives under a temp dir. +export const makeFakeWslHost = Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const mountRoot = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-wsl-mnt-" }); + const binDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-wsl-bin-" }); + const wslpath = path.join(binDir, "wslpath"); + yield* fileSystem + .writeFileString( + wslpath, + [ + "#!/bin/sh", + '[ "$1" = "-u" ] || exit 1', + "p=$(printf '%s' \"$2\" | tr '\\\\' '/')", + "drive=$(printf '%s' \"$p\" | cut -c1 | tr 'A-Z' 'a-z')", + `printf '%s/%s%s\\n' '${mountRoot}' "$drive" "\${p#?:}"`, + "", + ].join("\n"), + ) + .pipe(Effect.orDie); + yield* fileSystem.chmod(wslpath, 0o755).pipe(Effect.orDie); + const environment = { + ...HostProcess.Environment.defaultValue(), + PATH: `${binDir}:${HostProcess.Environment.defaultValue().PATH ?? ""}`, + }; + const underWsl = (effect: Effect.Effect) => + effect.pipe( + Effect.provideService(HostProcess.Platform, "linux"), + Effect.provideService(HostProcess.Environment, environment), + ); + return { mountRoot, underWsl }; +}); diff --git a/apps/server/src/textGeneration/ClaudeTextGeneration.test.ts b/apps/server/src/textGeneration/ClaudeTextGeneration.test.ts index a7041d5d249e5..d2e165ea3b2b5 100644 --- a/apps/server/src/textGeneration/ClaudeTextGeneration.test.ts +++ b/apps/server/src/textGeneration/ClaudeTextGeneration.test.ts @@ -234,6 +234,35 @@ it.layer(layerClaudeTextGenerationTest)("ClaudeTextGeneration", (it) => { }), ); + it.effect("passes --settings from the instance launch args to Claude text generation", () => + withFakeClaudeEnv( + { + output: JSON.stringify({ + structured_output: { + title: "Use instance settings", + }, + }), + argsMustContain: `"env":{"ANTHROPIC_VERTEX_PROJECT_ID":"vertex-project-a"}`, + claudeConfig: { + launchArgs: `--settings '{"env":{"ANTHROPIC_VERTEX_PROJECT_ID":"vertex-project-a"}}'`, + }, + }, + (textGeneration) => + Effect.gen(function* () { + const generated = yield* textGeneration.generateThreadTitle({ + cwd: process.cwd(), + message: "thread title", + modelSelection: { + instanceId: ProviderInstanceId.make("claudeAgent"), + model: SYNTHETIC_CLAUDE_STANDARD_MODEL, + }, + }); + + expect(generated.title).toBe(sanitizeThreadTitle("Use instance settings")); + }), + ), + ); + it.effect("retries without --no-session-persistence when the Claude CLI rejects it", () => withFakeClaudeEnv( { diff --git a/apps/server/src/textGeneration/ClaudeTextGeneration.ts b/apps/server/src/textGeneration/ClaudeTextGeneration.ts index 34e7415b3c9f4..a523a3873a2c5 100644 --- a/apps/server/src/textGeneration/ClaudeTextGeneration.ts +++ b/apps/server/src/textGeneration/ClaudeTextGeneration.ts @@ -1,7 +1,9 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Option from "effect/Option"; +import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; @@ -32,6 +34,7 @@ import { hasClaudeManagedMcpConfig, makeClaudeEnvironment, } from "../provider/Drivers/ClaudeHome.ts"; +import { readClaudeLaunchArgSettings } from "../orchestration-v2/Adapters/ClaudeAdapterV2.ts"; const CLAUDE_TIMEOUT_MS = 180_000; /** `--tools` takes bare names; patterns go in the allow list. */ @@ -97,9 +100,14 @@ export const makeClaudeTextGeneration = Effect.fn("makeClaudeTextGeneration")(fu claudeSettings: ClaudeSettings, environment?: NodeJS.ProcessEnv, modelCatalog: Effect.Effect = Effect.succeed(BUNDLED_CLAUDE_MODEL_CATALOG), + routeModelCatalog: ( + catalog: ClaudeModelCatalog, + cwd: string, + ) => Effect.Effect = Effect.succeed, ) { const commandSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; const claudeEnvironment = yield* makeClaudeEnvironment(claudeSettings, environment); const scopedModelCatalog = modelCatalog.pipe( Effect.map((catalog) => scopeClaudeModelCatalog(catalog, claudeSettings.customModels)), @@ -146,9 +154,19 @@ export const makeClaudeTextGeneration = Effect.fn("makeClaudeTextGeneration")(fu }: TextGenerationOperations.Request): Effect.fn.Return< S["Type"], TextGenerationError, - S["DecodingServices"] + S["DecodingServices"] | Scope.Scope > { - const catalog = yield* scopedModelCatalog; + const workingDirectory = + operation === "generateThreadTitle" + ? yield* fileSystem + .makeTempDirectoryScoped({ prefix: "t3code-claude-title-" }) + .pipe( + Effect.mapError((cause) => + normalizeCliError("claude", operation, cause, "Failed to create title directory"), + ), + ) + : cwd; + const catalog = yield* routeModelCatalog(yield* scopedModelCatalog, workingDirectory); const resolvedModelSelection = { ...modelSelection, model: resolveClaudeModelSlug(catalog, modelSelection.model), @@ -185,7 +203,18 @@ export const makeClaudeTextGeneration = Effect.fn("makeClaudeTextGeneration")(fu : undefined; const fastMode = fastModeDescriptor?.type === "boolean" ? fastModeDescriptor.currentValue : undefined; + const launchArgSettings = yield* readClaudeLaunchArgSettings({ + settings: claudeSettings, + cwd, + fileSystem, + path, + }).pipe( + Effect.mapError( + (cause) => new TextGenerationError({ operation, detail: cause.message, cause }), + ), + ); const settings = { + ...launchArgSettings, disableAllHooks: true, ...(typeof thinking === "boolean" ? { alwaysThinkingEnabled: thinking } : {}), ...(fastMode ? { fastMode: true } : {}), @@ -200,16 +229,6 @@ export const makeClaudeTextGeneration = Effect.fn("makeClaudeTextGeneration")(fu const runClaudeCommand = Effect.fn("runClaudeJson.runClaudeCommand")(function* ( noSessionPersistence: boolean, ) { - const workingDirectory = - operation === "generateThreadTitle" - ? yield* fileSystem - .makeTempDirectoryScoped({ prefix: "t3code-claude-title-" }) - .pipe( - Effect.mapError((cause) => - normalizeCliError("claude", operation, cause, "Failed to create title directory"), - ), - ) - : cwd; const managedMcpConfig = yield* hasClaudeManagedMcpConfig(); const spawnCommand = yield* resolveSpawnCommand( claudeSettings.binaryPath || "claude", @@ -344,5 +363,9 @@ export const makeClaudeTextGeneration = Effect.fn("makeClaudeTextGeneration")(fu ); }); - return TextGenerationOperations.fromRunner("ClaudeTextGeneration", runClaudeJson); + return TextGenerationOperations.fromRunner( + "ClaudeTextGeneration", + (request: TextGenerationOperations.Request) => + Effect.scoped(runClaudeJson(request)), + ); }); diff --git a/apps/server/src/textGeneration/CodexTextGeneration.test.ts b/apps/server/src/textGeneration/CodexTextGeneration.test.ts index e6e43d606e27f..1a1a5c43b0a0d 100644 --- a/apps/server/src/textGeneration/CodexTextGeneration.test.ts +++ b/apps/server/src/textGeneration/CodexTextGeneration.test.ts @@ -499,6 +499,26 @@ it.layer(layerCodexTextGenerationTest)("CodexTextGeneration", (it) => { ), ); + it.effect("keeps user hooks off over launch args that turn them on", () => + withFakeCodexEnv( + { + output: JSON.stringify({ title: "Hookless title" }), + launchArgs: "--config features.hooks=true --enable hooks", + requireArg: "--config features.codex_hooks=false --config features.hooks=false", + forbidArg: "--enable", + }, + (textGeneration) => + Effect.gen(function* () { + const result = yield* textGeneration.generateThreadTitle({ + cwd: process.cwd(), + message: "Describe this change", + modelSelection: DEFAULT_TEST_MODEL_SELECTION, + }); + expect(result.title).toBe("Hookless title"); + }), + ), + ); + it.effect("passes exec-safe launch args into codex exec", () => withFakeCodexEnv( { diff --git a/apps/server/src/textGeneration/CodexTextGeneration.ts b/apps/server/src/textGeneration/CodexTextGeneration.ts index 49c60166111ba..fc741aaaeb262 100644 --- a/apps/server/src/textGeneration/CodexTextGeneration.ts +++ b/apps/server/src/textGeneration/CodexTextGeneration.ts @@ -326,6 +326,11 @@ export const makeCodexTextGeneration = Effect.fn("makeCodexTextGeneration")(func model, "--config", `model_reasoning_effort="${reasoningEffort}"`, + // Current Codex gives hooks precedence over the historical codex_hooks alias. + "--config", + "features.codex_hooks=false", + "--config", + "features.hooks=false", ...(serviceTier ? ["--config", `service_tier="${serviceTier}"`] : []), "--output-schema", schemaPath, diff --git a/apps/server/src/textGeneration/GrokTextGeneration.test.ts b/apps/server/src/textGeneration/GrokTextGeneration.test.ts index 26ab9e79c7103..adb911623f800 100644 --- a/apps/server/src/textGeneration/GrokTextGeneration.test.ts +++ b/apps/server/src/textGeneration/GrokTextGeneration.test.ts @@ -35,7 +35,7 @@ function makeAcpGrokWrapper(dir: string, env: Record): string { env, source: execScriptSource({ scriptPath: mockAgentPath, - expectedArgs: ["agent", "stdio"], + expectedArgs: ["--permission-mode", "default", "agent", "stdio"], }), }); } @@ -58,14 +58,19 @@ function withFakeAcpGrok( }).pipe(Effect.scoped); } -function readJsonRpcRequests( - filePath: string, -): ReadonlyArray<{ readonly method?: string; readonly params?: Record }> { +function readJsonRpcRequests(filePath: string): ReadonlyArray<{ + readonly id?: unknown; + readonly method?: string; + readonly params?: Record; +}> { return NodeFS.readFileSync(filePath, "utf8") .trim() .split("\n") .filter((line) => line.length > 0) - .map((line) => JSON.parse(line) as { method?: string; params?: Record }); + .map( + (line) => + JSON.parse(line) as { id?: unknown; method?: string; params?: Record }, + ); } it.layer(layerGrokTextGenerationTest)("GrokTextGeneration", (it) => { @@ -78,6 +83,7 @@ it.layer(layerGrokTextGenerationTest)("GrokTextGeneration", (it) => { return withFakeAcpGrok( { T3_ACP_REQUEST_LOG_PATH: requestLogPath, + T3_ACP_REQUEST_PERMISSION: "1", T3_ACP_PROMPT_RESPONSE_TEXT: JSON.stringify({ subject: "Add Grok provider", body: "Wire up the ACP runtime and headless text generation path.", @@ -110,6 +116,9 @@ it.layer(layerGrokTextGenerationTest)("GrokTextGeneration", (it) => { request.params?.modelId === "grok-mock-alt", ), ).toBe(true); + expect(requests.find((request) => request.id === 9001)).toMatchObject({ + result: { outcome: { outcome: "cancelled" } }, + }); }), ); }); diff --git a/apps/server/src/textGeneration/TextGenerationPresets.ts b/apps/server/src/textGeneration/TextGenerationPresets.ts index b53ecc543597d..2dd3b95321290 100644 --- a/apps/server/src/textGeneration/TextGenerationPresets.ts +++ b/apps/server/src/textGeneration/TextGenerationPresets.ts @@ -3,7 +3,7 @@ import type { TextGenerationPolicy } from "@t3tools/provider-core/server/textGen export const conventionalCommitsTextGenerationPolicy: TextGenerationPolicy = { kind: "conventional_commits", commitInstructions: - "Use Conventional Commits when generating commit subjects. Prefer the narrowest accurate type and include a scope only when it is obvious from the diff.", + "Use Conventional Commits when generating commit subjects. Prefer the narrowest accurate type and include a scope only when it is obvious from the diff. Start the description after the colon with a lowercase word, never a proper noun or acronym, because commitlint rejects a capitalized description.", changeRequestInstructions: "Keep the change request title concise. Do not force Conventional Commit syntax into the title unless the repository already uses it.", inferRepositoryConventions: false, diff --git a/apps/server/src/usage/UsageLimitSources.test.ts b/apps/server/src/usage/UsageLimitSources.test.ts index d7e73a1da96c9..fc8a93f9e1bc6 100644 --- a/apps/server/src/usage/UsageLimitSources.test.ts +++ b/apps/server/src/usage/UsageLimitSources.test.ts @@ -44,6 +44,7 @@ it.effect("keeps malformed source URLs encodable for config subscribers", () => "https://hub.test:8317", undefined, ]); + expect(snapshots[3]?.endpoint).toBe("https://hub.test:8317/team-a"); expect(() => encodeConfigEvent({ version: 1, @@ -84,7 +85,7 @@ it.effect("keeps malformed source URLs encodable for config subscribers", () => }, [UsageLimitSourceId.make("valid-hub")]: { kind: "cliproxy", - url: "https://hub.test:8317", + url: "https://hub.test:8317/team-a/management.html", managementKey: "", enabled: true, }, diff --git a/apps/server/src/usage/UsageLimitSources.ts b/apps/server/src/usage/UsageLimitSources.ts index c591931843411..d6f867d621d23 100644 --- a/apps/server/src/usage/UsageLimitSources.ts +++ b/apps/server/src/usage/UsageLimitSources.ts @@ -86,7 +86,12 @@ export const make = Effect.gen(function* () { id, kind: config.kind, label: sourceLabel(id, config), - ...(url && /^https?:$/.test(url.protocol) ? { origin: url.origin } : {}), + ...(url && /^https?:$/.test(url.protocol) + ? { + origin: url.origin, + endpoint: `${url.origin}${url.pathname.replace(/(\/management\.html)?\/*$/i, "")}`, + } + : {}), checkedAt, } as const; const managementKey = yield* readManagementKey(config); diff --git a/apps/server/src/usage/UsageService.test.ts b/apps/server/src/usage/UsageService.test.ts index dc3f7ac927147..01cbf3a9f52ce 100644 --- a/apps/server/src/usage/UsageService.test.ts +++ b/apps/server/src/usage/UsageService.test.ts @@ -51,6 +51,7 @@ import * as AntigravityUsage from "../provider/Drivers/AntigravityUsage.ts"; import * as ProviderHostLive from "../provider/ProviderHostLive.ts"; import { totalTokens, type UsageRecord } from "@t3tools/provider-core/server/usage"; import * as UsageService from "./UsageService.ts"; +import { DESKTOP_HOST_SCOPE_ENV, physicalSourceIdOf } from "./usageTranscriptReader.ts"; import * as MuseSdk from "../../../../packages/provider-muse/src/server/sdk.ts"; import { MuseUsageHome } from "@t3tools/provider-muse/server"; import { makeScanCacheWriter, SCAN_CACHE_FILE_NAME } from "./usageScanCache.ts"; @@ -213,6 +214,7 @@ const layerService = (input: { Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), Layer.provideMerge(Layer.succeed(HostProcess.Platform, input.platform ?? "linux")), Layer.provideMerge(Layer.succeed(HostProcess.Hostname, input.hostname ?? "usage-test-host")), + Layer.provideMerge(Layer.succeed(HostProcess.HomeDirectory, input.home)), Layer.provideMerge(ServerSettings.layerTest(input.settings)), Layer.provideMerge( Layer.succeed( @@ -521,6 +523,73 @@ describe("Codex usage history", () => { }); describe("UsageService", () => { + it.effect("keeps pricing metadata paired with the rates used during a concurrent refresh", () => + Effect.gen(function* () { + const { home, settings } = yield* setup; + const sessions = NodePath.join(home, "codex", "sessions"); + yield* Effect.promise(async () => { + await NodeFSP.mkdir(sessions, { recursive: true }); + await NodeFSP.writeFile( + NodePath.join(sessions, "session.jsonl"), + [ + { type: "session_meta", payload: { id: "session", model_provider: "openai" } }, + { type: "turn_context", payload: { model: "gpt-6-astra", turn_id: "turn" } }, + { + type: "event_msg", + timestamp: "2026-08-01T10:00:00Z", + payload: { + type: "token_count", + info: { last_token_usage: { input_tokens: 0, output_tokens: 1 } }, + }, + }, + ] + .map((record) => encodeUnknownJsonString(record)) + .join("\n") + "\n", + ); + }); + let rate = 0; + yield* Effect.gen(function* () { + const resolving = yield* Deferred.make(); + const release = yield* Deferred.make(); + const service = yield* UsageService.make.pipe( + Effect.provideService(CodexUsageHistory.CodexUsageHistory, { + resolve: () => + Deferred.succeed(resolving, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.as(new Map()), + ), + }), + ); + const initialPricing = yield* service.refreshRates; + yield* TestClock.adjust(61_000); + const pending = yield* service.readSummary(WINDOW).pipe(Effect.forkChild); + yield* Deferred.await(resolving); + yield* service.refreshRates; + yield* Deferred.succeed(release, undefined); + const original = yield* Fiber.join(pending); + assert.strictEqual(original.buckets[0]?.costUsd, 1); + assert.strictEqual(original.pricing.fetchedAt, initialPricing.fetchedAt); + const updated = yield* service.readSummary(WINDOW); + assert.strictEqual(updated.buckets[0]?.costUsd, 2); + assert.notStrictEqual(updated.pricing.fetchedAt, initialPricing.fetchedAt); + }).pipe( + Effect.provide( + layerService({ + prefix: "concurrent-rates", + home, + settings, + ratesResponse: () => + Effect.succeed( + Response.json({ + "gpt-6-astra": { input_cost_per_token: 0, output_cost_per_token: ++rate }, + }), + ), + }), + ), + ); + }).pipe(Effect.scoped), + ); + it.effect("counts compressed Codex rollouts across refresh, rewrites, and cache restarts", () => Effect.gen(function* () { const { home, settings } = yield* setup; @@ -1158,6 +1227,76 @@ describe("UsageService", () => { }), ); + it.effect("prices Pi usage mapped onto a Pi model at that model's rate", () => + Effect.gen(function* () { + const { home, settings } = yield* setup; + const dir = NodePath.join(home, "pi", "sessions", "project"); + const message = (id: string, model: string) => ({ + type: "message", + id, + timestamp: "2026-08-01T10:00:00Z", + message: { + role: "assistant", + provider: "anthropic", + model, + usage: { input: 10, output: 5 }, + }, + }); + yield* Effect.promise(async () => { + await NodeFSP.mkdir(dir, { recursive: true }); + await NodeFSP.writeFile( + NodePath.join(dir, "session.jsonl"), + [ + { type: "session", id: "session" }, + message("released", "actual-model"), + message("preview", "actual-model-preview"), + ] + .map((line) => encodeUnknownJsonString(line)) + .join("\n") + "\n", + ); + }); + const summary = yield* Effect.gen(function* () { + const service = yield* UsageService.make; + return yield* service.readSummary(WINDOW); + }).pipe( + Effect.scoped, + Effect.provide( + layerService({ + prefix: "usage-pi-model-alias", + home, + ratesDocument: { + "actual-model": { input_cost_per_token: 1, output_cost_per_token: 4 }, + }, + settings: { + ...settings, + usageModelAliases: { "anthropic/actual-model-preview": "anthropic/actual-model" }, + }, + }), + ), + ); + assert.deepStrictEqual( + summary.buckets + .filter((bucket) => bucket.provider === "pi") + .map(({ model, records, unpricedRecords, costUsd, costSource }) => ({ + model, + records, + unpricedRecords, + costUsd, + costSource, + })), + [ + { + model: "anthropic/actual-model", + records: 2, + unpricedRecords: 0, + costUsd: 60, + costSource: "modelPriced", + }, + ], + ); + }), + ); + it.effect( "reads Muse inference history once, persists appended usage, and reprices unknown models", () => @@ -1440,6 +1579,39 @@ describe("UsageService", () => { }), ); + it.effect("identifies a transcript directory created after the server first read usage", () => + Effect.gen(function* () { + const { home, settings } = yield* setup; + const scope = "0123456789abcdef0123456789abcdef"; + const dir = NodePath.join(home, "pi", "sessions"); + yield* Effect.gen(function* () { + const service = yield* UsageService.make; + const piFingerprint = (summary: UsageSummary) => + summary.sources.find((source) => source.fingerprint.provider === "pi")?.fingerprint; + const missing = piFingerprint(yield* service.readSummary(WINDOW)); + assert.strictEqual(missing?.resolvedHomePath, dir); + assert.isUndefined(missing?.physicalSourceId); + yield* Effect.promise(() => NodeFSP.mkdir(dir, { recursive: true })); + assert.strictEqual( + piFingerprint(yield* service.readSummary(WINDOW))?.physicalSourceId, + physicalSourceIdOf(scope, yield* Effect.promise(() => NodeFSP.realpath(dir))), + ); + }).pipe( + Effect.scoped, + Effect.provide( + layerService({ + prefix: "usage-physical-source-created", + home, + settings, + // The native Windows server reads identity from realpath alone. + platform: "win32", + environment: { [DESKTOP_HOST_SCOPE_ENV]: scope }, + }), + ), + ); + }), + ); + it.effect( "reports missing Pi history and skips malformed records without losing valid usage", () => @@ -2392,6 +2564,34 @@ describe("UsageService", () => { }), ); + it.effect("keeps default-home history after the default account moves to a custom home", () => + Effect.gen(function* () { + const { transcript, settings, home } = yield* setup; + const defaultHome = NodePath.join(home, "default-claude"); + const defaultProjects = NodePath.join(defaultHome, "projects", "proj"); + yield* Effect.promise(async () => { + await NodeFSP.writeFile(transcript, claudeLine(1, 5)); + await NodeFSP.mkdir(defaultProjects, { recursive: true }); + await NodeFSP.writeFile( + NodePath.join(defaultProjects, "earlier.jsonl"), + claudeLine(2, 7, "claude-fable-5", "earlier-session"), + ); + }); + const summary = yield* UsageService.make.pipe( + Effect.flatMap((service) => service.readSummary(WINDOW)), + Effect.provide( + layerService({ + prefix: "usage-service-default-home-test", + home, + environment: { CLAUDE_CONFIG_DIR: defaultHome }, + settings, + }), + ), + ); + assert.strictEqual(totalOutputTokens(summary), 12); + }), + ); + it.effect( "uses explicit account settings before environment and default homes, then refreshes them", () => @@ -3106,7 +3306,7 @@ describe("UsageService", () => { 60 + 20 * (version === 4 ? 1 : 2), ); yield* upgraded.awaitPersisted; - assert.strictEqual(SCAN_CACHE_FILE_NAME, "usage-scan-cache-v16.json"); + assert.strictEqual(SCAN_CACHE_FILE_NAME, "usage-scan-cache-v17.json"); assert.deepStrictEqual( (yield* Effect.promise(() => NodeFSP.readdir(stateDir))) .filter((name) => name.startsWith("usage-scan-cache")) @@ -3670,6 +3870,56 @@ describe("UsageService", () => { }), ); + it.effect("refetches cached rates early for a model they do not list", () => + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse("2026-08-02T00:00:00Z")); + const { transcript, settings, home } = yield* setup; + yield* Effect.promise(() => + NodeFSP.writeFile( + transcript, + claudeLine(1, 5, "claude-new-model") + claudeLine(2, 5, "claude-unknown-model"), + ), + ); + const requests: string[] = []; + let ratesDocument: Record = { + "claude-fable-5": { input_cost_per_token: 1e-5, output_cost_per_token: 5e-5 }, + }; + const serviceLayer = layerService({ + prefix: "usage-service-unlisted-model-test", + home, + settings, + onRatesFetch: (request) => requests.push(request.url), + ratesResponse: () => Effect.succeed(Response.json(ratesDocument)), + }); + const newModelCost = (summary: UsageSummary) => + summary.buckets.find((bucket) => bucket.model === "claude-new-model")?.costUsd; + + // Rates fetched for this scan are fresh: no second request for the models they lack. + const first = yield* (yield* UsageService.make.pipe( + Effect.provide(serviceLayer), + )).readSummary(WINDOW); + assert.strictEqual(newModelCost(first), 0); + assert.strictEqual(requests.length, 1); + + // An hour later LiteLLM lists the new model; a restarted server starts from the cached table. + ratesDocument = { + ...ratesDocument, + "claude-new-model": { input_cost_per_token: 2e-5, output_cost_per_token: 1e-4 }, + }; + yield* TestClock.adjust(Duration.hours(1)); + const service = yield* UsageService.make.pipe(Effect.provide(serviceLayer)); + const refreshed = yield* service.readSummary(WINDOW); + assert.strictEqual(refreshed.pricing.status, "fresh"); + assert.closeTo(newModelCost(refreshed) ?? -1, 10 * 2e-5 + 5 * 1e-4, 1e-12); + assert.strictEqual(requests.length, 2); + + // A model the fresh table still lacks does not refetch again before the TTL. + yield* TestClock.adjust(Duration.hours(1)); + yield* service.readSummary(WINDOW); + assert.strictEqual(requests.length, 2); + }), + ); + it.effect("does not orphan an in-flight scan when its first caller is interrupted", () => Effect.gen(function* () { const { settings, home } = yield* setup; diff --git a/apps/server/src/usage/UsageService.ts b/apps/server/src/usage/UsageService.ts index a55532913c3c6..a59fbc66a02ca 100644 --- a/apps/server/src/usage/UsageService.ts +++ b/apps/server/src/usage/UsageService.ts @@ -40,10 +40,17 @@ import type { UsageRecord, } from "@t3tools/provider-core/server/usage"; import { canonicalModelFilter, resolveModelAliases, UsageAggregator } from "./usageAggregation.ts"; -import { createOverrideRateTable, parseRateTable, type RateTable } from "./usagePricing.ts"; import { + createOverrideRateTable, + isUnlistedModel, + parseRateTable, + type RateTable, +} from "./usagePricing.ts"; +import { + DESKTOP_HOST_SCOPE_ENV, listTranscriptFiles, readDirectoryVolumeId, + readPhysicalSourceId, readTranscriptRecords, } from "./usageTranscriptReader.ts"; import { @@ -83,6 +90,12 @@ const scanReaders = BUILT_IN_USAGE_DRIVERS.flatMap((driver) => driver.usage?.kind === "scan" ? [{ driver, reader: driver.usage }] : [], ); +const providerRateModels = new Map( + BUILT_IN_USAGE_DRIVERS.flatMap(({ usage }) => + usage?.rateModel === undefined ? [] : [[usage.provider, usage.rateModel] as const], + ), +); + const transcriptFormats = new Map( transcriptReaders.map(({ reader }) => [reader.provider, reader.format] as const), ); @@ -92,6 +105,7 @@ interface TranscriptSource { readonly format: TranscriptUsageFormat; readonly dir: string; readonly volumeId: string; + readonly physicalSourceId?: string; readonly fileName?: string; } @@ -158,6 +172,24 @@ export const make = Effect.gen(function* () { const settingsService = yield* ServerSettings.ServerSettingsService; const httpClient = yield* HttpClient.HttpClient; const hostEnvironment = yield* HostProcess.Environment; + const hostPlatform = yield* HostProcess.Platform; + const hostScope = hostEnvironment[DESKTOP_HOST_SCOPE_ENV]?.trim(); + // A mount or a directory's on-disk name does not change under a running server. + // A miss is not cached: the directory may not exist yet. + const physicalSourceIds = new Map>(); + const physicalSourceId = (dir: string) => + Effect.promise(() => { + const cached = physicalSourceIds.get(dir); + if (cached !== undefined) return cached; + const id = readPhysicalSourceId(dir, hostScope, hostPlatform); + physicalSourceIds.set(dir, id); + void id.then((resolved) => { + if (resolved === undefined && physicalSourceIds.get(dir) === id) { + physicalSourceIds.delete(dir); + } + }); + return id; + }); const hostname = yield* HostProcess.Hostname; const readerContext = yield* Effect.context(); @@ -291,6 +323,24 @@ export const make = Effect.gen(function* () { const ensureRates = (force: boolean) => ratesLock.withPermit(loadRates(force)); + // Rates are kept for RATES_TTL_MS, so a model released after the table was fetched would stay unpriced + // that long. A scan that meets a model the table does not list refetches early: once per model against + // a fresh table, with loadRates keeping RATES_REFRESH_FLOOR_MS between attempts. + const checkedUnlistedModels = new Set(); + const refreshForUnlistedModels = Effect.fn("UsageService.refreshForUnlistedModels")(function* ( + models: ReadonlySet, + ) { + const unlisted = [...models].filter( + (model) => !checkedUnlistedModels.has(model) && isUnlistedModel(rates, model), + ); + if (unlisted.length === 0) return; + yield* ensureRates(true); + const now = yield* Clock.currentTimeMillis; + // A failed fetch leaves the models unchecked, so a later scan tries again. + if (ratesFetchedAtMs === null || now - ratesFetchedAtMs >= RATES_REFRESH_FLOOR_MS) return; + for (const model of unlisted) checkedUnlistedModels.add(model); + }); + const refreshRates = ensureRates(true).pipe( Effect.map(pricing), Effect.withSpan("UsageService.refreshRates"), @@ -316,9 +366,8 @@ export const make = Effect.gen(function* () { > = Object.entries(settings.providerInstances) .filter(([, instance]) => instance.driver === driver.driverKind) .map(([id, instance]) => [ProviderInstanceId.make(id), instance, true] as const); - if (!Object.hasOwn(settings.providerInstances, driver.driverKind)) { - entries.push([ProviderInstanceId.make(driver.driverKind), {}, false]); - } + // History in the driver's default home stays counted after every instance moves elsewhere; readers dedupe shared directories. + entries.push([ProviderInstanceId.make(driver.driverKind), {}, false]); const decodeConfig = Schema.decodeUnknownOption(driver.configSchema); return yield* Effect.forEach( entries, @@ -385,11 +434,13 @@ export const make = Effect.gen(function* () { const key = `${provider}\0${dir}`; if (seen.has(key)) continue; seen.add(key); + const sourceId = yield* physicalSourceId(dir); dirs.push({ provider, format, dir, volumeId, + ...(sourceId === undefined ? {} : { physicalSourceId: sourceId }), ...(fileName === undefined ? {} : { fileName }), }); } @@ -629,6 +680,7 @@ export const make = Effect.gen(function* () { readonly provider: UsageProviderKind; readonly dir: string; readonly volumeId: string; + readonly physicalSourceId?: string; readonly hostId?: string; readonly status?: UsageSource["status"]; readonly message?: string; @@ -649,16 +701,22 @@ export const make = Effect.gen(function* () { windowStartMs: number, fromMemory: boolean, ) { - const { provider, format, dir, volumeId, fileName } = source; + const { provider, format, dir, volumeId, physicalSourceId, fileName } = source; + const identity = { + provider, + dir, + volumeId, + ...(physicalSourceId === undefined ? {} : { physicalSourceId }), + }; const exists = yield* fileSystem .exists(dir) .pipe(Effect.catchCause(() => Effect.succeed(false))); if (!exists) { cover(source, windowStartMs); - return { provider, dir, volumeId, files: null } satisfies ScannedDir; + return { ...identity, files: null } satisfies ScannedDir; } // scanSummary aggregates the saved files, as it does for a deleted directory. - if (fromMemory) return { provider, dir, volumeId, files: [] } satisfies ScannedDir; + if (fromMemory) return { ...identity, files: [] } satisfies ScannedDir; const { files, walked, failedPaths } = yield* Effect.promise(() => listTranscriptFiles(dir, windowStartMs, fileName === undefined ? undefined : { fileName }), ); @@ -704,9 +762,7 @@ export const make = Effect.gen(function* () { "usage.files_parsed": read.filter((file) => file.update !== undefined).length, }); return { - provider, - dir, - volumeId, + ...identity, files: parsedFiles, walked, ...(unread > 0 @@ -736,14 +792,19 @@ export const make = Effect.gen(function* () { Effect.flatMap((sources) => Effect.forEach(sources, ({ volumeId, ...source }) => Effect.map( + // A reader's own volumeId names a remote account, not a local directory. volumeId === undefined - ? Effect.promise(() => readDirectoryVolumeId(source.dir)) - : Effect.succeed(volumeId), - (resolved): ScannedDir => ({ + ? Effect.all([ + Effect.promise(() => readDirectoryVolumeId(source.dir)), + physicalSourceId(source.dir), + ]) + : Effect.succeed<[string, string | undefined]>([volumeId, undefined]), + ([resolved, sourceId]): ScannedDir => ({ ...source, files: source.files?.map((file) => ({ ...file, malformedRecords: 0 })) ?? null, provider: reader.provider, volumeId: resolved, + ...(sourceId === undefined ? {} : { physicalSourceId: sourceId }), }), ), ), @@ -848,18 +909,8 @@ export const make = Effect.gen(function* () { { concurrency: 2 }, ); - const aggregator = new UsageAggregator({ - timeZone: input.timeZone, - sinceDay: input.sinceDay, - untilDay: input.untilDay, - resolution: input.resolution ?? "day", - ...hourlyWindow, - rates, - priceOverrides: createOverrideRateTable(settings.usagePriceOverrides), - modelAliases: resolveModelAliases(settings.usageModelAliases), - ...(input.modelFilter === undefined ? {} : { modelFilter: input.modelFilter }), - }); - + const priceOverrides = createOverrideRateTable(settings.usagePriceOverrides); + const modelAliases = resolveModelAliases(settings.usageModelAliases); const sources: UsageSource[] = []; // Deleted transcripts' saved usage still aggregates and deduplicates normally. @@ -925,6 +976,37 @@ export const make = Effect.gen(function* () { for (const filePath of ordered) fileCache.set(filePath, entries.get(filePath)!); cacheDirty = true; } + + // The models priced below, as UsageAggregator resolves them: an alias replaces rateModel. + const rateModels = new Set(); + for (const files of filesByDir) { + for (const file of files) { + for (const record of file.records) { + const alias = modelAliases.get(record.model); + const model = alias ?? record.model; + if (priceOverrides.has(model.trim())) continue; + rateModels.add(alias === undefined ? (record.rateModel ?? model) : model); + } + } + } + // A saved answer prices with saved rates and the full scan after it refetches. + if (!fromMemory) yield* refreshForUnlistedModels(rateModels); + + const summaryPricing = pricing(); + + const aggregator = new UsageAggregator({ + timeZone: input.timeZone, + sinceDay: input.sinceDay, + untilDay: input.untilDay, + resolution: input.resolution ?? "day", + ...hourlyWindow, + rates, + priceOverrides, + modelAliases, + rateModel: (provider, model) => providerRateModels.get(provider)?.(model), + ...(input.modelFilter === undefined ? {} : { modelFilter: input.modelFilter }), + }); + const windowEndMs = hourlyWindow?.untilTimeMs ?? Date.parse(`${input.untilDay}T00:00:00Z`) + 86400000 + MTIME_SLACK_MS; @@ -972,7 +1054,17 @@ export const make = Effect.gen(function* () { for (const [ index, - { provider, dir, volumeId, files, status, message, refreshing, hostId: sourceHostId }, + { + provider, + dir, + volumeId, + physicalSourceId, + files, + status, + message, + refreshing, + hostId: sourceHostId, + }, ] of scannedDirs.entries()) { let scannedFiles = 0; let skippedFiles = 0; @@ -1019,7 +1111,13 @@ export const make = Effect.gen(function* () { const sourceMessage = message ?? (files === null ? "No transcript directory on this environment." : null); sources.push({ - fingerprint: { hostId: sourceHostId ?? hostId, provider, resolvedHomePath: dir, volumeId }, + fingerprint: { + hostId: sourceHostId ?? hostId, + provider, + resolvedHomePath: dir, + volumeId, + ...(physicalSourceId === undefined ? {} : { physicalSourceId }), + }, status: files === null && scannedFiles === 0 ? "missing" @@ -1067,7 +1165,7 @@ export const make = Effect.gen(function* () { untilDay: input.untilDay, buckets: aggregated.buckets, sources, - pricing: pricing(), + pricing: summaryPricing, scanDurationMs: Math.max(0, finishedAtMs - startedAtMs), ...(input.modelFilter === undefined ? {} : { modelFilter: input.modelFilter }), } satisfies UsageSummary; diff --git a/apps/server/src/usage/cliproxyApi.test.ts b/apps/server/src/usage/cliproxyApi.test.ts index 431af7b5971ff..5002d2223d1ba 100644 --- a/apps/server/src/usage/cliproxyApi.test.ts +++ b/apps/server/src/usage/cliproxyApi.test.ts @@ -437,6 +437,26 @@ describe("CLIProxyAPI built-in management API", () => { }), ); + it.effect("keeps a Claude account whose window reports no utilization", () => + Effect.gen(function* () { + const test = fixture({ + accounts: [{ ...accounts[0]!, provider: "claude" }], + upstream: () => ({ + status: 200, + body: { + five_hour: { utilization: null, resets_at: null }, + seven_day: { utilization: 50, resets_at: "2099-01-01T00:00:00Z" }, + }, + }), + }); + const result = yield* (yield* test.api).readAccounts(config); + expect(result[0]?.usageLimits.unavailable).toBeUndefined(); + expect( + result[0]?.usageLimits.windows.map((window) => [window.id, window.usedPercent]), + ).toEqual([["seven_day", 50]]); + }), + ); + const antigravitySummary = { buckets: [ { diff --git a/apps/server/src/usage/cliproxyApi.ts b/apps/server/src/usage/cliproxyApi.ts index 030983f50e33d..537a2b436f4a4 100644 --- a/apps/server/src/usage/cliproxyApi.ts +++ b/apps/server/src/usage/cliproxyApi.ts @@ -70,8 +70,9 @@ const CodexUsage = Schema.Struct({ }), ), }); +// The usage endpoint can report a window without a utilization, as the CLI's own schema allows. const ClaudeWindow = Schema.Struct({ - utilization: Schema.Number, + utilization: Schema.NullOr(Schema.Number), resets_at: Schema.NullOr(Schema.String), }); const ClaudeUsage = Schema.Struct({ diff --git a/apps/server/src/usage/usageAggregation.test.ts b/apps/server/src/usage/usageAggregation.test.ts index f99615085bd2a..58953fbcfafea 100644 --- a/apps/server/src/usage/usageAggregation.test.ts +++ b/apps/server/src/usage/usageAggregation.test.ts @@ -141,6 +141,32 @@ describe("UsageAggregator", () => { } }); + it("keeps a live Grok turn over a saved pre-v17 maximum in either source order", () => { + const turn = (timestampMs: number, outputTokens: number) => + record({ + provider: "grok", + model: "grok-4.5", + timestampMs, + dedupeKey: `s:p:${timestampMs}:grok-4.5`, + reportedCostUsd: outputTokens / 10, + totals: { ...record().totals, uncachedInputTokens: 0, outputTokens }, + }); + const first = turn(Date.parse("2026-08-08T10:00:00Z"), 10); + const resumed = turn(Date.parse("2026-08-08T11:00:00Z"), 20); + const savedMaximum = { ...first, totals: resumed.totals, reportedCostUsd: 2 }; + + const expected = aggregate([first, resumed]).buckets; + + for (const order of [ + [first, resumed, savedMaximum], + [savedMaximum, first, resumed], + ]) { + const result = aggregate(order); + expect(result.duplicatesDropped).toBe(1); + expect(result.buckets).toEqual(expected); + } + }); + it("does not count a key whose first copy fell outside the window", () => { const result = aggregate([ record({ dedupeKey: "msg_1:", timestampMs: Date.parse("2026-07-01T12:00:00Z") }), @@ -251,6 +277,7 @@ describe("UsageAggregator", () => { untilDay: "2026-08-31", rates, modelAliases: resolveModelAliases({ "example-preview": "claude-fable-5" }), + rateModel: () => "claude-fable", }); aggregator.add(record()); aggregator.add(record({ model: "example-preview", rateModel: "example-preview-high" })); @@ -266,20 +293,24 @@ describe("UsageAggregator", () => { describe("resolveModelAliases", () => { it("follows chains to the final model and drops chains that enter a loop", () => { - expect( - resolveModelAliases({ + expect([ + ...resolveModelAliases({ "preview[1m]": "preview", preview: "example-model", + shared: "preview", + intoLoop: "loop", loop: "back", back: "loop", - intoLoop: "loop", + laterIntoLoop: "back", + deepIntoLoop: "intoLoop", self: "self", + validAfterLoops: "shared", }), - ).toEqual( - new Map([ - ["preview[1m]", "example-model"], - ["preview", "example-model"], - ]), - ); + ]).toEqual([ + ["preview[1m]", "example-model"], + ["preview", "example-model"], + ["shared", "example-model"], + ["validAfterLoops", "example-model"], + ]); }); }); diff --git a/apps/server/src/usage/usageAggregation.ts b/apps/server/src/usage/usageAggregation.ts index 9ff9cb9369462..f4c39753548fd 100644 --- a/apps/server/src/usage/usageAggregation.ts +++ b/apps/server/src/usage/usageAggregation.ts @@ -10,7 +10,7 @@ import type { } from "@t3tools/contracts"; import { EMPTY_TOTALS, type UsageRecord } from "@t3tools/provider-core/server/usage"; -import { cacheSavingsUsd, priceUsage, type RateTable } from "./usagePricing.ts"; +import { cacheSavingsUsd, lookupRate, priceUsage, type RateTable } from "./usagePricing.ts"; // en-CA gives ISO-ordered parts in the requested zone; Date getters are host-local. function makeDayFormatter(timeZone: string): (timestampMs: number) => string { @@ -70,6 +70,7 @@ export interface AggregateOptions { readonly rates: RateTable; readonly priceOverrides?: RateTable; readonly modelAliases?: ReadonlyMap; + readonly rateModel?: (provider: UsageRecord["provider"], model: string) => string | undefined; readonly modelFilter?: UsageModelFilter; readonly resolution?: UsageResolution; readonly sinceTimeMs?: number; @@ -79,15 +80,26 @@ export interface AggregateOptions { export function resolveModelAliases( aliases: Readonly>, ): ReadonlyMap { - const resolved = new Map(); - for (const model of Object.keys(aliases)) { - const seen = new Set([model]); - let target = aliases[model]!; - while (Object.hasOwn(aliases, target) && !seen.has(target)) { - seen.add(target); + const models = Object.keys(aliases); + const memo = new Map(); + for (const model of models) { + if (memo.has(model)) continue; + const path: string[] = []; + let target = model; + while (Object.hasOwn(aliases, target) && !memo.has(target)) { + path.push(target); + // Null marks both the active walk and paths that end in a cycle. + memo.set(target, null); target = aliases[target]!; } - if (!Object.hasOwn(aliases, target)) resolved.set(model, target); + const terminal = Object.hasOwn(aliases, target) ? (memo.get(target) ?? null) : target; + for (const alias of path) memo.set(alias, terminal); + } + + const resolved = new Map(); + for (const model of models) { + const target = memo.get(model); + if (target !== null && target !== undefined) resolved.set(model, target); } return resolved; } @@ -170,6 +182,30 @@ export function mergeUsageCopies(held: UsageRecord, copy: UsageRecord): UsageRec }; } +/** + * Grok reports each turn once, complete, so real copies are equal. A larger one is a deleted + * transcript's pre-v17 maximum over turns that shared a prompt id, whichever source it came from. + */ +function smallestGrokCopy(held: UsageRecord, copy: UsageRecord): UsageRecord { + const a = held.totals; + const b = copy.totals; + const reportedCostUsd = + held.reportedCostUsd === null || copy.reportedCostUsd === null + ? (held.reportedCostUsd ?? copy.reportedCostUsd) + : Math.min(held.reportedCostUsd, copy.reportedCostUsd); + return { + ...held, + totals: { + uncachedInputTokens: Math.min(a.uncachedInputTokens, b.uncachedInputTokens), + cachedInputTokens: Math.min(a.cachedInputTokens, b.cachedInputTokens), + cacheCreationTokens: Math.min(a.cacheCreationTokens, b.cacheCreationTokens), + outputTokens: Math.min(a.outputTokens, b.outputTokens), + reasoningTokens: Math.min(a.reasoningTokens, b.reasoningTokens), + }, + reportedCostUsd, + }; +} + interface Contribution { record: UsageRecord; /** Null when the first copy fell outside the window. */ @@ -217,7 +253,12 @@ export class UsageAggregator { const held = this.#seen.get(record.dedupeKey); if (held !== undefined) { this.#duplicatesDropped += 1; - if (held.bucket !== null) held.record = mergeUsageCopies(held.record, record); + if (held.bucket !== null) { + held.record = + record.provider === "grok" + ? smallestGrokCopy(held.record, record) + : mergeUsageCopies(held.record, record); + } return false; } } @@ -301,12 +342,16 @@ export class UsageAggregator { if (priced.costSource === "providerReported") bucket.providerReportedRecords += 1; } - // Aliases use the target's rate, replacing provider-specific rateModel. + // Aliases use the target's rate. Only an unpriced target goes through the provider's own rate naming. #mapModel(record: UsageRecord): UsageRecord { const model = this.#options.modelAliases?.get(record.model); if (model === undefined) return record; const { rateModel: _rateModel, ...rest } = record; - return { ...rest, model }; + const rateModel = + lookupRate(this.#options.rates, model) === null + ? this.#options.rateModel?.(record.provider, model) + : undefined; + return rateModel === undefined ? { ...rest, model } : { ...rest, model, rateModel }; } // Time-ordered records usually reuse the previous bucket; avoid rebuilding its key. diff --git a/apps/server/src/usage/usagePricing.ts b/apps/server/src/usage/usagePricing.ts index 21a1bd02bb178..a9ffc691643fd 100644 --- a/apps/server/src/usage/usagePricing.ts +++ b/apps/server/src/usage/usagePricing.ts @@ -310,6 +310,14 @@ export function lookupRate(table: RateTable, model: string): ModelRate | null { return rate; } +/** A priceable model the table has no rate for; a newer table may list it. */ +export function isUnlistedModel(table: RateTable, model: string): boolean { + const bareName = bareModelName(stripVariantSuffix(normalizeRateKey(model))); + return ( + bareName.length > 0 && !UNPRICEABLE_MODELS.has(bareName) && lookupRate(table, model) === null + ); +} + function resolveRate(table: RateTable, model: string): ModelRate | null { const key = stripVariantSuffix(normalizeRateKey(model)); const bareName = bareModelName(key); diff --git a/apps/server/src/usage/usageScanCache.test.ts b/apps/server/src/usage/usageScanCache.test.ts index 09348d5818a2e..36864768eb525 100644 --- a/apps/server/src/usage/usageScanCache.test.ts +++ b/apps/server/src/usage/usageScanCache.test.ts @@ -339,6 +339,29 @@ describe("scan cache round trip", () => { }); }); + it.each(["s:p:grok-4.5", "s:p:1786000000000:grok-4.5"])( + "re-parses v16 Grok transcripts with key %s", + (dedupeKey) => { + const original = cacheWith([ + ["/grok/updates.jsonl", 100, [record({ provider: "grok", model: "grok-4.5", dedupeKey })]], + ]); + original.set("/grok/updates.jsonl", { + ...original.get("/grok/updates.jsonl")!, + provider: "grok", + }); + const encoded = parse(makeScanCacheWriter()(original, {})); + const restored = decodeScanCache({ ...encoded, version: 16 }, TEST_FORMAT_MAP); + expect(restored.get("/grok/updates.jsonl")).toEqual({ + ...original.get("/grok/updates.jsonl"), + size: -1, + position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null }, + records: [ + record({ provider: "grok", model: "grok-4.5", dedupeKey: "s:p:1786000000000:grok-4.5" }), + ], + }); + }, + ); + it("rewrites only changed entries and still restores the whole cache", () => { const write = makeScanCacheWriter(); const cache = cacheWith([ diff --git a/apps/server/src/usage/usageScanCache.ts b/apps/server/src/usage/usageScanCache.ts index e75691d2b8465..843b414b97f32 100644 --- a/apps/server/src/usage/usageScanCache.ts +++ b/apps/server/src/usage/usageScanCache.ts @@ -10,7 +10,7 @@ import { CLAUDE_SYNTHETIC_MODEL } from "../provider/Drivers/claudeUsage.ts"; import { mergeUsageCopies } from "./usageAggregation.ts"; import { GUARD_LENGTH, type TranscriptParsePosition } from "./usageTranscriptReader.ts"; -const USAGE_SCAN_CACHE_VERSION = 16 as const; +const USAGE_SCAN_CACHE_VERSION = 17 as const; const SPEED_COMPATIBLE_SINCE_VERSION = 4; /** @@ -18,8 +18,9 @@ const SPEED_COMPATIBLE_SINCE_VERSION = 4; * replace it, dropping saved usage for deleted transcripts. Separate files keep * the newer one; older files are removed once it is saved. */ -export const SCAN_CACHE_FILE_NAME = "usage-scan-cache-v16.json"; +export const SCAN_CACHE_FILE_NAME = "usage-scan-cache-v17.json"; export const LEGACY_SCAN_CACHE_FILE_NAMES = [ + "usage-scan-cache-v16.json", "usage-scan-cache-v15.json", "usage-scan-cache-v14.json", "usage-scan-cache-v13.json", @@ -264,7 +265,16 @@ export function decodeScanCache( ...(version >= 7 && unresolvedServiceTierTurnId != null ? { unresolvedServiceTierTurnId } : {}), - dedupeKey: typeof dedupeKey === "string" ? dedupeKey : null, + dedupeKey: + typeof dedupeKey !== "string" + ? null + : // Saved copies of moved transcripts must still match the re-parsed v17 key. + provider === "grok" && + version < 17 && + dedupeKey.endsWith(`:${model}`) && + !dedupeKey.endsWith(`:${timestampMs}:${model}`) + ? `${dedupeKey.slice(0, -model.length)}${timestampMs}:${model}` + : dedupeKey, }); } return records; @@ -301,8 +311,10 @@ export function decodeScanCache( // Keep history for deleted rollouts, but re-parse incompatible reducer state. // v13 adds Claude 1-hour cache writes; v14 uses Codex request ledgers. // v15 repairs skipped Codex requests whose counters rewound; clean v14 entries stay served. + // v17 keys Grok turns by time, restoring turns previously dropped as repeats. const legacy = (entry.p === "claude" && version < 13) || + (entry.p === "grok" && version < 17) || (format.state !== undefined && (version < 6 || (entry.p === "codex" && (version < 14 || (version < 15 && malformedRecords > 0))))); diff --git a/apps/server/src/usage/usageTranscriptReader.test.ts b/apps/server/src/usage/usageTranscriptReader.test.ts index f4cb72ec1e25c..ca994dd3ac5b2 100644 --- a/apps/server/src/usage/usageTranscriptReader.test.ts +++ b/apps/server/src/usage/usageTranscriptReader.test.ts @@ -13,6 +13,8 @@ import * as Schema from "effect/Schema"; import { TEST_FORMATS } from "./usageTestFormats.ts"; import { listTranscriptFiles, + physicalSourceIdOf, + readPhysicalSourceId, readTranscriptRecords, transcriptLineGate, } from "./usageTranscriptReader.ts"; @@ -577,3 +579,49 @@ describe("transcriptLineGate", () => { } }); }); + +describe("readPhysicalSourceId", () => { + const scope = "0123456789abcdef0123456789abcdef"; + const windowsPath = "C:\\Users\\User\\.claude\\projects"; + + // One NTFS directory the Windows server reads natively and the WSL server + // through drvfs. The temp dir stands in for the drive's mount point. + async function drvfsFixture(extraMounts: (escapedMountPoint: string) => string = () => "") { + const mountPoint = NodePath.join(dir, "drive c"); + await NodeFSP.mkdir(NodePath.join(mountPoint, "Users", "User", ".claude", "projects"), { + recursive: true, + }); + const escaped = mountPoint.replaceAll(" ", "\\040"); + const mountInfoPath = NodePath.join(dir, "mountinfo"); + await NodeFSP.writeFile( + mountInfoPath, + [ + "24 1 8:32 / / rw,relatime shared:1 - ext4 /dev/sdc rw,discard", + `81 24 0:61 / ${escaped} rw,noatime shared:2 - 9p C:\\134 rw,dirsync,aname=drvfs;path=C:\\;uid=1000;gid=1000;symlinkroot=/mnt/,mmap,access=client,msize=65536,trans=fd`, + extraMounts(escaped), + ].join("\n"), + ); + return { mountPoint, mountInfoPath }; + } + + it("names a drvfs directory as the Windows server names it, whatever its typed case", async () => { + const { mountPoint, mountInfoPath } = await drvfsFixture(); + const wslId = await readPhysicalSourceId( + `${mountPoint}/users/user/.claude/projects`, + scope, + "linux", + mountInfoPath, + ); + assert.strictEqual(wslId, physicalSourceIdOf(scope, windowsPath)); + }); + + it("gives no identity to native or shadowing Linux mounts, or without a desktop scope", async () => { + const { mountPoint, mountInfoPath } = await drvfsFixture( + (escaped) => `90 81 8:48 / ${escaped}/Users rw,relatime - ext4 /dev/sdd rw`, + ); + const projects = `${mountPoint}/Users/User/.claude/projects`; + assert.isUndefined(await readPhysicalSourceId(projects, scope, "linux", mountInfoPath)); + assert.isUndefined(await readPhysicalSourceId(dir, scope, "linux", mountInfoPath)); + assert.isUndefined(await readPhysicalSourceId(projects, undefined, "linux", mountInfoPath)); + }); +}); diff --git a/apps/server/src/usage/usageTranscriptReader.ts b/apps/server/src/usage/usageTranscriptReader.ts index a4b10a4df2045..5b92f48356aa4 100644 --- a/apps/server/src/usage/usageTranscriptReader.ts +++ b/apps/server/src/usage/usageTranscriptReader.ts @@ -1,5 +1,6 @@ // @effect-diagnostics nodeBuiltinImport:off // Cold windows span ~1.4 GB/1,500 files; node:fs buffer streaming is ~10x cheaper than materializing files. +import * as NodeCrypto from "node:crypto"; import * as NodeFS from "node:fs"; import * as NodeFSP from "node:fs/promises"; import * as NodePath from "node:path"; @@ -150,6 +151,137 @@ export async function readDirectoryVolumeId(path: string): Promise { } } +/** + * Set by the desktop app, from a value it persists, on its Windows server and + * the WSL server it manages, so both can name one Windows directory alike + * without merging separate machines that share a hostname and path. + */ +export const DESKTOP_HOST_SCOPE_ENV = "T3CODE_DESKTOP_HOST_SCOPE"; + +const MOUNT_FIELD_ESCAPE = /\\([0-7]{3})/g; +const WINDOWS_ROOT = /^(?:[A-Za-z]:|\\\\[^\\]+\\[^\\]+)\\?$/; + +const unescapeMountField = (field: string) => + field.replace(MOUNT_FIELD_ESCAPE, (_, octal: string) => + String.fromCharCode(Number.parseInt(octal, 8)), + ); + +const normalizeWindowsPath = (path: string) => + (/^[a-z]:/.test(path) ? path[0]!.toUpperCase() + path.slice(1) : path).replace(/\\$/, ""); + +/** + * The Windows directory behind the deepest mount holding `dir`, read from + * `/proc/self/mountinfo` so a custom automount root resolves and an ext4 + * mount under `/mnt` never passes for a drive. + */ +function windowsMountFor( + mountInfo: string, + dir: string, +): { readonly mountPoint: string; readonly windowsRoot: string } | undefined { + let deepest: { mountPoint: string; windowsRoot: string | undefined } | undefined; + for (const line of mountInfo.split("\n")) { + const fields = line.split(" "); + const separator = fields.indexOf("-", 6); + if (separator === -1) continue; + const mountPoint = unescapeMountField(fields[4] ?? ""); + const contains = + mountPoint === "/" || + dir === mountPoint || + dir.startsWith(mountPoint.endsWith("/") ? mountPoint : `${mountPoint}/`); + if (!contains || (deepest !== undefined && deepest.mountPoint.length > mountPoint.length)) { + continue; + } + // WSL2 mounts drives over 9p with `aname=drvfs;path=C:\`; WSL1 as `drvfs`. + const options = (fields[separator + 3] ?? "").split(/[,;]/); + const source = options.includes("aname=drvfs") + ? options.find((option) => option.startsWith("path="))?.slice("path=".length) + : fields[separator + 1] === "drvfs" + ? unescapeMountField(fields[separator + 2] ?? "") + : undefined; + const root = unescapeMountField(fields[3] ?? "/") + .split("/") + .filter(Boolean); + deepest = { + mountPoint, + windowsRoot: + source !== undefined && WINDOWS_ROOT.test(source) + ? [normalizeWindowsPath(source), ...root].join("\\") + : undefined, + }; + } + return deepest?.windowsRoot === undefined + ? undefined + : { mountPoint: deepest.mountPoint, windowsRoot: deepest.windowsRoot }; +} + +/** + * drvfs keeps the casing a path was typed with, while Windows reports the + * on-disk names. Read those names rather than folding case, which would join + * distinct entries of a case-sensitive directory. + */ +async function onDiskNames( + mountPoint: string, + parts: readonly string[], +): Promise { + const names: string[] = []; + let current = mountPoint; + for (const part of parts) { + let entries: string[]; + try { + entries = await NodeFSP.readdir(current); + } catch { + return undefined; + } + const folded = part.toUpperCase(); + const [name, ...others] = entries.includes(part) + ? [part] + : entries.filter((entry) => entry.toUpperCase() === folded); + if (name === undefined || others.length > 0) return undefined; + names.push(name); + current = NodePath.posix.join(current, name); + } + return names; +} + +export function physicalSourceIdOf(scope: string, windowsPath: string): string { + return NodeCrypto.createHash("sha256") + .update(`${scope}\0${windowsPath}`) + .digest("hex") + .slice(0, 32); +} + +/** + * Identifies a Windows directory the same way from the Windows server and from + * a WSL server reading it through drvfs. Directories that are not Windows + * backed, and servers the desktop app did not start, get none. + */ +export async function readPhysicalSourceId( + dir: string, + scope: string | undefined, + platform: NodeJS.Platform, + mountInfoPath = "/proc/self/mountinfo", +): Promise { + if (scope === undefined || scope.length === 0) return undefined; + try { + if (platform === "win32") { + // The native realpath reports on-disk casing; the JS one keeps the input's. + return physicalSourceIdOf(scope, normalizeWindowsPath(await NodeFSP.realpath(dir))); + } + if (platform !== "linux") return undefined; + const mount = windowsMountFor(await NodeFSP.readFile(mountInfoPath, "utf8"), dir); + if (mount === undefined) return undefined; + const names = await onDiskNames( + mount.mountPoint, + dir.slice(mount.mountPoint.length).split("/").filter(Boolean), + ); + return names === undefined + ? undefined + : physicalSourceIdOf(scope, [mount.windowsRoot, ...names].join("\\")); + } catch { + return undefined; + } +} + async function guardMatches( handle: NodeFSP.FileHandle, position: TranscriptParsePosition, diff --git a/apps/server/src/usage/usageTranscriptStreaming.test.ts b/apps/server/src/usage/usageTranscriptStreaming.test.ts index 856ba2ac5b9d7..635b5ae4e6bec 100644 --- a/apps/server/src/usage/usageTranscriptStreaming.test.ts +++ b/apps/server/src/usage/usageTranscriptStreaming.test.ts @@ -254,7 +254,7 @@ describe("large usage records", () => { outputTokens: 99, reasoningTokens: 10, }, - dedupeKey: "s1:p1:grok-4.5-build", + dedupeKey: "s1:p1:1785578400123:grok-4.5-build", reportedCostUsd: 0.25, }); }); diff --git a/apps/server/src/vcs/GitVcsDriver.test.ts b/apps/server/src/vcs/GitVcsDriver.test.ts index a8c0052452ed7..1acbf036fb52c 100644 --- a/apps/server/src/vcs/GitVcsDriver.test.ts +++ b/apps/server/src/vcs/GitVcsDriver.test.ts @@ -1,3 +1,5 @@ +import * as NodeNet from "node:net"; +import * as SecureRandom from "effect/crypto/SecureRandom"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Cause from "effect/Cause"; @@ -173,6 +175,392 @@ const makeCheckpointFixture = Effect.fn("makeCheckpointFixture")(function* ( return { git, checkpointRef }; }); +it.effect.each(["interrupt", "timeout", "success"] as const)( + "checkpoint bulk staging cleans operation-owned packs after $0 and can retry", + (stop) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const liveProcess = yield* VcsProcess.VcsProcess; + const sandbox = yield* fs.makeTempDirectoryScoped({ prefix: "t3-bulk-checkpoint-" }); + const cwd = path.join(sandbox, "repo"); + yield* fs.makeDirectory(cwd); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + const originalCheckpoint = (yield* git(["rev-parse", checkpointRef])).stdout; + const originalIndex = yield* fs.readFile(path.join(cwd, ".git/index")); + const originalHead = (yield* git(["rev-parse", "HEAD"])).stdout; + yield* git(["config", "core.bigFileThreshold", "1m"]); + const crypto = yield* SecureRandom.SecureRandom; + const payload = yield* crypto.randomBytes(2 * 1024 * 1024); + yield* fs.writeFile(path.join(cwd, "a-media.bin"), payload); + + // Git keeps bulk-checkin's pack open while the next path's clean filter runs. + // The socket is a barrier, not a race against the speed of compression. + const connected = Promise.withResolvers(); + const sockets = new Set(); + let released = false; + const server = yield* Effect.acquireRelease( + Effect.sync(() => + NodeNet.createServer((socket) => { + sockets.add(socket); + connected.resolve(socket); + // Git can run the filter again while refreshing the staged entry. + if (released) socket.end(); + }), + ), + (server) => + Effect.sync(() => { + for (const socket of sockets) socket.destroy(); + server.close(); + }), + ); + yield* Effect.promise( + () => + new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }), + ); + const address = server.address(); + assert.isTrue(address !== null && typeof address !== "string"); + if (address === null || typeof address === "string") return; + const filter = path.join(sandbox, "filter.cjs"); + yield* fs.writeFileString( + filter, + `const s = require('node:net').connect(${address.port}, '127.0.0.1'); s.resume(); process.stdin.resume(); s.on('end', () => process.exit(0));`, + ); + yield* git(["config", "filter.barrier.clean", `"${process.execPath}" "${filter}"`]); + yield* git(["config", "filter.barrier.required", "true"]); + yield* fs.writeFileString(path.join(cwd, ".gitattributes"), "z-gate filter=barrier\n"); + yield* fs.writeFileString(path.join(cwd, "z-gate"), "gate\n"); + const sharedPack = path.join(cwd, ".git/objects/pack/tmp_pack_other_operation"); + yield* fs.writeFileString(sharedPack, "owned by another Git operation"); + let objectDirectory = path.join(cwd, ".git/objects"); + const captureDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe( + Effect.provideService(VcsProcess.VcsProcess, { + run: (input) => { + if (input.args.includes("add") && input.env?.GIT_OBJECT_DIRECTORY) + objectDirectory = input.env.GIT_OBJECT_DIRECTORY; + return liveProcess.run(input); + }, + }), + ); + const fiber = yield* captureDriver.checkpoints + .captureCheckpoint({ cwd, checkpointRef }) + .pipe(Effect.exit, Effect.forkScoped); + const socket = yield* Effect.promise(() => connected.promise).pipe( + Effect.raceFirst( + Fiber.join(fiber).pipe( + Effect.flatMap((result) => + Exit.isFailure(result) + ? Effect.failCause(result.cause) + : Effect.die("capture ended before the filter connected"), + ), + ), + ), + ); + const packs = (yield* fs.readDirectory(path.join(objectDirectory, "pack"))).filter( + (name) => name.startsWith("tmp_pack_") && name !== "tmp_pack_other_operation", + ); + assert.strictEqual(packs.length, 1); + assert.isAbove( + Number((yield* fs.stat(path.join(objectDirectory, "pack", packs[0]!))).size), + 1024 * 1024, + ); + // An overlapping capture publishes the same large blob while the first + // still owns an unfinished pack. Its objects/ref must survive either exit. + const concurrentRef = CheckpointRef.make("refs/t3/checkpoints/concurrent"); + const concurrentDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe( + Effect.provideService(VcsProcess.VcsProcess, { + run: (input) => + liveProcess.run({ ...input, args: ["-c", "filter.barrier.clean=cat", ...input.args] }), + }), + ); + yield* concurrentDriver.checkpoints.captureCheckpoint({ cwd, checkpointRef: concurrentRef }); + const concurrentOid = (yield* git(["rev-parse", concurrentRef])).stdout; + const sharedPackNames = (yield* fs.readDirectory(path.join(cwd, ".git/objects/pack"))).sort(); + const packName = sharedPackNames.find((name) => name.endsWith(".pack")); + assert.isDefined(packName); + const sharedPackBytes = yield* fs.readFile(path.join(cwd, ".git/objects/pack", packName!)); + if (stop === "interrupt") yield* Fiber.interrupt(fiber); + else if (stop === "timeout") yield* TestClock.adjust("30 seconds"); + released = true; + socket.end(); + const result = yield* Fiber.await(fiber); + if (stop === "interrupt") + assert.isTrue(Exit.isFailure(result) && Cause.hasInterruptsOnly(result.cause)); + else if (stop === "timeout") { + assert.isTrue(Exit.isSuccess(result)); + if (Exit.isSuccess(result)) { + assert.isTrue(Exit.isFailure(result.value)); + if (Exit.isFailure(result.value)) { + const error = Cause.findErrorOption(result.value.cause); + assert.isTrue(Option.isSome(error) && error.value._tag === "VcsProcessTimeoutError"); + } + } + } else assert.isTrue(Exit.isSuccess(result) && Exit.isSuccess(result.value)); + assert.deepEqual( + (yield* fs.readDirectory(path.join(cwd, ".git/objects/pack"))).sort(), + sharedPackNames, + ); + assert.deepEqual( + yield* fs.readFile(path.join(cwd, ".git/objects/pack", packName!)), + sharedPackBytes, + ); + assert.strictEqual((yield* git(["rev-parse", concurrentRef])).stdout, concurrentOid); + assert.isFalse(yield* fs.exists(path.join(objectDirectory, "pack", packs[0]!))); + assert.isFalse(yield* fs.exists(objectDirectory)); + assert.strictEqual(yield* fs.readFileString(sharedPack), "owned by another Git operation"); + if (stop !== "success") + assert.strictEqual((yield* git(["rev-parse", checkpointRef])).stdout, originalCheckpoint); + assert.deepEqual(yield* fs.readFile(path.join(cwd, ".git/index")), originalIndex); + assert.strictEqual((yield* git(["rev-parse", "HEAD"])).stdout, originalHead); + assert.deepEqual(yield* fs.readFile(path.join(cwd, "a-media.bin")), payload); + + yield* git(["config", "filter.barrier.clean", "cat"]); + yield* captureDriver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + assert.isFalse(yield* fs.exists(objectDirectory)); + const blob = (yield* git(["hash-object", "a-media.bin"])).stdout.trim(); + assert.strictEqual( + (yield* git(["rev-parse", `${checkpointRef}:a-media.bin`])).stdout.trim(), + blob, + ); + yield* fs.remove(sharedPack); + yield* git(["fsck", "--full"]); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect.each(["failure", "interrupt"] as const)( + "checkpoint publication $0 preserves existing refs and permits retry", + (stop) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-checkpoint-publish-" }); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + const previous = (yield* git(["show-ref"])).stdout; + yield* fs.writeFileString(path.join(cwd, "new.txt"), "new checkpoint content\n"); + const published = yield* Deferred.make(); + const captureDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe( + Effect.provideService(FileSystem.FileSystem, { + ...fs, + link: (source, destination) => + fs.link(source, destination).pipe( + Effect.andThen(Deferred.succeed(published, undefined)), + Effect.andThen( + stop === "interrupt" + ? Effect.never + : Effect.fail( + PlatformError.systemError({ + _tag: "BadResource", + module: "FileSystem", + method: "link", + }), + ), + ), + ), + }), + ); + const fiber = yield* captureDriver.checkpoints + .captureCheckpoint({ cwd, checkpointRef }) + .pipe(Effect.exit, Effect.forkScoped); + yield* Deferred.await(published); + if (stop === "interrupt") yield* Fiber.interrupt(fiber); + const result = yield* Fiber.await(fiber); + if (stop === "interrupt") + assert.isTrue(Exit.isFailure(result) && Cause.hasInterruptsOnly(result.cause)); + else { + assert.isTrue(Exit.isSuccess(result) && Exit.isFailure(result.value)); + if (Exit.isSuccess(result) && Exit.isFailure(result.value)) { + const error = Cause.findErrorOption(result.value.cause); + assert.isTrue( + Option.isSome(error) && + error.value._tag === "VcsCheckpointStorageError" && + error.value.stage === "publish", + ); + } + } + assert.strictEqual((yield* git(["show-ref"])).stdout, previous); + assert.deepEqual( + (yield* fs.readDirectory(path.join(cwd, ".git/objects"))).filter((name) => + name.startsWith("tmp_objdir-t3-checkpoint-"), + ), + [], + ); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + assert.strictEqual( + (yield* git(["show", `${checkpointRef}:new.txt`])).stdout, + "new checkpoint content\n", + ); + yield* git(["fsck", "--full"]); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect.each([ + { stop: "before git restore spawns", gate: "restore", index: "staged", file: "later\n" }, + { stop: "after git restore ran", gate: "clean", index: "head", file: "unstaged\n" }, +] as const)( + "checkpoint restore interrupted $stop leaves the $index index", + ({ gate, index, file }) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const liveProcess = yield* VcsProcess.VcsProcess; + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-restore-interrupt-" }); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + yield* fs.writeFileString(path.join(cwd, "file.txt"), "later\n"); + yield* git(["add", "file.txt"]); + const stagedTree = (yield* git(["write-tree"])).stdout; + const entered = yield* Deferred.make(); + const gatedDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe( + Effect.provideService(VcsProcess.VcsProcess, { + run: (input) => + input.operation === "GitVcsDriver.checkpoints.restoreCheckpoint" && + input.args.includes(gate) + ? Deferred.succeed(entered, undefined).pipe(Effect.andThen(Effect.never)) + : liveProcess.run(input), + }), + ); + const fiber = yield* gatedDriver.checkpoints + .restoreCheckpoint({ cwd, checkpointRef }) + .pipe(Effect.exit, Effect.forkScoped); + yield* Deferred.await(entered); + yield* Fiber.interrupt(fiber); + const result = yield* Fiber.await(fiber); + assert.isTrue(Exit.isFailure(result) && Cause.hasInterruptsOnly(result.cause)); + + assert.strictEqual( + (yield* git(["write-tree"])).stdout, + index === "head" ? (yield* git(["rev-parse", "HEAD^{tree}"])).stdout : stagedTree, + ); + assert.strictEqual(yield* fs.readFileString(path.join(cwd, "file.txt")), file); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect("checkpoint publication falls back to rename where hardlinks are unavailable", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* SecureRandom.SecureRandom; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-publish-no-links-" }); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + yield* git(["config", "core.bigFileThreshold", "1m"]); + yield* fs.writeFile(path.join(cwd, "media.bin"), yield* crypto.randomBytes(2 * 1024 * 1024)); + const captureDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe( + Effect.provideService(FileSystem.FileSystem, { + ...fs, + link: () => + Effect.fail( + PlatformError.systemError({ + _tag: "PermissionDenied", + module: "FileSystem", + method: "link", + }), + ), + }), + ); + yield* captureDriver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + const files = yield* fs.readDirectory(path.join(cwd, ".git/objects/pack")); + assert.isTrue(files.some((name) => name.endsWith(".pack"))); + assert.isTrue(yield* driver.checkpoints.hasCheckpointRef({ cwd, checkpointRef })); + assert.match((yield* git(["count-objects", "-v"])).stdout, /^garbage: 0$/m); + yield* git(["fsck", "--full"]); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( + "checkpoint publication preserves Git's shared object directory permissions", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-publish-shared-" }); + const git = (args: ReadonlyArray) => + driver.execute({ operation: "test.shared", cwd, args }); + yield* git(["init", "--shared=0660"]); + const checkpointRef = CheckpointRef.make("refs/t3/checkpoints/shared"); + yield* fs.writeFileString(path.join(cwd, "shared.txt"), "new shared object\n"); + const blob = (yield* git(["hash-object", "-w", "shared.txt"])).stdout.trim(); + const directory = path.join(cwd, ".git/objects", blob.slice(0, 2)); + const expectedMode = (yield* fs.stat(directory)).mode & 0o7777; + yield* fs.remove(directory, { recursive: true }); + assert.isFalse(yield* fs.exists(directory)); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + assert.strictEqual((yield* fs.stat(directory)).mode & 0o7777, expectedMode); + yield* git(["fsck", "--full"]); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect("checkpoint capture ignores an inherited object directory", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const sandbox = yield* fs.makeTempDirectoryScoped({ prefix: "t3-inherited-objects-" }); + const cwd = path.join(sandbox, "repo"); + yield* fs.makeDirectory(cwd); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + const unrelated = path.join(sandbox, "unrelated"); + yield* git(["init", "--bare", unrelated]); + const unrelatedObjects = path.join(unrelated, "objects"); + // VcsProcess clears inherited selectors from git children; only the driver sees this one. + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }).pipe( + Effect.provideService(HostProcess.Environment, { + ...(yield* HostProcess.Environment), + GIT_OBJECT_DIRECTORY: unrelatedObjects, + }), + ); + assert.strictEqual((yield* git(["show", `${checkpointRef}:file.txt`])).stdout, "unstaged\n"); + assert.deepEqual( + (yield* fs.readDirectory(unrelatedObjects)).filter( + (name) => name !== "info" && name !== "pack", + ), + [], + ); + yield* git(["fsck", "--full"]); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + +it.effect.each(["sha1", "sha256"] as const)( + "checkpoint object isolation supports linked worktrees with $0 objects", + (format) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const sandbox = yield* fs.makeTempDirectoryScoped({ prefix: "t3-checkpoint-worktree-" }); + // A separator must remain part of the alternate's path, not split the list. + const platform = yield* HostProcess.Platform; + const cwd = path.join(sandbox, platform === "win32" ? "repo;space" : 'repo:space"\u0001'); + yield* fs.makeDirectory(cwd); + yield* driver.execute({ + operation: "test.init", + cwd, + args: ["init", `--object-format=${format}`], + }); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + const worktree = path.join(sandbox, "linked"); + yield* git(["worktree", "add", "-b", "linked", worktree]); + yield* fs.writeFileString(path.join(worktree, "untracked.txt"), "linked worktree\n"); + const originalIndex = yield* fs.readFile(path.join(cwd, ".git/index")); + yield* driver.checkpoints.captureCheckpoint({ cwd: worktree, checkpointRef }); + assert.strictEqual( + (yield* git(["show", `${checkpointRef}:untracked.txt`])).stdout, + "linked worktree\n", + ); + assert.deepEqual(yield* fs.readFile(path.join(cwd, ".git/index")), originalIndex); + yield* git(["fsck", "--full"]); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + it.effect("checkpoint capture skips untracked nested repositories without a commit", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; @@ -1174,6 +1562,8 @@ it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( "write-tree", "commit-tree", "update-ref", + // Without a parsed top level, capture cannot rule out submodules. + "submodule", ]); assert.strictEqual((yield* git(["show", `${checkpointRef}:file.txt`])).stdout, "unstaged\n"); assert.deepEqual(yield* fileSystem.readFile(path.join(cwd, ".git", "index")), originalIndex); @@ -1412,6 +1802,73 @@ it.effect("sparse restore rewrites manual skip-worktree files and keeps exclusio }).pipe(Effect.scoped, Effect.provide(layerGitContract)), ); +it.effect("restores and deletes checkpoints inside populated submodules", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const git = (cwd: string, args: ReadonlyArray) => + driver.execute({ operation: "test", cwd, args }); + const makeRepository = Effect.fn("makeRepository")(function* (prefix: string) { + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix }); + yield* git(cwd, ["init"]); + yield* git(cwd, ["config", "user.email", "test@test.com"]); + yield* git(cwd, ["config", "user.name", "Test"]); + yield* fileSystem.writeFileString(path.join(cwd, "file.txt"), "original\n"); + yield* git(cwd, ["add", "."]); + yield* git(cwd, ["commit", "-m", "initial"]); + return cwd; + }); + const nested = yield* makeRepository("t3-submodule-nested-"); + const library = yield* makeRepository("t3-submodule-library-"); + const addSubmodule = (cwd: string, url: string, name: string) => + git(cwd, ["-c", "protocol.file.allow=always", "submodule", "add", url, name]).pipe( + Effect.andThen(git(cwd, ["commit", "-m", `add ${name}`])), + ); + yield* addSubmodule(library, nested, "nested"); + const root = yield* makeRepository("t3-submodule-root-"); + yield* addSubmodule(root, library, "library"); + yield* git(root, [ + "-c", + "protocol.file.allow=always", + "submodule", + "update", + "--init", + "--recursive", + ]); + const files = ["file.txt", "library/file.txt", "library/nested/file.txt"].map((file) => + path.join(root, file), + ); + const checkpointRef = CheckpointRef.make("refs/t3/checkpoints/submodules"); + yield* fileSystem.writeFileString(files[1]!, "captured\n"); + yield* driver.checkpoints.captureCheckpoint({ cwd: root, checkpointRef }); + + for (const file of files) { + yield* fileSystem.writeFileString(file, "changed\n"); + yield* fileSystem.writeFileString(`${file}.added`, "new\n"); + } + assert.isTrue( + yield* driver.checkpoints.restoreCheckpoint({ + cwd: root, + checkpointRef, + fallbackToHead: false, + }), + ); + + assert.deepStrictEqual( + yield* Effect.forEach(files, (file) => fileSystem.readFileString(file)), + ["original\n", "captured\n", "original\n"], + ); + for (const file of files) assert.isFalse(yield* fileSystem.exists(`${file}.added`)); + + yield* driver.checkpoints.deleteCheckpointRefs({ cwd: root, checkpointRefs: [checkpointRef] }); + for (const cwd of [root, path.join(root, "library"), path.join(root, "library", "nested")]) { + const ref = yield* git(cwd, ["for-each-ref", "refs/t3"]); + assert.strictEqual(ref.stdout, ""); + } + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + // The real VCS process, spawning `command` wherever the driver runs git. const vcsProcessSpawning = (command: string) => Layer.effect( diff --git a/apps/server/src/vcs/GitVcsDriver.ts b/apps/server/src/vcs/GitVcsDriver.ts index 4084e531f2302..d1e9dae66f1fa 100644 --- a/apps/server/src/vcs/GitVcsDriver.ts +++ b/apps/server/src/vcs/GitVcsDriver.ts @@ -4,6 +4,7 @@ import * as Context from "effect/Context"; import * as SecureRandom from "effect/crypto/SecureRandom"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; @@ -12,6 +13,7 @@ import { ChildProcessSpawner } from "effect/process"; import { GitCommandError, + VcsCheckpointStorageError, VcsProcessExitError, VcsUnsupportedOperationError, type VcsSwitchRefInput, @@ -37,6 +39,7 @@ import { } from "@t3tools/contracts"; import { isMissingExecutable } from "@t3tools/source-control-core/server/discovery"; import * as HostProcess from "@t3tools/shared/HostProcess"; +import { quoteGitPatchPath } from "@t3tools/shared/gitPatchPath"; import { makeGitVcsDriverCore, PATCH_RENDER_PREFIX_ARGS, @@ -121,7 +124,10 @@ export interface ExecuteGitProgress { } export interface CreateWorktreeProgress { - /** Git refuses existing paths; claimed paths belong to this call and can be removed on cancel. */ + /** + * Runs uninterruptibly once the worktree exists; from then on the caller owns its cleanup. + * For new-branch calls on Git supporting add --reason, cancellation before the claim removes what it made. + */ readonly onWorktreeClaimed?: (path: string) => Effect.Effect; readonly onCheckoutProgress?: (input: { percent: number; @@ -177,12 +183,16 @@ export interface GitPushResult { status: "pushed" | "skipped_up_to_date"; branch: string; upstreamBranch?: string | undefined; + /** Set when `upstreamBranch` is on a configured push remote instead of the tracked upstream. */ + pushRemote?: string | undefined; setUpstream?: boolean | undefined; } export interface GitPushTarget { remoteName: string; branchName: string; + /** The tracked upstream is current, yet pushes go to this configured push remote. */ + viaPushRemote: boolean; } export interface GitRangeContext { @@ -327,6 +337,8 @@ export class GitVcsDriver extends Context.Service< cwd: string, options?: GitRemoteStatusOptions, ) => Effect.Effect; + /** Refs, HEAD and shallow boundaries. Null when the snapshot cannot be read. */ + readonly readRefSnapshot: (cwd: string) => Effect.Effect; readonly prepareCommitContext: ( cwd: string, filePaths?: readonly string[], @@ -563,6 +575,7 @@ const gitCommand = Effect.fn("GitVcsDriver.gitCommand")( args: ReadonlyArray, options?: { readonly stdin?: string; + readonly onSpawn?: () => void; readonly env?: NodeJS.ProcessEnv; readonly allowNonZeroExit?: boolean; readonly timeoutMs?: number; @@ -578,6 +591,7 @@ const gitCommand = Effect.fn("GitVcsDriver.gitCommand")( cwd, spawnCwd: globalThis.process.cwd(), ...(options?.stdin !== undefined ? { stdin: options.stdin } : {}), + ...(options?.onSpawn !== undefined ? { onSpawn: options.onSpawn } : {}), ...(options?.env !== undefined ? { env: options.env } : {}), ...(options?.allowNonZeroExit !== undefined ? { allowNonZeroExit: options.allowNonZeroExit } @@ -649,6 +663,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( const execute: VcsDriver.VcsDriver["Service"]["execute"] = (input) => gitCommand(vcsProcess, input.operation, input.cwd, input.args, { ...(input.stdin !== undefined ? { stdin: input.stdin } : {}), + ...(input.onSpawn !== undefined ? { onSpawn: input.onSpawn } : {}), ...(input.env !== undefined ? { env: input.env } : {}), ...(input.allowNonZeroExit !== undefined ? { allowNonZeroExit: input.allowNonZeroExit } : {}), ...(input.timeoutMs !== undefined ? { timeoutMs: input.timeoutMs } : {}), @@ -922,6 +937,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( "--git-common-dir", "--git-path", "index", + "--show-toplevel", "--show-prefix", "--verify", "--quiet", @@ -932,11 +948,12 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( const headExists = result.exitCode === 0; const headChecked = headExists || result.exitCode === 1; const lines = result.stdout.split("\n"); - const [gitCommonDir = "", indexPath = "", prefix = "", head = ""] = lines; - if (headChecked && lines.length === (headExists ? 5 : 4) && lines.at(-1) === "") { + const [gitCommonDir = "", indexPath = "", topLevel = "", prefix = "", head = ""] = lines; + if (headChecked && lines.length === (headExists ? 6 : 5) && lines.at(-1) === "") { return { gitCommonDir: gitCommonDir.trim(), indexPath: indexPath.trim(), + topLevel: topLevel.trim(), prefix, headExists, head: headExists ? head : undefined, @@ -953,12 +970,26 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( return { gitCommonDir: fallbackCommonDir, indexPath: undefined, + topLevel: undefined, prefix: prefixResult.stdout.replace(/\n$/, ""), headExists: headChecked ? headExists : yield* hasHeadCommit(cwd), head: undefined, }; }); + // $displaypath is relative to cwd, so "../" marks submodules outside a nested workspace. + const listSubmodules = Effect.fn("GitVcsDriver.listSubmodules")(function* (cwd: string) { + const result = yield* execute({ + operation: "GitVcsDriver.checkpoints.listSubmodules", + cwd, + args: ["submodule", "foreach", "--quiet", 'printf "%s\\0" "$displaypath"'], + }); + return result.stdout + .split("\0") + .filter((entry) => entry !== "" && !entry.startsWith("../")) + .map((entry) => path.join(cwd, entry)); + }); + // Git defaults and macOS writeout-only can leave empty refs after crashes; flush before publication. // Set fsync first: Git 2.36 ignores unknown batch without resetting the method. // Batch fully flushes refs and equals fsync on macOS HFS+/APFS and Windows NTFS/ReFS. @@ -1049,9 +1080,8 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( "-c", "splitIndex.sharedIndexExpire=never", ]; - const { gitCommonDir, indexPath, prefix, headExists, head } = yield* resolveCaptureRepository( - input.cwd, - ); + const { gitCommonDir, indexPath, topLevel, prefix, headExists, head } = + yield* resolveCaptureRepository(input.cwd); // Read before the snapshot and pinned to the same HEAD: a switch that lands while files are // being staged then belongs to the next checkpoint instead of being applied to this one. const headMoves = head === undefined ? [] : yield* readHeadMoves(input.cwd); @@ -1064,8 +1094,42 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( const indexId = yield* random.uuidV4(); const tempIndexPath = path.join(gitCommonDir, `t3-checkpoint-index-${indexId}`); const hostEnv = yield* HostProcess.Environment; + // VcsProcess clears inherited object selectors, so every other command uses this store. + const sharedObjects = path.join(gitCommonDir, "objects"); + const quotedObjects = quoteGitPatchPath(sharedObjects); + // Isolate bulk-checkin packs so cleanup cannot remove another Git writer's objects. + const privateObjects = yield* Effect.acquireRelease( + fileSystem + .makeTempDirectory({ directory: sharedObjects, prefix: "tmp_objdir-t3-checkpoint-" }) + .pipe( + Effect.mapError( + (cause) => + new VcsCheckpointStorageError({ + operation, + cwd: input.cwd, + stage: "isolate", + cause, + }), + ), + ), + (directory) => + fileSystem.remove(directory, { recursive: true, force: true }).pipe( + Effect.catch((cause) => + Effect.logWarning("Could not remove private checkpoint objects", { + reason: cause.reason._tag, + }), + ), + ), + ); const commitEnv: NodeJS.ProcessEnv = { GIT_INDEX_FILE: tempIndexPath, + GIT_OBJECT_DIRECTORY: privateObjects, + // A filter must not publish refs that depend on our disposable store. + GIT_QUARANTINE_PATH: privateObjects, + // Git accepts C-quoted alternate paths, including path-list separators. + GIT_ALTERNATE_OBJECT_DIRECTORIES: quotedObjects.startsWith('"') + ? quotedObjects + : `"${quotedObjects}"`, GIT_AUTHOR_NAME: "T3 Code", GIT_AUTHOR_EMAIL: "t3code@users.noreply.github.com", GIT_COMMITTER_NAME: "T3 Code", @@ -1369,6 +1433,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( GIT_INDEX_FILE: undefined, GIT_OBJECT_DIRECTORY: undefined, GIT_ALTERNATE_OBJECT_DIRECTORIES: undefined, + GIT_QUARANTINE_PATH: undefined, }; const exclusions: Array = []; for (const entry of candidates) { @@ -1566,13 +1631,134 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( }); } + // Git writers have exited; bound publication by fanout directory. + yield* fileSystem.readDirectory(privateObjects).pipe( + Effect.flatMap((directories) => + Effect.forEach( + directories.filter( + (directory) => directory === "pack" || /^[0-9a-f]{2}$/.test(directory), + ), + (directory) => + Effect.gen(function* () { + const source = path.join(privateObjects, directory); + const destination = path.join(sharedObjects, directory); + // Most fanouts already exist; avoid constructing an EEXIST error + // for each of them on every capture. mkdir still handles the race. + if (!(yield* fileSystem.exists(destination))) + yield* fileSystem.makeDirectory(destination).pipe( + Effect.andThen( + Effect.gen(function* () { + // Git has already applied core.sharedRepository to the source. + // chmod bypasses umask, but never changes a pre-existing bucket. + const { mode } = yield* fileSystem.stat(source); + yield* fileSystem.chmod(destination, mode & 0o7777); + }), + ), + Effect.catchIf( + (cause) => cause.reason._tag === "AlreadyExists", + () => Effect.void, + ), + Effect.uninterruptible, + ); + const names = yield* fileSystem.readDirectory(source); + // Per-object spans would swamp the capture span for large workspaces. + const linkObject = Effect.fnUntraced(function* (name: string) { + const from = path.join(source, name); + const to = path.join(destination, name); + yield* fileSystem.link(from, to).pipe( + Effect.catchIf( + (cause) => + cause.reason._tag === "PermissionDenied" || + cause.reason._tag === "Unknown", + () => + Effect.gen(function* () { + if (yield* fileSystem.exists(to)) return; + // Git also falls back to rename without hardlinks; identical object names are interchangeable. + yield* fileSystem.rename(from, to); + }), + ), + Effect.catchIf( + (cause) => cause.reason._tag === "AlreadyExists", + () => Effect.void, + ), + ); + }); + if (directory !== "pack") { + yield* Effect.forEach( + names.filter((name) => /^(?:[0-9a-f]{38}|[0-9a-f]{62})$/.test(name)), + linkObject, + { discard: true }, + ); + return; + } + const files = new Set(names); + for (const name of names) { + if (!/^pack-(?:[0-9a-f]{40}|[0-9a-f]{64})\.pack$/.test(name)) continue; + const base = name.slice(0, -5); + if (!files.has(`${base}.idx`)) continue; + // Link metadata before the pack so an I/O error cannot strand a + // large unindexed pack. Defer cancellation across this short group + // of links. Never roll back shared links: a concurrent writer may + // already depend on them. Ref publication still happens last. + yield* Effect.forEach( + [".keep", ".promisor", ".rev", ".idx", ".pack"] + .map((extension) => `${base}${extension}`) + .filter((file) => files.has(file)), + linkObject, + { discard: true }, + ).pipe(Effect.uninterruptible); + } + }), + { concurrency: 16, discard: true }, + ), + ), + Effect.mapError( + (cause) => + new VcsCheckpointStorageError({ operation, cwd: input.cwd, stage: "publish", cause }), + ), + ); + yield* execute({ operation, cwd: input.cwd, args: [...durableWrite, "update-ref", input.checkpointRef, commitOid], }); }).pipe(Effect.ensuring(cleanupTempIndex)); - }), + + // The tree records only each submodule's HEAD, so submodules keep their own ref. + const hasSubmodules = + topLevel === undefined || + (yield* fileSystem + .exists(path.join(topLevel, ".gitmodules")) + .pipe(Effect.orElseSucceed(() => true))); + if (!hasSubmodules) return; + // A submodule that cannot be captured drops its ref, so restore leaves it alone. + yield* listSubmodules(input.cwd).pipe( + Effect.flatMap((submodules) => + Effect.forEach( + submodules, + (cwd) => + checkpoints + .captureCheckpoint({ cwd, checkpointRef: input.checkpointRef }) + .pipe( + Effect.catch((cause) => + Effect.logWarning("submodule checkpoint capture failed", { cwd, cause }).pipe( + Effect.andThen( + checkpoints + .deleteCheckpointRefs({ cwd, checkpointRefs: [input.checkpointRef] }) + .pipe(Effect.ignore), + ), + ), + ), + ), + { discard: true }, + ), + ), + Effect.catch((cause) => + Effect.logWarning("submodule checkpoint capture failed", { cwd: input.cwd, cause }), + ), + ); + }, Effect.scoped), hasCheckpointRef: (input) => resolveCheckpointCommit(input.cwd, input.checkpointRef).pipe( @@ -1597,15 +1783,48 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( cwd: input.cwd, args: ["ls-files", "--cached", `--with-tree=${commitOid}`, "-z", "--", "."], }); - // An empty index and checkpoint have nothing for git restore's pathspec to match. - if (tracked.stdout.length > 0) { - const sparseConfig = yield* execute({ + const sparseConfig = + tracked.stdout.length > 0 + ? yield* execute({ + operation, + cwd: input.cwd, + args: ["config", "--bool", "core.sparseCheckout"], + allowNonZeroExit: true, + }) + : null; + const sparseCheckout = sparseConfig?.stdout.trim() === "true"; + // Restoring away the last tracked file can remove a nested workspace directory. + const recreateWorkspace = fileSystem.makeDirectory(input.cwd, { recursive: true }).pipe( + Effect.mapError( + (cause) => + new VcsProcessExitError({ + operation, + command: "git restore", + cwd: input.cwd, + exitCode: 0, + detail: `Could not recreate the checkpoint workspace: ${cause.message}`, + }), + ), + ); + const resetIndex = Effect.gen(function* () { + // On Windows clean can remove './' and exit 0. + yield* recreateWorkspace; + yield* execute({ operation, cwd: input.cwd, - args: ["config", "--bool", "core.sparseCheckout"], - allowNonZeroExit: true, + args: [...longPaths, "reset", "--quiet", "--", "."], }); - const sparseCheckout = sparseConfig.stdout.trim() === "true"; + }); + let mutationStarted = false; + const onSpawn = () => { + mutationStarted = true; + }; + // Queued commands preserve user staging; spawned children stop before this scope resets it. + yield* Effect.addFinalizer((exit) => + mutationStarted && Exit.hasInterrupts(exit) ? resetIndex.pipe(Effect.ignore) : Effect.void, + ); + // An empty index and checkpoint have nothing for git restore's pathspec to match. + if (tracked.stdout.length > 0) { const restore = [...longPaths, "restore", "--source", commitOid, "--worktree", "--staged"]; // restore leaves skip-worktree paths alone. Outside a sparse checkout every such bit was // set by hand and capture recorded the file, so override them all. @@ -1613,6 +1832,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( operation, cwd: input.cwd, args: [...restore, ...(sparseCheckout ? [] : ["--ignore-skip-worktree-bits"]), "--", "."], + onSpawn, }); if (sparseCheckout) { // Sparse exclusions stay excluded; only skip-worktree paths the rules select are manual. @@ -1684,28 +1904,6 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( } } } - // Restoring away the last tracked file can remove a nested workspace directory. - const recreateWorkspace = fileSystem.makeDirectory(input.cwd, { recursive: true }).pipe( - Effect.mapError( - (cause) => - new VcsProcessExitError({ - operation, - command: "git restore", - cwd: input.cwd, - exitCode: 0, - detail: `Could not recreate the checkpoint workspace: ${cause.message}`, - }), - ), - ); - const resetIndex = Effect.gen(function* () { - // On Windows clean can remove './' and exit 0. - yield* recreateWorkspace; - yield* execute({ - operation, - cwd: input.cwd, - args: [...longPaths, "reset", "--quiet", "--", "."], - }); - }); yield* recreateWorkspace; const clean = execute({ operation, @@ -1713,6 +1911,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( args: [...longPaths, "clean", "-fd", "--", "."], env: { LC_ALL: "C" }, allowNonZeroExit: true, + onSpawn, }); let cleaned = yield* clean; // Removing a new .gitignore exposes the files it hid; clean again until nothing is left. @@ -1765,8 +1964,34 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( } yield* resetIndex; + // A submodule without this ref was not captured, and restoring it to HEAD would lose edits. + // The root is already restored, so a failing submodule must not hide that from the caller. + yield* listSubmodules(input.cwd).pipe( + Effect.flatMap((submodules) => + Effect.forEach( + submodules, + (cwd) => + checkpoints + .restoreCheckpoint({ + cwd, + checkpointRef: input.checkpointRef, + fallbackToHead: false, + }) + .pipe( + Effect.catch((cause) => + Effect.logWarning("submodule checkpoint restore failed", { cwd, cause }), + ), + ), + { discard: true }, + ), + ), + Effect.catch((cause) => + Effect.logWarning("submodule checkpoint restore failed", { cwd: input.cwd, cause }), + ), + ); + return true; - }), + }, Effect.scoped), diffCheckpoints: Effect.fn("GitVcsDriver.checkpoints.diffCheckpoints")(function* (input) { const operation = "GitVcsDriver.checkpoints.diffCheckpoints"; @@ -1972,6 +2197,10 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( }), { discard: true }, ); + const submodules = yield* listSubmodules(input.cwd).pipe(Effect.orElseSucceed(() => [])); + for (const cwd of submodules) { + yield* checkpoints.deleteCheckpointRefs({ cwd, checkpointRefs: input.checkpointRefs }); + } }, ), }; diff --git a/apps/server/src/vcs/GitVcsDriverCore.test.ts b/apps/server/src/vcs/GitVcsDriverCore.test.ts index 060beadad39f7..9e5b20205fefb 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.test.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.test.ts @@ -1,5 +1,7 @@ // @effect-diagnostics nodeBuiltinImport:off - realpathSync.native resolves Windows 8.3 short names, which the Effect realPath does not. +import * as NodeChildProcess from "node:child_process"; import * as NodeFS from "node:fs"; +import * as NodeNet from "node:net"; import * as NodePath from "node:path"; import * as NodeURL from "node:url"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; @@ -10,6 +12,7 @@ import { afterAll, vi } from "vite-plus/test"; import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as Deferred from "effect/Deferred"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; @@ -48,6 +51,7 @@ import { windowsLongPathConfigEnv, } from "./GitVcsDriverCore.ts"; import * as GitVcsDriver from "./GitVcsDriver.ts"; +import { VcsAutomaticFetchInterval } from "./VcsAutomaticFetchInterval.ts"; import { gitConfigEnv } from "./VcsProcess.ts"; const encodeGitCommandError = Schema.encodeEffect(Schema.fromJsonString(GitCommandError)); @@ -342,6 +346,11 @@ const git = ( return result.stdout.trim(); }); +const [, gitMajor, gitMinor] = + /(\d+)\.(\d+)/u.exec(NodeChildProcess.execFileSync("git", ["--version"], { encoding: "utf8" })) ?? + []; +const gitHasReftable = Number(gitMajor) > 2 || (Number(gitMajor) === 2 && Number(gitMinor) >= 45); + const gitWithStdin = ( cwd: string, args: ReadonlyArray, @@ -388,14 +397,15 @@ const initRepoWithCommit = ( return { initialBranch: head.trim().replace(/^ref: refs\/heads\//, "") }; }); -const commitInitialRepo = (cwd: string) => +const commitInitialRepo = (cwd: string, initArgs: ReadonlyArray = []) => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; const pathService = yield* Path.Path; - yield* git(cwd, ["init"]); + yield* git(cwd, ["init", ...initArgs]); + // Newer Git detaches auto-maintenance after commit; its lock file would race the template copy. yield* fileSystem.writeFileString( pathService.join(cwd, ".git", "config"), - "[user]\n\temail = test@test.com\n\tname = Test\n", + "[user]\n\temail = test@test.com\n\tname = Test\n[maintenance]\n\tauto = false\n", { flag: "a" }, ); yield* writeTextFile(cwd, "README.md", "# test\n"); @@ -550,7 +560,10 @@ it.effect.each([ assert.equal(created.failure.exitCode, hookExitCode); } const metadataQueries = spawnedArgs.filter((args) => args.includes("--git-common-dir")); - assert.deepStrictEqual(metadataQueries, [["rev-parse", "--git-common-dir"]]); + assert.deepStrictEqual( + metadataQueries, + Array.from({ length: 2 }, () => ["rev-parse", "--git-common-dir"]), + ); assert.isFalse(spawnedArgs.some((args) => args.includes("symbolic-ref"))); invalidationFailure = "none"; @@ -613,7 +626,7 @@ it.effect.each(["branch", "detached", "nested", "aliased-root", "aliased-nested" newRefName: "feature/cold", }); - assert.equal(spawnedArgs.filter((args) => args.includes("--git-common-dir")).length, 1); + assert.equal(spawnedArgs.filter((args) => args.includes("--git-common-dir")).length, 2); assert.isFalse(spawnedArgs.some((args) => args.includes("--show-toplevel"))); assert.isFalse(spawnedArgs.some((args) => args.includes("symbolic-ref"))); spawnedArgs.length = 0; @@ -675,7 +688,10 @@ it.effect.each(["bare", "unborn"] as const)( assert.equal(result.failure.operation, "GitVcsDriver.createWorktree.createBranch"); assert.equal(result.failure.detail, "git branch failed"); } - assert.equal(spawnedArgs.filter((args) => args.includes("--git-common-dir")).length, 1); + assert.equal( + spawnedArgs.filter((args) => args.includes("--git-common-dir")).length, + state === "bare" ? 2 : 1, + ); assert.isFalse(spawnedArgs.some((args) => args[0] === "symbolic-ref")); spawnedArgs.length = 0; const refs = yield* driver.listRefs({ cwd }); @@ -722,24 +738,31 @@ it.effect("reads a branch whose name starts with a parenthesis as attached", () ).pipe(Effect.provide(layerTest)), ); -it.effect("follows a checkout made outside the driver after refs were cached", () => - Effect.scoped( - Effect.gen(function* () { - const cwd = yield* makeTmpDir(); - const { initialBranch } = yield* initRepoWithCommit(cwd); - yield* git(cwd, ["branch", "feature/outside"]); - const driver = yield* GitVcsDriver.GitVcsDriver; - yield* driver.listRefs({ cwd }); +it.effect.each(["an existing", "a new"] as const)( + "follows a checkout of %s branch made outside the driver after refs were cached", + (branch) => + Effect.scoped( + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + if (branch === "an existing") yield* git(cwd, ["branch", "feature/outside"]); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* driver.listRefs({ cwd }); - yield* git(cwd, ["switch", "feature/outside"]); + yield* git( + cwd, + branch === "an existing" + ? ["switch", "feature/outside"] + : ["switch", "-c", "feature/outside"], + ); - const refs = (yield* driver.listRefs({ cwd })).refs; - const previous = refs.find((ref) => ref.name === initialBranch); - assert.isTrue(refs.find((ref) => ref.name === "feature/outside")?.current); - assert.isFalse(previous?.current); - assert.isNull(previous?.worktreePath ?? null); - }), - ).pipe(Effect.provide(layerTest)), + const refs = (yield* driver.listRefs({ cwd })).refs; + const previous = refs.find((ref) => ref.name === initialBranch); + assert.isTrue(refs.find((ref) => ref.name === "feature/outside")?.current); + assert.isFalse(previous?.current); + assert.isNull(previous?.worktreePath ?? null); + }), + ).pipe(Effect.provide(layerTest)), ); it.effect.each(["on a branch", "detached", "unborn"] as const)( @@ -1101,6 +1124,287 @@ describe("pullCurrentBranch", () => { }; }); + it.effect("resolves a status waiter when Pull claims its queued fetch", () => + Effect.scoped( + Effect.gen(function* () { + const { cwd, remoteHead } = yield* cloneBehindRemote; + const blockers: string[] = []; + for (let index = 0; index < 4; index++) { + const blocker = yield* cloneBehindRemote; + blockers.push(NodeFS.realpathSync.native(blocker.cwd)); + } + const targetPath = NodeFS.realpathSync.native(cwd); + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const release = yield* Deferred.make(); + const blockersStarted = yield* Deferred.make(); + const targetEntered = yield* Deferred.make(); + let running = 0; + let targetFetches = 0; + let waiterSucceeded = false; + const spawner = ChildProcessSpawner.make((command) => + Effect.gen(function* () { + const handle = yield* delegate.spawn(command); + if (!ChildProcess.isStandardCommand(command) || !command.args.includes("fetch")) + return handle; + const commandCwd = String(command.options.cwd); + if (commandCwd === targetPath) targetFetches++; + if (blockers.includes(commandCwd)) { + running++; + if (running === 4) yield* Deferred.succeed(blockersStarted, undefined); + yield* Deferred.await(release); + } + return handle; + }), + ); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options); + if (options.name === "GitVcsDriver.fetchRemoteForStatus.wait") { + const attribute = span.attribute.bind(span); + span.attribute = (key, value) => { + attribute(key, value); + if (key === "statusFetch.joining") Deferred.doneUnsafe(targetEntered, Effect.void); + }; + const end = span.end.bind(span); + span.end = (endTime, exit) => { + waiterSucceeded = Exit.isSuccess(exit); + end(endTime, exit); + }; + } + return span; + }, + }); + yield* Effect.gen(function* () { + for (const blocker of blockers) { + yield* driver.statusDetailsRemote(blocker).pipe(Effect.forkChild); + } + yield* Deferred.await(blockersStarted); + const status = yield* driver + .statusDetailsRemote(cwd) + .pipe(Effect.withTracer(tracer), Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(targetEntered); + const pull = yield* driver.pullCurrentBranch(cwd); + const refreshed = yield* Fiber.join(status); + assert.equal(pull.status, "pulled"); + assert.equal(yield* git(cwd, ["rev-parse", "HEAD"]), remoteHead); + assert.isTrue(waiterSucceeded); + assert.equal(refreshed.behindCount, 0); + assert.equal(targetFetches, 0); + }).pipe(Effect.ensuring(Deferred.succeed(release, undefined))); + }), + ).pipe(Effect.provide(layerTest)), + ); + + const claimedFetchFallbacks = ["diverged", "interrupted", "upstream-only"] as const; + + it.effect.each(claimedFetchFallbacks)( + "falls back to the claimed status fetch when Pull is %s", + (failure) => + Effect.scoped( + Effect.gen(function* () { + const { cwd, updater, remoteHead } = yield* cloneBehindRemote; + if (failure === "upstream-only") { + yield* git(updater, ["push", "origin", "HEAD:refs/heads/other"]); + yield* writeTextFile(cwd, ".git/refs/remotes/origin/other.lock", ""); + } + if (failure === "diverged") { + yield* writeTextFile(cwd, "local.txt", "local\n"); + yield* git(cwd, ["add", "local.txt"]); + yield* git(cwd, ["commit", "-m", "local commit"]); + } + const blockers: string[] = []; + for (let index = 0; index < 4; index++) { + const blocker = yield* cloneBehindRemote; + blockers.push(NodeFS.realpathSync.native(blocker.cwd)); + } + const targetPath = NodeFS.realpathSync.native(cwd); + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const release = yield* Deferred.make(); + const blockersStarted = yield* Deferred.make(); + const targetEntered = yield* Deferred.make(); + const targetFetchStarted = yield* Deferred.make(); + const pullBeforeFetch = yield* Deferred.make(); + const releasePull = yield* Deferred.make(); + let running = 0; + let targetFetches = 0; + let waiterSucceeded = false; + let targetFetchExitCode: number | undefined; + const spawner = ChildProcessSpawner.make((command) => + Effect.gen(function* () { + if (ChildProcess.isStandardCommand(command)) { + const commandCwd = NodeFS.realpathSync.native(String(command.options.cwd)); + if ( + commandCwd === targetPath && + failure === "interrupted" && + command.args[0] === "rev-parse" && + command.args[1] === "HEAD" && + command.args.length === 2 + ) { + yield* Deferred.succeed(pullBeforeFetch, undefined); + yield* Deferred.await(releasePull); + } + } + const handle = yield* delegate.spawn(command); + if (!ChildProcess.isStandardCommand(command) || !command.args.includes("fetch")) + return handle; + const commandCwd = String(command.options.cwd); + if (commandCwd === targetPath) { + targetFetches++; + yield* Deferred.succeed(targetFetchStarted, undefined); + return ChildProcessSpawner.makeHandle({ + ...handle, + exitCode: handle.exitCode.pipe( + Effect.tap((exitCode) => + Effect.sync(() => { + targetFetchExitCode = exitCode; + }), + ), + ), + }); + } + if (blockers.includes(commandCwd)) { + running++; + if (running === 4) yield* Deferred.succeed(blockersStarted, undefined); + yield* Deferred.await(release); + } + return handle; + }), + ); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options); + if (options.name === "GitVcsDriver.fetchRemoteForStatus.wait") { + const attribute = span.attribute.bind(span); + span.attribute = (key, value) => { + attribute(key, value); + if (key === "statusFetch.joining") + Deferred.doneUnsafe(targetEntered, Effect.void); + }; + const end = span.end.bind(span); + span.end = (endTime, exit) => { + waiterSucceeded = Exit.isSuccess(exit); + end(endTime, exit); + }; + } + return span; + }, + }); + yield* Effect.gen(function* () { + for (const blocker of blockers) { + yield* driver.statusDetailsRemote(blocker).pipe(Effect.forkChild); + } + yield* Deferred.await(blockersStarted); + const status = yield* driver + .statusDetailsRemote(cwd) + .pipe(Effect.withTracer(tracer), Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(targetEntered); + if (failure === "interrupted") { + const pull = yield* driver.pullCurrentBranch(cwd).pipe(Effect.forkChild); + yield* Deferred.await(pullBeforeFetch); + yield* Fiber.interrupt(pull); + const exit = yield* Fiber.await(pull); + assert.isTrue(Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)); + } else if (failure === "upstream-only") { + assert.equal((yield* driver.pullCurrentBranch(cwd)).status, "pulled"); + yield* (yield* FileSystem.FileSystem).remove( + `${cwd}/.git/refs/remotes/origin/other.lock`, + ); + } else { + const error = yield* Effect.flip(driver.pullCurrentBranch(cwd)); + assert.equal(error.operation, "GitVcsDriver.pullCurrentBranch.pull"); + } + assert.equal(targetFetches, 0); + yield* Deferred.succeed(release, undefined); + yield* Deferred.await(targetFetchStarted); + assert.isTrue((yield* Fiber.join(status)).isRepo); + assert.isTrue(waiterSucceeded); + assert.equal(targetFetches, 1); + assert.equal(targetFetchExitCode, 0); + if (failure === "upstream-only") { + assert.equal(yield* git(cwd, ["rev-parse", "refs/remotes/origin/other"]), remoteHead); + } + }).pipe( + Effect.ensuring( + Effect.all([ + Deferred.succeed(release, undefined), + Deferred.succeed(releasePull, undefined), + ]), + ), + ); + }), + ).pipe(Effect.provide(layerTest)), + ); + + it.effect("waits for a running status fetch before starting Pull", () => + Effect.scoped( + Effect.gen(function* () { + const { cwd, remoteHead } = yield* cloneBehindRemote; + const targetPath = NodeFS.realpathSync.native(cwd); + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const fetchStarted = yield* Deferred.make(); + const releaseFetch = yield* Deferred.make(); + const pullResolvedCommonDir = yield* Deferred.make(); + const pullStarted = yield* Deferred.make(); + const events: string[] = []; + const spawner = ChildProcessSpawner.make((command) => + Effect.gen(function* () { + if (!ChildProcess.isStandardCommand(command)) return yield* delegate.spawn(command); + const target = NodeFS.realpathSync.native(String(command.options.cwd)) === targetPath; + if (target && command.args.includes("pull")) { + events.push("pull-spawn"); + yield* Deferred.succeed(pullStarted, undefined); + } + const handle = yield* delegate.spawn(command); + if (target && command.args.includes("fetch")) { + yield* Deferred.succeed(fetchStarted, undefined); + yield* Deferred.await(releaseFetch); + yield* handle.exitCode; + events.push("fetch-exit"); + } + return handle; + }), + ); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options); + if (options.name === "resolveGitCommonDir") { + const end = span.end.bind(span); + span.end = (endTime, exit) => { + end(endTime, exit); + Deferred.doneUnsafe(pullResolvedCommonDir, Effect.void); + }; + } + return span; + }, + }); + yield* Effect.gen(function* () { + yield* driver.statusDetailsRemote(cwd).pipe(Effect.forkChild); + yield* Deferred.await(fetchStarted); + const pull = yield* driver + .pullCurrentBranch(cwd) + .pipe(Effect.withTracer(tracer), Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(pullResolvedCommonDir); + assert.isFalse(yield* Deferred.isDone(pullStarted)); + assert.isUndefined(pull.pollUnsafe()); + yield* Deferred.succeed(releaseFetch, undefined); + const result = yield* Fiber.join(pull); + assert.equal(result.status, "pulled"); + assert.equal(yield* git(cwd, ["rev-parse", "HEAD"]), remoteHead); + assert.deepEqual(events, ["fetch-exit", "pull-spawn"]); + }).pipe(Effect.ensuring(Deferred.succeed(releaseFetch, undefined))); + }), + ).pipe(Effect.provide(layerTest)), + ); + it.effect("still fetches other branches on the first pull", () => Effect.gen(function* () { const driver = yield* GitVcsDriver.GitVcsDriver; @@ -1547,6 +1851,53 @@ it.effect("reads worktrees from git that rejects worktree list -z", () => ), ); +it.effect("commits selected files with git that rejects --pathspec-from-file", () => + Effect.scoped( + Effect.gen(function* () { + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + // Git before 2.25 has no `add --pathspec-from-file` and exits with a usage error. + const oldGitSpawner = ChildProcessSpawner.make((command) => { + if (!ChildProcess.isStandardCommand(command)) { + return Effect.die("expected a standard Git command"); + } + if (command.args.at(-1) === "version") { + return Effect.succeed(makeSuccessfulHandle("git version 2.24.4\n")); + } + if (command.args.some((arg) => String(arg).startsWith("--pathspec-from-file"))) { + return Effect.succeed(makeFailedHandle("error: unknown option `pathspec-from-file=-'")); + } + return delegate.spawn(command); + }); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, oldGitSpawner), + ); + yield* Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const selected = ["selected[1].txt", "-dash.txt", "with space.txt"]; + for (const file of selected) yield* writeTextFile(cwd, file, "selected\n"); + yield* writeTextFile(cwd, "left.txt", "left\n"); + + yield* driver.commit(cwd, "Selected", "", { stage: { filePaths: selected } }); + + assert.sameMembers( + (yield* git(cwd, ["show", "--pretty=", "--name-only", "-z", "HEAD"])) + .split("\0") + .filter(Boolean), + selected, + ); + assert.equal(yield* git(cwd, ["status", "--porcelain"]), "?? left.txt"); + }).pipe(Effect.provideService(GitVcsDriver.GitVcsDriver, driver)); + }), + ).pipe( + Effect.provide( + layerServerConfig.pipe( + Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), + ), + ), + ), +); + it.effect("does not report a separate git dir as the worktree of its branch", () => Effect.scoped( Effect.gen(function* () { @@ -1764,6 +2115,37 @@ it.effect("does not fetch for status when the branch tracks a local branch", () ).pipe(Effect.provide(layerTest)), ); +it.effect("does not fetch for status when the automatic fetch interval is zero", () => + Effect.scoped( + Effect.gen(function* () { + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const fetches: Array> = []; + const spawner = ChildProcessSpawner.make((command) => { + if (ChildProcess.isStandardCommand(command) && command.args.includes("fetch")) { + fetches.push(command.args.map(String)); + } + return delegate.spawn(command); + }); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provideService(VcsAutomaticFetchInterval, Effect.succeed(Duration.zero)), + ); + const cwd = yield* makeTmpDir(); + const remote = yield* makeTmpDir("git-vcs-driver-remote-"); + const { initialBranch } = yield* initRepoWithCommit(cwd); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["push", "--set-upstream", "origin", initialBranch]); + + yield* driver.statusDetails(cwd); + const status = yield* driver.statusDetailsRemote(cwd); + + assert.equal(status.upstreamRef, `origin/${initialBranch}`); + assert.deepStrictEqual(fetches, []); + }), + ).pipe(Effect.provide(layerTest)), +); + it.effect("lets a worktree checkout that is still working run past five minutes", () => Effect.gen(function* () { const checkoutStarted = yield* Deferred.make(); @@ -1772,7 +2154,8 @@ it.effect("lets a worktree checkout that is still working run past five minutes" Effect.gen(function* () { if (!ChildProcess.isStandardCommand(command)) return yield* Effect.die("expected Git command"); - if (!command.args.includes("worktree")) return makeSuccessfulHandle(""); + if (!command.args.includes("worktree") || command.args.includes("-h")) + return makeSuccessfulHandle(""); yield* Deferred.succeed(checkoutStarted, undefined); return ChildProcessSpawner.makeHandle({ ...makeSuccessfulHandle(""), @@ -1879,6 +2262,53 @@ it.effect("cancels a slow submodule checkout without reporting best-effort succe ), ); +// regression: #18537 the fetch before creating a worktree outgrew the 30s default +it.effect.each(["healthy", "hung"] as const)("bounds a %s remote fetch at five minutes", (fetch) => + Effect.gen(function* () { + const started = yield* Deferred.make(); + const finished = yield* Deferred.make(); + const spawner = ChildProcessSpawner.make((command) => + Effect.succeed( + ChildProcess.isStandardCommand(command) && command.args[0] === "fetch" + ? ChildProcessSpawner.makeHandle({ + ...makeSuccessfulHandle(""), + exitCode: Deferred.succeed(started, undefined).pipe( + Effect.andThen(Deferred.await(finished)), + Effect.as(ChildProcessSpawner.ExitCode(0)), + ), + }) + : makeNonRepositoryHandle(), + ), + ); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + const fetching = yield* driver + .fetchRemote({ cwd: yield* makeTmpDir(), remoteName: "origin" }) + .pipe(Effect.result, Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(started); + yield* TestClock.adjust("31 seconds"); + if (fetch === "healthy") yield* Deferred.succeed(finished, undefined); + else { + yield* TestClock.adjust("268 seconds"); + assert.isUndefined(fetching.pollUnsafe()); + yield* TestClock.adjust("1 second"); + } + + const result = yield* Fiber.join(fetching); + assert.equal( + Result.isFailure(result) ? result.failure.detail : "fetched", + fetch === "healthy" ? "fetched" : "Git command timed out.", + ); + }).pipe( + Effect.provide( + layerServerConfig.pipe( + Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), + ), + ), + ), +); + describe("rejected git refs", () => { const remoteUrl = "https://user:token-123@git.example.invalid/org/secret-repo.git"; const failedPush = `error: failed to push some refs to '${remoteUrl}'`; @@ -2175,6 +2605,7 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const spawner = ChildProcessSpawner.make((command) => { if ( ChildProcess.isStandardCommand(command) && + !command.args.includes("-h") && (command.args.includes("worktree") || command.args[0] === "branch" || command.args[0] === "checkout") @@ -2823,7 +3254,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }); const source = dirty.sources.find((source) => source.kind === "working-tree")!; assert.deepStrictEqual(source.files, [ - { path: "untracked.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "untracked.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ]); assert.include(source.diff, "+untracked content"); }), @@ -2929,7 +3366,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.isTrue(metadata.sources[0]!.patchesDeferred); assert.isEmpty(metadata.sources[0]!.diff); assert.deepStrictEqual(metadata.sources[0]!.files, [ - { path: "README.md", previousPath: null, additions: 1, deletions: 1 }, + { + path: "README.md", + previousPath: null, + changeType: "change", + additions: 1, + deletions: 1, + }, ]); }), ); @@ -2975,6 +3418,35 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("keeps the review working beside a nested repository with no commits", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* git(cwd, ["init", "scaffold"]); + const committed = `${cwd}/committed`; + yield* (yield* FileSystem.FileSystem).makeDirectory(committed); + yield* initRepoWithCommit(committed); + yield* writeTextFile(cwd, "README.md", "# tracked change\n"); + yield* writeTextFile(cwd, "untracked.txt", "untracked contents\n"); + + const preview = yield* driver.getReviewDiffPreview({ cwd, ignoreWhitespace: false }); + const diff = preview.sources.find((source) => source.kind === "working-tree")?.diff ?? ""; + + assert.include(diff, "+# tracked change"); + assert.include(diff, "+untracked contents"); + assert.notInclude(diff, "scaffold"); + assert.include(diff, "diff --git a/committed b/committed"); + + const status = yield* driver.statusDetailsLocal(cwd, { includeBranchChanges: true }); + assert.deepStrictEqual(status.branchChanges, { + baseRef: null, + insertions: 3, + deletions: 1, + }); + }), + ); + it.effect("keeps untracked filenames with pathspec magic in the review", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -2999,7 +3471,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }); const scopedSource = scoped.sources.find((source) => source.kind === "working-tree")!; assert.deepStrictEqual(scopedSource.files, [ - { path: ":(exclude)after.ts", previousPath: null, additions: 1, deletions: 0 }, + { + path: ":(exclude)after.ts", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ]); assert.include(scopedSource.diff, "+literal pathspec contents"); assert.notInclude(scopedSource.diff, "ordinary.ts"); @@ -3022,7 +3500,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const source = preview.sources.find((candidate) => candidate.kind === "working-tree")!; assert.deepStrictEqual(source.files, [ - { path: "excluded/new.ts", previousPath: null, additions: 1, deletions: 0 }, + { + path: "excluded/new.ts", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ]); assert.include(source.diff, "+outside the cone"); }), @@ -3065,7 +3549,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const source = preview.sources.find((candidate) => candidate.kind === "working-tree")!; assert.deepStrictEqual(source.files, [ - { path: "after.ts", previousPath: "before.ts", additions: 1, deletions: 1 }, + { + path: "after.ts", + previousPath: "before.ts", + changeType: "rename-changed", + additions: 1, + deletions: 1, + }, ]); const scoped = yield* driver.getReviewDiffPreview({ cwd, @@ -3185,7 +3675,15 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const preview = yield* driver.getReviewDiffPreview({ cwd }); assert.deepStrictEqual( preview.sources.find((source) => source.kind === "working-tree")!.files, - [{ path: "untracked.txt", previousPath: null, additions: 1, deletions: 0 }], + [ + { + path: "untracked.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, + ], ); assert.strictEqual(yield* fs.readFileString(path.join(cwd, ".git/filter-runs")), ""); assert.deepStrictEqual(yield* fs.readFile(indexPath), originalIndex); @@ -3225,8 +3723,20 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const preview = yield* driver.getReviewDiffPreview({ cwd }); const dirty = preview.sources.find((source) => source.kind === "working-tree")!; assert.deepStrictEqual(dirty.files, [ - { path: "tracked.txt", previousPath: null, additions: 1, deletions: 1 }, - { path: "untracked.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "tracked.txt", + previousPath: null, + changeType: "change", + additions: 1, + deletions: 1, + }, + { + path: "untracked.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ]); assert.include(dirty.diff, "-before"); assert.include(dirty.diff, "+after!"); @@ -3290,28 +3800,60 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { } assert.deepStrictEqual(branch.files, [ - { path: "a-large.txt", previousPath: null, additions: 4000, deletions: 0 }, - { path: "untracked.txt", previousPath: null, additions: 4000, deletions: 0 }, - { path: "z-last.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "a-large.txt", + previousPath: null, + changeType: "new", + additions: 4000, + deletions: 0, + }, + { + path: "untracked.txt", + previousPath: null, + changeType: "new", + additions: 4000, + deletions: 0, + }, + { path: "z-last.txt", previousPath: null, changeType: "new", additions: 1, deletions: 0 }, ]); assert.deepStrictEqual(dirty.files, [ - { path: "a-large.txt", previousPath: null, additions: 4000, deletions: 4000 }, - { path: "untracked.txt", previousPath: null, additions: 4000, deletions: 0 }, - { path: "z-last.txt", previousPath: null, additions: 1, deletions: 1 }, - ]); - }), - ); - - it.effect("preserves renames, unusual paths, modes, and binary statistics", () => - Effect.gen(function* () { - const cwd = yield* makeTmpDir(); - const { initialBranch } = yield* initRepoWithCommit(cwd); + { + path: "a-large.txt", + previousPath: null, + changeType: "change", + additions: 4000, + deletions: 4000, + }, + { + path: "untracked.txt", + previousPath: null, + changeType: "new", + additions: 4000, + deletions: 0, + }, + { + path: "z-last.txt", + previousPath: null, + changeType: "change", + additions: 1, + deletions: 1, + }, + ]); + }), + ); + + it.effect("preserves renames, unusual paths, modes, and binary statistics", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); const driver = yield* GitVcsDriver.GitVcsDriver; yield* writeTextFile(cwd, "mode-only.sh", "echo unchanged\n"); - yield* git(cwd, ["add", "mode-only.sh"]); + yield* writeTextFile(cwd, "removed.txt", "removed\n"); + yield* git(cwd, ["add", "mode-only.sh", "removed.txt"]); yield* git(cwd, ["commit", "-m", "add executable candidate"]); yield* git(cwd, ["checkout", "-b", "feature/paths"]); yield* git(cwd, ["mv", "README.md", "renamed.md"]); + yield* git(cwd, ["rm", "removed.txt"]); yield* writeTextFile(cwd, "[literal].txt", "literal\n"); yield* writeTextFile(cwd, " leading.txt", "whitespace path\n"); yield* writeTextFile(cwd, "l.txt", "other\n"); @@ -3351,12 +3893,28 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.deepInclude(branch.files ?? [], { path: "renamed.md", previousPath: "README.md", + changeType: "rename-pure", + additions: 0, + deletions: 0, + }); + assert.deepInclude(branch.files ?? [], { + path: "removed.txt", + previousPath: null, + changeType: "deleted", + additions: 0, + deletions: 1, + }); + assert.deepInclude(branch.files ?? [], { + path: "mode-only.sh", + previousPath: null, + changeType: "change", additions: 0, deletions: 0, }); assert.deepInclude(branch.files ?? [], { path: "binary.dat", previousPath: null, + changeType: "new", additions: 0, deletions: 0, }); @@ -3364,6 +3922,7 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.deepInclude(branch.files ?? [], { path: "tab\tand\nnewline.txt", previousPath: null, + changeType: "new", additions: 1, deletions: 0, }); @@ -3382,8 +3941,14 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const preview = yield* driver.getReviewDiffPreview({ cwd }); const dirty = preview.sources.find((source) => source.kind === "working-tree")!; assert.deepStrictEqual(dirty.files, [ - { path: "staged.txt", previousPath: null, additions: 1, deletions: 0 }, - { path: "untracked.txt", previousPath: null, additions: 1, deletions: 0 }, + { path: "staged.txt", previousPath: null, changeType: "new", additions: 1, deletions: 0 }, + { + path: "untracked.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ]); assert.include(dirty.diff, "b/staged.txt"); assert.include(dirty.diff, "b/untracked.txt"); @@ -3567,6 +4132,77 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("loads working-tree contents as Git diffs them under the ident attribute", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + const fileSystem = yield* FileSystem.FileSystem; + const filePath = (yield* Path.Path).join(cwd, "README.md"); + const committed = "$Id$\nfirst\nsecond\n"; + yield* writeTextFile(cwd, ".gitattributes", "README.md ident\n"); + yield* writeTextFile(cwd, "README.md", committed); + yield* writeTextFile(cwd, "PLAIN.md", "plain\n"); + yield* git(cwd, ["add", "."]); + yield* git(cwd, ["commit", "-m", "add ident keyword"]); + // Checkout smudges the keyword to `$Id: $`. + yield* fileSystem.remove(filePath); + yield* git(cwd, ["checkout-index", "--force", "README.md"]); + const smudged = yield* fileSystem.readFileString(filePath); + assert.match(smudged, /^\$Id: [0-9a-f]{40} \$\n/); + yield* writeTextFile(cwd, "README.md", smudged.replace("first", "edited")); + yield* writeTextFile(cwd, "PLAIN.md", "$Id: literal $\n"); + const disk = yield* fileSystem.readFile(filePath); + + const contents = yield* driver.getReviewDiffFileContents( + makeReviewDiffFileContentsInput(cwd), + ); + const plain = yield* driver.getReviewDiffFileContents( + makeReviewDiffFileContentsInput(cwd, { oldPath: "PLAIN.md", newPath: "PLAIN.md" }), + ); + + assert.deepStrictEqual(contents, { + oldContents: committed, + newContents: "$Id$\nedited\nsecond\n", + }); + assert.strictEqual(plain.newContents, "$Id: literal $\n"); + yield* writeTextFile(cwd, "README.md", "$Id: x $Id: y $\n"); + const sharedDollar = yield* driver.getReviewDiffFileContents( + makeReviewDiffFileContentsInput(cwd), + ); + assert.strictEqual(sharedDollar.newContents, "$Id$Id: y $\n"); + yield* git(cwd, ["config", "filter.unrelated.clean", "touch filter-ran; cat"]); + yield* writeTextFile(cwd, ".gitattributes", "README.md ident text eol=lf\n"); + yield* writeTextFile( + cwd, + "README.md", + smudged.replace("first", "edited").replaceAll("\n", "\r\n"), + ); + const crlf = yield* driver.getReviewDiffFileContents(makeReviewDiffFileContentsInput(cwd)); + assert.strictEqual(crlf.newContents, "$Id$\r\nedited\r\nsecond\r\n"); + yield* fileSystem.writeFile(filePath, disk); + yield* writeTextFile(cwd, ".gitattributes", "README.md ident=set\n"); + const literal = yield* driver.getReviewDiffFileContents( + makeReviewDiffFileContentsInput(cwd), + ); + assert.strictEqual(literal.newContents, smudged.replace("first", "edited")); + yield* writeTextFile(cwd, ".gitattributes", "README.md ident filter=probe\n"); + yield* git(cwd, ["config", "filter.probe.clean", "touch filter-ran; cat"]); + const filtered = yield* driver.getReviewDiffFileContents( + makeReviewDiffFileContentsInput(cwd), + ); + assert.strictEqual(filtered.newContents, literal.newContents); + yield* writeTextFile(cwd, ".gitattributes", "README.md ident filter=unspecified\n"); + yield* git(cwd, ["config", "filter.unspecified.clean", "touch filter-ran; cat"]); + const reservedFilter = yield* driver.getReviewDiffFileContents( + makeReviewDiffFileContentsInput(cwd), + ); + assert.strictEqual(reservedFilter.newContents, literal.newContents); + assert.isFalse(yield* fileSystem.exists((yield* Path.Path).join(cwd, "filter-ran"))); + assert.deepStrictEqual(yield* fileSystem.readFile(filePath), disk); + }), + ); + it.effect("Changes combines commits, uncommitted edits, and untracked files", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -3583,7 +4219,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const clean = yield* driver.getReviewDiffPreview({ cwd, baseRef: initialBranch }); assert.deepStrictEqual(sourceFiles(clean, "working-tree"), []); assert.deepStrictEqual(sourceFiles(clean, "branch-range"), [ - { path: "README.md", previousPath: null, additions: 1, deletions: 0 }, + { + path: "README.md", + previousPath: null, + changeType: "change", + additions: 1, + deletions: 0, + }, ]); yield* writeTextFile(cwd, "README.md", "# test\ncommitted\ndirty\n"); @@ -3591,14 +4233,26 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const dirty = yield* driver.getReviewDiffPreview({ cwd, baseRef: initialBranch }); const changes = dirty.sources.find((source) => source.kind === "branch-range")!; assert.deepStrictEqual(changes.files, [ - { path: "README.md", previousPath: null, additions: 2, deletions: 0 }, - { path: "new.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "README.md", + previousPath: null, + changeType: "change", + additions: 2, + deletions: 0, + }, + { path: "new.txt", previousPath: null, changeType: "new", additions: 1, deletions: 0 }, ]); assert.strictEqual(changes.diff.match(/^diff --git a\/README\.md /gm)?.length, 1); assert.include(changes.diff, "+dirty"); assert.deepStrictEqual(sourceFiles(dirty, "working-tree"), [ - { path: "README.md", previousPath: null, additions: 1, deletions: 0 }, - { path: "new.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "README.md", + previousPath: null, + changeType: "change", + additions: 1, + deletions: 0, + }, + { path: "new.txt", previousPath: null, changeType: "new", additions: 1, deletions: 0 }, ]); }), ); @@ -3627,7 +4281,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const changes = preview.sources.find((source) => source.kind === "branch-range")!; assert.strictEqual(changes.baseRef, "origin/main"); assert.deepStrictEqual(changes.files, [ - { path: "unpushed.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "unpushed.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ]); }), ); @@ -3666,7 +4326,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const changes = preview.sources.find((source) => source.kind === "branch-range")!; assert.strictEqual(changes.baseRef, baseRef); assert.deepStrictEqual(changes.files, [ - { path: "feature.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "feature.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ]); } @@ -3700,8 +4366,20 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const changes = preview.sources.find((source) => source.kind === "branch-range")!; assert.strictEqual(changes.baseRef, "refs/remotes/upstream/main"); assert.deepStrictEqual(changes.files, [ - { path: "feature.txt", previousPath: null, additions: 1, deletions: 0 }, - { path: "upstream.txt", previousPath: null, additions: 1, deletions: 1 }, + { + path: "feature.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, + { + path: "upstream.txt", + previousPath: null, + changeType: "change", + additions: 1, + deletions: 1, + }, ]); const contents = yield* driver.getReviewDiffFileContents( makeReviewDiffFileContentsInput(cwd, { @@ -3756,7 +4434,11 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { // Base config reads start only after status completes; hold every one until the rename lands. const spawner = ChildProcessSpawner.make((command) => Effect.gen(function* () { - if (ChildProcess.isStandardCommand(command) && command.args[0] === "config") { + if ( + ChildProcess.isStandardCommand(command) && + command.args[0] === "config" && + !command.args.includes("status.showUntrackedFiles") + ) { if (!(yield* Ref.getAndSet(renameClaimed, true))) { yield* git(cwd, ["branch", "-m", "t3/temporary", "feature/named"]).pipe( Effect.provideService(GitVcsDriver.GitVcsDriver, outer), @@ -3813,7 +4495,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const changes = preview.sources.find((source) => source.kind === "branch-range")!; assert.strictEqual(changes.baseRef, baseRef); assert.deepStrictEqual(changes.files, [ - { path: "README.md", previousPath: null, additions: 1, deletions: 0 }, + { + path: "README.md", + previousPath: null, + changeType: "change", + additions: 1, + deletions: 0, + }, ]); const scoped = yield* driver.getReviewDiffPreview({ cwd, @@ -3862,7 +4550,15 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const explicit = yield* driver.getReviewDiffPreview({ cwd, baseRef: initialBranch }); assert.deepStrictEqual( explicit.sources.find((source) => source.kind === "branch-range")!.files, - [{ path: "detached.txt", previousPath: null, additions: 1, deletions: 0 }], + [ + { + path: "detached.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, + ], ); const implicit = yield* driver.getReviewDiffPreview({ cwd }); assert.deepStrictEqual( @@ -3892,7 +4588,7 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const changes = preview.sources.find((source) => source.kind === "branch-range")!; assert.isNull(changes.baseRef); assert.deepStrictEqual(changes.files, [ - { path: "new.txt", previousPath: null, additions: 2, deletions: 0 }, + { path: "new.txt", previousPath: null, changeType: "new", additions: 2, deletions: 0 }, ]); const status = yield* driver.statusDetailsLocal(cwd, { includeBranchChanges: true }); assert.deepStrictEqual(status.branchChanges, { @@ -3923,7 +4619,15 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const preview = yield* driver.getReviewDiffPreview({ cwd, baseRef: initialBranch }); assert.deepStrictEqual( preview.sources.find((source) => source.kind === "branch-range")!.files, - [{ path: "feature.txt", previousPath: null, additions: 1, deletions: 0 }], + [ + { + path: "feature.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, + ], ); } }), @@ -4049,6 +4753,7 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { { path: "repeated.txt", previousPath: null, + changeType: "change", additions: Number(additions), deletions: Number(removals), }, @@ -4218,6 +4923,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { yield* git(cwd, ["checkout", "-b", "feature/many-untracked"]); yield* writeTextFile(cwd, ".git/info/exclude", "ignored.txt\n"); yield* writeTextFile(cwd, "bulk[1]/ignored.txt", "ignored\n"); + const clean = yield* driver.statusDetailsLocal(cwd, { includeBranchChanges: true }); + assert.deepStrictEqual(clean.branchChanges, { + baseRef: "main", + insertions: 0, + deletions: 0, + }); + assert.deepStrictEqual(untrackedScanRoots, []); yield* Effect.forEach( Array.from({ length: 1_001 }, (_, index) => `bulk[1]/${index}.txt`), (file) => writeTextFile(cwd, file, "one\ntwo\n"), @@ -4242,13 +4954,15 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { deletions: 0, }); assert.deepStrictEqual(untrackedScanRoots, [ - scenario === "normal" ? [":(top,literal)bulk[1]/"] : [], + scenario === "normal" || scenario === "no" ? [":(top,literal)bulk[1]/"] : [], ]); - if (scenario === "normal") { + if (scenario === "normal" || scenario === "no") { assert.deepStrictEqual( status.workingTree.files.map((file) => file.path), ["bulk[1]/"], ); + } else if (scenario === "all") { + assert.lengthOf(status.workingTree.files, 1_001); } } }), @@ -4347,8 +5061,20 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.deepStrictEqual( preview.sources.find((source) => source.kind === "working-tree")?.files, [ - { path: "asset.data", previousPath: null, additions: 0, deletions: 1 }, - { path: "ordinary.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "asset.data", + previousPath: null, + changeType: "deleted", + additions: 0, + deletions: 1, + }, + { + path: "ordinary.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ], ); }), @@ -4514,7 +5240,7 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); - it.effect("reports remote status on unborn HEAD without failing", () => + it.effect("reports status on unborn HEAD without failing or dropping its upstream", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); const driver = yield* GitVcsDriver.GitVcsDriver; @@ -4528,6 +5254,32 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.equal(status.hasUpstream, false); assert.equal(status.aheadCount, 0); assert.equal(status.behindCount, 0); + + const seed = yield* makeTmpDir(); + const remote = yield* makeTmpDir("git-remote-"); + yield* initRepoWithCommit(seed); + yield* git(remote, ["init", "--bare"]); + yield* git(seed, ["push", remote, `HEAD:refs/heads/${initialBranch}`]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["fetch", "origin"]); + yield* git(cwd, ["config", `branch.${initialBranch}.remote`, "origin"]); + yield* git(cwd, ["config", `branch.${initialBranch}.merge`, `refs/heads/${initialBranch}`]); + assert.equal((yield* driver.statusDetailsLocal(cwd)).hasUpstream, true); + }), + ); + + it.effect("reports no branch while a clone has not checked out HEAD yet", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* driver.initRepo({ cwd }); + // `git clone` points HEAD here until checkout; status then prints `branch.head (unknown)`. + yield* (yield* FileSystem.FileSystem).writeFileString( + NodePath.join(cwd, ".git", "HEAD"), + "ref: refs/heads/.invalid\n", + ); + + assert.equal((yield* driver.statusDetailsLocal(cwd)).branch, null); }), ); @@ -4598,6 +5350,100 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("a ref snapshot detects branch identity, ref and ancestry changes", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const remote = yield* makeTmpDir("git-vcs-driver-remote-"); + const updater = yield* makeTmpDir("git-vcs-driver-updater-"); + const { initialBranch } = yield* initRepoWithCommit(cwd); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["push", "-u", "origin", initialBranch]); + yield* git(updater, ["clone", remote, "."]); + yield* git(updater, ["config", "user.email", "test@test.com"]); + yield* git(updater, ["config", "user.name", "Test"]); + const driver = yield* GitVcsDriver.GitVcsDriver; + const snapshot = driver.readRefSnapshot(cwd); + + // Newer Git can create origin/HEAD on the first fetch. + yield* git(cwd, ["fetch", "origin"]); + const initial = yield* snapshot; + assert.isNotNull(initial); + yield* git(cwd, ["fetch", "origin"]); + assert.equal(yield* snapshot, initial); + + yield* git(cwd, ["branch", "same-commit"]); + const beforeCheckout = yield* snapshot; + const refsBeforeCheckout = yield* git(cwd, ["show-ref", "--head"]); + yield* git(cwd, ["checkout", "same-commit"]); + assert.equal(yield* git(cwd, ["show-ref", "--head"]), refsBeforeCheckout); + const afterCheckout = yield* snapshot; + assert.isNotNull(afterCheckout); + assert.notEqual(afterCheckout, beforeCheckout); + yield* git(cwd, ["checkout", "--detach"]); + assert.equal(yield* git(cwd, ["show-ref", "--head"]), refsBeforeCheckout); + const detached = yield* snapshot; + assert.isNotNull(detached); + assert.notEqual(detached, beforeCheckout); + assert.notEqual(detached, afterCheckout); + assert.equal(yield* snapshot, detached); + yield* git(cwd, ["checkout", initialBranch]); + assert.equal(yield* snapshot, beforeCheckout); + yield* git(cwd, ["branch", "-d", "same-commit"]); + + yield* writeTextFile(updater, "remote.txt", "remote\n"); + yield* git(updater, ["add", "remote.txt"]); + yield* git(updater, ["commit", "-m", "remote commit"]); + yield* git(updater, ["push", "origin", initialBranch]); + assert.equal(yield* snapshot, initial); + yield* git(cwd, ["fetch", "origin"]); + const fetched = yield* snapshot; + assert.notEqual(fetched, initial); + + const shallow = yield* makeTmpDir("git-vcs-driver-shallow-"); + yield* git(shallow, ["clone", "--depth=1", NodeURL.pathToFileURL(remote).href, "."]); + const shallowRefs = yield* git(shallow, ["show-ref", "--head"]); + const shallowSnapshot = yield* driver.readRefSnapshot(shallow); + assert.isNotNull(shallowSnapshot); + const shallowWorktree = yield* makeTmpDir("git-vcs-driver-shallow-worktree-"); + yield* git(shallow, ["worktree", "add", "--detach", shallowWorktree]); + const worktreeSnapshot = yield* driver.readRefSnapshot(shallowWorktree); + assert.isNotNull(worktreeSnapshot); + yield* git(shallow, ["fetch", "--unshallow", "origin"]); + assert.equal(yield* git(shallow, ["show-ref", "--head"]), shallowRefs); + const unshallowSnapshot = yield* driver.readRefSnapshot(shallow); + const unshallowWorktreeSnapshot = yield* driver.readRefSnapshot(shallowWorktree); + assert.isNotNull(unshallowSnapshot); + assert.isNotNull(unshallowWorktreeSnapshot); + assert.notEqual(unshallowSnapshot, shallowSnapshot); + assert.notEqual(unshallowWorktreeSnapshot, worktreeSnapshot); + + yield* git(cwd, ["checkout", "--detach"]); + yield* git(cwd, ["commit", "--allow-empty", "-m", "detached"]); + assert.notEqual(yield* snapshot, fetched); + + const head = yield* git(cwd, ["rev-parse", "HEAD"]); + yield* driver.execute({ + operation: "GitVcsDriver.test.git", + cwd, + args: ["update-ref", "--stdin"], + stdin: Array.from( + { length: 16_000 }, + (_, index) => `create refs/heads/snapshot-${String(index).padStart(5, "0")} ${head}\n`, + ).join(""), + }); + yield* git(cwd, ["pack-refs", "--all"]); + const large = yield* snapshot; + assert.isNotNull(large); + assert.isAbove(Buffer.byteLength(large ?? "", "utf8"), 1_000_000); + assert.equal(yield* snapshot, large); + yield* git(cwd, ["update-ref", "refs/heads/snapshot-00000", "HEAD~1"]); + assert.notEqual(yield* snapshot, large); + + assert.isNull(yield* driver.readRefSnapshot(yield* makeTmpDir())); + }), + ); + it.effect("does not start Git auto-maintenance from background upstream fetches", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -4708,7 +5554,12 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const defaultBranch = yield* driver.resolveDefaultBranchName(cwd, "origin"); assert.notEqual(defaultBranch, "development"); - assert.isNull(yield* driver.resolveWorktreeBaseRemote({ cwd, refName: "origin/HEAD" })); + for (const refName of ["origin/HEAD", "HEAD~1", "main^", "@{-1}", "-", ":/fix"]) + assert.isNull(yield* driver.resolveWorktreeBaseRemote({ cwd, refName })); + assert.deepEqual( + yield* driver.resolveWorktreeBaseRemote({ cwd, refName: "feature/a\u00a0b" }), + { remoteName: "origin", branchName: "feature/a\u00a0b" }, + ); }), ); @@ -4847,25 +5698,21 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); - it.effect("reports staged file counts on unborn HEAD without failing", () => + it.effect("reports net file counts from the empty tree on unborn HEAD", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); const driver = yield* GitVcsDriver.GitVcsDriver; yield* driver.initRepo({ cwd }); - yield* git(cwd, ["config", "user.email", "test@test.com"]); - yield* git(cwd, ["config", "user.name", "Test"]); yield* writeTextFile(cwd, "initial.ts", "// first file\n"); yield* git(cwd, ["add", "initial.ts"]); + yield* writeTextFile(cwd, "initial.ts", "// edited after staging\n"); - const status = yield* driver.statusDetails(cwd); + const status = yield* driver.statusDetailsLocal(cwd); assert.equal(status.isRepo, true); - assert.equal(status.workingTree.files.length, 1); - const file = status.workingTree.files[0]; - if (file) { - assert.equal(file.path, "initial.ts"); - assert.equal(file.insertions, 1); - } + assert.deepStrictEqual(status.workingTree.files, [ + { path: "initial.ts", insertions: 1, deletions: 0 }, + ]); }), ); }); @@ -5273,6 +6120,172 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }); describe("worktree operations", () => { + it.effect.each(["current", "legacy"] as const)( + "preserves another creator's worktree on its new branch at the same path (%s)", + (version) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const path = NodePath.join(yield* makeTmpDir("git-worktrees-"), "contested"); + const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; + const reachedAdd = yield* Deferred.make(); + let paused = false; + const spawner = ChildProcessSpawner.make((command) => + Effect.gen(function* () { + if ( + ChildProcess.isStandardCommand(command) && + command.args.includes("worktree") && + command.args.includes("add") + ) { + if (command.args.includes("-h")) { + return version === "legacy" + ? makeFailedHandle("usage: git worktree add [--lock]") + : yield* delegate.spawn(command); + } + if (!paused) { + paused = true; + yield* Deferred.succeed(reachedAdd, undefined); + return yield* Effect.never; + } + } + return yield* delegate.spawn(command); + }), + ); + const driver = yield* makeGitVcsDriverCore().pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provide(layerServerConfig), + ); + const creator = yield* driver + .createWorktree({ + cwd, + path, + refName: initialBranch, + newRefName: "contested", + }) + .pipe(Effect.forkChild); + yield* Deferred.await(reachedAdd); + yield* git(cwd, ["worktree", "add", "--", path, "contested"]); + yield* writeTextFile(path, "owned-by-B.txt", "keep this work\n"); + yield* Fiber.interrupt(creator); + assert.include(yield* git(cwd, ["worktree", "list", "--porcelain"]), path); + assert.equal(yield* git(path, ["branch", "--show-current"]), "contested"); + assert.equal( + yield* (yield* FileSystem.FileSystem).readFileString( + NodePath.join(path, "owned-by-B.txt"), + ), + "keep this work\n", + ); + const created = yield* driver.createWorktree({ + cwd, + path: `${path}-retry`, + refName: initialBranch, + newRefName: "retry", + }); + assert.equal(yield* git(created.worktree.path, ["branch", "--show-current"]), "retry"); + assert.notInclude(yield* git(cwd, ["worktree", "list", "--porcelain"]), "locked"); + }), + ); + + it.effect.each(["main", "linked"] as const)( + "removes its worktree and branch when cancelled before the claim (%s)", + (checkout) => + Effect.gen(function* () { + const repo = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(repo); + const cwd = + checkout === "main" + ? repo + : NodePath.join(yield* makeTmpDir("git-linked-"), "checkout"); + if (checkout === "linked") { + yield* git(repo, ["config", "worktree.useRelativePaths", "true"]); + yield* git(repo, ["worktree", "add", "--detach", cwd]); + } + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const driver = yield* GitVcsDriver.GitVcsDriver; + const worktreePath = path.join(yield* makeTmpDir("git-worktrees-"), "cancelled ü"); + const hookPath = path.join(repo, ".git/hooks/post-checkout"); + const connected = Promise.withResolvers(); + const sockets = new Set(); + const server = yield* Effect.acquireRelease( + Effect.sync(() => + NodeNet.createServer((socket) => { + sockets.add(socket); + connected.resolve(); + }), + ), + (server) => + Effect.sync(() => { + for (const socket of sockets) socket.destroy(); + server.close(); + }), + ); + yield* Effect.promise( + () => + new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }), + ); + const address = server.address(); + assert.isTrue(address !== null && typeof address !== "string"); + if (address === null || typeof address === "string") return; + const gate = path.join(cwd, "hook-gate.cjs"); + yield* fs.writeFileString( + gate, + `const s = require('node:net').connect(${address.port}, '127.0.0.1'); s.on('connect', () => process.stderr.write('Updating files: 50% (1/1000)\\n')); s.resume();`, + ); + yield* writeTextFile( + repo, + ".git/hooks/post-checkout", + `#!/bin/sh\nexec "${process.execPath.replaceAll("\\", "/")}" "${gate.replaceAll("\\", "/")}"\n`, + ); + yield* fs.chmod(hookPath, 0o755); + const hookStarted = yield* Deferred.make(); + let claims = 0; + const input = { + cwd, + path: worktreePath, + refName: initialBranch, + newRefName: "t3/cancel", + }; + const worktrees = git(cwd, ["worktree", "list", "--porcelain"]); + + const cancelled = yield* driver + .createWorktree(input, { + progress: { + onCheckoutProgress: ({ total }) => + total === 1000 ? Deferred.succeed(hookStarted, undefined) : Effect.void, + onWorktreeClaimed: () => Effect.sync(() => void claims++), + }, + }) + .pipe(Effect.forkChild); + yield* Deferred.await(hookStarted); + yield* Effect.promise(() => connected.promise); + assert.include(yield* worktrees, worktreePath); + yield* Fiber.interrupt(cancelled); + + assert.equal(claims, 0); + assert.notInclude(yield* worktrees, worktreePath); + assert.equal(yield* git(cwd, ["branch", "--list", "t3/cancel"]), ""); + assert.isFalse(yield* fs.exists(worktreePath)); + + // A retry gets the same path and branch, and a claimed worktree outlives a later cancel. + yield* fs.remove(hookPath); + const claimed = yield* Deferred.make(); + const retry = yield* driver + .createWorktree(input, { + progress: { onWorktreeClaimed: () => Deferred.succeed(claimed, undefined) }, + }) + .pipe(Effect.forkChild); + yield* Deferred.await(claimed); + yield* Fiber.interrupt(retry); + assert.include(yield* worktrees, worktreePath); + assert.notInclude(yield* worktrees, "locked"); + assert.equal(yield* git(worktreePath, ["branch", "--show-current"]), "t3/cancel"); + }), + ); + it.effect("uses parallel checkout without skipping filters or hooks", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -5329,6 +6342,19 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { `${expected}\n`, ); } + yield* writeTextFile( + cwd, + ".git/hooks/post-checkout", + '#!/bin/sh\ngit worktree unlock "$PWD" 2>/dev/null || :\ngit worktree lock --reason user-kept "$PWD"\n', + ); + const lockedPath = path.join(yield* makeTmpDir("git-worktrees-"), "hook-locked"); + yield* driver.createWorktree({ + cwd, + path: lockedPath, + refName: initialBranch, + newRefName: "feature/hook-locked", + }); + assert.include(yield* git(cwd, ["worktree", "list", "--porcelain"]), "locked user-kept"); }), ); @@ -5639,14 +6665,52 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); - it.effect.each(["release/v2", "refs/heads/release/v2", "deadbeef"])( - "records the branch name when creating a worktree from %s", - (baseRef) => - Effect.gen(function* () { - const cwd = yield* makeTmpDir(); - yield* initRepoWithCommit(cwd); - const baseBranch = baseRef.replace(/^refs\/heads\//, ""); - yield* git(cwd, ["branch", baseBranch]); + it.effect("gives parallel renames to one generated name distinct branches", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const path = yield* Path.Path; + const worktrees = yield* makeTmpDir("git-worktrees-"); + const driver = yield* GitVcsDriver.GitVcsDriver; + // A multi-model send launches one worktree per model with the same message, so their + // generated names often match and the renames can land together. + const temporaryBranches = ["t3/aaaa1111", "t3/bbbb2222", "t3/cccc3333"]; + yield* Effect.forEach(temporaryBranches, (branch) => + driver.createWorktree({ + cwd, + path: path.join(worktrees, branch.replace("/", "-")), + refName: initialBranch, + newRefName: branch, + }), + ); + + const renamed = yield* Effect.forEach( + temporaryBranches, + (branch) => + driver.renameBranch({ + cwd: path.join(worktrees, branch.replace("/", "-")), + oldBranch: branch, + newBranch: "feat/add-search", + }), + { concurrency: "unbounded" }, + ); + + assert.deepStrictEqual(renamed.map((result) => result.branch).toSorted(), [ + "feat/add-search", + "feat/add-search-1", + "feat/add-search-2", + ]); + }), + ); + + it.effect.each(["release/v2", "refs/heads/release/v2", "deadbeef"])( + "records the branch name when creating a worktree from %s", + (baseRef) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const baseBranch = baseRef.replace(/^refs\/heads\//, ""); + yield* git(cwd, ["branch", baseBranch]); const path = yield* Path.Path; const driver = yield* GitVcsDriver.GitVcsDriver; yield* driver.createWorktree({ @@ -5847,12 +6911,16 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const fileSystem = yield* FileSystem.FileSystem; const pathService = yield* Path.Path; + const submoduleRepo = yield* makeTmpDir("git-submodule-"); + yield* initRepoWithCommit(submoduleRepo); + const cwd = yield* makeTmpDir(); const { initialBranch } = yield* initRepoWithCommit(cwd); - yield* writeTextFile( - cwd, - ".gitmodules", - '[submodule "missing"]\n\tpath = missing\n\turl = /nonexistent/repo.git\n', + yield* git(cwd, ["submodule", "add", NodeURL.pathToFileURL(submoduleRepo).href, "shared"]); + yield* fileSystem.writeFileString( + pathService.join(cwd, ".gitmodules"), + (yield* fileSystem.readFileString(pathService.join(cwd, ".gitmodules"))) + + '[submodule "missing"]\n\tpath = missing\n\turl = /nonexistent/repo.git\n', ); yield* git(cwd, ["add", "."]); const gitlink = yield* git(cwd, ["rev-parse", "HEAD"]); @@ -5885,7 +6953,9 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.lengthOf(results, 1); assert.isFalse(results[0]!.ok); assert.include(results[0]!.detail, "GitVcsDriver.createWorktree.updateSubmodules"); - }), + // A submodule that did clone must not be left as a bare gitdir that reads as modified. + assert.equal(yield* git(worktreePath, ["status", "--porcelain"]), ""); + }).pipe(withGitFileProtocol), ); it.effect("creates worktrees under the configured worktrees directory", () => @@ -6207,6 +7277,41 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect.skipIf(!symlinksSupported)( + "removes a worktree whose .git file git-annex replaced with a symlink, keeping the symlink when refused", + () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + const pathService = yield* Path.Path; + const worktreePath = pathService.join(yield* makeTmpDir("git-worktrees-"), "annexed"); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* driver.createWorktree({ + cwd, + path: worktreePath, + refName: initialBranch, + newRefName: "feature/annexed", + }); + const gitFile = pathService.join(worktreePath, ".git"); + const adminDir = (yield* git(worktreePath, ["rev-parse", "--absolute-git-dir"])).trim(); + NodeFS.rmSync(gitFile); + NodeFS.symlinkSync(adminDir, gitFile); + yield* writeTextFile(worktreePath, "notes.txt", "draft\n"); + + const refused = yield* Effect.result(driver.removeWorktree({ cwd, path: worktreePath })); + + assert.isTrue(Result.isFailure(refused)); + assert.isTrue(NodeFS.lstatSync(gitFile).isSymbolicLink()); + assert.equal(NodeFS.realpathSync(gitFile), NodeFS.realpathSync(adminDir)); + NodeFS.rmSync(pathService.join(worktreePath, "notes.txt")); + + yield* driver.removeWorktree({ cwd, path: worktreePath }); + + assert.isFalse(NodeFS.existsSync(worktreePath)); + assert.isFalse(NodeFS.existsSync(adminDir)); + }), + ); + it.effect("leaves a folder alone that only shares the worktree's short name", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -6331,6 +7436,112 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }); describe("remote operations", () => { + it.effect.each([ + { + configuration: "ordinary", + fetchRefspecs: ["+refs/heads/*:refs/remotes/origin/*"], + addsMapping: false, + }, + { + configuration: "restricted", + fetchRefspecs: [ + "+refs/heads/main:refs/remotes/origin/main", + "+refs/heads/team/*:refs/remotes/origin/team/*", + "+refs/heads/release/release-*:refs/remotes/origin/release/release-*", + ], + addsMapping: true, + }, + ])( + "sets a fetched branch's upstream with $configuration fetch configuration", + ({ fetchRefspecs, addsMapping }) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const remote = yield* makeTmpDir("git-remote-"); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* git(cwd, ["config", "--unset-all", "remote.origin.fetch"]); + for (const refspec of fetchRefspecs) { + yield* git(cwd, ["config", "--add", "remote.origin.fetch", refspec]); + } + const remoteBranch = "feature/example-change"; + const branch = "review/example-change"; + const mapping = `+refs/heads/${remoteBranch}*:refs/remotes/origin/${remoteBranch}*`; + yield* git(cwd, [ + "push", + remote, + "HEAD:refs/heads/main", + `HEAD:refs/heads/${remoteBranch}`, + ]); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* driver.fetchRemoteTrackingBranch({ cwd, remoteName: "origin", remoteBranch }); + yield* git(cwd, ["checkout", "--no-track", "-b", branch, `origin/${remoteBranch}`]); + + const input = { cwd, branch, remoteName: "origin", remoteBranch }; + yield* driver.setBranchUpstream(input); + yield* driver.setBranchUpstream(input); + + assert.equal( + yield* git(cwd, ["rev-parse", "--abbrev-ref", `${branch}@{upstream}`]), + `origin/${remoteBranch}`, + ); + assert.equal(yield* git(cwd, ["config", `branch.${branch}.remote`]), "origin"); + assert.equal( + yield* git(cwd, ["config", `branch.${branch}.merge`]), + `refs/heads/${remoteBranch}`, + ); + assert.deepEqual( + (yield* git(cwd, ["config", "--get-all", "remote.origin.fetch"])).split("\n"), + addsMapping ? [...fetchRefspecs, mapping] : fetchRefspecs, + ); + // A merged PR's head is deleted; later fetches of the remote must keep working. + yield* git(remote, ["--git-dir", remote, "branch", "-D", remoteBranch]); + yield* git(cwd, ["fetch", "origin"]); + }), + ); + + it.effect.each(["ordinary", "restricted"] as const)( + "retries upstream configuration with a config lock and %s fetch mappings", + (configuration) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const { initialBranch } = yield* initRepoWithCommit(cwd); + yield* git(cwd, ["remote", "add", "origin", "https://example.invalid/repo.git"]); + if (configuration === "restricted") { + yield* git(cwd, [ + "config", + "remote.origin.fetch", + `+refs/heads/${initialBranch}:refs/remotes/origin/${initialBranch}`, + ]); + } + yield* git(cwd, ["branch", "review/pr-head"]); + yield* git(cwd, ["update-ref", "refs/remotes/origin/pr-head", "HEAD"]); + const lockPath = (yield* Path.Path).join(cwd, ".git", "config.lock"); + yield* (yield* FileSystem.FileSystem).writeFileString(lockPath, ""); + const { driver, failures } = yield* makeDriverReleasingLock( + configuration === "restricted" + ? (args) => args[0] === "config" && args[1] === "--add" + : (args) => args.includes("--set-upstream-to"), + lockPath, + ); + + yield* TestClock.withLive( + driver.setBranchUpstream({ + cwd, + branch: "review/pr-head", + remoteName: "origin", + remoteBranch: "pr-head", + }), + ); + + assert.equal(yield* Ref.get(failures), 1); + assert.equal( + yield* git(cwd, ["rev-parse", "--abbrev-ref", "review/pr-head@{upstream}"]), + "origin/pr-head", + ); + }), + ); + it.effect("ensureRemote reuses a partial clone remote and avoids its name", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -6888,30 +8099,49 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { ); it.effect.each([ - { location: "root", conflict: false }, - { location: "nested", conflict: false }, - { location: "worktree", conflict: false }, - { location: "root", conflict: true }, - { location: "nested", conflict: true }, - { location: "worktree", conflict: true }, + { location: "root", conflict: false, encoding: "utf8" }, + { location: "nested", conflict: false, encoding: "utf8" }, + { location: "worktree", conflict: false, encoding: "utf8" }, + { location: "root", conflict: true, encoding: "utf8" }, + { location: "nested", conflict: true, encoding: "utf8" }, + { location: "worktree", conflict: true, encoding: "utf8" }, + { location: "root", conflict: true, encoding: "utf16le" }, + { location: "root", conflict: true, encoding: "utf16le", large: true }, ])( - "rejects selected-file merge commits without mutation ($location, conflict: $conflict)", - ({ location, conflict }) => + "rejects selected-file merge commits without mutation ($location, conflict: $conflict, $encoding)", + ({ location, conflict, encoding, large }) => Effect.gen(function* () { let cwd = yield* makeTmpDir(); const { initialBranch } = yield* initRepoWithCommit(cwd); + const writeReadme = (text: string) => + Effect.sync(() => + NodeFS.writeFileSync( + NodePath.join(cwd, "README.md"), + encoding === "utf16le" ? Buffer.from(`\ufeff${text}`, "utf16le") : text, + ), + ); + if (encoding === "utf16le") { + yield* writeTextFile( + cwd, + ".gitattributes", + "README.md text working-tree-encoding=UTF-16\n", + ); + yield* writeReadme("base\n"); + yield* git(cwd, ["add", "."]); + yield* git(cwd, ["commit", "-m", "Declare UTF-16 encoding"]); + } const driver = yield* GitVcsDriver.GitVcsDriver; const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; yield* git(cwd, ["checkout", "-b", "feature"]); yield* writeTextFile(cwd, "feature.txt", "feature\n"); - if (conflict) yield* writeTextFile(cwd, "README.md", "feature version\n"); + if (conflict) yield* writeReadme("feature version\n"); yield* git(cwd, ["add", "."]); yield* git(cwd, ["commit", "-m", "Feature"]); const featureHead = yield* git(cwd, ["rev-parse", "HEAD"]); yield* git(cwd, ["checkout", initialBranch]); yield* writeTextFile(cwd, "main.txt", "main\n"); - if (conflict) yield* writeTextFile(cwd, "README.md", "main version\n"); + if (conflict) yield* writeReadme("main version\n"); yield* git(cwd, ["add", "."]); yield* git(cwd, ["commit", "-m", "Main"]); const mainHead = yield* git(cwd, ["rev-parse", "HEAD"]); @@ -6960,6 +8190,7 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.equal(yield* git(cwd, ["rev-parse", "HEAD"]), mainHead); assert.equal(yield* git(cwd, ["ls-files", "--unmerged"]), unmerged); + const resolvedText = `Heading\n=======\n\nresolved${large ? "x".repeat(17 * 1024 * 1024) : ""}\n`; if (conflict) { const unresolved = yield* driver .commit(commitCwd, "Unresolved merge", "", { stage: {} }) @@ -6968,8 +8199,10 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.equal(yield* git(cwd, ["rev-parse", "HEAD"]), mainHead); assert.equal(yield* git(cwd, ["ls-files", "--unmerged"]), unmerged); // A lone "=======" is a heading underline, not a conflict boundary. - yield* writeTextFile(cwd, "README.md", "Heading\n=======\n\nresolved\n"); - assert.isNotNull(yield* driver.prepareCommitContext(commitCwd)); + yield* writeReadme(resolvedText); + assert.isNotNull( + yield* driver.prepareCommitContext(commitCwd, undefined, { includePatch: false }), + ); } yield* driver.commit(commitCwd, "Finish merge", "", { stage: {} }); assert.equal( @@ -6979,7 +8212,12 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.isFalse(NodeFS.existsSync(path.join(gitDir, "MERGE_HEAD"))); assert.equal(yield* git(cwd, ["show", "HEAD:feature.txt"]), "feature"); assert.equal(yield* git(cwd, ["show", "HEAD:unrelated.txt"]), "unrelated"); - if (conflict) + if (large) { + assert.equal( + yield* git(cwd, ["cat-file", "-s", "HEAD:README.md"]), + String(Buffer.byteLength(resolvedText)), + ); + } else if (conflict) assert.equal( yield* git(cwd, ["show", "HEAD:README.md"]), "Heading\n=======\n\nresolved", @@ -7018,41 +8256,108 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { ); it.effect.each([ - { operation: "cherry-pick", head: "CHERRY_PICK_HEAD" }, - { operation: "revert", head: "REVERT_HEAD" }, - ])("rejects selected-file commits during a $operation", ({ operation, head }) => + { operation: "cherry-pick", head: "CHERRY_PICK_HEAD", refs: "files" }, + { operation: "revert", head: "REVERT_HEAD", refs: "files" }, + // Reftable keeps CHERRY_PICK_HEAD and REVERT_HEAD as refs with no file under .git. + { operation: "cherry-pick", head: "CHERRY_PICK_HEAD", refs: "reftable" }, + { operation: "revert", head: "REVERT_HEAD", refs: "reftable" }, + ])( + "rejects selected-file commits during a $operation ($refs)", + ({ operation, head, refs }, ctx) => + Effect.gen(function* () { + if (refs === "reftable" && !gitHasReftable) + return ctx.skip("Needs Git 2.45 or later for reftable."); + const cwd = yield* makeTmpDir(); + if (refs === "reftable") yield* commitInitialRepo(cwd, ["--ref-format=reftable"]); + else yield* initRepoWithCommit(cwd); + // Reftable leaves .git/HEAD as a stub, so ask Git for the branch. + const initialBranch = yield* git(cwd, ["symbolic-ref", "--short", "HEAD"]); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* git(cwd, ["checkout", "-b", "other"]); + yield* writeTextFile(cwd, "README.md", "other version\n"); + yield* writeTextFile(cwd, "picked.txt", "picked\n"); + yield* git(cwd, ["add", "."]); + yield* git(cwd, ["commit", "-m", "Other"]); + if (operation === "cherry-pick") yield* git(cwd, ["checkout", initialBranch]); + yield* writeTextFile(cwd, "README.md", "main version\n"); + yield* git(cwd, ["commit", "-am", "Main"]); + const originalHead = yield* git(cwd, ["rev-parse", "HEAD"]); + yield* git(cwd, ["branch", head]); + const result = yield* driver.execute({ + operation: `test.${operation}`, + cwd, + args: [operation, operation === "cherry-pick" ? "other" : "HEAD~1"], + allowNonZeroExit: true, + }); + assert.equal(result.exitCode, 1); + yield* writeTextFile(cwd, "README.md", "resolved version\n"); + yield* git(cwd, ["add", "README.md"]); + const selected = operation === "cherry-pick" ? "picked.txt" : "README.md"; + + const commit = yield* driver + .commit(cwd, "Partial", "", { stage: { filePaths: [selected] } }) + .pipe(Effect.result); + + assert.isTrue(Result.isFailure(commit), `Selected-file ${operation} commit succeeded`); + assert.equal(yield* git(cwd, ["rev-parse", "HEAD"]), originalHead); + assert.equal( + yield* git(cwd, [ + "-c", + "core.warnAmbiguousRefs=false", + "rev-parse", + "--verify", + "--symbolic-full-name", + head, + ]), + head, + ); + }), + ); + + it.effect("rejects selected-file commits during a multi-commit no-commit cherry-pick", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); const { initialBranch } = yield* initRepoWithCommit(cwd); const driver = yield* GitVcsDriver.GitVcsDriver; - yield* git(cwd, ["checkout", "-b", "other"]); - yield* writeTextFile(cwd, "README.md", "other version\n"); - yield* writeTextFile(cwd, "picked.txt", "picked\n"); + const path = yield* Path.Path; + yield* writeTextFile(cwd, "a.txt", "base a\n"); + yield* writeTextFile(cwd, "b.txt", "base b\n"); yield* git(cwd, ["add", "."]); - yield* git(cwd, ["commit", "-m", "Other"]); - if (operation === "cherry-pick") yield* git(cwd, ["checkout", initialBranch]); - yield* writeTextFile(cwd, "README.md", "main version\n"); + yield* git(cwd, ["commit", "-m", "Base"]); + yield* git(cwd, ["checkout", "-b", "feature"]); + yield* writeTextFile(cwd, "a.txt", "feature a\n"); + yield* writeTextFile(cwd, "b.txt", "feature b\n"); + yield* git(cwd, ["commit", "-am", "Feature one"]); + yield* writeTextFile(cwd, "later.txt", "later\n"); + yield* git(cwd, ["add", "."]); + yield* git(cwd, ["commit", "-m", "Feature two"]); + yield* git(cwd, ["checkout", initialBranch]); + yield* writeTextFile(cwd, "a.txt", "main a\n"); + yield* writeTextFile(cwd, "b.txt", "main b\n"); yield* git(cwd, ["commit", "-am", "Main"]); - const originalHead = yield* git(cwd, ["rev-parse", "HEAD"]); - const result = yield* driver.execute({ - operation: `test.${operation}`, + const head = yield* git(cwd, ["rev-parse", "HEAD"]); + const started = yield* driver.execute({ + operation: "test.sequencer", cwd, - args: [operation, operation === "cherry-pick" ? "other" : "HEAD~1"], + args: ["cherry-pick", "--no-commit", "feature~1", "feature"], allowNonZeroExit: true, }); - assert.equal(result.exitCode, 1); - yield* writeTextFile(cwd, "README.md", "resolved version\n"); - yield* git(cwd, ["add", "README.md"]); + assert.equal(started.exitCode, 1); + yield* writeTextFile(cwd, "a.txt", "resolved a\n"); const gitDir = yield* git(cwd, ["rev-parse", "--absolute-git-dir"]); - const selected = operation === "cherry-pick" ? "picked.txt" : "README.md"; - - const commit = yield* driver - .commit(cwd, "Partial", "", { stage: { filePaths: [selected] } }) - .pipe(Effect.result); + assert.isFalse(NodeFS.existsSync(path.join(gitDir, "CHERRY_PICK_HEAD"))); + const names = ["index", "sequencer/todo", "sequencer/head"]; + const before = names.map((name) => NodeFS.readFileSync(path.join(gitDir, name))); - assert.isTrue(Result.isFailure(commit), `Selected-file ${operation} commit succeeded`); - assert.equal(yield* git(cwd, ["rev-parse", "HEAD"]), originalHead); - assert.isTrue(NodeFS.existsSync(`${gitDir}/${head}`)); + const error = yield* driver + .commit(cwd, "Partial", "", { stage: { filePaths: ["a.txt"] } }) + .pipe(Effect.flip); + assert.include(error.detail, "Cannot commit selected files during a cherry-pick or revert"); + assert.deepEqual( + names.map((name) => NodeFS.readFileSync(path.join(gitDir, name))), + before, + ); + assert.equal(yield* git(cwd, ["rev-parse", "HEAD"]), head); }), ); @@ -7083,6 +8388,53 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("keeps other files conflicted through a selected-file commit", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + // git stash pop leaves README.md conflicted with no merge in progress. + yield* writeTextFile(cwd, "README.md", "stashed version\n"); + yield* git(cwd, ["stash"]); + yield* writeTextFile(cwd, "README.md", "committed version\n"); + yield* git(cwd, ["commit", "-am", "Change README"]); + const pop = yield* driver.execute({ + operation: "test.stashPop", + cwd, + args: ["stash", "pop"], + allowNonZeroExit: true, + }); + assert.equal(pop.exitCode, 1); + yield* writeTextFile(cwd, "other.txt", "other\n"); + const head = yield* git(cwd, ["rev-parse", "HEAD"]); + const unmerged = yield* git(cwd, ["ls-files", "--unmerged"]); + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const nested = path.join(cwd, "nested"); + yield* fs.makeDirectory(nested); + // A formatting hook stages its edit into the temporary index that --only commits from. + yield* writeTextFile( + cwd, + ".git/hooks/pre-commit", + "#!/bin/sh\necho formatted >> other.txt\ngit add other.txt\n", + ); + yield* fs.chmod(path.join(cwd, ".git/hooks/pre-commit"), 0o755); + + yield* driver.commit(cwd, "Add other", "", { stage: { filePaths: ["other.txt"] } }); + + assert.equal(yield* git(cwd, ["rev-parse", "HEAD~1"]), head); + assert.equal(yield* git(cwd, ["show", "--pretty=", "--name-only", "HEAD"]), "other.txt"); + assert.equal(yield* git(cwd, ["show", "HEAD:other.txt"]), "other\nformatted"); + assert.equal(yield* git(cwd, ["show", ":other.txt"]), "other\nformatted"); + assert.equal(yield* git(cwd, ["ls-files", "--unmerged"]), unmerged); + const all = yield* driver.prepareCommitContext(nested).pipe(Effect.flip); + assert.equal( + all.detail, + "Cannot commit with unresolved merge conflicts in README.md. Resolve them, then commit.", + ); + }), + ); + const commitWithHooks = (hooks: Record) => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -7377,7 +8729,13 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { const preview = yield* driver.getReviewDiffPreview({ cwd: worktreePath }); const changes = preview.sources.find((source) => source.kind === "branch-range")!; assert.deepStrictEqual(changes.files, [ - { path: "feature.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "feature.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ]); assert.include(changes.diff, "+feature work"); assert.notInclude(changes.diff, "upstream-only work"); @@ -7542,6 +8900,92 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { }), ); + it.effect("republishes a branch whose upstream was deleted", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const remote = yield* makeTmpDir("git-remote-"); + yield* initRepoWithCommit(cwd); + yield* git(remote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", remote]); + yield* (yield* GitVcsDriver.GitVcsDriver).createRef({ + cwd, + refName: "feature/push", + }); + yield* (yield* GitVcsDriver.GitVcsDriver).switchRef({ + cwd, + refName: "feature/push", + }); + yield* writeTextFile(cwd, "feature.txt", "feature\n"); + yield* git(cwd, ["add", "-A"]); + yield* (yield* GitVcsDriver.GitVcsDriver).commit(cwd, "Add feature", ""); + + yield* (yield* GitVcsDriver.GitVcsDriver).pushCurrentBranch(cwd, null); + + yield* git(cwd, ["push", "origin", "--delete", "feature/push"]); + yield* writeTextFile(cwd, "followup.txt", "followup\n"); + yield* git(cwd, ["add", "-A"]); + yield* (yield* GitVcsDriver.GitVcsDriver).commit(cwd, "Add followup", ""); + const republished = yield* (yield* GitVcsDriver.GitVcsDriver).pushCurrentBranch(cwd, null); + assert.deepInclude(republished, { status: "pushed", branch: "feature/push" }); + assert.equal( + yield* git(remote, ["--git-dir=.", "rev-parse", "refs/heads/feature/push"]), + yield* git(cwd, ["rev-parse", "HEAD"]), + ); + + const fork = yield* makeTmpDir("git-fork-"); + yield* git(fork, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "fork", fork]); + yield* git(cwd, ["remote", "add", "feature", fork]); + yield* git(cwd, ["push", "-u", "fork", "feature/push"]); + yield* git(cwd, ["push", "fork", "--delete", "feature/push"]); + yield* writeTextFile(cwd, "fork.txt", "fork\n"); + yield* git(cwd, ["add", "-A"]); + yield* (yield* GitVcsDriver.GitVcsDriver).commit(cwd, "Add fork change", ""); + assert.deepInclude( + yield* (yield* GitVcsDriver.GitVcsDriver).resolvePushTarget(cwd, { + branch: "feature/push", + hasUpstream: false, + }), + { remoteName: "fork", branchName: "feature/push" }, + ); + const forkRepublished = yield* (yield* GitVcsDriver.GitVcsDriver).pushCurrentBranch( + cwd, + null, + ); + assert.deepInclude(forkRepublished, { upstreamBranch: "fork/feature/push" }); + assert.equal( + yield* git(fork, ["--git-dir=.", "rev-parse", "refs/heads/feature/push"]), + yield* git(cwd, ["rev-parse", "HEAD"]), + ); + + yield* git(cwd, ["config", "branch.feature/push.remote", "."]); + yield* git(cwd, ["config", "branch.feature/push.merge", "refs/heads/push"]); + yield* writeTextFile(cwd, "local.txt", "local\n"); + yield* git(cwd, ["add", "-A"]); + yield* (yield* GitVcsDriver.GitVcsDriver).commit(cwd, "Add local change", ""); + const localUpstreamGone = yield* (yield* GitVcsDriver.GitVcsDriver).pushCurrentBranch( + cwd, + null, + ); + assert.deepInclude(localUpstreamGone, { upstreamBranch: "origin/feature/push" }); + + yield* git(cwd, ["push", "fork", "HEAD:refs/heads/release"]); + yield* git(cwd, ["branch", "-u", "fork/release"]); + yield* git(cwd, ["push", "fork", "--delete", "release"]); + yield* writeTextFile(cwd, "release.txt", "release\n"); + yield* git(cwd, ["add", "-A"]); + yield* (yield* GitVcsDriver.GitVcsDriver).commit(cwd, "Add release change", ""); + yield* (yield* GitVcsDriver.GitVcsDriver).pushCurrentBranch(cwd, null); + assert.equal( + yield* (yield* GitVcsDriver.GitVcsDriver).readConfigValue( + cwd, + "branch.feature/push.t3-merge-base", + ), + "release", + ); + }), + ); + it.effect("rebases a pushed branch's pulls when branch.autoSetupRebase asks for it", () => Effect.gen(function* () { const cwd = yield* makeTmpDir(); @@ -7645,6 +9089,16 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { yield* git(originRemote, ["--git-dir=.", "rev-parse", "refs/heads/feature/push"]), publishedHead, ); + + const pushOnlyRemote = yield* makeTmpDir("git-push-only-remote-"); + yield* git(pushOnlyRemote, ["init", "--bare"]); + yield* git(cwd, ["remote", "set-url", "--push", "origin", pushOnlyRemote]); + const pushedToPushUrl = yield* driver.pushCurrentBranch(cwd, null); + assert.deepInclude(pushedToPushUrl, { status: "pushed", branch }); + assert.equal( + yield* git(pushOnlyRemote, ["--git-dir=.", "rev-parse", "refs/heads/feature/push"]), + head, + ); }), ); @@ -7688,14 +9142,15 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { ); it.effect.each([ - { baseBranch: "dev", local: false }, - { baseBranch: "dev", local: true }, - { baseBranch: "feature/base", local: true }, - { baseBranch: "upstream", local: true }, - { baseBranch: "upstream/feature/x", local: true }, + { baseBranch: "dev", local: false, branch: "feature/x", deleted: false }, + { baseBranch: "dev", local: true, branch: "feature/x", deleted: false }, + { baseBranch: "feature/base", local: true, branch: "feature/x", deleted: false }, + { baseBranch: "upstream", local: true, branch: "feature/x", deleted: false }, + { baseBranch: "upstream/feature/x", local: true, branch: "feature/x", deleted: false }, + { baseBranch: "dev", local: false, branch: "origin/dev", deleted: true }, ])( - "publishes a branch tracking $baseBranch (local=$local) under its own name, not the base", - ({ baseBranch, local }) => + "publishes $branch tracking $baseBranch (local=$local, deleted=$deleted) without updating the base", + ({ baseBranch, local, branch, deleted }) => Effect.gen(function* () { const cwd = yield* makeTmpDir(); const remote = yield* makeTmpDir("git-remote-"); @@ -7720,55 +9175,78 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { "checkout", "--track", "-b", - "feature/x", + branch, local ? `refs/heads/${baseBranch}` : `refs/remotes/upstream/${baseBranch}`, ]); assert.equal( - yield* driver.readConfigValue(cwd, "branch.feature/x.remote"), + yield* driver.readConfigValue(cwd, `branch.${branch}.remote`), local ? "." : "upstream", ); yield* writeTextFile(cwd, "feature.txt", "feature\n"); yield* git(cwd, ["add", "-A"]); yield* driver.commit(cwd, "Add feature", ""); + if (deleted) { + yield* git(cwd, ["push", "upstream", "--delete", baseBranch]); + } + const publishBranch = branch === "origin/dev" ? "dev" : branch; + assert.deepInclude( + yield* driver.resolvePushTarget(cwd, { branch, hasUpstream: !deleted }), + { remoteName: "origin", branchName: publishBranch }, + ); const pushed = yield* driver.pushCurrentBranch(cwd, null); assert.deepInclude(pushed, { status: "pushed", - branch: "feature/x", - upstreamBranch: "origin/feature/x", + branch, + upstreamBranch: `origin/${publishBranch}`, setUpstream: true, }); assert.equal( - yield* git(remote, ["--git-dir=.", "log", "-1", "--pretty=%s", "feature/x"]), + yield* git(remote, ["--git-dir=.", "log", "-1", "--pretty=%s", publishBranch]), "Add feature", ); + if (publishBranch !== baseBranch) { + assert.equal( + yield* git(remote, ["--git-dir=.", "rev-parse", `refs/heads/${baseBranch}`]), + devSha, + ); + } assert.equal( - yield* git(remote, ["--git-dir=.", "rev-parse", `refs/heads/${baseBranch}`]), - devSha, + yield* git(upstream, [ + "--git-dir=.", + "for-each-ref", + "--format=%(objectname)", + `refs/heads/${baseBranch}`, + ]), + deleted ? "" : upstreamDevSha, ); assert.equal( - yield* git(upstream, ["--git-dir=.", "rev-parse", `refs/heads/${baseBranch}`]), - upstreamDevSha, + yield* git(cwd, ["rev-parse", "--symbolic-full-name", "@{upstream}"]), + `refs/remotes/origin/${publishBranch}`, ); assert.equal( - yield* git(cwd, ["rev-parse", "--abbrev-ref", "@{upstream}"]), - "origin/feature/x", - ); - assert.equal( - yield* driver.readConfigValue(cwd, "branch.feature/x.t3-merge-base"), + yield* driver.readConfigValue(cwd, `branch.${branch}.t3-merge-base`), baseBranch, ); assert.equal( - yield* driver.readConfigValue(cwd, "branch.feature/x.t3-merge-base-ref"), + yield* driver.readConfigValue(cwd, `branch.${branch}.t3-merge-base-ref`), local ? `refs/heads/${baseBranch}` : `refs/remotes/upstream/${baseBranch}`, ); assert.equal(yield* git(cwd, ["rev-parse", `refs/heads/${baseBranch}`]), upstreamDevSha); + if (deleted) return; + const preview = yield* driver.getReviewDiffPreview({ cwd }); const changes = preview.sources.find((source) => source.kind === "branch-range")!; assert.deepStrictEqual(changes.files, [ - { path: "feature.txt", previousPath: null, additions: 1, deletions: 0 }, + { + path: "feature.txt", + previousPath: null, + changeType: "new", + additions: 1, + deletions: 0, + }, ]); assert.equal( changes.baseRef, @@ -7913,6 +9391,32 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { yield* git(remote, ["--git-dir=.", "log", "-1", "--pretty=%s", "effect-atom"]), "Add alias update", ); + + yield* git(cwd, ["push", "my-org/upstream", "--delete", "effect-atom"]); + assert.deepInclude( + yield* driver.resolvePushTarget(cwd, { + branch: "upstream/effect-atom", + hasUpstream: false, + }), + { remoteName: "my-org/upstream", branchName: "effect-atom" }, + ); + assert.deepInclude(yield* driver.pushCurrentBranch(cwd, null), { + upstreamBranch: "my-org/upstream/effect-atom", + setUpstream: true, + }); + assert.equal( + yield* git(remote, ["--git-dir=.", "rev-parse", "refs/heads/effect-atom"]), + yield* git(cwd, ["rev-parse", "HEAD"]), + ); + assert.equal( + yield* git(remote, [ + "--git-dir=.", + "for-each-ref", + "--format=%(refname)", + "refs/heads/upstream/", + ]), + "", + ); }), ); @@ -7951,6 +9455,196 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => { assert.notEqual(originMain.exitCode, 0); }), ); + + it.effect.each(["branch.topic.pushRemote", "remote.pushDefault"] as const)( + "pushes a branch tracking its upstream head to the %s remote", + (configKey) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const upstreamRemote = yield* makeTmpDir("git-upstream-remote-"); + const forkRemote = yield* makeTmpDir("git-fork-remote-"); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* git(upstreamRemote, ["init", "--bare"]); + yield* git(forkRemote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "upstream", upstreamRemote]); + yield* git(cwd, ["remote", "add", "fork", forkRemote]); + yield* git(cwd, ["checkout", "-b", "topic"]); + yield* git(cwd, ["push", "upstream", "topic"]); + yield* git(cwd, ["push", "fork", "topic"]); + yield* git(cwd, ["branch", "--set-upstream-to=upstream/topic"]); + yield* git(cwd, ["config", configKey, "fork"]); + const upstreamTopicSha = yield* git(cwd, ["rev-parse", "HEAD"]); + yield* writeTextFile(cwd, "fork.txt", "fork\n"); + yield* git(cwd, ["add", "-A"]); + yield* driver.commit(cwd, "Add fork update", ""); + + const pushed = yield* driver.pushCurrentBranch(cwd, null); + + assert.deepInclude(pushed, { + status: "pushed", + branch: "topic", + upstreamBranch: "fork/topic", + pushRemote: "fork", + setUpstream: false, + }); + assert.equal( + yield* git(forkRemote, ["--git-dir=.", "log", "-1", "--pretty=%s", "topic"]), + "Add fork update", + ); + assert.equal( + yield* git(upstreamRemote, ["--git-dir=.", "rev-parse", "topic"]), + upstreamTopicSha, + ); + assert.equal( + yield* git(cwd, ["rev-parse", "--abbrev-ref", "@{upstream}"]), + "upstream/topic", + ); + }), + ); + + it.effect.each(["deleted", "rewound"] as const)( + "pushes an up-to-date tracked branch to the push remote when its branch there is %s", + (forkState) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const upstreamRemote = yield* makeTmpDir("git-upstream-remote-"); + const forkRemote = yield* makeTmpDir("git-fork-remote-"); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* git(upstreamRemote, ["init", "--bare"]); + yield* git(forkRemote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "upstream", upstreamRemote]); + yield* git(cwd, ["remote", "add", "fork", forkRemote]); + yield* git(cwd, ["checkout", "-b", "topic"]); + yield* writeTextFile(cwd, "upstream.txt", "upstream\n"); + yield* git(cwd, ["add", "-A"]); + yield* driver.commit(cwd, "Add upstream update", ""); + yield* git(cwd, ["push", "-u", "upstream", "topic"]); + yield* git(cwd, ["config", "remote.pushDefault", "fork"]); + // Another checkout changes the fork after this one last fetched it, + // leaving a local fork/topic that still contains HEAD. + yield* git(cwd, ["push", "fork", "topic"]); + yield* git( + forkRemote, + forkState === "deleted" + ? ["--git-dir=.", "update-ref", "-d", "refs/heads/topic"] + : ["--git-dir=.", "update-ref", "refs/heads/topic", "topic~1"], + ); + + const pushed = yield* driver.pushCurrentBranch(cwd, null); + + assert.deepInclude(pushed, { + status: "pushed", + upstreamBranch: "fork/topic", + pushRemote: "fork", + }); + assert.equal( + yield* git(forkRemote, ["--git-dir=.", "log", "-1", "--pretty=%s", "topic"]), + "Add upstream update", + ); + assert.equal( + yield* git(cwd, ["rev-parse", "--abbrev-ref", "@{upstream}"]), + "upstream/topic", + ); + }), + ); + + it.effect.each([ + { pushUrls: [], status: "skipped_up_to_date" }, + { pushUrls: ["mirror"], status: "pushed" }, + { pushUrls: ["fork", "mirror"], status: "pushed" }, + ] as const)( + "skips an up-to-date tracked branch only when every push URL has it (pushurl $pushUrls)", + ({ pushUrls, status }) => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const upstreamRemote = yield* makeTmpDir("git-upstream-remote-"); + const remotes = { + fork: yield* makeTmpDir("git-fork-remote-"), + mirror: yield* makeTmpDir("git-mirror-remote-"), + }; + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* git(upstreamRemote, ["init", "--bare"]); + yield* git(remotes.fork, ["init", "--bare"]); + yield* git(remotes.mirror, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "upstream", upstreamRemote]); + yield* git(cwd, ["remote", "add", "fork", remotes.fork]); + yield* git(cwd, ["checkout", "-b", "topic"]); + yield* git(cwd, ["push", "-u", "upstream", "topic"]); + yield* git(cwd, ["push", "fork", "topic"]); + yield* git(cwd, ["config", "remote.pushDefault", "fork"]); + for (const name of pushUrls) { + yield* git(cwd, ["remote", "set-url", "--add", "--push", "fork", remotes[name]]); + } + + const pushed = yield* driver.pushCurrentBranch(cwd, null); + + assert.deepInclude(pushed, { status, upstreamBranch: "fork/topic", pushRemote: "fork" }); + const head = yield* git(cwd, ["rev-parse", "HEAD"]); + for (const name of pushUrls) { + assert.equal( + yield* git(remotes[name], ["--git-dir=.", "rev-parse", "refs/heads/topic"]), + head, + ); + } + assert.equal( + yield* git(cwd, ["rev-parse", "--abbrev-ref", "@{upstream}"]), + "upstream/topic", + ); + }), + ); + + it.effect("pushes an up-to-date branch whose tracked remote has a separate push URL", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const fetchRemote = yield* makeTmpDir("git-fetch-remote-"); + const pushRemote = yield* makeTmpDir("git-push-remote-"); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* git(fetchRemote, ["init", "--bare"]); + yield* git(pushRemote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "origin", fetchRemote]); + yield* git(cwd, ["checkout", "-b", "topic"]); + yield* git(cwd, ["push", "-u", "origin", "topic"]); + yield* git(cwd, ["remote", "set-url", "--push", "origin", pushRemote]); + + const pushed = yield* driver.pushCurrentBranch(cwd, null); + + assert.deepInclude(pushed, { status: "pushed", upstreamBranch: "origin/topic" }); + assert.equal( + yield* git(pushRemote, ["--git-dir=.", "rev-parse", "refs/heads/topic"]), + yield* git(cwd, ["rev-parse", "HEAD"]), + ); + }), + ); + + it.effect("skips an up-to-date branch whose differently named upstream is its base", () => + Effect.gen(function* () { + const cwd = yield* makeTmpDir(); + const upstreamRemote = yield* makeTmpDir("git-upstream-remote-"); + const forkRemote = yield* makeTmpDir("git-fork-remote-"); + yield* initRepoWithCommit(cwd); + const driver = yield* GitVcsDriver.GitVcsDriver; + yield* git(upstreamRemote, ["init", "--bare"]); + yield* git(forkRemote, ["init", "--bare"]); + yield* git(cwd, ["remote", "add", "upstream", upstreamRemote]); + yield* git(cwd, ["remote", "add", "fork", forkRemote]); + yield* git(cwd, ["push", "upstream", "HEAD:refs/heads/release"]); + yield* git(cwd, ["fetch", "upstream"]); + yield* git(cwd, ["checkout", "-b", "topic", "--track", "upstream/release"]); + yield* git(cwd, ["config", "remote.pushDefault", "fork"]); + + const pushed = yield* driver.pushCurrentBranch(cwd, null); + + assert.deepInclude(pushed, { + status: "skipped_up_to_date", + upstreamBranch: "upstream/release", + }); + assert.equal(yield* git(forkRemote, ["--git-dir=.", "branch", "--list", "topic"]), ""); + }), + ); }); }); diff --git a/apps/server/src/vcs/GitVcsDriverCore.ts b/apps/server/src/vcs/GitVcsDriverCore.ts index 2a0c6772d8d28..da1794188b0b2 100644 --- a/apps/server/src/vcs/GitVcsDriverCore.ts +++ b/apps/server/src/vcs/GitVcsDriverCore.ts @@ -1,7 +1,10 @@ import * as Cache from "effect/Cache"; +import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as Data from "effect/Data"; +import * as Deferred from "effect/Deferred"; import * as Digest from "effect/crypto/Digest"; +import * as SecureRandom from "effect/crypto/SecureRandom"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; @@ -26,6 +29,7 @@ import { type GitCommandFailureReason, T3_PROJECT_FILE_NAME, type ReviewBranchCommitsInput, + type ReviewDiffChangeType, type ReviewDiffFileContentsInput, type ReviewDiffPreviewInput, type ReviewDiffFileStat, @@ -42,6 +46,7 @@ import { resolveProjectFileBackedSetting } from "@t3tools/shared/projectSettings import { gitCommandDuration, gitCommandsTotal, withMetrics } from "../observability/Metrics.ts"; import * as GitVcsDriver from "./GitVcsDriver.ts"; import { removeEmptyDirectory, type RemoveEmptyDirectoryError } from "./removeEmptyDirectory.ts"; +import { VcsAutomaticFetchInterval } from "./VcsAutomaticFetchInterval.ts"; import { explicitBareRepositoryEnv, gitConfigEnv, @@ -67,11 +72,19 @@ const branchBaseConfigKey = (branch: string) => `branch.${branch}.t3-merge-base` * CLI's key, which branches cut before then still carry instead. */ const legacyBranchBaseConfigKey = (branch: string) => `branch.${branch}.gh-merge-base`; +// Differently named upstreams are bases, not publish targets; pushing HEAD there would write features to shared branches. +const isAliasOfUpstreamHead = (branch: string, upstreamBranch: string, upstreamRef: string) => + branch === upstreamBranch || + (branch.endsWith(`/${upstreamBranch}`) && upstreamRef.endsWith(`/${branch}`)); const gitProcesses = Semaphore.makeUnsafe(8); const backgroundGitProcesses = Semaphore.makeUnsafe(2); +// The fetch before `git worktree add` downloads everything the remote gained since +// the last fetch; on busy repositories that alone outlasts the 30s default. +const FETCH_REMOTE_TIMEOUT_MS = Duration.toMillis(Duration.minutes(5)); const WORKTREE_INDEX_REFRESH_TIMEOUT_MS = Duration.toMillis(Duration.minutes(5)); const WORKTREE_REMOVE_TIMEOUT_MS = Duration.toMillis(Duration.minutes(5)); const DEFAULT_MAX_OUTPUT_BYTES = 1_000_000; +const REF_SNAPSHOT_MAX_OUTPUT_BYTES = 16 * 1024 * 1024; const OUTPUT_TRUNCATED_MARKER = "\n\n[truncated]"; const PREPARED_COMMIT_PATCH_MAX_OUTPUT_BYTES = 49_000; const RANGE_COMMIT_SUMMARY_MAX_OUTPUT_BYTES = 19_000; @@ -88,6 +101,8 @@ const REVIEW_UNTRACKED_MAX_BYTES = 256 * 1024 * 1024; const UNTRACKED_TOTALS_RETRY_MIN_MS = 60_000; const UNTRACKED_TOTALS_RETRY_MAX_MS = 30 * 60_000; const REVIEW_DIFF_FILE_MAX_OUTPUT_BYTES = 1024 * 1024; +// Git's ident clean step (convert.c ident_to_git): `$Id:` up to the next `$` on the same line. +const GIT_IDENT_KEYWORD = /\$Id:[^$\n]*\$/g; const REVIEW_BRANCH_COMMIT_LIMIT = 100; const REVIEW_BRANCH_SUBJECT_LIMIT = 300; const REVIEW_BRANCH_COMMIT_MAX_OUTPUT_BYTES = 128 * 1024; @@ -191,6 +206,7 @@ interface GitRepositoryPaths { } interface GitRefsSnapshot { + readonly defaultBranch: string | null; readonly localBranches: ReadonlyArray; readonly remoteBranches: ReadonlyArray; readonly hasPrimaryRemote: boolean; @@ -218,18 +234,41 @@ function parseBranchAb(value: string): { ahead: number; behind: number } { }; } +function rawChangeType(status: string): ReviewDiffChangeType { + switch (status[0]) { + case "A": + return "new"; + case "D": + return "deleted"; + case "R": + return status === "R100" ? "rename-pure" : "rename-changed"; + default: + return "change"; + } +} + +// Parses `--numstat -z`, optionally preceded by `--raw` records that carry each file's change type. function parseReviewNumstat(stdout: string): ReviewDiffFileStat[] { const fields = stdout.split("\0"); + const changeTypes = new Map(); const files: ReviewDiffFileStat[] = []; for (let index = 0; index < fields.length; index++) { const field = fields[index]!; + if (field.startsWith(":")) { + const status = field.slice(field.lastIndexOf(" ") + 1); + if (/^[RC]/.test(status)) index++; + changeTypes.set(fields[++index]!, rawChangeType(status)); + continue; + } const match = /^(\d+|-)\t(\d+|-)\t([\s\S]*)$/.exec(field); if (!match) continue; const previousPath = match[3] === "" ? fields[++index]! : null; const path = previousPath !== null ? fields[++index]! : match[3]!; + const changeType = changeTypes.get(path); files.push({ path, previousPath, + ...(changeType === undefined ? {} : { changeType }), additions: match[1] === "-" ? 0 : Number(match[1]), deletions: match[2] === "-" ? 0 : Number(match[2]), }); @@ -330,6 +369,14 @@ function splitNullSeparatedPaths(input: string, truncated: boolean): string[] { return parts.filter((value) => value.length > 0); } +// Parses "git version 2.24.3 (Apple Git-128)" or "git version 2.45.1.windows.1". +const gitSupportsPathspecFromFile = (versionOutput: string): boolean => { + const match = /^git version (\d+)\.(\d+)/.exec(versionOutput); + if (match === null) return false; + const major = Number(match[1]); + return major > 2 || (major === 2 && Number(match[2]) >= 25); +}; + export function splitNullSeparatedGitStdoutPaths( result: Pick, ): string[] { @@ -585,6 +632,10 @@ export function isNonRepositoryGitStderr(stderr: string): boolean { const normalized = stderr.toLowerCase(); return NON_REPOSITORY_STDERR_PHRASES.some((phrase) => normalized.includes(phrase)); } +const UNTRACKED_FILES_SHOWN_CONFIG_VALUES = new Set(["normal", "all", "true", "yes", "on", "1"]); +function untrackedFilesHiddenByConfig(value: string | null): boolean { + return value !== null && !UNTRACKED_FILES_SHOWN_CONFIG_VALUES.has(value.toLowerCase()); +} function isUnbornHeadStderr(stderr: string): boolean { const normalized = stderr.toLowerCase(); return ( @@ -969,9 +1020,22 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const { worktreesDir, worktreeCheckoutCacheDir } = yield* ServerConfig.ServerConfig; const isServerInternalPath = yield* makeIsServerInternalPath; const digest = yield* Digest.Digest; + const random = yield* SecureRandom.SecureRandom; const hostPlatform = yield* HostProcess.Platform; const serviceScope = yield* Effect.scope; - const statusFetches = new Map>(); + const automaticFetchInterval = yield* VcsAutomaticFetchInterval; + type StatusFetchState = { + phase: "queued" | "running" | "claimed"; + claimed: Deferred.Deferred; + finished: Deferred.Deferred; + }; + const statusFetches = new Map< + string, + { + state: StatusFetchState; + fiber: Fiber.Fiber; + } + >(); const statusFetchProcesses = yield* Semaphore.make(4); const executeRaw: GitVcsDriver.GitVcsDriver["Service"]["execute"] = Effect.fnUntraced( @@ -1398,11 +1462,14 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const fiber = yield* Effect.uninterruptible( Effect.gen(function* () { const pending = statusFetches.get(key); - if (pending) return pending; + if (pending) return pending.fiber; + const claimed = yield* Deferred.make(); + const finished = yield* Deferred.make(); + const state: StatusFetchState = { phase: "queued", claimed, finished }; // Frequent poll deadlines and disconnected clients must not interrupt a // healthy writer. A separate safety limit prevents a hung transport from // occupying its worker slot indefinitely. - const started = yield* executeGit("GitVcsDriver.fetchRemoteForStatus", fetchCwd, args, { + const fetch = executeGit("GitVcsDriver.fetchRemoteForStatus", fetchCwd, args, { env: STATUS_UPSTREAM_REFRESH_ENV, fallbackErrorDetail: "Background Git fetch exited with a non-zero status.", timeoutMs: Duration.toMillis(STATUS_UPSTREAM_FETCH_TIMEOUT), @@ -1415,18 +1482,39 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ), Effect.tapError(() => recordStatusRemoteRefreshFailure(key)), Effect.tapCause((cause) => Effect.logWarning("Background Git fetch failed", cause)), - statusFetchProcesses.withPermits(1), + ); + const started = yield* Effect.raceFirst( + Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + if (state.phase !== "queued") return false; + state.phase = "running"; + yield* restore(fetch); + return true; + }), + ).pipe(statusFetchProcesses.withPermits(1)), + Deferred.await(claimed).pipe(Effect.as(false)), + ).pipe( + Effect.flatMap((completed) => + completed + ? Effect.void + : Deferred.await(finished).pipe( + Effect.flatMap((succeeded) => + succeeded ? Effect.void : fetch.pipe(statusFetchProcesses.withPermits(1)), + ), + ), + ), Effect.onExit(() => Effect.sync(() => { - statusFetches.delete(key); + if (statusFetches.get(key)?.state === state) statusFetches.delete(key); }), ), Effect.forkIn(serviceScope, { startImmediately: false, uninterruptible: false }), ); - statusFetches.set(key, started); + statusFetches.set(key, { state, fiber: started }); return started; }), ); + yield* Effect.annotateCurrentSpan("statusFetch.joining", true); const result = yield* Fiber.join(fiber).pipe( Effect.timeoutOption(STATUS_UPSTREAM_REFRESH_TIMEOUT), ); @@ -1686,9 +1774,42 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ), }); + const readRefSnapshot: GitVcsDriver.GitVcsDriver["Service"]["readRefSnapshot"] = Effect.fn( + "GitVcsDriver.readRefSnapshot", + )( + function* (cwd) { + const result = yield* executeGit( + "GitVcsDriver.readRefSnapshot", + cwd, + ["show-ref", "--head"], + { + allowNonZeroExit: true, + maxOutputBytes: REF_SNAPSHOT_MAX_OUTPUT_BYTES, + }, + ); + if (result.exitCode !== 0) return null; + const symbolicHead = yield* executeGit( + "GitVcsDriver.readRefSnapshot", + cwd, + ["symbolic-ref", "--quiet", "HEAD"], + { allowNonZeroExit: true }, + ); + // Exit 1 means detached HEAD; other failures require a full status read. + if (symbolicHead.exitCode !== 0 && symbolicHead.exitCode !== 1) return null; + // Deepening a shallow clone changes ancestry without moving refs. + const shallowPath = path.join(yield* resolveGitCommonDir(cwd), "shallow"); + const shallow = (yield* fileSystem.exists(shallowPath)) + ? yield* fileSystem.readFileString(shallowPath) + : ""; + return `${result.stdout}\0${symbolicHead.stdout}\0${shallow}`; + }, + Effect.orElseSucceed(() => null), + ); + const refreshStatusUpstreamIfStale = Effect.fn("refreshStatusUpstreamIfStale")(function* ( cwd: string, ) { + if (Duration.isZero(yield* automaticFetchInterval)) return; const headRef = yield* runGitStdout( "GitVcsDriver.refreshStatusUpstreamIfStale.headRef", cwd, @@ -1816,7 +1937,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ), ); - const resolvePushRemoteName = Effect.fn("resolvePushRemoteName")(function* ( + const resolveConfiguredPushRemoteName = Effect.fn("resolveConfiguredPushRemoteName")(function* ( cwd: string, refName: string, ) { @@ -1836,13 +1957,65 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ["config", "--get", "remote.pushDefault"], true, ).pipe(Effect.map((stdout) => stdout.trim())); - if (pushDefaultRemote.length > 0) { - return pushDefaultRemote; + return pushDefaultRemote.length > 0 ? pushDefaultRemote : null; + }); + + const resolvePushRemoteName = Effect.fn("resolvePushRemoteName")(function* ( + cwd: string, + refName: string, + fallbackRemoteName: string | null = null, + ) { + const configuredRemote = yield* resolveConfiguredPushRemoteName(cwd, refName); + if (configuredRemote) { + return configuredRemote; + } + + if (fallbackRemoteName) { + return fallbackRemoteName; } return yield* resolvePrimaryRemoteName(cwd).pipe(Effect.orElseSucceed(() => null)); }); + const remoteRefContainsHead = (cwd: string, remoteRef: string) => + executeGit( + "GitVcsDriver.pushCurrentBranch.publishContainsHead", + cwd, + ["merge-base", "--is-ancestor", "HEAD", `refs/remotes/${remoteRef}`], + { allowNonZeroExit: true }, + ).pipe( + Effect.map((result) => result.exitCode === 0), + Effect.orElseSucceed(() => false), + ); + + // A fetch reads only the remote's first url, while a push writes every push URL (pushurl, else + // each url). A fetched ref is evidence of the push destination only when those are the same one. + const pushesOnlyToFetchUrl = Effect.fn("GitVcsDriver.pushesOnlyToFetchUrl")( + function* (cwd: string, remoteName: string) { + const [fetchUrl, pushUrls] = yield* Effect.all([ + runGitStdout("GitVcsDriver.pushesOnlyToFetchUrl.fetchUrl", cwd, [ + "remote", + "get-url", + "--", + remoteName, + ]), + runGitStdout("GitVcsDriver.pushesOnlyToFetchUrl.pushUrls", cwd, [ + "remote", + "get-url", + "--push", + "--all", + "--", + remoteName, + ]), + ]); + // Preserve whitespace in URLs and reject ambiguous multiline fetch URLs. + return ( + fetchUrl.endsWith("\n") && !fetchUrl.slice(0, -1).includes("\n") && pushUrls === fetchUrl + ); + }, + Effect.orElseSucceed(() => false), + ); + const ensureRemote: GitVcsDriver.GitVcsDriver["Service"]["ensureRemote"] = Effect.fn( "ensureRemote", )(function* (input) { @@ -2164,11 +2337,18 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* options?: GitVcsDriver.GitLocalStatusOptions, ) { const includeDivergence = options?.includeDivergence !== false; + // Hidden untracked files could not be committed; `all` stays as configured. + const untrackedFilesHidden = untrackedFilesHiddenByConfig( + yield* readConfigValue(cwd, "status.showUntrackedFiles").pipe( + Effect.orElseSucceed(() => null), + ), + ); const statusArgs = [ "status", "--porcelain=2", "--branch", "-z", + ...(untrackedFilesHidden ? ["--untracked-files=normal"] : []), ...(includeDivergence ? [] : ["--no-ahead-behind"]), ]; const indexResult = yield* executeGitWithStableDiagnostics( @@ -2225,41 +2405,25 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* isMissingGitCwdError(error) ? Effect.succeed(null) : Effect.fail(error), }), ); + // Porcelain diff rewrites indexes despite GIT_OPTIONAL_LOCKS=0; plumbing with -M preserves rename detection. + const readNumstat = (ref: string) => + executeGitWithStableDiagnostics( + "GitVcsDriver.statusDetails.numstat", + cwd, + ["diff-index", "-M", "--numstat", "-z", ref, "--"], + { allowNonZeroExit: true, maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES }, + ); // diff-index does not read status output, so both spawn together; a non-repository discards it. const [statusResult, numstatResult, defaultBranch, hasPrimaryRemote] = yield* Effect.all( [ readStatus, - // Porcelain diff rewrites indexes despite GIT_OPTIONAL_LOCKS=0; plumbing with -M preserves rename detection. - executeGitWithStableDiagnostics( - "GitVcsDriver.statusDetails.numstat", - cwd, - ["diff-index", "-M", "--numstat", "-z", "HEAD", "--"], - { allowNonZeroExit: true, maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES }, - ).pipe( + readNumstat("HEAD").pipe( + Effect.filterOrElse( + (result) => result.exitCode === 0 || !isUnbornHeadStderr(result.stderr), + () => Effect.flatMap(readEmptyTreeHash(cwd), readNumstat), + ), Effect.flatMap((result) => { if (result.exitCode === 0) return Effect.succeed(parseReviewNumstat(result.stdout)); - if (isUnbornHeadStderr(result.stderr)) { - return Effect.map( - Effect.all([ - runGitStdoutWithOptions( - "GitVcsDriver.statusDetails.numstat.unborn", - cwd, - ["diff-files", "--numstat", "-z"], - { maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES }, - ), - runGitStdoutWithOptions( - "GitVcsDriver.statusDetails.numstat.unborn.staged", - cwd, - ["diff", "--cached", "--numstat", "-z"], - { maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES }, - ), - ]), - ([unstagedStdout, stagedStdout]) => [ - ...parseReviewNumstat(stagedStdout), - ...parseReviewNumstat(unstagedStdout), - ], - ); - } return Effect.fail( new GitCommandError({ ...gitCommandContext({ @@ -2316,6 +2480,10 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* let refName: string | null = null; let upstreamRef: string | null = null; + // Git omits branch.ab when the upstream ref is gone, e.g. deleted after its PR merged, + // and also for an unborn HEAD, which can still track a fetched upstream. + let upstreamRefExists = false; + let isUnbornHead = false; let aheadCount = 0; let behindCount = 0; let aheadOfDefaultCount = 0; @@ -2326,9 +2494,14 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const statusRecords = statusStdout.split("\0"); for (let index = 0; index < statusRecords.length; index++) { const line = statusRecords[index]!; + if (line === "# branch.oid (initial)") { + isUnbornHead = true; + continue; + } if (line.startsWith("# branch.head ")) { const value = line.slice("# branch.head ".length).trim(); - refName = value === "(detached)" ? null : value; + // `(unknown)` is printed while HEAD names no resolvable branch, e.g. mid-`git clone`. + refName = value === "(detached)" || value === "(unknown)" ? null : value; continue; } if (line.startsWith("# branch.upstream ")) { @@ -2336,11 +2509,13 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* upstreamRef = value.length > 0 ? value : null; continue; } - if (includeDivergence && line.startsWith("# branch.ab ")) { - const value = line.slice("# branch.ab ".length).trim(); - const parsed = parseBranchAb(value); - aheadCount = parsed.ahead; - behindCount = parsed.behind; + if (line.startsWith("# branch.ab ")) { + upstreamRefExists = true; + if (includeDivergence) { + const parsed = parseBranchAb(line.slice("# branch.ab ".length).trim()); + aheadCount = parsed.ahead; + behindCount = parsed.behind; + } continue; } if (line.length > 0 && !line.startsWith("#")) { @@ -2359,6 +2534,8 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* } } + if (!upstreamRefExists && !isUnbornHead) upstreamRef = null; + const isDefaultBranch = refName !== null && (refName === defaultBranch || @@ -2483,6 +2660,18 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* })), ); + // git add takes --pathspec-from-file from Git 2.25; older Git keeps the argument form. + let pathspecFromFileSupported: boolean | undefined; + const supportsPathspecFromFile = Effect.fn("GitVcsDriver.supportsPathspecFromFile")(function* ( + cwd: string, + ) { + if (pathspecFromFileSupported !== undefined) return pathspecFromFileSupported; + const version = yield* executeGit("GitVcsDriver.version", cwd, ["version"]).pipe(Effect.option); + if (Option.isNone(version)) return false; + pathspecFromFileSupported = gitSupportsPathspecFromFile(version.value.stdout); + return pathspecFromFileSupported; + }); + const stageCommitChanges = Effect.fnUntraced(function* ( cwd: string, filePaths?: readonly string[], @@ -2491,6 +2680,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const config = env ? ["-c", "core.splitIndex=false", "-c", "splitIndex.sharedIndexExpire=never"] : []; + let preserveUnselectedConflicts = false; if (filePaths && filePaths.length > 0) { if (env) { const head = yield* executeGit( @@ -2511,14 +2701,40 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* { head: "MERGE_HEAD", operation: "merge" }, { head: "CHERRY_PICK_HEAD", operation: "cherry-pick" }, { head: "REVERT_HEAD", operation: "revert" }, + // A conflicted multi-commit `cherry-pick --no-commit` writes only the sequencer; reset + // would clear its unmerged entries, so `--continue` would walk past the conflict. + { head: "sequencer", operation: "cherry-pick or revert" }, ] as const; const headPaths = yield* runGitStdout("GitVcsDriver.commit.mergeHead", cwd, [ "rev-parse", ...inProgressHeads.flatMap(({ head }) => ["--git-path", head]), ]); const resolvedHeadPaths = headPaths.trim().split("\n"); - const inProgress = yield* Effect.findFirst(inProgressHeads, (_, index) => - fileSystem.exists(path.resolve(cwd, resolvedHeadPaths[index] ?? "")), + // Reftable keeps CHERRY_PICK_HEAD and REVERT_HEAD as refs with no file under .git. A + // same-named branch resolves to its full name; an ambiguity warning would blank stdout. + const operationRefs = new Set(["CHERRY_PICK_HEAD", "REVERT_HEAD"]); + const isOperationRef = (head: string) => + runGitStdout( + "GitVcsDriver.commit.operationRef", + cwd, + [ + "-c", + "core.warnAmbiguousRefs=false", + "rev-parse", + "--verify", + "--quiet", + "--symbolic-full-name", + head, + ], + true, + ).pipe(Effect.map((ref) => ref.trim() === head)); + const inProgress = yield* Effect.findFirst(inProgressHeads, ({ head }, index) => + fileSystem.exists(path.resolve(cwd, resolvedHeadPaths[index] ?? "")).pipe( + Effect.filterOrElse( + (exists) => exists || !operationRefs.has(head), + () => isOperationRef(head), + ), + ), ).pipe( Effect.mapError( (cause) => @@ -2541,16 +2757,33 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* detail: `Cannot commit selected files during a ${operation}. Commit all files to finish the ${operation}.`, }); } - yield* runGit("GitVcsDriver.commit.reset", cwd, ["reset", "--quiet"]); + const selectedFiles = new Set(filePaths); + const unmergedPaths = yield* runGitStdoutWithOptions( + "GitVcsDriver.commit.unmergedPaths", + cwd, + ["-c", "diff.relative=false", "diff", "--cached", "--name-only", "--diff-filter=U", "-z"], + { maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES }, + ); + // Reset drops every conflict stage, so keep the index while unselected files are conflicted. + preserveUnselectedConflicts = unmergedPaths + .split("\0") + .some((file) => file.length > 0 && !selectedFiles.has(file)); + if (!preserveUnselectedConflicts) { + yield* runGit("GitVcsDriver.commit.reset", cwd, ["reset", "--quiet"]); + } } + // Status paths are repository-relative, while cwd can be a subdirectory. + const pathspecs = filePaths.map((file) => `:(top,literal)${file}`); + const fromStdin = yield* supportsPathspecFromFile(cwd); yield* runGit( "GitVcsDriver.commit.addSelected", cwd, - [...config, "add", "-A", "--pathspec-from-file=-", "--pathspec-file-nul"], + fromStdin + ? [...config, "add", "-A", "--pathspec-from-file=-", "--pathspec-file-nul"] + : [...config, "add", "-A", "--", ...pathspecs], { ...(env ? { env } : {}), - // Status paths are repository-relative, while cwd can be a subdirectory. - stdin: filePaths.map((file) => `:(top,literal)${file}\0`).join(""), + ...(fromStdin ? { stdin: pathspecs.map((pathspec) => `${pathspec}\0`).join("") } : {}), }, ); } else { @@ -2585,7 +2818,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ); const reports = check.stdout.split("\n"); const conflicted: string[] = []; - let topLevel: string | undefined; + let emptyTree: string | undefined; for (const file of unmergedPaths) { const markerLines = new Set( reports.flatMap((line) => { @@ -2596,11 +2829,26 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* }), ); if (markerLines.size === 0) continue; - topLevel ??= (yield* runGitStdout("GitVcsDriver.commit.topLevel", cwd, [ - "rev-parse", - "--show-toplevel", - ])).trim(); - const lines = (yield* fileSystem.readFileString(path.join(topLevel, file)).pipe( + emptyTree ??= yield* readEmptyTreeHash(cwd); + // --check numbers lines after working-tree-encoding and clean filters; a diff from the + // empty tree prints that view. Read it in full so large resolved files can still commit. + const fileDiff = yield* runGitStdoutWithOptions( + "GitVcsDriver.commit.conflictMarkers", + cwd, + [ + "-c", + "diff.relative=false", + "diff", + "--no-color", + "--no-ext-diff", + "--no-textconv", + "--text", + emptyTree, + "--", + `:(top,literal)${file}`, + ], + { ...(env ? { env } : {}), maxOutputBytes: Number.POSITIVE_INFINITY }, + ).pipe( Effect.mapError( (cause) => new GitCommandError({ @@ -2611,7 +2859,12 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* cause, }), ), - )).split("\n"); + ); + const diffLines = fileDiff.split("\n"); + const lines = diffLines + .slice(diffLines.findIndex((line) => line.startsWith("@@")) + 1) + .filter((line) => line.startsWith("+")) + .map((line) => line.slice(1)); // Git also reports a lone "=======", which is valid content such as a heading underline. const reported = (marker: string) => lines.some((line, index) => markerLines.has(index) && line.startsWith(marker)); @@ -2633,6 +2886,9 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* env ? { env } : {}, ); } + return preserveUnselectedConflicts && filePaths + ? filePaths.map((file) => `:(top,literal)${file}`) + : undefined; }); const prepareCommitContext: GitVcsDriver.GitVcsDriver["Service"]["prepareCommitContext"] = @@ -2777,14 +3033,26 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* body, options?: GitVcsDriver.GitCommitOptions, ) { - if (options?.stage !== undefined) { - yield* stageCommitChanges(cwd, options.stage.filePaths); - } + const partialCommitPaths = + options?.stage !== undefined + ? yield* stageCommitChanges(cwd, options.stage.filePaths) + : undefined; const args = ["commit", "-m", subject]; const trimmedBody = body.trim(); if (trimmedBody.length > 0) { args.push("-m", trimmedBody); } + const onlyFromStdin = partialCommitPaths ? yield* supportsPathspecFromFile(cwd) : false; + const onlyPathspecArgs = onlyFromStdin + ? ["--pathspec-from-file=-", "--pathspec-file-nul"] + : ["--", ...(partialCommitPaths ?? [])]; + const onlyPathspecStdin = + partialCommitPaths && onlyFromStdin + ? { stdin: partialCommitPaths.map((pathspec) => `${pathspec}\0`).join("") } + : {}; + if (partialCommitPaths) { + args.push("--only", ...onlyPathspecArgs); + } const progress = options?.progress?.onOutputLine === undefined ? options?.progress @@ -2797,9 +3065,19 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* }; yield* executeGit("GitVcsDriver.commit.commit", cwd, args, { maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES, + ...onlyPathspecStdin, ...(options?.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}), ...(progress ? { progress } : {}), }).pipe(Effect.asVoid); + if (partialCommitPaths) { + // --only runs hooks on a temporary index, so a hook's `git add` never reaches the real one. + yield* runGit( + "GitVcsDriver.commit.syncSelectedIndex", + cwd, + ["reset", "--quiet", "HEAD", ...onlyPathspecArgs], + onlyPathspecStdin, + ); + } const commitSha = yield* runGitStdout("GitVcsDriver.commit.revParseHead", cwd, [ "rev-parse", "HEAD", @@ -2840,10 +3118,40 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* hasUpstream: boolean, ) { if (!hasUpstream) { - const remoteName = yield* resolvePushRemoteName(cwd, branch); + // A deleted upstream keeps its config: republish to it when it was this branch's own, + // otherwise it was the base, so keep that before publishing. + let goneUpstream: { remoteName: string; branchName: string } | null = null; + let goneBaseUpstream: { remoteName: string; branchName: string; upstreamRef: string } | null = + null; + const mergeRef = yield* readConfigValue(cwd, `branch.${branch}.merge`); + const mergeBranch = mergeRef?.startsWith("refs/heads/") + ? mergeRef.slice("refs/heads/".length) + : null; + const configuredRemoteName = yield* readConfigValue(cwd, `branch.${branch}.remote`); + if (mergeBranch && configuredRemoteName && configuredRemoteName !== ".") { + if (isAliasOfUpstreamHead(branch, mergeBranch, `${configuredRemoteName}/${mergeBranch}`)) { + goneUpstream = { remoteName: configuredRemoteName, branchName: mergeBranch }; + } else { + goneBaseUpstream = { + remoteName: configuredRemoteName, + branchName: mergeBranch, + upstreamRef: `${configuredRemoteName}/${mergeBranch}`, + }; + } + } + const remoteName = yield* resolvePushRemoteName(cwd, branch, goneUpstream?.remoteName); if (!remoteName) return null; - const branchName = yield* resolvePublishBranchName(cwd, branch); - return { remoteName, branchName, publishes: true, upstream: null }; + // A gone local upstream still gets the full local name, as below. + const branchName = + goneUpstream?.branchName ?? + (configuredRemoteName === "." ? branch : yield* resolvePublishBranchName(cwd, branch)); + return { + remoteName, + branchName, + publishes: true, + upstream: goneBaseUpstream, + pushRemote: null, + }; } if ((yield* readConfigValue(cwd, `branch.${branch}.remote`)) === ".") { @@ -2861,21 +3169,27 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* branchName: branch, publishes: true, upstream: { remoteName: ".", branchName: localUpstream, upstreamRef: localUpstream }, + pushRemote: null, }; } const upstream = yield* resolveCurrentUpstream(cwd).pipe(Effect.orElseSucceed(() => null)); if (!upstream) return null; - // Differently named upstreams are bases, not publish targets; pushing HEAD there would write features to shared branches. - const isAliasOfUpstreamHead = - branch === upstream.branchName || - (branch.endsWith(`/${upstream.branchName}`) && upstream.upstreamRef.endsWith(`/${branch}`)); - if (isAliasOfUpstreamHead) { + if (isAliasOfUpstreamHead(branch, upstream.branchName, upstream.upstreamRef)) { + // A fork workflow pushes to the configured push remote and keeps the upstream for pulls, as plain `git push` does. + const configuredPushRemote = yield* resolveConfiguredPushRemoteName(cwd, branch).pipe( + Effect.orElseSucceed(() => null), + ); + const pushRemote = + configuredPushRemote && configuredPushRemote !== upstream.remoteName + ? configuredPushRemote + : null; return { - remoteName: upstream.remoteName, + remoteName: pushRemote ?? upstream.remoteName, branchName: upstream.branchName, publishes: false, upstream, + pushRemote, }; } const publishRemoteName = yield* resolvePushRemoteName(cwd, branch).pipe( @@ -2887,6 +3201,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* branchName, publishes: true, upstream, + pushRemote: null, }; }); @@ -2895,7 +3210,11 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* )(function* (cwd, current) { const destination = yield* resolvePushDestination(cwd, current.branch, current.hasUpstream); return destination - ? { remoteName: destination.remoteName, branchName: destination.branchName } + ? { + remoteName: destination.remoteName, + branchName: destination.branchName, + viaPushRemote: destination.pushRemote !== null, + } : null; }); @@ -2933,16 +3252,41 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* } const hasNoLocalDelta = details.aheadCount === 0 && details.behindCount === 0; - if (hasNoLocalDelta && details.hasUpstream) { - return { - status: "skipped_up_to_date" as const, - branch, - ...(details.upstreamRef ? { upstreamBranch: details.upstreamRef } : {}), - }; - } - const destination = yield* resolvePushDestination(cwd, branch, details.hasUpstream); - if (hasNoLocalDelta) { + if (hasNoLocalDelta && details.hasUpstream) { + if (!destination?.pushRemote) { + // The tracking ref reflects the fetch URL, so it says nothing about a separate push URL. + if (!destination || (yield* pushesOnlyToFetchUrl(cwd, destination.remoteName))) { + return { + status: "skipped_up_to_date" as const, + branch, + ...(details.upstreamRef ? { upstreamBranch: details.upstreamRef } : {}), + }; + } + } else { + const { pushRemote, branchName } = destination; + const pushRemoteRef = `${pushRemote}/${branchName}`; + // The local ref can be stale when the branch was deleted or rewound elsewhere. + const pushRemoteHasHead = + (yield* pushesOnlyToFetchUrl(cwd, pushRemote)) && + (yield* fetchRemoteTrackingBranch({ + cwd, + remoteName: pushRemote, + remoteBranch: branchName, + }).pipe( + Effect.andThen(remoteRefContainsHead(cwd, pushRemoteRef)), + Effect.orElseSucceed(() => false), + )); + if (pushRemoteHasHead) { + return { + status: "skipped_up_to_date" as const, + branch, + upstreamBranch: pushRemoteRef, + pushRemote, + }; + } + } + } else if (hasNoLocalDelta) { const comparableBaseBranch = yield* resolveBaseBranchForNoUpstream(cwd, branch).pipe( Effect.orElseSucceed(() => null), ); @@ -2955,21 +3299,13 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* } // The configured diff base can contain HEAD while the publish branch is still behind. - const publishContainsHead = yield* executeGit( - "GitVcsDriver.pushCurrentBranch.publishContainsHead", - cwd, - [ - "merge-base", - "--is-ancestor", - "HEAD", - `refs/remotes/${destination.remoteName}/${destination.branchName}`, - ], - { allowNonZeroExit: true }, - ).pipe( - Effect.map((result) => result.exitCode === 0), - Effect.orElseSucceed(() => false), - ); - if (publishContainsHead) { + if ( + (yield* remoteRefContainsHead( + cwd, + `${destination.remoteName}/${destination.branchName}`, + )) && + (yield* pushesOnlyToFetchUrl(cwd, destination.remoteName)) + ) { return { status: "skipped_up_to_date" as const, branch, @@ -3038,7 +3374,12 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* return { status: "pushed" as const, branch, - upstreamBranch: destination.upstream.upstreamRef, + ...(destination.pushRemote + ? { + upstreamBranch: `${destination.remoteName}/${destination.branchName}`, + pushRemote: destination.pushRemote, + } + : { upstreamBranch: destination.upstream.upstreamRef }), setUpstream: false, }; } @@ -3080,12 +3421,31 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* }); } const remote = yield* readConfigValue(cwd, `branch.${refName}.remote`); + // The upstream-only retry leaves other remote-tracking refs stale, so a claimed status + // fetch must still run after it. + let fetchedUpstreamOnly = false; if (remote) { const gitCommonDir = yield* resolveGitCommonDir(cwd); const pending = statusFetches.get(`${gitCommonDir}\0${remote}`); // The status wait can finish while its fetch still owns reference locks. // Let our existing writer finish before starting the pull's fetch. - if (pending) yield* Fiber.await(pending); + if (pending) { + const claimed = yield* Effect.uninterruptible( + Effect.gen(function* () { + if (pending.state.phase !== "queued") return false; + pending.state.phase = "claimed"; + yield* Effect.addFinalizer((exit) => + Deferred.succeed( + pending.state.finished, + Exit.isSuccess(exit) && !fetchedUpstreamOnly, + ), + ); + yield* Deferred.succeed(pending.state.claimed, undefined); + return true; + }), + ); + if (!claimed) yield* Fiber.await(pending.fiber); + } } const beforeSha = yield* runGitStdout( "GitVcsDriver.pullCurrentBranch.beforeSha", @@ -3139,6 +3499,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* currentSha === beforeSha ) { args = ["pull", "--ff-only", "--", remote, mergeRef]; + fetchedUpstreamOnly = true; result = yield* executeGitWithStableDiagnostics( "GitVcsDriver.pullCurrentBranch.pull", cwd, @@ -3167,11 +3528,14 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const refreshed = yield* readStatusDetailsLocal(cwd); return { - status: beforeSha.length > 0 && beforeSha === afterSha ? "skipped_up_to_date" : "pulled", + status: + beforeSha.length > 0 && beforeSha === afterSha + ? ("skipped_up_to_date" as const) + : ("pulled" as const), refName, upstreamRef: refreshed.upstreamRef, }; - }); + }, Effect.scoped); const readRangeContext: GitVcsDriver.GitVcsDriver["Service"]["readRangeContext"] = Effect.fn( "readRangeContext", @@ -3277,7 +3641,21 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ); const stagedDeletions = new Set(stagedDeletionsStdout.split("\0").filter(Boolean)); // Staged deletions never enter the index, so their worktree copies must not spend the budget. - const candidates = untrackedPaths.filter((relativePath) => !stagedDeletions.has(relativePath)); + // A nested repository (listed with a trailing slash) with no commits cannot be indexed: + // Git before 2.53 fails the add, and later versions accept it and then fail every diff. + const candidates = yield* Effect.filter( + untrackedPaths.filter((relativePath) => !stagedDeletions.has(relativePath)), + (relativePath) => + relativePath.endsWith("/") + ? executeGit( + "GitVcsDriver.readUnifiedWorkingTreeReviewDiff.nestedRepositoryHead", + path.resolve(cwd, relativePath), + ["rev-parse", "--verify", "--quiet", "HEAD"], + { allowNonZeroExit: true }, + ).pipe(Effect.map((result) => result.exitCode !== 1)) + : Effect.succeed(true), + { concurrency: 16 }, + ); const sizes = yield* Effect.forEach( candidates, (relativePath) => @@ -3503,8 +3881,11 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* detail: remembered.detail, }); } - // Empty status can mean untracked files are hidden; retain the full scan. - const untracked = yield* prepareUntrackedReviewIndexOrReason(cwd, pathspecs, undefined, false); + // Status always lists untracked files, so no roots means there are none to scan. + const untracked = + roots.length === 0 + ? { env: undefined, omitted: false } + : yield* prepareUntrackedReviewIndexOrReason(cwd, pathspecs, undefined, false); if ("unavailable" in untracked) { const count = remembered?.roots === rootsKey ? remembered.count + 1 : 1; untrackedTotalsFailures.delete(cwd); @@ -3759,7 +4140,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const stat = yield* executeGit( "GitVcsDriver.getReviewDiffPreview.commitStat", cwd, - [...diffArgs, "--numstat", "-z", commit.parent, commit.sha, "--", ...pathArgs], + [...diffArgs, "--raw", "--numstat", "-z", commit.parent, commit.sha, "--", ...pathArgs], { maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES }, ); const files = parseReviewNumstat(stat.stdout); @@ -3809,7 +4190,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ref: string, env?: NodeJS.ProcessEnv, ) { - const args = [...diffArgs, "--numstat", "-z"]; + const args = [...diffArgs, "--raw", "--numstat", "-z"]; const result = yield* executeGit( "GitVcsDriver.getReviewDiffPreview.stat", cwd, @@ -4030,7 +4411,43 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* if (bytes.includes(0)) { return yield* fileError("fs.readFile", `Cannot expand binary file '${input.newPath}'.`); } - return new TextDecoder("utf-8", { ignoreBOM: true }).decode(bytes); + const contents = new TextDecoder("utf-8", { ignoreBOM: true }).decode(bytes); + if (!contents.includes("$Id:")) return contents; + const collapsed = contents.replace(GIT_IDENT_KEYWORD, () => "$Id$"); + if (collapsed === contents) return contents; + // check-attr reports both boolean ident and literal ident=set as "set". + const attribute = yield* runGitStdout( + "GitVcsDriver.getReviewDiffFileContents.workingTree.ident", + repositoryRoot, + ["check-attr", "-z", "ident", "filter", "--", input.newPath], + ); + const attributes = attribute.split("\0"); + if (attributes[2] !== "set" || attributes[5] !== "unspecified") return contents; + + // Literal filter=unspecified is indistinguishable from an absent filter in check-attr. + const filters = yield* executeGit( + "GitVcsDriver.getReviewDiffFileContents.workingTree.filters", + repositoryRoot, + ["config", "--name-only", "--get-regexp", "^filter\\.unspecified\\."], + { allowNonZeroExit: true }, + ); + if (filters.exitCode !== 1 || filters.stdout.length > 0) return contents; + // Probe a nonoverlapping keyword so ident conversion runs once on the actual file. + const [canonicalHash, collapsedHash] = yield* Effect.all([ + runGitStdoutWithOptions( + "GitVcsDriver.getReviewDiffFileContents.workingTree.canonicalHash", + repositoryRoot, + ["hash-object", "--stdin", `--path=${input.newPath}`], + { stdin: "$Id: probe $" }, + ), + runGitStdoutWithOptions( + "GitVcsDriver.getReviewDiffFileContents.workingTree.collapsedHash", + repositoryRoot, + ["hash-object", "--stdin", `--path=${input.newPath}`], + { stdin: "$Id$" }, + ), + ]); + return canonicalHash === collapsedHash ? collapsed : contents; }); const getReviewDiffFileContents = Effect.fn("getReviewDiffFileContents")(function* ( @@ -4338,6 +4755,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* : left.ref.name.localeCompare(right.ref.name); return { + defaultBranch, localBranches: localBranches.toSorted(byRecencyThenName).map(({ ref }) => ref), remoteBranches: remoteBranches.toSorted(byRecencyThenName).map(({ ref }) => ref), hasPrimaryRemote: remoteNames.length > 0, @@ -4478,7 +4896,24 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* { concurrency: 2 }, ); const { worktreeRoot } = repositoryPaths; - const localBranches = snapshot.localBranches.map((ref) => { + // The snapshot misses a branch created outside T3 Code until it refreshes. An unborn HEAD + // names a branch with no ref yet, so the ref must exist before it is listed. + const currentBranchUnlisted = + currentBranch !== null && !snapshot.localBranches.some((ref) => ref.name === currentBranch); + const unlistedCurrentBranch: ReadonlyArray = + currentBranchUnlisted && + (yield* branchExists(input.cwd, currentBranch).pipe(Effect.orElseSucceed(() => false))) + ? [ + { + name: currentBranch, + current: true, + isRemote: false, + isDefault: currentBranch === snapshot.defaultBranch, + worktreePath: worktreeRoot, + }, + ] + : []; + const listedBranches = snapshot.localBranches.map((ref) => { const current = ref.name === currentBranch; return { ...ref, @@ -4493,6 +4928,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* current, }; }); + const localBranches = [...unlistedCurrentBranch, ...listedBranches]; const combinedBranches = input.includeMatchingRemoteRefs ? [...localBranches, ...snapshot.remoteBranches] : dedupeRemoteBranchesWithLocalMatches([...localBranches, ...snapshot.remoteBranches]); @@ -4572,6 +5008,40 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* yield* refresh; }); + // Git writes the lock reason during registration, before checkout or user hooks can run. + const releaseWorktreeRegistration = Effect.fn("releaseWorktreeRegistration")(function* ( + cwd: string, + reason: string, + remove: boolean, + ) { + const commonDir = yield* readGitCommonDir(cwd); + if (commonDir === null) return; + const registrations = path.join(commonDir, "worktrees"); + const entries = yield* fileSystem.readDirectory(registrations).pipe( + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + () => Effect.succeed([]), + ), + ); + for (const entry of entries) { + const adminDir = path.join(registrations, entry); + const locked = yield* fileSystem + .readFileString(path.join(adminDir, "locked")) + .pipe(Effect.orElseSucceed(() => "")); + if (locked.replace(/\r?\n$/, "") !== reason) continue; + // gitdir can be relative to the registration with worktree.useRelativePaths enabled. + const gitdir = yield* fileSystem.readFileString(path.join(adminDir, "gitdir")); + const registered = path.dirname(path.resolve(adminDir, gitdir.replace(/\r?\n$/, ""))); + yield* runGit("GitVcsDriver.createWorktree.unlock", cwd, [ + "worktree", + "unlock", + "--", + registered, + ]); + if (remove) yield* removeWorktree({ cwd, path: registered, force: true }); + } + }); + const createWorktree: GitVcsDriver.GitVcsDriver["Service"]["createWorktree"] = Effect.fn( "createWorktree", )(function* (input, options) { @@ -4616,6 +5086,20 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* } } const newRefName = input.newRefName; + // Older Git still creates worktrees normally, but cannot prove ownership for cancellation. + const addHelp = newRefName + ? yield* executeGit( + "GitVcsDriver.createWorktree.capabilities", + input.cwd, + ["worktree", "add", "-h"], + { allowNonZeroExit: true }, + ) + : null; + const lockReason = + addHelp && `${addHelp.stdout}\n${addHelp.stderr}`.includes("--reason") + ? `t3-create-${yield* random.uuidV4()}` + : undefined; + if (newRefName) { // Kept apart from the checkout so the shared-config lock only covers the tracking write. yield* createBranchAt( @@ -4649,6 +5133,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* `checkout.workers=${checkoutWorkers}`, "worktree", "add", + ...(lockReason ? ["--lock", "--reason", lockReason] : []), "--", worktreePath, checkoutRef, @@ -4676,30 +5161,66 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* : {}), }, ); + // Set inside the claim's uninterruptible region: a pending cancel still surfaces as it ends. + let claimed = false; + const onWorktreeClaimed = progress?.onWorktreeClaimed; yield* ( fastCheckout ? checkoutCache - .create({ cwd: input.cwd, worktreePath, refName: checkoutRef, checkoutWorkers }) + .create({ + cwd: input.cwd, + worktreePath, + refName: checkoutRef, + checkoutWorkers, + lockReason, + }) .pipe(Effect.flatMap((cloned) => (cloned ? Effect.void : stockCheckout))) : stockCheckout ).pipe( - Effect.onError(() => - newRefName - ? deleteLocalBranch({ cwd: input.cwd, refName: newRefName, force: true }).pipe( + Effect.andThen( + fastCheckout ? checkoutCache.schedule(worktreePath, cacheGeneration) : Effect.void, + ), + Effect.andThen( + onWorktreeClaimed + ? Effect.uninterruptible( + onWorktreeClaimed(worktreePath).pipe( + Effect.andThen( + Effect.sync(() => { + claimed = true; + }), + ), + ), + ) + : Effect.void, + ), + Effect.onError((cause) => + newRefName && !claimed + ? // A failed hook keeps its worktree, as Git does; only a cancel abandons it. + (Cause.hasInterrupts(cause) + ? lockReason + ? releaseWorktreeRegistration(input.cwd, lockReason, true) + : Effect.void + : Effect.void + ).pipe( + Effect.andThen( + deleteLocalBranch({ cwd: input.cwd, refName: newRefName, force: true }), + ), + Effect.catch((cause) => + Effect.logWarning("could not clean up a failed worktree or its branch", { cause }), + ), + ) + : Effect.void, + ), + Effect.onExit(() => + lockReason + ? releaseWorktreeRegistration(input.cwd, lockReason, false).pipe( Effect.catch((cause) => - Effect.logWarning("could not remove the branch of a failed worktree", { cause }), + Effect.logWarning("could not unlock a created worktree", { cause }), ), ) : Effect.void, ), ); - if (fastCheckout) { - yield* checkoutCache.schedule(worktreePath, cacheGeneration); - } - - if (progress?.onWorktreeClaimed) { - yield* progress.onWorktreeClaimed(worktreePath); - } // worktree add leaves submodules empty, potentially omitting project skills, tools or source. const hasSubmodules = yield* fileSystem @@ -4734,6 +5255,61 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* yield* progress.onSubmodulesStarted(); } const onSubmoduleLine = progress?.onSubmoduleLine; + const updateSubmodules = (paths: readonly string[]) => + runGit( + "GitVcsDriver.createWorktree.updateSubmodules", + worktreePath, + [ + "submodule", + "update", + "--init", + ...(submoduleMode.value === "recursive" ? ["--recursive"] : []), + // Git reports transfer progress to a pipe only when forced. + ...(onSubmoduleLine ? ["--progress"] : []), + // A clone left by a failed update already has HEAD at the gitlink, so Git would skip it. + ...(paths.length > 0 ? ["--force", "--", ...paths] : []), + ], + { + // Cloning scales with remote size; let it finish or be cancelled, like worktree add. + timeoutMs: null, + ...(onSubmoduleLine + ? { + env: { LANGUAGE: "C" }, + progress: { onStdoutLine: onSubmoduleLine, onStderrLine: onSubmoduleLine }, + // Progress for many submodules can pass the output cap; truncate instead of failing. + appendTruncationMarker: true, + keepLineCallbacksAfterTruncation: true, + } + : {}), + }, + ); + // One failed clone stops Git before it checks out any submodule, leaving the cloned ones + // as empty working trees that read as modified; check those out on their own. + const checkOutClonedSubmodules = Effect.gen(function* () { + // Records are "submodule..path\n". + const submodulePaths = splitNullSeparatedPaths( + yield* runGitStdout("GitVcsDriver.createWorktree.listSubmodules", worktreePath, [ + "config", + "-z", + "--file", + ".gitmodules", + "--get-regexp", + "^submodule\\..*\\.path$", + ]), + false, + ).flatMap((record) => { + const newline = record.indexOf("\n"); + return newline === -1 ? [] : [record.slice(newline + 1)]; + }); + const clonedPaths = yield* Effect.filter(submodulePaths, (submodulePath) => + fileSystem + .exists(path.join(worktreePath, submodulePath, ".git")) + .pipe(Effect.orElseSucceed(() => false)), + ); + if (clonedPaths.length > 0) { + yield* updateSubmodules(clonedPaths); + } + }); // Registering submodule URLs writes the shared config; the clones that follow do not. yield* runGit( "GitVcsDriver.createWorktree.initSubmodules", @@ -4741,39 +5317,23 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ["submodule", "init"], { writesSharedConfig: true }, ).pipe( - Effect.andThen( - runGit( - "GitVcsDriver.createWorktree.updateSubmodules", - worktreePath, - [ - "submodule", - "update", - "--init", - ...(submoduleMode.value === "recursive" ? ["--recursive"] : []), - // Git reports transfer progress to a pipe only when forced. - ...(onSubmoduleLine ? ["--progress"] : []), - ], - { - // Cloning scales with remote size; let it finish or be cancelled, like worktree add. - timeoutMs: null, - ...(onSubmoduleLine - ? { - env: { LANGUAGE: "C" }, - progress: { onStdoutLine: onSubmoduleLine, onStderrLine: onSubmoduleLine }, - // Progress for many submodules can pass the output cap; truncate instead of failing. - appendTruncationMarker: true, - keepLineCallbacksAfterTruncation: true, - } - : {}), - }, - ), - ), + Effect.andThen(updateSubmodules([])), Effect.matchEffect({ onFailure: (cause) => Effect.logWarning("worktree submodule checkout failed; submodules may be incomplete", { worktreePath, cause, }).pipe( + Effect.andThen( + checkOutClonedSubmodules.pipe( + Effect.catch((retryCause) => + Effect.logWarning("could not check out the submodules that did clone", { + worktreePath, + cause: retryCause, + }), + ), + ), + ), Effect.andThen( progress?.onSubmodulesFinished ? progress.onSubmodulesFinished({ ok: false, detail: cause.message }) @@ -4940,6 +5500,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* const options = { env: STATUS_UPSTREAM_REFRESH_ENV, fallbackErrorDetail: `git fetch ${input.remoteName} failed`, + timeoutMs: FETCH_REMOTE_TIMEOUT_MS, }; // Another fetch in the same repository, such as the background status // poll, can win the race for a remote-tracking ref. The loser fails, but @@ -5065,8 +5626,12 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* remoteName: "origin", branchName: input.refName, }; - // `/HEAD` is the remote's default-branch pointer, not a branch it can serve. - return remoteNames.includes(remoteName) && branchName !== "HEAD" + // `/HEAD` is the remote's default-branch pointer, not a branch it can serve, and + // a revision expression such as `HEAD~1`, `@{-1}` or `:/text` is no branch name at all. + return remoteNames.includes(remoteName) && + branchName !== "HEAD" && + !branchName.startsWith("-") && + !/[\x00-\x20\x7f~^:?*[\\]|\.\.|@\{/.test(branchName) ? { remoteName, branchName } : null; }); @@ -5131,19 +5696,51 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* ]); }); - const setBranchUpstream: GitVcsDriver.GitVcsDriver["Service"]["setBranchUpstream"] = (input) => - executeGitWithStableDiagnostics( - "GitVcsDriver.setBranchUpstream", - input.cwd, - [ + const setBranchUpstream: GitVcsDriver.GitVcsDriver["Service"]["setBranchUpstream"] = Effect.fn( + "setBranchUpstream", + )( + function* (input) { + const args = [ "branch", "--set-upstream-to", - `${input.remoteName}/${input.remoteBranch}`, + `refs/remotes/${input.remoteName}/${input.remoteBranch}`, "--", input.branch, - ], - { writesSharedConfig: true }, - ).pipe(retryWhileConfigLocked, Effect.asVoid); + ]; + const result = yield* executeGitWithStableDiagnostics( + "GitVcsDriver.setBranchUpstream", + input.cwd, + args, + { allowNonZeroExit: true }, + ); + if (result.exitCode === 0) return; + // Git requires a configured fetch mapping even when the tracking ref already exists. The + // mapping is a glob because an exact source fails every later fetch once the branch is deleted. + if ( + /^fatal: cannot set up tracking information; starting point '.+' is not a branch$/m.test( + result.stderr, + ) + ) { + yield* executeGitWithStableDiagnostics( + "GitVcsDriver.setBranchUpstream.addFetchMapping", + input.cwd, + [ + "config", + "--add", + `remote.${input.remoteName}.fetch`, + `+refs/heads/${input.remoteBranch}*:refs/remotes/${input.remoteName}/${input.remoteBranch}*`, + ], + ).pipe(retryWhileConfigLocked); + } + yield* executeGitWithStableDiagnostics( + "GitVcsDriver.setBranchUpstream", + input.cwd, + args, + ).pipe(retryWhileConfigLocked); + }, + // Keep mapping detection and both writes under the same lock across sibling worktrees. + (effect, input) => withSharedConfigLock(input.cwd, effect), + ); /** * Unlinks links under `target` without following them and removes the @@ -5264,22 +5861,28 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* if (input.oldBranch === input.newBranch) { return { branch: input.newBranch }; } - const targetBranch = input.exactName - ? input.newBranch - : yield* resolveAvailableBranchName(input.cwd, input.newBranch); - - yield* executeGit( - "GitVcsDriver.renameBranch", + // Parallel launches often generate the same name; picking it under the lock that serializes + // the renames keeps a second launch from choosing a name the first is about to take. + return yield* withSharedConfigLock( input.cwd, - ["branch", "-m", "--", input.oldBranch, targetBranch], - { - timeoutMs: 10_000, - fallbackErrorDetail: "git branch rename failed", - writesSharedConfig: true, - }, - ); + Effect.gen(function* () { + const targetBranch = input.exactName + ? input.newBranch + : yield* resolveAvailableBranchName(input.cwd, input.newBranch); - return { branch: targetBranch }; + yield* executeGit( + "GitVcsDriver.renameBranch", + input.cwd, + ["branch", "-m", "--", input.oldBranch, targetBranch], + { + timeoutMs: 10_000, + fallbackErrorDetail: "git branch rename failed", + }, + ); + + return { branch: targetBranch }; + }), + ); }); const switchRef: GitVcsDriver.GitVcsDriver["Service"]["switchRef"] = Effect.fn("switchRef")( @@ -5521,6 +6124,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function* statusDetails, statusDetailsLocal, statusDetailsRemote, + readRefSnapshot, prepareCommitContext, commit: Effect.fnUntraced(function* (cwd, subject, body, options) { return yield* withListRefsInvalidation(cwd, commit(cwd, subject, body, options)); diff --git a/apps/server/src/vcs/VcsAutomaticFetchInterval.ts b/apps/server/src/vcs/VcsAutomaticFetchInterval.ts new file mode 100644 index 0000000000000..0f2c900b3ba11 --- /dev/null +++ b/apps/server/src/vcs/VcsAutomaticFetchInterval.ts @@ -0,0 +1,26 @@ +import * as Context from "effect/Context"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import { DEFAULT_AUTOMATIC_GIT_FETCH_INTERVAL } from "@t3tools/contracts"; +import { resolveServerBackgroundActivitySettings } from "@t3tools/shared/backgroundActivitySettings"; + +import * as ServerSettings from "../serverSettings.ts"; + +export class VcsAutomaticFetchInterval extends Context.Reference>( + "t3/vcs/VcsAutomaticFetchInterval", + { defaultValue: () => Effect.succeed(DEFAULT_AUTOMATIC_GIT_FETCH_INTERVAL) }, +) {} + +export const layer = Layer.effect( + VcsAutomaticFetchInterval, + Effect.gen(function* () { + const serverSettings = yield* ServerSettings.ServerSettingsService; + return serverSettings.getSettings.pipe( + Effect.map( + (settings) => resolveServerBackgroundActivitySettings(settings).automaticGitFetchInterval, + ), + Effect.orElseSucceed(() => DEFAULT_AUTOMATIC_GIT_FETCH_INTERVAL), + ); + }), +); diff --git a/apps/server/src/vcs/VcsProcess.ts b/apps/server/src/vcs/VcsProcess.ts index 4bfdcc90037ec..da2f988ce1ed8 100644 --- a/apps/server/src/vcs/VcsProcess.ts +++ b/apps/server/src/vcs/VcsProcess.ts @@ -33,6 +33,7 @@ export interface VcsProcessInput { readonly spawnCwd?: string; readonly stdin?: string; readonly onStdoutChunk?: (chunk: Uint8Array) => void; + readonly onSpawn?: () => void; readonly env?: NodeJS.ProcessEnv; readonly allowNonZeroExit?: boolean; readonly timeoutMs?: number; @@ -232,6 +233,7 @@ export const make = Effect.gen(function* () { ...(input.spawnCwd !== undefined ? { spawnCwd: input.spawnCwd } : {}), ...(input.stdin !== undefined ? { stdin: input.stdin } : {}), ...(input.onStdoutChunk !== undefined ? { onStdoutChunk: input.onStdoutChunk } : {}), + ...(input.onSpawn !== undefined ? { onSpawn: input.onSpawn } : {}), env: { ...(input.command === "git" ? unsetInheritedRepositoryEnv(process.env, hostPlatform) diff --git a/apps/server/src/vcs/VcsStatusBroadcaster.test.ts b/apps/server/src/vcs/VcsStatusBroadcaster.test.ts index 73093af6c750b..08fad4ae56aff 100644 --- a/apps/server/src/vcs/VcsStatusBroadcaster.test.ts +++ b/apps/server/src/vcs/VcsStatusBroadcaster.test.ts @@ -28,6 +28,7 @@ import type { import { GitCommandError, GitManagerError } from "@t3tools/contracts"; import { applyGitStatusStreamEvent } from "@t3tools/shared/git"; +import * as VcsAutomaticFetchInterval from "./VcsAutomaticFetchInterval.ts"; import * as VcsStatusBroadcaster from "./VcsStatusBroadcaster.ts"; import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; import * as GitWorkflowService from "../git/GitWorkflowService.ts"; @@ -77,10 +78,12 @@ function layerTestFor(state: { localInvalidationCalls: number; remoteInvalidationCalls: number; remoteStatusRefreshUpstreamValues?: Array; + localStatusBranchChanges?: Array; + refSnapshot?: string | null; remoteStatusError?: GitManagerError; backgroundWorkEnabled?: boolean; automaticFetchInterval?: Effect.Effect; - beforeRemoteStatus?: Effect.Effect; + beforeRemoteStatus?: Effect.Effect; beforeLocalStatus?: Effect.Effect; beforeCurrentBranch?: Effect.Effect; beforeInvalidate?: Effect.Effect; @@ -101,15 +104,19 @@ function layerTestFor(state: { Layer.provide(layerBackgroundPolicy(() => state.backgroundWorkEnabled !== false)), Layer.provide( Layer.succeed( - VcsStatusBroadcaster.VcsAutomaticFetchInterval, + VcsAutomaticFetchInterval.VcsAutomaticFetchInterval, state.automaticFetchInterval ?? Effect.succeed(Duration.seconds(30)), ), ), Layer.provide( Layer.mock(GitWorkflowService.GitWorkflowService)({ - localStatus: () => + localStatus: (_input, options) => Effect.suspend(() => { state.localStatusCalls += 1; + const includeBranchChanges = options?.includeBranchChanges !== false; + state.localStatusBranchChanges?.push(includeBranchChanges); + const { branchChanges: _branchChanges, ...withoutBranchChanges } = + state.currentLocalStatus; if (state.failLocalStatus) { return (state.onLocalStatusFailure ?? Effect.void).pipe( Effect.andThen( @@ -124,7 +131,7 @@ function layerTestFor(state: { ); } return (state.beforeLocalStatus ?? Effect.void).pipe( - Effect.as(state.currentLocalStatus), + Effect.as(includeBranchChanges ? state.currentLocalStatus : withoutBranchChanges), ); }), repositoryRoot: () => Effect.sync(() => (state.currentLocalStatus.isRepo ? "/repo" : null)), @@ -134,6 +141,8 @@ function layerTestFor(state: { : (state.beforeCurrentBranch ?? Effect.void).pipe( Effect.andThen(Effect.sync(() => state.currentLocalStatus.refName)), ), + readRefSnapshot: () => + Effect.sync(() => (state.refSnapshot === undefined ? "refs" : state.refSnapshot)), upstreamDivergence: () => Effect.sync(() => state.currentRemoteStatus?.hasUpstream && state.currentLocalStatus.refName !== null @@ -349,6 +358,80 @@ describe("VcsStatusBroadcaster", () => { }, ); + it.effect("publishes the new branch before pending detailed status completes", () => { + const state = newState(); + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const branches = yield* Stream.toPull(broadcaster.streamBranchStatus({ cwd: "/repo" })); + yield* branches; + const reading = yield* Deferred.make(); + const resume = yield* Deferred.make(); + state.beforeLocalStatus = Deferred.succeed(reading, undefined).pipe( + Effect.andThen(Deferred.await(resume)), + ); + const details = yield* Stream.runHead( + broadcaster.streamStatus({ cwd: "/repo", includeRemote: false }), + ).pipe(Effect.forkScoped); + yield* Deferred.await(reading); + state.currentLocalStatus = { ...baseLocalStatus, refName: "feature/next" }; + const update = yield* branches.pipe(Effect.forkScoped({ startImmediately: true })); + yield* broadcaster.refreshBranchStatus({ cwd: "/repo" }); + assert.deepEqual( + update.pollUnsafe(), + Exit.succeed([{ refName: "feature/next", isRepo: true }]), + ); + state.beforeLocalStatus = Effect.void; + yield* Deferred.succeed(resume, undefined); + yield* Fiber.join(details); + yield* broadcaster.getStatus({ cwd: "/repo" }); + state.currentLocalStatus = { ...baseLocalStatus, refName: "feature/latest" }; + yield* broadcaster.refreshBranchStatus({ cwd: "/repo" }); + assert.deepEqual(yield* branches, [{ refName: "feature/latest", isRepo: true }]); + }).pipe(Effect.provide(layerTestFor(state))); + }); + + it.effect("invalidates totals for a refresh arriving after detailed demand closes", () => { + const state = newState(); + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const detailScope = yield* Scope.make(); + const details = yield* Stream.toPull( + broadcaster.streamStatus({ cwd: "/repo", includeRemote: false }), + ).pipe(Scope.provide(detailScope)); + yield* details; + const reading = yield* Deferred.make(); + const resume = yield* Deferred.make(); + state.currentLocalStatus = { + ...baseLocalStatus, + refName: "feature/next", + workingTree: { files: [], insertions: 1, deletions: 0 }, + }; + state.beforeLocalStatus = Deferred.succeed(reading, undefined).pipe( + Effect.andThen(Deferred.await(resume)), + ); + yield* broadcaster.refreshBranchStatus({ cwd: "/repo" }); + yield* Deferred.await(reading); + yield* Scope.close(detailScope, Exit.void); + state.currentLocalStatus = { + ...state.currentLocalStatus, + workingTree: { files: [], insertions: 2, deletions: 0 }, + }; + const trailing = yield* broadcaster + .refreshBranchStatus({ cwd: "/repo" }) + .pipe(Effect.forkScoped({ startImmediately: true })); + state.beforeLocalStatus = Effect.void; + yield* Deferred.succeed(resume, undefined); + yield* Fiber.join(trailing); + const reopened = yield* Stream.runHead( + broadcaster.streamStatus({ cwd: "/repo", includeRemote: false }), + ); + assert.deepEqual( + reopened, + Option.some({ _tag: "snapshot" as const, local: state.currentLocalStatus, remote: null }), + ); + }).pipe(Effect.provide(layerTestFor(state))); + }); + it.effect("upgrades an in-flight branch refresh when detailed demand arrives", () => { const state: Parameters[0] = { currentLocalStatus: baseLocalStatus, @@ -673,6 +756,7 @@ describe("VcsStatusBroadcaster", () => { Layer.mock(GitWorkflowService.GitWorkflowService)({ localStatus: () => Effect.succeed(localStatus), currentBranch: () => Effect.succeed(localStatus.refName), + readRefSnapshot: () => Effect.succeed("refs"), remoteStatus: () => Effect.succeed(remoteStatus), invalidateLocalStatus: () => Effect.void, invalidateRemoteStatus: () => Effect.void, @@ -769,6 +853,7 @@ describe("VcsStatusBroadcaster", () => { Layer.mock(GitWorkflowService.GitWorkflowService)({ localStatus: () => Effect.succeed(baseLocalStatus), currentBranch: () => Effect.succeed(baseLocalStatus.refName), + readRefSnapshot: () => Effect.succeed("refs"), remoteStatus: () => Effect.gen(function* () { remoteReads += 1; @@ -815,6 +900,7 @@ describe("VcsStatusBroadcaster", () => { Layer.mock(GitWorkflowService.GitWorkflowService)({ localStatus: () => Effect.succeed(baseLocalStatus), currentBranch: () => Effect.succeed(baseLocalStatus.refName), + readRefSnapshot: () => Effect.succeed("refs"), remoteStatus: () => Effect.gen(function* () { remoteReads += 1; @@ -868,6 +954,7 @@ describe("VcsStatusBroadcaster", () => { return state.currentLocalStatus; }), currentBranch: () => Effect.succeed(state.currentLocalStatus.refName), + readRefSnapshot: () => Effect.succeed("refs"), remoteStatus: () => Effect.suspend(() => { state.remoteStatusCalls += 1; @@ -994,6 +1081,49 @@ describe("VcsStatusBroadcaster", () => { }, ); + it.effect("publishes a checkout's branch before returning, ahead of the remote read", () => { + const state: Parameters[0] = { + currentLocalStatus: baseLocalStatus, + currentRemoteStatus: baseRemoteStatus, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + }; + + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + yield* broadcaster.getStatus({ cwd: "/repo" }); + const subscribed = yield* Deferred.make(); + const localPublished = yield* Deferred.make(); + const remotePublished = yield* Deferred.make(); + yield* broadcaster.streamStatus({ cwd: "/repo", includeRemote: false }).pipe( + Stream.runForEach((event) => { + if (event._tag === "snapshot") return Deferred.succeed(subscribed, undefined); + if (event._tag === "localUpdated") return Deferred.succeed(localPublished, event.local); + return event.remote === null + ? Effect.void + : Deferred.succeed(remotePublished, event.remote); + }), + Effect.forkScoped, + ); + yield* Deferred.await(subscribed); + + const releaseRemote = yield* Deferred.make(); + state.beforeRemoteStatus = Deferred.await(releaseRemote); + state.currentLocalStatus = { ...baseLocalStatus, refName: "main" }; + state.currentRemoteStatus = remoteStatusWithPr; + yield* broadcaster.refreshAfterCheckout("/repo"); + + assert.equal((yield* Deferred.await(localPublished)).refName, "main"); + assert.isFalse(yield* Deferred.isDone(remotePublished)); + + yield* Deferred.succeed(releaseRemote, undefined); + assert.deepStrictEqual(yield* Deferred.await(remotePublished), remoteStatusWithPr); + assert.equal((yield* broadcaster.getStatus({ cwd: "/repo" })).pr?.number, 2978); + }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }); + it.effect("a remote poll refreshes Changes totals when divergence counts stay the same", () => { const state: Parameters[0] = { currentLocalStatus: { @@ -1033,6 +1163,116 @@ describe("VcsStatusBroadcaster", () => { }).pipe(Effect.provide(layerTestFor(state))); }); + it.effect("reads Changes totals only while a subscriber shows them", () => { + const branchChanges = { baseRef: "origin/main", insertions: 3, deletions: 1 }; + const state: Parameters[0] = { + currentLocalStatus: { ...baseLocalStatus, branchChanges }, + currentRemoteStatus: baseRemoteStatus, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + localStatusBranchChanges: [], + }; + + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const paletteEvents = yield* Queue.unbounded(); + yield* broadcaster.streamStatus({ cwd: "/repo", includeRemote: false }).pipe( + Stream.runForEach((event) => Queue.offer(paletteEvents, event)), + Effect.forkScoped, + ); + const paletteSnapshot = yield* Queue.take(paletteEvents); + assert.strictEqual(paletteSnapshot._tag, "snapshot"); + if (paletteSnapshot._tag === "snapshot") { + assert.isUndefined(paletteSnapshot.local.branchChanges); + } + yield* broadcaster.refreshLocalStatus("/repo"); + assert.deepStrictEqual(state.localStatusBranchChanges, [false, false]); + + const headerScope = yield* Scope.make(); + const headerEvents = yield* Queue.unbounded(); + yield* broadcaster.streamStatus({ cwd: "/repo" }).pipe( + Stream.runForEach((event) => Queue.offer(headerEvents, event)), + Effect.forkIn(headerScope), + ); + const headerSnapshot = yield* Queue.take(headerEvents); + assert.strictEqual(headerSnapshot._tag, "snapshot"); + if (headerSnapshot._tag === "snapshot") { + assert.deepStrictEqual(headerSnapshot.local.branchChanges, branchChanges); + } + const paletteUpdate = yield* Queue.take(paletteEvents); + assert.strictEqual(paletteUpdate._tag, "localUpdated"); + if (paletteUpdate._tag === "localUpdated") { + assert.deepStrictEqual(paletteUpdate.local.branchChanges, branchChanges); + } + // The header's first remote poll also rereads local status. + while ((yield* Queue.take(headerEvents))._tag !== "remoteUpdated") {} + const turnEnd = yield* broadcaster.refreshLocalStatus("/repo"); + assert.deepStrictEqual(turnEnd.branchChanges, branchChanges); + + yield* Scope.close(headerScope, Exit.void); + yield* broadcaster.refreshLocalStatus("/repo"); + assert.deepStrictEqual(state.localStatusBranchChanges, [ + false, + false, + true, + true, + true, + false, + ]); + }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }); + + it.effect("a light refresh that a header joins mid-read publishes Changes totals", () => { + const branchChanges = { baseRef: "origin/main", insertions: 3, deletions: 1 }; + const state: Parameters[0] = { + currentLocalStatus: { ...baseLocalStatus, branchChanges }, + currentRemoteStatus: baseRemoteStatus, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + localStatusBranchChanges: [], + fileSystem: FileSystem.makeNoop({ realPath: (cwd) => Effect.succeed(cwd) }), + }; + + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const subscribeHeader = Effect.fn("subscribeHeader")(function* (scope: Scope.Scope) { + const events = yield* Queue.unbounded(); + yield* broadcaster.streamStatus({ cwd: "/repo" }).pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkIn(scope, { startImmediately: true }), + ); + return events; + }); + const firstHeader = yield* Scope.make(); + const firstEvents = yield* subscribeHeader(firstHeader); + while ((yield* Queue.take(firstEvents))._tag !== "remoteUpdated") {} + yield* Scope.close(firstHeader, Exit.void); + + const readStarted = yield* Deferred.make(); + const finishRead = yield* Deferred.make(); + state.beforeLocalStatus = Deferred.succeed(readStarted, undefined).pipe( + Effect.andThen(Deferred.await(finishRead)), + ); + const turnEnd = yield* broadcaster.refreshLocalStatus("/repo").pipe(Effect.forkScoped); + yield* Deferred.await(readStarted); + // Starting immediately registers the header's demand before it waits behind the read. + const secondEvents = yield* subscribeHeader(yield* Scope.make()); + yield* Deferred.succeed(finishRead, undefined); + const snapshot = yield* Queue.take(secondEvents); + assert.strictEqual(snapshot._tag, "snapshot"); + if (snapshot._tag === "snapshot") { + assert.deepStrictEqual(snapshot.local.branchChanges, branchChanges); + } + + assert.deepStrictEqual((yield* Fiber.join(turnEnd)).branchChanges, branchChanges); + assert.deepStrictEqual(state.localStatusBranchChanges?.slice(-2), [false, true]); + }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }); + it.effect("an explicit refresh reads local totals after the fetch", () => { return Effect.gen(function* () { const fetchStarted = yield* Deferred.make(); @@ -1042,6 +1282,7 @@ describe("VcsStatusBroadcaster", () => { branchChanges: { baseRef: "origin/main", insertions: 745, deletions: 24 }, }; let cachedLocal: VcsStatusLocalResult | undefined; + let refSnapshot = "before"; const readLocalStatus = () => Effect.sync(() => (cachedLocal ??= local)); const layerTest = VcsStatusBroadcaster.layer.pipe( Layer.provideMerge(NodeServices.layer), @@ -1059,6 +1300,7 @@ describe("VcsStatusBroadcaster", () => { }), currentBranch: () => Effect.succeed(baseLocalStatus.refName), invalidateRemoteStatus: () => Effect.void, + readRefSnapshot: () => Effect.sync(() => refSnapshot), remoteStatus: () => Deferred.succeed(fetchStarted, undefined).pipe( Effect.andThen(Deferred.await(finishFetch)), @@ -1077,6 +1319,7 @@ describe("VcsStatusBroadcaster", () => { ...baseLocalStatus, branchChanges: { baseRef: "origin/main", insertions: 0, deletions: 0 }, }; + refSnapshot = "after"; yield* Deferred.succeed(finishFetch, undefined); const status = yield* Fiber.join(refresh); assert.deepStrictEqual(status.branchChanges, local.branchChanges); @@ -1084,6 +1327,59 @@ describe("VcsStatusBroadcaster", () => { }); }); + it.effect("an explicit refresh re-reads local totals only after refs moved", () => { + const totals = (insertions: number): VcsStatusLocalResult => ({ + ...baseLocalStatus, + branchChanges: { baseRef: "origin/main", insertions, deletions: 0 }, + }); + const state: Parameters[0] = { + currentLocalStatus: totals(745), + currentRemoteStatus: remoteStatusWithPr, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + }; + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const subscribed = yield* Deferred.make(); + const events = yield* broadcaster.streamStatus({ cwd: "/repo", includeRemote: false }).pipe( + Stream.tap((event) => + event._tag === "snapshot" ? Deferred.succeed(subscribed, undefined) : Effect.void, + ), + Stream.filter((event) => event._tag !== "snapshot"), + Stream.take(4), + Stream.runCollect, + Effect.forkScoped, + ); + yield* Deferred.await(subscribed); + + state.localStatusCalls = 0; + yield* broadcaster.refreshStatus("/repo"); + assert.equal(state.localStatusCalls, 1); + + state.localStatusCalls = 0; + state.currentRemoteStatus = baseRemoteStatus; + state.beforeRemoteStatus = Effect.sync(() => { + state.currentLocalStatus = totals(0); + state.refSnapshot = "moved"; + }); + const moved = yield* broadcaster.refreshStatus("/repo"); + assert.equal(state.localStatusCalls, 2); + assert.deepStrictEqual(moved.branchChanges, totals(0).branchChanges); + assert.deepStrictEqual( + [...(yield* Fiber.join(events))].map((event) => event._tag), + // The first refresh adds the totals this palette-style subscription skipped. + ["localUpdated", "remoteUpdated", "remoteUpdated", "localUpdated"], + ); + + state.localStatusCalls = 0; + state.refSnapshot = null; + yield* broadcaster.refreshStatus("/repo"); + assert.equal(state.localStatusCalls, 2); + }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }); + it.effect("publishes local status while an earlier lookup holds the remote lock", () => { const state: Parameters[0] = { currentLocalStatus: baseLocalStatus, @@ -1419,6 +1715,49 @@ describe("VcsStatusBroadcaster", () => { }).pipe(Effect.provide(NodeServices.layer)), ); + it.effect("a terminal checkout updates the branch before pending detailed status completes", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const gitDir = yield* fs.makeTempDirectoryScoped({ prefix: "vcs-head-watch-" }); + const headPath = path.join(gitDir, "HEAD"); + yield* fs.writeFileString(headPath, "ref: refs/heads/main\n"); + const state: Parameters[0] = { + currentLocalStatus: { ...baseLocalStatus, refName: "main" }, + currentRemoteStatus: null, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + headPath, + }; + + yield* Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const branches = yield* Stream.toPull(broadcaster.streamBranchStatus({ cwd: "/repo" })); + yield* branches; + const details = yield* Stream.toPull( + broadcaster.streamStatus({ cwd: "/repo", includeRemote: false }), + ); + yield* details; + + const resume = yield* Deferred.make(); + state.beforeLocalStatus = Deferred.await(resume); + state.currentLocalStatus = { ...baseLocalStatus, refName: "repro" }; + const lockPath = path.join(gitDir, "HEAD.lock"); + yield* fs.writeFileString(lockPath, "ref: refs/heads/repro\n"); + yield* fs.rename(lockPath, headPath); + + assert.deepEqual(yield* branches, [{ refName: "repro", isRepo: true }]); + state.currentLocalStatus = { ...baseLocalStatus, refName: "second" }; + yield* fs.writeFileString(lockPath, "ref: refs/heads/second\n"); + yield* fs.rename(lockPath, headPath); + assert.deepEqual(yield* branches, [{ refName: "second", isRepo: true }]); + yield* Deferred.succeed(resume, undefined); + }).pipe(Effect.provide(layerTestFor(state)), Effect.scoped); + }).pipe(Effect.provide(NodeServices.layer)), + ); + it.effect("a checkout read that fails while index.lock is held retries until it succeeds", () => Effect.gen(function* () { const events = yield* Queue.unbounded(); @@ -1724,6 +2063,92 @@ describe("VcsStatusBroadcaster", () => { assert.isTrue(Option.isSome(yield* Deferred.poll(remoteInterrupted))); }).pipe(Effect.provide(layerTest)); }); + + it.effect("refreshes requested during a refresh share one follow-up refresh", () => { + const state: Parameters[0] = { + currentLocalStatus: baseLocalStatus, + currentRemoteStatus: baseRemoteStatus, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + }; + + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const fetchStarted = yield* Deferred.make(); + const finishFetch = yield* Deferred.make(); + state.beforeRemoteStatus = Deferred.succeed(fetchStarted, undefined).pipe( + Effect.andThen(Deferred.await(finishFetch)), + ); + const first = yield* broadcaster.refreshStatus("/repo").pipe(Effect.forkScoped); + yield* Deferred.await(fetchStarted); + // Every read after the first sees a PR, so a caller that reused the + // first refresh would get a status from before its own request. + state.beforeRemoteStatus = Effect.sync(() => { + state.currentRemoteStatus = remoteStatusWithPr; + }); + const later = yield* Effect.all( + [broadcaster.refreshStatus("/repo"), broadcaster.refreshStatus("/repo")], + { concurrency: "unbounded" }, + ).pipe(Effect.forkScoped({ startImmediately: true })); + yield* Deferred.succeed(finishFetch, undefined); + + assert.isNull((yield* Fiber.join(first)).pr); + const [second, third] = yield* Fiber.join(later); + assert.deepStrictEqual(second.pr, remoteStatusWithPr.pr); + assert.deepStrictEqual(third.pr, remoteStatusWithPr.pr); + assert.equal(state.remoteStatusCalls, 2); + + yield* broadcaster.refreshStatus("/repo"); + assert.equal(state.remoteStatusCalls, 3); + }).pipe(Effect.provide(layerTestFor(state))); + }); + + it.effect("a failed refresh releases interrupted waiters and runs the shared follow-up", () => { + const state: Parameters[0] = { + currentLocalStatus: baseLocalStatus, + currentRemoteStatus: remoteStatusWithPr, + localStatusCalls: 0, + remoteStatusCalls: 0, + localInvalidationCalls: 0, + remoteInvalidationCalls: 0, + }; + + return Effect.gen(function* () { + const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; + const started = yield* Deferred.make(); + const release = yield* Deferred.make(); + state.beforeRemoteStatus = Deferred.succeed(started, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.andThen( + Effect.fail( + new GitManagerError({ + operation: "VcsStatusBroadcaster.test", + cwd: "/repo", + detail: "remote status failed", + }), + ), + ), + ); + const first = yield* broadcaster.refreshStatus("/repo").pipe(Effect.exit, Effect.forkScoped); + yield* Deferred.await(started); + const interrupted = yield* broadcaster + .refreshStatus("/repo") + .pipe(Effect.forkScoped({ startImmediately: true })); + const remaining = yield* broadcaster + .refreshStatus("/repo") + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Fiber.interrupt(interrupted); + state.beforeRemoteStatus = Effect.void; + yield* Deferred.succeed(release, undefined); + assert.isTrue(Exit.isFailure(yield* Fiber.join(first))); + assert.deepStrictEqual((yield* Fiber.join(remaining)).pr, remoteStatusWithPr.pr); + assert.equal(state.remoteStatusCalls, 1); + yield* broadcaster.refreshStatus("/repo"); + assert.equal(state.remoteStatusCalls, 2); + }).pipe(Effect.provide(layerTestFor(state))); + }); }); const newState = (): Parameters[0] => ({ @@ -1788,8 +2213,10 @@ it.effect("serializes branch reads with detailed refresh publication", () => { yield* Fiber.join(older); yield* Fiber.join(newer); state.branchRead = () => Effect.succeed("new"); - const result = yield* b.getStatus({ cwd: "/repo" }); - assert.equal(result.refName, "new"); + const result = yield* b + .streamStatus({ cwd: "/repo", includeRemote: false }) + .pipe(Stream.runHead, Effect.flatMap(Effect.fromOption)); + assert.equal(result._tag === "snapshot" && result.local.refName, "new"); assert.equal(state.localStatusCalls, 1); }).pipe(Effect.provide(layerTestFor(state))); }); diff --git a/apps/server/src/vcs/VcsStatusBroadcaster.ts b/apps/server/src/vcs/VcsStatusBroadcaster.ts index 28daef7d790ca..b691b9d2978a2 100644 --- a/apps/server/src/vcs/VcsStatusBroadcaster.ts +++ b/apps/server/src/vcs/VcsStatusBroadcaster.ts @@ -25,7 +25,6 @@ import type { VcsStatusSubscriptionInput, } from "@t3tools/contracts"; import { DEFAULT_AUTOMATIC_GIT_FETCH_INTERVAL } from "@t3tools/contracts"; -import { resolveServerBackgroundActivitySettings } from "@t3tools/shared/backgroundActivitySettings"; import { mergeGitStatusParts } from "@t3tools/shared/git"; import * as KeyedLock from "@t3tools/shared/KeyedLock"; import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; @@ -34,6 +33,7 @@ import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; import * as GitWorkflowService from "../git/GitWorkflowService.ts"; import * as ProjectStore from "../orchestration-v2/ProjectStore.ts"; import * as ServerSettings from "../serverSettings.ts"; +import { VcsAutomaticFetchInterval } from "./VcsAutomaticFetchInterval.ts"; const VCS_STATUS_REFRESH_FAILURE_BASE_DELAY = Duration.seconds(30); const VCS_STATUS_REFRESH_FAILURE_MAX_DELAY = Duration.minutes(15); @@ -143,8 +143,12 @@ interface CachedRemoteStatus extends CachedValue { readonly branch: string | null; } +interface CachedLocalStatus extends CachedValue { + readonly includesBranchChanges: boolean; +} + interface CachedVcsStatus { - readonly local: CachedValue | null; + readonly local: CachedLocalStatus | null; readonly remote: CachedRemoteStatus | null; } @@ -160,24 +164,6 @@ interface ActiveRemotePoller { readonly demandCwds: Ref.Ref>; } -export class VcsAutomaticFetchInterval extends Context.Reference>( - "t3/vcs/VcsAutomaticFetchInterval", - { defaultValue: () => Effect.succeed(DEFAULT_AUTOMATIC_GIT_FETCH_INTERVAL) }, -) {} - -export const layerAutomaticFetchInterval = Layer.effect( - VcsAutomaticFetchInterval, - Effect.gen(function* () { - const serverSettings = yield* ServerSettings.ServerSettingsService; - return serverSettings.getSettings.pipe( - Effect.map( - (settings) => resolveServerBackgroundActivitySettings(settings).automaticGitFetchInterval, - ), - Effect.orElseSucceed(() => DEFAULT_AUTOMATIC_GIT_FETCH_INTERVAL), - ); - }), -); - export class VcsAutoPullPolicy extends Context.Reference<{ readonly isEnabled: (cwd: string) => Effect.Effect; }>("t3/vcs/VcsAutoPullPolicy", { @@ -236,6 +222,11 @@ export class VcsStatusBroadcaster extends Context.Service< readonly refreshLocalStatus: ( cwd: string, ) => Effect.Effect; + /** + * For a checkout someone is waiting on. Returns once the new branch is published, so the + * caller's reply does not outrun it, and finishes the slower remote read in the background. + */ + readonly refreshAfterCheckout: (cwd: string) => Effect.Effect; readonly refreshStatus: ( cwd: string, options?: { readonly localOnly?: boolean }, @@ -286,17 +277,29 @@ export const make = Effect.gen(function* () { const detailedSubscribers = new Map(); const branches = new Map< string, - { readonly refName: string | null; readonly isRepo: boolean; readonly revision: number } + { + readonly refName: string | null; + readonly isRepo: boolean; + readonly revision: number; + readonly observation: number; + } >(); let branchRevision = 0; + // A slow detailed read must not replace a newer HEAD-only observation. + let branchObservation = 0; const publishBranch = Effect.fn("VcsStatusBroadcaster.publishBranch")(function* ( cwd: string, { refName, isRepo }: { readonly refName: string | null; readonly isRepo: boolean }, + observation: number, ) { const previous = branches.get(cwd); - if (previous?.refName === refName && previous.isRepo === isRepo) return; + if (previous && previous.observation > observation) return; + if (previous?.refName === refName && previous.isRepo === isRepo) { + branches.set(cwd, { ...previous, observation }); + return; + } const revision = ++branchRevision; - branches.set(cwd, { refName, isRepo, revision }); + branches.set(cwd, { refName, isRepo, revision, observation }); yield* PubSub.publish(branchChangesPubSub, { cwd, refName, isRepo, revision }); }); const localRefreshLocks = yield* KeyedLock.make(); @@ -316,9 +319,51 @@ export const make = Effect.gen(function* () { return yield* Ref.get(cacheRef).pipe(Effect.map((cache) => cache.get(cwd) ?? null)); }); + // Only full subscribers and direct reads show Changes totals. Command palette rows subscribe + // without remote status and skip them, which halves the git processes a burst of rows spawns. + const branchChangesDemand = new Map(); + const wantsBranchChanges = (cwd: string) => branchChangesDemand.has(cwd); + const acquireBranchChangesDemand = (cwd: string) => + Effect.acquireRelease( + Effect.sync(() => { + branchChangesDemand.set(cwd, (branchChangesDemand.get(cwd) ?? 0) + 1); + }), + () => + Effect.sync(() => { + const count = (branchChangesDemand.get(cwd) ?? 1) - 1; + if (count > 0) branchChangesDemand.set(cwd, count); + else branchChangesDemand.delete(cwd); + }), + ); + + const readLocalStatus = Effect.fn("VcsStatusBroadcaster.readLocalStatus")(function* ( + cwd: string, + ) { + const includeBranchChanges = wantsBranchChanges(cwd); + const local = yield* workflow.localStatus({ cwd }, { includeBranchChanges }); + // A subscriber that shows totals arrived during the read and must not receive it without them. + if (!includeBranchChanges && wantsBranchChanges(cwd)) { + return { + local: yield* workflow.localStatus({ cwd }, { includeBranchChanges: true }), + includesBranchChanges: true, + }; + } + return { local, includesBranchChanges: includeBranchChanges }; + }); + const updateCachedLocalStatus = Effect.fn("VcsStatusBroadcaster.updateCachedLocalStatus")( - function* (cwd: string, local: VcsStatusLocalResult, options?: { publish?: boolean }) { - const nextLocal = { fingerprint: fingerprintStatusPart(local), value: local }; + function* ( + cwd: string, + read: { readonly local: VcsStatusLocalResult; readonly includesBranchChanges: boolean }, + observation: number, + options?: { publish?: boolean }, + ) { + const { local, includesBranchChanges } = read; + const nextLocal = { + fingerprint: fingerprintStatusPart(local), + value: local, + includesBranchChanges, + }; const [localChanged, remoteCleared] = yield* Ref.modify(cacheRef, (cache) => { const previous = cache.get(cwd) ?? { local: null, remote: null }; const remote = previous.remote?.branch === local.refName ? previous.remote : null; @@ -327,7 +372,7 @@ export const make = Effect.gen(function* () { new Map(cache).set(cwd, { local: nextLocal, remote }), ] as const; }); - yield* publishBranch(cwd, local); + yield* publishBranch(cwd, local, observation); if (options?.publish && remoteCleared) { yield* PubSub.publish(changesPubSub, { cwd, @@ -366,21 +411,22 @@ export const make = Effect.gen(function* () { const loadLocalStatus = Effect.fn("VcsStatusBroadcaster.loadLocalStatus")(function* ( cwd: string, + options?: { publish?: boolean }, ) { - const local = yield* workflow.localStatus({ cwd }); - return yield* updateCachedLocalStatus(cwd, local); + const observation = ++branchObservation; + return yield* updateCachedLocalStatus(cwd, yield* readLocalStatus(cwd), observation, options); }); const getOrLoadLocalStatusUnlocked = Effect.fn("VcsStatusBroadcaster.getOrLoadLocalStatus")( function* (cwd: string) { const cached = yield* getCachedStatus(cwd); if (cached?.local) { - const branch = yield* workflow.currentBranch(cwd); - if (branch === cached.local.value.refName) return cached.local.value; + if (cached.local.includesBranchChanges || !wantsBranchChanges(cwd)) { + const branch = yield* workflow.currentBranch(cwd); + if (branch === cached.local.value.refName) return cached.local.value; + } yield* workflow.invalidateLocalStatus(cwd); - return yield* updateCachedLocalStatus(cwd, yield* workflow.localStatus({ cwd }), { - publish: true, - }); + return yield* loadLocalStatus(cwd, { publish: true }); } return yield* loadLocalStatus(cwd); }, @@ -403,6 +449,7 @@ export const make = Effect.gen(function* () { "VcsStatusBroadcaster.getStatus", )(function* (input) { const cwd = yield* withFileSystem(normalizeCwd(input.cwd)); + yield* acquireBranchChangesDemand(cwd); yield* getOrLoadLocalStatus(cwd); const cached = yield* getCachedStatus(cwd); if (cached?.local && cached.remote) { @@ -419,14 +466,18 @@ export const make = Effect.gen(function* () { return mergeGitStatusParts(parts.local, parts.remote); }), ); - }); + }, Effect.scoped); // A refresh that started after a request saw every change that request could mean, so queued // callers share it instead of each running git status again. const localRefreshRequests = new Map(); const localRefreshResults = new Map< string, - { readonly startedAt: number; readonly local: VcsStatusLocalResult } + { + readonly startedAt: number; + readonly local: VcsStatusLocalResult; + readonly includesBranchChanges: boolean; + } >(); const refreshLocalStatusCore = Effect.fn("VcsStatusBroadcaster.refreshLocalStatusCore")( function* (cwd: string) { @@ -436,12 +487,19 @@ export const make = Effect.gen(function* () { cwd, Effect.gen(function* () { const shared = localRefreshResults.get(cwd); - if (shared !== undefined && shared.startedAt >= requestedAt) return shared.local; + if ( + shared !== undefined && + shared.startedAt >= requestedAt && + (shared.includesBranchChanges || !wantsBranchChanges(cwd)) + ) { + return shared.local; + } const startedAt = localRefreshRequests.get(cwd) ?? requestedAt; yield* workflow.invalidateLocalStatus(cwd); - const local = yield* workflow.localStatus({ cwd }); - localRefreshResults.set(cwd, { startedAt, local }); - return yield* updateCachedLocalStatus(cwd, local, { publish: true }); + const observation = ++branchObservation; + const read = yield* readLocalStatus(cwd); + localRefreshResults.set(cwd, { startedAt, ...read }); + return yield* updateCachedLocalStatus(cwd, read, observation, { publish: true }); }), ); }, @@ -585,33 +643,63 @@ export const make = Effect.gen(function* () { ); }); + const refreshRemoteAndLocalTotals = Effect.fn("VcsStatusBroadcaster.refreshRemoteAndLocalTotals")( + function* (cwd: string, policyCwd: string, refsBeforeLocalRead: string | null) { + return yield* withRemoteWriteLock( + cwd, + Effect.gen(function* () { + const refreshUpstream = !Duration.isZero(yield* automaticFetchInterval); + // Bump the PR lookup epoch after earlier remote writers have finished. + yield* workflow.invalidateStatus(cwd); + const remoteRead = yield* readRemoteStatus(cwd, { refreshUpstream }); + // Without a fetch, the cached behind count cannot justify an automatic pull. + const pulled = refreshUpstream + ? yield* maybeAutoPull(cwd, remoteRead, [policyCwd]) + : null; + if (pulled !== null) return mergeGitStatusParts(pulled.local, pulled.remote); + yield* updateCachedRemoteStatus(cwd, remoteRead, { publish: true }); + // A fetch, ours or not, can move the base used for local Changes totals. + if ( + refsBeforeLocalRead === null || + (yield* workflow.readRefSnapshot(cwd)) !== refsBeforeLocalRead + ) { + yield* refreshLocalStatusCore(cwd); + } + const parts = yield* getStatusParts(cwd); + return mergeGitStatusParts(parts.local, parts.remote); + }), + ); + }, + ); + const refreshStatus: VcsStatusBroadcaster["Service"]["refreshStatus"] = Effect.fn( "VcsStatusBroadcaster.refreshStatus", )(function* (rawCwd, options) { const cwd = yield* withFileSystem(normalizeCwd(rawCwd)); + yield* acquireBranchChangesDemand(cwd); + const refsBeforeLocalRead = options?.localOnly ? null : yield* workflow.readRefSnapshot(cwd); yield* refreshLocalStatusCore(cwd); if (options?.localOnly) { // Never cache a remote here: readers take `value: null` as "read and found no PR". const parts = yield* getStatusParts(cwd); return mergeGitStatusParts(parts.local, parts.remote); } - return yield* withRemoteWriteLock( - cwd, - Effect.gen(function* () { - const refreshUpstream = !Duration.isZero(yield* automaticFetchInterval); - // Bump the PR lookup epoch after earlier remote writers have finished. - yield* workflow.invalidateStatus(cwd); - const remoteRead = yield* readRemoteStatus(cwd, { refreshUpstream }); - // Fetch can move the base used for local Changes totals. - yield* refreshLocalStatusCore(cwd); - // Without a fetch, the cached behind count cannot justify an automatic pull. - const pulled = refreshUpstream ? yield* maybeAutoPull(cwd, remoteRead, [rawCwd]) : null; - if (pulled !== null) return mergeGitStatusParts(pulled.local, pulled.remote); - yield* updateCachedRemoteStatus(cwd, remoteRead, { publish: true }); - const parts = yield* getStatusParts(cwd); - return mergeGitStatusParts(parts.local, parts.remote); - }), + return yield* refreshRemoteAndLocalTotals(cwd, rawCwd, refsBeforeLocalRead); + }, Effect.scoped); + + const refreshAfterCheckout: VcsStatusBroadcaster["Service"]["refreshAfterCheckout"] = Effect.fn( + "VcsStatusBroadcaster.refreshAfterCheckout", + )(function* (rawCwd) { + const cwd = yield* withFileSystem(normalizeCwd(rawCwd)); + const localDone = yield* Deferred.make(); + // Owned by the broadcaster so a client disconnecting mid-switch still leaves status correct. + yield* refreshLocalStatusCore(cwd).pipe( + Effect.ensuring(Deferred.succeed(localDone, undefined)), + Effect.andThen(refreshRemoteAndLocalTotals(cwd, rawCwd, null)), + Effect.ignoreCause({ log: true }), + Effect.forkIn(broadcasterScope), ); + yield* Deferred.await(localDone); }); const refreshPullRequestStatus: VcsStatusBroadcaster["Service"]["refreshPullRequestStatus"] = @@ -799,7 +887,66 @@ export const make = Effect.gen(function* () { } }); - // Serialize branch publication and cache invalidation with detailed reads and refreshes. + // A request made during a refresh needs a read that starts after it, so overlapping requests + // share one follow-up. Keyed by raw cwd because auto-pull policy depends on it. + // Missing means idle; null means running with no follow-up queued. + type SharedRefresh = Deferred.Deferred; + const queuedRefreshesRef = yield* Ref.make(new Map()); + + const runQueuedRefreshes = Effect.fn("VcsStatusBroadcaster.runQueuedRefreshes")(function* ( + rawCwd: string, + first: SharedRefresh, + ) { + let next: SharedRefresh | null = first; + while (next !== null) { + yield* Deferred.into(refreshStatus(rawCwd), next); + next = yield* Ref.modify(queuedRefreshesRef, (queued) => { + const followUp = queued.get(rawCwd) ?? null; + const nextQueued = new Map(queued); + if (followUp === null) { + nextQueued.delete(rawCwd); + } else { + nextQueued.set(rawCwd, null); + } + return [followUp, nextQueued] as const; + }); + } + }); + + const shareQueuedRefreshStatus: VcsStatusBroadcaster["Service"]["refreshStatus"] = Effect.fn( + "VcsStatusBroadcaster.shareQueuedRefreshStatus", + )(function* (rawCwd, options) { + if (options?.localOnly) return yield* refreshStatus(rawCwd, options); + const requested = yield* Deferred.make(); + // Registering and forking together keeps an interrupt from leaving an entry with no worker. + const shared = yield* Effect.uninterruptible( + Effect.gen(function* () { + const followUp = yield* Ref.modify( + queuedRefreshesRef, + (queued): readonly [SharedRefresh | undefined, Map] => { + if (!queued.has(rawCwd)) return [undefined, new Map(queued).set(rawCwd, null)]; + const followUp = queued.get(rawCwd) ?? requested; + return [followUp, new Map(queued).set(rawCwd, followUp)]; + }, + ); + if (followUp !== undefined) return followUp; + // The broadcaster scope owns the work, so one caller's interrupt does not cancel it for the rest. + yield* runQueuedRefreshes(rawCwd, requested).pipe(Effect.forkIn(broadcasterScope)); + return requested; + }), + ); + return yield* Deferred.await(shared); + }); + + const readHeadBranch = Effect.fn("VcsStatusBroadcaster.readHeadBranch")(function* (cwd: string) { + const observation = ++branchObservation; + const refName = yield* workflow.currentBranch(cwd); + // A null branch is a detached HEAD or a checkout that is gone, such as a removed worktree. + const isRepo = refName !== null || (yield* workflow.repositoryRoot(cwd)) !== null; + return { observation, refName, isRepo }; + }); + + // Keep cache invalidation serialized with detailed reads and refreshes. const refreshBranchStatusCore = Effect.fn("VcsStatusBroadcaster.refreshBranchStatusCore")( (cwd: string) => localRefreshLocks.withLock( @@ -807,15 +954,11 @@ export const make = Effect.gen(function* () { Effect.gen(function* () { if (detailedSubscribers.has(cwd)) { const local = yield* getOrLoadLocalStatusUnlocked(cwd); - return { - refName: local.refName, - isRepo: local.isRepo, - revision: branches.get(cwd)?.revision ?? 0, - }; + return ( + branches.get(cwd) ?? { refName: local.refName, isRepo: local.isRepo, revision: 0 } + ); } - const refName = yield* workflow.currentBranch(cwd); - // A null branch is a detached HEAD or a checkout that is gone, such as a removed worktree. - const isRepo = refName !== null || (yield* workflow.repositoryRoot(cwd)) !== null; + const { observation, refName, isRepo } = yield* readHeadBranch(cwd); // HEAD cannot certify working-tree totals, even when it has not moved. localRefreshResults.delete(cwd); yield* workflow.invalidateLocalStatus(cwd); @@ -824,34 +967,49 @@ export const make = Effect.gen(function* () { return previous ? new Map(cache).set(cwd, { ...previous, local: null }) : cache; }); if (detailedSubscribers.has(cwd)) { - const local = yield* workflow.localStatus({ cwd }); - yield* updateCachedLocalStatus(cwd, local, { publish: true }); - return { - refName: local.refName, - isRepo: local.isRepo, - revision: branches.get(cwd)?.revision ?? 0, - }; + const local = yield* loadLocalStatus(cwd, { publish: true }); + return ( + branches.get(cwd) ?? { refName: local.refName, isRepo: local.isRepo, revision: 0 } + ); } - yield* publishBranch(cwd, { refName, isRepo }); - return { refName, isRepo, revision: branches.get(cwd)?.revision ?? 0 }; + yield* publishBranch(cwd, { refName, isRepo }, observation); + return branches.get(cwd) ?? { refName, isRepo, revision: 0 }; }), ), ); // A checkout outside T3 Code, such as `git switch` in the terminal, calls no refresh RPC. // Rereading remote too, without a fetch, sends the new branch with its own PR and counts. - const refreshCheckedOutRef = Effect.fn("VcsStatusBroadcaster.refreshCheckedOutRef")(function* ( + const publishCheckedOutRef = Effect.fn("VcsStatusBroadcaster.publishCheckedOutRef")(function* ( cwd: string, ) { - if (!detailedSubscribers.has(cwd)) { - yield* refreshBranchStatusCore(cwd); - } else if ((yield* getCachedStatus(cwd))?.remote) { - yield* refreshRemoteStatus(cwd, { refreshUpstream: false }); - } else { - yield* refreshLocalStatusCore(cwd); - } + if (!detailedSubscribers.has(cwd)) return yield* refreshBranchStatusCore(cwd); + const { observation, refName, isRepo } = yield* readHeadBranch(cwd); + yield* publishBranch(cwd, { refName, isRepo }, observation); }); + const reconcileCheckedOutRef = Effect.fn("VcsStatusBroadcaster.reconcileCheckedOutRef")( + function* (cwd: string) { + if (!detailedSubscribers.has(cwd)) return; + if ((yield* getCachedStatus(cwd))?.remote) { + yield* refreshRemoteStatus(cwd, { refreshUpstream: false }); + } else { + yield* refreshLocalStatusCore(cwd); + } + }, + ); + + const retryCheckedOutRef = (cwd: string, refresh: Effect.Effect) => + refresh.pipe( + Effect.retry(CHECKOUT_REFRESH_RETRY), + Effect.catchCause((cause) => + Effect.logWarning("VCS checked-out ref refresh failed", { + cwdLength: cwd.length, + ...remoteRefreshFailureDiagnostics(cause), + }), + ), + ); + // Runs for the subscription lifetime, so tracing would leave a span open indefinitely. const watchHead = Effect.fnUntraced(function* (cwd: string, ready: Deferred.Deferred) { const headPath = yield* workflow.headPath(cwd); @@ -872,17 +1030,10 @@ export const make = Effect.gen(function* () { Stream.filter((event) => path.basename(event.path) === "HEAD"), // A rebase moves HEAD many times; refresh once for whatever arrived during a refresh. Stream.buffer({ capacity: 1, strategy: "sliding" }), - Stream.runForEach(() => - refreshCheckedOutRef(cwd).pipe( - Effect.retry(CHECKOUT_REFRESH_RETRY), - Effect.catchCause((cause) => - Effect.logWarning("VCS checked-out ref refresh failed", { - cwdLength: cwd.length, - ...remoteRefreshFailureDiagnostics(cause), - }), - ), - ), - ), + Stream.mapEffect(() => retryCheckedOutRef(cwd, publishCheckedOutRef(cwd))), + // Detailed status waits behind its own reads; the branch keeps up on the stage above. + Stream.buffer({ capacity: 1, strategy: "sliding" }), + Stream.runForEach(() => retryCheckedOutRef(cwd, reconcileCheckedOutRef(cwd))), ); }); @@ -958,6 +1109,7 @@ export const make = Effect.gen(function* () { else detailedSubscribers.set(cwd, remaining); }), ); + if (input.includeRemote !== false) yield* acquireBranchChangesDemand(cwd); const subscription = yield* PubSub.subscribe(changesPubSub); // Watch before reading so a checkout in between still publishes. yield* Effect.acquireRelease(retainHeadWatcher(cwd), () => releaseHeadWatcher(cwd)); @@ -993,8 +1145,21 @@ export const make = Effect.gen(function* () { const refreshBranchStatus: VcsStatusBroadcaster["Service"]["refreshBranchStatus"] = Effect.fn( "VcsStatusBroadcaster.refreshBranchStatus", )(function* (input) { - const { refName, isRepo } = yield* refreshBranchStatusCore( - yield* withFileSystem(normalizeCwd(input.cwd)), + const cwd = yield* withFileSystem(normalizeCwd(input.cwd)); + if (!detailedSubscribers.has(cwd)) { + const { refName, isRepo } = yield* refreshBranchStatusCore(cwd); + return { refName, isRepo }; + } + const { observation, refName, isRepo } = yield* readHeadBranch(cwd); + yield* publishBranch(cwd, { refName, isRepo }, observation); + yield* refreshBranchStatusCore(cwd).pipe( + Effect.catchCause((cause) => + Effect.logWarning("VCS branch refresh failed", { + cwdLength: cwd.length, + ...remoteRefreshFailureDiagnostics(cause), + }), + ), + Effect.forkIn(broadcasterScope), ); return { refName, isRepo }; }); @@ -1028,7 +1193,8 @@ export const make = Effect.gen(function* () { ), getStatus, refreshLocalStatus, - refreshStatus, + refreshAfterCheckout, + refreshStatus: shareQueuedRefreshStatus, refreshPullRequestStatus, streamStatus, }); diff --git a/apps/server/src/vcs/worktreeCheckoutCache.ts b/apps/server/src/vcs/worktreeCheckoutCache.ts index a72d6d859fc54..8a51189fe6d69 100644 --- a/apps/server/src/vcs/worktreeCheckoutCache.ts +++ b/apps/server/src/vcs/worktreeCheckoutCache.ts @@ -511,6 +511,7 @@ export const makeWorktreeCheckoutCache = Effect.fn("WorktreeCheckoutCache.make") readonly worktreePath: string; readonly refName: string; readonly checkoutWorkers: string; + readonly lockReason?: string | undefined; }) { if (!(yield* gitSupported)) return false; const worktreePath = path.resolve(input.cwd, input.worktreePath); @@ -613,13 +614,14 @@ export const makeWorktreeCheckoutCache = Effect.fn("WorktreeCheckoutCache.make") if (moved === null || (yield* fileSystem.exists(target.indexHook))) return null; return target; }); - const rollback = git("rollback", input.cwd, [ - "worktree", - "remove", - "--force", - "--", - worktreePath, - ]).pipe( + const rollback = ( + input.lockReason + ? git("unlockRollback", input.cwd, ["worktree", "unlock", "--", worktreePath]) + : Effect.void + ).pipe( + Effect.andThen( + git("rollback", input.cwd, ["worktree", "remove", "--force", "--", worktreePath]), + ), Effect.flatMap((removed) => removed === null ? Effect.logWarning("could not remove a partially created worktree", { worktreePath }) @@ -633,6 +635,7 @@ export const makeWorktreeCheckoutCache = Effect.fn("WorktreeCheckoutCache.make") "worktree", "add", "--no-checkout", + ...(input.lockReason ? ["--lock", "--reason", input.lockReason] : []), "--", worktreePath, input.refName, diff --git a/apps/server/src/vcs/worktreeRemoval.ts b/apps/server/src/vcs/worktreeRemoval.ts index f1140cdcc3a48..b48994873b282 100644 --- a/apps/server/src/vcs/worktreeRemoval.ts +++ b/apps/server/src/vcs/worktreeRemoval.ts @@ -9,6 +9,42 @@ import type * as GitVcsDriver from "./GitVcsDriver.ts"; const INDEX_LIST_MAX_BYTES = 64 * 1024 * 1024; +/** + * git-annex replaces a linked worktree's `.git` file with a symlink to its admin + * directory, so its symlinks into `.git/annex` resolve. Git then refuses to remove it. + */ +const annexedAdminDir = Effect.fn("WorktreeRemoval.annexedAdminDir")( + function* (worktreePath: string) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const adminDir = path.resolve( + worktreePath, + yield* fileSystem.readLink(path.join(worktreePath, ".git")), + ); + const pointsBackTo = path.dirname( + path.resolve( + adminDir, + (yield* fileSystem.readFileString(path.join(adminDir, "gitdir"))).trim(), + ), + ); + return (yield* fileSystem.realPath(pointsBackTo)) === (yield* fileSystem.realPath(worktreePath)) + ? Option.some(adminDir) + : Option.none(); + }, + Effect.orElseSucceed(() => Option.none()), +); + +export const isLinkedWorktree = Effect.fn("WorktreeRemoval.isLinkedWorktree")(function* ( + worktreePath: string, +) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + return ( + (yield* fileSystem.stat(path.join(worktreePath, ".git"))).type === "File" || + Option.isSome(yield* annexedAdminDir(worktreePath)) + ); +}); + /** * Git for Windows before 2.54 follows NTFS junctions; newer Git leaves them behind. * On Windows, mirror Git's removal checks, then delete with Node's `rm`, which @@ -33,6 +69,39 @@ export const removeLinkedWorktree = Effect.fn("WorktreeRemoval.removeLinkedWorkt // Git refuses to remove a worktree with untracked files unless forced, but // its check honors `status.showUntrackedFiles=no` and would delete them. const statusConfig = ["-c", "status.showUntrackedFiles=normal"]; + // Git's own refusals, worded the way it prints them. + const refuse = (message: string, cause?: unknown) => + new GitCommandError({ + operation: input.operation, + command: "git worktree remove", + cwd: input.cwd, + detail: `fatal: ${message}`, + ...(cause === undefined ? {} : { cause }), + }); + const annexed = yield* annexedAdminDir(worktreePath); + if (Option.isSome(annexed)) { + const adminDir = annexed.value; + const writeGitFile = fileSystem.writeFileString(gitFile, `gitdir: ${adminDir}\n`); + // A kept worktree's annexed files resolve only through the symlink; a gitdir + // file is the fallback when it cannot be recreated. + yield* Effect.addFinalizer(() => + fileSystem.exists(worktreePath).pipe( + Effect.flatMap((kept) => + kept + ? fileSystem.remove(gitFile, { force: true }).pipe( + Effect.andThen(fileSystem.symlink(adminDir, gitFile)), + Effect.catch(() => writeGitFile), + ) + : Effect.void, + ), + Effect.ignore, + ), + ); + yield* fileSystem.remove(gitFile).pipe( + Effect.andThen(writeGitFile), + Effect.mapError((cause) => refuse(`failed to replace '${gitFile}'`, cause)), + ); + } const linked = (yield* HostProcess.isWindows) && (yield* fileSystem.stat(gitFile).pipe( @@ -50,15 +119,6 @@ export const removeLinkedWorktree = Effect.fn("WorktreeRemoval.removeLinkedWorkt ]); } - // Git's own refusals, worded the way it prints them. - const refuse = (message: string, cause?: unknown) => - new GitCommandError({ - operation: input.operation, - command: "git worktree remove", - cwd: input.cwd, - detail: `fatal: ${message}`, - ...(cause === undefined ? {} : { cause }), - }); const gitOutput = (cwd: string, args: ReadonlyArray, env?: NodeJS.ProcessEnv) => input .execute({ @@ -153,4 +213,4 @@ export const removeLinkedWorktree = Effect.fn("WorktreeRemoval.removeLinkedWorkt ); // The folder is gone, so Git only drops the worktree's metadata. return yield* input.run(["worktree", "remove", "--force", "--force", "--", worktreePath]); -}); +}, Effect.scoped); diff --git a/apps/server/src/workspace/WorkspaceEntries.test.ts b/apps/server/src/workspace/WorkspaceEntries.test.ts index 521e127779984..1c50b94a7cbcc 100644 --- a/apps/server/src/workspace/WorkspaceEntries.test.ts +++ b/apps/server/src/workspace/WorkspaceEntries.test.ts @@ -688,6 +688,13 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { yield* git(cwd, ["commit", "-m", "init"], env); yield* git(cwd, ["worktree", "add", "-b", "x", ".worktrees/x"]); yield* writeTextFile(cwd, ".worktrees/x/src/foo.py", "print(1)"); + const expected = [".worktrees/x/src/foo.py"]; + // Windows file names cannot contain a newline. + if ((yield* HostProcess.Platform) !== "win32") { + yield* git(cwd, ["worktree", "add", "--detach", ".worktrees/line\nname"]); + yield* writeTextFile(cwd, ".worktrees/line\nname/src/foo.py", "print(2)"); + expected.unshift(".worktrees/line\nname/src/foo.py"); + } const exact = yield* searchWorkspaceEntries({ cwd, @@ -695,10 +702,14 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { limit: 25, kind: "file", }); - expect(exact.entries).toEqual([{ path: ".worktrees/x/src/foo.py", kind: "file" }]); + expect(exact.entries.toSorted((a, b) => (a.path < b.path ? -1 : 1))).toEqual( + expected.map((path) => ({ path, kind: "file" })), + ); const partial = yield* searchWorkspaceEntries({ cwd, query: "src/fo", limit: 25 }); - expect(partial.entries.map((entry) => entry.path)).not.toContain(".worktrees/x/src/foo.py"); + for (const path of expected) { + expect(partial.entries.map((entry) => entry.path)).not.toContain(path); + } }), ); }); diff --git a/apps/server/src/workspace/WorkspaceEntries.ts b/apps/server/src/workspace/WorkspaceEntries.ts index 43c8fae4dfcaf..51b1720126536 100644 --- a/apps/server/src/workspace/WorkspaceEntries.ts +++ b/apps/server/src/workspace/WorkspaceEntries.ts @@ -307,23 +307,30 @@ export const make = Effect.gen(function* () { ) { return []; } - const listed = yield* vcsProcess - .run({ - operation: "WorkspaceEntries.listWorktrees", - command: "git", - args: ["worktree", "list", "--porcelain"], - cwd, - allowNonZeroExit: true, - timeoutMs: 5_000, - }) - .pipe(Effect.orElseSucceed(() => undefined)); - if (!listed || listed.exitCode !== 0) return []; + const listWorktrees = (args: ReadonlyArray) => + vcsProcess + .run({ + operation: "WorkspaceEntries.listWorktrees", + command: "git", + args: ["worktree", "list", "--porcelain", ...args], + cwd, + allowNonZeroExit: true, + timeoutMs: 5_000, + }) + .pipe( + Effect.map((result) => (result.exitCode === 0 ? result.stdout : undefined)), + Effect.orElseSucceed(() => undefined), + ); + // Only NUL framing keeps a newline in a worktree path. Pre-2.36 Git rejects `-z`. + const listed = + (yield* listWorktrees(["-z"])) ?? (yield* listWorktrees([]))?.replaceAll("\n", "\0"); + if (listed === undefined) return []; const realCwd = yield* Effect.promise(() => NodeFSP.realpath(cwd).catch(() => cwd)); - const worktreeRoots = listed.stdout.split(/\n\s*\n/).flatMap((record) => { - const lines = record.split("\n"); - const worktree = lines.find((line) => line.startsWith("worktree ")); + const worktreeRoots = listed.split("\0\0").flatMap((record) => { + const fields = record.split("\0"); + const worktree = fields.find((field) => field.startsWith("worktree ")); // A `prunable` record's directory is gone; another checkout may now sit there. - if (worktree === undefined || lines.some((line) => line.startsWith("prunable"))) return []; + if (worktree === undefined || fields.some((field) => field.startsWith("prunable"))) return []; const root = worktree.slice("worktree ".length); const relativeRoot = path.relative(realCwd, root); return relativeRoot === "" || diff --git a/apps/server/src/workspace/WorkspaceFileSystem.test.ts b/apps/server/src/workspace/WorkspaceFileSystem.test.ts index dffb0701d68aa..bc22117ba74ab 100644 --- a/apps/server/src/workspace/WorkspaceFileSystem.test.ts +++ b/apps/server/src/workspace/WorkspaceFileSystem.test.ts @@ -12,6 +12,7 @@ import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as ServerConfig from "../config.ts"; +import { makeFakeWslHost } from "../testUtils/fakeWsl.ts"; import * as VcsDriverRegistry from "../vcs/VcsDriverRegistry.ts"; import * as VcsProcess from "../vcs/VcsProcess.ts"; import * as KnownWorkspaces from "./KnownWorkspaces.ts"; @@ -96,6 +97,14 @@ const writeTextFile = Effect.fn("writeTextFile")(function* ( yield* fileSystem.writeFileString(absolutePath, contents).pipe(Effect.orDie); }); +const makeWslHost = Effect.gen(function* () { + const { mountRoot, underWsl } = yield* makeFakeWslHost; + const service = yield* underWsl( + WorkspaceFileSystem.make.pipe(Effect.provide(layerKnownWorkspacesTest)), + ); + return { mountRoot, service, underWsl }; +}); + it.layer(layerTest, { excludeTestServices: true })("WorkspaceFileSystemLive", (it) => { describe("getMetadata", () => { it.effect( @@ -181,6 +190,31 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceFileSystemLive", (i expect(statted).toEqual([]); }), ); + + it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( + "reports a Windows drive path under WSL as the file it maps to", + () => + Effect.gen(function* () { + const { mountRoot, service, underWsl } = yield* makeWslHost; + yield* writeTextFile(mountRoot, "c/Users/me/Documents/notes.md", "# Notes\n"); + + const { entries } = yield* underWsl( + service.getMetadata({ + paths: [ + "C:\\Users\\me\\Documents\\notes.md", + "C:/Users/me/Documents/notes.md", + "C:/Users/me/Documents/notes\u0000.md", + ], + }), + ); + + expect(entries).toEqual([ + { kind: "file", byteLength: 8 }, + { kind: "file", byteLength: 8 }, + null, + ]); + }), + ); }); describe("readFile", () => { @@ -202,6 +236,18 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceFileSystemLive", (i truncated: false, }); + const bomContents = "\uFEFFexport const answer = 42;\n"; + yield* writeTextFile(cwd, "bom.ts", bomContents); + const bomRead = yield* workspaceFileSystem.readFile({ cwd, relativePath: "bom.ts" }); + yield* workspaceFileSystem.writeFile({ + cwd, + relativePath: "bom.ts", + contents: bomRead.contents.replace("42", "43"), + }); + const fs = yield* FileSystem.FileSystem; + const saved = yield* fs.readFile(`${cwd}/bom.ts`); + expect([...saved]).toEqual([...new TextEncoder().encode(bomContents.replace("42", "43"))]); + yield* writeTextFile(cwd, "report.txt", "ordinary file"); for (const relativePath of whitespaceFilePaths) { yield* writeTextFile(cwd, relativePath, "whitespace file"); @@ -274,25 +320,27 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceFileSystemLive", (i }), ); - it.effect("reads ~/ paths from the server home outside the workspace", () => - Effect.gen(function* () { - const workspaceFileSystem = yield* WorkspaceFileSystem.WorkspaceFileSystem; - const cwd = yield* makeTempDir; - const home = yield* makeTempDir; - yield* writeTextFile(home, ".claude/CLAUDE.md", "# Notes\n"); - yield* writeTextFile(cwd, "~/.claude/CLAUDE.md", "Workspace decoy\n"); + it.effect.each(["~/.claude/CLAUDE.md", "~\\.claude/CLAUDE.md"])( + "reads %s from the server home outside the workspace", + (relativePath) => + Effect.gen(function* () { + const workspaceFileSystem = yield* WorkspaceFileSystem.WorkspaceFileSystem; + const cwd = yield* makeTempDir; + const home = yield* makeTempDir; + yield* writeTextFile(home, ".claude/CLAUDE.md", "# Notes\n"); + yield* writeTextFile(cwd, "~/.claude/CLAUDE.md", "Workspace decoy\n"); - const result = yield* workspaceFileSystem - .readFile({ cwd, relativePath: "~/.claude/CLAUDE.md" }) - .pipe(Effect.provideService(HostProcess.HomeDirectory, home)); + const result = yield* workspaceFileSystem + .readFile({ cwd, relativePath }) + .pipe(Effect.provideService(HostProcess.HomeDirectory, home)); - expect(result).toEqual({ - relativePath: "~/.claude/CLAUDE.md", - contents: "# Notes\n", - byteLength: 8, - truncated: false, - }); - }), + expect(result).toEqual({ + relativePath, + contents: "# Notes\n", + byteLength: 8, + truncated: false, + }); + }), ); it.effect("does not read server state files", () => @@ -351,6 +399,24 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceFileSystemLive", (i }), ); + it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( + "reads a Windows drive path under WSL from where wslpath mounts it", + () => + Effect.gen(function* () { + const { mountRoot, service, underWsl } = yield* makeWslHost; + const cwd = yield* makeTempDir; + yield* writeTextFile(mountRoot, "c/Users/me/Documents/notes.md", "# Notes\n"); + + for (const relativePath of [ + "C:\\Users\\me\\Documents\\notes.md", + "C:/Users/me/Documents/notes.md", + ]) { + const result = yield* underWsl(service.readFile({ cwd, relativePath })); + expect(result).toMatchObject({ contents: "# Notes\n", truncated: false }); + } + }), + ); + it.effect("refuses a network path outside the workspace", () => Effect.gen(function* () { const workspaceFileSystem = yield* WorkspaceFileSystem.WorkspaceFileSystem; @@ -478,14 +544,17 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceFileSystemLive", (i }), ); - it.effect("rejects binary files without leaking their contents into the error", () => + it.effect.each([ + { name: "a NUL byte", bytes: [0x61, 0, 0x62] }, + { name: "invalid UTF-8", bytes: [0x63, 0x61, 0x66, 0xe9] }, + ])("rejects files with $name without leaking their contents", ({ bytes }) => Effect.gen(function* () { const workspaceFileSystem = yield* WorkspaceFileSystem.WorkspaceFileSystem; const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; const cwd = yield* makeTempDir; const absolutePath = path.join(cwd, "asset.bin"); - yield* fileSystem.writeFile(absolutePath, Uint8Array.from([0x61, 0, 0x62])); + yield* fileSystem.writeFile(absolutePath, Uint8Array.from(bytes)); const error = yield* workspaceFileSystem .readFile({ cwd, relativePath: "asset.bin" }) diff --git a/apps/server/src/workspace/WorkspaceFileSystem.ts b/apps/server/src/workspace/WorkspaceFileSystem.ts index 05409c6616ff0..494472e4ce68d 100644 --- a/apps/server/src/workspace/WorkspaceFileSystem.ts +++ b/apps/server/src/workspace/WorkspaceFileSystem.ts @@ -19,6 +19,7 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import * as Schema from "effect/Schema"; import { makeIsServerInternalPath } from "../auth/serverInternalPaths.ts"; @@ -26,6 +27,7 @@ import * as KnownWorkspaces from "./KnownWorkspaces.ts"; import * as WorkspaceEntries from "./WorkspaceEntries.ts"; import * as WorkspacePaths from "./WorkspacePaths.ts"; import { fileHeaderMimeType } from "./fileHeaderMimeType.ts"; +import { toWslDrivePath } from "./wslDrivePath.ts"; const PROJECT_READ_FILE_MAX_BYTES = 1024 * 1024; @@ -133,6 +135,27 @@ export class WorkspaceFileSystem extends Context.Service< } >()("t3/workspace/WorkspaceFileSystem") {} +const UTF8_BOM = "\uFEFF"; + +async function startsWithUtf8Bom(filePath: string): Promise { + const handle = await NodeFSP.open( + filePath, + // O_NONBLOCK keeps a FIFO with no writer from hanging the save. + NodeFS.constants.O_RDONLY | + (NodeFS.constants.O_NOFOLLOW ?? 0) | + (NodeFS.constants.O_NONBLOCK ?? 0), + ).catch(() => null); + if (!handle) return false; + try { + const { bytesRead, buffer } = await handle.read(Buffer.alloc(3), 0, 3, 0); + return bytesRead === 3 && buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf; + } catch { + return false; + } finally { + await handle.close(); + } +} + /** @public */ export const make = Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; @@ -141,9 +164,18 @@ export const make = Effect.gen(function* () { const knownWorkspaces = yield* KnownWorkspaces.KnownWorkspaces; const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; const isServerInternalPath = yield* makeIsServerInternalPath; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + + const toHostPath = (requestedPath: string) => + toWslDrivePath(requestedPath).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); const metadataForPath = Effect.fnUntraced(function* (requestedPath: string) { - const expandedPath = expandHomePath(requestedPath, yield* HostProcess.HomeDirectory); + const expandedPath = expandHomePath( + yield* toHostPath(requestedPath), + yield* HostProcess.HomeDirectory, + ); // Chips ask as soon as an agent's message renders, and touching a network // path makes Windows sign in to its host with the user's NTLM credentials. if (!path.isAbsolute(expandedPath) || isNetworkPath(expandedPath)) return null; @@ -228,9 +260,10 @@ export const make = Effect.gen(function* () { input: ProjectReadFileInput, ) { const requestedPath = input.relativePath; - const hostPath = requestedPath.startsWith("~/") - ? expandHomePath(requestedPath, yield* HostProcess.HomeDirectory) - : requestedPath; + const hostPath = + requestedPath.startsWith("~/") || requestedPath.startsWith("~\\") + ? expandHomePath(requestedPath, yield* HostProcess.HomeDirectory) + : yield* toHostPath(requestedPath); if ( isNetworkPath(hostPath) && !KnownWorkspaces.isWithinRoot(path, path.resolve(hostPath), path.resolve(input.cwd)) @@ -424,22 +457,22 @@ export const make = Effect.gen(function* () { offset += bytesRead; } const fileBytes = buffer.subarray(0, offset); - if (fileBytes.includes(0)) { - return yield* new WorkspaceBinaryFileError({ + const truncated = stat.size > PROJECT_READ_FILE_MAX_BYTES; + const binaryFileError = () => + new WorkspaceBinaryFileError({ workspaceRoot: input.cwd, relativePath: input.relativePath, resolvedPath: realTargetPath, }); - } - - const truncated = stat.size > PROJECT_READ_FILE_MAX_BYTES; - return { - relativePath: target.relativePath, - // Streaming decode drops a character the byte limit split, instead of ending in U+FFFD. - contents: new TextDecoder("utf-8").decode(fileBytes, { stream: truncated }), - byteLength: stat.size, - truncated, - }; + if (fileBytes.includes(0)) return yield* binaryFileError(); + // Fatal decoding rejects invalid UTF-8, since U+FFFD contents would corrupt the file on save. + // Streaming decode drops a character the byte limit split, instead of rejecting it. + const contents = yield* Effect.try({ + try: () => + new TextDecoder("utf-8", { fatal: true }).decode(fileBytes, { stream: truncated }), + catch: binaryFileError, + }); + return { relativePath: target.relativePath, contents, byteLength: stat.size, truncated }; }), (handle) => handle === null @@ -550,6 +583,12 @@ export const make = Effect.gen(function* () { const replacesFile = yield* fileSystem .exists(realTargetPath) .pipe(Effect.orElseSucceed(() => false)); + // readFile decodes the BOM away, so an editor save would otherwise drop it from the file. + const keepsBom = + replacesFile && + !input.contents.startsWith(UTF8_BOM) && + (yield* Effect.promise(() => startsWithUtf8Bom(realTargetPath))); + const contents = keepsBom ? UTF8_BOM + input.contents : input.contents; yield* fileSystem.makeDirectory(path.dirname(realTargetPath), { recursive: true }).pipe( Effect.mapError( @@ -567,7 +606,7 @@ export const make = Effect.gen(function* () { yield* Effect.tryPromise({ // A symlink swapped in after the check is not followed. Windows lacks O_NOFOLLOW. try: () => - NodeFSP.writeFile(realTargetPath, input.contents, { + NodeFSP.writeFile(realTargetPath, contents, { flag: NodeFS.constants.O_WRONLY | NodeFS.constants.O_CREAT | diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts index 45102e4c1cfea..6c4e873f12b47 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts @@ -9,6 +9,7 @@ import { import * as NodeServices from "@effect/platform-node/NodeServices"; import { afterEach, expect, it } from "@effect/vitest"; import * as HostProcess from "@t3tools/shared/HostProcess"; +import { PROJECT_FAVICON_EXTENSIONS, isProjectFaviconPath } from "@t3tools/shared/projectFavicon"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; @@ -40,44 +41,98 @@ function fileItem(relativePath: string): FileItem { }; } -it.effect("filters image searches before applying the result limit", () => - Effect.scoped( - Effect.gen(function* () { - const items = [ - ...Array.from({ length: 200 }, (_, index) => fileItem(`src/file-${index}.ts`)), - fileItem("public/icon.svg"), - ]; - const fileSearch = vi.fn(async () => ({ - ok: true as const, - value: { - items, - scores: [], - totalMatched: items.length, - totalFiles: items.length, - }, - })); - const finder = { - destroy: vi.fn(), - waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), - getScanProgressAsync: async () => ({ - ok: true as const, - value: { isScanning: false }, - }), - watch: () => ({ ok: true as const, value: () => undefined }), - fileSearchAsync: fileSearch, - } as unknown as FileFinder; - vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); - - const searchIndex = yield* WorkspaceSearchIndex.make("/workspace/project"); - const resultWithoutKind = yield* searchIndex.search("", 200, undefined, true); - const resultWithDirectoryKind = yield* searchIndex.search("", 200, "directory", true); - - expect(resultWithoutKind.entries).toEqual([{ kind: "file", path: "public/icon.svg" }]); - expect(resultWithDirectoryKind.entries).toEqual([{ kind: "file", path: "public/icon.svg" }]); - expect(fileSearch).toHaveBeenCalledTimes(2); - expect(fileSearch).toHaveBeenCalledWith("", { pageSize: 25_002 }); - }), - ).pipe(Effect.provide(testLayer)), +it.live("image searches match the image subset of a file search without fetching every file", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const vcsProcess = yield* VcsProcess.VcsProcess; + const writeFiles = Effect.fn(function* (root: string, relativePaths: ReadonlyArray) { + for (const relativePath of relativePaths) { + yield* fileSystem.makeDirectory(path.dirname(path.join(root, relativePath)), { + recursive: true, + }); + yield* fileSystem.writeFileString(path.join(root, relativePath), ""); + yield* fileSystem.utimes(path.join(root, relativePath), 0, 0); + } + }); + + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-image-search-" }); + yield* vcsProcess.run({ operation: "test.gitInit", command: "git", args: ["init"], cwd }); + yield* fileSystem.writeFileString(path.join(cwd, ".git/info/exclude"), "hidden/\n"); + yield* writeFiles(cwd, [ + ...Array.from({ length: 220 }, (_, index) => `assets/source-${index}.ts`), + ...PROJECT_FAVICON_EXTENSIONS.flatMap((extension) => [ + `assets/logo${extension}`, + `other/logo${extension.toUpperCase()}`, + ]), + "assets/logo.ts", + "assets/logotype.png", + "assetss/logotype.png", + "assets/images/logo.PNG", + "assets/foo{bar}.png", + "assets/[id].png", + "hidden/logo.png", + ]); + + const fileSearch = vi.spyOn(FileFinder.prototype, "fileSearchAsync"); + const searchIndex = yield* WorkspaceSearchIndex.make(cwd); + yield* searchIndex.search("", 200, undefined, true); + const nativeResult = yield* Effect.promise( + (): ReturnType => fileSearch.mock.results[0]?.value, + ); + if (!nativeResult.ok) return yield* Effect.die(nativeResult.error); + expect(nativeResult.value.items.length).toBeLessThanOrEqual(201); + + // The image glob must not change fff's parse of fuzzy paths, globs, negations or locations. + for (const query of [ + "", + "logo", + "lgo", + "*.png", + "!*.png", + "assets/**", + "!assets/", + "assets*/", + "/assets", + "assets/", + "logo:12", + "/assets/logotype.png", + "assets/images/", + "logo *.svg !other", + "foo{bar}.png", + "[id].png", + "*.{png,svg}", + "hidden/", + ]) { + const ordinary = yield* searchIndex.search(query, 1000, "file"); + const expected = ordinary.entries.filter((entry) => isProjectFaviconPath(entry.path)); + for (const limit of [1, 200]) { + const images = yield* searchIndex.search(query, limit, "directory", true); + expect(images, `query=${query} limit=${limit}`).toEqual({ + entries: expected.slice(0, limit), + truncated: expected.length > limit, + }); + } + } + + // Past 15,000 matches fff drops its filename fallback, which the image glob would otherwise + // reorder: "x/logo/logo.png" must keep its place relative to "a/xxlogo.png". + const largeCwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-image-rank-" }); + yield* writeFiles(largeCwd, [ + ...Array.from({ length: 14_998 }, (_, index) => `b/logo-${index}.png`), + "x/logo/source.ts", + "a/xxlogo.png", + "x/logo/logo.png", + ]); + const largeIndex = yield* WorkspaceSearchIndex.make(largeCwd); + const pairPaths = ["a/xxlogo.png", "x/logo/logo.png"]; + const ordinary = yield* largeIndex.search("logo", 20_000, "file"); + const images = yield* largeIndex.search("logo", 20_000, "file", true); + expect(images.entries).toHaveLength(15_000); + expect(images.entries.filter((entry) => pairPaths.includes(entry.path))).toEqual( + ordinary.entries.filter((entry) => pairPaths.includes(entry.path)), + ); + }).pipe(Effect.scoped, Effect.provide(testLayer)), ); it.effect("fetches past a first page of Git-ignored results", () => diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.ts b/apps/server/src/workspace/WorkspaceSearchIndex.ts index 22e307c182616..8d9798baadfd7 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.ts @@ -32,7 +32,7 @@ import type { ProjectSearchEntriesResult, } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; -import { isProjectFaviconPath } from "@t3tools/shared/projectFavicon"; +import { PROJECT_FAVICON_EXTENSIONS, isProjectFaviconPath } from "@t3tools/shared/projectFavicon"; import { scoreSubsequenceMatch } from "@t3tools/shared/searchRanking"; import * as VcsProcess from "../vcs/VcsProcess.ts"; @@ -42,6 +42,14 @@ import { type IgnoreDecision, type IgnoreFile, parseIgnoreFile } from "./IgnoreP const requireForFff = NodeModule.createRequire(import.meta.url); const { FileFinder } = requireForFff("@ff-labs/fff-node") as typeof import("@ff-labs/fff-node"); +// A glob intersects existing extension constraints; fff ORs bare `*.ext` constraints together. +const IMAGE_SEARCH_CONSTRAINT = `**/*.{${PROJECT_FAVICON_EXTENSIONS.map((extension) => + extension.slice(1).replace(/[a-z]/g, (letter) => `[${letter}${letter.toUpperCase()}]`), +).join(",")}}`; + +// fff 0.11 disables filename fallback above this match count per base/overflow segment. +const FFF_FILENAME_FALLBACK_MAX_MATCHES = 15_000; + const WORKSPACE_INDEX_MAX_ENTRIES = 25_000; const WORKSPACE_INDEX_PAGE_SIZE = WORKSPACE_INDEX_MAX_ENTRIES + 2; const WORKSPACE_INDEX_SCAN_TIMEOUT = "15 seconds"; @@ -175,6 +183,14 @@ function toDirectoryEntry(item: DirItem, hostIsWindows: boolean): ProjectEntry | return normalizedPath ? { path: normalizedPath, kind: "directory" } : null; } +function isLoneFuzzyPath(query: string): boolean { + const trimmed = query.trim(); + return ( + !/\s/u.test(trimmed) && + (trimmed.startsWith("/") || (trimmed.endsWith("/") && !/^[*!\\]/u.test(trimmed))) + ); +} + type SearchPage = { readonly items: ReadonlyArray; readonly totalMatched: number }; function mapFileSearchResult( @@ -841,13 +857,13 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( }); // Grows the first page until hidden entries can no longer crowd visible ones out of it. - const searchVisible = Effect.fn("WorkspaceSearchIndex.searchVisible")(function* ( + const searchVisible = Effect.fn("WorkspaceSearchIndex.searchVisible")(function* ( query: string, pageSize: number, operation: "directorySearch" | "fileSearch" | "mixedSearch", - execute: (pageSize: number) => Promise>>, + execute: (pageSize: number) => Promise & Metadata>>, pathOf: (item: I) => string, - ): Effect.fn.Return, WorkspaceSearchIndexSearchFailed> { + ): Effect.fn.Return & Metadata, WorkspaceSearchIndexSearchFailed> { let size = pageSize; while (true) { const fetchSize = size; @@ -862,6 +878,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( fetchSize >= WORKSPACE_INDEX_PAGE_SIZE ) { return { + ...result, items, totalMatched: result.totalMatched - (result.items.length - items.length), }; @@ -945,17 +962,40 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( ); // Index results that are filtered-out links get dropped in the merge, so fetch past them. const indexLimit = limit + linkPaths.size - symlinks.length; - const pageSize = imageOnly ? WORKSPACE_INDEX_PAGE_SIZE : Math.max(1, indexLimit + 1); + const pageSize = Math.max(1, indexLimit + 1); const merge = (indexed: ProjectSearchEntriesResult) => mergeSymlinkMatches(indexed, linkPaths, symlinks, query, limit); if (kind === "file" || imageOnly) { - const result = yield* searchVisible( - query, + const searchFiles = (fileQuery: string, size: number) => + searchVisible( + query, + size, + "fileSearch", + (fetchSize) => finder.fileSearchAsync(fileQuery, { pageSize: fetchSize }), + (item: FileItem) => item.relativePath, + ); + // Adding a constraint changes fff's lone-path parse; offset pages can skip tied matches. + if (imageOnly && isLoneFuzzyPath(query)) { + const result = yield* searchFiles(query, WORKSPACE_INDEX_PAGE_SIZE); + return merge(mapFileSearchResult(result, indexLimit, hostIsWindows, true)); + } + let result = yield* searchFiles( + imageOnly ? `${query} ${IMAGE_SEARCH_CONSTRAINT}` : query, pageSize, - "fileSearch", - (size) => finder.fileSearchAsync(query, { pageSize: size }), - (item) => item.relativePath, ); + if ( + imageOnly && + query.trim() && + result.totalFiles > FFF_FILENAME_FALLBACK_MAX_MATCHES && + result.totalMatched > 0 + ) { + const unfiltered = yield* runSearch(query, 1, "fileSearch", () => + finder.fileSearchAsync(query, { pageSize: 1 }), + ); + if (unfiltered.totalMatched > FFF_FILENAME_FALLBACK_MAX_MATCHES) { + result = yield* searchFiles(query, WORKSPACE_INDEX_PAGE_SIZE); + } + } return merge(mapFileSearchResult(result, indexLimit, hostIsWindows, imageOnly)); } if (kind === "directory") { diff --git a/apps/server/src/workspace/wslDrivePath.ts b/apps/server/src/workspace/wslDrivePath.ts new file mode 100644 index 0000000000000..188e1c85f45ec --- /dev/null +++ b/apps/server/src/workspace/wslDrivePath.ts @@ -0,0 +1,29 @@ +import * as HostProcess from "@t3tools/shared/HostProcess"; +import { isWindowsDrivePath } from "@t3tools/shared/path"; +import * as Effect from "effect/Effect"; +import * as ChildProcess from "effect/process/ChildProcess"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; + +// WSL mounts Windows drives under a configurable automount root, so only wslpath knows where. +// No WSL env check: servers started by systemd lack WSL_DISTRO_NAME, and off WSL the spawn just fails. +// Untraced: it runs for every chip path, and only drive paths on Linux do any work. +export const toWslDrivePath = Effect.fnUntraced(function* (requestedPath: string) { + // spawn throws on a NUL byte instead of failing. + if (!isWindowsDrivePath(requestedPath) || requestedPath.includes("\0")) return requestedPath; + if ((yield* HostProcess.Platform) !== "linux") return requestedPath; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const output = yield* spawner + .string( + ChildProcess.make("wslpath", ["-u", requestedPath], { + env: yield* HostProcess.Environment, + stdin: "ignore", + stderr: "ignore", + }), + ) + .pipe( + Effect.timeout("1 second"), + Effect.orElseSucceed(() => ""), + ); + const linuxPath = output.endsWith("\n") ? output.slice(0, -1) : output; + return linuxPath.startsWith("/") ? linuxPath : requestedPath; +}); diff --git a/apps/server/src/ws.test.ts b/apps/server/src/ws.test.ts index a6b2a819a472a..9d7931f103193 100644 --- a/apps/server/src/ws.test.ts +++ b/apps/server/src/ws.test.ts @@ -1205,6 +1205,7 @@ it.effect("access changes made while the snapshot loads reach the client after i Layer.mock(ServerEnvironment.ServerEnvironment)({}), Layer.mock(EnvironmentAuth.EnvironmentAuth)({ listPairingLinks: () => Effect.succeed([]), + listConnectDevices: () => Effect.succeed([]), // Another client revokes a session after the snapshot read it. listClientSessions: () => PubSub.publish(sessionChanges, { @@ -1248,6 +1249,18 @@ it.effect("access changes made while the snapshot loads reach the client after i type: "clientRemoved", payload: { sessionId: revokedElsewhere }, }); + yield* server.write(0, { _tag: "Ack", requestId }); + assert.equal((yield* Queue.take(responses)).type, "pairingLinkRemoved"); + yield* server.write(0, { _tag: "Ack", requestId }); + + // Device trust has no incremental event, so a change re-sends the whole snapshot. + yield* PubSub.publish(pairingChanges, { type: "connectDevicesChanged" }); + assert.deepEqual(yield* Queue.take(responses), { + version: 1, + revision: 4, + type: "snapshot", + payload: { pairingLinks: [], clientSessions: [], connectDevices: [] }, + }); yield* server.write(0, { _tag: "Interrupt", requestId, interruptors: [] }); }).pipe( Effect.provide( diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index b7d85fc318fc4..a022ca45c21fd 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -728,7 +728,10 @@ const serverWsRpcs = Array.from(ServerWsRpcGroup.requests.values()); const SplitWsRpcGroup = RpcGroup.make(...serverWsRpcs.filter(isSplitWsRpc)); const OtherWsRpcGroup = RpcGroup.make(...serverWsRpcs.filter(isOtherWsRpc)); function toAuthAccessStreamEvent( - change: PairingGrantStore.BootstrapCredentialChange | SessionStore.SessionCredentialChange, + change: Exclude< + PairingGrantStore.BootstrapCredentialChange | SessionStore.SessionCredentialChange, + { readonly type: "connectDevicesChanged" } + >, revision: number, currentSessionId: AuthSessionId, ): AuthAccessStreamEvent { @@ -1782,6 +1785,7 @@ export const layerWsRpc = ( Effect.all({ pairingLinks: serverAuth.listPairingLinks(), clientSessions: serverAuth.listClientSessions(currentSessionId), + connectDevices: serverAuth.listConnectDevices(), }).pipe( Effect.mapError( (error) => @@ -1887,6 +1891,7 @@ export const layerWsRpc = ( }), newProjectsRoot: managedFolders.namedProjectsRoot, ...(lastEventLoopStall === undefined ? {} : { lastEventLoopStall }), + serverTime: DateTime.formatIso(yield* DateTime.now), }; }); @@ -2654,6 +2659,10 @@ export const layerWsRpc = ( Effect.annotateCurrentSpan({ "orchestration_v2.thread_id": input.threadId }).pipe( Effect.andThen(peerLinkRequests.answer(input)), ), + [WS_METHODS.serverGetSettings]: (_input) => + Effect.map(serverSettings.getSettings, (settings) => + ServerSettings.redactServerSettingsForClient(settings, settingsClient), + ), [WS_METHODS.serverUpdateSettings]: /* Request instrumentation already owns the span; keep this helper in that context. */ Effect.fnUntraced(function* ({ patch, providerInstanceMutation }) { @@ -3174,9 +3183,19 @@ export const layerWsRpc = ( ), ), [WS_METHODS.vcsCreateRef]: (input) => - gitWorkflow.createRef(input).pipe(Effect.tap(() => refreshGitStatus(input.cwd))), + gitWorkflow + .createRef(input) + .pipe( + Effect.tap(() => + input.switchRef + ? vcsStatusBroadcaster.refreshAfterCheckout(input.cwd) + : refreshGitStatus(input.cwd), + ), + ), [WS_METHODS.vcsSwitchRef]: (input) => - gitWorkflow.switchRef(input).pipe(Effect.tap(() => refreshGitStatus(input.cwd))), + gitWorkflow + .switchRef(input) + .pipe(Effect.tap(() => vcsStatusBroadcaster.refreshAfterCheckout(input.cwd))), [WS_METHODS.vcsInit]: (input) => vcsProvisioning.initRepository(input).pipe(Effect.tap(() => refreshGitStatus(input.cwd))), [WS_METHODS.reviewGetBranchCommits]: (input) => review.getBranchCommits(input), @@ -3479,17 +3498,28 @@ export const layerWsRpc = ( const initialSnapshot = yield* loadAuthAccessSnapshot(); const revisionRef = yield* Ref.make(1); - const liveEvents: Stream.Stream = Stream.fromQueue( - liveBuffer, - ).pipe( - Stream.mapEffect((change) => - Ref.updateAndGet(revisionRef, (revision) => revision + 1).pipe( - Effect.map((revision) => - toAuthAccessStreamEvent(change, revision, currentSessionId), + const liveEvents: Stream.Stream = + Stream.fromQueue(liveBuffer).pipe( + Stream.mapEffect((change) => + Ref.updateAndGet(revisionRef, (revision) => revision + 1).pipe( + Effect.flatMap((revision) => + // Device trust has no incremental event older clients could decode. + change.type === "connectDevicesChanged" + ? loadAuthAccessSnapshot().pipe( + Effect.map((payload): AuthAccessStreamEvent => ({ + version: 1, + revision, + type: "snapshot", + payload, + })), + ) + : Effect.succeed( + toAuthAccessStreamEvent(change, revision, currentSessionId), + ), + ), ), ), - ), - ); + ); return Stream.concat( rpcInitialItems([ diff --git a/apps/web/index.html b/apps/web/index.html index e6dec0e60ad8c..6e61d7cd7381d 100644 --- a/apps/web/index.html +++ b/apps/web/index.html @@ -15,6 +15,7 @@ const LIGHT_BACKGROUND = "#ffffff"; const DARK_BACKGROUND = "#000000"; const CUSTOM_THEMES_STORAGE_KEY = "t3code:themes:v1"; + const ENVIRONMENT_THEMES_STORAGE_KEY = "t3code:environment-themes:v1"; const THEME_FOLLOW_SYSTEM_STORAGE_KEY = "t3code:theme-follow-system"; const THEME_APPEARANCE_MODE_STORAGE_KEY = "t3code:theme-appearance-mode"; const THEME_HALVES_STORAGE_KEY = "t3code:theme-halves:v1"; @@ -212,12 +213,9 @@ (appearance === value.appearance || isRecord(colors)), ); }; - const findCustomTheme = (themeId) => { - if (RESERVED_THEME_IDS.includes(themeId)) return null; + const findStoredTheme = (storageKey, themeId) => { try { - const parsed = JSON.parse( - window.localStorage.getItem(CUSTOM_THEMES_STORAGE_KEY) ?? "null", - ); + const parsed = JSON.parse(window.localStorage.getItem(storageKey) ?? "null"); if (!Array.isArray(parsed)) return null; return ( parsed.find((value) => isStoredCustomTheme(value) && value.id === themeId) ?? null @@ -226,6 +224,14 @@ return null; } }; + // Same precedence as getThemeDefinition: saved custom themes, then published ones. + const findCustomTheme = (themeId) => { + if (RESERVED_THEME_IDS.includes(themeId)) return null; + return ( + findStoredTheme(CUSTOM_THEMES_STORAGE_KEY, themeId) ?? + findStoredTheme(ENVIRONMENT_THEMES_STORAGE_KEY, themeId) + ); + }; try { const storedTheme = window.localStorage.getItem("t3code:theme"); diff --git a/apps/web/src/appearanceFonts.ts b/apps/web/src/appearanceFonts.ts index 19f2d88d3c29d..50a664fea336e 100644 --- a/apps/web/src/appearanceFonts.ts +++ b/apps/web/src/appearanceFonts.ts @@ -276,6 +276,18 @@ function resolveGenericFamilyLabel(generic: string): string | null { return null; } +// Synthetic Apple labels are not necessarily available as named CSS families. +export function displayFontFamily(family: string): string { + const label = family + .trim() + .replace(/^(['"])(.*)\1$/, "$2") + .trim() + .toLowerCase(); + if (label === "sf pro") return appearanceFontStack(family, DEFAULT_SANS_FONT_STACK); + if (label === "sf mono") return DEFAULT_CODE_FONT_STACK; + return cssFontFamilies(family) ?? ""; +} + export function resolveDefaultFamilyLabel(stack: string): string | null { for (const raw of stack.split(",")) { const family = raw.trim().replace(/^(['"])(.*)\1$/, "$2"); diff --git a/apps/web/src/authBootstrap.test.ts b/apps/web/src/authBootstrap.test.ts index 65c6f8edd3424..f444c64cb16ad 100644 --- a/apps/web/src/authBootstrap.test.ts +++ b/apps/web/src/authBootstrap.test.ts @@ -8,6 +8,7 @@ import { import { createBrowserHistory } from "@tanstack/react-router"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; +import { HttpClientError, HttpClientRequest } from "effect/http"; import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; import { installEnvironmentHttpTest } from "../test/environmentHttpTest"; @@ -165,6 +166,54 @@ describe("resolveInitialServerAuthGateState", () => { expect(testApi.calls.browserSession).toEqual([{ credential: "desktop-bootstrap-token" }]); }); + it("retries the desktop bootstrap while the backend is still unreachable", async () => { + const nextSession = sequence( + unauthenticatedSession(DESKTOP_AUTH), + authenticatedSession(DESKTOP_AUTH), + ); + const testApi = await installAuthApi({ + session: nextSession, + browserSession: () => Effect.succeed(browserSession(["orchestration:read", "access:write"])), + }); + // What FetchHttpClient yields when nothing listens on the backend port. + const connectionRefused = new HttpClientError.HttpClientError({ + reason: new HttpClientError.TransportError({ + request: HttpClientRequest.get("http://localhost:3773/api/auth/session"), + cause: new TypeError("Failed to fetch"), + }), + }); + let unreachableAttempts = 1; + __setPrimaryHttpRunnerForTests((effect) => { + if (unreachableAttempts > 0) { + unreachableAttempts -= 1; + return Promise.reject(connectionRefused); + } + return testApi.run(effect); + }); + + const testWindow = installTestBrowser("http://localhost/"); + testWindow.desktopBridge = { + getLocalEnvironmentBootstraps: () => [ + { + id: "primary", + label: "Local", + httpBaseUrl: "http://localhost:3773", + wsBaseUrl: "ws://localhost:3773", + bootstrapToken: "desktop-bootstrap-token", + }, + ], + } as unknown as DesktopBridge; + + const { resolveInitialServerAuthGateState } = await import("./environments/primary"); + + vi.useFakeTimers({ toFake: ["setTimeout", "Date"] }); + await Promise.all([ + expect(resolveInitialServerAuthGateState()).resolves.toEqual({ status: "authenticated" }), + vi.advanceTimersByTimeAsync(1_000), + ]); + expect(testApi.calls.browserSession).toEqual([{ credential: "desktop-bootstrap-token" }]); + }); + it("takes a pairing token from the location hash and strips it immediately", async () => { const testWindow = installTestBrowser("http://localhost/#token=pairing-token"); const { takePairingTokenFromUrl } = await import("./environments/primary"); diff --git a/apps/web/src/browser/HostedBrowserWebview.tsx b/apps/web/src/browser/HostedBrowserWebview.tsx index 28edc42d406a8..023fb03c3aa17 100644 --- a/apps/web/src/browser/HostedBrowserWebview.tsx +++ b/apps/web/src/browser/HostedBrowserWebview.tsx @@ -20,6 +20,7 @@ import { useActiveBrowserRecordingTabIds } from "./browserRecording"; import { browserViewportSettingKey, resolveBrowserViewportLayout, + resolveBrowserViewportTransform, resolveFittedBrowserViewport, } from "./browserViewportLayout"; import { BrowserDeviceToolbar } from "./BrowserDeviceToolbar"; @@ -313,6 +314,23 @@ export function HostedBrowserWebview(props: { return () => withDesktopTab(runtimeTabId, () => bridge.setShown(runtimeTabId, false)); }, [clientSettingsHydrated, runtimeTabId, serverDriven, shown]); + // Captures copy the guest's raster, which a scaled guest only keeps at full detail on request. + const fullDetail = backgroundActivity || pictureInPicture || recordingActive; + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- tells the desktop host once the guest is drawn at full detail, so its screenshot waits for that frame + useEffect(() => { + const bridge = window.desktopBridge?.preview; + if (!clientSettingsHydrated || !bridge || !serverDriven || !fullDetail) return; + let frame = requestAnimationFrame(() => { + frame = requestAnimationFrame(() => + withDesktopTab(runtimeTabId, () => bridge.setCaptureDrawn(runtimeTabId, true)), + ); + }); + return () => { + cancelAnimationFrame(frame); + withDesktopTab(runtimeTabId, () => bridge.setCaptureDrawn(runtimeTabId, false)); + }; + }, [clientSettingsHydrated, fullDetail, runtimeTabId, serverDriven]); + if (!clientSettingsHydrated || !config) return null; // This fixed wrapper sits outside the right panel shell, so it takes thumb presses itself. @@ -322,6 +340,7 @@ export function HostedBrowserWebview(props: { }; const renderingActive = active || backgroundActivity || pictureInPicture || recordingActive; + const viewportTransform = resolveBrowserViewportTransform(layout.viewportScale, fullDetail); const wrapperStyle = resolveHostedBrowserWebviewWrapperStyle({ active, renderingActive, @@ -354,47 +373,56 @@ export function HostedBrowserWebview(props: { onChange={commitViewportChange} /> ) : null} - + > + + {active && effectiveViewport._tag !== "fill" && !fittedSourceViewport ? ( <> { ).toEqual({ width: 460, height: 409 }); }); }); + +describe("resolveBrowserViewportTransform", () => { + it("keeps a full-detail guest at raster scale 1 while it draws at the fitted scale", () => { + expect(resolveBrowserViewportTransform(1, true)).toBeUndefined(); + expect(resolveBrowserViewportTransform(0.6, false)).toBe("scale(0.6)"); + const values = resolveBrowserViewportTransform(0.6, true) + ?.match(/^matrix3d\((.*)\)$/)?.[1] + ?.split(",") + .map(Number); + // Column-major: the x and y columns keep their 0.6 screen terms and reach length 1 through z. + expect(values?.slice(0, 2)).toEqual([0.6, 0]); + expect(values?.slice(4, 6)).toEqual([0, 0.6]); + expect(Math.hypot(...(values?.slice(0, 3) ?? []))).toBeCloseTo(1); + expect(Math.hypot(...(values?.slice(4, 7) ?? []))).toBeCloseTo(1); + }); +}); diff --git a/apps/web/src/browser/browserViewportLayout.ts b/apps/web/src/browser/browserViewportLayout.ts index 2fe4e035100ab..7fe6b170c671d 100644 --- a/apps/web/src/browser/browserViewportLayout.ts +++ b/apps/web/src/browser/browserViewportLayout.ts @@ -125,6 +125,22 @@ export function resolveBrowserViewportLayout( }; } +/** + * Chromium rasters a CSS-scaled guest at its on-screen scale, so captures of it lose detail. + * Its raster scale also counts x/y-to-z terms, which a `preserve-3d` parent keeps and the + * screen drops, so `fullDetail` draws like `scale()` but keeps full-resolution frames. + */ +export function resolveBrowserViewportTransform( + scale: number, + fullDetail: boolean, +): string | undefined { + if (scale >= 1) return undefined; + if (!fullDetail) return `scale(${scale})`; + // Positive depth keeps the guest in front of its parent for hit testing. + const depth = Math.sqrt(1 - scale * scale); + return `matrix3d(${scale}, 0, ${depth}, 0, 0, ${scale}, ${depth}, 0, 0, 0, 1, 0, 0, 0, 0, 1)`; +} + export function resolveBrowserDeviceViewportLayout( container: { readonly width: number; readonly height: number }, setting: Exclude, diff --git a/apps/web/src/cloud/t3ConnectState.ts b/apps/web/src/cloud/t3ConnectState.ts index c468c73d965ea..67fef2eaee1a6 100644 --- a/apps/web/src/cloud/t3ConnectState.ts +++ b/apps/web/src/cloud/t3ConnectState.ts @@ -3,6 +3,7 @@ import type { RelayManagedEndpoint, RelayManagedEndpointProviderKind, } from "@t3tools/contracts/relay"; +import { parseUrl } from "@t3tools/shared/url"; // Require managed discovery: stale tunnel config after a publish-only switch causes endpoint_provider_not_managed. export function isManagedTunnelActive( @@ -31,9 +32,9 @@ export function t3ConnectMcpUrl(input: { readonly relayEndpoint: RelayManagedEndpoint | undefined; }): string | null { const url = input.thisMachine - ? URL.parse(`https://${input.tunnelHostname ?? ""}`) + ? parseUrl(`https://${input.tunnelHostname ?? ""}`) : input.relayEndpoint?.providerKind === "cloudflare_tunnel" - ? URL.parse(input.relayEndpoint.httpBaseUrl) + ? parseUrl(input.relayEndpoint.httpBaseUrl) : null; return url?.protocol === "https:" && TUNNEL_HOSTNAME.test(url.hostname) ? `${url.origin}/mcp` diff --git a/apps/web/src/components/AppSidebarLayout.browser.test.tsx b/apps/web/src/components/AppSidebarLayout.browser.test.tsx new file mode 100644 index 0000000000000..c97ea5e6412aa --- /dev/null +++ b/apps/web/src/components/AppSidebarLayout.browser.test.tsx @@ -0,0 +1,42 @@ +import { page, userEvent } from "vite-plus/test/browser"; +import { describe, expect, it, vi } from "vite-plus/test"; + +import { render } from "../../test/render"; + +vi.mock(import("../state/server"), async (original) => { + const actual = await original(); + const { Atom } = await import("effect/reactivity"); + const { compileResolvedKeybindingsConfig } = await import("@t3tools/shared/keybindings"); + const keybindings = compileResolvedKeybindingsConfig([ + { key: "mod+b", command: "sidebar.toggle", when: "!terminalFocus" }, + ]); + return { ...actual, primaryServerKeybindingsAtom: Atom.make(keybindings) }; +}); +// The thread list needs live environments; the shortcut lives in the shell around it. +vi.mock(import("./Sidebar"), () => ({ default: (() => null) as never })); + +import { AppSidebarLayout } from "./AppSidebarLayout"; + +describe("sidebar toggle shortcut", () => { + it("leaves the key to a focused terminal when its rule excludes terminalFocus", async () => { + await page.viewport(1280, 800); + using _platform = vi.spyOn(navigator, "platform", "get").mockReturnValue("Linux x86_64"); + await using screen = await render( + +
    +