diff --git a/apps/server/src/provider/Drivers/CodexHomeLayout.test.ts b/apps/server/src/provider/Drivers/CodexHomeLayout.test.ts index 2bbcea20faa91..9db6dd1c8bb36 100644 --- a/apps/server/src/provider/Drivers/CodexHomeLayout.test.ts +++ b/apps/server/src/provider/Drivers/CodexHomeLayout.test.ts @@ -38,6 +38,39 @@ const writeTextFile = Effect.fn("CodexHomeLayout.test.writeTextFile")(function* yield* fileSystem.writeFileString(filePath, contents); }); +// Collects real (non-symlink) file contents so data preservation can be asserted without +// prescribing where a reconciliation keeps or backs up a conflicting shadow entry. +const readRegularFileContents = Effect.fn("CodexHomeLayout.test.readRegularFileContents")( + function* (roots: ReadonlyArray) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const contents: Array = []; + for (const root of roots) { + for (const relativePath of yield* fileSystem.readDirectory(root, { recursive: true })) { + const entryPath = path.join(root, relativePath); + const isSymlink = yield* fileSystem.readLink(entryPath).pipe( + Effect.as(true), + Effect.orElseSucceed(() => false), + ); + if (isSymlink) continue; + const info = yield* fileSystem.stat(entryPath); + if (info.type === "File") { + contents.push(yield* fileSystem.readFileString(entryPath)); + } + } + } + return contents; + }, +); + +const isSymlink = Effect.fn("CodexHomeLayout.test.isSymlink")(function* (entryPath: string) { + const fileSystem = yield* FileSystem.FileSystem; + return yield* fileSystem.readLink(entryPath).pipe( + Effect.as(true), + Effect.orElseSucceed(() => false), + ); +}); + it.layer(NodeServices.layer)("CodexHomeLayout", (it) => { describe("resolveCodexHomeLayout", () => { it.effect("uses the shared home for continuation and the shadow home for runtime", () => @@ -363,5 +396,124 @@ it.layer(NodeServices.layer)("CodexHomeLayout", (it) => { ); }), ); + + it.effect.skipIf(!symlinksSupported)( + "keeps working when Codex creates new entries in both homes between materializations", + () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const sharedHome = yield* makeTempDir("t3code-codex-shared-"); + const shadowRoot = yield* makeTempDir("t3code-codex-shadow-root-"); + const shadowHome = path.join(shadowRoot, "shadow"); + const layout = yield* resolveCodexHomeLayout( + decodeCodexSettings({ homePath: sharedHome, shadowHomePath: shadowHome }), + ); + + yield* materializeCodexShadowHome(layout); + + // Entries Codex 0.163 creates on its own after the first materialization, plus a + // name no Codex release uses yet: the fix must not depend on an allowlist. + const runtimeFiles = [ + ".tmp/runtime.lock", + "installation_id", + "queue_1.sqlite", + "goals_1.sqlite", + "future_state_42.sqlite", + ]; + for (const runtimeFile of runtimeFiles) { + yield* writeTextFile(path.join(sharedHome, runtimeFile), `shared ${runtimeFile}`); + yield* writeTextFile(path.join(shadowHome, runtimeFile), `shadow ${runtimeFile}`); + } + + yield* materializeCodexShadowHome(layout); + + const shadowRootContents = yield* readRegularFileContents([shadowRoot]); + for (const runtimeFile of runtimeFiles) { + expect(yield* fileSystem.readFileString(path.join(sharedHome, runtimeFile))).toBe( + `shared ${runtimeFile}`, + ); + expect(shadowRootContents).toContain(`shadow ${runtimeFile}`); + } + }), + ); + + it.effect.skipIf(!symlinksSupported)( + "leaves regular shadow WAL and SHM sidecars of a linked SQLite database in place", + () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const sharedHome = yield* makeTempDir("t3code-codex-shared-"); + const shadowRoot = yield* makeTempDir("t3code-codex-shadow-root-"); + const shadowHome = path.join(shadowRoot, "shadow"); + yield* writeTextFile(path.join(sharedHome, "logs_2.sqlite"), "shared logs db"); + const layout = yield* resolveCodexHomeLayout( + decodeCodexSettings({ homePath: sharedHome, shadowHomePath: shadowHome }), + ); + + yield* materializeCodexShadowHome(layout); + expect(yield* isSymlink(path.join(shadowHome, "logs_2.sqlite"))).toBe(true); + + // SQLite's Windows VFS opens the journal next to the path it was given rather than + // the link target, so Codex there writes regular sidecars beside the linked base + // database while the shared home has its own. + for (const suffix of ["-wal", "-shm"]) { + const name = `logs_2.sqlite${suffix}`; + yield* writeTextFile(path.join(sharedHome, name), `shared ${name}`); + yield* writeTextFile(path.join(shadowHome, name), `shadow ${name}`); + } + + yield* materializeCodexShadowHome(layout); + + // Folding a WAL into another home's database needs SQLite, so the existing topology is + // kept byte for byte instead of being migrated. + expect(yield* fileSystem.readLink(path.join(shadowHome, "logs_2.sqlite"))).toBe( + path.join(sharedHome, "logs_2.sqlite"), + ); + for (const suffix of ["-wal", "-shm"]) { + const name = `logs_2.sqlite${suffix}`; + expect(yield* isSymlink(path.join(shadowHome, name))).toBe(false); + expect(yield* fileSystem.readFileString(path.join(shadowHome, name))).toBe( + `shadow ${name}`, + ); + expect(yield* fileSystem.readFileString(path.join(sharedHome, name))).toBe( + `shared ${name}`, + ); + } + }), + ); + + it.effect.skipIf(!symlinksSupported)( + "never links shared sidecars next to a shadow-local SQLite database", + () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const sharedHome = yield* makeTempDir("t3code-codex-shared-"); + const shadowRoot = yield* makeTempDir("t3code-codex-shadow-root-"); + const shadowHome = path.join(shadowRoot, "shadow"); + for (const name of ["queue_1.sqlite", "queue_1.sqlite-wal", "queue_1.sqlite-shm"]) { + yield* writeTextFile(path.join(sharedHome, name), `shared ${name}`); + } + // A sidecar whose base the shared home does not have must not be linked either, or a + // database the shadow home creates later would journal into the shared file. + yield* writeTextFile(path.join(sharedHome, "orphan.sqlite-wal"), "shared orphan wal"); + yield* writeTextFile(path.join(shadowHome, "queue_1.sqlite"), "shadow queue db"); + const layout = yield* resolveCodexHomeLayout( + decodeCodexSettings({ homePath: sharedHome, shadowHomePath: shadowHome }), + ); + + yield* materializeCodexShadowHome(layout); + + expect(yield* fileSystem.readFileString(path.join(shadowHome, "queue_1.sqlite"))).toBe( + "shadow queue db", + ); + for (const name of ["queue_1.sqlite-wal", "queue_1.sqlite-shm", "orphan.sqlite-wal"]) { + expect(yield* fileSystem.exists(path.join(shadowHome, name))).toBe(false); + expect(yield* isSymlink(path.join(shadowHome, name))).toBe(false); + } + }), + ); }); }); diff --git a/apps/server/src/provider/Drivers/CodexHomeLayout.ts b/apps/server/src/provider/Drivers/CodexHomeLayout.ts index f6d2ba9d44cf5..507955ac4d9e9 100644 --- a/apps/server/src/provider/Drivers/CodexHomeLayout.ts +++ b/apps/server/src/provider/Drivers/CodexHomeLayout.ts @@ -39,6 +39,19 @@ const REPLACEABLE_SHARED_RUNTIME_ENTRIES = new Set([ "thread-writer-locks", ".sqlite-maintenance.lock", ]); +// A local copy of these defeats the shared home, so it is reported instead of kept. Any other +// regular shadow entry is state Codex created for this account (new releases keep adding +// databases such as queue_1.sqlite), and it stays local rather than disabling the provider. +const REQUIRED_SHARED_ENTRY_NAMES = new Set([...KNOWN_SHARED_DIRECTORIES, "config.toml"]); +const SQLITE_SIDECAR_SUFFIXES = ["-wal", "-shm", "-journal"] as const; + +// Groups a SQLite database with its sidecars, which must all live in the same home. +function sqliteFamilyName(entryName: string): string { + const suffix = SQLITE_SIDECAR_SUFFIXES.find( + (candidate) => entryName.length > candidate.length && entryName.endsWith(candidate), + ); + return suffix === undefined ? entryName : entryName.slice(0, -suffix.length); +} function resolveHomePath(path: Path.Path, home: string, value: string | undefined): string { const expanded = @@ -388,25 +401,56 @@ export const materializeCodexShadowHome = Effect.fn("materializeCodexShadowHome" { concurrency: "unbounded" }, ); - const sharedEntryNames = yield* fileSystem.readDirectory(layout.sharedHomePath).pipe( - Effect.catchTags({ - PlatformError: (cause) => - new CodexShadowHomeFileSystemError({ - sharedHomePath: layout.sharedHomePath, - effectiveHomePath, - operation: "readDirectory", - path: layout.sharedHomePath, - cause, - }), - }), - ); + const readDirectory = (directoryPath: string) => + fileSystem.readDirectory(directoryPath).pipe( + Effect.catchTags({ + PlatformError: (cause) => + new CodexShadowHomeFileSystemError({ + sharedHomePath: layout.sharedHomePath, + effectiveHomePath, + operation: "readDirectory", + path: directoryPath, + cause, + }), + }), + ); + const [sharedEntryNames, shadowEntryNames] = yield* Effect.all([ + readDirectory(layout.sharedHomePath), + readDirectory(effectiveHomePath), + ]); + const isSharable = (entryName: string) => + !PRIVATE_ENTRY_NAMES.has(entryName) && !SHADOW_LOCAL_ENTRY_NAMES.has(entryName); const entries = new Set(KNOWN_SHARED_DIRECTORIES); for (const entryName of sharedEntryNames) { - if (!PRIVATE_ENTRY_NAMES.has(entryName) && !SHADOW_LOCAL_ENTRY_NAMES.has(entryName)) { + if (isSharable(entryName)) { entries.add(entryName); } } + // A family with any regular shadow member stays exactly as it is: linking the rest would + // pair a local database with shared sidecars, or the reverse, and only SQLite can merge them. + const shadowLocalFamilies = new Set(); + yield* Effect.forEach( + shadowEntryNames, + (entryName) => + isSharable(entryName) + ? readLinkState({ + fileSystem, + sharedHomePath: layout.sharedHomePath, + effectiveHomePath, + entryName, + linkPath: path.join(effectiveHomePath, entryName), + }).pipe( + Effect.map((state) => { + if (state._tag === "NotSymlink") { + shadowLocalFamilies.add(sqliteFamilyName(entryName)); + } + }), + ) + : Effect.void, + { discard: true }, + ); + yield* Effect.forEach( PRIVATE_ENTRY_NAMES, (entryName) => @@ -427,6 +471,19 @@ export const materializeCodexShadowHome = Effect.fn("materializeCodexShadowHome" if (PRIVATE_ENTRY_NAMES.has(entryName)) { return Effect.void; } + const familyName = sqliteFamilyName(entryName); + // Sidecars follow their base, so they are not linked next to a database the shadow home + // may create later. + if (familyName !== entryName && !entries.has(familyName)) { + return Effect.void; + } + if ( + shadowLocalFamilies.has(familyName) && + !REQUIRED_SHARED_ENTRY_NAMES.has(entryName) && + !REPLACEABLE_SHARED_RUNTIME_ENTRIES.has(entryName) + ) { + return Effect.void; + } return ensureSymlink({ fileSystem, sharedHomePath: layout.sharedHomePath,