From b913ce512ebc44d664beef5cff0f7de83ce6fd92 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 22 Oct 2026 06:42:23 +0000 Subject: [PATCH 1/3] test(server): reproduce issue 5817 Codex shadow-home runtime entry conflicts Co-Authored-By: Claude Opus 5.5 --- .../provider/Drivers/CodexHomeLayout.test.ts | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/apps/server/src/provider/Drivers/CodexHomeLayout.test.ts b/apps/server/src/provider/Drivers/CodexHomeLayout.test.ts index 093777e67db5d..bfdefa6cad643 100644 --- a/apps/server/src/provider/Drivers/CodexHomeLayout.test.ts +++ b/apps/server/src/provider/Drivers/CodexHomeLayout.test.ts @@ -38,6 +38,39 @@ const writeTextFile = Effect.fn("CodexHomeLayout.test.writeTextFile")(function* yield* fileSystem.writeFileString(filePath, contents); }); +// Collects real (non-symlink) file contents so data preservation can be asserted without +// prescribing where a reconciliation keeps or backs up a conflicting shadow entry. +const readRegularFileContents = Effect.fn("CodexHomeLayout.test.readRegularFileContents")( + function* (roots: ReadonlyArray) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const contents: Array = []; + for (const root of roots) { + for (const relativePath of yield* fileSystem.readDirectory(root, { recursive: true })) { + const entryPath = path.join(root, relativePath); + const isSymlink = yield* fileSystem.readLink(entryPath).pipe( + Effect.as(true), + Effect.orElseSucceed(() => false), + ); + if (isSymlink) continue; + const info = yield* fileSystem.stat(entryPath); + if (info.type === "File") { + contents.push(yield* fileSystem.readFileString(entryPath)); + } + } + } + return contents; + }, +); + +const isSymlink = Effect.fn("CodexHomeLayout.test.isSymlink")(function* (entryPath: string) { + const fileSystem = yield* FileSystem.FileSystem; + return yield* fileSystem.readLink(entryPath).pipe( + Effect.as(true), + Effect.orElseSucceed(() => false), + ); +}); + it.layer(NodeServices.layer)("CodexHomeLayout", (it) => { describe("resolveCodexHomeLayout", () => { it.effect("uses the shared home for continuation and the shadow home for runtime", () => @@ -323,5 +356,87 @@ it.layer(NodeServices.layer)("CodexHomeLayout", (it) => { ); }), ); + + it.effect.skipIf(!symlinksSupported)( + "keeps working when Codex creates new entries in both homes between materializations", + () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const sharedHome = yield* makeTempDir("t3code-codex-shared-"); + const shadowRoot = yield* makeTempDir("t3code-codex-shadow-root-"); + const shadowHome = path.join(shadowRoot, "shadow"); + const layout = yield* resolveCodexHomeLayout( + decodeCodexSettings({ homePath: sharedHome, shadowHomePath: shadowHome }), + ); + + yield* materializeCodexShadowHome(layout); + + // Entries Codex 0.163 creates on its own after the first materialization, plus a + // name no Codex release uses yet: the fix must not depend on an allowlist. + const runtimeFiles = [ + ".tmp/runtime.lock", + "installation_id", + "queue_1.sqlite", + "goals_1.sqlite", + "future_state_42.sqlite", + ]; + for (const runtimeFile of runtimeFiles) { + yield* writeTextFile(path.join(sharedHome, runtimeFile), `shared ${runtimeFile}`); + yield* writeTextFile(path.join(shadowHome, runtimeFile), `shadow ${runtimeFile}`); + } + + yield* materializeCodexShadowHome(layout); + + const shadowRootContents = yield* readRegularFileContents([shadowRoot]); + for (const runtimeFile of runtimeFiles) { + expect(yield* fileSystem.readFileString(path.join(sharedHome, runtimeFile))).toBe( + `shared ${runtimeFile}`, + ); + expect(shadowRootContents).toContain(`shadow ${runtimeFile}`); + } + }), + ); + + it.effect.skipIf(!symlinksSupported)( + "reconciles regular shadow WAL and SHM sidecars of a linked SQLite database as one family", + () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const sharedHome = yield* makeTempDir("t3code-codex-shared-"); + const shadowRoot = yield* makeTempDir("t3code-codex-shadow-root-"); + const shadowHome = path.join(shadowRoot, "shadow"); + yield* writeTextFile(path.join(sharedHome, "logs_2.sqlite"), "shared logs db"); + const layout = yield* resolveCodexHomeLayout( + decodeCodexSettings({ homePath: sharedHome, shadowHomePath: shadowHome }), + ); + + yield* materializeCodexShadowHome(layout); + expect(yield* isSymlink(path.join(shadowHome, "logs_2.sqlite"))).toBe(true); + + // SQLite opens its journal next to the path it was given, so Codex writes regular + // sidecars beside the linked base database while the shared home has its own. + for (const suffix of ["-wal", "-shm"]) { + const name = `logs_2.sqlite${suffix}`; + yield* writeTextFile(path.join(sharedHome, name), `shared ${name}`); + yield* writeTextFile(path.join(shadowHome, name), `shadow ${name}`); + } + + yield* materializeCodexShadowHome(layout); + + const familyLinks = yield* Effect.forEach( + ["logs_2.sqlite", "logs_2.sqlite-wal", "logs_2.sqlite-shm"], + (name) => isSymlink(path.join(shadowHome, name)), + ); + expect(new Set(familyLinks).size).toBe(1); + expect(yield* fileSystem.readFileString(path.join(sharedHome, "logs_2.sqlite-wal"))).toBe( + "shared logs_2.sqlite-wal", + ); + expect(yield* readRegularFileContents([shadowRoot])).toContain( + "shadow logs_2.sqlite-wal", + ); + }), + ); }); }); From 51cf491624be9461b77412c3e30dbe91a33d93e2 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 22 Oct 2026 07:14:20 +0000 Subject: [PATCH 2/3] fix(server): new Codex runtime entries no longer disable shadow-home providers A regular shadow-home entry that is not core shared state (config.toml or the known shared directories) now stays local instead of failing materialization, so databases future Codex releases add (queue_1.sqlite, ...) keep working. SQLite base, -wal, -shm and -journal entries are treated as one family: if any member is local to the shadow home, no other member is linked, and sidecars are never linked without their shared base. Existing topology is left untouched; nothing is moved, deleted or migrated. Co-Authored-By: Claude Opus 5.5 --- .../provider/Drivers/CodexHomeLayout.test.ts | 54 ++++++++++-- .../src/provider/Drivers/CodexHomeLayout.ts | 83 ++++++++++++++++--- 2 files changed, 115 insertions(+), 22 deletions(-) diff --git a/apps/server/src/provider/Drivers/CodexHomeLayout.test.ts b/apps/server/src/provider/Drivers/CodexHomeLayout.test.ts index bfdefa6cad643..676f1d46d9ab6 100644 --- a/apps/server/src/provider/Drivers/CodexHomeLayout.test.ts +++ b/apps/server/src/provider/Drivers/CodexHomeLayout.test.ts @@ -399,7 +399,7 @@ it.layer(NodeServices.layer)("CodexHomeLayout", (it) => { ); it.effect.skipIf(!symlinksSupported)( - "reconciles regular shadow WAL and SHM sidecars of a linked SQLite database as one family", + "leaves regular shadow WAL and SHM sidecars of a linked SQLite database in place", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; @@ -425,17 +425,53 @@ it.layer(NodeServices.layer)("CodexHomeLayout", (it) => { yield* materializeCodexShadowHome(layout); - const familyLinks = yield* Effect.forEach( - ["logs_2.sqlite", "logs_2.sqlite-wal", "logs_2.sqlite-shm"], - (name) => isSymlink(path.join(shadowHome, name)), + // Folding a WAL into another home's database needs SQLite, so the existing topology is + // kept byte for byte instead of being migrated. + expect(yield* fileSystem.readLink(path.join(shadowHome, "logs_2.sqlite"))).toBe( + path.join(sharedHome, "logs_2.sqlite"), ); - expect(new Set(familyLinks).size).toBe(1); - expect(yield* fileSystem.readFileString(path.join(sharedHome, "logs_2.sqlite-wal"))).toBe( - "shared logs_2.sqlite-wal", + for (const suffix of ["-wal", "-shm"]) { + const name = `logs_2.sqlite${suffix}`; + expect(yield* isSymlink(path.join(shadowHome, name))).toBe(false); + expect(yield* fileSystem.readFileString(path.join(shadowHome, name))).toBe( + `shadow ${name}`, + ); + expect(yield* fileSystem.readFileString(path.join(sharedHome, name))).toBe( + `shared ${name}`, + ); + } + }), + ); + + it.effect.skipIf(!symlinksSupported)( + "never links shared sidecars next to a shadow-local SQLite database", + () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const sharedHome = yield* makeTempDir("t3code-codex-shared-"); + const shadowRoot = yield* makeTempDir("t3code-codex-shadow-root-"); + const shadowHome = path.join(shadowRoot, "shadow"); + for (const name of ["queue_1.sqlite", "queue_1.sqlite-wal", "queue_1.sqlite-shm"]) { + yield* writeTextFile(path.join(sharedHome, name), `shared ${name}`); + } + // A sidecar whose base the shared home does not have must not be linked either, or a + // database the shadow home creates later would journal into the shared file. + yield* writeTextFile(path.join(sharedHome, "orphan.sqlite-wal"), "shared orphan wal"); + yield* writeTextFile(path.join(shadowHome, "queue_1.sqlite"), "shadow queue db"); + const layout = yield* resolveCodexHomeLayout( + decodeCodexSettings({ homePath: sharedHome, shadowHomePath: shadowHome }), ); - expect(yield* readRegularFileContents([shadowRoot])).toContain( - "shadow logs_2.sqlite-wal", + + yield* materializeCodexShadowHome(layout); + + expect(yield* fileSystem.readFileString(path.join(shadowHome, "queue_1.sqlite"))).toBe( + "shadow queue db", ); + for (const name of ["queue_1.sqlite-wal", "queue_1.sqlite-shm", "orphan.sqlite-wal"]) { + expect(yield* fileSystem.exists(path.join(shadowHome, name))).toBe(false); + expect(yield* isSymlink(path.join(shadowHome, name))).toBe(false); + } }), ); }); diff --git a/apps/server/src/provider/Drivers/CodexHomeLayout.ts b/apps/server/src/provider/Drivers/CodexHomeLayout.ts index b0bb7df4e2d46..26facd9fb7d43 100644 --- a/apps/server/src/provider/Drivers/CodexHomeLayout.ts +++ b/apps/server/src/provider/Drivers/CodexHomeLayout.ts @@ -34,6 +34,19 @@ const SHADOW_LOCAL_ENTRY_NAMES = new Set(["log", "memories", "tmp", ".DS_Store"] // Codex creates these runtime locks itself when it starts in a shadow home before the shared // home has one. They guard shared state, so a local copy is replaced with the shared link. const REPLACEABLE_SHARED_RUNTIME_ENTRIES = new Set(["mcp-oauth-locks", ".sqlite-maintenance.lock"]); +// A local copy of these defeats the shared home, so it is reported instead of kept. Any other +// regular shadow entry is state Codex created for this account (new releases keep adding +// databases such as queue_1.sqlite), and it stays local rather than disabling the provider. +const REQUIRED_SHARED_ENTRY_NAMES = new Set([...KNOWN_SHARED_DIRECTORIES, "config.toml"]); +const SQLITE_SIDECAR_SUFFIXES = ["-wal", "-shm", "-journal"] as const; + +// Groups a SQLite database with its sidecars, which must all live in the same home. +function sqliteFamilyName(entryName: string): string { + const suffix = SQLITE_SIDECAR_SUFFIXES.find( + (candidate) => entryName.length > candidate.length && entryName.endsWith(candidate), + ); + return suffix === undefined ? entryName : entryName.slice(0, -suffix.length); +} function resolveHomePath(path: Path.Path, home: string, value: string | undefined): string { const expanded = @@ -359,25 +372,56 @@ export const materializeCodexShadowHome = Effect.fn("materializeCodexShadowHome" { concurrency: "unbounded" }, ); - const sharedEntryNames = yield* fileSystem.readDirectory(layout.sharedHomePath).pipe( - Effect.catchTags({ - PlatformError: (cause) => - new CodexShadowHomeFileSystemError({ - sharedHomePath: layout.sharedHomePath, - effectiveHomePath, - operation: "readDirectory", - path: layout.sharedHomePath, - cause, - }), - }), - ); + const readDirectory = (directoryPath: string) => + fileSystem.readDirectory(directoryPath).pipe( + Effect.catchTags({ + PlatformError: (cause) => + new CodexShadowHomeFileSystemError({ + sharedHomePath: layout.sharedHomePath, + effectiveHomePath, + operation: "readDirectory", + path: directoryPath, + cause, + }), + }), + ); + const [sharedEntryNames, shadowEntryNames] = yield* Effect.all([ + readDirectory(layout.sharedHomePath), + readDirectory(effectiveHomePath), + ]); + const isSharable = (entryName: string) => + !PRIVATE_ENTRY_NAMES.has(entryName) && !SHADOW_LOCAL_ENTRY_NAMES.has(entryName); const entries = new Set(KNOWN_SHARED_DIRECTORIES); for (const entryName of sharedEntryNames) { - if (!PRIVATE_ENTRY_NAMES.has(entryName) && !SHADOW_LOCAL_ENTRY_NAMES.has(entryName)) { + if (isSharable(entryName)) { entries.add(entryName); } } + // A family with any regular shadow member stays exactly as it is: linking the rest would + // pair a local database with shared sidecars, or the reverse, and only SQLite can merge them. + const shadowLocalFamilies = new Set(); + yield* Effect.forEach( + shadowEntryNames, + (entryName) => + isSharable(entryName) + ? readLinkState({ + fileSystem, + sharedHomePath: layout.sharedHomePath, + effectiveHomePath, + entryName, + linkPath: path.join(effectiveHomePath, entryName), + }).pipe( + Effect.map((state) => { + if (state._tag === "NotSymlink") { + shadowLocalFamilies.add(sqliteFamilyName(entryName)); + } + }), + ) + : Effect.void, + { discard: true }, + ); + yield* Effect.forEach( PRIVATE_ENTRY_NAMES, (entryName) => @@ -398,6 +442,19 @@ export const materializeCodexShadowHome = Effect.fn("materializeCodexShadowHome" if (PRIVATE_ENTRY_NAMES.has(entryName)) { return Effect.void; } + const familyName = sqliteFamilyName(entryName); + // Sidecars follow their base, so they are not linked next to a database the shadow home + // may create later. + if (familyName !== entryName && !entries.has(familyName)) { + return Effect.void; + } + if ( + shadowLocalFamilies.has(familyName) && + !REQUIRED_SHARED_ENTRY_NAMES.has(entryName) && + !REPLACEABLE_SHARED_RUNTIME_ENTRIES.has(entryName) + ) { + return Effect.void; + } return ensureSymlink({ fileSystem, sharedHomePath: layout.sharedHomePath, From c3fd7542373b59bc457b5493edf7cbc711fec361 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 22 Oct 2026 08:17:21 +0000 Subject: [PATCH 3/3] test(server): the linked-base WAL fixture names SQLite's Windows journal behavior Co-Authored-By: Claude Opus 5.5 --- apps/server/src/provider/Drivers/CodexHomeLayout.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/server/src/provider/Drivers/CodexHomeLayout.test.ts b/apps/server/src/provider/Drivers/CodexHomeLayout.test.ts index 676f1d46d9ab6..25b51fde78d06 100644 --- a/apps/server/src/provider/Drivers/CodexHomeLayout.test.ts +++ b/apps/server/src/provider/Drivers/CodexHomeLayout.test.ts @@ -415,8 +415,9 @@ it.layer(NodeServices.layer)("CodexHomeLayout", (it) => { yield* materializeCodexShadowHome(layout); expect(yield* isSymlink(path.join(shadowHome, "logs_2.sqlite"))).toBe(true); - // SQLite opens its journal next to the path it was given, so Codex writes regular - // sidecars beside the linked base database while the shared home has its own. + // SQLite's Windows VFS opens the journal next to the path it was given rather than + // the link target, so Codex there writes regular sidecars beside the linked base + // database while the shared home has its own. for (const suffix of ["-wal", "-shm"]) { const name = `logs_2.sqlite${suffix}`; yield* writeTextFile(path.join(sharedHome, name), `shared ${name}`);