Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions packages/codev-upload/src/db.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { existsSync } from "node:fs"
import { partToMarkdown, safeParse, unixToISO } from "./parts"
import { openSessionDb, type SessionDb } from "#sqlite"
import { projectFolderName, sessionDbPath } from "./paths"
import { networkVolumeFilter } from "./network-volumes"
import { buildSessionStats, type SessionStats, type StatsMessage } from "./stats"
import type { Message, Session } from "./types"

Expand Down Expand Up @@ -169,8 +170,10 @@ export function readAllSessions(dbPath = sessionDbPath()): Session[] {

// Statistics for every top-level session in the store, subagent descendants
// folded in. Same session selection and project keying as readAllSessions, so
// a session's stats and its markdown land under one project.
export function readAllSessionStats(dbPath = sessionDbPath()): SessionStats[] {
// a session's stats and its markdown land under one project — except that a
// project on a network volume is named as stored rather than resolved, since
// this runs at every launch and must not touch the share.
export function readAllSessionStats(dbPath = sessionDbPath(), onNetworkVolume = networkVolumeFilter()): SessionStats[] {
if (!existsSync(dbPath)) return []
const db = openSessionDb(dbPath)
try {
Expand All @@ -195,7 +198,7 @@ export function readAllSessionStats(dbPath = sessionDbPath()): SessionStats[] {
const stats = buildSessionStats({
id: row.id,
origin: "local",
project: projectFolderName(projectDir),
project: projectFolderName(projectDir, { resolve: !onNetworkVolume(projectDir) }),
messages: readStatsMessages(db, row.id),
descendants: descendantsOf(row.id).map((id) => readStatsMessages(db, id)),
})
Expand Down
47 changes: 47 additions & 0 deletions packages/codev-upload/src/network-volumes.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
import { execFileSync } from "node:child_process"

// macOS asks the user before an app — or any process it spawns — touches a
// file on a network volume ("… would like to access files on a network
// volume"), and an unsigned build cannot keep the answer across updates. The
// stats push reads everything it sends from the local session store; the
// only thing it would touch on a share is the project directory it resolves
// for naming. This tells it which directories to leave alone, from the
// kernel's mount table — listing mounts never opens one.

const NETWORK_FS_TYPES = new Set(["smbfs", "afpfs", "nfs", "webdav", "ftp", "autofs"])

// The data volume's firmlinked and synthetic paths ("/System/Volumes/Data/home"
// is also "/home") reach the same mounts under their short names.
const DATA_VOLUME = "/System/Volumes/Data"

/**
* Returns a predicate saying whether a directory lives on a network mount
* (SMB, AFP, NFS, WebDAV, FTP, or an automounted map such as /net or the
* network home). Only macOS has the prompt this avoids; elsewhere, and when
* the mount table cannot be read, nothing counts as a network path.
*/
export function networkVolumeFilter(
platform: NodeJS.Platform = process.platform,
readMounts: () => string = () => execFileSync("/sbin/mount", { encoding: "utf8", timeout: 5000 }),
): (dir: string) => boolean {
if (platform !== "darwin") return () => false
let table: string
try {
table = readMounts()
} catch {
return () => false
}
const roots = parseNetworkMounts(table).flatMap((mount) =>
mount.startsWith(`${DATA_VOLUME}/`) ? [mount, mount.slice(DATA_VOLUME.length)] : [mount],
)
return (dir) => roots.some((root) => dir === root || dir.startsWith(`${root}/`))
}

/** Mount points of network filesystems in `mount` output ("<source> on <point> (<type>, …)"). */
export function parseNetworkMounts(table: string): string[] {
return table.split("\n").flatMap((line) => {
const match = / on (.+) \(([^,)]+)/.exec(line)
if (!match?.[1] || !match[2] || !NETWORK_FS_TYPES.has(match[2].trim())) return []
return [match[1]]
})
}
28 changes: 17 additions & 11 deletions packages/codev-upload/src/paths.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,17 +74,12 @@ export function statsSentPath(): string {

// Maps a project directory to its per-project export subfolder. Byte-for-byte
// port of the hub's projectFolderName (src/lib/paths.ts) — the folder name is
// half of the absolute local_file_path the backend dedups on.
export function projectFolderName(dir: string): string {
let real: string
try {
real = realpathSync(dir)
} catch {
// The stored project dir may no longer exist (or be a POSIX-style Windows
// path). Normalize to platform separators + drive-letter case so the
// mangled name still matches what the hub computes from a live cwd.
real = platformNormalize(dir)
}
// half of the absolute local_file_path the backend dedups on. `resolve: false`
// names the directory as stored instead of resolving it on disk — for callers
// that must not touch it (a project on a network volume; see
// network-volumes.ts). Paths without symlinks name the same either way.
export function projectFolderName(dir: string, options: { resolve?: boolean } = {}): string {
const real = options.resolve === false ? platformNormalize(dir) : resolveDir(dir)
const h = home()
if (real === h) return "home"
let stripped = real
Expand All @@ -101,6 +96,17 @@ export function projectFolderName(dir: string): string {
// ("C:\Users\me\project"). Without unifying, the home-prefix strip above
// misses on Windows and the two sides export to different folders — which the
// backend then treats as different conversations.
function resolveDir(dir: string): string {
try {
return realpathSync(dir)
} catch {
// The stored project dir may no longer exist (or be a POSIX-style Windows
// path). Normalize to platform separators + drive-letter case so the
// mangled name still matches what the hub computes from a live cwd.
return platformNormalize(dir)
}
}

function platformNormalize(p: string): string {
const drive = p.replace(/^([a-zA-Z]):/, (_m, d: string) => `${d.toUpperCase()}:`)
if (sep === "\\") return drive.replace(/\//g, "\\")
Expand Down
7 changes: 5 additions & 2 deletions packages/codev-upload/src/push.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { readCodevCodeBaseUrl } from "./config"
import { readAllSessionStats } from "./db"
import { statsLog } from "./log"
import { sessionDbPath, statsSentPath } from "./paths"
import { networkVolumeFilter } from "./network-volumes"
import { readSessionStatsFromServer, type ServerSource } from "./server-source"
import { STATS_VERSION, type SessionStats } from "./stats"

Expand Down Expand Up @@ -109,15 +110,17 @@ export async function pushSessionStats(options: PushOptions): Promise<PushSummar
// Local store first, then each sandbox. A sandbox that cannot be read (went
// down mid-run) is skipped — its sessions push on the next launch.
async function collectStats(servers: ServerSource[], fetchImpl: typeof fetch): Promise<SessionStats[]> {
// Read the mount table once per push, not once per source.
const onNetworkVolume = networkVolumeFilter()
const records: SessionStats[] = []
try {
records.push(...readAllSessionStats(sessionDbPath()))
records.push(...readAllSessionStats(sessionDbPath(), onNetworkVolume))
} catch (err) {
statsLog.warn("session store read failed", { err, extra: { db_path: sessionDbPath() } })
}
for (const server of servers) {
try {
records.push(...(await readSessionStatsFromServer(server, fetchImpl)))
records.push(...(await readSessionStatsFromServer(server, fetchImpl, onNetworkVolume)))
} catch (err) {
statsLog.warn("sandbox server read failed", { err })
}
Expand Down
6 changes: 5 additions & 1 deletion packages/codev-upload/src/server-source.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { networkVolumeFilter } from "./network-volumes"
import { projectFolderName } from "./paths"
import { buildSessionStats, type SessionStats, type StatsMessage } from "./stats"

Expand Down Expand Up @@ -94,6 +95,7 @@ async function readMessages(
export async function readSessionStatsFromServer(
source: ServerSource,
fetchImpl: typeof fetch = fetch,
onNetworkVolume = networkVolumeFilter(),
): Promise<SessionStats[]> {
const all = await listAllSessions(source, fetchImpl)
const childrenOf = new Map<string, ApiSessionInfo[]>()
Expand All @@ -104,7 +106,9 @@ export async function readSessionStatsFromServer(
const descendantsOf = (id: string): ApiSessionInfo[] =>
(childrenOf.get(id) ?? []).flatMap((child) => [child, ...descendantsOf(child.id)])

const project = projectFolderName(source.projectDir)
// The host project dir names the project; one on a share is not resolved
// (see network-volumes.ts).
const project = projectFolderName(source.projectDir, { resolve: !onNetworkVolume(source.projectDir) })
const stats: SessionStats[] = []
for (const info of all) {
if (info.parentID) continue
Expand Down
118 changes: 118 additions & 0 deletions packages/codev-upload/test/network-volumes.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
import { Database } from "bun:sqlite"
import { afterEach, beforeEach, describe, expect, test } from "bun:test"
import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync } from "node:fs"
import { tmpdir } from "node:os"
import { join } from "node:path"
import { readAllSessionStats } from "../src/db"
import { networkVolumeFilter, parseNetworkMounts } from "../src/network-volumes"
import { projectFolderName, sessionDbPath } from "../src/paths"

// Trimmed `mount` output from a Mac with an SMB share, an NFS export, the
// default automounts, and local volumes.
const MOUNT_TABLE = [
"/dev/disk3s1s1 on / (apfs, sealed, local, read-only, journaled)",
"/dev/disk3s5 on /System/Volumes/Data (apfs, local, journaled, nobrowse, protect, root data)",
"map auto_home on /System/Volumes/Data/home (autofs, automounted, nobrowse)",
"map -hosts on /net (autofs, nosuid, automounted, nobrowse)",
"//alice@files.corp/Team Share on /Volumes/Team Share (smbfs, nodev, nosuid, mounted by alice)",
"nas:/export/src on /Volumes/src (nfs, nodev, nosuid, mounted by alice)",
"/dev/disk5s1 on /Volumes/USB Drive (msdos, local, nodev, nosuid, noowners)",
].join("\n")

describe("parseNetworkMounts", () => {
test("keeps network and automounted filesystems, drops local volumes", () => {
expect(parseNetworkMounts(MOUNT_TABLE)).toEqual([
"/System/Volumes/Data/home",
"/net",
"/Volumes/Team Share",
"/Volumes/src",
])
})
})

describe("networkVolumeFilter", () => {
test("matches paths on or under a network mount, including the data volume's short names", () => {
const onNetworkVolume = networkVolumeFilter("darwin", () => MOUNT_TABLE)
expect(onNetworkVolume("/Volumes/Team Share")).toBe(true)
expect(onNetworkVolume("/Volumes/Team Share/app")).toBe(true)
expect(onNetworkVolume("/Volumes/src/repo")).toBe(true)
expect(onNetworkVolume("/net/nas/export")).toBe(true)
// /home is the synthetic alias of /System/Volumes/Data/home.
expect(onNetworkVolume("/home/alice/project")).toBe(true)
expect(onNetworkVolume("/Volumes/USB Drive/project")).toBe(false)
expect(onNetworkVolume("/Volumes/Team Shared/project")).toBe(false)
expect(onNetworkVolume("/Users/alice/project")).toBe(false)
})

test("off macOS, or without a readable mount table, nothing counts as a network path", () => {
expect(networkVolumeFilter("linux", () => MOUNT_TABLE)("/Volumes/src/repo")).toBe(false)
const unreadable = networkVolumeFilter("darwin", () => {
throw new Error("mount: not found")
})
expect(unreadable("/Volumes/src/repo")).toBe(false)
})
})

describe("naming without resolving", () => {
let tempDir: string
let originalTestHome: string | undefined
let originalXdgData: string | undefined

beforeEach(() => {
// Resolved up front: macOS's temp dir is itself behind a symlink (/var ->
// /private/var), and only the project link should differ when resolved.
tempDir = realpathSync(mkdtempSync(join(tmpdir(), "codev-upload-netvol-")))
originalTestHome = process.env.CODEV_TEST_HOME
originalXdgData = process.env.XDG_DATA_HOME
process.env.CODEV_TEST_HOME = tempDir
process.env.XDG_DATA_HOME = join(tempDir, "share")
})

afterEach(() => {
if (originalTestHome === undefined) delete process.env.CODEV_TEST_HOME
else process.env.CODEV_TEST_HOME = originalTestHome
if (originalXdgData === undefined) delete process.env.XDG_DATA_HOME
else process.env.XDG_DATA_HOME = originalXdgData
rmSync(tempDir, { recursive: true, force: true })
})

// A symlinked project dir is the observable difference: resolving follows
// the link, naming as stored does not touch the filesystem at all.
function linkedProject() {
const target = join(tempDir, "real-proj")
const link = join(tempDir, "linked-proj")
mkdirSync(target, { recursive: true })
symlinkSync(target, link)
return link
}

test("projectFolderName names the stored path when told not to resolve", () => {
const link = linkedProject()
expect(projectFolderName(link)).toBe("real-proj")
expect(projectFolderName(link, { resolve: false })).toBe("linked-proj")
})

test("readAllSessionStats leaves network projects unresolved", () => {
const link = linkedProject()
mkdirSync(join(tempDir, "share", "codev"), { recursive: true })
const db = new Database(sessionDbPath(), { create: true })
db.run("CREATE TABLE project (id TEXT PRIMARY KEY, worktree TEXT)")
db.run(
"CREATE TABLE session (id TEXT PRIMARY KEY, project_id TEXT, title TEXT, directory TEXT, parent_id TEXT, time_created INTEGER, time_updated INTEGER)",
)
db.run("CREATE TABLE message (id TEXT PRIMARY KEY, session_id TEXT, time_created INTEGER, data TEXT)")
db.run("CREATE TABLE part (id TEXT PRIMARY KEY, session_id TEXT, message_id TEXT, time_created INTEGER, data TEXT)")
db.run("INSERT INTO project (id, worktree) VALUES ('prj1', ?)", [link])
db.run(
"INSERT INTO session (id, project_id, title, directory, parent_id, time_created, time_updated) VALUES ('ses1', 'prj1', 't', ?, NULL, 1, 1)",
[link],
)
db.run("INSERT INTO message (id, session_id, time_created, data) VALUES ('m1', 'ses1', 1, ?)", [
JSON.stringify({ role: "assistant", modelID: "m", time: { created: 1_700_000_000_000 } }),
])
db.close()

expect(readAllSessionStats(sessionDbPath(), () => false)[0]?.project).toBe("real-proj")
expect(readAllSessionStats(sessionDbPath(), (dir) => dir === link)[0]?.project).toBe("linked-proj")
})
})
Loading