From bafbb213e04f2040c53b2c55197c5ef292313649 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 12 Nov 2026 13:36:07 -0800 Subject: [PATCH 01/15] refactor(server): share Claude and Pi transcript projections Co-Authored-By: Claude Fable 5.2 --- .../server/src/project/AgentSessionScanner.ts | 164 ++-------------- .../src/project/agentTranscripts.test.ts | 57 ++++++ apps/server/src/project/agentTranscripts.ts | 179 ++++++++++++++++++ 3 files changed, 254 insertions(+), 146 deletions(-) create mode 100644 apps/server/src/project/agentTranscripts.test.ts create mode 100644 apps/server/src/project/agentTranscripts.ts diff --git a/apps/server/src/project/AgentSessionScanner.ts b/apps/server/src/project/AgentSessionScanner.ts index a3ebadf8d1122..b5a507c6c1073 100644 --- a/apps/server/src/project/AgentSessionScanner.ts +++ b/apps/server/src/project/AgentSessionScanner.ts @@ -1,3 +1,16 @@ +import { + TranscriptRecord, + conversationItem, + activePiBranch, + emptyClaudeFields, + extractText, + foldClaudeFields, + isHiddenClaudeRecord, + isPiConversationRecord, + isTextContentBlock, + type ClaudeFields, + type DecodedTranscriptRecord, +} from "./agentTranscripts.ts"; import { resolveClaudeRuntime } from "../provider/Drivers/ClaudeHome.ts"; import * as NodeOS from "node:os"; import * as NodeSqlite from "node:sqlite"; @@ -103,52 +116,10 @@ const GIT_FILE_READ_CONCURRENCY = 16; const BLOCKED_GIT_FILE_RETRY_MS = 5 * 60 * 1000; const MAX_BLOCKED_GIT_FILE_PATHS = 1024; -const TranscriptContentBlock = Schema.Struct({ - type: Schema.optional(Schema.String), - text: Schema.optional(Schema.String), -}); - -const TranscriptMessage = Schema.Struct({ - role: Schema.optional(Schema.String), - content: Schema.optional(Schema.Union([Schema.String, Schema.Array(TranscriptContentBlock)])), - model: Schema.optional(Schema.String), -}); - const CodexTurnMetadata = Schema.Struct({ turn_id: Schema.optional(Schema.Union([Schema.String, Schema.Null])), }); -const TranscriptRecord = Schema.Struct({ - type: Schema.optional(Schema.String), - // Pi entries form a tree through id/parentId; session_info entries carry its name. - id: Schema.optional(Schema.String), - parentId: Schema.optional(Schema.NullOr(Schema.String)), - name: Schema.optional(Schema.String), - timestamp: Schema.optional(Schema.String), - cwd: Schema.optional(Schema.String), - sessionId: Schema.optional(Schema.String), - aiTitle: Schema.optional(Schema.String), - customTitle: Schema.optional(Schema.String), - isSidechain: Schema.optional(Schema.Boolean), - isMeta: Schema.optional(Schema.Boolean), - isCompactSummary: Schema.optional(Schema.Boolean), - message: Schema.optional(TranscriptMessage), - payload: Schema.optional( - Schema.Struct({ - id: Schema.optional(Schema.String), - session_id: Schema.optional(Schema.String), - type: Schema.optional(Schema.String), - role: Schema.optional(Schema.String), - message: Schema.optional(Schema.String), - model: Schema.optional(Schema.String), - cwd: Schema.optional(Schema.String), - history_base: Schema.optional(Schema.Unknown), - content: Schema.optional(Schema.Array(TranscriptContentBlock)), - internal_chat_message_metadata_passthrough: Schema.optional(Schema.Unknown), - }), - ), -}); - const CodexSessionIndexEntry = Schema.Struct({ id: Schema.String, thread_name: Schema.String, @@ -185,8 +156,6 @@ const decodeCodexSessionIndexEntry = Schema.decodeUnknownOption( Schema.fromJsonString(CodexSessionIndexEntry), ); -type DecodedTranscriptRecord = typeof TranscriptRecord.Type; - interface AgentSessionTranscriptMetadata { readonly source: AgentSessionSource; readonly providerInstanceId: ProviderInstanceId; @@ -290,22 +259,6 @@ function splitTranscriptRecords(contents: string, limit: number): string[] { return records.split("\n", limit); } -function isTextContentBlock(block: typeof TranscriptContentBlock.Type): boolean { - return block.type === "text" || block.type === "input_text" || block.type === "output_text"; -} - -function extractText( - content: string | ReadonlyArray | undefined, -): string { - if (typeof content === "string") return content.trim(); - if (content === undefined) return ""; - return content - .filter(isTextContentBlock) - .map((block) => block.text?.trim() ?? "") - .filter((text) => text.length > 0) - .join("\n"); -} - function normalizeTimestamp(value: string | undefined, fallback: string): string { if (value === undefined) return fallback; const parsed = DateTime.make(value); @@ -324,40 +277,6 @@ function codexTurnId(metadata: unknown): string | null { return decoded.value.turn_id; } -function isHiddenClaudeRecord(record: DecodedTranscriptRecord): boolean { - return record.isSidechain === true || record.isMeta === true || record.isCompactSummary === true; -} - -// Claude repeats session fields on records that need not be retained. -interface ClaudeFields { - readonly sessionId: string | undefined; - readonly title: string | undefined; - readonly customTitle: string | null; - readonly model: string | undefined; -} - -const emptyClaudeFields: ClaudeFields = { - sessionId: undefined, - title: undefined, - customTitle: null, - model: undefined, -}; - -function foldClaudeFields(fields: ClaudeFields, record: DecodedTranscriptRecord): ClaudeFields { - if (isHiddenClaudeRecord(record)) return fields; - const model = record.message?.model?.trim(); - return { - sessionId: record.sessionId?.trim() || fields.sessionId, - title: record.aiTitle?.trim() || fields.title, - // Claude flushes the AI title after the user's rename; an empty rename clears it. - customTitle: - record.customTitle === undefined ? fields.customTitle : record.customTitle.trim() || null, - // Claude uses this sentinel for local error responses. It is not a - // model ID that can be selected when the imported session resumes. - model: model && model !== "" ? model : fields.model, - }; -} - // Codex records setup context as user text ahead of the real request. Skip // these leading blocks when deriving a title; imported messages keep them. const CODEX_TITLE_PREAMBLE = @@ -394,39 +313,6 @@ function deriveTitle(source: AgentSessionSource, text: string): string | undefin return request.trim().split("\n")[0]?.slice(0, 100).trim() || undefined; } -// Pi keeps abandoned branches in the file; its active branch runs from the last entry to the root. -function activePiBranch( - records: ReadonlyArray, -): ReadonlyArray { - const entries = records.filter((record) => record.type !== "session"); - const leaf = entries.at(-1); - // Version 1 sessions predate the tree and are linear. - if (leaf?.id === undefined) return entries; - const byId = new Map( - entries.flatMap((entry) => (entry.id === undefined ? [] : [[entry.id, entry] as const])), - ); - const branch: Array = []; - for ( - let entry: DecodedTranscriptRecord | undefined = leaf; - entry !== undefined && branch.length < entries.length; - entry = entry.parentId ? byId.get(entry.parentId) : undefined - ) { - branch.push(entry); - } - return branch.toReversed(); -} - -function isPiConversationRecord( - record: DecodedTranscriptRecord, -): record is DecodedTranscriptRecord & { - readonly message: typeof TranscriptMessage.Type & { readonly role: "user" | "assistant" }; -} { - return ( - record.type === "message" && - (record.message?.role === "user" || record.message?.role === "assistant") - ); -} - export function parseAgentSessionTranscript( input: AgentSessionTranscriptMetadata & { readonly contents: string; @@ -556,26 +442,12 @@ function parseAgentSessionRecords( let recordIndex = -1; for (const record of input.source === "pi" ? activePiBranch(records) : records) { recordIndex += 1; - if (input.source === "pi") { - if (!isPiConversationRecord(record)) continue; - const text = extractText(record.message.content); - if (text.length === 0) continue; - retainMessage({ - role: record.message.role, - text, - createdAt: normalizeTimestamp(record.timestamp, fallbackTimestamp), - codexResponseUser: false, - }); - continue; - } - if (input.source === "claudeAgent") { - if (isHiddenClaudeRecord(record)) continue; - if (record.type !== "user" && record.type !== "assistant") continue; - const text = extractText(record.message?.content); - if (text.length === 0) continue; + if (input.source === "pi" || input.source === "claudeAgent") { + const item = conversationItem(record, input.source); + if (item === null) continue; retainMessage({ - role: record.type, - text, + role: item.role, + text: item.text, createdAt: normalizeTimestamp(record.timestamp, fallbackTimestamp), codexResponseUser: false, }); diff --git a/apps/server/src/project/agentTranscripts.test.ts b/apps/server/src/project/agentTranscripts.test.ts new file mode 100644 index 0000000000000..8576ef713547e --- /dev/null +++ b/apps/server/src/project/agentTranscripts.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from "vitest"; +import * as Schema from "effect/Schema"; +import { activePiBranch, TranscriptRecord, transcriptItems } from "./agentTranscripts.ts"; + +const decode = Schema.decodeUnknownSync(TranscriptRecord); + +describe("transcript evidence", () => { + it("keeps native Claude identities and tool evidence without thinking or hidden records", () => { + const record = decode({ + uuid: "native-uuid", + type: "assistant", + message: { + content: [ + { type: "thinking", text: "private reasoning" }, + { type: "text", text: "Visible answer" }, + { type: "tool_use", name: "read", input: { path: "src/index.ts" } }, + { type: "tool_result", content: [{ type: "text", text: "File contents" }] }, + ], + }, + }); + expect(transcriptItems(record, "claudeAgent")).toEqual([ + { id: "native-uuid", role: "assistant", text: "Visible answer" }, + { id: "native-uuid/tool/2", role: "tool", text: 'read\n{"path":"src/index.ts"}' }, + { id: "native-uuid/tool/3", role: "tool", text: "File contents" }, + ]); + for (const flag of ["isSidechain", "isMeta", "isCompactSummary"]) + expect(transcriptItems(decode({ ...record, [flag]: true }), "claudeAgent")).toEqual([]); + }); + + it("keeps only the active Pi lineage with its original entry IDs", () => { + const records = [ + { type: "session", id: "session" }, + { + type: "message", + id: "first", + parentId: null, + message: { role: "user", content: "Question" }, + }, + { + type: "message", + id: "abandoned", + parentId: "first", + message: { role: "assistant", content: "Old answer" }, + }, + { + type: "message", + id: "active", + parentId: "first", + message: { role: "assistant", content: "New answer" }, + }, + ].map(decode); + expect(activePiBranch(records).flatMap((r) => transcriptItems(r, "pi"))).toEqual([ + { id: "first", role: "user", text: "Question" }, + { id: "active", role: "assistant", text: "New answer" }, + ]); + }); +}); diff --git a/apps/server/src/project/agentTranscripts.ts b/apps/server/src/project/agentTranscripts.ts new file mode 100644 index 0000000000000..24f66e996dcd1 --- /dev/null +++ b/apps/server/src/project/agentTranscripts.ts @@ -0,0 +1,179 @@ +import * as Schema from "effect/Schema"; + +export const TranscriptContentBlock = Schema.Struct({ + type: Schema.optional(Schema.String), + text: Schema.optional(Schema.String), + name: Schema.optional(Schema.String), + input: Schema.optional(Schema.Unknown), + content: Schema.optional( + Schema.Union([ + Schema.String, + Schema.Array( + Schema.Struct({ + type: Schema.optional(Schema.String), + text: Schema.optional(Schema.String), + }), + ), + ]), + ), +}); + +export const TranscriptMessage = Schema.Struct({ + role: Schema.optional(Schema.String), + content: Schema.optional(Schema.Union([Schema.String, Schema.Array(TranscriptContentBlock)])), + model: Schema.optional(Schema.String), +}); + +export const TranscriptRecord = Schema.Struct({ + type: Schema.optional(Schema.String), + uuid: Schema.optional(Schema.String), + entrypoint: Schema.optional(Schema.String), + id: Schema.optional(Schema.String), + parentId: Schema.optional(Schema.NullOr(Schema.String)), + name: Schema.optional(Schema.String), + timestamp: Schema.optional(Schema.String), + cwd: Schema.optional(Schema.String), + sessionId: Schema.optional(Schema.String), + aiTitle: Schema.optional(Schema.String), + customTitle: Schema.optional(Schema.String), + isSidechain: Schema.optional(Schema.Boolean), + isMeta: Schema.optional(Schema.Boolean), + isCompactSummary: Schema.optional(Schema.Boolean), + message: Schema.optional(TranscriptMessage), + payload: Schema.optional( + Schema.Struct({ + id: Schema.optional(Schema.String), + session_id: Schema.optional(Schema.String), + type: Schema.optional(Schema.String), + role: Schema.optional(Schema.String), + message: Schema.optional(Schema.String), + model: Schema.optional(Schema.String), + cwd: Schema.optional(Schema.String), + history_base: Schema.optional(Schema.Unknown), + content: Schema.optional(Schema.Array(TranscriptContentBlock)), + internal_chat_message_metadata_passthrough: Schema.optional(Schema.Unknown), + }), + ), +}); + +export function isTextContentBlock(block: typeof TranscriptContentBlock.Type): boolean { + return block.type === "text" || block.type === "input_text" || block.type === "output_text"; +} + +export function extractText( + content: string | ReadonlyArray | undefined, +): string { + if (typeof content === "string") return content.trim(); + if (content === undefined) return ""; + return content + .filter(isTextContentBlock) + .map((block) => block.text?.trim() ?? "") + .filter((text) => text.length > 0) + .join("\n"); +} + +export function isHiddenClaudeRecord(record: DecodedTranscriptRecord): boolean { + return record.isSidechain === true || record.isMeta === true || record.isCompactSummary === true; +} + +export interface ClaudeFields { + readonly sessionId: string | undefined; + readonly title: string | undefined; + readonly customTitle: string | null; + readonly model: string | undefined; +} + +export const emptyClaudeFields: ClaudeFields = { + sessionId: undefined, + title: undefined, + customTitle: null, + model: undefined, +}; + +export function foldClaudeFields( + fields: ClaudeFields, + record: DecodedTranscriptRecord, +): ClaudeFields { + if (isHiddenClaudeRecord(record)) return fields; + const model = record.message?.model?.trim(); + return { + sessionId: record.sessionId?.trim() || fields.sessionId, + title: record.aiTitle?.trim() || fields.title, + // Claude flushes the AI title after the user's rename; an empty rename clears it. + customTitle: + record.customTitle === undefined ? fields.customTitle : record.customTitle.trim() || null, + model: model && model !== "" ? model : fields.model, + }; +} + +// Pi keeps abandoned branches in the file; its active branch runs from the last entry to the root. +export function activePiBranch( + records: ReadonlyArray, +): ReadonlyArray { + const entries = records.filter((record) => record.type !== "session"); + const leaf = entries.at(-1); + // Version 1 sessions predate the tree and are linear. + if (leaf?.id === undefined) return entries; + const byId = new Map( + entries.flatMap((entry) => (entry.id === undefined ? [] : [[entry.id, entry] as const])), + ); + const branch: Array = []; + const seen = new Set(); + for ( + let entry: DecodedTranscriptRecord | undefined = leaf; + entry !== undefined && branch.length < entries.length; + entry = entry.parentId ? byId.get(entry.parentId) : undefined + ) { + if (entry.id && seen.has(entry.id)) break; + if (entry.id) seen.add(entry.id); + branch.push(entry); + } + return branch.toReversed(); +} + +export type DecodedTranscriptRecord = typeof TranscriptRecord.Type; + +export interface TranscriptItem { + readonly id: string; + readonly role: "user" | "assistant" | "tool"; + readonly text: string; +} + +export function conversationItem(record: DecodedTranscriptRecord, source: "claudeAgent" | "pi") { + const role = source === "pi" ? record.message?.role : record.type; + if (source === "claudeAgent" && isHiddenClaudeRecord(record)) return null; + if (source === "pi" && record.type !== "message") return null; + if (role !== "user" && role !== "assistant") return null; + const text = extractText(record.message?.content); + return text ? { id: source === "pi" ? record.id : record.uuid, role, text } : null; +} + +export function transcriptItems( + record: DecodedTranscriptRecord, + source: "claudeAgent" | "pi", +): ReadonlyArray { + if (source === "claudeAgent" && isHiddenClaudeRecord(record)) return []; + const nativeId = source === "pi" ? record.id : record.uuid; + if (!nativeId) return []; + const conversation = conversationItem(record, source); + const items: TranscriptItem[] = conversation ? [{ ...conversation, id: nativeId }] : []; + const content = record.message?.content; + if (typeof content === "string" || content === undefined) return items; + for (const [index, block] of content.entries()) { + const text = + block.type === "tool_result" + ? extractText(block.content) + : block.type === "tool_use" + ? [block.name, JSON.stringify(block.input)].filter(Boolean).join("\n") + : ""; + if (text) items.push({ id: `${nativeId}/tool/${index}`, role: "tool", text }); + } + return items; +} + +export function isPiConversationRecord(record: DecodedTranscriptRecord): boolean { + return ( + record.type === "message" && + (record.message?.role === "user" || record.message?.role === "assistant") + ); +} From d1ca88013b1a8e749c890150e64559bf009ff9be Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 12 Nov 2026 13:38:52 -0800 Subject: [PATCH 02/15] feat(contracts): describe discovered external history sources Co-Authored-By: Claude Fable 5.2 --- .../contracts/src/externalHistory.test.ts | 94 +++++++++++++++++++ packages/contracts/src/externalHistory.ts | 21 ++++- 2 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 packages/contracts/src/externalHistory.test.ts diff --git a/packages/contracts/src/externalHistory.test.ts b/packages/contracts/src/externalHistory.test.ts new file mode 100644 index 0000000000000..b5d290c0ce234 --- /dev/null +++ b/packages/contracts/src/externalHistory.test.ts @@ -0,0 +1,94 @@ +import * as Schema from "effect/Schema"; +import { describe, expect, it } from "vite-plus/test"; +import { ExternalHistoryRequest, ExternalHistoryResponse } from "./externalHistory.ts"; + +const decode = Schema.decodeUnknownSync(ExternalHistoryResponse); +const encode = Schema.encodeSync(ExternalHistoryResponse); +const decodeRequest = Schema.decodeUnknownSync(ExternalHistoryRequest); +const legacySources = Schema.Struct({ + operation: Schema.Literal("sources"), + enabled: Schema.Boolean, + sources: Schema.Array( + Schema.Struct({ + id: Schema.String, + label: Schema.String, + kind: Schema.Literals(["codex", "chatgpt_export"]), + }), + ), +}); + +const decodeLegacy = Schema.decodeUnknownSync(legacySources); + +describe("external history catalogs", () => { + it("defaults legacy-server capabilities without inferring them from the source kind", () => { + expect( + decode({ + operation: "sources", + enabled: true, + sources: [{ id: "old", label: "Codex", kind: "codex" }], + }), + ).toEqual({ + operation: "sources", + enabled: true, + unsupported: [], + sources: [ + { + id: "old", + label: "Codex", + kind: "codex", + status: "ready", + capabilities: { cwdFilter: false, archived: false }, + }, + ], + }); + expect(decodeRequest({ operation: "sources" })).toEqual({ + operation: "sources", + }); + expect(decodeRequest({ operation: "sources", catalogVersion: 2 })).toEqual({ + operation: "sources", + catalogVersion: 2, + }); + }); + + it("round-trips the new catalog while a legacy catalog still decodes with the old source union", () => { + const response = decode({ + operation: "sources", + enabled: true, + sources: [ + { + id: "claude", + label: "Claude Code", + kind: "local_transcript", + provider: "claudeAgent", + capabilities: { cwdFilter: true, archived: false }, + status: "ready", + }, + { + id: "opencode", + label: "OpenCode", + kind: "local_transcript", + provider: "opencode", + capabilities: { cwdFilter: true, archived: false }, + status: "unsupported_format", + message: "Unsupported OpenCode history format.", + }, + { + id: "codex", + label: "Codex", + kind: "codex", + capabilities: { cwdFilter: true, archived: true }, + status: "unavailable", + }, + ], + unsupported: [{ provider: "grok", label: "Grok", reason: "No verified local reader." }], + }); + expect(decode(encode(response))).toEqual(response); + if (response.operation !== "sources") throw new Error("Wrong response"); + expect( + decodeLegacy({ + ...response, + sources: response.sources.filter((s) => s.kind !== "local_transcript"), + }).sources, + ).toHaveLength(1); + }); +}); diff --git a/packages/contracts/src/externalHistory.ts b/packages/contracts/src/externalHistory.ts index 7db0f00e10149..78895365246d0 100644 --- a/packages/contracts/src/externalHistory.ts +++ b/packages/contracts/src/externalHistory.ts @@ -1,5 +1,6 @@ import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; +import { ProviderDriverKind } from "./providerInstance.ts"; import { NonNegativeInt } from "./baseSchemas.ts"; const Identifier = Schema.String.check(Schema.isNonEmpty(), Schema.isMaxLength(512)); @@ -23,7 +24,10 @@ export const ExternalHistorySourceRef = Schema.Struct({ export type ExternalHistorySourceRef = typeof ExternalHistorySourceRef.Type; export const ExternalHistoryRequest = Schema.Union([ - Schema.Struct({ operation: Schema.Literal("sources") }), + Schema.Struct({ + operation: Schema.Literal("sources"), + catalogVersion: Schema.optionalKey(Schema.Literal(2)), + }), Schema.Struct({ operation: Schema.Literal("search"), sourceId: Identifier, @@ -46,8 +50,18 @@ export type ExternalHistoryRequest = typeof ExternalHistoryRequest.Type; export const ExternalHistorySource = Schema.Struct({ id: Identifier, label: Schema.String, - kind: Schema.Literals(["codex", "chatgpt_export"]), + kind: Schema.Literals(["codex", "chatgpt_export", "local_transcript"]), + provider: Schema.optionalKey(ProviderDriverKind), + capabilities: Schema.Struct({ cwdFilter: Schema.Boolean, archived: Schema.Boolean }).pipe( + Schema.withDecodingDefault(Effect.succeed({ cwdFilter: false, archived: false })), + ), + status: Schema.Literals(["ready", "unavailable", "unsupported_format"]).pipe( + Schema.withDecodingDefault(Effect.succeed("ready" as const)), + ), + message: Schema.optionalKey(Schema.String), }); +export type ExternalHistorySource = typeof ExternalHistorySource.Type; + export const ExternalHistoryMatch = Schema.Struct({ ref: ExternalHistorySourceRef, threadTitle: Schema.String, @@ -64,6 +78,9 @@ export const ExternalHistoryResponse = Schema.Union([ operation: Schema.Literal("sources"), enabled: Schema.Boolean, sources: Schema.Array(ExternalHistorySource), + unsupported: Schema.Array( + Schema.Struct({ provider: ProviderDriverKind, label: Schema.String, reason: Schema.String }), + ).pipe(Schema.withDecodingDefault(Effect.succeed([]))), }), Schema.Struct({ operation: Schema.Literal("search"), From 5477f471c94413c3b0c19386abfb346de0ef9125 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 12 Nov 2026 13:45:56 -0800 Subject: [PATCH 03/15] refactor(server): unify local external history search and paging Co-Authored-By: Claude Fable 5.2 --- .../ExternalHistoryToolkit.test.ts | 11 +- .../src/project/agentTranscripts.test.ts | 4 +- .../ChatGptExportHistory.test.ts | 71 ++--- .../externalHistory/ChatGptExportHistory.ts | 281 +++++++----------- .../externalHistory/ExternalHistory.test.ts | 2 + .../externalHistory/ExternalHistory.ts | 121 ++++++-- .../project/externalHistory/LocalHistory.ts | 194 ++++++++++++ .../externalHistory/LocalHistoryStore.ts | 73 +++++ .../externalHistory/historySupport.test.ts | 18 ++ .../project/externalHistory/historySupport.ts | 89 ++++++ 10 files changed, 618 insertions(+), 246 deletions(-) create mode 100644 apps/server/src/project/externalHistory/LocalHistory.ts create mode 100644 apps/server/src/project/externalHistory/LocalHistoryStore.ts create mode 100644 apps/server/src/project/externalHistory/historySupport.test.ts create mode 100644 apps/server/src/project/externalHistory/historySupport.ts diff --git a/apps/server/src/mcp/toolkits/externalHistory/ExternalHistoryToolkit.test.ts b/apps/server/src/mcp/toolkits/externalHistory/ExternalHistoryToolkit.test.ts index 110eb687419d3..f24b23153614c 100644 --- a/apps/server/src/mcp/toolkits/externalHistory/ExternalHistoryToolkit.test.ts +++ b/apps/server/src/mcp/toolkits/externalHistory/ExternalHistoryToolkit.test.ts @@ -51,7 +51,16 @@ it.effect( ? Effect.succeed({ operation: "sources", enabled: true, - sources: [{ id: "source", label: "Codex", kind: "codex" }], + sources: [ + { + id: "source", + label: "Codex", + kind: "codex", + capabilities: { cwdFilter: true, archived: true }, + status: "ready", + }, + ], + unsupported: [], }) : Effect.fail( new ExternalHistoryError({ diff --git a/apps/server/src/project/agentTranscripts.test.ts b/apps/server/src/project/agentTranscripts.test.ts index 8576ef713547e..1efced40a40dd 100644 --- a/apps/server/src/project/agentTranscripts.test.ts +++ b/apps/server/src/project/agentTranscripts.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it } from "vite-plus/test"; import * as Schema from "effect/Schema"; import { activePiBranch, TranscriptRecord, transcriptItems } from "./agentTranscripts.ts"; @@ -48,7 +48,7 @@ describe("transcript evidence", () => { parentId: "first", message: { role: "assistant", content: "New answer" }, }, - ].map(decode); + ].map((record) => decode(record)); expect(activePiBranch(records).flatMap((r) => transcriptItems(r, "pi"))).toEqual([ { id: "first", role: "user", text: "Question" }, { id: "active", role: "assistant", text: "New answer" }, diff --git a/apps/server/src/project/externalHistory/ChatGptExportHistory.test.ts b/apps/server/src/project/externalHistory/ChatGptExportHistory.test.ts index 9aa92dcfd9d81..c5f3bbb115f9a 100644 --- a/apps/server/src/project/externalHistory/ChatGptExportHistory.test.ts +++ b/apps/server/src/project/externalHistory/ChatGptExportHistory.test.ts @@ -5,7 +5,8 @@ import { expect, it } from "@effect/vitest"; import type { ExternalHistoryResponse } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; -import { makeChatGptExportHistory } from "./ChatGptExportHistory.ts"; +import { makeChatGptExportStore } from "./ChatGptExportHistory.ts"; +import { makeLocalHistory } from "./LocalHistory.ts"; const conversation = (texts: ReadonlyArray, id = "conversation-1") => ({ id, @@ -30,15 +31,16 @@ const fixture = Effect.fn("ChatGptExportHistory.test.fixture")(function* (value: const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3-external-export-" }); const filePath = `${directory}/conversations.json`; yield* fs.writeFileString(filePath, JSON.stringify(value)); - return { history: yield* makeChatGptExportHistory, fs, filePath }; + const history = yield* makeLocalHistory; + return { history, store: yield* makeChatGptExportStore(filePath, history.cache), fs, filePath }; }); it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("ChatGPT export history", (it) => { it.effect("finds suffix matches and pages digest-verified exact Unicode text", () => Effect.gen(function* () { const text = "İ😀" + "a".repeat(3_000) + "NEEDLE [literal]"; - const { history, filePath } = yield* fixture([conversation([text])]); - const result = yield* history.search(filePath, { + const { history, store } = yield* fixture([conversation([text])]); + const result = yield* history.search(store, { operation: "search", sourceId: "export", query: "needle [literal]", @@ -51,7 +53,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("ChatGPT export h let offset: number | null = 0; while (offset !== null) { const page: Extract = yield* history.read( - filePath, + store, { operation: "read", ref, textOffset: offset, maxChars: 101 }, ); actual += page.text; @@ -59,11 +61,11 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("ChatGPT export h } expect(actual).toBe(text); const split = yield* history - .read(filePath, { operation: "read", ref, textOffset: 2 }) + .read(store, { operation: "read", ref, textOffset: 2 }) .pipe(Effect.flip); expect(split.reason).toBe("invalid_cursor"); const wrongDigest = yield* history - .read(filePath, { operation: "read", ref: { ...ref, digest: "0".repeat(64) } }) + .read(store, { operation: "read", ref: { ...ref, digest: "0".repeat(64) } }) .pipe(Effect.flip); expect(wrongDigest.reason).toBe("changed"); }).pipe(Effect.scoped), @@ -100,32 +102,33 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("ChatGPT export h }, conversation(["needle second conversation"], "conversation-2"), ]; - const { history, filePath } = yield* fixture(exportValue); + const { history, store } = yield* fixture(exportValue); const request = { operation: "search", sourceId: "export", query: "needle" } as const; - const page1 = yield* history.search(filePath, request); + const page1 = yield* history.search(store, request); expect(page1.matches).toEqual([]); expect(page1.examinedItems).toBe(20); expect(page1.nextCursor).not.toBeNull(); - const page2 = yield* history.search(filePath, { ...request, cursor: page1.nextCursor! }); + const page2 = yield* history.search(store, { ...request, cursor: page1.nextCursor! }); expect(page2.matches.map((entry) => entry.itemId)).toEqual(["message-20"]); - const page3 = yield* history.search(filePath, { ...request, cursor: page2.nextCursor! }); - expect(page3.matches.map((entry) => entry.ref.threadId)).toEqual(["conversation-2"]); - const end = yield* history.search(filePath, { ...request, cursor: page3.nextCursor! }); - expect(end.nextCursor).toBeNull(); + const page3 = yield* history.search(store, { ...request, cursor: page2.nextCursor! }); + expect(page3.matches.map((entry) => entry.ref.threadId)).toEqual(["1:conversation-2"]); + expect(page3.nextCursor).toBeNull(); }).pipe(Effect.scoped), ); it.effect("binds search cursors and exact source reads to the current file", () => Effect.gen(function* () { - const { history, fs, filePath } = yield* fixture([conversation(["needle"])]); + const { history, store, fs, filePath } = yield* fixture([ + conversation(["needle", ...Array.from({ length: 20 }, () => "other")]), + ]); const request = { operation: "search", sourceId: "export", query: "needle" } as const; - const page = yield* history.search(filePath, request); + const page = yield* history.search(store, request); const wrongQuery = yield* history - .search(filePath, { ...request, query: "other", cursor: page.nextCursor! }) + .search(store, { ...request, query: "other", cursor: page.nextCursor! }) .pipe(Effect.flip); expect(wrongQuery.reason).toBe("invalid_cursor"); const wrongSource = yield* history - .read(filePath, { operation: "read", ref: { ...page.matches[0]!.ref, sourceId: "other" } }) + .read(store, { operation: "read", ref: { ...page.matches[0]!.ref, sourceId: "other" } }) .pipe(Effect.flip); expect(wrongSource.reason).toBe("invalid_cursor"); yield* fs.writeFileString( @@ -133,13 +136,11 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("ChatGPT export h JSON.stringify([conversation(["needle changed source"])]), ); const changed = yield* history - .read(filePath, { operation: "read", ref: page.matches[0]!.ref }) + .read(store, { operation: "read", ref: page.matches[0]!.ref }) .pipe(Effect.flip); expect(changed.reason).toBe("changed"); - const stalePage = yield* history - .search(filePath, { ...request, cursor: page.nextCursor! }) - .pipe(Effect.flip); - expect(stalePage.reason).toBe("changed"); + const restartedPage = yield* history.search(store, { ...request, cursor: page.nextCursor! }); + expect(restartedPage.matches[0]?.excerpt).toBe("needle changed source"); }).pipe(Effect.scoped), ); @@ -160,25 +161,23 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("ChatGPT export h }, }, }; - const { history, fs, filePath } = yield* fixture([media]); + const { history, store, fs, filePath } = yield* fixture([media]); const request = { operation: "search", sourceId: "export", query: "needle" } as const; - expect((yield* history.search(filePath, request)).matches[0]?.excerpt).toBe("needle"); + expect((yield* history.search(store, request)).matches[0]?.excerpt).toBe("needle"); yield* fs.writeFileString( filePath, JSON.stringify([conversation(["x".repeat(5_000_000) + "needle"])]), ); - const error = yield* history.search(filePath, request).pipe(Effect.flip); + const error = yield* history.search(store, request).pipe(Effect.flip); expect(error.reason).toBe("limit_exceeded"); }).pipe(Effect.scoped), ); it.effect("fails explicitly for unsupported exports and ambiguous branches", () => Effect.gen(function* () { - const { history, fs, filePath } = yield* fixture({ conversations: [] }); + const { history, store, fs, filePath } = yield* fixture({ conversations: [] }); const request = { operation: "search", sourceId: "export", query: "needle" } as const; - expect((yield* history.search(filePath, request).pipe(Effect.flip)).reason).toBe( - "unsupported", - ); + expect((yield* history.search(store, request).pipe(Effect.flip)).reason).toBe("unsupported"); const source = conversation(["needle"]); yield* fs.writeFileString( filePath, @@ -186,19 +185,13 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("ChatGPT export h { ...source, mapping: { "node-0": { ...source.mapping["node-0"], parent: "node-0" } } }, ]), ); - expect((yield* history.search(filePath, request).pipe(Effect.flip)).reason).toBe( - "unsupported", - ); + expect((yield* history.search(store, request).pipe(Effect.flip)).reason).toBe("unsupported"); const duplicate = conversation(["first message", "needle second message"]); duplicate.mapping["node-1"]!.message.id = duplicate.mapping["node-0"]!.message.id; yield* fs.writeFileString(filePath, JSON.stringify([duplicate])); - expect((yield* history.search(filePath, request).pipe(Effect.flip)).reason).toBe( - "unsupported", - ); + expect((yield* history.search(store, request).pipe(Effect.flip)).reason).toBe("unsupported"); yield* fs.writeFileString(filePath, '[{"bad":'); - expect((yield* history.search(filePath, request).pipe(Effect.flip)).reason).toBe( - "unsupported", - ); + expect((yield* history.search(store, request).pipe(Effect.flip)).reason).toBe("unsupported"); }).pipe(Effect.scoped), ); }); diff --git a/apps/server/src/project/externalHistory/ChatGptExportHistory.ts b/apps/server/src/project/externalHistory/ChatGptExportHistory.ts index ab9750ee2ecb9..592d0b525c116 100644 --- a/apps/server/src/project/externalHistory/ChatGptExportHistory.ts +++ b/apps/server/src/project/externalHistory/ChatGptExportHistory.ts @@ -1,28 +1,20 @@ -import { - ExternalHistoryError, - type ExternalHistoryMatch, - type ExternalHistoryRequest, - type ExternalHistoryResponse, -} from "@t3tools/contracts"; -import * as Digest from "effect/crypto/Digest"; +import { ExternalHistoryError } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; -import * as Hex from "effect/encoding/Hex"; import * as Option from "effect/Option"; import * as PlatformError from "effect/PlatformError"; import * as Schema from "effect/Schema"; import { createTranscriptJsonReader, TranscriptJsonLimitError } from "../AgentSessionJson.ts"; import { - boundedTextEnd, - isTextBoundary, -} from "../conversationSummaries/conversationSummarySources.ts"; + fileVersion, + type LocalHistoryStore, + type LocalThreadRef, + type ThreadCache, +} from "./LocalHistoryStore.ts"; const MAX_CONVERSATION_BYTES = 8 * 1024 * 1024; const CHUNK_BYTES = 64 * 1024; -const PAGE_ITEMS = 20; -const MAX_EXCERPT_CHARS = 2_000; -const Index = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)); const Message = Schema.Struct({ id: Schema.String.check(Schema.isNonEmpty(), Schema.isMaxLength(512)), author: Schema.Struct({ role: Schema.String }), @@ -49,39 +41,15 @@ const Conversation = Schema.Struct({ }), ), }); -const SearchCursor = Schema.Struct({ - version: Schema.Literal(1), - sourceId: Schema.String, - file: Schema.String, - query: Schema.String, - conversation: Index, - message: Index, -}); -const ReadCursor = Schema.Struct({ - version: Schema.Literal(1), - sourceId: Schema.String, - file: Schema.String, - conversation: Index, - threadId: Schema.String, - itemId: Schema.String, -}); const decodeConversation = Schema.decodeUnknownEffect(Conversation); -const decodeSearchCursor = Schema.decodeUnknownEffect(Schema.fromJsonString(SearchCursor)); -const encodeSearchCursor = Schema.encodeSync(Schema.fromJsonString(SearchCursor)); -const decodeReadCursor = Schema.decodeUnknownEffect(Schema.fromJsonString(ReadCursor)); -const encodeReadCursor = Schema.encodeSync(Schema.fromJsonString(ReadCursor)); const isHistoryError = Schema.is(ExternalHistoryError); const isTranscriptJsonLimitError = Schema.is(TranscriptJsonLimitError); -const encodeFileIdentity = Schema.encodeSync( - Schema.fromJsonString( - Schema.Struct({ - size: Schema.String, - dev: Schema.String, - ino: Schema.NullOr(Schema.String), - mtime: Schema.NullOr(Schema.Number), - birthtime: Schema.NullOr(Schema.Number), - }), - ), +const decodeMetadata = Schema.decodeUnknownSync( + Schema.Struct({ + id: Schema.optionalKey(Schema.String), + conversation_id: Schema.optionalKey(Schema.String), + title: Schema.String, + }), ); interface VisibleMessage { @@ -159,27 +127,11 @@ function selectExportPath(path: ReadonlyArray, index: nu ); } -function fileIdentity(info: FileSystem.File.Info) { - return encodeFileIdentity({ - size: String(info.size), - dev: String(info.dev), - ino: Option.map(info.ino, String).pipe(Option.getOrNull), - mtime: Option.map(info.mtime, (date) => date.getTime()).pipe(Option.getOrNull), - birthtime: Option.map(info.birthtime, (date) => date.getTime()).pipe(Option.getOrNull), - }); -} - -type SearchRequest = Extract; -type ReadRequest = Extract; - -export const makeChatGptExportHistory = Effect.gen(function* () { +export const makeChatGptExportStore = Effect.fn("ChatGptExportHistory.open")(function* ( + filePath: string, + cache: ThreadCache, +) { const fs = yield* FileSystem.FileSystem; - const crypto = yield* Digest.Digest; - const digest = (text: string) => - crypto.digest(Digest.sha256, new TextEncoder().encode(text)).pipe( - Effect.map(Hex.encode), - Effect.mapError((cause) => new ExternalHistoryError({ reason: "unavailable", cause })), - ); const mapError = (cause: unknown) => isHistoryError(cause) ? cause @@ -202,8 +154,8 @@ export const makeChatGptExportHistory = Effect.gen(function* () { const file = yield* fs.open(filePath, { flag: "r" }); const stat = yield* file.stat; if (stat.type !== "File") return yield* new ExternalHistoryError({ reason: "unsupported" }); - const identity = fileIdentity(stat); - const version = yield* digest(identity); + const identity = fileVersion(stat); + const version = identity; if (expected !== undefined && expected !== version) return yield* new ExternalHistoryError({ reason: "changed" }); let value: unknown; @@ -247,8 +199,8 @@ export const makeChatGptExportHistory = Effect.gen(function* () { return yield* new ExternalHistoryError({ reason: "unsupported" }); } if ( - identity !== fileIdentity(yield* file.stat) || - identity !== fileIdentity(yield* fs.stat(filePath)) + identity !== fileVersion(yield* file.stat) || + identity !== fileVersion(yield* fs.stat(filePath)) ) return yield* new ExternalHistoryError({ reason: "changed" }); if (!found) return { file: version, conversation: null }; @@ -270,110 +222,97 @@ export const makeChatGptExportHistory = Effect.gen(function* () { ).pipe(Effect.mapError(mapError)); }); - const search = Effect.fn("ChatGptExportHistory.search")(function* ( - filePath: string, - input: SearchRequest, - ) { - if (input.filterCwd) return yield* new ExternalHistoryError({ reason: "unsupported" }); - const cursor = - input.cursor === undefined - ? undefined - : yield* decodeSearchCursor(input.cursor).pipe( - Effect.mapError(() => new ExternalHistoryError({ reason: "invalid_cursor" })), - ); - if (cursor && (cursor.sourceId !== input.sourceId || cursor.query !== input.query)) - return yield* new ExternalHistoryError({ reason: "invalid_cursor" }); - const index = cursor?.conversation ?? 0; - const result = yield* readConversation(filePath, index, cursor?.file); - if (!result.conversation) - return { - operation: "search", - matches: [], - examinedItems: 0, - nextCursor: null, - } satisfies ExternalHistoryResponse; - const { threadId, title, messages } = result.conversation; - const offset = cursor?.message ?? 0; - if (offset > messages.length) - return yield* new ExternalHistoryError({ reason: "invalid_cursor" }); - const end = Math.min(messages.length, offset + PAGE_ITEMS); - const pattern = new RegExp(input.query.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "iu"); - const matches: Array = []; - for (const message of messages.slice(offset, end)) { - const match = pattern.exec(message.text); - if (!match) continue; - let start = Math.max(0, match.index - 200); - if (!isTextBoundary(message.text, start)) start--; - matches.push({ - ref: { - sourceId: input.sourceId, - threadId, - readCursor: encodeReadCursor({ - version: 1, - sourceId: input.sourceId, - file: result.file, - conversation: index, - threadId, - itemId: message.id, - }), - digest: yield* digest(`chatgpt-visible-evidence-v1\n${message.role}\n${message.text}`), - }, - threadTitle: title, - itemId: message.id, - role: message.role, - excerpt: message.text.slice(start, boundedTextEnd(message.text, start, MAX_EXCERPT_CHARS)), - textOffset: start, - totalChars: message.text.length, - }); - } - return { - operation: "search", - matches, - examinedItems: end - offset, - nextCursor: encodeSearchCursor({ - version: 1, - sourceId: input.sourceId, - file: result.file, - query: input.query, - conversation: end < messages.length ? index : index + 1, - message: end < messages.length ? end : 0, + let listing: { version: string; refs: ReadonlyArray } | undefined; + const listThreads = Effect.gen(function* () { + const stat = yield* fs.stat(filePath); + if (stat.type !== "File") return yield* new ExternalHistoryError({ reason: "unsupported" }); + const version = fileVersion(stat); + if (listing?.version === version) return listing.refs; + const refs: LocalThreadRef[] = []; + yield* Effect.scoped( + Effect.gen(function* () { + const file = yield* fs.open(filePath, { flag: "r" }); + if (fileVersion(yield* file.stat) !== version) + return yield* new ExternalHistoryError({ reason: "changed" }); + let allocated = 0; + const reader = createTranscriptJsonReader( + (bytes) => { + allocated += bytes; + if (allocated > MAX_CONVERSATION_BYTES) + throw new TranscriptJsonLimitError({ message: "Export metadata limit" }); + }, + (path) => + path.length <= 1 || + (path.length === 2 && + (path[1] === "id" || path[1] === "conversation_id" || path[1] === "title")), + { + onArrayItem: (index, value) => { + allocated = 0; + const metadata = decodeMetadata(value); + const id = metadata.id ?? metadata.conversation_id; + if (!id) throw new ExternalHistoryError({ reason: "unsupported" }); + const key = `${index}:${id}`; + if (key.length > 512 || refs.length >= 50_000) + throw new ExternalHistoryError({ reason: "limit_exceeded" }); + refs.push({ + key, + title: metadata.title, + updatedAt: Option.map(stat.mtime, (d) => d.getTime()).pipe( + Option.getOrElse(() => 0), + ), + version, + }); + return true; + }, + }, + ); + const decoder = new TextDecoder("utf-8", { fatal: true }); + while (true) { + const bytes = yield* file.readAlloc(CHUNK_BYTES); + if (Option.isNone(bytes)) break; + yield* Effect.try({ + try: () => reader.write(decoder.decode(bytes.value, { stream: true })), + catch: mapError, + }); + } + const result = yield* Effect.try({ + try: () => { + reader.write(decoder.decode()); + return reader.finish(); + }, + catch: mapError, + }); + if (!Array.isArray(result)) + return yield* new ExternalHistoryError({ reason: "unsupported" }); + if ( + fileVersion(yield* file.stat) !== version || + fileVersion(yield* fs.stat(filePath)) !== version + ) + return yield* new ExternalHistoryError({ reason: "changed" }); }), - } satisfies ExternalHistoryResponse; - }); - - const read = Effect.fn("ChatGptExportHistory.read")(function* ( - filePath: string, - input: ReadRequest, - ) { - const cursor = yield* decodeReadCursor(input.ref.readCursor).pipe( - Effect.mapError(() => new ExternalHistoryError({ reason: "invalid_cursor" })), ); - if (cursor.sourceId !== input.ref.sourceId || cursor.threadId !== input.ref.threadId) - return yield* new ExternalHistoryError({ reason: "invalid_cursor" }); - const result = yield* readConversation(filePath, cursor.conversation, cursor.file); - if (result.conversation?.threadId !== cursor.threadId) - return yield* new ExternalHistoryError({ reason: "changed" }); - const message = result.conversation.messages.find((entry) => entry.id === cursor.itemId); - if ( - !message || - (yield* digest(`chatgpt-visible-evidence-v1\n${message.role}\n${message.text}`)) !== - input.ref.digest - ) - return yield* new ExternalHistoryError({ reason: "changed" }); - const start = input.textOffset ?? 0; - if (start > message.text.length || !isTextBoundary(message.text, start)) + listing = { version, refs }; + return refs; + }).pipe(Effect.mapError(mapError)); + const loadThread = Effect.fn("ChatGptExportHistory.loadThread")(function* (ref: LocalThreadRef) { + const version = fileVersion(yield* fs.stat(filePath)); + const match = /^(0|[1-9][0-9]*):(.+)$/.exec(ref.key); + if (!match || !Number.isSafeInteger(Number(match[1]))) return yield* new ExternalHistoryError({ reason: "invalid_cursor" }); - const end = boundedTextEnd(message.text, start, input.maxChars ?? 12_000); - return { - operation: "read", - ref: input.ref, - threadTitle: result.conversation.title, - role: message.role, - text: message.text.slice(start, end), - textOffset: start, - totalChars: message.text.length, - nextTextOffset: end < message.text.length ? end : null, - } satisfies ExternalHistoryResponse; - }); - return { search, read }; + return yield* cache( + `${filePath}\0${ref.key}\0${version}`, + Effect.gen(function* () { + const result = yield* readConversation(filePath, Number(match[1]), version); + if (!result.conversation || result.conversation.threadId !== match[2]) + return yield* new ExternalHistoryError({ reason: "changed" }); + return { + version, + title: result.conversation.title, + cwd: null, + items: result.conversation.messages, + }; + }), + ); + }, Effect.mapError(mapError)); + return { listThreads, loadThread } satisfies LocalHistoryStore; }); diff --git a/apps/server/src/project/externalHistory/ExternalHistory.test.ts b/apps/server/src/project/externalHistory/ExternalHistory.test.ts index 34a8e7a136587..63446ebfc0630 100644 --- a/apps/server/src/project/externalHistory/ExternalHistory.test.ts +++ b/apps/server/src/project/externalHistory/ExternalHistory.test.ts @@ -159,6 +159,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("external history operation: "sources", enabled: false, sources: [], + unsupported: [], }); expect(state.registryReads).toBe(0); const error = yield* service @@ -283,6 +284,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("external history operation: "sources", enabled: true, sources: [], + unsupported: [], }); }), ); diff --git a/apps/server/src/project/externalHistory/ExternalHistory.ts b/apps/server/src/project/externalHistory/ExternalHistory.ts index 346aa3e958205..c3a3898801afc 100644 --- a/apps/server/src/project/externalHistory/ExternalHistory.ts +++ b/apps/server/src/project/externalHistory/ExternalHistory.ts @@ -2,6 +2,8 @@ import { ExternalHistoryError, type ExternalHistoryRequest, type ExternalHistoryResponse, + type ExternalHistorySource, + ProviderDriverKind, } from "@t3tools/contracts"; import type { ProviderNativeHistory, @@ -11,6 +13,7 @@ import * as Context from "effect/Context"; import * as Digest from "effect/crypto/Digest"; import * as Effect from "effect/Effect"; import * as Hex from "effect/encoding/Hex"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; @@ -18,7 +21,9 @@ import * as Schema from "effect/Schema"; import * as ServerConfig from "../../config.ts"; import * as ServerSettings from "../../serverSettings.ts"; import * as ProviderInstanceRegistry from "../../provider/ProviderInstanceRegistry.ts"; -import { makeChatGptExportHistory } from "./ChatGptExportHistory.ts"; +import { makeChatGptExportStore } from "./ChatGptExportHistory.ts"; +import { makeLocalHistory } from "./LocalHistory.ts"; +import type { LocalHistoryStore } from "./LocalHistoryStore.ts"; export class ExternalHistory extends Context.Service< ExternalHistory, @@ -49,6 +54,7 @@ const encodeCursor = (cursor: typeof NativeCursor.Type) => ? Effect.succeed(encoded) : Effect.fail(new ExternalHistoryError({ reason: "limit_exceeded" })); }); +const encodeFingerprint = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); const encodeIdentity = Schema.encodeSync( Schema.fromJsonString( Schema.Struct({ @@ -58,12 +64,9 @@ const encodeIdentity = Schema.encodeSync( ), ); -type Source = { - readonly id: string; - readonly label: string; -} & ( - | { readonly kind: "codex"; readonly native: ProviderNativeHistory; readonly readers: number } - | { readonly kind: "chatgpt_export"; readonly filePath: string } +type Source = { readonly info: ExternalHistorySource } & ( + | { readonly _tag: "native"; readonly native: ProviderNativeHistory; readonly readers: number } + | { readonly _tag: "local"; readonly store: LocalHistoryStore } ); // A signed-out or older instance must not hide a shared home that another enabled instance @@ -95,24 +98,31 @@ const make = Effect.gen(function* () { const config = yield* ServerConfig.ServerConfig; const crypto = yield* Digest.Digest; const path = yield* Path.Path; - const exports = yield* makeChatGptExportHistory; + const fs = yield* FileSystem.FileSystem; + const local = yield* makeLocalHistory; + let localCatalog = new Map(); + let fingerprint: string | undefined; const digest = (text: string) => crypto.digest(Digest.sha256, new TextEncoder().encode(text)).pipe( Effect.map(Hex.encode), Effect.mapError((cause) => new ExternalHistoryError({ reason: "unavailable", cause })), ); - const sources = Effect.fn("ExternalHistory.sources")(function* () { + const sources = Effect.fn("ExternalHistory.sources")(function* (refresh: boolean) { const value = yield* settings.getSettings.pipe( Effect.mapError((cause) => new ExternalHistoryError({ reason: "unavailable", cause })), ); - if (!value.externalHistory.enabled) return { enabled: false, sources: [] as Array }; + if (!value.externalHistory.enabled) { + localCatalog = new Map(); + fingerprint = undefined; + return { enabled: false, sources: [] as Array }; + } const result = new Map(); for (const instance of yield* registry.listInstances) { if (!instance.enabled || instance.driverKind !== "codex" || !instance.nativeHistory) continue; const id = `codex:${yield* digest(encodeIdentity(instance.continuationIdentity))}`; const existing = result.get(id); - if (existing?.kind === "codex" && existing.readers < 3) { + if (existing?._tag === "native" && existing.readers < 3) { result.set(id, { ...existing, native: alternateNativeReader(existing.native, instance.nativeHistory), @@ -120,28 +130,51 @@ const make = Effect.gen(function* () { }); } else if (existing === undefined) result.set(id, { - id, - label: instance.displayName ?? "Codex", - kind: "codex", + _tag: "native", + info: { + id, + label: instance.displayName ?? "Codex", + kind: "codex", + provider: ProviderDriverKind.make("codex"), + capabilities: { cwdFilter: true, archived: true }, + status: "ready", + }, native: instance.nativeHistory, readers: 1, }); } - for (const configured of value.externalHistory.chatGptExportPaths) { - const filePath = path.resolve(config.cwd, configured); - const id = `chatgpt:${yield* digest(filePath)}`; - result.set(id, { - id, - label: `ChatGPT · ${path.basename(filePath)}`, - kind: "chatgpt_export", - filePath, - }); + const nextFingerprint = encodeFingerprint([value.externalHistory, value.providerInstances]); + if (refresh || fingerprint !== nextFingerprint) { + const next = new Map(); + for (const configured of value.externalHistory.chatGptExportPaths) { + const filePath = path.resolve(config.cwd, configured); + const id = `chatgpt:${yield* digest(filePath)}`; + next.set( + id, + localCatalog.get(id) ?? { + _tag: "local", + info: { + id, + label: `ChatGPT · ${path.basename(filePath)}`, + kind: "chatgpt_export", + capabilities: { cwdFilter: false, archived: false }, + status: "ready", + }, + store: yield* makeChatGptExportStore(filePath, local.cache).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + ), + }, + ); + } + localCatalog = next; + fingerprint = nextFingerprint; } + for (const [id, source] of localCatalog) result.set(id, source); return { enabled: true, sources: [...result.values()] }; }); const nativeSearch = Effect.fn("ExternalHistory.nativeSearch")(function* ( - source: Extract, + source: Extract, input: Extract, ) { const decoded = @@ -152,7 +185,7 @@ const make = Effect.gen(function* () { ); let cursor: typeof NativeCursor.Type = decoded ?? { version: 1, - sourceId: source.id, + sourceId: source.info.id, query: input.query, filterCwd: input.filterCwd ?? null, includeArchived: input.includeArchived ?? false, @@ -162,7 +195,7 @@ const make = Effect.gen(function* () { itemCursor: null, }; if ( - cursor.sourceId !== source.id || + cursor.sourceId !== source.info.id || cursor.query !== input.query || cursor.filterCwd !== (input.filterCwd ?? null) || cursor.includeArchived !== (input.includeArchived ?? false) || @@ -226,7 +259,7 @@ const make = Effect.gen(function* () { nextCursor, matches: page.items.map((item) => ({ ref: { - sourceId: source.id, + sourceId: source.info.id, threadId: page.thread.nativeThreadId, readCursor: item.readCursor, digest: item.digest, @@ -244,22 +277,44 @@ const make = Effect.gen(function* () { const query = Effect.fn("ExternalHistory.query")(function* ( input: ExternalHistoryRequest, ): Effect.fn.Return { - const current = yield* sources(); + let current = yield* sources(input.operation === "sources"); + const requestedId = + input.operation === "read" + ? input.ref.sourceId + : input.operation === "search" + ? input.sourceId + : undefined; + if ( + current.enabled && + requestedId && + !current.sources.some((source) => source.info.id === requestedId) + ) + current = yield* sources(true); if (input.operation === "sources") return { operation: "sources", enabled: current.enabled, - sources: current.sources.map(({ id, label, kind }) => ({ id, label, kind })), + sources: current.sources + .map(({ info }) => info) + .filter((info) => input.catalogVersion === 2 || info.kind !== "local_transcript"), + unsupported: [], }; if (!current.enabled) return yield* new ExternalHistoryError({ reason: "disabled" }); const source = current.sources.find( - (entry) => entry.id === (input.operation === "read" ? input.ref.sourceId : input.sourceId), + (entry) => + entry.info.id === (input.operation === "read" ? input.ref.sourceId : input.sourceId), ); if (!source) return yield* new ExternalHistoryError({ reason: "missing" }); - if (source.kind === "chatgpt_export") + if ( + input.operation === "search" && + ((input.filterCwd && !source.info.capabilities.cwdFilter) || + (input.includeArchived && !source.info.capabilities.archived)) + ) + return yield* new ExternalHistoryError({ reason: "unsupported" }); + if (source._tag === "local") return yield* input.operation === "search" - ? exports.search(source.filePath, input) - : exports.read(source.filePath, input); + ? local.search(source.store, input) + : local.read(source.store, input); if (input.operation === "search") return yield* nativeSearch(source, input); const result = yield* source.native .readSource({ diff --git a/apps/server/src/project/externalHistory/LocalHistory.ts b/apps/server/src/project/externalHistory/LocalHistory.ts new file mode 100644 index 0000000000000..1c9e31ec75a07 --- /dev/null +++ b/apps/server/src/project/externalHistory/LocalHistory.ts @@ -0,0 +1,194 @@ +import { + ExternalHistoryError, + type ExternalHistoryMatch, + type ExternalHistoryRequest, + type ExternalHistoryResponse, +} from "@t3tools/contracts"; +import { normalizeProjectPathForComparison } from "@t3tools/shared/path"; +import * as Digest from "effect/crypto/Digest"; +import * as Hex from "effect/encoding/Hex"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; +import { + boundedTextEnd, + isTextBoundary, +} from "../conversationSummaries/conversationSummarySources.ts"; +import { + makeThreadCache, + type LocalHistoryStore, + type LocalThreadRef, +} from "./LocalHistoryStore.ts"; + +const SearchCursor = Schema.Struct({ + v: Schema.Literal(2), + sourceId: Schema.String, + query: Schema.String, + filterCwd: Schema.NullOr(Schema.String), + afterKey: Schema.NullOr(Schema.String), + itemOffset: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + version: Schema.NullOr(Schema.String), +}); +const ReadCursor = Schema.Struct({ + v: Schema.Literal(2), + sourceId: Schema.String, + key: Schema.String, + version: Schema.String, + itemId: Schema.String, +}); +const decodeSearchCursor = Schema.decodeUnknownEffect(Schema.fromJsonString(SearchCursor)); +const encodeSearchCursor = Schema.encodeSync(Schema.fromJsonString(SearchCursor)); +const decodeReadCursor = Schema.decodeUnknownEffect(Schema.fromJsonString(ReadCursor)); +const encodeReadCursor = Schema.encodeSync(Schema.fromJsonString(ReadCursor)); +const invalidCursor = () => new ExternalHistoryError({ reason: "invalid_cursor" }); +const evidence = (item: { readonly role: string; readonly text: string }) => + `external-history-v2\n${item.role}\n${item.text}`; + +export const makeLocalHistory = Effect.gen(function* () { + const crypto = yield* Digest.Digest; + const cache = yield* makeThreadCache; + const digest = (text: string) => + crypto.digest(Digest.sha256, new TextEncoder().encode(text)).pipe( + Effect.map(Hex.encode), + Effect.mapError((cause) => new ExternalHistoryError({ reason: "unavailable", cause })), + ); + const search = Effect.fn("ExternalHistory.searchLocal")(function* ( + store: LocalHistoryStore, + input: Extract, + ) { + const cursor = + input.cursor === undefined + ? undefined + : yield* decodeSearchCursor(input.cursor).pipe(Effect.mapError(invalidCursor)); + if ( + cursor && + (cursor.sourceId !== input.sourceId || + cursor.query !== input.query || + cursor.filterCwd !== (input.filterCwd ?? null)) + ) + return yield* invalidCursor(); + const refs = yield* store.listThreads; + const index = + cursor?.afterKey == null ? 0 : refs.findIndex((ref) => ref.key === cursor.afterKey); + if (index < 0) return yield* new ExternalHistoryError({ reason: "changed" }); + const ref = refs[index]; + if (!ref) + return { + operation: "search", + matches: [], + examinedItems: 0, + nextCursor: null, + } satisfies ExternalHistoryResponse; + const nextThread = () => + refs[index + 1] + ? encodeSearchCursor({ + v: 2, + sourceId: input.sourceId, + query: input.query, + filterCwd: input.filterCwd ?? null, + afterKey: refs[index + 1]!.key, + itemOffset: 0, + version: null, + }) + : null; + const thread = yield* store.loadThread(ref); + if ( + thread.origin === "t3" || + (input.filterCwd && + (thread.cwd === null || + normalizeProjectPathForComparison(thread.cwd) !== + normalizeProjectPathForComparison(input.filterCwd))) + ) + return { + operation: "search", + matches: [], + examinedItems: 0, + nextCursor: nextThread(), + } satisfies ExternalHistoryResponse; + const offset = cursor?.version === thread.version ? cursor.itemOffset : 0; + if (offset > thread.items.length) return yield* invalidCursor(); + const end = Math.min(thread.items.length, offset + 20); + const pattern = new RegExp(input.query.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "iu"); + const matches: ExternalHistoryMatch[] = []; + for (const item of thread.items.slice(offset, end)) { + const match = pattern.exec(item.text); + if (!match) continue; + let start = Math.max(0, match.index - 200); + if (!isTextBoundary(item.text, start)) start--; + const readCursor = encodeReadCursor({ + v: 2, + sourceId: input.sourceId, + key: ref.key, + version: thread.version, + itemId: item.id, + }); + if (readCursor.length > 16_384 || item.id.length > 512) + return yield* new ExternalHistoryError({ reason: "limit_exceeded" }); + matches.push({ + ref: { + sourceId: input.sourceId, + threadId: ref.key, + readCursor, + digest: yield* digest(evidence(item)), + }, + threadTitle: thread.title, + itemId: item.id, + role: item.role, + excerpt: item.text.slice(start, boundedTextEnd(item.text, start, 2_000)), + textOffset: start, + totalChars: item.text.length, + }); + } + return { + operation: "search", + matches, + examinedItems: end - offset, + nextCursor: + end < thread.items.length + ? encodeSearchCursor({ + v: 2, + sourceId: input.sourceId, + query: input.query, + filterCwd: input.filterCwd ?? null, + afterKey: ref.key, + itemOffset: end, + version: thread.version, + }) + : nextThread(), + } satisfies ExternalHistoryResponse; + }); + const read = Effect.fn("ExternalHistory.readLocal")(function* ( + store: LocalHistoryStore, + input: Extract, + ) { + const cursor = yield* decodeReadCursor(input.ref.readCursor).pipe( + Effect.mapError(invalidCursor), + ); + if (cursor.sourceId !== input.ref.sourceId || cursor.key !== input.ref.threadId) + return yield* invalidCursor(); + const ref: LocalThreadRef = { key: cursor.key, version: cursor.version, updatedAt: 0 }; + const thread = yield* store.loadThread(ref); + const item = thread.items.find((item) => item.id === cursor.itemId); + if ( + thread.version !== cursor.version || + thread.origin === "t3" || + !item || + (yield* digest(evidence(item))) !== input.ref.digest + ) + return yield* new ExternalHistoryError({ reason: "changed" }); + const start = input.textOffset ?? 0; + if (start > item.text.length || !isTextBoundary(item.text, start)) + return yield* invalidCursor(); + const end = boundedTextEnd(item.text, start, input.maxChars ?? 12_000); + return { + operation: "read", + ref: input.ref, + threadTitle: thread.title, + role: item.role, + text: item.text.slice(start, end), + textOffset: start, + totalChars: item.text.length, + nextTextOffset: end < item.text.length ? end : null, + } satisfies ExternalHistoryResponse; + }); + return { search, read, cache }; +}); diff --git a/apps/server/src/project/externalHistory/LocalHistoryStore.ts b/apps/server/src/project/externalHistory/LocalHistoryStore.ts new file mode 100644 index 0000000000000..72a0cc5b1a5b5 --- /dev/null +++ b/apps/server/src/project/externalHistory/LocalHistoryStore.ts @@ -0,0 +1,73 @@ +import type { ExternalHistoryError } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import type * as FileSystem from "effect/FileSystem"; +import type { TranscriptItem } from "../agentTranscripts.ts"; + +export interface LocalThreadRef { + readonly key: string; + readonly title?: string; + readonly cwd?: string; + readonly updatedAt: number; + readonly version: string; +} + +export interface LocalThread { + readonly version: string; + readonly title: string; + readonly cwd: string | null; + readonly items: ReadonlyArray; + readonly origin?: "t3"; +} + +export interface LocalHistoryStore { + readonly listThreads: Effect.Effect, ExternalHistoryError>; + readonly loadThread: (ref: LocalThreadRef) => Effect.Effect; +} + +export type ThreadCache = ( + key: string, + load: Effect.Effect, +) => Effect.Effect; + +export const makeThreadCache = Effect.sync((): ThreadCache => { + const entries = new Map(); + let bytes = 0; + return Effect.fn("ExternalHistory.cachedThread")(function* (key, load) { + const cached = entries.get(key); + if (cached) { + entries.delete(key); + entries.set(key, cached); + return cached.thread; + } + const thread = yield* load; + const size = thread.items.reduce( + (sum, item) => sum + 2 * (item.text.length + item.id.length), + 0, + ); + if (size <= 32 * 1024 * 1024) { + const replaced = entries.get(key); + if (replaced) bytes -= replaced.bytes; + entries.delete(key); + while (entries.size >= 64 || bytes + size > 32 * 1024 * 1024) { + const oldest = entries.entries().next().value; + if (!oldest) break; + entries.delete(oldest[0]); + bytes -= oldest[1].bytes; + } + entries.set(key, { thread, bytes: size }); + bytes += size; + } + return thread; + }); +}); + +export function fileVersion(info: FileSystem.File.Info): string { + return JSON.stringify({ + size: String(info.size), + dev: String(info.dev), + ino: Option.map(info.ino, String).pipe(Option.getOrNull), + mtime: Option.map(info.mtime, (date) => date.getTime()).pipe(Option.getOrNull), + birthtime: Option.map(info.birthtime, (date) => date.getTime()).pipe(Option.getOrNull), + }); +} diff --git a/apps/server/src/project/externalHistory/historySupport.test.ts b/apps/server/src/project/externalHistory/historySupport.test.ts new file mode 100644 index 0000000000000..af9fecd81f689 --- /dev/null +++ b/apps/server/src/project/externalHistory/historySupport.test.ts @@ -0,0 +1,18 @@ +import { expect, it } from "vite-plus/test"; +import { ExternalHistoryError } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import { BUILT_IN_DRIVERS } from "../../provider/builtInDrivers.ts"; +import { makeHistorySupport } from "./historySupport.ts"; + +it("requires a history decision for every built-in driver", () => { + const open = () => Effect.fail(new ExternalHistoryError({ reason: "missing" })); + const table = makeHistorySupport({ + claudeAgent: open, + pi: open, + opencode: open, + openCodeRoots: () => Effect.succeed([]), + }); + expect(Object.keys(table).toSorted()).toEqual( + BUILT_IN_DRIVERS.map((driver) => driver.driverKind).toSorted(), + ); +}); diff --git a/apps/server/src/project/externalHistory/historySupport.ts b/apps/server/src/project/externalHistory/historySupport.ts new file mode 100644 index 0000000000000..195c138afeb9d --- /dev/null +++ b/apps/server/src/project/externalHistory/historySupport.ts @@ -0,0 +1,89 @@ +import { ClaudeSettings, ProviderInstanceId } from "@t3tools/contracts"; +import type { ExternalHistoryError } from "@t3tools/contracts"; +import { piHome, resolvePiSessionsDir } from "@t3tools/provider-pi/server/usage"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import { resolveClaudeRuntime } from "../../provider/Drivers/ClaudeHome.ts"; +import type { LocalHistoryStore, ThreadCache } from "./LocalHistoryStore.ts"; + +interface Profile { + readonly instanceId: ProviderInstanceId; + readonly config: unknown; + readonly environment: NodeJS.ProcessEnv; + readonly stateDir: string; +} + +type StoreFactory = ( + root: string, + cache: ThreadCache, +) => Effect.Effect; +type HistorySupport = { readonly label: string } & ( + | { readonly kind: "native" } + | { + readonly kind: "local"; + readonly roots: ( + profile: Profile, + ) => Effect.Effect, never, FileSystem.FileSystem | Path.Path>; + readonly open: StoreFactory; + } + | { readonly kind: "unsupported"; readonly reason: string } +); + +const decodeClaudeSettings = Schema.decodeUnknownSync(ClaudeSettings); +const unsupported = (label: string): HistorySupport => ({ + kind: "unsupported", + label, + reason: "History isn't stored on this machine in a readable format yet.", +}); + +export function makeHistorySupport(stores: { + readonly claudeAgent: StoreFactory; + readonly pi: StoreFactory; + readonly opencode: StoreFactory; + readonly openCodeRoots: ( + environment: NodeJS.ProcessEnv, + ) => Effect.Effect, never, FileSystem.FileSystem | Path.Path>; +}): Readonly> { + return { + codex: { kind: "native", label: "Codex" }, + claudeAgent: { + kind: "local", + label: "Claude Code", + open: stores.claudeAgent, + roots: Effect.fn("ExternalHistory.claudeRoots")(function* (profile) { + const runtime = yield* resolveClaudeRuntime({ + ...profile, + config: decodeClaudeSettings(profile.config ?? {}), + }); + return [runtime.configDir]; + }), + }, + pi: { + kind: "local", + label: "Pi", + open: stores.pi, + roots: Effect.fn("ExternalHistory.piRoots")(function* ({ environment }) { + const path = yield* Path.Path; + const home = yield* HostProcess.HomeDirectory; + const platform = yield* HostProcess.Platform; + return [resolvePiSessionsDir(environment, piHome(environment, home, platform), path)]; + }), + }, + opencode: { + kind: "local", + label: "OpenCode", + open: stores.opencode, + roots: (profile) => stores.openCodeRoots(profile.environment), + }, + grok: unsupported("Grok"), + antigravity: unsupported("Antigravity"), + cursor: unsupported("Cursor"), + kiro: unsupported("Kiro"), + muse: unsupported("Muse"), + fx: unsupported("FX"), + acpRegistry: unsupported("ACP agents"), + }; +} From 7a264be7fc47b0985651a5a6b2a2d6437bbb82e0 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:04:43 -0800 Subject: [PATCH 04/15] feat(server): discover Claude Pi and OpenCode external history Co-Authored-By: Claude Fable 5.2 --- apps/server/src/project/agentTranscripts.ts | 6 + .../externalHistory/ChatGptExportHistory.ts | 5 +- .../externalHistory/ExternalHistory.test.ts | 140 +++++++++- .../externalHistory/ExternalHistory.ts | 155 ++++++++++- .../externalHistory/JsonlHistoryStore.test.ts | 172 ++++++++++++ .../externalHistory/JsonlHistoryStore.ts | 256 ++++++++++++++++++ .../OpenCodeHistoryStore.test.ts | 118 ++++++++ .../externalHistory/OpenCodeHistoryStore.ts | 194 +++++++++++++ packages/provider-opencode/package.json | 4 + .../provider-opencode/src/server/usage.ts | 2 +- 10 files changed, 1038 insertions(+), 14 deletions(-) create mode 100644 apps/server/src/project/externalHistory/JsonlHistoryStore.test.ts create mode 100644 apps/server/src/project/externalHistory/JsonlHistoryStore.ts create mode 100644 apps/server/src/project/externalHistory/OpenCodeHistoryStore.test.ts create mode 100644 apps/server/src/project/externalHistory/OpenCodeHistoryStore.ts diff --git a/apps/server/src/project/agentTranscripts.ts b/apps/server/src/project/agentTranscripts.ts index 24f66e996dcd1..d930bd383f478 100644 --- a/apps/server/src/project/agentTranscripts.ts +++ b/apps/server/src/project/agentTranscripts.ts @@ -153,8 +153,14 @@ export function transcriptItems( source: "claudeAgent" | "pi", ): ReadonlyArray { if (source === "claudeAgent" && isHiddenClaudeRecord(record)) return []; + if (source === "pi" && record.type !== "message") return []; + if (source === "claudeAgent" && record.type !== "user" && record.type !== "assistant") return []; const nativeId = source === "pi" ? record.id : record.uuid; if (!nativeId) return []; + if (source === "pi" && record.message?.role === "toolResult") { + const text = extractText(record.message.content); + return text ? [{ id: nativeId, role: "tool", text }] : []; + } const conversation = conversationItem(record, source); const items: TranscriptItem[] = conversation ? [{ ...conversation, id: nativeId }] : []; const content = record.message?.content; diff --git a/apps/server/src/project/externalHistory/ChatGptExportHistory.ts b/apps/server/src/project/externalHistory/ChatGptExportHistory.ts index 592d0b525c116..f45c9a43a5357 100644 --- a/apps/server/src/project/externalHistory/ChatGptExportHistory.ts +++ b/apps/server/src/project/externalHistory/ChatGptExportHistory.ts @@ -299,7 +299,7 @@ export const makeChatGptExportStore = Effect.fn("ChatGptExportHistory.open")(fun const match = /^(0|[1-9][0-9]*):(.+)$/.exec(ref.key); if (!match || !Number.isSafeInteger(Number(match[1]))) return yield* new ExternalHistoryError({ reason: "invalid_cursor" }); - return yield* cache( + const thread = yield* cache( `${filePath}\0${ref.key}\0${version}`, Effect.gen(function* () { const result = yield* readConversation(filePath, Number(match[1]), version); @@ -313,6 +313,9 @@ export const makeChatGptExportStore = Effect.fn("ChatGptExportHistory.open")(fun }; }), ); + if (fileVersion(yield* fs.stat(filePath)) !== version) + return yield* new ExternalHistoryError({ reason: "changed" }); + return thread; }, Effect.mapError(mapError)); return { listThreads, loadThread } satisfies LocalHistoryStore; }); diff --git a/apps/server/src/project/externalHistory/ExternalHistory.test.ts b/apps/server/src/project/externalHistory/ExternalHistory.test.ts index 63446ebfc0630..e4d70119f0a59 100644 --- a/apps/server/src/project/externalHistory/ExternalHistory.test.ts +++ b/apps/server/src/project/externalHistory/ExternalHistory.test.ts @@ -16,6 +16,7 @@ import { import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Stream from "effect/Stream"; import * as ServerConfig from "../../config.ts"; @@ -123,17 +124,32 @@ const makeFixture = Effect.fn("ExternalHistory.test.fixture")(function* () { instances: [instance] as Array, settings: { enabled: true, chatGptExportPaths: [] } as ExternalHistorySettings, registryReads: 0, + providerInstances: DEFAULT_SERVER_SETTINGS.providerInstances, }; + const fs = yield* FileSystem.FileSystem; + const enumerations: string[] = []; + const countingFs = FileSystem.make({ + ...fs, + readDirectory: (path) => { + enumerations.push(path); + return fs.readDirectory(path); + }, + }); + const home = yield* fs.makeTempDirectoryScoped({ prefix: "t3-history-home-" }); const service = yield* ExternalHistory.ExternalHistory.pipe( Effect.provide( ExternalHistory.layer.pipe( Layer.provide( Layer.mergeAll( + Layer.succeed(FileSystem.FileSystem, countingFs), + Layer.succeed(HostProcess.HomeDirectory, home), + Layer.succeed(HostProcess.Environment, { HOME: home }), ServerConfig.layerTest("/operational-project", { prefix: "t3-external-config-" }), Layer.mock(ServerSettings.ServerSettingsService)({ getSettings: Effect.sync(() => ({ ...DEFAULT_SERVER_SETTINGS, externalHistory: state.settings, + providerInstances: state.providerInstances, })), }), Layer.mock(ProviderInstanceRegistry.ProviderInstanceRegistry)({ @@ -147,19 +163,99 @@ const makeFixture = Effect.fn("ExternalHistory.test.fixture")(function* () { ), ), ); - return { service, state, calls, instance, native }; + return { service, state, calls, instance, native, home, enumerations }; }); it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("external history service", (it) => { + it.effect( + "discovers default and disabled profiles with no paths, excludes T3 homes, and keeps page requests out of discovery", + () => + Effect.gen(function* () { + const { service, state, home, enumerations } = yield* makeFixture(); + const fs = yield* FileSystem.FileSystem; + const claude = `${home}/.claude`; + const pi = `${home}/.pi/agent/sessions`; + const managed = `${home}/.t3/userdata/providers/inference/claude`; + for (const root of [claude, managed]) { + yield* fs.makeDirectory(`${root}/projects/p`, { recursive: true }); + yield* fs.writeFileString( + `${root}/projects/p/session.jsonl`, + JSON.stringify({ type: "user", uuid: "claude", message: { content: "needle" } }) + "\n", + ); + } + yield* fs.makeDirectory(`${pi}/p`, { recursive: true }); + yield* fs.writeFileString( + `${pi}/p/pi.jsonl`, + JSON.stringify({ + type: "message", + id: "pi", + message: { role: "user", content: "needle" }, + }) + "\n", + ); + yield* fs.makeDirectory(`${home}/Downloads`, { recursive: true }); + state.providerInstances = { + ...state.providerInstances, + [ProviderInstanceId.make("disabled-claude")]: { + driver: ProviderDriverKind.make("claudeAgent"), + enabled: false, + config: { homePath: claude }, + }, + [ProviderInstanceId.make("opencode-downloads")]: { + driver: ProviderDriverKind.make("opencode"), + environment: [ + { name: "OPENCODE_DATA_DIR", value: `${home}/Downloads`, sensitive: false }, + ], + }, + [ProviderInstanceId.make("opencode-inference")]: { + driver: ProviderDriverKind.make("opencode"), + environment: [{ name: "OPENCODE_DATA_DIR", value: managed, sensitive: false }], + }, + [ProviderInstanceId.make("managed-claude")]: { + driver: ProviderDriverKind.make("claudeAgent"), + config: { homePath: managed }, + }, + }; + const catalog = yield* service.query({ operation: "sources", catalogVersion: 2 }); + if (catalog.operation !== "sources") return yield* Effect.die("wrong response"); + expect(catalog.sources.map((source) => source.provider).toSorted()).toEqual([ + "claudeAgent", + "codex", + "pi", + ]); + expect(catalog.unsupported.find((source) => source.provider === "grok")?.reason).toContain( + "readable format", + ); + const legacy = yield* service.query({ operation: "sources" }); + expect( + legacy.operation === "sources" && legacy.sources.map((source) => source.kind), + ).toEqual(["codex"]); + const source = catalog.sources.find((source) => source.provider === "claudeAgent")!; + const page = yield* service.query({ + operation: "search", + sourceId: source.id, + query: "needle", + }); + if (page.operation !== "search") return yield* Effect.die("wrong response"); + const count = enumerations.length; + yield* service.query({ operation: "search", sourceId: source.id, query: "other" }); + yield* service.query({ operation: "read", ref: page.matches[0]!.ref }); + expect(enumerations).toHaveLength(count); + expect( + enumerations.some( + (path) => path.includes("Downloads") || path.includes("providers/inference"), + ), + ).toBe(false); + }), + ); + it.effect("checks the environment opt-in before touching providers and on every read", () => Effect.gen(function* () { const { service, state, calls } = yield* makeFixture(); state.settings = { enabled: false, chatGptExportPaths: [] }; - expect(yield* service.query({ operation: "sources" })).toEqual({ + expect(yield* service.query({ operation: "sources" })).toMatchObject({ operation: "sources", enabled: false, sources: [], - unsupported: [], }); expect(state.registryReads).toBe(0); const error = yield* service @@ -280,12 +376,8 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("external history .pipe(Effect.flip); expect(unsupported.reason).toBe("unsupported"); state.instances = [{ ...instance, enabled: false }]; - expect(yield* service.query({ operation: "sources" })).toEqual({ - operation: "sources", - enabled: true, - sources: [], - unsupported: [], - }); + const disabledInstance = yield* service.query({ operation: "sources" }); + expect(disabledInstance.operation === "sources" && disabledInstance.sources.length).toBe(1); }), ); @@ -333,6 +425,36 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("external history }).pipe(Effect.scoped), ); + it.effect("excludes native Codex sessions marked as T3 conversations", () => + Effect.gen(function* () { + const { service, state, instance, native, calls } = yield* makeFixture(); + state.instances = [ + { + ...instance, + nativeHistory: { + ...native, + list: () => + Effect.succeed({ threads: [{ ...thread, originator: "T3 Code" }], nextCursor: null }), + }, + }, + ]; + const catalog = yield* service.query({ operation: "sources" }); + if (catalog.operation !== "sources") return yield* Effect.die("wrong response"); + const result = yield* service.query({ + operation: "search", + sourceId: catalog.sources[0]!.id, + query: "needle", + }); + expect(result).toEqual({ + operation: "search", + matches: [], + examinedItems: 0, + nextCursor: null, + }); + expect(calls.items).toEqual([]); + }), + ); + it.effect("rejects a native continuation that cannot fit the public cursor budget", () => Effect.gen(function* () { const { service, state, instance, native } = yield* makeFixture(); diff --git a/apps/server/src/project/externalHistory/ExternalHistory.ts b/apps/server/src/project/externalHistory/ExternalHistory.ts index c3a3898801afc..f59c5b36fd3c8 100644 --- a/apps/server/src/project/externalHistory/ExternalHistory.ts +++ b/apps/server/src/project/externalHistory/ExternalHistory.ts @@ -4,6 +4,7 @@ import { type ExternalHistoryResponse, type ExternalHistorySource, ProviderDriverKind, + ProviderInstanceId, } from "@t3tools/contracts"; import type { ProviderNativeHistory, @@ -17,12 +18,20 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; +import * as Option from "effect/Option"; +import * as Semaphore from "effect/Semaphore"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import { mergeProviderInstanceEnvironment } from "@t3tools/provider-core/server/instanceEnvironment"; +import { resolveOpenCodeDataDirs } from "@t3tools/provider-opencode/server/usage"; import * as ServerConfig from "../../config.ts"; import * as ServerSettings from "../../serverSettings.ts"; import * as ProviderInstanceRegistry from "../../provider/ProviderInstanceRegistry.ts"; import { makeChatGptExportStore } from "./ChatGptExportHistory.ts"; import { makeLocalHistory } from "./LocalHistory.ts"; +import { makeHistorySupport } from "./historySupport.ts"; +import { makeClaudeHistoryStore, makePiHistoryStore } from "./JsonlHistoryStore.ts"; +import { makeOpenCodeHistoryStore } from "./OpenCodeHistoryStore.ts"; import type { LocalHistoryStore } from "./LocalHistoryStore.ts"; export class ExternalHistory extends Context.Service< @@ -67,6 +76,7 @@ const encodeIdentity = Schema.encodeSync( type Source = { readonly info: ExternalHistorySource } & ( | { readonly _tag: "native"; readonly native: ProviderNativeHistory; readonly readers: number } | { readonly _tag: "local"; readonly store: LocalHistoryStore } + | { readonly _tag: "unavailable" } ); // A signed-out or older instance must not hide a shared home that another enabled instance @@ -100,6 +110,39 @@ const make = Effect.gen(function* () { const path = yield* Path.Path; const fs = yield* FileSystem.FileSystem; const local = yield* makeLocalHistory; + const home = yield* HostProcess.HomeDirectory; + const environment = yield* HostProcess.Environment; + const catalogLock = yield* Semaphore.make(1); + const excludedRoots = Effect.forEach( + [config.baseDir, config.stateDir, path.join(home, ".t3"), path.join(home, "Downloads")], + (dir) => fs.realPath(dir).pipe(Effect.orElseSucceed(() => path.resolve(dir))), + ); + const inside = (root: string, parent: string) => { + const relative = path.relative(parent, root); + return ( + relative === "" || + (!relative.startsWith(`..${path.sep}`) && relative !== ".." && !path.isAbsolute(relative)) + ); + }; + const supportTable = makeHistorySupport({ + claudeAgent: makeClaudeHistoryStore, + pi: makePiHistoryStore, + opencode: makeOpenCodeHistoryStore, + openCodeRoots: Effect.fn("ExternalHistory.openCodeRoots")(function* (env) { + const roots: string[] = []; + const excluded = yield* excludedRoots; + for (const root of yield* resolveOpenCodeDataDirs(env)) { + const canonical = yield* fs.realPath(root).pipe(Effect.option); + if (Option.isNone(canonical) || excluded.some((parent) => inside(canonical.value, parent))) + continue; + const dir = canonical.value; + const names = yield* fs.readDirectory(dir).pipe(Effect.orElseSucceed(() => [] as string[])); + const databases = names.filter((name) => /^opencode(?:-[a-zA-Z0-9_-]+)?\.db$/.test(name)); + roots.push(...(databases.length ? databases.map((name) => path.join(dir, name)) : [dir])); + } + return roots; + }), + }); let localCatalog = new Map(); let fingerprint: string | undefined; const digest = (text: string) => @@ -119,7 +162,7 @@ const make = Effect.gen(function* () { } const result = new Map(); for (const instance of yield* registry.listInstances) { - if (!instance.enabled || instance.driverKind !== "codex" || !instance.nativeHistory) continue; + if (supportTable[instance.driverKind]?.kind !== "native" || !instance.nativeHistory) continue; const id = `codex:${yield* digest(encodeIdentity(instance.continuationIdentity))}`; const existing = result.get(id); if (existing?._tag === "native" && existing.readers < 3) { @@ -146,6 +189,87 @@ const make = Effect.gen(function* () { const nextFingerprint = encodeFingerprint([value.externalHistory, value.providerInstances]); if (refresh || fingerprint !== nextFingerprint) { const next = new Map(); + const excluded = yield* excludedRoots; + const seen = new Set(); + for (const [driver, support] of Object.entries(supportTable)) { + if (support.kind !== "local") continue; + const profiles = [ + { + instanceId: ProviderInstanceId.make(driver), + config: undefined, + environment: undefined, + }, + ...Object.entries(value.providerInstances) + .filter(([, profile]) => profile.driver === driver) + .map(([id, profile]) => ({ + instanceId: ProviderInstanceId.make(id), + config: profile.config, + environment: profile.environment, + })), + ]; + for (const profile of profiles) { + const env = yield* mergeProviderInstanceEnvironment( + profile.environment, + environment, + ).pipe(Effect.provideService(HostProcess.HomeDirectory, home)); + const roots = yield* support + .roots({ + instanceId: profile.instanceId, + config: profile.config, + environment: env, + stateDir: config.stateDir, + }) + .pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(HostProcess.HomeDirectory, home), + ); + for (const root of roots) { + const canonical = yield* fs.realPath(root).pipe(Effect.option); + if (Option.isNone(canonical) || excluded.some((dir) => inside(canonical.value, dir))) + continue; + const stat = yield* fs.stat(canonical.value).pipe(Effect.option); + if (Option.isNone(stat)) continue; + const identity = `${driver}:${stat.value.dev}:${Option.map(stat.value.ino, String).pipe(Option.getOrElse(() => canonical.value))}`; + if (seen.has(identity)) continue; + seen.add(identity); + const id = `local:${driver}:${yield* digest(canonical.value)}`; + const existing = localCatalog.get(id); + if (existing?._tag === "local") { + next.set(id, existing); + continue; + } + const opened = yield* support + .open(canonical.value, local.cache) + .pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.result, + ); + if (opened._tag === "Failure" && opened.failure.reason === "missing") continue; + const info: ExternalHistorySource = { + id, + label: `${support.label} · ${canonical.value}`, + kind: "local_transcript", + provider: ProviderDriverKind.make(driver), + capabilities: { cwdFilter: true, archived: false }, + status: + opened._tag === "Success" + ? "ready" + : opened.failure.reason === "unsupported" + ? "unsupported_format" + : "unavailable", + ...(opened._tag === "Failure" ? { message: opened.failure.message } : {}), + }; + next.set( + id, + opened._tag === "Success" + ? { _tag: "local", info, store: opened.success } + : { _tag: "unavailable", info }, + ); + } + } + } for (const configured of value.externalHistory.chatGptExportPaths) { const filePath = path.resolve(config.cwd, configured); const id = `chatgpt:${yield* digest(filePath)}`; @@ -171,7 +295,7 @@ const make = Effect.gen(function* () { } for (const [id, source] of localCatalog) result.set(id, source); return { enabled: true, sources: [...result.values()] }; - }); + }, catalogLock.withPermit); const nativeSearch = Effect.fn("ExternalHistory.nativeSearch")(function* ( source: Extract, @@ -236,6 +360,15 @@ const make = Effect.gen(function* () { ? afterPhase() : encodeCursor({ ...cursor, listCursor: page.nextCursor }), } satisfies ExternalHistoryResponse; + if (thread.originator === "T3 Code") + return { + operation: "search", + matches: [], + examinedItems: 0, + nextCursor: yield* page.nextCursor === null + ? afterPhase() + : encodeCursor({ ...cursor, listCursor: page.nextCursor }), + } satisfies ExternalHistoryResponse; cursor = { ...cursor, threadId: thread.nativeThreadId, listCursor: page.nextCursor }; } const page = yield* source.native @@ -297,7 +430,17 @@ const make = Effect.gen(function* () { sources: current.sources .map(({ info }) => info) .filter((info) => input.catalogVersion === 2 || info.kind !== "local_transcript"), - unsupported: [], + unsupported: Object.entries(supportTable).flatMap(([provider, support]) => + support.kind === "unsupported" + ? [ + { + provider: ProviderDriverKind.make(provider), + label: support.label, + reason: support.reason, + }, + ] + : [], + ), }; if (!current.enabled) return yield* new ExternalHistoryError({ reason: "disabled" }); const source = current.sources.find( @@ -305,6 +448,10 @@ const make = Effect.gen(function* () { entry.info.id === (input.operation === "read" ? input.ref.sourceId : input.sourceId), ); if (!source) return yield* new ExternalHistoryError({ reason: "missing" }); + if (source._tag === "unavailable") + return yield* new ExternalHistoryError({ + reason: source.info.status === "unsupported_format" ? "unsupported" : "unavailable", + }); if ( input.operation === "search" && ((input.filterCwd && !source.info.capabilities.cwdFilter) || @@ -326,6 +473,8 @@ const make = Effect.gen(function* () { ...(input.maxChars === undefined ? {} : { maxChars: input.maxChars }), }) .pipe(Effect.mapError((cause) => new ExternalHistoryError({ reason: cause.reason, cause }))); + if (result.thread.originator === "T3 Code") + return yield* new ExternalHistoryError({ reason: "missing" }); return { operation: "read", ref: input.ref, diff --git a/apps/server/src/project/externalHistory/JsonlHistoryStore.test.ts b/apps/server/src/project/externalHistory/JsonlHistoryStore.test.ts new file mode 100644 index 0000000000000..43205c76a2c58 --- /dev/null +++ b/apps/server/src/project/externalHistory/JsonlHistoryStore.test.ts @@ -0,0 +1,172 @@ +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import { makeClaudeHistoryStore, makePiHistoryStore } from "./JsonlHistoryStore.ts"; +import { makeLocalHistory } from "./LocalHistory.ts"; + +it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("local JSONL histories", (it) => { + it.effect( + "finds old Claude evidence before the import window and preserves verified native IDs without re-enumeration", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + const project = `${root}/projects/project`; + yield* fs.makeDirectory(project, { recursive: true }); + const file = `${project}/session.jsonl`; + const records = [ + { + type: "user", + uuid: "old-evidence", + sessionId: "session", + cwd: "/project", + timestamp: "2020-01-01T00:00:00Z", + message: { content: "needle from the distant past" }, + }, + ...Array.from({ length: 230 }, (_, index) => ({ + type: "assistant", + uuid: `new-${index}`, + message: { content: `Other message ${index}` }, + })), + { type: "custom-title", customTitle: "Renamed" }, + { type: "ai-title", aiTitle: "Generated" }, + ]; + yield* fs.writeFileString(file, records.map((r) => JSON.stringify(r)).join("\n") + "\n"); + let enumerations = 0; + const counting = FileSystem.make({ + ...fs, + readDirectory: (path) => { + enumerations++; + return fs.readDirectory(path); + }, + }); + const history = yield* makeLocalHistory; + const store = yield* makeClaudeHistoryStore(root, history.cache).pipe( + Effect.provideService(FileSystem.FileSystem, counting), + ); + const request = { + operation: "search", + sourceId: "claude", + query: "needle", + filterCwd: "/project", + } as const; + const first = yield* history.search(store, request); + expect(first.matches.map((m) => m.itemId)).toEqual(["old-evidence"]); + expect(first.matches[0]?.threadTitle).toBe("Renamed"); + const enumerated = enumerations; + const second = yield* history.search(store, { ...request, cursor: first.nextCursor! }); + expect(second.examinedItems).toBe(20); + const read = yield* history.read(store, { operation: "read", ref: first.matches[0]!.ref }); + expect(read.text).toBe("needle from the distant past"); + expect(enumerations).toBe(enumerated); + expect( + (yield* history + .read(store, { + operation: "read", + ref: { ...first.matches[0]!.ref, digest: "0".repeat(64) }, + }) + .pipe(Effect.flip)).reason, + ).toBe("changed"); + yield* fs.writeFileString( + file, + records.map((r) => JSON.stringify(r)).join("\n") + + "\n" + + JSON.stringify({ + type: "assistant", + uuid: "append", + message: { content: "needle appended" }, + }) + + "\n", + ); + expect( + (yield* history + .read(store, { operation: "read", ref: first.matches[0]!.ref }) + .pipe(Effect.flip)).reason, + ).toBe("changed"); + const restarted = yield* history.search(store, { ...request, cursor: second.nextCursor! }); + expect(restarted.matches.map((m) => m.itemId)).toEqual(["old-evidence"]); + expect( + (yield* store + .loadThread({ key: "../../outside.jsonl", version: "", updatedAt: 0 }) + .pipe(Effect.flip)).reason, + ).toBe("invalid_cursor"); + }).pipe(Effect.scoped), + ); + + it.effect("excludes T3 Claude sessions and follows the active Pi branch", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + const project = `${root}/projects/project`; + yield* fs.makeDirectory(project, { recursive: true }); + yield* fs.writeFileString( + `${project}/t3.jsonl`, + JSON.stringify({ + type: "user", + uuid: "t3", + entrypoint: "t3-code", + message: { content: "needle" }, + }) + "\n", + ); + const history = yield* makeLocalHistory; + const claude = yield* makeClaudeHistoryStore(root, history.cache); + expect( + (yield* history.search(claude, { + operation: "search", + sourceId: "claude", + query: "needle", + })).matches, + ).toEqual([]); + const sessions = `${root}/sessions/project`; + yield* fs.makeDirectory(sessions, { recursive: true }); + const piFile = `${sessions}/pi.jsonl`; + yield* fs.writeFileString( + piFile, + [ + { type: "session", id: "pi", cwd: "/project" }, + { + type: "message", + id: "user", + parentId: null, + message: { role: "user", content: "Question" }, + }, + { + type: "message", + id: "abandoned", + parentId: "user", + message: { role: "assistant", content: "needle abandoned" }, + }, + { + type: "message", + id: "active", + parentId: "user", + message: { role: "assistant", content: "needle active" }, + }, + { type: "session_info", id: "name", parentId: "active", name: "Pi renamed" }, + ] + .map((r) => JSON.stringify(r)) + .join("\n") + "\n", + ); + const pi = yield* makePiHistoryStore(`${root}/sessions`, history.cache); + const page = yield* history.search(pi, { + operation: "search", + sourceId: "pi", + query: "needle", + }); + expect(page.matches.map((m) => m.itemId)).toEqual(["active"]); + expect(page.matches[0]?.threadTitle).toBe("Pi renamed"); + yield* fs.writeFileString( + piFile, + JSON.stringify({ type: "message", message: { role: "user", content: "legacy" } }) + "\n", + ); + expect( + (yield* history + .search(pi, { operation: "search", sourceId: "pi", query: "legacy" }) + .pipe(Effect.flip)).reason, + ).toBe("unsupported"); + }).pipe(Effect.scoped), + ); +}); diff --git a/apps/server/src/project/externalHistory/JsonlHistoryStore.ts b/apps/server/src/project/externalHistory/JsonlHistoryStore.ts new file mode 100644 index 0000000000000..16448d5ae3c13 --- /dev/null +++ b/apps/server/src/project/externalHistory/JsonlHistoryStore.ts @@ -0,0 +1,256 @@ +import { ExternalHistoryError } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as PlatformError from "effect/PlatformError"; +import * as Schema from "effect/Schema"; +import { + TranscriptRecord, + conversationItem, + activePiBranch, + emptyClaudeFields, + foldClaudeFields, + transcriptItems, + type DecodedTranscriptRecord, + type TranscriptItem, +} from "../agentTranscripts.ts"; +import { + createTranscriptJsonReader, + createTranscriptJsonSelector, + TranscriptJsonLimitError, +} from "../AgentSessionJson.ts"; +import { + fileVersion, + type LocalThreadRef, + type LocalHistoryStore, + type ThreadCache, +} from "./LocalHistoryStore.ts"; + +const decodeRecord = Schema.decodeUnknownOption(TranscriptRecord); +const selectRecord = createTranscriptJsonSelector(TranscriptRecord); +const isHistoryError = Schema.is(ExternalHistoryError); +const isLimitError = Schema.is(TranscriptJsonLimitError); +const mapError = (cause: unknown) => + isHistoryError(cause) + ? cause + : new ExternalHistoryError({ + reason: isLimitError(cause) + ? "limit_exceeded" + : PlatformError.isPlatformError(cause) + ? cause.reason._tag === "NotFound" + ? "missing" + : "unavailable" + : "unsupported", + cause, + }); + +function projectThread( + records: ReadonlyArray, + source: "claudeAgent" | "pi", + version: string, +) { + const origin = source === "claudeAgent" && records.some((r) => r.entrypoint === "t3-code"); + const fields = records.reduce(foldClaudeFields, emptyClaudeFields); + const items = new Map(); + for (const record of source === "pi" ? activePiBranch(records) : records) { + const visible = transcriptItems(record, source); + const conversation = conversationItem(record, source); + if (!origin && conversation && !conversation.id) + throw new ExternalHistoryError({ reason: "unsupported" }); + for (const item of visible) { + const previous = items.get(item.id); + if (previous && (previous.role !== item.role || previous.text !== item.text)) + throw new ExternalHistoryError({ reason: "unsupported" }); + items.set(item.id, item); + } + } + const firstUser = [...items.values()].find((item) => item.role === "user"); + const title = + source === "pi" + ? records.findLast((r) => r.type === "session_info")?.name?.trim() + : (fields.customTitle ?? fields.title); + return { + version, + title: title || firstUser?.text.split("\n")[0]?.slice(0, 100) || "Conversation", + cwd: records.find((r) => r.cwd?.trim())?.cwd ?? null, + items: origin ? [] : [...items.values()], + ...(origin ? { origin: "t3" as const } : {}), + }; +} + +const makeJsonlHistoryStore = Effect.fn("ExternalHistory.openJsonlStore")(function* ( + root: string, + source: "claudeAgent" | "pi", + cache: ThreadCache, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const directory = yield* fs.realPath( + source === "claudeAgent" ? path.join(root, "projects") : root, + ); + if ((yield* fs.stat(directory)).type !== "Directory") + return yield* new ExternalHistoryError({ reason: "unsupported" }); + let listing: + | { directories: ReadonlyMap; refs: ReadonlyArray } + | undefined; + const listThreads = Effect.gen(function* () { + if (listing) { + let unchanged = true; + for (const [dir, version] of listing.directories) { + const stat = yield* fs.stat(dir).pipe(Effect.option); + if (Option.isNone(stat) || fileVersion(stat.value) !== version) { + unchanged = false; + break; + } + } + if (unchanged) return listing.refs; + } + const directories = new Map(); + const refs: LocalThreadRef[] = []; + const rootInfo = yield* fs.stat(directory); + directories.set(directory, fileVersion(rootInfo)); + const children = yield* fs.readDirectory(directory); + if (children.length > 50_000) + return yield* new ExternalHistoryError({ reason: "limit_exceeded" }); + for (const child of children) { + const childPath = path.join(directory, child); + if (yield* Effect.isSuccess(fs.readLink(childPath))) continue; + const info = yield* fs.stat(childPath); + const files = + info.type === "Directory" + ? yield* fs.readDirectory(childPath) + : source === "pi" && child.endsWith(".jsonl") + ? [child] + : []; + if (info.type === "Directory") directories.set(childPath, fileVersion(info)); + if (directories.size > 50_000 || files.length + refs.length > 50_000) + return yield* new ExternalHistoryError({ reason: "limit_exceeded" }); + for (const name of files) { + if (!name.endsWith(".jsonl")) continue; + const filePath = info.type === "Directory" ? path.join(childPath, name) : childPath; + if (yield* Effect.isSuccess(fs.readLink(filePath))) continue; + const stat = yield* fs.stat(filePath); + if (stat.type !== "File") continue; + const key = path.relative(directory, filePath).split(path.sep).join("/"); + if (key.length > 512) return yield* new ExternalHistoryError({ reason: "limit_exceeded" }); + refs.push({ + key, + updatedAt: Option.map(stat.mtime, (d) => d.getTime()).pipe(Option.getOrElse(() => 0)), + version: fileVersion(stat), + }); + } + } + refs.sort((a, b) => b.updatedAt - a.updatedAt || a.key.localeCompare(b.key)); + listing = { directories, refs }; + return refs; + }).pipe(Effect.mapError(mapError)); + + const loadThread = Effect.fn("ExternalHistory.loadJsonlThread")(function* (ref: LocalThreadRef) { + if ( + ref.key.includes("\\") || + ref.key.split("/").some((part) => !part || part === "." || part === "..") || + path.isAbsolute(ref.key) || + !ref.key.endsWith(".jsonl") + ) + return yield* new ExternalHistoryError({ reason: "invalid_cursor" }); + const filePath = path.join(directory, ...ref.key.split("/")); + const canonical = yield* fs.realPath(filePath); + if (canonical !== filePath) return yield* new ExternalHistoryError({ reason: "changed" }); + const stat = yield* fs.stat(filePath); + if (stat.type !== "File") return yield* new ExternalHistoryError({ reason: "unsupported" }); + const version = fileVersion(stat); + const thread = yield* cache( + `${source}\0${filePath}\0${version}`, + Effect.scoped( + Effect.gen(function* () { + const file = yield* fs.open(filePath, { flag: "r" }); + if (fileVersion(yield* file.stat) !== version) + return yield* new ExternalHistoryError({ reason: "changed" }); + const records: DecodedTranscriptRecord[] = []; + let allocated = 0; + let reader = createTranscriptJsonReader( + (bytes) => { + allocated += bytes; + if (allocated > 32 * 1024 * 1024) + throw new TranscriptJsonLimitError({ message: "Transcript text limit" }); + }, + (path) => path[0] !== "payload" && selectRecord(path), + ); + const finishRecord = () => { + const decoded = decodeRecord(reader.finish()); + if (Option.isNone(decoded)) throw new ExternalHistoryError({ reason: "unsupported" }); + if (records.length >= 100_000) + throw new ExternalHistoryError({ reason: "limit_exceeded" }); + records.push(decoded.value); + reader = createTranscriptJsonReader( + (bytes) => { + allocated += bytes; + if (allocated > 32 * 1024 * 1024) + throw new TranscriptJsonLimitError({ message: "Transcript text limit" }); + }, + (path) => path[0] !== "payload" && selectRecord(path), + ); + }; + const decoder = new TextDecoder("utf-8", { fatal: true }); + let pending = false; + let readBytes = 0; + const consume = (text: string) => { + let start = 0; + for (let index = 0; index < text.length; index++) { + if (text[index] !== "\n") continue; + const part = text.slice(start, index); + if (pending || part.trim()) { + reader.write(part); + finishRecord(); + } + pending = false; + start = index + 1; + } + const tail = text.slice(start); + if (tail.trim() || pending) { + reader.write(tail); + pending = true; + } + }; + while (true) { + const bytes = yield* file.readAlloc(64 * 1024); + if (Option.isNone(bytes)) break; + readBytes += bytes.value.length; + if (readBytes > 4 * 1024 * 1024 * 1024) + return yield* new ExternalHistoryError({ reason: "limit_exceeded" }); + yield* Effect.try({ + try: () => consume(decoder.decode(bytes.value, { stream: true })), + catch: mapError, + }); + } + yield* Effect.try({ + try: () => { + consume(decoder.decode()); + if (pending) finishRecord(); + }, + catch: mapError, + }); + if ( + fileVersion(yield* file.stat) !== version || + fileVersion(yield* fs.stat(filePath)) !== version + ) + return yield* new ExternalHistoryError({ reason: "changed" }); + return yield* Effect.try({ + try: () => projectThread(records, source, version), + catch: mapError, + }); + }), + ).pipe(Effect.mapError(mapError)), + ); + if (fileVersion(yield* fs.stat(filePath)) !== version) + return yield* new ExternalHistoryError({ reason: "changed" }); + return thread; + }, Effect.mapError(mapError)); + return { listThreads, loadThread } satisfies LocalHistoryStore; +}, Effect.mapError(mapError)); + +export const makeClaudeHistoryStore = (root: string, cache: ThreadCache) => + makeJsonlHistoryStore(root, "claudeAgent", cache); +export const makePiHistoryStore = (root: string, cache: ThreadCache) => + makeJsonlHistoryStore(root, "pi", cache); diff --git a/apps/server/src/project/externalHistory/OpenCodeHistoryStore.test.ts b/apps/server/src/project/externalHistory/OpenCodeHistoryStore.test.ts new file mode 100644 index 0000000000000..09bad8c783fcf --- /dev/null +++ b/apps/server/src/project/externalHistory/OpenCodeHistoryStore.test.ts @@ -0,0 +1,118 @@ +import * as NodeSqlite from "node:sqlite"; +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import { makeOpenCodeHistoryStore } from "./OpenCodeHistoryStore.ts"; +import { makeLocalHistory } from "./LocalHistory.ts"; + +it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("OpenCode local history", (it) => { + it.effect( + "reads current-schema user, assistant and tool text without reasoning or database writes", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + const file = `${root}/opencode.db`; + yield* Effect.sync(() => { + const db = new NodeSqlite.DatabaseSync(file); + try { + db.exec( + "CREATE TABLE session_v2 (id TEXT PRIMARY KEY, directory TEXT, title TEXT, parent_id TEXT, time_updated INTEGER); CREATE TABLE session_message (id TEXT PRIMARY KEY, session_id TEXT, type TEXT, seq INTEGER, data TEXT)", + ); + db.prepare("INSERT INTO session_v2 VALUES (?, ?, ?, ?, ?)").run( + "session", + "/project", + "OpenCode conversation", + null, + 1, + ); + const insert = db.prepare("INSERT INTO session_message VALUES (?, 'session', ?, ?, ?)"); + insert.run("user", "user", 0, JSON.stringify({ text: "needle user" })); + insert.run( + "assistant", + "assistant", + 1, + JSON.stringify({ + content: [ + { + type: "reasoning", + text: "hidden needle reasoning", + native: { opaque: "x".repeat(20_000_000) }, + }, + { type: "text", text: "needle answer" }, + { + type: "tool", + id: "call", + name: "read", + state: { + input: { path: "src/index.ts" }, + content: [{ type: "text", text: "needle tool output" }], + }, + }, + ], + }), + ); + } finally { + db.close(); + } + }); + const before = yield* fs.readFile(file); + const history = yield* makeLocalHistory; + const store = yield* makeOpenCodeHistoryStore(file, history.cache); + const result = yield* history.search(store, { + operation: "search", + sourceId: "opencode", + query: "needle", + }); + expect(result.matches.map((m) => [m.itemId, m.role])).toEqual([ + ["user", "user"], + ["assistant", "assistant"], + ["assistant/tool/call", "tool"], + ]); + expect(result.matches.map((m) => m.excerpt).join("\n")).not.toContain("reasoning"); + expect( + (yield* history.read(store, { operation: "read", ref: result.matches[2]!.ref })).text, + ).toBe("read\nneedle tool output"); + expect(Buffer.from(yield* fs.readFile(file)).equals(Buffer.from(before))).toBe(true); + expect(yield* fs.readDirectory(root)).toEqual(["opencode.db"]); + yield* Effect.sync(() => { + const db = new NodeSqlite.DatabaseSync(file); + try { + db.prepare("UPDATE session_message SET data = ? WHERE id = 'user'").run( + JSON.stringify({ text: "Changed" }), + ); + } finally { + db.close(); + } + }); + expect( + (yield* history + .read(store, { operation: "read", ref: result.matches[0]!.ref }) + .pipe(Effect.flip)).reason, + ).toBe("changed"); + }).pipe(Effect.scoped), + ); + + it.effect("rejects older OpenCode databases as an unsupported format", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + const file = `${root}/opencode.db`; + yield* Effect.sync(() => { + const db = new NodeSqlite.DatabaseSync(file); + try { + db.exec("CREATE TABLE message (id TEXT, data TEXT)"); + } finally { + db.close(); + } + }); + const history = yield* makeLocalHistory; + expect((yield* makeOpenCodeHistoryStore(file, history.cache).pipe(Effect.flip)).reason).toBe( + "unsupported", + ); + }).pipe(Effect.scoped), + ); +}); diff --git a/apps/server/src/project/externalHistory/OpenCodeHistoryStore.ts b/apps/server/src/project/externalHistory/OpenCodeHistoryStore.ts new file mode 100644 index 0000000000000..f87ffb67f97b8 --- /dev/null +++ b/apps/server/src/project/externalHistory/OpenCodeHistoryStore.ts @@ -0,0 +1,194 @@ +import * as NodeSqlite from "node:sqlite"; +import { ExternalHistoryError } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Schema from "effect/Schema"; +import type { TranscriptItem } from "../agentTranscripts.ts"; +import { + fileVersion, + type LocalHistoryStore, + type LocalThreadRef, + type ThreadCache, +} from "./LocalHistoryStore.ts"; + +const decodeSessions = Schema.decodeUnknownSync( + Schema.Array( + Schema.Struct({ + id: Schema.String, + title: Schema.NullOr(Schema.String), + directory: Schema.String, + time_updated: Schema.Number, + }), + ), +); +const decodeMessages = Schema.decodeUnknownSync( + Schema.Array( + Schema.Struct({ + id: Schema.String, + type: Schema.Literals(["user", "assistant"]), + visible: Schema.String, + }), + ), +); +const decodeVisible = Schema.decodeUnknownSync( + Schema.fromJsonString( + Schema.Struct({ + text: Schema.optionalKey(Schema.String), + content: Schema.optionalKey( + Schema.Array( + Schema.Struct({ + type: Schema.Literals(["text", "tool"]), + text: Schema.NullOr(Schema.String), + id: Schema.NullOr(Schema.String), + name: Schema.NullOr(Schema.String), + output: Schema.Array(Schema.String), + }), + ), + ), + }), + ), +); +const isHistoryError = Schema.is(ExternalHistoryError); +const mapError = (cause: unknown) => + isHistoryError(cause) ? cause : new ExternalHistoryError({ reason: "unavailable", cause }); + +export const makeOpenCodeHistoryStore = Effect.fn("ExternalHistory.openOpenCodeStore")(function* ( + filePath: string, + cache: ThreadCache, +) { + const fs = yield* FileSystem.FileSystem; + if ((yield* fs.stat(filePath)).type !== "File") + return yield* new ExternalHistoryError({ reason: "unsupported" }); + const withDatabase = (read: (db: NodeSqlite.DatabaseSync) => A) => + Effect.try({ + try: () => { + const db = new NodeSqlite.DatabaseSync(filePath, { readOnly: true }); + try { + db.exec("PRAGMA busy_timeout = 100"); + return read(db); + } finally { + db.close(); + } + }, + catch: mapError, + }); + yield* withDatabase((db) => { + for (const [table, required] of [ + ["session_v2", ["id", "directory", "title", "parent_id", "time_updated"]], + ["session_message", ["id", "session_id", "type", "seq", "data"]], + ] as const) { + const columns = new Set( + db + .prepare(`PRAGMA table_info(${table})`) + .all() + .map((row) => row["name"]), + ); + if (required.some((column) => !columns.has(column))) + throw new ExternalHistoryError({ reason: "unsupported" }); + } + }); + const version = Effect.gen(function* () { + const database = fileVersion(yield* fs.stat(filePath)); + const wal = yield* fs.stat(`${filePath}-wal`).pipe( + Effect.map(fileVersion), + Effect.catchTags({ + PlatformError: (error) => + error.reason._tag === "NotFound" ? Effect.succeed("") : Effect.fail(error), + }), + ); + return database + wal; + }).pipe(Effect.mapError(mapError)); + let listing: { version: string; refs: ReadonlyArray } | undefined; + const listThreads = Effect.gen(function* () { + const current = yield* version; + if (listing?.version === current) return listing.refs; + const rows = yield* withDatabase((db) => + decodeSessions( + db + .prepare( + "SELECT id, title, directory, time_updated FROM session_v2 WHERE parent_id IS NULL ORDER BY time_updated DESC, id DESC LIMIT 50001", + ) + .all(), + ), + ); + if (rows.length > 50_000) return yield* new ExternalHistoryError({ reason: "limit_exceeded" }); + const refs = rows.map((row) => ({ + key: row.id, + title: row.title ?? "Conversation", + cwd: row.directory, + updatedAt: row.time_updated, + version: current, + })); + listing = { version: current, refs }; + return refs; + }); + const loadThread = Effect.fn("ExternalHistory.loadOpenCodeThread")(function* ( + ref: LocalThreadRef, + ) { + const current = yield* version; + const thread = yield* cache( + `${filePath}\0${ref.key}\0${current}`, + withDatabase((db) => { + db.exec("BEGIN"); + const session = decodeSessions( + db + .prepare( + "SELECT id, title, directory, time_updated FROM session_v2 WHERE id = ? AND parent_id IS NULL", + ) + .all(ref.key), + )[0]; + if (!session) throw new ExternalHistoryError({ reason: "missing" }); + const rows = decodeMessages( + db + .prepare(`SELECT id, type, CASE WHEN json_valid(data) THEN + CASE WHEN type = 'user' THEN json_object('text', data ->> '$.text') + WHEN json_type(data, '$.content') = 'array' THEN json_object('content', json(( + SELECT coalesce(json_group_array(json_object( + 'type', c.value ->> '$.type', 'text', c.value ->> '$.text', + 'id', c.value ->> '$.id', 'name', c.value ->> '$.name', + 'output', json((SELECT coalesce(json_group_array(p.value ->> '$.text'), '[]') + FROM json_each(c.value, '$.state.content') p WHERE p.value ->> '$.type' = 'text')) + )), '[]') FROM json_each(data, '$.content') c WHERE c.value ->> '$.type' IN ('text', 'tool') + ))) END END AS visible + FROM session_message WHERE session_id = ? AND type IN ('user', 'assistant') ORDER BY seq, id LIMIT 100001`) + .all(ref.key), + ); + if (rows.length > 100_000) throw new ExternalHistoryError({ reason: "limit_exceeded" }); + const items: TranscriptItem[] = []; + let chars = 0; + for (const row of rows) { + if (row.visible.length > 16 * 1024 * 1024) + throw new ExternalHistoryError({ reason: "limit_exceeded" }); + const visible = decodeVisible(row.visible); + const text = + row.type === "user" + ? visible.text + : visible.content + ?.filter((part) => part.type === "text") + .map((part) => part.text ?? "") + .join("\n"); + if (text) items.push({ id: row.id, role: row.type, text }); + for (const part of visible.content ?? []) { + if (part.type !== "tool" || !part.id) continue; + const text = [part.name, ...part.output].filter(Boolean).join("\n"); + if (text) items.push({ id: `${row.id}/tool/${part.id}`, role: "tool", text }); + } + chars += + (text?.length ?? 0) + + (visible.content?.reduce((sum, part) => sum + part.output.join("\n").length, 0) ?? 0); + if (chars > 16 * 1024 * 1024) + throw new ExternalHistoryError({ reason: "limit_exceeded" }); + } + return { + version: current, + title: session.title ?? "Conversation", + cwd: session.directory, + items, + }; + }), + ); + if ((yield* version) !== current) return yield* new ExternalHistoryError({ reason: "changed" }); + return thread; + }); + return { listThreads, loadThread } satisfies LocalHistoryStore; +}, Effect.mapError(mapError)); diff --git a/packages/provider-opencode/package.json b/packages/provider-opencode/package.json index 3f569ca2c508b..7289e293a352a 100644 --- a/packages/provider-opencode/package.json +++ b/packages/provider-opencode/package.json @@ -19,6 +19,10 @@ "types": "./src/testing.ts", "import": "./src/testing.ts" }, + "./server/usage": { + "types": "./src/server/usage.ts", + "import": "./src/server/usage.ts" + }, "./server/OpenCodeRuntime": { "types": "./src/server/OpenCodeRuntime.ts", "import": "./src/server/OpenCodeRuntime.ts" diff --git a/packages/provider-opencode/src/server/usage.ts b/packages/provider-opencode/src/server/usage.ts index 4ae7b6df358e9..8358f14af10bb 100644 --- a/packages/provider-opencode/src/server/usage.ts +++ b/packages/provider-opencode/src/server/usage.ts @@ -314,7 +314,7 @@ export const readOpenCodeUsage = Effect.fn("readOpenCodeUsage")(function* ( return { files, missing: !found && !error, error }; }); -const resolveOpenCodeDataDirs = Effect.fn("resolveOpenCodeDataDirs")(function* ( +export const resolveOpenCodeDataDirs = Effect.fn("resolveOpenCodeDataDirs")(function* ( environment: NodeJS.ProcessEnv, ) { const fileSystem = yield* FileSystem.FileSystem; From 74b3e408701ea2e8a6191eeb44fa987c9f34cc6f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:27:39 -0800 Subject: [PATCH 05/15] feat(clients): lead external history with discovered conversations Show source availability and supported filters before search on web and mobile. Keep manual ChatGPT exports in a collapsed section and describe automatic discovery to agents and users. Co-Authored-By: Claude Fable 5.2 --- .../ExternalHistoryRouteScreen.test.tsx | 13 +- .../ExternalHistoryRouteScreen.tsx | 282 +++++++++------- .../SettingsServerControlsRouteScreen.tsx | 2 +- .../src/mcp/toolkits/externalHistory/tools.ts | 2 +- .../externalHistory/ExternalHistory.test.ts | 10 +- .../externalHistory/ExternalHistory.ts | 14 +- .../src/provider/agentContextBudget.json | 2 +- .../ExternalHistoryDialog.browser.test.tsx | 39 ++- .../externalHistory/ExternalHistoryDialog.tsx | 305 +++++++++++------- .../src/components/settings/ZackSettings.tsx | 2 +- docs/user/external-history.md | 30 +- .../src/externalHistory/index.test.ts | 16 + .../src/externalHistory/index.ts | 26 +- 13 files changed, 479 insertions(+), 264 deletions(-) create mode 100644 packages/client-runtime/src/externalHistory/index.test.ts diff --git a/apps/mobile/src/features/external-history/ExternalHistoryRouteScreen.test.tsx b/apps/mobile/src/features/external-history/ExternalHistoryRouteScreen.test.tsx index 4ebbd9d8a99c0..a0be498c637e5 100644 --- a/apps/mobile/src/features/external-history/ExternalHistoryRouteScreen.test.tsx +++ b/apps/mobile/src/features/external-history/ExternalHistoryRouteScreen.test.tsx @@ -117,7 +117,16 @@ vi.mock("../../state/query", () => ({ ? { operation: "sources", enabled: true, - sources: [{ id: "codex-history", label: "Native Codex", kind: "codex" }], + sources: [ + { + id: "codex-history", + label: "Native Codex", + kind: "codex", + capabilities: { cwdFilter: true, archived: true }, + status: "ready", + }, + ], + unsupported: [], } : request.operation === "search" ? { @@ -200,6 +209,8 @@ it("keeps remote history read-only without settings permission and stops reads w expect(container.textContent).toContain("Update this server"); act(() => registry.set(config, configFor(true))); expect(button("Allow external history search").disabled).toBe(true); + expect(container.querySelector('[aria-label="ChatGPT export paths"]')).toBeNull(); + await click("Add a ChatGPT export"); expect( container.querySelector('[aria-label="ChatGPT export paths"]')?.disabled, ).toBe(true); diff --git a/apps/mobile/src/features/external-history/ExternalHistoryRouteScreen.tsx b/apps/mobile/src/features/external-history/ExternalHistoryRouteScreen.tsx index eb23d85ad52fc..d89bed200e468 100644 --- a/apps/mobile/src/features/external-history/ExternalHistoryRouteScreen.tsx +++ b/apps/mobile/src/features/external-history/ExternalHistoryRouteScreen.tsx @@ -1,15 +1,20 @@ import { useAtomValue } from "@effect/atom-react"; import { useIsFocused, useNavigation, type StaticScreenProps } from "@react-navigation/native"; -import { externalHistoryRangeLabel } from "@t3tools/client-runtime/externalHistory"; +import { + externalHistoryRangeLabel, + externalHistorySearchFilters, + externalHistorySourceStatusLabel, +} from "@t3tools/client-runtime/externalHistory"; import { squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime"; import { EnvironmentId, type ExternalHistoryMatch, type ExternalHistoryRequest, type ExternalHistorySettings, + type ExternalHistorySource, type ProjectId, } from "@t3tools/contracts"; -import { useState } from "react"; +import { useState, type ReactNode } from "react"; import { ScrollView, View } from "react-native"; import { AppBar } from "../../components/ui/AppBar"; @@ -70,8 +75,8 @@ export function ExternalHistoryRouteScreen( contentContainerStyle={{ padding: 20, paddingBottom: 40, gap: 16 }} > - Search Codex conversations and configured ChatGPT exports on this environment. Open - the original evidence behind each match. + Search agent conversations found on this environment. Open the original evidence + behind each match. - {config.settings.externalHistory.enabled ? ( - - ) : ( - - External history search is off for this environment. - - )} + > + {config.settings.externalHistory.enabled ? ( + + ) : ( + + External history search is off for this environment. + + )} + ) : !supported ? ( @@ -108,11 +116,13 @@ export function ExternalHistoryRouteScreen( function HistorySettings(props: { readonly environmentId: EnvironmentId; readonly settings: ExternalHistorySettings; + readonly children: ReactNode; }) { const canWrite = useAtomValue( serverEnvironment.updateSettings.permissionAtom(props.environmentId), ); const update = useAtomCommand(serverEnvironment.updateSettings, { reportFailure: false }); + const [exportsOpen, setExportsOpen] = useState(false); const [paths, setPaths] = useState(props.settings.chatGptExportPaths.join("\n")); const [pending, setPending] = useState(false); const [error, setError] = useState(null); @@ -128,60 +138,74 @@ function HistorySettings(props: { if (result._tag === "Failure") setError(String(squashAtomCommandFailure(result))); }; return ( - - void save({ ...props.settings, enabled })} - /> - - Makes these sources available to this environment's connected clients and agents with thread - access. Source files stay in their existing locations. - - ChatGPT export paths - - - Paths refer to files on the T3 server. Remove a path to stop offering that export. - + + + void save({ ...props.settings, enabled })} + /> + + Makes these sources available to this environment's connected clients and agents with + thread access. Source files stay in their existing locations. + + {!canWrite ? ( + + This connection cannot change environment settings. + + ) : null} + {error ? ( + + {error} + + ) : null} + + {props.children}