diff --git a/packages/opencode/src/altimate/telemetry/index.ts b/packages/opencode/src/altimate/telemetry/index.ts index 0e43e0099f..07f1508d39 100644 --- a/packages/opencode/src/altimate/telemetry/index.ts +++ b/packages/opencode/src/altimate/telemetry/index.ts @@ -639,6 +639,66 @@ export namespace Telemetry { kind: "skills" | "memory" reason: "auth" | "not_found" | "network" | "other" } + // Workspace engine (see workspace/engine-telemetry.ts) + | { + /** A session's engine attach settled at a new outcome. */ + type: "workspace_engine_attach" + timestamp: number + session_id: string + outcome: "attached" | "engine_missing" | "engine_too_old" | "connect_failed" | "disabled" | "unbound" + /** The workspace the boundary settled for; absent for unbound and disabled. */ + workspace_id?: number + available?: number + declared?: number + missing?: number + extensions?: number + engine_version?: string + found_version?: string + error_class?: "link_unreadable" | "probe_failed" | "not_connected" | "other" + } + | { + type: "workspace_engine_install_prompt" + timestamp: number + session_id: string + action: "shown" | "accepted" | "installed" | "install_failed" | "copied" | "declined" + reason: "engine-missing" | "engine-too-old" + workspace_id?: number + declared?: number + found_version?: string + surface?: "headless" | "dialog" | "bus" | "toast" + node_detected?: boolean + node_major?: number + } + | { + /** A session's precedence snapshot settled at a new state. */ + type: "workspace_engine_precedence" + timestamp: number + session_id: string + enabled: boolean + reason?: string + shadowed_types: number + shadowed_connections: number + extensions: number + escape_hatch: boolean + workspace_id?: number + } + | { + /** A datamate tool call refused because the engine is off for that workspace. */ + type: "workspace_engine_redirect" + timestamp: number + session_id: string + tool: string + operation: string + outcome: "refused" + workspace_id?: number + } + | { + type: "workspace_engine_error" + timestamp: number + session_id: string + stage: "attach" | "install" | "precedence" + message: string + } // altimate_change end | { type: "mcp_server_census" diff --git a/packages/opencode/src/altimate/tools/datamate.ts b/packages/opencode/src/altimate/tools/datamate.ts index e7f2da3827..b2ea4c4d8d 100644 --- a/packages/opencode/src/altimate/tools/datamate.ts +++ b/packages/opencode/src/altimate/tools/datamate.ts @@ -79,14 +79,14 @@ export const DatamateManagerTool = Tool.define("datamate_manager", { .optional() .describe("Server name to remove (for 'remove'). Use 'list-config' or 'status' to find names."), }), - async execute(args): Promise<{ title: string; metadata: Record; output: string }> { + async execute(args, ctx): Promise<{ title: string; metadata: Record; output: string }> { // altimate_change start — a linked project gets its integrations from the // workspace's own engine only. The tool is kept out of the model's catalog // there (datamate-manager-gate.ts); this covers callers that run it // directly. Every operation is refused before anything is looked up or // written, so the refusal does not depend on the API being reachable. const workspace = await disablingWorkspace() - if (workspace) return refusal(args.operation, workspace) + if (workspace) return refusal(args.operation, workspace, ctx.sessionID) // altimate_change end if (args.operation !== "status" && args.operation !== "list-config") { const configured = await AltimateApi.isConfigured() @@ -106,9 +106,9 @@ export const DatamateManagerTool = Tool.define("datamate_manager", { case "list-integrations": return handleListIntegrations() case "add": - return handleAdd(args) + return handleAdd(args, "add", ctx.sessionID) case "create": - return handleCreate(args) + return handleCreate(args, ctx.sessionID) case "edit": return handleEdit(args) case "delete": @@ -116,7 +116,7 @@ export const DatamateManagerTool = Tool.define("datamate_manager", { case "status": return handleStatus() case "remove": - return handleRemove(args) + return handleRemove(args, ctx.sessionID) case "list-config": return handleListConfig() } @@ -250,6 +250,8 @@ function mergeRefreshedEntry( async function handleAdd( args: { datamate_id?: string; name?: string; scope?: "project" | "global" }, operation = "add", + /** The calling session, for the refusal's telemetry row. */ + sessionID?: string, ) { if (!args.datamate_id) { return { @@ -315,7 +317,7 @@ async function handleAdd( // it, as a boundary starts the engine's, or it could replace the engine's. using lock = await holdDirectoryLock() const off = await disablingWorkspace() - if (off) return refusal(operation, off) + if (off) return refusal(operation, off, sessionID) // altimate_change end if (transport !== null) { @@ -452,7 +454,7 @@ async function handleAdd( else await removeMcpFromConfig(serverName, configPath) again.release() if (prior && prior.enabled !== false) await MCP.add(serverName, prior).catch(() => {}) - return refusal(operation, linked) + return refusal(operation, linked, sessionID) } } // altimate_change end @@ -492,14 +494,17 @@ async function handleAdd( } } -async function handleCreate(args: { - name?: string - description?: string - integration_ids?: string[] - memory_enabled?: boolean - privacy?: string - scope?: "project" | "global" -}) { +async function handleCreate( + args: { + name?: string + description?: string + integration_ids?: string[] + memory_enabled?: boolean + privacy?: string + scope?: "project" | "global" + }, + sessionID?: string, +) { if (!args.name) { return { title: "Datamate create: FAILED", @@ -514,7 +519,7 @@ async function handleCreate(args: { // altimate_change start — checked again before the datamate is created: a // boundary may have linked the project since the check in `execute`. const off = await disablingWorkspace() - if (off) return refusal("create", off) + if (off) return refusal("create", off, sessionID) // The account the datamate is created in, kept for deleting it again: ids // are per tenant, and the credentials can change while the request runs. const creds = await AltimateApi.getCredentials() @@ -535,6 +540,7 @@ async function handleCreate(args: { const added = await handleAdd( { datamate_id: created.id, name: `datamate-${slugify(args.name)}`, scope: args.scope }, "create", + sessionID, ) if (!("managedBy" in added.metadata)) return added const rolledBack = await AltimateApi.deleteDatamate(created.id, creds).then( @@ -691,7 +697,7 @@ async function handleStatus() { } } -async function handleRemove(args: { server_name?: string; scope?: "project" | "global" }) { +async function handleRemove(args: { server_name?: string; scope?: "project" | "global" }, sessionID?: string) { if (!args.server_name) { return { title: "Datamate remove: FAILED", @@ -706,7 +712,7 @@ async function handleRemove(args: { server_name?: string; scope?: "project" | "g // check at the top of `execute` must not have its engine's key removed. using _lock = await holdDirectoryLock() const off = await disablingWorkspace() - if (off) return refusal("remove", off) + if (off) return refusal("remove", off, sessionID) // altimate_change end // Fully remove from runtime state (disconnect + purge from MCP list) // altimate_change start — MCP.remove (was disconnect): delete the status entry + publish diff --git a/packages/opencode/src/altimate/workspace/datamate-manager-gate.ts b/packages/opencode/src/altimate/workspace/datamate-manager-gate.ts index 1fdfcf28ef..2c94bc2f7b 100644 --- a/packages/opencode/src/altimate/workspace/datamate-manager-gate.ts +++ b/packages/opencode/src/altimate/workspace/datamate-manager-gate.ts @@ -19,6 +19,7 @@ // key. Null for an unlinked project, an unreadable binding with no engine // running, workspaces disabled, `altimate serve`, and organisation-managed // config that owns the `datamate` key. +import { trackRedirect } from "./engine-telemetry" import { linkedWorkspaceLoaded, managedWorkspace, settledOutcome, settledWorkspace } from "./engine-overlay" import { installCommand } from "./engine-offer" import { log } from "./engine-seams" @@ -52,7 +53,8 @@ export async function hiddenToolIds(): Promise> { /** What `datamate_manager` answers in a linked project, whatever the operation. * Nothing is looked up or written before it. */ -export function refusal(operation: string, workspace: { id: string; name: string }) { +export function refusal(operation: string, workspace: { id: string; name: string }, sessionID?: string) { + trackRedirect("datamate", operation, Number(workspace.id), sessionID) // telemetry: the engine is off for this workspace return { title: `Datamate ${operation}: off in a project linked to a workspace`, metadata: { operation, managedBy: workspace.id }, diff --git a/packages/opencode/src/altimate/workspace/engine-offer.ts b/packages/opencode/src/altimate/workspace/engine-offer.ts index 4e282e1471..5d2b83983f 100644 --- a/packages/opencode/src/altimate/workspace/engine-offer.ts +++ b/packages/opencode/src/altimate/workspace/engine-offer.ts @@ -7,6 +7,7 @@ // the detail here from disk and PATH rather than from the overlay's memory. // Offer, never install on the flow's own account — `installEngine` only ever // runs from an explicit "Install now". +import { trackInstallPrompt } from "./engine-telemetry" import { execFile, type ChildProcess } from "node:child_process" import launch from "cross-spawn" import { Process } from "@/util/process" @@ -272,15 +273,61 @@ export async function installEngine(): Promise { return { ok: false, error: detail || `npm exited with code ${run.code ?? "unknown"}` } } +/** What the bus carries with the offer command: the offer's telemetry fields, so the surface that + * prints the offer in another process (an attached run) reports `shown` with them. No node probe + * rides along: it would sit on the dispatch path, ahead of the turn boundary. */ +export type OfferDetail = Pick + +export function encodeOfferDetail(offer: EngineOffer): string { + const detail: OfferDetail = { + reason: offer.reason, + workspaceId: offer.workspaceId, + declared: offer.declared, + found: offer.found, + } + return JSON.stringify(detail) +} + +/** The detail back from the bus, or undefined for anything that is not one. */ +export function decodeOfferDetail(raw: unknown): OfferDetail | undefined { + if (typeof raw !== "string") return undefined + try { + const v = JSON.parse(raw) as Partial> + const reason = v.reason === "engine-missing" || v.reason === "engine-too-old" ? v.reason : undefined + const workspaceId = typeof v.workspaceId === "string" ? v.workspaceId : undefined + const declared = typeof v.declared === "number" ? v.declared : v.declared === undefined ? undefined : NaN + const found = typeof v.found === "string" ? v.found : v.found === undefined ? undefined : "" + if (!reason || workspaceId === undefined || Number.isNaN(declared) || found === "") { + return undefined + } + return { reason, workspaceId, declared, found } + } catch { + return undefined + } +} + +/** An attached run printed the offer it read off the bus: that process is the surface that showed + * it, so it reports `shown` (surface `bus`) from the fields the event carried. False when the + * event carried none that decode, and nothing is reported. */ +export function trackPrintedOffer(raw: unknown, sessionID: string): boolean { + const detail = decodeOfferDetail(raw) + if (!detail) return false + trackInstallPrompt("shown", detail, { surface: "bus" }, sessionID) + return true +} + /** Ask the TUI to raise the offer. False when the bus is unavailable. The * session is carried so an attached headless run, which reads the same event - * stream, prints the offer raised for its own session only. */ -async function publishOffer(sessionID: string): Promise { - if (syncInternals.publishOffer) return syncInternals.publishOffer(sessionID) + * stream, prints the offer raised for its own session only; the offer's fields + * ride along so that run reports `shown` from them (the TUI dialog re-derives + * the offer and reports its own). */ +async function publishOffer(offer: EngineOffer, sessionID: string): Promise { + const detail = encodeOfferDetail(offer) + if (syncInternals.publishOffer) return syncInternals.publishOffer(sessionID, detail) try { await AppRuntime.runPromise( EventV2Bridge.Service.use((events) => - events.publish(TuiEvent.CommandExecute, { command: OFFER_COMMAND, sessionID }), + events.publish(TuiEvent.CommandExecute, { command: OFFER_COMMAND, sessionID, arguments: detail }), ), ) return true @@ -320,11 +367,20 @@ export function toolsNeed(declared: number | undefined): string { /** Offer via the dialog surface when there is one; otherwise print (headless) * or toast (bus unavailable). Exactly one of these happens. */ export async function offerOrNotify(offer: EngineOffer, toast: Toast, sessionID: string): Promise { + // Telemetry: the offer was shown, on which surface, and whether Node was there to install with. + const shown = (surface: "headless" | "dialog" | "bus" | "toast") => + void nodeMajor() + .catch(() => null) + .then((major) => trackInstallPrompt("shown", offer, { surface, nodeMajor: major }, sessionID)) if (isHeadless()) { printLine(describeOfferLine(offer)) + shown("headless") return } + // A dialog, whether taken here or over the bus, reports `shown` itself when it renders: the TUI + // may hold the offer back (its 7-day latch), so dispatch is not display. The toast reports here + // only when it was shown. if (offerInstall(offer)) return - if (await publishOffer(sessionID)) return - await notify(toast) + if (await publishOffer(offer, sessionID)) return + if (await notify(toast)) shown("toast") } diff --git a/packages/opencode/src/altimate/workspace/engine-overlay.ts b/packages/opencode/src/altimate/workspace/engine-overlay.ts index 074f940ae8..3d535c41ef 100644 --- a/packages/opencode/src/altimate/workspace/engine-overlay.ts +++ b/packages/opencode/src/altimate/workspace/engine-overlay.ts @@ -44,6 +44,7 @@ import { } from "./engine-seams" import { declaredBounded, fingerprint, notify, printLine, resolveBinding, versionOf, which } from "./engine-probes" import { OFFER_RECHECK_MS, OFFER_SKIP_TTL_MS, installCommand, offerOrNotify, type EngineOffer } from "./engine-offer" +import { attachKey, trackAttach, trackEngineError } from "./engine-telemetry" import { ENGINE_BINARY, INSTALL_HELPS, @@ -293,6 +294,7 @@ export async function overlay( log.warn("workspace engine overlay failed; leaving the MCP config as loaded", { err: String(err) }) state.current = null state.failedAt = now() + trackEngineError("attach", String(err)) } } @@ -335,6 +337,12 @@ export function linkedWorkspace(directory: string | null = currentDirectory()): return linked ? { id: linked.id, name: linked.name } : null } +/** The linked workspace's account-scoped identity (credential scope plus id), for telemetry's dedupe. */ +export function linkedWorkspaceKey(directory: string | null = currentDirectory()): string | undefined { + if (!directory) return undefined + return directories.get(directory)?.linked?.key +} + /** `linkedWorkspace` once the overlay has run for this instance. */ export async function linkedWorkspaceLoaded( directory: string | null = currentDirectory(), @@ -358,17 +366,33 @@ type SessionRecord = { /** The workspace the directory was linked to when the boundary settled this * outcome. */ linked?: { id: string; name: string } | null + /** The key of the last attach reported to telemetry (workspace and outcome); a transient record + * carries the previous one. */ + tracked?: string } const sessions = new Map() const declaredCache = new Map() /** Verdict signatures a headless process has already printed to stderr. */ const headlessPrinted = new Set() -function record(sessionID: string, outcome: Outcome): SessionRecord { +function record( + sessionID: string, + outcome: Outcome, + extra: { engineVersion?: string | null; transient?: boolean } = {}, +): SessionRecord { const previous = sessions.get(sessionID) + // telemetry: a settled outcome, when it or the workspace it settled for changed. A transient + // record (the retry sentinel) reports nothing and keeps the last reported key for the next + // comparison. + const workspaceId = Number(linkedWorkspace()?.id) || undefined + const tracked = extra.transient ? previous?.tracked : attachKey(outcome, linkedWorkspaceKey()) + if (!extra.transient) { + trackAttach(outcome, previous?.tracked, { ...extra, workspaceId, workspaceIdentity: linkedWorkspaceKey() }, sessionID) + } sessions.delete(sessionID) const next: SessionRecord = { outcome, + tracked, announced: previous?.announced, announcedAt: previous?.announcedAt, retried: previous?.retried, @@ -490,6 +514,7 @@ export async function atTurnStart(sessionID: string, body: () => Promise): await reconcile(sessionID, directory, state) } catch (err) { log.warn("workspace engine turn hook failed", { sessionID, err: String(err) }) + trackEngineError("attach", String(err), sessionID) } // Kept with the outcome, under the lock: the directory's link moves with any // later config load, and a turn's notice must name the workspace its own @@ -828,14 +853,20 @@ async function reconcile(sessionID: string, directory: string, state: DirectoryS let status = statusMap[DATAMATE_KEY] const session = sessions.get(sessionID) if (status?.status !== "connected" && !session?.retried) { - ;(session ?? record(sessionID, { kind: "connect-failed", error: "retrying" })).retried = true + ;(session ?? record(sessionID, { kind: "connect-failed", error: "retrying" }, { transient: true })).retried = true log.info("workspace engine not connected; retrying once for this session", { workspaceId: workspace.id, sessionID, status: status?.status, }) - await mcp().add(DATAMATE_KEY, overlayNow.entry) - status = (await mcp().status())[DATAMATE_KEY] + try { + await mcp().add(DATAMATE_KEY, overlayNow.entry) + status = (await mcp().status())[DATAMATE_KEY] + } catch (err) { + // A retry that throws settles as a failure for this session, rather than leaving the turn + // hook to swallow it with no outcome recorded. + status = { status: "failed", error: String(err) } + } } if (status?.status !== "connected") { const outcome: Outcome = { @@ -873,7 +904,7 @@ async function reconcile(sessionID: string, directory: string, state: DirectoryS ...(unfulfilled === undefined ? {} : { unfulfilled }), ...(declared?.extensions?.length ? { extensions: declared.extensions } : {}), } - const rec = record(sessionID, outcome) + const rec = record(sessionID, outcome, { engineVersion: overlayNow.version }) const snapshot: AttachSnapshot = { workspace: { id: workspace.id, name: workspace.name, key: workspace.key }, engineVersion: overlayNow.version, diff --git a/packages/opencode/src/altimate/workspace/engine-seams.ts b/packages/opencode/src/altimate/workspace/engine-seams.ts index 73d6609159..4ab0553a47 100644 --- a/packages/opencode/src/altimate/workspace/engine-seams.ts +++ b/packages/opencode/src/altimate/workspace/engine-seams.ts @@ -33,7 +33,7 @@ export const syncInternals: { printLine?: (line: string) => void /** Install-offer seams (see engine-offer.ts). */ offer?: (offer: EngineOffer) => boolean - publishOffer?: (sessionID: string) => Promise + publishOffer?: (sessionID: string, detail: string) => Promise runInstall?: ( argv: string[], timeoutMs: number, diff --git a/packages/opencode/src/altimate/workspace/engine-telemetry.ts b/packages/opencode/src/altimate/workspace/engine-telemetry.ts new file mode 100644 index 0000000000..75f34962d0 --- /dev/null +++ b/packages/opencode/src/altimate/workspace/engine-telemetry.ts @@ -0,0 +1,245 @@ +// altimate_change - new file +// +// Telemetry for the workspace engine, per the shared event contract: one name per fact, fired +// on the outcome rather than the attempt, the workspace id on the row where the code has it +// (every row of a bound session also carries it from the telemetry context). Each builder is +// pure so it can be pinned by tests; each `track*` wrapper never throws into its caller. +import { Telemetry } from "@/altimate/telemetry" +import type { Outcome } from "./engine-types" +import type { EngineOffer } from "./engine-offer" +import type { Precedence } from "./precedence" + +export type AttachOutcome = "attached" | "engine_missing" | "engine_too_old" | "connect_failed" | "disabled" | "unbound" +export type InstallPromptAction = "shown" | "accepted" | "installed" | "install_failed" | "copied" | "declined" +export type InstallPromptSurface = "headless" | "dialog" | "bus" | "toast" +export type EngineStage = "attach" | "install" | "precedence" + +const sessionId = () => Telemetry.getContext().sessionId + +function outcomeName(kind: Outcome["kind"]): AttachOutcome { + return kind.replace(/-/g, "_") as AttachOutcome +} + +/** The class of a connect failure, from the fixed messages the overlay produces. */ +export function connectErrorClass(error: string): "link_unreadable" | "probe_failed" | "not_connected" | "other" { + const e = error.toLowerCase() + if (e.includes("link could not be read")) return "link_unreadable" + if (e.includes("could not be checked")) return "probe_failed" + if (e.includes("not connected") || e.includes("did not connect") || e.includes("handshake")) return "not_connected" + return "other" +} + +/** What makes two settled outcomes the same attach for telemetry: the kind and its counts. */ +export function attachSignature(outcome: Outcome): string { + switch (outcome.kind) { + case "attached": + return `attached:${outcome.available}:${outcome.declared ?? ""}:${outcome.missing?.length ?? ""}:${outcome.extensions?.length ?? 0}` + case "engine-missing": + return `engine-missing:${outcome.declared ?? ""}` + case "engine-too-old": + return `engine-too-old:${outcome.found ?? ""}` + case "connect-failed": + return `connect-failed:${connectErrorClass(outcome.error)}` + default: + return outcome.kind + } +} + +export function attachEvent( + outcome: Outcome, + extra: { engineVersion?: string | null; workspaceId?: number } = {}, + sessionID?: string, +): Telemetry.Event { + // The boundary's own workspace, named on the row rather than left to the process context; an + // unbound or disabled outcome has none, and says so with an explicit undefined so the session's + // entry is not stamped onto it either (the envelope drops an undefined value). + const settledFor = outcome.kind !== "unbound" && outcome.kind !== "disabled" ? extra.workspaceId : undefined + const base = { + type: "workspace_engine_attach" as const, + timestamp: Date.now(), + session_id: sessionID ?? sessionId(), + workspace_id: settledFor !== undefined && Number.isInteger(settledFor) && settledFor > 0 ? settledFor : undefined, + } + switch (outcome.kind) { + case "attached": + return { + ...base, + outcome: "attached", + available: outcome.available, + ...(outcome.declared !== undefined ? { declared: outcome.declared } : {}), + ...(outcome.missing !== undefined ? { missing: outcome.missing.length } : {}), + ...(outcome.extensions?.length ? { extensions: outcome.extensions.length } : {}), + ...(extra.engineVersion ? { engine_version: extra.engineVersion } : {}), + } + case "engine-missing": + return { + ...base, + outcome: "engine_missing", + ...(outcome.declared !== undefined ? { declared: outcome.declared } : {}), + } + case "engine-too-old": + return { ...base, outcome: "engine_too_old", ...(outcome.found ? { found_version: outcome.found } : {}) } + case "connect-failed": + return { ...base, outcome: "connect_failed", error_class: connectErrorClass(outcome.error) } + default: + return { ...base, outcome: outcomeName(outcome.kind) } + } +} + +/** Track the attach when the session's outcome changed (not on every turn boundary). */ +/** What an attach row is deduped on: the directory's workspace identity (the credential scope plus the + * tenant-local id, as the overlay keys it, so the same id under another account is another workspace) + * and the outcome's signature; the same-shaped attach to another workspace after a relink is a change. */ +export function attachKey(outcome: Outcome, workspaceIdentity?: string): string { + return `${workspaceIdentity ?? ""}|${attachSignature(outcome)}` +} + +/** Track a session's attach when its key differs from the last one reported for the session. */ +export function trackAttach( + outcome: Outcome, + previousKey: string | undefined, + extra: { engineVersion?: string | null; workspaceId?: number; workspaceIdentity?: string } = {}, + sessionID?: string, +): boolean { + try { + const identity = + extra.workspaceIdentity ?? (extra.workspaceId !== undefined ? String(extra.workspaceId) : undefined) + if (previousKey !== undefined && previousKey === attachKey(outcome, identity)) return false + Telemetry.track(attachEvent(outcome, extra, sessionID)) + return true + } catch { + return false + } +} + +export function installPromptEvent( + action: InstallPromptAction, + offer: Pick, + opts: { surface?: InstallPromptSurface; nodeMajor?: number | null } = {}, + sessionID?: string, +): Telemetry.Event { + const id = Number(offer.workspaceId) + return { + type: "workspace_engine_install_prompt", + timestamp: Date.now(), + session_id: sessionID ?? sessionId(), + action, + reason: offer.reason, + ...(Number.isInteger(id) && id > 0 ? { workspace_id: id } : {}), + ...(offer.declared !== undefined ? { declared: offer.declared } : {}), + ...(offer.found ? { found_version: offer.found } : {}), + ...(opts.surface ? { surface: opts.surface } : {}), + ...(opts.nodeMajor !== undefined ? { node_detected: opts.nodeMajor !== null } : {}), + ...(typeof opts.nodeMajor === "number" ? { node_major: opts.nodeMajor } : {}), + } +} + +export function trackInstallPrompt( + action: InstallPromptAction, + offer: Pick, + opts: { surface?: InstallPromptSurface; nodeMajor?: number | null } = {}, + sessionID?: string, +): void { + try { + Telemetry.track(installPromptEvent(action, offer, opts, sessionID)) + } catch { + // never block the offer + } +} + +export function precedenceSignature( + p: Pick, +): string { + const shadowed = [...p.shadowed.entries()] + .map(([type, caps]) => `${type}=${caps.size}`) + .sort() + .join(",") + return `${p.workspaceId ?? ""}|${p.enabled}|${p.disabledReason ?? ""}|${shadowed}|${p.extensions?.length ?? 0}` +} + +export function precedenceEvent( + p: Pick, + opts: { escapeHatch: boolean; connections: number; types: number; sessionID?: string; workspaceId?: number }, +): Telemetry.Event { + // The snapshot's own workspace, else the directory's linked one passed by refresh(): a disabled + // snapshot (escape hatch, unattributed) names no workspace itself, and the row must not be left + // to the process context. + const own = Number(p.workspaceId) + const id = Number.isInteger(own) && own > 0 ? own : opts.workspaceId + return { + type: "workspace_engine_precedence", + timestamp: Date.now(), + session_id: opts.sessionID ?? sessionId(), + enabled: p.enabled, + ...(p.disabledReason ? { reason: p.disabledReason } : {}), + // Types the session can route through the engine (the inventory line's count), not every + // materialised type: a session whose rules deny the engine keys shadows nothing. + shadowed_types: opts.types, + shadowed_connections: opts.connections, + extensions: p.extensions?.length ?? 0, + escape_hatch: opts.escapeHatch, + // Present and undefined for an unbound or disabled snapshot with no linked workspace, as on the + // attach row: the row names no workspace, and nothing else may name one for it. + workspace_id: id !== undefined && Number.isInteger(id) && id > 0 ? id : undefined, + } +} + +const precedenceSeen = new Map() +/** Sessions remembered for the dedupe; the oldest is forgotten past this, and reports again on its next refresh. */ +const MAX_SEEN_SESSIONS = 256 + +/** Track a session's precedence snapshot when it changed since the last refresh. */ +export function trackPrecedence( + sessionID: string, + p: Pick, + opts: { escapeHatch: boolean; connections: number; types: number; identity?: string; workspaceId?: number }, +): boolean { + try { + // The local connections shadowed and the escape hatch are part of what is reported, so a change + // in either re-reports the same snapshot; the account-scoped identity (credential scope plus + // id) tells the same id under another account apart. + const signature = `${precedenceSignature(p)}|c=${opts.connections}|t=${opts.types}|e=${opts.escapeHatch}|w=${opts.identity ?? ""}` + if (precedenceSeen.get(sessionID) === signature) return false + precedenceSeen.set(sessionID, signature) + if (precedenceSeen.size > MAX_SEEN_SESSIONS) precedenceSeen.delete(precedenceSeen.keys().next().value!) + Telemetry.track(precedenceEvent(p, { ...opts, sessionID })) + return true + } catch { + return false + } +} + +export function resetEngineTelemetryForTests(): void { + precedenceSeen.clear() +} + +export function trackEngineError(stage: EngineStage, message: string, sessionID?: string): void { + try { + Telemetry.track({ + type: "workspace_engine_error", + timestamp: Date.now(), + session_id: sessionID ?? sessionId(), + stage, + message: Telemetry.maskString(message).slice(0, 200), + }) + } catch { + // never surface + } +} + +/** A datamate tool call refused because the engine is off for that workspace. */ +export function trackRedirect(tool: string, operation: string, workspaceId: number, sessionID?: string): void { + try { + Telemetry.track({ + type: "workspace_engine_redirect", + timestamp: Date.now(), + session_id: sessionID ?? sessionId(), + tool, + operation, + outcome: "refused", + ...(Number.isInteger(workspaceId) && workspaceId > 0 ? { workspace_id: workspaceId } : {}), + }) + } catch { + // never surface + } +} diff --git a/packages/opencode/src/altimate/workspace/precedence.ts b/packages/opencode/src/altimate/workspace/precedence.ts index ff070b578e..bb3d9f1fa1 100644 --- a/packages/opencode/src/altimate/workspace/precedence.ts +++ b/packages/opencode/src/altimate/workspace/precedence.ts @@ -48,6 +48,7 @@ // that cache sooner, so it makes the next re-derivation see the change earlier — but the // per-turn re-derivation is the mechanism, not the notification. import { Config } from "@/config/config" +import { resetEngineTelemetryForTests, trackEngineError, trackPrecedence } from "./engine-telemetry" import { AppRuntime } from "@/effect/app-runtime" import { EventV2Bridge } from "@/event-v2-bridge" import { TuiEvent } from "@/server/tui-event" @@ -65,6 +66,8 @@ import { settledOutcome, type EntryLike, type Outcome, + linkedWorkspace, + linkedWorkspaceKey, } from "./engine-overlay" import { readLocalBindingScopedStrict, resolvePinnedBindingForRouting } from "./state" import { liveBridge } from "./engine-probes" @@ -532,10 +535,22 @@ export async function refresh( result = await derive(sessionID, tools) } catch (err) { log.warn("precedence could not be derived; running locally with a notice", { sessionID, err: String(err) }) + trackEngineError("precedence", String(err), sessionID) result = EMPTY("derive-failed") } if (ruleset) result.ruleset = ruleset remember(sessionID, result) + // telemetry: only when the snapshot (or the local connections it shadows) changed + trackPrecedence(sessionID, result, { + escapeHatch: escapeHatchOn(), + // The registry is read only when something can be shadowed: a disabled snapshot shadows nothing. + connections: result.enabled ? countShadowedConnections(result) : 0, + // Types the session can route through the engine, as the inventory line counts them. + types: result.enabled ? servedInventory(result).length : 0, + identity: linkedWorkspaceKey(), + // Named explicitly for disabled snapshots that carry no binding of their own. + workspaceId: Number(linkedWorkspace()?.id) || undefined, + }) // Mechanism 6 — say once, per session, what is now served where. Silence is the one // thing this design does not allow, but repeating it every turn would be noise. // A session that never had routing is told nothing — there is nothing to say. But a @@ -689,6 +704,7 @@ export function announcedSessionCount(): number { } export function resetForTests(): void { + resetEngineTelemetryForTests() bySession.clear() incarnations.clear() announced.clear() @@ -1141,7 +1157,7 @@ export function inventoryLine(precedence: Precedence): string { return `Workspace integrations — ${parts.join("; ")}. ${shadowedCount} local connection${shadowedCount === 1 ? "" : "s"} shadowed.` } -function countShadowedConnections(precedence: Precedence): number { +export function countShadowedConnections(precedence: Precedence): number { try { return Registry.list().warehouses.filter((w) => { const type = canonicalType(w.type) diff --git a/packages/opencode/src/cli/cmd/run.ts b/packages/opencode/src/cli/cmd/run.ts index 36d1c31b11..1c92eecb7b 100644 --- a/packages/opencode/src/cli/cmd/run.ts +++ b/packages/opencode/src/cli/cmd/run.ts @@ -62,7 +62,7 @@ import { SessionTermination } from "../../session/termination" // altimate_change start — upstream_fix: type-only import for the tracing-config cast (see tracer setup below) import type { ConfigV1 } from "@opencode-ai/core/v1/config/config" // altimate_change - render the workspace engine offer in an attached run -import { OFFER_COMMAND, installCommand } from "@/altimate/workspace/engine-offer" +import { OFFER_COMMAND, installCommand, trackPrintedOffer } from "@/altimate/workspace/engine-offer" // altimate_change end // When a tool's parameters can't be statically inferred (legacy fork tools whose @@ -998,6 +998,9 @@ You are speaking to a non-technical business executive. Follow these rules stric `This workspace's integration tools need the local engine on the server. Install it there with: ${installCommand()}` + EOL, ) + // This process is the surface that showed the offer, so it reports `shown` (surface + // `bus`) from the fields the event carried; the server's publish is not a display. + if (sessionID) trackPrintedOffer((event.properties as { arguments?: unknown }).arguments, sessionID) continue } // altimate_change end diff --git a/packages/opencode/src/plugin/tui/altimate/workspace.tsx b/packages/opencode/src/plugin/tui/altimate/workspace.tsx index 0895f828d3..8473cfc68b 100644 --- a/packages/opencode/src/plugin/tui/altimate/workspace.tsx +++ b/packages/opencode/src/plugin/tui/altimate/workspace.tsx @@ -42,6 +42,7 @@ import { type Namesakes, } from "@/altimate/workspace/workspace-name" import { currentAttachSnapshot } from "@/altimate/workspace/attach-snapshot" +import { trackEngineError, trackInstallPrompt } from "@/altimate/workspace/engine-telemetry" import { accountScope, loadStatusView, @@ -97,7 +98,7 @@ import { OFFER_SKIP_TTL_MS, type EngineOffer, } from "@/altimate/workspace/engine-offer" -import { useClipboard } from "@opencode-ai/tui/context/clipboard" +import { useClipboard, type ClipboardService } from "@opencode-ai/tui/context/clipboard" import { AltimateApi } from "@/altimate/api/client" import { Log } from "@/altimate/util/log" @@ -1779,6 +1780,8 @@ function EngineInstallOfferDialog(props: EngineOfferProps) { // opened in our place. So completion reports through a toast when we are // gone, and only touches the dialog while we still own it. let mounted = true + // The offer was shown: reported from the surface that rendered it, not from the dispatch. + onMount(() => trackInstallPrompt("shown", props.offer, { surface: "dialog", nodeMajor: props.nodeMajor })) onCleanup(() => { mounted = false // Release the single-offer latch however this dialog goes away — chosen, @@ -1865,6 +1868,8 @@ function EngineInstallOfferDialog(props: EngineOfferProps) { const performInstall = async () => { const result = await installEngine() if (!result.ok) { + trackInstallPrompt("install_failed", props.offer) + trackEngineError("install", result.error) installing = false if (!mounted) { // Dismissed mid-install: the failed-phase rows have nowhere to render, @@ -1880,6 +1885,7 @@ function EngineInstallOfferDialog(props: EngineOfferProps) { setPhase("failed") return } + trackInstallPrompt("installed", props.offer) // Only clear a dialog we still own — by now the user may have opened // another, and clearing the stack would take theirs down instead. if (mounted) props.api.ui.dialog.clear() @@ -1900,21 +1906,12 @@ function EngineInstallOfferDialog(props: EngineOfferProps) { const copyCommand = () => { const cmd = command() void (async () => { - try { - await clipboard.write?.(cmd) - // A resolved write is NOT proof of a copy. The host's writer picks - // xclip/xsel on Linux and otherwise falls back to clipboardy, and it - // swallows backend failures (`.catch(() => undefined)`), so on the many - // Linux/WSL boxes with neither tool installed the write silently does - // nothing. Read back and compare before claiming success. (Caught by - // E2E: the toast said "Copied:" while the clipboard was untouched.) - const back = await clipboard.read?.() - if (back?.data.trim() === cmd) { - props.api.ui.toast({ variant: "info", message: `Copied: ${cmd}` }) - return - } - } catch { - // Unreadable or unwritable clipboard — fall through and show it. + if (await confirmedCopy(clipboard, cmd)) { + // `copied` only once the read-back confirms the clipboard holds the command: the + // selection itself proves nothing (an unconfirmed copy tracks no row). + trackInstallPrompt("copied", props.offer) + props.api.ui.toast({ variant: "info", message: `Copied: ${cmd}` }) + return } props.api.ui.toast({ variant: "warning", @@ -1942,8 +1939,13 @@ function EngineInstallOfferDialog(props: EngineOfferProps) { if (option.value === "install") { if (installing) return installing = true + trackInstallPrompt("accepted", props.offer) void runInstall().catch((err) => { const message = err instanceof Error ? err.message : String(err) + // A rejection (a throw past installEngine's own result) is the same failure to the + // funnel as a reported one. + trackInstallPrompt("install_failed", props.offer) + trackEngineError("install", message) installing = false if (!mounted) { // Same as a reported failure after dismissal: the rows are gone, @@ -1966,6 +1968,7 @@ function EngineInstallOfferDialog(props: EngineOfferProps) { return } if (option.value === "skip") { + trackInstallPrompt("declined", props.offer) recordEngineSkip(props.api, props.offer.workspaceId, props.latchScope, Date.now()) } props.api.ui.dialog.clear() @@ -1974,6 +1977,25 @@ function EngineInstallOfferDialog(props: EngineOfferProps) { ) } +/** Write `text` to the clipboard and read it back: true only when the clipboard then holds it. + * + * A resolved write is NOT proof of a copy. The host's writer picks xclip/xsel on Linux and + * otherwise falls back to clipboardy, and it swallows backend failures (`.catch(() => undefined)`), + * so on the many Linux/WSL boxes with neither tool installed the write silently does nothing. + * (Caught by E2E: the toast said "Copied:" while the clipboard was untouched.) An unreadable or + * unwritable clipboard is the same unconfirmed outcome. */ +export async function confirmedCopy(clipboard: Pick, text: string) { + // No writer means nothing was copied, whatever the clipboard happens to hold. + if (!clipboard.write) return false + try { + await clipboard.write(text) + const back = await clipboard.read?.() + return back?.data.trim() === text + } catch { + return false + } +} + /** Show the offer unless the 7-day latch suppresses it. * * The overlay raises this as a bare command over the event bus — it cannot hand diff --git a/packages/opencode/src/server/tui-event.ts b/packages/opencode/src/server/tui-event.ts index 93d3f7e6df..c7e62862d4 100644 --- a/packages/opencode/src/server/tui-event.ts +++ b/packages/opencode/src/server/tui-event.ts @@ -36,6 +36,8 @@ export const TuiEvent = { // same stream: the session it was raised for lets that run print only its // own offer, not another session's in the same directory. sessionID: Schema.optional(Schema.String), + // The offer's telemetry fields (JSON), for the surface that prints it to report `shown` with. + arguments: Schema.optional(Schema.String), // altimate_change end }, }), diff --git a/packages/opencode/test/altimate/plugin/workspace.test.ts b/packages/opencode/test/altimate/plugin/workspace.test.ts index 9670a24113..fad6d04623 100644 --- a/packages/opencode/test/altimate/plugin/workspace.test.ts +++ b/packages/opencode/test/altimate/plugin/workspace.test.ts @@ -35,6 +35,7 @@ const { recordEngineSkip, awaitKvReady, canInstallWith, + confirmedCopy, engineOfferInternals, showEngineInstallOffer, linkDialogInternals, @@ -1249,3 +1250,40 @@ describe("/integrations", () => { expect(h.selects[1]!.options[0]!.description).toContain("No session has attached yet") }) }) + +describe("confirmedCopy", () => { + const cmd = "npm i -g @altimateai/altimate-code-engine" + test("true only when the read-back holds the command", async () => { + let held = "" + const clipboard = { + write: async (data: string) => { + held = data + }, + read: async () => ({ data: `${held}\n`, mime: "text/plain" }), + } + expect(await confirmedCopy(clipboard, cmd)).toBe(true) + }) + test("false when the writer silently did nothing", async () => { + const clipboard = { + write: async () => {}, + read: async () => ({ data: "something else", mime: "text/plain" }), + } + expect(await confirmedCopy(clipboard, cmd)).toBe(false) + }) + test("false when the clipboard throws or cannot be read", async () => { + expect( + await confirmedCopy( + { + write: async () => { + throw new Error("no backend") + }, + read: async () => ({ data: cmd, mime: "text/plain" }), + }, + cmd, + ), + ).toBe(false) + expect(await confirmedCopy({ write: async () => {}, read: undefined }, cmd)).toBe(false) + // No writer: the clipboard holding the command already is not a copy. + expect(await confirmedCopy({ write: undefined, read: async () => ({ data: cmd, mime: "text/plain" }) }, cmd)).toBe(false) + }) +}) diff --git a/packages/opencode/test/altimate/workspace/datamate-manager-gate.test.ts b/packages/opencode/test/altimate/workspace/datamate-manager-gate.test.ts index 4db9c869f9..301b511517 100644 --- a/packages/opencode/test/altimate/workspace/datamate-manager-gate.test.ts +++ b/packages/opencode/test/altimate/workspace/datamate-manager-gate.test.ts @@ -38,8 +38,10 @@ import { DATAMATE_MANAGER_TOOL_ID, engineNotice, hiddenToolIds, + refusal, turnNotice, } from "../../../src/altimate/workspace/datamate-manager-gate" +import { Telemetry } from "../../../src/altimate/telemetry" const ORIGINAL_FLAG = process.env.ALTIMATE_DISABLE_WORKSPACE const ORIGINAL_SPEC = process.env.ALTIMATE_ENGINE_INSTALL_SPEC @@ -815,3 +817,28 @@ describe("older datamate- entries in a linked project", () => { }) } }) + +describe("telemetry — the refusal", () => { + test("a refused datamate_manager call is reported as a redirect naming the operation and workspace", () => { + const rows: Array> = [] + const spy = spyOn(Telemetry, "track").mockImplementation((e) => { + rows.push(e as Record) + }) + try { + const out = refusal("status", { id: "42", name: "analytics" }, "ses_r") + expect(out.metadata).toEqual({ operation: "status", managedBy: "42" }) + expect(rows).toEqual([ + expect.objectContaining({ + type: "workspace_engine_redirect", + tool: "datamate", + operation: "status", + outcome: "refused", + workspace_id: 42, + session_id: "ses_r", + }), + ]) + } finally { + spy.mockRestore() + } + }) +}) diff --git a/packages/opencode/test/altimate/workspace/engine-install-offer.test.ts b/packages/opencode/test/altimate/workspace/engine-install-offer.test.ts index d4b75772cd..d99aaf2be4 100644 --- a/packages/opencode/test/altimate/workspace/engine-install-offer.test.ts +++ b/packages/opencode/test/altimate/workspace/engine-install-offer.test.ts @@ -22,8 +22,14 @@ import { type EngineOffer, type Toast, } from "../../../src/altimate/workspace/engine-overlay" -import { OFFER_RECHECK_MS, OFFER_SKIP_TTL_MS } from "../../../src/altimate/workspace/engine-offer" +import { + OFFER_RECHECK_MS, + OFFER_SKIP_TTL_MS, + decodeOfferDetail, + trackPrintedOffer, +} from "../../../src/altimate/workspace/engine-offer" import type { CachedBinding } from "../../../src/altimate/workspace/state" +import { Telemetry } from "../../../src/altimate/telemetry" const DIR = "/tmp/analytics" const ORIGINAL_FLAG = process.env.ALTIMATE_DISABLE_WORKSPACE @@ -37,7 +43,14 @@ const binding: CachedBinding = { linkedAt: 0, } as CachedBinding -type Harness = { offers: EngineOffer[]; toasts: Toast[]; printed: string[]; published: number; publishedFor: string[] } +type Harness = { + offers: EngineOffer[] + toasts: Toast[] + printed: string[] + published: number + publishedFor: string[] + publishedDetail: string[] +} /** No engine on PATH (or an old one) plus captured surfaces. */ function install(opts: { @@ -49,7 +62,7 @@ function install(opts: { surface?: boolean bound?: boolean }): Harness { - const h: Harness = { offers: [], toasts: [], printed: [], published: 0, publishedFor: [] } + const h: Harness = { offers: [], toasts: [], printed: [], published: 0, publishedFor: [], publishedDetail: [] } delete process.env.ALTIMATE_DISABLE_WORKSPACE syncInternals.serve = () => false syncInternals.headless = () => opts.headless === true @@ -67,10 +80,11 @@ function install(opts: { syncInternals.printLine = (line) => { h.printed.push(line) } - syncInternals.publishOffer = async (sessionID) => { + syncInternals.publishOffer = async (sessionID, detail) => { if (opts.bus === false) return false h.published += 1 h.publishedFor.push(sessionID) + h.publishedDetail.push(detail) return true } if (opts.surface) { @@ -160,6 +174,43 @@ describe("offer routing — engine missing", () => { expect(h.toasts).toEqual([]) expect(h.printed).toEqual([]) }) + test("the published offer carries its fields, and the surface that prints them reports shown", async () => { + const h = install({}) + await beforeTurn("s1") + expect(decodeOfferDetail(h.publishedDetail[0])).toEqual({ + reason: "engine-missing", + workspaceId: "42", + declared: 2, + found: undefined, + }) + const rows: Record[] = [] + const spy = spyOn(Telemetry, "track").mockImplementation((e) => { + rows.push(e as Record) + }) + try { + // An attached run prints the offer from this event in its own process and reports from it. + expect(trackPrintedOffer(h.publishedDetail[0], "s1")).toBe(true) + expect(trackPrintedOffer(undefined, "s1")).toBe(false) + expect(trackPrintedOffer("not json", "s1")).toBe(false) + expect(trackPrintedOffer(JSON.stringify({ reason: "other", workspaceId: "42" }), "s1")).toBe(false) + expect(trackPrintedOffer(JSON.stringify({ reason: "engine-missing", declared: "2" }), "s1")).toBe(false) + } finally { + spy.mockRestore() + } + expect(rows).toEqual([ + expect.objectContaining({ + type: "workspace_engine_install_prompt", + action: "shown", + surface: "bus", + session_id: "s1", + workspace_id: 42, + reason: "engine-missing", + declared: 2, + }), + ]) + // No node fields: the probe stays off the dispatch path, so the event carries none. + expect("node_detected" in rows[0]!).toBe(false) + }) test("falls back to the toast only when the bus is unavailable", async () => { const h = install({ bus: false }) await beforeTurn("s1") @@ -460,3 +511,57 @@ describe("install success is verified, not assumed", () => { expect(await installEngine()).toEqual({ ok: false, error: "boom EACCES denied" }) }) }) + +describe("telemetry — the install prompt", () => { + const rows: Array> = [] + let spy: ReturnType + beforeEach(() => { + rows.length = 0 + spy = spyOn(Telemetry, "track").mockImplementation((e) => { + rows.push(e as Record) + }) + }) + afterEach(() => spy.mockRestore()) + const settle = () => new Promise((resolve) => setTimeout(resolve, 0)) + const prompts = () => rows.filter((r) => r.type === "workspace_engine_install_prompt") + + test("headless: shown once per session, naming the surface and the node that could install", async () => { + install({ headless: true }) + syncInternals.nodeMajor = async () => 22 + await beforeTurn("s1") + await beforeTurn("s1") + await settle() + expect(prompts()).toEqual([ + expect.objectContaining({ + session_id: "s1", + action: "shown", + reason: "engine-missing", + workspace_id: 42, + declared: 2, + surface: "headless", + node_detected: true, + node_major: 22, + }), + ]) + // The missing engine is the attach outcome behind the prompt, reported once as well. + expect(rows.filter((r) => r.type === "workspace_engine_attach")).toEqual([ + expect.objectContaining({ outcome: "engine_missing", declared: 2 }), + ]) + }) + + test("a dialog reports shown itself when it renders, so dispatch over the bus reports nothing here", async () => { + install({}) + await beforeTurn("s1") + await settle() + expect(prompts()).toHaveLength(0) + }) + + test("the toast reports shown once shown, and no node on PATH is said rather than guessed", async () => { + install({ bus: false }) + await beforeTurn("s1") + await settle() + expect(prompts()).toHaveLength(1) + expect(prompts()[0]).toEqual(expect.objectContaining({ surface: "toast", node_detected: false })) + expect("node_major" in prompts()[0]).toBe(false) + }) +}) diff --git a/packages/opencode/test/altimate/workspace/engine-overlay.test.ts b/packages/opencode/test/altimate/workspace/engine-overlay.test.ts index ca845e28f5..601a853366 100644 --- a/packages/opencode/test/altimate/workspace/engine-overlay.test.ts +++ b/packages/opencode/test/altimate/workspace/engine-overlay.test.ts @@ -29,6 +29,7 @@ import { } from "../../../src/altimate/workspace/engine-overlay" import type { ScopedBinding } from "../../../src/altimate/workspace/engine-seams" import type { AttachSnapshot } from "../../../src/altimate/workspace/attach-snapshot" +import { Telemetry } from "../../../src/altimate/telemetry" import { log } from "../../../src/altimate/workspace/engine-seams" import { DATAMATE_KEY } from "../../../src/altimate/datamate-transport" @@ -1262,3 +1263,100 @@ describe("beforeTurn — what a turn boundary does", () => { expect(managedWorkspace()).toEqual({ id: "42", name: "analytics" }) }) }) + +describe("telemetry — the attach", () => { + const rows: Array> = [] + let spy: ReturnType + beforeEach(() => { + rows.length = 0 + spy = spyOn(Telemetry, "track").mockImplementation((e) => { + rows.push(e as Record) + }) + }) + afterEach(() => spy.mockRestore()) + const attaches = () => rows.filter((r) => r.type === "workspace_engine_attach") + + test("a settled attach is reported once with its inventory; a repeated boundary adds nothing; a change reports again", async () => { + const report = [{ key: "dbt_execute_sql", integrationId: "dbt", reason: "invalid-connection" }] + const h = install({ meta: { [UNFULFILLED_META_KEY]: report } }) + await beforeTurn("s1") + await beforeTurn("s1") + expect(attaches()).toEqual([ + expect.objectContaining({ + session_id: "s1", + workspace_id: 42, + outcome: "attached", + available: 2, + declared: 3, + missing: 1, + engine_version: "0.7.3", + }), + ]) + h.binding = null + await beforeTurn("s1") + expect(attaches().map((r) => r.outcome)).toEqual(["attached", "unbound"]) + }) + + test("the retry sentinel is not a settled attach: a retry that succeeds reports one attach, one that fails reports one failure", async () => { + const h = install({ status: "failed" }) + h.onAdd = () => { + h.status = "connected" + } + await beforeTurn("s1") + await beforeTurn("s1") + expect(attaches().map((r) => r.outcome)).toEqual(["attached"]) + rows.length = 0 + install({ status: "failed", statusError: "Connection closed" }) + await beforeTurn("s2") + await beforeTurn("s2") + expect(attaches().map((r) => r.outcome)).toEqual(["connect_failed"]) + // A retry whose add throws settles as a failure too, rather than vanishing into the turn hook. + rows.length = 0 + const g = install({ status: "failed" }) + g.onAdd = () => { + throw new Error("spawn failed") + } + await beforeTurn("s3") + expect(settledOutcome("s3")?.kind).toBe("connect-failed") + expect(attaches().map((r) => r.outcome)).toEqual(["connect_failed"]) + }) + + test("a relink to another workspace reports the attach again, named for the new workspace", async () => { + const h = install({}) + await beforeTurn("s1") + h.binding = bound(7, "growth") + await beforeTurn("s1") + await beforeTurn("s1") + expect(attaches().map((r) => [r.outcome, r.workspace_id])).toEqual([ + ["attached", 42], + ["attached", 7], + ]) + // The same id under another account is another workspace: reported again. + h.binding = bound(7, "growth", "other|https://api.other.example") + await beforeTurn("s1") + expect(attaches().map((r) => [r.outcome, r.workspace_id])).toEqual([ + ["attached", 42], + ["attached", 7], + ["attached", 7], + ]) + }) + + test("an engine below the floor reports the version it found", async () => { + install({ version: "0.1.0" }) + await beforeTurn("s1") + expect(attaches()).toEqual([expect.objectContaining({ outcome: "engine_too_old", found_version: "0.1.0" })]) + }) + + test("a probe that throws during the overlay is an attach-stage error with a masked message", async () => { + const h = install({ mcp: { datamate: IDE_ENTRY } }) + syncInternals.versionOf = async () => { + throw new Error("probe exploded at /home/someone/bin/datamate") + } + await overlay(DIR, h.config) + const errors = rows.filter((r) => r.type === "workspace_engine_error") + expect(errors).toHaveLength(1) + expect(errors[0]).toEqual(expect.objectContaining({ stage: "attach" })) + expect(String(errors[0].message)).toContain("probe exploded") + expect(String(errors[0].message)).not.toContain("/home/someone") + }) +}) diff --git a/packages/opencode/test/altimate/workspace/engine-telemetry.test.ts b/packages/opencode/test/altimate/workspace/engine-telemetry.test.ts new file mode 100644 index 0000000000..4c0e84ce6b --- /dev/null +++ b/packages/opencode/test/altimate/workspace/engine-telemetry.test.ts @@ -0,0 +1,189 @@ +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test" +import { Telemetry } from "../../../src/altimate/telemetry" +import { + attachEvent, + attachKey, + attachSignature, + connectErrorClass, + installPromptEvent, + precedenceEvent, + resetEngineTelemetryForTests, + trackAttach, + trackEngineError, + trackPrecedence, + trackRedirect, +} from "../../../src/altimate/workspace/engine-telemetry" + +describe("workspace engine telemetry builders", () => { + let events: any[] + let spy: ReturnType + beforeEach(() => { + events = [] + spy = spyOn(Telemetry, "track").mockImplementation((e) => { + events.push(e) + }) + resetEngineTelemetryForTests() + }) + afterEach(() => { + spy.mockRestore() + resetEngineTelemetryForTests() + }) + + test("attach outcomes map to the event with counts, versions and an error class", () => { + expect( + attachEvent({ kind: "attached", available: 3, declared: 4, missing: ["x"] }, { engineVersion: "1.2.3" }), + ).toEqual( + expect.objectContaining({ + type: "workspace_engine_attach", + outcome: "attached", + available: 3, + declared: 4, + missing: 1, + engine_version: "1.2.3", + }), + ) + expect(attachEvent({ kind: "engine-missing", declared: 2 })).toEqual( + expect.objectContaining({ outcome: "engine_missing", declared: 2 }), + ) + expect(attachEvent({ kind: "engine-too-old", found: "0.9.0" })).toEqual( + expect.objectContaining({ outcome: "engine_too_old", found_version: "0.9.0" }), + ) + expect(attachEvent({ kind: "connect-failed", error: "the workspace link could not be read" })).toEqual( + expect.objectContaining({ outcome: "connect_failed", error_class: "link_unreadable" }), + ) + expect(attachEvent({ kind: "unbound" })).toEqual(expect.objectContaining({ outcome: "unbound" })) + expect(connectErrorClass("the workspace engine could not be checked")).toBe("probe_failed") + expect((attachEvent({ kind: "unbound" }, {}, "ses_x") as any).session_id).toBe("ses_x") + expect(connectErrorClass("something else")).toBe("other") + }) + + test("an attach is tracked when the outcome changed, not on every turn boundary", () => { + const attached = { kind: "attached", available: 2 } as const + expect(trackAttach(attached, undefined)).toBe(true) + expect(trackAttach(attached, attachKey(attached))).toBe(false) + expect(trackAttach({ kind: "attached", available: 3 }, attachKey(attached))).toBe(true) + expect(trackAttach({ kind: "engine-missing" }, attachKey({ kind: "attached", available: 3 }))).toBe(true) + // The same-shaped attach to another workspace (a relink) is a change, and the identity is the + // account-scoped key: the same id under another account is another workspace. + expect( + trackAttach(attached, attachKey(attached, "acme|42"), { workspaceId: 42, workspaceIdentity: "acme|42" }), + ).toBe(false) + expect(trackAttach(attached, attachKey(attached, "acme|42"), { workspaceId: 7, workspaceIdentity: "acme|7" })).toBe( + true, + ) + expect( + trackAttach(attached, attachKey(attached, "acme|42"), { workspaceId: 42, workspaceIdentity: "other|42" }), + ).toBe(true) + expect(events.map((e) => e.outcome)).toEqual(["attached", "attached", "engine_missing", "attached", "attached"]) + expect(attachSignature({ kind: "connect-failed", error: "a" })).toBe( + attachSignature({ kind: "connect-failed", error: "b" }), + ) + expect(attachSignature({ kind: "attached", available: 2, extensions: [{ integration: "x" } as any] })).not.toBe( + attachSignature({ kind: "attached", available: 2 }), + ) + expect((attachEvent({ kind: "attached", available: 1 }, { workspaceId: 42 }) as any).workspace_id).toBe(42) + // An unbound attach names no workspace, explicitly: the key is present and undefined, so the + // session's entry cannot be stamped onto it. + const unbound = attachEvent({ kind: "unbound" }, { workspaceId: 42 }) as any + expect("workspace_id" in unbound).toBe(true) + expect(unbound.workspace_id).toBeUndefined() + }) + + test("the install prompt event carries the offer, the surface and node detection", () => { + const offer = { reason: "engine-missing" as const, workspaceId: "17", declared: 2 } + expect(installPromptEvent("shown", offer, { surface: "headless", nodeMajor: 22 })).toEqual( + expect.objectContaining({ + type: "workspace_engine_install_prompt", + action: "shown", + reason: "engine-missing", + workspace_id: 17, + declared: 2, + surface: "headless", + node_detected: true, + node_major: 22, + }), + ) + const shown = installPromptEvent( + "shown", + { reason: "engine-too-old", workspaceId: "x", found: "0.9.0" }, + { nodeMajor: null }, + ) as any + expect(shown.node_detected).toBe(false) + expect("node_major" in shown).toBe(false) + expect("workspace_id" in shown).toBe(false) + expect(shown.found_version).toBe("0.9.0") + expect(installPromptEvent("declined", offer)).toEqual( + expect.objectContaining({ action: "declined", workspace_id: 17 }), + ) + }) + + test("precedence snapshots are tracked per session when they change", () => { + const shadowed = new Map([ + [ + "snowflake", + new Map([ + ["sql_execute", {} as any], + ["sql_explain", {} as any], + ]), + ], + ["dbt", new Map([["dbt_run", {} as any]])], + ]) + const on: any = { enabled: true, shadowed, extensions: [{ integration: "x", tools: [] }], workspaceId: "7" } + // A disabled snapshot carries no binding; the directory's linked workspace names the row. + const offSnap: any = { enabled: false, disabledReason: "escape-hatch" as const, shadowed: new Map() } + expect( + (precedenceEvent(offSnap, { escapeHatch: true, connections: 0, types: 0, workspaceId: 7 }) as any).workspace_id, + ).toBe(7) + // Unbound and no linked workspace: the key is present and undefined, so the row names none. + const unboundRow = precedenceEvent(offSnap, { escapeHatch: true, connections: 0, types: 0 }) as any + expect("workspace_id" in unboundRow).toBe(true) + expect(unboundRow.workspace_id).toBeUndefined() + expect(precedenceEvent(on, { escapeHatch: false, connections: 1, types: 2 })).toEqual( + expect.objectContaining({ + type: "workspace_engine_precedence", + enabled: true, + shadowed_types: 2, + shadowed_connections: 1, + extensions: 1, + escape_hatch: false, + workspace_id: 7, + }), + ) + expect(trackPrecedence("s1", on, { escapeHatch: false, connections: 1, types: 2 })).toBe(true) + expect(trackPrecedence("s1", on, { escapeHatch: false, connections: 1, types: 2 })).toBe(false) // unchanged + expect(trackPrecedence("s1", on, { escapeHatch: false, connections: 2, types: 2 })).toBe(true) // a connection was added locally + expect(trackPrecedence("s2", on, { escapeHatch: false, connections: 2, types: 2 })).toBe(true) // another session + const off: any = { enabled: false, disabledReason: "escape-hatch" as const, shadowed: new Map(), workspaceId: "7" } + expect(trackPrecedence("s1", off, { escapeHatch: true, connections: 0, types: 0 })).toBe(true) + expect(events.at(-1)).toEqual( + expect.objectContaining({ enabled: false, reason: "escape-hatch", shadowed_types: 0, escape_hatch: true }), + ) + expect(trackPrecedence("s1", off, { escapeHatch: false, connections: 0, types: 0 })).toBe(true) // the hatch toggled + // The same id under another account is another workspace: reported again. + expect(trackPrecedence("s3", on, { escapeHatch: false, connections: 1, types: 2, identity: "acme|7" })).toBe(true) + expect(trackPrecedence("s3", on, { escapeHatch: false, connections: 1, types: 2, identity: "acme|7" })).toBe(false) + expect(trackPrecedence("s3", on, { escapeHatch: false, connections: 1, types: 2, identity: "other|7" })).toBe(true) + expect((events.at(-1) as any).escape_hatch).toBe(false) + }) + + test("errors are masked and bounded; a refusal names the operation and workspace", () => { + trackEngineError("attach", "ENOENT: /home/someone/.altimate/engine " + "x".repeat(500)) + const err = events.at(-1) + expect(err.type).toBe("workspace_engine_error") + expect(err.stage).toBe("attach") + expect(err.message.length).toBeLessThanOrEqual(200) + expect(err.message).not.toContain("/home/someone") + trackEngineError("precedence", "derive boom", "ses_p") + expect(events.at(-1)).toEqual(expect.objectContaining({ stage: "precedence", session_id: "ses_p" })) + trackRedirect("datamate", "status", 9) + expect(events.at(-1)).toEqual( + expect.objectContaining({ + type: "workspace_engine_redirect", + tool: "datamate", + operation: "status", + outcome: "refused", + workspace_id: 9, + }), + ) + }) +}) diff --git a/packages/opencode/test/altimate/workspace/precedence.test.ts b/packages/opencode/test/altimate/workspace/precedence.test.ts index 680b916965..93060cc717 100644 --- a/packages/opencode/test/altimate/workspace/precedence.test.ts +++ b/packages/opencode/test/altimate/workspace/precedence.test.ts @@ -4,7 +4,7 @@ // bound workspace's engine has attached. The binding and the engine-attribution read // both go through `precedenceInternals`, so these exercise the decision logic without // booting an instance, reading config, or touching MCP state. -import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test" import { INTEGRATION_TYPE, MAX_TRACKED_SESSIONS, @@ -26,6 +26,7 @@ import { servedInventory, } from "../../../src/altimate/workspace/precedence" import * as Registry from "../../../src/altimate/native/connections/registry" +import { Telemetry } from "../../../src/altimate/telemetry" // altimate_change - shared with awareness.test.ts; see precedence-fixture.ts import { BIGQUERY_TOOLS, SNOWFLAKE_TOOLS, WAREHOUSE_CONFIGS, bindTo } from "./precedence-fixture" import { canonicalType } from "../../../src/altimate/native/connections/registry" @@ -1403,3 +1404,32 @@ describe("servedInventory — what the model will be told is routed", () => { }) }) // altimate_change end + +describe("telemetry — the precedence snapshot", () => { + test("a snapshot is reported when it changes for the session, not on every refresh", async () => { + const rows: Array> = [] + const spy = spyOn(Telemetry, "track").mockImplementation((e) => { + rows.push(e as Record) + }) + try { + await refresh(SESSION, SNOWFLAKE_TOOLS) + await refresh(SESSION, SNOWFLAKE_TOOLS) + const snapshots = () => rows.filter((r) => r.type === "workspace_engine_precedence") + expect(snapshots()).toEqual([ + expect.objectContaining({ enabled: true, shadowed_types: 1, escape_hatch: false, workspace_id: 42 }), + ]) + // One local snowflake connection in the fixture registry is shadowed; the four served + // capabilities are not what is counted. + expect(snapshots()[0].shadowed_connections).toBe(1) + await refresh(SESSION, {}) + expect(snapshots().at(-1)).toEqual( + expect.objectContaining({ enabled: false, reason: "nothing-materialised", shadowed_types: 0 }), + ) + await refresh("ses_other", SNOWFLAKE_TOOLS) + expect(snapshots()).toHaveLength(3) + expect(snapshots()[2].session_id).toBe("ses_other") + } finally { + spy.mockRestore() + } + }) +})