From 9a5af9efa499d05b3f54f3b99f183f07e37da235 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Wed, 11 Nov 2026 12:19:50 +1100 Subject: [PATCH 1/7] feat(e2e): add isolated Claude Migrator adapter Run the trusted model CLI with a fresh API-key-only configuration and a single MCP exec bridge into the existing isolation broker. Validate runtime tools and permission mode, enforce turn limits and record the resolved model without interpreting model text as commands. Refs #1537 --- .../__tests__/claude-adapter.test.ts | 184 ++++++++++++ .../__tests__/migrator.test.ts | 31 ++ e2e/migration-harness/run/claude-adapter.md | 68 +++++ e2e/migration-harness/run/claude-adapter.mjs | 275 ++++++++++++++++++ e2e/migration-harness/run/claude-mcp.mjs | 119 ++++++++ e2e/migration-harness/run/claude-wire.mjs | 44 +++ e2e/migration-harness/run/migrator.mjs | 42 +++ e2e/migration-harness/types.d.ts | 7 + 8 files changed, 770 insertions(+) create mode 100644 e2e/migration-harness/__tests__/claude-adapter.test.ts create mode 100644 e2e/migration-harness/run/claude-adapter.md create mode 100644 e2e/migration-harness/run/claude-adapter.mjs create mode 100644 e2e/migration-harness/run/claude-mcp.mjs create mode 100644 e2e/migration-harness/run/claude-wire.mjs diff --git a/e2e/migration-harness/__tests__/claude-adapter.test.ts b/e2e/migration-harness/__tests__/claude-adapter.test.ts new file mode 100644 index 000000000..0c0dfb5c1 --- /dev/null +++ b/e2e/migration-harness/__tests__/claude-adapter.test.ts @@ -0,0 +1,184 @@ +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, expect, test, vi } from 'vitest' +import { runMigrator } from '../run/migrator.mjs' + +const directories: string[] = [] +afterEach(async () => { + vi.unstubAllEnvs() + for (const directory of directories.splice(0)) + await rm(directory, { recursive: true, force: true }) +}) + +async function fixture(mode = 'success') { + const directory = await mkdtemp(join(tmpdir(), 'claude-adapter-test-')) + directories.push(directory) + const executable = join(directory, 'claude') + const report = join(directory, 'launch.json') + // This fake speaks the real CLI stream and launches the production MCP child. + // No model API, Docker, or application command is run in the host process. + await writeFile( + executable, + `#!${process.execPath} +import { spawn } from 'node:child_process' +import { readFile, writeFile } from 'node:fs/promises' +import { createInterface } from 'node:readline' +const mode = ${JSON.stringify(mode)} +if (process.argv.includes('--version') && mode === 'version-failure') { console.error(process.env.ANTHROPIC_API_KEY); process.exit(1) } +if (process.argv.includes('--version')) { console.log('2.1.300 (Claude Code)'); process.exit(0) } +const args = process.argv.slice(2) +const config = JSON.parse(await readFile(args[args.indexOf('--mcp-config') + 1], 'utf8')) +const entry = config.mcpServers.migration +await writeFile(${JSON.stringify(report)}, JSON.stringify({args,env:process.env,cwd:process.cwd(),config})) +const emit = (value) => console.log(JSON.stringify(value)) +emit({ type:'system',subtype:'init',model:'claude-sonnet-resolved',permissionMode:mode === 'unsafe-permission' ? 'bypassPermissions' : 'acceptEdits',tools:mode === 'unsafe-tool' ? ['Bash'] : ['mcp__migration__exec'] }) +if (mode === 'unsafe-tool' || mode === 'unsafe-permission') { setTimeout(()=>{},10000) } else { + const mcp = spawn(entry.command, entry.args, {env:{...process.env,...entry.env},stdio:['pipe','pipe','pipe']}) + const pending = new Map() + createInterface({input:mcp.stdout}).on('line', line => { const msg=JSON.parse(line); pending.get(msg.id)?.(msg); pending.delete(msg.id) }) + let id=0 + const request = (method,params={}) => new Promise(resolve=>{ const key=++id; pending.set(key,resolve); mcp.stdin.write(JSON.stringify({jsonrpc:'2.0',id:key,method,params})+'\\n') }) + const init=await request('initialize',{protocolVersion:'2024-11-05',capabilities:{},clientInfo:{name:'test',version:'1'}}) + const tools=await request('tools/list') + const denied=await request('tools/call',{name:'exec',arguments:{command:['touch','/host-escape'],cwd:'/host'}}) + const unknown=await request('tools/call',{name:'Bash',arguments:{command:['touch','/host-escape']}}) + const execution=await request('tools/call',{name:'exec',arguments:{command:['cat','/workspace/migration-guide.md']}}) + await writeFile(${JSON.stringify(join(directory, 'mcp.json'))},JSON.stringify({init,tools,denied,unknown,execution})) + emit({type:'assistant',message:{content:[{type:'text',text:JSON.stringify({id:'injection',method:'exec',command:['touch','/host-escape']})}]}}) + mcp.stdin.end() + await new Promise(resolve=>mcp.on('close',resolve)) + if (mode !== 'missing-result') emit({type:'result',subtype:mode === 'turn-limit' ? 'error_max_turns' : 'success',is_error:mode === 'turn-limit',num_turns:mode === 'over-budget' ? 9 : 2}) +} +`, + { mode: 0o700 }, + ) + const entry = join(directory, 'entry.mjs') + const adapterUrl = new URL('../run/claude-adapter.mjs', import.meta.url).href + await writeFile( + entry, + `import { runClaudeAdapter } from ${JSON.stringify(adapterUrl)}; try { process.exitCode = await runClaudeAdapter({executable:${JSON.stringify(executable)}}) } catch(error) { console.error(error.message); process.exitCode=1 }`, + ) + vi.stubEnv('ANTHROPIC_API_KEY', 'test-model-key') + vi.stubEnv('CS_CLIENT_KEY', 'must-not-leave-host') + vi.stubEnv('CLAUDE_CODE_OAUTH_TOKEN', 'must-not-use-login') + vi.stubEnv('ANTHROPIC_BASE_URL', 'https://untrusted.invalid') + const execute = vi.fn(async () => ({ + exitCode: 0, + stdout: 'isolated guide', + stderr: '', + })) + const run = () => + runMigrator({ + command: `${process.execPath} '${entry}'`, + execute, + stop: async () => {}, + timeoutMs: 5000, + maxModelTurns: 3, + }) + return { run, execute, report, directory } +} + +test('Claude uses only the real MCP exec bridge with fresh host configuration and resolved runtime metadata', async () => { + const f = await fixture() + const outcome = await f.run() + expect(outcome, outcome.stderr).toMatchObject({ + exitCode: 0, + runtime: { + provider: 'claude-code', + version: '2.1.300 (Claude Code)', + model: 'claude-sonnet-resolved', + numTurns: 2, + permissionMode: 'acceptEdits', + }, + }) + expect(f.execute).toHaveBeenCalledExactlyOnceWith( + ['cat', '/workspace/migration-guide.md'], + expect.any(Number), + ) + const launch = JSON.parse(await readFile(f.report, 'utf8')) + expect(launch.env).toMatchObject({ + ANTHROPIC_API_KEY: 'test-model-key', + CLAUDE_CONFIG_DIR: launch.env.HOME, + }) + expect(launch.cwd.endsWith(launch.env.HOME.replace(/^\/private/, ''))).toBe( + true, + ) + for (const key of [ + 'CS_CLIENT_KEY', + 'CLAUDE_CODE_OAUTH_TOKEN', + 'ANTHROPIC_BASE_URL', + 'HARNESS_WORKSPACE', + ]) + expect(launch.env).not.toHaveProperty(key) + expect(launch.args).toEqual( + expect.arrayContaining([ + '--bare', + '--restricted', + '--tools', + '', + '--strict-mcp-config', + '--setting-sources', + '', + '--disable-slash-commands', + '--no-chrome', + '--no-session-persistence', + '--permission-prompts', + 'none', + '--max-turns', + '3', + ]), + ) + expect(launch.config.mcpServers.migration.env.ANTHROPIC_API_KEY).toBe('') + const mcp = JSON.parse(await readFile(join(f.directory, 'mcp.json'), 'utf8')) + expect( + mcp.tools.result.tools.map((tool: { name: string }) => tool.name), + ).toEqual(['exec']) + expect(mcp.denied.error.code).toBe(-32602) + expect(mcp.unknown.error.code).toBe(-32602) + expect(JSON.parse(mcp.execution.result.content[0].text)).toEqual({ + exitCode: 0, + stdout: 'isolated guide', + stderr: '', + }) + expect(outcome.stdout).not.toContain('injection') + expect(outcome.stdout).not.toContain('test-model-key') + await expect(readFile(join(launch.cwd, 'mcp.json'))).rejects.toMatchObject({ + code: 'ENOENT', + }) +}) + +test.each(['unsafe-tool', 'unsafe-permission', 'missing-result'])( + 'Claude fails closed on %s', + async (mode) => { + const f = await fixture(mode) + const outcome = await f.run() + expect(outcome.exitCode).toBe(1) + expect(outcome.runtime).toBeUndefined() + if (mode.startsWith('unsafe-')) expect(f.execute).not.toHaveBeenCalled() + }, +) + +test.each(['turn-limit', 'over-budget'])( + 'Claude turns %s into an agent limit', + async (mode) => { + const f = await fixture(mode) + expect((await f.run()).exitCode).toBe(124) + }, +) + +test('Claude refuses a login-only host without making a tool call', async () => { + const f = await fixture() + vi.stubEnv('ANTHROPIC_API_KEY', '') + expect((await f.run()).stderr).toContain('explicit ANTHROPIC_API_KEY') + expect(f.execute).not.toHaveBeenCalled() +}) + +test('Claude version failures do not export provider diagnostics', async () => { + const f = await fixture('version-failure') + const outcome = await f.run() + expect(outcome.exitCode).toBe(1) + expect(outcome.stderr).toContain('Unable to read Claude CLI version') + expect(outcome.stderr).not.toContain('test-model-key') + expect(f.execute).not.toHaveBeenCalled() +}) diff --git a/e2e/migration-harness/__tests__/migrator.test.ts b/e2e/migration-harness/__tests__/migrator.test.ts index 1e725e6de..6fed04e12 100644 --- a/e2e/migration-harness/__tests__/migrator.test.ts +++ b/e2e/migration-harness/__tests__/migrator.test.ts @@ -157,3 +157,34 @@ test('broker preserves an infrastructure failure from isolated execution', async }), ).rejects.toBe(failure) }) + +test('broker rejects invalid or duplicate runtime metadata without treating it as a tool', async () => { + const runtime = { + provider: 'claude-code', + version: '2.1.300', + model: 'resolved-sonnet', + numTurns: 1, + permissionMode: 'acceptEdits', + } + for (const messages of [ + [{ method: 'runtime', runtime: { ...runtime, numTurns: -1 } }], + [{ method: 'runtime', runtime: { ...runtime, apiKey: 'not-exportable' } }], + [ + { method: 'runtime', runtime }, + { method: 'runtime', runtime }, + ], + ]) { + const execute = vi.fn(async () => result) + await expect( + runMigrator({ + command: await adapter( + `process.stdout.write(${JSON.stringify(`${messages.map((message) => JSON.stringify(message)).join('\n')}\n`)});setTimeout(()=>{},10000)`, + ), + execute, + stop: async () => {}, + timeoutMs: 1000, + }), + ).rejects.toThrow('invalid or duplicate runtime metadata') + expect(execute).not.toHaveBeenCalled() + } +}) diff --git a/e2e/migration-harness/run/claude-adapter.md b/e2e/migration-harness/run/claude-adapter.md new file mode 100644 index 000000000..d07fe91b4 --- /dev/null +++ b/e2e/migration-harness/run/claude-adapter.md @@ -0,0 +1,68 @@ +# Claude Migrator adapter + +The trusted host runs Claude Code. The model gets one MCP tool, `exec`, which +forwards argument arrays to the harness's existing isolation broker. Commands, +file reads and file writes therefore execute in the Migrator Environment, with +the same filesystem and network policy as Fake Migrators. + +## Run + +Install a recent Claude Code CLI supporting `--bare`, `--restricted` and +`--permission-prompts none` on the trusted host. Supply `ANTHROPIC_API_KEY` through +the host's secret environment. The adapter deliberately does not use a Claude +login session, OAuth token, keychain, provider overrides, or an API-key helper. +CipherStash and store credentials remain owned by the harness. + +Pass an **absolute** adapter path because the broker starts it in an empty +working directory: + +```sh +pnpm --filter @cipherstash/e2e harness:run \ + --migrator 'node /absolute/path/to/stack/e2e/migration-harness/run/claude-adapter.mjs' \ + [other harness options] +``` + +The same adapter works with each supported migration procedure. It follows the +Run's staged guide at `/workspace/migration-guide.md`, upgrades the Fixture App, +and writes `/workspace/friction.json`. It receives no previous Run Reports, +Friction Logs, Oracle results, or Reference Migration. + +## Configuration and evidence + +The default model is `sonnet`, with `acceptEdits` permission mode, a 200-turn +budget and a 40-minute outer deadline. The broker passes the configured turn +budget to the adapter; Claude's documented `--max-turns` enforces it during print +mode, even though some CLI versions omit the flag from `--help`. The broker +separately enforces tool-call and wall-clock limits. A Claude turn-limit result +returns exit code 124. See the [Claude CLI reference](https://code.claude.com/docs/en/cli-reference). + +Each invocation uses an empty temporary working/configuration directory and an +allowlisted environment. Built-in tools, implicit MCP servers, settings sources, +slash commands, hooks, automatic memory, browser access and persistent sessions +are disabled. The adapter checks Claude's initialization record for the expected +tool set and permission mode, then records the CLI version, resolved model +identifier and final turn count in `manifest.agents.runtime`. Model text never +becomes a broker request. CLI stderr is consumed but not exported, because it can +contain provider details. The broker transcript records the actual isolated +command requests and runtime metadata; the broker also returns tool-call exit codes. + +The MCP bridge connects through a private Unix socket and a per-run token. It +accepts only `exec` with a string argument array; the model cannot select a host +working directory, environment, mount or policy. The model API key is supplied +only to the host CLI and explicitly removed from the bridge's environment. The +adapter and CLI are trusted host code, not an additional security boundary +against a compromised host executable. Temporary configuration and socket files +are removed after execution. + +## Verification + +`harness:selftest` runs a fake CLI through the production MCP bridge and broker. +It checks isolated command dispatch, rejected execution-context overrides, +configuration/credential separation, resolved runtime metadata, malformed runtime +completion, unexpected tool/permission settings, and turn limits. These tests +make no model calls and need no Docker. They do need permission to create a local +Unix socket. + +A real model Run for each procedure still belongs in credentialed CI. Fake CLI +coverage does not prove that an installed Claude version can complete a +migration or that the guide is sufficient for a real model. diff --git a/e2e/migration-harness/run/claude-adapter.mjs b/e2e/migration-harness/run/claude-adapter.mjs new file mode 100644 index 000000000..591fbaea3 --- /dev/null +++ b/e2e/migration-harness/run/claude-adapter.mjs @@ -0,0 +1,275 @@ +import { execFile, spawn } from 'node:child_process' +import { randomBytes } from 'node:crypto' +import { chmod, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { createServer } from 'node:net' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' +import { promisify } from 'node:util' +import { readMessages, validCommand } from './claude-wire.mjs' + +/** Run the trusted host CLI; its MCP bridge is the only application tool. + * Options are trusted operator configuration, never model-provided. + * @param {{executable?:string, model?:string, input?:import('node:stream').Readable, output?:import('node:stream').Writable}} options + */ +export async function runClaudeAdapter({ + executable = 'claude', + model = 'sonnet', + input = process.stdin, + output = process.stdout, +} = {}) { + if (process.env.HARNESS_BROKER_PROTOCOL !== 'json-lines-exec-v1') + throw Error('Claude adapter requires the harness broker') + if (!process.env.ANTHROPIC_API_KEY) + throw Error( + 'Claude adapter requires an explicit ANTHROPIC_API_KEY; login sessions are not used', + ) + const maxTurns = Number(process.env.HARNESS_MAX_MODEL_TURNS) + if (!Number.isSafeInteger(maxTurns) || maxTurns < 1) + throw Error('Invalid model-turn budget') + const directory = await mkdtemp(join(tmpdir(), 'mh-claude-')) + const socketPath = join(directory, 'exec.sock') + const token = randomBytes(32).toString('hex') + // In particular, do not inherit NODE_OPTIONS, CLAUDE_*, cloud auth, CS_*, or + // ANTHROPIC_BASE_URL. The only model provider is the public Anthropic API. + const env = { + PATH: process.env.PATH, + HOME: directory, + TMPDIR: directory, + CLAUDE_CONFIG_DIR: directory, + ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY, + } + /** @type {Map} */ + const pending = new Map() + /** @type {Set} */ + const sockets = new Set() + let counter = 0 + /** @type {import('node:child_process').ChildProcess | undefined} */ + let child + /** @type {Error|undefined} */ + let failure + /** @param {Error} error */ + function fail(error) { + failure ??= error + child?.kill('SIGKILL') + for (const socket of sockets) socket.destroy() + } + /** @param {unknown} message */ + function send(message) { + output.write(`${JSON.stringify(message)}\n`) + } + const server = createServer((socket) => { + sockets.add(socket) + socket.on('close', () => { + sockets.delete(socket) + if ([...pending.values()].some((item) => item.socket === socket)) + fail(Error('MCP bridge disconnected during a tool call')) + }) + readMessages( + socket, + (message) => { + if ( + message?.token !== token || + !validCommand(message.command) || + !['string', 'number'].includes(typeof message.id) || + Object.keys(message).some( + (key) => !['token', 'id', 'command'].includes(key), + ) + ) { + fail(Error('Invalid MCP bridge request')) + return + } + const id = `claude-${++counter}` + pending.set(id, { socket, id: message.id }) + send({ id, method: 'exec', command: message.command }) + }, + fail, + ) + }) + try { + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(socketPath, () => resolve(undefined)) + }) + await chmod(socketPath, 0o600) + const config = join(directory, 'mcp.json') + await writeFile( + config, + JSON.stringify({ + mcpServers: { + migration: { + command: process.execPath, + args: [fileURLToPath(new URL('./claude-mcp.mjs', import.meta.url))], + env: { + MIGRATION_BRIDGE_SOCKET: socketPath, + MIGRATION_BRIDGE_TOKEN: token, + ANTHROPIC_API_KEY: '', + }, + }, + }, + }), + { mode: 0o600 }, + ) + const { stdout: versionOutput } = await promisify(execFile)( + executable, + ['--version'], + { cwd: directory, env, timeout: 10000, maxBuffer: 4096 }, + ).catch(() => { + throw Error('Unable to read Claude CLI version') + }) + const version = versionOutput.trim() + if (!version || version.length > 256) + throw Error('Invalid Claude CLI version') + readMessages( + input, + (message) => { + const request = pending.get(message?.id) + if ( + !request || + !message.result || + !Number.isInteger(message.result.exitCode) || + typeof message.result.stdout !== 'string' || + typeof message.result.stderr !== 'string' + ) { + fail(Error('Invalid harness broker response')) + return + } + pending.delete(message.id) + request.socket.write( + `${JSON.stringify({ id: request.id, result: message.result })}\n`, + ) + }, + fail, + ) + const prompt = + 'Upgrade the Fixture App in /workspace using only /workspace/migration-guide.md. Read that guide through the exec tool, follow its applicable migration procedure, and verify the migrated application. All commands and file access must use the exec tool in the isolated Migrator Environment. Record guide friction in /workspace/friction.json as a JSON array of objects with section, issue, actionable (boolean), description, and workaround. Write [] when there is no friction. Do not invent prior-run context.' + const args = [ + '--bare', + '--restricted', + '--tools', + '', + '--strict-mcp-config', + '--mcp-config', + config, + '--setting-sources', + '', + '--disable-slash-commands', + '--no-chrome', + '--no-session-persistence', + '--permission-mode', + 'acceptEdits', + '--allowedTools', + 'mcp__migration__exec', + '--permission-prompts', + 'none', + '--model', + model, + '--max-turns', + String(maxTurns), + '--output-format', + 'stream-json', + '--verbose', + '-p', + prompt, + ] + child = spawn(executable, args, { + cwd: directory, + env, + stdio: ['ignore', 'pipe', 'pipe'], + }) + /** @type {string|undefined} */ + let resolvedModel + /** @type {{num_turns:number,subtype:string,is_error:boolean}|undefined} */ + let result + let diagnosticBytes = 0 + child.stderr?.on('data', (data) => { + // CLI diagnostics can contain environment/provider details; do not export them. + diagnosticBytes += data.length + if (diagnosticBytes > 1024 * 1024) + fail(Error('Claude diagnostic output limit exceeded')) + }) + if (!child.stdout) throw Error('Missing Claude output channel') + readMessages( + child.stdout, + (message) => { + if (message?.type === 'system' && message.subtype === 'init') { + if ( + resolvedModel || + message.permissionMode !== 'acceptEdits' || + typeof message.model !== 'string' || + !message.model || + message.model.length > 256 || + !Array.isArray(message.tools) || + message.tools.some( + (/** @type {unknown} */ tool) => tool !== 'mcp__migration__exec', + ) || + !message.tools.includes('mcp__migration__exec') + ) { + fail(Error('Claude initialized with unexpected model or tools')) + return + } + resolvedModel = message.model + } else if (message?.type === 'result') { + if ( + result || + !Number.isSafeInteger(message.num_turns) || + message.num_turns < 0 || + typeof message.subtype !== 'string' || + typeof message.is_error !== 'boolean' + ) { + fail(Error('Invalid Claude result')) + return + } + result = message + } + // Assistant text, tool arguments and tool output are never parsed as broker requests. + }, + fail, + ) + const code = await new Promise((resolve, reject) => { + child?.once('error', reject) + child?.once('close', resolve) + }) + if (failure) throw failure + if (pending.size || !resolvedModel || !result) + throw Error('Claude exited without a complete runtime result') + send({ + method: 'runtime', + runtime: { + provider: 'claude-code', + version, + model: resolvedModel, + numTurns: result.num_turns, + permissionMode: 'acceptEdits', + }, + }) + if (result.num_turns > maxTurns || result.subtype === 'error_max_turns') + return 124 + return code === 0 && !result.is_error && result.subtype === 'success' + ? 0 + : 1 + } finally { + input.pause() + child?.kill('SIGKILL') + for (const socket of sockets) socket.destroy() + await new Promise((resolve) => server.close(resolve)) + await rm(directory, { recursive: true, force: true }) + } +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + runClaudeAdapter().then( + (code) => { + process.exitCode = code + }, + (error) => { + process.stderr.write( + `${error instanceof Error ? error.message : 'Claude adapter failed'}\n`, + ) + process.exitCode = 1 + }, + ) +} diff --git a/e2e/migration-harness/run/claude-mcp.mjs b/e2e/migration-harness/run/claude-mcp.mjs new file mode 100644 index 000000000..7e755cf4f --- /dev/null +++ b/e2e/migration-harness/run/claude-mcp.mjs @@ -0,0 +1,119 @@ +import { connect } from 'node:net' +import { readMessages, validCommand } from './claude-wire.mjs' + +// This trusted stdio MCP server never spawns commands. Its only capability is +// forwarding argument arrays to the host adapter's fixed isolation broker. +const token = process.env.MIGRATION_BRIDGE_TOKEN +const path = process.env.MIGRATION_BRIDGE_SOCKET +if (!token || !path) throw Error('Missing migration bridge configuration') +const socket = connect(path) +/** @type {Set} */ +const pending = new Set() +/** @param {unknown} message */ +function send(message) { + process.stdout.write(`${JSON.stringify(message)}\n`) +} +/** @param {string|number|null} id @param {number} code @param {string} message */ +function error(id, code, message) { + send({ jsonrpc: '2.0', id, error: { code, message } }) +} +function fail() { + socket.destroy() + process.exitCode = 1 + process.stdin.destroy() +} +readMessages( + socket, + (message) => { + if (!pending.delete(message.id) || !message.result) { + fail() + return + } + send({ + jsonrpc: '2.0', + id: message.id, + result: { + content: [{ type: 'text', text: JSON.stringify(message.result) }], + isError: message.result.exitCode !== 0, + }, + }) + }, + fail, +) +socket.on('close', () => { + if (pending.size) fail() + process.stdin.destroy() +}) +readMessages( + process.stdin, + (message) => { + const id = message?.id + if (message?.jsonrpc !== '2.0') { + error(null, -32600, 'Invalid request') + return + } + if (id === undefined && message.method?.startsWith('notifications/')) return + if ((typeof id !== 'string' && typeof id !== 'number') || pending.has(id)) { + error(null, -32600, 'Invalid request ID') + return + } + if (message.method === 'initialize') { + send({ + jsonrpc: '2.0', + id, + result: { + protocolVersion: '2024-11-05', + capabilities: { tools: {} }, + serverInfo: { name: 'migration-exec', version: '1.0.0' }, + }, + }) + } else if (message.method === 'ping') { + send({ jsonrpc: '2.0', id, result: {} }) + } else if (message.method === 'tools/list') { + send({ + jsonrpc: '2.0', + id, + result: { + tools: [ + { + name: 'exec', + description: + 'Run an argument array in the isolated Migrator Environment at /workspace. Use this for every file operation and command. No host paths or environment overrides are accepted.', + inputSchema: { + type: 'object', + properties: { + command: { + type: 'array', + items: { type: 'string' }, + minItems: 1, + }, + }, + required: ['command'], + additionalProperties: false, + }, + }, + ], + }, + }) + } else if (message.method === 'tools/call') { + const args = message.params?.arguments + if ( + message.params?.name !== 'exec' || + !args || + !validCommand(args.command) || + Object.keys(args).some((key) => key !== 'command') + ) { + error( + id, + -32602, + 'Only exec with a command argument array is supported', + ) + return + } + pending.add(id) + socket.write(`${JSON.stringify({ token, id, command: args.command })}\n`) + } else error(id, -32601, 'Method not found') + }, + fail, +) +process.stdin.on('end', () => socket.end()) diff --git a/e2e/migration-harness/run/claude-wire.mjs b/e2e/migration-harness/run/claude-wire.mjs new file mode 100644 index 000000000..afbec4973 --- /dev/null +++ b/e2e/migration-harness/run/claude-wire.mjs @@ -0,0 +1,44 @@ +/** Bounded newline-delimited JSON framing shared by the trusted adapter and MCP bridge. + * @param {import('node:stream').Readable} stream + * @param {(message:any)=>void} receive + * @param {(error:Error)=>void} fail + */ +export function readMessages(stream, receive, fail) { + let buffer = '' + let total = 0 + stream.setEncoding('utf8') + stream.on('data', (chunk) => { + total += chunk.length + buffer += chunk + if (buffer.length > 1024 * 1024 || total > 16 * 1024 * 1024) { + fail(Error('Protocol output limit exceeded')) + return + } + for (;;) { + const end = buffer.indexOf('\n') + if (end < 0) break + const line = buffer.slice(0, end) + buffer = buffer.slice(end + 1) + try { + receive(JSON.parse(line)) + } catch { + fail(Error('Invalid protocol message')) + return + } + } + }) + stream.on('end', () => { + if (buffer.trim()) fail(Error('Unfinished protocol message')) + }) + stream.on('error', fail) +} + +/** @param {unknown} command @returns {command is string[]} */ +export function validCommand(command) { + return ( + Array.isArray(command) && + command.length > 0 && + Boolean(command[0]) && + command.every((arg) => typeof arg === 'string' && !arg.includes('\0')) + ) +} diff --git a/e2e/migration-harness/run/migrator.mjs b/e2e/migration-harness/run/migrator.mjs index 6184a3adf..ee506e7ef 100644 --- a/e2e/migration-harness/run/migrator.mjs +++ b/e2e/migration-harness/run/migrator.mjs @@ -62,6 +62,8 @@ export async function runMigrator({ pending = 0 /** @type {string|undefined} */ let limit + /** @type {import('../types.js').MigratorResult['runtime']} */ + let runtime /** @type {Error|undefined} */ let failure let queue = Promise.resolve() @@ -131,6 +133,45 @@ export async function runMigrator({ invalid('invalid JSON') return } + if (request?.method === 'runtime') { + const value = request.runtime + if ( + runtime || + !value || + value.provider !== 'claude-code' || + value.permissionMode !== 'acceptEdits' || + !Number.isSafeInteger(value.numTurns) || + value.numTurns < 0 || + ![value.model, value.version].every( + (item) => + typeof item === 'string' && + item.length > 0 && + item.length <= 256, + ) || + Object.keys(request).some( + (key) => !['method', 'runtime'].includes(key), + ) || + Object.keys(value).some( + (key) => + ![ + 'provider', + 'permissionMode', + 'numTurns', + 'model', + 'version', + ].includes(key), + ) + ) { + invalid('invalid or duplicate runtime metadata') + return + } + runtime = value + if (value.numTurns > maxModelTurns) { + limit = 'model turn limit reached' + terminate() + } + continue + } if ( request?.method !== 'exec' || typeof request.id !== 'string' || @@ -194,6 +235,7 @@ export async function runMigrator({ exitCode: limit ? 124 : (code ?? 1), limit, toolCalls: [...toolCalls], + ...(runtime ? { runtime } : {}), stdout, stderr, }) diff --git a/e2e/migration-harness/types.d.ts b/e2e/migration-harness/types.d.ts index 3166fff08..5268bde1a 100644 --- a/e2e/migration-harness/types.d.ts +++ b/e2e/migration-harness/types.d.ts @@ -21,6 +21,13 @@ export interface OracleCheck { cause?: string } export interface MigratorResult { + runtime?: { + provider: 'claude-code' + version: string + model: string + numTurns: number + permissionMode: 'acceptEdits' + } exitCode: number limit?: string toolCalls?: Array<{ id: string; exitCode: number }> From fab51cff7f578023b651cb0f7c78b6b60a6e6852 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Wed, 11 Nov 2026 13:59:32 +1100 Subject: [PATCH 2/7] docs(e2e): keep real agent runs outside CI Align adapter instructions with the accepted isolation ADR and the pending remote-host selection. Deterministic CI coverage does not establish guide sufficiency. Refs #1537. --- e2e/migration-harness/run/claude-adapter.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/e2e/migration-harness/run/claude-adapter.md b/e2e/migration-harness/run/claude-adapter.md index d07fe91b4..a60961dc3 100644 --- a/e2e/migration-harness/run/claude-adapter.md +++ b/e2e/migration-harness/run/claude-adapter.md @@ -63,6 +63,8 @@ completion, unexpected tool/permission settings, and turn limits. These tests make no model calls and need no Docker. They do need permission to create a local Unix socket. -A real model Run for each procedure still belongs in credentialed CI. Fake CLI -coverage does not prove that an installed Claude version can complete a -migration or that the guide is sufficient for a real model. +A real model Run for each procedure remains pending on a disposable remote host +selected by the maintainer. Real agents remain outside CI under the ADR; CI runs +deterministic Fake Migrators. Fake CLI coverage does not prove that an installed +Claude version can complete a migration or that the guide is sufficient for a +real model. From bbe2262b1c9af66c8cd0c3dbd0c81639fb01146d Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Wed, 11 Nov 2026 14:04:12 +1100 Subject: [PATCH 3/7] docs(e2e): record acceptance of the migration harness design The design PR is merged, but proposed status obscures the accepted contract. Record acceptance separately from the remaining real-agent and automation evidence. Refs #1540. --- .../docs/adr/0001-isolated-agent-migration-harness.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/e2e/migration-harness/docs/adr/0001-isolated-agent-migration-harness.md b/e2e/migration-harness/docs/adr/0001-isolated-agent-migration-harness.md index 0f73d8d35..3d35ea329 100644 --- a/e2e/migration-harness/docs/adr/0001-isolated-agent-migration-harness.md +++ b/e2e/migration-harness/docs/adr/0001-isolated-agent-migration-harness.md @@ -1,5 +1,5 @@ --- -status: proposed +status: accepted date: 2026-11-09 --- @@ -11,7 +11,10 @@ Migration Harness: a Migrator upgrades a fresh Fixture App, a deterministic Orac checks it through a restricted App Host, and an Editor revises the guide from Iteration Results. This ADR is the authoritative architecture and implementation contract for [epic #1540](https://github.com/cipherstash/stack/issues/1540), -using the [context glossary](../../CONTEXT.md). Status remains proposed pending review. +using the [context glossary](../../CONTEXT.md). The design was accepted in +[PR #1575](https://github.com/cipherstash/stack/pull/1575), merged on 2026-11-11. +Acceptance of the design does not imply implementation completion: real Migrator +evidence, Editor/Iteration automation and Loop convergence remain outstanding. The full implementation contract remains in this ADR at the maintainer's direction, as an explicit exception to the usual short ADR format. From 60db09993048c9a675f5a83ea47fc430e385219b Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Wed, 11 Nov 2026 15:17:30 +1100 Subject: [PATCH 4/7] feat(e2e): let the Claude adapter authenticate with a subscription token The adapter accepted only ANTHROPIC_API_KEY, which bills an API account. A maintainer running real Runs on a Claude subscription can now supply CLAUDE_CODE_OAUTH_TOKEN from `claude setup-token` instead. The CLI's --bare mode never reads OAuth credentials, so that path runs without --bare; the restricted tool, settings, MCP and slash-command flags, the empty configuration directory and the initialization check still apply. An API key wins when both are set. The token is removed from the MCP bridge environment, and the isolation startup probe now fails if it reaches the Migrator Environment. No login session or keychain is read on either path. Refs #1537 --- .../__tests__/claude-adapter.test.ts | 39 ++++++++++++++++++- e2e/migration-harness/isolation/startup.mjs | 1 + e2e/migration-harness/run/claude-adapter.md | 17 ++++++-- e2e/migration-harness/run/claude-adapter.mjs | 19 +++++++-- 4 files changed, 67 insertions(+), 9 deletions(-) diff --git a/e2e/migration-harness/__tests__/claude-adapter.test.ts b/e2e/migration-harness/__tests__/claude-adapter.test.ts index 0c0dfb5c1..c1e3f1a8a 100644 --- a/e2e/migration-harness/__tests__/claude-adapter.test.ts +++ b/e2e/migration-harness/__tests__/claude-adapter.test.ts @@ -170,10 +170,47 @@ test.each(['turn-limit', 'over-budget'])( test('Claude refuses a login-only host without making a tool call', async () => { const f = await fixture() vi.stubEnv('ANTHROPIC_API_KEY', '') - expect((await f.run()).stderr).toContain('explicit ANTHROPIC_API_KEY') + vi.stubEnv('CLAUDE_CODE_OAUTH_TOKEN', '') + expect((await f.run()).stderr).toContain( + 'explicit ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN', + ) expect(f.execute).not.toHaveBeenCalled() }) +// --bare reads only ANTHROPIC_API_KEY or apiKeyHelper, never OAuth, so a +// subscription token from `claude setup-token` runs without --bare. Isolation +// then rests on the other flags, the empty config directory and the init check. +test('Claude can authenticate with an explicit subscription token', async () => { + const f = await fixture() + vi.stubEnv('ANTHROPIC_API_KEY', '') + vi.stubEnv('CLAUDE_CODE_OAUTH_TOKEN', 'test-subscription-token') + const outcome = await f.run() + expect(outcome, outcome.stderr).toMatchObject({ exitCode: 0 }) + const launch = JSON.parse(await readFile(f.report, 'utf8')) + expect(launch.env).toMatchObject({ + CLAUDE_CODE_OAUTH_TOKEN: 'test-subscription-token', + CLAUDE_CONFIG_DIR: launch.env.HOME, + }) + expect(launch.env).not.toHaveProperty('ANTHROPIC_API_KEY') + expect(launch.env).not.toHaveProperty('CS_CLIENT_KEY') + expect(launch.args).not.toContain('--bare') + expect(launch.args).toEqual( + expect.arrayContaining([ + '--restricted', + '--strict-mcp-config', + '--setting-sources', + '', + '--disable-slash-commands', + '--no-session-persistence', + ]), + ) + expect(launch.config.mcpServers.migration.env).toMatchObject({ + ANTHROPIC_API_KEY: '', + CLAUDE_CODE_OAUTH_TOKEN: '', + }) + expect(outcome.stdout).not.toContain('test-subscription-token') +}) + test('Claude version failures do not export provider diagnostics', async () => { const f = await fixture('version-failure') const outcome = await f.run() diff --git a/e2e/migration-harness/isolation/startup.mjs b/e2e/migration-harness/isolation/startup.mjs index d79a224a3..45d654b47 100644 --- a/e2e/migration-harness/isolation/startup.mjs +++ b/e2e/migration-harness/isolation/startup.mjs @@ -9,6 +9,7 @@ for (const name of [ 'HARNESS_CONFIG', 'OPENAI_API_KEY', 'ANTHROPIC_API_KEY', + 'CLAUDE_CODE_OAUTH_TOKEN', 'HARNESS_MODEL_CANARY', ]) { if (process.env[name]) diff --git a/e2e/migration-harness/run/claude-adapter.md b/e2e/migration-harness/run/claude-adapter.md index a60961dc3..67377a347 100644 --- a/e2e/migration-harness/run/claude-adapter.md +++ b/e2e/migration-harness/run/claude-adapter.md @@ -8,9 +8,18 @@ the same filesystem and network policy as Fake Migrators. ## Run Install a recent Claude Code CLI supporting `--bare`, `--restricted` and -`--permission-prompts none` on the trusted host. Supply `ANTHROPIC_API_KEY` through -the host's secret environment. The adapter deliberately does not use a Claude -login session, OAuth token, keychain, provider overrides, or an API-key helper. +`--permission-prompts none` on the trusted host. Supply one model credential +through the host's secret environment: + +- `ANTHROPIC_API_KEY`, billed to an API account. The CLI runs with `--bare`. +- `CLAUDE_CODE_OAUTH_TOKEN`, a long-lived Claude subscription token created + with `claude setup-token`. `--bare` never reads OAuth credentials, so the CLI + runs without it; the remaining flags, the empty configuration directory and + the initialization check below still apply. Usage counts against the + subscription's limits. + +When both are set the API key is used. The adapter deliberately does not use a +Claude login session, keychain, provider overrides, or an API-key helper. CipherStash and store credentials remain owned by the harness. Pass an **absolute** adapter path because the broker starts it in an empty @@ -48,7 +57,7 @@ command requests and runtime metadata; the broker also returns tool-call exit co The MCP bridge connects through a private Unix socket and a per-run token. It accepts only `exec` with a string argument array; the model cannot select a host -working directory, environment, mount or policy. The model API key is supplied +working directory, environment, mount or policy. The model credential is supplied only to the host CLI and explicitly removed from the bridge's environment. The adapter and CLI are trusted host code, not an additional security boundary against a compromised host executable. Temporary configuration and socket files diff --git a/e2e/migration-harness/run/claude-adapter.mjs b/e2e/migration-harness/run/claude-adapter.mjs index 591fbaea3..35d701ec7 100644 --- a/e2e/migration-harness/run/claude-adapter.mjs +++ b/e2e/migration-harness/run/claude-adapter.mjs @@ -20,9 +20,15 @@ export async function runClaudeAdapter({ } = {}) { if (process.env.HARNESS_BROKER_PROTOCOL !== 'json-lines-exec-v1') throw Error('Claude adapter requires the harness broker') - if (!process.env.ANTHROPIC_API_KEY) + // An API key wins when both are set. A subscription token comes from + // `claude setup-token`; neither path reads a login session or keychain. + const apiKey = process.env.ANTHROPIC_API_KEY + const subscriptionToken = apiKey + ? undefined + : process.env.CLAUDE_CODE_OAUTH_TOKEN + if (!apiKey && !subscriptionToken) throw Error( - 'Claude adapter requires an explicit ANTHROPIC_API_KEY; login sessions are not used', + 'Claude adapter requires an explicit ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN; login sessions are not used', ) const maxTurns = Number(process.env.HARNESS_MAX_MODEL_TURNS) if (!Number.isSafeInteger(maxTurns) || maxTurns < 1) @@ -37,7 +43,9 @@ export async function runClaudeAdapter({ HOME: directory, TMPDIR: directory, CLAUDE_CONFIG_DIR: directory, - ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY, + ...(apiKey + ? { ANTHROPIC_API_KEY: apiKey } + : { CLAUDE_CODE_OAUTH_TOKEN: subscriptionToken }), } /** @type {Map} */ const pending = new Map() @@ -104,6 +112,7 @@ export async function runClaudeAdapter({ MIGRATION_BRIDGE_SOCKET: socketPath, MIGRATION_BRIDGE_TOKEN: token, ANTHROPIC_API_KEY: '', + CLAUDE_CODE_OAUTH_TOKEN: '', }, }, }, @@ -143,8 +152,10 @@ export async function runClaudeAdapter({ ) const prompt = 'Upgrade the Fixture App in /workspace using only /workspace/migration-guide.md. Read that guide through the exec tool, follow its applicable migration procedure, and verify the migrated application. All commands and file access must use the exec tool in the isolated Migrator Environment. Record guide friction in /workspace/friction.json as a JSON array of objects with section, issue, actionable (boolean), description, and workaround. Write [] when there is no friction. Do not invent prior-run context.' + // --bare never reads OAuth, so a subscription token runs without it. The + // flags below, the empty config directory and the init check still apply. const args = [ - '--bare', + ...(apiKey ? ['--bare'] : []), '--restricted', '--tools', '', From 88e34ee276623d20e768c8ce72349d469cce0858 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Thu, 12 Nov 2026 10:02:56 +1100 Subject: [PATCH 5/7] fix(e2e): retain private broker response evidence Command-only transcripts cannot establish which supplementary instructions reached a Migrator. Preserve bounded command and response records privately, record incomplete evidence explicitly, and report conservative documentation exposure signals for human review. Refs #1537 --- .../__tests__/broker-evidence.test.ts | 141 ++++++++++++++++++ .../__tests__/review-driver.test.ts | 60 ++++++-- e2e/migration-harness/run/claude-adapter.md | 25 +++- e2e/migration-harness/run/migrator.mjs | 43 +++++- e2e/migration-harness/run/real-driver.mjs | 22 +++ e2e/migration-harness/run/run.mjs | 22 +++ e2e/migration-harness/types.d.ts | 20 ++- 7 files changed, 316 insertions(+), 17 deletions(-) create mode 100644 e2e/migration-harness/__tests__/broker-evidence.test.ts diff --git a/e2e/migration-harness/__tests__/broker-evidence.test.ts b/e2e/migration-harness/__tests__/broker-evidence.test.ts new file mode 100644 index 000000000..914fee8ba --- /dev/null +++ b/e2e/migration-harness/__tests__/broker-evidence.test.ts @@ -0,0 +1,141 @@ +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, expect, test } from 'vitest' +import { requiredOracleChecks } from '../run/migration-checks.mjs' +import { runMigrator } from '../run/migrator.mjs' +import { run } from '../run/run.mjs' +import type { Manifest } from '../types.js' + +const directories: string[] = [] +afterEach(async () => { + for (const directory of directories.splice(0)) + await rm(directory, { recursive: true, force: true }) +}) + +async function invoke( + command: string[], + response: { exitCode: number; stdout: string; stderr: string }, + maxEvidenceBytes?: number, +) { + const directory = await mkdtemp(join(tmpdir(), 'mh-evidence-test-')) + directories.push(directory) + const adapter = join(directory, 'adapter.mjs') + await writeFile( + adapter, + ` + console.log(JSON.stringify(${JSON.stringify({ id: 'request-1', method: 'exec', command })})); + process.stdin.once('data', () => process.exit(0)); + `, + ) + return runMigrator({ + command: `node '${adapter}'`, + execute: async () => response, + stop: async () => {}, + timeoutMs: 2000, + maxEvidenceBytes, + }) +} + +test('broker retains responses with the command and request identity for private evidence', async () => { + const command = ['npm', 'view', '@cipherstash/stack', 'readme'] + const response = { + exitCode: 0, + stdout: 'DOCUMENTATION-RESPONSE-CANARY', + stderr: 'DIAGNOSTIC-RESPONSE-CANARY', + } + const result = await invoke(command, response) + expect(result.exitCode).toBe(0) + expect(result).toMatchObject({ + brokerEvidence: [{ id: 'request-1', command, result: response }], + }) + // Raw command responses must not silently become inline Editor evidence. + expect(result.toolCalls).toEqual([{ id: 'request-1', exitCode: 0 }]) +}) + +test('explicit documentation retrieval is flagged for evidence review', async () => { + const result = await invoke(['npm', 'view', '@cipherstash/stack', 'readme'], { + exitCode: 0, + stdout: 'package instructions', + stderr: '', + }) + expect(result).toMatchObject({ + documentationExposure: { status: 'observed' }, + }) +}) + +test('an arbitrary shell command cannot establish the absence of supplementary documentation', async () => { + const result = await invoke(['sh', '-c', 'node inspect.mjs'], { + exitCode: 0, + stdout: 'application output', + stderr: '', + }) + expect(result).toMatchObject({ documentationExposure: { status: 'unknown' } }) +}) + +test('evidence retention has a byte budget and never labels incomplete records complete', async () => { + const result = await invoke( + ['true'], + { exitCode: 0, stdout: 'x'.repeat(100), stderr: '' }, + 64, + ) + expect(result).toMatchObject({ + brokerEvidence: [], + documentationExposure: { + status: 'unknown', + evidenceComplete: false, + reviewRequired: true, + }, + }) +}) + +test.each(['openat', 'stat'])( + 'documentation exposure distinguishes a successful %s from metadata inspection', + async (operation) => { + const manifest: Manifest = { + guideHash: 'guide', + documentationExposure: { + status: 'unknown', + reviewRequired: true, + evidenceComplete: true, + }, + } + let sent = false + const report = await run({ + id: 'docs', + manifest, + driver: { + provision: async () => {}, + installUnmigrated: async () => {}, + initializeLegacyData: async () => {}, + snapshot: async () => {}, + migrate: async () => ({ exitCode: 0 }), + friction: async () => [], + installAppHost: async () => {}, + behaviour: async () => + requiredOracleChecks.map((id) => ({ id, passed: true })), + }, + supervision: { + stop: async () => {}, + drain: () => { + if (sent) return [] + sent = true + return [ + { + actor: 'migrator', + stage: 'command', + operation, + resource: '/workspace/node_modules/example/README.md', + decision: 'allowed', + result: 'success', + }, + ] + }, + }, + }) + expect(report.manifest.documentationExposure?.status).toBe( + operation === 'openat' ? 'possible' : 'unknown', + ) + expect(report.manifest.documentationExposure?.reviewRequired).toBe(true) + }, +) diff --git a/e2e/migration-harness/__tests__/review-driver.test.ts b/e2e/migration-harness/__tests__/review-driver.test.ts index 31c735f19..be1b28714 100644 --- a/e2e/migration-harness/__tests__/review-driver.test.ts +++ b/e2e/migration-harness/__tests__/review-driver.test.ts @@ -1,5 +1,5 @@ import { execFile } from 'node:child_process' -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { fileURLToPath } from 'node:url' @@ -10,7 +10,7 @@ import { migrationChecks, noRewriteCheck } from '../run/migration-checks.mjs' import { createRealDriver } from '../run/real-driver.mjs' import { run } from '../run/run.mjs' import { dynamoSnapshotError } from '../run/snapshot-errors.mjs' -import type { IsolationOptions } from '../types.js' +import type { IsolationOptions, Manifest } from '../types.js' const observed = vi.hoisted(() => ({ dispose: vi.fn(async () => {}), @@ -97,6 +97,7 @@ async function setup( gitFailure?: 'add' | 'diff' stopFails?: boolean timeoutMs?: number + toolResponse?: { exitCode: number; stdout: string; stderr: string } } = {}, ) { const output = await mkdtemp(join(tmpdir(), 'mh-review-driver-')) @@ -124,7 +125,7 @@ async function setup( const result = await execute(file, args, { cwd: invocation.workspace }) return { exitCode: 0, stdout: result.stdout, stderr: result.stderr } } - return { exitCode: 0, stdout: '', stderr: '' } + return options.toolResponse ?? { exitCode: 0, stdout: '', stderr: '' } }, stop: async (stopOptions?: { expected?: boolean }) => { cancellations.push(stopOptions) @@ -148,6 +149,11 @@ async function setup( close: async () => ({ exitCode: 0, stdout: '', stderr: '' }), }), } + const manifest: Manifest & { stack: string } = { + guideHash: 'guide', + stack: '1.2.1', + agents: { limits: { timeMs: options.timeoutMs ?? 5000, turns: 5 } }, + } const driver = createRealDriver({ id: 'review', root, @@ -155,14 +161,10 @@ async function setup( command: options.command ?? 'true', isolation, credentials: { CS_CLIENT_KEY: 'app-secret' }, - manifest: { - guideHash: 'guide', - stack: '1.2.1', - agents: { limits: { timeMs: options.timeoutMs ?? 5000, turns: 5 } }, - }, + manifest, }) await driver.provision() - return { driver, output, commands, calls, cancellations } + return { driver, output, commands, calls, cancellations, manifest } } test('F1 records a committed migration relative to the prepared fixture', async () => { @@ -577,3 +579,43 @@ test.each([ (await migrationChecks(options)).find((check) => check.id === id)?.passed, ).toBe(false) }) + +test('N3 persists private broker responses without exporting their contents in the manifest', async () => { + const adapterDir = await mkdtemp(join(tmpdir(), 'mh-evidence-driver-')) + outputs.push(adapterDir) + const adapter = join(adapterDir, 'adapter.mjs') + await writeFile( + adapter, + `console.log(JSON.stringify({id:'doc',method:'exec',command:['npm','view','example','readme']}));process.stdin.once('data',()=>process.exit(0));`, + ) + const response = { + exitCode: 0, + stdout: 'PRIVATE-RESPONSE-CANARY', + stderr: 'PRIVATE-DIAGNOSTIC-CANARY', + } + const { driver, output, manifest } = await setup({ + command: `node '${adapter}'`, + toolResponse: response, + }) + await driver.migrate() + const path = join(output, 'broker-evidence.json') + expect(JSON.parse(await readFile(path, 'utf8'))).toMatchObject({ + complete: true, + records: [ + { + id: 'doc', + command: ['npm', 'view', 'example', 'readme'], + result: response, + }, + ], + }) + expect((await stat(path)).mode & 0o777).toBe(0o600) + expect(manifest.artifacts?.brokerEvidence).toBe('broker-evidence.json') + expect(manifest.documentationExposure).toMatchObject({ + status: 'observed', + evidenceComplete: true, + reviewRequired: true, + }) + expect(JSON.stringify(manifest)).not.toContain('PRIVATE-RESPONSE-CANARY') + expect(JSON.stringify(manifest)).not.toContain('PRIVATE-DIAGNOSTIC-CANARY') +}) diff --git a/e2e/migration-harness/run/claude-adapter.md b/e2e/migration-harness/run/claude-adapter.md index 67377a347..92aec27b3 100644 --- a/e2e/migration-harness/run/claude-adapter.md +++ b/e2e/migration-harness/run/claude-adapter.md @@ -54,6 +54,20 @@ identifier and final turn count in `manifest.agents.runtime`. Model text never becomes a broker request. CLI stderr is consumed but not exported, because it can contain provider details. The broker transcript records the actual isolated command requests and runtime metadata; the broker also returns tool-call exit codes. +`broker-evidence.json` separately retains request IDs, commands and responses as +private evidence (mode `0600`), referenced by `manifest.artifacts.brokerEvidence`. +It is not inline Editor evidence and must be redacted before any future export. +Retention is limited to 8 MiB per Run; records that exceed the remaining budget +are omitted and the artifact and manifest report incomplete evidence. + +`manifest.documentationExposure` reports `observed` for explicit successful npm +README retrieval. Successful Migrator opens of installed Markdown/README files +report `possible`: observer events omit open flags, so extraction writes cannot +be distinguished from reads. Metadata-only inspection does not set this signal. +Other execution remains `unknown`, +never "no exposure". `reviewRequired` is always true. Inspect the private responses +and supervisor events before making a guide-only claim; incomplete evidence cannot +establish one. These signals do not detect every way documentation can be accessed. The MCP bridge connects through a private Unix socket and a per-run token. It accepts only `exec` with a string argument array; the model cannot select a host @@ -72,8 +86,9 @@ completion, unexpected tool/permission settings, and turn limits. These tests make no model calls and need no Docker. They do need permission to create a local Unix socket. -A real model Run for each procedure remains pending on a disposable remote host -selected by the maintainer. Real agents remain outside CI under the ADR; CI runs -deterministic Fake Migrators. Fake CLI coverage does not prove that an installed -Claude version can complete a migration or that the guide is sufficient for a -real model. +The workstream handoff reports one successful guide-only sample per procedure at +guide hash `1a3a83d1…` and harness commit `d6feaca69`. The retained final artifacts +have not been audited here, and command responses missing from those historical +Runs cannot be reconstructed by this change. Those samples do not establish ADR +convergence. Real agents remain outside CI; CI runs deterministic Fake Migrators. +Fake CLI coverage does not prove guide sufficiency for a real model. diff --git a/e2e/migration-harness/run/migrator.mjs b/e2e/migration-harness/run/migrator.mjs index ee506e7ef..fdfc9bfe1 100644 --- a/e2e/migration-harness/run/migrator.mjs +++ b/e2e/migration-harness/run/migrator.mjs @@ -5,7 +5,7 @@ import { join } from 'node:path' /** Run a trusted model adapter on the host. Its only application tool is the * JSON-lines exec broker; the caller supplies the fixed isolated execution context. - * @param {{command:string, execute:(command:string[],timeoutMs:number)=>Promise, stop:()=>Promise, timeoutMs?:number, maxToolCalls?:number, maxModelTurns?:number}} options + * @param {{command:string, execute:(command:string[],timeoutMs:number)=>Promise, stop:()=>Promise, timeoutMs?:number, maxToolCalls?:number, maxModelTurns?:number, maxEvidenceBytes?:number}} options * @returns {Promise} */ export async function runMigrator({ @@ -15,6 +15,7 @@ export async function runMigrator({ timeoutMs = 40 * 60 * 1000, maxToolCalls = 200, maxModelTurns = 200, + maxEvidenceBytes = 8 * 1024 * 1024, }) { if ( !Number.isFinite(timeoutMs) || @@ -22,7 +23,9 @@ export async function runMigrator({ !Number.isSafeInteger(maxToolCalls) || maxToolCalls <= 0 || !Number.isSafeInteger(maxModelTurns) || - maxModelTurns <= 0 + maxModelTurns <= 0 || + !Number.isSafeInteger(maxEvidenceBytes) || + maxEvidenceBytes <= 0 ) throw Error('Invalid Migrator limits') const directory = await mkdtemp(join(tmpdir(), 'migration-adapter-')) @@ -54,6 +57,17 @@ export async function runMigrator({ const seen = new Set() /** @type {Array<{id:string,exitCode:number}>} */ const toolCalls = [] + /** @type {NonNullable} */ + const brokerEvidence = [] + let evidenceBytes = 0 + let evidenceComplete = true + /** @type {import('../types.js').DocumentationExposure} */ + const documentationExposure = { + status: 'unknown', + reviewRequired: true, + evidenceComplete: true, + } + let buffer = '', stdout = '', stderr = '' @@ -207,6 +221,29 @@ export async function runMigrator({ Math.max(1, deadline - Date.now()), ) toolCalls.push({ id: request.id, exitCode: result.exitCode }) + // Keep raw responses private, separate from exportable tool metadata. + const record = { + id: request.id, + command: request.command, + result, + } + const bytes = Buffer.byteLength(JSON.stringify(record)) + if (evidenceBytes + bytes <= maxEvidenceBytes) { + brokerEvidence.push(record) + evidenceBytes += bytes + } else evidenceComplete = false + // This recognizes explicit retrieval only; unrecognized commands never + // prove absence. File-access evidence supplies another positive signal. + if ( + result.exitCode === 0 && + request.command[0] === 'npm' && + ['view', 'info', 'show'].includes(request.command[1]) && + request.command.some((/** @type {string} */ arg) => + /^readme(?:\.|$)/i.test(arg), + ) + ) + documentationExposure.status = 'observed' + documentationExposure.evidenceComplete = evidenceComplete if (!stopped && !closed) child.stdin.write( `${JSON.stringify({ id: request.id, result })}\n`, @@ -235,6 +272,8 @@ export async function runMigrator({ exitCode: limit ? 124 : (code ?? 1), limit, toolCalls: [...toolCalls], + brokerEvidence, + documentationExposure, ...(runtime ? { runtime } : {}), stdout, stderr, diff --git a/e2e/migration-harness/run/real-driver.mjs b/e2e/migration-harness/run/real-driver.mjs index c2983bcd1..67dfe65f6 100644 --- a/e2e/migration-harness/run/real-driver.mjs +++ b/e2e/migration-harness/run/real-driver.mjs @@ -290,6 +290,11 @@ export function createRealDriver({ ) }, async migrate() { + manifest.documentationExposure = { + status: 'unknown', + reviewRequired: true, + evidenceComplete: false, + } if (!stackSource) throw Error('Stack source not resolved') if (stackSource.tarball) await cp(stackSource.tarball, resolve(workspace, 'resolved-stack.tgz')) @@ -317,6 +322,17 @@ export function createRealDriver({ timeoutMs, }), }) + if ('brokerEvidence' in result && result.brokerEvidence) { + await writeFile( + resolve(output, 'broker-evidence.json'), + JSON.stringify({ + complete: result.documentationExposure?.evidenceComplete ?? false, + records: result.brokerEvidence, + }), + { mode: 0o600, flag: 'wx' }, + ) + manifest.documentationExposure = result.documentationExposure + } if ('runtime' in result && result.runtime) manifest.agents = { ...manifest.agents, runtime: result.runtime } if (fakeMigration && fakeMigration !== 'noop') @@ -327,6 +343,9 @@ export function createRealDriver({ ) manifest.artifacts = { transcript: 'migrator-transcript.log', + ...('brokerEvidence' in result + ? { brokerEvidence: 'broker-evidence.json' } + : {}), snapshot: 'legacy-snapshot.json', } // A timed-out broker has stopped its isolated tools and proof relays. @@ -370,6 +389,9 @@ export function createRealDriver({ await writeFile(resolve(output, 'migration.diff'), diff.stdout) manifest.artifacts = { transcript: 'migrator-transcript.log', + ...('brokerEvidence' in result + ? { brokerEvidence: 'broker-evidence.json' } + : {}), diff: 'migration.diff', snapshot: 'legacy-snapshot.json', } diff --git a/e2e/migration-harness/run/run.mjs b/e2e/migration-harness/run/run.mjs index eb8915590..ffa1238ca 100644 --- a/e2e/migration-harness/run/run.mjs +++ b/e2e/migration-harness/run/run.mjs @@ -100,6 +100,28 @@ export async function run({ id, manifest, driver, supervision = null }) { } for (const event of events) { evidence(name, 'isolation-event', event.actor, event, false) + if ( + event.actor === 'migrator' && + event.stage === 'command' && + event.result === 'success' && + event.decision === 'allowed' && + ['open', 'openat', 'openat2'].includes(event.operation ?? '') && + /\/node_modules\/.*\/(?:readme(?:\.[^/]*)?|[^/]+\.md)$/i.test( + event.resource, + ) + ) { + // Opens may be package extraction writes: the observer does not + // export open flags, so this is a review signal rather than proof of reading. + manifest.documentationExposure = { + status: + manifest.documentationExposure?.status === 'observed' + ? 'observed' + : 'possible', + reviewRequired: true, + evidenceComplete: + manifest.documentationExposure?.evidenceComplete ?? false, + } + } if (event.result === 'success' && event.decision === 'forbidden') { stopped = true report.failures.push({ diff --git a/e2e/migration-harness/types.d.ts b/e2e/migration-harness/types.d.ts index 75b8c4f25..569acbac4 100644 --- a/e2e/migration-harness/types.d.ts +++ b/e2e/migration-harness/types.d.ts @@ -20,7 +20,19 @@ export interface OracleCheck { passed: boolean cause?: string } +export interface DocumentationExposure { + status: 'observed' | 'possible' | 'unknown' + reviewRequired: true + evidenceComplete: boolean +} export interface MigratorResult { + /** Private raw evidence; never put these records into exported Run evidence. */ + brokerEvidence?: Array<{ + id: string + command: string[] + result: IsolationResult + }> + documentationExposure?: DocumentationExposure runtime?: { provider: 'claude-code' version: string @@ -83,7 +95,13 @@ export interface Manifest { } isolation?: unknown fixtureRevision?: string - artifacts?: { transcript: string; diff?: string; snapshot: string } + artifacts?: { + transcript: string + diff?: string + snapshot: string + brokerEvidence?: string + } + documentationExposure?: DocumentationExposure storeCapabilities?: unknown [key: string]: unknown } From 5261d3a5c0ae6a4750a7a1102d08436a1d314725 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Thu, 12 Nov 2026 12:25:22 +1100 Subject: [PATCH 6/7] fix(e2e): direct migrators to the staged friction schema The staged schema contains the exact guide section identifiers accepted by the harness. Point the default adapter at that contract so models do not infer a different Markdown slug convention when recording friction. Refs #1537. --- e2e/migration-harness/run/claude-adapter.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/migration-harness/run/claude-adapter.mjs b/e2e/migration-harness/run/claude-adapter.mjs index 35d701ec7..1b199dec4 100644 --- a/e2e/migration-harness/run/claude-adapter.mjs +++ b/e2e/migration-harness/run/claude-adapter.mjs @@ -151,7 +151,7 @@ export async function runClaudeAdapter({ fail, ) const prompt = - 'Upgrade the Fixture App in /workspace using only /workspace/migration-guide.md. Read that guide through the exec tool, follow its applicable migration procedure, and verify the migrated application. All commands and file access must use the exec tool in the isolated Migrator Environment. Record guide friction in /workspace/friction.json as a JSON array of objects with section, issue, actionable (boolean), description, and workaround. Write [] when there is no friction. Do not invent prior-run context.' + 'Upgrade the Fixture App in /workspace using only /workspace/migration-guide.md. Read that guide through the exec tool, follow its applicable migration procedure, and verify the migrated application. All commands and file access must use the exec tool in the isolated Migrator Environment. Read /workspace/friction-schema.json for the report schema and exact accepted section IDs. Record guide friction in /workspace/friction.json as a JSON array of objects with section, issue, actionable (boolean), description, and workaround; copy section exactly from the schema enum. Write [] when there is no friction. Do not invent prior-run context.' // --bare never reads OAuth, so a subscription token runs without it. The // flags below, the empty config directory and the init check still apply. const args = [ From e8e7f6165fbe1e01c33c147f1f41857fafc4b56c Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Thu, 12 Nov 2026 12:39:32 +1100 Subject: [PATCH 7/7] fix(e2e): address review findings on the Claude migration adapter - Bound each command result the broker returns to the adapter (192 Ki escaped characters per stream, with a truncation marker) and drop the 16 MiB lifetime cap in readMessages. One large `cat` used to kill the adapter and end the Run with no runtime record. The private broker evidence still keeps the full response. - Record an adapter exit 124 with runtime metadata as `model turn limit reached`, so a Claude turn-limit stop is not reported as a crash. - Point the adapter's TMPDIR at the broker-owned directory. A SIGKILL on a limit skipped the adapter's cleanup and left mcp.json, which holds the bridge token, in the host tmpdir. - Forward the operator's proxy variables and NODE_EXTRA_CA_CERTS to the Claude CLI so it can reach the API from a proxied host. - Carry completed tool records through a broker failure and write them to broker-evidence.json marked incomplete, instead of discarding them. - Count documentation exposure only when an npm view/info/show requests a readme field and returns content; `npm view readme version` asks for a package named readme. --- .../__tests__/broker-evidence.test.ts | 19 ++++ .../__tests__/claude-adapter.test.ts | 78 +++++++++++++++-- .../__tests__/claude-wire.test.ts | 29 +++++++ .../__tests__/migrator.test.ts | 31 +++++++ .../__tests__/review-driver.test.ts | 29 +++++++ e2e/migration-harness/run/README.md | 11 ++- e2e/migration-harness/run/claude-adapter.md | 16 ++-- e2e/migration-harness/run/claude-adapter.mjs | 14 +++ e2e/migration-harness/run/claude-wire.mjs | 7 +- e2e/migration-harness/run/migrator.mjs | 86 ++++++++++++++++--- e2e/migration-harness/run/real-driver.mjs | 59 +++++++------ 11 files changed, 327 insertions(+), 52 deletions(-) create mode 100644 e2e/migration-harness/__tests__/claude-wire.test.ts diff --git a/e2e/migration-harness/__tests__/broker-evidence.test.ts b/e2e/migration-harness/__tests__/broker-evidence.test.ts index 914fee8ba..9f3108e35 100644 --- a/e2e/migration-harness/__tests__/broker-evidence.test.ts +++ b/e2e/migration-harness/__tests__/broker-evidence.test.ts @@ -64,6 +64,25 @@ test('explicit documentation retrieval is flagged for evidence review', async () }) }) +// `npm view readme version` asks for the `version` field of a package named +// `readme`. Only a requested readme field, answered with content, is exposure. +test.each([ + [['npm', 'view', 'readme', 'version'], 'stack-docs-1.0.0', 'unknown'], + [['npm', 'view', 'readme'], '{"name":"readme"}', 'unknown'], + [['npm', 'view', '@cipherstash/stack', 'readme'], '', 'unknown'], + [ + ['npm', 'info', '--json', '@cipherstash/stack', 'readme'], + 'docs', + 'observed', + ], +])( + 'documentation exposure from %j answering %j is %s', + async (command, stdout, status) => { + const result = await invoke(command, { exitCode: 0, stdout, stderr: '' }) + expect(result.documentationExposure?.status).toBe(status) + }, +) + test('an arbitrary shell command cannot establish the absence of supplementary documentation', async () => { const result = await invoke(['sh', '-c', 'node inspect.mjs'], { exitCode: 0, diff --git a/e2e/migration-harness/__tests__/claude-adapter.test.ts b/e2e/migration-harness/__tests__/claude-adapter.test.ts index c1e3f1a8a..468f19649 100644 --- a/e2e/migration-harness/__tests__/claude-adapter.test.ts +++ b/e2e/migration-harness/__tests__/claude-adapter.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, expect, test, vi } from 'vitest' @@ -11,8 +11,17 @@ afterEach(async () => { await rm(directory, { recursive: true, force: true }) }) -async function fixture(mode = 'success') { - const directory = await mkdtemp(join(tmpdir(), 'claude-adapter-test-')) +async function fixture( + mode = 'success', + options: { + stdout?: string + timeoutMs?: number + parent?: string + } = {}, +) { + const directory = await mkdtemp( + join(options.parent ?? tmpdir(), 'claude-adapter-test-'), + ) directories.push(directory) const executable = join(directory, 'claude') const report = join(directory, 'launch.json') @@ -33,7 +42,7 @@ const entry = config.mcpServers.migration await writeFile(${JSON.stringify(report)}, JSON.stringify({args,env:process.env,cwd:process.cwd(),config})) const emit = (value) => console.log(JSON.stringify(value)) emit({ type:'system',subtype:'init',model:'claude-sonnet-resolved',permissionMode:mode === 'unsafe-permission' ? 'bypassPermissions' : 'acceptEdits',tools:mode === 'unsafe-tool' ? ['Bash'] : ['mcp__migration__exec'] }) -if (mode === 'unsafe-tool' || mode === 'unsafe-permission') { setTimeout(()=>{},10000) } else { +if (mode === 'unsafe-tool' || mode === 'unsafe-permission' || mode === 'hang') { setTimeout(()=>{},10000) } else { const mcp = spawn(entry.command, entry.args, {env:{...process.env,...entry.env},stdio:['pipe','pipe','pipe']}) const pending = new Map() createInterface({input:mcp.stdout}).on('line', line => { const msg=JSON.parse(line); pending.get(msg.id)?.(msg); pending.delete(msg.id) }) @@ -65,7 +74,7 @@ if (mode === 'unsafe-tool' || mode === 'unsafe-permission') { setTimeout(()=>{}, vi.stubEnv('ANTHROPIC_BASE_URL', 'https://untrusted.invalid') const execute = vi.fn(async () => ({ exitCode: 0, - stdout: 'isolated guide', + stdout: options.stdout ?? 'isolated guide', stderr: '', })) const run = () => @@ -73,7 +82,7 @@ if (mode === 'unsafe-tool' || mode === 'unsafe-permission') { setTimeout(()=>{}, command: `${process.execPath} '${entry}'`, execute, stop: async () => {}, - timeoutMs: 5000, + timeoutMs: options.timeoutMs ?? 5000, maxModelTurns: 3, }) return { run, execute, report, directory } @@ -163,10 +172,65 @@ test.each(['turn-limit', 'over-budget'])( 'Claude turns %s into an agent limit', async (mode) => { const f = await fixture(mode) - expect((await f.run()).exitCode).toBe(124) + const outcome = await f.run() + expect(outcome.exitCode).toBe(124) + // run.mjs reports `limit`; without it a turn-limit stop reads as a crash. + expect(outcome.limit).toBe('model turn limit reached') }, ) +// One command result over the 1 MiB protocol line limit used to kill the +// adapter. The broker now bounds what it returns and marks the cut, so the +// model can retry with a narrower read and the Run continues. +test('Claude receives an oversized command result truncated with a marker instead of failing the Run', async () => { + const f = await fixture('success', { stdout: 'x'.repeat(2_000_000) }) + const outcome = await f.run() + expect(outcome, outcome.stderr).toMatchObject({ exitCode: 0 }) + const mcp = JSON.parse(await readFile(join(f.directory, 'mcp.json'), 'utf8')) + const delivered = JSON.parse(mcp.execution.result.content[0].text) + expect(delivered.stdout).toMatch(/output truncated/) + expect(delivered.stdout.length).toBeLessThan(1024 * 1024) + // The private evidence keeps the full response. + expect(outcome.brokerEvidence?.[0]?.result.stdout).toHaveLength(2_000_000) +}) + +// A SIGKILL on a time or tool-call limit skips the adapter's `finally`. Its +// temporary CLI home holds mcp.json with the bridge token, so it must live in +// the broker-owned directory the surviving broker always removes. +test('a Claude adapter killed on a limit leaves no temporary home behind', async () => { + // A short parent keeps the nested Unix socket path under the macOS limit. + const parent = await mkdtemp('/tmp/mh-t-') + directories.push(parent) + vi.stubEnv('TMPDIR', parent) + const f = await fixture('hang', { timeoutMs: 3000, parent }) + const outcome = await f.run() + expect(outcome.limit).toBe('time limit reached') + const launch = JSON.parse(await readFile(f.report, 'utf8')) + expect(launch.env.HOME).toBeTruthy() + expect( + (await readdir(parent)).filter((name) => name.startsWith('mh-claude-')), + ).toEqual([]) + await expect( + readFile(join(launch.env.HOME, 'mcp.json')), + ).rejects.toMatchObject({ code: 'ENOENT' }) +}) + +test('Claude receives the operator proxy and CA settings it needs to reach the API', async () => { + const f = await fixture() + vi.stubEnv('HTTPS_PROXY', 'http://proxy.internal:3128') + vi.stubEnv('NO_PROXY', 'localhost') + vi.stubEnv('NODE_EXTRA_CA_CERTS', '/etc/ssl/corp.pem') + const outcome = await f.run() + expect(outcome, outcome.stderr).toMatchObject({ exitCode: 0 }) + const launch = JSON.parse(await readFile(f.report, 'utf8')) + expect(launch.env).toMatchObject({ + HTTPS_PROXY: 'http://proxy.internal:3128', + NO_PROXY: 'localhost', + NODE_EXTRA_CA_CERTS: '/etc/ssl/corp.pem', + }) + expect(launch.env).not.toHaveProperty('ANTHROPIC_BASE_URL') +}) + test('Claude refuses a login-only host without making a tool call', async () => { const f = await fixture() vi.stubEnv('ANTHROPIC_API_KEY', '') diff --git a/e2e/migration-harness/__tests__/claude-wire.test.ts b/e2e/migration-harness/__tests__/claude-wire.test.ts new file mode 100644 index 000000000..d354a47e6 --- /dev/null +++ b/e2e/migration-harness/__tests__/claude-wire.test.ts @@ -0,0 +1,29 @@ +import { PassThrough } from 'node:stream' +import { expect, test, vi } from 'vitest' +import { readMessages } from '../run/claude-wire.mjs' + +// Each message is bounded; the stream as a whole is not. Claude's output and +// the broker's responses repeat many valid results over a Run, and a lifetime +// cap used to end the Run after about 18 results of 900 KiB. +test('readMessages bounds each message but not the stream as a whole', async () => { + const stream = new PassThrough() + const receive = vi.fn() + const fail = vi.fn() + readMessages(stream, receive, fail) + const line = `${JSON.stringify({ text: 'x'.repeat(900 * 1024) })}\n` + for (let i = 0; i < 20; i++) stream.write(line) + stream.end() + await new Promise((resolve) => stream.on('end', resolve)) + expect(fail).not.toHaveBeenCalled() + expect(receive).toHaveBeenCalledTimes(20) +}) + +test('readMessages still fails on a single message over 1 MiB', async () => { + const stream = new PassThrough() + const fail = vi.fn() + readMessages(stream, () => {}, fail) + stream.write('x'.repeat(1024 * 1024 + 1)) + stream.end() + await new Promise((resolve) => stream.on('end', resolve)) + expect(fail).toHaveBeenCalledWith(Error('Protocol output limit exceeded')) +}) diff --git a/e2e/migration-harness/__tests__/migrator.test.ts b/e2e/migration-harness/__tests__/migrator.test.ts index 6fed04e12..6b68e88a7 100644 --- a/e2e/migration-harness/__tests__/migrator.test.ts +++ b/e2e/migration-harness/__tests__/migrator.test.ts @@ -140,6 +140,37 @@ test.each([0, -1, 1.5, Number.NaN])( }, ) +// A failure after an earlier tool call succeeded must not discard that call's +// record: the driver writes it to incomplete broker evidence before rethrowing. +test('broker carries completed tool records through an isolation failure', async () => { + const messages = ['first', 'second'] + .map((id) => JSON.stringify({ id, method: 'exec', command: ['true'] })) + .join('\n') + let calls = 0 + const execute = vi.fn(async () => { + if (++calls === 2) + throw Object.assign(Error('isolation broke'), { infrastructure: true }) + return result + }) + const failure = await runMigrator({ + command: await adapter( + `process.stdout.write(${JSON.stringify(`${messages}\n`)});setTimeout(()=>{},10000)`, + ), + execute, + stop: async () => {}, + timeoutMs: 5000, + }).then( + () => undefined, + (error) => error, + ) + expect(failure?.message).toBe('isolation broke') + expect(failure?.migratorResult).toMatchObject({ + toolCalls: [{ id: 'first', exitCode: 0 }], + brokerEvidence: [{ id: 'first', command: ['true'], result }], + documentationExposure: { evidenceComplete: false }, + }) +}) + test('broker preserves an infrastructure failure from isolated execution', async () => { const failure = Object.assign(Error('trusted stream failed'), { infrastructure: true, diff --git a/e2e/migration-harness/__tests__/review-driver.test.ts b/e2e/migration-harness/__tests__/review-driver.test.ts index a4da12e40..144dfd3b0 100644 --- a/e2e/migration-harness/__tests__/review-driver.test.ts +++ b/e2e/migration-harness/__tests__/review-driver.test.ts @@ -106,6 +106,7 @@ async function setup( stopFails?: boolean timeoutMs?: number toolResponse?: { exitCode: number; stdout: string; stderr: string } + toolFailsOnCall?: number } = {}, ) { const output = await mkdtemp(join(tmpdir(), 'mh-review-driver-')) @@ -113,6 +114,7 @@ async function setup( const commands: IsolationOptions[] = [] const calls: string[] = [] const cancellations: Array<{ expected?: boolean } | undefined> = [] + let toolCalls = 0 const isolation = { prove: async () => ({ verified: true, @@ -133,6 +135,8 @@ async function setup( const result = await execute(file, args, { cwd: invocation.workspace }) return { exitCode: 0, stdout: result.stdout, stderr: result.stderr } } + if (++toolCalls === options.toolFailsOnCall) + throw Object.assign(Error('isolation broke'), { infrastructure: true }) return options.toolResponse ?? { exitCode: 0, stdout: '', stderr: '' } }, stop: async (stopOptions?: { expected?: boolean }) => { @@ -624,3 +628,28 @@ test('N3 persists private broker responses without exporting their contents in t expect(JSON.stringify(manifest)).not.toContain('PRIVATE-RESPONSE-CANARY') expect(JSON.stringify(manifest)).not.toContain('PRIVATE-DIAGNOSTIC-CANARY') }) + +test('an isolation failure after a completed tool call still persists that call as incomplete evidence', async () => { + const adapterDir = await mkdtemp(join(tmpdir(), 'mh-evidence-driver-')) + outputs.push(adapterDir) + const adapter = join(adapterDir, 'adapter.mjs') + await writeFile( + adapter, + `console.log(JSON.stringify({id:'first',method:'exec',command:['cat','guide.md']}));process.stdin.once('data',()=>{console.log(JSON.stringify({id:'second',method:'exec',command:['true']}));setTimeout(()=>{},10000)});`, + ) + const response = { exitCode: 0, stdout: 'guide text', stderr: '' } + const { driver, output, manifest } = await setup({ + command: `node '${adapter}'`, + toolResponse: response, + toolFailsOnCall: 2, + }) + await expect(driver.migrate()).rejects.toThrow('isolation broke') + expect( + JSON.parse(await readFile(join(output, 'broker-evidence.json'), 'utf8')), + ).toEqual({ + complete: false, + records: [{ id: 'first', command: ['cat', 'guide.md'], result: response }], + }) + expect(manifest.artifacts?.brokerEvidence).toBe('broker-evidence.json') + expect(manifest.documentationExposure?.evidenceComplete).toBe(false) +}) diff --git a/e2e/migration-harness/run/README.md b/e2e/migration-harness/run/README.md index 41117b439..bdf827960 100644 --- a/e2e/migration-harness/run/README.md +++ b/e2e/migration-harness/run/README.md @@ -86,6 +86,12 @@ a response to the adapter's stdin: {"id":"read-guide","result":{"exitCode":0,"stdout":"...","stderr":""}} ``` +The harness bounds each of `stdout` and `stderr` in a response to 192 Ki +characters of escaped JSON. Longer output is cut and ends with a +`[harness: output truncated; …]` marker, so the adapter can retry with a +narrower command; the private broker evidence keeps the full response. Each +protocol line is limited to 1 MiB. + Only `id`, `method` and `command` are accepted in requests. IDs must be unique nonempty strings of at most 128 characters. Commands are nonempty arrays of strings. Requests are processed in order; nonzero tool exits are returned so the @@ -105,7 +111,10 @@ does not claim those restrictions exist merely because a command was supplied. The harness enforces the wall-clock deadline and a separate limit on broker requests (`agents.limits.toolCalls`, default 200). On timeout it terminates the adapter process group and isolated tools, preserves `agent-limit`, and skips -post-agent Git commands. `HARNESS_MAX_MODEL_TURNS` communicates +post-agent Git commands. Adapter temporary files go under `TMPDIR`, a directory +the harness owns and always removes, because a killed adapter cannot clean up. +If the broker fails after earlier tool calls completed, their records are still +written to `broker-evidence.json`, marked incomplete. `HARNESS_MAX_MODEL_TURNS` communicates `agents.limits.turns` (default 200) to the trusted adapter, which must configure its model runtime to enforce that budget. Tool-call counts are not model-turn counts. `HARNESS_MAX_TOOL_CALLS` and diff --git a/e2e/migration-harness/run/claude-adapter.md b/e2e/migration-harness/run/claude-adapter.md index 92aec27b3..79922b7ec 100644 --- a/e2e/migration-harness/run/claude-adapter.md +++ b/e2e/migration-harness/run/claude-adapter.md @@ -43,10 +43,13 @@ budget and a 40-minute outer deadline. The broker passes the configured turn budget to the adapter; Claude's documented `--max-turns` enforces it during print mode, even though some CLI versions omit the flag from `--help`. The broker separately enforces tool-call and wall-clock limits. A Claude turn-limit result -returns exit code 124. See the [Claude CLI reference](https://code.claude.com/docs/en/cli-reference). +returns exit code 124, which the broker records as `model turn limit reached`. See the [Claude CLI reference](https://code.claude.com/docs/en/cli-reference). Each invocation uses an empty temporary working/configuration directory and an -allowlisted environment. Built-in tools, implicit MCP servers, settings sources, +allowlisted environment. The allowlist forwards the operator's `HTTPS_PROXY`, +`HTTP_PROXY`, `NO_PROXY` (either case) and `NODE_EXTRA_CA_CERTS` when set, so +the CLI can reach the Anthropic API through a proxy or custom CA; it never +forwards `ANTHROPIC_BASE_URL`. Built-in tools, implicit MCP servers, settings sources, slash commands, hooks, automatic memory, browser access and persistent sessions are disabled. The adapter checks Claude's initialization record for the expected tool set and permission mode, then records the CLI version, resolved model @@ -60,8 +63,10 @@ It is not inline Editor evidence and must be redacted before any future export. Retention is limited to 8 MiB per Run; records that exceed the remaining budget are omitted and the artifact and manifest report incomplete evidence. -`manifest.documentationExposure` reports `observed` for explicit successful npm -README retrieval. Successful Migrator opens of installed Markdown/README files +`manifest.documentationExposure` reports `observed` for a successful npm +`view`/`info`/`show` that requests a `readme` field of a package and returns +content (`npm view readme version` asks for a package named `readme`, and is +not counted). Successful Migrator opens of installed Markdown/README files report `possible`: observer events omit open flags, so extraction writes cannot be distinguished from reads. Metadata-only inspection does not set this signal. Other execution remains `unknown`, @@ -75,7 +80,8 @@ working directory, environment, mount or policy. The model credential is supplie only to the host CLI and explicitly removed from the bridge's environment. The adapter and CLI are trusted host code, not an additional security boundary against a compromised host executable. Temporary configuration and socket files -are removed after execution. +are created under the broker's own temporary directory (`TMPDIR`), so they are +removed after execution even when the broker kills the adapter on a limit. ## Verification diff --git a/e2e/migration-harness/run/claude-adapter.mjs b/e2e/migration-harness/run/claude-adapter.mjs index 1b199dec4..4de76dff7 100644 --- a/e2e/migration-harness/run/claude-adapter.mjs +++ b/e2e/migration-harness/run/claude-adapter.mjs @@ -38,7 +38,21 @@ export async function runClaudeAdapter({ const token = randomBytes(32).toString('hex') // In particular, do not inherit NODE_OPTIONS, CLAUDE_*, cloud auth, CS_*, or // ANTHROPIC_BASE_URL. The only model provider is the public Anthropic API. + // The operator's proxy and CA settings are forwarded only so the CLI can + // reach that API from a host whose route needs them. + const network = Object.fromEntries( + [ + 'HTTPS_PROXY', + 'https_proxy', + 'HTTP_PROXY', + 'http_proxy', + 'NO_PROXY', + 'no_proxy', + 'NODE_EXTRA_CA_CERTS', + ].flatMap((key) => (process.env[key] ? [[key, process.env[key]]] : [])), + ) const env = { + ...network, PATH: process.env.PATH, HOME: directory, TMPDIR: directory, diff --git a/e2e/migration-harness/run/claude-wire.mjs b/e2e/migration-harness/run/claude-wire.mjs index afbec4973..3b7544d5e 100644 --- a/e2e/migration-harness/run/claude-wire.mjs +++ b/e2e/migration-harness/run/claude-wire.mjs @@ -1,16 +1,17 @@ /** Bounded newline-delimited JSON framing shared by the trusted adapter and MCP bridge. + * Each message is bounded at 1 MiB (UTF-16 units). The stream as a whole is + * not: a Run repeats many valid results, and the broker bounds each one well + * under this limit (`migrator.mjs`), so a lifetime cap only ended valid Runs. * @param {import('node:stream').Readable} stream * @param {(message:any)=>void} receive * @param {(error:Error)=>void} fail */ export function readMessages(stream, receive, fail) { let buffer = '' - let total = 0 stream.setEncoding('utf8') stream.on('data', (chunk) => { - total += chunk.length buffer += chunk - if (buffer.length > 1024 * 1024 || total > 16 * 1024 * 1024) { + if (buffer.length > 1024 * 1024) { fail(Error('Protocol output limit exceeded')) return } diff --git a/e2e/migration-harness/run/migrator.mjs b/e2e/migration-harness/run/migrator.mjs index fdfc9bfe1..e4528e2d3 100644 --- a/e2e/migration-harness/run/migrator.mjs +++ b/e2e/migration-harness/run/migrator.mjs @@ -3,6 +3,55 @@ import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' +/** Escaped JSON length allowed for each of a result's stdout and stderr. + * An adapter may wrap a result in one more JSON string (the Claude MCP bridge + * does), which at most doubles an escaped string's length. Two streams at + * this bound stay under the 1 MiB protocol line limit after that wrapping. + */ +export const RESULT_STREAM_CHARS = 192 * 1024 + +/** Bound one stream by its JSON-escaped length, cutting at a code point and + * appending a marker so the model can retry with a narrower command. + * @param {string} text + * @param {number} [limit] + */ +export function boundStream(text, limit = RESULT_STREAM_CHARS) { + if (JSON.stringify(text).length - 2 <= limit) return text + const marker = (/** @type {number} */ omitted) => + `\n[harness: output truncated; ${omitted} of ${text.length} characters omitted. Use a narrower command.]` + const budget = limit - (JSON.stringify(marker(text.length)).length - 2) + let used = 0 + let end = 0 + for (const point of text) { + const cost = JSON.stringify(point).length - 2 + if (used + cost > budget) break + used += cost + end += point.length + } + return `${text.slice(0, end)}${marker(text.length - end)}` +} + +/** @param {import('../types.js').IsolationResult} result */ +function boundResult(result) { + return { + ...result, + stdout: boundStream(result.stdout), + stderr: boundStream(result.stderr), + } +} + +/** @param {string[]} command */ +function requestsReadme(command) { + if (command[0] !== 'npm' || !['view', 'info', 'show'].includes(command[1])) + return false + // `npm view [field...]`: the first positional names the package. + const fields = command + .slice(2) + .filter((arg) => !arg.startsWith('-')) + .slice(1) + return fields.some((field) => /^readme(?:\.|$)/i.test(field)) +} + /** Run a trusted model adapter on the host. Its only application tool is the * JSON-lines exec broker; the caller supplies the fixed isolated execution context. * @param {{command:string, execute:(command:string[],timeoutMs:number)=>Promise, stop:()=>Promise, timeoutMs?:number, maxToolCalls?:number, maxModelTurns?:number, maxEvidenceBytes?:number}} options @@ -44,6 +93,10 @@ export async function runMigrator({ HARNESS_FRICTION_LOG: '/workspace/friction.json', HARNESS_MAX_TOOL_CALLS: String(maxToolCalls), HARNESS_MAX_MODEL_TURNS: String(maxModelTurns), + // terminate() SIGKILLs the adapter before its own cleanup runs. Its + // temporary files (Claude's holds the bridge token) land here instead, + // and the finally below always removes this directory. + TMPDIR: directory, }) try { return await new Promise((resolve, reject) => { @@ -236,17 +289,14 @@ export async function runMigrator({ // prove absence. File-access evidence supplies another positive signal. if ( result.exitCode === 0 && - request.command[0] === 'npm' && - ['view', 'info', 'show'].includes(request.command[1]) && - request.command.some((/** @type {string} */ arg) => - /^readme(?:\.|$)/i.test(arg), - ) + result.stdout.trim() && + requestsReadme(request.command) ) documentationExposure.status = 'observed' documentationExposure.evidenceComplete = evidenceComplete if (!stopped && !closed) child.stdin.write( - `${JSON.stringify({ id: request.id, result })}\n`, + `${JSON.stringify({ id: request.id, result: boundResult(result) })}\n`, ) } catch (error) { failure = error instanceof Error ? error : Error(String(error)) @@ -264,11 +314,11 @@ export async function runMigrator({ invalid('adapter exited with an unfinished request') await cancellation await queue - if (failure) { - reject(failure) - return - } - resolve({ + // An adapter that reported runtime metadata and exits 124 stopped on + // its model-turn limit, even when its own turn count is within budget. + if (!limit && code === 124 && runtime) + limit = 'model turn limit reached' + const outcome = { exitCode: limit ? 124 : (code ?? 1), limit, toolCalls: [...toolCalls], @@ -277,7 +327,19 @@ export async function runMigrator({ ...(runtime ? { runtime } : {}), stdout, stderr, - }) + } + if (failure) { + // Completed tool calls still happened; carry them to the caller, + // marked incomplete, so a later failure does not erase them. + documentationExposure.evidenceComplete = false + Object.defineProperty(failure, 'migratorResult', { + value: outcome, + configurable: true, + }) + reject(failure) + return + } + resolve(outcome) }) }) } finally { diff --git a/e2e/migration-harness/run/real-driver.mjs b/e2e/migration-harness/run/real-driver.mjs index 40751b368..647b62478 100644 --- a/e2e/migration-harness/run/real-driver.mjs +++ b/e2e/migration-harness/run/real-driver.mjs @@ -291,6 +291,35 @@ export function createRealDriver({ ) }, async migrate() { + /** Write the Migrator's evidence, runtime and transcript artifacts. + * @param {import('../types.js').MigratorResult & {stdout?:string,stderr?:string}} result + */ + async function record(result) { + if ('brokerEvidence' in result && result.brokerEvidence) { + await writeFile( + resolve(output, 'broker-evidence.json'), + JSON.stringify({ + complete: result.documentationExposure?.evidenceComplete ?? false, + records: result.brokerEvidence, + }), + { mode: 0o600, flag: 'wx' }, + ) + manifest.documentationExposure = result.documentationExposure + } + if ('runtime' in result && result.runtime) + manifest.agents = { ...manifest.agents, runtime: result.runtime } + await writeFile( + resolve(output, 'migrator-transcript.log'), + `${result.stdout ?? ''}\n${result.stderr ?? ''}`, + ) + manifest.artifacts = { + transcript: 'migrator-transcript.log', + ...('brokerEvidence' in result + ? { brokerEvidence: 'broker-evidence.json' } + : {}), + snapshot: 'legacy-snapshot.json', + } + } manifest.documentationExposure = { status: 'unknown', reviewRequired: true, @@ -322,33 +351,15 @@ export function createRealDriver({ forwards: forwards(migratorStores), timeoutMs, }), + }).catch(async (error) => { + // The broker attaches the tool calls that completed before it + // failed. Persist them, marked incomplete, before rethrowing. + if (error?.migratorResult) await record(error.migratorResult) + throw error }) - if ('brokerEvidence' in result && result.brokerEvidence) { - await writeFile( - resolve(output, 'broker-evidence.json'), - JSON.stringify({ - complete: result.documentationExposure?.evidenceComplete ?? false, - records: result.brokerEvidence, - }), - { mode: 0o600, flag: 'wx' }, - ) - manifest.documentationExposure = result.documentationExposure - } - if ('runtime' in result && result.runtime) - manifest.agents = { ...manifest.agents, runtime: result.runtime } + await record(result) if (fakeMigration && fakeMigration !== 'noop') await execute('migrator', workspace, migratorStores, ['npm', 'install']) - await writeFile( - resolve(output, 'migrator-transcript.log'), - `${result.stdout ?? ''}\n${result.stderr ?? ''}`, - ) - manifest.artifacts = { - transcript: 'migrator-transcript.log', - ...('brokerEvidence' in result - ? { brokerEvidence: 'broker-evidence.json' } - : {}), - snapshot: 'legacy-snapshot.json', - } // A timed-out broker has stopped its isolated tools and proof relays. // Preserve its agent-limit result rather than restarting work for a diff. if (result.exitCode !== 0 || result.limit) return result