diff --git a/e2e/migration-harness/__tests__/behaviour.test.ts b/e2e/migration-harness/__tests__/behaviour.test.ts index d9790473c..18347ec4a 100644 --- a/e2e/migration-harness/__tests__/behaviour.test.ts +++ b/e2e/migration-harness/__tests__/behaviour.test.ts @@ -107,3 +107,32 @@ test('fresh-write lookup failure is visible as stable Oracle evidence', async () }, ) }) +test('Mutation Data is inspected after setActive and before a read can repair it', async () => { + const order: string[] = [] + const result = await behaviour( + async (method: string) => { + order.push(method) + return method.startsWith('find') ? [] : null + }, + async () => { + order.push('inspect') + }, + ) + await result.mutate() + const setActive = order.indexOf('setActive') + assert.notEqual(setActive, -1, 'setActive must be called') + assert.equal( + order[setActive + 1], + 'inspect', + `inspection must follow setActive directly: ${order.slice(setActive).join(',')}`, + ) +}) +test('a Mutation Data inspection failure still aborts the Oracle', async () => { + const result = await behaviour( + async (method: string) => (method.startsWith('find') ? [] : null), + async () => { + throw Object.assign(Error('store unavailable'), { infrastructure: true }) + }, + ) + await assert.rejects(result.mutate(), /store unavailable/) +}) diff --git a/e2e/migration-harness/__tests__/cli-fake-manifest.test.ts b/e2e/migration-harness/__tests__/cli-fake-manifest.test.ts new file mode 100644 index 000000000..d2af3fddb --- /dev/null +++ b/e2e/migration-harness/__tests__/cli-fake-manifest.test.ts @@ -0,0 +1,48 @@ +import { mkdtemp, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { expect, test, vi } from 'vitest' +import { requiredOracleChecks } from '../run/migration-checks.mjs' + +vi.mock('../run/real-driver.mjs', () => ({ + createRealDriver: () => ({ + provision: async () => {}, + installUnmigrated: async () => {}, + initializeLegacyData: async () => {}, + snapshot: async () => {}, + migrate: async () => ({ exitCode: 0 }), + friction: async () => [], + installAppHost: async () => {}, + behaviour: async () => [ + { id: 'behaviour.cli', passed: true }, + ...requiredOracleChecks.map((id) => ({ id, passed: true })), + ], + close: async () => {}, + }), +})) +vi.mock('../isolation.mjs', () => ({ + createIsolation: () => ({ drain: () => [], stop: async () => {} }), +})) + +test('a Fake Migrator Run records its variant rather than a command adapter', async () => { + const output = await mkdtemp(join(tmpdir(), 'mh-cli-fake-')) + const argv = process.argv + const exitCode = process.exitCode + const diagnostics = vi.spyOn(console, 'error').mockImplementation(() => {}) + try { + process.argv = ['node', 'cli.mjs', '--fake', 'downcase', '--output', output] + await import('../run/cli.mjs') + const report = JSON.parse( + await readFile(join(output, 'run-report.json'), 'utf8'), + ) + expect(report.manifest.agents).toMatchObject({ + migrator: 'fake:downcase', + model: 'fake-migrator', + }) + } finally { + process.argv = argv + process.exitCode = exitCode + diagnostics.mockRestore() + await rm(output, { recursive: true, force: true }) + } +}) diff --git a/e2e/migration-harness/__tests__/migration-live.test.ts b/e2e/migration-harness/__tests__/migration-live.test.ts new file mode 100644 index 000000000..a00c697c8 --- /dev/null +++ b/e2e/migration-harness/__tests__/migration-live.test.ts @@ -0,0 +1,127 @@ +import 'dotenv/config' +import assert from 'node:assert/strict' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { test } from 'vitest' +import { createIsolation } from '../isolation.mjs' +import { createRealDriver } from '../run/real-driver.mjs' +import { run } from '../run/run.mjs' + +const keys = [ + 'CS_WORKSPACE_CRN', + 'CS_CLIENT_ID', + 'CS_CLIENT_KEY', + 'CS_CLIENT_ACCESS_KEY', +] +const enabled = + process.env.HARNESS_LIVE === '1' && keys.every((key) => process.env[key]) +const cases = [ + ['reference', 'pass', null], + ['noop', 'behaviour', 'migration.protect-dependencies-absent'], + ['downcase', 'behaviour', 'behaviour.downcase-new-write-lookup'], + ['stale-hmac', 'behaviour', 'migration.active-hmac-absent'], + ['bigint', 'behaviour', 'behaviour.balance-decimal-string'], + ['rewrite', 'behaviour', 'no-rewrite.legacy-bytes'], + ['missing-log', 'invalid-friction-log', null], + ['legacy-ffi', 'behaviour', 'migration.protect-ffi-version'], + ['denied-app', 'pass', null], + ['denied-lifecycle', 'pass', null], + ['tree', 'pass', null], +] as const +for (const [kind, outcome, checkId] of cases) { + test.skipIf(!enabled)( + `real Run self-test: ${kind}`, + async () => { + const output = await mkdtemp(resolve(tmpdir(), 'mh-selftest-')) + const isolation = createIsolation() + const manifest = { + stack: kind === 'tree' ? 'tree' : '^1.2.1', + guideHash: null, + } + const driver = createRealDriver({ + id: `selftest-${Date.now()}`, + root: fileURLToPath(new URL('../../..', import.meta.url)), + output, + command: '', + fakeMigration: kind === 'tree' ? 'reference' : kind, + isolation, + credentials: Object.fromEntries( + keys.map((key) => [key, process.env[key] ?? '']), + ), + manifest, + }) + try { + const report = await run({ + id: kind, + manifest, + driver, + supervision: isolation, + }) + const diagnostics = JSON.stringify({ + outcome: report.outcome, + failures: report.failures, + checks: report.evidence + .filter( + (e) => + e.kind === 'oracle-check' && Object(e.payload).passed === false, + ) + .map((e) => e.payload), + }) + assert.equal(report.outcome, outcome, diagnostics) + // Every isolated exec starts with a startup probe that reads the same + // canary, so only a command-stage denial from the expected actor and + // Run stage proves the migrated code itself was exercised. + const deniedBy = (actor: string, stage?: string) => + report.evidence.some( + (e) => + e.kind === 'isolation-event' && + (stage === undefined || e.stage === stage) && + Object(e.payload).actor === actor && + Object(e.payload).stage === 'command' && + Object(e.payload).result === 'denied' && + Object(e.payload).decision === 'forbidden' && + Object(e.payload).resource === '/trace/canary', + ) + if (kind === 'denied-app') + assert.ok(deniedBy('app-host'), 'createApp probe was not denied') + // Only the Migrator's bare `npm install` runs the root install hook; + // the App Host names a package spec, and npm skips root lifecycle + // scripts for `npm install `. + if (kind === 'denied-lifecycle') + assert.ok( + deniedBy('migrator', 'migrator'), + 'Migrator install hook was not denied', + ) + if (checkId) + assert.equal( + Object(report.evidence.find((e) => e.checkId === checkId)?.payload) + .passed, + false, + diagnostics, + ) + if (kind === 'noop') + assert.ok( + report.evidence + .filter( + (e) => + e.kind === 'oracle-check' && + !e.checkId.startsWith('migration.'), + ) + .every((e) => Object(e.payload).passed), + ) + if (kind === 'reference') + assert.ok( + report.evidence.some( + (e) => e.checkId === 'no-rewrite.legacy-bytes', + ), + ) + } finally { + await driver.close() + await rm(output, { recursive: true, force: true }) + } + }, + 600_000, + ) +} diff --git a/e2e/migration-harness/__tests__/migration.test.ts b/e2e/migration-harness/__tests__/migration.test.ts new file mode 100644 index 000000000..523b40e1c --- /dev/null +++ b/e2e/migration-harness/__tests__/migration.test.ts @@ -0,0 +1,77 @@ +import assert from 'node:assert/strict' +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { test } from 'vitest' +import { migrationChecks } from '../run/migration-checks.mjs' +import { run } from '../run/run.mjs' + +for (const stale of [false, true]) { + test(`Run migration evidence ${stale ? 'rejects stale' : 'accepts removed'} boolean HMAC`, async () => { + const workspace = await mkdtemp(join(tmpdir(), 'mh-migration-')) + await mkdir(join(workspace, 'src')) + await writeFile( + join(workspace, 'src/app.mjs'), + 'import { Encryption } from "@cipherstash/stack/v3"', + ) + await writeFile( + join(workspace, 'package.json'), + JSON.stringify({ dependencies: { '@cipherstash/stack': '1.2.1' } }), + ) + await writeFile( + join(workspace, 'package-lock.json'), + JSON.stringify({ + packages: { + '': { dependencies: { '@cipherstash/stack': '1.2.1' } }, + 'node_modules/@cipherstash/stack': { + version: '1.2.1', + dependencies: { '@cipherstash/protect-ffi': '0.31.0' }, + }, + 'node_modules/@cipherstash/protect-ffi': { version: '0.31.0' }, + }, + }), + ) + try { + const report = await run({ + id: 'migration', + manifest: { guideHash: 'guide' }, + driver: { + provision: async () => {}, + installUnmigrated: async () => {}, + initializeLegacyData: async () => {}, + snapshot: async () => {}, + migrate: async () => ({ exitCode: 0 }), + friction: async () => [], + installAppHost: async () => {}, + behaviour: async () => [ + { id: 'no-rewrite.legacy-bytes', passed: true }, + ...(await migrationChecks({ + workspace, + installedWorkspace: workspace, + ffiRange: '0.31.0', + mutationSnapshot: { + dynamo: { + Item: stale + ? { active__source: 'cipher', active__hmac: 'old' } + : { active__source: 'cipher' }, + }, + }, + stackVersion: '1.2.1', + })), + ], + }, + }) + assert.equal(report.outcome, stale ? 'behaviour' : 'pass') + assert.equal( + Object( + report.evidence.find( + (e) => e.checkId === 'migration.active-hmac-absent', + )?.payload, + ).passed, + !stale, + ) + } finally { + await rm(workspace, { recursive: true, force: true }) + } + }) +} diff --git a/e2e/migration-harness/__tests__/review-driver.test.ts b/e2e/migration-harness/__tests__/review-driver.test.ts index 83c17bc5c..b55639730 100644 --- a/e2e/migration-harness/__tests__/review-driver.test.ts +++ b/e2e/migration-harness/__tests__/review-driver.test.ts @@ -1,4 +1,5 @@ import { execFile } from 'node:child_process' +import { createHash } from 'node:crypto' import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -91,6 +92,8 @@ async function setup( gitFailure?: 'add' | 'diff' stopFails?: boolean timeoutMs?: number + stack?: string + npm?: (invocation: IsolationOptions) => Promise } = {}, ) { const output = await mkdtemp(join(tmpdir(), 'mh-review-driver-')) @@ -118,6 +121,7 @@ async function setup( const result = await execute(file, args, { cwd: invocation.workspace }) return { exitCode: 0, stdout: result.stdout, stderr: result.stderr } } + if (file === 'npm') await options.npm?.(invocation) return { exitCode: 0, stdout: '', stderr: '' } }, stop: async (stopOptions?: { expected?: boolean }) => { @@ -151,7 +155,7 @@ async function setup( credentials: { CS_CLIENT_KEY: 'app-secret' }, manifest: { guideHash: 'guide', - stack: '1.2.1', + stack: options.stack ?? '1.2.1', agents: { limits: { timeMs: options.timeoutMs ?? 5000, turns: 5 } }, }, }) @@ -391,6 +395,137 @@ test.each([ }, ) +test.each([ + [Object.assign(Error('missing table'), { code: '42P01' }), 'behaviour'], + [ + Object.assign(Error('missing DynamoDB table'), { + name: 'ResourceNotFoundException', + }), + 'behaviour', + ], + [ + Object.assign(Error('connection lost'), { code: '08006' }), + 'infrastructure', + ], +] as const)( + 'Mutation Data inspection classifies %s', + async (error, outcome) => { + const { driver, calls } = await setup() + const baseline = await observed.snapshot() + observed.snapshot + .mockResolvedValueOnce(baseline) + .mockResolvedValueOnce(baseline) + .mockRejectedValueOnce(error) + try { + const report = await run({ + id: 'mutation-snapshot', + manifest: { guideHash: 'guide' }, + driver: { + ...driver, + provision: async () => {}, + migrate: async () => ({ exitCode: 0 }), + friction: async () => [], + }, + }) + expect(report.outcome).toBe(outcome) + expect(report.editorEligible).toBe(outcome === 'behaviour') + expect(observed.snapshot).toHaveBeenLastCalledWith('mutation') + expect(calls).toContain('setActive') + expect(calls.slice(calls.lastIndexOf('setActive'))).not.toContain( + 'copyToPostgres', + ) + } finally { + await driver.close() + } + }, +) + +/** Pack a minimal Stack tarball that resolveStack accepts. */ +async function stackTarball(directory: string, marker: string) { + const source = await mkdtemp(join(tmpdir(), 'mh-stack-tgz-')) + outputs.push(source) + await mkdir(join(source, 'package')) + await writeFile( + join(source, 'package/package.json'), + JSON.stringify({ + name: '@cipherstash/stack', + version: '1.2.1', + description: marker, + dependencies: { '@cipherstash/protect-ffi': '0.33.0' }, + }), + ) + const tarball = join(directory, `${marker}.tgz`) + await execute('tar', ['-czf', tarball, '-C', source, 'package']) + return tarball +} +/** Record the lock entry npm writes for a local tarball install. */ +async function npmRecords(workspace: string, tarball: string) { + const integrity = `sha512-${createHash('sha512') + .update(await readFile(tarball)) + .digest('base64')}` + await writeFile( + join(workspace, 'package-lock.json'), + JSON.stringify({ + packages: { + 'node_modules/@cipherstash/stack': { + version: '1.2.1', + resolved: 'file:resolved-stack.tgz', + integrity, + }, + }, + }), + ) +} + +test('App Host installs the host-resolved Stack tarball, not a Migrator substitute', async () => { + const sources = await mkdtemp(join(tmpdir(), 'mh-stack-sources-')) + outputs.push(sources) + const original = await stackTarball(sources, 'original') + const substitute = await stackTarball(sources, 'substitute') + const { driver, output } = await setup({ + stack: original, + npm: async ({ actor, workspace }) => { + if (actor === 'app-host') + await npmRecords(workspace, join(workspace, 'resolved-stack.tgz')) + }, + }) + try { + await driver.snapshot() + // The Migrator controls its workspace, including the copied tarball. + await writeFile( + join(output, 'migrator/resolved-stack.tgz'), + await readFile(substitute), + ) + await driver.installAppHost() + expect(await readFile(join(output, 'app-host/resolved-stack.tgz'))).toEqual( + await readFile(original), + ) + } finally { + await driver.close() + } +}) + +test('App Host install fails when the installed Stack is not the resolved tarball', async () => { + const sources = await mkdtemp(join(tmpdir(), 'mh-stack-sources-')) + outputs.push(sources) + const original = await stackTarball(sources, 'original') + const substitute = await stackTarball(sources, 'substitute') + const { driver } = await setup({ + stack: original, + npm: async ({ actor, workspace }) => { + if (actor === 'app-host') await npmRecords(workspace, substitute) + }, + }) + try { + await driver.snapshot() + await expect(driver.installAppHost()).rejects.toThrow( + 'Installed Stack is not the resolved tarball', + ) + } finally { + await driver.close() + } +}) + async function runReal(driver: Awaited>['driver']) { try { return await run({ diff --git a/e2e/migration-harness/__tests__/run.test.ts b/e2e/migration-harness/__tests__/run.test.ts index ea827cdf1..4a81ad3dc 100644 --- a/e2e/migration-harness/__tests__/run.test.ts +++ b/e2e/migration-harness/__tests__/run.test.ts @@ -221,6 +221,10 @@ test('App Host install failure outranks invalid friction', async () => { }), }) assert.equal(report.outcome, 'install') + assert.deepEqual( + report.failures.map((f) => f.category), + ['invalid-friction-log', 'install'], + ) }) test('trusted successful forbidden access stops later calls and excludes Editor', async () => { let later = false diff --git a/e2e/migration-harness/__tests__/stack-source.test.ts b/e2e/migration-harness/__tests__/stack-source.test.ts new file mode 100644 index 000000000..6e773b328 --- /dev/null +++ b/e2e/migration-harness/__tests__/stack-source.test.ts @@ -0,0 +1,46 @@ +import { afterEach, expect, test, vi } from 'vitest' + +const npmView = vi.hoisted(() => new Map()) + +vi.mock('node:child_process', () => ({ + execFile: ( + _command: string, + args: string[], + callback: (error: Error | null, result?: { stdout: string }) => void, + ) => { + const stdout = npmView.get(args.slice(1, 3).join(' ')) + if (stdout === undefined) callback(Error(`unexpected npm ${args}`)) + else callback(null, { stdout }) + }, +})) + +const { resolveStack } = await import('../run/stack-source.mjs') + +afterEach(() => npmView.clear()) + +// npm 12 answers `npm view --json` with an array even when the +// spec names one exact version; older npm answers with the bare value. +test.each([ + ['npm 10 bare values', '"1.2.1"', '"0.33.0"'], + ['npm 12 single-element arrays', '["1.2.1"]', '["0.33.0"]'], +])('resolves a registry Stack from %s', async (_npm, version, ffi) => { + npmView.set('@cipherstash/stack@^1.2.1 version', version) + npmView.set( + '@cipherstash/stack@1.2.1 dependencies.@cipherstash/protect-ffi', + ffi, + ) + await expect( + resolveStack({ source: '^1.2.1', root: '/', output: '/' }), + ).resolves.toEqual({ version: '1.2.1', install: '1.2.1', ffiRange: '0.33.0' }) +}) + +test('rejects an ambiguous FFI dependency answer', async () => { + npmView.set('@cipherstash/stack@1.2.1 version', '"1.2.1"') + npmView.set( + '@cipherstash/stack@1.2.1 dependencies.@cipherstash/protect-ffi', + '["0.33.0","0.34.0"]', + ) + await expect( + resolveStack({ source: '1.2.1', root: '/', output: '/' }), + ).rejects.toThrow('Could not resolve Stack FFI dependency') +}) diff --git a/e2e/migration-harness/isolation/Dockerfile b/e2e/migration-harness/isolation/Dockerfile index 2a1a5b1c4..61e9a0b60 100644 --- a/e2e/migration-harness/isolation/Dockerfile +++ b/e2e/migration-harness/isolation/Dockerfile @@ -1,5 +1,5 @@ FROM node:22-bookworm-slim -RUN apt-get update && apt-get install -y --no-install-recommends strace socat python3 util-linux git && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates strace socat python3 util-linux git && rm -rf /var/lib/apt/lists/* # A bind mount's root can appear root-owned (Colima virtiofs on macOS) while the # actor runs as the host uid; Git would then refuse /workspace as dubious ownership. RUN git config --system --add safe.directory /workspace diff --git a/e2e/migration-harness/reference/src/app.mjs b/e2e/migration-harness/reference/src/app.mjs new file mode 100644 index 000000000..84d71d85d --- /dev/null +++ b/e2e/migration-harness/reference/src/app.mjs @@ -0,0 +1,140 @@ +import { encryptedDynamoDB } from '@cipherstash/stack/dynamodb' +import { Encryption, encryptedTable, types } from '@cipherstash/stack/v3' +import { createStores } from './stores.mjs' + +const customers = encryptedTable('customers', { + email: types.TextEq('email'), + emailLower: types.TextEq('emailLower'), + age: types.IntegerEq('age'), + balance: types.BigintEq('balance'), + active: types.Boolean('active'), + notes: types.Text('notes'), +}) +const postgresColumns = ['email', 'age', 'notes'] +function unwrap(result) { + if (result.failure) throw Error(result.failure.message) + return result.data +} +function publicCustomer(model) { + return 'balance' in model + ? { ...model, balance: String(model.balance) } + : model +} +function balance(value) { + if ( + typeof value !== 'string' || + !/^-?(0|[1-9]\d*)$/.test(value) || + !Number.isSafeInteger(Number(value)) + ) + throw Error('Balance must be a safe decimal integer string') + return BigInt(value) +} +export async function createApp(env = process.env) { + const client = await Encryption({ schemas: [customers] }) + const encrypted = encryptedDynamoDB({ encryptionClient: client }) + const stores = await createStores(env) + async function decrypt(item) { + return item === null + ? null + : publicCustomer(unwrap(await encrypted.decryptModel(item, customers))) + } + async function term(column, value) { + const result = unwrap( + await client.encryptQuery(value, { + table: customers, + column: customers[column], + }), + ) + if (!result || typeof result !== 'object' || typeof result.hm !== 'string') + throw Error('Missing equality term') + return result.hm + } + async function find(column, value) { + return Promise.all( + (await stores.query(column, await term(column, value))).map(decrypt), + ) + } + function fromPostgres(row) { + return row === null + ? null + : Object.assign( + { pk: row.pk }, + ...postgresColumns.map((column) => ({ + [`${column}__source`]: row[column].c, + ...(typeof row[column].hm === 'string' + ? { [`${column}__hmac`]: row[column].hm } + : {}), + })), + ) + } + async function findPostgres(column, value) { + const rows = await stores.findPostgres(column, { + v: 3, + i: { t: 'customers', c: column }, + hm: await term(column, value), + }) + return Promise.all(rows.map((row) => decrypt(fromPostgres(row)))) + } + return { + async put(customer) { + await stores.put( + unwrap( + await encrypted.encryptModel( + { + ...customer, + emailLower: customer.emailLower.toLowerCase(), + balance: balance(customer.balance), + }, + customers, + ), + ), + ) + }, + get: async (pk) => decrypt(await stores.get(pk)), + findEmail: (value) => find('email', value), + findEmailLower: (value) => find('emailLower', value.toLowerCase()), + findAge: (value) => find('age', value), + findBalance: (value) => find('balance', balance(value)), + async findActive(value) { + return (await Promise.all((await stores.scan()).map(decrypt))).filter( + (item) => item.active === value, + ) + }, + async setActive(pk, active) { + await stores.updateActive( + pk, + unwrap(await encrypted.encryptModel({ active }, customers)), + ) + }, + async copyToPostgres(pk) { + const item = await stores.get(pk) + if (!item) throw Error('Customer not found') + await stores.putPostgres({ + pk, + ...Object.fromEntries( + postgresColumns.map((column) => [ + column, + { + v: 3, + i: { t: 'customers', c: column }, + c: item[`${column}__source`], + ...(typeof item[`${column}__hmac`] === 'string' + ? { hm: item[`${column}__hmac`] } + : {}), + }, + ]), + ), + }) + }, + getPostgres: async (pk) => + decrypt(fromPostgres(await stores.getPostgres(pk))), + findPostgresEmail: (value) => findPostgres('email', value), + findPostgresAge: (value) => findPostgres('age', value), + async copyToDynamo(pk) { + const row = await stores.getPostgres(pk) + if (!row) throw Error('Customer not found') + await stores.put({ ...(await stores.get(pk)), ...fromPostgres(row) }) + }, + close: () => stores.close(), + } +} diff --git a/e2e/migration-harness/reference/src/stores.mjs b/e2e/migration-harness/reference/src/stores.mjs new file mode 100644 index 000000000..f02ae647c --- /dev/null +++ b/e2e/migration-harness/reference/src/stores.mjs @@ -0,0 +1,181 @@ +import { + CreateTableCommand, + DescribeTableCommand, + DynamoDBClient, + waitUntilTableExists, +} from '@aws-sdk/client-dynamodb' +import { + DeleteCommand, + DynamoDBDocumentClient, + GetCommand, + PutCommand, + QueryCommand, + ScanCommand, + UpdateCommand, +} from '@aws-sdk/lib-dynamodb' +import postgres from 'postgres' + +export async function createStores(env = process.env) { + for (const key of ['HARNESS_DYNAMO_URL', 'HARNESS_TABLE', 'DATABASE_URL']) { + if (!env[key]) throw new Error(`Missing ${key}`) + } + const endpoint = new URL(env.HARNESS_DYNAMO_URL) + const database = new URL(env.DATABASE_URL) + if ( + ![endpoint, database].every((url) => + ['localhost', '127.0.0.1', '[::1]'].includes(url.hostname), + ) + ) { + throw new Error('Fixture stores must be local disposable databases') + } + const raw = new DynamoDBClient({ + endpoint: endpoint.href, + region: 'us-east-1', + credentials: { + accessKeyId: 'migrationfixture', + secretAccessKey: 'migrationfixture', + }, + }) + const dynamo = DynamoDBDocumentClient.from(raw) + const TableName = env.HARNESS_TABLE + const sql = postgres(env.DATABASE_URL, { prepare: false, onnotice: () => {} }) + + async function pages(Command, input) { + const items = [] + let cursor + do { + const page = await dynamo.send( + new Command({ + TableName, + Limit: 2, + ...input, + ...(cursor ? { ExclusiveStartKey: cursor } : {}), + }), + ) + items.push(...(page.Items ?? [])) + cursor = page.LastEvaluatedKey + } while (cursor) + return items.sort((a, b) => a.pk.localeCompare(b.pk)) + } + + async function scan() { + return pages(ScanCommand, { ConsistentRead: true }) + } + + return { + async init() { + try { + await raw.send(new DescribeTableCommand({ TableName })) + } catch (error) { + if (error.name !== 'ResourceNotFoundException') throw error + await raw.send( + new CreateTableCommand({ + TableName, + BillingMode: 'PAY_PER_REQUEST', + AttributeDefinitions: [ + { AttributeName: 'pk', AttributeType: 'S' }, + { AttributeName: 'email__hmac', AttributeType: 'S' }, + ], + KeySchema: [{ AttributeName: 'pk', KeyType: 'HASH' }], + GlobalSecondaryIndexes: [ + { + IndexName: 'email-index', + KeySchema: [{ AttributeName: 'email__hmac', KeyType: 'HASH' }], + Projection: { ProjectionType: 'ALL' }, + }, + ], + }), + ) + } + await waitUntilTableExists( + { client: raw, maxWaitTime: 60 }, + { TableName }, + ) + await sql`CREATE TABLE IF NOT EXISTS customers ( + pk text PRIMARY KEY, + email public.eql_v3_text_eq, + age public.eql_v3_integer_eq, + notes public.eql_v3_text + )` + }, + async put(item) { + await dynamo.send(new PutCommand({ TableName, Item: item })) + }, + async get(pk) { + const result = await dynamo.send( + new GetCommand({ TableName, Key: { pk }, ConsistentRead: true }), + ) + return result.Item ?? null + }, + scan, + async query(column, hmac) { + if ( + !['email', 'emailLower', 'age', 'balance', 'active'].includes(column) + ) { + throw new Error('Unsupported lookup column') + } + const expression = { + ExpressionAttributeNames: { '#term': `${column}__hmac` }, + ExpressionAttributeValues: { ':term': hmac }, + } + return column === 'email' + ? pages(QueryCommand, { + ...expression, + IndexName: 'email-index', + KeyConditionExpression: '#term = :term', + }) + : pages(ScanCommand, { + ...expression, + ConsistentRead: true, + FilterExpression: '#term = :term', + }) + }, + async updateActive(pk, attributes) { + await dynamo.send( + new UpdateCommand({ + TableName, + Key: { pk }, + ConditionExpression: 'attribute_exists(pk)', + UpdateExpression: 'SET active__source = :source REMOVE active__hmac', + ExpressionAttributeValues: { + ':source': attributes.active__source, + }, + }), + ) + }, + async putPostgres(row) { + await sql`INSERT INTO customers (pk, email, age, notes) + VALUES (${row.pk}, ${sql.json(row.email)}::jsonb, ${sql.json(row.age)}::jsonb, ${sql.json(row.notes)}::jsonb) + ON CONFLICT (pk) DO UPDATE SET email = excluded.email, age = excluded.age, notes = excluded.notes` + }, + async getPostgres(pk) { + const rows = + await sql`SELECT pk, email::jsonb AS email, age::jsonb AS age, notes::jsonb AS notes + FROM customers WHERE pk = ${pk}` + return rows[0] ?? null + }, + async findPostgres(column, term) { + if (column === 'email') { + return sql`SELECT pk, email::jsonb AS email, age::jsonb AS age, notes::jsonb AS notes + FROM customers WHERE eql_v3.eq(email, ${sql.json(term)}::jsonb::eql_v3.query_text_eq) ORDER BY pk` + } + if (column === 'age') { + return sql`SELECT pk, email::jsonb AS email, age::jsonb AS age, notes::jsonb AS notes + FROM customers WHERE eql_v3.eq(age, ${sql.json(term)}::jsonb::eql_v3.query_integer_eq) ORDER BY pk` + } + throw new Error('Unsupported Postgres lookup column') + }, + async clear() { + for (const item of await scan()) { + await dynamo.send( + new DeleteCommand({ TableName, Key: { pk: item.pk } }), + ) + } + await sql`DELETE FROM customers` + }, + async close() { + raw.destroy() + await sql.end() + }, + } +} diff --git a/e2e/migration-harness/run/README.md b/e2e/migration-harness/run/README.md index 2b8a0b6f6..55d9e7ae7 100644 --- a/e2e/migration-harness/run/README.md +++ b/e2e/migration-harness/run/README.md @@ -93,3 +93,16 @@ Diff evidence uses the host-recorded fixture commit, so an adapter can commit its changes without hiding them. Failed Git commands fail the migration stage instead of publishing an empty diff. Raw transcripts and snapshots remain local Run artifacts; this command does not export them to an Editor. + +## Deterministic acceptance matrix + +Use `harness:run --fake reference` for the known-correct migration. Other variants +are `noop`, `downcase`, `stale-hmac`, `bigint`, `rewrite`, `missing-log`, +`legacy-ffi`, `denied-app` and `denied-lifecycle`. These harness-owned substitutes +copy application files; dependency installation and application execution stay +inside isolation. The hidden Reference Migration is never mounted into an actor. + +`--stack` also accepts `tree` or a `.tgz` path. Tree mode requires Stack build +outputs and packs them with pnpm. Exact versions and tarball SHA-256 digests are +recorded in the manifest. Set `HARNESS_LIVE=1` and the four `CS_*` credentials +to run the credentialed self-test matrix; CI runs it without model calls. diff --git a/e2e/migration-harness/run/behaviour.mjs b/e2e/migration-harness/run/behaviour.mjs index a8cf23783..01f5337e7 100644 --- a/e2e/migration-harness/run/behaviour.mjs +++ b/e2e/migration-harness/run/behaviour.mjs @@ -59,8 +59,9 @@ function orderedRows(value) { .sort((left, right) => String(left.pk).localeCompare(String(right.pk))) } /** Calls only the App Host JSON RPC boundary; never imports application code. - * @param {AppCall} call */ -export async function behaviour(call) { + * @param {AppCall} call + * @param {()=>Promise} afterSetActive */ +export async function behaviour(call, afterSetActive = async () => {}) { /** @type {import('../types.js').OracleCheck[]} */ const checks = [] /** @param {string} id @param {()=>Promise} action */ @@ -125,9 +126,7 @@ export async function behaviour(call) { // Mutation checks are deliberately separate so #1535 can insert no-rewrite here. return { checks, - /** @param {{afterSetActive?: ()=>Promise}} [hooks] afterSetActive - * runs before any later full put can overwrite the setActive result. */ - async mutate(hooks = {}) { + async mutate() { await check('behaviour.fresh-write', async () => { equal(await call('get', [fresh.pk]), null) await call('put', [fresh]) @@ -161,11 +160,25 @@ export async function behaviour(call) { await check('behaviour.balance-decimal-string', async () => equal(record(await call('get', [fresh.pk])).balance, fresh.balance), ) + // Inspect the stored shape before any read: a `get` that repairs a stale + // active__hmac would otherwise hide it from migration.active-hmac-absent. + // Inspection failures abort the Oracle rather than fail this check. + /** @type {unknown[]} */ + const inspectionFailure = [] await check('behaviour.set-active', async () => { - await call('setActive', [mutation.pk, false]) + try { + await call('setActive', [mutation.pk, false]) + } finally { + try { + await afterSetActive() + } catch (error) { + inspectionFailure.push(error) + } + } + if (inspectionFailure.length) return equal(record(await call('get', [mutation.pk])).active, false) }) - await hooks.afterSetActive?.() + if (inspectionFailure.length) throw inspectionFailure[0] const copiedMutation = { ...mutation, active: false, diff --git a/e2e/migration-harness/run/cli.mjs b/e2e/migration-harness/run/cli.mjs index c29bb2f38..c4f7c7dc2 100644 --- a/e2e/migration-harness/run/cli.mjs +++ b/e2e/migration-harness/run/cli.mjs @@ -11,14 +11,21 @@ const { values } = parseArgs({ args: process.argv.slice(2), options: { migrator: { type: 'string' }, + fake: { type: 'string' }, output: { type: 'string' }, stack: { type: 'string' }, }, strict: true, }) -const command = values.migrator -if (!command || values.output === '' || values.stack === '') - throw Error('Usage: harness:run --migrator ') +const command = values.migrator ?? '' +const fakeMigration = values.fake +if ( + (!command && !fakeMigration) || + (command && fakeMigration) || + values.output === '' || + values.stack === '' +) + throw Error('Usage: harness:run --migrator | --fake ') const id = randomUUID() const root = resolve(fileURLToPath(new URL('../../..', import.meta.url))) const output = resolve(values.output ?? `.migration-harness-runs/${id}`) @@ -30,8 +37,9 @@ const manifest = { await promisify(execFile)('git', ['rev-parse', 'HEAD'], { cwd: root }) ).stdout.trim(), agents: { - migrator: command, - model: 'command-adapter', + // A deterministic Fake Migrator Run must not read as real agent evidence. + migrator: fakeMigration ? `fake:${fakeMigration}` : command, + model: fakeMigration ? 'fake-migrator' : 'command-adapter', limits: { timeMs: 40 * 60 * 1000, turns: 200 }, allowlist: ['brokered isolated exec'], }, @@ -59,6 +67,7 @@ try { root, output, command, + fakeMigration, isolation, credentials, manifest, diff --git a/e2e/migration-harness/run/fake-migrator.mjs b/e2e/migration-harness/run/fake-migrator.mjs new file mode 100644 index 000000000..850e7ca48 --- /dev/null +++ b/e2e/migration-harness/run/fake-migrator.mjs @@ -0,0 +1,89 @@ +import { cp, readFile, writeFile } from 'node:fs/promises' +import { resolve } from 'node:path' + +/** Harness-owned deterministic command substitute. The hidden reference is never mounted. + * @param {{root:string,workspace:string,kind:string,stack:string}} options + * @returns {Promise} */ +export async function fakeMigrate({ root, workspace, kind, stack }) { + const variants = [ + 'reference', + 'downcase', + 'stale-hmac', + 'bigint', + 'rewrite', + 'missing-log', + 'legacy-ffi', + 'noop', + 'denied-app', + 'denied-lifecycle', + ] + if (!variants.includes(kind)) throw Error('Unknown Fake Migrator') + if (kind !== 'noop') { + await cp( + resolve(root, 'e2e/migration-harness/reference/src'), + resolve(workspace, 'src'), + { recursive: true }, + ) + const path = resolve(workspace, 'src/app.mjs') + let source = await readFile(path, 'utf8') + if (kind === 'denied-app') + source = `import { readFile } from 'node:fs/promises'\n${source}`.replace( + 'export async function createApp(env = process.env) {', + 'export async function createApp(env = process.env) { await readFile("/trace/canary").catch(() => {});', + ) + if (kind === 'downcase') + source = source.replace( + 'emailLower: customer.emailLower.toLowerCase()', + 'emailLower: customer.emailLower', + ) + if (kind === 'bigint') + source = source.replace( + 'balance: String(model.balance)', + 'balance: Number(model.balance)', + ) + if (kind === 'rewrite') + source = source.replace( + /get:\s*async\s*\(?pk\)?\s*=>\s*decrypt\(await stores.get\(pk\)\),/, + 'async get(pk) { const value = await decrypt(await stores.get(pk)); if (pk === "legacy" && value) await this.put(value); return value },', + ) + await writeFile(path, source) + if (kind === 'stale-hmac') { + const stores = resolve(workspace, 'src/stores.mjs') + let text = await readFile(stores, 'utf8') + text = text.replace( + 'SET active__source = :source REMOVE active__hmac', + 'SET active__source = :source', + ) + await writeFile(stores, text) + } + const manifestPath = resolve(workspace, 'package.json') + const manifest = JSON.parse(await readFile(manifestPath, 'utf8')) + for (const group of [ + 'dependencies', + 'devDependencies', + 'optionalDependencies', + ]) { + for (const name of [ + '@cipherstash/protect', + '@cipherstash/protect-dynamodb', + '@cipherstash/schema', + '@cipherstash/protect-ffi', + ]) + delete manifest[group]?.[name] + } + manifest.dependencies['@cipherstash/stack'] = stack + if (kind === 'denied-lifecycle') + manifest.scripts = { + ...manifest.scripts, + install: `node -e "try { require('node:fs').readFileSync('/trace/canary') } catch {}"`, + } + if (kind === 'legacy-ffi') + manifest.dependencies['@cipherstash/protect-ffi'] = '0.18.1' + await writeFile(manifestPath, JSON.stringify(manifest, null, 2)) + // npm must resolve the new graph rather than retain the legacy lock graph. + await writeFile(resolve(workspace, 'package-lock.json'), '{}') + } + if (kind !== 'missing-log') + await writeFile(resolve(workspace, 'friction.json'), '[]') + return { exitCode: 0, stdout: `Fake Migrator: ${kind}`, stderr: '' } +} diff --git a/e2e/migration-harness/run/real-driver.mjs b/e2e/migration-harness/run/real-driver.mjs index 74270bb46..104a1382e 100644 --- a/e2e/migration-harness/run/real-driver.mjs +++ b/e2e/migration-harness/run/real-driver.mjs @@ -5,6 +5,7 @@ import { cp, mkdir, open, readFile, rm, writeFile } from 'node:fs/promises' import { relative, resolve, sep } from 'node:path' import { promisify } from 'node:util' import { behaviour, legacy, mutation } from './behaviour.mjs' +import { fakeMigrate } from './fake-migrator.mjs' import { migrationChecks } from './migration-checks.mjs' import { runMigrator } from './migrator.mjs' import { noRewrite, validateLegacySnapshot } from './no-rewrite.mjs' @@ -13,17 +14,19 @@ import { isSnapshotInfrastructureError, snapshotErrorCode, } from './snapshot-errors.mjs' +import { resolveStack, verifyInstalledTarball } from './stack-source.mjs' import { prepareWorkspace } from './workspace.mjs' /** @typedef {Awaited>} StoreSet * @typedef {Pick, 'prove'|'exec'|'openAppHost'|'stop'>} Isolation */ /** Real application execution has exactly one route: the proven isolation host. - * @param {{id:string,root:string,output:string,command:string,isolation:Isolation,credentials:Record,manifest:import('../types.js').Manifest & {stack:string}}} options */ + * @param {{id:string,root:string,output:string,command:string,fakeMigration?:string,isolation:Isolation,credentials:Record,manifest:import('../types.js').Manifest & {stack:string}}} options */ export function createRealDriver({ id, root, output, command, + fakeMigration, isolation, credentials, manifest, @@ -40,6 +43,8 @@ export function createRealDriver({ let host = null let baseCommit = '' let ffiRange = '' + /** @type {Awaited>|undefined} */ + let stackSource /** @type {unknown} */ let legacySnapshot /** @param {StoreSet|null} stores */ @@ -100,34 +105,10 @@ export function createRealDriver({ if (proof?.verified !== true) throw Error('Isolation capability proof failed') manifest.isolation = proof - const resolved = JSON.parse( - ( - await promisify(execFile)('npm', [ - 'view', - `@cipherstash/stack@${manifest.stack}`, - 'version', - '--json', - ]) - ).stdout, - ) - manifest.stack = Array.isArray(resolved) ? resolved.at(-1) : resolved - if ( - typeof manifest.stack !== 'string' || - !/^\d+\.\d+\.\d+(?:-[\w.-]+)?$/.test(manifest.stack) - ) - throw Error('Could not resolve exact Stack version') - ffiRange = JSON.parse( - ( - await promisify(execFile)('npm', [ - 'view', - `@cipherstash/stack@${manifest.stack}`, - 'dependencies.@cipherstash/protect-ffi', - '--json', - ]) - ).stdout, - ) - if (typeof ffiRange !== 'string' || !ffiRange) - throw Error('Could not resolve Stack FFI dependency') + stackSource = await resolveStack({ source: manifest.stack, root, output }) + manifest.stack = stackSource.version + manifest.stackSource = stackSource + ffiRange = stackSource.ffiRange const guide = resolve(root, 'docs/migration/protect-dynamodb-to-stack.md') const content = await readFile(guide, 'utf8') manifest.guideHash = createHash('sha256').update(content).digest('hex') @@ -217,22 +198,34 @@ export function createRealDriver({ ) }, async migrate() { - const result = await runMigrator({ - command, - timeoutMs: manifest.agents?.limits?.timeMs, - maxToolCalls: manifest.agents?.limits?.toolCalls, - maxModelTurns: manifest.agents?.limits?.turns, - stop: () => isolation.stop({ expected: true }), - execute: (argv, timeoutMs) => - isolation.exec({ - actor: 'migrator', + if (!stackSource) throw Error('Stack source not resolved') + if (stackSource.tarball) + await cp(stackSource.tarball, resolve(workspace, 'resolved-stack.tgz')) + const result = fakeMigration + ? await fakeMigrate({ + root, workspace, - env: env(migratorStores), - command: argv, - forwards: forwards(migratorStores), - timeoutMs, - }), - }) + kind: fakeMigration, + stack: stackSource.install, + }) + : await runMigrator({ + command, + timeoutMs: manifest.agents?.limits?.timeMs, + maxToolCalls: manifest.agents?.limits?.toolCalls, + maxModelTurns: manifest.agents?.limits?.turns, + stop: () => isolation.stop({ expected: true }), + execute: (argv, timeoutMs) => + isolation.exec({ + actor: 'migrator', + workspace, + env: env(migratorStores), + command: argv, + forwards: forwards(migratorStores), + timeoutMs, + }), + }) + if (fakeMigration && fakeMigration !== 'noop') + await execute('migrator', workspace, migratorStores, ['npm', 'install']) await writeFile( resolve(output, 'migrator-transcript.log'), `${result.stdout ?? ''}\n${result.stderr ?? ''}`, @@ -315,12 +308,25 @@ export function createRealDriver({ .split(sep) .some((part) => ['node_modules', '.git'].includes(part)), }) + // The Migrator controls its copy of the tarball. Supply the App Host with + // the host-resolved artifact, removing first so a planted symlink cannot + // redirect the write, and confirm npm installed exactly that artifact. + const tarball = stackSource?.tarball + const installed = resolve(oracleWorkspace, 'resolved-stack.tgz') + if (tarball) { + await rm(installed, { force: true }) + await cp(tarball, installed) + } await execute('app-host', oracleWorkspace, oracleStores, [ 'npm', 'install', '--save-exact', - `@cipherstash/stack@${manifest.stack}`, + tarball + ? 'file:./resolved-stack.tgz' + : `@cipherstash/stack@${manifest.stack}`, ]) + if (tarball) + await verifyInstalledTarball({ workspace: oracleWorkspace, tarball }) host = await isolation.openAppHost({ actor: 'app-host', workspace: oracleWorkspace, @@ -332,8 +338,25 @@ export function createRealDriver({ const application = host if (!application) throw Error('App Host has not started') try { - const result = await behaviour((method, args) => - application.call(method, args), + /** @type {{snapshot: unknown} | {cause: string}} */ + let mutationRead = { snapshot: undefined } + const result = await behaviour( + (method, args) => application.call(method, args), + async () => { + mutationRead = await read(mutation.pk) + if ('cause' in mutationRead) + throw Object.assign(new Error(mutationRead.cause), { + infrastructure: false, + checks: [ + ...checks, + { + id: 'migration.active-hmac-absent', + passed: false, + cause: mutationRead.cause, + }, + ], + }) + }, ) const checks = result.checks /** A store outage stays infrastructure and carries the checks already @@ -361,14 +384,7 @@ export function createRealDriver({ ? { id: 'no-rewrite.legacy-bytes', passed: false, ...current } : noRewrite(legacySnapshot, current.snapshot), ) - // Inspect setActive before later full writes can repair stale HMACs. - /** @type {{snapshot: unknown} | {cause: string}} */ - let mutationRead = { snapshot: undefined } - await result.mutate({ - afterSetActive: async () => { - mutationRead = await read(mutation.pk) - }, - }) + await result.mutate() const migration = await migrationChecks({ workspace, installedWorkspace: oracleWorkspace, diff --git a/e2e/migration-harness/run/stack-source.mjs b/e2e/migration-harness/run/stack-source.mjs new file mode 100644 index 000000000..cc234bb07 --- /dev/null +++ b/e2e/migration-harness/run/stack-source.mjs @@ -0,0 +1,102 @@ +import { execFile } from 'node:child_process' +import { createHash } from 'node:crypto' +import { access, cp, lstat, readFile } from 'node:fs/promises' +import { resolve } from 'node:path' +import { promisify } from 'node:util' + +/** Resolve once per invocation; Iteration/Loop callers reuse this value. + * @param {{source:string,root:string,output:string}} options */ +export async function resolveStack({ source, root, output }) { + const exec = promisify(execFile) + if (source === 'tree' || source.endsWith('.tgz')) { + let tarball = resolve(source) + if (source === 'tree') { + const directory = resolve(root, 'languages/typescript/packages/stack') + await access(resolve(directory, 'dist/index.js')) + tarball = resolve(output, 'tree-stack.tgz') + await exec('pnpm', ['pack', '--out', tarball], { cwd: directory }) + } + const digest = createHash('sha256') + .update(await readFile(tarball)) + .digest('hex') + const result = await exec('tar', ['-xOf', tarball, 'package/package.json']) + const manifest = JSON.parse(result.stdout) + if ( + manifest.name !== '@cipherstash/stack' || + typeof manifest.version !== 'string' + ) + throw Error('Tarball is not @cipherstash/stack') + if (typeof manifest.dependencies?.['@cipherstash/protect-ffi'] !== 'string') + throw Error('Tarball has no FFI dependency') + const destination = resolve(output, 'resolved-stack.tgz') + if (tarball !== destination) await cp(tarball, destination) + return { + version: manifest.version, + ffiRange: manifest.dependencies?.['@cipherstash/protect-ffi'], + digest, + tarball: destination, + install: 'file:./resolved-stack.tgz', + } + } + const result = await exec('npm', [ + 'view', + `@cipherstash/stack@${source}`, + 'version', + '--json', + ]) + const resolved = JSON.parse(result.stdout) + const version = Array.isArray(resolved) ? resolved.at(-1) : resolved + if ( + typeof version !== 'string' || + !/^\d+\.\d+\.\d+(?:-[\w.-]+)?$/.test(version) + ) + throw Error('Could not resolve exact Stack version') + const ffiAnswer = JSON.parse( + ( + await exec('npm', [ + 'view', + `@cipherstash/stack@${version}`, + 'dependencies.@cipherstash/protect-ffi', + '--json', + ]) + ).stdout, + ) + // npm 12 wraps even an exact version's field in an array; older npm does not. + const ffiRange = Array.isArray(ffiAnswer) + ? ffiAnswer.length === 1 + ? ffiAnswer[0] + : undefined + : ffiAnswer + if (typeof ffiRange !== 'string' || !ffiRange) + throw Error('Could not resolve Stack FFI dependency') + return { version, install: version, ffiRange } +} + +/** The Subresource Integrity string npm records for a tarball. + * @param {string} tarball */ +async function integrity(tarball) { + const digest = createHash('sha512') + .update(await readFile(tarball)) + .digest('base64') + return `sha512-${digest}` +} + +/** Confirm the App Host lockfile records the host-resolved tarball for Stack. + * The workspace is actor-written, so its lockfile must be a bounded regular file. + * @param {{workspace:string,tarball:string}} options */ +export async function verifyInstalledTarball({ workspace, tarball }) { + const path = resolve(workspace, 'package-lock.json') + const stat = await lstat(path) + if (!stat.isFile() || stat.size > 8 * 1024 * 1024) + throw Error('Installed Stack is not the resolved tarball') + let entry + try { + entry = JSON.parse(await readFile(path, 'utf8'))?.packages?.[ + 'node_modules/@cipherstash/stack' + ] + } catch { + throw Error('Installed Stack is not the resolved tarball') + } + if (entry?.integrity !== (await integrity(tarball))) + throw Error('Installed Stack is not the resolved tarball') +} diff --git a/e2e/vitest.harness.config.ts b/e2e/vitest.harness.config.ts index 110e88be6..78f5e3fc0 100644 --- a/e2e/vitest.harness.config.ts +++ b/e2e/vitest.harness.config.ts @@ -2,6 +2,9 @@ import { defineConfig } from 'vitest/config' export default defineConfig({ test: { + // Real Runs provision disposable stores; fail promptly rather than spend + // minutes on later variants when the reference or infrastructure is broken. + bail: 1, include: ['migration-harness/__tests__/**/*.test.ts'], exclude: ['**/node_modules/**'], },