diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 97cbf4c0c..e2fbe903f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -503,6 +503,18 @@ jobs: env: DATABASE_URL: ${{ steps.db.outputs.database-url }} + # Deterministic Fake Migrators only: no model credentials or token spend. + # Only the live file reads HARNESS_LIVE; migration-harness.yml runs the + # rest of the self-test on harness changes. Independent of the steps + # above: the harness installs @cipherstash/stack from npm inside its + # isolated containers and never loads this job's native bindings, so an + # unrelated failure earlier in the job must not hide its result. + - name: Run the credentialed Migration Harness self-test + if: ${{ !cancelled() }} + run: pnpm exec turbo run harness:selftest --filter @cipherstash/e2e --env-mode=loose -- migration-harness/__tests__/no-rewrite-live.test.ts + env: + HARNESS_LIVE: '1' + # Guarded on the project being set: if the stack never came up there is # nothing to tear down, and an unguarded `-p ""` would mask the real error. - name: Stop postgres diff --git a/e2e/migration-harness/__tests__/no-rewrite-live.test.ts b/e2e/migration-harness/__tests__/no-rewrite-live.test.ts new file mode 100644 index 000000000..0eace880b --- /dev/null +++ b/e2e/migration-harness/__tests__/no-rewrite-live.test.ts @@ -0,0 +1,86 @@ +import 'dotenv/config' +import assert from 'node:assert/strict' +import { randomUUID } from 'node:crypto' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { test } from 'vitest' +import { createIsolation } from '../isolation.mjs' +import { createRealDriver } from '../run/real-driver.mjs' +import { run } from '../run/run.mjs' + +const keys = [ + 'CS_WORKSPACE_CRN', + 'CS_CLIENT_ID', + 'CS_CLIENT_KEY', + 'CS_CLIENT_ACCESS_KEY', +] +// HARNESS_LIVE=1 asks for these Runs, so a missing credential fails them +// rather than skipping them silently. +const enabled = process.env.HARNESS_LIVE === '1' +const missing = keys.filter((key) => !process.env[key]) +for (const rewrite of [false, true]) { + test.skipIf(!enabled)( + `real Run ${rewrite ? 'detects a read rewrite' : 'keeps Legacy Data unchanged through mutations'}`, + async () => { + assert.deepEqual( + missing, + [], + `HARNESS_LIVE=1 requires ${keys.join(', ')}`, + ) + const output = await mkdtemp(resolve(tmpdir(), 'mh-no-rewrite-')) + const isolation = createIsolation() + const manifest = { stack: '^1.2.1', guideHash: null } + const replacement = + 'async get(pk) { if (pk === "legacy") { const customer = await decrypt(await stores.get(pk)); await this.put(customer); }' + const script = `import {readFile,writeFile} from "node:fs/promises"; const p="src/app.mjs"; let s=await readFile(p,"utf8"); s=s.replace("async get(pk) {", ${JSON.stringify(replacement)}); await writeFile(p,s); await writeFile("friction.json","[]")` + const tool = rewrite + ? `node --input-type=module -e '${script}'` + : `printf '[]' > friction.json` + const adapter = resolve(output, 'adapter.mjs') + await (await import('node:fs/promises')).writeFile( + adapter, + ` + import {createInterface} from 'node:readline'; + const lines=createInterface({input:process.stdin}); + console.log(JSON.stringify({id:'fake',method:'exec',command:['sh','-c',${JSON.stringify(tool)}]})); + for await(const line of lines){ const response=JSON.parse(line); process.exitCode=response.result.exitCode; break; } + lines.close(); + `, + ) + const command = `node '${adapter}'` + const driver = createRealDriver({ + // Store names use the first eight characters, so they must vary per Run. + id: randomUUID(), + root: fileURLToPath(new URL('../../..', import.meta.url)), + output, + command, + isolation, + credentials: Object.fromEntries( + keys.map((key) => [key, process.env[key] ?? '']), + ), + manifest, + }) + try { + const report = await run({ + id: 'live-no-rewrite', + manifest, + driver, + supervision: isolation, + }) + const check = report.evidence.find( + (e) => e.checkId === 'no-rewrite.legacy-bytes', + ) + assert.ok(check, JSON.stringify(report)) + assert.equal(Object(check.payload).passed, !rewrite) + if (rewrite) + assert.match(JSON.stringify(check.payload), /legacy.*__source/) + } finally { + await driver.close() + await rm(output, { recursive: true, force: true }) + } + }, + 600_000, + ) +} diff --git a/e2e/migration-harness/__tests__/no-rewrite.test.ts b/e2e/migration-harness/__tests__/no-rewrite.test.ts new file mode 100644 index 000000000..2397d1f0d --- /dev/null +++ b/e2e/migration-harness/__tests__/no-rewrite.test.ts @@ -0,0 +1,102 @@ +import assert from 'node:assert/strict' +import { test } from 'vitest' +import { requiredOracleChecks } from '../run/migration-checks.mjs' +import { noRewrite } from '../run/no-rewrite.mjs' +import { run } from '../run/run.mjs' + +const baseline = { + dynamo: { + Item: { + pk: { S: 'legacy' }, + email__source: { S: 'cipher' }, + email__hmac: { S: 'term' }, + }, + }, + postgres: [ + { + pk: 'legacy', + email: { c: 'pg-cipher', hm: 'pg-term' }, + notes: { c: 'note' }, + }, + ], +} +for (const changed of [false, true]) { + test(`Run records ${changed ? 'rewritten' : 'unchanged'} Legacy Data`, async () => { + const current = structuredClone(baseline) + if (changed) current.dynamo.Item.email__source.S = 'rewritten' + const report = await run({ + id: 'rewrite', + manifest: { guideHash: 'guide' }, + driver: { + provision: async () => {}, + installUnmigrated: async () => {}, + initializeLegacyData: async () => {}, + snapshot: async () => {}, + migrate: async () => ({ exitCode: 0 }), + friction: async () => [], + installAppHost: async () => {}, + behaviour: async () => [ + noRewrite(baseline, current), + ...requiredOracleChecks + .filter((id) => id !== 'no-rewrite.legacy-bytes') + .map((id) => ({ id, passed: true })), + ], + }, + }) + assert.equal(report.outcome, changed ? 'behaviour' : 'pass') + const check = report.evidence.find( + (e) => e.checkId === 'no-rewrite.legacy-bytes', + ) + assert.ok(check) + if (changed) + assert.match( + JSON.stringify(check.payload), + /dynamo.*legacy.*email__source/, + ) + }) +} + +test('Run rejects a rewritten duplicate Legacy Data row even when the original is last', async () => { + const current = structuredClone(baseline) + const rewritten = structuredClone(current.postgres[0]) + rewritten.email.c = 'rewritten' + current.postgres.unshift(rewritten) + const report = await run({ + id: 'duplicate-legacy', + manifest: { guideHash: 'guide' }, + driver: { + provision: async () => {}, + installUnmigrated: async () => {}, + initializeLegacyData: async () => {}, + snapshot: async () => {}, + migrate: async () => ({ exitCode: 0 }), + friction: async () => [], + installAppHost: async () => {}, + behaviour: async () => [ + noRewrite(baseline, current), + ...requiredOracleChecks + .filter((id) => id !== 'no-rewrite.legacy-bytes') + .map((id) => ({ id, passed: true })), + ], + }, + }) + assert.equal(report.outcome, 'behaviour') + assert.equal(report.editorEligible, true) + const check = report.evidence.find( + (e) => e.checkId === 'no-rewrite.legacy-bytes', + ) + assert.ok(check, 'a malformed current snapshot still reports the check') + assert.equal(Object(check.payload).passed, false) +}) + +test('a deleted Legacy Data row is a failed check, while a malformed baseline still throws', () => { + const current = structuredClone(baseline) + current.postgres = [] + const check = noRewrite(baseline, current) + assert.equal(check.id, 'no-rewrite.legacy-bytes') + assert.equal(check.passed, false) + assert.throws( + () => noRewrite(current, baseline), + /exactly one Legacy Data Postgres row/, + ) +}) diff --git a/e2e/migration-harness/__tests__/review-driver.test.ts b/e2e/migration-harness/__tests__/review-driver.test.ts index 1aa99f377..83c17bc5c 100644 --- a/e2e/migration-harness/__tests__/review-driver.test.ts +++ b/e2e/migration-harness/__tests__/review-driver.test.ts @@ -6,17 +6,38 @@ import { fileURLToPath } from 'node:url' import { promisify } from 'node:util' import { afterEach, beforeEach, expect, test, vi } from 'vitest' import { legacy } from '../run/behaviour.mjs' +import { dynamoColumns } from '../run/legacy-columns.mjs' import { migrationChecks, noRewriteCheck } from '../run/migration-checks.mjs' import { createRealDriver } from '../run/real-driver.mjs' import { run } from '../run/run.mjs' +import { dynamoSnapshotError } from '../run/snapshot-errors.mjs' import type { IsolationOptions } from '../types.js' const observed = vi.hoisted(() => ({ dispose: vi.fn(async () => {}), - snapshot: vi.fn(async () => ({ - dynamo: { Item: { email__source: { S: 'original' } } }, - postgres: [{ pk: 'legacy', email: { c: 'original', hm: 'term' } }], - })), + snapshot: vi.fn( + async (): Promise => ({ + dynamo: { + Item: { + pk: { S: 'legacy' }, + ...Object.fromEntries( + dynamoColumns.flatMap((key) => [ + [`${key}__source`, { S: 'original' }], + [`${key}__hmac`, { S: 'term' }], + ]), + ), + }, + }, + postgres: [ + { + pk: 'legacy', + email: { c: 'original', hm: 'term' }, + age: { c: 'original', hm: 'term' }, + notes: { c: 'original' }, + }, + ], + }), + ), })) vi.mock('../run/real-stores.mjs', () => ({ @@ -273,6 +294,191 @@ test('F5 checks legacy ciphertext after reads and before mutations', async () => expect(mutationBeforeSnapshot).toBe(false) }) +test.each([ + [ + 'dropped table', + Object.assign(Error('missing table'), { code: '42P01' }), + 'behaviour', + ], + [ + 'renamed column', + Object.assign(Error('missing column'), { code: '42703' }), + 'behaviour', + ], + [ + 'invalid payload cast', + Object.assign(Error('invalid cast'), { code: '42846' }), + 'behaviour', + ], + ['deleted DynamoDB table', dynamoSnapshotError(400), 'behaviour'], + ['DynamoDB service unavailable', dynamoSnapshotError(503), 'infrastructure'], + ['DynamoDB throttling', dynamoSnapshotError(429), 'infrastructure'], + [ + 'unknown snapshot failure', + Error('unrecognized snapshot failure'), + 'behaviour', + ], + [ + 'refused connection', + Object.assign(Error('connection refused'), { code: 'ECONNREFUSED' }), + 'infrastructure', + ], + [ + 'reset connection', + Object.assign(Error('connection reset'), { code: 'ECONNRESET' }), + 'infrastructure', + ], + [ + 'connection timeout', + Object.assign(Error('connection timed out'), { code: 'ETIMEDOUT' }), + 'infrastructure', + ], + [ + 'broken socket pipe', + Object.assign(Error('broken pipe'), { code: 'EPIPE' }), + 'infrastructure', + ], + [ + 'Postgres.js closed connection', + Object.assign(Error('connection closed'), { code: 'CONNECTION_CLOSED' }), + 'infrastructure', + ], + [ + 'Postgres connection loss', + Object.assign(Error('connection failure'), { code: '08006' }), + 'infrastructure', + ], + [ + 'Postgres shutdown', + Object.assign(Error('admin shutdown'), { code: '57P01' }), + 'infrastructure', + ], + [ + 'fetch transport failure', + new TypeError('fetch failed', { + cause: Object.assign(Error('socket closed'), { code: 'UND_ERR_SOCKET' }), + }), + 'infrastructure', + ], +] as const)( + 'post-read snapshot classifies %s', + async (_scenario, error, outcome) => { + const { driver, calls } = await setup() + const baseline = await observed.snapshot() + observed.snapshot + .mockResolvedValueOnce(baseline) + .mockRejectedValueOnce(error) + try { + const report = await run({ + id: 'post-read-snapshot', + manifest: { guideHash: 'guide' }, + driver: { + ...driver, + provision: async () => {}, + migrate: async () => ({ exitCode: 0 }), + friction: async () => [], + }, + }) + expect(report.outcome).toBe(outcome) + expect(report.editorEligible).toBe(outcome === 'behaviour') + expect(calls).toContain('findPostgresAge') + expect( + report.stages.find((stage) => stage.name === 'behaviour')?.status, + ).toBe('failed') + } finally { + await driver.close() + } + }, +) + +async function runReal(driver: Awaited>['driver']) { + try { + return await run({ + id: 'snapshot-evidence', + manifest: { guideHash: 'guide' }, + driver: { + ...driver, + provision: async () => {}, + migrate: async () => ({ exitCode: 0 }), + friction: async () => [], + }, + }) + } finally { + await driver.close() + } +} +function oracleChecks(report: Awaited>) { + return report.evidence + .filter((item) => item.kind === 'oracle-check') + .map((item) => Object(item.payload)) +} + +test('a post-read schema error records a failed no-rewrite check and keeps the read checks', async () => { + const { driver } = await setup() + const baseline = await observed.snapshot() + observed.snapshot + .mockResolvedValueOnce(baseline) + .mockRejectedValueOnce( + Object.assign(Error('column "email" does not exist'), { code: '42703' }), + ) + const report = await runReal(driver) + expect(report.outcome).toBe('behaviour') + const checks = oracleChecks(report) + expect(checks).toContainEqual( + expect.objectContaining({ + id: 'no-rewrite.legacy-bytes', + passed: false, + cause: expect.stringContaining('42703'), + }), + ) + expect(checks).toContainEqual( + expect.objectContaining({ id: 'behaviour.legacy-findPostgresAge' }), + ) +}) + +test('a reset connection on the Mutation Data snapshot is infrastructure and keeps completed checks', async () => { + const { driver } = await setup() + const baseline = await observed.snapshot() + observed.snapshot + .mockResolvedValueOnce(baseline) + .mockResolvedValueOnce(baseline) + .mockRejectedValueOnce( + Object.assign(Error('connection reset'), { code: 'ECONNRESET' }), + ) + const report = await runReal(driver) + expect(report.outcome).toBe('infrastructure') + expect(report.editorEligible).toBe(false) + expect(oracleChecks(report)).toContainEqual( + expect.objectContaining({ id: 'no-rewrite.legacy-bytes', passed: true }), + ) +}) + +test('a schema error on the Mutation Data snapshot fails the active-HMAC check with its cause', async () => { + const { driver } = await setup() + const baseline = await observed.snapshot() + observed.snapshot + .mockResolvedValueOnce(baseline) + .mockResolvedValueOnce(baseline) + .mockRejectedValueOnce( + Object.assign(Error('relation "customers" does not exist'), { + code: '42P01', + }), + ) + const report = await runReal(driver) + expect(report.outcome).toBe('behaviour') + const checks = oracleChecks(report) + expect(checks).toContainEqual( + expect.objectContaining({ + id: 'migration.active-hmac-absent', + passed: false, + cause: expect.stringContaining('42P01'), + }), + ) + expect(checks).toContainEqual( + expect.objectContaining({ id: 'no-rewrite.legacy-bytes', passed: true }), + ) +}) + test('inspects the stored active field before later full puts can overwrite it', async () => { const { driver, calls } = await setup() await driver.snapshot() diff --git a/e2e/migration-harness/__tests__/review-stores.test.ts b/e2e/migration-harness/__tests__/review-stores.test.ts index 352eccf76..067e0b7a8 100644 --- a/e2e/migration-harness/__tests__/review-stores.test.ts +++ b/e2e/migration-harness/__tests__/review-stores.test.ts @@ -12,6 +12,10 @@ vi.mock('node:child_process', () => ({ })) vi.mock('postgres', () => ({ default: connect })) +import { + postgresColumns, + postgresSnapshotQuery, +} from '../run/legacy-columns.mjs' import { createStoreSet, disposeStores } from '../run/real-stores.mjs' let dockerCalls: string[][] @@ -71,8 +75,8 @@ beforeEach(() => { return [] } return Object.assign(query, { - unsafe: async (text: string) => { - statements.push({ database, text, values: [] }) + unsafe: async (text: string, values: unknown[] = []) => { + statements.push({ database, text, values }) if (cleanupFailure && text.startsWith(cleanupFailure)) throw Error(`${cleanupFailure} failed`) return [] @@ -165,9 +169,10 @@ test('the Oracle snapshot reads the selected record from both assigned stores', expect(JSON.parse(requests[0]).Key).toEqual({ pk: { S: 'mutation' } }) expect( - statements.find((query) => query.text.includes('SELECT pk, email::jsonb')) - ?.values, + statements.find((query) => query.text === postgresSnapshotQuery)?.values, ).toEqual(['mutation']) + for (const column of postgresColumns) + expect(postgresSnapshotQuery).toContain(`"${column}"::jsonb`) } finally { await disposeStores(resources) } diff --git a/e2e/migration-harness/run/legacy-columns.mjs b/e2e/migration-harness/run/legacy-columns.mjs new file mode 100644 index 000000000..35651db79 --- /dev/null +++ b/e2e/migration-harness/run/legacy-columns.mjs @@ -0,0 +1,24 @@ +/** The Fixture App's Legacy Data columns, as the harness checks them. The + * harness never imports app code, so `fixture/src/stores.mjs` keeps its own + * copy; every harness site reads this one. + * `notes` is free text with no lookup, so it has no HMAC term. */ +export const dynamoColumns = /** @type {const} */ ([ + 'email', + 'emailLower', + 'age', + 'balance', + 'active', + 'notes', +]) +export const postgresColumns = /** @type {const} */ (['email', 'age', 'notes']) + +/** @param {string} column */ +export function hasHmac(column) { + return column !== 'notes' +} + +/** The Oracle's Legacy Data read. Column names are the constants above; only + * the key is a parameter. */ +export const postgresSnapshotQuery = `SELECT pk, ${postgresColumns + .map((column) => `"${column}"::jsonb`) + .join(', ')} FROM customers WHERE pk = $1` diff --git a/e2e/migration-harness/run/migration-checks.mjs b/e2e/migration-harness/run/migration-checks.mjs index f6c5980de..46d96d1ab 100644 --- a/e2e/migration-harness/run/migration-checks.mjs +++ b/e2e/migration-harness/run/migration-checks.mjs @@ -1,8 +1,9 @@ import { lstat, readdir, readFile } from 'node:fs/promises' import { join } from 'node:path' -import { isDeepStrictEqual } from 'node:util' import { satisfies, valid } from 'semver' import ts from 'typescript' +import { noRewrite } from './no-rewrite.mjs' +import { object } from './record.mjs' export const requiredOracleChecks = [ 'no-rewrite.legacy-bytes', @@ -15,57 +16,11 @@ const legacyPackage = /@cipherstash\/(?:protect-dynamodb|protect|schema)(?![\w-])/ const hostSetting = /^(CS_CTS_HOST|CS_ZEROKMS_HOST)$/ -/** @param {unknown} value @returns {Record} */ -function object(value) { - if (!value || typeof value !== 'object' || Array.isArray(value)) - throw Error('Expected a stored record or package object') - return Object.fromEntries(Object.entries(value)) -} - -/** Project only the encrypted bytes and equality terms covered by the ADR. - * Metadata and unrelated stored fields are not part of the no-rewrite contract. - * @param {unknown} snapshot */ -export function legacyBytes(snapshot) { - const value = object(snapshot) - const item = object(object(value.dynamo).Item) - const dynamo = Object.fromEntries( - Object.entries(item).filter(([key]) => /__(source|hmac)$/.test(key)), - ) - if (!Object.keys(dynamo).some((key) => key.endsWith('__source'))) - throw Error('Legacy DynamoDB ciphertext is missing') - if (!Array.isArray(value.postgres) || value.postgres.length !== 1) - throw Error('Legacy Postgres record is missing') - const row = object(value.postgres[0]) - const postgres = Object.fromEntries( - Object.entries(row).flatMap(([key, payload]) => { - if (key === 'pk') return [] - const envelope = object(payload) - if (typeof envelope.c !== 'string') - throw Error('Legacy Postgres ciphertext is missing') - return [ - [ - key, - { c: envelope.c, ...('hm' in envelope ? { hm: envelope.hm } : {}) }, - ], - ] - }), - ) - if (!Object.keys(postgres).length) - throw Error('Legacy Postgres ciphertext is missing') - return { dynamo, postgres } -} - -/** @param {unknown} before @param {unknown} after @returns {import('../types.js').OracleCheck} */ +/** Compatibility entry point for check-level callers. + * @param {unknown} before @param {unknown} after @returns {import('../types.js').OracleCheck} */ export function noRewriteCheck(before, after) { try { - const passed = isDeepStrictEqual(legacyBytes(before), legacyBytes(after)) - return { - id: 'no-rewrite.legacy-bytes', - passed, - ...(!passed - ? { cause: 'Legacy ciphertext or equality terms changed during reads' } - : {}), - } + return noRewrite(before, after) } catch { return { id: 'no-rewrite.legacy-bytes', diff --git a/e2e/migration-harness/run/no-rewrite.mjs b/e2e/migration-harness/run/no-rewrite.mjs new file mode 100644 index 000000000..d543858fb --- /dev/null +++ b/e2e/migration-harness/run/no-rewrite.mjs @@ -0,0 +1,95 @@ +import { isDeepStrictEqual } from 'node:util' +import { dynamoColumns, hasHmac, postgresColumns } from './legacy-columns.mjs' +import { object } from './record.mjs' + +/** Compare stored bytes through admin snapshots, without exporting their values. + * @param {unknown} before @param {unknown} after + * @returns {import('../types.js').OracleCheck} */ +export function noRewrite(before, after) { + /** @param {unknown} snapshot */ + function fields(snapshot, baseline = false) { + /** @type {Map} */ + const result = new Map() + const data = object(snapshot) + const item = object(object(data.dynamo ?? {}).Item ?? {}) + if (!Array.isArray(data.postgres) || data.postgres.length !== 1) + throw Error('Expected exactly one Legacy Data Postgres row') + for (const [key, value] of Object.entries(item)) + if (/__(source|hmac)$/.test(key)) + result.set( + `dynamo/${object(item.pk ?? {}).S ?? 'legacy'}/${key}`, + value, + ) + for (const raw of data.postgres) { + const row = object(raw) + for (const [column, payload] of Object.entries(row)) { + if (column === 'pk' || !payload || typeof payload !== 'object') continue + const attributes = object(payload) + for (const attribute of ['c', 'hm']) + if (attribute in attributes) + result.set( + `postgres/${row.pk}/${column}.${attribute}`, + attributes[attribute], + ) + } + } + if (baseline && !result.size) + throw Error('Legacy Data snapshot contains no encrypted attributes') + return result + } + // A malformed baseline is infrastructure; a malformed current snapshot is + // app damage, so it fails this check rather than hiding it from the report. + const expected = fields(before, true) + /** @type {Map} */ + let actual + try { + actual = fields(after) + } catch { + return { + id: 'no-rewrite.legacy-bytes', + passed: false, + cause: 'Current Legacy Data snapshot does not match the baseline shape', + } + } + const changed = [...new Set([...expected.keys(), ...actual.keys()])].filter( + (key) => + !expected.has(key) || + !actual.has(key) || + !isDeepStrictEqual(expected.get(key), actual.get(key)), + ) + return { + id: 'no-rewrite.legacy-bytes', + passed: changed.length === 0, + ...(changed.length + ? { cause: `Changed Legacy Data attributes: ${changed.join(', ')}` } + : {}), + } +} + +/** Validate the known Fixture App baseline before any Migrator invocation. + * @param {unknown} snapshot */ +export function validateLegacySnapshot(snapshot) { + const data = object(snapshot) + const item = object(object(data.dynamo).Item) + if (!Array.isArray(data.postgres) || data.postgres.length !== 1) + throw Error('Expected exactly one Legacy Data Postgres row') + const row = object(data.postgres[0]) + if (object(item.pk).S !== 'legacy' || row.pk !== 'legacy') + throw Error('Missing Legacy Data record') + for (const column of dynamoColumns) { + if ( + typeof object(item[`${column}__source`] ?? {}).S !== 'string' || + (hasHmac(column) && + typeof object(item[`${column}__hmac`] ?? {}).S !== 'string') + ) + throw Error(`Incomplete Legacy Data DynamoDB column: ${column}`) + } + for (const column of postgresColumns) { + const attributes = object(row[column] ?? {}) + if ( + typeof attributes.c !== 'string' || + (hasHmac(column) && typeof attributes.hm !== 'string') + ) + throw Error(`Incomplete Legacy Data Postgres column: ${column}`) + } +} diff --git a/e2e/migration-harness/run/real-driver.mjs b/e2e/migration-harness/run/real-driver.mjs index 8eda370b3..74270bb46 100644 --- a/e2e/migration-harness/run/real-driver.mjs +++ b/e2e/migration-harness/run/real-driver.mjs @@ -5,13 +5,14 @@ import { cp, mkdir, open, readFile, rm, writeFile } from 'node:fs/promises' import { relative, resolve, sep } from 'node:path' import { promisify } from 'node:util' import { behaviour, legacy, mutation } from './behaviour.mjs' -import { - legacyBytes, - migrationChecks, - noRewriteCheck, -} from './migration-checks.mjs' +import { migrationChecks } from './migration-checks.mjs' import { runMigrator } from './migrator.mjs' +import { noRewrite, validateLegacySnapshot } from './no-rewrite.mjs' import { createStoreSet, disposeStores } from './real-stores.mjs' +import { + isSnapshotInfrastructureError, + snapshotErrorCode, +} from './snapshot-errors.mjs' import { prepareWorkspace } from './workspace.mjs' /** @typedef {Awaited>} StoreSet @@ -209,7 +210,7 @@ export function createRealDriver({ }, async snapshot() { legacySnapshot = await assigned(oracleStores).snapshot() - legacyBytes(legacySnapshot) + validateLegacySnapshot(legacySnapshot) await writeFile( resolve(output, 'legacy-snapshot.json'), JSON.stringify(legacySnapshot), @@ -334,31 +335,55 @@ export function createRealDriver({ const result = await behaviour((method, args) => application.call(method, args), ) - const comparison = noRewriteCheck( - legacySnapshot, - await assigned(oracleStores).snapshot(), + const checks = result.checks + /** A store outage stays infrastructure and carries the checks already + * completed; app-caused damage (a dropped table, a renamed column) + * becomes a failed check, so the Editor sees what broke. + * @param {string} [pk] + * @returns {Promise<{snapshot: unknown} | {cause: string}>} */ + async function read(pk) { + try { + return { snapshot: await assigned(oracleStores).snapshot(pk) } + } catch (error) { + if (isSnapshotInfrastructureError(error)) + throw Object.assign( + new Error('Oracle snapshot read failed', { cause: error }), + { infrastructure: true, checks: [...checks] }, + ) + return { + cause: `Oracle snapshot read failed: ${snapshotErrorCode(error)}`, + } + } + } + const current = await read() + checks.push( + 'cause' in current + ? { id: 'no-rewrite.legacy-bytes', passed: false, ...current } + : noRewrite(legacySnapshot, current.snapshot), ) - result.checks.push(comparison) - // Inspect the stored active field straight after setActive: the copy - // checks then put the full record twice, which would replace a stale - // legacy HMAC left by a broken setActive before it could be seen. - /** @type {unknown} */ - let mutationSnapshot - const checks = await result.mutate({ + // Inspect setActive before later full writes can repair stale HMACs. + /** @type {{snapshot: unknown} | {cause: string}} */ + let mutationRead = { snapshot: undefined } + await result.mutate({ afterSetActive: async () => { - mutationSnapshot = await assigned(oracleStores).snapshot( - mutation.pk, - ) + mutationRead = await read(mutation.pk) }, }) + const migration = await migrationChecks({ + workspace, + installedWorkspace: oracleWorkspace, + stackVersion: manifest.stack, + ffiRange, + mutationSnapshot: + 'snapshot' in mutationRead ? mutationRead.snapshot : undefined, + }) checks.push( - ...(await migrationChecks({ - workspace, - installedWorkspace: oracleWorkspace, - stackVersion: manifest.stack, - ffiRange, - mutationSnapshot, - })), + ...migration.map((check) => + 'cause' in mutationRead && + check.id === 'migration.active-hmac-absent' + ? { ...check, passed: false, cause: mutationRead.cause } + : check, + ), ) return checks } finally { diff --git a/e2e/migration-harness/run/real-stores.mjs b/e2e/migration-harness/run/real-stores.mjs index 99bcd4f52..072609e57 100644 --- a/e2e/migration-harness/run/real-stores.mjs +++ b/e2e/migration-harness/run/real-stores.mjs @@ -3,6 +3,8 @@ import { randomBytes } from 'node:crypto' import { readFile } from 'node:fs/promises' import { promisify } from 'node:util' import postgres from 'postgres' +import { postgresSnapshotQuery } from './legacy-columns.mjs' +import { dynamoSnapshotError } from './snapshot-errors.mjs' import { provisionPostgres } from './stores.mjs' const exec = promisify(execFile) @@ -329,12 +331,10 @@ export async function createStoreSet({ runId, actor, eqlBundle, resources }) { ConsistentRead: true, }), }) - if (!response.ok) - throw Error(`Dynamo snapshot failed: ${response.status}`) + if (!response.ok) throw dynamoSnapshotError(response.status) return { dynamo: await response.json(), - postgres: - await snapshotSql`SELECT pk, email::jsonb, age::jsonb, notes::jsonb FROM customers WHERE pk = ${pk}`, + postgres: await snapshotSql.unsafe(postgresSnapshotQuery, [pk]), } }, } diff --git a/e2e/migration-harness/run/record.mjs b/e2e/migration-harness/run/record.mjs new file mode 100644 index 000000000..ab0e5eaa1 --- /dev/null +++ b/e2e/migration-harness/run/record.mjs @@ -0,0 +1,7 @@ +/** Narrow a stored record or package object without importing application code. + * @param {unknown} value @returns {Record} */ +export function object(value) { + if (!value || typeof value !== 'object' || Array.isArray(value)) + throw Error('Expected a stored record or package object') + return Object.fromEntries(Object.entries(value)) +} diff --git a/e2e/migration-harness/run/run.mjs b/e2e/migration-harness/run/run.mjs index d199b8a09..4963b8bec 100644 --- a/e2e/migration-harness/run/run.mjs +++ b/e2e/migration-harness/run/run.mjs @@ -210,7 +210,20 @@ export async function run({ id, manifest, driver, supervision = null }) { stages[6], 'behaviour', async () => { - const checks = await driver.behaviour() + let checks + try { + checks = await driver.behaviour() + } catch (error) { + // A store outage after checks ran keeps their evidence in the report. + const completed = + error && typeof error === 'object' && 'checks' in error + ? error.checks + : [] + if (Array.isArray(completed)) + for (const check of completed) + evidence('behaviour', 'oracle-check', check.id, check) + throw error + } if (!Array.isArray(checks) || checks.length === 0) throw new Error('Behaviour checks must not be empty') for (const check of checks) diff --git a/e2e/migration-harness/run/snapshot-errors.mjs b/e2e/migration-harness/run/snapshot-errors.mjs new file mode 100644 index 000000000..c3a4428f7 --- /dev/null +++ b/e2e/migration-harness/run/snapshot-errors.mjs @@ -0,0 +1,67 @@ +const connectionCodes = new Set([ + 'ECONNREFUSED', + 'ECONNRESET', + 'ETIMEDOUT', + 'EPIPE', + 'ENETUNREACH', + 'EHOSTUNREACH', + 'ENOTFOUND', + 'EAI_AGAIN', + 'CONNECT_TIMEOUT', + 'CONNECTION_CLOSED', + 'CONNECTION_DESTROYED', + 'CONNECTION_ENDED', + 'UND_ERR_CONNECT_TIMEOUT', + 'UND_ERR_HEADERS_TIMEOUT', + 'UND_ERR_BODY_TIMEOUT', + 'UND_ERR_SOCKET', +]) + +/** Preserve retryable store transport failures without hiding app-caused damage. + * PostgreSQL class 08 is connection errors; 57P01-03 are server shutdown/startup. + * HTTP 5xx and 429 store responses are availability failures. + * Schema/data errors and unrecognized failures remain behaviour failures. + * @param {unknown} error @returns {boolean} */ +export function isSnapshotInfrastructureError(error) { + const seen = new Set() + while (error && typeof error === 'object' && !seen.has(error)) { + seen.add(error) + // A store that answered 5xx or 429 was unavailable; a 4xx is app damage. + if ( + 'status' in error && + typeof error.status === 'number' && + (error.status >= 500 || error.status === 429) + ) + return true + if ('code' in error && typeof error.code === 'string') { + if (connectionCodes.has(error.code)) return true + if (/^[0-9A-Z]{5}$/.test(error.code)) + return ( + /^08[0-9A-Z]{3}$/.test(error.code) || /^57P0[123]$/.test(error.code) + ) + } + error = 'cause' in error ? error.cause : undefined + } + return false +} + +/** The failed DynamoDB GetItem response, keeping its HTTP status for classification. + * @param {number} status @returns {Error & { status: number }} */ +export function dynamoSnapshotError(status) { + return Object.assign(Error(`Dynamo snapshot failed: ${status}`), { status }) +} + +/** Name a snapshot failure by its code or HTTP status, never its message, which + * can quote stored values. + * @param {unknown} error @returns {string} */ +export function snapshotErrorCode(error) { + const seen = new Set() + while (error && typeof error === 'object' && !seen.has(error)) { + seen.add(error) + if ('code' in error && typeof error.code === 'string') return error.code + if ('status' in error && typeof error.status === 'number') + return `HTTP ${error.status}` + error = 'cause' in error ? error.cause : undefined + } + return 'unrecognized error' +}