feat(e2e): verify the migration Oracle with reference and broken applications

e2e/migration-harness/__tests__/behaviour.test.ts

@@ -107,3 +107,32 @@ test('fresh-write lookup failure is visible as stable Oracle evidence', async ()
     },
   )
 })
+test('Mutation Data is inspected after setActive and before a read can repair it', async () => {
+  const order: string[] = []
+  const result = await behaviour(
+    async (method: string) => {
+      order.push(method)
+      return method.startsWith('find') ? [] : null
+    },
+    async () => {
+      order.push('inspect')
+    },
+  )
+  await result.mutate()
+  const setActive = order.indexOf('setActive')
+  assert.notEqual(setActive, -1, 'setActive must be called')
+  assert.equal(
+    order[setActive + 1],
+    'inspect',
+    `inspection must follow setActive directly: ${order.slice(setActive).join(',')}`,
+  )
+})
+test('a Mutation Data inspection failure still aborts the Oracle', async () => {
+  const result = await behaviour(
+    async (method: string) => (method.startsWith('find') ? [] : null),
+    async () => {
+      throw Object.assign(Error('store unavailable'), { infrastructure: true })
+    },
+  )
+  await assert.rejects(result.mutate(), /store unavailable/)
+})

e2e/migration-harness/__tests__/cli-fake-manifest.test.ts

@@ -0,0 +1,48 @@
+import { mkdtemp, readFile, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { expect, test, vi } from 'vitest'
+import { requiredOracleChecks } from '../run/migration-checks.mjs'
+
+vi.mock('../run/real-driver.mjs', () => ({
+  createRealDriver: () => ({
+    provision: async () => {},
+    installUnmigrated: async () => {},
+    initializeLegacyData: async () => {},
+    snapshot: async () => {},
+    migrate: async () => ({ exitCode: 0 }),
+    friction: async () => [],
+    installAppHost: async () => {},
+    behaviour: async () => [
+      { id: 'behaviour.cli', passed: true },
+      ...requiredOracleChecks.map((id) => ({ id, passed: true })),
+    ],
+    close: async () => {},
+  }),
+}))
+vi.mock('../isolation.mjs', () => ({
+  createIsolation: () => ({ drain: () => [], stop: async () => {} }),
+}))
+
+test('a Fake Migrator Run records its variant rather than a command adapter', async () => {
+  const output = await mkdtemp(join(tmpdir(), 'mh-cli-fake-'))
+  const argv = process.argv
+  const exitCode = process.exitCode
+  const diagnostics = vi.spyOn(console, 'error').mockImplementation(() => {})
+  try {
+    process.argv = ['node', 'cli.mjs', '--fake', 'downcase', '--output', output]
+    await import('../run/cli.mjs')
+    const report = JSON.parse(
+      await readFile(join(output, 'run-report.json'), 'utf8'),
+    )
+    expect(report.manifest.agents).toMatchObject({
+      migrator: 'fake:downcase',
+      model: 'fake-migrator',
+    })
+  } finally {
+    process.argv = argv
+    process.exitCode = exitCode
+    diagnostics.mockRestore()
+    await rm(output, { recursive: true, force: true })
+  }
+})

e2e/migration-harness/__tests__/migration-live.test.ts

@@ -0,0 +1,127 @@
+import 'dotenv/config'
+import assert from 'node:assert/strict'
+import { mkdtemp, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { test } from 'vitest'
+import { createIsolation } from '../isolation.mjs'
+import { createRealDriver } from '../run/real-driver.mjs'
+import { run } from '../run/run.mjs'
+
+const keys = [
+  'CS_WORKSPACE_CRN',
+  'CS_CLIENT_ID',
+  'CS_CLIENT_KEY',
+  'CS_CLIENT_ACCESS_KEY',
+]
+const enabled =
+  process.env.HARNESS_LIVE === '1' && keys.every((key) => process.env[key])
+const cases = [
+  ['reference', 'pass', null],
+  ['noop', 'behaviour', 'migration.protect-dependencies-absent'],
+  ['downcase', 'behaviour', 'behaviour.downcase-new-write-lookup'],
+  ['stale-hmac', 'behaviour', 'migration.active-hmac-absent'],
+  ['bigint', 'behaviour', 'behaviour.balance-decimal-string'],
+  ['rewrite', 'behaviour', 'no-rewrite.legacy-bytes'],
+  ['missing-log', 'invalid-friction-log', null],
+  ['legacy-ffi', 'behaviour', 'migration.protect-ffi-version'],
+  ['denied-app', 'pass', null],
+  ['denied-lifecycle', 'pass', null],
+  ['tree', 'pass', null],
+] as const
+for (const [kind, outcome, checkId] of cases) {
+  test.skipIf(!enabled)(
+    `real Run self-test: ${kind}`,
+    async () => {
+      const output = await mkdtemp(resolve(tmpdir(), 'mh-selftest-'))
+      const isolation = createIsolation()
+      const manifest = {
+        stack: kind === 'tree' ? 'tree' : '^1.2.1',
+        guideHash: null,
+      }
+      const driver = createRealDriver({
+        id: `selftest-${Date.now()}`,
+        root: fileURLToPath(new URL('../../..', import.meta.url)),
+        output,
+        command: '',
+        fakeMigration: kind === 'tree' ? 'reference' : kind,
+        isolation,
+        credentials: Object.fromEntries(
+          keys.map((key) => [key, process.env[key] ?? '']),
+        ),
+        manifest,
+      })
+      try {
+        const report = await run({
+          id: kind,
+          manifest,
+          driver,
+          supervision: isolation,
+        })
+        const diagnostics = JSON.stringify({
+          outcome: report.outcome,
+          failures: report.failures,
+          checks: report.evidence
+            .filter(
+              (e) =>
+                e.kind === 'oracle-check' && Object(e.payload).passed === false,
+            )
+            .map((e) => e.payload),
+        })
+        assert.equal(report.outcome, outcome, diagnostics)
+        // Every isolated exec starts with a startup probe that reads the same
+        // canary, so only a command-stage denial from the expected actor and
+        // Run stage proves the migrated code itself was exercised.
+        const deniedBy = (actor: string, stage?: string) =>
+          report.evidence.some(
+            (e) =>
+              e.kind === 'isolation-event' &&
+              (stage === undefined || e.stage === stage) &&
+              Object(e.payload).actor === actor &&
+              Object(e.payload).stage === 'command' &&
+              Object(e.payload).result === 'denied' &&
+              Object(e.payload).decision === 'forbidden' &&
+              Object(e.payload).resource === '/trace/canary',
+          )
+        if (kind === 'denied-app')
+          assert.ok(deniedBy('app-host'), 'createApp probe was not denied')
+        // Only the Migrator's bare `npm install` runs the root install hook;
+        // the App Host names a package spec, and npm skips root lifecycle
+        // scripts for `npm install <spec>`.
+        if (kind === 'denied-lifecycle')
+          assert.ok(
+            deniedBy('migrator', 'migrator'),
+            'Migrator install hook was not denied',
+          )
+        if (checkId)
+          assert.equal(
+            Object(report.evidence.find((e) => e.checkId === checkId)?.payload)
+              .passed,
+            false,
+            diagnostics,
+          )
+        if (kind === 'noop')
+          assert.ok(
+            report.evidence
+              .filter(
+                (e) =>
+                  e.kind === 'oracle-check' &&
+                  !e.checkId.startsWith('migration.'),
+              )
+              .every((e) => Object(e.payload).passed),
+          )
+        if (kind === 'reference')
+          assert.ok(
+            report.evidence.some(
+              (e) => e.checkId === 'no-rewrite.legacy-bytes',
+            ),
+          )
+      } finally {
+        await driver.close()
+        await rm(output, { recursive: true, force: true })
+      }
+    },
+    600_000,
+  )
+}

e2e/migration-harness/__tests__/migration.test.ts

@@ -0,0 +1,77 @@
+import assert from 'node:assert/strict'
+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { test } from 'vitest'
+import { migrationChecks } from '../run/migration-checks.mjs'
+import { run } from '../run/run.mjs'
+
+for (const stale of [false, true]) {
+  test(`Run migration evidence ${stale ? 'rejects stale' : 'accepts removed'} boolean HMAC`, async () => {
+    const workspace = await mkdtemp(join(tmpdir(), 'mh-migration-'))
+    await mkdir(join(workspace, 'src'))
+    await writeFile(
+      join(workspace, 'src/app.mjs'),
+      'import { Encryption } from "@cipherstash/stack/v3"',
+    )
+    await writeFile(
+      join(workspace, 'package.json'),
+      JSON.stringify({ dependencies: { '@cipherstash/stack': '1.2.1' } }),
+    )
+    await writeFile(
+      join(workspace, 'package-lock.json'),
+      JSON.stringify({
+        packages: {
+          '': { dependencies: { '@cipherstash/stack': '1.2.1' } },
+          'node_modules/@cipherstash/stack': {
+            version: '1.2.1',
+            dependencies: { '@cipherstash/protect-ffi': '0.31.0' },
+          },
+          'node_modules/@cipherstash/protect-ffi': { version: '0.31.0' },
+        },
+      }),
+    )
+    try {
+      const report = await run({
+        id: 'migration',
+        manifest: { guideHash: 'guide' },
+        driver: {
+          provision: async () => {},
+          installUnmigrated: async () => {},
+          initializeLegacyData: async () => {},
+          snapshot: async () => {},
+          migrate: async () => ({ exitCode: 0 }),
+          friction: async () => [],
+          installAppHost: async () => {},
+          behaviour: async () => [
+            { id: 'no-rewrite.legacy-bytes', passed: true },
+            ...(await migrationChecks({
+              workspace,
+              installedWorkspace: workspace,
+              ffiRange: '0.31.0',
+              mutationSnapshot: {
+                dynamo: {
+                  Item: stale
+                    ? { active__source: 'cipher', active__hmac: 'old' }
+                    : { active__source: 'cipher' },
+                },
+              },
+              stackVersion: '1.2.1',
+            })),
+          ],
+        },
+      })
+      assert.equal(report.outcome, stale ? 'behaviour' : 'pass')
+      assert.equal(
+        Object(
+          report.evidence.find(
+            (e) => e.checkId === 'migration.active-hmac-absent',
+          )?.payload,
+        ).passed,
+        !stale,
+      )
+    } finally {
+      await rm(workspace, { recursive: true, force: true })
+    }
+  })
+}