diff --git a/docs/implementation/persistent-review-store.md b/docs/implementation/persistent-review-store.md index 4e3c5ab..79b9a85 100644 --- a/docs/implementation/persistent-review-store.md +++ b/docs/implementation/persistent-review-store.md @@ -8,9 +8,11 @@ Only the trusted runner calls `Store`. Web handlers must send commands through t SQLite owns plan revision allocation. Stable repository/task/plan identity scopes every record; the selected issue is fixed at plan creation. Imports start at revision 1 regardless of the uploaded revision and increment only after validation. Historical revisions and base/head snapshots are append-only. Review writes compare both revision and snapshot ID; imports compare revision. Returned objects are decoded copies. -Suggestion requests receive opaque UUIDs before an agent response exists. Completion, cancellation, and Apply consult the saved identity, revision, and lifecycle state. Apply loads the saved reply and binding; it atomically saves the next revision, consumes that request, and invalidates pending/ready siblings. Invalid edits roll back without consuming the request. A reply arriving after cancellation or revision change cannot reactivate a request. `getSuggestions` recovers request state and cards after restart. +Suggestion requests receive opaque UUID attempt IDs before an agent response exists. Each request captures both the plan revision and repository snapshot. Completion and Apply atomically compare that binding with the current plan; a response for an older snapshot cannot become ready or apply. Apply loads the saved reply and binding, atomically saves the next revision, consumes that request, and invalidates pending/ready siblings. Invalid edits roll back without consuming the request. -Each write transaction takes SQLite's immediate write lock. WAL plus FULL synchronization provides committed recovery; the lock has a five-second busy timeout. Contending processes either serialize or fail explicitly. There is no asynchronous callback inside a transaction. Schema version 1 is installed atomically; unknown versions fail rather than being migrated implicitly. Require Node 26.7.0 or later, matching the CI baseline. +The request lifecycle is `pending` to `ready` to `consumed`, or `pending` to a terminal `failed`, `cancelled`, or `invalidated` state. Plan and snapshot changes invalidate active requests while retaining completed replies as stale history. Runner cleanup uses `settleSuggestion`, which compares the opaque attempt ID, expected binding, and `pending` state; if another process has already completed the request, cleanup loses without altering the ready result. Explicit user cancellation may cancel pending or ready work. Every non-success terminal transition persists a bounded reason, and `getSuggestions` recovers the binding, reason, and reply after restart. A reply arriving after any terminal transition cannot reactivate a request. + +Each write transaction takes SQLite's immediate write lock. WAL plus FULL synchronization provides committed recovery; the lock has a five-second busy timeout. Contending processes either serialize or fail explicitly. There is no asynchronous callback inside a transaction. Schema migrations advance through explicit versions; v4 adds request snapshot bindings and terminal reasons, and invalidates active legacy requests that cannot be bound safely. Unknown versions fail. Require Node 26.7.0 or later, matching the CI baseline. Ledger entries are immutable within a plan identity. Entries retain full SHA, nullable owner, owned/foreign origin, and immediate source SHA. Rebase records the new base/head, one-to-one mappings, and inherited ownership atomically. A missing or explicitly foreign source yields a foreign destination with null owner. No trailer is consulted. Historical owners survive plan amendments; new normal entries must name a current item. Replaying incompatible ownership is an error, never an upsert. @@ -20,7 +22,7 @@ Execution checkpoints retain the audited snapshot, revision, executed prefix, ac ## Validation -`npm test` includes disk-backed SQLite integration tests for revision allocation, cancelled/delayed/cross-plan/replayed suggestions, two independent processes racing Apply, late transaction rollback, abrupt process exit with committed and uncommitted writes, identity isolation, foreign/owned rebase chains, real-Git linking from stored mappings, stale review writes, typed metadata fingerprints, and checkpoint preservation. A child-process startup test treats any SQLite warning as a failure. `npm run typecheck` includes `runner`. +`npm test` includes disk-backed SQLite integration tests for revision allocation, cancelled/delayed/cross-plan/replayed suggestions, independent connections racing completion against cleanup and snapshot movement, two independent processes racing Apply, terminal-reason restart recovery, v3 request migration, late transaction rollback, abrupt process exit with committed and uncommitted writes, identity isolation, foreign/owned rebase chains, real-Git linking from stored mappings, stale review writes, typed metadata fingerprints, and checkpoint preservation. A child-process startup test treats any SQLite warning as a failure. `npm run typecheck` includes `runner`. Baseline: 136 tests. Final counts and CI evidence are recorded in the PR. diff --git a/runner/store.ts b/runner/store.ts index 899a5ea..a838568 100644 --- a/runner/store.ts +++ b/runner/store.ts @@ -12,6 +12,8 @@ export function requireSupportedNode(version = process.versions.node): void { } export interface Snapshot { id: string; base: string; head: string } export interface ReviewState { revision: number; snapshotId: string; reviewVersion?: number } +export type SuggestionState = 'pending' | 'ready' | 'failed' | 'cancelled' | 'invalidated' | 'consumed'; +export interface SuggestionRequest { state: SuggestionState; revision: number; snapshotId: string | null; reply: EditReply | null; reason: string | null } export interface SnippetReference { key: string; path: string; side: 'old' | 'new'; start: number; end: number; text: string; head: string; base: string } export interface QuestionAnswer { provider?: 'claude' | 'codex'; attempt: string; contextId?: string; status: 'pending' | 'complete' | 'failed'; expiresAt: number; text?: string; error?: string } export interface ReviewNote { id: string; item: string; kind: 'question' | 'change'; text: string; reference?: SnippetReference; answer?: QuestionAnswer; createdAt: string; revision: number; snapshotId: string } @@ -40,8 +42,8 @@ export class Store { this.#db.exec('PRAGMA foreign_keys=ON; PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL;'); this.#transaction(() => { const version = this.#get('PRAGMA user_version')!.user_version; - if (version !== 0 && version !== 1 && version !== 2 && version !== 3) throw new Error('Unsupported store schema version.'); - if (version === 3) return; + if (version !== 0 && version !== 1 && version !== 2 && version !== 3 && version !== 4) throw new Error('Unsupported store schema version.'); + if (version === 4) return; if (version === 0) this.#db.exec(` CREATE TABLE plans (key TEXT PRIMARY KEY, issue INTEGER NOT NULL, revision INTEGER NOT NULL, snapshot_id TEXT); CREATE TABLE revisions (key TEXT NOT NULL REFERENCES plans(key), revision INTEGER NOT NULL, data TEXT NOT NULL, PRIMARY KEY(key,revision)); @@ -55,10 +57,14 @@ export class Store { CREATE TABLE continuations (key TEXT NOT NULL REFERENCES plans(key), checkpoint_id TEXT NOT NULL, revision INTEGER NOT NULL, PRIMARY KEY(key,checkpoint_id,revision)); PRAGMA user_version=1; `); - if (version !== 2) this.#db.exec(`ALTER TABLE plans ADD COLUMN review_version INTEGER NOT NULL DEFAULT 0; + if (version < 2) this.#db.exec(`ALTER TABLE plans ADD COLUMN review_version INTEGER NOT NULL DEFAULT 0; CREATE TABLE review_notes (key TEXT NOT NULL REFERENCES plans(key), id TEXT NOT NULL, data TEXT NOT NULL, PRIMARY KEY(key,id)); PRAGMA user_version=2;`); - this.#db.exec("CREATE TABLE IF NOT EXISTS app_settings (key TEXT PRIMARY KEY, value TEXT NOT NULL); PRAGMA user_version=3;"); + if (version < 3) this.#db.exec("CREATE TABLE IF NOT EXISTS app_settings (key TEXT PRIMARY KEY, value TEXT NOT NULL); PRAGMA user_version=3;"); + if (version < 4) this.#db.exec(`ALTER TABLE requests ADD COLUMN snapshot_id TEXT; + ALTER TABLE requests ADD COLUMN reason TEXT; + UPDATE requests SET state='invalidated', reason='Request predates snapshot binding.' WHERE state IN ('pending','ready'); + PRAGMA user_version=4;`); }); } catch (error) { this.#db.close(); throw error; } } @@ -94,7 +100,7 @@ export class Store { #savePlan(key: string, plan: Plan, expected: number): void { if (this.#run('UPDATE plans SET revision=? WHERE key=? AND revision=?', plan.revision, key, expected).changes !== 1) throw new Error('Stale plan revision.'); this.#run('INSERT INTO revisions VALUES (?,?,?)', key, plan.revision, encode(plan)); - this.#run("UPDATE requests SET state='invalidated' WHERE key=? AND state IN ('pending','ready')", key); + this.#run("UPDATE requests SET state='invalidated',reason='Plan revision changed.' WHERE key=? AND state IN ('pending','ready')", key); const items = new Set(plan.items.map(item => item.id)); for (const row of this.#db.prepare('SELECT item FROM approvals WHERE key=?').all(key)) { if (!items.has(row.item as string)) this.#run('DELETE FROM approvals WHERE key=? AND item=?', key, row.item!); @@ -135,6 +141,7 @@ export class Store { const snapshot = { id: randomUUID(), base, head }; this.#run('INSERT INTO snapshots VALUES (?,?,?)', key, snapshot.id, encode(snapshot)); this.#run('UPDATE plans SET snapshot_id=? WHERE key=?', snapshot.id, key); + this.#run("UPDATE requests SET state='invalidated',reason='Repository snapshot changed.' WHERE key=? AND state IN ('pending','ready')", key); return snapshot; } getSnapshot(identity: PlanIdentity, id?: string): Snapshot { @@ -143,12 +150,12 @@ export class Store { if (!row) throw new Error('Unknown snapshot.'); return decode(row.data); } - beginSuggestions(identity: PlanIdentity, expectedRevision: number): string { + beginSuggestions(identity: PlanIdentity, expected: ReviewState): string { const key = identityKey(identity); return this.#transaction(() => { - if (this.#current(key).revision !== expectedRevision) throw new Error('Stale plan revision.'); + this.#expect(key, expected); const id = randomUUID(); - this.#run("INSERT INTO requests VALUES (?,?,?,'pending',NULL)", id, key, expectedRevision); return id; + this.#run("INSERT INTO requests (id,key,revision,state,reply,snapshot_id,reason) VALUES (?,?,?,'pending',NULL,?,NULL)", id, key, expected.revision, expected.snapshotId); return id; }); } completeSuggestions(identity: PlanIdentity, id: string, reply: unknown): void { @@ -156,17 +163,26 @@ export class Store { const key = identityKey(identity); this.#transaction(() => { const current = this.#current(key); - if (reply.base_revision !== current.revision || this.#run("UPDATE requests SET state='ready',reply=? WHERE id=? AND key=? AND revision=? AND state='pending'", encode(reply), id, key, current.revision!).changes !== 1) + if (reply.base_revision !== current.revision || this.#run("UPDATE requests SET state='ready',reply=?,reason=NULL WHERE id=? AND key=? AND revision=? AND snapshot_id=? AND state='pending'", encode(reply), id, key, current.revision!, current.snapshot_id!).changes !== 1) throw new Error('Suggestion request is stale, cancelled, or complete.'); }); } - cancelSuggestions(identity: PlanIdentity, id: string): void { - this.#run("UPDATE requests SET state='cancelled' WHERE id=? AND key=? AND state IN ('pending','ready')", id, identityKey(identity)); + settleSuggestion(identity: PlanIdentity, id: string, expected: ReviewState, outcome: { state: 'failed' | 'cancelled' | 'invalidated'; reason: string }): boolean { + if (!['failed', 'cancelled', 'invalidated'].includes(outcome.state) || typeof outcome.reason !== 'string' || !outcome.reason.trim() || outcome.reason.length > 4000) throw new Error('Invalid suggestion outcome.'); + const key = identityKey(identity); + return this.#transaction(() => this.#run(`UPDATE requests SET state=?,reason=? + WHERE id=? AND key=? AND revision=? AND snapshot_id=? AND state='pending' + AND EXISTS (SELECT 1 FROM plans WHERE key=? AND revision=? AND snapshot_id=?)`, + outcome.state, outcome.reason.trim(), id, key, expected.revision, expected.snapshotId, key, expected.revision, expected.snapshotId).changes === 1); + } + cancelSuggestions(identity: PlanIdentity, id: string, reason = 'Suggestion cancelled.'): void { + if (typeof reason !== 'string' || !reason.trim() || reason.length > 4000) throw new Error('Invalid cancellation reason.'); + this.#run("UPDATE requests SET state='cancelled',reason=? WHERE id=? AND key=? AND state IN ('pending','ready')", reason.trim(), id, identityKey(identity)); } - getSuggestions(identity: PlanIdentity, id: string): { state: string; revision: number; reply: EditReply | null } { + getSuggestions(identity: PlanIdentity, id: string): SuggestionRequest { const row = this.#get('SELECT * FROM requests WHERE key=? AND id=?', identityKey(identity), id); if (!row) throw new Error('Unknown suggestion request.'); - return { state: row.state as string, revision: row.revision as number, reply: row.reply === null ? null : decode(row.reply) }; + return { state: row.state as SuggestionState, revision: row.revision as number, snapshotId: row.snapshot_id as string | null, reply: row.reply === null ? null : decode(row.reply), reason: row.reason as string | null }; } applySuggestion(identity: PlanIdentity, id: string, index: number, context: PlanContext): Plan { const key = identityKey(identity); @@ -174,12 +190,14 @@ export class Store { this.#context(key, context); const request = this.#get("SELECT * FROM requests WHERE id=? AND key=? AND state='ready'", id, key); if (!request) throw new Error('Suggestion is unavailable.'); + const current = this.#current(key); + if (request.revision !== current.revision || request.snapshot_id !== current.snapshot_id) throw new Error('Suggestion is unavailable.'); const plan = this.getPlan(identity); const next = applySuggestion(plan, decode(request.reply), index, context, { identity, schemaVersion: plan.schema_version, baseRevision: request.revision as number, issue: plan.issue, }); this.#savePlan(key, next, request.revision as number); - this.#run("UPDATE requests SET state='consumed' WHERE id=?", id); return next; + this.#run("UPDATE requests SET state='consumed',reason=NULL WHERE id=?", id); return next; }); } #entry(key: string, entry: LedgerEntry): void { diff --git a/test/review.test.ts b/test/review.test.ts index 0f9f69a..944aba8 100644 --- a/test/review.test.ts +++ b/test/review.test.ts @@ -49,7 +49,7 @@ it('persists bounded per-item notes without creating a plan revision',()=>{ }); it('migrates a v1 database without losing its plans or ledger',()=>{ const {service,config}=fixture();service.close();services.splice(services.indexOf(service),1); - const db=new DatabaseSync(config.database);db.exec('ALTER TABLE plans DROP COLUMN review_version; DROP TABLE review_notes; PRAGMA user_version=1;');db.close(); + const db=new DatabaseSync(config.database);db.exec('ALTER TABLE plans DROP COLUMN review_version; DROP TABLE review_notes; DROP TABLE app_settings; ALTER TABLE requests DROP COLUMN reason; ALTER TABLE requests DROP COLUMN snapshot_id; PRAGMA user_version=1;');db.close(); const migrated=new ReviewService(config);services.push(migrated);expect(migrated.load().plan.revision).toBe(1);expect(migrated.store.getLedger(config.identity)).toHaveLength(2);expect(migrated.load().notes).toEqual([]); }); it('rejects a concurrent store review edit through the atomic review counter',()=>{ diff --git a/test/store.test.ts b/test/store.test.ts index 22bd31b..5f76ec9 100644 --- a/test/store.test.ts +++ b/test/store.test.ts @@ -3,6 +3,7 @@ import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { execFileSync, spawn } from 'node:child_process'; import { once } from 'node:events'; +import { DatabaseSync } from 'node:sqlite'; import { afterEach, expect, it } from 'vitest'; import { Store, requireSupportedNode } from '../runner/store.ts'; import type { Plan, PlanContext, EditReply } from '../core/plan.ts'; @@ -18,9 +19,10 @@ const dirs: string[] = [], stores: Store[] = []; afterEach(() => { for (const store of stores.splice(0)) store.close(); for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); }); function directory() { const dir = mkdtempSync(join(tmpdir(), 'codeboost-store-')); dirs.push(dir); return dir; } function open(path: string) { const store = new Store(path); stores.push(store); return store; } +function close(store: Store) { store.close(); stores.splice(stores.indexOf(store), 1); } function fixture() { const path = join(directory(), 'state.sqlite'); const store = open(path); store.createPlan(JSON.stringify(plan()), 'json', context, oid(1), oid(2)); return { path, store }; } const state = (store: Store) => ({ revision: store.getPlan(identity).revision, snapshotId: store.getSnapshot(identity).id }); -function ready(store: Store) { const id = store.beginSuggestions(identity, 1); store.completeSuggestions(identity, id, reply()); return id; } +function ready(store: Store) { const id = store.beginSuggestions(identity, state(store)); store.completeSuggestions(identity, id, reply()); return id; } it('allocates revisions in SQLite, survives reopen, and keeps old revisions and snapshots immutable', () => { const { store, path } = fixture(); const first = store.getSnapshot(identity); expect(store.getPlan(identity).revision).toBe(1); @@ -36,14 +38,62 @@ it('binds suggestion requests before the reply and rejects cross-plan, cancelled const { store } = fixture(); const id = ready(store), sibling = ready(store); const other = { ...identity, planId: 'other' }; store.createPlan(JSON.stringify(plan()), 'json', { ...context, identity: other }, oid(1), oid(2)); expect(() => store.applySuggestion(other, id, 0, { ...context, identity: other })).toThrow(/unavailable/); - const cancelled = store.beginSuggestions(identity, 1); store.cancelSuggestions(identity, cancelled); + const cancelled = store.beginSuggestions(identity, state(store)); store.cancelSuggestions(identity, cancelled); + expect(store.getSuggestions(identity, cancelled)).toMatchObject({ state: 'cancelled', reason: 'Suggestion cancelled.' }); expect(() => store.completeSuggestions(identity, cancelled, reply())).toThrow(/stale|cancelled/); - const delayed = store.beginSuggestions(identity, 1); + const delayed = store.beginSuggestions(identity, state(store)); expect(store.applySuggestion(identity, id, 0, context).revision).toBe(2); expect(() => store.applySuggestion(identity, id, 0, context)).toThrow(/unavailable/); expect(() => store.applySuggestion(identity, sibling, 0, context)).toThrow(/unavailable/); + expect(store.getSuggestions(identity, sibling)).toMatchObject({ state: 'invalidated', reason: 'Plan revision changed.', reply: reply() }); expect(() => store.completeSuggestions(identity, delayed, reply())).toThrow(/stale/); }); +it('preserves a completed request when stale pending cleanup loses the race', () => { + const { store, path } = fixture(); const other = open(path), expected = state(store); + const id = store.beginSuggestions(identity, expected); + other.completeSuggestions(identity, id, reply()); + expect(store.settleSuggestion(identity, id, expected, { state: 'failed', reason: 'Provider failed.' })).toBe(false); + expect(store.getSuggestions(identity, id)).toMatchObject({ state: 'ready', snapshotId: expected.snapshotId, reason: null, reply: reply() }); + expect(store.applySuggestion(identity, id, 0, context).revision).toBe(2); +}); +it('persists the winning terminal reason and prevents a late completion from reviving it', () => { + const { store, path } = fixture(); const other = open(path), expected = state(store); + const id = store.beginSuggestions(identity, expected); + expect(store.settleSuggestion(identity, id, expected, { state: 'failed', reason: 'Provider exited.' })).toBe(true); + expect(() => other.completeSuggestions(identity, id, reply())).toThrow(/stale|cancelled|complete/); + const recovered = open(path); + expect(recovered.getSuggestions(identity, id)).toMatchObject({ state: 'failed', snapshotId: expected.snapshotId, reason: 'Provider exited.', reply: null }); + expect(() => recovered.applySuggestion(identity, id, 0, context)).toThrow(/unavailable/); +}); +it('retains cancellation reasons across restart and never revives cancelled work', () => { + const { store, path } = fixture(), expected = state(store); + const id = store.beginSuggestions(identity, expected); + expect(store.settleSuggestion(identity, id, expected, { state: 'cancelled', reason: 'Runner shut down.' })).toBe(true); + close(store); + const recovered = open(path); + expect(recovered.getSuggestions(identity, id)).toMatchObject({ state: 'cancelled', reason: 'Runner shut down.', snapshotId: expected.snapshotId }); + expect(() => recovered.completeSuggestions(identity, id, reply())).toThrow(/stale|cancelled|complete/); +}); +it('migrates unbound active requests to terminal history instead of reviving them', () => { + const { store, path } = fixture(); const id = ready(store); + close(store); + const legacy = new DatabaseSync(path); + legacy.exec('ALTER TABLE requests DROP COLUMN reason; ALTER TABLE requests DROP COLUMN snapshot_id; PRAGMA user_version=3;'); + legacy.close(); + const recovered = open(path); + expect(recovered.getSuggestions(identity, id)).toEqual({ state: 'invalidated', revision: 1, snapshotId: null, reply: reply(), reason: 'Request predates snapshot binding.' }); + expect(() => recovered.applySuggestion(identity, id, 0, context)).toThrow(/unavailable/); +}); +it('invalidates pending and ready requests when another connection advances the snapshot', () => { + const { store, path } = fixture(); const other = open(path), expected = state(store); + const pending = store.beginSuggestions(identity, expected), completed = store.beginSuggestions(identity, expected); + store.completeSuggestions(identity, completed, reply()); + other.recordHistory(identity, expected, oid(1), oid(3), []); + expect(store.getSuggestions(identity, pending)).toMatchObject({ state: 'invalidated', snapshotId: expected.snapshotId, reason: 'Repository snapshot changed.', reply: null }); + expect(store.getSuggestions(identity, completed)).toMatchObject({ state: 'invalidated', snapshotId: expected.snapshotId, reason: 'Repository snapshot changed.', reply: reply() }); + expect(() => store.completeSuggestions(identity, pending, reply())).toThrow(/stale|cancelled|complete/); + expect(() => store.applySuggestion(identity, completed, 0, context)).toThrow(/unavailable/); +}); it('rolls back invalid edits without consuming the request or allocating a revision', () => { const { store } = fixture(); const id = ready(store); expect(() => store.applySuggestion(identity, id, 3, context)).toThrow(/index/); @@ -139,7 +189,7 @@ it('recovers a committed suggestion after abrupt process exit and discards an in const store = new Store(process.argv[1]); const context = {...${JSON.stringify(context)}, pathKey: p => p}; store.createPlan(${JSON.stringify(JSON.stringify(plan()))}, 'json', context, '${oid(1)}', '${oid(2)}'); - const id = store.beginSuggestions(context.identity, 1); + const id = store.beginSuggestions(context.identity, {revision: 1, snapshotId: store.getSnapshot(context.identity).id}); store.completeSuggestions(context.identity, id, ${JSON.stringify(reply())}); process.stdout.write(id); process.exit(0);`; const id = execFileSync(process.execPath, ['--input-type=module', '-e', source, path], { encoding: 'utf8' });