From 4514754de246bb1aa09b60155b4fb232705b909b Mon Sep 17 00:00:00 2001 From: Aaron Queen Date: Fri, 11 Sep 2026 21:42:44 -0600 Subject: [PATCH] test(eval): precision scoring and a pinned-corpus edge gate for resolution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 0 of docs/design/resolution-binding-model-plan.md. The evaluation runner scored recall only, so a wrong edge — the thing the resolution PR chain (#1713, #1718, #1746, #1844) exists to remove — was invisible to it. scoring.ts gains scoreEdgeCase: an EdgeCase names a kind, a target endpoint (file suffix + symbol) and optionally a source, and expects the edge absent (known-wrong) or present (control). evaluation-edge-scoring.test.ts pins it on a synthetic project with the bare-import and sealed-module shapes and proves it flags a violated case and a missing endpoint. edge-cases.ts encodes the edges the PR bodies named on the commits they measured (full SHAs): vite 8492422b (#1713 self-import + getEnv, #1718 two fuzzy nested calls, #1746 sealed defineConfig -> test-stacktrace.js vite, one relative-import control), vitest 7c818153 (evaluatedModules), svelte 5895c637 (bundle). npm run eval:precision -- fetches the pinned commit, indexes, scores, prints the resolved-edge histogram by resolver (the LOST/GAINED methodology) and writes a report; the three baseline reports at codegraph 1498c2ff are committed: vite 28,893 edges / fuzzy 16, vitest 75,035 / 28, svelte 70,432 / 159; every absent case held, the control held. --- __tests__/evaluation-edge-scoring.test.ts | 80 +++++++++++ __tests__/evaluation/edge-cases.ts | 132 ++++++++++++++++++ __tests__/evaluation/precision-runner.ts | 130 +++++++++++++++++ .../precision-svelte-5895c63-1498c2ff.json | 35 +++++ .../precision-vite-8492422-1498c2ff.json | 76 ++++++++++ .../precision-vitest-7c81815-1498c2ff.json | 35 +++++ __tests__/evaluation/scoring.ts | 49 ++++++- __tests__/evaluation/types.ts | 57 ++++++++ docs/design/resolution-binding-model-plan.md | 20 ++- package.json | 1 + 10 files changed, 610 insertions(+), 5 deletions(-) create mode 100644 __tests__/evaluation-edge-scoring.test.ts create mode 100644 __tests__/evaluation/edge-cases.ts create mode 100644 __tests__/evaluation/precision-runner.ts create mode 100644 __tests__/evaluation/results/precision-svelte-5895c63-1498c2ff.json create mode 100644 __tests__/evaluation/results/precision-vite-8492422-1498c2ff.json create mode 100644 __tests__/evaluation/results/precision-vitest-7c81815-1498c2ff.json diff --git a/__tests__/evaluation-edge-scoring.test.ts b/__tests__/evaluation-edge-scoring.test.ts new file mode 100644 index 000000000..9fb7a5674 --- /dev/null +++ b/__tests__/evaluation-edge-scoring.test.ts @@ -0,0 +1,80 @@ +/** + * The precision scorer (__tests__/evaluation/scoring.ts, scoreEdgeCase) on a + * synthetic project that reproduces the shapes the resolution PR chain + * removed: a bare npm import that must not fuzzy-bind to a same-named + * project symbol (#1713) and a sealed module whose locals must not be + * cross-file candidates (#1746), plus a relative import as the `present` + * control. This pins that the scorer can SEE a false edge — the recall + * scorers cannot — before the binding-model plan changes any resolver. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import CodeGraph from '../src/index'; +import { scoreEdgeCase } from './evaluation/scoring'; +import type { EdgeCase } from './evaluation/types'; + +let dir: string; +let cg: CodeGraph; + +beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-precision-')); + const write = (rel: string, body: string) => { + fs.mkdirSync(path.dirname(path.join(dir, rel)), { recursive: true }); + fs.writeFileSync(path.join(dir, rel), body); + }; + write('package.json', JSON.stringify({ name: 'precision-fixture', dependencies: { 'magic-string': '1' } })); + // #1713 shape: a bare import whose name collides with a project function. + write('src/consumer.ts', "import { bundle } from 'magic-string';\nimport { helper } from './util';\nexport function run() { return bundle(helper()); }\n"); + write('scripts/build.ts', 'export function bundle(): string { return "x"; }\n'); + write('src/util.ts', 'export function helper(): number { return 1; }\n'); + // #1746 shape: a module that imports but exports nothing. + write('sealed.ts', "import { helper } from './src/util';\nconst widget = helper();\n"); + write('src/other.ts', "import { helper } from './util';\nexport function use() { return widget + helper(); }\n"); + cg = await CodeGraph.init(dir, { index: true }); + cg.resolveReferences(); +}); +afterAll(() => { + cg.close(); + fs.rmSync(dir, { recursive: true, force: true }); +}); + +const base = { corpus: 'synthetic', source: 'test', why: '' } as const; + +describe('scoreEdgeCase', () => { + it('holds an absent case when the bare import does not bind to the project symbol', () => { + const c: EdgeCase = { ...base, id: 'bare', kind: 'imports', from: { file: 'src/consumer.ts', name: 'consumer.ts' }, to: { file: 'scripts/build.ts', name: 'bundle' }, expect: 'absent' }; + const r = scoreEdgeCase(c, cg); + expect(r.found).toEqual([]); + expect(r.pass).toBe(true); + }); + + it('holds an absent case when a sealed module local is not reached cross-file', () => { + const c: EdgeCase = { ...base, id: 'sealed', kind: 'references', from: { file: 'src/other.ts', name: 'use' }, to: { file: 'sealed.ts', name: 'widget' }, expect: 'absent' }; + const r = scoreEdgeCase(c, cg); + expect(r.pass).toBe(true); + }); + + it('holds a present control for a relative import', () => { + const c: EdgeCase = { ...base, id: 'control', kind: 'imports', from: { file: 'src/consumer.ts', name: 'consumer.ts' }, to: { file: 'src/util.ts', name: 'helper' }, expect: 'present' }; + const r = scoreEdgeCase(c, cg); + expect(r.found.length).toBeGreaterThan(0); + expect(r.pass).toBe(true); + }); + + it('reports a violated absent case when the edge exists', () => { + // Use the control edge as if it were known-wrong: the scorer must flag it. + const c: EdgeCase = { ...base, id: 'violated', kind: 'imports', from: { file: 'src/consumer.ts', name: 'consumer.ts' }, to: { file: 'src/util.ts', name: 'helper' }, expect: 'absent' }; + const r = scoreEdgeCase(c, cg); + expect(r.pass).toBe(false); + expect(r.found[0]?.resolvedBy).toBeDefined(); + }); + + it('fails a present control whose endpoint is not in the graph, and says which', () => { + const c: EdgeCase = { ...base, id: 'missing', kind: 'imports', from: { file: 'src/consumer.ts', name: 'consumer.ts' }, to: { file: 'src/nowhere.ts', name: 'nothing' }, expect: 'present' }; + const r = scoreEdgeCase(c, cg); + expect(r.pass).toBe(false); + expect(r.missingEndpoints).toEqual(['to src/nowhere.ts:nothing']); + }); +}); diff --git a/__tests__/evaluation/edge-cases.ts b/__tests__/evaluation/edge-cases.ts new file mode 100644 index 000000000..6f3a2d265 --- /dev/null +++ b/__tests__/evaluation/edge-cases.ts @@ -0,0 +1,132 @@ +/** + * Known-wrong and known-right edges on pinned real repositories + * (docs/design/resolution-binding-model-plan.md, Phase 0). + * + * Every `absent` case is an edge a resolution PR measured and removed on a + * real corpus; the endpoints are the ones named in that PR's body. They must + * stay gone. The `present` cases are controls: edges the same PRs reported + * unchanged, so a change that silently kills resolution fails here rather + * than looking like a precision win. + * + * Run with `npm run eval:precision -- ` (precision-runner.ts), which + * fetches the pinned commit, indexes it, scores every case, and records the + * resolved-edge histogram for LOST/GAINED comparison across builds. + */ +import type { EdgeCase } from './types.js'; + +export interface PrecisionCorpus { + key: string; + repo: string; + /** Full SHA: GitHub only serves an arbitrary commit to `git fetch` by its full id. */ + commit: string; + note: string; +} + +export const PRECISION_CORPORA: Record = { + vite: { + key: 'vite', + repo: 'https://github.com/vitejs/vite.git', + commit: '8492422b8f110625a90c702f42f30784e8cf19dc', + note: 'The corpus #1713, #1718 and #1746 measured (LOST 4 / 12 / 157, GAINED 0).', + }, + vitest: { + key: 'vitest', + repo: 'https://github.com/vitest-dev/vitest.git', + commit: '7c818153add03b0bca54453a54e76961dd5be18d', + note: '#1713: 34 fuzzy edges from bare imports removed, GAINED 0.', + }, + svelte: { + key: 'svelte', + repo: 'https://github.com/sveltejs/svelte.git', + commit: '5895c637b04dc8667020c8d326807c3f3a984472', + note: '#1713: 5 fuzzy edges removed, GAINED 0.', + }, +}; + +export const edgeCases: EdgeCase[] = [ + // --- #1713: a bare (npm / builtin) import must never fuzzy-match a project symbol --- + { + id: 'vite-bare-import-self-edge', + corpus: 'vite', + kind: 'imports', + from: { file: 'packages/vite/src/node/optimizer/scan.ts', name: 'scan.ts' }, + to: { file: 'packages/vite/src/node/optimizer/scan.ts', name: 'scan' }, + expect: 'absent', + source: '#1713', + why: "`import { scan } from 'rolldown/experimental'` resolved onto the importing file's OWN `scan` — a self-edge.", + }, + { + id: 'vite-bare-import-getEnv', + corpus: 'vite', + kind: 'imports', + from: { file: 'packages/vite/src/node/config.ts', name: 'config.ts' }, + to: { file: 'packages/vite/src/node/plugins/importAnalysis.ts', name: 'getEnv' }, + expect: 'absent', + source: '#1713', + why: "`import { getEnv } from '@vitejs/devtools/config'` resolved onto an unrelated project `getEnv`.", + }, + { + id: 'vitest-bare-import-evaluatedModules', + corpus: 'vitest', + kind: 'imports', + to: { file: '.ts', name: 'evaluatedModules' }, + expect: 'absent', + source: '#1713', + why: "`import type { EvaluatedModules } from 'vite/module-runner'` resolved (case-insensitively) onto the method VitestMocker::evaluatedModules in 17 files.", + }, + { + id: 'svelte-bare-import-bundle', + corpus: 'svelte', + kind: 'imports', + to: { file: 'scripts/generate-browser-support.ts', name: 'bundle' }, + expect: 'absent', + source: '#1713', + why: "`import MagicString, { Bundle } from 'magic-string'` resolved onto the script function `bundle` in 5 files.", + }, + + // --- #1718 (supersedes #1709): fuzzy may reject a unique guess, never manufacture one --- + { + id: 'vite-fuzzy-nested-resolveConfig-getEnv', + corpus: 'vite', + kind: 'calls', + from: { file: 'packages/vite/src/node/config.ts', name: 'resolveConfig' }, + to: { file: 'packages/vite/src/node/plugins/importAnalysis.ts', name: 'getEnv' }, + expect: 'absent', + source: '#1718', + why: 'A fuzzy call edge onto a nested function the call site cannot lexically reach.', + }, + { + id: 'vite-fuzzy-nested-build-scan', + corpus: 'vite', + kind: 'calls', + from: { file: 'packages/vite/src/node/optimizer/scan.ts', name: 'build' }, + to: { file: 'packages/vite/src/node/optimizer/scan.ts', name: 'scan' }, + expect: 'absent', + source: '#1718', + why: 'The `calls` edge that followed the self-import above.', + }, + + // --- #1746 / #1719: a module that imports but exports nothing is sealed --- + { + id: 'vite-sealed-module-defineConfig', + corpus: 'vite', + kind: 'imports', + from: { file: 'vite.config.ts', name: 'vite.config.ts' }, + to: { file: 'playground/ssr-html/test-stacktrace.js', name: 'vite' }, + expect: 'absent', + source: '#1746', + why: "Every `import { defineConfig } from 'vite'` across the playground resolved onto a module-scope `const vite` in a file that exports nothing (157 edges).", + }, + + // --- Controls: edges the PRs reported unchanged --- + { + id: 'vite-control-relative-import', + corpus: 'vite', + kind: 'imports', + from: { file: 'packages/vite/src/node/config.ts', name: 'config.ts' }, + to: { file: 'packages/vite/src/node/logger.ts', name: 'createLogger' }, + expect: 'present', + source: 'control', + why: "`import { createLogger } from './logger'` is a relative project import; the import resolver must still bind it.", + }, +]; diff --git a/__tests__/evaluation/precision-runner.ts b/__tests__/evaluation/precision-runner.ts new file mode 100644 index 000000000..05de8bf0e --- /dev/null +++ b/__tests__/evaluation/precision-runner.ts @@ -0,0 +1,130 @@ +/** + * Precision gate on a pinned real repository + * (docs/design/resolution-binding-model-plan.md, Phase 0). + * + * npm run eval:precision -- vite # fetch + index + score + * EVAL_REPOS=/path/to/dir npm run eval:precision -- vite + * + * Fetches PRECISION_CORPORA[] at its pinned commit (shallow, into + * $EVAL_REPOS or $TMPDIR/codegraph-precision/), indexes it with the + * library, scores every edge case for the corpus, prints the resolved-edge + * histogram by resolver (the LOST/GAINED methodology the PRs used), and + * writes a JSON report next to the recall reports. Exit 1 when any `absent` + * case is violated or any `present` control is missing. + */ +import { execFileSync } from 'child_process'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { DatabaseSync } from 'node:sqlite'; +import { CodeGraph } from '../../src/index.js'; +import { scoreEdgeCase } from './scoring.js'; +import { edgeCases, PRECISION_CORPORA } from './edge-cases.js'; +import type { EdgeCaseResult, PrecisionReport } from './types.js'; + +const corpusKey = process.argv[2]; +const corpus = corpusKey ? PRECISION_CORPORA[corpusKey] : undefined; +if (!corpus) { + console.error(`usage: npx tsx __tests__/evaluation/precision-runner.ts <${Object.keys(PRECISION_CORPORA).join('|')}>`); + process.exit(2); +} + +const reposDir = process.env.EVAL_REPOS ?? path.join(os.tmpdir(), 'codegraph-precision'); +const repoDir = path.join(reposDir, corpus.key); + +function sh(cmd: string, args: string[], cwd?: string, quiet = false): string { + return execFileSync(cmd, args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', quiet ? 'ignore' : 'inherit'] }).trim(); +} + +function fetchPinned(): void { + if (!fs.existsSync(path.join(repoDir, '.git'))) { + fs.mkdirSync(repoDir, { recursive: true }); + sh('git', ['init', '-q'], repoDir); + sh('git', ['remote', 'add', 'origin', corpus!.repo], repoDir); + } + const head = (() => { try { return sh('git', ['rev-parse', 'HEAD'], repoDir, true); } catch { return ''; } })(); + if (!head.startsWith(corpus!.commit)) { + console.log(`fetching ${corpus!.repo} @ ${corpus!.commit}`); + sh('git', ['fetch', '-q', '--depth', '1', 'origin', corpus!.commit], repoDir); + sh('git', ['checkout', '-q', '--force', 'FETCH_HEAD'], repoDir); + } +} + +function resolvedByHistogram(): { resolvedBy: Record; edges: number } { + const db = new DatabaseSync(path.join(repoDir, '.codegraph', 'codegraph.db'), { readOnly: true }); + try { + const rows = db + .prepare(`SELECT COALESCE(json_extract(metadata, '$.resolvedBy'), '(none)') AS r, COUNT(*) AS n FROM edges GROUP BY r ORDER BY n DESC`) + .all() as Array<{ r: string; n: number }>; + const resolvedBy: Record = {}; + let edges = 0; + for (const { r, n } of rows) { resolvedBy[r] = n; edges += n; } + return { resolvedBy, edges }; + } finally { + db.close(); + } +} + +async function run(): Promise { + fetchPinned(); + const commit = sh('git', ['rev-parse', '--short', 'HEAD'], repoDir); + let codegraphSha = 'unknown'; + try { codegraphSha = sh('git', ['rev-parse', '--short', 'HEAD']); } catch {} + + console.log(`\nCodeGraph precision — ${corpus!.key} @ ${commit} (codegraph ${codegraphSha})`); + console.log(`${corpus!.note}\n`); + + fs.rmSync(path.join(repoDir, '.codegraph'), { recursive: true, force: true }); + const cg = CodeGraph.initSync(repoDir); + const t0 = performance.now(); + await cg.indexAll(); + console.log(`indexed in ${((performance.now() - t0) / 1000).toFixed(1)} s`); + + const results: EdgeCaseResult[] = []; + for (const c of edgeCases.filter((c) => c.corpus === corpus!.key)) { + const r = scoreEdgeCase(c, cg); + results.push(r); + const status = r.pass ? '\x1b[32mHELD\x1b[0m' : '\x1b[31mVIOLATED\x1b[0m'; + console.log(` ${c.id.padEnd(44)} ${c.expect.padEnd(7)} ${status} (${c.source})`); + for (const f of r.found) { + const from = cg.getNode(f.from); const to = cg.getNode(f.to); + console.log(` ${from?.filePath}:${from?.name} → ${to?.filePath}:${to?.name} [${f.resolvedBy ?? '?'}]`); + } + if (r.missingEndpoints.length) console.log(` endpoints not in graph: ${r.missingEndpoints.join('; ')}`); + } + cg.close(); + + const { resolvedBy, edges } = resolvedByHistogram(); + console.log(`\nedges: ${edges}`); + for (const [k, v] of Object.entries(resolvedBy)) console.log(` ${k.padEnd(18)} ${v}`); + + const absent = results.filter((r) => r.expect === 'absent'); + const present = results.filter((r) => r.expect === 'present'); + const report: PrecisionReport = { + timestamp: new Date().toISOString(), + corpus: corpus!.key, + repo: corpus!.repo, + commit, + codegraphSha, + summary: { + absentCases: absent.length, + absentHeld: absent.filter((r) => r.pass).length, + presentCases: present.length, + presentHeld: present.filter((r) => r.pass).length, + unjudgeable: results.filter((r) => r.missingEndpoints.length > 0 && r.expect === 'absent' && !r.pass).length, + }, + resolvedBy, + edges, + results, + }; + const resultsDir = path.join(__dirname, 'results'); + fs.mkdirSync(resultsDir, { recursive: true }); + const file = path.join(resultsDir, `precision-${corpus!.key}-${commit}-${codegraphSha}.json`); + fs.writeFileSync(file, JSON.stringify(report, null, 2)); + const failed = results.filter((r) => !r.pass).length; + console.log(`\nSUMMARY: absent ${report.summary.absentHeld}/${absent.length} held, present ${report.summary.presentHeld}/${present.length} held`); + console.log(`Report saved: ${file}`); + process.exit(failed > 0 ? 1 : 0); +} + +run().catch((err) => { console.error(err); process.exit(1); }); diff --git a/__tests__/evaluation/results/precision-svelte-5895c63-1498c2ff.json b/__tests__/evaluation/results/precision-svelte-5895c63-1498c2ff.json new file mode 100644 index 000000000..b8821e959 --- /dev/null +++ b/__tests__/evaluation/results/precision-svelte-5895c63-1498c2ff.json @@ -0,0 +1,35 @@ +{ + "timestamp": "2026-09-12T03:42:07.350Z", + "corpus": "svelte", + "repo": "https://github.com/sveltejs/svelte.git", + "commit": "5895c63", + "codegraphSha": "1498c2ff", + "summary": { + "absentCases": 1, + "absentHeld": 1, + "presentCases": 0, + "presentHeld": 0, + "unjudgeable": 0 + }, + "resolvedBy": { + "(none)": 37891, + "exact-match": 16731, + "import": 8468, + "qualified-name": 4751, + "instance-method": 1234, + "file-path": 685, + "function-ref": 299, + "framework": 214, + "fuzzy": 159 + }, + "edges": 70432, + "results": [ + { + "caseId": "svelte-bare-import-bundle", + "expect": "absent", + "pass": true, + "found": [], + "missingEndpoints": [] + } + ] +} \ No newline at end of file diff --git a/__tests__/evaluation/results/precision-vite-8492422-1498c2ff.json b/__tests__/evaluation/results/precision-vite-8492422-1498c2ff.json new file mode 100644 index 000000000..7067d1fde --- /dev/null +++ b/__tests__/evaluation/results/precision-vite-8492422-1498c2ff.json @@ -0,0 +1,76 @@ +{ + "timestamp": "2026-09-12T03:41:35.187Z", + "corpus": "vite", + "repo": "https://github.com/vitejs/vite.git", + "commit": "8492422", + "codegraphSha": "1498c2ff", + "summary": { + "absentCases": 5, + "absentHeld": 5, + "presentCases": 1, + "presentHeld": 1, + "unjudgeable": 0 + }, + "resolvedBy": { + "(none)": 13098, + "exact-match": 7250, + "import": 5410, + "qualified-name": 1791, + "instance-method": 827, + "file-path": 258, + "function-ref": 202, + "framework": 41, + "fuzzy": 16 + }, + "edges": 28893, + "results": [ + { + "caseId": "vite-bare-import-self-edge", + "expect": "absent", + "pass": true, + "found": [], + "missingEndpoints": [] + }, + { + "caseId": "vite-bare-import-getEnv", + "expect": "absent", + "pass": true, + "found": [], + "missingEndpoints": [] + }, + { + "caseId": "vite-fuzzy-nested-resolveConfig-getEnv", + "expect": "absent", + "pass": true, + "found": [], + "missingEndpoints": [] + }, + { + "caseId": "vite-fuzzy-nested-build-scan", + "expect": "absent", + "pass": true, + "found": [], + "missingEndpoints": [] + }, + { + "caseId": "vite-sealed-module-defineConfig", + "expect": "absent", + "pass": true, + "found": [], + "missingEndpoints": [] + }, + { + "caseId": "vite-control-relative-import", + "expect": "present", + "pass": true, + "found": [ + { + "from": "file:packages/vite/src/node/config.ts", + "to": "function:0c2f36c32c5742aee412978220fd5b23", + "resolvedBy": "import" + } + ], + "missingEndpoints": [] + } + ] +} \ No newline at end of file diff --git a/__tests__/evaluation/results/precision-vitest-7c81815-1498c2ff.json b/__tests__/evaluation/results/precision-vitest-7c81815-1498c2ff.json new file mode 100644 index 000000000..d1383b306 --- /dev/null +++ b/__tests__/evaluation/results/precision-vitest-7c81815-1498c2ff.json @@ -0,0 +1,35 @@ +{ + "timestamp": "2026-09-12T03:41:57.403Z", + "corpus": "vitest", + "repo": "https://github.com/vitest-dev/vitest.git", + "commit": "7c81815", + "codegraphSha": "1498c2ff", + "summary": { + "absentCases": 1, + "absentHeld": 1, + "presentCases": 0, + "presentHeld": 0, + "unjudgeable": 0 + }, + "resolvedBy": { + "import": 31966, + "(none)": 23342, + "exact-match": 13647, + "instance-method": 2863, + "qualified-name": 2230, + "file-path": 463, + "function-ref": 410, + "framework": 86, + "fuzzy": 28 + }, + "edges": 75035, + "results": [ + { + "caseId": "vitest-bare-import-evaluatedModules", + "expect": "absent", + "pass": true, + "found": [], + "missingEndpoints": [] + } + ] +} \ No newline at end of file diff --git a/__tests__/evaluation/scoring.ts b/__tests__/evaluation/scoring.ts index b20f604c4..0452f4fd7 100644 --- a/__tests__/evaluation/scoring.ts +++ b/__tests__/evaluation/scoring.ts @@ -1,4 +1,4 @@ -import type { EvalResult } from './types.js'; +import type { EvalResult, EdgeCase, EdgeCaseResult, EdgeEndpoint } from './types.js'; export const PASS_THRESHOLD = 0.5; @@ -80,3 +80,50 @@ export function scoreFindRelevantContext( latencyMs, }; } + +/** + * Precision scoring (docs/design/resolution-binding-model-plan.md, Phase 0). + * + * The recall scorers above ask "did the expected symbols show up"; nothing in + * them penalizes a wrong edge, which is exactly what the resolution PR chain + * (#1713, #1718, #1746, #1844) exists to remove. This scores an {@link EdgeCase} + * against a graph: for an `absent` case the edge must not exist between any + * node pair matching the endpoints; for a `present` control it must. + */ +export function scoreEdgeCase( + edgeCase: EdgeCase, + graph: { + getNodesByName(name: string): Array<{ id: string; name: string; filePath: string }>; + getOutgoingEdgesFrom(ids: readonly string[], kinds?: Array): Array<{ source: string; target: string; metadata?: Record | null }>; + getIncomingEdgesTo(ids: readonly string[], kinds?: Array): Array<{ source: string; target: string; metadata?: Record | null }>; + getNode(id: string): { id: string; name: string; filePath: string } | null; + } +): EdgeCaseResult { + const pick = (ep: EdgeEndpoint) => + graph.getNodesByName(ep.name).filter((n) => n.filePath.endsWith(ep.file) || n.filePath.replace(/\\/g, '/').endsWith(ep.file)); + const fromNodes = edgeCase.from ? pick(edgeCase.from) : null; + const toNodes = pick(edgeCase.to); + const missingEndpoints: string[] = []; + if (fromNodes && fromNodes.length === 0) missingEndpoints.push(`from ${edgeCase.from!.file}:${edgeCase.from!.name}`); + if (toNodes.length === 0) missingEndpoints.push(`to ${edgeCase.to.file}:${edgeCase.to.name}`); + + const found: EdgeCaseResult['found'] = []; + if (toNodes.length && (fromNodes === null || fromNodes.length)) { + const toIds = new Set(toNodes.map((n) => n.id)); + const edges = fromNodes + ? graph.getOutgoingEdgesFrom(fromNodes.map((n) => n.id), [edgeCase.kind]).filter((e) => toIds.has(e.target)) + : graph.getIncomingEdgesTo([...toIds], [edgeCase.kind]); + for (const e of edges) { + const meta = (e.metadata ?? {}) as { resolvedBy?: string }; + found.push({ from: e.source, to: e.target, resolvedBy: meta.resolvedBy }); + } + } + + // A missing FROM endpoint makes an `absent` case vacuous and a `present` + // case a failure; a missing TO endpoint is fine for `absent` (the wrong + // target may simply not exist at this commit) and a failure for `present`. + let pass: boolean; + if (edgeCase.expect === 'absent') pass = found.length === 0 && (fromNodes === null || fromNodes.length > 0); + else pass = found.length > 0; + return { caseId: edgeCase.id, expect: edgeCase.expect, pass, found, missingEndpoints }; +} diff --git a/__tests__/evaluation/types.ts b/__tests__/evaluation/types.ts index 64a24270d..43bed076d 100644 --- a/__tests__/evaluation/types.ts +++ b/__tests__/evaluation/types.ts @@ -35,3 +35,60 @@ export interface EvalReport { }; results: EvalResult[]; } + +// --------------------------------------------------------------------------- +// Precision: known-wrong and known-right edges on pinned real repositories +// (docs/design/resolution-binding-model-plan.md, Phase 0). The recall cases +// above cannot see a false edge; these can. +// --------------------------------------------------------------------------- + +export interface EdgeEndpoint { + /** Suffix of the file path as stored in the graph (matched with endsWith). */ + file: string; + name: string; +} + +export interface EdgeCase { + id: string; + /** Corpus key — see PRECISION_CORPORA in edge-cases.ts. */ + corpus: string; + kind: 'calls' | 'imports' | 'references'; + /** Omitted = any source node (the PR named only the wrong target). */ + from?: EdgeEndpoint; + to: EdgeEndpoint; + /** `absent`: a known-wrong edge that must stay gone. `present`: a known-right control. */ + expect: 'absent' | 'present'; + /** Which PR or issue established it. */ + source: string; + why: string; +} + +export interface EdgeCaseResult { + caseId: string; + expect: 'absent' | 'present'; + /** Whether the graph matches the expectation. */ + pass: boolean; + /** Edges found between the endpoints (empty when none). */ + found: Array<{ from: string; to: string; resolvedBy?: string }>; + /** Endpoints that did not resolve to any node at all — the case cannot be judged. */ + missingEndpoints: string[]; +} + +export interface PrecisionReport { + timestamp: string; + corpus: string; + repo: string; + commit: string; + codegraphSha: string; + summary: { + absentCases: number; + absentHeld: number; + presentCases: number; + presentHeld: number; + unjudgeable: number; + }; + /** Resolved-edge histogram by resolver, for LOST/GAINED-style comparison across builds. */ + resolvedBy: Record; + edges: number; + results: EdgeCaseResult[]; +} diff --git a/docs/design/resolution-binding-model-plan.md b/docs/design/resolution-binding-model-plan.md index faf1c88b1..c1ed3912a 100644 --- a/docs/design/resolution-binding-model-plan.md +++ b/docs/design/resolution-binding-model-plan.md @@ -1,6 +1,6 @@ # Resolution binding model — one source of truth for exports and bindings -**Status:** plan, not started. Written 2026-09-11. Companion to [kernel-only-extraction-plan.md](kernel-only-extraction-plan.md) (which should land first, so there is one extractor to emit the new facts) and [greenfield-rust-core-sketch.md](greenfield-rust-core-sketch.md). Closes upstream issue #1721 and ends the fix cycle behind #1566, #1790, #1794 and #1844. +**Status:** Phase 0 done (2026-09-12); Phases 1 to 4 not started. Written 2026-09-11. Companion to [kernel-only-extraction-plan.md](kernel-only-extraction-plan.md) (which should land first, so there is one extractor to emit the new facts) and [greenfield-rust-core-sketch.md](greenfield-rust-core-sketch.md). Closes upstream issue #1721 and ends the fix cycle behind #1566, #1790, #1794 and #1844. **Goal:** extraction emits a per-file binding table. Resolution consumes it and never rescans raw source to answer "is X exported", "what does N bind to in F", or "is this receiver a known thing". Every resolver predicate that reads source today is replaced by a lookup. @@ -97,11 +97,23 @@ The binding table is emitted by the kernel walkers only. This is why the kernel- ## 3. Phases -### Phase 0: precision in the eval runner +### Phase 0: precision in the eval runner — DONE 2026-09-12 -- Add a precision score to `__tests__/evaluation/scoring.ts`: for a fixed corpus (vite, vitest, svelte, rollup, the ones the PR bodies used), a checked-in list of known-wrong edges that must stay absent and known-right edges that must stay present. This is the gate every later phase runs against. +- `__tests__/evaluation/scoring.ts` gains `scoreEdgeCase`: an `EdgeCase` names a `kind`, a target endpoint (file suffix + symbol name) and optionally a source endpoint, and expects the edge `absent` (known-wrong) or `present` (control). The recall scorers cannot see a false edge; this can. `__tests__/evaluation-edge-scoring.test.ts` pins the scorer on a synthetic project with the #1713 bare-import and #1746 sealed-module shapes, and proves it flags a violated case and a missing endpoint. +- `__tests__/evaluation/edge-cases.ts` encodes the edges the PR bodies named, on the exact commits they measured (full SHAs, since `git fetch` of an arbitrary commit needs one): vite `8492422b` (#1713 self-import and `getEnv`, #1718 the two fuzzy nested calls, #1746 the sealed `defineConfig` → `test-stacktrace.js::vite`, plus a relative-import control), vitest `7c818153` (`evaluatedModules`), svelte `5895c637` (`bundle`). #1844's regression is already pinned by `ts-chained-receiver.test.ts`; rollup's PR table listed counts only, no endpoints, so it has no cases yet. +- `npm run eval:precision -- ` (`precision-runner.ts`) fetches the pinned commit, indexes it, scores the cases, prints the resolved-edge histogram by resolver (the LOST/GAINED methodology the PRs used) and writes `results/precision---.json`. The reports are committed as the baseline. -Exit: the LOST/GAINED tables from #1713, #1718, #1746 and #1844 are encoded as tests. +Baseline at codegraph `1498c2ff` (kernel-only branch, after Phase 5): + +| Corpus | Index | Edges | fuzzy | Cases | +|---|---|---|---|---| +| vite | 3.3 s | 28,893 | 16 | 5 absent held, 1 control held | +| vitest | 5.8 s | 75,035 | 28 | 1 absent held | +| svelte | 7.9 s | 70,432 | 159 | 1 absent held | + +Reading it: the resolution PRs' removals hold on the current engine. The `fuzzy` counts are the number to watch through Phases 1 to 3; the plan's claim is that they fall toward zero as bindings replace guesses, and any `absent` case flipping to found is a regression the recall scorers would never report. + +Exit met: the LOST/GAINED tables that named endpoints are encoded and green; the runner reports the histogram every later phase compares against. ### Phase 1: emit bindings for TS/JS diff --git a/package.json b/package.json index 03cde2572..4903f7eed 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "test:watch": "vitest", "test:eval": "vitest run __tests__/evaluation/", "eval": "npm run build && npx tsx __tests__/evaluation/runner.ts", + "eval:precision": "npx tsx __tests__/evaluation/precision-runner.ts", "clean": "node -e \"const fs=require('fs');fs.rmSync('dist',{recursive:true,force:true})\"" }, "keywords": [