test(eval): precision scoring and a pinned-corpus edge gate for resolution

__tests__/evaluation-edge-scoring.test.ts

@@ -0,0 +1,80 @@
+/**
+ * The precision scorer (__tests__/evaluation/scoring.ts, scoreEdgeCase) on a
+ * synthetic project that reproduces the shapes the resolution PR chain
+ * removed: a bare npm import that must not fuzzy-bind to a same-named
+ * project symbol (#1713) and a sealed module whose locals must not be
+ * cross-file candidates (#1746), plus a relative import as the `present`
+ * control. This pins that the scorer can SEE a false edge — the recall
+ * scorers cannot — before the binding-model plan changes any resolver.
+ */
+import { describe, it, expect, beforeAll, afterAll } from 'vitest';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import CodeGraph from '../src/index';
+import { scoreEdgeCase } from './evaluation/scoring';
+import type { EdgeCase } from './evaluation/types';
+
+let dir: string;
+let cg: CodeGraph;
+
+beforeAll(async () => {
+  dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-precision-'));
+  const write = (rel: string, body: string) => {
+    fs.mkdirSync(path.dirname(path.join(dir, rel)), { recursive: true });
+    fs.writeFileSync(path.join(dir, rel), body);
+  };
+  write('package.json', JSON.stringify({ name: 'precision-fixture', dependencies: { 'magic-string': '1' } }));
+  // #1713 shape: a bare import whose name collides with a project function.
+  write('src/consumer.ts', "import { bundle } from 'magic-string';\nimport { helper } from './util';\nexport function run() { return bundle(helper()); }\n");
+  write('scripts/build.ts', 'export function bundle(): string { return "x"; }\n');
+  write('src/util.ts', 'export function helper(): number { return 1; }\n');
+  // #1746 shape: a module that imports but exports nothing.
+  write('sealed.ts', "import { helper } from './src/util';\nconst widget = helper();\n");
+  write('src/other.ts', "import { helper } from './util';\nexport function use() { return widget + helper(); }\n");
+  cg = await CodeGraph.init(dir, { index: true });
+  cg.resolveReferences();
+});
+afterAll(() => {
+  cg.close();
+  fs.rmSync(dir, { recursive: true, force: true });
+});
+
+const base = { corpus: 'synthetic', source: 'test', why: '' } as const;
+
+describe('scoreEdgeCase', () => {
+  it('holds an absent case when the bare import does not bind to the project symbol', () => {
+    const c: EdgeCase = { ...base, id: 'bare', kind: 'imports', from: { file: 'src/consumer.ts', name: 'consumer.ts' }, to: { file: 'scripts/build.ts', name: 'bundle' }, expect: 'absent' };
+    const r = scoreEdgeCase(c, cg);
+    expect(r.found).toEqual([]);
+    expect(r.pass).toBe(true);
+  });
+
+  it('holds an absent case when a sealed module local is not reached cross-file', () => {
+    const c: EdgeCase = { ...base, id: 'sealed', kind: 'references', from: { file: 'src/other.ts', name: 'use' }, to: { file: 'sealed.ts', name: 'widget' }, expect: 'absent' };
+    const r = scoreEdgeCase(c, cg);
+    expect(r.pass).toBe(true);
+  });
+
+  it('holds a present control for a relative import', () => {
+    const c: EdgeCase = { ...base, id: 'control', kind: 'imports', from: { file: 'src/consumer.ts', name: 'consumer.ts' }, to: { file: 'src/util.ts', name: 'helper' }, expect: 'present' };
+    const r = scoreEdgeCase(c, cg);
+    expect(r.found.length).toBeGreaterThan(0);
+    expect(r.pass).toBe(true);
+  });
+
+  it('reports a violated absent case when the edge exists', () => {
+    // Use the control edge as if it were known-wrong: the scorer must flag it.
+    const c: EdgeCase = { ...base, id: 'violated', kind: 'imports', from: { file: 'src/consumer.ts', name: 'consumer.ts' }, to: { file: 'src/util.ts', name: 'helper' }, expect: 'absent' };
+    const r = scoreEdgeCase(c, cg);
+    expect(r.pass).toBe(false);
+    expect(r.found[0]?.resolvedBy).toBeDefined();
+  });
+
+  it('fails a present control whose endpoint is not in the graph, and says which', () => {
+    const c: EdgeCase = { ...base, id: 'missing', kind: 'imports', from: { file: 'src/consumer.ts', name: 'consumer.ts' }, to: { file: 'src/nowhere.ts', name: 'nothing' }, expect: 'present' };
+    const r = scoreEdgeCase(c, cg);
+    expect(r.pass).toBe(false);
+    expect(r.missingEndpoints).toEqual(['to src/nowhere.ts:nothing']);
+  });
+});

__tests__/evaluation/edge-cases.ts

@@ -0,0 +1,132 @@
+/**
+ * Known-wrong and known-right edges on pinned real repositories
+ * (docs/design/resolution-binding-model-plan.md, Phase 0).
+ *
+ * Every `absent` case is an edge a resolution PR measured and removed on a
+ * real corpus; the endpoints are the ones named in that PR's body. They must
+ * stay gone. The `present` cases are controls: edges the same PRs reported
+ * unchanged, so a change that silently kills resolution fails here rather
+ * than looking like a precision win.
+ *
+ * Run with `npm run eval:precision -- <corpus>` (precision-runner.ts), which
+ * fetches the pinned commit, indexes it, scores every case, and records the
+ * resolved-edge histogram for LOST/GAINED comparison across builds.
+ */
+import type { EdgeCase } from './types.js';
+
+export interface PrecisionCorpus {
+  key: string;
+  repo: string;
+  /** Full SHA: GitHub only serves an arbitrary commit to `git fetch` by its full id. */
+  commit: string;
+  note: string;
+}
+
+export const PRECISION_CORPORA: Record<string, PrecisionCorpus> = {
+  vite: {
+    key: 'vite',
+    repo: 'https://github.com/vitejs/vite.git',
+    commit: '8492422b8f110625a90c702f42f30784e8cf19dc',
+    note: 'The corpus #1713, #1718 and #1746 measured (LOST 4 / 12 / 157, GAINED 0).',
+  },
+  vitest: {
+    key: 'vitest',
+    repo: 'https://github.com/vitest-dev/vitest.git',
+    commit: '7c818153add03b0bca54453a54e76961dd5be18d',
+    note: '#1713: 34 fuzzy edges from bare imports removed, GAINED 0.',
+  },
+  svelte: {
+    key: 'svelte',
+    repo: 'https://github.com/sveltejs/svelte.git',
+    commit: '5895c637b04dc8667020c8d326807c3f3a984472',
+    note: '#1713: 5 fuzzy edges removed, GAINED 0.',
+  },
+};
+
+export const edgeCases: EdgeCase[] = [
+  // --- #1713: a bare (npm / builtin) import must never fuzzy-match a project symbol ---
+  {
+    id: 'vite-bare-import-self-edge',
+    corpus: 'vite',
+    kind: 'imports',
+    from: { file: 'packages/vite/src/node/optimizer/scan.ts', name: 'scan.ts' },
+    to: { file: 'packages/vite/src/node/optimizer/scan.ts', name: 'scan' },
+    expect: 'absent',
+    source: '#1713',
+    why: "`import { scan } from 'rolldown/experimental'` resolved onto the importing file's OWN `scan` — a self-edge.",
+  },
+  {
+    id: 'vite-bare-import-getEnv',
+    corpus: 'vite',
+    kind: 'imports',
+    from: { file: 'packages/vite/src/node/config.ts', name: 'config.ts' },
+    to: { file: 'packages/vite/src/node/plugins/importAnalysis.ts', name: 'getEnv' },
+    expect: 'absent',
+    source: '#1713',
+    why: "`import { getEnv } from '@vitejs/devtools/config'` resolved onto an unrelated project `getEnv`.",
+  },
+  {
+    id: 'vitest-bare-import-evaluatedModules',
+    corpus: 'vitest',
+    kind: 'imports',
+    to: { file: '.ts', name: 'evaluatedModules' },
+    expect: 'absent',
+    source: '#1713',
+    why: "`import type { EvaluatedModules } from 'vite/module-runner'` resolved (case-insensitively) onto the method VitestMocker::evaluatedModules in 17 files.",
+  },
+  {
+    id: 'svelte-bare-import-bundle',
+    corpus: 'svelte',
+    kind: 'imports',
+    to: { file: 'scripts/generate-browser-support.ts', name: 'bundle' },
+    expect: 'absent',
+    source: '#1713',
+    why: "`import MagicString, { Bundle } from 'magic-string'` resolved onto the script function `bundle` in 5 files.",
+  },
+
+  // --- #1718 (supersedes #1709): fuzzy may reject a unique guess, never manufacture one ---
+  {
+    id: 'vite-fuzzy-nested-resolveConfig-getEnv',
+    corpus: 'vite',
+    kind: 'calls',
+    from: { file: 'packages/vite/src/node/config.ts', name: 'resolveConfig' },
+    to: { file: 'packages/vite/src/node/plugins/importAnalysis.ts', name: 'getEnv' },
+    expect: 'absent',
+    source: '#1718',
+    why: 'A fuzzy call edge onto a nested function the call site cannot lexically reach.',
+  },
+  {
+    id: 'vite-fuzzy-nested-build-scan',
+    corpus: 'vite',
+    kind: 'calls',
+    from: { file: 'packages/vite/src/node/optimizer/scan.ts', name: 'build' },
+    to: { file: 'packages/vite/src/node/optimizer/scan.ts', name: 'scan' },
+    expect: 'absent',
+    source: '#1718',
+    why: 'The `calls` edge that followed the self-import above.',
+  },
+
+  // --- #1746 / #1719: a module that imports but exports nothing is sealed ---
+  {
+    id: 'vite-sealed-module-defineConfig',
+    corpus: 'vite',
+    kind: 'imports',
+    from: { file: 'vite.config.ts', name: 'vite.config.ts' },
+    to: { file: 'playground/ssr-html/test-stacktrace.js', name: 'vite' },
+    expect: 'absent',
+    source: '#1746',
+    why: "Every `import { defineConfig } from 'vite'` across the playground resolved onto a module-scope `const vite` in a file that exports nothing (157 edges).",
+  },
+
+  // --- Controls: edges the PRs reported unchanged ---
+  {
+    id: 'vite-control-relative-import',
+    corpus: 'vite',
+    kind: 'imports',
+    from: { file: 'packages/vite/src/node/config.ts', name: 'config.ts' },
+    to: { file: 'packages/vite/src/node/logger.ts', name: 'createLogger' },
+    expect: 'present',
+    source: 'control',
+    why: "`import { createLogger } from './logger'` is a relative project import; the import resolver must still bind it.",
+  },
+];

__tests__/evaluation/precision-runner.ts

@@ -0,0 +1,130 @@
+/**
+ * Precision gate on a pinned real repository
+ * (docs/design/resolution-binding-model-plan.md, Phase 0).
+ *
+ *   npm run eval:precision -- vite            # fetch + index + score
+ *   EVAL_REPOS=/path/to/dir npm run eval:precision -- vite
+ *
+ * Fetches PRECISION_CORPORA[<corpus>] at its pinned commit (shallow, into
+ * $EVAL_REPOS or $TMPDIR/codegraph-precision/<corpus>), indexes it with the
+ * library, scores every edge case for the corpus, prints the resolved-edge
+ * histogram by resolver (the LOST/GAINED methodology the PRs used), and
+ * writes a JSON report next to the recall reports. Exit 1 when any `absent`
+ * case is violated or any `present` control is missing.
+ */
+import { execFileSync } from 'child_process';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { DatabaseSync } from 'node:sqlite';
+import { CodeGraph } from '../../src/index.js';
+import { scoreEdgeCase } from './scoring.js';
+import { edgeCases, PRECISION_CORPORA } from './edge-cases.js';
+import type { EdgeCaseResult, PrecisionReport } from './types.js';
+
+const corpusKey = process.argv[2];
+const corpus = corpusKey ? PRECISION_CORPORA[corpusKey] : undefined;
+if (!corpus) {
+  console.error(`usage: npx tsx __tests__/evaluation/precision-runner.ts <${Object.keys(PRECISION_CORPORA).join('|')}>`);
+  process.exit(2);
+}
+
+const reposDir = process.env.EVAL_REPOS ?? path.join(os.tmpdir(), 'codegraph-precision');
+const repoDir = path.join(reposDir, corpus.key);
+
+function sh(cmd: string, args: string[], cwd?: string, quiet = false): string {
+  return execFileSync(cmd, args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', quiet ? 'ignore' : 'inherit'] }).trim();
+}
+
+function fetchPinned(): void {
+  if (!fs.existsSync(path.join(repoDir, '.git'))) {
+    fs.mkdirSync(repoDir, { recursive: true });
+    sh('git', ['init', '-q'], repoDir);
+    sh('git', ['remote', 'add', 'origin', corpus!.repo], repoDir);
+  }
+  const head = (() => { try { return sh('git', ['rev-parse', 'HEAD'], repoDir, true); } catch { return ''; } })();
+  if (!head.startsWith(corpus!.commit)) {
+    console.log(`fetching ${corpus!.repo} @ ${corpus!.commit}`);
+    sh('git', ['fetch', '-q', '--depth', '1', 'origin', corpus!.commit], repoDir);
+    sh('git', ['checkout', '-q', '--force', 'FETCH_HEAD'], repoDir);
+  }
+}
+
+function resolvedByHistogram(): { resolvedBy: Record<string, number>; edges: number } {
+  const db = new DatabaseSync(path.join(repoDir, '.codegraph', 'codegraph.db'), { readOnly: true });
+  try {
+    const rows = db
+      .prepare(`SELECT COALESCE(json_extract(metadata, '$.resolvedBy'), '(none)') AS r, COUNT(*) AS n FROM edges GROUP BY r ORDER BY n DESC`)
+      .all() as Array<{ r: string; n: number }>;
+    const resolvedBy: Record<string, number> = {};
+    let edges = 0;
+    for (const { r, n } of rows) { resolvedBy[r] = n; edges += n; }
+    return { resolvedBy, edges };
+  } finally {
+    db.close();
+  }
+}
+
+async function run(): Promise<void> {
+  fetchPinned();
+  const commit = sh('git', ['rev-parse', '--short', 'HEAD'], repoDir);
+  let codegraphSha = 'unknown';
+  try { codegraphSha = sh('git', ['rev-parse', '--short', 'HEAD']); } catch {}
+
+  console.log(`\nCodeGraph precision — ${corpus!.key} @ ${commit} (codegraph ${codegraphSha})`);
+  console.log(`${corpus!.note}\n`);
+
+  fs.rmSync(path.join(repoDir, '.codegraph'), { recursive: true, force: true });
+  const cg = CodeGraph.initSync(repoDir);
+  const t0 = performance.now();
+  await cg.indexAll();
+  console.log(`indexed in ${((performance.now() - t0) / 1000).toFixed(1)} s`);
+
+  const results: EdgeCaseResult[] = [];
+  for (const c of edgeCases.filter((c) => c.corpus === corpus!.key)) {
+    const r = scoreEdgeCase(c, cg);
+    results.push(r);
+    const status = r.pass ? '\x1b[32mHELD\x1b[0m' : '\x1b[31mVIOLATED\x1b[0m';
+    console.log(`  ${c.id.padEnd(44)} ${c.expect.padEnd(7)} ${status}  (${c.source})`);
+    for (const f of r.found) {
+      const from = cg.getNode(f.from); const to = cg.getNode(f.to);
+      console.log(`      ${from?.filePath}:${from?.name} → ${to?.filePath}:${to?.name} [${f.resolvedBy ?? '?'}]`);
+    }
+    if (r.missingEndpoints.length) console.log(`      endpoints not in graph: ${r.missingEndpoints.join('; ')}`);
+  }
+  cg.close();
+
+  const { resolvedBy, edges } = resolvedByHistogram();
+  console.log(`\nedges: ${edges}`);
+  for (const [k, v] of Object.entries(resolvedBy)) console.log(`  ${k.padEnd(18)} ${v}`);
+
+  const absent = results.filter((r) => r.expect === 'absent');
+  const present = results.filter((r) => r.expect === 'present');
+  const report: PrecisionReport = {
+    timestamp: new Date().toISOString(),
+    corpus: corpus!.key,
+    repo: corpus!.repo,
+    commit,
+    codegraphSha,
+    summary: {
+      absentCases: absent.length,
+      absentHeld: absent.filter((r) => r.pass).length,
+      presentCases: present.length,
+      presentHeld: present.filter((r) => r.pass).length,
+      unjudgeable: results.filter((r) => r.missingEndpoints.length > 0 && r.expect === 'absent' && !r.pass).length,
+    },
+    resolvedBy,
+    edges,
+    results,
+  };
+  const resultsDir = path.join(__dirname, 'results');
+  fs.mkdirSync(resultsDir, { recursive: true });
+  const file = path.join(resultsDir, `precision-${corpus!.key}-${commit}-${codegraphSha}.json`);
+  fs.writeFileSync(file, JSON.stringify(report, null, 2));
+  const failed = results.filter((r) => !r.pass).length;
+  console.log(`\nSUMMARY: absent ${report.summary.absentHeld}/${absent.length} held, present ${report.summary.presentHeld}/${present.length} held`);
+  console.log(`Report saved: ${file}`);
+  process.exit(failed > 0 ? 1 : 0);
+}
+
+run().catch((err) => { console.error(err); process.exit(1); });

__tests__/evaluation/results/precision-svelte-5895c63-1498c2ff.json

@@ -0,0 +1,35 @@
+{
+  "timestamp": "2026-09-12T03:42:07.350Z",
+  "corpus": "svelte",
+  "repo": "https://github.com/sveltejs/svelte.git",
+  "commit": "5895c63",
+  "codegraphSha": "1498c2ff",
+  "summary": {
+    "absentCases": 1,
+    "absentHeld": 1,
+    "presentCases": 0,
+    "presentHeld": 0,
+    "unjudgeable": 0
+  },
+  "resolvedBy": {
+    "(none)": 37891,
+    "exact-match": 16731,
+    "import": 8468,
+    "qualified-name": 4751,
+    "instance-method": 1234,
+    "file-path": 685,
+    "function-ref": 299,
+    "framework": 214,
+    "fuzzy": 159
+  },
+  "edges": 70432,
+  "results": [
+    {
+      "caseId": "svelte-bare-import-bundle",
+      "expect": "absent",
+      "pass": true,
+      "found": [],
+      "missingEndpoints": []
+    }
+  ]
+}