Add unattended scalability and generation fidelity benchmarks

.gitignore

@@ -41,3 +41,7 @@ spatialdsl_deploy.key
 # Claude Code
 .claude/
 CLAUDE.md
+
+# Benchmark outputs and the local Python environment
+benchmarks/results/
+benchmarks/.venv/

benchmarks/README.md

@@ -0,0 +1,55 @@
+# Evaluation scripts
+
+Two unattended scripts measure SpatialDSL Studio without a browser, an HTTP
+server, or a database. Both call the backend engines in-process and write their
+results to `benchmarks/results/` (git-ignored).
+
+## Prerequisites
+
+```bash
+cd backend && npm ci                       # the scripts reuse the backend's ts-node and engines
+cd ../benchmarks
+python3 -m venv .venv && .venv/bin/pip install usd-core   # fidelity only; provides pxr
+```
+
+## Scalability
+
+```bash
+cd benchmarks && npm run scalability
+```
+
+Generates seeded synthetic Smart Warehouse models with 100, 500, 1,000, 2,500,
+5,000, and 10,000 placed elements on a grid with random yaw, about 5 percent of
+them placed to violate one of the example spatial rules. For each size it
+reports the median of 5 runs (after one warm-up run) of spatial rule
+evaluation, the full backend model validation engine, and code generation of
+the Smart Warehouse OpenUSD project, plus the serialized model size. The script
+fails if the number of rule issues differs from the number of seeded
+violations.
+
+Outputs: `results/scalability.csv`, `results/scalability.md`,
+`results/scalability.json` (including CPU, RAM, OS, and Node.js version).
+Expected runtime: about 20 seconds on an Apple M2.
+
+## Generation fidelity
+
+```bash
+cd benchmarks && npm run fidelity          # same as: .venv/bin/python fidelity.py [K] [seed]
+```
+
+Applies K = 200 seeded pose edits (move, rotate, elevate) to the bundled Smart
+Warehouse model and regenerates after each edit. Every generated layered USD
+stage is opened with pxr; each element's `xformOp:translate`, composed world
+translation, `xformOp:rotateZ`, `xformOp:scale`, and `spatialDsl:*` attributes
+are compared with the model after the millimetre-to-metre conversion
+(tolerance 1e-6). The Visual Components location files and the Omniverse
+`warehouse_layout.json` are compared the same way; the layout JSON gives a yaw
+only for drones (the other groups are axis-aligned footprints for the
+simulator), so their yaw is compared in USD and Visual Components only. The
+first and last composed stages are checked with the `UsdValidation` framework,
+and the unedited model is generated twice to confirm byte-identical outputs.
+The script exits non-zero on any mismatch, nondeterminism, or USD validation
+error or warning.
+
+Outputs: `results/fidelity.json`, `results/fidelity.md`, and the generated
+bundles in `results/fidelity/`. Expected runtime: under 10 seconds.

benchmarks/fidelity-generate.ts

@@ -0,0 +1,140 @@
+/**
+ * Generation half of the fidelity check (run by fidelity.py).
+ *
+ * Applies K seeded pose edits (move, rotate, elevate) to the bundled Smart
+ * Warehouse model one after another, regenerates the layered OpenUSD bundle
+ * and the Visual Components location files through the backend generation
+ * engine after each edit, and writes the outputs plus the expected poses.
+ * It also generates the unedited model twice and records whether the outputs
+ * are byte-identical.
+ *
+ *   npm run fidelity:generate -- <out-dir> [K] [seed]
+ */
+import './lib/offline-env';
+import fs from 'fs';
+import path from 'path';
+import { CodeGenerationTemplate, Metamodel, Model, ModelElement } from '../shared/types';
+import { codeGenerationEngine } from '../backend/src/services/pipeline/code-generation.engine';
+import { seededRandom } from './lib/synthetic-warehouse';
+
+const repoRoot = path.resolve(__dirname, '..');
+const outDir = path.resolve(process.argv[2] || path.join(__dirname, 'results', 'fidelity'));
+const editCount = Number(process.argv[3] || 200);
+const seed = Number(process.argv[4] || 20261011);
+
+const readJson = (relativePath: string): any => JSON.parse(fs.readFileSync(path.join(repoRoot, relativePath), 'utf8'));
+const metamodel = readJson('frontend/src/examples/data/smart-warehouse-metamodel.json') as Metamodel;
+const baseModel = readJson('frontend/src/examples/data/smart-warehouse-model.json') as Model;
+const omniverseTemplates = readJson('examples/codegen-projects/smart-warehouse-omniverse-project.json').templates as CodeGenerationTemplate[];
+const visualComponentsTemplates = readJson('examples/codegen-projects/smart-warehouse-visual-components-project.json').templates as CodeGenerationTemplate[];
+
+const USD_LAYERS = ['warehouse_layout.usda', 'warehouse_assets.usda', 'warehouse_physics.usda', 'warehouse_simulation.usda'];
+const OMNIVERSE_FILES = [...USD_LAYERS, 'warehouse_layout.json'];
+const VISUAL_COMPONENTS_FILES = [
+  'PathwayAreaLocation.json', 'RobotLocation.json', 'ConveyorLocation.json', 'OutputLocation.json', 'ChargingStationLocation.json',
+];
+
+const classNames = new Map(metamodel.classes.map(metaClass => [metaClass.id, metaClass.name]));
+const isPlaced = (element: ModelElement): boolean => Boolean(element.presentation?.position3D && element.presentation?.size3D);
+const round3 = (value: number): number => Math.round(value * 1000) / 1000;
+
+const generateAll = (model: Model) => ({
+  omniverse: codeGenerationEngine.generate(model, metamodel, omniverseTemplates),
+  visualComponents: codeGenerationEngine.generate(model, metamodel, visualComponentsTemplates),
+});
+
+const expectedPoses = (model: Model) => model.elements.filter(isPlaced).map(element => ({
+  id: element.id,
+  name: String(element.style?.name || element.name || element.id),
+  className: classNames.get(element.modelElementId) || element.modelElementId,
+  x: element.presentation!.position3D!.x,
+  y: element.presentation!.position3D!.y,
+  z: element.presentation!.position3D!.z,
+  rotationZ: element.presentation!.rotationZ ?? 0,
+  widthMm: element.presentation!.size3D!.widthMm,
+  heightMm: element.presentation!.size3D!.heightMm,
+  depthMm: element.presentation!.size3D!.depthMm,
+}));
+
+const writeOutputs = (directory: string, model: Model, outputs: ReturnType<typeof generateAll>): void => {
+  fs.mkdirSync(path.join(directory, 'visual-components'), { recursive: true });
+  const byName = (files: typeof outputs.omniverse) => new Map(files.map(file => [file.filename, file.content]));
+  const omniverse = byName(outputs.omniverse);
+  const visualComponents = byName(outputs.visualComponents);
+  OMNIVERSE_FILES.forEach(name => fs.writeFileSync(path.join(directory, name), omniverse.get(name)!));
+  VISUAL_COMPONENTS_FILES.forEach(name => fs.writeFileSync(path.join(directory, 'visual-components', name), visualComponents.get(name)!));
+  // Layers reference assets as @omniverse-assets/...@ relative to the bundle root.
+  const assetLink = path.join(directory, 'omniverse-assets');
+  if (!fs.existsSync(assetLink)) fs.symlinkSync(path.join(repoRoot, 'examples', 'omniverse-assets'), assetLink, 'dir');
+  fs.writeFileSync(path.join(directory, 'expected.json'), `${JSON.stringify(expectedPoses(model), null, 2)}\n`);
+};
+
+fs.rmSync(outDir, { recursive: true, force: true });
+fs.mkdirSync(outDir, { recursive: true });
+
+// Determinism: the unedited model generated twice must give byte-identical files.
+const first = generateAll(baseModel);
+const second = generateAll(JSON.parse(JSON.stringify(baseModel)));
+const allFiles = (outputs: ReturnType<typeof generateAll>) => [...outputs.omniverse, ...outputs.visualComponents];
+const differing = allFiles(first)
+  .filter((file, index) => {
+    const other = allFiles(second)[index];
+    return !other || other.filename !== file.filename || other.content !== file.content || other.contentHash !== file.contentHash;
+  })
+  .map(file => file.filename);
+const determinism = {
+  files: allFiles(first).length,
+  byteIdentical: differing.length === 0 && allFiles(first).length === allFiles(second).length,
+  differing,
+  contentHashes: Object.fromEntries(allFiles(first).map(file => [file.filename, file.contentHash])),
+};
+
+const random = seededRandom(seed);
+const model: Model = JSON.parse(JSON.stringify(baseModel));
+const placed = model.elements.filter(isPlaced);
+const edits: Array<Record<string, unknown>> = [];
+const started = Date.now();
+writeOutputs(path.join(outDir, 'edit-000'), model, first);
+
+for (let index = 1; index <= editCount; index += 1) {
+  const element = placed[Math.floor(random() * placed.length)];
+  const presentation = element.presentation!;
+  const before = { ...presentation.position3D!, rotationZ: presentation.rotationZ ?? 0 };
+  const roll = random();
+  let kind: string;
+  if (roll < 1 / 3) {
+    kind = 'move';
+    presentation.position3D = {
+      ...presentation.position3D!,
+      x: round3(presentation.position3D!.x + (random() * 6000 - 3000)),
+      y: round3(presentation.position3D!.y + (random() * 6000 - 3000)),
+    };
+  } else if (roll < 2 / 3) {
+    kind = 'rotate';
+    presentation.rotationZ = round3(random() * 360 - 180);
+  } else {
+    kind = 'elevate';
+    presentation.position3D = { ...presentation.position3D!, z: round3(random() * 3000) };
+  }
+  edits.push({
+    index,
+    kind,
+    element: element.style?.name,
+    className: classNames.get(element.modelElementId),
+    before,
+    after: { ...presentation.position3D!, rotationZ: presentation.rotationZ ?? 0 },
+  });
+  writeOutputs(path.join(outDir, `edit-${String(index).padStart(3, '0')}`), model, generateAll(model));
+}
+
+fs.writeFileSync(path.join(outDir, 'manifest.json'), `${JSON.stringify({
+  seed,
+  edits: editCount,
+  placedElements: placed.length,
+  omniverseFiles: OMNIVERSE_FILES,
+  visualComponentsFiles: VISUAL_COMPONENTS_FILES,
+  determinism,
+  generationSeconds: Math.round((Date.now() - started) / 100) / 10,
+  editLog: edits,
+}, null, 2)}\n`);
+process.stdout.write(`Generated ${editCount + 1} bundles in ${outDir}; determinism byte-identical: ${determinism.byteIdentical}\n`);