From b248519a6e90b81771a32b8d70ee595c3a7fdba0 Mon Sep 17 00:00:00 2001 From: John Travers Date: Tue, 27 Oct 2026 06:24:58 +0000 Subject: [PATCH 01/13] SEP 38: track complete runs per hardware; retire SEP 30's standard runner Replace the standard benchmark runner with python -m benchmarks track: the tracking set's complete runs, each in its own worker with the library's default integrator and tolerances and a fixed noise seed, timed through the driver's run loop after an untimed warm-up, at one thread and the production width. A hardware slug (CPU model and cores, no host name) names a history CSV under benchmarks/history/. ab moves onto the same worker and gains --length. Every SEP 30 case stays runnable by name; standard mode, references, native coverage, export and the batch exercise are removed with their tests. Co-Authored-By: Claude Opus 5.5 --- benchmarks/__main__.py | 553 +---------- benchmarks/ab.py | 231 ++--- benchmarks/ab_worker.py | 118 --- benchmarks/batching.py | 261 ----- benchmarks/builders.py | 344 ++++--- benchmarks/collection.py | 784 --------------- benchmarks/comparison.py | 413 -------- benchmarks/coverage_reports.py | 189 ---- benchmarks/coverage_worker.py | 81 -- benchmarks/guides.py | 16 +- benchmarks/history.py | 142 +++ benchmarks/native_coverage.py | 517 ---------- benchmarks/planning.py | 198 ---- benchmarks/publication.py | 319 ------ benchmarks/qualification.py | 419 -------- benchmarks/records.py | 169 +--- benchmarks/references.py | 604 ------------ benchmarks/registry.py | 923 +++++++----------- benchmarks/runner.py | 183 ++++ benchmarks/schedules.py | 455 --------- benchmarks/system.py | 288 ++---- benchmarks/telemetry.py | 64 -- benchmarks/track.py | 302 ++++++ benchmarks/views.py | 199 ---- benchmarks/worker.py | 598 +++--------- .../dev/seps/sep-0038-performance-tracking.md | 286 ++++++ pyproject.toml | 2 +- tests/test_benchmark_ab.py | 104 +- tests/test_benchmark_cases.py | 158 +++ tests/test_benchmark_coverage.py | 309 ------ tests/test_benchmark_exercises.py | 177 +--- tests/test_benchmark_guides.py | 78 +- tests/test_benchmark_publication.py | 162 --- tests/test_benchmark_references.py | 456 --------- tests/test_benchmark_telemetry.py | 87 +- tests/test_benchmark_track.py | 225 +++++ tests/test_standard_benchmark_cases.py | 103 -- tests/test_standard_benchmarks.py | 724 -------------- 38 files changed, 2268 insertions(+), 8973 deletions(-) delete mode 100644 benchmarks/ab_worker.py delete mode 100644 benchmarks/batching.py delete mode 100644 benchmarks/collection.py delete mode 100644 benchmarks/comparison.py delete mode 100644 benchmarks/coverage_reports.py delete mode 100644 benchmarks/coverage_worker.py create mode 100644 benchmarks/history.py delete mode 100644 benchmarks/native_coverage.py delete mode 100644 benchmarks/planning.py delete mode 100644 benchmarks/publication.py delete mode 100644 benchmarks/qualification.py delete mode 100644 benchmarks/references.py create mode 100644 benchmarks/runner.py delete mode 100644 benchmarks/schedules.py create mode 100644 benchmarks/track.py delete mode 100644 benchmarks/views.py create mode 100644 docs/dev/seps/sep-0038-performance-tracking.md create mode 100644 tests/test_benchmark_cases.py delete mode 100644 tests/test_benchmark_coverage.py delete mode 100644 tests/test_benchmark_publication.py delete mode 100644 tests/test_benchmark_references.py create mode 100644 tests/test_benchmark_track.py delete mode 100644 tests/test_standard_benchmark_cases.py delete mode 100644 tests/test_standard_benchmarks.py diff --git a/benchmarks/__main__.py b/benchmarks/__main__.py index a8838ae5..ca53439a 100644 --- a/benchmarks/__main__.py +++ b/benchmarks/__main__.py @@ -2,539 +2,76 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -"""Expose the standard benchmark command without importing scientific libraries.""" +"""Expose the benchmark commands without importing scientific libraries (SEP 38).""" from __future__ import annotations import argparse -import dataclasses import sys -import uuid -from pathlib import Path +from benchmarks import history from benchmarks.ab import ab, ab_options -from benchmarks.collection import Collection, collect, prepared_python -from benchmarks.comparison import compare, completed -from benchmarks.planning import Selection, matrix -from benchmarks.records import ( - LOCAL_ROOT, - Record, - SuiteError, - attempts, - canonical, - distribution, - read_record, - write_csv, -) -from benchmarks.registry import ADAPTIVE, CASES, SETS, TRACKS, select -from benchmarks.system import profile_for, save_profile +from benchmarks.records import SuiteError +from benchmarks.registry import select +from benchmarks.runner import plan_rows, show_plan +from benchmarks.system import physical_cores +from benchmarks.track import selection_options, steppers, track, track_options -__all__ = [ - "compare_paths", - "coverage_options", - "describe", - "diagnose", - "diagnostic_options", - "main", - "manage", - "new_collection", - "parser_for", - "resume_request", - "selected_plan", - "selectors", - "show_comparison", - "show_results", -] - - -def selectors(parser: argparse.ArgumentParser) -> None: - """Share selectors and measurement policy between planning and collection.""" - parser.add_argument("--case", action="append", default=[]) - parser.add_argument("--set", action="append", choices=SETS, default=[]) - parser.add_argument("--all", action="store_true") - parser.add_argument("--mode", choices=("quick", "standard"), default="quick") - parser.add_argument( - "--backend", choices=("numpy_ref", "numba_cpu", "native"), default="native" - ) - parser.add_argument("--threads") - parser.add_argument("--track", action="append", choices=TRACKS, default=[]) - parser.add_argument("--machine-profile", type=Path) - parser.add_argument("--references", type=Path, default=LOCAL_ROOT / "references") - parser.add_argument("--diagnostic", action="store_true") - - -def diagnostic_options(parser: argparse.ArgumentParser) -> None: - """Declare the separate thread/binding diagnostic's short workload and limits.""" - parser.add_argument("--case", default="smoke", choices=tuple(CASES)) - parser.add_argument( - "--backend", choices=("numpy_ref", "numba_cpu", "native"), default="native" - ) - parser.add_argument("--threads", default="1,production") - parser.add_argument("--machine-profile", type=Path) - parser.add_argument("--out", type=Path, required=True) - parser.add_argument("--timeout", type=float, default=300) - parser.add_argument("--memory-limit", type=int) +__all__ = ["main", "parser_for"] def parser_for() -> argparse.ArgumentParser: - """Declare the CLI and its explicit qualification/profile operations.""" - parser = argparse.ArgumentParser(description="SEP 30: bounded standard benchmarks.") - commands = parser.add_subparsers(dest="command", required=True) - listing = commands.add_parser( - "list", help="plan without scientific imports or translation" - ) - selectors(listing) - listing.add_argument("--json", action="store_true") - run = commands.add_parser( - "run", help="collect quick or standard fresh-process samples" - ) - selectors(run) - run.add_argument("--out", type=Path) - run.add_argument("--resume", type=Path) - run.add_argument("--dry-run", action="store_true") - run.add_argument("--baseline-python") - run.add_argument("--candidate-python", default=sys.executable) - run.add_argument("--seed", type=int, default=30) - run.add_argument( - "--timeout", type=float, default=1800.0, help="attempt limit, seconds" - ) - run.add_argument("--memory-limit", type=int, help="attempt memory limit, bytes") - run.add_argument("--allow-unestimated", action="store_true") - run.add_argument( - "--allow-accuracy-failure", - action="store_true", - help="retain failed accuracy checks and continue timing; exploratory only", - ) - run.add_argument( - "--force", - action="store_true", - help="skip guard waiting; exploratory results only", - ) - comparison = commands.add_parser( - "compare", help="compare complete stored standard selections" - ) - comparison.add_argument("baseline", type=Path) - comparison.add_argument("candidate", type=Path, nargs="?") - comparison.add_argument("--aa-control", type=Path) - comparison.add_argument("--json", action="store_true") - publication = commands.add_parser( - "export", help="write a sanitized public bundle without publishing it" - ) - publication.add_argument("source", type=Path) - publication.add_argument("--out", type=Path, required=True) - publication.add_argument("--alias", required=True, help="public machine alias") - diagnostic = commands.add_parser( - "diagnose", - help="inspect threads and binding separately from timing comparisons", - ) - diagnostic_options(diagnostic) - profile = commands.add_parser( - "profile", help="save a declared production width before collecting" - ) - profile.add_argument("out", type=Path) - profile.add_argument("--alias", required=True) - profile.add_argument("--production", type=int, required=True) - profile.add_argument( - "--policy", - choices=("budget", "all-pools-N", "blas1", "as-installed"), - default="budget", - ) - qualification = commands.add_parser( - "qualify", help="prepare/check numerical references separately from timing" - ) - selectors(qualification) - qualification.add_argument("--out", type=Path) - qualification.add_argument("--timeout", type=float, default=3600.0) - qualification.add_argument("--memory-limit", type=int) - qualification.add_argument( - "--accuracy-bound", - type=float, - help="reviewed field/band envelope; defaults to approved case adaptive bounds", - ) - qualification.add_argument("--reference-revision", type=int) - qualification.add_argument("--policy-revision", type=int) - qualification.add_argument( - "--recheck", - type=Path, - help="reassess retained reference evidence with a revised bound; no simulation", - ) - qualification.add_argument( - "--supersedes", - type=Path, - help="previous reference bundle; preserves its receipts", + """Declare the list, track, ab and history commands.""" + parser = argparse.ArgumentParser( + description="SEP 38: track complete runs per hardware, or A/B two builds." ) - qualification.add_argument( - "--revision-note", help="reason for replacing a reference or accuracy policy" - ) - qualification.add_argument( - "--reviewed", - action="store_true", - help="with --recheck, confirm retained reset/history/statistics evidence " - "was reviewed for standard comparison", - ) - ab_options( - commands.add_parser( - "ab", help="time two builds on a few cases quickly; exploratory only" - ) + commands = parser.add_subparsers(dest="command", required=True) + selection_options( + commands.add_parser("list", help="print the plan and its estimated time") ) - coverage_options( - commands.add_parser( - "coverage", help="collect separate Clang CPU coverage and execution labels" - ) + track_options( + commands.add_parser("track", help="time complete runs; record per hardware") ) + ab_options(commands.add_parser("ab", help="compare two builds; exploratory")) + recorded = commands.add_parser("history", help="summarise recorded runs") + recorded.add_argument("slug", nargs="?", help="hardware slug; default: all") + recorded.add_argument("--threads", help="one width only") return parser -def coverage_options(coverage: argparse.ArgumentParser) -> None: - """Declare the explicit native coverage diagnostic.""" - coverage.add_argument("--case", action="append", default=[]) - coverage.add_argument("--set", action="append", choices=SETS, default=[]) - coverage.add_argument("--all", action="store_true") - coverage.add_argument("--track", action="append", choices=TRACKS, default=[]) - coverage.add_argument("--threads", default="1") - coverage.add_argument("--machine-profile", type=Path) - coverage.add_argument("--diagnostic", action="store_true") - coverage.add_argument("--native-root", type=Path, required=True) - coverage.add_argument("--llvm-bin", type=Path, required=True) - coverage.add_argument("--python", default=sys.executable) - coverage.add_argument( - "--out", - type=Path, - help="local report directory; defaults beneath ignored .benchmarks/", - ) - coverage.add_argument("--timeout", type=float, default=1800) - coverage.add_argument("--memory-limit", type=int) - - -def describe(plan: dict) -> None: - """Print admitted and excluded configurations without pretending costs are known.""" - print( - f"{plan['mode']}: production width {plan['profile']['production']}; " - f"{plan['rounds']} round(s), at most {plan['maximum_rounds']}" - ) - for row in plan["rows"]: - values = row["configuration"] - estimate = row["estimate_bytes"] - memory = "unknown" if estimate is None else f"{estimate / 2**30:.2f} GiB" - detail = row["reason"] or f"reference {row['reference']}; cost unknown" - print( - f"{values['case']:24} {values['track']:20} {values['threads']:3} threads " - f"{row['status']}: {detail}; memory budget {memory}" - ) - print( - "Estimates require compatible pilot receipts; " - "reference preparation is separate." - ) - - -def show_results(plan: Record, rows: list[Record]) -> None: - """Show all-sample timings and failures without hiding unqualified results.""" - groups: dict[tuple[str, str], list[Record]] = {} - for row in completed(rows): - groups.setdefault((row["configuration_id"], row["arm"]), []).append(row) - print("\nWarm times are all-sample medians; startup includes the first window.") - print( - f"{'case':24} {'track':20} {'N':>3} {'arm':9} " - f"{'warm / s':>10} {'start / s':>10} {'RSS / GiB':>10} {'clean':>7} check" - ) - for item in plan["rows"]: - configuration = item["configuration"] - for (key, arm), group in groups.items(): - if key != item["id"]: - continue - timed = [row for row in group if row.get("seconds") is not None] - warm = distribution([row["seconds"] for row in timed]).median - start = distribution( - [row["startup_seconds"] for row in timed if row.get("startup_seconds")] - ).median - peaks = [ - row["peak_rss_bytes"] for row in timed if row.get("peak_rss_bytes") - ] - peak = max(peaks) / 2**30 if peaks else None - values = [ - "-" if value is None else f"{value:.4f}" - for value in (warm, start, peak) - ] - clean = sum(row.get("clean") is True for row in group) - checks = sorted( - {row.get("accuracy", {}).get("status", row["status"]) for row in group} - ) - print( - f"{configuration['case']:24} {configuration['track']:20} " - f"{configuration['threads']:3} {arm:9} " - f"{values[0]:>10} {values[1]:>10} {values[2]:>10} " - f"{clean:3}/{len(group):<3} {','.join(checks)}" - ) - for row in group: - if row["status"] != "ok": - detail = row.get("error", row.get("admission", "see accuracy.csv")) - print(f" {row['status']}: {detail}") - print( - "Missing references do not block timing. " - "Quick results have no adoption verdict." - ) - - -def show_comparison(result: Record) -> None: - """Print descriptive changes even when adoption evidence is unavailable.""" - print(f"Comparison: {result['status']}") - print("Changes use all-sample medians; negative means less time.") - for row in result.get("configurations", []): - ratio = row.get("all_ratio") - change = "unavailable" if ratio is None else f"{100 * (ratio - 1):+.2f}%" - print( - f"{row['case']:24} {row['track']:20} {row['threads']:3} threads " - f"{change:>12} {row['status']}" - ) - if result.get("reason"): - print(result["reason"]) - - -def compare_paths( - baseline: Path, - candidate: Path | None, - aa_path: Path | None = None, - *, - as_json: bool = False, -) -> int: - """Refuse incompatible manifests before comparing their durable samples.""" - before = read_record(baseline / "manifest.json")["plan"] - rows = attempts(baseline) - if candidate is not None: - after = read_record(candidate / "manifest.json")["plan"] - if before["profile"] != after["profile"] or [ - row["id"] for row in before["rows"] - ] != [row["id"] for row in after["rows"]]: - print( - canonical( - { - "status": "incompatible", - "reason": "selected workload or machine profile differs", - } - ) - ) - return 1 - rows = [{**row, "arm": "baseline"} for row in rows] + [ - {**row, "arm": "candidate"} for row in attempts(candidate) - ] - before = { - **before, - "exploratory": before.get("exploratory", False) - or after.get("exploratory", False), - } - bands = None - if aa_path is not None: - control = read_record(aa_path / "aa.json") - environment = before["environments"].get( - "baseline", before["environments"]["candidate"] - ) - if ( - control["profile"] != before["profile"] - or control["baseline"] != environment - ): - raise SuiteError("A/A profile or baseline environment identity differs") - bands = control["configurations"] - result = compare(before, rows, paired=candidate is None, aa_bands=bands) - if as_json: - print(canonical(result)) - else: - show_comparison(result) - usable = result["status"] in ("passed", "exploratory") and all( - row["status"] not in ("incomplete", "incompatible", "numerically-invalid") - for row in result.get("configurations", []) - ) - return 0 if usable else 1 - - -def resume_request( - args: argparse.Namespace, tokens: list[str] -) -> tuple[Record, Collection]: - """Restore a saved request while refusing explicit changes to its policy.""" - path: Path = args.resume - saved = read_record(path / "manifest.json")["plan"] - explicit = {token.split("=", 1)[0] for token in tokens if token.startswith("--")} - selection = { - "--case", - "--set", - "--all", - "--mode", - "--backend", - "--threads", - "--track", - "--machine-profile", - "--diagnostic", - "--out", - } - if explicit & selection: - raise SuiteError( - "--resume restores the saved selection and profile; " - "use a new output to change them" - ) - options = dict(saved["options"]) - for key, value in options.items(): - if f"--{key.replace('_', '-')}" not in explicit: - continue - supplied = getattr(args, key) - if key == "references": - supplied = str(supplied.resolve()) - elif key in ("baseline_python", "candidate_python"): - supplied = prepared_python(supplied) - if supplied != value: - raise SuiteError(f"resume {key} differs; use a new output") - options["output"] = path - options["references"] = Path(options["references"]) - options["resume"] = True - return saved, Collection(**options) - - -def selected_plan(args: argparse.Namespace) -> Record: - """Resolve fresh selectors without constructing a simulation.""" - cases = select(tuple(args.case), tuple(args.set), all_cases=args.all) - profile = profile_for(args.machine_profile, standard=args.mode == "standard") - plan = matrix( - cases, - profile, - Selection( - args.mode, - args.backend, - args.threads, - tuple(args.track), - args.references, - args.diagnostic, - ), - ) - plan["fixed_assessment"] = bool(args.track) and all( - track.startswith("fixed-") for track in args.track - ) - return plan - - -def new_collection(args: argparse.Namespace) -> Collection: - """Validate execution limits before admitting a new collection.""" - if args.timeout <= 0 or (args.memory_limit is not None and args.memory_limit <= 0): - raise SuiteError("timeout and memory limit must be positive") - if args.allow_unestimated and args.memory_limit is None: - raise SuiteError("--allow-unestimated requires --memory-limit") - return Collection( - args.out or LOCAL_ROOT / "runs" / uuid.uuid4().hex, - args.baseline_python, - args.candidate_python, - args.seed, - args.timeout, - args.memory_limit, - args.force, - args.allow_unestimated, - False, - args.references, - args.allow_accuracy_failure, - ) - - -def diagnose(args: argparse.Namespace) -> int: - """Collect one short diagnostic per width without an adoption comparison.""" - if args.timeout <= 0 or (args.memory_limit is not None and args.memory_limit <= 0): - raise SuiteError("timeout and memory limit must be positive") - case = CASES[args.case] - profile = profile_for(args.machine_profile, standard=False) - plan = matrix( - (case,), - profile, - Selection( - backend=args.backend, - threads=args.threads, - tracks=(ADAPTIVE[1] if ADAPTIVE[1] in case.tracks else case.tracks[0],), - diagnostic=True, - ), - ) - plan.update(purpose="thread-binding-diagnostic", thread_diagnostics=True) - result = collect( - plan, - Collection( - args.out, timeout=args.timeout, memory_limit=args.memory_limit, force=True - ), - ) - rows = attempts(args.out) - write_csv( - args.out / "threads.csv", - [ - { - "attempt_id": row["attempt_id"], - "configuration_id": row["configuration_id"], - **thread, - } - for row in rows - for thread in (row.get("thread_diagnostics") or {}).get("threads", []) - ], - ) - print(f"Thread/binding diagnostics: {args.out.resolve()}") - return 0 if result["execution_ok"] else 1 - - -def manage(args: argparse.Namespace) -> int | None: - """Handle stored-bundle and profile commands before constructing a plan.""" - if args.command == "export": - from benchmarks.publication import export_public # noqa: PLC0415 - - export_public(args.source, args.out, args.alias) - print(f"Public export: {args.out.resolve()}") - return 0 - if args.command == "profile": - print( - canonical( - dataclasses.asdict( - save_profile(args.out, args.alias, args.production, args.policy) - ) - ) - ) - return 0 - if args.command == "compare": - return compare_paths( - args.baseline, args.candidate, args.aa_control, as_json=args.json - ) - if args.command == "diagnose": - return diagnose(args) - if args.command == "coverage": - from benchmarks.native_coverage import coverage # noqa: PLC0415 - - return coverage(args) - return None - - def main(argv: list[str] | None = None) -> int: - """Default to a quick collection, with explicit planning and review commands.""" + """Run one command; with none, plan the default tracking run.""" tokens = list(sys.argv[1:] if argv is None else argv) - if not tokens or (tokens[0].startswith("-") and tokens[0] not in ("-h", "--help")): - tokens.insert(0, "run") + if not tokens: + tokens = ["list"] args = parser_for().parse_args(tokens) try: - managed = manage(args) - if managed is not None: - return managed + if args.command == "track": + return track(args) if args.command == "ab": return ab(args) - options = None - if args.command == "run" and args.resume is not None: - plan, options = resume_request(args, tokens) - else: - plan = selected_plan(args) - if args.command == "list" or (args.command == "run" and args.dry_run): - if getattr(args, "json", False): - print(canonical(plan)) - else: - describe(plan) + if args.command == "history": + paths = ( + [history.HISTORY / f"{args.slug}.csv"] + if args.slug + else sorted(history.HISTORY.glob("*.csv")) + ) + if not paths: + print("No recorded runs.") + for path in paths: + history.show(path, args.threads) return 0 - if args.command == "qualify": - from benchmarks.qualification import qualify # noqa: PLC0415 - - return qualify(plan, args) - if options is None: - options = new_collection(args) - result = collect(plan, options) - show_results(plan, attempts(options.output)) - print(f"Results: {options.output.resolve()} ({result['status']})") - return 0 if result["execution_ok"] else 1 + rows, skipped = plan_rows( + select(tuple(args.case), tuple(args.set), all_cases=args.all), + steppers(args.stepper), + args.threads, + args.production or physical_cores(), + backend=args.backend, + ) + show_plan(rows, skipped, args.repeats) + return 0 except (SuiteError, OSError, ValueError) as error: - print(f"benchmark suite: {error}", file=sys.stderr) + print(f"benchmarks: {error}", file=sys.stderr) return 2 diff --git a/benchmarks/ab.py b/benchmarks/ab.py index c82e188f..28bceebb 100644 --- a/benchmarks/ab.py +++ b/benchmarks/ab.py @@ -2,24 +2,20 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -"""Compare two builds quickly: one worker per build per round, cases built once.""" +"""Compare two builds quickly: alternating workers per configuration and round.""" from __future__ import annotations import argparse -import json import math import os import random import statistics -import subprocess import sys import time import uuid from pathlib import Path -from benchmarks.collection import freeze_harness, prepared_python -from benchmarks.planning import Selection, matrix from benchmarks.records import ( LOCAL_ROOT, Record, @@ -28,17 +24,23 @@ canonical, now, ) -from benchmarks.registry import SETS, TRACKS, select -from benchmarks.system import admission, guard, profile_for, thread_environment +from benchmarks.registry import select +from benchmarks.runner import ( + freeze_harness, + launch_worker, + plan_rows, + prepared_python, + receipt, +) +from benchmarks.system import eligible_cpus, guard, physical_cores, thread_environment +from benchmarks.track import selection_options, steppers __all__ = [ "ARMS", "ab", "ab_options", - "admitted", "arm_order", "field_difference", - "launch_worker", "show_summary", "summarise", ] @@ -47,27 +49,21 @@ def ab_options(parser: argparse.ArgumentParser) -> None: - """Declare the exploratory A/B selection, builds, rounds and repetitions.""" - parser.add_argument("--case", action="append", default=[]) - parser.add_argument("--set", action="append", choices=SETS, default=[]) - parser.add_argument("--all", action="store_true") - parser.add_argument( - "--backend", choices=("numpy_ref", "numba_cpu", "native"), default="native" - ) - parser.add_argument("--threads") - parser.add_argument("--track", action="append", choices=TRACKS, default=[]) - parser.add_argument("--machine-profile", type=Path) - parser.add_argument("--diagnostic", action="store_true") + """Declare the A/B selection, builds, length, rounds and repetitions.""" + selection_options(parser) parser.add_argument("--baseline-python", required=True) parser.add_argument("--candidate-python", default=sys.executable) - parser.add_argument("--rounds", type=int, default=3) parser.add_argument( - "--repeats", type=int, default=5, help="timed windows per case per worker" + "--length", + type=float, + default=1.0, + help="fraction of each guide to run, for quick iteration", ) - parser.add_argument("--seed", type=int, default=30) + parser.add_argument("--rounds", type=int, default=3) + parser.add_argument("--seed", type=int, default=30, help="shuffles the case order") parser.add_argument("--out", type=Path) parser.add_argument( - "--timeout", type=float, default=1800.0, help="worker limit, seconds" + "--timeout", type=float, default=3600.0, help="per-worker limit, seconds" ) parser.add_argument("--json", action="store_true") @@ -77,65 +73,8 @@ def arm_order(round_index: int) -> tuple[str, ...]: return ARMS if round_index % 2 == 0 else ARMS[::-1] -def admitted(plan: Record) -> tuple[list[Record], list[str]]: - """Keep ready singleton rows that fit in memory; explain every other row.""" - rows: list[Record] = [] - skipped: list[str] = [] - for row in plan["rows"]: - values = row["configuration"] - label = f"{values['case']} {values['track']} {values['threads']} threads" - if row["status"] != "ready": - skipped.append(f"{label}: {row['reason']}") - elif values["case"] == "batch": - skipped.append(f"{label}: the batch exercise needs the standard runner") - elif not (memory := admission(row["estimate_bytes"], None))["admitted"]: - skipped.append(f"{label}: {memory['reason']}") - else: - rows.append(row) - return rows, skipped - - -def launch_worker( - python: str, - request: Record, - environment: dict[str, str], - directory: Path, - timeout: float, -) -> list[Record]: - """Run one worker to completion and return its protocol events.""" - directory.mkdir(parents=True, exist_ok=True) - path = directory / "request.json" - atomic_record(path, request, immutable=True) - harness = Path(request["harness_root"]).resolve() - with ( - (directory / "stdout.log").open("w", encoding="utf-8") as stdout, - (directory / "stderr.log").open("w", encoding="utf-8") as stderr, - ): - try: - subprocess.run( - [python, "-m", "benchmarks.ab_worker", str(path.resolve())], - cwd=harness, - env={**environment, "PYTHONPATH": str(harness)}, - stdout=stdout, - stderr=stderr, - timeout=timeout, - check=False, - ) - except subprocess.TimeoutExpired: - stdout.write(canonical({"event": "failed", "error": "worker timeout"})) - events = [] - for line in (directory / "stdout.log").read_text(encoding="utf-8").splitlines(): - try: - event = json.loads(line) - except json.JSONDecodeError: - continue - if isinstance(event, dict) and "event" in event: - events.append(event) - return events - - def field_difference(baseline: Path, candidate: Path) -> float | None: - """Return the relative L2 difference of two builds' retained fields.""" + """Return the relative L2 difference of two builds' final fields.""" if not (baseline.exists() and candidate.exists()): return None import numpy as np # noqa: PLC0415 - only after both workers saved fields @@ -171,7 +110,7 @@ def summarise(rows: list[Record], samples: list[Record], fields: Path) -> list[R { "id": row["id"], "case": values["case"], - "track": values["track"], + "stepper": values["stepper"], "threads": values["threads"], "rounds": len(rounds), "baseline_seconds": statistics.median(medians["baseline"].values()) @@ -200,13 +139,13 @@ def summarise(rows: list[Record], samples: list[Record], fields: Path) -> list[R def show_summary(summary: list[Record]) -> None: """Print one line per configuration; the ratio is exploratory, not a verdict.""" print( - f"\n{'case':22} {'track':20} {'N':>3} {'base / ms':>10} {'cand / ms':>10} " - f"{'ratio':>7} {'range':>15} {'field diff':>10} work" + f"\n{'case':26} {'stepper':12} {'N':>3} {'base / s':>9} {'cand / s':>9} " + f"{'ratio':>7} {'range':>13} {'field diff':>10} work" ) for row in summary: - def ms(value: float | None) -> str: - return "-" if value is None else f"{1e3 * value:.3f}" + def seconds(value: float | None) -> str: + return "-" if value is None else f"{value:.3f}" ratio = "-" if row["ratio"] is None else f"{row['ratio']:.4f}" spread = ( @@ -218,9 +157,10 @@ def ms(value: float | None) -> str: "-" if row["field_difference"] is None else f"{row['field_difference']:.1e}" ) print( - f"{row['case']:22} {row['track']:20} {row['threads']:3} " - f"{ms(row['baseline_seconds']):>10} {ms(row['candidate_seconds']):>10} " - f"{ratio:>7} {spread:>15} {field:>10} " + f"{row['case']:26} {row['stepper']:12} {row['threads']:3} " + f"{seconds(row['baseline_seconds']):>9} " + f"{seconds(row['candidate_seconds']):>9} " + f"{ratio:>7} {spread:>13} {field:>10} " f"{'same' if row['same_work'] else 'differs'}" ) for error in row["failures"]: @@ -236,29 +176,22 @@ def ms(value: float | None) -> str: def ab(args: argparse.Namespace) -> int: - """Collect and summarise a fast paired comparison of two prepared builds.""" + """Collect and summarise a paired comparison of two prepared builds.""" if args.rounds < 1 or args.repeats < 1 or args.timeout <= 0: raise SuiteError("rounds, repeats and timeout must be positive") started = time.perf_counter() - cases = select(tuple(args.case), tuple(args.set), all_cases=args.all) - profile = profile_for(args.machine_profile, standard=False) - plan = matrix( - cases, - profile, - Selection( - "quick", - args.backend, - args.threads, - tuple(args.track), - None, - args.diagnostic, - ), + rows, skipped = plan_rows( + select(tuple(args.case), tuple(args.set), all_cases=args.all), + steppers(args.stepper), + args.threads, + args.production or physical_cores(), + backend=args.backend, + fraction=args.length, ) - rows, skipped = admitted(plan) for reason in skipped: print(f"skipped {reason}") if not rows: - raise SuiteError("no admitted configuration in the selection") + raise SuiteError("no configuration in the selection") output: Path = args.out or LOCAL_ROOT / "ab" / uuid.uuid4().hex if output.exists() and any(output.iterdir()): raise SuiteError("output already exists; choose a new --out") @@ -275,8 +208,8 @@ def ab(args: argparse.Namespace) -> int: "pythons": pythons, "rounds": args.rounds, "repeats": args.repeats, + "length": args.length, "seed": args.seed, - "profile": plan["profile"], "rows": rows, "skipped": skipped, "harness_files": freeze_harness(harness), @@ -285,66 +218,47 @@ def ab(args: argparse.Namespace) -> int: ) samples: list[Record] = [] provenance: dict[str, list[Record]] = {arm: [] for arm in ARMS} - diagnostics: list[Record] = [] - widths = list(dict.fromkeys(row["configuration"]["threads"] for row in rows)) + allocation = eligible_cpus() for round_index in range(args.rounds): shuffled = list(rows) random.Random(args.seed + round_index).shuffle(shuffled) - for width in widths: - selected = [ - row for row in shuffled if row["configuration"]["threads"] == width - ] + shuffled.sort(key=lambda row: row["configuration"]["threads"]) + for row in shuffled: + width = row["configuration"]["threads"] for arm in arm_order(round_index): - quiet = guard(tuple(profile.allocation), standard=False) - request = { - "rows": selected, - "repeats": args.repeats, - "harness_root": str(harness.resolve()), - "fields": str((output / "fields" / arm).resolve()) - if round_index == 0 - else None, - } - launched = time.perf_counter() - events = launch_worker( + quiet = guard(allocation)["quiet"] + events, elapsed = launch_worker( pythons[arm], - request, - thread_environment(width, profile.policy, dict(os.environ)), - output / "raw" / f"r{round_index}-t{width}-{arm}", + { + "configuration": row["configuration"], + "repeats": args.repeats, + "fields": str( + (output / "fields" / arm / f"{row['id']}.npy").resolve() + ) + if round_index == 0 + else None, + }, + thread_environment(width, dict(os.environ)), + output / "raw" / f"r{round_index}-{arm}-{row['id']}", args.timeout, + harness, ) - elapsed = time.perf_counter() - launched - received = {e["id"]: e for e in events if e["event"] == "row"} - failure = next( - (e["error"] for e in events if e["event"] == "failed"), - "no receipt from the worker", - ) - provenance[arm].extend( - e["provenance"] for e in events if e["event"] == "provenance" - ) - diagnostics.extend( - {"arm": arm, "round": round_index, **item} - for e in events - if e["event"] == "diagnostics" - for item in e["diagnostics"] + result = receipt(events) + if result.get("provenance"): + provenance[arm].append(result["provenance"]) + samples.append( + { + **{k: v for k, v in result.items() if k != "provenance"}, + "id": row["id"], + "arm": arm, + "round": round_index, + "quiet": quiet, + } ) - for row in selected: - event = received.get( - row["id"], {"status": "failed", "error": failure} - ) - samples.append( - { - **{k: v for k, v in event.items() if k != "event"}, - "id": row["id"], - "arm": arm, - "round": round_index, - "quiet": quiet["quiet"], - } - ) - done = sum(row["id"] in received for row in selected) print( - f"round {round_index + 1}/{args.rounds}, {width} threads, " - f"{arm}: {done}/{len(selected)} cases in {elapsed:.1f} s" - + ("" if quiet["quiet"] else " (machine busy at launch)"), + f"round {round_index + 1}/{args.rounds} {row['id']} {arm}: " + f"{result['status']} in {elapsed:.1f} s" + + ("" if quiet else " (machine busy at launch)"), flush=True, ) summary = summarise(rows, samples, output / "fields") @@ -354,7 +268,6 @@ def ab(args: argparse.Namespace) -> int: "samples": samples, "summary": summary, "provenance": provenance, - "diagnostics": diagnostics, "collection_seconds": time.perf_counter() - started, "busy_samples": sum(not item["quiet"] for item in samples), }, diff --git a/benchmarks/ab_worker.py b/benchmarks/ab_worker.py deleted file mode 100644 index e0fc6f01..00000000 --- a/benchmarks/ab_worker.py +++ /dev/null @@ -1,118 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Measure several configurations of one build in one process for a fast A/B.""" - -from __future__ import annotations - -import sys -import time -import traceback -from pathlib import Path - -from benchmarks.records import Record, read_record -from benchmarks.worker import ( - Diagnostics, - emit, - measure, - prepare, - retained_fields, - runtime, - save_plane, -) - -__all__ = ["execute", "main", "measure_configuration"] - - -def measure_configuration( - configuration: Record, repeats: int, fields: Path | None -) -> Record: - """Construct once, warm up once, then time ``repeats`` resets of the window. - - Each repetition restores the initial field and the declared lead-in outside - the timer, exactly as a standard sample's warm repetitions do. The retained - fields of the last repetition are saved to ``fields`` when it is given. - """ - import numpy as np # noqa: PLC0415 - - from benchmarks.builders import build # noqa: PLC0415 - from solprop.backends import select # noqa: PLC0415 - - started = time.perf_counter() - simulation = build(configuration) - problem = simulation.translate() - engine = select( - configuration["backend"], - problem, - grid=simulation.grid, - modes=simulation.modes, - profiles=simulation.profiles, - ) - construction = time.perf_counter() - started - try: - output = prepare(simulation, problem, engine, configuration) - first = measure(engine, problem, configuration, output) - output.close() - repetitions = [] - for _ in range(repeats): - output = prepare(simulation, problem, engine, configuration) - repetitions.append(measure(engine, problem, configuration, output)) - if len(repetitions) < repeats: - output.close() - if not configuration["track"].startswith("adaptive-"): - # Fixed tracks observe only after timing, as in a standard sample. - save_plane(engine, problem, output) - result = retained_fields(output) - output.close() - if fields is not None: - fields.parent.mkdir(parents=True, exist_ok=True) - np.save(fields, result, allow_pickle=False) - return { - "construction_seconds": construction, - "first_window_seconds": first["seconds"], - "repetitions": repetitions, - "finite": bool(np.all(np.isfinite(result))), - } - finally: - engine.close() - - -def execute(request: Record, diagnostics: Diagnostics) -> None: - """Emit the runtime identity, then one receipt per configuration as it ends.""" - rows = request["rows"] - provenance = runtime(rows[0]["configuration"]["backend"]) - emit("provenance", provenance=provenance) - fields = request.get("fields") - for row in rows: - diagnostics.case = row["configuration"]["case"] - diagnostics.phase = "measurement" - try: - result = measure_configuration( - row["configuration"], - request["repeats"], - None if fields is None else Path(fields) / f"{row['id']}.npy", - ) - emit("row", id=row["id"], status="ok", **result) - except Exception as error: # one failed configuration keeps the others - traceback.print_exc(file=sys.stderr) - emit("row", id=row["id"], status="failed", error=f"{error!r}") - - -def main() -> int: - """Execute the JSON request passed by the A/B controller.""" - diagnostics = Diagnostics() - try: - request = read_record(Path(sys.argv[1])) - with diagnostics.capture(): - execute(request, diagnostics) - except Exception as error: # worker boundary: the controller records the failure - traceback.print_exc(file=sys.stderr) - emit("failed", error=f"{type(error).__name__}: {error}") - return 1 - emit("diagnostics", diagnostics=diagnostics.records()) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/batching.py b/benchmarks/batching.py deleted file mode 100644 index ebd9d687..00000000 --- a/benchmarks/batching.py +++ /dev/null @@ -1,261 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Measure the real fixed-schedule batch route with independent member evidence.""" - -from __future__ import annotations - -import time -from dataclasses import dataclass, replace -from typing import cast - -import numpy as np - -from benchmarks.builders import fixed_schedule -from benchmarks.exercises import fiber -from benchmarks.records import Record, SuiteError, identity, metadata_value -from benchmarks.references import field_errors -from benchmarks.registry import CASES -from benchmarks.worker import ( - Diagnostics, - ResetWitness, - accuracy, - emit, - observed_runtime, - reset_columns, - reset_differences, - resolved, - source_calls, - timer_resolution, -) -from solprop.backends import select, select_batch -from solprop.backends.batch import LockstepRunner -from solprop.backends.driver import StepDriver -from solprop.backends.protocol import Backend, BatchBackend -from solprop.driver import Simulation -from solprop.driver.executors.pool import peak_rss -from solprop.driver.run import _power_observation, plane_from_observation -from solprop.output import MemoryOutput -from solprop.output.schema import serialise -from solprop.stepper import PIController, RungeKutta -from solprop.translate.problem import KernelProblem - -__all__ = ["BatchCase", "construct", "execute_batch"] - - -@dataclass(frozen=True) -class BatchCase: - """Retain independently translated members and their shared batch controller.""" - - simulations: tuple[Simulation, ...] - problems: tuple[KernelProblem, ...] - members: tuple[Backend, ...] - engine: BatchBackend - - def reset(self) -> None: - """Reset members and initialise the base march without recreating buffers. - - The batch seam accepts independently loaded member states. Its common - controller can be reinitialised with those same owners after their ordinary - ``set_initial`` calls. Calling only the base initializer preserves Numba's - fused buffers and the native backend's resident descriptors; backend-specific - constructors, FFT planning and translation are not repeated. - """ - for member, problem in zip(self.members, self.problems, strict=True): - member.set_initial(problem.initial.field0, None) - if not isinstance(self.engine, LockstepRunner): - raise SuiteError("batch reset requires the known CPU lockstep controller") - LockstepRunner.__init__( - self.engine, [cast(StepDriver[object], member) for member in self.members] - ) - - def observe(self) -> tuple[MemoryOutput, ...]: - """Convert every member's own field, statistics and history separately.""" - outputs = [] - for simulation, problem, member, observation in zip( - self.simulations, - self.problems, - self.members, - self.engine.observe_members(), - strict=True, - ): - output = MemoryOutput() - output.open(simulation.metadata()) - output.save( - plane_from_observation( - _power_observation(problem, observation), - member.stats(problem.stats_spec), - ) - ) - output.close() - outputs.append(output) - return tuple(outputs) - - def measure(self, stop: float) -> Record: - """Time propagation only and retain actual source counts for every member.""" - before = [source_calls(member) for member in self.members] - started = time.perf_counter_ns() - summary = self.engine.propagate_to(stop) - seconds = (time.perf_counter_ns() - started) * 1e-9 - if summary.z != stop: - raise SuiteError("batch did not reach the declared endpoint") - calls = [ - source_calls(member) - initial - for member, initial in zip(self.members, before, strict=True) - ] - return { - "seconds": seconds, - "accepted": summary.n_steps - summary.n_rejected, - "rejected": summary.n_rejected, - "source_evaluations": sum(calls), - "member_source_evaluations": calls, - "endpoint": stop, - "observation_seconds": None, - } - - -def construct(configuration: Record) -> BatchCase: - """Translate two distinct GNLSE members and select the actual batch backend.""" - simulations, problems, members = [], [], [] - try: - for index in range(2): - simulation = fiber(index) - simulation.settings = replace( - simulation.settings, - controller=PIController( - rtol=configuration["rtol"], atol=configuration["atol"] - ), - integrator=RungeKutta(), - schedule=fixed_schedule(CASES["batch"], configuration), - save_planes=[float(configuration["stop"])], - backend=configuration["backend"], - fields="none", - ) - problem = simulation.translate() - member = select( - configuration["backend"], - problem, - grid=simulation.grid, - modes=simulation.modes, - profiles=simulation.profiles, - ) - members.append(member) - member.set_initial(problem.initial.field0, None) - simulations.append(simulation) - problems.append(problem) - engine = select_batch(members) - return BatchCase(tuple(simulations), tuple(problems), tuple(members), engine) - except BaseException: - for member in members: - member.close() - raise - - -def execute_batch( - request: Record, diagnostics: Diagnostics, provenance: Record, import_seconds: float -) -> Record: - """Use the ordinary receipt protocol for one fixed two-member batch sample.""" - configuration = request["configuration"] - diagnostics.phase = "construction" - started = time.perf_counter() - case = construct(configuration) - construction_seconds = time.perf_counter() - started - try: - diagnostics.phase = "first-window" - first = case.measure(float(configuration["stop"])) - emit("first-window", seconds=first["seconds"]) - fresh = case.observe() if request.get("verify_reset") else None - repetitions = [] - for _ in range(request.get("repetitions", 1)): - diagnostics.phase = "reset" - case.reset() - diagnostics.phase = "measurement" - repetitions.append(case.measure(float(configuration["stop"]))) - peak = peak_rss() - emit("measurement", peak_rss_bytes=peak) - diagnostics.phase = "validation" - started = time.perf_counter() - outputs = case.observe() - repetitions[-1]["observation_seconds"] = time.perf_counter() - started - # One endpoint per member; the leading axis scores members separately, - # so a bright member cannot hide an inaccurate weaker member. - fields = np.concatenate([output.fields(convention="fft") for output in outputs]) - check = accuracy(fields, configuration, request) - overridden = check["status"] == "invalid" and request.get( - "allow_accuracy_failure", False - ) - reset = None - if fresh is not None: - truth = np.concatenate( - [output.fields(convention="fft") for output in fresh] - ) - reset = { - "plane_errors": field_errors(fields, truth), - "history_statistics_noise": { - str(index): reset_differences( - ResetWitness(old.fields(convention="fft"), reset_columns(old)), - new, - ) - for index, (old, new) in enumerate(zip(fresh, outputs, strict=True)) - }, - "fresh_counts": first, - "reset_counts": repetitions, - } - if request.get("save_fields"): - np.save(request["save_fields"], fields, allow_pickle=False) - np.save( - f"{request['save_fields']}.band.npy", - case.problems[0].grids.inband_mask, - allow_pickle=False, - ) - np.save( - f"{request['save_fields']}.omega.npy", - case.problems[0].grids.omega, - allow_pickle=False, - ) - resolution = { - "configuration": configuration, - "field_axis": "member, mode, polarization, frequency", - "members": [ - resolved(simulation, problem, configuration) - for simulation, problem in zip( - case.simulations, case.problems, strict=True - ) - ], - } - return { - "status": "invalid" - if check["status"] == "invalid" and not overridden - else "ok", - "accuracy_overridden": overridden, - "accuracy": check, - "seconds": sum(item["seconds"] for item in repetitions) / len(repetitions), - "repetitions": repetitions, - "peak_rss_bytes": peak, - "reset_evidence": reset, - "pilot_edges": [], - "timer_resolution": timer_resolution(), - "resolution": resolution, - "resolution_hash": identity(resolution), - "provenance": observed_runtime(provenance), - "metadata": metadata_value( - serialise( - { - **case.engine.metadata(), - "members": case.engine.member_metadata(), - "source_evaluations": sum( - source_calls(member) for member in case.members - ), - } - ) - ), - "ledger": case.engine.memory_ledger(), - "first_window_seconds": first["seconds"], - "import_seconds": import_seconds, - "construction_seconds": construction_seconds, - "backend_seconds": None, - "backend_seconds_reason": "included in member construction", - } - finally: - case.engine.close() diff --git a/benchmarks/builders.py b/benchmarks/builders.py index e0a6f4c9..bbfb418a 100644 --- a/benchmarks/builders.py +++ b/benchmarks/builders.py @@ -4,8 +4,10 @@ """Build declared cases after the worker has configured its scientific runtimes. -Definitions are extracted from the public sources recorded by the registry. No -gallery or test module executes here, and no warning filter hides model warnings. +Each case is the interface call a user would write: no tolerance or integrator is +passed where the interface has a default (SEP 37), and every noisy launch fixes +:data:`~benchmarks.registry.SEED`. No gallery or test module executes here, and +no warning filter hides model warnings. """ from __future__ import annotations @@ -14,56 +16,36 @@ from typing import TYPE_CHECKING from benchmarks.records import Record, SuiteError -from benchmarks.registry import ADAPTIVE, CASES, KERR, Case +from benchmarks.registry import CASES, SEED -__all__ = ["build", "capillary", "fixed_schedule", "gnlse", "kerr", "spatial"] +__all__ = ["build", "capillary", "gnlse", "integrator", "kerr", "spatial"] if TYPE_CHECKING: from solprop.driver import Simulation, SimulationSettings - from solprop.stepper import FixedSchedule + from solprop.stepper import Multistep, RungeKutta +FREE_RTOL = 1e-6 +"""The controller tolerance of the Cartesian and line cases. -def fixed_schedule(case: Case, configuration: Record) -> FixedSchedule: - """Keep the exact measured step count and any unexecuted full-guide tail.""" - import numpy as np # noqa: PLC0415 - worker-only imports +Free space has no default tolerance (SEP 37 measured none), so these cases keep the +value SEP 30 ran them at. +""" - from solprop.stepper import FixedSchedule # noqa: PLC0415 - stop = float(configuration["stop"]) - edges = configuration.get("schedule", []) - if edges: - expected_steps = 4 if case.name == "envelope_plasma" else 10 - if len(edges) != expected_steps + 1 or edges[0] != 0 or edges[-1] > case.length: - raise SuiteError( - f"a fixed pilot schedule needs {expected_steps} " - "entrance steps within the guide" - ) - else: - if case.step is None: - raise SuiteError("fixed track requires a qualified schedule") - # Freeze the integer count instead of allowing a rounded quotient to add - # a step. The Kerr count includes its separate one-step lead-in. - edges = np.linspace(0.0, stop, round(stop / case.step) + 1).tolist() - lengths = np.diff(edges) - if not np.all(np.isfinite(lengths)) or np.any(lengths <= 0): - raise SuiteError("fixed schedule edges must be finite and strictly increasing") - if edges[-1] < case.length: - corners = (5e-3,) if case.name == "moving_solid_guide" else () - tail = FixedSchedule.uniform( - edges[-1], - case.length, - min(lengths), - save_planes=[ - *(corner for corner in corners if corner > edges[-1]), - case.length, - ], - ) - lengths = np.concatenate((lengths, tail.steps)) - return FixedSchedule(lengths) +def integrator(stepper: str) -> Multistep | RungeKutta | None: + """Return the integrator a stepper override names; ``None`` keeps the default.""" + from solprop.stepper import Multistep, RungeKutta # noqa: PLC0415 + + return { + "default": None, + "multistep": Multistep(), + "multistep-interaction": Multistep(formulation="interaction"), + "rk": RungeKutta(), + }[stepper] def capillary(name: str) -> Simulation: - """Build the original full guide, independently of the measured interval.""" + """Build a capillary case on its resolved or mode-averaged route.""" from solprop.stats import default_spec # noqa: PLC0415 simulation = _resolved_capillary(name) @@ -94,37 +76,43 @@ def _resolved_capillary(name: str) -> Simulation: fwhm=7.5e-15, energy=275e-6, modes=4 if name == "rdw" else 1, - rtol=1e-6, - atol=1e-10, - backend="numpy_ref", - seed=20261027, + seed=SEED, + ) + if name == "ctc": + return Simulation.from_capillary( + 16e-6, + 0.13, + "H2", + 40.0, + lambda0=515e-9, + wavelength_limits=(220e-9, 4e-6), + trange=2e-12, + fwhm=400e-15, + energy=10e-6, + modes=3, + loss=0.0, + seed=SEED, ) - if name in ("ctc", "ctc_reduced"): - full = name == "ctc" + if name == "ctc_reduced": return Simulation.from_capillary( 16e-6, - 0.2 if full else 0.1, + 0.1, "H2", 40.0, lambda0=515e-9, - wavelength_limits=(170e-9, 4e-6) if full else (250e-9, 2e-6), - # Revision 2 of the reduced case: 2 ps, the window a production run of - # this pulse takes (1 ps crowded its 400 fs pulse). - trange=4e-12 if full else 2e-12, + wavelength_limits=(250e-9, 2e-6), + trange=2e-12, fwhm=400e-15, energy=10e-6, - modes=6 if full else 2, + modes=2, raman=True, temperature=294.0, - rtol=1e-6, - atol=1e-10, - backend="numpy_ref", - seed=20261027, + seed=SEED, ) - if name in ("argon_blueshift", "argon_blueshift_averaged"): + if name.startswith("argon_blueshift"): return Simulation.from_capillary( 13e-6, - 0.4, + 0.2, "Ar", 1.7, lambda0=800e-9, @@ -132,12 +120,26 @@ def _resolved_capillary(name: str) -> Simulation: energy=3e-6, loss=False, trange=400e-15, - wavelength_limits=(300e-9, 4e-6), + wavelength_limits=(160e-9, 4e-6), modes=1, - rtol=1e-6, - atol=1e-10, - backend="numpy_ref", - seed=20261027, + envelope=name == "argon_blueshift_envelope", + seed=SEED, + ) + if name == "h2_raman_averaged": + return Simulation.from_capillary( + 20e-6, + 0.4, + "H2", + 10.0, + lambda0=1030e-9, + fwhm=40e-15, + energy=3e-6, + loss=False, + plasma=False, + trange=800e-15, + wavelength_limits=(200e-9, 4e-6), + modes=1, + seed=SEED, ) if name in ("h2_raman_6ps", "h2_raman_6ps_averaged"): return Simulation.from_capillary( @@ -154,10 +156,7 @@ def _resolved_capillary(name: str) -> Simulation: trange=6e-12, wavelength_limits=(400e-9, 4e-6), modes=1, - rtol=1e-6, - atol=1e-10, - backend="numpy_ref", - seed=20261027, + seed=SEED, ) raise SuiteError(f"capillary builder not implemented for {name}") @@ -188,10 +187,7 @@ def gnlse(name: str) -> Simulation: pulseshape="sech", raman=False, shock=False, - rtol=1e-6, - atol=1e-10, - backend="numpy_ref", - seed=20261027, + seed=SEED, ) if name == "gnlse_raman_shock": return Simulation.from_gnlse( @@ -218,10 +214,7 @@ def gnlse(name: str) -> Simulation: peak_power=10e3, raman=True, shock=True, - noise=None, - rtol=1e-6, - atol=1e-10, - backend="numpy_ref", + seed=SEED, ) raise SuiteError(f"GNLSE builder not implemented for {name}") @@ -273,7 +266,7 @@ def kerr(name: str, settings: SimulationSettings) -> Simulation: [KerrResponse(chi3=2.2e-22 * MU0 * EPS0 * C**2, thg=False)], launch, settings=settings, - seed=20261027, + seed=SEED, ) @@ -320,6 +313,8 @@ def spatial(name: str, settings: SimulationSettings) -> Simulation: responses: list[KerrResponse | PlasmaResponse] = [KerrResponse(chi3, thg=False)] if plasma: responses.append(PlasmaResponse("Ar", OptmatRate("Ar", 800e-9))) + if plasma: + settings = dataclasses.replace(settings, ionisation_field_range=(1e7, 1e11)) if name.startswith("radial") or name == "spatial_absorber": return Simulation.from_radial( grid, @@ -330,15 +325,25 @@ def spatial(name: str, settings: SimulationSettings) -> Simulation: responses, pulse, settings=settings, - seed=20261027, + seed=SEED, ) if name == "line" or name.startswith("cartesian_filament"): + from solprop.stepper import PIController # noqa: PLC0415 + size = case.dimensions[0] space = ( LineGrid(1e-3, size) if name == "line" else FreeGrid(6e-4, size, 6e-4, size) ) return Simulation.from_free( - grid, space, medium, responses, pulse, settings=settings, seed=20261027 + grid, + space, + medium, + responses, + pulse, + settings=dataclasses.replace( + settings, controller=PIController(rtol=FREE_RTOL, atol=0.0) + ), + seed=SEED, ) raise SuiteError(f"spatial builder not implemented for {name}") @@ -371,112 +376,93 @@ def _planar(settings: SimulationSettings) -> Simulation: ) +def smoke() -> Simulation: + """Build the runner's own short noiseless fibre.""" + from solprop.driver import Simulation # noqa: PLC0415 + + return Simulation.from_gnlse( + 0.11, + CASES["smoke"].length, + [0.0, 0.0, -11.83e-27], + lambda0=835e-9, + wavelength_limits=(600e-9, 1200e-9), + trange=200e-15, + width=20e-15, + peak_power=10.0, + pulseshape="sech", + raman=False, + shock=False, + n_save=2, + noise=None, + ) + + def build(configuration: Record) -> Simulation: - """Resolve the exact declared integrator and bounded observation itinerary.""" + """Build a case for the configured backend, stepper and observations. + + The tolerances are left to the interface. A stepper override replaces only the + integrator. A case with a fixed step runs its declared schedule with the + low-storage RK4IP march, as free space has no default tolerance. + """ import numpy as np # noqa: PLC0415 from solprop.driver import SimulationSettings # noqa: PLC0415 - from solprop.stepper import ( # noqa: PLC0415 - LowStorageRungeKutta, - Multistep, - PIController, - RungeKutta, - ) + from solprop.stepper import FixedSchedule, LowStorageRungeKutta # noqa: PLC0415 case = CASES[configuration["case"]] - track = configuration["track"] - if case.status != "implemented": - raise SuiteError(f"{case.name}: {case.pending_reason}") - start, stop = float(configuration["start"]), float(configuration["stop"]) - planes = np.linspace(start, stop, int(configuration["observations"]) + 1)[1:] - # Translation sees the original guide endpoint; the benchmark loop only - # visits the selected planes. No gradient or gas-flow law is rescaled. - itinerary: list[float] = sorted( - {*(float(value) for value in planes), case.length, *([start] if start else [])} - ) - integrator = ( - Multistep() - if track == ADAPTIVE[0] - else LowStorageRungeKutta(scheme="rk4ip") - if track == KERR - else RungeKutta() - ) - controller = PIController( - rtol=configuration.get("reference_rtol") or configuration["rtol"] or 1e-6, - atol=configuration["atol"] or 0.0, - ) - schedule = fixed_schedule(case, configuration) if track not in ADAPTIVE else None - settings = SimulationSettings( - controller, - save_planes=itinerary, - dz_max=case.length, - dz_init=case.length * 1e-4, - auto_init=True, - integrator=integrator, - schedule=schedule, - backend=configuration["backend"], - fields=configuration["fields"], - field_planes=(len(planes),) if configuration["fields"] == "planes" else (), - propagator_mode="tabulate" if track == KERR else "auto", - ionisation_field_range=(1e7, 1e11) - if case.builder == "spatial" and "filament" in case.name - else None, - ) - if case.builder in ("capillary", "gnlse", "advanced", "exercise"): - if case.builder == "exercise": - from benchmarks.exercises import exercise # noqa: PLC0415 - - simulation = exercise(case.name) - elif case.builder == "advanced": - from benchmarks.guides import advanced # noqa: PLC0415 - - simulation = advanced(case.name) - else: - simulation = ( - capillary(case.name) - if case.builder == "capillary" - else gnlse(case.name) - ) - simulation.settings = dataclasses.replace( - simulation.settings, - controller=controller, - integrator=integrator, - schedule=schedule, - save_planes=itinerary, - backend=configuration["backend"], - fields=configuration["fields"], - field_planes=(len(planes),) if configuration["fields"] == "planes" else (), - ) - return simulation - if case.builder == "kerr": - return kerr(case.name, settings) - if case.builder == "spatial": - return spatial(case.name, settings) - if case.builder == "smoke": - from solprop.driver import Simulation # noqa: PLC0415 - - simulation = Simulation.from_gnlse( - 0.11, - case.length, - [0.0, 0.0, -11.83e-27], - lambda0=835e-9, - wavelength_limits=(600e-9, 1200e-9), - trange=200e-15, - width=20e-15, - peak_power=10.0, - pulseshape="sech", - raman=False, - shock=False, - n_save=2, - noise=None, - ) - simulation.settings = dataclasses.replace( - simulation.settings, - controller=controller, - integrator=integrator, - schedule=schedule, + stop = float(configuration["stop"]) + # A shortened run (ab --length) saves its last plane where it stops. + itinerary = np.linspace(0.0, stop, int(configuration["planes"]) + 1)[1:].tolist() + fields = configuration["fields"] + observed = { + "save_planes": itinerary, + "backend": configuration["backend"], + "fields": fields, + "field_planes": (len(itinerary),) if fields == "planes" else (), + } + chosen = integrator(configuration["stepper"]) + if case.builder in ("kerr", "spatial"): + settings = SimulationSettings( save_planes=itinerary, + dz_max=case.length, + dz_init=case.length * 1e-4, + auto_init=True, + integrator=chosen, backend=configuration["backend"], + fields=fields, + field_planes=observed["field_planes"], ) - return simulation - raise SuiteError(f"unknown builder {case.builder}") + if case.step is not None: + from solprop.stepper import PIController # noqa: PLC0415 + + # The controller is required but unused by a fixed schedule. + settings = dataclasses.replace( + settings, + controller=PIController(rtol=FREE_RTOL, atol=0.0), + integrator=LowStorageRungeKutta(scheme="rk4ip"), + schedule=FixedSchedule( + np.diff(np.linspace(0.0, stop, max(1, round(stop / case.step)) + 1)) + ), + propagator_mode="tabulate", + ) + return (kerr if case.builder == "kerr" else spatial)(case.name, settings) + if case.builder == "exercise": + from benchmarks.exercises import exercise # noqa: PLC0415 + + simulation = exercise(case.name) + elif case.builder == "advanced": + from benchmarks.guides import advanced # noqa: PLC0415 + + simulation = advanced(case.name, case.length) + elif case.builder == "capillary": + simulation = capillary(case.name) + elif case.builder == "gnlse": + simulation = gnlse(case.name) + elif case.builder == "smoke": + simulation = smoke() + else: + raise SuiteError(f"unknown builder {case.builder}") + if chosen is not None: + observed["integrator"] = chosen + simulation.settings = dataclasses.replace(simulation.settings, **observed) + return simulation diff --git a/benchmarks/collection.py b/benchmarks/collection.py deleted file mode 100644 index 0f7a9784..00000000 --- a/benchmarks/collection.py +++ /dev/null @@ -1,784 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Collect serial fresh-process attempts and retain every completed receipt.""" - -from __future__ import annotations - -import contextlib -import dataclasses -import json -import math -import os -import queue -import random -import shutil -import subprocess -import sys -import threading -import time -from pathlib import Path - -import psutil - -from benchmarks.comparison import aa_control, compare, completed, needs_more -from benchmarks.records import ( - Record, - SuiteError, - atomic_record, - attempts, - export_tables, - file_hash, - identity, - now, - read_record, - run_lock, -) -from benchmarks.system import ( - admission, - counters, - git_receipt, - guard, - interval, - thread_environment, -) - -__all__ = [ - "ROOT", - "Attempt", - "Collection", - "Monitor", - "calibrate", - "calibration", - "collect", - "collect_round", - "environment_identity", - "freeze_harness", - "freeze_manifest", - "inventory", - "launch", - "ordered", - "persist", - "prepared_python", - "prior_validation", - "reference_for", - "stop_tree", -] - -ROOT = Path(__file__).resolve().parents[1] - - -@dataclasses.dataclass(frozen=True) -class Collection: - """Freeze execution limits and prepared interpreters for an entire collection.""" - - output: Path - baseline_python: str | None = None - candidate_python: str = sys.executable - seed: int = 30 - timeout: float = 1800.0 - memory_limit: int | None = None - force: bool = False - allow_unestimated: bool = False - resume: bool = False - references: Path = Path(".benchmarks/suite/references") - allow_accuracy_failure: bool = False - - def __post_init__(self) -> None: - """Keep executable paths valid when a worker changes its directory.""" - for name in ("baseline_python", "candidate_python"): - value = getattr(self, name) - if value is not None: - object.__setattr__(self, name, prepared_python(value)) - - -def prepared_python(value: str) -> str: - """Locate an interpreter without resolving a virtual environment's symlink.""" - found = shutil.which(value) - if found is None: - raise SuiteError(f"prepared Python executable does not exist: {value}") - return str(Path(found).absolute()) - - -@dataclasses.dataclass(frozen=True) -class Attempt: - """Identify one immutable attempt within a balanced round.""" - - arm: str - round_index: int - repetitions: int = 1 - phase: str = "measurement" - - -@dataclasses.dataclass -class Monitor: - """Accumulate parent observations while its one worker remains alive.""" - - started: float - allocation: tuple[int, ...] - startup: float | None = None - result: Record | None = None - own_cpu: float | None = None - sampled_peak: int | None = None - samples: int = 0 - missed_samples: int = 0 - measurement_peak: int | None = None - measurement_seen: bool = False - measurement_system: Record | None = None - measurement_cpu: float | None = None - exceeded: str | None = None - - def guard_receipt(self, before: Record) -> Record: - """Score load through measurement, retaining missing worker CPU confidence.""" - during = interval( - before, - self.measurement_system or counters(), - self.allocation, - self.measurement_cpu - if self.measurement_cpu is not None - else self.own_cpu or 0.0, - ) - if self.measurement_cpu is None: - during.update( - confidence="unknown", - quiet=False, - reason="worker CPU usage unavailable at the measurement boundary", - ) - return during - - def event(self, received: float, line: str) -> None: - """Accept only declared protocol events; keep other stdout as diagnostics.""" - try: - event = json.loads(line) - except json.JSONDecodeError: - return - if not isinstance(event, dict): - return - if event.get("event") == "first-window": - self.startup = received - self.started - elif event.get("event") == "measurement": - self.measurement_peak = event.get("peak_rss_bytes") - self.measurement_seen = True - self.measurement_system = counters() - self.measurement_cpu = self.own_cpu - elif event.get("event") == "result": - self.result = event["result"] - - def sample( - self, owner: psutil.Process | None, limits: tuple[float, int | None] - ) -> None: - """Sample only the owned process, without inventing unavailable counters.""" - if owner is not None: - try: - times = owner.cpu_times() - self.own_cpu = times.user + times.system - self.sampled_peak = max(self.sampled_peak or 0, owner.memory_info().rss) - self.samples += 1 - except psutil.Error: - self.missed_samples += 1 - else: - self.missed_samples += 1 - if time.monotonic() - self.started > limits[0]: - self.exceeded = "timeout" - if ( - limits[1] is not None - and self.sampled_peak is not None - and self.sampled_peak > limits[1] - ): - self.exceeded = "memory-limit" - - -def stop_tree(process: subprocess.Popen[str], born: float | None) -> None: - """Terminate only processes belonging to this attempt, checking PID reuse.""" - try: - owner = psutil.Process(process.pid) - if born is not None and owner.create_time() != born: - return - try: - children = owner.children(recursive=True) - except psutil.Error, OSError: - # Restricted process inventories can still signal the directly - # owned child. Never substitute a machine-wide process-name kill. - children = [] - for child in reversed(children): - with contextlib.suppress(psutil.NoSuchProcess): - child.terminate() - owner.terminate() - _, alive = psutil.wait_procs([*children, owner], timeout=2) - for child in alive: - with contextlib.suppress(psutil.NoSuchProcess): - child.kill() - except psutil.NoSuchProcess: - pass - - -def launch( - python: str, - request: Record, - environment: dict[str, str], - directory: Path, - limits: tuple[float, int | None], - allocation: tuple[int, ...], -) -> Record: - """Launch a worker, timestamp receipts and enforce limits from its parent.""" - directory.mkdir(parents=True, exist_ok=True) - request_path = directory / "request.json" - atomic_record(request_path, request, immutable=True) - events: queue.Queue[tuple[float, str]] = queue.Queue() - harness_root = Path(request.get("harness_root", ROOT)).resolve() - environment = {**environment, "PYTHONPATH": str(harness_root)} - started, controller_cpu = time.monotonic(), time.process_time() - before = counters() - monitor = Monitor(started, allocation) - with ( - (directory / "stderr.log").open("w", encoding="utf-8") as errors, - (directory / "stdout.log").open("w", encoding="utf-8") as output, - ): - process = subprocess.Popen( - [python, "-m", "benchmarks.worker", str(request_path.resolve())], - cwd=harness_root, - env=environment, - stdout=subprocess.PIPE, - stderr=errors, - text=True, - encoding="utf-8", - start_new_session=os.name != "nt", - ) - owner: psutil.Process | None = None - try: - owner = psutil.Process(process.pid) - born = owner.create_time() - except psutil.NoSuchProcess: - born = None - - def read_events() -> None: - assert process.stdout is not None - with process.stdout as stream: - for line in stream: - events.put((time.monotonic(), line)) - - reader = threading.Thread(target=read_events, daemon=True) - reader.start() - while process.poll() is None or reader.is_alive() or not events.empty(): - try: - received, line = events.get(timeout=0.1) - output.write(line) - output.flush() - monitor.sample(owner, limits) - monitor.event(received, line) - except queue.Empty: - pass - if process.poll() is None: - monitor.sample(owner, limits) - if monitor.exceeded: - stop_tree(process, born) - break - reader.join(timeout=2) - returncode = process.wait(timeout=5) - during = monitor.guard_receipt(before) - result = monitor.result - if monitor.exceeded: - result = { - "status": monitor.exceeded, - "error": f"attempt exceeded its {monitor.exceeded}", - } - elif result is None: - result = { - "status": "failed", - "error": f"missing result receipt; exit {returncode}", - } - elif returncode != 0 and result.get("status") == "ok": - result = { - **result, - "status": "failed", - "error": f"worker exited {returncode} after receipt", - } - elif ( - result.get("status") == "ok" - and not request.get("inventory") - and (monitor.startup is None or not monitor.measurement_seen) - ): - result = {**result, "status": "failed", "error": "missing boundary receipt"} - return { - **result, - "startup_seconds": monitor.startup, - "returncode": returncode, - "elapsed_seconds": time.monotonic() - started, - "controller_cpu_seconds": time.process_time() - controller_cpu, - "sampled_peak_bytes": monitor.sampled_peak, - "sampled_peak_reason": None - if monitor.sampled_peak is not None - else "no successful process RSS poll", - "measurement_peak_bytes": monitor.measurement_peak, - "process": { - "pid": process.pid, - "created": born, - "cpu_seconds": monitor.own_cpu, - "cpu_reason": None - if monitor.own_cpu is not None - else "process CPU counters unavailable", - "rss_samples": monitor.samples, - "missed_samples": monitor.missed_samples, - "poll_interval_seconds": 0.1, - }, - "during_guard": during, - } - - -def ordered( - rows: list[Record], seed: int, round_index: int, paired: bool -) -> list[tuple[Record, tuple[str, ...]]]: - """Shuffle each balanced round while keeping reversed A/B pairs adjacent.""" - shuffled = list(rows) - random.Random(seed + round_index).shuffle(shuffled) - arms = ( - ("baseline", "candidate") if round_index % 2 == 0 else ("candidate", "baseline") - ) - return [(row, arms if paired else ("candidate",)) for row in shuffled] - - -def calibration(times: list[float], resolution: float) -> tuple[int, bool]: - """Freeze a common repetition count from paired pilots, without a time floor.""" - target = max(0.1, 1000 * resolution) - proposed = max(1, math.ceil(target / min(times))) - return min(10, proposed), proposed > 10 - - -def reference_for(directory: Path, configuration: Record) -> str | None: - """Find a qualification receipt for this exact measurement identity.""" - path = directory / f"{configuration['case']}-{configuration['track']}.json" - return str(path.resolve()) if path.exists() else None - - -def prior_validation( - previous: list[Record], configuration_id: str, arm: str, reference: str | None -) -> Record | None: - """Reuse a successful check only within the same arm and reference revision.""" - if reference is None: - return None - try: - reference_id = identity(read_record(Path(reference))) - except OSError, ValueError: - return None - for sample in reversed(previous): - if sample["configuration_id"] != configuration_id or sample["arm"] != arm: - continue - check = sample.get("accuracy", {}) - if ( - sample["status"] != "ok" - or sample.get("accuracy_overridden") - or check.get("status") != "passed" - or check.get("reference_id") != reference_id - or not check.get("field_shape") - ): - return None - return { - "status": "passed", - "reference_id": reference_id, - "qualified_for_adoption": check.get("qualified_for_adoption", False), - "reference_revision": check.get("reference_revision", 1), - "policy_revision": check.get("policy_revision", 1), - "field_shape": check["field_shape"], - "validation_sample_id": check.get( - "validation_sample_id", sample["sample_id"] - ), - "validation": "prior sample; current shape and finiteness checked", - } - return None - - -def _attempt( - row: Record, - attempt: Attempt, - options: Collection, - plan: Record, - guard_receipt: Record, -) -> Record: - previous = attempts(options.output) - attempt_id = max((item["attempt_id"] for item in previous), default=-1) + 1 - # A killed collector can leave worker files without a completed shard. - # Keep those diagnostics intact and reserve a fresh directory on resume. - while (options.output / "raw" / f"{attempt_id:06d}").exists(): - attempt_id += 1 - configuration = row["configuration"] - arm, round_index = attempt.arm, attempt.round_index - sample_id = identity([row["id"], arm, round_index, attempt.phase]) - base = { - "attempt_id": attempt_id, - "sample_id": sample_id, - "configuration_id": row["id"], - "arm": arm, - "round": round_index, - "phase": attempt.phase, - "created": now(), - "guard": guard_receipt, - } - result: Record - if row["status"] != "ready": - result = {"status": "excluded", "error": row["reason"]} - else: - estimate_invalid = any( - item.get("estimator_defect") - for item in previous - if item["configuration_id"] == row["id"] - ) - admit = admission( - None if estimate_invalid else row["estimate_bytes"], - options.memory_limit, - allow_unestimated=options.allow_unestimated, - ) - if not admit["admitted"]: - result = {"status": "refused", "admission": admit} - else: - profile = plan["profile"] - environment = thread_environment( - configuration["threads"], profile["policy"], dict(os.environ) - ) - reference = reference_for(options.references, configuration) - request = { - "configuration": configuration, - "repetitions": attempt.repetitions, - "reference": reference, - "prior_validation": prior_validation( - previous, row["id"], arm, reference - ), - "harness_root": str((options.output / "harness").resolve()), - "allow_accuracy_failure": options.allow_accuracy_failure, - "thread_diagnostics": plan.get("thread_diagnostics", False), - } - python = ( - options.baseline_python - if arm == "baseline" - else options.candidate_python - ) - assert python is not None - try: - result = launch( - python, - request, - environment, - options.output / "raw" / f"{attempt_id:06d}", - (options.timeout, admit["limit_bytes"]), - tuple(profile["allocation"]), - ) - except (OSError, SuiteError) as error: - result = {"status": "failed", "error": f"worker launch: {error}"} - if result["status"] == "ok" and environment_identity( - result["provenance"] - ) != (environment_identity(plan["environments"][arm])): - result.update( - status="failed", error="worker environment identity changed" - ) - result["admission"] = admit - result["incoming_threads"] = { - key: value - for key, value in os.environ.items() - if key in environment - and (key.endswith("THREADS") or key.startswith("SOLPROP_FFT")) - } - result["effective_threads"] = { - key: value - for key, value in environment.items() - if key.endswith("THREADS") or key.startswith("SOLPROP_FFT") - } - peak = result.get("peak_rss_bytes") - result["estimator_defect"] = ( - peak is not None - and row["estimate_bytes"] is not None - and peak > row["estimate_bytes"] - ) - result["clean"] = bool( - guard_receipt.get("quiet") - and result.get("during_guard", {}).get("quiet") - and not options.force - ) - result["warning_unreviewed"] = any( - not item["expected"] for item in result.get("diagnostics", []) - ) - return {**base, **result} - - -def freeze_harness(directory: Path) -> Record: - """Copy the small harness so later workers cannot observe mid-run edits.""" - files = {} - for source in sorted((ROOT / "benchmarks").glob("*.py")): - target = directory / source.relative_to(ROOT) - target.parent.mkdir(parents=True, exist_ok=True) - if target.exists(): - if file_hash(target) != file_hash(source): - raise SuiteError("stored harness differs from current source") - else: - target.write_bytes(source.read_bytes()) - files[str(source.relative_to(ROOT))] = file_hash(target) - return files - - -def persist(options: Collection, result: Record) -> None: - """Commit one immutable shard before refreshing derived views.""" - atomic_record( - options.output / "attempts" / f"{result['attempt_id']:06d}.json", - result, - immutable=True, - ) - export_tables(options.output) - - -def inventory(plan: Record, options: Collection) -> Record: - """Identify prepared environments once, before any timed samples.""" - result: Record = {} - for arm, python in ( - ("baseline", options.baseline_python), - ("candidate", options.candidate_python), - ): - if python is None: - continue - profile = plan["profile"] - record = launch( - python, - { - "configuration": plan["rows"][0]["configuration"], - "inventory": True, - "harness_root": str((options.output / "harness").resolve()), - }, - thread_environment( - profile["production"], profile["policy"], dict(os.environ) - ), - options.output / "inventories" / f"{arm}-{time.time_ns()}", - (options.timeout, options.memory_limit), - tuple(profile["allocation"]), - ) - if record["status"] != "ok": - raise SuiteError(f"{arm} inventory failed: {record.get('error')}") - result[arm] = record["provenance"] - return result - - -def environment_identity(provenance: Record) -> Record: - """Separate source/build identity from runtime telemetry and workload identity.""" - result = { - key: provenance.get(key) - for key in ( - "packages", - "solprop_path", - "python_executable", - "extension_hash", - "build_info", - ) - } - if result["build_info"] is not None: - # BuildInfo also contains process-local pool widths and policies. Inventory - # runs at production width; a one-thread sample, whose pools are all set - # explicitly, must still identify the same extension. Keep its actual - # widths in the unmodified sample receipt. - runtime_widths = { - "explicit_thread_pools", - "blas_thread_policy", - "requested_threads", - "runtime_threads", - "thread_count", - "fftw_threads", - "raman_fft_threads", - "raman_tail_fft_threads", - "blas_threads", - } - result["build_info"] = { - key: value - for key, value in result["build_info"].items() - if key not in runtime_widths - } - return result - - -def freeze_manifest(plan: Record, options: Collection) -> Record: - """Validate resume against the live harness and actual imported environments.""" - selected_options = dataclasses.asdict(options) - for key in ("output", "references"): - selected_options[key] = str(selected_options[key].resolve()) - selected_options.pop("resume") - plan = { - **plan, - "options": selected_options, - "exploratory": ( - options.force or options.allow_unestimated or options.allow_accuracy_failure - ), - "harness": git_receipt(ROOT), - "harness_files": freeze_harness(options.output / "harness"), - "environments": inventory(plan, options), - } - manifest_path = options.output / "manifest.json" - if manifest_path.exists(): - if not options.resume: - raise SuiteError("output already exists; select --resume or a new output") - saved = read_record(manifest_path) - if saved["plan_hash"] != identity(plan): - raise SuiteError( - "resume plan, profile, environment or harness identity differs" - ) - else: - if options.resume: - raise SuiteError("cannot resume a run without a manifest") - atomic_record( - manifest_path, - {"created": now(), "plan": plan, "plan_hash": identity(plan)}, - immutable=True, - ) - return plan - - -def calibrate(plan: Record, options: Collection) -> Record: - """Persist pilots and freeze common repetition counts before standard rounds.""" - path = options.output / "repetitions.json" - counts = read_record(path) if path.exists() else {} - paired = options.baseline_python is not None - for row, arms in ordered(plan["rows"], options.seed, -1, paired): - if row["id"] in counts: - continue - count, capped = 1, False - if plan["mode"] == "standard" and row["status"] == "ready": - pilots = [] - for arm in arms: - previous = [ - item - for item in attempts(options.output) - if item.get("phase") == "pilot" - and item["configuration_id"] == row["id"] - and item["arm"] == arm - and item["status"] == "ok" - ] - if previous: - pilots.append(previous[-1]) - continue - configuration = row["configuration"] - print( - f"pilot: {configuration['case']} {configuration['track']} " - f"{configuration['threads']} threads {arm}", - flush=True, - ) - receipt = guard( - tuple(plan["profile"]["allocation"]), - standard=True, - force=options.force, - ) - result = _attempt( - row, Attempt(arm, -1, phase="pilot"), options, plan, receipt - ) - persist(options, result) - if result["status"] == "ok": - pilots.append(result) - if len(pilots) != len(arms): - raise SuiteError( - "calibration failed; attempts are retained; resume to retry " - "before collecting measurements" - ) - count, capped = calibration( - [item["seconds"] for item in pilots], - max(item["timer_resolution"] for item in pilots), - ) - counts[row["id"]] = {"count": count, "capped": capped} - atomic_record(path, counts) - return counts - - -def collect_round( - plan: Record, options: Collection, round_index: int, counts: Record -) -> None: - """Run one balanced round with at most one whole-pair guard deferral.""" - successful = { - row["sample_id"] - for row in completed(attempts(options.output)) - if row["status"] == "ok" - } - jobs = [ - (row, arms, False) - for row, arms in ordered( - plan["rows"], options.seed, round_index, options.baseline_python is not None - ) - ] - while jobs: - row, arms, deferred = jobs.pop(0) - remaining = [ - arm - for arm in arms - if identity([row["id"], arm, round_index, "measurement"]) not in successful - ] - if not remaining: - continue - guard_receipt = guard( - tuple(plan["profile"]["allocation"]), - standard=plan["mode"] == "standard", - force=options.force, - ) - if ( - plan["mode"] == "standard" - and not guard_receipt["quiet"] - and not deferred - and not options.force - ): - jobs.append((row, arms, True)) - continue - for arm in remaining: - result = _attempt( - row, - Attempt(arm, round_index, counts[row["id"]]["count"]), - options, - plan, - guard_receipt, - ) - persist(options, result) - configuration = row["configuration"] - print( - f"round {round_index + 1}: {configuration['case']} " - f"{configuration['track']} {configuration['threads']} threads " - f"{arm}: {result['status']}", - flush=True, - ) - - -def collect(plan: Record, options: Collection) -> Record: - """Collect/resume a frozen plan with durable failure rows and bounded retries.""" - started = time.perf_counter() - if not plan["rows"]: - raise SuiteError("selection is empty") - paired = options.baseline_python is not None - configuration_ids = [row["id"] for row in plan["rows"]] - with run_lock(options.output): - plan = freeze_manifest(plan, options) - counts = calibrate(plan, options) - for round_index in range(plan["maximum_rounds"]): - rows = attempts(options.output) - if round_index >= plan["rounds"] and not needs_more( - rows, - configuration_ids, - ("baseline", "candidate") if paired else ("candidate",), - ): - break - collect_round(plan, options, round_index, counts) - # Explicit resume retries a failure; automatic extra rounds address - # dispersion and contention only, never silently discard a crash. - if any( - row["status"] != "ok" for row in completed(attempts(options.output)) - ): - break - rows = attempts(options.output) - result: Record = ( - compare(plan, rows) - if paired - else {"status": "collected", "adoption": "requires comparison"} - ) - selected = completed(rows) - result["accuracy_failures"] = sum( - row.get("accuracy", {}).get("status") == "invalid" for row in selected - ) - result["accuracy_override"] = options.allow_accuracy_failure - result["collection_seconds"] = time.perf_counter() - started - if options.allow_accuracy_failure: - result.update(status="exploratory", adoption="accuracy override enabled") - result["execution_ok"] = bool(selected) and all( - row["status"] == "ok" for row in selected - ) - atomic_record(options.output / "comparison.json", result) - if paired and options.baseline_python == options.candidate_python: - atomic_record(options.output / "aa.json", aa_control(plan, rows)) - export_tables(options.output) - return result diff --git a/benchmarks/comparison.py b/benchmarks/comparison.py deleted file mode 100644 index 3c7ad1da..00000000 --- a/benchmarks/comparison.py +++ /dev/null @@ -1,413 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Apply SEP 30's descriptive comparison rules to complete persisted selections.""" - -from __future__ import annotations - -import math -import statistics -from collections.abc import Sequence - -from benchmarks.records import Record, distribution - -__all__ = [ - "aa_control", - "clean_pairs", - "compare", - "compare_configuration", - "completed", - "geometric", - "needs_more", - "noisy", - "repeatability", - "resource_flags", -] - - -def completed(rows: Sequence[Record]) -> list[Record]: - """Select the latest successful attempt per sample, retaining failures on disk.""" - selected: dict[str, Record] = {} - for row in rows: - if row.get("phase", "measurement") != "measurement": - continue - previous = selected.get(row["sample_id"]) - if previous is None or row["status"] == "ok" or previous["status"] != "ok": - selected[row["sample_id"]] = row - return list(selected.values()) - - -def clean_pairs( - baseline: Sequence[Record], candidate: Sequence[Record] -) -> tuple[list[Record], list[Record]]: - """Drop both members of a contaminated pair without joining round indices.""" - left = { - row["round"]: row - for row in baseline - if row.get("clean") and row["status"] == "ok" - } - right = { - row["round"]: row - for row in candidate - if row.get("clean") and row["status"] == "ok" - } - rounds = sorted(left.keys() & right.keys()) - return [left[index] for index in rounds], [right[index] for index in rounds] - - -def repeatability(rows: Sequence[Record]) -> float | None: - """Use only adjacent original clean rounds, with at least four differences.""" - times = {row["round"]: row["seconds"] for row in rows} - changes = [ - abs(math.log(times[index + 1] / value)) - for index, value in times.items() - if index + 1 in times - ] - return 2 * statistics.median(changes) if len(changes) >= 4 else None - - -def noisy(rows: Sequence[Record]) -> bool: - """Decide whether more complete standard rounds are required.""" - values = distribution([row["seconds"] for row in rows]) - return values.count < 5 or ( - values.mad is not None - and values.median is not None - and 2 * values.mad / values.median > 0.05 - ) - - -def needs_more( - rows: Sequence[Record], configuration_ids: Sequence[str], arms: tuple[str, ...] -) -> bool: - """Extend for dispersion, eligibility or unavailable adjacent-round evidence.""" - active = completed(rows) - for configuration in configuration_ids: - groups = [ - [ - row - for row in active - if row["configuration_id"] == configuration and row["arm"] == arm - ] - for arm in arms - ] - if any( - group and sum(not row.get("clean") for row in group) / len(group) > 0.2 - for group in groups - ): - return True - if len(groups) == 2: - groups[0], groups[1] = clean_pairs(*groups) - if repeatability(groups[0]) is None: - return True - else: - groups = [ - [row for row in group if row.get("clean") and row["status"] == "ok"] - for group in groups - ] - if any(noisy(group) for group in groups): - return True - return False - - -def resource_flags(baseline: Sequence[Record], candidate: Sequence[Record]) -> Record: - """Compare matched sample-count memory maxima and startup medians separately.""" - result: Record = {} - for name, reduction, threshold in ( - ("peak_rss_bytes", max, 0.05), - ("startup_seconds", statistics.median, 0.1), - ): - left = [row[name] for row in baseline if row.get(name) is not None] - right = [row[name] for row in candidate if row.get(name) is not None] - if not left or len(left) != len(right): - result[name] = { - "status": "unavailable", - "reason": "missing telemetry or unmatched counts", - } - continue - before, after = reduction(left), reduction(right) - change = after - before - ratio = after / before if before else None - result[name] = { - "baseline": before, - "candidate": after, - "difference": change, - "ratio": ratio, - "flag": change > 0 - and ( - (ratio is not None and ratio > 1 + threshold) - or (name == "peak_rss_bytes" and change > 67108864) - ), - } - return result - - -def compare_configuration( - baseline: Sequence[Record], - candidate: Sequence[Record], - *, - paired: bool = True, - aa_band: float | None = None, -) -> Record: - """Evaluate one configuration without hiding failures or contamination.""" - if any( - row["status"] == "invalid" or row.get("accuracy", {}).get("status") == "invalid" - for row in (*baseline, *candidate) - ): - return {"status": "numerically-invalid"} - left, right = completed(baseline), completed(candidate) - if ( - any(row["status"] == "refused" for row in right) - and any(row["status"] == "ok" for row in left) - and not any(row["status"] == "refused" for row in left) - ): - return { - "status": "regression", - "resource_regression": ( - "candidate memory admission refused; baseline succeeded" - ), - } - if not left or not right or any(row["status"] != "ok" for row in (*left, *right)): - return {"status": "incomplete"} - if any( - row.get("resolution_hash") != left[0].get("resolution_hash") - for row in (*left, *right) - ): - return { - "status": "incompatible", - "reason": "resolved method, inputs or output differ", - } - references = { - row.get("accuracy", {}).get("reference_id") - for row in (*left, *right) - if row.get("accuracy", {}).get("reference_id") is not None - } - if len(references) > 1: - return { - "status": "incompatible", - "reason": ( - "reference or accuracy-policy revision differs; revalidate both arms" - ), - } - if paired: - clean_left, clean_right = clean_pairs(left, right) - else: - clean_left, clean_right = ( - [row for row in group if row.get("clean")] for group in (left, right) - ) - band = repeatability(clean_left) if paired else aa_band - before = distribution([row["seconds"] for row in clean_left]) - after = distribution([row["seconds"] for row in clean_right]) - ratio = after.median / before.median if before.median and after.median else None - conservative = None - if ( - before.median is not None - and before.mad is not None - and after.median is not None - and after.mad is not None - ): - denominator = before.median - 2 * before.mad - if denominator > 0: - conservative = (after.median + 2 * after.mad) / denominator - eligible = ( - min(before.count, after.count) >= 5 - and not noisy(clean_left) - and not noisy(clean_right) - and all( - sum(not row.get("clean", False) for row in group) / len(group) <= 0.2 - for group in (left, right) - ) - ) - qualified = all( - row.get("accuracy", {}).get("status") == "passed" - and row.get("accuracy", {}).get("reference_id") is not None - and row.get("accuracy", {}).get("qualified_for_adoption", False) - and not row.get("warning_unreviewed") - and not row.get("estimator_defect") - for row in (*left, *right) - ) - status = ( - "eligible" - if eligible and qualified and band is not None and conservative is not None - else "unresolved" - ) - return { - "status": status, - "ratio": ratio, - "conservative_ratio": conservative, - "repeatability_band": band, - "clean_counts": [before.count, after.count], - "all_counts": [len(left), len(right)], - "qualified": qualified, - "timing_regression": ratio is not None and ratio > 1.1, - "all_resources": resource_flags(left, right), - "clean_resources": resource_flags(clean_left, clean_right), - "all_ratio": statistics.median(row["seconds"] for row in right) - / statistics.median(row["seconds"] for row in left), - } - - -def geometric(values: Sequence[float]) -> float: - """Give every declared member equal weight.""" - return math.exp(statistics.fmean(math.log(value) for value in values)) - - -def aa_control(plan: Record, rows: Sequence[Record]) -> Record: - """Retain matched clean A/A log-ratio bands for compatible later comparisons.""" - bands = {} - for configuration in plan["rows"]: - members = [ - row - for row in completed(rows) - if row["configuration_id"] == configuration["id"] - ] - left, right = clean_pairs( - [row for row in members if row["arm"] == "baseline"], - [row for row in members if row["arm"] == "candidate"], - ) - eligible = ( - plan["mode"] == "standard" - and min(len(left), len(right)) >= 5 - and all( - sum(not row.get("clean") for row in members if row["arm"] == arm) - <= 0.2 * sum(row["arm"] == arm for row in members) - for arm in ("baseline", "candidate") - ) - ) - ratios = [b["seconds"] / a["seconds"] for a, b in zip(left, right, strict=True)] - bands[configuration["id"]] = { - "status": "measured" if eligible else "unresolved", - "clean_pairs": len(left), - "median_ratio": statistics.median(ratios) if ratios else None, - "band": 2 * statistics.median(abs(math.log(value)) for value in ratios) - if eligible - else None, - } - return { - "profile": plan["profile"], - "baseline": plan["environments"]["baseline"], - "configurations": bands, - } - - -def compare( - plan: Record, - rows: Sequence[Record], - *, - paired: bool = True, - aa_bands: Record | None = None, -) -> Record: - """Compare the complete selected matrix, with separate production aggregates.""" - details = [] - for row in plan["rows"]: - configuration = row["configuration"] - members = [item for item in rows if item["configuration_id"] == row["id"]] - verdict = compare_configuration( - [item for item in members if item["arm"] == "baseline"], - [item for item in members if item["arm"] == "candidate"], - paired=paired, - aa_band=None - if aa_bands is None - else aa_bands.get(row["id"], {}).get("band"), - ) - details.append( - { - "configuration_id": row["id"], - "case": configuration["case"], - "threads": configuration["threads"], - "track": configuration["track"], - **verdict, - } - ) - if plan["mode"] != "standard" or plan.get("exploratory"): - return { - "status": "exploratory", - "reason": "descriptive comparison only; quick or overridden collection", - "configurations": details, - } - for failure in ("numerically-invalid", "incompatible", "regression", "incomplete"): - if any(row["status"] == failure for row in details): - return {"status": failure, "configurations": details} - aggregates = [] - for width in sorted({row["threads"] for row in details}): - groups = { - track: [ - row - for row in details - if row["threads"] == width and row["track"] == track - ] - for track in {row["track"] for row in details} - } - groups["adaptive-combined"] = [ - row - for row in details - if row["threads"] == width and row["track"].startswith("adaptive-") - ] - for track, group in groups.items(): - if not group: - continue - if any(row["status"] != "eligible" for row in group): - aggregates.append( - {"threads": width, "track": track, "status": "unresolved"} - ) - continue - tracks = {item["track"] for item in group} - weights = [ - 1 - / len(tracks) - / sum(other["track"] == item["track"] for other in group) - for item in group - ] - ratio = math.exp( - sum( - weight * math.log(item["ratio"]) - for weight, item in zip(weights, group, strict=True) - ) - ) - conservative = math.exp( - sum( - weight * math.log(item["conservative_ratio"]) - for weight, item in zip(weights, group, strict=True) - ) - ) - band = sum( - weight * item["repeatability_band"] - for weight, item in zip(weights, group, strict=True) - ) - passed = ratio < 0.95 and conservative < 1 and -math.log(ratio) > band - aggregates.append( - { - "threads": width, - "track": track, - "ratio": ratio, - "conservative_ratio": conservative, - "repeatability_band": band, - "status": "passed" - if passed - else "regression" - if ratio > 1.05 - else "unresolved", - } - ) - adoption = [ - row - for row in aggregates - if row["threads"] == plan["profile"]["production"] - and row["track"] == "adaptive-combined" - ] - if not adoption and plan.get("fixed_assessment"): - adoption = [ - row for row in aggregates if row["threads"] == plan["profile"]["production"] - ] - status = ( - "unresolved" - if not adoption - else "passed" - if all(row["status"] == "passed" for row in adoption) - else ( - "regression" - if any(row["status"] == "regression" for row in adoption) - else "unresolved" - ) - ) - return {"status": status, "configurations": details, "aggregates": aggregates} diff --git a/benchmarks/coverage_reports.py b/benchmarks/coverage_reports.py deleted file mode 100644 index 67473d63..00000000 --- a/benchmarks/coverage_reports.py +++ /dev/null @@ -1,189 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Filter LLVM coverage to project sources and preserve its counting semantics.""" - -from __future__ import annotations - -import hashlib -from pathlib import Path - -from benchmarks.records import Record, SuiteError, file_hash, write_csv -from benchmarks.registry import CASES - -__all__ = [ - "capability_tags", - "coverage_tables", - "lcov_lines", - "project_file", - "source_digest", -] - - -def capability_tags(case: str) -> tuple[str, ...]: - """Declare intended exercise capabilities, separately from observed labels.""" - definition = CASES[case] - tags = {definition.builder, *definition.sets} - if case.startswith("kerr_beam"): - tags.update(("cartesian", "envelope", "kerr", "rk4ip")) - if "filament" in case: - tags.update(("plasma", "field", "kerr")) - if "averaged" in case or case.startswith("gnlse"): - tags.add("mode-averaged") - if definition.builder == "advanced": - tags.add( - "general-modal" - if case.startswith(("grin", "step_index", "moving")) - else "coupled-flow" - ) - special = { - "rdw": ("field", "modal", "plasma"), - "ctc": ("field", "modal", "raman"), - "vector_raman": ("tensor-raman", "envelope", "vector"), - "envelope_plasma": ("envelope", "plasma"), - "mixture_gradient": ("mixture", "z-dependent", "plasma"), - "dispersive_kerr": ("dispersive-kerr",), - "plasma_band": ("plasma", "band-integration"), - "plasma_trapezoid": ("plasma", "trapezoid-integration"), - "batch": ("batch", "member-statistics"), - "noise": ("physical-noise",), - "dense_output": ("dense-output",), - "absorber_schemes": ("split-step-absorber",), - "ordinary_stats": ("ordinary-statistics", "plasma"), - "far_field_stats": ( - "far-field", - "apertures", - "spectral-regions", - "periodic-statistics", - ), - } - tags.update(special.get(case, ())) - return tuple(sorted(tags)) - - -def source_digest(root: Path) -> str: - """Match the coverage build's ordered C++ source/header identity.""" - paths = sorted((*root.glob("src/**/*.cpp"), *root.glob("include/**/*.hpp"))) - if not paths: - raise SuiteError("native source root contains no project C++ sources") - record = "".join( - f"{p.relative_to(root).as_posix()}:{file_hash(p)}\n" for p in paths - ) - return hashlib.sha256(record.encode()).hexdigest() - - -def project_file(name: str, root: Path) -> str | None: - """Accept core, bindings and project headers; exclude dependencies and tools.""" - path = Path(name).resolve() - if not path.is_relative_to(root.resolve()): - return None - relative = path.relative_to(root.resolve()) - if relative.parts[0] == "src" and relative.suffix == ".cpp": - if relative.name in ("benchmark_gemm.cpp", "coverage_labels.cpp"): - return None - return relative.as_posix() - if ( - relative.parts[:2] == ("include", "solprop_native") - and relative.suffix == ".hpp" - ): - return relative.as_posix() - return None - - -def lcov_lines(lcov: str, root: Path) -> list[Record]: - """Read LLVM line aggregation while excluding dependency files.""" - lines = [] - selected = None - for line in lcov.splitlines(): - if line.startswith("SF:"): - selected = project_file(line[3:], root) - elif line == "end_of_record": - selected = None - elif selected is not None and line.startswith("DA:"): - number, count, *_ = line[3:].split(",") - lines.append({"file": selected, "line": int(number), "count": int(count)}) - return lines - - -def coverage_tables(exported: Record, lcov: str, root: Path, output: Path) -> Record: - """Write exact LLVM line counts and function/branch views with project totals. - - LCOV supplies LLVM's line-count aggregation, avoiding a second interpretation - of nested coverage segments. Branch rows retain true and false counts. LLVM - omits constant-folded conditions and uninstantiated templates from its maps; - their absence is reported explicitly, never as zero coverage. - """ - if exported.get("type") != "llvm.coverage.json.export": - raise SuiteError("unsupported LLVM coverage export") - files, functions, branches = [], [], [] - for data in exported["data"]: - for item in data["files"]: - name = project_file(item["filename"], root) - if name is None: - continue - files.append({"file": name, **item["summary"]}) - for index, branch in enumerate(item.get("branches", [])): - branches.append( - { - "file": name, - "branch": index, - "line": branch[0], - "column": branch[1], - "end_line": branch[2], - "end_column": branch[3], - "true_count": branch[4], - "false_count": branch[5], - } - ) - for item in data["functions"]: - name = project_file(item["filenames"][0], root) - if name is not None: - functions.append( - { - "file": name, - "function": item["name"], - "count": item["count"], - "status": "covered" if item["count"] else "uncovered", - "line": item["regions"][0][0] if item["regions"] else None, - } - ) - lines = lcov_lines(lcov, root) - if not files or not lines: - raise SuiteError("coverage export contains no mapped project lines") - for name, rows in ( - ("lines", lines), - ("functions", functions), - ("branches", branches), - ): - write_csv(output / f"{name}.csv", rows) - totals = {} - for metric in ("lines", "functions", "instantiations", "regions", "branches"): - count = sum(item[metric]["count"] for item in files) - covered = sum(item[metric]["covered"] for item in files) - totals[metric] = { - "count": count, - "covered": covered, - "percent": 100 * covered / count if count else None, - } - return { - "files": files, - "totals": totals, - "uninstantiated": { - "count": None, - "reason": "templates absent from LLVM maps have no measurable denominator", - }, - "compiled_out": { - "count": None, - "reason": "preprocessor and constant-folded paths are absent from the map; " - "inspect build capabilities", - }, - "unmapped_sources": [ - p.relative_to(root).as_posix() - for p in sorted( - (*root.glob("src/*.cpp"), *root.glob("include/solprop_native/*.hpp")) - ) - if project_file(str(p), root) is not None - and p.relative_to(root).as_posix() not in {f["file"] for f in files} - ], - } diff --git a/benchmarks/coverage_worker.py b/benchmarks/coverage_worker.py deleted file mode 100644 index eef83bcd..00000000 --- a/benchmarks/coverage_worker.py +++ /dev/null @@ -1,81 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Execute one declared window for coverage, without warmups or repetitions.""" - -from benchmarks.records import Record, SuiteError, metadata_value -from benchmarks.worker import ( - Diagnostics, - accuracy, - emit, - measure, - prepare, - retained_fields, - save_plane, -) - -__all__ = ["execute_coverage"] - - -def execute_coverage( - request: Record, diagnostics: Diagnostics, provenance: Record -) -> Record: - """Run construction, lead-in, one window and its declared observations.""" - import numpy as np # noqa: PLC0415 - pre-import environment already installed - - from benchmarks.builders import build # noqa: PLC0415 - from solprop.backends import select # noqa: PLC0415 - from solprop.driver.executors.pool import peak_rss # noqa: PLC0415 - from solprop.output.schema import serialise # noqa: PLC0415 - - if provenance.get("extension_hash") != request["extension_hash"]: - raise SuiteError("coverage worker imported a different native binary") - configuration = request["configuration"] - diagnostics.phase = "construction" - if configuration["case"] == "batch": - from benchmarks.batching import construct # noqa: PLC0415 - - case = construct(configuration) - try: - diagnostics.phase = "measurement" - work = case.measure(float(configuration["stop"])) - outputs = case.observe() - fields = np.concatenate([retained_fields(output) for output in outputs]) - metadata = case.engine.metadata() - finally: - case.engine.close() - else: - simulation = build(configuration) - problem = simulation.translate() - engine = select( - "native", - problem, - grid=simulation.grid, - modes=simulation.modes, - profiles=simulation.profiles, - ) - try: - output = prepare(simulation, problem, engine, configuration) - diagnostics.phase = "measurement" - work = measure(engine, problem, configuration, output) - if not configuration["track"].startswith("adaptive-"): - save_plane(engine, problem, output) - fields = retained_fields(output) - output.close() - metadata = engine.metadata() - finally: - engine.close() - # The common parent needs completion and memory boundaries. None of these - # diagnostic wall times become a warm sample or enter timing comparisons. - emit("first-window") - emit("measurement", peak_rss_bytes=peak_rss()) - check = accuracy(fields, configuration, request) - return { - "purpose": "coverage", - "status": "invalid" if check["status"] == "invalid" else "ok", - "provenance": provenance, - "accuracy": check, - "work": {k: v for k, v in work.items() if not k.endswith("seconds")}, - "metadata": metadata_value(serialise(metadata)), - } diff --git a/benchmarks/guides.py b/benchmarks/guides.py index f931a881..98b521d5 100644 --- a/benchmarks/guides.py +++ b/benchmarks/guides.py @@ -170,12 +170,13 @@ def step_index() -> Simulation: ) -def grin(*, nonlinear: bool) -> Simulation: +def grin(*, nonlinear: bool, length: float = 0.1) -> Simulation: """Preserve the linear lesson or the four-channel dispersive GRIN example. The nonlinear source describes the GPI model and its chosen Kerr coefficient alongside Krupa et al., doi:10.1103/PhysRevLett.116.183901. This extraction retains its four-channel truncation, not a mode-count convergence claim. + `length` shortens the nonlinear guide, which is uniform along z. """ radius, contrast = (50e-6, 0.0095) if nonlinear else (25e-6, 0.005) wavelength = 1.064e-6 if nonlinear else 1.03e-6 @@ -200,7 +201,7 @@ def grin(*, nonlinear: bool) -> Simulation: core_responses=(MaterialKerr(n2=2.7e-20, at=wavelength),) if nonlinear else (), ) grid = FieldGrid.from_limits( - 0.1 if nonlinear else 1e-3, + length if nonlinear else 1e-3, wavelength, (450e-9, 1.8e-6) if nonlinear else (1e-6, 1.06e-6), 1.2e-12 if nonlinear else 100e-15, @@ -238,7 +239,7 @@ def grin(*, nonlinear: bool) -> Simulation: ) return Simulation.from_waveguide( (), - 0.1, + length, provider=guide, grid=grid, incident=beam, @@ -320,14 +321,17 @@ def moving() -> Simulation: ) -def advanced(name: str) -> Simulation: - """Select a declared guide before applying the suite's measurement settings.""" +def advanced(name: str, length: float) -> Simulation: + """Select a declared guide before applying the suite's measurement settings. + + `length` is the registered guide length; only the nonlinear GRIN guide takes it. + """ if name in ("scalar_wall", "vector_wall", "taper", "taper_wall"): return coupled(name) if name == "step_index_pair": return step_index() if name in ("grin_linear", "grin_nonlinear"): - return grin(nonlinear=name == "grin_nonlinear") + return grin(nonlinear=name == "grin_nonlinear", length=length) if name == "moving_solid_guide": return moving() raise SuiteError(f"advanced guide builder not implemented for {name}") diff --git a/benchmarks/history.py b/benchmarks/history.py new file mode 100644 index 00000000..e76d8fbd --- /dev/null +++ b/benchmarks/history.py @@ -0,0 +1,142 @@ +# SPDX-FileCopyrightText: 2026 John C. Travers and contributors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Keep and summarise the per-hardware tracking history (SEP 38, section 5.3). + +Each hardware set has one CSV under ``benchmarks/history/``, named by its slug, with +one row per run, case, stepper and width. Rows compare only within one hardware set, +width, case revision and resolved integrator. +""" + +from __future__ import annotations + +import csv +import math +import statistics +from collections.abc import Sequence +from pathlib import Path + +from benchmarks.records import Record, SuiteError +from benchmarks.runner import ROOT + +__all__ = ["COLUMNS", "HISTORY", "append", "comparable", "read", "show", "summary"] + +HISTORY = ROOT / "benchmarks" / "history" +COLUMNS = ( + "date", + "solprop", + "commit", + "native", + "native_build", + "slug", + "cpu_model", + "instance_type", + "system", + "threads", + "case", + "revision", + "stepper", + "integrator", + "rtol", + "atol", + "noise", + "median_s", + "min_s", + "max_s", + "repeats", + "accepted", + "rejected", + "source_evaluations", + "peak_rss_gib", + "construction_s", + "busy", +) + + +def append(path: Path, rows: Sequence[Record]) -> None: + """Append rows, writing the header to a new file and refusing a changed one.""" + path.parent.mkdir(parents=True, exist_ok=True) + new = not path.exists() + if not new: + with path.open(encoding="utf-8", newline="") as stream: + header = next(csv.reader(stream), None) + if tuple(header or ()) != COLUMNS: + raise SuiteError(f"{path}: columns differ from this version's") + with path.open("a", encoding="utf-8", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=list(COLUMNS), lineterminator="\n") + if new: + writer.writeheader() + for row in rows: + if set(row) != set(COLUMNS): + raise SuiteError("a history row must give exactly the declared columns") + writer.writerow( + {key: "" if row[key] is None else row[key] for key in COLUMNS} + ) + + +def read(path: Path) -> list[Record]: + """Read a history file's rows as strings.""" + with path.open(encoding="utf-8", newline="") as stream: + return list(csv.DictReader(stream)) + + +def comparable(row: Record) -> tuple[str, ...]: + """Key the rows that measure the same thing on one hardware set.""" + return ( + str(row["threads"]), + row["case"], + str(row["revision"]), + row["stepper"], + row["integrator"], + ) + + +def summary(rows: Sequence[Record]) -> list[Record]: + """Compare each recorded run with the first one, per width. + + A run is the rows sharing a date and commit. Its change is the geometric mean, + over the configurations it shares with the first run at that width, of the + ratio of median times; below one is faster. + """ + runs: dict[tuple[str, str, str], dict[tuple[str, ...], float]] = {} + labels: dict[tuple[str, str, str], str] = {} + for row in rows: + key = (str(row["threads"]), row["date"], row["commit"]) + runs.setdefault(key, {})[comparable(row)] = float(row["median_s"]) + labels[key] = row["solprop"] + result = [] + first: dict[str, dict[tuple[str, ...], float]] = {} + for (threads, date, commit), times in runs.items(): + reference = first.setdefault(threads, times) + shared = sorted(set(times) & set(reference)) + ratios = [times[key] / reference[key] for key in shared] + result.append( + { + "threads": threads, + "date": date, + "commit": commit, + "solprop": labels[(threads, date, commit)], + "cases": len(times), + "shared": len(shared), + "ratio": math.exp(statistics.fmean(map(math.log, ratios))) + if ratios + else None, + } + ) + return result + + +def show(path: Path, threads: str | None = None) -> None: + """Print one line per recorded run and width.""" + print(f"{path.stem}") + print(f"{'threads':>7} {'date':25} {'solprop':26} {'cases':>5} {'vs first':>9}") + for row in summary(read(path)): + if threads is not None and row["threads"] != threads: + continue + ratio = "-" if row["ratio"] is None else f"{row['ratio']:.3f}" + print( + f"{row['threads']:>7} {row['date']:25} {row['solprop']:26} " + f"{row['cases']:>5} {ratio:>9}" + ) + print("vs first: geometric mean of median-time ratios over shared cases.") diff --git a/benchmarks/native_coverage.py b/benchmarks/native_coverage.py deleted file mode 100644 index 999c3040..00000000 --- a/benchmarks/native_coverage.py +++ /dev/null @@ -1,517 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Collect native CPU profiles and separate Kokkos execution-label evidence.""" - -from __future__ import annotations - -import argparse -import json -import os -import re -import shutil -import subprocess -import uuid -from dataclasses import dataclass -from pathlib import Path - -from benchmarks.collection import launch, prepared_python -from benchmarks.coverage_reports import ( - capability_tags, - coverage_tables, - project_file, - source_digest, -) -from benchmarks.planning import Selection, matrix -from benchmarks.records import ( - LOCAL_ROOT, - Record, - SuiteError, - atomic_record, - file_hash, - run_lock, - write_csv, -) -from benchmarks.registry import CASES, select -from benchmarks.system import admission, profile_for, thread_environment - -__all__ = [ - "CoverageOptions", - "collect_coverage", - "command", - "coverage", - "profile_groups", - "read_labels", - "render_report", - "toolchain", - "verified_profiles", -] - - -@dataclass(frozen=True) -class CoverageOptions: - """Fix the diagnostic binary, tools, source tree and execution limits.""" - - output: Path - native_root: Path - llvm_bin: Path - python: str - timeout: float - memory_limit: int | None - - -def command(arguments: list[str], directory: Path, name: str) -> str: - """Retain tool output and refuse failed or mismatched profile processing.""" - directory.mkdir(parents=True, exist_ok=True) - result = subprocess.run( - arguments, capture_output=True, text=True, check=False, timeout=300 - ) - (directory / f"{name}.stderr.log").write_text(result.stderr, encoding="utf-8") - (directory / f"{name}.stdout.log").write_text(result.stdout, encoding="utf-8") - if result.returncode or re.search( - r"mismatch|malformed|out of date|truncated|error:", result.stderr, re.I - ): - raise SuiteError(f"{name} failed; see {directory / f'{name}.stderr.log'}") - return result.stdout - - -def toolchain(directory: Path, compiler: str) -> Record: - """Require matching LLVM releases for Clang's versioned raw-profile format.""" - match = re.search(r"\b(\d+\.\d+\.\d+)\b", compiler) - if match is None: - raise SuiteError("coverage compiler has no identifiable LLVM version") - result = {} - for name in ("llvm-profdata", "llvm-cov"): - path = directory / name - version = subprocess.run( - [str(path), "--version"], capture_output=True, text=True, check=True - ).stdout - found = re.search(r"version (\d+\.\d+\.\d+)", version) - if found is None or found[1] != match[1]: - raise SuiteError(f"{name} must match coverage compiler {match[1]}") - result[name] = { - "path": str(path), - "version": found[1], - "sha256": file_hash(path), - } - return result - - -def verified_profiles(rows: list[Record], binary_hash: str) -> list[Path]: - """Admit successful raw profiles only with their recorded binary and hashes.""" - result = [] - for row in rows: - if row["status"] != "ok": - continue - if row["extension_hash"] != binary_hash: - raise SuiteError("coverage profile belongs to a different binary") - if not row["profiles"]: - raise SuiteError("successful coverage worker has no raw profiles") - for profile in row["profiles"]: - path = Path(profile["path"]) - if ( - not path.is_file() - or not path.stat().st_size - or file_hash(path) != profile["sha256"] - ): - raise SuiteError("raw coverage profile is missing, empty or changed") - result.append(path) - return sorted(set(result)) - - -def profile_groups(rows: list[Record]) -> dict[str, list[Record]]: - """Union successful configurations by case and declared set, without repeats.""" - groups: dict[str, list[Record]] = {} - for row in rows: - if row["status"] != "ok": - continue - case = CASES[row["configuration"]["case"]] - for key in ( - "union", - f"cases/{case.name}", - *(f"sets/{name}" for name in case.sets), - ): - groups.setdefault(key, []).append(row) - return groups - - -def render_report( - options: CoverageOptions, name: str, rows: list[Record], binary: Record -) -> Record: - """Merge a verified profile union and export HTML plus normalized CSV views.""" - directory = options.output / "reports" / name - directory.mkdir(parents=True, exist_ok=True) - path = Path(binary["path"]) - if file_hash(path) != binary["sha256"]: - raise SuiteError("coverage binary snapshot changed") - if source_digest(options.native_root) != binary["source_hash"]: - raise SuiteError("native source changed since the coverage build") - profiles = verified_profiles(rows, binary["sha256"]) - if not profiles: - raise SuiteError("coverage report has no successful profiles") - profile = directory / "merged.profdata" - command( - [ - str(options.llvm_bin / "llvm-profdata"), - "merge", - "-sparse", - "--failure-mode=any", - *map(str, profiles), - "-o", - str(profile), - ], - directory, - "merge", - ) - base = [ - str(options.llvm_bin / "llvm-cov"), - "export", - str(path), - f"-instr-profile={profile}", - ] - exported = json.loads(command(base, directory, "export")) - lcov = command([*base, "-format=lcov"], directory, "lcov") - summary = coverage_tables(exported, lcov, options.native_root, directory) - sources = sorted( - { - item["filename"] - for data in exported["data"] - for item in data["files"] - if project_file(item["filename"], options.native_root) is not None - } - ) - command( - [ - str(options.llvm_bin / "llvm-cov"), - "show", - str(path), - f"-instr-profile={profile}", - "-format=html", - f"-output-dir={directory / 'html'}", - "-show-line-counts-or-regions", - "-show-branches=count", - "-show-instantiations", - *sources, - ], - directory, - "html", - ) - result = { - "group": name, - "binary": binary, - "configuration_ids": [row["id"] for row in rows], - **summary, - } - atomic_record(directory / "summary.json", result, immutable=True) - return result - - -def _environment(configuration: Record, directory: Path) -> dict[str, str]: - environment = thread_environment( - configuration["threads"], configuration["policy"], dict(os.environ) - ) - # Neither inherited tools nor an old profile destination may contaminate this - # process. Labels get their own second pass and raw-profile directory. - environment.pop("KOKKOS_TOOLS_LIBS", None) - environment.pop("SOLPROP_NATIVE_LABEL_FILE", None) - raw = directory / "raw" - raw.mkdir(parents=True, exist_ok=True) - environment["LLVM_PROFILE_FILE"] = str((raw / "%p-%m.profraw").resolve()) - return environment - - -def _one_case( - row: Record, plan: Record, options: CoverageOptions, binary: Record -) -> Record: - configuration = row["configuration"] - directory = options.output / "configurations" / row["id"] - receipt = { - "id": row["id"], - "configuration": configuration, - "extension_hash": binary["sha256"], - "profiles": [], - "declared_capabilities": capability_tags(configuration["case"]), - } - admit = admission(row["estimate_bytes"], options.memory_limit) - if row["status"] != "ready" or not admit["admitted"]: - receipt.update( - status="excluded" if row["status"] != "ready" else "refused", - reason=row["reason"], - admission=admit, - ) - else: - request = { - "configuration": configuration, - "coverage": True, - "extension_hash": binary["sha256"], - } - result = launch( - options.python, - request, - _environment(configuration, directory), - directory / "worker", - (options.timeout, admit["limit_bytes"]), - tuple(plan["profile"]["allocation"]), - ) - receipt.update(status=result["status"], result=result) - receipt["capability_status"] = ( - "executed" - if result["status"] == "ok" - else "unsupported" - if "UnsupportedFeatureError:" in result.get("error", "") - else "failed" - ) - profiles = sorted((directory / "raw").glob("*.profraw")) - receipt["profiles"] = [ - {"path": str(path.resolve()), "sha256": file_hash(path)} - for path in profiles - ] - if receipt["status"] == "ok" and ( - not profiles or any(not path.stat().st_size for path in profiles) - ): - receipt.update( - status="failed", error="missing or empty raw coverage profiles" - ) - if receipt["status"] == "ok": - if file_hash(Path(binary["labels_library"])) != binary["labels_hash"]: - raise SuiteError("Kokkos label module changed during collection") - labels = directory / "labels" - environment = _environment(configuration, labels) - environment.update( - KOKKOS_TOOLS_LIBS=binary["labels_library"], - SOLPROP_NATIVE_LABEL_FILE=str((labels / "events.jsonl").resolve()), - ) - receipt["label_result"] = launch( - options.python, - request, - environment, - labels / "worker", - (options.timeout, admit["limit_bytes"]), - tuple(plan["profile"]["allocation"]), - ) - events = labels / "events.jsonl" - try: - receipt["labels"] = read_labels(events) - except (OSError, ValueError) as error: - receipt["labels"] = [] - receipt["labels_error"] = str(error) - receipt["labels_status"] = ( - "ok" - if not receipt.get("labels_error") - and receipt["label_result"]["status"] == "ok" - else "failed" - ) - atomic_record(directory / "receipt.json", receipt, immutable=True) - return receipt - - -def read_labels(path: Path) -> list[Record]: - """Reject malformed execution-label receipts without interpreting them as zero.""" - result = [] - for line in path.read_text(encoding="utf-8").splitlines(): - item = json.loads(line) - if ( - not isinstance(item, dict) - or not isinstance(item.get("label"), str) - or not isinstance(item.get("kind"), str) - or type(item.get("count")) is not int - or item["count"] < 1 - ): - raise SuiteError("invalid Kokkos execution-label receipt") - result.append(item) - if not result: - raise SuiteError("Kokkos label collector produced no execution labels") - return result - - -def collect_coverage(plan: Record, options: CoverageOptions) -> Record: - """Collect each selected configuration serially, retaining failures as rows.""" - with run_lock(options.output): - if (options.output / "manifest.json").exists(): - raise SuiteError("coverage output already exists; choose a new bundle") - inventory_configuration = { - "case": "smoke", - "backend": "native", - "threads": 1, - "policy": "budget", - } - directory = options.output / "inventory" - inventory = launch( - options.python, - { - "configuration": inventory_configuration, - "inventory": True, - "coverage": True, - }, - _environment(inventory_configuration, directory), - directory / "worker", - (options.timeout, options.memory_limit), - tuple(plan["profile"]["allocation"]), - ) - atomic_record(directory / "receipt.json", inventory, immutable=True) - if inventory["status"] != "ok": - raise SuiteError(f"coverage inventory failed: {inventory.get('error')}") - provenance = inventory["provenance"] - build = provenance["build_info"] - digest = source_digest(options.native_root) - if digest != build.get("coverage_source_hash"): - raise SuiteError("native source does not match the instrumented binary") - tools = toolchain(options.llvm_bin, build["compiler"]) - extension = Path(provenance["extension_path"]) - libraries = list(extension.parent.glob("solprop_native_labels.*")) - if len(libraries) != 1: - raise SuiteError("coverage build must provide one Kokkos label module") - snapshot = options.output / f"native-core{extension.suffix}" - shutil.copyfile(extension, snapshot) - binary = { - "path": str(snapshot.resolve()), - "sha256": file_hash(snapshot), - "source_hash": digest, - "labels_library": str(libraries[0]), - "labels_hash": file_hash(libraries[0]), - } - if binary["sha256"] != provenance["extension_hash"]: - raise SuiteError("native binary changed during coverage inventory") - atomic_record( - options.output / "manifest.json", - { - "schema": "solprop.native-coverage.v1", - "plan": plan, - "provenance": provenance, - "binary": binary, - "tools": tools, - "scope": "construction, lead-in, one window and observations; " - "separate label pass; no timing samples", - }, - immutable=True, - ) - rows = [] - for row in plan["rows"]: - print( - f"coverage {row['configuration']['case']} " - f"{row['configuration']['track']} " - f"{row['configuration']['threads']} threads", - flush=True, - ) - try: - rows.append(_one_case(row, plan, options, binary)) - except (OSError, ValueError, subprocess.SubprocessError) as error: - failure = { - "id": row["id"], - "configuration": row["configuration"], - "status": "failed", - "error": str(error), - } - atomic_record( - options.output / "configurations" / row["id"] / "failure.json", - failure, - immutable=True, - ) - rows.append(failure) - reports = {} - for name, selected in profile_groups(rows).items(): - try: - reports[name] = { - "status": "ok", - **render_report(options, name, selected, binary), - } - except (OSError, ValueError, subprocess.SubprocessError) as error: - reports[name] = {"status": "failed", "error": str(error)} - atomic_record( - options.output / "reports" / name / "failure.json", - reports[name], - immutable=True, - ) - write_csv( - options.output / "configurations.csv", - [ - { - "id": r["id"], - **r["configuration"], - "status": r["status"], - "labels_status": r.get("labels_status"), - "capabilities": r.get("declared_capabilities", ()), - "capability_status": r.get("capability_status", r["status"]), - } - for r in rows - ], - ) - write_csv( - options.output / "labels.csv", - [ - { - "id": r["id"], - "case": r["configuration"]["case"], - "declared_capabilities": r.get("declared_capabilities", ()), - **label, - } - for r in rows - for label in r.get("labels", []) - ], - ) - result = { - "status": "complete" - if all(report["status"] == "ok" for report in reports.values()) - and all( - r["status"] in ("ok", "excluded") - and r.get("labels_status", "ok") == "ok" - for r in rows - ) - else "incomplete", - "rows": rows, - "reports": reports, - "instrumented_cpu_only": True, - "compiled_out_capabilities": ["CUDA", "HIP"], - "uninstantiated_templates": None, - "uninstantiated_reason": "not enumerated by LLVM coverage maps; " - "not counted as missed executable code", - } - atomic_record(options.output / "summary.json", result, immutable=True) - return result - - -def coverage(args: argparse.Namespace) -> int: - """Resolve coverage selections independently of timing modes and rounds.""" - profile = profile_for(args.machine_profile, standard=False) - plan = matrix( - select(tuple(args.case), tuple(args.set), all_cases=args.all), - profile, - Selection( - mode="standard", - backend="native", - threads=args.threads, - tracks=tuple(args.track), - diagnostic=args.diagnostic, - ), - ) - for key in ("mode", "rounds", "maximum_rounds", "noise_extension_cost"): - plan.pop(key, None) - options = CoverageOptions( - (args.out or LOCAL_ROOT / "coverage" / uuid.uuid4().hex).resolve(), - args.native_root.resolve(), - args.llvm_bin.resolve(), - prepared_python(args.python), - args.timeout, - args.memory_limit, - ) - result = collect_coverage(plan, options) - union = result["reports"].get("union", {}) - if union.get("status") == "ok": - print("Native project coverage (compiled CPU paths):") - for name in ("lines", "functions", "branches"): - metric = union["totals"][name] - percent = ( - f"{metric['percent']:.2f}%" - if metric["percent"] is not None - else "unavailable" - ) - print(f" {name}: {metric['covered']}/{metric['count']} ({percent})") - failures = sum(row["status"] not in ("ok", "excluded") for row in result["rows"]) - if failures: - print( - f" {failures} configuration(s) failed or were refused; " - "retained in the report" - ) - print(f"Native coverage: {options.output} ({result['status']})") - return 0 if result["status"] == "complete" else 1 diff --git a/benchmarks/planning.py b/benchmarks/planning.py deleted file mode 100644 index 159e5a66..00000000 --- a/benchmarks/planning.py +++ /dev/null @@ -1,198 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Expand an immutable measurement matrix without constructing a simulation.""" - -from __future__ import annotations - -import dataclasses -from pathlib import Path - -from benchmarks.records import ( - SCHEMA_VERSION, - SUITE_VERSION, - Record, - SuiteError, - identity, - read_record, -) -from benchmarks.references import measurement_identity -from benchmarks.registry import ADAPTIVE, FIXED, KERR, TRACKS, Case -from benchmarks.system import Profile, topology -from benchmarks.telemetry import allocation_details, hardware_details - -__all__ = ["Selection", "estimate", "matrix", "reference_status", "widths"] - - -def reference_status(directory: Path | None, configuration: Record) -> str: - """Inspect receipt metadata without loading arrays or constructing a case.""" - if directory is None or "case_hash" not in configuration: - return "not-qualified" - path = directory / f"{configuration['case']}-{configuration['track']}.json" - if not path.exists(): - return "not-qualified" - try: - receipt = read_record(path) - except OSError, ValueError: - return "invalid receipt" - if ( - receipt.get("revision") != 1 - or receipt.get("case_hash") != configuration["case_hash"] - or receipt.get("measurement_hash") != measurement_identity(configuration) - ): - return "incompatible receipt" - return ( - "reviewed receipt; arrays checked by worker" - if receipt.get("qualified_for_adoption") - else "available receipt; review pending" - ) - - -def widths(value: str | None, profile: Profile, mode: str) -> tuple[int, ...]: - """Resolve positive thread budgets without silently clamping them.""" - tokens = (value or ("production" if mode == "quick" else "1,production")).split(",") - try: - result = tuple( - dict.fromkeys( - profile.production if item == "production" else int(item) - for item in tokens - ) - ) - except ValueError as error: - raise SuiteError("threads must be positive integers or production") from error - if not result or any(item < 1 or item > len(profile.allocation) for item in result): - raise SuiteError( - "thread width exceeds the profile allocation or is not positive" - ) - return result - - -def estimate(case: Case, track: str) -> int | None: - """Bound core construction conservatively until empirical receipts supersede it. - - Cartesian RK4IP allowance: 24 complex field volumes plus 4 GiB for libraries, - FFT plans, tables and transient copies. SEP 28's retained low-storage ledger - holds five field volumes; the additional nineteen cover launch, translation, - NumPy temporaries, output conversion and qualification copies. This does not - admit an Adams history at the same size. - Other core cases reserve 8 GiB, above their documented array inventories. - These are explicitly provisional admission budgets, not measured peak RSS. - """ - if case.estimate_bytes is not None: - return case.estimate_bytes - if case.builder == "pending": - return None - if case.builder == "kerr": - if track != KERR: - return None - return 24 * 16 * case.dimensions[0] ** 2 * case.dimensions[-1] + 4 * 2**30 - if case.name.startswith("cartesian_filament"): - # Full field state plus RK registers or Adams state/source histories, - # propagator tables, launch/output copies and FFT temporaries. Reserve - # 96 volumes for adaptive history and 48 for fixed RK, plus runtime and - # node-chunk work. Empirical peaks still validate this provisional cap. - volumes = 96 if track == ADAPTIVE[0] else 48 - bins = case.dimensions[-2] // 2 + 1 - return volumes * 16 * case.dimensions[0] ** 2 * bins + 8 * 2**30 - return 8 * 2**30 - - -@dataclasses.dataclass(frozen=True) -class Selection: - """Freeze matrix selection separately from machine and case identities.""" - - mode: str = "quick" - backend: str = "native" - threads: str | None = None - tracks: tuple[str, ...] = () - receipts: Path | None = None - diagnostic: bool = False - - -def matrix(cases: tuple[Case, ...], profile: Profile, selection: Selection) -> Record: - """Resolve selections and exclusions into a persistable collection plan.""" - mode, backend = selection.mode, selection.backend - if mode not in ("quick", "standard") or backend not in ( - "numpy_ref", - "numba_cpu", - "native", - ): - raise SuiteError("unknown mode or backend") - if any(track not in TRACKS for track in selection.tracks): - raise SuiteError("unknown track") - rows: list[Record] = [] - for case in cases: - chosen = selection.tracks or ( - ( - tuple( - track for track in case.tracks if track in ADAPTIVE or track == KERR - ) - or case.tracks - ) - if mode == "quick" - else case.tracks - ) - for track in dict.fromkeys(chosen): - for width in widths(selection.threads, profile, mode): - reason = case.pending_reason - if track not in case.tracks: - reason = "track not admitted by this case" - if ( - case.name == "kerr_beam_768" - and width == 1 - and not selection.diagnostic - ): - reason = "one thread requires explicit diagnostic selection" - schedule: tuple[float, ...] = () - if track == FIXED and case.step is None: - if track in case.tracks: - schedule = tuple(case.measurement(track)["schedule"]) - schedule_path = ( - None - if selection.receipts is None - else selection.receipts / f"{case.name}-schedule.json" - ) - if schedule_path is not None and schedule_path.exists(): - receipt = read_record(schedule_path) - if receipt["case_hash"] == case.digest: - schedule = tuple(receipt["edges"]) - if not schedule: - reason = "fixed schedule pending separate qualification" - values: Record = ( - case.measurement(track, schedule) - if track in case.tracks - else {"case": case.name, "track": track} - ) - values.update( - backend=backend, - threads=width, - policy=profile.policy, - profile_hash=profile.digest, - ) - key = identity(values) - rows.append( - { - "id": key, - "configuration": values, - "status": "excluded" if reason else "ready", - "reason": reason, - "estimate_bytes": estimate(case, track), - "estimate_status": "provisional", - "reference": reference_status(selection.receipts, values), - "pilot_seconds": None, - "pilot_reason": "no compatible pilot receipt", - } - ) - return { - "schema_version": SCHEMA_VERSION, - "suite_version": SUITE_VERSION, - "mode": mode, - "profile": dataclasses.asdict(profile), - "system": {**topology(), **hardware_details(profile.allocation)}, - "allocation": allocation_details(), - "rows": rows, - "rounds": 1 if mode == "quick" else 5, - "maximum_rounds": 1 if mode == "quick" else 11, - "noise_extension_cost": "unknown until compatible pilots exist", - } diff --git a/benchmarks/publication.py b/benchmarks/publication.py deleted file mode 100644 index 0301ead8..00000000 --- a/benchmarks/publication.py +++ /dev/null @@ -1,319 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Export an explicit public view without copying private bundle contents.""" - -from __future__ import annotations - -import re -from dataclasses import asdict -from pathlib import Path - -from benchmarks.comparison import completed -from benchmarks.records import ( - Record, - SuiteError, - atomic_record, - attempts, - distribution, - identity, - read_record, - run_lock, - write_csv, -) - -__all__ = ["export_public", "public_record", "scalar_columns", "summaries"] - -CONFIGURATION_COLUMNS = ( - "case", - "revision", - "case_hash", - "track", - "backend", - "threads", - "policy", - "start", - "stop", - "step", - "rtol", - "atol", - "observations", - "fields", - "statistics", - "noise_seed", - "reset", - "measurement_version", -) -SAMPLE_COLUMNS = ( - "attempt_id", - "sample_id", - "arm", - "round", - "configuration_id", - "created", - "phase", - "status", - "clean", - "seconds", - "startup_seconds", - "peak_rss_bytes", - "sampled_peak_bytes", - "measurement_peak_bytes", - "estimator_defect", - "accuracy_overridden", - "warning_unreviewed", - "returncode", -) -OMISSIONS = ( - "hostnames and original machine aliases", - "local paths, executables, source locations and raw logs", - "process identities, command arguments and environment values", - "free-form diagnostics, errors and compiler/link flags", -) - - -def scalar_columns(record: Record, names: tuple[str, ...]) -> Record: - """Retain named scalar fields; nested details stay in their JSON records.""" - return { - name: record[name] - for name in names - if name in record - and isinstance(record[name], (str, int, float, bool, type(None))) - } - - -def _label(value: object) -> str | None: - # Machine/build labels cannot contain paths, shell fragments or CSV formulae. - return ( - value - if isinstance(value, str) - and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9 .+_-]*", value) - else None - ) - - -def public_record(manifest: Record, rows: list[Record], alias: str) -> Record: - """Create a whitelisted publication snapshot, retaining failures and unknowns.""" - if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", alias): - raise SuiteError( - "public alias must use letters, digits, dots, dashes or underscores" - ) - plan = manifest["plan"] - configurations = [] - for row in plan["rows"]: - values = scalar_columns(row["configuration"], CONFIGURATION_COLUMNS) - if any( - _label(values.get(name)) is None for name in ("case", "track", "backend") - ): - raise SuiteError("configuration has an invalid public label") - configurations.append( - { - "id": row["id"], - "configuration": values, - "schedule": row["configuration"].get("schedule", []), - "status": row["status"], - "estimate_bytes": row.get("estimate_bytes"), - "reference": row.get("reference"), - } - ) - environments = {} - for arm, environment in plan.get("environments", {}).items(): - build = environment.get("build_info", {}) - environments[arm] = { - "packages": { - name: _label(version) - for name, version in environment.get("packages", {}).items() - }, - "source": scalar_columns( - environment.get("source") or {}, ("revision", "dirty", "dirty_hash") - ), - "extension_hash": environment.get("extension_hash"), - "build": { - key: value - for key in ( - "compiler_id", - "compiler_version", - "execution_space", - "kokkos_version", - "build_type", - "fftw_provider", - "blas_provider", - "coverage_instrumented", - ) - if (value := build.get(key)) is not None - and (isinstance(value, (bool, int, float)) or _label(value) is not None) - }, - } - samples = [] - for row in rows: - sample = scalar_columns(row, SAMPLE_COLUMNS) - accuracy = row.get("accuracy", {}) - sample.update( - accuracy_status=accuracy.get("status", "unavailable"), - reference_id=accuracy.get("reference_id"), - qualified_for_adoption=accuracy.get("qualified_for_adoption", False), - peak_rss_unavailable_reason=None - if row.get("peak_rss_bytes") is not None - else "not recorded by worker", - repetitions=[ - scalar_columns( - repetition, - ( - "seconds", - "accepted", - "rejected", - "source_evaluations", - "endpoint", - "observation_seconds", - ), - ) - for repetition in row.get("repetitions", []) - ], - ) - samples.append(sample) - hardware = scalar_columns( - plan.get("system", {}), - ( - "system", - "machine", - "logical_cpus", - "physical_cores", - "memory_bytes", - "processor", - "cpu_model", - ), - ) - return { - "publication_schema_version": 1, - "suite_version": plan["suite_version"], - "result_schema_version": plan["schema_version"], - "created": manifest["created"], - "bundle_id": identity(manifest), - "alias": alias, - "mode": plan["mode"], - "exploratory": plan.get("exploratory", False), - "profile": scalar_columns( - plan["profile"], - ("revision", "production", "policy", "explicit", "topology_hash"), - ), - "hardware": hardware, - "hardware_unavailable_reason": None - if hardware - else "not retained by this bundle version", - "environments": environments, - "configurations": configurations, - "samples": samples, - "omissions": list(OMISSIONS), - } - - -def summaries(rows: list[Record]) -> list[Record]: - """Summarize final attempts with separate all-sample and clean populations.""" - groups: dict[tuple[str, str], list[Record]] = {} - for row in completed(rows): - groups.setdefault((row["configuration_id"], row["arm"]), []).append(row) - result = [] - for (configuration, arm), group in groups.items(): - for population in ("all", "clean"): - selected = [ - row - for row in group - if row["status"] == "ok" - and (population == "all" or row.get("clean") is True) - ] - result.append( - { - "configuration_id": configuration, - "arm": arm, - "population": population, - **asdict(distribution([row["seconds"] for row in selected])), - "failed_samples": sum(row["status"] != "ok" for row in group), - } - ) - return result - - -def export_public(source: Path, destination: Path, alias: str) -> Record: - """Write a separate immutable public JSON snapshot and flat CSV views. - - Publication does not copy raw files, alter the local bundle, or commit files. - The caller chooses a public machine alias independently of its local profile. - """ - if source.resolve() == destination.resolve(): - raise SuiteError("publication destination must differ from the local bundle") - if destination.exists() and any(destination.iterdir()): - raise SuiteError("publication destination must be empty or new") - with run_lock(source): - result = public_record( - read_record(source / "manifest.json"), attempts(source), alias - ) - with run_lock(destination): - atomic_record(destination / "manifest.json", result, immutable=True) - write_csv( - destination / "samples.csv", - [ - {key: value for key, value in row.items() if key != "repetitions"} - for row in result["samples"] - ], - ) - write_csv( - destination / "repetitions.csv", - [ - {"attempt_id": row["attempt_id"], "repetition": index, **repetition} - for row in result["samples"] - for index, repetition in enumerate(row["repetitions"]) - ], - ) - write_csv( - destination / "configurations.csv", - [ - { - "configuration_id": row["id"], - **row["configuration"], - "status": row["status"], - "estimate_bytes": row["estimate_bytes"], - } - for row in result["configurations"] - ], - ) - write_csv(destination / "summary.csv", summaries(result["samples"])) - write_csv( - destination / "cases.csv", - list( - { - row["configuration"]["case_hash"]: scalar_columns( - row["configuration"], ("case", "revision", "case_hash") - ) - for row in result["configurations"] - }.values() - ), - ) - write_csv( - destination / "runs.csv", - [ - scalar_columns( - result, - ( - "bundle_id", - "created", - "mode", - "alias", - "suite_version", - "result_schema_version", - "exploratory", - ), - ) - ], - ) - write_csv( - destination / "system.csv", - [ - { - "alias": alias, - **result["hardware"], - "unavailable_reason": result["hardware_unavailable_reason"], - } - ], - ) - return result diff --git a/benchmarks/qualification.py b/benchmarks/qualification.py deleted file mode 100644 index 2ba1f0e8..00000000 --- a/benchmarks/qualification.py +++ /dev/null @@ -1,419 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Prepare independently retained reference, refinement and reset evidence.""" - -from __future__ import annotations - -import argparse -import dataclasses -import math -import os -import shutil -import sys -import uuid -from pathlib import Path - -from benchmarks.collection import launch -from benchmarks.records import ( - Record, - SuiteError, - atomic_record, - file_hash, - identity, - now, - read_record, - run_lock, -) -from benchmarks.references import ( - ReferenceReceipt, - approved_envelope, - field_errors, - measurement_identity, - reference_path, - spectral_errors, -) -from benchmarks.registry import ADAPTIVE -from benchmarks.system import admission, thread_environment - -__all__ = [ - "Qualification", - "envelope_failures", - "qualify", - "qualify_row", - "recheck_row", - "reference_lineage", - "reviewed_reference", - "run_reference", -] - - -@dataclasses.dataclass(frozen=True) -class Qualification: - """Declare the bounded qualification run and optional reviewed envelope.""" - - output: Path - timeout: float - memory_limit: int | None - accuracy_bound: float | None - reference_revision: int | None = None - policy_revision: int | None = None - supersedes: Path | None = None - revision_note: str | None = None - reviewed: bool = False - - -def reviewed_reference(evidence: Record, bound: float | None) -> bool: - """Verify the evidence required by an explicit retained-reference review. - - The caller confirms review of the history, statistics and work-count - differences. These diagnostics have case-dependent conditioning; this helper - does not invent a universal rounding tolerance or demand identical histories. - The existing field ceiling also applies to the reset field witness. - """ - if bound is None or not math.isfinite(bound) or bound <= 0: - raise SuiteError("reviewed comparison requires a positive accuracy bound") - records = evidence.get("records", {}) - for name in ("tight", "coarse", "matched", "candidate"): - record = records.get(name, {}) - reset = record.get("reset_evidence") or {} - errors = reset.get("plane_errors", []) - if ( - record.get("status") != "ok" - or not errors - or not all(math.isfinite(value) and 0 <= value <= bound for value in errors) - or not reset.get("history_statistics_noise") - or not reset.get("fresh_counts") - or not reset.get("reset_counts") - ): - raise SuiteError( - f"{name} lacks successful reset/reference evidence " - "within the field ceiling" - ) - return True - - -def reference_lineage(configuration: Record, options: Qualification) -> Record: - """Version a replacement without changing the workload or old receipts.""" - if any( - value is not None and value < 1 - for value in (options.reference_revision, options.policy_revision) - ): - raise SuiteError("reference and policy revisions must be positive") - if options.accuracy_bound is not None and ( - not math.isfinite(options.accuracy_bound) or options.accuracy_bound <= 0 - ): - raise SuiteError("accuracy bound must be positive and finite") - previous = None - if options.supersedes is not None: - previous = read_record( - options.supersedes - / f"{configuration['case']}-{configuration['track']}.json" - ) - if previous["measurement_hash"] != measurement_identity(configuration): - raise SuiteError("superseded reference belongs to a different measurement") - if not options.revision_note or not options.revision_note.strip(): - raise SuiteError("superseding a reference requires --revision-note") - old_reference = previous.get("reference_revision", 1) if previous else 0 - old_policy = previous.get("policy_revision", 1) if previous else 0 - changed_bound = ( - previous is not None - and previous.get("accuracy_bound") != options.accuracy_bound - ) - reference_revision = options.reference_revision or old_reference + 1 - approved = approved_envelope(configuration) - approved_revision = ( - approved.revision - if approved and approved.bound == options.accuracy_bound - else 1 - ) - policy_revision = options.policy_revision or max( - approved_revision, old_policy + int(changed_bound) - ) - if reference_revision <= old_reference or policy_revision < max(1, old_policy): - raise SuiteError( - "reference revisions must advance and policy revisions cannot decrease" - ) - if changed_bound and policy_revision <= old_policy: - raise SuiteError("a changed accuracy bound requires a new policy revision") - return { - "reference_revision": reference_revision, - "policy_revision": policy_revision, - "supersedes": identity(previous) if previous else None, - "revision_note": options.revision_note, - } - - -def recheck_row(row: Record, source: Path, options: Qualification) -> Record: - """Apply a revised bound to retained evidence without running a simulation.""" - configuration = row["configuration"] - prefix = f"{configuration['case']}-{configuration['track']}" - previous = read_record(source / f"{prefix}.json") - envelope = approved_envelope(configuration) - bound = options.accuracy_bound - if bound is None: - bound = envelope.bound if envelope else previous.get("accuracy_bound") - options = dataclasses.replace(options, accuracy_bound=bound, supersedes=source) - lineage = reference_lineage(configuration, options) - evidence = read_record(reference_path(previous, "evidence", source)) - metrics = evidence.get("spectral_metrics_against_tight", {}) - if configuration["case"] in ("rdw", "ctc") and not metrics: - raise SuiteError( - "retained evidence lacks required band metrics; prepare a new reference" - ) - failures = ( - [] - if bound is None - else envelope_failures( - evidence["errors_against_tight"], - metrics, - bound, - adaptive=configuration["track"] in ADAPTIVE, - ) - ) - reviewed = { - **evidence, - "created": now(), - "rechecked_from": identity(previous), - "lineage": lineage, - "accuracy_bound": bound, - "bound_source": "explicit reviewed recheck" if options.accuracy_bound else None, - "status": "envelope-failed" - if failures - else "envelope-passed" - if bound - else "measured-unreviewed", - "failures": failures, - } - evidence_path = options.output / f"{prefix}-evidence.json" - atomic_record(evidence_path, reviewed, immutable=True) - if failures: - raise SuiteError("; ".join(failures) + "; recheck evidence retained") - replacement = { - **previous, - **lineage, - "accuracy_bound": bound, - "qualified_for_adoption": reviewed_reference(evidence, bound) - if options.reviewed - else False, - } - for artifact in ("fields", "band", "omega"): - if previous.get(f"{artifact}_file") is None: - continue - original = reference_path(previous, artifact, source) - target = options.output / "arrays" / f"{prefix}-{artifact}.npy" - target.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(original, target) - replacement[f"{artifact}_file"] = str(target.relative_to(options.output)) - replacement[f"{artifact}_hash"] = file_hash(target) - replacement.update( - evidence_file=evidence_path.name, evidence_hash=file_hash(evidence_path) - ) - atomic_record(options.output / f"{prefix}.json", replacement, immutable=True) - return reviewed - - -def envelope_failures( - errors: Record, metrics: Record, bound: float, *, adaptive: bool -) -> list[str]: - """Check a reviewed envelope and all retained refinement diagnostics.""" - failures = [] - if adaptive and bound < 10 * errors["coarse"]: - failures.append("field bound is below ten times the reference refinement floor") - if any(errors[name] > bound for name in ("matched", "candidate")): - failures.append("reviewed field envelope is not met") - for suffix, measured in metrics.items(): - for name, item in measured.items(): - if name == "energy": - continue - limit = bound - if suffix == "coarse": - if not adaptive: - continue - limit /= 10 - if item["value"] is None or item["value"] > limit: - failures.append( - f"{suffix} {name} exceeds its reviewed envelope/refinement limit" - ) - return failures - - -def run_reference( - configuration: Record, - row: Record, - options: Qualification, - profile: Record, - name: str, -) -> Record: - """Run one separately admitted reference or matched-method check.""" - directory = options.output / name - directory.mkdir(parents=True, exist_ok=True) - admit = admission(row["estimate_bytes"], options.memory_limit) - if not admit["admitted"]: - record = {"status": "refused", "admission": admit} - else: - fields_path = directory / "fields.npy" - record = launch( - sys.executable, - { - "configuration": configuration, - "repetitions": 1, - "verify_reset": True, - "save_fields": str(fields_path.resolve()), - }, - thread_environment( - configuration["threads"], profile["policy"], dict(os.environ) - ), - directory / "worker", - (options.timeout, admit["limit_bytes"]), - tuple(profile["allocation"]), - ) - atomic_record(directory / "result.json", record, immutable=True) - if record["status"] != "ok": - raise SuiteError( - f"qualification {name}: {record['status']}: {record.get('error', admit)}" - ) - return record - - -def qualify_row(row: Record, plan: Record, options: Qualification) -> Record: - """Preserve measured evidence; never manufacture an accuracy envelope.""" - import numpy as np # noqa: PLC0415 - explicit qualification only - - configuration = row["configuration"] - envelope = approved_envelope(configuration) - if options.accuracy_bound is None and envelope is not None: - options = dataclasses.replace(options, accuracy_bound=envelope.bound) - lineage = reference_lineage(configuration, options) - prefix = f"{configuration['case']}-{configuration['track']}" - oracle = {**configuration, "backend": "numpy_ref", "threads": 1} - adaptive = configuration["track"] in ADAPTIVE - tight = ( - {**oracle, "track": ADAPTIVE[1], "reference_rtol": 1e-10} - if adaptive - else oracle - ) - coarse = {**tight, "reference_rtol": 1e-9} if adaptive else oracle - records = {} - for suffix, settings in ( - ("tight", tight), - ("coarse", coarse), - ("matched", oracle), - ("candidate", configuration), - ): - print(f"qualify {prefix}: {suffix}", flush=True) - records[suffix] = run_reference( - settings, row, options, plan["profile"], f"{prefix}-{suffix}" - ) - paths = { - suffix: options.output / f"{prefix}-{suffix}" / "fields.npy" - for suffix in records - } - fields = { - suffix: np.load(path, allow_pickle=False, mmap_mode="r") - for suffix, path in paths.items() - } - band = np.load(f"{paths['tight']}.band.npy", allow_pickle=False) - errors = { - suffix: max(field_errors(value[..., band], fields["tight"][..., band])) - for suffix, value in fields.items() - if suffix != "tight" - } - omega_path = Path(f"{paths['tight']}.omega.npy") - omega = np.load(omega_path, allow_pickle=False) - metrics = { - suffix: spectral_errors(value, fields["tight"], omega, configuration["case"]) - for suffix, value in fields.items() - if suffix != "tight" - } - parity = max( - field_errors(fields["candidate"][..., band], fields["matched"][..., band]) - ) - evidence = { - "created": now(), - "configuration": configuration, - "records": records, - "errors_against_tight": errors, - "spectral_metrics_against_tight": metrics, - "matched_method_difference": parity, - "bound_source": dataclasses.asdict(envelope) - if envelope is not None and envelope.bound == options.accuracy_bound - else "explicit reviewed CLI envelope" - if options.accuracy_bound - else None, - "status": "measured-unreviewed", - "lineage": lineage, - } - bound = options.accuracy_bound - failures = [] - if bound is not None: - if not np.isfinite(bound) or bound <= 0: - raise SuiteError("accuracy bound must be positive and finite") - failures = envelope_failures(errors, metrics, bound, adaptive=adaptive) - evidence["status"] = "envelope-failed" if failures else "envelope-passed" - evidence["failures"] = failures - evidence_path = options.output / f"{prefix}-evidence.json" - atomic_record(evidence_path, evidence, immutable=True) - if failures: - raise SuiteError("; ".join(failures) + "; evidence retained") - receipt = ReferenceReceipt( - 1, - configuration["case_hash"], - measurement_identity(configuration), - str(paths["tight"].relative_to(options.output)), - file_hash(paths["tight"]), - str(paths["tight"].relative_to(options.output)) + ".band.npy", - file_hash(Path(f"{paths['tight']}.band.npy")), - bound, - errors["coarse"] if adaptive else None, - evidence_path.name, - file_hash(evidence_path), - "tight-rk-1e-10-refined-1e-9" if adaptive else "matched-fixed-numpy", - omega_file=str(omega_path.relative_to(options.output)), - omega_hash=file_hash(omega_path), - **lineage, - ) - atomic_record( - options.output / f"{prefix}.json", dataclasses.asdict(receipt), immutable=True - ) - return evidence - - -def qualify(plan: Record, args: argparse.Namespace) -> int: - """Run explicit qualification separately from ordinary measurement bundles.""" - output = args.out or args.references / uuid.uuid4().hex - options = Qualification( - output, - args.timeout, - args.memory_limit, - args.accuracy_bound, - args.reference_revision, - args.policy_revision, - args.supersedes, - args.revision_note, - args.reviewed, - ) - if args.reviewed and args.recheck is None: - raise SuiteError("--reviewed applies to retained evidence with --recheck") - if args.recheck is not None and args.supersedes not in (None, args.recheck): - raise SuiteError("--recheck and --supersedes must name the same source bundle") - with run_lock(output): - atomic_record(output / "plan.json", plan, immutable=True) - seen = set() - for row in plan["rows"]: - key = measurement_identity(row["configuration"]) - if key in seen: - continue - seen.add(key) - if row["status"] != "ready": - atomic_record( - output / f"excluded-{identity(row)}.json", row, immutable=True - ) - continue - if args.recheck is None: - qualify_row(row, plan, options) - else: - recheck_row(row, args.recheck, options) - print(f"Qualification evidence: {output.resolve()}") - return 0 diff --git a/benchmarks/records.py b/benchmarks/records.py index a552a9d9..3a1b91f7 100644 --- a/benchmarks/records.py +++ b/benchmarks/records.py @@ -7,47 +7,33 @@ from __future__ import annotations import contextlib -import csv -import dataclasses import hashlib import json import math import os -import statistics import tempfile -from collections.abc import Iterator, Sequence from datetime import UTC, datetime from pathlib import Path from typing import Any __all__ = [ "LOCAL_ROOT", - "SCHEMA_VERSION", - "SUITE_VERSION", - "Distribution", "Record", "SuiteError", "atomic_record", - "attempts", "canonical", - "distribution", - "export_tables", "file_hash", "identity", "metadata_value", "now", "read_record", - "run_lock", - "write_csv", ] # JSON is the boundary to independently installed worker environments and native # build records. Dynamic fields stay in these records, never in the case builders. type Record = dict[str, Any] -SUITE_VERSION = 1 -SCHEMA_VERSION = 1 -LOCAL_ROOT = Path(".benchmarks") / "suite" +LOCAL_ROOT = Path(".benchmarks") class SuiteError(ValueError): @@ -113,160 +99,7 @@ def atomic_record(path: Path, value: Record, *, immutable: bool = False) -> None stream.write(canonical(value) + "\n") stream.flush() os.fsync(stream.fileno()) - # The controller holds the run lock, so no other owner can create path - # between the existence check and this same-filesystem rename. Path(temporary).replace(path) finally: with contextlib.suppress(FileNotFoundError): Path(temporary).unlink() - - -@contextlib.contextmanager -def run_lock(directory: Path) -> Iterator[None]: - """Hold a kernel-released lock, including across a controller crash.""" - directory.mkdir(parents=True, exist_ok=True) - with (directory / ".lock").open("a+b") as stream: - if os.name == "nt": - import msvcrt # noqa: PLC0415 - platform-specific - - stream.seek(0) - stream.write(b"0") - stream.flush() - stream.seek(0) - try: - msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1) - except OSError as error: - raise SuiteError(f"run already locked: {directory}") from error - try: - yield - finally: - stream.seek(0) - msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1) - else: - import fcntl # noqa: PLC0415 - POSIX lock - - try: - fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB) - except BlockingIOError as error: - raise SuiteError(f"run already locked: {directory}") from error - try: - yield - finally: - fcntl.flock(stream, fcntl.LOCK_UN) - - -def write_csv(path: Path, rows: Sequence[Record]) -> None: - """Write a derived UTF-8 table, with explicit JSON for nested values.""" - keys = sorted({key for row in rows for key in row}) - descriptor, temporary = tempfile.mkstemp(dir=path.parent, prefix=".csv-") - try: - with os.fdopen(descriptor, "w", encoding="utf-8", newline="") as stream: - writer = csv.DictWriter(stream, fieldnames=keys) - writer.writeheader() - for row in rows: - writer.writerow( - { - key: canonical(value) - if isinstance(value, (dict, list, tuple)) - else "" - if value is None - else value - for key, value in row.items() - } - ) - stream.flush() - os.fsync(stream.fileno()) - Path(temporary).replace(path) - finally: - with contextlib.suppress(FileNotFoundError): - Path(temporary).unlink() - - -@dataclasses.dataclass(frozen=True) -class Distribution: - """Keep descriptive statistics; one observation has no dispersion estimate.""" - - count: int - median: float | None - mad: float | None - - -def distribution(values: Sequence[float]) -> Distribution: - """Summarize positive finite times without selecting best observations.""" - if any(not math.isfinite(value) or value <= 0 for value in values): - raise SuiteError("sample times must be finite and positive") - if not values: - return Distribution(0, None, None) - median = statistics.median(values) - mad = ( - None - if len(values) < 2 - else statistics.median(abs(value - median) for value in values) - ) - return Distribution(len(values), median, mad) - - -def attempts(directory: Path) -> list[Record]: - """Read only completed immutable shards; temporary writes are ignored.""" - return [ - read_record(path) for path in sorted((directory / "attempts").glob("*.json")) - ] - - -def export_tables(directory: Path) -> None: - """Rebuild derived tables from durable attempts after collection or resume.""" - rows = attempts(directory) - write_csv(directory / "samples.csv", rows) - write_csv( - directory / "accuracy.csv", - [ - { - "attempt_id": row["attempt_id"], - "sample_id": row["sample_id"], - "arm": row["arm"], - "configuration_id": row["configuration_id"], - **row.get("accuracy", {"status": "unavailable"}), - } - for row in rows - ], - ) - from benchmarks.comparison import completed # noqa: PLC0415 - - groups: dict[tuple[str, str], list[Record]] = {} - for row in completed(rows): - groups.setdefault((row["configuration_id"], row["arm"]), []).append(row) - summaries: list[Record] = [] - for (configuration, arm), group in groups.items(): - valid = [row for row in group if row["status"] == "ok"] - for population in ("all", "clean"): - selected = ( - valid - if population == "all" - else [row for row in valid if row.get("clean") is True] - ) - times = distribution([row["seconds"] for row in selected]) - peaks = [ - row["peak_rss_bytes"] - for row in selected - if row.get("peak_rss_bytes") is not None - ] - starts = [ - row["startup_seconds"] - for row in selected - if row.get("startup_seconds") is not None - ] - summaries.append( - { - "configuration_id": configuration, - "arm": arm, - "population": population, - **dataclasses.asdict(times), - "peak_rss_bytes": max(peaks) if peaks else None, - "startup_seconds": statistics.median(starts) if starts else None, - "failed_attempts": len(group) - len(valid), - } - ) - write_csv(directory / "summary.csv", summaries) - from benchmarks.views import write_views # noqa: PLC0415 - - write_views(directory, rows) diff --git a/benchmarks/references.py b/benchmarks/references.py deleted file mode 100644 index 63120621..00000000 --- a/benchmarks/references.py +++ /dev/null @@ -1,604 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Identify reference receipts and evaluate the declared worst-plane field metric.""" - -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path -from typing import TYPE_CHECKING - -from benchmarks.records import ( - Record, - SuiteError, - file_hash, - identity, - read_record, -) - -__all__ = [ - "ADVANCED_ENVELOPES", - "CORE_ENVELOPES", - "EXERCISE_ENVELOPES", - "MODAL_ENVELOPES", - "AccuracyEnvelope", - "ReferenceReceipt", - "approved_envelope", - "check", - "field_errors", - "measurement_identity", - "reference_path", - "spectral_errors", -] - -if TYPE_CHECKING: - import numpy as np - from jaxtyping import Complex, Float - - -@dataclass(frozen=True) -class AccuracyEnvelope: - """Freeze a reviewed acceptance ceiling independently of solver tolerance. - - Attributes - ---------- - case : str - Public case alias. - revision : int - Accuracy-policy revision, independent of the case and result schemas. - bound : float - Dimensionless relative field and retained-band error ceiling. - measurements : tuple of str - Exact workload/measurement identities covered by this review. - """ - - case: str - revision: int - bound: float - measurements: tuple[str, ...] - - -# SEP 30 Part II records the independent reference/refinement pilot and its -# review. The ceilings use source-campaign target levels with headroom, not a fit -# to the fastest candidate. Hashes prevent a changed window inheriting this review. -CORE_ENVELOPES = ( - AccuracyEnvelope( - "rdw", - 1, - 1e-4, - ( - "6ea41171e90bd551e8b3afcf05dcf1d75f570f8db36f75f193302b2ea5e31732", - "a2c01b6756f5257aecfbdfa1d801a27d790df0c033fbfc3f80a2a6f6195d64af", - ), - ), - AccuracyEnvelope( - "ctc", - 1, - 1e-5, - ( - "b9bb84f300848a0fb3a7132c92039697f977f6c1ebf9eb08f98f98a8c1c547de", - "e43c31b1ed8784d9d85e210336a7484752aa29782da5439928c1ddfb3f5eb21e", - ), - ), - AccuracyEnvelope( - "radial_filament_256", - 1, - 1e-4, - ( - "fd4f0b9918208908821ebad776d54b34ff476f752bb17ad06b5c4c8d9d1c7982", - "27a930d6d7849dc5422a6cb053def410743b2d5890adbbf8dcc951d95ae9abdb", - ), - ), -) - - -# Reviewed phase 2 ceilings; see SEP 30 Part II for measured evidence. -MODAL_ENVELOPES = ( - AccuracyEnvelope( - "rdw_one_mode", - 1, - 1e-4, - ( - "84398731ca1744959ce10c617def158c21c3103589fa2d1708c0af7d79dabcd1", - "bc171d288de1c2889b0023888f375b4d2327ddf6fc64d49da27fb72757a30a85", - ), - ), - # Workload revision 2 (the 2 ps window): the same 1e-4 ceiling, qualified anew - # (candidates 3.3e-7 and 1.1e-7 against tight RK, refinement witness 1.1e-11). - AccuracyEnvelope( - "ctc_reduced", - 1, - 1e-4, - ( - "f26795def59044337a7f89e3ebf8138eccf255cb2c5ea55168b07e4f8c0c1cf7", - "c82eda51f852d80f400c5097db873f5049287461d2789a2e90a766b91fdc18e5", - ), - ), - AccuracyEnvelope( - "argon_blueshift", - 1, - 1e-4, - ( - "535113368d35b09ad660d445aee2233020f9c374a876142b35c873378569fc7d", - "a98e65bf58f35de0f4a456c42b6a7f149166f5e3b495f6fa143fc4f0812b2ce8", - ), - ), - AccuracyEnvelope( - "h2_raman_6ps", - 1, - 1e-4, - ( - "6a17ad8758d995c1458fec33ad01536b26d2fe03a3fb46ea9964ac30e7b7f2c6", - "0679ec539ff730fe2aae493e9d807cffb5f012179b8c98be7b3e5c1bf608298e", - ), - ), - AccuracyEnvelope( - "he_rdw_averaged", - 1, - 1e-4, - ( - "888f3999189c641288bcc860fbddcf4624db391e5578805d4c494496d2919c83", - "9c8963f38ea41269e16a523be941159d9c75cb5baa8607b1077a9c9abcea46ff", - ), - ), - AccuracyEnvelope( - "argon_blueshift_averaged", - 1, - 1e-4, - ( - "ab37a5d92bd7e06c1df1a0a14796f69af327d1992033acfe209cca544db43114", - "74837207a40ba6312b0c495771993123e6e70a6d86bb4e51032b7c2a98a76636", - ), - ), - AccuracyEnvelope( - "h2_raman_6ps_averaged", - 1, - 1e-4, - ( - "518d3484e2f2f3da16db6819464eaf6d08976574b13e89d9f08bde274aa0d851", - "980f036d6e7bedfab73abc814f03edd1d5cfedaf5145db8cfd0287dfbb19306c", - ), - ), - AccuracyEnvelope( - "gnlse_soliton", - 1, - 1e-4, - ( - "aa895940a728b6e3b57472987b29dc740e06087cd694b58342436cfc39af9296", - "0780a2bbbf0b678986662eda981391524d0385524a7d8a8fb365e0711e1166a0", - ), - ), - AccuracyEnvelope( - "gnlse_raman_shock", - 1, - 1e-4, - ( - "eadafdd81e90c9038c378fb450e4eb6092d4dd8100cf371bc0a4a8cc28519320", - "aded17e77fe4c1a7838d8aca5cfc1729301dda9fbd6cf3c17d242f76f8e4ea95", - ), - ), -) - - -# Reviewed advanced and spatial ceilings, with measured headroom. -ADVANCED_ENVELOPES = ( - AccuracyEnvelope( - "scalar_wall", - 1, - 1e-4, - ( - "9ba5e7e873ed929e2b7db006932ebee35ca873548f29851fc18aaa0422ba9caa", - "86687e068e8218247971ae5623cc969ce18f3de1ba1ca66a27e282f4fd7aa9e1", - ), - ), - AccuracyEnvelope( - "vector_wall", - 1, - 1e-4, - ( - "1c38a9c2d6a6a39caf2b8b06cb6cdf4b1f4bb785c305a20ec1f3342f0484ba85", - "94433388be568d010b2800fbd3fa0d17c3319069e3eb7fd6077050f10f65c48c", - ), - ), - AccuracyEnvelope( - "taper", - 1, - 1e-4, - ( - "9deef7413f17aba0e269726bb6a54900d588bc9dff6ed96979dc2b9114522f35", - "592a601bd2e5504a13c345227947ae10ce6a1ac809b793f158b179ed2f5458aa", - ), - ), - AccuracyEnvelope( - "taper_wall", - 1, - 1e-4, - ( - "3bbe22681b246cb1a77257208090f8d2bc20046a139b697035bb63ecacd3b6ae", - "473f17d1d87579fe1eeda62eaffc5bf54e3f7135043fd9977aff0f28131f774b", - ), - ), - AccuracyEnvelope( - "step_index_pair", - 1, - 1e-4, - ( - "81627ea4c59490d57102307eed5671ca58418f6f53722388a9a78d60ee6902b1", - "a7864758611f50c7cc5fa8d08cffa9d3995933c21edcf6b322d34bc2fcf6b92d", - ), - ), - AccuracyEnvelope( - "grin_linear", - 1, - 1e-4, - ( - "967d370e58f4c9f737177826d7ccae67adaa042206847d3715767dcabaecd47e", - "fb5ab666bacc8d6927571dabcdad939b344d1a65cbb5e7d35041285272a38c5f", - ), - ), - AccuracyEnvelope( - "grin_nonlinear", - 1, - 1e-4, - ( - "dc66ca6ccc1a65baadec48f7c96e26e37b2d199d57dba41e4500e119fc0121c7", - "d49290aca313beff4ebc73243cd2d9d186cba2bf9d52669d1ff65e16659d9bb3", - ), - ), - AccuracyEnvelope( - "moving_solid_guide", - 1, - 1e-4, - ( - "129cd4088e49bf0c1c5331e9749cd36cd46ec312727e35409238daef177b78e4", - "f0b8f82edd4d13115be421e1810c7ef209aa8e81007507812a076a3f85e7ec63", - ), - ), - AccuracyEnvelope( - "line", - 1, - 1e-4, - ( - "8bf25ceed3203b27ec13865165ef6cea225c1286cc7525429e24fec86ed612c1", - "980eecce15e9ec1945afeceb1f78cb664b8772244d31d494eeccd056af778c5b", - ), - ), - AccuracyEnvelope( - "planar", - 1, - 1e-4, - ( - "0e4d9b2bf5cdc30fad613f55f907c4f33afdf1d4ace3f11233149024223d1c81", - "a41e0224f29efeb3de357cf4dc9cf5f16bf5c3ad0c11e21df44be5073b7be14e", - ), - ), - AccuracyEnvelope( - "spatial_absorber", - 1, - 1e-4, - ( - "7de22f4914a486c40a822928232299064908a0f3debb5af1aaf623b68d729ee1", - "0c007223550acdbb8c4eeb6a997af893bd955f0470a687a3954f2a58d9f37f80", - ), - ), -) - - -# Reviewed backend-exercise ceilings, independent of solver settings. -EXERCISE_ENVELOPES = ( - AccuracyEnvelope( - "vector_raman", - 1, - 1e-4, - ( - "0b4c6af3f20ade9ecec1072f1eac075928ad22c14e6318a2da9f95e5031fe3ba", - "b1a6e6fdd496a92a335aa8007567145fa26320c3f0221a3bed57c2a4bcc584d0", - ), - ), - AccuracyEnvelope( - "envelope_plasma", - 1, - 1e-4, - ( - "a7192b6564f5080e6b6c1b8547664724c37831bb401061ccc7f2fcb84df37948", - "5018094657908f68afe83e89a27fb3ad869c8008ed6a250a4d81b8baddc1caae", - ), - ), - AccuracyEnvelope( - "mixture_gradient", - 1, - 1e-4, - ( - "2d1c9a98977d3e4839cc5558e318c10548b9bad9d8df16581fb5772c89138e60", - "6dea7e88f7cfdffc39be63c097a2966a05134e9e71d0e7471c4c55ad6dd6973c", - ), - ), - AccuracyEnvelope( - "dispersive_kerr", - 1, - 1e-4, - ( - "588813bb72fc29f6b3834ded9c7078a8a1a5746eef2680c75ad65e5e5a272c96", - "97688f847d3f35cb2be89782cd16aef024201a814da67bf4b4da67a19cc67f19", - ), - ), - AccuracyEnvelope( - "plasma_band", - 1, - 1e-4, - ( - "7a9effb9c27b84f72195703d8e302ba5684861461af079ba1af26d1e66b7c8cf", - "d19a47f0d7595f813c452a8d21ae826c606d9b179f7e7963a2efddbd18abffab", - ), - ), - AccuracyEnvelope( - "plasma_trapezoid", - 1, - 1e-4, - ( - "abda1dacc9a31f448e8b3690bc8c55f6fb1954e652d1d20cc9e081bc6aed41df", - "1bcf695c1d051f8b5c211292642f0e726412067f22f8dd79bdda801336f5d286", - ), - ), - AccuracyEnvelope( - "noise", - 1, - 1e-4, - ( - "5d3495b4fd41c594e540e16df660d76e7550d68a93e923b1fb5a916f0fe9a002", - "ab15d2ce3fdb08db0e9da45629045f6d895fb619d2bdbe3c18b7f3073adaccf2", - ), - ), - AccuracyEnvelope( - "dense_output", - 1, - 1e-4, - ( - "c6c68a6abd1d74cb374c726452ca8e44a9ac752fd3278ce50c94a54fcadcac1b", - "8bd833422a9dab440b53eb4e7fa10972b9d4db9e184dace1908745bfcd429255", - ), - ), - AccuracyEnvelope( - "absorber_schemes", - 1, - 1e-4, - ( - "a7d905d7c129c6019fd9deee3ab6dc45b30f75d2dc3bbe5337d03f72a5f32ff1", - "8d3fd4c34ccbbeba9a7858521de43bd338139c89619e6a2a8c322a96d4de25c0", - ), - ), - AccuracyEnvelope( - "ordinary_stats", - 1, - 1e-4, - ( - "4dabb07cb9e633d7ac63577fb19172c86848d5d865078d0feaa8f46397ab6334", - "0900e26546e1db9bc6c852a3fe9bc8794a79e8e309d4fa48a09801235982547f", - ), - ), - AccuracyEnvelope( - "far_field_stats", - 1, - 1e-4, - ( - "d1de6a66bc4d147aabe4d590f7255e7d0c55ebf8682b55c58b88052af3d5a989", - "0d107f4509d4d8854093958451882b59b6a0e04c4e52274d175d49f7e76c34aa", - ), - ), -) - - -@dataclass(frozen=True) -class ReferenceReceipt: - """Freeze a numerical reference independently of timing source/build identity.""" - - revision: int - case_hash: str - measurement_hash: str - fields_file: str - fields_hash: str - band_file: str - band_hash: str - accuracy_bound: float | None - refinement_floor: float | None - evidence_file: str - evidence_hash: str - protocol: str - qualified_for_adoption: bool = False - omega_file: str | None = None - omega_hash: str | None = None - reference_revision: int = 1 - policy_revision: int = 1 - supersedes: str | None = None - revision_note: str | None = None - - -def measurement_identity(configuration: Record) -> str: - """Exclude execution choices while retaining method, tolerance and window.""" - return identity( - { - key: value - for key, value in configuration.items() - if key not in ("backend", "threads", "profile_hash", "policy") - } - ) - - -def approved_envelope(configuration: Record) -> AccuracyEnvelope | None: - """Return a reviewed bound only for its exact frozen measurement.""" - key = measurement_identity(configuration) - return next( - ( - item - for item in ( - *CORE_ENVELOPES, - *MODAL_ENVELOPES, - *ADVANCED_ENVELOPES, - *EXERCISE_ENVELOPES, - ) - if key in item.measurements - ), - None, - ) - - -def field_errors( - fields: Complex[np.ndarray, "n_z ..."], truth: Complex[np.ndarray, "n_z ..."] -) -> list[float]: - """Return relative L2 errors for each plane, refusing malformed references.""" - import numpy as np # noqa: PLC0415 - qualification/worker only - - if fields.shape != truth.shape or len(fields) == 0: - raise SuiteError("reference shape mismatch or empty fields") - errors = [] - for found, expected in zip(fields, truth, strict=True): - denominator = float(np.linalg.norm(expected.ravel())) - if denominator == 0 or not np.isfinite(denominator): - raise SuiteError("reference field norm is zero or non-finite") - error = float(np.linalg.norm((found - expected).ravel())) / denominator - if not np.isfinite(error): - raise SuiteError("non-finite field error") - errors.append(error) - return errors - - -def spectral_errors( - fields: Complex[np.ndarray, "n_z n_modes n_pol n_omega"], - truth: Complex[np.ndarray, "n_z n_modes n_pol n_omega"], - omega: Float[np.ndarray, " n_omega"], - case: str, -) -> Record: - """Retain the source campaigns' weak-bin, UV and modal-energy diagnostics. - - The weak bins are frozen from the reference's final-plane power, between - 1e-9 and 1e-3 of its peak. Band norms have the source campaign's 1e-3 floor - across planes. RDW additionally uses 100--150 nm. See SEP 16's - ``campaign.py::score`` and ``varying_campaign.py::score``. At a common grid, - Parseval's constant cancels from the relative modal-energy difference, whose - endpoint weights follow ``solprop.fourier.energy_omega`` (conventions §4). - Energy is a diagnostic, not a separate acceptance threshold. - """ - import numpy as np # noqa: PLC0415 - - from solprop.constants import C # noqa: PLC0415 - worker/qualification only - from solprop.fourier import energy_omega # noqa: PLC0415 - - if case not in ("rdw", "ctc"): - return {} - if ( - fields.ndim != 4 - or fields.shape != truth.shape - or not len(fields) - or omega.shape != (fields.shape[-1],) - or not np.all(np.isfinite(omega)) - or not np.all(np.isfinite(fields)) - or not np.all(np.isfinite(truth)) - ): - raise SuiteError("spectral metric shape or finite-value contract is invalid") - if np.any(np.linalg.norm(truth.reshape(len(truth), -1), axis=1) == 0): - raise SuiteError("spectral reference has a zero-norm plane") - found, reference = fields.reshape(len(fields), -1), truth.reshape(len(truth), -1) - power = np.abs(reference[-1]) ** 2 - masks = {"weak": (power > 1e-9 * power.max()) & (power < 1e-3 * power.max())} - if case == "rdw": - ultraviolet = (omega >= 2 * np.pi * C / 150e-9) & ( - omega <= 2 * np.pi * C / 100e-9 - ) - masks["uv"] = np.broadcast_to(ultraviolet, truth.shape[1:]).reshape(-1) - result: Record = {} - for name, mask in masks.items(): - norms = np.linalg.norm(reference[:, mask], axis=1) - scale = float(norms.max()) - if not scale > 0: - result[name] = {"value": None, "reason": "reference band has zero norm"} - continue - differences = np.linalg.norm((found - reference)[:, mask], axis=1) - values = differences / np.maximum(norms, 1e-3 * scale) - result[name] = { - "value": float(values.max()), - "planes": values.tolist(), - "selected_bins": int(mask.sum()), - } - # Both core modal cases use field grids and saved true-power amplitudes. - # The common dt cancels; the endpoint/interior weights must still be retained. - reference_energy = energy_omega(truth, 1.0, kind="field").sum(axis=(1, 2)) - energy = energy_omega(fields, 1.0, kind="field").sum(axis=(1, 2)) - values = np.abs(energy / reference_energy - 1) - result["energy"] = { - "value": float(values.max()), - "planes": values.tolist(), - "role": "diagnostic", - } - return result - - -def reference_path(receipt: Record, key: str, directory: Path) -> Path: - """Validate a stored artifact's relative path and content identity.""" - path = (directory / receipt[f"{key}_file"]).resolve() - if not path.is_relative_to(directory.resolve()): - raise SuiteError("reference artifact escapes its directory") - if file_hash(path) != receipt[f"{key}_hash"]: - raise SuiteError(f"reference {key} hash mismatch") - return path - - -def check(fields: np.ndarray, configuration: Record, path: Path) -> Record: - """Compare after measurement, retaining full-band and scored-band errors.""" - import numpy as np # noqa: PLC0415 - - receipt = read_record(path) - if receipt.get("revision") != 1: - raise SuiteError("unsupported reference receipt revision") - if receipt["case_hash"] != configuration["case_hash"] or receipt[ - "measurement_hash" - ] != measurement_identity(configuration): - raise SuiteError("reference measurement identity mismatch") - truth = np.load( - reference_path(receipt, "fields", path.parent), - allow_pickle=False, - mmap_mode="r", - ) - band = np.load(reference_path(receipt, "band", path.parent), allow_pickle=False) - reference_path(receipt, "evidence", path.parent) - if band.dtype != np.bool_ or band.shape != (fields.shape[-1],) or not band.any(): - raise SuiteError("reference band shape or values invalid") - full_errors = field_errors(fields, truth) - errors = field_errors(fields[..., band], truth[..., band]) - bound = receipt.get("accuracy_bound") - if bound is not None and (not np.isfinite(bound) or bound <= 0): - raise SuiteError("reference accuracy bound must be positive and finite") - worst = max(errors) - metrics = {} - if receipt.get("omega_file") is not None: - omega = np.load( - reference_path(receipt, "omega", path.parent), allow_pickle=False - ) - metrics = spectral_errors(fields, truth, omega, configuration["case"]) - missing_metrics = configuration["case"] in ("rdw", "ctc") and not metrics - metrics_pass = bound is not None and all( - item["value"] is not None and item["value"] <= bound - for name, item in metrics.items() - if name != "energy" - ) - return { - "status": "not-qualified" - if bound is None or missing_metrics - else "passed" - if worst <= bound and metrics_pass - else "invalid", - "worst_plane_relative_field_error": worst, - "all_bins_error": max(full_errors), - "plane_errors": errors, - "accuracy_bound": bound, - "reference_id": identity(receipt), - "reference_revision": receipt.get("reference_revision", 1), - "policy_revision": receipt.get("policy_revision", 1), - "spectral_metrics": metrics, - "qualified_for_adoption": receipt.get("qualified_for_adoption", False) - and not missing_metrics - and metrics_pass - and bound is not None - and worst <= bound, - } diff --git a/benchmarks/registry.py b/benchmarks/registry.py index dfdd4782..94f322c7 100644 --- a/benchmarks/registry.py +++ b/benchmarks/registry.py @@ -2,238 +2,303 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -"""Declare immutable benchmark workloads without importing their builders.""" +"""Declare immutable benchmark workloads without importing their builders. + +Every case is a complete guide run with its interface's default integrator and +tolerances (SEP 37), a fixed noise seed and the native backend unless asked +otherwise (SEP 38). The ``tracking`` set is what ``python -m benchmarks track`` +times by default; every other case runs only when named. +""" from __future__ import annotations from dataclasses import asdict, dataclass from benchmarks.records import Record, SuiteError, identity -from benchmarks.schedules import ENTRANCE_SCHEDULES __all__ = [ - "ADAPTIVE", "CASES", - "FIXED", - "KERR", + "SEED", "SETS", - "TRACKS", + "STEPPERS", + "TRACKING", + "WIDTHS", "Case", - "ResolvedConfiguration", - "WarningRule", "select", ] -ADAPTIVE = ("adaptive-multistep", "adaptive-rk") -FIXED = "fixed-rk" -KERR = "fixed-rk4ip" -TRACKS = (*ADAPTIVE, FIXED, KERR) SETS = ( - "core", + "tracking", "modal-capillary", "free-space", "mode-averaged", "advanced", "backend-exercise", ) - -# First ten accepted entrance steps of the independent NumPy default-RK pilot -# at rtol=1e-6. SEP 30 Part II records the pilot and verification. Keep these -# separate from Case so freezing a diagnostic does not rename adaptive windows. -_FIXED_SCHEDULES = { - **{ - (name, 1): edges - for name, edges in ENTRANCE_SCHEDULES.items() - if name - not in ("ctc_reduced", "scalar_wall", "vector_wall", "taper", "taper_wall") - }, - ("radial_filament_256", 1): ( - 0.0, - 3e-6, - 3.7247892534444475e-5, - 6.857136108217572e-5, - 9.611546370159467e-5, - 0.00012252784589431744, - 0.00014998314854780485, - 0.00017906017049297354, - 0.0002095001546572779, - 0.00024121867141398048, - 0.00027452363203191686, - ), -} - - -@dataclass(frozen=True) -class WarningRule: - """Declare one reviewed warning by phase, category and narrow message pattern.""" - - phase: str - category: str - pattern: str - - -@dataclass(frozen=True) -class ResolvedConfiguration: - """Freeze planned measurement semantics before constructing scientific objects.""" - - case: str - revision: int - case_hash: str - track: str - start: float - stop: float | None - schedule: tuple[float, ...] - step: float | None - rtol: float | None - atol: float | None - observations: int - fields: str - statistics: str = "source-default" - noise_seed: int | None = None - reset: str = "launch-and-lead-in-v1" - measurement_version: int = 1 +TRACKING = "tracking" +STEPPERS = ("default", "multistep", "multistep-interaction", "rk") +WIDTHS = ("1", "production") +SEED = 20261027 +"""The noise seed every case fixes, so that its repeats take the same steps.""" @dataclass(frozen=True) class Case: - """Name the definition, supported measurements and qualification status.""" + """Name one complete workload, how it is observed and what it costs. + + Attributes + ---------- + name : str + Registry key; a changed definition keeps the name and raises `revision`. + revision : int + Definition revision; history rows compare only within one. + sets : tuple of str + Selection groups; ``tracking`` puts the case in the default run. + interface : str + The ``prop_*`` interface whose defaults the case runs with. + builder : str + Builder family in ``benchmarks.builders``. + length : float + Guide length in metres; a run propagates all of it. + dimensions : tuple of int + Main array sizes, for the plan only. + source : str + Where the definition comes from. + widths : tuple of str + Thread widths the case runs at by default: ``"1"`` and ``"production"``. + planes : int + Evenly spaced observations, the last at `length`. + fields : str + ``"all"`` keeps every observed field, ``"planes"`` only the last. + step : float or None + A fixed step in metres; the case then runs its declared fixed schedule and + admits no stepper override. + cost : tuple of float or None + Seconds per complete run at 1 and 16 threads on an AMD EPYC 7763 node with + SEP 37's defaults (its phase 0), for the plan's estimate; ``None`` if unknown. + """ name: str revision: int sets: tuple[str, ...] + interface: str builder: str - tracks: tuple[str, ...] length: float - start: float - stop: float - step: float | None dimensions: tuple[int, ...] source: str - status: str = "implemented" - pending_reason: str | None = None - estimate_bytes: int | None = None - atol: float = 1e-10 - planes: int = 1 - warnings: tuple[WarningRule, ...] = () + widths: tuple[str, ...] = WIDTHS + planes: int = 11 + fields: str = "all" + step: float | None = None + cost: tuple[float | None, float | None] = (None, None) + + @property + def tracked(self) -> bool: + """Whether the default ``track`` selection includes the case.""" + return TRACKING in self.sets + + @property + def steppers(self) -> tuple[str, ...]: + """Stepper choices the case admits; a fixed schedule admits its own only.""" + return ("default",) if self.step is not None else STEPPERS @property def digest(self) -> str: - """Identify immutable definition values, excluding selection aliases.""" + """Identify the definition, excluding selection groups and estimates.""" record = asdict(self) - for key in ("sets", "status", "pending_reason", "estimate_bytes"): + for key in ("sets", "widths", "cost"): record.pop(key) return identity(record) - def measurement(self, track: str, schedule: tuple[float, ...] = ()) -> Record: - """Resolve a measurement without constructing any scientific object.""" - if track not in self.tracks: - raise SuiteError(f"{self.name} does not admit {track}") - if track == FIXED and not schedule: - schedule = _FIXED_SCHEDULES.get((self.name, self.revision), ()) - adaptive = track in ADAPTIVE - start = self.start if adaptive or track == KERR else 0.0 - stop = ( - self.stop - if adaptive or track == KERR - else ( - schedule[-1] - if schedule - else 10 * self.step - if self.step is not None - else None - ) - ) - return asdict( - ResolvedConfiguration( - self.name, - self.revision, - self.digest, - track, - start, - stop, - schedule, - self.step, - 1e-6 if track != KERR else None, - self.atol if track != KERR else None, - self.planes if adaptive else 1, - "planes" - if adaptive and self.name == "far_field_stats" - else "all" - if adaptive - else "none", - noise_seed=41 - if self.name == "grin_nonlinear" - else 13 - if self.name == "noise" - else None, - ) - ) + def configuration( + self, + stepper: str, + width: int, + backend: str = "native", + fraction: float = 1.0, + warmup: float = 0.02, + ) -> Record: + """Resolve one measurement without constructing a scientific object.""" + if stepper not in self.steppers: + raise SuiteError(f"{self.name} admits only {', '.join(self.steppers)}") + if not 0 < fraction <= 1 or not 0 <= warmup < 1: + raise SuiteError("length fraction must be in (0, 1], warm-up in [0, 1)") + return { + "case": self.name, + "revision": self.revision, + "case_hash": self.digest, + "stepper": stepper, + "threads": width, + "backend": backend, + "stop": self.length * fraction, + "fraction": fraction, + "warmup": warmup, + "planes": self.planes, + "fields": self.fields, + } -_CORE = ( +_TRACKED = ( + Case( + "gnlse_raman_shock", + 1, + ("tracking", "mode-averaged"), + "prop_gnlse", + "gnlse", + 0.15, + (1, 16384, 16384), + "Dudley, Genty and Coen, Rev. Mod. Phys. 78, 1135 (2006), section III", + cost=(80.0, 30.0), + ), + Case( + "gnlse_soliton", + 1, + ("tracking", "mode-averaged"), + "prop_gnlse", + "gnlse", + 1.3278075458959397, + (1, 4096, 4096), + "docs/introduction/tutorials/plot_02_soliton.py; one soliton period", + cost=(8.0, 5.0), + ), Case( "rdw", 1, - ("core", "modal-capillary"), + ("tracking", "modal-capillary"), + "prop_capillary", "capillary", - (*ADAPTIVE, FIXED), 1.5, - 0.02, - 0.04, - 0.002, (4, 4096, 8192), - "benchmarks/cases.py:rdw", + "He resonant dispersive wave, four resolved modes", + cost=(19.0, 10.0), ), Case( - "ctc", + "he_rdw_averaged", 1, - ("core", "modal-capillary"), + ("tracking", "mode-averaged"), + "prop_capillary", + "capillary", + 1.5, + (1, 4096, 8192), + "rdw on the mode-averaged route", + cost=(3.0, 3.0), + ), + # Revision 2 (SEP 38): 0.2 m and 160 nm-4 um, as SEP 37 phase 0 measured it; + # revision 1 was the 0.4 m blue-shift example with a 300 nm edge. + Case( + "argon_blueshift", + 2, + ("tracking", "modal-capillary"), + "prop_capillary", "capillary", - (*ADAPTIVE, FIXED), 0.2, - 0.00017, - 0.00034, - 0.000017, - (6, 32768, 65536), - "benchmarks/cases.py:ctc", + (1, 4096, 8192), + "docs/examples/processes/plot_06_ionisation_blue_shift.py, shortened", + cost=(67.0, 34.0), ), Case( - "kerr_beam_384", + "argon_blueshift_averaged", + 2, + ("tracking", "mode-averaged"), + "prop_capillary", + "capillary", + 0.2, + (1, 4096, 8192), + "argon_blueshift on the mode-averaged route", + cost=(15.0, 15.0), + ), + Case( + "argon_blueshift_envelope", 1, - ("core", "free-space"), - "kerr", - (KERR,), - 5.5e-6, - 0.5e-6, - 5.5e-6, - 0.5e-6, - (384, 384, 600), - "benchmarks/sep28/cases.py:kerr_beam_384", - atol=0.0, - warnings=( - WarningRule( - "construction", - "MaterialRangeWarning", - r"malitson 'Malitson 1965' is valid over 210-3710 nm; " - r"the requested spectrum spans 130\.2-1200 nm\. Source: " - r"Malitson, JOSA 55, 1205 \(1965\), doi:10\.1364/JOSA\.55\.001205", - ), - ), + ("tracking", "modal-capillary"), + "prop_capillary", + "capillary", + 0.2, + (1, 2048, 2048), + "argon_blueshift on the envelope grid", + cost=(3.2, 3.1), + ), + Case( + "h2_raman_averaged", + 1, + ("tracking", "mode-averaged"), + "prop_capillary", + "capillary", + 0.4, + (1, 4096, 8192), + "docs/examples/processes/plot_05_raman_self_frequency_shift.py, 10 bar", + cost=(16.0, 16.0), + ), + # Revision 2 (SEP 38): three modes, 0.13 m, 2 ps and 220 nm-4 um, the + # definition SEP 37 phase 0 measured; revision 1 had six modes over 0.2 m. + Case( + "ctc", + 2, + ("tracking", "modal-capillary"), + "prop_capillary", + "capillary", + 0.13, + (3, 8192, 16384), + "H2 comb-to-continuum, three modes", + widths=("production",), + cost=(605.0, 290.0), ), Case( "radial_filament_256", 1, - ("core", "free-space"), + ("tracking", "free-space"), + "prop_radial", "spatial", - (*ADAPTIVE, FIXED), - 0.03, - 0.0, 0.03, - None, (256, 250), - "benchmarks/sep16/free_space.py:radial_filament", - atol=0.0, + "Ar filament with plasma, 256 radial nodes", planes=4, + cost=(10.0, 4.0), + ), + # Revision 2 (SEP 38): a 2 cm guide, the window SEP 37 phase 0 measured; + # revision 1 was the example's 0.1 m. + Case( + "grin_nonlinear", + 2, + ("tracking", "advanced"), + "prop_waveguide", + "advanced", + 0.02, + (4, 4096, 8192), + "docs/examples/waveguides/plot_04_graded_index_multimode.py, 2 cm", + widths=("production",), + planes=5, + cost=(102.0, 35.0), + ), + Case( + "kerr_beam_384", + 1, + ("tracking", "free-space"), + "prop_free", + "kerr", + 5.5e-6, + (384, 384, 600), + "fused-silica Kerr beam, fixed RK4IP steps", + widths=("production",), + planes=1, + fields="planes", + step=0.5e-6, + cost=(91.0, 10.0), + ), + Case( + "kerr_beam_192", + 1, + ("tracking", "free-space"), + "prop_free", + "kerr", + 5.5e-6, + (192, 192, 300), + "fused-silica Kerr beam, fixed RK4IP steps", + widths=("1",), + planes=1, + fields="planes", + step=0.5e-6, + cost=(10.0, None), ), ) @@ -242,120 +307,41 @@ def measurement(self, track: str, schedule: tuple[float, ...] = ()) -> Record: "rdw_one_mode", 1, ("modal-capillary",), + "prop_capillary", "capillary", - (*ADAPTIVE, FIXED), 1.5, - 0.0, - ENTRANCE_SCHEDULES["rdw_one_mode"][-1], - None, (1, 4096, 8192), - "benchmarks/cases.py:rdw with one resolved mode", + "rdw with one resolved mode", ), - # Revision 2 (SEP 35 phase 4): a 2 ps window, as a production run of this pulse - # takes; revision 1's 1 ps crowded the 400 fs pulse. Its entrance re-piloted. Case( "ctc_reduced", 2, ("modal-capillary",), + "prop_capillary", "capillary", - (*ADAPTIVE, FIXED), 0.1, - 0.0, - ENTRANCE_SCHEDULES["ctc_reduced"][-1], - 0.00012, (2, 8192, 16384), - "benchmarks/cases.py:ctc_reduced with a 2 ps window", - ), - Case( - "argon_blueshift", - 1, - ("modal-capillary",), - "capillary", - (*ADAPTIVE, FIXED), - 0.4, - 0.0, - ENTRANCE_SCHEDULES["argon_blueshift"][-1], - None, - (1, 2048, 4096), - "docs/examples/processes/plot_06_ionisation_blue_shift.py", + "ctc with two modes over 0.1 m and a 2 ps window", ), Case( "h2_raman_6ps", 1, ("modal-capillary",), + "prop_capillary", "capillary", - (*ADAPTIVE, FIXED), 2.0, - 0.0, - ENTRANCE_SCHEDULES["h2_raman_6ps"][-1], - None, (1, 16384, 32768), "docs/examples/processes/plot_05_raman_self_frequency_shift.py; 6 ps window", ), - Case( - "he_rdw_averaged", - 1, - ("mode-averaged",), - "capillary", - (*ADAPTIVE, FIXED), - 1.5, - 0.0, - ENTRANCE_SCHEDULES["he_rdw_averaged"][-1], - None, - (1, 4096, 8192), - "benchmarks/cases.py:rdw; benchmarks/sep25/cases.py averaged route", - ), - Case( - "argon_blueshift_averaged", - 1, - ("mode-averaged",), - "capillary", - (*ADAPTIVE, FIXED), - 0.4, - 0.0, - ENTRANCE_SCHEDULES["argon_blueshift_averaged"][-1], - None, - (1, 2048, 4096), - "docs/examples/processes/plot_06_ionisation_blue_shift.py", - ), Case( "h2_raman_6ps_averaged", 1, ("mode-averaged",), + "prop_capillary", "capillary", - (*ADAPTIVE, FIXED), 2.0, - 0.0, - ENTRANCE_SCHEDULES["h2_raman_6ps_averaged"][-1], - None, (1, 16384, 32768), - "docs/examples/processes/plot_05_raman_self_frequency_shift.py; 6 ps window", - ), - Case( - "gnlse_soliton", - 1, - ("mode-averaged",), - "gnlse", - (*ADAPTIVE, FIXED), - 1.3278075458959397, - 0.0, - ENTRANCE_SCHEDULES["gnlse_soliton"][-1], - None, - (1, 4096, 4096), - "docs/introduction/tutorials/plot_02_soliton.py", - ), - Case( - "gnlse_raman_shock", - 1, - ("mode-averaged",), - "gnlse", - (*ADAPTIVE, FIXED), - 0.15, - 0.0, - ENTRANCE_SCHEDULES["gnlse_raman_shock"][-1], - None, - (1, 16384, 16384), - "benchmarks/sep13/gnlse.py", + "h2_raman_6ps on the mode-averaged route", ), ) @@ -364,209 +350,81 @@ def measurement(self, track: str, schedule: tuple[float, ...] = ()) -> Record: "radial_filament_512", 1, ("free-space",), + "prop_radial", "spatial", - (*ADAPTIVE, FIXED), 0.03, - 0.0, - ENTRANCE_SCHEDULES["radial_filament_512"][-1], - None, (512, 1024, 2048), - "benchmarks/sep16/free_space.py:radial_filament_large", - atol=0.0, - ), - Case( - "cartesian_filament_128", - 1, - ("free-space",), - "spatial", - (*ADAPTIVE, FIXED), - 0.03, - 0.0, - ENTRANCE_SCHEDULES["cartesian_filament_128"][-1], - None, - (128, 128, 1024, 2048), - "benchmarks/sep16/free_space.py:cartesian_filament_large", - atol=0.0, - ), - Case( - "cartesian_filament_256", - 1, - ("free-space",), - "spatial", - (*ADAPTIVE, FIXED), - 0.03, - 0.0, - ENTRANCE_SCHEDULES["cartesian_filament_256"][-1], - None, - (256, 256, 1024, 2048), - "benchmarks/sep28/cases.py:cartesian_filament_256", - atol=0.0, + "radial_filament_256 at 512 nodes", ), - Case( - "kerr_beam_192", - 1, - ("free-space",), - "kerr", - (KERR,), - 5.5e-6, - 0.5e-6, - 5.5e-6, - 0.5e-6, - (192, 192, 300), - "benchmarks/sep28/cases.py:kerr_beam_192", - atol=0.0, - warnings=( - WarningRule( - "construction", - "MaterialRangeWarning", - r"malitson 'Malitson 1965' is valid over 210-3710 nm; " - r"the requested spectrum spans 130\.4-1182 nm\. Source: " - r"Malitson, JOSA 55, 1205 \(1965\), doi:10\.1364/JOSA\.55\.001205", - ), - ), + *( + Case( + f"cartesian_filament_{size}", + 1, + ("free-space",), + "prop_free", + "spatial", + 0.03, + (size, size, 1024, 2048), + "the Ar filament on a Cartesian grid", + ) + for size in (128, 256) ), Case( "kerr_beam_768", 1, ("free-space",), + "prop_free", "kerr", - (KERR,), 3e-6, - 0.5e-6, - 3e-6, - 0.5e-6, (768, 768, 600), - "benchmarks/sep28/cases.py:kerr_beam_768", - atol=0.0, - warnings=_CORE[2].warnings, + "fused-silica Kerr beam, fixed RK4IP steps", + planes=1, + fields="planes", + step=0.5e-6, ), Case( "line", 1, ("free-space",), + "prop_free", "spatial", - (*ADAPTIVE, FIXED), 0.08, - 0.0, - ENTRANCE_SCHEDULES["line"][-1], - None, (128, 512, 1024), - "benchmarks/sep16/free_space.py:line", - atol=0.0, + "Ar Kerr beam on a line grid", ), Case( "planar", 1, ("free-space",), + "prop_planar", "spatial", - (*ADAPTIVE, FIXED), 0.05, - 0.0, - ENTRANCE_SCHEDULES["planar"][-1], - None, (2, 32, 512, 512), - "benchmarks/sep16/free_space.py:planar", - atol=0.0, + "two-mode Ar slab", ), Case( "spatial_absorber", 1, ("free-space",), + "prop_radial", "spatial", - (*ADAPTIVE, FIXED), 0.08, - 0.0, - ENTRANCE_SCHEDULES["spatial_absorber"][-1], - None, (64, 512, 1024), - "benchmarks/sep16/free_space.py:radial_boundary", - atol=0.0, + "radial Kerr beam meeting its absorbing boundary", ), ) -_GUIDE_WARNINGS = { - "scalar_wall": ( - WarningRule( - "construction", - "SolpropWarning", - "the temporal absorber attenuates a packet walking off by 0\\.15" - " fs over 0\\.0002 m by only 0\\.00065 nepers, below the 6\\.9 " - "that stops it wrapping around the time window; widen trange, " - "or raise boundary_N above 2\\.12e\\+05", - ), - WarningRule( - "construction", - "GrazingIncidenceWarning", - "the coherent wall block's quasi_static reaches 0\\.0449, within" - " a factor ten of the 0\\.25 limit; the condensed paraxial model" - " is marginal over this band", - ), - ), - "vector_wall": ( - WarningRule( - "construction", - "SolpropWarning", - "the temporal absorber attenuates a packet walking off by " - "0\\.141 fs over 0\\.0002 m by only 0\\.00069 nepers, below the " - "6\\.9 that stops it wrapping around the time window; widen " - "trange, or raise boundary_N above 2e\\+05", - ), - WarningRule( - "construction", - "GrazingIncidenceWarning", - "the coherent wall block's quasi_static reaches 0\\.06069, " - "within a factor ten of the 0\\.25 limit; the condensed paraxial" - " model is marginal over this band", - ), - ), - "taper": ( - WarningRule( - "construction", - "SolpropWarning", - "the temporal absorber attenuates a packet walking off by 0\\.19" - " fs over 0\\.0002 m by only 0\\.00051 nepers, below the 6\\.9 " - "that stops it wrapping around the time window; widen trange, " - "or raise boundary_N above 2\\.69e\\+05", - ), - ), - "taper_wall": ( - WarningRule( - "construction", - "SolpropWarning", - "the temporal absorber attenuates a packet walking off by 0\\.19" - " fs over 0\\.0002 m by only 0\\.00051 nepers, below the 6\\.9 " - "that stops it wrapping around the time window; widen trange, " - "or raise boundary_N above 2\\.69e\\+05", - ), - WarningRule( - "construction", - "GrazingIncidenceWarning", - "the coherent wall block's quasi_static reaches 0\\.05613, " - "within a factor ten of the 0\\.25 limit; the condensed paraxial" - " model is marginal over this band", - ), - ), - "step_index_pair": (), - "grin_linear": (), - "grin_nonlinear": (), - "moving_solid_guide": (), -} - _ADVANCED = ( *( Case( name, 1, ("advanced",), + "prop_capillary", "advanced", - (*ADAPTIVE, FIXED), 2e-4, - 0.0, - ENTRANCE_SCHEDULES[name][-1], - 2e-5, (4 if name == "vector_wall" else 3, 1024, 2048), f"docs/tutorials/capillaries/{source}", - warnings=_GUIDE_WARNINGS[name], ) for name, source in ( ("scalar_wall", "plot_07_coherent_wall.py"), @@ -576,39 +434,15 @@ def measurement(self, track: str, schedule: tuple[float, ...] = ()) -> Record: ) ), *( - Case( - name, - 1, - ("advanced",), - "advanced", - (*ADAPTIVE, FIXED), - length, - 0.0, - ENTRANCE_SCHEDULES[name][-1], - None, - dimensions, - source, - warnings=_GUIDE_WARNINGS[name], - ) - for name, length, dimensions, source in ( - ( - "step_index_pair", - 1e-3, - (2, 1024, 2048), - "benchmarks/sep25/cases.py:_general", - ), + Case(name, 1, ("advanced",), "prop_waveguide", "advanced", length, size, source) + for name, length, size, source in ( + ("step_index_pair", 1e-3, (2, 1024, 2048), "nonlinear HE11 parity pair"), ( "grin_linear", 0.0007846800630498693, (2, 128, 256), "docs/tutorials/waveguides/plot_06_graded_index.py", ), - ( - "grin_nonlinear", - 0.1, - (4, 4096, 8192), - "docs/examples/waveguides/plot_04_graded_index_multimode.py", - ), ( "moving_solid_guide", 0.015, @@ -619,148 +453,111 @@ def measurement(self, track: str, schedule: tuple[float, ...] = ()) -> Record: ), ) -CASES = {case.name: case for case in (*_CORE, *_MODAL, *_FREE, *_ADVANCED)} +_EXERCISES = tuple( + Case( + name, + 1, + ("backend-exercise",), + interface, + "exercise", + length, + dimensions, + source, + planes=8 if name in ("dense_output", "far_field_stats") else 1, + fields="planes" if name == "far_field_stats" else "all", + ) + for name, interface, length, dimensions, source in ( + ( + "vector_raman", + "prop_capillary", + 2e-4, + (1, 2, 256, 256), + "the vector Raman tutorial's circular pump and delayed probe", + ), + ( + "envelope_plasma", + "prop_capillary", + 8e-6, + (1, 1, 256, 256), + "docs/tutorials/capillaries/plot_05_envelope_plasma.py", + ), + ( + "mixture_gradient", + "prop_capillary", + 1.5, + (1, 1, 512, 1024), + "docs/guide/media/mixtures.md", + ), + ( + "dispersive_kerr", + "prop_capillary", + 0.5, + (1, 1, 2048, 4096), + "docs/tutorials/capillaries/plot_11_dispersive_kerr.py", + ), + ("plasma_band", "prop_capillary", 1e-3, (2, 1, 512, 1024), "band plasma"), + ( + "plasma_trapezoid", + "prop_capillary", + 1e-3, + (2, 1, 512, 1024), + "trapezoid plasma", + ), + ("noise", "prop_gnlse", 1e-3, (1, 1, 1024, 1024), "modified-noise fibre"), + ("dense_output", "prop_gnlse", 1.0, (1, 1, 1024, 1024), "dense output"), + ( + "absorber_schemes", + "prop_gnlse", + 1e-3, + (1, 1, 1024, 1024), + "split-step absorber", + ), + ( + "ordinary_stats", + "prop_capillary", + 1e-3, + (2, 1, 512, 1024), + "ordinary statistics on two modes", + ), + ( + "far_field_stats", + "prop_free", + 1e-3, + (224, 64, 128, 128), + "docs/tutorials/free_space/plot_07_far_field_statistics.py", + ), + ) +) + +CASES = { + case.name: case for case in (*_TRACKED, *_MODAL, *_FREE, *_ADVANCED, *_EXERCISES) +} CASES["smoke"] = Case( "smoke", 1, (), + "prop_gnlse", "smoke", - (*ADAPTIVE, FIXED), - 1e-4, - 0.0, - 1e-4, - 1e-5, - (), - "suite integration smoke; not an adoption workload", -) -CASES["batch"] = Case( - "batch", - 1, - ("backend-exercise",), - "batch", - (FIXED,), - 1e-3, - 0.0, - 1e-3, 1e-4, - (2, 1, 1024, 1024), - "tests/backends/test_batch_lockstep.py:simulation; ten fixed RK steps", + (1, 256, 256), + "a short noiseless fibre for testing the runner; not a workload", + planes=2, ) -# Original guides with frozen default-RK entrance pilots; the 8 µm plasma -# tutorial ends after four steps, a window retained by design. -for _name, _length, _dimensions, _source in ( - ( - "vector_raman", - 2e-4, - (1, 2, 256, 256), - "tests/vector_raman_cases.py and vector Raman tutorial", - ), - ( - "envelope_plasma", - 8e-6, - (1, 1, 256, 256), - "docs/tutorials/capillaries/plot_05_envelope_plasma.py", - ), - ( - "mixture_gradient", - 1.5, - (1, 1, 512, 1024), - "docs/guide/media/mixtures.md; capillary interface case", - ), - ( - "dispersive_kerr", - 0.5, - (1, 1, 2048, 4096), - "docs/tutorials/capillaries/plot_11_dispersive_kerr.py", - ), - ( - "plasma_band", - 1e-3, - (2, 1, 512, 1024), - "tests/backends/test_band_plasma_parity.py:modal", - ), - ( - "plasma_trapezoid", - 1e-3, - (2, 1, 512, 1024), - "tests/backends/test_band_plasma_parity.py:modal", - ), - ( - "noise", - 1e-3, - (1, 1, 1024, 1024), - "tests/backends/test_batch_lockstep.py:simulation", - ), - ("dense_output", 1.0, (1, 1, 1024, 1024), "tests/backends/test_dense_output.py"), - ( - "absorber_schemes", - 1e-3, - (1, 1, 1024, 1024), - "tests/backends/test_batch_lockstep.py:simulation", - ), - ( - "ordinary_stats", - 1e-3, - (2, 1, 512, 1024), - "tests/driver/test_prop_capillary.py:build", - ), - ( - "far_field_stats", - 1e-3, - (224, 64, 128, 128), - "docs/tutorials/free_space/plot_07_far_field_statistics.py", - ), -): - CASES[_name] = Case( - _name, - 1, - ("backend-exercise",), - "exercise", - ADAPTIVE if _name == "dense_output" else (*ADAPTIVE, FIXED), - _length, - 0.0, - ENTRANCE_SCHEDULES[_name][-1], - None, - _dimensions, - _source, - atol=1e-14 - if _name == "envelope_plasma" - else 1e-18 - if _name == "dense_output" - else 1e-12 - if _name == "far_field_stats" - else 1e-10, - planes=8 if _name in ("dense_output", "far_field_stats") else 1, - warnings=( - WarningRule( - "construction", - "MaterialRangeWarning", - r"H2: optmat's Raman response raman_response/H2@293\.15K,800nm " - r"is used outside its validated density range " - r"\(0\.3722 to 0\.3722 amagat\)\. optmat: density below 0\.94 amagat, " - r"where the motion of the molecules sets the widths: " - r"the Dicke terms are capped at the Doppler widths \(TC-\d+\)", - ), - ) - if _name == "vector_raman" - else (), - ) - def select( cases: tuple[str, ...] = (), sets: tuple[str, ...] = (), *, all_cases: bool = False ) -> tuple[Case, ...]: - """Expand selectors once, retaining declared order and excluding no failures.""" - if any(name not in CASES for name in cases): - raise SuiteError( - f"unknown case: {', '.join(name for name in cases if name not in CASES)}" - ) + """Expand selectors in declared order; no selection means the tracking set.""" + if unknown := [name for name in cases if name not in CASES]: + raise SuiteError(f"unknown case: {', '.join(unknown)}") if any(name not in SETS for name in sets): - raise SuiteError("unknown set; use --all for the union") - selected_sets = SETS if all_cases else sets or (() if cases else ("core",)) + raise SuiteError("unknown set; use --all for every case") + if all_cases: + return tuple(case for case in CASES.values() if case.name != "smoke") + selected = sets or (() if cases else (TRACKING,)) return tuple( case for case in CASES.values() - if case.name in cases or set(case.sets).intersection(selected_sets) + if case.name in cases or set(case.sets).intersection(selected) ) diff --git a/benchmarks/runner.py b/benchmarks/runner.py new file mode 100644 index 00000000..f764e640 --- /dev/null +++ b/benchmarks/runner.py @@ -0,0 +1,183 @@ +# SPDX-FileCopyrightText: 2026 John C. Travers and contributors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Plan configurations and launch one worker per configuration (SEP 38). + +Shared by ``track`` and ``ab``. Nothing here imports a scientific library. +""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import time +from pathlib import Path + +from benchmarks.records import Record, SuiteError, atomic_record, canonical, file_hash +from benchmarks.registry import Case +from benchmarks.system import widths + +__all__ = [ + "ROOT", + "estimate", + "freeze_harness", + "launch_worker", + "plan_rows", + "prepared_python", + "receipt", + "show_plan", +] + +ROOT = Path(__file__).resolve().parents[1] +CONSTRUCTION_SECONDS = 10.0 +"""A rough per-configuration allowance for import, construction and translation.""" + + +def prepared_python(value: str) -> str: + """Locate an interpreter without resolving a virtual environment's symlink.""" + found = shutil.which(value) + if found is None: + raise SuiteError(f"Python executable does not exist: {value}") + return str(Path(found).absolute()) + + +def freeze_harness(directory: Path) -> Record: + """Copy the harness so that every worker of a run uses the same benchmark code.""" + files = {} + for source in sorted((ROOT / "benchmarks").glob("*.py")): + target = directory / source.relative_to(ROOT) + target.parent.mkdir(parents=True, exist_ok=True) + if not target.exists(): + target.write_bytes(source.read_bytes()) + elif file_hash(target) != file_hash(source): + raise SuiteError("stored harness differs from the current source") + files[str(source.relative_to(ROOT))] = file_hash(target) + return files + + +def plan_rows( + cases: tuple[Case, ...], + steppers: tuple[str, ...], + threads: str | None, + production: int, + *, + backend: str = "native", + fraction: float = 1.0, +) -> tuple[list[Record], list[str]]: + """Resolve configurations in run order; explain every one left out. + + Without `threads`, each case runs at its declared widths; with it, every case + runs at the widths it names. + """ + rows: list[Record] = [] + skipped: list[str] = [] + for case in cases: + declared = ",".join(case.widths) + for width in widths(threads or declared, production): + for stepper in steppers: + if stepper not in case.steppers: + skipped.append( + f"{case.name} {stepper}: a fixed-step case runs its own " + "schedule only" + ) + continue + rows.append( + { + "id": f"{case.name}.{stepper}.t{width}", + "configuration": case.configuration( + stepper, width, backend, fraction + ), + "cost": case.cost[0 if width == 1 else 1], + } + ) + rows.sort(key=lambda row: row["configuration"]["threads"]) + return rows, skipped + + +def estimate(rows: list[Record], runs: int) -> float | None: + """Estimate seconds for `runs` complete runs per row, or ``None`` if unknown.""" + costs = [row["cost"] for row in rows] + if any(cost is None for cost in costs): + return None + return sum(CONSTRUCTION_SECONDS + cost * (runs + 0.02) for cost in costs) + + +def show_plan(rows: list[Record], skipped: list[str], runs: int) -> None: + """Print the configurations and the estimated time, labelled as an estimate.""" + print(f"{'case':26} {'stepper':22} {'threads':>7} {'est. s/run':>10}") + for row in rows: + values = row["configuration"] + cost = "?" if row["cost"] is None else f"{row['cost']:.0f}" + print( + f"{values['case']:26} {values['stepper']:22} {values['threads']:>7} " + f"{cost:>10}" + ) + for reason in skipped: + print(f"skipped {reason}") + total = estimate(rows, runs) + print( + f"{len(rows)} configurations, {runs} timed run(s) each; estimated " + + ("time unknown" if total is None else f"{total / 60:.0f} min") + + " (complete-run costs on an AMD EPYC 7763 node; other machines differ)" + ) + + +def launch_worker( + python: str, + request: Record, + environment: dict[str, str], + directory: Path, + timeout: float, + harness: Path = ROOT, +) -> tuple[list[Record], float]: + """Run one worker to completion; return its events and its wall time.""" + directory.mkdir(parents=True, exist_ok=True) + path = directory / "request.json" + atomic_record(path, request, immutable=True) + started = time.perf_counter() + with ( + (directory / "stdout.log").open("w", encoding="utf-8") as stdout, + (directory / "stderr.log").open("w", encoding="utf-8") as stderr, + ): + try: + subprocess.run( + [python, "-m", "benchmarks.worker", str(path.resolve())], + cwd=harness, + env={**environment, "PYTHONPATH": str(harness)}, + stdout=stdout, + stderr=stderr, + timeout=timeout, + check=False, + ) + except subprocess.TimeoutExpired: + stdout.write( + canonical( + {"event": "result", "status": "failed", "error": "worker timeout"} + ) + + "\n" + ) + events = [] + for line in (directory / "stdout.log").read_text(encoding="utf-8").splitlines(): + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(event, dict) and "event" in event: + events.append(event) + return events, time.perf_counter() - started + + +def receipt(events: list[Record]) -> Record: + """Return the worker's result, or a failure saying why there is none.""" + result = next((e for e in reversed(events) if e["event"] == "result"), None) + if result is None: + return {"status": "failed", "error": "no receipt from the worker"} + provenance = next( + (e["provenance"] for e in events if e["event"] == "provenance"), None + ) + return { + **{key: value for key, value in result.items() if key != "event"}, + "provenance": provenance, + } diff --git a/benchmarks/schedules.py b/benchmarks/schedules.py deleted file mode 100644 index 0504a824..00000000 --- a/benchmarks/schedules.py +++ /dev/null @@ -1,455 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Freeze ten accepted entrance steps from independent NumPy RK pilots. - -SEP 30 Part II records the pilot protocol: the original full guide, default RK, -rtol=1e-6, with atol=1e-10 for modal/GNLSE and zero for spatial cases, and -no intervening observation landings. These edges -select bounded workloads; they are not an accuracy claim or a timing result. -Reduced CtC retains its already specified uniform fixed diagnostic instead. -""" - -__all__ = ["ENTRANCE_SCHEDULES"] - -ENTRANCE_SCHEDULES = { - "rdw_one_mode": ( - 0.0, - 0.00015000000000000001, - 0.00736828643250081, - 0.013970204136183006, - 0.019399093125691977, - 0.02367940390805454, - 0.027181846939439522, - 0.030313668538471426, - 0.033306614204046237, - 0.036240794574350796, - 0.039137683386366584, - ), - # Revision 2's 2 ps window, piloted by the same protocol (which reproduced - # revision 1's 1 ps edges exactly). - "ctc_reduced": ( - 0.0, - 1e-05, - 0.00019683137535397063, - 0.00036770923896977325, - 0.0005205083194341789, - 0.000662020084178603, - 0.0007976102875723777, - 0.0009294561225711036, - 0.0010584439105383128, - 0.0011856223236902477, - 0.0013123521892400005, - ), - "argon_blueshift": ( - 0.0, - 4e-05, - 0.00036333936283839205, - 0.000659068811064311, - 0.0009284014327881641, - 0.0011831189910154237, - 0.0014332989224319186, - 0.0016825466870057796, - 0.0019313531065438426, - 0.0021797408267138912, - 0.0024271711683208604, - ), - "h2_raman_6ps": ( - 0.0, - 0.0002, - 0.002566320815563155, - 0.004730581742438799, - 0.006658751627861455, - 0.008220868436456403, - 0.00948054717280201, - 0.010549630294383069, - 0.011508737109621326, - 0.01239967654731973, - 0.013240857906639918, - ), - "he_rdw_averaged": ( - 0.0, - 0.00015000000000000001, - 0.0074580960397082376, - 0.014142154517565564, - 0.01960441912911167, - 0.023886886582792106, - 0.027382722121571224, - 0.03050969067834002, - 0.033500902042873267, - 0.03643462104145296, - 0.03933104059820758, - ), - "argon_blueshift_averaged": ( - 0.0, - 4e-05, - 0.0003719003577908754, - 0.0006754597784993843, - 0.0009475081828661954, - 0.0012019325759404286, - 0.0014509623894948384, - 0.0016988917680546838, - 0.0019454460069577777, - 0.0021901409308761905, - 0.0024333553851389363, - ), - "h2_raman_6ps_averaged": ( - 0.0, - 0.0002, - 0.0025663208061068262, - 0.004730581724333616, - 0.006658751605875426, - 0.008220868413740498, - 0.009480547150538575, - 0.010549630272896989, - 0.011508737088939743, - 0.012399676527387129, - 0.013240857887389613, - ), - "gnlse_soliton": ( - 0.0, - 0.00013278075458959397, - 0.012981788223734362, - 0.024733620279518063, - 0.035032763237968466, - 0.044530063225890704, - 0.05381350583997682, - 0.06306052546058549, - 0.07221107065839219, - 0.08120184417344199, - 0.09003161010327049, - ), - "gnlse_raman_shock": ( - 0.0, - 1.5e-05, - 0.0001464519258857109, - 0.00026667918547193504, - 0.00037198047481855356, - 0.0004692247288644632, - 0.0005647220111853069, - 0.0006605094496714045, - 0.0007560992329590459, - 0.0008509654072051106, - 0.0009452748610032289, - ), - "radial_filament_512": ( - 0.0, - 3e-06, - 3.724785372100234e-05, - 6.857128676956723e-05, - 9.611537688505799e-05, - 0.00012252775784117446, - 0.0001499830586688221, - 0.00017906007406975379, - 0.00020950004917735438, - 0.00024121856001387253, - 0.0002745235180576067, - ), - "line": ( - 0.0, - 8.000000000000001e-06, - 0.0008080000000000001, - 0.0016880000000000003, - 0.0026560000000000004, - 0.0037208000000000007, - 0.004892080000000002, - 0.006180488000000003, - 0.0075977368000000045, - 0.009156710480000005, - 0.010871581528000007, - ), - "planar": ( - 0.0, - 5e-06, - 0.000505, - 0.001055, - 0.00166, - 0.0023255000000000003, - 0.0030575500000000005, - 0.003862805000000001, - 0.004748585500000001, - 0.005722944050000001, - 0.006794738455000002, - ), - "spatial_absorber": ( - 0.0, - 8.000000000000001e-06, - 0.0008080000000000001, - 0.0016880000000000003, - 0.0026560000000000004, - 0.0037208000000000007, - 0.004892080000000002, - 0.006180488000000003, - 0.0075977368000000045, - 0.009156710480000005, - 0.010871581528000007, - ), - "cartesian_filament_128": ( - 0.0, - 3e-06, - 3.152186164627756e-05, - 5.760824448870751e-05, - 8.053333523675468e-05, - 0.00010239462031135691, - 0.0001249292676211788, - 0.00014860342277138442, - 0.00017320545161684928, - 0.0001986410378670889, - 0.00022511039009170193, - ), - "cartesian_filament_256": ( - 0.0, - 3e-06, - 3.1521923856426404e-05, - 5.760836359688705e-05, - 8.053349685479316e-05, - 0.00010239480250257994, - 0.0001249294574021636, - 0.0001486036112420747, - 0.00017320565016913476, - 0.00019864126629267663, - 0.0002251106200216726, - ), - "scalar_wall": ( - 0.0, - 2e-05, - 4e-05, - 6.000000000000001e-05, - 8e-05, - 0.0001, - 0.00012, - 0.00014000000000000001, - 0.00016, - 0.00018, - 0.0002, - ), - "vector_wall": ( - 0.0, - 2e-05, - 4e-05, - 6.000000000000001e-05, - 8e-05, - 0.0001, - 0.00012, - 0.00014000000000000001, - 0.00016, - 0.00018, - 0.0002, - ), - "taper": ( - 0.0, - 2e-05, - 4e-05, - 6e-05, - 8e-05, - 0.0001, - 0.00010617537272306832, - 0.00011290365417353301, - 0.00012030476376904418, - 0.00012844598432410647, - 0.000137401326934675, - ), - "taper_wall": ( - 0.0, - 2e-05, - 4e-05, - 6e-05, - 8e-05, - 0.0001, - 0.00010625650263163623, - 0.00011307230979494334, - 0.00012056969767458116, - 0.00012881682434218275, - 0.0001378886636765445, - ), - "step_index_pair": ( - 0.0, - 1.0000000000000001e-07, - 1.0100000000000002e-05, - 2.1100000000000005e-05, - 3.320000000000001e-05, - 4.6510000000000014e-05, - 6.115100000000002e-05, - 7.725610000000003e-05, - 9.497171000000005e-05, - 0.00011445888100000007, - 0.0001358947691000001, - ), - "grin_linear": ( - 0.0, - 2.1386070884935598e-06, - 4.2772141769871196e-06, - 6.24353965842701e-06, - 8.01643371429857e-06, - 9.731217513831928e-06, - 1.1497566981689743e-05, - 1.3343410725016612e-05, - 1.5253939802989334e-05, - 1.722227346676146e-05, - 1.926009921846214e-05, - ), - "grin_nonlinear": ( - 0.0, - 1e-05, - 7.557464016841108e-05, - 0.00013554986862307302, - 0.00020152261992320114, - 0.0002740926463533421, - 0.0003525155103824647, - 0.0004321507646046046, - 0.0005038149145445943, - 0.0005651642418415654, - 0.0006200574504726586, - ), - "moving_solid_guide": ( - 0.0, - 1.5e-06, - 3e-06, - 4.65e-06, - 6.465000000000001e-06, - 8.4615e-06, - 1.0657650000000001e-05, - 1.3073415000000002e-05, - 1.5730756500000003e-05, - 1.8653832150000003e-05, - 2.1869215365000004e-05, - ), - "vector_raman": ( - 0.0, - 2e-08, - 4e-08, - 6.2e-08, - 8.62e-08, - 1.1282000000000001e-07, - 1.4210200000000004e-07, - 1.7431220000000004e-07, - 2.0974342000000007e-07, - 2.487177620000001e-07, - 2.9158953820000015e-07, - ), - "envelope_plasma": ( - 0.0, - 2e-06, - 4e-06, - 6e-06, - 8e-06, - ), - "mixture_gradient": ( - 0.0, - 0.00015000000000000001, - 0.0032122048994991095, - 0.006012928440654216, - 0.00846842973675812, - 0.01073850311643064, - 0.012968673605987035, - 0.015206027973528376, - 0.017440231962271457, - 0.019660237278656602, - 0.02187030041559376, - ), - "dispersive_kerr": ( - 0.0, - 5e-05, - 0.002171822274785093, - 0.004112462365887402, - 0.005818693060289439, - 0.007401949406261684, - 0.008962996587770014, - 0.01053407178358141, - 0.01210653672003907, - 0.013668482355647282, - 0.015218912539647148, - ), - "plasma_band": ( - 0.0, - 1.0000000000000001e-07, - 1.0100000000000002e-05, - 2.1100000000000005e-05, - 3.320000000000001e-05, - 4.6510000000000014e-05, - 6.115100000000002e-05, - 7.725610000000003e-05, - 9.497171000000005e-05, - 0.00011445888100000007, - 0.0001358947691000001, - ), - "plasma_trapezoid": ( - 0.0, - 1.0000000000000001e-07, - 1.0100000000000002e-05, - 2.1100000000000005e-05, - 3.320000000000001e-05, - 4.6510000000000014e-05, - 6.115100000000002e-05, - 7.725610000000003e-05, - 9.497171000000005e-05, - 0.00011445888100000007, - 0.0001358947691000001, - ), - "noise": ( - 0.0, - 1.0000000000000001e-07, - 1.0100000000000002e-05, - 2.1100000000000005e-05, - 3.320000000000001e-05, - 4.6510000000000014e-05, - 6.115100000000002e-05, - 7.725610000000003e-05, - 9.497171000000005e-05, - 0.00011445888100000007, - 0.0001358947691000001, - ), - "dense_output": ( - 0.0, - 0.0001, - 0.0002, - 0.00031000000000000005, - 0.00043100000000000007, - 0.0005641000000000001, - 0.0007105100000000001, - 0.0008715610000000002, - 0.0010487171000000003, - 0.0012435888100000003, - 0.0014579476910000005, - ), - "absorber_schemes": ( - 0.0, - 1.0000000000000001e-07, - 1.0100000000000002e-05, - 2.1100000000000005e-05, - 3.320000000000001e-05, - 4.6510000000000014e-05, - 6.115100000000002e-05, - 7.725610000000003e-05, - 9.497171000000005e-05, - 0.00011445888100000007, - 0.0001358947691000001, - ), - "ordinary_stats": ( - 0.0, - 1.0000000000000001e-07, - 1.0100000000000002e-05, - 2.1100000000000005e-05, - 3.320000000000001e-05, - 4.6510000000000014e-05, - 6.115100000000002e-05, - 7.725610000000003e-05, - 9.497171000000005e-05, - 0.00011445888100000007, - 0.0001358947691000001, - ), - "far_field_stats": ( - 0.0, - 0.000125, - 0.00025, - 0.0003627672240109631, - 0.00045829952601817635, - 0.0005381823413533154, - 0.0006078656659498561, - 0.0006729248417794652, - 0.0007366526108033569, - 0.0008001930842129548, - 0.0008638007330063497, - ), -} diff --git a/benchmarks/system.py b/benchmarks/system.py index 2e42df75..ad6bd6db 100644 --- a/benchmarks/system.py +++ b/benchmarks/system.py @@ -2,11 +2,14 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -"""Record allocations, thread policies and bounded contention observations.""" +"""Identify the hardware, set thread pools and observe contention. + +Nothing here records a host name: the hardware is named by its CPU model and the +cores the run may use (SEP 38, section 5.3). +""" from __future__ import annotations -import dataclasses import os import platform import re @@ -16,28 +19,21 @@ import psutil -from benchmarks.records import ( - Record, - SuiteError, - atomic_record, - identity, - read_record, -) +from benchmarks.records import Record, SuiteError, identity +from benchmarks.telemetry import allocation_details, hardware_details __all__ = [ "POOL_VARIABLES", - "Profile", - "admission", "counters", - "effective_memory", "eligible_cpus", "git_receipt", "guard", + "hardware", "interval", - "profile_for", - "save_profile", + "physical_cores", + "slug", "thread_environment", - "topology", + "widths", ] POOL_VARIABLES = ( @@ -68,108 +64,84 @@ "KOKKOS_TOOLS_LIBS", ) +_MODEL_NOISE = re.compile( + r"\((?:r|tm)\)|\bcpu\b|\bprocessor\b|\b\d+-core\b|@.*$", re.IGNORECASE +) + def eligible_cpus() -> tuple[int, ...]: - """Read the process allocation, never an OpenMP worker's narrowed placement.""" + """Read the process's CPU allocation.""" try: return tuple(psutil.Process().cpu_affinity()) except AttributeError, psutil.Error: return tuple(range(os.process_cpu_count() or os.cpu_count() or 1)) -def topology() -> Record: - """Collect inexpensive portable hardware facts without a hostname.""" +def physical_cores() -> int: + """Count the physical cores in the allocation, assuming even SMT.""" + logical = psutil.cpu_count() or 1 + physical = psutil.cpu_count(logical=False) or logical + return max(1, len(eligible_cpus()) * physical // logical) + + +def widths(value: str | None, production: int) -> tuple[int, ...]: + """Resolve ``1``, ``production`` and integers, in order and without repeats.""" + if production < 1 or production > len(eligible_cpus()): + raise SuiteError("the production width must fit the CPU allocation") + resolved: list[int] = [] + for token in (value or "1,production").split(","): + width = production if token.strip() == "production" else int(token) + if width < 1: + raise SuiteError("thread counts must be positive") + if width not in resolved: + resolved.append(width) + return tuple(resolved) + + +def _instance_type() -> str | None: + """Read a cloud instance type from the Linux DMI table, where it names one.""" + root = Path("/sys/class/dmi/id") + try: + vendor = (root / "sys_vendor").read_text(encoding="utf-8").strip() + product = (root / "product_name").read_text(encoding="utf-8").strip() + except OSError: + return None + if vendor.startswith("Amazon"): + return f"aws {product}" + return None + + +def hardware(production: int, instance_type: str | None = None) -> Record: + """Describe the hardware without a host name, and derive its slug.""" + details = hardware_details(eligible_cpus()) + model = details["cpu_model"] or platform.processor() or platform.machine() + cores = physical_cores() return { + "slug": slug(model, cores), + "cpu_model": model, + "physical_cores": cores, + "logical_cpus": len(eligible_cpus()), + "production": production, + "memory_bytes": psutil.virtual_memory().total, "system": platform.system(), + "release": platform.release(), "machine": platform.machine(), - "logical_cpus": psutil.cpu_count(), - "physical_cores": psutil.cpu_count(logical=False), - "allocation": list(eligible_cpus()), - "memory_bytes": psutil.virtual_memory().total, - "processor": platform.processor(), - "core_residency": None, - "core_residency_reason": "not measured by ordinary collection", + "instance_type": instance_type or _instance_type(), + "allocation": allocation_details(), } -@dataclasses.dataclass(frozen=True) -class Profile: - """Freeze production width, allocation and policy before measurement.""" - - alias: str - revision: int - production: int - allocation: tuple[int, ...] - topology_hash: str - policy: str = "budget" - explicit: bool = True - - def __post_init__(self) -> None: - if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", self.alias): - raise SuiteError("machine alias must be a public-safe identifier") - if self.production < 1 or self.production > len(self.allocation): - raise SuiteError("production width exceeds the declared CPU allocation") - if self.policy not in ("budget", "all-pools-N", "blas1", "as-installed"): - raise SuiteError(f"unsupported thread policy {self.policy}") - - @property - def digest(self) -> str: - """Identify the full profile revision.""" - return identity(dataclasses.asdict(self)) - - -def profile_for(path: Path | None, *, standard: bool) -> Profile: - """Load an explicit profile or make a clearly provisional quick profile.""" - current = topology() - if path is not None: - values = read_record(path) - values["allocation"] = tuple(values["allocation"]) - profile = Profile(**values) - if profile.topology_hash != identity(current): - raise SuiteError( - "machine topology/allocation differs from the saved profile" - ) - if standard and not profile.explicit: - raise SuiteError( - "standard collection needs a profile with an explicit production width" - ) - return profile - if standard: - raise SuiteError( - "standard collection requires --machine-profile with a production width" - ) - # An unidentified machine is useful for quick diagnostics but cannot become - # an adoption baseline solely because another host has the same CPU model. - return Profile( - os.environ.get("SOLPROP_BENCH_MACHINE", "unidentified"), - 1, - len(eligible_cpus()), - eligible_cpus(), - identity(current), - explicit=False, - ) - - -def save_profile( - path: Path, alias: str, production: int, policy: str = "budget" -) -> Profile: - """Write an explicit profile without replacing an earlier revision.""" - profile = Profile( - alias, 1, production, eligible_cpus(), identity(topology()), policy - ) - atomic_record(path, dataclasses.asdict(profile), immutable=True) - return profile +def slug(model: str, cores: int) -> str: + """Name a hardware set by CPU model and core count, e.g. ``apple-m5-max-18c``.""" + words = re.sub(r"[^a-z0-9]+", "-", _MODEL_NOISE.sub(" ", model).lower()) + return f"{words.strip('-') or 'unknown-cpu'}-{cores}c" -def thread_environment( - width: int, policy: str, incoming: dict[str, str] -) -> dict[str, str]: - """Set pools before importing their libraries, retaining size-based choices.""" +def thread_environment(width: int, incoming: dict[str, str]) -> dict[str, str]: + """Set every pool to `width` before its library is imported.""" if width < 1: raise SuiteError("thread counts must be positive") environment = incoming.copy() - if policy == "as-installed": - return environment for key in POOL_VARIABLES: if key not in ("SOLPROP_FFTW_PLAN", "KOKKOS_TOOLS_LIBS"): environment.pop(key, None) @@ -179,16 +151,13 @@ def thread_environment( "SOLPROP_NUMBA_THREADS", "NUMBA_NUM_THREADS", "OMP_NUM_THREADS", - ): - environment[key] = str(width) - for key in ( "OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", "BLIS_NUM_THREADS", "VECLIB_MAXIMUM_THREADS", ): - environment[key] = str(1 if policy == "blas1" else width) - if width == 1 or policy == "all-pools-N": + environment[key] = str(width) + if width == 1: for key in ( "SOLPROP_NATIVE_KERNEL_THREADS", "SOLPROP_NATIVE_FFT_THREADS", @@ -198,9 +167,7 @@ def thread_environment( "SOLPROP_TRANSVERSE_FFTW_THREADS", "SOLPROP_BLAS_THREADS", ): - environment[key] = str(width) - if policy == "blas1": - environment["SOLPROP_BLAS_THREADS"] = "1" + environment[key] = "1" environment.update( OMP_PROC_BIND="false", OMP_WAIT_POLICY="PASSIVE", @@ -210,71 +177,22 @@ def thread_environment( return environment -def effective_memory() -> int: - """Constrain currently available RAM by a Linux cgroup when observable.""" - available = int(psutil.virtual_memory().available) - # Unified cgroups: a container's current usage must be subtracted from its - # limit. The host's free memory alone is not its allocation. - for parent in (Path("/sys/fs/cgroup"),): - try: - limit = (parent / "memory.max").read_text(encoding="utf-8").strip() - used = int((parent / "memory.current").read_text(encoding="utf-8")) - if limit != "max": - available = min(available, max(0, int(limit) - used)) - except OSError, ValueError: - pass - return available - - -def admission( - estimate: int | None, limit: int | None, *, allow_unestimated: bool = False -) -> Record: - """Refuse unknown or excessive allocations before expensive construction.""" - available = effective_memory() - budget = min(int(0.8 * available), limit if limit is not None else available) - admitted = estimate is not None and estimate <= budget - if estimate is None and allow_unestimated and limit is not None: - admitted = limit <= int(0.8 * available) - return { - "admitted": admitted, - "estimate_bytes": estimate, - "limit_bytes": budget, - "available_bytes": available, - "reserve_fraction": 0.2, - "exploratory": estimate is None, - "reason": None - if admitted - else "unestimated" - if estimate is None - else "insufficient memory", - } - - def counters() -> Record: - """Read counters with explicit unknowns for unsupported OS observations.""" + """Read CPU and swap counters, with explicit unknowns where unsupported.""" times = psutil.cpu_times(percpu=True) busy = [float(item.user + item.system) for item in times] try: swap = psutil.swap_memory() swapping: int | None = int(swap.sin + swap.sout) - reason = None if os.name == "nt": - swapping, reason = None, "Windows does not expose swap I/O counters" + swapping = None except psutil.Error, OSError: - swapping, reason = None, "swap I/O counters unavailable" - return { - "time": time.monotonic(), - "busy": busy, - "swap_bytes": swapping, - "swap_reason": reason, - "available_bytes": effective_memory(), - } + swapping = None + return {"time": time.monotonic(), "busy": busy, "swap_bytes": swapping} -def interval( - before: Record, after: Record, allocation: tuple[int, ...], own_cpu: float = 0.0 -) -> Record: - """Score external CPU use over an actual elapsed counter interval.""" +def interval(before: Record, after: Record, allocation: tuple[int, ...]) -> Record: + """Score other processes' CPU use over an elapsed counter interval.""" elapsed = after["time"] - before["time"] valid = elapsed > 0 and all(index < len(after["busy"]) for index in allocation) deltas = ( @@ -283,57 +201,27 @@ def interval( else [] ) valid = valid and all(value >= 0 for value in deltas) - usage = ( - max(0.0, sum(deltas) - own_cpu) / (elapsed * len(allocation)) if valid else None - ) + usage = sum(deltas) / (elapsed * len(allocation)) if valid else None before_swap, after_swap = before["swap_bytes"], after["swap_bytes"] swap_delta = ( - int(after_swap) - int(before_swap) + after_swap - before_swap if before_swap is not None and after_swap is not None else None ) - confidence = ( - "known" if valid and swap_delta is not None and swap_delta >= 0 else "unknown" - ) + known = usage is not None and swap_delta is not None and swap_delta >= 0 return { "external_cpu_fraction": usage, "swap_bytes": swap_delta, - "confidence": confidence, - "quiet": confidence == "known" - and usage is not None - and usage <= 0.1 - and swap_delta == 0, + "confidence": "known" if known else "unknown", + "quiet": known and usage is not None and usage <= 0.1 and swap_delta == 0, } -def guard( - allocation: tuple[int, ...], *, standard: bool, force: bool = False -) -> Record: - """Take one quick interval or bounded standard counter intervals.""" - if force: - return {"quiet": False, "confidence": "unknown", "forced": True, "seconds": 0.0} - started = time.monotonic() - observations: list[Record] = [] +def guard(allocation: tuple[int, ...]) -> Record: + """Take one one-second reading; it labels a busy machine and never waits.""" before = counters() - for _ in range(5 if standard else 1): - time.sleep(1.0) - after = counters() - observations.append(interval(before, after, allocation)) - before = after - if not standard or ( - len(observations) >= 2 and all(row["quiet"] for row in observations[-2:]) - ): - break - selected = observations[-2:] if standard else observations - return { - "quiet": all(row["quiet"] for row in selected), - "confidence": "known" - if all(row["confidence"] == "known" for row in selected) - else "unknown", - "seconds": time.monotonic() - started, - "intervals": observations, - "threshold_fraction": 0.1, - } + time.sleep(1.0) + return interval(before, counters(), allocation) def git_receipt(directory: Path) -> Record: diff --git a/benchmarks/telemetry.py b/benchmarks/telemetry.py index 07399eda..aecb755d 100644 --- a/benchmarks/telemetry.py +++ b/benchmarks/telemetry.py @@ -12,8 +12,6 @@ import subprocess from pathlib import Path -import psutil - from benchmarks.records import Record, SuiteError __all__ = [ @@ -21,8 +19,6 @@ "cpu_list", "hardware_details", "linux_topology", - "numa_pages", - "process_binding", ] @@ -134,63 +130,3 @@ def allocation_details() -> Record: } result["exclusivity"] = "unverified; requires the scheduler allocation record" return result - - -def numa_pages(contents: str) -> list[Record]: - """Aggregate resident mapping bytes by NUMA node, excluding mapping paths.""" - totals: dict[int, int] = {} - for line in contents.splitlines(): - size = re.search(r"\bkernelpagesize_kB=(\d+)\b", line) - if size is None: - continue - for node, count in re.findall(r"\bN(\d+)=(\d+)\b", line): - key = int(node) - totals[key] = totals.get(key, 0) + int(count) * int(size[1]) * 1024 - return [ - {"node": node, "resident_mapping_bytes": value} - for node, value in sorted(totals.items()) - ] - - -def process_binding() -> Record: - """Inspect live thread affinities and NUMA placement after diagnostic work. - - Allowed CPU sets do not establish execution residency. This deliberately - separate diagnostic neither samples inside kernels nor changes affinity. - """ - try: - threads = psutil.Process().threads() - except psutil.Error: - threads = None - rows = [] - for thread in threads or []: - affinity = None - reason = "per-thread affinity unavailable on this OS" - if hasattr(os, "sched_getaffinity"): - try: - affinity = sorted(os.sched_getaffinity(thread.id)) - reason = None - except OSError: - reason = "thread exited or affinity query denied" - rows.append( - { - "thread_id": thread.id, - "user_seconds": thread.user_time, - "system_seconds": thread.system_time, - "allowed_cpus": affinity, - "affinity_reason": reason, - } - ) - maps = _read(Path("/proc/self/numa_maps")) - placement = numa_pages(maps) if maps else [] - return { - "allocation": allocation_details(), - "threads": rows, - "threads_reason": None - if threads is not None - else "thread inventory unavailable", - "numa": placement, - "numa_reason": None if placement else "NUMA mapping sizes unavailable", - "residency": None, - "residency_reason": "affinity is allowed placement, not sampled CPU residency", - } diff --git a/benchmarks/track.py b/benchmarks/track.py new file mode 100644 index 00000000..a7d0c4e1 --- /dev/null +++ b/benchmarks/track.py @@ -0,0 +1,302 @@ +# SPDX-FileCopyrightText: 2026 John C. Travers and contributors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Time the tracking set's complete runs and record them per hardware (SEP 38).""" + +from __future__ import annotations + +import argparse +import os +import shutil +import statistics +import sys +import time +from datetime import UTC, datetime +from pathlib import Path + +from benchmarks import history +from benchmarks.records import LOCAL_ROOT, Record, SuiteError, atomic_record, now +from benchmarks.registry import SETS, STEPPERS, select +from benchmarks.runner import ( + ROOT, + launch_worker, + plan_rows, + receipt, + show_plan, +) +from benchmarks.system import ( + eligible_cpus, + git_receipt, + guard, + hardware, + physical_cores, + thread_environment, +) + +__all__ = [ + "history_rows", + "selection_options", + "show_results", + "steppers", + "summarise", + "track", + "track_options", +] + + +def selection_options(parser: argparse.ArgumentParser) -> None: + """Declare the case, stepper and width selection shared by every command.""" + parser.add_argument("--case", action="append", default=[]) + parser.add_argument("--set", action="append", choices=SETS, default=[]) + parser.add_argument("--all", action="store_true", help="every registered case") + parser.add_argument( + "--stepper", + default="default", + help="default, multistep, multistep-interaction or rk; comma-separated to " + "compare steppers in one run", + ) + parser.add_argument( + "--threads", help="widths, e.g. 1,production; default: each case's own" + ) + parser.add_argument( + "--production", type=int, help="production width; default: physical cores" + ) + parser.add_argument( + "--backend", choices=("numpy_ref", "numba_cpu", "native"), default="native" + ) + parser.add_argument("--repeats", type=int, default=3, help="timed runs per case") + + +def track_options(parser: argparse.ArgumentParser) -> None: + """Declare the tracking run's output and history options.""" + selection_options(parser) + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--out", type=Path) + parser.add_argument("--instance-type", help="cloud instance type, if any") + parser.add_argument( + "--append-history", + action="store_true", + help="add the run to benchmarks/history/.csv", + ) + parser.add_argument( + "--allow-dirty", + action="store_true", + help="with --append-history, accept uncommitted changes", + ) + parser.add_argument("--archive", type=Path, help="copy the run's bundle here") + parser.add_argument( + "--timeout", type=float, default=3600.0, help="per-worker limit, seconds" + ) + + +def steppers(value: str) -> tuple[str, ...]: + """Parse a comma-separated stepper list.""" + names = tuple(dict.fromkeys(item.strip() for item in value.split(","))) + if unknown := [name for name in names if name not in STEPPERS]: + raise SuiteError(f"unknown stepper: {', '.join(unknown)}") + return names + + +def summarise(row: Record, result: Record, busy: bool) -> Record: + """Reduce one worker's receipt to the tracked statistics.""" + values = row["configuration"] + base = { + "id": row["id"], + "case": values["case"], + "revision": values["revision"], + "stepper": values["stepper"], + "threads": values["threads"], + "status": result["status"], + "busy": busy, + } + if result["status"] != "ok": + return {**base, "error": result.get("error")} + times = [item["seconds"] for item in result["repetitions"]] + last = result["repetitions"][-1] + return { + **base, + "median_s": statistics.median(times), + "min_s": min(times), + "max_s": max(times), + "repeats": len(times), + "accepted": last["accepted"], + "rejected": last["rejected"], + "same_steps": len( + {(r["accepted"], r["rejected"]) for r in result["repetitions"]} + ) + == 1, + "source_evaluations": last["source_evaluations"], + "peak_rss_gib": result["peak_rss_bytes"] / 2**30, + "construction_s": result["construction_seconds"] + result["backend_seconds"], + "warmup_s": result["warmup_seconds"], + "integrator": result["integrator"], + "rtol": result["rtol"], + "atol": result["atol"], + "noise": result["noise"], + "finite": result["finite"], + } + + +def show_results(rows: list[Record]) -> None: + """Print one line per configuration.""" + print( + f"\n{'case':26} {'integrator':28} {'N':>3} {'median s':>9} " + f"{'range %':>8} {'steps':>7} {'GiB':>6}" + ) + for row in rows: + if row["status"] != "ok": + print( + f"{row['case']:26} {'':28} {row['threads']:>3} failed: {row['error']}" + ) + continue + spread = 100 * (row["max_s"] - row["min_s"]) / row["median_s"] + print( + f"{row['case']:26} {row['integrator']:28} {row['threads']:>3} " + f"{row['median_s']:9.2f} {spread:8.1f} {row['accepted']:7d} " + f"{row['peak_rss_gib']:6.2f}" + + ("" if row["same_steps"] else " (steps differ between repeats)") + + (" (busy)" if row["busy"] else "") + ) + + +def history_rows( + rows: list[Record], machine: Record, source: Record, provenance: Record +) -> list[Record]: + """Shape a complete run's results as history rows.""" + build = provenance.get("build_info") or {} + date = datetime.now(UTC).strftime("%Y-%m-%dT%H:%MZ") + return [ + { + "date": date, + "solprop": provenance["packages"]["solprop"], + "commit": (source.get("revision") or "")[:12], + "native": build.get("package_version"), + "native_build": "; ".join( + str(build[key]) + for key in ("build_profile", "compiler", "fft_backend", "blas_backend") + if build.get(key) + ) + or None, + "slug": machine["slug"], + "cpu_model": machine["cpu_model"], + "instance_type": machine["instance_type"], + "system": f"{machine['system']} {machine['machine']}", + "threads": row["threads"], + "case": row["case"], + "revision": row["revision"], + "stepper": row["stepper"], + "integrator": row["integrator"], + "rtol": row["rtol"], + "atol": row["atol"], + "noise": row["noise"], + "median_s": f"{row['median_s']:.4f}", + "min_s": f"{row['min_s']:.4f}", + "max_s": f"{row['max_s']:.4f}", + "repeats": row["repeats"], + "accepted": row["accepted"], + "rejected": row["rejected"], + "source_evaluations": row["source_evaluations"], + "peak_rss_gib": f"{row['peak_rss_gib']:.3f}", + "construction_s": f"{row['construction_s']:.2f}", + "busy": int(row["busy"]), + } + for row in rows + ] + + +def track(args: argparse.Namespace) -> int: + """Run the selected configurations, one worker each, and record the results.""" + if args.repeats < 1 or args.timeout <= 0: + raise SuiteError("repeats and timeout must be positive") + production = args.production or physical_cores() + rows, skipped = plan_rows( + select(tuple(args.case), tuple(args.set), all_cases=args.all), + steppers(args.stepper), + args.threads, + production, + backend=args.backend, + ) + if not rows: + raise SuiteError("no configuration in the selection") + show_plan(rows, skipped, args.repeats) + if args.dry_run: + return 0 + machine = hardware(production, args.instance_type) + source = git_receipt(ROOT) + if args.append_history: + if args.backend != "native": + raise SuiteError("the history records the native backend only") + if source.get("dirty") and not args.allow_dirty: + raise SuiteError( + "uncommitted changes: commit them, or pass --allow-dirty to record " + "this run anyway" + ) + stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + output: Path = args.out or LOCAL_ROOT / "track" / f"{stamp}-{machine['slug']}" + if output.exists() and any(output.iterdir()): + raise SuiteError("output already exists; choose a new --out") + atomic_record( + output / "plan.json", + { + "created": now(), + "hardware": machine, + "source": source, + "repeats": args.repeats, + "rows": rows, + "skipped": skipped, + }, + immutable=True, + ) + started = time.perf_counter() + results: list[Record] = [] + provenance: Record = {} + allocation = eligible_cpus() + for index, row in enumerate(rows): + busy = not guard(allocation)["quiet"] + width = row["configuration"]["threads"] + events, elapsed = launch_worker( + sys.executable, + {"configuration": row["configuration"], "repeats": args.repeats}, + thread_environment(width, dict(os.environ)), + output / "raw" / row["id"], + args.timeout, + ) + result = receipt(events) + provenance = result.get("provenance") or provenance + results.append(summarise(row, result, busy)) + state = ( + f"{results[-1]['median_s']:.2f} s" + if result["status"] == "ok" + else result["status"] + ) + print( + f"[{index + 1}/{len(rows)}] {row['id']}: {state} " + f"({elapsed:.0f} s with construction)" + (" busy" if busy else ""), + flush=True, + ) + atomic_record( + output / "results.json", + { + "rows": results, + "provenance": provenance, + "collection_seconds": time.perf_counter() - started, + }, + ) + show_results(results) + complete = all(row["status"] == "ok" and row["finite"] for row in results) + print( + f"Results: {output.resolve()} ({(time.perf_counter() - started) / 60:.0f} min)" + ) + if args.append_history: + if not complete: + print("Not added to the history: a configuration failed.", file=sys.stderr) + return 1 + path = history.HISTORY / f"{machine['slug']}.csv" + history.append(path, history_rows(results, machine, source, provenance)) + print(f"History: {path}") + if args.archive is not None: + target = args.archive / output.name + shutil.copytree(output, target) + print(f"Archived: {target}") + return 0 if complete else 1 diff --git a/benchmarks/views.py b/benchmarks/views.py deleted file mode 100644 index 7796f3f2..00000000 --- a/benchmarks/views.py +++ /dev/null @@ -1,199 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Derive normalized local CSV views from the manifest and immutable attempts.""" - -from __future__ import annotations - -from pathlib import Path - -from benchmarks.publication import CONFIGURATION_COLUMNS, scalar_columns -from benchmarks.records import Record, identity, read_record, write_csv - -__all__ = ["write_views"] - - -def write_views(directory: Path, rows: list[Record]) -> None: - """Keep scalar tables linked by bundle, configuration and attempt identities.""" - path = directory / "manifest.json" - if not path.exists(): - return - manifest = read_record(path) - plan = manifest["plan"] - run_id = identity(manifest) - comparison_path = directory / "comparison.json" - comparison = read_record(comparison_path) if comparison_path.exists() else {} - write_csv( - directory / "runs.csv", - [ - { - "run_id": run_id, - "created": manifest["created"], - **scalar_columns( - plan, ("schema_version", "suite_version", "mode", "exploratory") - ), - "collection_seconds": comparison.get("collection_seconds"), - "collection_seconds_reason": None - if "collection_seconds" in comparison - else "collection incomplete", - } - ], - ) - configurations = [ - { - "configuration_id": row["id"], - **scalar_columns(row["configuration"], CONFIGURATION_COLUMNS), - "status": row["status"], - "reason": row.get("reason"), - "estimate_bytes": row.get("estimate_bytes"), - "estimate_status": row.get("estimate_status"), - } - for row in plan["rows"] - ] - write_csv(directory / "configurations.csv", configurations) - write_csv( - directory / "cases.csv", - list( - { - row.get("case_hash", row["case"]): scalar_columns( - row, ("case", "revision", "case_hash") - ) - for row in configurations - }.values() - ), - ) - system = plan.get("system", {}) - write_csv( - directory / "system.csv", - [ - { - "run_id": run_id, - **scalar_columns( - system, - ( - "system", - "machine", - "logical_cpus", - "physical_cores", - "memory_bytes", - "processor", - "cpu_model", - "cpu_model_reason", - "cpu_topology_reason", - "core_residency", - "core_residency_reason", - ), - ), - "unavailable_reason": None - if system - else "not retained by this bundle version", - } - ], - ) - write_csv( - directory / "allocations.csv", - [ - { - "run_id": run_id, - "cpu": cpu, - "production": plan["profile"]["production"], - "policy": plan["profile"]["policy"], - "profile_explicit": plan["profile"]["explicit"], - **next( - ( - scalar_columns( - item, ("core", "socket", "core_reason", "socket_reason") - ) - for item in system.get("cpu_topology") or [] - if item["cpu"] == cpu - ), - {}, - ), - } - for cpu in plan["profile"]["allocation"] - ], - ) - write_csv( - directory / "processes.csv", - [ - { - "attempt_id": row["attempt_id"], - **scalar_columns( - row.get("process", {}), - ( - "pid", - "created", - "cpu_seconds", - "cpu_reason", - "rss_samples", - "missed_samples", - "poll_interval_seconds", - ), - ), - "unavailable_reason": None - if row.get("process") - else "no retained process receipt", - } - for row in rows - ], - ) - libraries = [] - for arm, environment in plan.get("environments", {}).items(): - found = environment.get("libraries", []) - for library in found or [{}]: - libraries.append( - { - "arm": arm, - "scope": "inventory", - **scalar_columns( - library, - ( - "user_api", - "internal_api", - "prefix", - "filepath", - "version", - "num_threads", - "threading_layer", - "architecture", - ), - ), - "unavailable_reason": None - if library - else "no supported pool observed", - } - ) - for row in rows: - for library in row.get("provenance", {}).get("libraries", []): - libraries.append( - { - "arm": row["arm"], - "attempt_id": row["attempt_id"], - "scope": "after measurement", - **scalar_columns( - library, - ( - "user_api", - "internal_api", - "prefix", - "filepath", - "version", - "num_threads", - "threading_layer", - "architecture", - ), - ), - } - ) - write_csv(directory / "libraries.csv", libraries) - write_csv( - directory / "coverage.csv", - [ - { - "run_id": run_id, - "status": "not-collected", - "reason": "native coverage requires its own collection", - } - ], - ) diff --git a/benchmarks/worker.py b/benchmarks/worker.py index 0bd3e309..57b22c44 100644 --- a/benchmarks/worker.py +++ b/benchmarks/worker.py @@ -2,7 +2,15 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -"""Execute one configuration in a fresh process with receipt-delimited measurement.""" +"""Time complete runs of one configuration in a fresh process (SEP 38). + +The controller sets the thread pools in the environment and passes a JSON request: +one configuration, the number of timed repeats and, optionally, where to save the +final field. The worker builds and translates the case and constructs its backend, +runs an untimed warm-up over the first part of the guide, then times complete runs +through the driver's own loop, each restarted from the translated launch. It prints +one JSON event per line; the last is the receipt. +""" from __future__ import annotations @@ -10,15 +18,12 @@ import importlib import importlib.metadata import logging -import re -import statistics import sys import time import traceback import warnings from collections.abc import Iterator from contextlib import contextmanager -from itertools import pairwise from pathlib import Path from typing import TYPE_CHECKING @@ -27,220 +32,69 @@ SuiteError, canonical, file_hash, - identity, metadata_value, read_record, ) -from benchmarks.registry import CASES -from benchmarks.telemetry import process_binding __all__ = [ "Diagnostics", - "ResetWitness", - "accuracy", "emit", - "execute", - "execute_single", + "integrator_label", "main", "measure", - "observed_runtime", - "prepare", - "reset_columns", - "reset_differences", - "reset_reference", - "resolved", - "retained_fields", "runtime", - "save_plane", "source_calls", - "timer_resolution", - "validation_artifacts", ] if TYPE_CHECKING: - import numpy as np - from solprop.backends.protocol import Backend - from solprop.driver import Simulation - from solprop.output import MemoryOutput - from solprop.translate.problem import KernelProblem - - -@dataclasses.dataclass(frozen=True) -class ResetWitness: - """Keep fresh fields and numerical history only in a qualification worker.""" - - fields: np.ndarray - columns: dict[str, np.ndarray] - - -def retained_fields(output: MemoryOutput) -> np.ndarray: - """Read the fields actually saved without fabricating statistics-only planes.""" - import numpy as np # noqa: PLC0415 - worker-only array access - - if output.retention.fields == "all": - return output.fields(convention="fft") - indices = output["field_planes"] - if len(indices) == 0: - raise SuiteError("validation needs at least one retained field") - return np.stack([output.plane_field(int(index), "fft") for index in indices]) - - -def reset_columns(output: MemoryOutput) -> dict[str, np.ndarray]: - """Copy observed decisions, controller history, statistics and noise state.""" - names = [ - "step_history/z", - "step_history/h", - "step_history/norm", - "step_history/accepted", - *(f"stats/{name}" for name in output.stat_names), - ] - if output.step_stat_names: - names.extend( - [ - "step_stats/z", - "step_stats/step", - *(f"step_stats/{name}" for name in output.step_stat_names), - ] - ) - columns = {name: output[name].copy() for name in names} - if output.noise_phase is not None: - columns["noise_phase"] = output.noise_phase.copy() - return columns - - -def reset_differences(witness: ResetWitness, output: MemoryOutput) -> Record: - """Report numerical history differences without imposing bitwise agreement.""" - import numpy as np # noqa: PLC0415 - qualification only - - found = reset_columns(output) - result: Record = {"same_columns": found.keys() == witness.columns.keys()} - for name, expected in witness.columns.items(): - actual = found.get(name) - if actual is None or actual.shape != expected.shape: - result[name] = {"same_shape": False} - elif expected.dtype == np.bool_: - result[name] = {"same_decisions": bool(np.array_equal(actual, expected))} - else: - # Empty modal columns have no centroid; a CW has no FWHM. These - # statistics deliberately use NaN (stats.py), unlike state/history. - # Compare their defined entries and retain the undefined masks - # explicitly, without allowing a NaN into the JSON metric itself. - undefined = np.isnan(expected) - found_undefined = np.isnan(actual) - if ( - np.isinf(expected).any() - or np.isinf(actual).any() - or ( - not ( - name.startswith(("stats/", "step_stats/")) - and name not in ("step_stats/z", "step_stats/step") - ) - and (undefined.any() or found_undefined.any()) - ) - ): - raise SuiteError(f"non-finite reset history or statistic: {name}") - defined = ~(undefined | found_undefined) - difference = ( - float(np.linalg.norm(actual[defined] - expected[defined])) - if defined.any() - else None - ) - norm = float(np.linalg.norm(expected[defined])) - result[name] = { - "same_shape": True, - "same_undefined": bool(np.array_equal(undefined, found_undefined)), - "reference_undefined_count": int(np.count_nonzero(undefined)), - "actual_undefined_count": int(np.count_nonzero(found_undefined)), - "compared_finite_count": int(np.count_nonzero(defined)), - "absolute_l2": difference, - "relative_l2": difference / norm - if norm and difference is not None - else None, - "relative_unavailable": "no jointly defined statistic entries" - if difference is None - else "zero reference norm" - if norm == 0 - else None, - } - return result def emit(kind: str, **values: object) -> None: - """Flush a protocol event; the parent timestamps its receipt.""" + """Flush a protocol event; the parent reads it after the worker exits.""" print(canonical({"event": kind, **values}), flush=True) -def runtime(backend: str, *, inventory: bool = False, coverage: bool = False) -> Record: - """Import in the required order and identify the actual loaded packages.""" +def runtime(backend: str) -> Record: + """Import in the required order and identify the loaded packages and pools.""" info: Record = {} if backend == "native": # solprop_native must be imported before pyFFTW. import solprop_native # noqa: PLC0415 # pyright: ignore[reportMissingImports] build = dataclasses.asdict(solprop_native.build_info()) - if build.get("coverage_instrumented", False) and not coverage: - raise SuiteError("instrumented native builds cannot collect timing samples") - if coverage and not build.get("coverage_instrumented", False): - raise SuiteError("native coverage requires an instrumented native build") + if build.get("coverage_instrumented", False): + raise SuiteError("an instrumented native build cannot be timed") if build.get("execution_space") not in ("OpenMP", "Threads", "Serial"): - raise SuiteError("this suite is qualified only for native CPU builds") + raise SuiteError("the benchmarks time native CPU builds only") extension = Path(importlib.import_module("solprop_native._core").__file__ or "") - info.update( - build_info=build, - extension_hash=file_hash(extension), - extension_path=str(extension), - wheel_hash=None, - wheel_hash_reason="original wheel unavailable from installed distribution", - ) + info.update(build_info=build, extension_hash=file_hash(extension)) from threadpoolctl import threadpool_info # noqa: PLC0415 import solprop # noqa: PLC0415 - from benchmarks.system import git_receipt # noqa: PLC0415 packages = {} - for name in ( - "solprop", - "numpy", - "scipy", - "optmat", - "numba", - "pyfftw", - "solprop-native", - ): + for name in ("solprop", "numpy", "scipy", "optmat", "numba", "pyfftw"): try: packages[name] = importlib.metadata.version(name) except importlib.metadata.PackageNotFoundError: packages[name] = None - source = Path(solprop.__file__).resolve().parents[2] info.update( packages=packages, solprop_path=str(Path(solprop.__file__).resolve()), - source=git_receipt(source) if inventory else None, - source_reason=None if inventory else "recorded once in collection inventory", libraries=threadpool_info(), - filesystem_cache="unknown", - numba_cache="uncontrolled-existing", python=sys.version, python_executable=sys.executable, ) return info -def observed_runtime(provenance: Record) -> Record: - """Observe lazily loaded library pools after timing and peak RSS collection.""" - from threadpoolctl import threadpool_info # noqa: PLC0415 - - return {**provenance, "libraries": threadpool_info()} - - class Diagnostics(logging.Handler): - """Aggregate emitted warnings and logs without suppressing their occurrences.""" + """Count emitted warnings and logs by phase without suppressing them.""" def __init__(self) -> None: super().__init__(logging.WARNING) self.phase = "import" - self.case: str | None = None self.rows: dict[tuple[str, str, str], int] = {} def emit(self, record: logging.LogRecord) -> None: @@ -272,27 +126,15 @@ def record( logger.removeHandler(self) def records(self) -> list[Record]: - """Return all occurrences; qualification reviews expected patterns.""" + """Return every distinct occurrence with its count.""" return [ - { - "phase": phase, - "category": category, - "message": message, - "count": count, - "expected": self.case is not None - and any( - rule.phase == phase - and rule.category == category - and re.fullmatch(rule.pattern, message) is not None - for rule in CASES[self.case].warnings - ), - } + {"phase": phase, "category": category, "message": message, "count": count} for (phase, category, message), count in self.rows.items() ] -def source_calls(engine: Backend) -> int: - """Read physical/logical evaluation ledgers without step-count inference.""" +def source_calls(engine: Backend) -> int | None: + """Read the backend's source-evaluation ledger, or ``None`` if it has none.""" metadata = engine.metadata() value = metadata.get("source_evaluations") if isinstance(value, int): @@ -300,249 +142,45 @@ def source_calls(engine: Backend) -> int: ledger = metadata.get("response_calls") if isinstance(ledger, dict): return sum(int(ledger[key]) for key in ("startup", "predictor", "corrected")) - raise SuiteError( - f"{engine.name} has no source evaluation counter; rebuild the backend" - ) - - -def prepare( - simulation: Simulation, - problem: KernelProblem, - engine: Backend, - configuration: Record, -) -> MemoryOutput: - """Restore launch and method/controller history through the declared lead-in.""" - from solprop.output import MemoryOutput # noqa: PLC0415 - from solprop.output.schema import FieldRetention # noqa: PLC0415 - - engine.set_initial(problem.initial.field0, None) - if configuration["start"]: - engine.propagate_to(float(configuration["start"])) - engine.observe(field=False) - output = MemoryOutput() - if configuration["fields"] == "planes": - output.retain( - FieldRetention("planes", (int(configuration["observations"]) - 1,)), - problem.shape[1:], - ) - output.open(simulation.metadata()) - output.reserve(int(configuration["observations"])) - return output - - -def save_plane( - engine: Backend, problem: KernelProblem, output: MemoryOutput, *, keep: bool = True -) -> None: - """Use the ordinary amplitude and saved-plane conversions.""" - from solprop.driver.run import ( # noqa: PLC0415 - _power_observation, - plane_from_observation, - ) - - observed = _power_observation(problem, engine.observe(field=keep)) - output.save(plane_from_observation(observed, engine.stats(problem.stats_spec))) - - -def measure( - engine: Backend, problem: KernelProblem, configuration: Record, output: MemoryOutput -) -> Record: - """Time the declared work, preserving every repetition's actual counters.""" - import numpy as np # noqa: PLC0415 - - adaptive = configuration["track"].startswith("adaptive-") - planes = np.linspace( - configuration["start"], - configuration["stop"], - int(configuration["observations"]) + 1, - )[1:] - before = source_calls(engine) - steps = rejected = 0 - started = time.perf_counter_ns() - for index, plane in enumerate(planes): - summary = engine.propagate_to(float(plane)) - steps += summary.n_steps - rejected += summary.n_rejected - if summary.z != float(plane): - raise SuiteError("backend did not reach the declared endpoint") - if adaptive: - save_plane( - engine, - problem, - output, - keep=configuration["fields"] != "planes" or index == len(planes) - 1, - ) - seconds = (time.perf_counter_ns() - started) * 1e-9 - return { - "seconds": seconds, - "accepted": steps - rejected, - "rejected": rejected, - "source_evaluations": source_calls(engine) - before, - "endpoint": float(planes[-1]), - "observation_seconds": None, - } - - -def resolved( - simulation: Simulation, problem: KernelProblem, configuration: Record -) -> Record: - """Record actual method defaults and material/initial-state identities.""" - - from solprop.output.schema import serialise # noqa: PLC0415 - - return metadata_value( - { - "configuration": configuration, - "simulation": serialise(simulation.metadata().configuration), - "integrator": serialise(problem.stepper.integrator), - "controller": serialise(problem.stepper.controller), - "shape": list(problem.shape), - "n_t": problem.grids.n_t, - "n_to": problem.grids.n_to, - "statistics": serialise(problem.stats_spec), - "measurement_planes": configuration["observations"], - "fields": configuration["fields"], - } - ) - - -def accuracy(fields: np.ndarray, configuration: Record, request: Record) -> Record: - """Check finiteness and a frozen independent reference after the RSS receipt.""" - import numpy as np # noqa: PLC0415 + return None - if not np.all(np.isfinite(fields)): - return {"status": "invalid", "reason": "non-finite output"} - previous = request.get("prior_validation") - if previous is not None: - if list(fields.shape) != previous["field_shape"]: - return {"status": "invalid", "reason": "validated output shape changed"} - return dict(previous) - reference = request.get("reference") - if not reference: - return {"status": "not-qualified", "reason": "no frozen qualified reference"} - from benchmarks.references import check # noqa: PLC0415 - try: - return { - **check(fields, configuration, Path(reference)), - "field_shape": list(fields.shape), - "validation": "current sample", - } - except (SuiteError, OSError, ValueError) as error: - return {"status": "invalid", "reason": str(error)} +def integrator_label(integrator: object) -> str: + """Name an integrator with the option that distinguishes its variants. + A translated problem holds ``None`` for the interaction-picture DOPRI5 march. + """ + if integrator is None: + return "RungeKutta" + name = type(integrator).__name__ + for option in ("formulation", "scheme"): + value = getattr(integrator, option, None) + if isinstance(value, str): + return f"{name}({value})" + return name -def timer_resolution() -> float: - """Measure clock increments and call overhead without a minimum time loop.""" - ticks = [time.perf_counter_ns() for _ in range(1001)] - differences = [right - left for left, right in pairwise(ticks) if right > left] - return max( - time.get_clock_info("perf_counter").resolution, - statistics.median(differences) * 1e-9, - ) +def measure(configuration: Record, repeats: int, fields: Path | None) -> Record: + """Build once, warm up once, then time ``repeats`` complete runs. -def validation_artifacts( - fields: np.ndarray, - fresh_fields: ResetWitness | None, - output: MemoryOutput, - problem: KernelProblem, - request: Record, - measurements: tuple[Record, list[Record]], -) -> Record: - """Save qualification arrays and reset evidence after the measurement receipt.""" + Each run restarts the backend from the translated launch and goes through + :func:`solprop.driver.run.run_problem`, the loop of ``Simulation.run``, so it + includes the plane observations and statistics. Construction, translation and + backend construction are timed separately. + """ import numpy as np # noqa: PLC0415 - from benchmarks.references import field_errors # noqa: PLC0415 - - first, repetitions = measurements - if request.get("save_fields"): - np.save(request["save_fields"], fields, allow_pickle=False) - np.save( - f"{request['save_fields']}.band.npy", - problem.grids.inband_mask, - allow_pickle=False, - ) - np.save( - f"{request['save_fields']}.omega.npy", - problem.grids.omega, - allow_pickle=False, - ) - reset = None - if fresh_fields is not None: - keys = ("accepted", "rejected", "source_evaluations") - reset = { - "plane_errors": field_errors(fields, fresh_fields.fields), - "history_statistics_noise": reset_differences(fresh_fields, output), - "fresh_counts": {key: first[key] for key in keys}, - "reset_counts": [{key: item[key] for key in keys} for item in repetitions], - } - edges = [ - 0.0, - *( - float(z + h) - for z, h, accepted in zip( - output["step_history/z"], - output["step_history/h"], - output["step_history/accepted"], - strict=True, - ) - if accepted - ), - ] - return {"reset_evidence": reset, "pilot_edges": edges[:11]} - - -def reset_reference( - engine: Backend, problem: KernelProblem, output: MemoryOutput, request: Record -) -> ResetWitness | None: - """Capture the fresh construction result only during separate qualification.""" - if not request.get("verify_reset"): - return None - if not request["configuration"]["track"].startswith("adaptive-"): - save_plane(engine, problem, output) - return ResetWitness(retained_fields(output).copy(), reset_columns(output)) - - -def execute(request: Record, diagnostics: Diagnostics) -> Record: - """Run one complete fresh-process sample and return its validated receipt.""" - configuration = request["configuration"] - diagnostics.case = configuration["case"] - started = time.perf_counter() - provenance = runtime( - configuration["backend"], - inventory=request.get("inventory", False), - coverage=request.get("coverage", False), - ) - import_seconds = time.perf_counter() - started - if request.get("inventory"): - return {"status": "ok", "provenance": provenance} - if request.get("coverage"): - from benchmarks.coverage_worker import execute_coverage # noqa: PLC0415 - - return execute_coverage(request, diagnostics, provenance) - if configuration["case"] == "batch": - from benchmarks.batching import execute_batch # noqa: PLC0415 - - return execute_batch(request, diagnostics, provenance, import_seconds) - return execute_single(request, diagnostics, provenance, import_seconds) - - -def execute_single( - request: Record, diagnostics: Diagnostics, provenance: Record, import_seconds: float -) -> Record: - """Measure one singleton backend after the common runtime inventory.""" - configuration = request["configuration"] from benchmarks.builders import build # noqa: PLC0415 from solprop.backends import select # noqa: PLC0415 from solprop.driver.executors.pool import peak_rss # noqa: PLC0415 + from solprop.driver.run import field_retention, run_problem # noqa: PLC0415 + from solprop.output import MemoryOutput # noqa: PLC0415 from solprop.output.schema import serialise # noqa: PLC0415 - diagnostics.phase = "construction" started = time.perf_counter() simulation = build(configuration) problem = simulation.translate() - resolution = resolved(simulation, problem, configuration) - construction_seconds = time.perf_counter() - started + construction = time.perf_counter() - started started = time.perf_counter() engine = select( configuration["backend"], @@ -552,89 +190,105 @@ def execute_single( profiles=simulation.profiles, ) backend_seconds = time.perf_counter() - started - repetitions: list[Record] = [] + retention = field_retention( + problem, simulation.settings.fields, simulation.settings.field_planes + ) + metadata = simulation.metadata() + controller = problem.stepper.controller try: - diagnostics.phase = "first-window" - output = prepare(simulation, problem, engine, configuration) - first = measure(engine, problem, configuration, output) - emit("first-window", seconds=first["seconds"]) - fresh_fields = reset_reference(engine, problem, output, request) - output.close() - for _ in range(request.get("repetitions", 1)): - output.close() - del output - diagnostics.phase = "reset" - output = prepare(simulation, problem, engine, configuration) - diagnostics.phase = "measurement" - repetitions.append(measure(engine, problem, configuration, output)) - peak = peak_rss() - emit("measurement", peak_rss_bytes=peak) - diagnostics.phase = "validation" - if not configuration["track"].startswith("adaptive-"): - # The fixed track's observation is separate, and cannot increase the - # recorded measurement peak or the propagation timer. - started = time.perf_counter() - save_plane(engine, problem, output) - repetitions[-1]["observation_seconds"] = time.perf_counter() - started - output.close() - fields = retained_fields(output) - check = accuracy(fields, configuration, request) - overridden = check["status"] == "invalid" and request.get( - "allow_accuracy_failure", False - ) - artifacts = validation_artifacts( - fields, fresh_fields, output, problem, request, (first, repetitions) - ) - output.close() + started = time.perf_counter() + engine.set_initial(problem.initial.field0, None, z=problem.initial.z) + if configuration["warmup"]: + engine.propagate_to(configuration["warmup"] * configuration["stop"]) + warmup = time.perf_counter() - started + repetitions = [] + output = None + for _ in range(repeats): + engine.set_initial(problem.initial.field0, None, z=problem.initial.z) + output = MemoryOutput() + before = source_calls(engine) + started = time.perf_counter_ns() + run_problem( + problem, + engine, + output, + metadata=metadata, + stats_spec=problem.stats_spec, + progress="none", + resume_history=simulation.settings.resume_history, + fields=retention.fields, + field_planes=retention.field_planes, + ) + seconds = (time.perf_counter_ns() - started) * 1e-9 + after = source_calls(engine) + accepted = np.asarray(output["step_history/accepted"], dtype=bool) + repetitions.append( + { + "seconds": seconds, + "accepted": int(accepted.sum()), + "rejected": int((~accepted).sum()), + "source_evaluations": None + if before is None or after is None + else after - before, + "z": float(output["z"][-1]), + } + ) + assert output is not None + final = output.plane_field(output.n_saved - 1, "fft") + if fields is not None: + fields.parent.mkdir(parents=True, exist_ok=True) + np.save(fields, final, allow_pickle=False) return { - "status": "invalid" - if check["status"] == "invalid" and not overridden - else "ok", - "accuracy_overridden": overridden, - "seconds": sum(item["seconds"] for item in repetitions) / len(repetitions), - "repetitions": repetitions, - "peak_rss_bytes": peak, - "accuracy": check, - **artifacts, - "timer_resolution": timer_resolution(), - "resolution": resolution, - "resolution_hash": identity(resolution), - "provenance": observed_runtime(provenance), - "metadata": metadata_value(serialise(engine.metadata())), - "ledger": engine.memory_ledger(), - "first_window_seconds": first["seconds"], - "import_seconds": import_seconds, - "construction_seconds": construction_seconds, + "construction_seconds": construction, "backend_seconds": backend_seconds, - "thread_diagnostics": process_binding() - if request.get("thread_diagnostics") + "warmup_seconds": warmup, + "repetitions": repetitions, + "peak_rss_bytes": peak_rss(), + "finite": bool(np.all(np.isfinite(final))), + "integrator": integrator_label(problem.stepper.integrator), + "rtol": controller.rtol, + "atol": controller.atol, + "noise": simulation.noise_resolution.resolved + if simulation.noise_resolution is not None else None, + "shape": list(problem.shape), + "resolution": metadata_value( + { + "integrator": serialise(problem.stepper.integrator), + "controller": serialise(controller), + } + ), } finally: engine.close() def main() -> int: - """Execute the JSON worker request passed by the controller.""" + """Execute the JSON request passed by the controller.""" diagnostics = Diagnostics() try: request = read_record(Path(sys.argv[1])) + configuration = request["configuration"] with diagnostics.capture(): - result = execute(request, diagnostics) - result["diagnostics"] = diagnostics.records() - emit("result", result=result) - return 0 if result.get("status", "ok") == "ok" else 1 - except Exception as error: # worker boundary: preserve a failed-attempt receipt + emit("provenance", provenance=runtime(configuration["backend"])) + diagnostics.phase = "measurement" + fields = request.get("fields") + result = measure( + configuration, + int(request["repeats"]), + None if fields is None else Path(fields), + ) + emit("result", status="ok", diagnostics=diagnostics.records(), **result) + except Exception as error: # worker boundary: the controller records the failure traceback.print_exc(file=sys.stderr) emit( "result", - result={ - "status": "failed", - "error": f"{type(error).__name__}: {error}", - "diagnostics": diagnostics.records(), - }, + status="failed", + error=f"{type(error).__name__}: {error}", + diagnostics=diagnostics.records(), ) return 1 + return 0 if __name__ == "__main__": diff --git a/docs/dev/seps/sep-0038-performance-tracking.md b/docs/dev/seps/sep-0038-performance-tracking.md new file mode 100644 index 00000000..8bdb4dd7 --- /dev/null +++ b/docs/dev/seps/sep-0038-performance-tracking.md @@ -0,0 +1,286 @@ +# SEP 38 — Performance tracking: complete runs, about an hour per machine + +| | | +|---|---| +| **SEP** | 38 | +| **Title** | Performance tracking: complete runs, about an hour per machine | +| **Author** | John Travers | +| **Drafting agent** | Opus 5.5 (2026-10-27) | +| **Status** | Draft | +| **Review status** | Not reviewed | +| **Reviewers** | — | +| **Type** | Process | +| **Created** | 2026-10-27 | +| **Resolution** | — | +| **Requires** | [SEP 37](sep-0037-noise-and-stepper-defaults.md) (the per-interface defaults the tracked cases run with) | +| **Supersedes** | [SEP 30](archive/sep-0030-standard-benchmark-suite.md) | +| **Superseded-by** | — | +| **Implementation** | Opus 5.5 (AI) for John Travers, branch `feature/sep-0038-performance-tracking`, drafted together with its implementation, stacked on SEP 37's branch | +| **Manual pages** | — | + +## Part I — Proposal + +### 1. Abstract + +Replace SEP 30's standard benchmark runner with a small tracking command, +`python -m benchmarks track`. It times **complete propagations** of about a dozen +representative workloads at one thread and at the machine's production width, each with +the library's own default integrator and tolerances for its interface. On one machine it +takes about an hour. Each run appends a few lines per case to a small CSV kept in the +repository, one file per hardware set, so that every release can be compared with the +previous ones on the same hardware. The exploratory `ab` command stays, for comparing two +builds while a change is being made. SEP 30's standard paired mode, its numerical +reference qualification, its native coverage collector and its public export are +removed. Success means the command is run before each release on the maintainer's +machines, and a regression shows in its history. + +### 2. Motivation and scope + +SEP 30 built a rigorous suite: 41 cases, adaptive and fixed tracks over short entrance +windows, five to eleven fresh-process rounds, frozen references and a sanitized export. +As specified it is too heavy to be used. The core set took 2.5–3.3 h per backend on an +Apple M5 Max and about 6 h on an Intel Xeon Gold 6240; the advanced set at two widths +about 5 h. Yet most measured windows lasted 0.05–6 s: the cost was construction, fresh +processes, guards and rounds. The windows are entrance windows, before compression, +ionisation or the Raman shift develop, so they do not represent a production run's cost +either. The suite has not been run since it merged. + +A release needs something smaller: one command, an hour per machine, numbers that mean +"how long does the usual simulation take now", kept where the next release can find +them. + +**In scope:** + +- the `track` command, its case set and its measurement protocol (§4, §5); +- a hardware identity without host names, and a per-hardware history CSV in the + repository (§5.3); +- keeping every SEP 30 case runnable by name, outside the default selection; +- `ab` reworked onto the same worker, with a length option for quick iteration; +- removing SEP 30's standard mode, references, coverage and export, and their tests. + +**Out of scope:** + +- accuracy qualification. SEP 37 measured the defaults' accuracy; tracking times the + default, it does not certify it. A numerical regression is caught by the test suite; +- GPU tracking (a later SEP, once the GPU builds are released); +- statistical adoption verdicts. The history shows changes; the + [benchmark rules](../benchmarks.md) still say how a performance claim is judged; +- continuous integration. Shared CI runners are too noisy for timing. + +### 3. Context + +The [benchmark rules](../benchmarks.md) and the +[runner's manual](../benchmarking.md) describe SEP 30's suite. Its modules are under +`benchmarks/`: the registry (`registry.py`), case builders (`builders.py`, +`exercises.py`, `guides.py`, `detectors.py`, `batching.py`), the controller and worker +(`collection.py`, `worker.py`, `planning.py`), comparison and qualification +(`comparison.py`, `qualification.py`, `references.py`), native coverage +(`native_coverage.py`, `coverage_worker.py`, `coverage_reports.py`), export +(`publication.py`, `views.py`) and machine records (`system.py`, `telemetry.py`). The +`ab` mode (`ab.py`, `ab_worker.py`) was added after SEP 30 to time two builds quickly. + +[SEP 37](sep-0037-noise-and-stepper-defaults.md) made input noise the default and chose +an integrator and tolerances for each `prop_*` interface. Its phase 0 timed complete +runs of the cases proposed here on an AMD EPYC 7763 node: these give the cost estimates +below. With noise on by default, a timed case needs a fixed seed to be repeatable. + +### 4. Theory + +There is no physics here; this section states the measurement. + +**What is timed.** One complete propagation, from the launch to the end of the guide, +through the backend's `propagate_to` and the ordinary plane observations and statistics, +as `Simulation.run` performs them. Import, construction, translation and backend +construction are timed separately and reported, not tracked. A run restarts from the +translated initial field (`set_initial`), so its repeats take the same steps. + +**Warm-up.** Before the timed repeats, one untimed partial run to 2 % of the length (at +least one step) builds FFT plans and fills caches. A cold first run is not tracked: its +cost is dominated by one-off work the history should not mix with propagation. + +**Repeats and statistic.** Three timed repeats per configuration (`--repeats`). The +tracked value is their median; the minimum and maximum are kept beside it. On a quiet +machine the spread of complete runs of tens of seconds is a few per cent, against the +tens of per cent of the millisecond windows SEP 30 measured. + +**One process per configuration.** Each case at each width runs in its own worker, +whose thread pools are set in the environment before import. Peak resident memory is +then the configuration's own. + +**Aggregate.** For one hardware set and width, the change between two runs is the +geometric mean, over the cases both ran with the same case revision and resolved +integrator, of the ratio of their medians. A changed default integrator is a different +configuration, and is shown as such rather than averaged in. + +### 5. Proposed interface + +#### 5.1 Commands + +```text +python -m benchmarks list [--all] [--case NAME] [--stepper S] [--threads W] +python -m benchmarks track [--case NAME] [--all] [--stepper S] [--threads W] + [--repeats 3] [--production N] [--instance-type TYPE] + [--append-history] [--archive DIR] [--out DIR] +python -m benchmarks ab --baseline-python PY [--candidate-python PY] + [--case NAME] [--stepper S] [--threads W] + [--length FRACTION] [--rounds 3] [--repeats 3] +python -m benchmarks history [SLUG] [--width W] +``` + +- `list` prints the plan and its estimated time; it imports no scientific library. +- `track` with no selection runs the tracking set (§5.2) at `1,production`. + `--case` selects any registered case, tracked or not. +- `--stepper` is `default` (the interface's own, SEP 37), `multistep`, + `multistep-interaction` or `rk`, or a comma-separated list of them to A/B steppers in + one run. An override replaces the integrator only; the tolerances stay the + interface's. A case on a fixed schedule admits only `default`. +- `--production` gives the production width; otherwise it is the number of physical + cores in the process's CPU allocation. +- `--append-history` adds the run to `benchmarks/history/.csv`. It refuses a + dirty source tree, an incomplete run and a non-native backend. +- `history` prints, per width, one line per recorded run: date, version, commit, the + number of cases, and the geometric mean relative to the first recorded run. + +#### 5.2 The tracking set + +The cases represent the main uses of solprop at the sizes they are used. Lengths are +complete guides. Costs are SEP 37 phase 0's medians for complete runs on an AMD EPYC +7763, at 1 and 16 threads, with that branch's defaults; "—" marks a width the case does +not run at. + +| Case | Interface | Problem | Widths | Cost (s) | +|---|---|---|---|---| +| `gnlse_raman_shock` | `prop_gnlse` | Dudley–Genty–Coen supercontinuum, Raman and shock, 0.15 m [1] | 1, P | ~80 / ~30 | +| `gnlse_soliton` | `prop_gnlse` | ideal N = 3 soliton over one soliton period | 1, P | ~8 / ~5 | +| `rdw` | `prop_capillary` | He resonant dispersive wave, 4 resolved modes, 1.5 m, 0.8→0 bar gradient | 1, P | 19 / 10 | +| `he_rdw_averaged` | `prop_capillary` | the same, mode-averaged | 1, P | 3 / 3 | +| `argon_blueshift` (rev. 2) | `prop_capillary` | Ar ionisation blueshift, 13 µm, 1.7 bar, 0.2 m, 1 resolved mode | 1, P | 67 / 34 | +| `argon_blueshift_averaged` (rev. 2) | `prop_capillary` | the same, mode-averaged | 1, P | 15 / 15 | +| `argon_blueshift_envelope` (new) | `prop_capillary` | `argon_blueshift` on the envelope grid | 1, P | 3 / 3 | +| `h2_raman_averaged` (new) | `prop_capillary` | H2 soliton self-frequency shift, 10 bar, 0.4 m, mode-averaged | 1, P | 16 / 16 | +| `ctc` (rev. 2) | `prop_capillary` | H2 comb-to-continuum, 3 modes, 40 bar, 0.13 m, 2 ps window | P | — / 290 | +| `radial_filament_256` | `prop_radial` | Ar filament, 256 radial nodes, 3 cm | 1, P | ~10 / ~4 | +| `grin_nonlinear` (rev. 2) | `prop_waveguide` | four-channel graded-index fibre, 2 cm | P | — / ~35 | +| `kerr_beam_384` | `prop_free` | fused-silica Kerr beam, 384² transverse grid, fixed RK4IP | P | — / ~10 | +| `kerr_beam_192` | `prop_free` | the same at 192², fixed RK4IP | 1 | ~10 / — | + +The sum is about 220 s at one thread and 460 s at 16 threads per pass. Three repeats, the +warm-ups and construction bring it to about 40 minutes on that node; the target is under +an hour on any machine in use. The Phase 2 calibration run fixes the final lengths. + +The three most expensive cases run at the production width only: `ctc` (about 600 s at +one thread), the 2 cm GRIN window (about 100 s) and the 384² Kerr beam (about 90 s). The +192² Kerr beam keeps a one-thread free-space row. + +Every case keeps its interface's default noise (`"input"` after SEP 37) with a fixed +seed. Every other registered case stays runnable by name; none is in the default +selection. + +#### 5.3 Hardware identity and history + +The **hardware slug** is the CPU model reduced to lower-case words, followed by the number +of physical cores in the allocation, for example `apple-m5-max-18c` or +`amd-epyc-7763-16c`. It names no host. A cloud instance type, from `--instance-type` or +the DMI product name on Linux, is recorded beside it but is not part of the slug. + +Each history row is one (run, case, stepper, width): date, solprop version, commit, +native build identity (version, profile, compiler), slug, CPU model, instance type, OS, +width, case, case revision, requested stepper, resolved integrator and tolerances, noise +model, median, minimum and maximum seconds, repeats, accepted and rejected steps, source +evaluations, peak resident memory, and whether the machine was busy when a worker +started. That is a few kilobytes per release per machine. + +The full bundle of a run (plan, worker requests and logs, results, environment) stays in +the output directory, which git ignores. `--archive DIR` copies it to a directory of the +maintainer's choosing. + +### 6. Implementation strategies + +**Trim and reuse, or rewrite.** The registry, builders, telemetry and `ab` are sound and +are kept. The controller of the standard mode (`collection.py`, 780 lines, with +monitoring, resume and admission) has nothing `track` needs: `track` launches one worker +per configuration and reads its receipt, as `ab` already does. Rewriting the worker once +for both `track` and `ab` is smaller than adapting the standard worker, which is entwined +with references and reset witnesses. + +**Library defaults.** The builders stop passing tolerances and integrators. A case is +the interface call a user would write, plus a seed and the backend. Where the interface +has no default controller (free space) or the case builds a `Simulation` directly, the +case declares its own, as now. + +**Case revisions.** A case whose definition changes keeps its name and increments its +revision; history rows carry the revision and are compared only within one. The +capillary cases revised here take the definitions SEP 37 phase 0 measured. + +**Removed.** `collection.py`, `comparison.py`, `qualification.py`, `references.py`, +`planning.py`, `schedules.py`, `native_coverage.py`, `coverage_worker.py`, +`coverage_reports.py`, `publication.py`, `views.py`, `ab_worker.py`, and the `batch` +exercise (`batching.py`), whose lockstep route is covered by +`tests/backends/test_batch_lockstep.py`. Their tests go with them. + +### 7. Verification and validation + +- Unit tests, run in the default lane: the registry (every case resolves, the tracking + set, revisions), the plan and its estimate without scientific imports, the hardware + slug (formats, no host name), the history writer and summary, the stepper override, + and `ab`'s pairing. +- An end-to-end test runs `track` and `ab` on the `smoke` case with one repeat. +- Calibration: one complete `track` run on an AMD EPYC 7763 node, recorded in the phase + report with each case's time, and the lengths adjusted if the total exceeds an hour. +- First history: the Apple M5 Max and the EPYC node, appended at the release commit. + +### 8. Documentation + +Rewrite [Running the benchmark suite](../benchmarking.md) as the manual for `track`, `ab` +and `history`. In [Benchmarks](../benchmarks.md), replace "Measurement windows" with +the tracking protocol, keep the rules and the indicative results, and point to the +history files. Update `benchmarks/README.md`, the `poe bench` task, `AGENTS.md` and the +performance guide's link. + +### 9. Compatibility and impact + +No change to solprop itself. The SEP 30 commands `run`, `compare`, `qualify`, `profile`, +`diagnose`, `coverage` and `export` are removed; nothing outside the benchmark package +uses them. SEP 30's archived baselines are not comparable with tracking rows (windows +against complete runs) and are not migrated. SEP 30 becomes `Superseded` when this SEP +is final. + +### 10. Alternatives considered + +- **Keep SEP 30 with lighter defaults.** Fewer rounds and samples would still time + entrance windows, which say little about production cost, and keep about 5000 lines + that nothing uses. +- **Time `Simulation.run` end to end.** Simple, but each repeat would reconstruct the + backend and its plans, and that construction would vary with the build in ways that + are not propagation performance. +- **A single width.** One thread isolates per-core regressions; the production width + shows threading regressions. Both are cheap except on the three cases kept at the + production width only. +- **Track in CI.** Shared runners vary by tens of per cent between jobs. + +### 11. Phases + +1. **Implementation:** the registry, builders on library defaults, the worker, `track`, + `history`, `ab` on the new worker, the removals, tests and documentation. +2. **Calibration and first data:** a complete run on the EPYC node, lengths adjusted, + then the first history rows on the maintainer's machines. + +### 12. Open questions + +- Q1. If SEP 37's capillary default becomes `RungeKutta()` at 1e-5, the capillary costs + fall; should the freed time go to a longer `ctc`, or should `ctc` run at one thread + too? +- Q2. Should a cloud instance (for example AWS `c8a` or GCP `c4d`) be a standing + history file, re-rented per release, or measured only when a hardware question arises? + +### 13. References + +1. J. M. Dudley, G. Genty and S. Coen, "Supercontinuum generation in photonic crystal + fiber," *Rev. Mod. Phys.* **78**, 1135 (2006). doi:10.1103/RevModPhys.78.1135 + +## Part II — Implementation record + +## Part III — History + +- 2026-10-27: Drafted, together with its implementation on the same branch, as the + maintainer asked; it supersedes SEP 30 at Final. diff --git a/pyproject.toml b/pyproject.toml index db2ad2de..0dfafa10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -280,7 +280,7 @@ docs-clean = "python -c \"import shutil; [shutil.rmtree(p, ignore_errors=True) f [tool.poe.tasks.bench] cmd = "python -m benchmarks" -help = "Run the bounded core quick suite; pass list, run, compare or qualify options." +help = "Plan the tracking run; pass track, ab, history or list options (SEP 38)." # The test lanes run under xdist: one worker per core, with `loadgroup` and a default # group per module assigned by tests/conftest.py. Module fixtures stay on one worker; diff --git a/tests/test_benchmark_ab.py b/tests/test_benchmark_ab.py index 61eb04d6..f7711e4e 100644 --- a/tests/test_benchmark_ab.py +++ b/tests/test_benchmark_ab.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -"""Exercise the fast exploratory A/B mode on synthetic receipts and a small case.""" +"""Exercise the exploratory A/B mode on synthetic receipts and a small case.""" import math import sys @@ -10,26 +10,8 @@ import numpy as np import pytest from benchmarks import __main__ as cli -from benchmarks import ab, system -from benchmarks.collection import ROOT -from benchmarks.planning import Selection, matrix +from benchmarks import ab from benchmarks.records import read_record -from benchmarks.registry import ADAPTIVE, FIXED, select - - -@pytest.fixture(autouse=True) -def _ample_memory(monkeypatch): - """Admit by the cases' estimates on a fixed machine, not this one's free memory. - - A loaded 16 GB runner has too little free memory for the smoke case's estimate, - which would skip the rows these tests exercise; admission is patched where tested. - """ - monkeypatch.setattr(system, "effective_memory", lambda: 64 * 2**30) - - -@pytest.fixture -def profile(): - return system.Profile("test-machine", 1, 2, (0, 1), "topology") def receipt(identifier, arm, round_index, seconds, *, work=(10, 1)): @@ -51,30 +33,10 @@ def test_build_order_alternates_between_rounds(): assert ab.arm_order(2) == ab.arm_order(0) -def test_admission_explains_every_skipped_row(profile, monkeypatch): - plan = matrix( - select(("smoke", "batch", "kerr_beam_768")), - profile, - Selection(threads="1", tracks=(ADAPTIVE[1], FIXED)), - ) - rows, skipped = ab.admitted(plan) - assert [row["configuration"]["case"] for row in rows] == ["smoke", "smoke"] - assert any("batch exercise needs the standard runner" in item for item in skipped) - assert any("kerr_beam_768" in item for item in skipped) - monkeypatch.setattr( - ab, "admission", lambda *_args: {"admitted": False, "reason": "no memory"} - ) - rows, skipped = ab.admitted(plan) - assert not rows - assert any( - item.endswith("smoke adaptive-rk 1 threads: no memory") for item in skipped - ) - - def test_summary_pairs_rounds_and_reports_spread_and_fields(tmp_path): rows = [ - {"id": "a", "configuration": {"case": "x", "track": "t", "threads": 1}}, - {"id": "b", "configuration": {"case": "y", "track": "t", "threads": 1}}, + {"id": "a", "configuration": {"case": "x", "stepper": "rk", "threads": 1}}, + {"id": "b", "configuration": {"case": "y", "stepper": "rk", "threads": 1}}, ] samples = [ receipt("a", "baseline", 0, [1.0, 1.2, 1.1]), @@ -122,9 +84,9 @@ def test_summary_output_states_ratio_direction_without_a_verdict(capsys): [ { "case": "x", - "track": "t", + "stepper": "rk", "threads": 1, - "baseline_seconds": 1e-3, + "baseline_seconds": 1.0, "candidate_seconds": None, "ratio": None, "field_difference": None, @@ -133,10 +95,10 @@ def test_summary_output_states_ratio_direction_without_a_verdict(capsys): }, { "case": "y", - "track": "t", + "stepper": "rk", "threads": 1, - "baseline_seconds": 2e-3, - "candidate_seconds": 1e-3, + "baseline_seconds": 2.0, + "candidate_seconds": 1.0, "ratio": 0.5, "ratio_min": 0.4, "ratio_max": 0.6, @@ -156,26 +118,21 @@ def test_summary_output_states_ratio_direction_without_a_verdict(capsys): def test_invalid_options_and_existing_output_are_refused(tmp_path, capsys): base = ["ab", "--baseline-python", sys.executable, "--threads", "1"] assert cli.main([*base, "--case", "smoke", "--rounds", "0"]) == 2 - assert cli.main([*base, "--case", "batch", "--out", str(tmp_path / "b")]) == 2 + assert ( + cli.main([*base, "--case", "kerr_beam_192", "--stepper", "rk", "--out", "x"]) + == 2 + ) (tmp_path / "kept").write_text("x") assert cli.main([*base, "--case", "smoke", "--out", str(tmp_path)]) == 2 errors = capsys.readouterr().err assert "must be positive" in errors - assert "no admitted configuration" in errors + assert "no configuration in the selection" in errors assert "output already exists" in errors -def test_worker_timeout_and_missing_receipts_are_failures(tmp_path, monkeypatch): - events = ab.launch_worker( - sys.executable, - {"rows": [], "repeats": 1, "harness_root": str(ROOT)}, - {"PATH": ""}, - tmp_path / "slow", - 1e-3, - ) - assert events == [{"event": "failed", "error": "worker timeout"}] - monkeypatch.setattr(ab, "launch_worker", lambda *_args: []) - monkeypatch.setattr(ab, "guard", lambda *_args, **_kw: {"quiet": False}) +def test_missing_receipts_are_failures(tmp_path, monkeypatch): + monkeypatch.setattr(ab, "launch_worker", lambda *_args: ([], 0.0)) + monkeypatch.setattr(ab, "guard", lambda *_args: {"quiet": False}) out = tmp_path / "run" status = cli.main( [ @@ -184,8 +141,6 @@ def test_worker_timeout_and_missing_receipts_are_failures(tmp_path, monkeypatch) "smoke", "--threads", "1", - "--track", - ADAPTIVE[1], "--rounds", "1", "--baseline-python", @@ -205,7 +160,7 @@ def test_worker_timeout_and_missing_receipts_are_failures(tmp_path, monkeypatch) def test_small_case_runs_both_builds_in_alternating_order( tmp_path, monkeypatch, capsys ): - monkeypatch.setattr(ab, "guard", lambda *_args, **_kw: {"quiet": True}) + monkeypatch.setattr(ab, "guard", lambda *_args: {"quiet": True}) out = tmp_path / "run" status = cli.main( [ @@ -216,10 +171,8 @@ def test_small_case_runs_both_builds_in_alternating_order( "numpy_ref", "--threads", "1", - "--track", - ADAPTIVE[1], - "--track", - FIXED, + "--length", + "0.5", "--rounds", "2", "--repeats", @@ -241,12 +194,17 @@ def test_small_case_runs_both_builds_in_alternating_order( "baseline", ] results = read_record(out / "results.json") - assert len(results["samples"]) == 8 + assert len(results["samples"]) == 4 assert all(len(item["repetitions"]) == 2 for item in results["samples"]) - for row in results["summary"]: - assert row["rounds"] == 2 - assert row["same_work"] - # The same interpreter and build repeat the same operations exactly. - assert row["field_difference"] == 0.0 + assert all( + rep["z"] == pytest.approx(0.5e-4) + for item in results["samples"] + for rep in item["repetitions"] + ) + (row,) = results["summary"] + assert row["rounds"] == 2 + assert row["same_work"] + # The same interpreter and build repeat the same operations exactly. + assert row["field_difference"] == 0.0 assert {item["python_executable"] for item in results["provenance"]["baseline"]} assert read_record(out / "plan.json")["harness_files"] diff --git a/tests/test_benchmark_cases.py b/tests/test_benchmark_cases.py new file mode 100644 index 00000000..91033edf --- /dev/null +++ b/tests/test_benchmark_cases.py @@ -0,0 +1,158 @@ +# SPDX-FileCopyrightText: 2026 John C. Travers and contributors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Check the benchmark cases' definitions without timing them (SEP 38).""" + +import numpy as np +import pytest +from benchmarks.builders import build, capillary, gnlse +from benchmarks.records import SuiteError +from benchmarks.registry import CASES, SEED, STEPPERS, select + +from solprop.driver.simulation import CAPILLARY_RTOL, DEFAULT_RTOL +from solprop.exceptions import MaterialRangeWarning +from solprop.nonlinear.responses import PlasmaResponse +from solprop.stepper import LowStorageRungeKutta, Multistep, RungeKutta + + +def configuration(name, stepper="default", fraction=1.0): + return CASES[name].configuration(stepper, 1, "numpy_ref", fraction) + + +@pytest.mark.parametrize( + ("name", "integrator", "rtol"), + [ + ("gnlse_raman_shock", RungeKutta, DEFAULT_RTOL), + ("rdw", Multistep, CAPILLARY_RTOL), + ("he_rdw_averaged", Multistep, CAPILLARY_RTOL), + ], +) +def test_cases_run_with_the_interface_defaults_and_a_fixed_seed(name, integrator, rtol): + simulation = build(configuration(name)) + assert isinstance(simulation.settings.integrator, integrator) + assert simulation.settings.controller is not None + assert simulation.settings.controller.rtol == rtol + assert simulation.noise is not None + assert simulation.noise.seed == SEED + assert simulation.noise.model == "input" + + +@pytest.mark.parametrize( + ("stepper", "integrator"), + [ + ("multistep", Multistep), + ("multistep-interaction", Multistep), + ("rk", RungeKutta), + ], +) +def test_a_stepper_override_replaces_the_integrator_only(stepper, integrator): + default = build(configuration("rdw")) + chosen = build(configuration("rdw", stepper)) + assert isinstance(chosen.settings.integrator, integrator) + assert chosen.settings.controller is not None + assert default.settings.controller is not None + assert chosen.settings.controller.rtol == default.settings.controller.rtol + assert chosen.settings.controller.atol == default.settings.controller.atol + if stepper == "multistep-interaction": + assert isinstance(chosen.settings.integrator, Multistep) + assert chosen.settings.integrator.formulation == "interaction" + + +def test_runs_save_evenly_to_their_stop_and_a_fraction_stops_early(): + case = CASES["argon_blueshift"] + full = build(configuration("argon_blueshift")) + np.testing.assert_allclose( + full.settings.save_planes, np.linspace(0, case.length, case.planes + 1)[1:] + ) + short = build(configuration("argon_blueshift", fraction=0.25)) + assert short.settings.save_planes[-1] == pytest.approx(case.length / 4) + assert short.grid.length == case.length + + +def test_fixed_step_cases_run_their_schedule_and_refuse_overrides(): + case = CASES["kerr_beam_192"] + assert case.steppers == ("default",) + with pytest.raises(SuiteError, match="admits only default"): + case.configuration("rk", 1) + # The case keeps SEP 28's ultraviolet extrapolation of the silica fit. + with pytest.warns(MaterialRangeWarning, match="Malitson"): + simulation = build(configuration("kerr_beam_192")) + assert isinstance(simulation.settings.integrator, LowStorageRungeKutta) + schedule = simulation.settings.schedule + assert schedule is not None + assert len(schedule.steps) == 11 + np.testing.assert_allclose(schedule.edges[-1], case.length, rtol=1e-12) + assert simulation.settings.fields == "planes" + + +@pytest.mark.parametrize( + ("resolved", "averaged"), + [ + ("rdw_one_mode", "he_rdw_averaged"), + ("argon_blueshift", "argon_blueshift_averaged"), + ("h2_raman_6ps", "h2_raman_6ps_averaged"), + ], +) +def test_averaging_is_a_distinct_route_with_the_original_grid(resolved, averaged): + first, second = capillary(resolved), capillary(averaged) + assert first.settings.quadrature is not None + assert second.settings.quadrature is None + assert second.settings.components is None + # A route change must not change the launch, guide or temporal axes. + assert first.grid.length == second.grid.length + np.testing.assert_array_equal(first.grid.omega, second.grid.omega) + assert first.settings.stats != second.settings.stats + if resolved == "h2_raman_6ps": + assert not any( + isinstance(response, PlasmaResponse) for response in first.responses + ) + + +def test_the_envelope_case_differs_from_its_field_twin_only_in_the_grid(): + field = capillary("argon_blueshift") + envelope = capillary("argon_blueshift_envelope") + assert field.grid.length == envelope.grid.length + assert type(field.grid).__name__ == "FieldGrid" + assert type(envelope.grid).__name__ == "EnvelopeGrid" + + +def test_the_tracking_set_is_the_default_selection(): + tracked = select() + assert [case.name for case in tracked] == [ + "gnlse_raman_shock", + "gnlse_soliton", + "rdw", + "he_rdw_averaged", + "argon_blueshift", + "argon_blueshift_averaged", + "argon_blueshift_envelope", + "h2_raman_averaged", + "ctc", + "radial_filament_256", + "grin_nonlinear", + "kerr_beam_384", + "kerr_beam_192", + ] + # Every tracked width has a cost, so the default plan has an estimate. + assert all( + case.tracked and case.cost[0 if width == "1" else 1] is not None + for case in tracked + for width in case.widths + ) + assert {case.name for case in select(all_cases=True)} == set(CASES) - {"smoke"} + assert len({case.digest for case in CASES.values()}) == len(CASES) + assert STEPPERS[0] == "default" + + +def test_selection_and_builders_refuse_unknown_names(): + with pytest.raises(SuiteError, match="unknown case"): + select(("absent",)) + with pytest.raises(SuiteError, match="unknown set"): + select(sets=("absent",)) + with pytest.raises(SuiteError, match="fraction"): + CASES["rdw"].configuration("default", 1, fraction=0.0) + with pytest.raises(SuiteError, match="not implemented"): + capillary("absent") + with pytest.raises(SuiteError, match="not implemented"): + gnlse("absent") diff --git a/tests/test_benchmark_coverage.py b/tests/test_benchmark_coverage.py deleted file mode 100644 index c98e10be..00000000 --- a/tests/test_benchmark_coverage.py +++ /dev/null @@ -1,309 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Check native report semantics and failed-profile refusal without a campaign.""" - -import csv -import json -import os -import subprocess -import sys -from dataclasses import dataclass, replace -from pathlib import Path -from types import SimpleNamespace - -import pytest -from benchmarks.__main__ import parser_for -from benchmarks.coverage_reports import ( - capability_tags, - coverage_tables, - project_file, - source_digest, -) -from benchmarks.native_coverage import ( - CoverageOptions, - command, - profile_groups, - read_labels, - render_report, - toolchain, - verified_profiles, -) -from benchmarks.records import SuiteError, file_hash -from benchmarks.worker import runtime - - -def test_llvm_views_preserve_hits_misses_and_exclude_dependency_templates(tmp_path): - root = tmp_path / "native" - source = root / "src/core.cpp" - source.parent.mkdir(parents=True) - source.write_text("hit\nmiss\n", encoding="utf-8") - dependency = root / "build/_deps/kokkos.cpp" - metrics = { - key: {"count": 2, "covered": 1} - for key in ("lines", "functions", "instantiations", "regions", "branches") - } - exported = { - "type": "llvm.coverage.json.export", - "data": [ - { - "files": [ - { - "filename": str(p), - "summary": metrics, - "branches": [[1, 1, 1, 5, 3, 0, 0, 0, 4]], - } - for p in (source, dependency) - ], - "functions": [ - { - "name": name, - "count": count, - "filenames": [str(p)], - "regions": [[1, 1, 1, 5, count, 0, 0, 0]], - } - for name, count, p in ( - ("hit", 3, source), - ("miss", 0, source), - ("dependency", 100, dependency), - ) - ], - } - ], - } - lcov = ( - f"SF:{source}\nDA:1,3\nDA:2,0\nend_of_record\n" - f"SF:{dependency}\nDA:1,100\nend_of_record\n" - ) - output = tmp_path / "report" - output.mkdir() - result = coverage_tables(exported, lcov, root, output) - assert result["totals"]["lines"] == {"count": 2, "covered": 1, "percent": 50} - for name in ("lines", "functions", "branches"): - with (output / f"{name}.csv").open(encoding="utf-8") as stream: - rows = list(csv.DictReader(stream)) - assert {row["file"] for row in rows} == {"src/core.cpp"} - assert result["uninstantiated"]["count"] is None - assert result["compiled_out"]["count"] is None - with pytest.raises(SuiteError, match="no mapped"): - coverage_tables(exported, "", root, output) - with pytest.raises(SuiteError, match="unsupported"): - coverage_tables({}, lcov, root, output) - - -def test_sources_and_profile_receipts_bind_the_exact_build(tmp_path): - root = tmp_path / "native" - root.mkdir() - with pytest.raises(SuiteError, match="no project"): - source_digest(root) - source = root / "include/solprop_native/core.hpp" - source.parent.mkdir(parents=True) - source.write_text("first", encoding="utf-8") - original = source_digest(root) - source.write_text("changed", encoding="utf-8") - assert source_digest(root) != original - assert project_file(str(source), root) == "include/solprop_native/core.hpp" - assert project_file(str(root / "src/coverage_labels.cpp"), root) is None - assert project_file(str(tmp_path / "other.cpp"), root) is None - assert project_file(str(root / "src/python.py"), root) is None - profile = tmp_path / "123-module.profraw" - profile.write_bytes(b"profile") - row = { - "status": "ok", - "extension_hash": "binary", - "profiles": [{"path": str(profile), "sha256": file_hash(profile)}], - } - assert verified_profiles([row, row, {"status": "failed"}], "binary") == [profile] - with pytest.raises(SuiteError, match="different binary"): - verified_profiles([row], "other") - with pytest.raises(SuiteError, match="no raw profiles"): - verified_profiles([row | {"profiles": []}], "binary") - profile.write_bytes(b"changed") - with pytest.raises(SuiteError, match="changed"): - verified_profiles([row], "binary") - - -def test_case_set_union_does_not_turn_failures_into_coverage(): - rows = [ - {"status": "ok", "configuration": {"case": "rdw"}}, - {"status": "ok", "configuration": {"case": "ctc"}}, - {"status": "failed", "configuration": {"case": "line"}}, - ] - groups = profile_groups(rows) - assert groups["union"] == rows[:2] - assert groups["sets/core"] == rows[:2] - assert groups["cases/rdw"] == rows[:1] - assert "cases/line" not in groups - assert "tensor-raman" in capability_tags("vector_raman") - assert "general-modal" in capability_tags("grin_linear") - assert "rk4ip" in capability_tags("kerr_beam_192") - - -def test_tool_versions_and_failure_logs_are_explicit(tmp_path, monkeypatch): - for name in ("llvm-cov", "llvm-profdata"): - (tmp_path / name).write_text("tool", encoding="utf-8") - monkeypatch.setattr( - subprocess, - "run", - lambda *args, **kwargs: subprocess.CompletedProcess( - [], 0, "LLVM version 23.1.3", "" - ), - ) - assert toolchain(tmp_path, "Clang 23.1.3")["llvm-cov"]["version"] == "23.1.3" - with pytest.raises(SuiteError, match="must match"): - toolchain(tmp_path, "Clang 22.0.0") - with pytest.raises(SuiteError, match="no identifiable"): - toolchain(tmp_path, "unknown") - assert command(["tool"], tmp_path, "success") == "LLVM version 23.1.3" - monkeypatch.setattr( - subprocess, - "run", - lambda *args, **kwargs: subprocess.CompletedProcess( - [], 1, "", "corrupt profile" - ), - ) - with pytest.raises(SuiteError, match="failed"): - command(["tool"], tmp_path, "merge") - assert (tmp_path / "merge.stderr.log").read_text( - encoding="utf-8" - ) == "corrupt profile" - - -def test_binary_and_source_changes_refuse_report_before_merging(tmp_path): - binary = tmp_path / "native.so" - binary.write_bytes(b"native") - options = CoverageOptions( - tmp_path / "output", tmp_path, tmp_path, "python", 30, None - ) - with pytest.raises(SuiteError, match="snapshot changed"): - render_report(options, "union", [], {"path": str(binary), "sha256": "other"}) - source = tmp_path / "src/core.cpp" - source.parent.mkdir() - source.write_text("source", encoding="utf-8") - record = {"path": str(binary), "sha256": file_hash(binary), "source_hash": "other"} - with pytest.raises(SuiteError, match="source changed"): - render_report(replace(options, output=tmp_path / "second"), "union", [], record) - - -@pytest.mark.parametrize( - "value", ["", "broken", "{}", '{"kind":"for","label":"x","count":0}'] -) -def test_bad_labels_are_unknown_not_zero(tmp_path, value): - path = tmp_path / "labels.jsonl" - path.write_text(value, encoding="utf-8") - with pytest.raises(ValueError): - read_labels(path) - - -def test_coverage_is_an_explicit_command_and_labels_are_lossless(tmp_path): - args = parser_for().parse_args( - [ - "coverage", - "--case", - "smoke", - "--native-root", - ".", - "--llvm-bin", - ".", - "--out", - "output", - ] - ) - assert args.threads == "1" - assert not hasattr(args, "mode") - assert not hasattr(args, "repetitions") - label = {"kind": "parallel_for", "label": 'a\\b"\n', "count": 2} - path = tmp_path / "labels.jsonl" - path.write_text(json.dumps(label) + "\n", encoding="utf-8") - assert read_labels(path) == [label] - - -@pytest.mark.parametrize("instrumented", [False, True]) -def test_timing_and_coverage_refuse_the_wrong_build_before_construction( - monkeypatch, instrumented -): - @dataclass - class Build: - coverage_instrumented: bool = instrumented - - package = SimpleNamespace(build_info=Build, _core=SimpleNamespace()) - monkeypatch.setitem(sys.modules, "solprop_native", package) - with pytest.raises(SuiteError, match="instrumented"): - runtime("native", coverage=not instrumented) - - -@pytest.mark.medium -def test_real_clang_known_hits_misses_and_invalid_profile(tmp_path): - location = os.environ.get("SOLPROP_COVERAGE_LLVM") - if not location: - pytest.skip( - "explicit matching LLVM toolchain required for coverage integration" - ) - tools = Path(location) - root = tmp_path / "native" - source = root / "src/core.cpp" - source.parent.mkdir(parents=True) - source.write_text( - "int choose(int n) {\n if (n > 1) return 2;\n return 1;\n}\n" - "int unused(int n) { return n * 4; }\n" - "int main(int argc, char**) { return choose(argc) == 1 ? 0 : 1; }\n", - encoding="utf-8", - ) - binary = tmp_path / "fixture" - command( - [ - str(tools / "clang++"), - "-O1", - "-fprofile-instr-generate", - "-fcoverage-mapping", - "-fprofile-update=atomic", - str(source), - "-o", - str(binary), - ], - tmp_path, - "compile", - ) - profile = tmp_path / "fixture.profraw" - subprocess.run( - [str(binary)], env={**os.environ, "LLVM_PROFILE_FILE": str(profile)}, check=True - ) - digest = file_hash(binary) - row = { - "id": "fixture", - "status": "ok", - "extension_hash": digest, - "profiles": [{"path": str(profile), "sha256": file_hash(profile)}], - } - options = CoverageOptions( - tmp_path / "output", root, tools, sys.executable, 60, None - ) - summary = render_report( - options, - "union", - [row], - {"path": str(binary), "sha256": digest, "source_hash": source_digest(root)}, - ) - assert summary["totals"]["functions"]["count"] == 3 - assert summary["totals"]["functions"]["covered"] == 2 - assert ( - 0 - < summary["totals"]["branches"]["covered"] - < summary["totals"]["branches"]["count"] - ) - assert (options.output / "reports/union/html/index.html").is_file() - bad = tmp_path / "bad.profraw" - bad.write_bytes(b"invalid profile") - with pytest.raises(SuiteError, match="failed"): - command( - [ - str(tools / "llvm-profdata"), - "merge", - str(bad), - "-o", - str(tmp_path / "bad.profdata"), - ], - tmp_path, - "bad-profile", - ) diff --git a/tests/test_benchmark_exercises.py b/tests/test_benchmark_exercises.py index 4fc53292..b0c0518a 100644 --- a/tests/test_benchmark_exercises.py +++ b/tests/test_benchmark_exercises.py @@ -2,164 +2,57 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -"""Exercise batch reset and observation boundaries on short CPU problems.""" +"""Check that the backend exercises build, and that a worker's repeats agree.""" -import importlib.util -from dataclasses import replace +import contextlib import numpy as np import pytest -from benchmarks.batching import construct -from benchmarks.builders import build, fixed_schedule +from benchmarks.builders import build from benchmarks.exercises import exercise, plasma -from benchmarks.planning import Selection, matrix from benchmarks.records import SuiteError -from benchmarks.registry import ADAPTIVE, CASES, FIXED -from benchmarks.system import Profile -from benchmarks.worker import measure, prepare, reset_columns, retained_fields +from benchmarks.registry import CASES, select +from benchmarks.worker import measure -from solprop.backends import select -from solprop.stats import StatsSpec +from solprop.exceptions import MaterialRangeWarning @pytest.mark.medium -@pytest.mark.parametrize("backend", ["numpy_ref", "numba_cpu", "native"]) -def test_batch_reuses_workspace_and_resets_each_member(backend): - if backend == "native" and importlib.util.find_spec("solprop_native") is None: - pytest.skip("optional native CPU backend is not installed") - configuration = CASES["batch"].measurement(FIXED) | {"backend": backend} - case = construct(configuration) - try: - ledger = case.engine.memory_ledger() - first = case.measure(float(configuration["stop"])) - original = case.observe() - case.reset() - assert case.engine.memory_ledger() == ledger - second = case.measure(float(configuration["stop"])) - repeated = case.observe() - assert first["accepted"] == second["accepted"] == 10 - assert first["member_source_evaluations"] == second["member_source_evaluations"] - assert first["source_evaluations"] == sum(first["member_source_evaluations"]) - for before, after in zip(original, repeated, strict=True): - # Matched ten-step computations: 32 epsilon per step covers stage - # reductions and field conversions; reset need not be bitwise. - np.testing.assert_allclose( - after.fields(convention="fft"), - before.fields(convention="fft"), - rtol=320 * np.finfo(float).eps, - atol=0, - ) - assert after.step_stat_names == before.step_stat_names == ("energy",) - np.testing.assert_array_equal( - after["step_stats/step"], before["step_stats/step"] - ) - np.testing.assert_allclose( - after["stats/energy"], - before["stats/energy"], - rtol=320 * np.finfo(float).eps, - atol=0, - ) - assert original[1]["stats/energy"][0, 0] > original[0]["stats/energy"][0, 0] - finally: - case.engine.close() - - -@pytest.mark.medium -def test_statistics_only_planes_avoid_field_saves_but_keep_periodic_reset_evidence(): - configuration = CASES["smoke"].measurement(ADAPTIVE[1]) | { - "backend": "numpy_ref", - "fields": "planes", - "observations": 4, - } - simulation = build(configuration) - simulation.settings = replace( - simulation.settings, - stats=StatsSpec(names=("energy",), periodic=("energy",), period=1), +@pytest.mark.parametrize( + "name", [case.name for case in select(sets=("backend-exercise",))] +) +def test_every_exercise_builds_and_translates(name): + case = CASES[name] + expected = ( + # optmat's H2 Raman response, used below its validated density. + pytest.warns(MaterialRangeWarning, match="validated density range") + if name == "vector_raman" + else contextlib.nullcontext() ) - problem = simulation.translate() - engine = select("numpy_ref", problem, grid=simulation.grid, modes=simulation.modes) - try: - output = prepare(simulation, problem, engine, configuration) - result = measure(engine, problem, configuration, output) - assert result["endpoint"] == configuration["stop"] - assert len(output["z"]) == 4 and len(output["stats/energy"]) == 4 - assert len(retained_fields(output)) == 1 - columns = reset_columns(output) - assert len(columns["step_stats/energy"]) > 0 - assert len(columns["step_stats/step"]) == len(columns["step_stats/energy"]) - output.close() - finally: - engine.close() + with expected: + simulation = build(case.configuration("default", 1, "numpy_ref")) + problem = simulation.translate() + planes = np.asarray(problem.stepper.save_planes) + assert planes[-1] == pytest.approx(case.length) + assert len(planes) == case.planes -def test_fixed_only_batch_is_visible_in_quick_and_unsupported_tracks_are_explicit(): - profile = Profile("test", 1, 1, (0,), "test") - plan = matrix((CASES["batch"],), profile, Selection(backend="numpy_ref")) - assert len(plan["rows"]) == 1 - assert plan["rows"][0]["configuration"]["track"] == FIXED - assert plan["rows"][0]["status"] == "ready" - other = matrix((CASES["batch"],), profile, Selection(tracks=ADAPTIVE)) - assert all(row["status"] == "excluded" for row in other["rows"]) +def test_exercise_builders_refuse_unknown_names(): for builder in (exercise, plasma): with pytest.raises(SuiteError, match="not implemented"): builder("absent") -def test_short_plasma_schedule_stops_at_the_original_guide_exit(): - case = CASES["envelope_plasma"] - configuration = case.measurement(FIXED) - schedule = fixed_schedule(case, configuration) - assert len(schedule.steps) == 4 - # Four subtraction/addition operations reconstruct the frozen pilot edges. - np.testing.assert_allclose( - np.cumsum(schedule.steps), - np.array([2, 4, 6, 8]) * 1e-6, - rtol=4 * np.finfo(float).eps, - atol=0, - ) - with pytest.raises(SuiteError, match="4 entrance steps"): - fixed_schedule(case, configuration | {"schedule": [0, 8e-6]}) - - @pytest.mark.medium -def test_dense_observations_keep_the_integrator_trajectory_between_planes(): - configuration = CASES["dense_output"].measurement(ADAPTIVE[1]) | { - "backend": "numpy_ref" - } - simulation = build(configuration) - assert simulation.settings.interpolate_planes - assert simulation.grid.length == 1.0 - problem = simulation.translate() - engine = select("numpy_ref", problem, grid=simulation.grid, modes=simulation.modes) - try: - output = prepare(simulation, problem, engine, configuration) - measure(engine, problem, configuration, output) - planes = output["z"] - assert len(planes) == 8 - # Dense observations need not be accepted step endpoints. At least one - # plane lies inside a step, proving this exercises interpolation. - endpoints = output["step_history/z"] + output["step_history/h"] - assert any( - not np.any( - np.isclose(plane, endpoints, rtol=8 * np.finfo(float).eps, atol=0) - ) - for plane in planes[:-1] - ) - calls = engine.metadata()["source_evaluations"] - first = retained_fields(output) - one_plane = configuration | {"observations": 1} - repeated = prepare(simulation, problem, engine, one_plane) - measure(engine, problem, one_plane, repeated) - assert engine.metadata()["source_evaluations"] == calls - # Same discrete march with two observation itineraries: 32 epsilon for - # each of the ten steps plus their amplitude/output conversions. - np.testing.assert_allclose( - retained_fields(repeated)[-1], - first[-1], - rtol=320 * np.finfo(float).eps, - atol=0, - ) - output.close() - repeated.close() - finally: - engine.close() +def test_repeats_restart_from_the_launch_and_take_the_same_steps(tmp_path): + configuration = CASES["noise"].configuration("default", 1, "numpy_ref") + result = measure(configuration, 2, tmp_path / "field.npy") + first, second = result["repetitions"] + assert (first["accepted"], first["rejected"]) == ( + second["accepted"], + second["rejected"], + ) + assert first["z"] == second["z"] == pytest.approx(CASES["noise"].length) + assert result["finite"] + assert result["noise"] == "modified" + assert np.isfinite(np.load(tmp_path / "field.npy")).all() diff --git a/tests/test_benchmark_guides.py b/tests/test_benchmark_guides.py index fb888092..ec37db18 100644 --- a/tests/test_benchmark_guides.py +++ b/tests/test_benchmark_guides.py @@ -2,59 +2,57 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -"""Verify bounded advanced definitions without running a benchmark campaign.""" +"""Check the advanced guide cases without running them.""" + +import warnings import numpy as np import pytest -from benchmarks.builders import build, fixed_schedule +from benchmarks.builders import build from benchmarks.guides import ScaledPermittivity, advanced, coupled from benchmarks.records import SuiteError -from benchmarks.registry import ADAPTIVE, CASES, FIXED, select -from benchmarks.worker import Diagnostics +from benchmarks.registry import CASES, select +from solprop.exceptions import GrazingIncidenceWarning, SolpropWarning from solprop.modes import IndexMaterial -from solprop.stepper import Multistep, RungeKutta +from solprop.stepper import Multistep + + +def configuration(name, stepper="default"): + return CASES[name].configuration(stepper, 1, "numpy_ref") @pytest.mark.medium @pytest.mark.parametrize("name", ["scalar_wall", "vector_wall", "taper", "taper_wall"]) -def test_wall_cases_keep_full_profiles_and_the_original_fixed_itinerary(name): - case = CASES[name] - diagnostics = Diagnostics() - diagnostics.case, diagnostics.phase = name, "construction" - with diagnostics.capture(): - simulation = build(case.measurement(FIXED) | {"backend": "numpy_ref"}) +def test_wall_cases_keep_full_profiles(name): + # The tutorials' short guides under-absorb walk-off, and the walls are near + # the paraxial limit: the warnings SEP 30 reviewed for these cases. + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + simulation = build(configuration(name)) problem = simulation.translate() + assert {type(item.message) for item in caught} == ( + {SolpropWarning} + if name == "taper" + else {SolpropWarning, GrazingIncidenceWarning} + ) assert simulation.grid.length == 2e-4 assert problem.shape[1] == (4 if name == "vector_wall" else 3) assert (simulation.wall is None) == (name == "taper") assert simulation.taper_coupling - schedule = problem.stepper.schedule - assert schedule is not None - assert len(schedule.steps) == 10 - assert diagnostics.records() and all( - row["expected"] for row in diagnostics.records() - ) - if name.startswith("taper"): - # The fixed schedule must land on both original profile corners even - # though the adaptive diagnostic ends during the taper. - edges = schedule.edges - for corner in (6e-5, 1.4e-4): - assert np.min(abs(edges - corner)) <= 10 * np.finfo(float).eps * corner @pytest.mark.medium -@pytest.mark.parametrize("track", [*ADAPTIVE, FIXED]) -def test_linear_grin_launch_is_configurable_before_translation(track): +@pytest.mark.parametrize("stepper", ["default", "multistep"]) +def test_linear_grin_launch_is_configurable_before_translation(stepper): case = CASES["grin_linear"] - simulation = build(case.measurement(track) | {"backend": "numpy_ref"}) + simulation = build(configuration("grin_linear", stepper)) assert simulation.grid.length == 1e-3 waveguide = simulation.configuration["waveguide"] assert isinstance(waveguide, dict) assert waveguide["length"] == case.length - assert isinstance( - simulation.settings.integrator, - Multistep if track == ADAPTIVE[0] else RungeKutta, + assert isinstance(simulation.settings.integrator, Multistep) == ( + stepper == "multistep" ) centre = int(np.argmin(abs(simulation.grid.omega - simulation.grid.omega0))) launch = simulation.initial @@ -64,24 +62,14 @@ def test_linear_grin_launch_is_configurable_before_translation(track): simulation.translate() -def test_advanced_selection_keeps_seeds_windows_and_unexecuted_profile_corners(): +def test_advanced_selection_and_unknown_guides(): cases = select(sets=("advanced",)) assert len(cases) == 8 - assert all(case.status == "implemented" for case in cases) - assert all(0 < case.stop <= case.length for case in cases) - assert CASES["grin_nonlinear"].measurement(ADAPTIVE[0])["noise_seed"] == 41 - case = CASES["moving_solid_guide"] - schedule = fixed_schedule(case, case.measurement(FIXED)) - # Schedule arithmetic may round once per edge; its unexecuted tail still - # has to land on the original taper/waist corner for translation to admit it. - assert ( - np.min(abs(schedule.edges - 5e-3)) - <= len(schedule.steps) * np.finfo(float).eps * case.length - ) - assert schedule.edges[10] < 5e-3 - for builder in (advanced, coupled): - with pytest.raises(SuiteError, match="not implemented"): - builder("absent") + assert CASES["grin_nonlinear"].length == 0.02 + with pytest.raises(SuiteError, match="not implemented"): + advanced("absent", 1.0) + with pytest.raises(SuiteError, match="not implemented"): + coupled("absent") def test_scaled_cladding_preserves_the_material_identity_and_domain(): diff --git a/tests/test_benchmark_publication.py b/tests/test_benchmark_publication.py deleted file mode 100644 index 3d0c3661..00000000 --- a/tests/test_benchmark_publication.py +++ /dev/null @@ -1,162 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Verify public export omissions, failure retention and missing telemetry.""" - -import csv -import dataclasses -import time - -import pytest -from benchmarks.__main__ import main -from benchmarks.collection import Monitor -from benchmarks.planning import Selection, matrix -from benchmarks.publication import export_public, public_record, summaries -from benchmarks.records import ( - SuiteError, - atomic_record, - attempts, - canonical, - read_record, -) -from benchmarks.registry import select -from benchmarks.system import Profile -from benchmarks.views import write_views - - -@pytest.fixture -def local_bundle(tmp_path): - source = tmp_path / "local" - plan = matrix( - select(("smoke",)), - Profile("private-machine", 1, 1, (0,), "topology"), - Selection(backend="numpy_ref"), - ) - plan["environments"] = { - "candidate": { - "solprop_path": "/private/source/solprop", - "python_executable": "/private/python", - "packages": {"numpy": "2.5.2"}, - "source": {"revision": "abc", "location": "/private/source"}, - "build_info": { - "execution_space": "OpenMP", - "compiler_flags": "-I/private/include", - }, - } - } - plan["system"]["hostname"] = "private-host" - plan["options"] = {"references": "/private/reference"} - manifest = {"created": "2026-10-21T12:00:00Z", "plan": plan} - atomic_record(source / "manifest.json", manifest) - for index, status in enumerate(("ok", "failed")): - atomic_record( - source / "attempts" / f"{index}.json", - { - "attempt_id": index, - "sample_id": str(index), - "arm": "candidate", - "round": index, - "configuration_id": plan["rows"][0]["id"], - "status": status, - "phase": "measurement", - "seconds": 1.0 if status == "ok" else None, - "clean": False, - "peak_rss_bytes": None, - "error": "/private/worker crashed on private-host", - "diagnostics": [{"message": "/private/log"}], - "process": {"pid": 123, "command": "/private/command"}, - "repetitions": [{"seconds": 1.0, "source_evaluations": 7}] - if status == "ok" - else [], - }, - ) - return source - - -def test_public_export_omits_private_details_and_preserves_failures( - local_bundle, tmp_path -): - destination = tmp_path / "public" - original = read_record(local_bundle / "manifest.json") - result = export_public(local_bundle, destination, "public-machine") - contents = canonical(result) - assert "private" not in contents.replace("private bundle", "").replace( - "private paths", "" - ) - assert "pid" not in contents - assert result["environments"]["candidate"]["build"]["execution_space"] == "OpenMP" - assert [row["status"] for row in result["samples"]] == ["ok", "failed"] - assert all(row["peak_rss_bytes"] is None for row in result["samples"]) - assert all(row["peak_rss_unavailable_reason"] for row in result["samples"]) - assert read_record(local_bundle / "manifest.json") == original - summary = summaries(result["samples"]) - assert summary[0]["count"] == 1 and summary[0]["mad"] is None - assert summary[1]["count"] == 0 and summary[1]["median"] is None - assert all(row["failed_samples"] == 1 for row in summary) - for name in ( - "samples", - "summary", - "configurations", - "cases", - "runs", - "system", - "repetitions", - ): - assert (destination / f"{name}.csv").exists() - with pytest.raises(SuiteError, match="empty or new"): - export_public(local_bundle, destination, "public-machine") - with pytest.raises(SuiteError, match="must differ"): - export_public(local_bundle, local_bundle, "public-machine") - - -def test_export_command_and_public_alias_validation(local_bundle, tmp_path): - assert ( - main( - [ - "export", - str(local_bundle), - "--out", - str(tmp_path / "export"), - "--alias", - "cpu", - ] - ) - == 0 - ) - manifest = read_record(local_bundle / "manifest.json") - with pytest.raises(SuiteError, match="public alias"): - public_record(manifest, [], "../private") - del manifest["plan"]["system"] - assert public_record(manifest, [], "cpu")["hardware_unavailable_reason"] - - -def test_missing_process_telemetry_is_unknown_not_zero(): - monitor = Monitor(time.monotonic(), (0,)) - monitor.sample(None, (60.0, None)) - record = dataclasses.asdict(monitor) - assert record["own_cpu"] is None - assert record["sampled_peak"] is None - assert record["missed_samples"] == 1 - - -def test_local_views_link_attempts_and_explain_missing_telemetry(local_bundle): - rows = attempts(local_bundle) - rows[0]["process"] = {"pid": 123, "created": 123.5, "cpu_seconds": 0.25} - rows[0]["provenance"] = { - "libraries": [{"internal_api": "openblas", "num_threads": 1}] - } - write_views(local_bundle, rows) - - def table(name): - with (local_bundle / f"{name}.csv").open(encoding="utf-8") as source: - return list(csv.DictReader(source)) - - assert table("runs")[0]["collection_seconds_reason"] == "collection incomplete" - assert table("configurations")[0]["case"] == "smoke" - assert table("allocations")[0]["cpu"] == "0" - assert table("processes")[0]["cpu_seconds"] == "0.25" - assert table("libraries")[0]["unavailable_reason"] == "no supported pool observed" - assert table("libraries")[1]["scope"] == "after measurement" - assert table("coverage")[0]["status"] == "not-collected" - assert table("system")[0]["run_id"] == table("runs")[0]["run_id"] diff --git a/tests/test_benchmark_references.py b/tests/test_benchmark_references.py deleted file mode 100644 index 963a8739..00000000 --- a/tests/test_benchmark_references.py +++ /dev/null @@ -1,456 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Verify reference invalidation and retained diagnostics on synthetic fields.""" - -import dataclasses -import os -import sys - -import numpy as np -import pytest -from benchmarks import collection, system, worker -from benchmarks.builders import fixed_schedule -from benchmarks.qualification import ( - Qualification, - envelope_failures, - recheck_row, - reference_lineage, - reviewed_reference, -) -from benchmarks.records import SuiteError, atomic_record, canonical, file_hash -from benchmarks.references import ( - ADVANCED_ENVELOPES, - EXERCISE_ENVELOPES, - MODAL_ENVELOPES, - ReferenceReceipt, - approved_envelope, - check, - field_errors, - measurement_identity, - reference_path, - spectral_errors, -) -from benchmarks.registry import ADAPTIVE, CASES, FIXED, KERR - -from solprop.constants import C -from solprop.output import MemoryOutput - - -def test_reset_evidence_retains_undefined_statistics_without_nonfinite_json( - monkeypatch, -): - expected = { - "stats/omega0": np.array([1.0, np.nan]), - "stats/fwhm_t": np.array([np.nan]), - } - monkeypatch.setattr(worker, "reset_columns", lambda _: expected) - witness = worker.ResetWitness(np.ones((1, 1), dtype=complex), expected) - output = MemoryOutput() - result = worker.reset_differences(witness, output) - assert result["stats/omega0"]["same_undefined"] - assert result["stats/omega0"]["compared_finite_count"] == 1 - assert result["stats/fwhm_t"]["absolute_l2"] is None - canonical(result) - monkeypatch.setattr( - worker, - "reset_columns", - lambda _: { - "stats/omega0": np.array([1.0, 2.0]), - "stats/fwhm_t": np.array([np.nan]), - }, - ) - difference = worker.reset_differences(witness, output)["stats/omega0"] - assert not difference["same_undefined"] - for name, value in (("stats/energy", np.inf), ("step_history/norm", np.nan)): - bad = {name: np.array([value])} - monkeypatch.setattr(worker, "reset_columns", lambda _, values=bad: values) - with pytest.raises(SuiteError, match="non-finite reset"): - worker.reset_differences(worker.ResetWitness(witness.fields, bad), output) - - -@pytest.fixture -def receipt(tmp_path): - configuration = {"case": "smoke", "case_hash": "test-case", "track": "adaptive-rk"} - fields = np.ones((2, 1, 1, 3), dtype=complex) - np.save(tmp_path / "fields.npy", fields) - np.save(tmp_path / "band.npy", np.array([True, True, False])) - atomic_record(tmp_path / "evidence.json", {"status": "synthetic-test"}) - record = ReferenceReceipt( - revision=1, - case_hash=configuration["case_hash"], - measurement_hash=measurement_identity(configuration), - fields_file="fields.npy", - fields_hash=file_hash(tmp_path / "fields.npy"), - band_file="band.npy", - band_hash=file_hash(tmp_path / "band.npy"), - accuracy_bound=1e-5, - refinement_floor=1e-9, - evidence_file="evidence.json", - evidence_hash=file_hash(tmp_path / "evidence.json"), - protocol="synthetic-test", - qualified_for_adoption=True, - ) - path = tmp_path / "reference.json" - atomic_record(path, dataclasses.asdict(record)) - return configuration, fields, record, path - - -def test_reference_envelope_and_identity_invalidation(receipt): - configuration, fields, record, path = receipt - # Multiplicative perturbations have exactly their magnitude as relative L2 - # error, up to the rounding in the three-bin norm. - assert check(fields * (1 + 1e-6), configuration, path)["status"] == "passed" - failed = check(fields * (1 + 1e-4), configuration, path) - assert failed["status"] == "invalid" - assert not failed["qualified_for_adoption"] - with pytest.raises(SuiteError, match="identity"): - check(fields, {**configuration, "case_hash": "revised"}, path) - atomic_record(path, dataclasses.asdict(dataclasses.replace(record, revision=2))) - with pytest.raises(SuiteError, match="revision"): - check(fields, configuration, path) - - -def test_later_samples_reuse_only_validated_arm_and_reference(receipt): - configuration, fields, record, path = receipt - validated = worker.accuracy(fields, configuration, {"reference": str(path)}) - sample = { - "configuration_id": "configuration", - "arm": "candidate", - "sample_id": "first", - "status": "ok", - "accuracy": validated, - } - previous = collection.prior_validation( - [sample], "configuration", "candidate", str(path) - ) - assert previous is not None - assert previous["validation_sample_id"] == "first" - assert "plane_errors" not in previous # These were not measured again. - request = {"prior_validation": previous} - assert worker.accuracy(fields, configuration, request)["status"] == "passed" - assert worker.accuracy(fields[:, :0], configuration, request)["status"] == "invalid" - assert ( - worker.accuracy(fields * np.nan, configuration, request)["status"] == "invalid" - ) - for changed in ( - {"arm": "baseline"}, - {"configuration_id": "different"}, - {"status": "invalid"}, - {"accuracy_overridden": True}, - {"accuracy": {"status": "not-qualified"}}, - ): - assert ( - collection.prior_validation( - [sample | changed], "configuration", "candidate", str(path) - ) - is None - ) - repeated = sample | {"sample_id": "second", "accuracy": previous} - cached = collection.prior_validation( - [repeated], "configuration", "candidate", str(path) - ) - assert cached is not None - assert cached["validation_sample_id"] == "first" - atomic_record( - path, dataclasses.asdict(dataclasses.replace(record, policy_revision=2)) - ) - assert ( - collection.prior_validation([sample], "configuration", "candidate", str(path)) - is None - ) - path.write_text("broken", encoding="utf-8") - assert ( - collection.prior_validation([sample], "configuration", "candidate", str(path)) - is None - ) - assert ( - collection.prior_validation([sample], "configuration", "candidate", None) - is None - ) - - -@pytest.mark.parametrize("artifact", ["fields", "band", "evidence"]) -def test_reference_artifact_hashes_are_checked(receipt, artifact): - configuration, fields, record, path = receipt - (path.parent / getattr(record, f"{artifact}_file")).write_bytes(b"changed") - with pytest.raises(SuiteError, match="hash mismatch"): - check(fields, configuration, path) - - -def test_reference_path_cannot_escape_bundle(tmp_path): - with pytest.raises(SuiteError, match="escapes"): - reference_path( - {"fields_file": "../outside.npy", "fields_hash": "unused"}, - "fields", - tmp_path, - ) - - -def test_field_metric_refuses_invalid_or_zero_reference(): - truth = np.ones((1, 2), complex) - for fields, reference in ( - (truth, truth[:, :1]), - (truth, truth * 0), - (truth * np.nan, truth), - ): - with pytest.raises(SuiteError): - field_errors(fields, reference) - - -def test_spectral_diagnostics_retain_weak_bins_uv_and_parseval_energy(): - truth = np.array([[[[1.0, 1e-2, 1e-3]]]], dtype=complex) - omega = 2 * np.pi * C / np.array([100e-9, 125e-9, 200e-9]) - epsilon = 1e-6 - measured = spectral_errors(truth * (1 + epsilon), truth, omega, "rdw") - assert measured["weak"]["selected_bins"] == 2 - assert measured["uv"]["selected_bins"] == 2 - # These synthetic scalings give analytic norm and energy differences; - # 8 epsilon_machine bounds the small sums and subtraction of unit energies. - rounding = 8 * np.finfo(float).eps - assert measured["weak"]["value"] == pytest.approx(epsilon, abs=rounding) - assert measured["uv"]["value"] == pytest.approx(epsilon, abs=rounding) - assert measured["energy"]["value"] == pytest.approx( - 2 * epsilon + epsilon**2, abs=rounding - ) - missing = spectral_errors(truth, truth, omega / 10, "rdw") - assert missing["uv"]["value"] is None - assert missing["uv"]["reason"] - with pytest.raises(SuiteError, match="contract"): - spectral_errors(truth, truth, omega[:1], "ctc") - with pytest.raises(SuiteError, match="zero-norm"): - spectral_errors(truth * 0, truth * 0, omega, "ctc") - - -def test_refinement_floor_includes_retained_bands(): - errors = {"coarse": 1e-10, "matched": 1e-7, "candidate": 1e-7} - metrics = {"coarse": {"uv": {"value": 2e-5}}, "candidate": {"uv": {"value": 3e-5}}} - failures = envelope_failures(errors, metrics, 1e-4, adaptive=True) - assert len(failures) == 1 - assert "coarse uv" in failures[0] - assert not envelope_failures(errors, metrics, 1e-4, adaptive=False) - - -def test_approved_limits_apply_only_to_reviewed_measurements(): - limits = (("rdw", 1e-4), ("ctc", 1e-5), ("radial_filament_256", 1e-4)) - for case, bound in ( - *limits, - *( - (item.case, 1e-4) - for item in (*MODAL_ENVELOPES, *ADVANCED_ENVELOPES, *EXERCISE_ENVELOPES) - ), - ): - for track in ADAPTIVE: - configuration = CASES[case].measurement(track) - policy = approved_envelope(configuration) - assert policy is not None - assert policy.bound == bound - assert approved_envelope({**configuration, "stop": 0.5}) is None - # A later workload revision never inherits the review. - later = {**configuration, "revision": configuration["revision"] + 1} - assert approved_envelope(later) is None - if FIXED in CASES[case].tracks: - assert approved_envelope(CASES[case].measurement(FIXED)) is None - - -def test_energy_remains_diagnostic_during_qualification(): - errors = {"coarse": 1e-10, "matched": 1e-7, "candidate": 1e-7} - metrics = {"candidate": {"energy": {"value": 0.1, "role": "diagnostic"}}} - assert not envelope_failures(errors, metrics, 1e-4, adaptive=True) - - -@pytest.mark.parametrize( - "name,track,count", - [ - ("rdw", FIXED, 10), - ("ctc", FIXED, 10), - ("radial_filament_256", FIXED, 10), - ("kerr_beam_384", KERR, 11), - ], -) -def test_fixed_schedule_has_declared_entrance_count_without_a_zero_tail( - name, track, count -): - case = CASES[name] - configuration = case.measurement(track) - schedule = fixed_schedule(case, configuration) - # The saved endpoint is a sum of the prescribed steps: a handful of additions - # permits 8 eps of relative rounding, never a tolerance-sized extra step. - rounding = 8 * np.finfo(float).eps - assert schedule.edges[count] == pytest.approx(configuration["stop"], rel=rounding) - assert schedule.z_end == pytest.approx(case.length, rel=rounding) - if track == KERR: - assert schedule.n_steps == 11 - assert schedule.edges[1] == pytest.approx(configuration["start"], rel=rounding) - - -def test_reference_revisions_preserve_prior_evidence_and_advance_policy( - receipt, tmp_path -): - configuration, _, record, path = receipt - options = Qualification( - tmp_path / "replacement", - 60, - None, - 2e-5, - supersedes=path.parent, - revision_note="Reviewed wider accuracy margin", - ) - # The fixture alias must match the named bundle receipt used by the CLI. - named = path.with_name("smoke-adaptive-rk.json") - named.write_bytes(path.read_bytes()) - original = named.read_bytes() - lineage = reference_lineage(configuration, options) - assert lineage["reference_revision"] == 2 - assert lineage["policy_revision"] == 2 - assert lineage["supersedes"] - assert named.read_bytes() == original - with pytest.raises(SuiteError, match="new policy revision"): - reference_lineage( - configuration, dataclasses.replace(options, policy_revision=1) - ) - with pytest.raises(SuiteError, match="revision-note"): - reference_lineage( - configuration, dataclasses.replace(options, revision_note=None) - ) - with pytest.raises(SuiteError, match="positive"): - reference_lineage( - configuration, dataclasses.replace(options, reference_revision=0) - ) - with pytest.raises(SuiteError, match="different measurement"): - reference_lineage({**configuration, "stop": 1.0}, options) - assert record.accuracy_bound == 1e-5 - - -@pytest.mark.parametrize("reviewed", [False, True]) -def test_recheck_changes_policy_without_propagation(receipt, tmp_path, reviewed): - configuration, fields, record, path = receipt - evidence_path = path.parent / record.evidence_file - atomic_record( - evidence_path, - { - "errors_against_tight": { - "coarse": 1e-10, - "matched": 1e-7, - "candidate": 1e-7, - }, - "spectral_metrics_against_tight": {}, - "records": { - name: { - "status": "ok", - "reset_evidence": { - "plane_errors": [1e-12], - "history_statistics_noise": {"same_columns": True}, - "fresh_counts": {"accepted": 10}, - "reset_counts": [{"accepted": 10}], - }, - } - for name in ("tight", "coarse", "matched", "candidate") - }, - }, - ) - updated = dataclasses.replace(record, evidence_hash=file_hash(evidence_path)) - named = path.with_name("smoke-adaptive-rk.json") - atomic_record(named, dataclasses.asdict(updated)) - before = named.read_bytes() - options = Qualification( - tmp_path / "rechecked", - 60, - None, - 2e-5, - revision_note="Margin review", - reviewed=reviewed, - ) - report = recheck_row({"configuration": configuration}, path.parent, options) - assert report["status"] == "envelope-passed" - result = check(fields, configuration, options.output / named.name) - assert result["status"] == "passed" - assert result["reference_revision"] == 2 - assert result["policy_revision"] == 2 - assert result["qualified_for_adoption"] is reviewed - assert named.read_bytes() == before - - -def test_reviewed_reference_requires_retained_reset_evidence(): - reset = { - "plane_errors": [1e-12], - "history_statistics_noise": {"same_columns": True}, - "fresh_counts": {"accepted": 10}, - "reset_counts": [{"accepted": 10}], - } - records = { - name: {"status": "ok", "reset_evidence": reset} - for name in ("tight", "coarse", "matched", "candidate") - } - assert reviewed_reference({"records": records}, 1e-4) - with pytest.raises(SuiteError, match="positive accuracy"): - reviewed_reference({"records": records}, None) - for bad in ( - {"status": "failed"}, - {"status": "ok", "reset_evidence": {**reset, "plane_errors": [1e-3]}}, - {"status": "ok", "reset_evidence": {**reset, "reset_counts": []}}, - ): - with pytest.raises(SuiteError, match="candidate lacks"): - reviewed_reference({"records": {**records, "candidate": bad}}, 1e-4) - - -@pytest.mark.medium -def test_real_worker_accuracy_override_retains_failed_check_and_timing(tmp_path): - configuration = { - **CASES["smoke"].measurement(ADAPTIVE[1]), - "backend": "numpy_ref", - "threads": 1, - } - environment = system.thread_environment(1, "budget", dict(os.environ)) - original = tmp_path / "original.npy" - initial = collection.launch( - sys.executable, - {"configuration": configuration, "save_fields": str(original)}, - environment, - tmp_path / "initial", - (60, 2 * 2**30), - system.eligible_cpus(), - ) - assert initial["status"] == "ok", initial - # A deliberately incorrect reference creates a large finite mismatch, without - # modifying the simulation or relying on round-off at the acceptance boundary. - truth_path = tmp_path / "incorrect.npy" - np.save(truth_path, 2 * np.load(original, allow_pickle=False)) - evidence_path = tmp_path / "evidence.json" - atomic_record(evidence_path, {"test": "intentional finite reference mismatch"}) - receipt = ReferenceReceipt( - 1, - configuration["case_hash"], - measurement_identity(configuration), - truth_path.name, - file_hash(truth_path), - original.name + ".band.npy", - file_hash(tmp_path / (original.name + ".band.npy")), - 1e-5, - None, - evidence_path.name, - file_hash(evidence_path), - "synthetic-test", - ) - path = tmp_path / "reference.json" - atomic_record(path, dataclasses.asdict(receipt)) - for override in (False, True): - result = collection.launch( - sys.executable, - { - "configuration": configuration, - "reference": str(path), - "allow_accuracy_failure": override, - }, - environment, - tmp_path / str(override), - (60, 2 * 2**30), - system.eligible_cpus(), - ) - assert result["status"] == ("ok" if override else "invalid") - assert result["returncode"] == (0 if override else 1) - assert result["accuracy"]["status"] == "invalid" - assert result["accuracy_overridden"] is override - assert result["seconds"] > 0 - assert result["measurement_peak_bytes"] > 0 diff --git a/tests/test_benchmark_telemetry.py b/tests/test_benchmark_telemetry.py index f0e689b1..265a2dba 100644 --- a/tests/test_benchmark_telemetry.py +++ b/tests/test_benchmark_telemetry.py @@ -2,16 +2,11 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -"""Check topology identities, missing telemetry and separate binding diagnostics.""" +"""Check topology identities and missing telemetry.""" -from types import SimpleNamespace - -import psutil import pytest -from benchmarks import __main__ as cli from benchmarks import telemetry -from benchmarks.records import SuiteError, atomic_record -from benchmarks.system import Profile +from benchmarks.records import SuiteError def test_linux_topology_distinguishes_smt_socket_and_missing_nodes(tmp_path): @@ -45,83 +40,11 @@ def test_linux_topology_distinguishes_smt_socket_and_missing_nodes(tmp_path): telemetry.cpu_list(invalid) -def test_numa_mapping_counts_use_each_mapping_page_size_and_omit_paths(): - contents = ( - "abc default file=/private/library N0=2 N1=3 kernelpagesize_kB=4\n" - "def bind:1 anon=1 N1=1 kernelpagesize_kB=2048\n" - "ghi default file=/private/unknown N0=10\n" - ) - assert telemetry.numa_pages(contents) == [ - {"node": 0, "resident_mapping_bytes": 8192}, - {"node": 1, "resident_mapping_bytes": 2109440}, - ] - - -def test_missing_hardware_and_process_counters_keep_reasons(monkeypatch): +def test_missing_hardware_and_allocation_details_keep_reasons(monkeypatch): monkeypatch.setattr(telemetry.platform, "system", lambda: "unknown") record = telemetry.hardware_details((0,)) assert record["cpu_model"] is None and record["cpu_model_reason"] assert record["cpu_topology"] is None and record["cpu_topology_reason"] monkeypatch.setattr(telemetry, "_read", lambda _: None) - - def denied(): - raise psutil.AccessDenied() - - monkeypatch.setattr(telemetry.psutil, "Process", denied) - record = telemetry.process_binding() - assert record["threads"] == [] and record["threads_reason"] - assert record["numa"] == [] and record["numa_reason"] - assert record["allocation"]["allowed_cpus"] is None - assert record["residency"] is None and record["residency_reason"] - - -def test_binding_snapshot_does_not_claim_execution_residency(monkeypatch): - monkeypatch.setattr(telemetry, "_read", lambda _: None) - thread = SimpleNamespace(id=123, user_time=1.0, system_time=0.25) - monkeypatch.setattr( - telemetry.psutil, "Process", lambda: SimpleNamespace(threads=lambda: [thread]) - ) - monkeypatch.setattr( - telemetry.os, "sched_getaffinity", lambda _: {2, 4}, raising=False - ) - record = telemetry.process_binding() - assert record["threads"][0]["allowed_cpus"] == [2, 4] - assert record["residency"] is None - - -def test_diagnostic_command_uses_serial_exploratory_quick_collection( - tmp_path, monkeypatch -): - seen = [] - monkeypatch.setattr( - cli, - "profile_for", - lambda *_args, **_kw: Profile("test", 1, 2, (0, 1), "topology"), - ) - - def collect(plan, options): - seen.append((plan, options)) - atomic_record( - options.output / "attempts" / "0.json", - { - "attempt_id": 0, - "configuration_id": plan["rows"][0]["id"], - "thread_diagnostics": { - "threads": [{"thread_id": 1, "allowed_cpus": [0, 1]}] - }, - }, - ) - return {"execution_ok": True} - - monkeypatch.setattr(cli, "collect", collect) - assert cli.main(["diagnose", "--out", str(tmp_path / "diagnostic")]) == 0 - plan, options = seen[0] - assert plan["purpose"] == "thread-binding-diagnostic" and plan["thread_diagnostics"] - assert plan["mode"] == "quick" and plan["rounds"] == 1 - assert [row["configuration"]["threads"] for row in plan["rows"]] == [1, 2] - assert options.force - assert (options.output / "threads.csv").exists() - assert ( - cli.main(["diagnose", "--out", str(tmp_path / "invalid"), "--timeout", "0"]) - == 2 - ) + record = telemetry.allocation_details() + assert record["allowed_cpus"] is None and record["allowed_cpus_reason"] diff --git a/tests/test_benchmark_track.py b/tests/test_benchmark_track.py new file mode 100644 index 00000000..266ff377 --- /dev/null +++ b/tests/test_benchmark_track.py @@ -0,0 +1,225 @@ +# SPDX-FileCopyrightText: 2026 John C. Travers and contributors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Exercise the tracking command's plan, hardware identity and history (SEP 38).""" + +import csv +import socket +import sys + +import pytest +from benchmarks import __main__ as cli +from benchmarks import history, runner, system, track +from benchmarks.records import SuiteError, read_record +from benchmarks.registry import CASES, select + + +def test_the_default_plan_runs_each_case_at_its_declared_widths(): + rows, skipped = runner.plan_rows(select(), ("default",), None, 8) + assert not skipped + widths = {} + for row in rows: + widths.setdefault(row["configuration"]["case"], []).append( + row["configuration"]["threads"] + ) + assert widths["rdw"] == [1, 8] + assert widths["ctc"] == [8] + assert widths["kerr_beam_192"] == [1] + # One-thread rows first, so a run's widths are not interleaved. + threads = [row["configuration"]["threads"] for row in rows] + assert threads == sorted(threads) + assert runner.estimate(rows, 3) is not None + + +def test_named_widths_and_steppers_apply_to_every_case_and_explain_skips(): + cases = (CASES["rdw"], CASES["kerr_beam_192"]) + rows, skipped = runner.plan_rows(cases, ("default", "rk"), "2", 4) + assert [row["id"] for row in rows] == [ + "rdw.default.t2", + "rdw.rk.t2", + "kerr_beam_192.default.t2", + ] + assert skipped == ["kerr_beam_192 rk: a fixed-step case runs its own schedule only"] + assert runner.estimate(rows, 1) is None + assert track.steppers("rk, default,rk") == ("rk", "default") + with pytest.raises(SuiteError, match="unknown stepper"): + track.steppers("adams") + with pytest.raises(SuiteError, match="must fit"): + system.widths("1", 10**6) + with pytest.raises(SuiteError, match="positive"): + system.widths("0", 1) + + +@pytest.mark.parametrize( + ("model", "cores", "expected"), + [ + ("Apple M5 Max", 18, "apple-m5-max-18c"), + ("AMD EPYC 7763 64-Core Processor", 16, "amd-epyc-7763-16c"), + ("Intel(R) Xeon(R) Gold 6240 CPU @ 2.60GHz", 36, "intel-xeon-gold-6240-36c"), + ("", 1, "unknown-cpu-1c"), + ], +) +def test_the_hardware_slug_names_the_cpu_model_and_cores(model, cores, expected): + assert system.slug(model, cores) == expected + + +def test_the_hardware_record_names_no_host(): + record = system.hardware(1, "aws c8a.4xlarge") + assert record["instance_type"] == "aws c8a.4xlarge" + assert record["slug"].endswith(f"-{system.physical_cores()}c") + text = repr(record).lower() + assert socket.gethostname().split(".")[0].lower() not in text + + +def result_row(case="rdw", threads=1, median=2.0): + return { + "case": case, + "revision": CASES[case].revision, + "stepper": "default", + "threads": threads, + "status": "ok", + "busy": False, + "median_s": median, + "min_s": median * 0.99, + "max_s": median * 1.01, + "repeats": 3, + "accepted": 100, + "rejected": 2, + "same_steps": True, + "source_evaluations": 104, + "peak_rss_gib": 0.5, + "construction_s": 3.0, + "warmup_s": 0.1, + "integrator": "Multistep(exponential)", + "rtol": 1e-6, + "atol": 1e-10, + "noise": "input", + "finite": True, + } + + +def test_history_rows_append_read_and_summarise(tmp_path): + machine = { + "slug": "test-cpu-2c", + "cpu_model": "Test CPU", + "instance_type": None, + "system": "Linux", + "machine": "x86_64", + } + provenance = { + "packages": {"solprop": "1.0"}, + "build_info": {"package_version": "1.0", "build_profile": "local"}, + } + path = tmp_path / "test-cpu-2c.csv" + first = track.history_rows( + [result_row(), result_row("ctc", 2, 10.0)], + machine, + {"revision": "a" * 40}, + provenance, + ) + assert all(tuple(row) == history.COLUMNS for row in first) + history.append(path, first) + later = [ + {**row, "date": "2099-01-01T00:00Z", "commit": "b" * 12, "median_s": value} + for row, value in zip(first, ("1.0", "5.0"), strict=True) + ] + history.append(path, later) + rows = history.read(path) + assert len(rows) == 4 + assert rows[0]["native_build"] == "local" + assert rows[0]["commit"] == "a" * 12 + summary = history.summary(rows) + assert [(row["threads"], row["ratio"]) for row in summary] == [ + ("1", 1.0), + ("2", 1.0), + ("1", 0.5), + ("2", 0.5), + ] + with path.open("w", encoding="utf-8", newline="") as stream: + csv.writer(stream).writerow(["date"]) + with pytest.raises(SuiteError, match="columns differ"): + history.append(path, first) + with pytest.raises(SuiteError, match="exactly the declared columns"): + history.append(tmp_path / "new.csv", [{"date": "x"}]) + + +def test_a_failed_worker_is_summarised_without_statistics(): + row = {"id": "x", "configuration": CASES["rdw"].configuration("default", 1)} + result = track.summarise(row, {"status": "failed", "error": "boom"}, busy=True) + assert result["status"] == "failed" + assert result["error"] == "boom" + assert "median_s" not in result + + +def test_worker_timeout_and_missing_receipts_are_failures(tmp_path): + events, _ = runner.launch_worker( + sys.executable, {"repeats": 1}, {"PATH": ""}, tmp_path / "slow", 1e-3 + ) + assert runner.receipt(events)["error"] == "worker timeout" + assert runner.receipt([])["error"] == "no receipt from the worker" + + +def test_list_and_dry_run_plan_without_running(capsys): + assert cli.main([]) == 0 + assert cli.main(["track", "--dry-run", "--case", "rdw", "--threads", "1"]) == 0 + text = capsys.readouterr().out + assert "configurations" in text + assert "estimated" in text + assert cli.main(["track", "--case", "absent", "--dry-run"]) == 2 + assert "unknown case" in capsys.readouterr().err + + +@pytest.mark.medium +def test_a_tracking_run_records_complete_runs_and_appends_history( + tmp_path, monkeypatch, capsys +): + monkeypatch.setattr(history, "HISTORY", tmp_path / "history") + monkeypatch.setattr(track, "guard", lambda *_args: {"quiet": True}) + out = tmp_path / "run" + status = cli.main( + [ + "track", + "--case", + "smoke", + "--stepper", + "default,multistep", + "--backend", + "numpy_ref", + "--threads", + "1", + "--repeats", + "2", + "--out", + str(out), + ] + ) + text = capsys.readouterr().out + assert status == 0, text + results = read_record(out / "results.json") + assert [row["integrator"] for row in results["rows"]] == [ + "RungeKutta", + "Multistep(exponential)", + ] + for row in results["rows"]: + assert row["repeats"] == 2 + assert row["same_steps"] + assert row["finite"] + assert read_record(out / "plan.json")["hardware"]["slug"] + # The history takes the native backend only. + assert ( + cli.main( + [ + "track", + "--case", + "smoke", + "--backend", + "numpy_ref", + "--append-history", + "--out", + str(tmp_path / "refused"), + ] + ) + == 2 + ) + assert "native backend only" in capsys.readouterr().err diff --git a/tests/test_standard_benchmark_cases.py b/tests/test_standard_benchmark_cases.py deleted file mode 100644 index b64da7b3..00000000 --- a/tests/test_standard_benchmark_cases.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Check extracted workload definitions without running benchmark campaigns.""" - -import numpy as np -import pytest -from benchmarks.builders import build, capillary, gnlse -from benchmarks.planning import Selection, matrix -from benchmarks.records import SuiteError -from benchmarks.registry import ADAPTIVE, CASES, FIXED, select -from benchmarks.schedules import ENTRANCE_SCHEDULES -from benchmarks.system import Profile - -from solprop.nonlinear.responses import PlasmaResponse -from solprop.stepper import Multistep, RungeKutta - - -@pytest.mark.parametrize( - "name", - [ - name - for name in ENTRANCE_SCHEDULES - if CASES[name].builder not in ("advanced", "exercise") - ], -) -def test_windows_preserve_full_guides_and_frozen_fixed_steps(name): - case = CASES[name] - adaptive = case.measurement(ADAPTIVE[1]) - assert 0 == adaptive["start"] < adaptive["stop"] < case.length - configuration = case.measurement(FIXED) | {"backend": "numpy_ref"} - simulation = build(configuration) - assert simulation.grid.length == case.length - assert isinstance(simulation.settings.integrator, RungeKutta) - assert simulation.settings.fields == "none" - schedule = simulation.settings.schedule - assert schedule is not None - expected = ( - np.linspace(0.0, 0.0012, 11) - if name == "ctc_reduced" - else np.array(ENTRANCE_SCHEDULES[name]) - ) - # Schedule construction subtracts stored edges, and cumulative addition can - # round once per edge. This tests their frozen itinerary, not a solver bound. - np.testing.assert_allclose( - np.cumsum(schedule.steps[:10]), - expected[1:], - rtol=10 * np.finfo(float).eps, - atol=0, - ) - assert simulation.settings.save_planes[-1] == case.length - assert simulation.grid.n_t == case.dimensions[-2] - assert simulation.grid.n_to == case.dimensions[-1] - - -@pytest.mark.parametrize( - ("resolved", "averaged"), - [ - ("rdw_one_mode", "he_rdw_averaged"), - ("argon_blueshift", "argon_blueshift_averaged"), - ("h2_raman_6ps", "h2_raman_6ps_averaged"), - ], -) -def test_averaging_is_a_distinct_route_with_the_original_grid(resolved, averaged): - first, second = capillary(resolved), capillary(averaged) - assert first.settings.quadrature is not None - assert second.settings.quadrature is None - assert second.settings.components is None - # A route change must not change the launch, guide or temporal axes. - assert first.grid.length == second.grid.length - np.testing.assert_array_equal(first.grid.omega, second.grid.omega) - assert first.settings.stats != second.settings.stats - if resolved == "h2_raman_6ps": - assert not any( - isinstance(response, PlasmaResponse) for response in first.responses - ) - - -def test_new_sets_are_ready_for_timing_without_invented_references(): - cases = select(sets=("modal-capillary", "mode-averaged")) - plan = matrix( - cases, Profile("test", 1, 1, (0,), "topology"), Selection(backend="numba_cpu") - ) - assert len(cases) == 11 - assert len(plan["rows"]) == 22 - assert all(row["status"] == "ready" for row in plan["rows"]) - assert all(row["reference"] == "not-qualified" for row in plan["rows"]) - for name in ENTRANCE_SCHEDULES: - if CASES[name].builder in ("advanced", "exercise"): - continue - simulation = build( - CASES[name].measurement(ADAPTIVE[0]) | {"backend": "numpy_ref"} - ) - assert isinstance(simulation.settings.integrator, Multistep) - assert simulation.settings.fields == "all" - - -def test_named_builders_refuse_unknown_workloads(): - with pytest.raises(SuiteError, match="not implemented"): - capillary("absent") - with pytest.raises(SuiteError, match="not implemented"): - gnlse("absent") diff --git a/tests/test_standard_benchmarks.py b/tests/test_standard_benchmarks.py deleted file mode 100644 index 6a44bbf9..00000000 --- a/tests/test_standard_benchmarks.py +++ /dev/null @@ -1,724 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Exercise standard-suite control and recovery without a benchmark campaign.""" - -import dataclasses -import json -import math -import os -import subprocess -import sys - -import pytest -from benchmarks import __main__ as cli -from benchmarks import collection, system -from benchmarks.__main__ import parser_for, resume_request -from benchmarks.comparison import ( - aa_control, - clean_pairs, - compare, - compare_configuration, - completed, - needs_more, - repeatability, - resource_flags, -) -from benchmarks.planning import Selection, matrix, widths -from benchmarks.records import ( - SuiteError, - atomic_record, - attempts, - canonical, - distribution, - identity, - metadata_value, - read_record, - run_lock, -) -from benchmarks.references import measurement_identity -from benchmarks.registry import ADAPTIVE, CASES, KERR, select - - -@pytest.fixture -def profile(): - return system.Profile("test-machine", 1, 2, (0, 1), "topology") - - -def samples(arm, *, count=5, seconds=1.0): - return [ - { - "sample_id": f"{arm}-{index}", - "arm": arm, - "round": index, - "configuration_id": "case", - "status": "ok", - "seconds": seconds, - "clean": True, - "resolution_hash": "same", - "accuracy": { - "status": "passed", - "qualified_for_adoption": True, - "reference_id": "same-reference-and-policy", - }, - "peak_rss_bytes": 2**30, - "startup_seconds": 2.0, - } - for index in range(count) - ] - - -def test_registry_selection_deduplicates_and_preserves_pending(): - assert len(select()) == 4 - assert select(("rdw",), ("core",)) == select() - union = select(all_cases=True) - assert len(union) == len({case.name for case in union}) - assert "smoke" not in {case.name for case in union} - assert CASES["vector_wall"] in union - with pytest.raises(SuiteError, match="unknown set"): - select(sets=("all",)) - with pytest.raises(SuiteError, match="unknown case"): - select(("absent",)) - - -def test_metadata_import_and_dry_run_cannot_construct_simulations(): - script = """ -import importlib.abc -import sys -class Refuse(importlib.abc.MetaPathFinder): - def find_spec(self, fullname, path=None, target=None): - roots = ('numpy', 'solprop', 'solprop_native', 'numba', 'pyfftw') - if fullname.split('.')[0] in roots: - raise AssertionError('scientific import: ' + fullname) -sys.meta_path.insert(0, Refuse()) -from benchmarks.__main__ import main -assert main(['list', '--set', 'core']) == 0 -assert main(['run', '--dry-run', '--case', 'rdw']) == 0 -assert main(['--dry-run', '--case', 'rdw']) == 0 -""" - result = subprocess.run( - [sys.executable, "-c", script], capture_output=True, text=True, check=False - ) - assert result.returncode == 0, result.stderr - - -def test_matrix_core_seven_quick_and_twenty_standard(profile): - quick = matrix(select(), profile, Selection()) - assert len(quick["rows"]) == 7 - assert all(row["configuration"]["threads"] == 2 for row in quick["rows"]) - standard = matrix(select(), profile, Selection(mode="standard")) - assert len(standard["rows"]) == 20 - assert all(row["status"] == "ready" for row in standard["rows"]) - radial = CASES["radial_filament_256"].measurement("fixed-rk") - assert len(radial["schedule"]) == 11 - assert radial["schedule"][0] == 0.0 - assert radial["stop"] == radial["schedule"][-1] - assert 0 < radial["stop"] < CASES["radial_filament_256"].length - refused = matrix( - select(("kerr_beam_384",)), profile, Selection(tracks=(ADAPTIVE[0],)) - ) - assert refused["rows"][0]["status"] == "excluded" - assert CASES["kerr_beam_384"].tracks == (KERR,) - assert widths("1,production,1", profile, "quick") == (1, 2) - with pytest.raises(SuiteError): - widths("3", profile, "standard") - - -@pytest.mark.parametrize("reviewed", [False, True]) -def test_planning_reports_receipt_compatibility_without_loading_arrays( - tmp_path, profile, reviewed, capsys -): - selection = Selection(receipts=tmp_path, tracks=(ADAPTIVE[1],)) - cases = select(("rdw",)) - configuration = matrix(cases, profile, selection)["rows"][0]["configuration"] - receipt = { - "revision": 1, - "case_hash": configuration["case_hash"], - "measurement_hash": measurement_identity(configuration), - "qualified_for_adoption": reviewed, - "fields_file": "not-present.npy", - } - path = tmp_path / "rdw-adaptive-rk.json" - atomic_record(path, receipt) - planned = matrix(cases, profile, selection) - assert planned["rows"][0]["reference"].startswith( - "reviewed receipt" if reviewed else "available receipt" - ) - cli.describe(planned) - assert "memory budget 8.00 GiB" in capsys.readouterr().out - atomic_record(path, {**receipt, "measurement_hash": "changed-window"}) - assert matrix(cases, profile, selection)["rows"][0]["reference"] == ( - "incompatible receipt" - ) - path.write_text("broken JSON", encoding="utf-8") - assert matrix(cases, profile, selection)["rows"][0]["reference"] == ( - "invalid receipt" - ) - - -def test_default_command_runs_quick_and_prints_failures(monkeypatch, tmp_path, capsys): - observed = [] - - def collect_stub(plan, options): - observed.append((plan, options)) - return {"status": "collected", "execution_ok": True} - - monkeypatch.setattr(cli, "collect", collect_stub) - monkeypatch.setattr(cli, "attempts", lambda path: []) - monkeypatch.setattr( - cli, "new_collection", lambda args: collection.Collection(tmp_path) - ) - assert cli.main([]) == 0 - plan = observed[0][0] - assert plan["mode"] == "quick" - assert len(plan["rows"]) == 7 - assert all(row["configuration"]["backend"] == "native" for row in plan["rows"]) - row = samples("candidate", count=1)[0] - row.update( - configuration_id=plan["rows"][0]["id"], status="failed", error="worker crashed" - ) - cli.show_results(plan, [row]) - output = capsys.readouterr().out - assert "worker crashed" in output - assert "warm / s" in output - - -def test_child_environment_overrides_incoming_pools_before_import(): - incoming = { - "PATH": "test", - "OMP_PROC_BIND": "close", - "OMP_PLACES": "cores", - "OPENBLAS_NUM_THREADS": "72", - "SOLPROP_THREADS": "72", - "SOLPROP_OVERSUBSCRIBE": "1", - "SOLPROP_FFTW_PLAN": "estimate", - } - result = system.thread_environment(1, "budget", incoming) - assert result["OPENBLAS_NUM_THREADS"] == "1" - assert result["SOLPROP_THREADS"] == "1" - assert "SOLPROP_OVERSUBSCRIBE" not in result - assert result["SOLPROP_FFTW_THREADS"] == "1" - assert result["SOLPROP_FFTW_PLAN"] == "estimate" - assert result["OMP_PROC_BIND"] == "false" - assert "OMP_PLACES" not in result - assert incoming["OMP_PROC_BIND"] == "close" - - -@pytest.mark.parametrize("policy", ["budget", "blas1", "all-pools-N", "as-installed"]) -def test_child_environment_handles_native_pool_overrides(policy): - pools = ( - "SOLPROP_NATIVE_KERNEL_THREADS", - "SOLPROP_NATIVE_FFT_THREADS", - "SOLPROP_NATIVE_RAMAN_FFT_THREADS", - "SOLPROP_NATIVE_RAMAN_TAIL_FFT_THREADS", - ) - threshold = "SOLPROP_NATIVE_THREADED_FFT_MIN_SAMPLES" - incoming = dict.fromkeys(pools, "99") | {threshold: "12345"} - result = system.thread_environment(4, policy, incoming) - if policy == "as-installed": - assert result == incoming - else: - assert threshold not in result - for name in pools: - if policy == "all-pools-N": - assert result[name] == "4" - else: - assert name not in result - assert incoming == dict.fromkeys(pools, "99") | {threshold: "12345"} - - -def test_environment_identity_accepts_runtime_widths_but_refuses_changed_builds(): - inventory = { - "packages": {"solprop": "test"}, - "extension_hash": "original-binary", - "build_info": { - "compiler": "Clang", - "runtime_threads": 18, - "fftw_threads": 18, - "blas_threads": 18, - "explicit_thread_pools": 0, - "blas_thread_policy": "the Kokkos thread count", - }, - } - sample = { - **inventory, - "build_info": { - **inventory["build_info"], - "runtime_threads": 1, - "fftw_threads": 1, - "blas_threads": 1, - "explicit_thread_pools": 15, - "blas_thread_policy": "OMP_NUM_THREADS at process start", - }, - } - assert collection.environment_identity( - inventory - ) == collection.environment_identity(sample) - changed = {**sample, "extension_hash": "replacement-binary"} - assert collection.environment_identity( - inventory - ) != collection.environment_identity(changed) - changed = {**sample, "build_info": {**sample["build_info"], "compiler": "GCC"}} - assert collection.environment_identity( - inventory - ) != collection.environment_identity(changed) - assert sample["build_info"]["runtime_threads"] == 1 - - -def test_calibration_and_balanced_adjacent_rounds(): - rows = [{"id": index} for index in range(8)] - first = collection.ordered(rows, 30, 0, True) - second = collection.ordered(rows, 30, 1, True) - assert first == collection.ordered(rows, 30, 0, True) - assert {row["id"] for row, _ in first} == set(range(8)) - assert all(arms == ("baseline", "candidate") for _, arms in first) - assert all(arms == ("candidate", "baseline") for _, arms in second) - assert collection.calibration([0.2, 0.3], 1e-9) == (1, False) - assert collection.calibration([0.02, 0.04], 1e-9) == (5, False) - assert collection.calibration([0.001, 0.01], 1e-9) == (10, True) - - -def test_contention_extends_whole_rounds_without_joining_gaps(): - left, right = samples("baseline", count=7), samples("candidate", count=7) - right[2]["clean"] = False - a, b = clean_pairs(left, right) - assert [row["round"] for row in a] == [0, 1, 3, 4, 5, 6] - assert len(b) == 6 - assert repeatability(a) == 0 - assert repeatability(a[:4]) is None - assert needs_more(left[:5] + right[:5], ["case"], ("baseline", "candidate")) - assert not needs_more(left + right, ["case"], ("baseline", "candidate")) - for index in (0, 1): - right[index]["clean"] = False - assert needs_more(left + right, ["case"], ("baseline", "candidate")) - - -def test_comparison_requires_complete_qualified_production_set(): - rows = samples("baseline") + samples("candidate", seconds=0.9) - plan = { - "mode": "standard", - "profile": {"production": 2}, - "rows": [ - { - "id": "case", - "configuration": {"case": "rdw", "threads": 2, "track": "adaptive-rk"}, - } - ], - } - assert compare(plan, rows)["status"] == "passed" - quick = compare({**plan, "mode": "quick"}, rows) - assert quick["status"] == "exploratory" - # Exact synthetic times: this ratio is descriptive even without adoption. - assert quick["configurations"][0]["all_ratio"] == pytest.approx(0.9, abs=1e-15) - assert ( - compare({**plan, "profile": {"production": 4}}, rows)["status"] == "unresolved" - ) - assert compare(plan, rows[:5])["status"] == "incomplete" - assert ( - compare_configuration(rows[:5], rows[5:], paired=False)["status"] - == "unresolved" - ) - rows[-1]["resolution_hash"] = "changed" - assert compare(plan, rows)["status"] == "incompatible" - rows[-1]["status"] = "invalid" - assert compare(plan, rows)["status"] == "numerically-invalid" - - -def test_resource_flags_and_single_sample_dispersion_are_explicit(): - left, right = samples("baseline"), samples("candidate") - right[0]["peak_rss_bytes"] += 70 * 2**20 - for row in right: - row["startup_seconds"] = 2.3 - flags = resource_flags(left, right) - assert flags["peak_rss_bytes"]["flag"] - assert flags["startup_seconds"]["flag"] - assert resource_flags(left, right[:4])["peak_rss_bytes"]["status"] == "unavailable" - assert distribution([1.0]).mad is None - with pytest.raises(SuiteError, match="finite"): - distribution([math.inf]) - - -def test_atomic_shards_ignore_interruption_and_preserve_retries(tmp_path, monkeypatch): - shard = tmp_path / "attempts" / "000000.json" - failed = {**samples("candidate")[0], "status": "failed"} - atomic_record(shard, failed, immutable=True) - with pytest.raises(SuiteError, match="replace"): - atomic_record(shard, samples("candidate")[0], immutable=True) - atomic_record( - shard.with_name("000001.json"), samples("candidate")[0], immutable=True - ) - shard.with_name(".pending-interrupted").write_text( - '{"incomplete":', encoding="utf-8" - ) - assert len(attempts(tmp_path)) == 2 - assert completed(attempts(tmp_path))[0]["status"] == "ok" - - def fail_sync(_descriptor): - raise OSError("interrupted flush") - - monkeypatch.setattr(os, "fsync", fail_sync) - with pytest.raises(OSError, match="interrupted"): - atomic_record(shard.with_name("000002.json"), failed) - assert not shard.with_name("000002.json").exists() - assert len(attempts(tmp_path)) == 2 - - -def test_lock_is_exclusive_and_released_after_error(tmp_path): - with pytest.raises(RuntimeError), run_lock(tmp_path): - with pytest.raises(SuiteError, match="locked"), run_lock(tmp_path): - pytest.fail("second writer acquired a held lock") - raise RuntimeError("interrupted controller") - with run_lock(tmp_path): - atomic_record(tmp_path / "recovered.json", {"ok": True}) - - -def test_synthetic_excessive_adams_memory_is_refused_before_launch(monkeypatch): - monkeypatch.setattr(system, "effective_memory", lambda: 64 * 2**30) - assert not system.admission(200 * 2**30, None)["admitted"] - assert not system.admission(None, None)["admitted"] - assert system.admission(None, 16 * 2**30, allow_unestimated=True)["admitted"] - assert not system.admission(None, None, allow_unestimated=True)["admitted"] - - -def test_counter_guard_does_not_treat_unknown_as_zero(): - before = {"time": 1.0, "busy": [1.0, 2.0], "swap_bytes": None} - after = {"time": 2.0, "busy": [1.1, 2.1], "swap_bytes": None} - assert system.interval(before, after, (0, 1))["confidence"] == "unknown" - assert not system.interval(before, after, (0, 1))["quiet"] - before["swap_bytes"], after["swap_bytes"] = 0, 0 - assert system.interval(before, after, (0, 1), own_cpu=0.1)["quiet"] - - -def test_strict_json_distinguishes_metadata_infinity_from_samples(tmp_path): - with pytest.raises(ValueError): - canonical({"seconds": math.inf}) - material = metadata_value({"validity": [0.0, math.inf]}) - assert json.loads(canonical(material))["validity"][1] == { - "nonfinite_metadata": "inf" - } - path = tmp_path / "invalid.json" - path.write_text('{"seconds": NaN}', encoding="utf-8") - with pytest.raises(SuiteError, match="non-finite"): - read_record(path) - assert identity(dataclasses.asdict(distribution([1.0]))) - - -@pytest.fixture -def fake_worker(tmp_path): - package = tmp_path / "fake" / "benchmarks" - package.mkdir(parents=True) - (package / "worker.py").write_text( - "import json, os, sys, time\n" - "r = json.load(open(sys.argv[1], encoding='utf-8'))\n" - "if r['scenario'] == 'crash': os._exit(7)\n" - "if r['scenario'] == 'timeout': time.sleep(100)\n" - "event = {'event': 'result', 'result': {'status': 'ok'}}\n" - "print(json.dumps(event), flush=True)\n", - encoding="utf-8", - ) - return package.parent - - -@pytest.mark.parametrize( - "scenario, status, limit", - [ - ("crash", "failed", 60.0), - ("timeout", "timeout", 0.3), - ("missing", "failed", 60.0), - ], -) -def test_worker_crash_timeout_and_missing_receipts_are_rows( - tmp_path, - fake_worker, - scenario, - status, - limit, -): - # Only the timeout case runs into its limit; the others get room for a slow - # interpreter start (a loaded Windows runner can take longer than 0.3 s). - result = collection.launch( - sys.executable, - {"scenario": scenario, "harness_root": str(fake_worker)}, - dict(os.environ), - tmp_path / "attempt", - (limit, None), - system.eligible_cpus(), - ) - assert result["status"] == status - assert result["error"] - assert result["controller_cpu_seconds"] >= 0 - assert (tmp_path / "attempt" / "request.json").exists() - - -def test_resume_preserves_failure_then_skips_completed_sample( - tmp_path, monkeypatch, profile -): - calls = [] - provenance = {"packages": {"solprop": "test"}} - - def launch(_python, request, _environment, _directory, _limits, _allocation): - if request.get("inventory"): - return {"status": "ok", "provenance": provenance} - calls.append(request) - if len(calls) == 1: - return {"status": "failed", "error": "synthetic crash"} - return { - "status": "ok", - "provenance": provenance, - "seconds": 1.0, - "peak_rss_bytes": 1024, - "during_guard": {"quiet": True}, - "accuracy": {"status": "not-qualified"}, - "timer_resolution": 1e-9, - } - - monkeypatch.setattr(collection, "launch", launch) - monkeypatch.setattr(collection, "guard", lambda *a, **k: {"quiet": True}) - monkeypatch.setattr(collection, "git_receipt", lambda _root: {"head": "same"}) - monkeypatch.setattr(system, "effective_memory", lambda: 64 * 2**30) - plan = matrix(select(("smoke",)), profile, Selection(tracks=(ADAPTIVE[1],))) - options = collection.Collection(tmp_path / "run") - assert not collection.collect(plan, options)["execution_ok"] - resume = dataclasses.replace(options, resume=True) - assert collection.collect(plan, resume)["execution_ok"] - assert collection.collect(plan, resume)["execution_ok"] - assert len(calls) == 2 - assert [row["status"] for row in attempts(options.output)] == ["failed", "ok"] - with pytest.raises(SuiteError, match="identity differs"): - collection.collect({**plan, "suite_version": 2}, resume) - - -def test_resume_preserves_interrupted_worker_files_and_uses_fresh_attempt_ids( - tmp_path, monkeypatch, profile -): - provenance = {"packages": {"solprop": "test"}} - directories = [] - - def launch(_python, request, _environment, directory, _limits, _allocation): - if request.get("inventory"): - return {"status": "ok", "provenance": provenance} - atomic_record(directory / "request.json", request, immutable=True) - directories.append(directory.name) - if len(directories) == 2: - raise KeyboardInterrupt - return { - "status": "ok", - "provenance": provenance, - "seconds": 1.0, - "during_guard": {"quiet": True}, - "accuracy": {"status": "not-qualified"}, - } - - monkeypatch.setattr(collection, "launch", launch) - monkeypatch.setattr(collection, "guard", lambda *a, **k: {"quiet": True}) - monkeypatch.setattr(collection, "git_receipt", lambda _root: {"head": "same"}) - monkeypatch.setattr(system, "effective_memory", lambda: 64 * 2**30) - plan = matrix(select(("smoke",)), profile, Selection(tracks=(ADAPTIVE[1],))) - plan.update(rounds=3, maximum_rounds=3) - options = collection.Collection(tmp_path / "run") - with pytest.raises(KeyboardInterrupt): - collection.collect(plan, options) - interrupted = options.output / "raw" / "000001" / "request.json" - original = interrupted.read_bytes() - resume = dataclasses.replace(options, resume=True) - assert collection.collect(plan, resume)["execution_ok"] - assert collection.collect(plan, resume)["execution_ok"] - assert interrupted.read_bytes() == original - assert directories == ["000000", "000001", "000002", "000003"] - assert [row["attempt_id"] for row in attempts(options.output)] == [0, 2, 3] - - -@pytest.mark.medium -@pytest.mark.parametrize("backend", ["numpy_ref", "numba_cpu"]) -@pytest.mark.parametrize("track", ADAPTIVE) -def test_small_worker_reset_and_history_conventions(tmp_path, profile, backend, track): - configuration = matrix( - select(("smoke",)), - profile, - Selection(backend=backend, threads="1", tracks=(track,)), - )["rows"][0]["configuration"] - result = collection.launch( - sys.executable, - {"configuration": configuration, "repetitions": 2, "verify_reset": True}, - system.thread_environment(1, "budget", dict(os.environ)), - tmp_path / "worker", - (180.0, 2 * 2**30), - system.eligible_cpus(), - ) - assert result["status"] == "ok", result - reset = result["reset_evidence"] - # Identical discrete reference runs: 64 rounding operations per stage and - # seven stages per accepted step give a conservative accumulated L2 budget. - bound = 64 * 7 * result["repetitions"][0]["accepted"] * sys.float_info.epsilon - assert max(reset["plane_errors"]) <= bound - assert all(counts == reset["fresh_counts"] for counts in reset["reset_counts"]) - history = reset["history_statistics_noise"] - assert history["same_columns"] - assert history["step_history/accepted"]["same_decisions"] - for name, item in history.items(): - if name in ("same_columns", "step_history/accepted"): - continue - assert item["same_shape"] - # Repeated histories use the same operations and the field budget above. - assert (item["relative_l2"] or item["absolute_l2"]) <= bound - assert result["pilot_edges"][0] == 0 - assert all( - right > left - for left, right in zip( - result["pilot_edges"], result["pilot_edges"][1:], strict=False - ) - ) - assert result["peak_rss_bytes"] == result["measurement_peak_bytes"] - assert result["startup_seconds"] > 0 - - -def test_harness_snapshot_is_frozen_before_workers(tmp_path, monkeypatch): - root = tmp_path / "source" - source = root / "benchmarks" / "worker.py" - source.parent.mkdir(parents=True) - source.write_text("original", encoding="utf-8") - monkeypatch.setattr(collection, "ROOT", root) - snapshot = tmp_path / "snapshot" - first = collection.freeze_harness(snapshot) - assert first == collection.freeze_harness(snapshot) - source.write_text("changed", encoding="utf-8") - with pytest.raises(SuiteError, match="stored harness differs"): - collection.freeze_harness(snapshot) - - -def test_aa_bands_supply_unpaired_repeatability_only_with_clean_counts(): - rows = samples("baseline") + samples("candidate", seconds=1.01) - plan = { - "mode": "standard", - "profile": {"production": 2}, - "environments": {"baseline": {"build": "same"}}, - "rows": [ - { - "id": "case", - "configuration": {"case": "rdw", "threads": 2, "track": "adaptive-rk"}, - } - ], - } - report = aa_control(plan, rows) - band = report["configurations"]["case"]["band"] - assert band == pytest.approx(2 * math.log(1.01), rel=4 * sys.float_info.epsilon) - faster = samples("baseline") + samples("candidate", seconds=0.9) - assert ( - compare(plan, faster, paired=False, aa_bands=report["configurations"])["status"] - == "passed" - ) - rows[5]["clean"] = False - assert aa_control(plan, rows)["configurations"]["case"]["band"] is None - - -def test_asymmetric_memory_refusal_is_a_resource_regression(): - left, right = samples("baseline"), samples("candidate") - right[0]["status"] = "refused" - report = compare_configuration(left, right) - assert report["status"] == "regression" - assert "memory admission" in report["resource_regression"] - left[0]["status"] = "refused" - assert compare_configuration(left, right)["status"] == "incomplete" - - -def test_failed_calibration_requires_retry_and_reuses_successful_arm( - tmp_path, monkeypatch, profile -): - calls = [] - - def attempt(row, sample, options, plan, receipt): - calls.append(sample.arm) - return { - "attempt_id": len(attempts(options.output)), - "sample_id": sample.arm, - "configuration_id": row["id"], - "arm": sample.arm, - "phase": "pilot", - "status": "failed" if len(calls) == 2 else "ok", - "seconds": 0.02, - "timer_resolution": 1e-9, - } - - monkeypatch.setattr(collection, "_attempt", attempt) - monkeypatch.setattr(collection, "guard", lambda *a, **k: {"quiet": True}) - plan = matrix( - select(("smoke",)), - profile, - Selection(mode="standard", tracks=(ADAPTIVE[1],), threads="1"), - ) - options = collection.Collection(tmp_path, baseline_python=sys.executable) - with pytest.raises(SuiteError, match="calibration failed"): - collection.calibrate(plan, options) - assert not (tmp_path / "repetitions.json").exists() - counts = collection.calibrate(plan, options) - assert len(calls) == 3 - assert list(counts.values()) == [{"count": 5, "capped": False}] - assert len(attempts(tmp_path)) == 3 - - -def test_resume_cli_restores_saved_options_and_rejects_changes(tmp_path): - options = dataclasses.asdict(collection.Collection(tmp_path, seed=123, timeout=9.0)) - options.pop("resume") - options["output"] = str(tmp_path) - options["references"] = str(options["references"].resolve()) - atomic_record(tmp_path / "manifest.json", {"plan": {"options": options}}) - tokens = ["run", "--resume", str(tmp_path)] - _, restored = resume_request(parser_for().parse_args(tokens), tokens) - assert restored.resume - assert restored.seed == 123 - assert restored.timeout == 9.0 - changed = [*tokens, "--seed", "1"] - with pytest.raises(SuiteError, match="resume seed differs"): - resume_request(parser_for().parse_args(changed), changed) - changed = [*tokens, "--case", "rdw"] - with pytest.raises(SuiteError, match="restores the saved selection"): - resume_request(parser_for().parse_args(changed), changed) - - -def test_comparison_requires_the_same_reference_and_policy_revision(): - left, right = samples("baseline"), samples("candidate", seconds=0.9) - right[0]["accuracy"]["reference_id"] = "corrected-reference" - result = compare_configuration(left, right) - assert result["status"] == "incompatible" - assert "revalidate both arms" in result["reason"] - - -@pytest.mark.parametrize("crash", [False, True]) -def test_accuracy_override_continues_rounds_but_does_not_hide_crashes( - tmp_path, monkeypatch, profile, crash -): - calls = [] - provenance = {"packages": {"solprop": "test"}} - - def launch(_python, request, _environment, _directory, _limits, _allocation): - if request.get("inventory"): - return {"status": "ok", "provenance": provenance} - calls.append(request) - assert request["allow_accuracy_failure"] - return { - "status": "failed" if crash else "ok", - "provenance": provenance, - "accuracy": {"status": "invalid", "reason": "synthetic accuracy failure"}, - "accuracy_overridden": not crash, - "seconds": 1.0, - "timer_resolution": 1e-9, - "during_guard": {"quiet": True}, - } - - monkeypatch.setattr(collection, "launch", launch) - monkeypatch.setattr(collection, "guard", lambda *a, **k: {"quiet": True}) - monkeypatch.setattr(collection, "git_receipt", lambda _root: {"head": "same"}) - monkeypatch.setattr(system, "effective_memory", lambda: 64 * 2**30) - plan = matrix(select(("smoke",)), profile, Selection(tracks=(ADAPTIVE[1],))) - plan.update(rounds=5, maximum_rounds=5) - options = collection.Collection(tmp_path, allow_accuracy_failure=True) - result = collection.collect(plan, options) - assert result["execution_ok"] is not crash - assert len(calls) == (1 if crash else 5) - assert result["status"] == "exploratory" - assert result["accuracy_failures"] == len(calls) - saved = read_record(tmp_path / "manifest.json")["plan"] - assert saved["exploratory"] - assert saved["options"]["allow_accuracy_failure"] - assert len(attempts(tmp_path)) == len(calls) From 6fd3c3feda461ee55ae215f1963caeaf41256849 Mon Sep 17 00:00:00 2001 From: John Travers Date: Tue, 27 Oct 2026 06:25:44 +0000 Subject: [PATCH 02/13] Count allocated cores from the Linux topology, so one thread per core counts each core Co-Authored-By: Claude Opus 5.5 --- benchmarks/system.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/benchmarks/system.py b/benchmarks/system.py index ad6bd6db..b102b9a6 100644 --- a/benchmarks/system.py +++ b/benchmarks/system.py @@ -20,7 +20,7 @@ import psutil from benchmarks.records import Record, SuiteError, identity -from benchmarks.telemetry import allocation_details, hardware_details +from benchmarks.telemetry import allocation_details, hardware_details, linux_topology __all__ = [ "POOL_VARIABLES", @@ -78,10 +78,21 @@ def eligible_cpus() -> tuple[int, ...]: def physical_cores() -> int: - """Count the physical cores in the allocation, assuming even SMT.""" + """Count the physical cores in the allocation. + + On Linux the distinct (socket, core) pairs of the allocated CPUs, so that an + allocation of one hardware thread per core counts every core; elsewhere the + allocation scaled by the machine's ratio of cores to hardware threads. + """ + allocation = eligible_cpus() + if platform.system() == "Linux": + rows = linux_topology(Path("/sys/devices/system/cpu"), allocation) + pairs = {(row["socket"], row["core"]) for row in rows} + if rows and all(None not in pair for pair in pairs): + return len(pairs) logical = psutil.cpu_count() or 1 physical = psutil.cpu_count(logical=False) or logical - return max(1, len(eligible_cpus()) * physical // logical) + return max(1, len(allocation) * physical // logical) def widths(value: str | None, production: int) -> tuple[int, ...]: From 046d33a37f7ef840030697dfa8c72a954ac78af2 Mon Sep 17 00:00:00 2001 From: John Travers Date: Tue, 27 Oct 2026 06:31:52 +0000 Subject: [PATCH 03/13] Document the tracking suite: manual, rules, SEP index and roadmap (SEP 38) Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- CHANGELOG.md | 8 + benchmarks/README.md | 21 +- docs/dev/backends.md | 6 +- docs/dev/benchmarking.md | 634 ++++++-------------------------- docs/dev/benchmarks.md | 82 ++--- docs/dev/dependencies.md | 4 +- docs/dev/roadmap.md | 19 +- docs/dev/seps/index.md | 2 + docs/dev/testing.md | 4 +- docs/dev/tooling.md | 2 +- docs/guide/performance/index.md | 2 +- 12 files changed, 187 insertions(+), 599 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 2eb9e112..441845a8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -53,7 +53,7 @@ rather than style, ask rather than guess. | Slow lane | `uv run poe test-slow` | | Coverage lane (100 % required) | `uv run poe cov` (kernel bodies interpreted first, remaining suite compiled) | | GPU tests | `uv run pytest -m gpu` (needs a GPU build of solprop-native and a device) | -| Benchmarks | `uv run poe bench` (on an otherwise idle machine) | +| Benchmarks | `uv run poe bench` plans; `uv run python -m benchmarks track` runs (on an otherwise idle machine) | | Build docs | `uv run poe docs` (never executes the worked examples) | | Run the worked examples | `uv run poe docs-examples [plot_NN_name]` (slow; commit `docs/auto_examples/`) | diff --git a/CHANGELOG.md b/CHANGELOG.md index ed380570..40e92940 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -310,6 +310,14 @@ The first public release. It contains: `"none"` shows nothing. Plane stores keep it as the `run_summary` annotation (`solprop.driver.summary`; a reporter receives it through the optional `SummarisingReporter.summary` call). +- **Performance tracking** for developers + ([SEP 38](https://solprop.readthedocs.io/en/latest/dev/seps/sep-0038-performance-tracking.html)): + `python -m benchmarks track` times complete runs of about a dozen representative + workloads, each with its interface's default integrator and tolerances, at one + thread and the production width in about an hour per machine, and records them in + `benchmarks/history/` by hardware. It replaces SEP 30's standard suite, whose + reference qualification, native coverage and export are removed; `ab` compares two + builds on the same worker. - **Documentation.** An introduction with executable lessons, a user guide, a long-form primer on the physics and numerics, focused tutorials and worked examples, the API reference, and a developer's guide with the design records (SEPs and ADRs). diff --git a/benchmarks/README.md b/benchmarks/README.md index e214ac6e..114cbac4 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,18 +1,19 @@ # Benchmarks -The standard benchmark suite -([SEP 30](../docs/dev/seps/archive/sep-0030-standard-benchmark-suite.md)); its manual is +Release performance tracking +([SEP 38](../docs/dev/seps/sep-0038-performance-tracking.md)); its manual is `docs/dev/benchmarking.md`. Run it on a quiet machine: ```bash -uv run poe bench # core set, quick mode, native CPU -uv run poe bench run --mode standard --backend numba_cpu -uv run python -m benchmarks compare RUN_A RUN_B +uv run poe bench # the plan and its estimate +uv run python -m benchmarks track --append-history # about an hour; records history/ +uv run python -m benchmarks history uv run python -m benchmarks ab --case CASE --baseline-python OLD/bin/python # fast A/B ``` -Runs and machine profiles go to `.benchmarks/suite/`, which git ignores; the benchmark rules -and indicative results are in `docs/dev/benchmarks.md`. What has already been tried for CPU -performance is in `docs/dev/cpu-performance.md`. Earlier campaign harnesses, the retired -version-2 suite and their raw data are kept with the project's development records, not -in this repository. +`history/` holds one CSV per hardware set, a few rows per release. Run bundles go to +`.benchmarks/`, which git ignores; the benchmark rules and indicative results are in +`docs/dev/benchmarks.md`. What has already been tried for CPU performance is in +`docs/dev/cpu-performance.md`. Earlier campaign harnesses, the retired standard suite's +baselines and their raw data are kept with the project's development records, not in +this repository. diff --git a/docs/dev/backends.md b/docs/dev/backends.md index ea35d0c0..069e3131 100644 --- a/docs/dev/backends.md +++ b/docs/dev/backends.md @@ -1662,9 +1662,9 @@ L2 across a stage only if the working set is small. ## 7. Benchmarks -`benchmarks/` holds the standard suite (see [benchmarks.md](benchmarks.md)): bounded windows -of full-resolution problems, run in fresh processes, whose results stay on the machine that -ran them. Every performance claim in this document must point at a measurement recorded +`benchmarks/` holds the performance tracking suite (see [benchmarks.md](benchmarks.md)): +complete runs of representative problems, each in its own process, recorded per hardware +set in `benchmarks/history/`. Every performance claim in this document must point at a measurement recorded under [benchmarks.md](benchmarks.md#reporting-a-performance-claim)'s rules or be marked as an estimate. diff --git a/docs/dev/benchmarking.md b/docs/dev/benchmarking.md index 38f175ad..e9a07206 100644 --- a/docs/dev/benchmarking.md +++ b/docs/dev/benchmarking.md @@ -1,546 +1,136 @@ -# Running the benchmark suite +# Running the benchmarks -Run the four-case core suite with `uv run poe bench`. It measures bounded windows -of full-resolution problems and prints warm time, startup time, peak memory and -check status. Numerical reference preparation is optional for exploratory timing. -The runner is specified by -[SEP 30](seps/archive/sep-0030-standard-benchmark-suite.md). The rules it applies, -the measurement windows and what a performance claim reports are in -[Benchmarks](benchmarks.md); this page is the runner's manual. +`benchmarks/` times **complete runs** of a small set of representative workloads. Each +release adds a few lines per case to a history file kept per hardware set, so that +regressions and improvements show from one release to the next. The design is +[SEP 38](seps/sep-0038-performance-tracking.md), which replaced SEP 30's standard +suite. The rules for a performance claim and the indicative results are in +[Benchmarks](benchmarks.md); this page is the manual. ```sh -uv run poe bench --dry-run -uv run poe bench --out .benchmarks/suite/runs/first-look -uv run poe bench --case rdw --threads 1 +uv run poe bench # the plan and its estimated time +uv run python -m benchmarks track # the tracking run, about an hour +uv run python -m benchmarks history # what earlier runs recorded +uv run python -m benchmarks ab --case rdw --baseline-python OLD/bin/python ``` -The default uses the native CPU backend and requires it to be installed in the -environment. For a small first check without the native backend, use -`uv run poe bench --case rdw --backend numpy_ref --threads 1`. -The 384-square Kerr case is a substantial memory workload: its conservative -admission budget is about 36 GiB. The controller records a refusal if available -memory is insufficient; select individual cases on smaller machines. - -`poe bench` accepts the same arguments as `python -m benchmarks`. -Omitting a subcommand means `run`. No profile, baseline environment or reference -preparation is required for a quick run. Missing references are labelled -`not-qualified` and do not prevent timing. - -## Select a measurement - -Planning imports metadata only; it does not import NumPy, initialise a backend, -translate a simulation or prepare references: - -```sh -uv run python -m benchmarks list -uv run python -m benchmarks run --dry-run --set core -uv run python -m benchmarks list --all -``` - -The table includes conservative memory budgets and reference receipt status. -Planning checks reference metadata compatibility only; the worker verifies array -hashes and numerical results. Missing cost measurements are shown as unknown. - -The default selection is `core`, `quick`, `native`, at the available production -width. Declare a machine profile before a standard comparison. The alias must be -suitable for a public report; the width must fit the current allocation: +Every command uses the native CPU backend unless `--backend` says otherwise, and needs +it installed (`uv sync --extra native`). Run on a quiet machine: a dedicated +workstation, or an exclusive allocation on a cluster node. + +## The tracking set + +`python -m benchmarks list` prints the plan: one line per case, stepper and width, and +the estimated time from the costs each case declares (complete runs on an AMD EPYC 7763 +node; other machines differ). With no selection it is the tracking set: + +| Case | Interface | Problem | Widths | +|---|---|---|---| +| `gnlse_raman_shock` | `prop_gnlse` | Dudley–Genty–Coen supercontinuum, 0.15 m | 1, production | +| `gnlse_soliton` | `prop_gnlse` | N = 3 soliton, one soliton period | 1, production | +| `rdw` | `prop_capillary` | He resonant dispersive wave, four modes, 1.5 m | 1, production | +| `he_rdw_averaged` | `prop_capillary` | the same, mode-averaged | 1, production | +| `argon_blueshift` | `prop_capillary` | Ar ionisation blueshift, 0.2 m | 1, production | +| `argon_blueshift_averaged` | `prop_capillary` | the same, mode-averaged | 1, production | +| `argon_blueshift_envelope` | `prop_capillary` | the same on the envelope grid | 1, production | +| `h2_raman_averaged` | `prop_capillary` | H2 soliton self-frequency shift, 0.4 m | 1, production | +| `ctc` | `prop_capillary` | H2 comb-to-continuum, three modes, 0.13 m | production | +| `radial_filament_256` | `prop_radial` | Ar filament, 256 radial nodes, 3 cm | 1, production | +| `grin_nonlinear` | `prop_waveguide` | four-channel graded-index fibre, 2 cm | production | +| `kerr_beam_384` | `prop_free` | fused-silica Kerr beam, 384², fixed RK4IP steps | production | +| `kerr_beam_192` | `prop_free` | the same at 192² | 1 | + +The definitions are in `benchmarks/registry.py` and `benchmarks/builders.py`. Every +other registered case (the remaining modal, free-space and advanced-guide cases, and +the backend exercises) runs when named with `--case`, `--set` or `--all`; none is in +the default selection. A case whose definition changes keeps its name and raises its +revision, and the history compares rows only within one revision. + +## What is timed + +Each configuration (case, stepper, width) runs in its own worker process, whose thread +pools are set in the environment before any scientific import. The worker: + +1. builds the case, translates it and constructs the backend, timing these separately; +2. runs an untimed warm-up over the first 2 % of the guide, which builds FFT plans and + fills caches; +3. times `--repeats` complete runs (default 3). Each restarts the backend from the + translated launch and goes through `solprop.driver.run.run_problem`, the loop of + `Simulation.run`, so it includes the plane observations and statistics. + +The result is the median of the repeats, with the minimum and maximum beside it, the +accepted and rejected steps, the source evaluations, the peak resident memory, and the +integrator and tolerances the run actually used. Repeats of one configuration take the +same steps; the summary says so when they do not. + +**Library defaults.** A case is the interface call a user would write. It passes no +tolerance or integrator, so it runs with its interface's defaults +([SEP 37](seps/sep-0037-noise-and-stepper-defaults.md)). A changed default therefore +shows in the history as a new integrator or tolerance on that row. Every noisy launch +fixes the same seed, so its repeats and its releases are comparable. The free-space +cases, whose interface has no default tolerance, keep their own controller; the Kerr +beams run their declared fixed RK4IP schedule. + +**Steppers.** `--stepper` takes `default`, `multistep`, `multistep-interaction` or `rk`, +or a comma-separated list to compare steppers in one run: ```sh -uv run python -m benchmarks profile .benchmarks/suite/profiles/workstation.json \ - --alias workstation --production 8 -uv run python -m benchmarks run --set core \ - --machine-profile .benchmarks/suite/profiles/workstation.json +uv run python -m benchmarks track --case rdw --case ctc --stepper default,rk ``` -Use `--case` or `--set` repeatedly to select a deduplicated union. `--all` selects -all sets, including visible pending definitions. A pending definition produces an -explicit exclusion; it never becomes a successful measurement. `--threads` accepts -positive integers and `production`, for example `1,production`. Quick mode defaults -to production; standard mode defaults to both widths. Unsupported tracks and widths -are reported explicitly. - -| Core case | Headline measurement | Fixed diagnostic | -|---|---|---| -| Four-mode RDW | 0.02–0.04 m, following a 0–0.02 m lead-in | Ten 2 mm entrance steps | -| Six-mode CtC | 0.00017–0.00034 m, following its lead-in | Ten 17 micrometre entrance steps | -| 384-square Kerr beam | Fixed RK4IP only, one lead-in step and ten 0.5 micrometre steps | The same row | -| 256-node radial filament | 0–0.03 m, four observation planes | Ten frozen RK pilot steps, ending at 0.274524 mm | +An override replaces the integrator only; the tolerances stay the interface's. A +fixed-step case admits `default` only, and the plan says which it skipped. -RDW, CtC and radial headline rows each use default `Multistep()` and `RungeKutta()`. -The original guides, gas-flow profiles and grids remain intact. The measurement -window does not redefine a pressure-gradient guide's endpoint. -The radial diagnostic uses the first ten accepted steps of the NumPy default-RK -pilot at `rtol=1e-6`; its edges are stored with the registry and included in the -measurement identity. It does not run a pilot during ordinary collection. +**Widths.** Without `--threads`, each case runs at its declared widths: one thread and +the production width, or one of them for the most expensive cases. `--threads 1,8` +names the widths for every selected case. The production width is the number of +physical cores in the process's CPU allocation unless `--production` gives it; on a +cluster, give it explicitly. -The `modal-capillary` and `mode-averaged` sets add nine bounded cases. Both -adaptive methods and a ten-step fixed RK diagnostic are available on NumPy, -Numba and native CPU. Select Numba with `--backend numba_cpu`; its first window -includes JIT compilation or cache loading in startup, before warm timing. -The existing on-disk Numba cache is recorded as uncontrolled, so startup should -not be presented as a reproducible cold compilation measurement. +## Recording a run ```sh -uv run poe bench run --set mode-averaged --backend numba_cpu -uv run poe bench run --case argon_blueshift --track fixed-rk --threads 1 +uv run python -m benchmarks track --append-history ``` -| Additional case | Adaptive entrance interval | Fixed diagnostic | -|---|---|---| -| One resolved mode RDW | 0–39.1377 mm | Ten frozen pilot steps | -| Reduced two-mode CtC, 2 ps (revision 2) | 0–1.31235 mm | Ten 120 micrometre steps | -| Resolved argon blueshift | 0–2.42717 mm | Ten frozen pilot steps | -| Resolved hydrogen Raman, 6 ps | 0–13.2409 mm | Ten frozen pilot steps | -| Averaged helium RDW | 0–39.3310 mm | Ten frozen pilot steps | -| Averaged argon blueshift | 0–2.43336 mm | Ten frozen pilot steps | -| Averaged hydrogen Raman, 6 ps | 0–13.2409 mm | Ten frozen pilot steps | -| Ideal N=3 GNLSE soliton | 0–90.0316 mm | Ten frozen pilot steps | -| Raman/shock GNLSE | 0–0.945275 mm | Ten frozen pilot steps | - -These windows end at the tenth accepted step of independent NumPy RK pilots at -`rtol=1e-6`, `atol=1e-10`. The registry retains the full precision of every edge. -Each adaptive window saves its endpoint with ordinary statistics. The full guides, -including the helium pressure profile over 1.5 m, are preserved. The averaged -cases use the effective-area route; the resolved cases keep transverse quadrature -even with one mode. The two hydrogen cases retain the explicitly named 6 ps grids. -The ideal soliton retains the lesson's full-period guide but propagates only the -short entrance interval. These nine adaptive cases have a reviewed relative field -ceiling of `1e-4`, independent of the solver tolerance. Preparing or selecting -reference receipts remains separate from ordinary timing. - -The `free-space` set also adds these windows. The six adaptive additions use -the same ten-step NumPy pilot protocol, with `atol=0`; both default methods and -the frozen fixed-RK diagnostic are available. Kerr remains fixed-RK4IP only. - -| Additional case | Measured interval | Fixed diagnostic | -|---|---|---| -| Radial filament, 512 nodes | 0–274.524 micrometres | Ten frozen pilot steps | -| Cartesian filament, 128-square | 0–225.110 micrometres | Ten frozen pilot steps | -| Cartesian filament, 256-square | 0–225.111 micrometres | Ten frozen pilot steps | -| Line, 128 points | 0–10.8716 mm | Ten frozen pilot steps | -| Planar, two modes by 32 points | 0–6.79474 mm | Ten frozen pilot steps | -| Radial absorber, 64 nodes | 0–10.8716 mm | Ten frozen pilot steps | -| Kerr beam, 192-square | 0.5–5.5 micrometres | Ten 0.5 micrometre steps | -| Kerr beam, 768-square | 0.5–3 micrometres | Five 0.5 micrometre steps | - -Kerr windows follow one lead-in step. The 768-square case excludes one-thread -selection unless `--diagnostic` is explicit. Memory admission still applies: -large Cartesian cases can be refused before construction on a smaller machine. -Line, planar and radial-absorber cases have reviewed `1e-4` relative field -ceilings. The larger filament additions remain unqualified for accuracy. - -The `advanced` set preserves the source guides, coherent wall models and taper -corners. Each case supports both adaptive methods and a fixed RK diagnostic: - -| Additional case | Adaptive entrance endpoint | Fixed diagnostic | -|---|---:|---| -| Scalar coherent wall | 200 micrometres | Ten 20 micrometre steps | -| Vector TE/TM coherent wall | 200 micrometres | Ten 20 micrometre steps | -| Taper without wall | 137.401 micrometres | Ten 20 micrometre steps | -| Taper with scalar wall | 137.889 micrometres | Ten 20 micrometre steps | -| General step-index HE11 pair | 135.895 micrometres | Ten frozen pilot steps | -| Linear two-channel GRIN | 19.2601 micrometres | Ten frozen pilot steps | -| Nonlinear four-channel GRIN | 620.057 micrometres | Ten frozen pilot steps | -| Moving solid guide | 21.8692 micrometres | Ten frozen pilot steps | - -The linear GRIN case keeps the continuous-wave two-channel launch and its -0.784680 mm self-imaging guide; its temporal grid retains the source's 1 mm -length. Its ten accepted pilot steps include three rejected trials. Nonlinear -GRIN retains input-noise seed 41. The moving guide retains its 5 mm taper corner -in the unexecuted schedule tail. Weak-absorber and grazing-incidence warnings -from the wall tutorials are declared by category and message. General-guide -construction can dominate startup, independently of the short measured interval. -All eight advanced cases have reviewed `1e-4` relative field ceilings, with -reference and policy revisions independent of the workload. - -The `backend-exercise` set covers particular execution paths on small problems: - -| Exercise | Measured entrance endpoint | Tracks and observations | -|---|---:|---| -| Vector rotational Raman | 0.291590 micrometres | Both adaptive methods; ten fixed pilot steps | -| Envelope plasma | 8 micrometres | Both adaptive methods; four original 2 micrometre fixed steps | -| Helium/argon pressure gradient | 21.8703 mm | Both adaptive methods; ten fixed pilot steps | -| Dispersive Kerr | 15.2189 mm | Both adaptive methods; ten fixed pilot steps | -| Band/trapezoid plasma integration | 135.895 micrometres | Two cases, both adaptive methods; ten fixed pilot steps | -| Batch | 1 mm | Two distinct GNLSE members, ten fixed RK steps | -| Physical noise | 135.895 micrometres | Both adaptive methods; ten fixed pilot steps; seed 13 | -| Dense output | 1.45795 mm | Both adaptive methods; eight interpolated observations | -| Split-step absorber | 135.895 micrometres | Both adaptive methods; ten fixed pilot steps | -| Ordinary statistics | 135.895 micrometres | Both adaptive methods; ten fixed pilot steps | -| Far-field statistics | 863.801 micrometres | Both adaptive methods; eight observations with only the exit field retained; ten fixed pilot steps | - -Vector Raman uses the current optmat rotational hydrogen response with vibration -disabled, retaining the circular pump and delayed oblique probe launch. Its -low-density linewidth warning is recorded. The envelope-plasma guide ends after -four source steps, so that shorter interval is retained. The other new windows -use ten accepted default-RK pilot steps. Dense output visits intermediate planes -without forcing the integrator to land there. Far-field observations include -apertures, spectral regions and periodic absorbed power. Fixed diagnostics time -propagation alone and report observation costs separately. - -The batch workload uses the actual backend batch implementation, retaining each -member's own statistics, source counts and field check. Reset reuses member -buffers and FFT plans. These exercises can be timed before reference review; -their numerical evidence is collected separately through `qualify`. The eleven -adaptive exercises have reviewed `1e-4` relative field ceilings. The fixed batch -check remains a matched-method diagnostic. - -## Timing boundaries and accounting - -These boundaries implement the -[measurement windows](benchmarks.md#measurement-windows) of the benchmark policy. -Each sample runs in a fresh process. Its thread environment is set before scientific -imports, with `solprop_native` loaded before pyFFTW. Budget policies clear inherited Kokkos -pool overrides and the FFT-size override. `all-pools-N` explicitly forces the pools; -`as-installed` preserves the environment. This keeps an automatic-policy measurement -separate from a forced-width measurement. The backend and FFT plans are constructed -once. After a first complete window, the worker calls `set_initial`, repeats the -bounded lead-in outside warm timing, and measures the declared window again. - -Adaptive timing includes propagation, declared observations, ordinary statistics -and in-memory output. Fixed timing includes propagation; the final observation and -statistics are measured separately. Fresh-process startup is measured by the parent -through receipt of the first completed window. Process startup and warm propagation -answer different questions and are reported separately. - -`source_evaluations` counts actual Runge–Kutta source dispatches since the most recent -`set_initial`. Rejected trials contribute work; a reused stage contributes no new -evaluation. NumPy may therefore report more evaluations than a backend that reuses -DOPRI5's last stage. Adams retains its existing response-call ledger, and low-storage -methods retain their existing evaluation counters. These are execution facts, not -changes to the simulation or kernel-contract schemas. +The run's bundle goes to `.benchmarks/track/-/` (git ignores it): the +plan, each worker's request and logs, and `results.json`. `--archive DIR` copies the +bundle to a directory outside the repository. -The worker reports lifetime peak RSS after measurement and before reference loading. -Allocation-ledger bytes are reported separately. Unsupported telemetry carries an -unavailable reason; a missing counter is not zero. Admission estimates are conservative -provisional budgets until compatible empirical receipts are qualified. An unknown -estimate requires `--allow-unestimated` and an explicit `--memory-limit` in bytes; -such a collection is exploratory. -Each attempt also reports `controller_cpu_seconds`: CPU time consumed by the parent -while launching, monitoring and reading the worker. It includes process sampling -and receipt handling; it is separate from the child's warm propagation time. +`--append-history` adds one row per configuration to +`benchmarks/history/.csv`, which is committed. It refuses a run with a +failed configuration, a backend other than native, and a source tree with uncommitted +changes (unless `--allow-dirty`). The hardware name is the CPU model and the physical +cores used, for example `apple-m5-max-18c` or `amd-epyc-7763-16c`; it never contains a +host name. A cloud instance type, from `--instance-type` or the machine's DMI record, +goes in its own column. -## Collection, recovery and comparison +`python -m benchmarks history` prints, per hardware and width, one line per recorded +run: its date, solprop version, case count, and the geometric mean of its median times +over the first recorded run's, over the configurations both ran with the same case +revision and integrator. Below one is faster. -Quick mode takes one sample per configuration and has no adoption verdict or MAD -estimate. Standard mode starts with five samples per configuration, uses balanced -shuffled rounds, and alternates adjacent baseline/candidate order. A short separate -pilot can select up to ten individually timed repetitions; the same count is frozen -for both arms. No repetition is selected merely because it was fastest. +## Comparing two builds while iterating -For a first look, collect the same selection before and after a change, then compare -the saved bundles. Quick comparison prints descriptive percentage changes even -without qualified references or a clean baseline; it does not approve adoption: +`ab` times the same configurations with two prepared environments, for a change in +progress: ```sh -uv run poe bench compare .benchmarks/suite/runs/before .benchmarks/suite/runs/after -``` - -Use `compare --json` for the detailed machine-readable result, including resource -flags. For a controlled paired comparison, prepare both Python environments first: - -```sh -uv run python -m benchmarks run --mode standard --set core \ - --machine-profile .benchmarks/suite/profiles/workstation.json \ - --baseline-python /path/to/baseline/bin/python \ - --candidate-python /path/to/candidate/bin/python \ - --out .benchmarks/suite/runs/comparison -uv run python -m benchmarks compare .benchmarks/suite/runs/comparison -``` - -The controller admits one measurement worker at a time. A guard miss defers the -whole pair once in standard mode, then records contention if it persists. Contended -samples remain in all-sample tables. Clean paired summaries omit both members if -either is contended. Unknown guard confidence cannot establish a clean comparison. -`--force` skips waiting and explicitly makes the collection exploratory. - -Run bundles live below the ignored `.benchmarks/suite/` directory unless `--out` -overrides it. They contain a manifest, immutable attempt shards, raw worker logs, -and derived `samples.csv`, `summary.csv` and `accuracy.csv` tables. A run lock prevents -concurrent writers. Temporary partial files do not count as completed attempts. -Use `run --resume RESULTS` to restore the saved selection and options and retry an -interrupted run; completed sample identities are skipped and previous failures remain -on disk. Worker files left by a canceled collector remain intact; a retry uses a -fresh attempt number. An explicit conflicting option refuses resume. -Changed harness, environment, selection or machine identity refuses resume. - -Comparison requires the complete affected selection and compatible workload, -resolved method, measurement and profile identities. Build identities may differ. -Adaptive and fixed tracks, and different widths, have separate aggregates. Standard -adoption applies the aggregate rule of [the benchmark rules](benchmarks.md#rules) and -additionally requires qualified numerical references, sufficient clean paired rounds -and a gain exceeding measured adjacent-round repeatability. More than 20% contention, -insufficient adjacent clean rounds, or missing qualification leaves the result -unresolved. Missing rows are never removed to improve the aggregate. Peak-memory -and startup regressions have separate advisory flags. - -Use `--allow-accuracy-failure` when timing should continue despite failed numerical -checks. It preserves each failed check and its measured error, completes timing -rounds, and labels the entire collection exploratory. Successful execution then -exits zero even if an accuracy check failed; crashes, timeouts and memory refusals -still fail. This option does not turn a failed reference check into a pass, and an -override collection cannot establish adoption: - -```sh -uv run python -m benchmarks run --case rdw --allow-accuracy-failure \ - --out .benchmarks/suite/runs/rdw-exploratory -``` - -## Fast A/B comparison while iterating - -A standard paired comparison is a final check, not an iteration tool. Each sample is -a fresh process that imports, constructs the case, runs a first window and only then -its warm repetitions; standard mode adds paired pilots, a counter guard before every -pair and five to eleven rounds. Construction dominates: in a standard run of the -advanced set on a 36-core workstation it took about 70 % of the collection time, the -first and timed windows about 9 % each, and guards, process start and validation the -rest. The whole set at two widths took about five hours, though most windows last -milliseconds to seconds. - -While developing a performance change, time only the cases it should affect with the -`ab` command, and keep the standard comparison for the settled change: - -```sh -uv run python -m benchmarks ab --case step_index_pair --case taper_wall \ - --track adaptive-rk --threads 1 \ - --baseline-python /path/to/baseline/bin/python \ +uv run python -m benchmarks ab --case rdw --case argon_blueshift --threads 1 \ + --length 0.2 --baseline-python /path/to/baseline/bin/python \ --candidate-python /path/to/candidate/bin/python ``` -Selection (`--case`, `--set`, `--all`, `--track`, `--threads`, `--backend`, -`--machine-profile`) works as for `run`, defaulting like a quick run to the adaptive -tracks at the production width. Cases, measurement windows and lead-ins come from the -registry unchanged. Each round starts one worker per build and thread width. The -worker constructs each selected case once, runs one untimed warm-up window, then -times `--repeats` windows (default 5), each after the usual reset and lead-in. There -are `--rounds` rounds (default 3), and the build that runs first alternates between -rounds. A selection of a few cases therefore costs a few constructions per build -instead of one per sample: four advanced cases at one track compare in a few minutes. - -The command prints, per configuration, each build's median window time, the median -over rounds of the paired candidate/baseline ratio of per-round medians (below 1 means -the candidate is faster), the range of those ratios over rounds, whether both builds -took the same accepted and rejected steps, and the relative L2 difference between the -two builds' retained fields from the first round. A geometric mean of the ratios -follows. Passing the same interpreter for both builds gives an A/A control, whose -ratio range shows the noise to expect on that machine. - -Results go to `.benchmarks/suite/ab/` unless `--out` names a new directory: -`plan.json`, `results.json` with every timed window and the runtime identity of each -worker, the raw worker logs and the saved fields. The guard takes one one-second -reading before each worker and labels a busy machine, but never waits. There is no -pilot, reference check, memory sampling, resume or adoption verdict; the batch -exercise is not supported. Use a quiet machine, and confirm a promising result with a -standard comparison. - -## Numerical qualification - -Reference preparation is explicit and runs outside ordinary timing collection: - -```sh -uv run python -m benchmarks qualify --case rdw --threads 1 \ - --out .benchmarks/suite/references/rdw-review -uv run python -m benchmarks run --case rdw \ - --references .benchmarks/suite/references/rdw-review -``` - -Qualification is a one-time preparation step for each reference revision. Routine -timing validates the first successful sample for each arm/configuration after the -timer and memory receipt. Later samples check finite output and the validated shape, -and link to the original validation rather than repeating its error measurements. -A changed reference/policy or a failed sample requires a fresh check. Timing does not -run a reference convergence campaign. The qualification path retains tight RK at relative tolerance `1e-10`, a -`1e-9` refinement witness, a matched-method NumPy result, a candidate result, reset -evidence and array hashes. It evaluates the worst-plane relative field error in the -requested band and preserves the full-band error. Fixed methods use a same-schedule -NumPy reference. Qualification runs also retain fresh-versus-reset fields, work -counts, observed controller histories, statistics and noise state. RDW retains its -weak-bin and 100–150 nm band metrics; CtC retains its weak-bin metric. Modal energy -differences use the ordinary Parseval weights and are diagnostic only. - -An empty mode can have an undefined spectral centroid, and a continuous wave -can have an undefined pulse width. Reset evidence records these statistic masks -and compares the finite entries separately. Non-finite fields, histories or -infinities remain errors. - -The benchmark's solver tolerance remains `rtol=1e-6`. Its separate accuracy -envelope bounds the measured field and retained-band errors against the reference: - -| Adaptive case | Largest pilot error over both integrators | Reviewed ceiling | -|---|---:|---:| -| RDW | 2.045 × 10⁻⁵ (UV band) | 1 × 10⁻⁴ | -| CtC | 2.055 × 10⁻⁶ (weak bins) | 1 × 10⁻⁵ | -| Radial 256 | 1.206 × 10⁻⁶ (worst-plane field) | 1 × 10⁻⁴ | -| Nine added modal/GNLSE cases | 6.013 × 10⁻⁷ (relative field) | 1 × 10⁻⁴ | - -These ceilings reuse target levels from the source campaigns with headroom for -ordinary rounding and solver variation; they are not fitted to timing results. -The approved defaults match exact measurement identities. A changed workload must -be reviewed again. Other measurements without a reviewed positive envelope remain -`not-qualified`. The explicit `--accuracy-bound` option supplies a reviewed -replacement ceiling. Adaptive field and band ceilings must remain at least ten -times their respective measured reference refinement floors. -An envelope pass alone does not qualify adoption. After reviewing the retained -reset fields, controller histories, statistics and work counts, use -`qualify --recheck ... --reviewed` to mark that reference eligible for standard -comparison. This checks that all four reference runs succeeded and retained reset -evidence, with reset fields inside the same approved field ceiling. History and -statistics need their case-specific numerical interpretation; this option confirms -that review without imposing identical adaptive trajectories. A changed policy does -not inherit a previous review automatically. Ordinary timing needs neither this -review nor a reference. Clean-pair, warning and resource checks remain part of each -standard comparison. - -### Revise an accuracy policy or reference - -Workload, reference, accuracy-policy and result-schema revisions are independent. -An improved reference or a revised accuracy limit does not rename an unchanged -benchmark case. Receipts are immutable, carry the source/build evidence, and link -to the receipt they supersede with a revision note. - -To reassess retained evidence with a reviewed ceiling, without executing another -simulation: - -```sh -uv run python -m benchmarks qualify --case rdw \ - --recheck .benchmarks/suite/references/rdw-v1 --accuracy-bound 2e-4 \ - --revision-note "Reviewed margin after numerical investigation" \ - --out .benchmarks/suite/references/rdw-v2 -``` - -The ceiling in this example illustrates the command; it is not an additional -approved RDW default. Rechecking verifies artifact hashes, reevaluates retained -errors and refinement floors, and writes a new receipt. It cannot fix an incorrect -reference field. After fixing a reference implementation, prepare fresh evidence: - -```sh -uv run python -m benchmarks qualify --case rdw \ - --supersedes .benchmarks/suite/references/rdw-v1 \ - --revision-note "Recomputed after correcting the reference implementation" \ - --out .benchmarks/suite/references/rdw-v2 -``` - -Replacement reference revisions advance automatically; a changed ceiling also -advances the policy revision. Explicit `--reference-revision` and `--policy-revision` -values may advance these further. Both comparison arms must be validated against -the same receipt to support adoption. Old bundles remain historical evidence under -their original policy; changing a ceiling does not rewrite their accuracy results. -Revalidation against a replacement needs retained output fields or a new collection. - -Warnings are captured with phase, category, message and occurrence count. A case -may declare a narrow expected pattern; undeclared warnings prevent adoption. -The Kerr recipe retains its source's Malitson UV extrapolation warning explicitly. - -## Native source coverage - -Coverage is an explicit developer diagnostic. It reports how much of the compiled -native CPU backend the selected benchmark cases execute; it is not run by ordinary -timing commands and has no automatic notifications or numerical target. - -Prepare a separate Clang CPU build of `native/` with -`SOLPROP_NATIVE_ENABLE_COVERAGE=ON` as described in `native/README.md`. Use LLVM tools from -the same compiler release. Keep the ordinary timing environment separate: timing -commands refuse an instrumented build. Then, from the repository root: - -```sh -uv run python -m benchmarks coverage --set backend-exercise \ - --python /path/to/coverage-environment/bin/python \ - --native-root native \ - --llvm-bin /path/to/matching/llvm/bin > coverage.log -``` - -The command prints a redirectable line/function/branch summary and the local report -path. Defaults are the core set and one thread. All admitted tracks of each case -are included; `--case`, `--set`, `--all`, `--track`, `--threads` and `--diagnostic` -select a narrower or broader diagnostic. Memory admission and timeouts still apply. -Failed, refused and excluded configurations remain visible. - -Each configuration runs once, including construction, its declared lead-in and -observations. There are no warmups, calibration loops or timing samples. A separate -second execution collects Kokkos Tools labels. Capability tags describe intended -workloads; observed labels show which named operations executed. Neither implies -that every branch of a capability was exercised. - -Reports default to ignored `.benchmarks/suite/coverage/`; `--out` chooses another -local directory. The bundle contains per-case, per-set and union HTML reports, -line/function/branch CSV views, labels, raw profiles and a copy of the exact binary. -They are local development artifacts, never committed automatically. Delete them -when no longer useful. Generated benchmark or coverage results do not belong in -the repository; only definitions, tests, policies and concise implementation notes -are versioned. `export` below writes an explicitly requested sanitized snapshot; -it does not publish or commit one. - -Atomic Clang counters support threaded CPU execution. Process/module-specific -profile names avoid collisions; source digests, binary hashes and profile hashes -prevent mixing builds. Dependency code is excluded by source path, including -dependency templates instantiated in a project translation unit. LLVM supplies -the line-count aggregation. Uncovered compiled functions and branch outcomes are -reported as misses. Unsupported configurations and compiled-out device capabilities -are separate. Uninstantiated templates and preprocessor-elided paths have no LLVM -coverage-map denominator; their counts remain unavailable, not zero. See the -[Clang coverage workflow](https://clang.llvm.org/docs/SourceBasedCodeCoverage.html). - -Coverage is informational and has no required percentage. A timeout or memory -refusal leaves the selection incomplete and returns a nonzero exit status, while -reports for successful configurations remain available. Inspect `summary.json` -and `configurations.csv` alongside the HTML union to see its actual scope. -Instrumented execution can be much slower than ordinary propagation; its elapsed -time is not a benchmark result. A retry at another thread width is a separate -collection and does not erase the original failure. - -## Local tables and public export - -Collection writes normalized CSV views for runs, cases, configurations, systems, -CPU allocations, measured processes and observed libraries alongside the existing -sample, summary and accuracy tables. Records link through run, configuration and -attempt identifiers. Missing telemetry carries a reason rather than a measured -zero. Library records distinguish the initial inventory from pools observed after -the measurement, including libraries loaded lazily by a backend. - -Linux allocation records include physical cores, sockets, sibling CPUs and NUMA -nodes when the operating system exposes them. Scheduler variables describe the -allocation but do not prove exclusivity. Missing topology or residency information -retains an explicit reason. - -Collect thread and binding details separately from a timing campaign: - -```sh -uv run poe bench diagnose --threads 1,production \ - --out .benchmarks/suite/thread-diagnostics -``` - -This runs the small diagnostic case at each width serially, recording per-thread -CPU time and, on Linux, allowed CPUs and resident NUMA pages. It does not change -affinity. An allowed CPU set is not evidence of where a thread actually executed. -These exploratory results cannot establish adoption. - -Export a separate public snapshot explicitly: - -```sh -uv run poe bench export .benchmarks/suite/runs/my-run \ - --out .benchmarks/public/my-run --alias apple-silicon-development -``` - -The destination must be new or empty. The export retains timings, failures, -contention, work counts, safe build identities and the selected configurations. -It omits hostnames, original machine aliases, local paths, process identities, -environment values, raw logs and free-form diagnostics. Its manifest states those -omissions. The command writes files; it does not commit or upload them. - -The runner and explicit native coverage command are ready for development use. -Clean machine baselines and complete reference checks for the largest spatial cases -remain follow-up work. Planning currently uses conservative memory budgets and -reports unknown runtime costs; automatic reuse of measured memory and cost receipts -is also a follow-up. These limits do not prevent exploratory timing. - -A performance baseline belongs to a particular source and build. Prepare a fresh -one after integrating performance changes, using matched workloads and thread -budgets when comparing Numba and native CPU. This does not require repeating -numerical reference preparation unless the reference's workload or numerical -assumptions changed. - -Indicative results of earlier measurements are summarised under -[indicative results](benchmarks.md#indicative-results). - -See [SEP 30, Part II](seps/archive/sep-0030-standard-benchmark-suite.md#part-ii--implementation-record) -for the implementation decisions, measured evidence and remaining limitations. +Selection works as for `track`. `--length` runs that fraction of each guide, from the +entrance, for quick iteration. Each round runs every configuration once with each +build, one worker each, and alternates which build goes first; `--rounds` sets the +count (default 3) and `--repeats` the timed runs per worker (default 3). The harness +is copied into the output directory first, so both builds run the same benchmark code. + +The summary gives, per configuration, each build's median time, the median over rounds +of the candidate/baseline ratio of the per-round medians (below one is faster), the +range of those ratios, whether both builds took the same steps, and the relative L2 +difference of their final fields. Passing the same interpreter twice is an A/A +control: its ratio range shows the noise to expect on that machine. A one-second +reading before each worker labels a busy machine but never waits. `ab` gives no +adoption verdict; the [rules](benchmarks.md#rules) say how a claim is judged. diff --git a/docs/dev/benchmarks.md b/docs/dev/benchmarks.md index f4f4904e..b2d20175 100644 --- a/docs/dev/benchmarks.md +++ b/docs/dev/benchmarks.md @@ -1,32 +1,23 @@ # Benchmarks How solprop's performance is measured and judged, and what the measurements so far -indicate. A performance claim names its measured window, build, machine and result; -the rules below say how to take it. The numbers in the last two sections are indicative: -each was measured once, on one machine, for one change. Re-measure with the standard -suite before relying on any of them. +indicate. A performance claim names what it measured, the build, the machine and the +result; the rules below say how to take it. The numbers in the indicative sections are +indicative: each was measured once, on one machine, for one change. Re-measure before +relying on any of them. ## Where things are -- **The standard benchmark suite** is `benchmarks/` - ([SEP 30](seps/archive/sep-0030-standard-benchmark-suite.md)), run with - `uv run poe bench` (`python -m benchmarks`). It measures bounded windows of - full-resolution problems in fresh processes and reports warm time, startup time, - peak memory and numerical check status. Its default is a quick Kokkos CPU run of the - four core cases (four-mode RDW, six-mode comb-to-continuum, a 384² Kerr beam and a - 256-node radial filament); further sets cover modal capillaries, mode-averaged and - GNLSE cases, free space, coherent walls, tapers, the general modal route and - particular backend paths. Quick mode takes one sample and gives no verdict; standard - mode takes paired, shuffled fresh-process samples at one thread and at the machine's - declared production width and applies the adoption rule below. Cases are versioned - by their workload, so a changed definition is a new revision and is never compared - with the old one. [Running the benchmark suite](benchmarking.md) is the manual: case - sets, timing boundaries, collection, numerical qualification, native coverage and - export. -- **Results stay local.** Runs, machine profiles, references and coverage reports go to - the ignored `.benchmarks/suite/` directory; no generated timing, reference or - coverage report is committed. `poe bench export` writes a sanitised snapshot when - one is wanted for a report. +- **The benchmarks** are `benchmarks/` + ([SEP 38](seps/sep-0038-performance-tracking.md)), run with `uv run poe bench` + (`python -m benchmarks`). `track` times complete runs of the tracking set (about a + dozen representative workloads, with each interface's default integrator and + tolerances) at one thread and the production width, in about an hour per machine; + `ab` compares two builds while a change is made. [Running the benchmarks](benchmarking.md) + is the manual. +- **The history** is `benchmarks/history/`: one CSV per hardware set, named by its CPU + model and core count, with a few rows per release. `python -m benchmarks history` + summarises it. Full run bundles stay in the ignored `.benchmarks/` directory. - **[CPU performance: what was tried](cpu-performance.md)** records the compiled CPU backend's performance experiments: what was adopted, measured and rejected, and what remains open. Read it before repeating an experiment. @@ -50,31 +41,26 @@ suite before relying on any of them. denominator counting as unresolved. Small regressions on individual cases, or at one thread, are accepted when the aggregate improves; they stay visible in the table and do not veto the change. Numerical bounds are never relaxed to make a change pass. - The standard suite's comparison adds its clean-pair, repeatability and qualified - reference requirements - ([collection and comparison](benchmarking.md#collection-recovery-and-comparison)). + The A/A spread of the same comparison says how large a difference has to be to count + ([comparing two builds](benchmarking.md#comparing-two-builds-while-iterating)). (benchmark-windows)= -## Measurement windows - -Timing is measured on short windows, not on complete propagations: a complete -comb-to-continuum run takes hours, and its cost is the window's times a step count the -window already predicts. - -- A timing window is about ten accepted steps of the adaptive run, frozen from an - untimed pilot so that every repeat takes the same prescribed steps, or a fibre - short enough that a complete run takes seconds. -- Report the window that was measured: its z extent, accepted steps, what the timed - region includes (propagation, observation and public statistics) and excludes - (translation, backend construction, HDF5), and the A/A resolution indicator of the - alternating pairs. Do not extrapolate a window to a complete-run cost. -- A later state of a long propagation is timed by replaying a saved witness into a - short window, not by regenerating the propagation. -- Compare two source trees with alternating fresh-process pairs (A/B/A/B…), and report the - A/A spread beside the ratio; a difference inside the spread is not a result. - -The suite's windows and what its timers include are listed under -[timing boundaries and accounting](benchmarking.md#timing-boundaries-and-accounting). +## What is timed + +- **Release tracking times complete runs.** The tracking set's cases are complete + guides at production sizes, with the library's defaults, so the history says how long + the usual simulations take now. Construction, translation and backend construction + are timed separately; an untimed warm-up builds plans first + ([what is timed](benchmarking.md#what-is-timed)). +- **A change in progress may time less.** `ab --length` runs part of each guide from + the entrance, which is fine for comparing two builds on the same work. Report the + fraction run, and do not extrapolate an entrance window to a complete-run cost: the + expensive part of a propagation (compression, ionisation, the Raman shift) often comes + late. +- **A later state of a long propagation** is timed by replaying a saved witness into a + short run, not by regenerating the propagation. +- **Compare two source trees with alternating pairs** (A/B/A/B…), and report the A/A + spread beside the ratio; a difference inside the spread is not a result. (benchmark-reporting)= ## Reporting a performance claim @@ -82,8 +68,8 @@ The suite's windows and what its timers include are listed under **The machine record.** Every measurement carries the CPU (or device) model, core count, memory, operating system, Python and dependency versions (including the Kokkos build and its profile), thread counts, FFTW plan flags and wisdom state, the source revision and -whether the tree was dirty, and the load before and after. The suite's machine profile -and run manifest record these; a hand-run measurement states them. Results are compared +whether the tree was dirty, and the load before and after. A tracking run's plan and +results record these; a hand-run measurement states them. Results are compared only within one machine. **A quiet machine.** Absolute numbers are taken on a dedicated workstation or an diff --git a/docs/dev/dependencies.md b/docs/dev/dependencies.md index 0cec795f..8e30cf8a 100644 --- a/docs/dev/dependencies.md +++ b/docs/dev/dependencies.md @@ -78,8 +78,8 @@ solprop-native has no runtime dependency on solprop. ## Development -The standard benchmark controller uses **psutil ≥ 7.1.0** for process ownership, -CPU counters, memory admission and sampled RSS. This floor includes the macOS C +The benchmark controller uses **psutil ≥ 7.1.0** for the CPU allocation, core +count, memory size and the busy-machine reading. This floor includes the macOS C safety fixes and Windows ARM64 wheels recorded in the [upstream 7.1.0 changelog](https://github.com/giampaolo/psutil/blob/master/docs/changelog.rst). Collection records the installed version. Unsupported or sandbox-denied counters diff --git a/docs/dev/roadmap.md b/docs/dev/roadmap.md index 8a23d214..d7366597 100644 --- a/docs/dev/roadmap.md +++ b/docs/dev/roadmap.md @@ -106,6 +106,10 @@ The [SEP index](seps/index.md) lists every proposal and its current status. subtraction, a weak-response limit and local temporal integration. Its ensemble, data and integration questions must be settled before acceptance; shared-level ladders, vector saturation and repeated-window histories remain future work. +- [SEP 38 — Performance tracking: complete runs, about an hour per machine](seps/sep-0038-performance-tracking.md): + replaces SEP 30's standard suite with a release-tracking command that times + complete runs with the library's defaults and keeps a small history per hardware + set; drafted with its implementation. The informal items below record work not fully covered by these proposals. When adding or revising a SEP, remove an informal item only if the proposal covers it in full; @@ -621,8 +625,8 @@ GPUs with slow double precision. Apple GPUs on the fibre, radial and Cartesian routes with every response but the refused ones. Its open questions: whether a rule comparing binary32 Adams with binary64 Adams should admit `Multistep()` under single precision (binary64 Adams has - the same errors at the tolerances the study used); `mlx` cases in the standard - benchmark suite (SEP 30); and a fused stage kernel for the launch-bound small plasma + the same errors at the tolerances the study used); `mlx` cases in the + benchmarks (SEP 38); and a fused stage kernel for the launch-bound small plasma capillaries. Open beyond it: single precision on `native` and `numba_cpu` (Kokkos, Kokkos Kernels and Kokkos-FFT support it on every backend in use), on the routes SEP 33 refuses, and noise models under single @@ -631,13 +635,10 @@ GPUs with slow double precision. would need a single-source C++ compiler for Metal (the runtime part is modest: 5 to 10 thousand lines for each existing GPU backend). -**Benchmark platform qualification beyond the initial CPU suite.** -[SEP 30](seps/archive/sep-0030-standard-benchmark-suite.md) covers the standard suite, portable -records and durable incremental exports, with CPU timing collections on macOS and -Linux and an initial native coverage collection on macOS. Linux native coverage, -Windows execution, cloud launch/collection recipes and GPU timing and coverage -need separate qualification. Keep future platform results in distinct machine/build -records; successful CPU coverage does not establish coverage of device code. +**Benchmark platforms beyond CPU tracking.** +[SEP 38](seps/sep-0038-performance-tracking.md) tracks native CPU runs per hardware set. +GPU tracking, Windows execution and cloud launch recipes need their own work; keep +their results in distinct hardware records. **Remaining benchmark references and estimates.** Complete independent reference checks for the 512-node radial filament, 128/256-square Cartesian filaments and diff --git a/docs/dev/seps/index.md b/docs/dev/seps/index.md index 14c2fe53..679eafca 100644 --- a/docs/dev/seps/index.md +++ b/docs/dev/seps/index.md @@ -37,6 +37,7 @@ Proposals that are active, in draft, accepted, in progress or deferred. | [21](sep-0021-absorbing-boundaries.md) | Temporal boundaries: wider windows, absorption and the escaped field | Draft | Change | | [32](sep-0032-raman-maxwell-bloch.md) | Raman Maxwell–Bloch dynamics | Draft | Feature | | [37](sep-0037-noise-and-stepper-defaults.md) | Shot noise on by default, and an integrator and tolerances chosen for each interface | In progress | Change | +| [38](sep-0038-performance-tracking.md) | Performance tracking: complete runs, about an hour per machine | Draft | Process | SEP 13, the standalone numerical prerequisite for SEP 12, SEP 14, which carried it to the compiled backends, and SEP 16, which unified the multistep family and made it the @@ -92,6 +93,7 @@ sep-0017-gas-two-photon-ionisation sep-0021-absorbing-boundaries sep-0032-raman-maxwell-bloch sep-0037-noise-and-stepper-defaults +sep-0038-performance-tracking template archive/index ``` diff --git a/docs/dev/testing.md b/docs/dev/testing.md index 78866f7b..f360c50c 100644 --- a/docs/dev/testing.md +++ b/docs/dev/testing.md @@ -63,8 +63,8 @@ instead of a tier marker: - `benchmark` is run by **no lane at all**. It is where a study goes whose numbers are recorded in its docstring and on which nothing in the default configuration depends — the full-size CtC case, the gigabyte Φ table, the polarisation taper's recurrence - floor. Each says in its own docstring how to run it by hand; the benchmark suite - (`poe bench`) is a separate program and does not reach them. + floor. Each says in its own docstring how to run it by hand; the benchmarks + (`poe bench`) are a separate program and do not reach them. ### The lanes diff --git a/docs/dev/tooling.md b/docs/dev/tooling.md index 1ca48636..a1bc2dc5 100644 --- a/docs/dev/tooling.md +++ b/docs/dev/tooling.md @@ -38,7 +38,7 @@ sequence plus `poe test-slow` and the CPU coverage lane; keep the two identical. Other tasks: `poe test-medium` (the medium tier alone, timed against its budget), `poe test-slow`, `poe cov` (interpreted bodies, then the compiled suite), `poe bench` -(the standard benchmark suite in `benchmarks/`, on a quiet machine; +(the benchmark plan in `benchmarks/`; `track` runs it on a quiet machine; [manual](benchmarking.md)), `poe docs`, `poe docs-clean`, `poe figures` (redraw the committed structure and geometry SVGs) and `poe docs-examples` (execute the worked examples and refresh their committed output). diff --git a/docs/guide/performance/index.md b/docs/guide/performance/index.md index 0dc48653..89490fc5 100644 --- a/docs/guide/performance/index.md +++ b/docs/guide/performance/index.md @@ -9,7 +9,7 @@ cover the optional native build, memory, CPU threads, and GPU execution. To measure a change's performance, or to see roughly what each backend and method costs, read the developer pages: [Benchmarks](../../dev/benchmarks.md) gives the measurement rules and indicative results, and -[Running the benchmark suite](../../dev/benchmarking.md) is the manual for +[Running the benchmarks](../../dev/benchmarking.md) is the manual for `uv run poe bench`. ```{toctree} From 91113b4dd88d8296f5b86e2c86e5b20580325d0a Mon Sep 17 00:00:00 2001 From: John Travers Date: Tue, 27 Oct 2026 07:15:39 +0000 Subject: [PATCH 04/13] Give the nonlinear GRIN case its planes and stepper at construction Its coupled launch translates the run when it is built, so the save planes and an integrator override cannot be set afterwards. A slow test covers it. Co-Authored-By: Claude Opus 5.5 --- benchmarks/builders.py | 17 +++++++++++++++-- benchmarks/guides.py | 27 ++++++++++++++++++++++----- tests/test_benchmark_guides.py | 14 ++++++++++++++ 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/benchmarks/builders.py b/benchmarks/builders.py index bbfb418a..3169909a 100644 --- a/benchmarks/builders.py +++ b/benchmarks/builders.py @@ -13,7 +13,7 @@ from __future__ import annotations import dataclasses -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from benchmarks.records import Record, SuiteError from benchmarks.registry import CASES, SEED @@ -453,7 +453,20 @@ def build(configuration: Record) -> Simulation: elif case.builder == "advanced": from benchmarks.guides import advanced # noqa: PLC0415 - simulation = advanced(case.name, case.length) + options: dict[str, Any] = {"save_planes": itinerary} + if chosen is not None: + options["integrator"] = chosen + simulation = advanced(case.name, case.length, options) + if case.name == "grin_nonlinear": + # Translated at construction (its launch is coupled): only the backend + # and the field retention may still change. + simulation.settings = dataclasses.replace( + simulation.settings, + backend=configuration["backend"], + fields=fields, + field_planes=observed["field_planes"], + ) + return simulation elif case.builder == "capillary": simulation = capillary(case.name) elif case.builder == "gnlse": diff --git a/benchmarks/guides.py b/benchmarks/guides.py index 98b521d5..76bb2d72 100644 --- a/benchmarks/guides.py +++ b/benchmarks/guides.py @@ -13,6 +13,8 @@ from collections.abc import Mapping from dataclasses import dataclass +from types import MappingProxyType +from typing import Any import numpy as np from jaxtyping import Complex @@ -170,13 +172,20 @@ def step_index() -> Simulation: ) -def grin(*, nonlinear: bool, length: float = 0.1) -> Simulation: +def grin( + *, + nonlinear: bool, + length: float = 0.1, + options: Mapping[str, Any] = MappingProxyType({}), +) -> Simulation: """Preserve the linear lesson or the four-channel dispersive GRIN example. The nonlinear source describes the GPI model and its chosen Kerr coefficient alongside Krupa et al., doi:10.1103/PhysRevLett.116.183901. This extraction retains its four-channel truncation, not a mode-count convergence claim. - `length` shortens the nonlinear guide, which is uniform along z. + `length` shortens the nonlinear guide, which is uniform along z. Its launch is + coupled at construction, which translates the run, so `options` (save planes and + an integrator) go to the constructor rather than to the settings afterwards. """ radius, contrast = (50e-6, 0.0095) if nonlinear else (25e-6, 0.005) wavelength = 1.064e-6 if nonlinear else 1.03e-6 @@ -249,6 +258,7 @@ def grin(*, nonlinear: bool, length: float = 0.1) -> Simulation: seed=41, dz_max=period / 10, backend="numpy_ref", + **options, ) q = float(grid.omega[centre]) / C * radius * 1.45 beta = np.sqrt(q * q - 2 * q * np.sqrt(2 * contrast) * np.array([1, 3])) / radius @@ -321,17 +331,24 @@ def moving() -> Simulation: ) -def advanced(name: str, length: float) -> Simulation: +def advanced( + name: str, length: float, options: Mapping[str, Any] = MappingProxyType({}) +) -> Simulation: """Select a declared guide before applying the suite's measurement settings. - `length` is the registered guide length; only the nonlinear GRIN guide takes it. + `length` is the registered guide length and `options` the run's save planes and + integrator; only the nonlinear GRIN guide, translated at construction, takes them. """ if name in ("scalar_wall", "vector_wall", "taper", "taper_wall"): return coupled(name) if name == "step_index_pair": return step_index() if name in ("grin_linear", "grin_nonlinear"): - return grin(nonlinear=name == "grin_nonlinear", length=length) + return ( + grin(nonlinear=True, length=length, options=options) + if name == "grin_nonlinear" + else grin(nonlinear=False) + ) if name == "moving_solid_guide": return moving() raise SuiteError(f"advanced guide builder not implemented for {name}") diff --git a/tests/test_benchmark_guides.py b/tests/test_benchmark_guides.py index ec37db18..efacd289 100644 --- a/tests/test_benchmark_guides.py +++ b/tests/test_benchmark_guides.py @@ -85,3 +85,17 @@ def test_scaled_cladding_preserves_the_material_identity_and_domain(): ) assert scaled.omega_domain == law.omega_domain assert scaled.identity == {"law": dict(law.identity), "factor": 0.981} + + +@pytest.mark.slow +def test_the_nonlinear_grin_guide_takes_its_planes_and_stepper_at_construction(): + # Its launch is coupled at construction, which translates the run, so the + # builder passes the planes and integrator to the constructor. + case = CASES["grin_nonlinear"] + simulation = build(configuration("grin_nonlinear", "multistep")) + problem = simulation.translate() + np.testing.assert_allclose( + problem.stepper.save_planes, np.linspace(0, case.length, case.planes + 1)[1:] + ) + assert isinstance(problem.stepper.integrator, Multistep) + assert simulation.settings.backend == "numpy_ref" From 0ec790929b84041c41f773de2224e4635fa4a9e7 Mon Sep 17 00:00:00 2001 From: John Travers Date: Tue, 27 Oct 2026 07:22:19 +0000 Subject: [PATCH 05/13] SEP 38: calibrate on an EPYC 7763 node; costs and phase 1 report Co-Authored-By: Claude Opus 5.5 --- benchmarks/registry.py | 31 ++++---- .../dev/seps/sep-0038-performance-tracking.md | 70 ++++++++++++++----- 2 files changed, 68 insertions(+), 33 deletions(-) diff --git a/benchmarks/registry.py b/benchmarks/registry.py index 94f322c7..532a9cc9 100644 --- a/benchmarks/registry.py +++ b/benchmarks/registry.py @@ -74,8 +74,9 @@ class Case: A fixed step in metres; the case then runs its declared fixed schedule and admits no stepper override. cost : tuple of float or None - Seconds per complete run at 1 and 16 threads on an AMD EPYC 7763 node with - SEP 37's defaults (its phase 0), for the plan's estimate; ``None`` if unknown. + Seconds per complete run at 1 and 16 threads on 16 cores of an AMD EPYC 7763 + node, from SEP 38's calibration run, for the plan's estimate; ``None`` if + unknown. """ name: str @@ -148,7 +149,7 @@ def configuration( 0.15, (1, 16384, 16384), "Dudley, Genty and Coen, Rev. Mod. Phys. 78, 1135 (2006), section III", - cost=(80.0, 30.0), + cost=(84.5, 47.2), ), Case( "gnlse_soliton", @@ -159,7 +160,7 @@ def configuration( 1.3278075458959397, (1, 4096, 4096), "docs/introduction/tutorials/plot_02_soliton.py; one soliton period", - cost=(8.0, 5.0), + cost=(12.6, 8.5), ), Case( "rdw", @@ -170,7 +171,7 @@ def configuration( 1.5, (4, 4096, 8192), "He resonant dispersive wave, four resolved modes", - cost=(19.0, 10.0), + cost=(19.2, 12.6), ), Case( "he_rdw_averaged", @@ -181,7 +182,7 @@ def configuration( 1.5, (1, 4096, 8192), "rdw on the mode-averaged route", - cost=(3.0, 3.0), + cost=(2.7, 2.7), ), # Revision 2 (SEP 38): 0.2 m and 160 nm-4 um, as SEP 37 phase 0 measured it; # revision 1 was the 0.4 m blue-shift example with a 300 nm edge. @@ -194,7 +195,7 @@ def configuration( 0.2, (1, 4096, 8192), "docs/examples/processes/plot_06_ionisation_blue_shift.py, shortened", - cost=(67.0, 34.0), + cost=(65.8, 35.5), ), Case( "argon_blueshift_averaged", @@ -205,7 +206,7 @@ def configuration( 0.2, (1, 4096, 8192), "argon_blueshift on the mode-averaged route", - cost=(15.0, 15.0), + cost=(15.2, 15.1), ), Case( "argon_blueshift_envelope", @@ -216,7 +217,7 @@ def configuration( 0.2, (1, 2048, 2048), "argon_blueshift on the envelope grid", - cost=(3.2, 3.1), + cost=(3.3, 3.4), ), Case( "h2_raman_averaged", @@ -227,7 +228,7 @@ def configuration( 0.4, (1, 4096, 8192), "docs/examples/processes/plot_05_raman_self_frequency_shift.py, 10 bar", - cost=(16.0, 16.0), + cost=(16.2, 16.2), ), # Revision 2 (SEP 38): three modes, 0.13 m, 2 ps and 220 nm-4 um, the # definition SEP 37 phase 0 measured; revision 1 had six modes over 0.2 m. @@ -241,7 +242,7 @@ def configuration( (3, 8192, 16384), "H2 comb-to-continuum, three modes", widths=("production",), - cost=(605.0, 290.0), + cost=(605.0, 291.5), ), Case( "radial_filament_256", @@ -253,7 +254,7 @@ def configuration( (256, 250), "Ar filament with plasma, 256 radial nodes", planes=4, - cost=(10.0, 4.0), + cost=(16.4, 6.1), ), # Revision 2 (SEP 38): a 2 cm guide, the window SEP 37 phase 0 measured; # revision 1 was the example's 0.1 m. @@ -268,7 +269,7 @@ def configuration( "docs/examples/waveguides/plot_04_graded_index_multimode.py, 2 cm", widths=("production",), planes=5, - cost=(102.0, 35.0), + cost=(102.0, 37.8), ), Case( "kerr_beam_384", @@ -283,7 +284,7 @@ def configuration( planes=1, fields="planes", step=0.5e-6, - cost=(91.0, 10.0), + cost=(None, 133.6), ), Case( "kerr_beam_192", @@ -298,7 +299,7 @@ def configuration( planes=1, fields="planes", step=0.5e-6, - cost=(10.0, None), + cost=(36.7, None), ), ) diff --git a/docs/dev/seps/sep-0038-performance-tracking.md b/docs/dev/seps/sep-0038-performance-tracking.md index 8bdb4dd7..3e6db72d 100644 --- a/docs/dev/seps/sep-0038-performance-tracking.md +++ b/docs/dev/seps/sep-0038-performance-tracking.md @@ -15,7 +15,7 @@ | **Requires** | [SEP 37](sep-0037-noise-and-stepper-defaults.md) (the per-interface defaults the tracked cases run with) | | **Supersedes** | [SEP 30](archive/sep-0030-standard-benchmark-suite.md) | | **Superseded-by** | — | -| **Implementation** | Opus 5.5 (AI) for John Travers, branch `feature/sep-0038-performance-tracking`, drafted together with its implementation, stacked on SEP 37's branch | +| **Implementation** | Opus 5.5 (AI) for John Travers, branch `feature/sep-0038-performance-tracking`, stacked on SEP 37's branch: phase 1 implemented and calibrated 2026-10-27; the first history rows wait for the release | | **Manual pages** | — | ## Part I — Proposal @@ -144,33 +144,33 @@ python -m benchmarks history [SLUG] [--width W] #### 5.2 The tracking set The cases represent the main uses of solprop at the sizes they are used. Lengths are -complete guides. Costs are SEP 37 phase 0's medians for complete runs on an AMD EPYC -7763, at 1 and 16 threads, with that branch's defaults; "—" marks a width the case does -not run at. +complete guides. Costs are the medians of a complete `track` run (Part II, phase 1) on +16 cores of an AMD EPYC 7763 node, at 1 and 16 threads, with SEP 37's defaults; "—" +marks a width the case does not run at. | Case | Interface | Problem | Widths | Cost (s) | |---|---|---|---|---| -| `gnlse_raman_shock` | `prop_gnlse` | Dudley–Genty–Coen supercontinuum, Raman and shock, 0.15 m [1] | 1, P | ~80 / ~30 | -| `gnlse_soliton` | `prop_gnlse` | ideal N = 3 soliton over one soliton period | 1, P | ~8 / ~5 | -| `rdw` | `prop_capillary` | He resonant dispersive wave, 4 resolved modes, 1.5 m, 0.8→0 bar gradient | 1, P | 19 / 10 | +| `gnlse_raman_shock` | `prop_gnlse` | Dudley–Genty–Coen supercontinuum, Raman and shock, 0.15 m [1] | 1, P | 84 / 47 | +| `gnlse_soliton` | `prop_gnlse` | ideal N = 3 soliton over one soliton period | 1, P | 13 / 8 | +| `rdw` | `prop_capillary` | He resonant dispersive wave, 4 resolved modes, 1.5 m, 0.8→0 bar gradient | 1, P | 19 / 13 | | `he_rdw_averaged` | `prop_capillary` | the same, mode-averaged | 1, P | 3 / 3 | -| `argon_blueshift` (rev. 2) | `prop_capillary` | Ar ionisation blueshift, 13 µm, 1.7 bar, 0.2 m, 1 resolved mode | 1, P | 67 / 34 | +| `argon_blueshift` (rev. 2) | `prop_capillary` | Ar ionisation blueshift, 13 µm, 1.7 bar, 0.2 m, 1 resolved mode | 1, P | 66 / 36 | | `argon_blueshift_averaged` (rev. 2) | `prop_capillary` | the same, mode-averaged | 1, P | 15 / 15 | | `argon_blueshift_envelope` (new) | `prop_capillary` | `argon_blueshift` on the envelope grid | 1, P | 3 / 3 | | `h2_raman_averaged` (new) | `prop_capillary` | H2 soliton self-frequency shift, 10 bar, 0.4 m, mode-averaged | 1, P | 16 / 16 | -| `ctc` (rev. 2) | `prop_capillary` | H2 comb-to-continuum, 3 modes, 40 bar, 0.13 m, 2 ps window | P | — / 290 | -| `radial_filament_256` | `prop_radial` | Ar filament, 256 radial nodes, 3 cm | 1, P | ~10 / ~4 | -| `grin_nonlinear` (rev. 2) | `prop_waveguide` | four-channel graded-index fibre, 2 cm | P | — / ~35 | -| `kerr_beam_384` | `prop_free` | fused-silica Kerr beam, 384² transverse grid, fixed RK4IP | P | — / ~10 | -| `kerr_beam_192` | `prop_free` | the same at 192², fixed RK4IP | 1 | ~10 / — | +| `ctc` (rev. 2) | `prop_capillary` | H2 comb-to-continuum, 3 modes, 40 bar, 0.13 m, 2 ps window | P | — / 291 | +| `radial_filament_256` | `prop_radial` | Ar filament, 256 radial nodes, 3 cm | 1, P | 16 / 6 | +| `grin_nonlinear` (rev. 2) | `prop_waveguide` | four-channel graded-index fibre, 2 cm | P | — / 38 | +| `kerr_beam_384` | `prop_free` | fused-silica Kerr beam, 384² transverse grid, fixed RK4IP | P | — / 134 | +| `kerr_beam_192` | `prop_free` | the same at 192², fixed RK4IP | 1 | 37 / — | -The sum is about 220 s at one thread and 460 s at 16 threads per pass. Three repeats, the -warm-ups and construction bring it to about 40 minutes on that node; the target is under -an hour on any machine in use. The Phase 2 calibration run fixes the final lengths. +The sum is about 270 s at one thread and 620 s at 16 threads per pass. With three +repeats, the warm-ups and construction, the run took about 50 minutes on that node; the +target is under an hour on any machine in use. The three most expensive cases run at the production width only: `ctc` (about 600 s at -one thread), the 2 cm GRIN window (about 100 s) and the 384² Kerr beam (about 90 s). The -192² Kerr beam keeps a one-thread free-space row. +one thread in SEP 37's phase 0), the 2 cm GRIN window (about 100 s) and the 384² Kerr +beam. The 192² Kerr beam keeps a one-thread free-space row. Every case keeps its interface's default noise (`"input"` after SEP 37) with a fixed seed. Every other registered case stays runnable by name; none is in the default @@ -270,6 +270,9 @@ is final. - Q1. If SEP 37's capillary default becomes `RungeKutta()` at 1e-5, the capillary costs fall; should the freed time go to a longer `ctc`, or should `ctc` run at one thread too? +- Q3. The 384² Kerr beam costs 134 s per run at 16 threads on the EPYC node (25 GiB), + a fifth of the run's time for one free-space case. Keep it, or track the 192² beam at + both widths instead? - Q2. Should a cloud instance (for example AWS `c8a` or GCP `c4d`) be a standing history file, re-rented per release, or measured only when a hardware question arises? @@ -280,6 +283,37 @@ is final. ## Part II — Implementation record +### Phase 1 report — the tracking command and its calibration + +- **Scope:** phases 1 and 2's calibration, 2026-10-27, Opus 5.5 (AI) for John Travers, + branch `feature/sep-0038-performance-tracking`, stacked on SEP 37's pull request + (its defaults are provisional until the maintainer approves them). +- **Delivered:** `benchmarks/registry.py` (cases as complete guides, the `tracking` + set, widths, observation planes and costs), `builders.py` (interface calls on the + library's defaults with a fixed seed), `worker.py` (one configuration per process, + timed through `run_problem` after a warm-up), `runner.py`, `track.py`, + `history.py`, `ab.py` (with `--length`) and `__main__.py` (`list`, `track`, `ab`, + `history`); tests `tests/test_benchmark_{track,cases,ab,exercises,guides,telemetry}.py`; + the manual `docs/dev/benchmarking.md` and the rules in `docs/dev/benchmarks.md`. + Removed as §6 proposed, with their tests. +- **Deviations:** the nonlinear GRIN case couples its launch at construction, which + translates it, so its save planes and any stepper override go to its constructor + (`benchmarks/guides.py`); a slow test covers it. The production width counts the + distinct cores of the allocation from the Linux topology, so an allocation of one + hardware thread per core counts every core. +- **Calibration:** one complete default run on 16 cores (one NUMA domain) of an AMD EPYC + 7763 cluster node, native backend (LOCAL profile, GCC 16.2, FFTW 3.3.11, OpenBLAS + 0.3.34), three repeats: 22 configurations, about 50 minutes including construction. + The three repeats of every configuration took the same steps and agreed within 0.7 %, + except `gnlse_raman_shock` at 16 threads (2.2 %), `ctc` (1.4 %) and + `radial_filament_256` at 16 threads (26 %, a 6 s run on a node shared with other + jobs). The costs in §5.2 are this run's medians. The run bundle is kept with the + project's development records. +- **Not run:** no history row was recorded: the defaults are provisional and the + branch is not merged, so the first rows wait for the release commit. The Apple M5 + Max was not timed (it was busy). +- **Open questions:** Q1–Q3 (§12). + ## Part III — History - 2026-10-27: Drafted, together with its implementation on the same branch, as the From cdad4a9d210b241b1811e5f25b477eacb1ece7a3 Mon Sep 17 00:00:00 2001 From: John Travers Date: Tue, 27 Oct 2026 08:33:05 +0000 Subject: [PATCH 06/13] Plan the width tests on a fixed 16-CPU allocation, as CI runners have fewer Co-Authored-By: Claude Opus 5.5 --- tests/test_benchmark_track.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_benchmark_track.py b/tests/test_benchmark_track.py index 266ff377..a63826dd 100644 --- a/tests/test_benchmark_track.py +++ b/tests/test_benchmark_track.py @@ -15,6 +15,13 @@ from benchmarks.registry import CASES, select +@pytest.fixture +def sixteen_cpus(monkeypatch): + """Plan for a 16-CPU allocation, whatever this runner has.""" + monkeypatch.setattr(system, "eligible_cpus", lambda: tuple(range(16))) + + +@pytest.mark.usefixtures("sixteen_cpus") def test_the_default_plan_runs_each_case_at_its_declared_widths(): rows, skipped = runner.plan_rows(select(), ("default",), None, 8) assert not skipped @@ -32,6 +39,7 @@ def test_the_default_plan_runs_each_case_at_its_declared_widths(): assert runner.estimate(rows, 3) is not None +@pytest.mark.usefixtures("sixteen_cpus") def test_named_widths_and_steppers_apply_to_every_case_and_explain_skips(): cases = (CASES["rdw"], CASES["kerr_beam_192"]) rows, skipped = runner.plan_rows(cases, ("default", "rk"), "2", 4) From 8d3b3f51e26bb5bc176db9fe6dfce3cae0e1c245 Mon Sep 17 00:00:00 2001 From: John Travers Date: Tue, 27 Oct 2026 08:46:11 +0000 Subject: [PATCH 07/13] Track the Kerr beams over four fixed steps; name Arm and Windows CPUs; a threading option The Kerr beams cost the same per fixed step, so four steps track the same work in a third of the time (revision 2). lscpu names Arm cores, whose /proc/cpuinfo has no model name, and the registry names Windows processors. --threading picks OpenMP placement and waiting; the policy is a history column and part of what rows compare by. Co-Authored-By: Claude Opus 5.5 --- benchmarks/ab.py | 3 +- benchmarks/history.py | 2 + benchmarks/registry.py | 18 ++++---- benchmarks/system.py | 43 ++++++++++++++---- benchmarks/telemetry.py | 73 +++++++++++++++++++++++++------ benchmarks/track.py | 18 ++++++-- tests/test_benchmark_cases.py | 2 +- tests/test_benchmark_telemetry.py | 16 +++++++ tests/test_benchmark_track.py | 15 +++++++ 9 files changed, 155 insertions(+), 35 deletions(-) diff --git a/benchmarks/ab.py b/benchmarks/ab.py index 28bceebb..883fe99f 100644 --- a/benchmarks/ab.py +++ b/benchmarks/ab.py @@ -209,6 +209,7 @@ def ab(args: argparse.Namespace) -> int: "rounds": args.rounds, "repeats": args.repeats, "length": args.length, + "threading": args.threading, "seed": args.seed, "rows": rows, "skipped": skipped, @@ -238,7 +239,7 @@ def ab(args: argparse.Namespace) -> int: if round_index == 0 else None, }, - thread_environment(width, dict(os.environ)), + thread_environment(width, dict(os.environ), args.threading), output / "raw" / f"r{round_index}-{arm}-{row['id']}", args.timeout, harness, diff --git a/benchmarks/history.py b/benchmarks/history.py index e76d8fbd..495d3604 100644 --- a/benchmarks/history.py +++ b/benchmarks/history.py @@ -34,6 +34,7 @@ "instance_type", "system", "threads", + "threading", "case", "revision", "stepper", @@ -85,6 +86,7 @@ def comparable(row: Record) -> tuple[str, ...]: """Key the rows that measure the same thing on one hardware set.""" return ( str(row["threads"]), + row["threading"], row["case"], str(row["revision"]), row["stepper"], diff --git a/benchmarks/registry.py b/benchmarks/registry.py index 532a9cc9..9a9862b4 100644 --- a/benchmarks/registry.py +++ b/benchmarks/registry.py @@ -271,35 +271,37 @@ def configuration( planes=5, cost=(102.0, 37.8), ), + # Revision 2 (SEP 38): four 0.5 um steps; revision 1 took eleven over 5.5 um. + # A fixed-step run costs the same per step, so fewer steps track the same work. Case( "kerr_beam_384", - 1, + 2, ("tracking", "free-space"), "prop_free", "kerr", - 5.5e-6, + 2e-6, (384, 384, 600), - "fused-silica Kerr beam, fixed RK4IP steps", + "fused-silica Kerr beam, four fixed RK4IP steps", widths=("production",), planes=1, fields="planes", step=0.5e-6, - cost=(None, 133.6), + cost=(None, 48.6), ), Case( "kerr_beam_192", - 1, + 2, ("tracking", "free-space"), "prop_free", "kerr", - 5.5e-6, + 2e-6, (192, 192, 300), - "fused-silica Kerr beam, fixed RK4IP steps", + "fused-silica Kerr beam, four fixed RK4IP steps", widths=("1",), planes=1, fields="planes", step=0.5e-6, - cost=(36.7, None), + cost=(13.3, None), ), ) diff --git a/benchmarks/system.py b/benchmarks/system.py index b102b9a6..77afb0ba 100644 --- a/benchmarks/system.py +++ b/benchmarks/system.py @@ -24,6 +24,7 @@ __all__ = [ "POOL_VARIABLES", + "THREADING", "counters", "eligible_cpus", "git_receipt", @@ -64,6 +65,29 @@ "KOKKOS_TOOLS_LIBS", ) +THREADING: dict[str, dict[str, str]] = { + # Unpinned, and idle threads sleep at once: robust on a shared machine. + "passive": { + "OMP_PROC_BIND": "false", + "OMP_WAIT_POLICY": "PASSIVE", + "KMP_BLOCKTIME": "0", + "GOMP_SPINCOUNT": "0", + }, + # Unpinned, with each OpenMP runtime's own waiting (a short spin, then sleep). + "default": {"OMP_PROC_BIND": "false"}, + # Unpinned, and idle threads spin between parallel regions. + "active": {"OMP_PROC_BIND": "false", "OMP_WAIT_POLICY": "ACTIVE"}, + # Threads pinned to neighbouring cores, with the runtime's own waiting. + "bound": {"OMP_PROC_BIND": "close", "OMP_PLACES": "cores"}, + # Pinned, and spinning. + "bound-active": { + "OMP_PROC_BIND": "close", + "OMP_PLACES": "cores", + "OMP_WAIT_POLICY": "ACTIVE", + }, +} +"""OpenMP placement and waiting policies a run may choose (SEP 38).""" + _MODEL_NOISE = re.compile( r"\((?:r|tm)\)|\bcpu\b|\bprocessor\b|\b\d+-core\b|@.*$", re.IGNORECASE ) @@ -148,10 +172,18 @@ def slug(model: str, cores: int) -> str: return f"{words.strip('-') or 'unknown-cpu'}-{cores}c" -def thread_environment(width: int, incoming: dict[str, str]) -> dict[str, str]: - """Set every pool to `width` before its library is imported.""" +def thread_environment( + width: int, incoming: dict[str, str], threading: str = "passive" +) -> dict[str, str]: + """Set every pool to `width`, and OpenMP's placement and waiting by `threading`. + + The pools are set before their libraries are imported. `threading` is a key of + :data:`THREADING`; its OpenMP variables replace any the caller had. + """ if width < 1: raise SuiteError("thread counts must be positive") + if threading not in THREADING: + raise SuiteError(f"unknown threading policy {threading}") environment = incoming.copy() for key in POOL_VARIABLES: if key not in ("SOLPROP_FFTW_PLAN", "KOKKOS_TOOLS_LIBS"): @@ -179,12 +211,7 @@ def thread_environment(width: int, incoming: dict[str, str]) -> dict[str, str]: "SOLPROP_BLAS_THREADS", ): environment[key] = "1" - environment.update( - OMP_PROC_BIND="false", - OMP_WAIT_POLICY="PASSIVE", - KMP_BLOCKTIME="0", - GOMP_SPINCOUNT="0", - ) + environment.update(THREADING[threading]) return environment diff --git a/benchmarks/telemetry.py b/benchmarks/telemetry.py index aecb755d..84eeb091 100644 --- a/benchmarks/telemetry.py +++ b/benchmarks/telemetry.py @@ -19,6 +19,7 @@ "cpu_list", "hardware_details", "linux_topology", + "lscpu_model", ] @@ -73,29 +74,73 @@ def linux_topology(root: Path, allocation: tuple[int, ...]) -> list[Record]: return result +def _command(arguments: list[str]) -> str | None: + """Return a short query's standard output, or ``None`` if it fails.""" + try: + result = subprocess.run( + arguments, + capture_output=True, + text=True, + encoding="utf-8", + timeout=5, + check=False, + ) + except OSError, subprocess.TimeoutExpired: + return None + return result.stdout if result.returncode == 0 else None + + +def lscpu_model(output: str) -> str | None: + """Name an Arm or other CPU from ``lscpu``, e.g. ``ARM Neoverse-V2``. + + Arm Linux kernels give ``/proc/cpuinfo`` no model name; ``lscpu`` decodes the + implementer and part numbers into a vendor and a core name. + """ + fields = { + key.strip(): value.strip() + for key, _, value in (line.partition(":") for line in output.splitlines()) + if value.strip() + } + model = fields.get("Model name") + if not model or model == "-": + return None + vendor = fields.get("Vendor ID", "") + # x86 vendor IDs (GenuineIntel, AuthenticAMD) are not names to prefix. + if not vendor or vendor.startswith(("Genuine", "Authentic")) or vendor in model: + return model + return f"{vendor} {model}" + + +def _windows_model() -> str | None: + """Read the processor name Windows keeps in its registry.""" + try: + import winreg # noqa: PLC0415 # pyright: ignore[reportMissingModuleSource] + + with winreg.OpenKey( # pyright: ignore[reportAttributeAccessIssue] + winreg.HKEY_LOCAL_MACHINE, # pyright: ignore[reportAttributeAccessIssue] + r"HARDWARE\DESCRIPTION\System\CentralProcessor\0", + ) as key: + value, _ = winreg.QueryValueEx(key, "ProcessorNameString") # pyright: ignore[reportAttributeAccessIssue] + except ImportError, OSError: + return None + return str(value).strip() or None + + def hardware_details(allocation: tuple[int, ...]) -> Record: - """Record richer static hardware without changing existing profile identities.""" + """Record the CPU model and, on Linux, the per-CPU topology.""" system = platform.system() model = None cores: list[Record] = [] if system == "Linux": contents = _read(Path("/proc/cpuinfo")) or "" match = re.search(r"^model name\s*:\s*(.+)$", contents, re.MULTILINE) - model = match[1] if match else None + model = match[1] if match else lscpu_model(_command(["lscpu"]) or "") cores = linux_topology(Path("/sys/devices/system/cpu"), allocation) elif system == "Darwin": - try: - result = subprocess.run( - ["sysctl", "-n", "machdep.cpu.brand_string"], - capture_output=True, - text=True, - encoding="utf-8", - timeout=2, - check=False, - ) - model = result.stdout.strip() if result.returncode == 0 else None - except OSError, subprocess.TimeoutExpired: - pass + output = _command(["sysctl", "-n", "machdep.cpu.brand_string"]) + model = output.strip() if output else None + elif system == "Windows": + model = _windows_model() return { "cpu_model": model, "cpu_model_reason": None if model else "model query unavailable", diff --git a/benchmarks/track.py b/benchmarks/track.py index a7d0c4e1..2197fd9e 100644 --- a/benchmarks/track.py +++ b/benchmarks/track.py @@ -26,6 +26,7 @@ show_plan, ) from benchmarks.system import ( + THREADING, eligible_cpus, git_receipt, guard, @@ -66,6 +67,12 @@ def selection_options(parser: argparse.ArgumentParser) -> None: "--backend", choices=("numpy_ref", "numba_cpu", "native"), default="native" ) parser.add_argument("--repeats", type=int, default=3, help="timed runs per case") + parser.add_argument( + "--threading", + choices=tuple(THREADING), + default="passive", + help="OpenMP placement and waiting (benchmarks/system.py)", + ) def track_options(parser: argparse.ArgumentParser) -> None: @@ -98,7 +105,9 @@ def steppers(value: str) -> tuple[str, ...]: return names -def summarise(row: Record, result: Record, busy: bool) -> Record: +def summarise( + row: Record, result: Record, busy: bool, threading: str = "passive" +) -> Record: """Reduce one worker's receipt to the tracked statistics.""" values = row["configuration"] base = { @@ -107,6 +116,7 @@ def summarise(row: Record, result: Record, busy: bool) -> Record: "revision": values["revision"], "stepper": values["stepper"], "threads": values["threads"], + "threading": threading, "status": result["status"], "busy": busy, } @@ -183,6 +193,7 @@ def history_rows( "instance_type": machine["instance_type"], "system": f"{machine['system']} {machine['machine']}", "threads": row["threads"], + "threading": row["threading"], "case": row["case"], "revision": row["revision"], "stepper": row["stepper"], @@ -243,6 +254,7 @@ def track(args: argparse.Namespace) -> int: "hardware": machine, "source": source, "repeats": args.repeats, + "threading": args.threading, "rows": rows, "skipped": skipped, }, @@ -258,13 +270,13 @@ def track(args: argparse.Namespace) -> int: events, elapsed = launch_worker( sys.executable, {"configuration": row["configuration"], "repeats": args.repeats}, - thread_environment(width, dict(os.environ)), + thread_environment(width, dict(os.environ), args.threading), output / "raw" / row["id"], args.timeout, ) result = receipt(events) provenance = result.get("provenance") or provenance - results.append(summarise(row, result, busy)) + results.append(summarise(row, result, busy, args.threading)) state = ( f"{results[-1]['median_s']:.2f} s" if result["status"] == "ok" diff --git a/tests/test_benchmark_cases.py b/tests/test_benchmark_cases.py index 91033edf..115b3d4b 100644 --- a/tests/test_benchmark_cases.py +++ b/tests/test_benchmark_cases.py @@ -81,7 +81,7 @@ def test_fixed_step_cases_run_their_schedule_and_refuse_overrides(): assert isinstance(simulation.settings.integrator, LowStorageRungeKutta) schedule = simulation.settings.schedule assert schedule is not None - assert len(schedule.steps) == 11 + assert len(schedule.steps) == 4 np.testing.assert_allclose(schedule.edges[-1], case.length, rtol=1e-12) assert simulation.settings.fields == "planes" diff --git a/tests/test_benchmark_telemetry.py b/tests/test_benchmark_telemetry.py index 265a2dba..c8222c9d 100644 --- a/tests/test_benchmark_telemetry.py +++ b/tests/test_benchmark_telemetry.py @@ -48,3 +48,19 @@ def test_missing_hardware_and_allocation_details_keep_reasons(monkeypatch): monkeypatch.setattr(telemetry, "_read", lambda _: None) record = telemetry.allocation_details() assert record["allowed_cpus"] is None and record["allowed_cpus_reason"] + + +@pytest.mark.parametrize( + ("output", "model"), + [ + ( + "Architecture: aarch64\nVendor ID: ARM\nModel name: Neoverse-V2\n", + "ARM Neoverse-V2", + ), + ("Vendor ID: AuthenticAMD\nModel name: AMD EPYC 9R45\n", "AMD EPYC 9R45"), + ("Vendor ID: ARM\nModel name: -\n", None), + ("", None), + ], +) +def test_lscpu_names_arm_cores_by_vendor_and_part(output, model): + assert telemetry.lscpu_model(output) == model diff --git a/tests/test_benchmark_track.py b/tests/test_benchmark_track.py index a63826dd..27c7d955 100644 --- a/tests/test_benchmark_track.py +++ b/tests/test_benchmark_track.py @@ -86,6 +86,7 @@ def result_row(case="rdw", threads=1, median=2.0): "revision": CASES[case].revision, "stepper": "default", "threads": threads, + "threading": "passive", "status": "ok", "busy": False, "median_s": median, @@ -231,3 +232,17 @@ def test_a_tracking_run_records_complete_runs_and_appends_history( == 2 ) assert "native backend only" in capsys.readouterr().err + + +def test_threading_policies_replace_the_callers_openmp_settings(): + incoming = {"OMP_WAIT_POLICY": "ACTIVE", "OMP_PLACES": "threads", "PATH": "/bin"} + passive = system.thread_environment(4, incoming) + assert passive["OMP_WAIT_POLICY"] == "PASSIVE" + assert passive["SOLPROP_NATIVE_THREADS"] == "4" + assert "OMP_PLACES" not in passive + assert passive["PATH"] == "/bin" + bound = system.thread_environment(4, incoming, "bound") + assert (bound["OMP_PROC_BIND"], bound["OMP_PLACES"]) == ("close", "cores") + assert "OMP_WAIT_POLICY" not in bound + with pytest.raises(SuiteError, match="unknown threading"): + system.thread_environment(4, incoming, "spin") From 50a4f3dd6867ddd03a3205b3622b2b1cd69d289d Mon Sep 17 00:00:00 2001 From: John Travers Date: Tue, 27 Oct 2026 08:56:54 +0000 Subject: [PATCH 08/13] SEP 38: act on the independent review History rows now compare by threading, tolerances and noise model as well as the integrator, and summaries show the change against the first and previous runs with the configurations shared. Only a complete default run of committed code, timing this checkout's src with a native build of the same commit, is recorded; each repeat must take the same steps and reach the same final field. Older history files gain new columns; dirty and case-hash columns are added. The slug counts the production width and names a cloud family; the busy flag judges by CPU use where swap is not reported; Kokkos tool libraries are stripped and each worker's environment is saved; results are rewritten after every worker. Tests cover the refusals, migration and a native smoke run. Co-Authored-By: Claude Opus 5.5 --- benchmarks/__init__.py | 6 +- benchmarks/history.py | 128 ++++++++++++------ benchmarks/runner.py | 11 +- benchmarks/system.py | 33 +++-- benchmarks/track.py | 117 ++++++++++++---- benchmarks/worker.py | 103 +++++++++----- docs/dev/benchmarking.md | 45 ++++-- docs/dev/benchmarks.md | 2 +- .../dev/seps/sep-0038-performance-tracking.md | 95 +++++++++---- native/README.md | 3 +- tests/test_benchmark_ab.py | 4 +- tests/test_benchmark_cases.py | 24 +++- tests/test_benchmark_exercises.py | 18 ++- tests/test_benchmark_track.py | 120 ++++++++++++++-- 14 files changed, 536 insertions(+), 173 deletions(-) diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py index b19dedb4..ba92d5c0 100644 --- a/benchmarks/__init__.py +++ b/benchmarks/__init__.py @@ -2,10 +2,10 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -"""Collect bounded, independently qualified benchmarks (SEP 30). +"""Track complete runs per hardware, and compare two builds (SEP 38). -Importing the registry or planner never imports a scientific library. Workers -load those libraries only after their thread environment has been established. +Importing the registry or the controller never imports a scientific library. +Workers load those libraries only after their thread environment is set. """ __all__: list[str] = [] diff --git a/benchmarks/history.py b/benchmarks/history.py index 495d3604..a606e6f3 100644 --- a/benchmarks/history.py +++ b/benchmarks/history.py @@ -27,7 +27,9 @@ "date", "solprop", "commit", + "dirty", "native", + "native_commit", "native_build", "slug", "cpu_model", @@ -37,6 +39,7 @@ "threading", "case", "revision", + "case_hash", "stepper", "integrator", "rtol", @@ -53,26 +56,38 @@ "construction_s", "busy", ) +"""History columns. New ones go anywhere; older files gain them, empty, on append.""" def append(path: Path, rows: Sequence[Record]) -> None: - """Append rows, writing the header to a new file and refusing a changed one.""" + """Append rows, giving an older file any columns it lacks. + + A file whose header has a column this version does not know is refused: it + was written by a later version. + """ + for row in rows: + if set(row) != set(COLUMNS): + raise SuiteError("a history row must give exactly the declared columns") path.parent.mkdir(parents=True, exist_ok=True) - new = not path.exists() - if not new: + existing: list[Record] = [] + if path.exists(): with path.open(encoding="utf-8", newline="") as stream: - header = next(csv.reader(stream), None) - if tuple(header or ()) != COLUMNS: - raise SuiteError(f"{path}: columns differ from this version's") - with path.open("a", encoding="utf-8", newline="") as stream: + reader = csv.DictReader(stream) + if unknown := set(reader.fieldnames or ()) - set(COLUMNS): + raise SuiteError( + f"{path}: columns {', '.join(sorted(unknown))} are from a later " + "version of the benchmarks" + ) + if tuple(reader.fieldnames or ()) != COLUMNS: + existing = list(reader) + mode = "w" if existing or not path.exists() else "a" + with path.open(mode, encoding="utf-8", newline="") as stream: writer = csv.DictWriter(stream, fieldnames=list(COLUMNS), lineterminator="\n") - if new: + if mode == "w": writer.writeheader() - for row in rows: - if set(row) != set(COLUMNS): - raise SuiteError("a history row must give exactly the declared columns") + for row in (*existing, *rows): writer.writerow( - {key: "" if row[key] is None else row[key] for key in COLUMNS} + {key: "" if row.get(key) is None else row[key] for key in COLUMNS} ) @@ -83,47 +98,71 @@ def read(path: Path) -> list[Record]: def comparable(row: Record) -> tuple[str, ...]: - """Key the rows that measure the same thing on one hardware set.""" - return ( - str(row["threads"]), - row["threading"], - row["case"], - str(row["revision"]), - row["stepper"], - row["integrator"], + """Key the rows that measure the same thing on one hardware set. + + Two rows compare only with the same width, threading, case definition, + stepper, resolved integrator, tolerances and noise model: a changed library + default is a new configuration, not a speed-up. + """ + return tuple( + str(row.get(key) or "") + for key in ( + "threads", + "threading", + "case", + "revision", + "stepper", + "integrator", + "rtol", + "atol", + "noise", + ) ) +def _ratio( + times: dict[tuple[str, ...], float], before: dict +) -> tuple[int, float | None]: + shared = sorted(set(times) & set(before)) + ratios = [times[key] / before[key] for key in shared] + mean = math.exp(statistics.fmean(map(math.log, ratios))) if ratios else None + return len(shared), mean + + def summary(rows: Sequence[Record]) -> list[Record]: - """Compare each recorded run with the first one, per width. + """Compare each recorded run with the first and with the previous, per width. - A run is the rows sharing a date and commit. Its change is the geometric mean, - over the configurations it shares with the first run at that width, of the - ratio of median times; below one is faster. + A run is the rows sharing a date and commit. Each change is the geometric + mean, over the configurations the two runs share, of the ratio of median + times; below one is faster. Busy rows count, and are counted. """ runs: dict[tuple[str, str, str], dict[tuple[str, ...], float]] = {} - labels: dict[tuple[str, str, str], str] = {} + labels: dict[tuple[str, str, str], tuple[str, int]] = {} for row in rows: key = (str(row["threads"]), row["date"], row["commit"]) runs.setdefault(key, {})[comparable(row)] = float(row["median_s"]) - labels[key] = row["solprop"] + label, busy = labels.get(key, (row["solprop"], 0)) + labels[key] = (label, busy + int(str(row.get("busy") or "0") == "1")) result = [] first: dict[str, dict[tuple[str, ...], float]] = {} + previous: dict[str, dict[tuple[str, ...], float]] = {} for (threads, date, commit), times in runs.items(): - reference = first.setdefault(threads, times) - shared = sorted(set(times) & set(reference)) - ratios = [times[key] / reference[key] for key in shared] + shared_first, versus_first = _ratio(times, first.setdefault(threads, times)) + shared_previous, versus_previous = _ratio(times, previous.get(threads, times)) + previous[threads] = times + label, busy = labels[(threads, date, commit)] result.append( { "threads": threads, "date": date, "commit": commit, - "solprop": labels[(threads, date, commit)], + "solprop": label, "cases": len(times), - "shared": len(shared), - "ratio": math.exp(statistics.fmean(map(math.log, ratios))) - if ratios - else None, + "busy": busy, + "shared_first": shared_first, + "ratio_first": versus_first, + "shared_previous": shared_previous, + "ratio_previous": versus_previous, } ) return result @@ -132,13 +171,24 @@ def summary(rows: Sequence[Record]) -> list[Record]: def show(path: Path, threads: str | None = None) -> None: """Print one line per recorded run and width.""" print(f"{path.stem}") - print(f"{'threads':>7} {'date':25} {'solprop':26} {'cases':>5} {'vs first':>9}") + print( + f"{'threads':>7} {'date':18} {'solprop':26} {'cases':>5} {'busy':>4} " + f"{'vs first':>13} {'vs previous':>13}" + ) for row in summary(read(path)): if threads is not None and row["threads"] != threads: continue - ratio = "-" if row["ratio"] is None else f"{row['ratio']:.3f}" + + def change(ratio: float | None, shared: int) -> str: + return "-" if ratio is None else f"{ratio:.3f} ({shared})" + print( - f"{row['threads']:>7} {row['date']:25} {row['solprop']:26} " - f"{row['cases']:>5} {ratio:>9}" + f"{row['threads']:>7} {row['date']:18} {row['solprop']:26} " + f"{row['cases']:>5} {row['busy']:>4} " + f"{change(row['ratio_first'], row['shared_first']):>13} " + f"{change(row['ratio_previous'], row['shared_previous']):>13}" ) - print("vs first: geometric mean of median-time ratios over shared cases.") + print( + "Changes are geometric means of median-time ratios (below 1: faster) over " + "the configurations both runs share, counted in brackets." + ) diff --git a/benchmarks/runner.py b/benchmarks/runner.py index f764e640..4722e13c 100644 --- a/benchmarks/runner.py +++ b/benchmarks/runner.py @@ -17,7 +17,7 @@ from benchmarks.records import Record, SuiteError, atomic_record, canonical, file_hash from benchmarks.registry import Case -from benchmarks.system import widths +from benchmarks.system import POOL_VARIABLES, widths __all__ = [ "ROOT", @@ -136,6 +136,15 @@ def launch_worker( directory.mkdir(parents=True, exist_ok=True) path = directory / "request.json" atomic_record(path, request, immutable=True) + atomic_record( + directory / "environment.json", + { + key: value + for key, value in environment.items() + if key in POOL_VARIABLES or key.startswith(("SOLPROP_", "OMP_", "KMP_")) + }, + immutable=True, + ) started = time.perf_counter() with ( (directory / "stdout.log").open("w", encoding="utf-8") as stdout, diff --git a/benchmarks/system.py b/benchmarks/system.py index 77afb0ba..36bfdd59 100644 --- a/benchmarks/system.py +++ b/benchmarks/system.py @@ -151,8 +151,11 @@ def hardware(production: int, instance_type: str | None = None) -> Record: details = hardware_details(eligible_cpus()) model = details["cpu_model"] or platform.processor() or platform.machine() cores = physical_cores() + instance = instance_type or _instance_type() return { - "slug": slug(model, cores), + # The cores a run uses name the hardware set: on a cluster the CPU mask can + # show the whole node while the allocation is a part of it. + "slug": slug(model, production, instance), "cpu_model": model, "physical_cores": cores, "logical_cpus": len(eligible_cpus()), @@ -161,13 +164,20 @@ def hardware(production: int, instance_type: str | None = None) -> Record: "system": platform.system(), "release": platform.release(), "machine": platform.machine(), - "instance_type": instance_type or _instance_type(), + "instance_type": instance, "allocation": allocation_details(), } -def slug(model: str, cores: int) -> str: - """Name a hardware set by CPU model and core count, e.g. ``apple-m5-max-18c``.""" +def slug(model: str, cores: int, instance: str | None = None) -> str: + """Name a hardware set by CPU model and core count, e.g. ``apple-m5-max-18c``. + + A cloud instance adds its family (``aws m8g.4xlarge`` gives ``-aws-m8g``): one + core design, such as Arm's Neoverse-V2, is sold in several clouds' instances. + """ + if instance: + family = re.sub(r"[^a-z0-9]+", "-", instance.lower().split(".")[0]).strip("-") + return f"{slug(model, cores)}-{family}" words = re.sub(r"[^a-z0-9]+", "-", _MODEL_NOISE.sub(" ", model).lower()) return f"{words.strip('-') or 'unknown-cpu'}-{cores}c" @@ -186,7 +196,8 @@ def thread_environment( raise SuiteError(f"unknown threading policy {threading}") environment = incoming.copy() for key in POOL_VARIABLES: - if key not in ("SOLPROP_FFTW_PLAN", "KOKKOS_TOOLS_LIBS"): + # A Kokkos tool left in the shell would profile the timed runs. + if key != "SOLPROP_FFTW_PLAN": environment.pop(key, None) for key in ( "SOLPROP_THREADS", @@ -246,12 +257,13 @@ def interval(before: Record, after: Record, allocation: tuple[int, ...]) -> Reco if before_swap is not None and after_swap is not None else None ) - known = usage is not None and swap_delta is not None and swap_delta >= 0 + # Where swap traffic is not reported (Windows), CPU use alone decides. + known = usage is not None and (swap_delta is None or swap_delta >= 0) return { "external_cpu_fraction": usage, "swap_bytes": swap_delta, "confidence": "known" if known else "unknown", - "quiet": known and usage is not None and usage <= 0.1 and swap_delta == 0, + "quiet": known and usage is not None and usage <= 0.1 and not swap_delta, } @@ -273,7 +285,12 @@ def git(*arguments: str) -> bytes: try: revision = git("rev-parse", "HEAD").decode().strip() changes = git("diff", "HEAD", "--binary") - untracked = git("ls-files", "--others", "--exclude-standard", "-z").split(b"\0") + # Untracked files count only where the timed code lives, so an editor's + # settings directory does not make every run dirty. + untracked = git( + "ls-files", "--others", "--exclude-standard", "-z", "--", + "src", "native", "benchmarks", + ).split(b"\0") # fmt: skip from benchmarks.records import file_hash # noqa: PLC0415 fingerprints = { diff --git a/benchmarks/track.py b/benchmarks/track.py index 2197fd9e..33934554 100644 --- a/benchmarks/track.py +++ b/benchmarks/track.py @@ -17,7 +17,7 @@ from benchmarks import history from benchmarks.records import LOCAL_ROOT, Record, SuiteError, atomic_record, now -from benchmarks.registry import SETS, STEPPERS, select +from benchmarks.registry import CASES, SETS, STEPPERS, select from benchmarks.runner import ( ROOT, launch_worker, @@ -35,8 +35,15 @@ thread_environment, ) +REPEAT_TOLERANCE = 1e-9 +"""The relative L2 difference two repeats' final fields may have: rounding only.""" + __all__ = [ + "REPEAT_TOLERANCE", + "admissible", "history_rows", + "imported_elsewhere", + "refuse_partial", "selection_options", "show_results", "steppers", @@ -108,7 +115,11 @@ def steppers(value: str) -> tuple[str, ...]: def summarise( row: Record, result: Record, busy: bool, threading: str = "passive" ) -> Record: - """Reduce one worker's receipt to the tracked statistics.""" + """Reduce one worker's receipt to the tracked statistics. + + A row is consistent when its repeats took the same steps and reached the same + final field to rounding, and the native backend ran the threads it was given. + """ values = row["configuration"] base = { "id": row["id"], @@ -124,6 +135,9 @@ def summarise( return {**base, "error": result.get("error")} times = [item["seconds"] for item in result["repetitions"]] last = result["repetitions"][-1] + build = (result.get("provenance") or {}).get("build_info") or {} + native_threads = build.get("runtime_threads") + same_steps = len({(r["accepted"], r["rejected"]) for r in result["repetitions"]}) return { **base, "median_s": statistics.median(times), @@ -132,10 +146,12 @@ def summarise( "repeats": len(times), "accepted": last["accepted"], "rejected": last["rejected"], - "same_steps": len( - {(r["accepted"], r["rejected"]) for r in result["repetitions"]} - ) - == 1, + "same_steps": same_steps == 1, + "repeat_difference": result["repeat_difference"], + "native_threads": native_threads, + "consistent": same_steps == 1 + and result["repeat_difference"] <= REPEAT_TOLERANCE + and native_threads in (None, values["threads"]), "source_evaluations": last["source_evaluations"], "peak_rss_gib": result["peak_rss_bytes"] / 2**30, "construction_s": result["construction_seconds"] + result["backend_seconds"], @@ -145,6 +161,7 @@ def summarise( "atol": result["atol"], "noise": result["noise"], "finite": result["finite"], + "solprop_path": (result.get("provenance") or {}).get("solprop_path"), } @@ -165,7 +182,7 @@ def show_results(rows: list[Record]) -> None: f"{row['case']:26} {row['integrator']:28} {row['threads']:>3} " f"{row['median_s']:9.2f} {spread:8.1f} {row['accepted']:7d} " f"{row['peak_rss_gib']:6.2f}" - + ("" if row["same_steps"] else " (steps differ between repeats)") + + ("" if row["consistent"] else " (repeats or threads inconsistent)") + (" (busy)" if row["busy"] else "") ) @@ -181,7 +198,9 @@ def history_rows( "date": date, "solprop": provenance["packages"]["solprop"], "commit": (source.get("revision") or "")[:12], + "dirty": int(bool(source.get("dirty"))), "native": build.get("package_version"), + "native_commit": (build.get("git_commit") or "")[:12] or None, "native_build": "; ".join( str(build[key]) for key in ("build_profile", "compiler", "fft_backend", "blas_backend") @@ -196,6 +215,7 @@ def history_rows( "threading": row["threading"], "case": row["case"], "revision": row["revision"], + "case_hash": CASES[row["case"]].digest[:12], "stepper": row["stepper"], "integrator": row["integrator"], "rtol": row["rtol"], @@ -216,6 +236,55 @@ def history_rows( ] +def refuse_partial( + args: argparse.Namespace, rows: list[Record], production: int, source: Record +) -> None: + """Admit to the history only a complete default run of committed code.""" + if args.backend != "native": + raise SuiteError("the history records the native backend only") + if source.get("dirty") is None: + raise SuiteError("the source state is unknown (no Git checkout or no git)") + if source["dirty"] and not args.allow_dirty: + raise SuiteError( + "uncommitted changes: commit them, or pass --allow-dirty to record " + "this run anyway (its rows are marked dirty)" + ) + default, _ = plan_rows(select(), ("default",), None, production) + if [row["id"] for row in rows] != [row["id"] for row in default]: + raise SuiteError("the history records the whole default plan only") + if args.repeats < 3: + raise SuiteError("the history records runs of at least three repeats") + + +def admissible(results: list[Record]) -> str | None: + """Say why a finished run may not enter the history, or ``None``.""" + if any(row["status"] != "ok" or not row["finite"] for row in results): + return "a configuration failed" + if any(not row["consistent"] for row in results): + return "repeats took different steps or fields, or threads differed" + return None + + +def imported_elsewhere( + results: list[Record], provenance: Record, source_state: Record +) -> str | None: + """Say if the workers timed code other than this checkout's commit.""" + commit = (provenance.get("build_info") or {}).get("git_commit") + if commit and commit != source_state.get("revision"): + return ( + f"the native backend was built from {commit[:12]}, not this checkout's " + "commit; rebuild it (uv sync --extra native)" + ) + source = ROOT / "src" + if any( + row["solprop_path"] is None + or not Path(row["solprop_path"]).is_relative_to(source) + for row in results + ): + return f"the workers imported solprop from outside {source}" + return None + + def track(args: argparse.Namespace) -> int: """Run the selected configurations, one worker each, and record the results.""" if args.repeats < 1 or args.timeout <= 0: @@ -236,13 +305,7 @@ def track(args: argparse.Namespace) -> int: machine = hardware(production, args.instance_type) source = git_receipt(ROOT) if args.append_history: - if args.backend != "native": - raise SuiteError("the history records the native backend only") - if source.get("dirty") and not args.allow_dirty: - raise SuiteError( - "uncommitted changes: commit them, or pass --allow-dirty to record " - "this run anyway" - ) + refuse_partial(args, rows, production, source) stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") output: Path = args.out or LOCAL_ROOT / "track" / f"{stamp}-{machine['slug']}" if output.exists() and any(output.iterdir()): @@ -287,22 +350,26 @@ def track(args: argparse.Namespace) -> int: f"({elapsed:.0f} s with construction)" + (" busy" if busy else ""), flush=True, ) - atomic_record( - output / "results.json", - { - "rows": results, - "provenance": provenance, - "collection_seconds": time.perf_counter() - started, - }, - ) + # Rewritten after every worker, so an interrupted run keeps what it did. + atomic_record( + output / "results.json", + { + "rows": results, + "provenance": provenance, + "collection_seconds": time.perf_counter() - started, + "complete": index + 1 == len(rows), + }, + ) show_results(results) - complete = all(row["status"] == "ok" and row["finite"] for row in results) + refusal = admissible(results) + complete = refusal is None print( f"Results: {output.resolve()} ({(time.perf_counter() - started) / 60:.0f} min)" ) if args.append_history: - if not complete: - print("Not added to the history: a configuration failed.", file=sys.stderr) + refusal = refusal or imported_elsewhere(results, provenance, source) + if refusal is not None: + print(f"Not added to the history: {refusal}.", file=sys.stderr) return 1 path = history.HISTORY / f"{machine['slug']}.csv" history.append(path, history_rows(results, machine, source, provenance)) diff --git a/benchmarks/worker.py b/benchmarks/worker.py index 57b22c44..812491f9 100644 --- a/benchmarks/worker.py +++ b/benchmarks/worker.py @@ -44,10 +44,16 @@ "measure", "runtime", "source_calls", + "timed_run", ] if TYPE_CHECKING: + import numpy as np + from solprop.backends.protocol import Backend + from solprop.driver import Simulation + from solprop.output.schema import FieldRetention + from solprop.translate.problem import KernelProblem def emit(kind: str, **values: object) -> None: @@ -160,6 +166,48 @@ def integrator_label(integrator: object) -> str: return name +def timed_run( + simulation: Simulation, + problem: KernelProblem, + engine: Backend, + retention: FieldRetention, +) -> tuple[Record, np.ndarray]: + """Time one complete run from the launch; return its counts and final field.""" + import numpy as np # noqa: PLC0415 + + from solprop.driver.run import run_problem # noqa: PLC0415 + from solprop.output import MemoryOutput # noqa: PLC0415 + + engine.set_initial(problem.initial.field0, None, z=problem.initial.z) + output = MemoryOutput() + before = source_calls(engine) + started = time.perf_counter_ns() + run_problem( + problem, + engine, + output, + metadata=simulation.metadata(), + stats_spec=problem.stats_spec, + progress="none", + resume_history=simulation.settings.resume_history, + fields=retention.fields, + field_planes=retention.field_planes, + ) + seconds = (time.perf_counter_ns() - started) * 1e-9 + after = source_calls(engine) + accepted = np.asarray(output["step_history/accepted"], dtype=bool) + record = { + "seconds": seconds, + "accepted": int(accepted.sum()), + "rejected": int((~accepted).sum()), + "source_evaluations": None + if before is None or after is None + else after - before, + "z": float(output["z"][-1]), + } + return record, output.plane_field(output.n_saved - 1, "fft") + + def measure(configuration: Record, repeats: int, fields: Path | None) -> Record: """Build once, warm up once, then time ``repeats`` complete runs. @@ -173,8 +221,7 @@ def measure(configuration: Record, repeats: int, fields: Path | None) -> Record: from benchmarks.builders import build # noqa: PLC0415 from solprop.backends import select # noqa: PLC0415 from solprop.driver.executors.pool import peak_rss # noqa: PLC0415 - from solprop.driver.run import field_retention, run_problem # noqa: PLC0415 - from solprop.output import MemoryOutput # noqa: PLC0415 + from solprop.driver.run import field_retention # noqa: PLC0415 from solprop.output.schema import serialise # noqa: PLC0415 started = time.perf_counter() @@ -193,48 +240,33 @@ def measure(configuration: Record, repeats: int, fields: Path | None) -> Record: retention = field_retention( problem, simulation.settings.fields, simulation.settings.field_planes ) - metadata = simulation.metadata() controller = problem.stepper.controller try: started = time.perf_counter() engine.set_initial(problem.initial.field0, None, z=problem.initial.z) if configuration["warmup"]: engine.propagate_to(configuration["warmup"] * configuration["stop"]) + # Observe once, so that the observation path's first use is warm too. + engine.observe(field=True) + engine.stats(problem.stats_spec) warmup = time.perf_counter() - started repetitions = [] - output = None + first = final = None + difference = 0.0 for _ in range(repeats): - engine.set_initial(problem.initial.field0, None, z=problem.initial.z) - output = MemoryOutput() - before = source_calls(engine) - started = time.perf_counter_ns() - run_problem( - problem, - engine, - output, - metadata=metadata, - stats_spec=problem.stats_spec, - progress="none", - resume_history=simulation.settings.resume_history, - fields=retention.fields, - field_planes=retention.field_planes, - ) - seconds = (time.perf_counter_ns() - started) * 1e-9 - after = source_calls(engine) - accepted = np.asarray(output["step_history/accepted"], dtype=bool) - repetitions.append( - { - "seconds": seconds, - "accepted": int(accepted.sum()), - "rejected": int((~accepted).sum()), - "source_evaluations": None - if before is None or after is None - else after - before, - "z": float(output["z"][-1]), - } - ) - assert output is not None - final = output.plane_field(output.n_saved - 1, "fft") + repetition, final = timed_run(simulation, problem, engine, retention) + repetitions.append(repetition) + # Every repeat restarts from the same launch and seed, so its final + # field may differ from the first repeat's by rounding only. + if first is None: + first = final + else: + scale = float(np.linalg.norm(first)) or 1.0 + difference = max( + difference, float(np.linalg.norm(final - first)) / scale + ) + assert first is not None + final = first if final is None else final if fields is not None: fields.parent.mkdir(parents=True, exist_ok=True) np.save(fields, final, allow_pickle=False) @@ -245,6 +277,7 @@ def measure(configuration: Record, repeats: int, fields: Path | None) -> Record: "repetitions": repetitions, "peak_rss_bytes": peak_rss(), "finite": bool(np.all(np.isfinite(final))), + "repeat_difference": difference, "integrator": integrator_label(problem.stepper.integrator), "rtol": controller.rtol, "atol": controller.atol, diff --git a/docs/dev/benchmarking.md b/docs/dev/benchmarking.md index e9a07206..f8f5d217 100644 --- a/docs/dev/benchmarking.md +++ b/docs/dev/benchmarking.md @@ -37,7 +37,7 @@ node; other machines differ). With no selection it is the tracking set: | `ctc` | `prop_capillary` | H2 comb-to-continuum, three modes, 0.13 m | production | | `radial_filament_256` | `prop_radial` | Ar filament, 256 radial nodes, 3 cm | 1, production | | `grin_nonlinear` | `prop_waveguide` | four-channel graded-index fibre, 2 cm | production | -| `kerr_beam_384` | `prop_free` | fused-silica Kerr beam, 384², fixed RK4IP steps | production | +| `kerr_beam_384` | `prop_free` | fused-silica Kerr beam, 384², four fixed RK4IP steps | production | | `kerr_beam_192` | `prop_free` | the same at 192² | 1 | The definitions are in `benchmarks/registry.py` and `benchmarks/builders.py`. Every @@ -87,6 +87,12 @@ names the widths for every selected case. The production width is the number of physical cores in the process's CPU allocation unless `--production` gives it; on a cluster, give it explicitly. +**Threading.** `--threading` sets OpenMP's placement and waiting in every worker: +`passive` (unpinned, idle threads sleep at once), `default` (the runtime's own short +spin before sleeping), `active` (idle threads spin), `bound` (threads pinned to +neighbouring cores) or `bound-active`. The policy is recorded with each row, and rows +compare only within one policy. + ## Recording a run ```sh @@ -94,21 +100,38 @@ uv run python -m benchmarks track --append-history ``` The run's bundle goes to `.benchmarks/track/-/` (git ignores it): the -plan, each worker's request and logs, and `results.json`. `--archive DIR` copies the +plan, each worker's request, environment and logs, and `results.json`, which is +rewritten after every configuration so that an interrupted run keeps what it +measured. `--archive DIR` copies the bundle to a directory outside the repository. `--append-history` adds one row per configuration to -`benchmarks/history/.csv`, which is committed. It refuses a run with a -failed configuration, a backend other than native, and a source tree with uncommitted -changes (unless `--allow-dirty`). The hardware name is the CPU model and the physical -cores used, for example `apple-m5-max-18c` or `amd-epyc-7763-16c`; it never contains a -host name. A cloud instance type, from `--instance-type` or the machine's DMI record, -goes in its own column. +`benchmarks/history/.csv`, which is committed. It records only the whole +default plan with at least three repeats, on the native backend, and only when: + +- every configuration finished, and its repeats took the same steps and reached the + same final field to rounding, with the native backend running the threads it was + given; +- the workers imported solprop from this checkout's `src/` (run from the checkout's + own environment, `uv sync --extra native`, not an installed copy) with a native + build of the same commit; +- nothing under `src/`, `native/` or `benchmarks/` is uncommitted, unless + `--allow-dirty`, whose rows are marked dirty. + +The hardware name is the CPU model and the production width, plus the instance family +on a cloud machine, for example `apple-m5-max-18c`, `amd-epyc-7763-16c` or +`arm-neoverse-v2-16c-aws-m8g`; it never contains a host name. Give a cloud machine's +instance type with `--instance-type` where its DMI record does not name it (AWS's +does). `python -m benchmarks history` prints, per hardware and width, one line per recorded -run: its date, solprop version, case count, and the geometric mean of its median times -over the first recorded run's, over the configurations both ran with the same case -revision and integrator. Below one is faster. +run: its date, solprop version, the configurations it ran and how many started on a +busy machine, and its change against the first and the previous recorded runs. A +change is the geometric mean of median-time ratios (below one is faster) over the +configurations both runs share, counted in brackets. Two rows are the same +configuration only with the same threading policy, case revision, stepper, resolved +integrator, tolerances and noise model, so a changed library default starts a new +series rather than reading as a speed-up. ## Comparing two builds while iterating diff --git a/docs/dev/benchmarks.md b/docs/dev/benchmarks.md index b2d20175..9cb985f6 100644 --- a/docs/dev/benchmarks.md +++ b/docs/dev/benchmarks.md @@ -80,7 +80,7 @@ baseline runs beside every ratio; a difference inside it is not a result. **What goes in a pull request.** The measured window and what it includes, the build and machine record, the per-case table with medians and spreads, the declared aggregate and its verdict, and any case that regressed. Concise evidence belongs in the change -description; raw runs stay local or go into an exported snapshot. +description; raw runs stay local. **Numba's on-disk cache.** Clear it (`find src -name '__pycache__' -type d -prune -exec rm -rf {} +`) before timing in a diff --git a/docs/dev/seps/sep-0038-performance-tracking.md b/docs/dev/seps/sep-0038-performance-tracking.md index 3e6db72d..eb8b2069 100644 --- a/docs/dev/seps/sep-0038-performance-tracking.md +++ b/docs/dev/seps/sep-0038-performance-tracking.md @@ -107,24 +107,33 @@ tens of per cent of the millisecond windows SEP 30 measured. whose thread pools are set in the environment before import. Peak resident memory is then the configuration's own. +**Consistency.** A run's repeats restart from the same seeded launch, so they must take +the same steps and reach the same final field to rounding (a relative L2 difference +below 1e-9), and the native backend must run the threads it was given. A configuration +that fails this is reported and kept out of the history. + **Aggregate.** For one hardware set and width, the change between two runs is the -geometric mean, over the cases both ran with the same case revision and resolved -integrator, of the ratio of their medians. A changed default integrator is a different -configuration, and is shown as such rather than averaged in. +geometric mean, over the configurations both ran, of the ratio of their medians. Two +rows are the same configuration only with the same threading policy, case revision, +stepper, resolved integrator, tolerances and noise model: a changed default is a new +configuration, shown as such rather than averaged in. Each run is compared with the +first recorded and with the previous one, with the number of shared configurations. ### 5. Proposed interface #### 5.1 Commands ```text -python -m benchmarks list [--all] [--case NAME] [--stepper S] [--threads W] -python -m benchmarks track [--case NAME] [--all] [--stepper S] [--threads W] - [--repeats 3] [--production N] [--instance-type TYPE] - [--append-history] [--archive DIR] [--out DIR] -python -m benchmarks ab --baseline-python PY [--candidate-python PY] - [--case NAME] [--stepper S] [--threads W] - [--length FRACTION] [--rounds 3] [--repeats 3] -python -m benchmarks history [SLUG] [--width W] +python -m benchmarks list [SELECTION] +python -m benchmarks track [SELECTION] [--dry-run] [--instance-type TYPE] + [--append-history [--allow-dirty]] [--archive DIR] + [--out DIR] [--timeout SECONDS] +python -m benchmarks ab --baseline-python PY [--candidate-python PY] [SELECTION] + [--length FRACTION] [--rounds 3] [--out DIR] [--json] +python -m benchmarks history [SLUG] [--threads W] + +SELECTION: [--case NAME]... [--set SET]... [--all] [--stepper S] [--threads W] + [--production N] [--backend B] [--repeats 3] [--threading POLICY] ``` - `list` prints the plan and its estimated time; it imports no scientific library. @@ -136,10 +145,21 @@ python -m benchmarks history [SLUG] [--width W] interface's. A case on a fixed schedule admits only `default`. - `--production` gives the production width; otherwise it is the number of physical cores in the process's CPU allocation. -- `--append-history` adds the run to `benchmarks/history/.csv`. It refuses a - dirty source tree, an incomplete run and a non-native backend. -- `history` prints, per width, one line per recorded run: date, version, commit, the - number of cases, and the geometric mean relative to the first recorded run. +- `--threading` sets OpenMP's placement and waiting for every worker: `passive` + (unpinned; idle threads sleep at once), `default` (the runtime's own waiting), + `active` (idle threads spin), `bound` (pinned to neighbouring cores) or + `bound-active`. The policy is part of what history rows compare by; the default is + set by phase 2's measurement. +- `--append-history` adds the run to `benchmarks/history/.csv`. It records only + the whole default plan with at least three repeats, on the native backend, every + configuration complete and consistent (§4), timing the checkout's own `src/` with a + native build of the same commit. It refuses uncommitted changes under `src/`, + `native/` or `benchmarks/` unless `--allow-dirty`, whose rows are marked dirty. +- `history` prints, per width, one line per recorded run: date, version, the number of + configurations and of busy ones, and the change against the first and the previous + recorded runs with the number of configurations each compares. +- Each worker's thread and OpenMP environment is saved beside its request; a Kokkos + tool library in the caller's environment is removed. #### 5.2 The tracking set @@ -161,16 +181,18 @@ marks a width the case does not run at. | `ctc` (rev. 2) | `prop_capillary` | H2 comb-to-continuum, 3 modes, 40 bar, 0.13 m, 2 ps window | P | — / 291 | | `radial_filament_256` | `prop_radial` | Ar filament, 256 radial nodes, 3 cm | 1, P | 16 / 6 | | `grin_nonlinear` (rev. 2) | `prop_waveguide` | four-channel graded-index fibre, 2 cm | P | — / 38 | -| `kerr_beam_384` | `prop_free` | fused-silica Kerr beam, 384² transverse grid, fixed RK4IP | P | — / 134 | -| `kerr_beam_192` | `prop_free` | the same at 192², fixed RK4IP | 1 | 37 / — | +| `kerr_beam_384` (rev. 2) | `prop_free` | fused-silica Kerr beam, 384² transverse grid, four fixed RK4IP steps | P | — / ~49 | +| `kerr_beam_192` (rev. 2) | `prop_free` | the same at 192², four fixed RK4IP steps | 1 | ~13 / — | -The sum is about 270 s at one thread and 620 s at 16 threads per pass. With three -repeats, the warm-ups and construction, the run took about 50 minutes on that node; the -target is under an hour on any machine in use. +The sum is about 250 s at one thread and 530 s at 16 threads per pass. With three +repeats, the warm-ups and construction, the run took about 50 minutes on that node +before the Kerr beams were cut from eleven fixed steps to four (their cost per step is +unchanged, so they track the same work); about 40 minutes now. The target is under an +hour on any machine in use. The three most expensive cases run at the production width only: `ctc` (about 600 s at one thread in SEP 37's phase 0), the 2 cm GRIN window (about 100 s) and the 384² Kerr -beam. The 192² Kerr beam keeps a one-thread free-space row. +beam (25 GiB). The 192² Kerr beam keeps a one-thread free-space row. Every case keeps its interface's default noise (`"input"` after SEP 37) with a fixed seed. Every other registered case stays runnable by name; none is in the default @@ -178,9 +200,15 @@ selection. #### 5.3 Hardware identity and history -The **hardware slug** is the CPU model reduced to lower-case words, followed by the number -of physical cores in the allocation, for example `apple-m5-max-18c` or -`amd-epyc-7763-16c`. It names no host. A cloud instance type, from `--instance-type` or +The **hardware slug** is the CPU model reduced to lower-case words, followed by the +production width in cores and, on a cloud instance, its family, for example `apple-m5-max-18c`, +`amd-epyc-7763-16c` or `arm-neoverse-v2-16c-aws-m8g`. It names no host. The model comes from +`/proc/cpuinfo` on x86 Linux, `lscpu` on Arm Linux (whose `/proc/cpuinfo` names no +model), `sysctl` on macOS and the registry on Windows. + +The standing history files are an Apple M5 Max, a 16-core allocation of an AMD EPYC +7763 cluster node and a cloud Arm instance (Neoverse-V2, for example AWS Graviton4), +with Windows x86-64 measured occasionally. GPU tracking is left to a later SEP. A cloud instance type, from `--instance-type` or the DMI product name on Linux, is recorded beside it but is not part of the slug. Each history row is one (run, case, stepper, width): date, solprop version, commit, @@ -270,11 +298,9 @@ is final. - Q1. If SEP 37's capillary default becomes `RungeKutta()` at 1e-5, the capillary costs fall; should the freed time go to a longer `ctc`, or should `ctc` run at one thread too? -- Q3. The 384² Kerr beam costs 134 s per run at 16 threads on the EPYC node (25 GiB), - a fifth of the run's time for one free-space case. Keep it, or track the 192² beam at - both widths instead? -- Q2. Should a cloud instance (for example AWS `c8a` or GCP `c4d`) be a standing - history file, re-rented per release, or measured only when a hardware question arises? +- Q2. Decided: a cloud Arm instance is a standing history file, re-rented per + release, and Windows x86-64 is measured occasionally (§5.3). +- Q3. Decided: both Kerr beams stay, over four fixed steps instead of eleven. ### 13. References @@ -317,4 +343,13 @@ is final. ## Part III — History - 2026-10-27: Drafted, together with its implementation on the same branch, as the - maintainer asked; it supersedes SEP 30 at Final. + maintainer asked; it supersedes SEP 30 at Final. This departs from SEP 0, which + implements a SEP after acceptance: the maintainer chose to draft, implement and + calibrate together, and to review the whole before acceptance. +- 2026-10-27: Independent AI review of the draft and its code (Opus 5.5). Its findings + were fixed on the branch: history rows compare by tolerance and noise model too, a + run's repeats must agree in steps and final field, only a complete default run of + committed code with a matching native build is recorded, older history files gain + new columns, the slug counts the production width and names a cloud family, and the + busy flag judges by CPU use where swap is not reported. Its open questions went to + the maintainer. diff --git a/native/README.md b/native/README.md index e912efb6..eaa4525b 100644 --- a/native/README.md +++ b/native/README.md @@ -145,8 +145,7 @@ Kokkos Kernels and other dependencies are not project coverage. The coverage build also installs `solprop_native_labels`, a host-only Kokkos Tools module that counts parallel-for/reduce/scan labels and profile regions without requesting fences. Set `KOKKOS_TOOLS_LIBS` to this module and `SOLPROP_NATIVE_LABEL_FILE` -to a new JSON-lines output path for a separate label pass. The benchmark suite's -`coverage` command does this itself. Reports, raw profiles and binary snapshots belong +to a new JSON-lines output path for a separate label pass. Reports, raw profiles and binary snapshots belong in ignored local `.benchmarks/` storage, not in git. An uninstantiated template or compiled-out path has no counter and is not an executed path with a zero count; CPU coverage has no target percentage. diff --git a/tests/test_benchmark_ab.py b/tests/test_benchmark_ab.py index f7711e4e..e3baabba 100644 --- a/tests/test_benchmark_ab.py +++ b/tests/test_benchmark_ab.py @@ -176,7 +176,7 @@ def test_small_case_runs_both_builds_in_alternating_order( "--rounds", "2", "--repeats", - "2", + "1", "--baseline-python", sys.executable, "--out", @@ -195,7 +195,7 @@ def test_small_case_runs_both_builds_in_alternating_order( ] results = read_record(out / "results.json") assert len(results["samples"]) == 4 - assert all(len(item["repetitions"]) == 2 for item in results["samples"]) + assert all(len(item["repetitions"]) == 1 for item in results["samples"]) assert all( rep["z"] == pytest.approx(0.5e-4) for item in results["samples"] diff --git a/tests/test_benchmark_cases.py b/tests/test_benchmark_cases.py index 115b3d4b..ebe86484 100644 --- a/tests/test_benchmark_cases.py +++ b/tests/test_benchmark_cases.py @@ -47,8 +47,8 @@ def test_cases_run_with_the_interface_defaults_and_a_fixed_seed(name, integrator ], ) def test_a_stepper_override_replaces_the_integrator_only(stepper, integrator): - default = build(configuration("rdw")) - chosen = build(configuration("rdw", stepper)) + default = build(configuration("he_rdw_averaged")) + chosen = build(configuration("he_rdw_averaged", stepper)) assert isinstance(chosen.settings.integrator, integrator) assert chosen.settings.controller is not None assert default.settings.controller is not None @@ -156,3 +156,23 @@ def test_selection_and_builders_refuse_unknown_names(): capillary("absent") with pytest.raises(SuiteError, match="not implemented"): gnlse("absent") + + +def test_tracked_definitions_change_only_with_a_new_revision(): + # A changed definition must raise its revision: rows compare within one. The + # digest covers the registry's declared fields; builder edits need the same. + assert [(case.name, case.revision, case.digest[:12]) for case in select()] == [ + ("gnlse_raman_shock", 1, "b3f7fc864691"), + ("gnlse_soliton", 1, "c842ded9a2c7"), + ("rdw", 1, "883433a7ddb1"), + ("he_rdw_averaged", 1, "677a784eeb82"), + ("argon_blueshift", 2, "a080d3f0462f"), + ("argon_blueshift_averaged", 2, "66e834d1cf14"), + ("argon_blueshift_envelope", 1, "a11d59421661"), + ("h2_raman_averaged", 1, "7dbd829ef3bf"), + ("ctc", 2, "089a0f9e01e3"), + ("radial_filament_256", 1, "99fd7698eaaf"), + ("grin_nonlinear", 2, "b0e47fd41668"), + ("kerr_beam_384", 2, "44905b3e6d02"), + ("kerr_beam_192", 2, "726f5965e965"), + ] diff --git a/tests/test_benchmark_exercises.py b/tests/test_benchmark_exercises.py index b0c0518a..0b3390cb 100644 --- a/tests/test_benchmark_exercises.py +++ b/tests/test_benchmark_exercises.py @@ -17,7 +17,7 @@ from solprop.exceptions import MaterialRangeWarning -@pytest.mark.medium +@pytest.mark.slow @pytest.mark.parametrize( "name", [case.name for case in select(sets=("backend-exercise",))] ) @@ -56,3 +56,19 @@ def test_repeats_restart_from_the_launch_and_take_the_same_steps(tmp_path): assert result["finite"] assert result["noise"] == "modified" assert np.isfinite(np.load(tmp_path / "field.npy")).all() + + +@pytest.mark.medium +def test_repeats_of_an_input_noise_run_reach_the_same_field(): + # The tracked default noise: each repeat restarts from the same seeded launch. + configuration = CASES["gnlse_soliton"].configuration( + "default", 1, "numpy_ref", 0.002 + ) + result = measure(configuration, 2, None) + first, second = result["repetitions"] + assert result["noise"] == "input" + assert (first["accepted"], first["rejected"]) == ( + second["accepted"], + second["rejected"], + ) + assert result["repeat_difference"] <= 1e-12 diff --git a/tests/test_benchmark_track.py b/tests/test_benchmark_track.py index 27c7d955..80f25a3f 100644 --- a/tests/test_benchmark_track.py +++ b/tests/test_benchmark_track.py @@ -4,7 +4,9 @@ """Exercise the tracking command's plan, hardware identity and history (SEP 38).""" +import argparse import csv +import importlib.util import socket import sys @@ -73,9 +75,10 @@ def test_the_hardware_slug_names_the_cpu_model_and_cores(model, cores, expected) def test_the_hardware_record_names_no_host(): - record = system.hardware(1, "aws c8a.4xlarge") - assert record["instance_type"] == "aws c8a.4xlarge" - assert record["slug"].endswith(f"-{system.physical_cores()}c") + record = system.hardware(1, "aws m8g.4xlarge") + assert record["instance_type"] == "aws m8g.4xlarge" + # The slug counts the cores the run uses, then names the instance family. + assert record["slug"].endswith("-1c-aws-m8g") text = repr(record).lower() assert socket.gethostname().split(".")[0].lower() not in text @@ -96,6 +99,10 @@ def result_row(case="rdw", threads=1, median=2.0): "accepted": 100, "rejected": 2, "same_steps": True, + "repeat_difference": 0.0, + "native_threads": threads, + "consistent": True, + "solprop_path": str(runner.ROOT / "src" / "solprop" / "__init__.py"), "source_evaluations": 104, "peak_rss_gib": 0.5, "construction_s": 3.0, @@ -139,20 +146,91 @@ def test_history_rows_append_read_and_summarise(tmp_path): assert rows[0]["native_build"] == "local" assert rows[0]["commit"] == "a" * 12 summary = history.summary(rows) - assert [(row["threads"], row["ratio"]) for row in summary] == [ - ("1", 1.0), - ("2", 1.0), - ("1", 0.5), - ("2", 0.5), - ] - with path.open("w", encoding="utf-8", newline="") as stream: - csv.writer(stream).writerow(["date"]) - with pytest.raises(SuiteError, match="columns differ"): - history.append(path, first) + assert [ + (row["threads"], row["ratio_first"], row["ratio_previous"]) for row in summary + ] == [("1", 1.0, 1.0), ("2", 1.0, 1.0), ("1", 0.5, 0.5), ("2", 0.5, 0.5)] + # A changed tolerance is a new configuration, not a speed-up. + changed = [{**row, "date": "2100-01-01T00:00Z", "rtol": "1e-05"} for row in later] + history.append(path, changed) + assert history.summary(history.read(path))[-1]["shared_previous"] == 0 with pytest.raises(SuiteError, match="exactly the declared columns"): history.append(tmp_path / "new.csv", [{"date": "x"}]) +def test_older_history_files_gain_new_columns_and_newer_ones_are_refused(tmp_path): + path = tmp_path / "old.csv" + with path.open("w", encoding="utf-8", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(["date", "case", "median_s"]) + writer.writerow(["2026-01-01T00:00Z", "rdw", "2.0"]) + row: dict[str, object] = dict.fromkeys(history.COLUMNS, "x") + history.append(path, [row]) + rows = history.read(path) + assert tuple(rows[0]) == history.COLUMNS + assert (rows[0]["case"], rows[0]["commit"]) == ("rdw", "") + with path.open("w", encoding="utf-8", newline="") as stream: + csv.writer(stream).writerow([*history.COLUMNS, "future"]) + with pytest.raises(SuiteError, match="later version"): + history.append(path, [row]) + + +def namespace(**values): + defaults = {"backend": "native", "allow_dirty": False, "repeats": 3} + return argparse.Namespace(**(defaults | values)) + + +@pytest.mark.usefixtures("sixteen_cpus") +def test_the_history_takes_only_a_complete_default_run_of_committed_code(): + rows, _ = runner.plan_rows(select(), ("default",), None, 16) + clean = {"revision": "a" * 40, "dirty": False} + track.refuse_partial(namespace(), rows, 16, clean) + for args, source, message in ( + (namespace(backend="numpy_ref"), clean, "native backend only"), + (namespace(), {"revision": None, "dirty": None}, "source state is unknown"), + (namespace(), {**clean, "dirty": True}, "uncommitted changes"), + (namespace(repeats=2), clean, "three repeats"), + ): + with pytest.raises(SuiteError, match=message): + track.refuse_partial(args, rows, 16, source) + track.refuse_partial( + namespace(allow_dirty=True), rows, 16, {**clean, "dirty": True} + ) + with pytest.raises(SuiteError, match="whole default plan"): + track.refuse_partial(namespace(), rows[:-1], 16, clean) + + +def test_inconsistent_or_foreign_runs_are_not_recorded(): + good = [result_row()] + assert track.admissible(good) is None + assert "failed" in str(track.admissible([{**good[0], "status": "failed"}])) + assert "repeats" in str(track.admissible([{**good[0], "consistent": False}])) + source = {"revision": "a" * 40} + native = {"build_info": {"git_commit": "a" * 40}} + assert track.imported_elsewhere(good, native, source) is None + stale = {"build_info": {"git_commit": "b" * 40}} + assert "rebuild" in str(track.imported_elsewhere(good, stale, source)) + elsewhere = [{**good[0], "solprop_path": "/elsewhere/solprop/__init__.py"}] + assert "outside" in str(track.imported_elsewhere(elsewhere, native, source)) + row = {"id": "x", "configuration": CASES["rdw"].configuration("default", 4)} + receipt = { + "status": "ok", + "repetitions": [{"seconds": 1.0, "accepted": 5, "rejected": 0, + "source_evaluations": 6, "z": 1.0}], + "repeat_difference": 0.0, "peak_rss_bytes": 1, "construction_seconds": 1.0, + "backend_seconds": 0.0, "warmup_seconds": 0.0, "integrator": "RungeKutta", + "rtol": 1e-6, "atol": 0.0, "noise": None, "finite": True, + "provenance": {"build_info": {"runtime_threads": 2}}, + } # fmt: skip + # The thread budget gave the native backend fewer threads than asked for. + assert not track.summarise(row, receipt, busy=False)["consistent"] + + +def test_a_machine_without_swap_counters_is_judged_by_cpu_alone(): + before = {"time": 0.0, "busy": [0.0], "swap_bytes": None} + after = {"time": 1.0, "busy": [0.05], "swap_bytes": None} + assert system.interval(before, after, (0,))["quiet"] + + def test_a_failed_worker_is_summarised_without_statistics(): row = {"id": "x", "configuration": CASES["rdw"].configuration("default", 1)} result = track.summarise(row, {"status": "failed", "error": "boom"}, busy=True) @@ -246,3 +324,19 @@ def test_threading_policies_replace_the_callers_openmp_settings(): assert "OMP_WAIT_POLICY" not in bound with pytest.raises(SuiteError, match="unknown threading"): system.thread_environment(4, incoming, "spin") + + +@pytest.mark.medium +@pytest.mark.skipif( + importlib.util.find_spec("solprop_native") is None, + reason="the native backend is not installed", +) +def test_a_native_tracking_run_records_its_build(tmp_path, monkeypatch): + monkeypatch.setattr(track, "guard", lambda *_args: {"quiet": True}) + out = tmp_path / "native" + arguments = ["track", "--case", "smoke", "--threads", "1", "--repeats", "1"] + assert cli.main([*arguments, "--out", str(out)]) == 0 + results = read_record(out / "results.json") + assert results["provenance"]["build_info"]["package_version"] + assert results["rows"][0]["native_threads"] == 1 + assert (out / "raw" / "smoke.default.t1" / "environment.json").exists() From ef819ca50872dbf02e43aec8f794e697c15fe17c Mon Sep 17 00:00:00 2001 From: John Travers Date: Tue, 27 Oct 2026 09:12:56 +0000 Subject: [PATCH 09/13] Compare history rows by the memory domains spanned; rebuild native only when native/ changed Co-Authored-By: Claude Opus 5.5 --- benchmarks/history.py | 5 ++- benchmarks/system.py | 42 +++++++++++++++++++ benchmarks/track.py | 15 +++++-- docs/dev/benchmarking.md | 13 +++--- .../dev/seps/sep-0038-performance-tracking.md | 13 ++++-- tests/test_benchmark_track.py | 14 +++++++ 6 files changed, 89 insertions(+), 13 deletions(-) diff --git a/benchmarks/history.py b/benchmarks/history.py index a606e6f3..a0d5a641 100644 --- a/benchmarks/history.py +++ b/benchmarks/history.py @@ -35,6 +35,7 @@ "cpu_model", "instance_type", "system", + "numa_domains", "threads", "threading", "case", @@ -100,13 +101,15 @@ def read(path: Path) -> list[Record]: def comparable(row: Record) -> tuple[str, ...]: """Key the rows that measure the same thing on one hardware set. - Two rows compare only with the same width, threading, case definition, + Two rows compare only with the same memory domains spanned, width, threading, + case definition, stepper, resolved integrator, tolerances and noise model: a changed library default is a new configuration, not a speed-up. """ return tuple( str(row.get(key) or "") for key in ( + "numa_domains", "threads", "threading", "case", diff --git a/benchmarks/system.py b/benchmarks/system.py index 36bfdd59..45cb1809 100644 --- a/benchmarks/system.py +++ b/benchmarks/system.py @@ -31,6 +31,8 @@ "guard", "hardware", "interval", + "native_changed", + "numa_domains", "physical_cores", "slug", "thread_environment", @@ -153,6 +155,7 @@ def hardware(production: int, instance_type: str | None = None) -> Record: cores = physical_cores() instance = instance_type or _instance_type() return { + "numa_domains": numa_domains(details["cpu_topology"]), # The cores a run uses name the hardware set: on a cluster the CPU mask can # show the whole node while the allocation is a part of it. "slug": slug(model, production, instance), @@ -169,6 +172,45 @@ def hardware(production: int, instance_type: str | None = None) -> Record: } +def numa_domains(topology: list[Record] | None) -> int: + """Count the memory (NUMA) domains the allocated CPUs span; 1 where unknown. + + A 16-core allocation inside one domain and one spread over eight share a CPU + model and core count but not memory bandwidth, so rows compare by this too. + """ + nodes = {node for row in topology or () for node in row.get("numa_nodes") or ()} + return max(1, len(nodes)) + + +def native_changed(directory: Path, commit: str, revision: str) -> bool: + """Whether ``native/`` differs between a native build's commit and `revision`. + + Its documentation and tests do not count; a commit Git cannot find does. + """ + try: + result = subprocess.run( + [ + "git", + "-C", + str(directory), + "diff", + "--quiet", + commit, + revision, + "--", + "native", + ":(exclude,glob)native/**/*.md", + ":(exclude)native/tests", + ], + capture_output=True, + timeout=30, + check=False, + ) + except OSError, subprocess.TimeoutExpired: + return True + return result.returncode != 0 + + def slug(model: str, cores: int, instance: str | None = None) -> str: """Name a hardware set by CPU model and core count, e.g. ``apple-m5-max-18c``. diff --git a/benchmarks/track.py b/benchmarks/track.py index 33934554..0c65e0bf 100644 --- a/benchmarks/track.py +++ b/benchmarks/track.py @@ -31,6 +31,7 @@ git_receipt, guard, hardware, + native_changed, physical_cores, thread_environment, ) @@ -211,6 +212,7 @@ def history_rows( "cpu_model": machine["cpu_model"], "instance_type": machine["instance_type"], "system": f"{machine['system']} {machine['machine']}", + "numa_domains": machine["numa_domains"], "threads": row["threads"], "threading": row["threading"], "case": row["case"], @@ -270,10 +272,17 @@ def imported_elsewhere( ) -> str | None: """Say if the workers timed code other than this checkout's commit.""" commit = (provenance.get("build_info") or {}).get("git_commit") - if commit and commit != source_state.get("revision"): + revision = source_state.get("revision") + # A build from an earlier commit is current if native/ has not changed since. + if ( + commit + and revision + and commit != revision + and native_changed(ROOT, commit, revision) + ): return ( - f"the native backend was built from {commit[:12]}, not this checkout's " - "commit; rebuild it (uv sync --extra native)" + f"native/ changed since the native backend's build ({commit[:12]}); " + "rebuild it (uv sync --extra native)" ) source = ROOT / "src" if any( diff --git a/docs/dev/benchmarking.md b/docs/dev/benchmarking.md index f8f5d217..f0e0ae1d 100644 --- a/docs/dev/benchmarking.md +++ b/docs/dev/benchmarking.md @@ -113,8 +113,9 @@ default plan with at least three repeats, on the native backend, and only when: same final field to rounding, with the native backend running the threads it was given; - the workers imported solprop from this checkout's `src/` (run from the checkout's - own environment, `uv sync --extra native`, not an installed copy) with a native - build of the same commit; + own environment, `uv sync --extra native`, not an installed copy), with a native + build from a commit after which nothing under `native/` changed but its + documentation and tests; - nothing under `src/`, `native/` or `benchmarks/` is uncommitted, unless `--allow-dirty`, whose rows are marked dirty. @@ -129,9 +130,11 @@ run: its date, solprop version, the configurations it ran and how many started o busy machine, and its change against the first and the previous recorded runs. A change is the geometric mean of median-time ratios (below one is faster) over the configurations both runs share, counted in brackets. Two rows are the same -configuration only with the same threading policy, case revision, stepper, resolved -integrator, tolerances and noise model, so a changed library default starts a new -series rather than reading as a speed-up. +configuration only with the same number of memory (NUMA) domains spanned, threading +policy, case revision, stepper, resolved integrator, tolerances and noise model. A +changed library default therefore starts a new series rather than reading as a +speed-up, and so does an allocation spread over more memory domains than usual: on a +cluster, keep the cores together (`--distribution=block:block` under Slurm). ## Comparing two builds while iterating diff --git a/docs/dev/seps/sep-0038-performance-tracking.md b/docs/dev/seps/sep-0038-performance-tracking.md index eb8b2069..9cbb4e2c 100644 --- a/docs/dev/seps/sep-0038-performance-tracking.md +++ b/docs/dev/seps/sep-0038-performance-tracking.md @@ -114,7 +114,8 @@ that fails this is reported and kept out of the history. **Aggregate.** For one hardware set and width, the change between two runs is the geometric mean, over the configurations both ran, of the ratio of their medians. Two -rows are the same configuration only with the same threading policy, case revision, +rows are the same configuration only with the same number of memory (NUMA) domains +spanned by the allocation, threading policy, case revision, stepper, resolved integrator, tolerances and noise model: a changed default is a new configuration, shown as such rather than averaged in. Each run is compared with the first recorded and with the previous one, with the number of shared configurations. @@ -153,7 +154,8 @@ SELECTION: [--case NAME]... [--set SET]... [--all] [--stepper S] [--threads W] - `--append-history` adds the run to `benchmarks/history/.csv`. It records only the whole default plan with at least three repeats, on the native backend, every configuration complete and consistent (§4), timing the checkout's own `src/` with a - native build of the same commit. It refuses uncommitted changes under `src/`, + native build from a commit after which `native/` (but its documentation and tests) + has not changed. It refuses uncommitted changes under `src/`, `native/` or `benchmarks/` unless `--allow-dirty`, whose rows are marked dirty. - `history` prints, per width, one line per recorded run: date, version, the number of configurations and of busy ones, and the change against the first and the previous @@ -202,7 +204,9 @@ selection. The **hardware slug** is the CPU model reduced to lower-case words, followed by the production width in cores and, on a cloud instance, its family, for example `apple-m5-max-18c`, -`amd-epyc-7763-16c` or `arm-neoverse-v2-16c-aws-m8g`. It names no host. The model comes from +`amd-epyc-7763-16c` or `arm-neoverse-v2-16c-aws-m8g`. It names no host. The memory +domains the allocation spans are a column beside it: sixteen cores packed in one EPYC +domain and sixteen spread over eight share a name but not memory bandwidth. The model comes from `/proc/cpuinfo` on x86 Linux, `lscpu` on Arm Linux (whose `/proc/cpuinfo` names no model), `sysctl` on macOS and the registry on Windows. @@ -352,4 +356,5 @@ is final. committed code with a matching native build is recorded, older history files gain new columns, the slug counts the production width and names a cloud family, and the busy flag judges by CPU use where swap is not reported. Its open questions went to - the maintainer. + the maintainer, who added the memory domains spanned to what rows compare by and + asked that a native rebuild be required only when `native/` changed. diff --git a/tests/test_benchmark_track.py b/tests/test_benchmark_track.py index 80f25a3f..e5e0edd1 100644 --- a/tests/test_benchmark_track.py +++ b/tests/test_benchmark_track.py @@ -122,6 +122,7 @@ def test_history_rows_append_read_and_summarise(tmp_path): "instance_type": None, "system": "Linux", "machine": "x86_64", + "numa_domains": 1, } provenance = { "packages": {"solprop": "1.0"}, @@ -340,3 +341,16 @@ def test_a_native_tracking_run_records_its_build(tmp_path, monkeypatch): assert results["provenance"]["build_info"]["package_version"] assert results["rows"][0]["native_threads"] == 1 assert (out / "raw" / "smoke.default.t1" / "environment.json").exists() + + +def test_memory_domains_count_from_the_topology_and_separate_series(): + assert system.numa_domains(None) == 1 + rows = [{"numa_nodes": [0]}, {"numa_nodes": [0]}, {"numa_nodes": [3]}] + assert system.numa_domains(rows) == 2 + row: dict[str, object] = dict.fromkeys(history.COLUMNS, "1") + assert history.comparable(row) != history.comparable({**row, "numa_domains": "8"}) + + +def test_a_native_build_counts_as_current_unless_native_changed(): + assert not system.native_changed(runner.ROOT, "HEAD", "HEAD") + assert system.native_changed(runner.ROOT, "0" * 40, "HEAD") From 2332187d9b65982bd86c20ca1efd1b022280b9d5 Mon Sep 17 00:00:00 2001 From: John Travers Date: Tue, 27 Oct 2026 10:29:34 +0000 Subject: [PATCH 10/13] Benchmarks: a --device mode that tracks a native GPU build Each case runs once on the device with the production width of host threads; the history names the hardware by its GPU, records the device memory the run holds and keeps device rows apart from CPU rows. Co-Authored-By: Claude Opus 5.5 --- benchmarks/ab.py | 1 + benchmarks/history.py | 2 ++ benchmarks/registry.py | 9 +++++- benchmarks/runner.py | 20 ++++++++---- benchmarks/system.py | 35 +++++++++++++++++--- benchmarks/telemetry.py | 32 +++++++++++++++++++ benchmarks/track.py | 29 +++++++++++++---- benchmarks/worker.py | 45 +++++++++++++++++++++++--- tests/test_benchmark_track.py | 60 +++++++++++++++++++++++++++++++++-- 9 files changed, 207 insertions(+), 26 deletions(-) diff --git a/benchmarks/ab.py b/benchmarks/ab.py index 883fe99f..de1e3f29 100644 --- a/benchmarks/ab.py +++ b/benchmarks/ab.py @@ -187,6 +187,7 @@ def ab(args: argparse.Namespace) -> int: args.production or physical_cores(), backend=args.backend, fraction=args.length, + device=args.device, ) for reason in skipped: print(f"skipped {reason}") diff --git a/benchmarks/history.py b/benchmarks/history.py index a0d5a641..4e690a90 100644 --- a/benchmarks/history.py +++ b/benchmarks/history.py @@ -33,6 +33,7 @@ "native_build", "slug", "cpu_model", + "gpu_model", "instance_type", "system", "numa_domains", @@ -54,6 +55,7 @@ "rejected", "source_evaluations", "peak_rss_gib", + "device_gib", "construction_s", "busy", ) diff --git a/benchmarks/registry.py b/benchmarks/registry.py index 9a9862b4..03cf9af2 100644 --- a/benchmarks/registry.py +++ b/benchmarks/registry.py @@ -118,8 +118,14 @@ def configuration( backend: str = "native", fraction: float = 1.0, warmup: float = 0.02, + *, + device: bool = False, ) -> Record: - """Resolve one measurement without constructing a scientific object.""" + """Resolve one measurement without constructing a scientific object. + + On a device build, `width` is the host's thread count and `device` says the + native backend must run on the accelerator. + """ if stepper not in self.steppers: raise SuiteError(f"{self.name} admits only {', '.join(self.steppers)}") if not 0 < fraction <= 1 or not 0 <= warmup < 1: @@ -131,6 +137,7 @@ def configuration( "stepper": stepper, "threads": width, "backend": backend, + "device": device, "stop": self.length * fraction, "fraction": fraction, "warmup": warmup, diff --git a/benchmarks/runner.py b/benchmarks/runner.py index 4722e13c..93688cad 100644 --- a/benchmarks/runner.py +++ b/benchmarks/runner.py @@ -57,7 +57,9 @@ def freeze_harness(directory: Path) -> Record: return files -def plan_rows( +# The suppression below is for PLR0913: the keyword options are the shared +# command-line selection, passed through unchanged. +def plan_rows( # noqa: PLR0913 cases: tuple[Case, ...], steppers: tuple[str, ...], threads: str | None, @@ -65,16 +67,18 @@ def plan_rows( *, backend: str = "native", fraction: float = 1.0, + device: bool = False, ) -> tuple[list[Record], list[str]]: """Resolve configurations in run order; explain every one left out. Without `threads`, each case runs at its declared widths; with it, every case - runs at the widths it names. + runs at the widths it names. On a `device`, each case runs once, with the + production width of host threads, and its CPU cost does not apply. """ rows: list[Record] = [] skipped: list[str] = [] for case in cases: - declared = ",".join(case.widths) + declared = "production" if device else ",".join(case.widths) for width in widths(threads or declared, production): for stepper in steppers: if stepper not in case.steppers: @@ -85,11 +89,12 @@ def plan_rows( continue rows.append( { - "id": f"{case.name}.{stepper}.t{width}", + "id": f"{case.name}.{stepper}." + + ("device" if device else f"t{width}"), "configuration": case.configuration( - stepper, width, backend, fraction + stepper, width, backend, fraction, device=device ), - "cost": case.cost[0 if width == 1 else 1], + "cost": None if device else case.cost[0 if width == 1 else 1], } ) rows.sort(key=lambda row: row["configuration"]["threads"]) @@ -111,7 +116,8 @@ def show_plan(rows: list[Record], skipped: list[str], runs: int) -> None: values = row["configuration"] cost = "?" if row["cost"] is None else f"{row['cost']:.0f}" print( - f"{values['case']:26} {values['stepper']:22} {values['threads']:>7} " + f"{values['case']:26} {values['stepper']:22} " + f"{'device' if values.get('device') else values['threads']:>7} " f"{cost:>10}" ) for reason in skipped: diff --git a/benchmarks/system.py b/benchmarks/system.py index 45cb1809..502968a4 100644 --- a/benchmarks/system.py +++ b/benchmarks/system.py @@ -20,12 +20,18 @@ import psutil from benchmarks.records import Record, SuiteError, identity -from benchmarks.telemetry import allocation_details, hardware_details, linux_topology +from benchmarks.telemetry import ( + allocation_details, + gpu_details, + hardware_details, + linux_topology, +) __all__ = [ "POOL_VARIABLES", "THREADING", "counters", + "device_slug", "eligible_cpus", "git_receipt", "guard", @@ -148,18 +154,30 @@ def _instance_type() -> str | None: return None -def hardware(production: int, instance_type: str | None = None) -> Record: - """Describe the hardware without a host name, and derive its slug.""" +def hardware( + production: int, instance_type: str | None = None, *, device: bool = False +) -> Record: + """Describe the hardware without a host name, and derive its slug. + + A `device` run is named by its GPU, which then sets the pace; the host CPU is + recorded beside it. + """ details = hardware_details(eligible_cpus()) model = details["cpu_model"] or platform.processor() or platform.machine() cores = physical_cores() instance = instance_type or _instance_type() + gpu = gpu_details() if device else {"gpu_model": None} + if device and not isinstance(gpu["gpu_model"], str): + raise SuiteError(f"cannot name the GPU: {gpu['gpu_model_reason']}") return { "numa_domains": numa_domains(details["cpu_topology"]), # The cores a run uses name the hardware set: on a cluster the CPU mask can # show the whole node while the allocation is a part of it. - "slug": slug(model, production, instance), + "slug": device_slug(gpu["gpu_model"], instance) + if isinstance(gpu["gpu_model"], str) + else slug(model, production, instance), "cpu_model": model, + "gpu_model": gpu["gpu_model"], "physical_cores": cores, "logical_cpus": len(eligible_cpus()), "production": production, @@ -224,6 +242,15 @@ def slug(model: str, cores: int, instance: str | None = None) -> str: return f"{words.strip('-') or 'unknown-cpu'}-{cores}c" +def device_slug(model: str, instance: str | None = None) -> str: + """Name a GPU hardware set by its model (``nvidia-a40``), plus any cloud family.""" + words = re.sub(r"[^a-z0-9]+", "-", model.lower()).strip("-") or "unknown-gpu" + if instance: + family = re.sub(r"[^a-z0-9]+", "-", instance.lower().split(".")[0]).strip("-") + return f"{words}-{family}" + return words + + def thread_environment( width: int, incoming: dict[str, str], threading: str = "passive" ) -> dict[str, str]: diff --git a/benchmarks/telemetry.py b/benchmarks/telemetry.py index 84eeb091..fb71b537 100644 --- a/benchmarks/telemetry.py +++ b/benchmarks/telemetry.py @@ -17,6 +17,7 @@ __all__ = [ "allocation_details", "cpu_list", + "gpu_model", "hardware_details", "linux_topology", "lscpu_model", @@ -175,3 +176,34 @@ def allocation_details() -> Record: } result["exclusivity"] = "unverified; requires the scheduler allocation record" return result + + +def gpu_model(nvidia: str | None, amd: str | None) -> str | None: + """Name the first visible GPU from ``nvidia-smi`` or ``rocm-smi`` output. + + `nvidia` is ``nvidia-smi --query-gpu=name --format=csv,noheader``; `amd` is + ``rocm-smi --showproductname --csv``, whose ``Card Series`` column names the part. + """ + for line in (nvidia or "").splitlines(): + if line.strip(): + return line.strip() + lines = [line for line in (amd or "").splitlines() if line.strip()] + if len(lines) > 1: + header = [cell.strip() for cell in lines[0].split(",")] + if "Card Series" in header: + value = lines[1].split(",")[header.index("Card Series")].strip() + if value: + return value if value.startswith("AMD") else f"AMD {value}" + return None + + +def gpu_details() -> Record: + """Record the first visible GPU's model, or why it is unknown.""" + model = gpu_model( + _command(["nvidia-smi", "--query-gpu=name", "--format=csv,noheader"]), + _command(["rocm-smi", "--showproductname", "--csv"]), + ) + return { + "gpu_model": model, + "gpu_model_reason": None if model else "no nvidia-smi or rocm-smi answer", + } diff --git a/benchmarks/track.py b/benchmarks/track.py index 0c65e0bf..6656efe2 100644 --- a/benchmarks/track.py +++ b/benchmarks/track.py @@ -74,6 +74,11 @@ def selection_options(parser: argparse.ArgumentParser) -> None: parser.add_argument( "--backend", choices=("numpy_ref", "numba_cpu", "native"), default="native" ) + parser.add_argument( + "--device", + action="store_true", + help="run each case once on the GPU of a native device build", + ) parser.add_argument("--repeats", type=int, default=3, help="timed runs per case") parser.add_argument( "--threading", @@ -120,14 +125,16 @@ def summarise( A row is consistent when its repeats took the same steps and reached the same final field to rounding, and the native backend ran the threads it was given. + A device row's width is ``device``: its host threads do not set its pace. """ values = row["configuration"] + device = bool(values.get("device")) base = { "id": row["id"], "case": values["case"], "revision": values["revision"], "stepper": values["stepper"], - "threads": values["threads"], + "threads": "device" if device else values["threads"], "threading": threading, "status": result["status"], "busy": busy, @@ -152,9 +159,12 @@ def summarise( "native_threads": native_threads, "consistent": same_steps == 1 and result["repeat_difference"] <= REPEAT_TOLERANCE - and native_threads in (None, values["threads"]), + and (device or native_threads in (None, values["threads"])), "source_evaluations": last["source_evaluations"], "peak_rss_gib": result["peak_rss_bytes"] / 2**30, + "device_gib": None + if result.get("device_bytes") is None + else result["device_bytes"] / 2**30, "construction_s": result["construction_seconds"] + result["backend_seconds"], "warmup_s": result["warmup_seconds"], "integrator": result["integrator"], @@ -175,14 +185,14 @@ def show_results(rows: list[Record]) -> None: for row in rows: if row["status"] != "ok": print( - f"{row['case']:26} {'':28} {row['threads']:>3} failed: {row['error']}" + f"{row['case']:26} {'':28} {row['threads']!s:>3} failed: {row['error']}" ) continue spread = 100 * (row["max_s"] - row["min_s"]) / row["median_s"] print( - f"{row['case']:26} {row['integrator']:28} {row['threads']:>3} " + f"{row['case']:26} {row['integrator']:28} {row['threads']!s:>3} " f"{row['median_s']:9.2f} {spread:8.1f} {row['accepted']:7d} " - f"{row['peak_rss_gib']:6.2f}" + f"{row['device_gib'] or row['peak_rss_gib']:6.2f}" + ("" if row["consistent"] else " (repeats or threads inconsistent)") + (" (busy)" if row["busy"] else "") ) @@ -210,6 +220,7 @@ def history_rows( or None, "slug": machine["slug"], "cpu_model": machine["cpu_model"], + "gpu_model": machine.get("gpu_model"), "instance_type": machine["instance_type"], "system": f"{machine['system']} {machine['machine']}", "numa_domains": machine["numa_domains"], @@ -231,6 +242,9 @@ def history_rows( "rejected": row["rejected"], "source_evaluations": row["source_evaluations"], "peak_rss_gib": f"{row['peak_rss_gib']:.3f}", + "device_gib": None + if row.get("device_gib") is None + else f"{row['device_gib']:.3f}", "construction_s": f"{row['construction_s']:.2f}", "busy": int(row["busy"]), } @@ -251,7 +265,7 @@ def refuse_partial( "uncommitted changes: commit them, or pass --allow-dirty to record " "this run anyway (its rows are marked dirty)" ) - default, _ = plan_rows(select(), ("default",), None, production) + default, _ = plan_rows(select(), ("default",), None, production, device=args.device) if [row["id"] for row in rows] != [row["id"] for row in default]: raise SuiteError("the history records the whole default plan only") if args.repeats < 3: @@ -305,13 +319,14 @@ def track(args: argparse.Namespace) -> int: args.threads, production, backend=args.backend, + device=args.device, ) if not rows: raise SuiteError("no configuration in the selection") show_plan(rows, skipped, args.repeats) if args.dry_run: return 0 - machine = hardware(production, args.instance_type) + machine = hardware(production, args.instance_type, device=args.device) source = git_receipt(ROOT) if args.append_history: refuse_partial(args, rows, production, source) diff --git a/benchmarks/worker.py b/benchmarks/worker.py index 812491f9..cc5aafc6 100644 --- a/benchmarks/worker.py +++ b/benchmarks/worker.py @@ -37,6 +37,7 @@ ) __all__ = [ + "HOST_SPACES", "Diagnostics", "emit", "integrator_label", @@ -61,9 +62,19 @@ def emit(kind: str, **values: object) -> None: print(canonical({"event": kind, **values}), flush=True) -def runtime(backend: str) -> Record: - """Import in the required order and identify the loaded packages and pools.""" +HOST_SPACES = ("OpenMP", "Threads", "Serial") +"""Kokkos execution spaces of a native CPU build; any other is an accelerator's.""" + + +def runtime(backend: str, device: bool = False) -> Record: + """Import in the required order and identify the loaded packages and pools. + + A `device` run needs a native accelerator build with a visible device; any + other run needs a CPU build. + """ info: Record = {} + if device and backend != "native": + raise SuiteError("only the native backend runs on a device") if backend == "native": # solprop_native must be imported before pyFFTW. import solprop_native # noqa: PLC0415 # pyright: ignore[reportMissingImports] @@ -71,8 +82,11 @@ def runtime(backend: str) -> Record: build = dataclasses.asdict(solprop_native.build_info()) if build.get("coverage_instrumented", False): raise SuiteError("an instrumented native build cannot be timed") - if build.get("execution_space") not in ("OpenMP", "Threads", "Serial"): - raise SuiteError("the benchmarks time native CPU builds only") + on_host = build.get("execution_space") in HOST_SPACES + if device and (on_host or not build.get("device_count")): + raise SuiteError("a device run needs a native GPU build and a visible GPU") + if not device and not on_host: + raise SuiteError("a native GPU build times device runs only (--device)") extension = Path(importlib.import_module("solprop_native._core").__file__ or "") info.update(build_info=build, extension_hash=file_hash(extension)) from threadpoolctl import threadpool_info # noqa: PLC0415 @@ -208,6 +222,15 @@ def timed_run( return record, output.plane_field(output.n_saved - 1, "fft") +def _device_free(configuration: Record) -> int | None: + """Read the accelerator's free bytes on a device run, else ``None``.""" + if not configuration.get("device"): + return None + import solprop_native # noqa: PLC0415 # pyright: ignore[reportMissingImports] + + return solprop_native.available_device_memory() + + def measure(configuration: Record, repeats: int, fields: Path | None) -> Record: """Build once, warm up once, then time ``repeats`` complete runs. @@ -224,6 +247,7 @@ def measure(configuration: Record, repeats: int, fields: Path | None) -> Record: from solprop.driver.run import field_retention # noqa: PLC0415 from solprop.output.schema import serialise # noqa: PLC0415 + free_before = _device_free(configuration) started = time.perf_counter() simulation = build(configuration) problem = simulation.translate() @@ -250,6 +274,7 @@ def measure(configuration: Record, repeats: int, fields: Path | None) -> Record: engine.observe(field=True) engine.stats(problem.stats_spec) warmup = time.perf_counter() - started + free_warm = _device_free(configuration) repetitions = [] first = final = None difference = 0.0 @@ -276,6 +301,11 @@ def measure(configuration: Record, repeats: int, fields: Path | None) -> Record: "warmup_seconds": warmup, "repetitions": repetitions, "peak_rss_bytes": peak_rss(), + # The run's workspace stays allocated, so the drop in free memory from + # before construction to after the warm-up is what the run holds. + "device_bytes": None + if free_before is None or free_warm is None + else free_before - free_warm, "finite": bool(np.all(np.isfinite(final))), "repeat_difference": difference, "integrator": integrator_label(problem.stepper.integrator), @@ -303,7 +333,12 @@ def main() -> int: request = read_record(Path(sys.argv[1])) configuration = request["configuration"] with diagnostics.capture(): - emit("provenance", provenance=runtime(configuration["backend"])) + emit( + "provenance", + provenance=runtime( + configuration["backend"], bool(configuration.get("device")) + ), + ) diagnostics.phase = "measurement" fields = request.get("fields") result = measure( diff --git a/tests/test_benchmark_track.py b/tests/test_benchmark_track.py index e5e0edd1..cdb9fff0 100644 --- a/tests/test_benchmark_track.py +++ b/tests/test_benchmark_track.py @@ -12,7 +12,7 @@ import pytest from benchmarks import __main__ as cli -from benchmarks import history, runner, system, track +from benchmarks import history, runner, system, telemetry, track from benchmarks.records import SuiteError, read_record from benchmarks.registry import CASES, select @@ -176,7 +176,12 @@ def test_older_history_files_gain_new_columns_and_newer_ones_are_refused(tmp_pat def namespace(**values): - defaults = {"backend": "native", "allow_dirty": False, "repeats": 3} + defaults = { + "backend": "native", + "allow_dirty": False, + "repeats": 3, + "device": False, + } return argparse.Namespace(**(defaults | values)) @@ -354,3 +359,54 @@ def test_memory_domains_count_from_the_topology_and_separate_series(): def test_a_native_build_counts_as_current_unless_native_changed(): assert not system.native_changed(runner.ROOT, "HEAD", "HEAD") assert system.native_changed(runner.ROOT, "0" * 40, "HEAD") + + +@pytest.mark.usefixtures("sixteen_cpus") +def test_a_device_plan_runs_each_case_once_and_its_rows_stand_apart(): + rows, _ = runner.plan_rows(select(), ("default",), None, 8, device=True) + assert len(rows) == len(select()) + assert all(row["id"].endswith(".device") for row in rows) + assert {row["configuration"]["threads"] for row in rows} == {8} + assert all(row["configuration"]["device"] for row in rows) + assert runner.estimate(rows, 3) is None + clean = {"revision": "a" * 40, "dirty": False} + track.refuse_partial(namespace(device=True), rows, 8, clean) + with pytest.raises(SuiteError, match="whole default plan"): + track.refuse_partial(namespace(), rows, 8, clean) + receipt = { + "status": "ok", + "repetitions": [{"seconds": 1.0, "accepted": 5, "rejected": 0, + "source_evaluations": 6, "z": 1.0}], + "repeat_difference": 0.0, "peak_rss_bytes": 1, "device_bytes": 2**31, + "construction_seconds": 1.0, "backend_seconds": 0.0, "warmup_seconds": 0.0, + "integrator": "RungeKutta", "rtol": 1e-6, "atol": 0.0, "noise": None, + "finite": True, "provenance": {"build_info": {"runtime_threads": 1}}, + } # fmt: skip + # The host threads of a device build do not set its pace, so are not checked. + summary = track.summarise(rows[0], receipt, busy=False) + assert summary["consistent"] + assert summary["threads"] == "device" + assert summary["device_gib"] == 2.0 + + +@pytest.mark.parametrize( + ("nvidia", "amd", "model"), + [ + ("NVIDIA A40\nNVIDIA A40\n", None, "NVIDIA A40"), + (None, "device,Card Series,Card Model\ncard0,MI325X,0x74a5\n", "AMD MI325X"), + ( + None, + "device,Card Series\ncard0,AMD Instinct MI300X\n", + "AMD Instinct MI300X", + ), + (None, None, None), + ("", "device,Card Model\ncard0,0x74a5\n", None), + ], +) +def test_the_gpu_is_named_from_the_vendor_query(nvidia, amd, model): + assert telemetry.gpu_model(nvidia, amd) == model + + +def test_a_device_slug_names_the_gpu_and_any_cloud_family(): + assert system.device_slug("NVIDIA H200") == "nvidia-h200" + assert system.device_slug("NVIDIA A40", "aws g5.xlarge") == "nvidia-a40-aws-g5" From 479ee731198d2ee96e8b3f205e816309006bc9d4 Mon Sep 17 00:00:00 2001 From: John Travers Date: Tue, 27 Oct 2026 10:31:08 +0000 Subject: [PATCH 11/13] SEP 38: document device mode Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 ++- docs/dev/benchmarking.md | 16 +++++++++++++ .../dev/seps/sep-0038-performance-tracking.md | 24 +++++++++++++++---- 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40e92940..e9d4b30d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -315,7 +315,8 @@ The first public release. It contains: `python -m benchmarks track` times complete runs of about a dozen representative workloads, each with its interface's default integrator and tolerances, at one thread and the production width in about an hour per machine, and records them in - `benchmarks/history/` by hardware. It replaces SEP 30's standard suite, whose + `benchmarks/history/` by hardware; `--device` runs the same cases on a native GPU + build. It replaces SEP 30's standard suite, whose reference qualification, native coverage and export are removed; `ab` compares two builds on the same worker. - **Documentation.** An introduction with executable lessons, a user guide, a diff --git a/docs/dev/benchmarking.md b/docs/dev/benchmarking.md index f0e0ae1d..badb22ab 100644 --- a/docs/dev/benchmarking.md +++ b/docs/dev/benchmarking.md @@ -93,6 +93,22 @@ spin before sleeping), `active` (idle threads spin), `bound` (threads pinned to neighbouring cores) or `bound-active`. The policy is recorded with each row, and rows compare only within one policy. +## On a GPU + +With a native GPU build installed (CUDA or HIP; see the native backend's build guide), +`--device` runs every selected case once on the device: + +```sh +uv run python -m benchmarks track --device +``` + +The host still runs the production width of threads, but each row's width is `device`. +The hardware name is the GPU model, for example `nvidia-a40` or `nvidia-h200`, plus the +instance family on a cloud machine, and the host CPU model is recorded beside it. Each +row also records the device memory the run holds. The worker refuses a GPU build +without `--device` and a CPU build with it. The case costs are CPU costs, so a device +plan gives no time estimate. + ## Recording a run ```sh diff --git a/docs/dev/seps/sep-0038-performance-tracking.md b/docs/dev/seps/sep-0038-performance-tracking.md index 9cbb4e2c..c8eb6938 100644 --- a/docs/dev/seps/sep-0038-performance-tracking.md +++ b/docs/dev/seps/sep-0038-performance-tracking.md @@ -62,7 +62,8 @@ them. - accuracy qualification. SEP 37 measured the defaults' accuracy; tracking times the default, it does not certify it. A numerical regression is caught by the test suite; -- GPU tracking (a later SEP, once the GPU builds are released); +- GPU qualification and tuning. `--device` (§5.1) times a native GPU build on the same + cases, but which GPUs get a standing history file is the maintainer's choice; - statistical adoption verdicts. The history shows changes; the [benchmark rules](../benchmarks.md) still say how a performance claim is judged; - continuous integration. Shared CI runners are too noisy for timing. @@ -134,7 +135,8 @@ python -m benchmarks ab --baseline-python PY [--candidate-python PY] [SELEC python -m benchmarks history [SLUG] [--threads W] SELECTION: [--case NAME]... [--set SET]... [--all] [--stepper S] [--threads W] - [--production N] [--backend B] [--repeats 3] [--threading POLICY] + [--production N] [--backend B] [--device] [--repeats 3] + [--threading POLICY] ``` - `list` prints the plan and its estimated time; it imports no scientific library. @@ -151,6 +153,13 @@ SELECTION: [--case NAME]... [--set SET]... [--all] [--stepper S] [--threads W] `active` (idle threads spin), `bound` (pinned to neighbouring cores) or `bound-active`. The policy is part of what history rows compare by; the default is set by phase 2's measurement. +- `--device` runs each case once on the GPU of a native CUDA or HIP build, with the + production width of host threads. The worker refuses a device build without + `--device` and a CPU build with it. A device row's width is `device`, the slug is + the GPU model (for example `nvidia-a40`, plus any cloud family), and the row records + the device memory the run holds: the drop in the device's free memory from before + construction to after the warm-up. The CPU costs of §5.2 do not apply, so a device + plan has no time estimate. - `--append-history` adds the run to `benchmarks/history/.csv`. It records only the whole default plan with at least three repeats, on the native backend, every configuration complete and consistent (§4), timing the checkout's own `src/` with a @@ -212,14 +221,16 @@ model), `sysctl` on macOS and the registry on Windows. The standing history files are an Apple M5 Max, a 16-core allocation of an AMD EPYC 7763 cluster node and a cloud Arm instance (Neoverse-V2, for example AWS Graviton4), -with Windows x86-64 measured occasionally. GPU tracking is left to a later SEP. A cloud instance type, from `--instance-type` or +with Windows x86-64 measured occasionally; a GPU file can be added with `--device` +(§5.1). A cloud instance type, from `--instance-type` or the DMI product name on Linux, is recorded beside it but is not part of the slug. Each history row is one (run, case, stepper, width): date, solprop version, commit, -native build identity (version, profile, compiler), slug, CPU model, instance type, OS, +native build identity (version, profile, compiler), slug, CPU model, GPU model (device +rows), instance type, OS, width, case, case revision, requested stepper, resolved integrator and tolerances, noise model, median, minimum and maximum seconds, repeats, accepted and rejected steps, source -evaluations, peak resident memory, and whether the machine was busy when a worker +evaluations, peak resident memory, device memory (device rows), and whether the machine was busy when a worker started. That is a few kilobytes per release per machine. The full bundle of a run (plan, worker requests and logs, results, environment) stays in @@ -358,3 +369,6 @@ is final. busy flag judges by CPU use where swap is not reported. Its open questions went to the maintainer, who added the memory domains spanned to what rows compare by and asked that a native rebuild be required only when `native/` changed. +- 2026-10-27: At the maintainer's request, device mode (`--device`) was added before + the first release instead of in a later SEP, so that a GPU history line can start + with the first tagged release. From 924e91ce1239a6de5aae0059b4d6c2a6fe20db43 Mon Sep 17 00:00:00 2001 From: John Travers Date: Tue, 27 Oct 2026 12:19:33 +0000 Subject: [PATCH 12/13] Benchmarks: a passive-bound threading policy Co-Authored-By: Claude Opus 5.5 --- benchmarks/system.py | 8 ++++++++ docs/dev/benchmarking.md | 3 ++- docs/dev/seps/sep-0038-performance-tracking.md | 3 ++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/benchmarks/system.py b/benchmarks/system.py index 502968a4..4b021978 100644 --- a/benchmarks/system.py +++ b/benchmarks/system.py @@ -81,6 +81,14 @@ "KMP_BLOCKTIME": "0", "GOMP_SPINCOUNT": "0", }, + # Pinned to neighbouring cores, and idle threads sleep at once. + "passive-bound": { + "OMP_PROC_BIND": "close", + "OMP_PLACES": "cores", + "OMP_WAIT_POLICY": "PASSIVE", + "KMP_BLOCKTIME": "0", + "GOMP_SPINCOUNT": "0", + }, # Unpinned, with each OpenMP runtime's own waiting (a short spin, then sleep). "default": {"OMP_PROC_BIND": "false"}, # Unpinned, and idle threads spin between parallel regions. diff --git a/docs/dev/benchmarking.md b/docs/dev/benchmarking.md index 0e650e6c..0ea7e0b1 100644 --- a/docs/dev/benchmarking.md +++ b/docs/dev/benchmarking.md @@ -87,7 +87,8 @@ physical cores in the process's CPU allocation unless `--production` gives it; o cluster, give it explicitly. **Threading.** `--threading` sets OpenMP's placement and waiting in every worker: -`passive` (unpinned, idle threads sleep at once), `default` (the runtime's own short +`passive` (unpinned, idle threads sleep at once), `passive-bound` (the same, pinned +to neighbouring cores), `default` (the runtime's own short spin before sleeping), `active` (idle threads spin), `bound` (threads pinned to neighbouring cores) or `bound-active`. The policy is recorded with each row, and rows compare only within one policy. diff --git a/docs/dev/seps/sep-0038-performance-tracking.md b/docs/dev/seps/sep-0038-performance-tracking.md index 5307076a..e5a8b485 100644 --- a/docs/dev/seps/sep-0038-performance-tracking.md +++ b/docs/dev/seps/sep-0038-performance-tracking.md @@ -149,7 +149,8 @@ SELECTION: [--case NAME]... [--set SET]... [--all] [--stepper S] [--threads W] - `--production` gives the production width; otherwise it is the number of physical cores in the process's CPU allocation. - `--threading` sets OpenMP's placement and waiting for every worker: `passive` - (unpinned; idle threads sleep at once), `default` (the runtime's own waiting), + (unpinned; idle threads sleep at once), `passive-bound` (pinned, sleeping at + once), `default` (the runtime's own waiting), `active` (idle threads spin), `bound` (pinned to neighbouring cores) or `bound-active`. The policy is part of what history rows compare by; the default is set by phase 2's measurement. From 89faeb1f18bcbe6a0354fbe65c0798661bb3eb33 Mon Sep 17 00:00:00 2001 From: John Travers Date: Tue, 27 Oct 2026 14:24:20 +0000 Subject: [PATCH 13/13] SEP 38 Final: archive, indexes, roadmap and closing record; SEP 30 superseded The default and bound threading policies spell out the runtimes' own waiting, so the native backend's passive default (#33) cannot change what they measure. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- benchmarks/README.md | 2 +- benchmarks/system.py | 10 +++- docs/dev/benchmarking.md | 2 +- docs/dev/benchmarks.md | 2 +- docs/dev/roadmap.md | 12 ++-- docs/dev/seps/archive/index.md | 1 + .../sep-0030-standard-benchmark-suite.md | 7 ++- .../sep-0038-performance-tracking.md | 59 ++++++++++++++----- docs/dev/seps/index.md | 5 +- tests/test_benchmark_track.py | 5 ++ 11 files changed, 72 insertions(+), 35 deletions(-) rename docs/dev/seps/{ => archive}/sep-0038-performance-tracking.md (86%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23ed49d4..db073ff5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -315,7 +315,7 @@ The first public release. It contains: (`solprop.driver.summary`; a reporter receives it through the optional `SummarisingReporter.summary` call). - **Performance tracking** for developers - ([SEP 38](https://solprop.readthedocs.io/en/latest/dev/seps/sep-0038-performance-tracking.html)): + ([SEP 38](https://solprop.readthedocs.io/en/latest/dev/seps/archive/sep-0038-performance-tracking.html)): `python -m benchmarks track` times complete runs of about a dozen representative workloads, each with its interface's default integrator and tolerances, at one thread and the production width in about an hour per machine, and records them in diff --git a/benchmarks/README.md b/benchmarks/README.md index 114cbac4..dcf94ae2 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,7 +1,7 @@ # Benchmarks Release performance tracking -([SEP 38](../docs/dev/seps/sep-0038-performance-tracking.md)); its manual is +([SEP 38](../docs/dev/seps/archive/sep-0038-performance-tracking.md)); its manual is `docs/dev/benchmarking.md`. Run it on a quiet machine: ```bash diff --git a/benchmarks/system.py b/benchmarks/system.py index 4b021978..8dbb0d21 100644 --- a/benchmarks/system.py +++ b/benchmarks/system.py @@ -73,6 +73,9 @@ "KOKKOS_TOOLS_LIBS", ) +_RUNTIME_WAITING = {"GOMP_SPINCOUNT": "300000", "KMP_BLOCKTIME": "200"} +"""GNU's and LLVM's own spin before an idle thread sleeps: 300,000 spins, 200 ms.""" + THREADING: dict[str, dict[str, str]] = { # Unpinned, and idle threads sleep at once: robust on a shared machine. "passive": { @@ -89,12 +92,13 @@ "KMP_BLOCKTIME": "0", "GOMP_SPINCOUNT": "0", }, - # Unpinned, with each OpenMP runtime's own waiting (a short spin, then sleep). - "default": {"OMP_PROC_BIND": "false"}, + # Unpinned, with each OpenMP runtime's own waiting (a short spin, then sleep), + # spelled out: the native backend makes idle threads sleep when nothing is set. + "default": {"OMP_PROC_BIND": "false", **_RUNTIME_WAITING}, # Unpinned, and idle threads spin between parallel regions. "active": {"OMP_PROC_BIND": "false", "OMP_WAIT_POLICY": "ACTIVE"}, # Threads pinned to neighbouring cores, with the runtime's own waiting. - "bound": {"OMP_PROC_BIND": "close", "OMP_PLACES": "cores"}, + "bound": {"OMP_PROC_BIND": "close", "OMP_PLACES": "cores", **_RUNTIME_WAITING}, # Pinned, and spinning. "bound-active": { "OMP_PROC_BIND": "close", diff --git a/docs/dev/benchmarking.md b/docs/dev/benchmarking.md index 0ea7e0b1..0417764b 100644 --- a/docs/dev/benchmarking.md +++ b/docs/dev/benchmarking.md @@ -3,7 +3,7 @@ `benchmarks/` times **complete runs** of a small set of representative workloads. Each release adds a few lines per case to a history file kept per hardware set, so that regressions and improvements show from one release to the next. The design is -[SEP 38](seps/sep-0038-performance-tracking.md), which replaced SEP 30's standard +[SEP 38](seps/archive/sep-0038-performance-tracking.md), which replaced SEP 30's standard suite. The rules for a performance claim and the indicative results are in [Benchmarks](benchmarks.md); this page is the manual. diff --git a/docs/dev/benchmarks.md b/docs/dev/benchmarks.md index 9cb985f6..b237c516 100644 --- a/docs/dev/benchmarks.md +++ b/docs/dev/benchmarks.md @@ -9,7 +9,7 @@ relying on any of them. ## Where things are - **The benchmarks** are `benchmarks/` - ([SEP 38](seps/sep-0038-performance-tracking.md)), run with `uv run poe bench` + ([SEP 38](seps/archive/sep-0038-performance-tracking.md)), run with `uv run poe bench` (`python -m benchmarks`). `track` times complete runs of the tracking set (about a dozen representative workloads, with each interface's default integrator and tolerances) at one thread and the production width, in about an hour per machine; diff --git a/docs/dev/roadmap.md b/docs/dev/roadmap.md index e1a8f67d..6c37c79c 100644 --- a/docs/dev/roadmap.md +++ b/docs/dev/roadmap.md @@ -98,10 +98,6 @@ The [SEP index](seps/index.md) lists every proposal and its current status. subtraction, a weak-response limit and local temporal integration. Its ensemble, data and integration questions must be settled before acceptance; shared-level ladders, vector saturation and repeated-window histories remain future work. -- [SEP 38 — Performance tracking: complete runs, about an hour per machine](seps/sep-0038-performance-tracking.md): - replaces SEP 30's standard suite with a release-tracking command that times - complete runs with the library's defaults and keeps a small history per hardware - set; drafted with its implementation. The informal items below record work not fully covered by these proposals. When adding or revising a SEP, remove an informal item only if the proposal covers it in full; @@ -633,9 +629,11 @@ GPUs with slow double precision. 10 thousand lines for each existing GPU backend). **Benchmark platforms beyond CPU tracking.** -[SEP 38](seps/sep-0038-performance-tracking.md) tracks native CPU runs per hardware set. -GPU tracking, Windows execution and cloud launch recipes need their own work; keep -their results in distinct hardware records. +[SEP 38](seps/archive/sep-0038-performance-tracking.md) (final) tracks native CPU and +GPU runs per hardware set. The first history rows are recorded at the release commit on +an Apple M5 Max, a 16-core AMD EPYC 7763 allocation and a cloud Arm Neoverse-V2 +instance, with Windows x86-64 occasionally; a GPU file, cloud launch recipes and an +`mlx` backend in the tracking set remain to do. **Remaining benchmark references and estimates.** Complete independent reference checks for the 512-node radial filament, 128/256-square Cartesian filaments and diff --git a/docs/dev/seps/archive/index.md b/docs/dev/seps/archive/index.md index 8a940824..f0587a40 100644 --- a/docs/dev/seps/archive/index.md +++ b/docs/dev/seps/archive/index.md @@ -31,4 +31,5 @@ sep-0033-single-precision-mlx sep-0034-medium-flux sep-0035-carrier-frame-exponential-adams sep-0037-noise-and-stepper-defaults +sep-0038-performance-tracking ``` diff --git a/docs/dev/seps/archive/sep-0030-standard-benchmark-suite.md b/docs/dev/seps/archive/sep-0030-standard-benchmark-suite.md index c71e8eba..8dc630c4 100644 --- a/docs/dev/seps/archive/sep-0030-standard-benchmark-suite.md +++ b/docs/dev/seps/archive/sep-0030-standard-benchmark-suite.md @@ -6,7 +6,7 @@ | **Title** | Standard benchmark suite | | **Author** | John Travers | | **Drafting agent** | GPT-6 (2026-10-21) | -| **Status** | Final | +| **Status** | Superseded | | **Review status** | Reviewed | | **Reviewers** | John Travers (human); Opus 5.5 (AI) | | **Type** | Process | @@ -14,7 +14,7 @@ | **Resolution** | Accepted 2026-10-21 by John Travers after reading the revised proposal; maintainer decision recorded in Part III. | | **Requires** | [SEP 25](sep-0025-rounding-level-reproducibility.md) (numerical comparison policy); [SEP 28](sep-0028-free-space-evaluation.md) (bounded spatial workloads) | | **Supersedes** | — | -| **Superseded-by** | — | +| **Superseded-by** | [SEP 38](sep-0038-performance-tracking.md) | | **Implementation** | Phase 1 merged; phases 2–4 implemented on `feature/sep-0030-phase-2`, `feature/sep-0030-phase-3` and `feature/sep-0030-phase-4`; consolidated for merge on the phase 4 branches after closing validation | | **Manual pages** | [Running the benchmark suite](../../benchmarking.md) | @@ -1295,6 +1295,9 @@ branches are unnecessary. No main-branch merge or publication is claimed here. requested its merge to main. Status is Accepted and Part I is frozen. The qualification questions assigned to implementation phases in §12 remain there; acceptance does not claim measured timings or completed implementation. +- 2026-10-27: Superseded by [SEP 38](sep-0038-performance-tracking.md), which retired the + standard mode, reference qualification, native coverage and export for complete-run + tracking. ### AI review — 2026-10-21 diff --git a/docs/dev/seps/sep-0038-performance-tracking.md b/docs/dev/seps/archive/sep-0038-performance-tracking.md similarity index 86% rename from docs/dev/seps/sep-0038-performance-tracking.md rename to docs/dev/seps/archive/sep-0038-performance-tracking.md index e5a8b485..d7e6c809 100644 --- a/docs/dev/seps/sep-0038-performance-tracking.md +++ b/docs/dev/seps/archive/sep-0038-performance-tracking.md @@ -6,17 +6,17 @@ | **Title** | Performance tracking: complete runs, about an hour per machine | | **Author** | John Travers | | **Drafting agent** | Opus 5.5 (2026-10-27) | -| **Status** | Draft | -| **Review status** | Not reviewed | -| **Reviewers** | — | +| **Status** | Final | +| **Review status** | Reviewed | +| **Reviewers** | John Travers (human); Opus 5.5 (AI) | | **Type** | Process | | **Created** | 2026-10-27 | -| **Resolution** | — | -| **Requires** | [SEP 37](archive/sep-0037-noise-and-stepper-defaults.md) (the per-interface defaults the tracked cases run with) | -| **Supersedes** | [SEP 30](archive/sep-0030-standard-benchmark-suite.md) | +| **Resolution** | Accepted and Final 2026-10-27 by John Travers, after the independent AI review and his decisions on the Kerr beams, the standing hardware files, the memory domains, the native rebuild rule and device mode (Part III), with its implementation in one pull request ([LupoLab/solprop#32](https://github.com/LupoLab/solprop/pull/32)). | +| **Requires** | [SEP 37](sep-0037-noise-and-stepper-defaults.md) (the per-interface defaults the tracked cases run with) | +| **Supersedes** | [SEP 30](sep-0030-standard-benchmark-suite.md) | | **Superseded-by** | — | -| **Implementation** | Opus 5.5 (AI) for John Travers, branch `feature/sep-0038-performance-tracking`, stacked on SEP 37's branch: phase 1 implemented and calibrated 2026-10-27; the first history rows wait for the release | -| **Manual pages** | — | +| **Implementation** | Opus 5.5 (AI) for John Travers, branch `feature/sep-0038-performance-tracking` ([LupoLab/solprop#32](https://github.com/LupoLab/solprop/pull/32)): implemented and calibrated 2026-10-27, device mode run on an NVIDIA A40; the first history rows wait for the release | +| **Manual pages** | [Running the benchmarks](../../benchmarking.md); [Benchmarks](../../benchmarks.md) | ## Part I — Proposal @@ -65,13 +65,13 @@ them. - GPU qualification and tuning. `--device` (§5.1) times a native GPU build on the same cases, but which GPUs get a standing history file is the maintainer's choice; - statistical adoption verdicts. The history shows changes; the - [benchmark rules](../benchmarks.md) still say how a performance claim is judged; + [benchmark rules](../../benchmarks.md) still say how a performance claim is judged; - continuous integration. Shared CI runners are too noisy for timing. ### 3. Context -The [benchmark rules](../benchmarks.md) and the -[runner's manual](../benchmarking.md) describe SEP 30's suite. Its modules are under +The [benchmark rules](../../benchmarks.md) and the +[runner's manual](../../benchmarking.md) describe SEP 30's suite. Its modules are under `benchmarks/`: the registry (`registry.py`), case builders (`builders.py`, `exercises.py`, `guides.py`, `detectors.py`, `batching.py`), the controller and worker (`collection.py`, `worker.py`, `planning.py`), comparison and qualification @@ -80,7 +80,7 @@ The [benchmark rules](../benchmarks.md) and the (`publication.py`, `views.py`) and machine records (`system.py`, `telemetry.py`). The `ab` mode (`ab.py`, `ab_worker.py`) was added after SEP 30 to time two builds quickly. -[SEP 37](archive/sep-0037-noise-and-stepper-defaults.md) made input noise the default and chose +[SEP 37](sep-0037-noise-and-stepper-defaults.md) made input noise the default and chose an integrator and tolerances for each `prop_*` interface. Its phase 0 timed complete runs of the cases proposed here on an AMD EPYC 7763 node: these give the cost estimates below. With noise on by default, a timed case needs a fixed seed to be repeatable. @@ -152,8 +152,9 @@ SELECTION: [--case NAME]... [--set SET]... [--all] [--stepper S] [--threads W] (unpinned; idle threads sleep at once), `passive-bound` (pinned, sleeping at once), `default` (the runtime's own waiting), `active` (idle threads spin), `bound` (pinned to neighbouring cores) or - `bound-active`. The policy is part of what history rows compare by; the default is - set by phase 2's measurement. + `bound-active`. Every policy names its waiting, so a library default never decides + it. The policy is part of what history rows compare by; the default is `passive`, + from the measurement in Part II. - `--device` runs each case once on the GPU of a native CUDA or HIP build, with the production width of host threads. The worker refuses a device build without `--device` and a CPU build with it. A device row's width is `device`, the slug is @@ -274,8 +275,8 @@ exercise (`batching.py`), whose lockstep route is covered by ### 8. Documentation -Rewrite [Running the benchmark suite](../benchmarking.md) as the manual for `track`, `ab` -and `history`. In [Benchmarks](../benchmarks.md), replace "Measurement windows" with +Rewrite [Running the benchmark suite](../../benchmarking.md) as the manual for `track`, `ab` +and `history`. In [Benchmarks](../../benchmarks.md), replace "Measurement windows" with the tracking protocol, keep the rules and the indicative results, and point to the history files. Update `benchmarks/README.md`, the `poe bench` task, `AGENTS.md` and the performance guide's link. @@ -356,6 +357,29 @@ is final. Max was not timed (it was busy). - **Open questions:** none open (§12). +### Closing — threading, device mode and Final + +- **Scope:** 2026-10-27, Opus 5.5 (AI) for John Travers, after SEP 37 reached Final + and the branch was merged with it and retargeted to `main`. +- **Threading:** six OpenMP policies at 16 threads on 16 cores of the EPYC 7763 + allocation, on six tracked cases. Letting idle threads spin, whether by the runtime's + own waiting or by an explicit spin of 10,000 or more, made the cases that run + several thread teams 1.4 to 37 times slower (`rdw` 334 s against 12.7 s with + `passive`); the single-team supercontinuum gained up to 20 %. Pinning with idle + threads sleeping (`passive-bound`) was 6 to 30 % faster than `passive` on the + multi-team cases, over two alternating runs each. `passive` stays the default: it + is the library's own default too (ADR-056, which this measurement informed), and + `passive-bound` is for a dedicated allocation. On 36 cores of two Intel Xeon Gold + 6240, one thread per core beat two on every case by 6 to 31 %. +- **Device mode:** `--device` was added and run end to end on an NVIDIA A40 (CUDA + 12.8): all thirteen tracked cases in 19 minutes, every repeat bitwise identical, so + the repeat tolerance of §4 holds on the device unchanged. +- **Free space after SEP 37:** free space gained a default controller and no noise by + default, so the Cartesian and line cases take the library's controller and pass no + seed; the tracked Kerr beams keep their fixed schedule. +- **Not done:** no history row; the first rows are recorded at the release commit on + the standing machines (§5.3). + ## Part III — History - 2026-10-27: Drafted, together with its implementation on the same branch, as the @@ -373,3 +397,6 @@ is final. - 2026-10-27: At the maintainer's request, device mode (`--device`) was added before the first release instead of in a later SEP, so that a GPU history line can start with the first tagged release. +- 2026-10-27: Final, by John Travers, with SEP 30 marked Superseded; the proposal and + its implementation land in one pull request + ([LupoLab/solprop#32](https://github.com/LupoLab/solprop/pull/32)). diff --git a/docs/dev/seps/index.md b/docs/dev/seps/index.md index 786a6122..bd2ca52f 100644 --- a/docs/dev/seps/index.md +++ b/docs/dev/seps/index.md @@ -36,7 +36,6 @@ Proposals that are active, in draft, accepted, in progress or deferred. | [17](sep-0017-gas-two-photon-ionisation.md) | Broadband two-photon continuum response beside strong-field ionisation | Accepted | Feature | | [21](sep-0021-absorbing-boundaries.md) | Temporal boundaries: wider windows, absorption and the escaped field | Draft | Change | | [32](sep-0032-raman-maxwell-bloch.md) | Raman Maxwell–Bloch dynamics | Draft | Feature | -| [38](sep-0038-performance-tracking.md) | Performance tracking: complete runs, about an hour per machine | Draft | Process | SEP 13, the standalone numerical prerequisite for SEP 12, SEP 14, which carried it to the compiled backends, and SEP 16, which unified the multistep family and made it the @@ -68,12 +67,13 @@ Proposals that are final, rejected, withdrawn or superseded; the | [27](archive/sep-0027-held-plasma-current.md) | An edge-free field-grid plasma source: hold the current at the temporal collar | Final | Change | | [28](archive/sep-0028-free-space-evaluation.md) | Free-space evaluation at the memory bound | Final | Change | | [29](archive/sep-0029-compiled-backend-distribution.md) | The native backend in solprop, distributed as binaries | Final | Change | -| [30](archive/sep-0030-standard-benchmark-suite.md) | Standard benchmark suite | Final | Process | +| [30](archive/sep-0030-standard-benchmark-suite.md) | Standard benchmark suite | Superseded | Process | | [31](archive/sep-0031-simulation-coupling.md) | Coupling simulations and transporting ultrafast fields | Final | Feature | | [33](archive/sep-0033-single-precision-mlx.md) | Single-precision propagation, and an MLX backend for Apple GPUs | Final | Feature | | [34](archive/sep-0034-medium-flux.md) | Medium flux for free-space propagation | Final | Change | | [35](archive/sep-0035-carrier-frame-exponential-adams.md) | A carrier frame for the exponential Adams history | Final | Change | | [37](archive/sep-0037-noise-and-stepper-defaults.md) | Shot noise on by default, and an integrator and tolerances chosen for each interface | Final | Change | +| [38](archive/sep-0038-performance-tracking.md) | Performance tracking: complete runs, about an hour per machine | Final | Process | ```{toctree} :hidden: @@ -92,7 +92,6 @@ sep-0011-finite-density-gas-index sep-0017-gas-two-photon-ionisation sep-0021-absorbing-boundaries sep-0032-raman-maxwell-bloch -sep-0038-performance-tracking template archive/index ``` diff --git a/tests/test_benchmark_track.py b/tests/test_benchmark_track.py index cdb9fff0..1f82b3d6 100644 --- a/tests/test_benchmark_track.py +++ b/tests/test_benchmark_track.py @@ -328,6 +328,11 @@ def test_threading_policies_replace_the_callers_openmp_settings(): bound = system.thread_environment(4, incoming, "bound") assert (bound["OMP_PROC_BIND"], bound["OMP_PLACES"]) == ("close", "cores") assert "OMP_WAIT_POLICY" not in bound + assert bound["GOMP_SPINCOUNT"] == "300000" + # Every policy names its waiting, so the native backend's own default never applies. + waiting = {"OMP_WAIT_POLICY", "GOMP_SPINCOUNT", "KMP_BLOCKTIME"} + for policy in system.THREADING: + assert waiting & set(system.thread_environment(4, {}, policy)), policy with pytest.raises(SuiteError, match="unknown threading"): system.thread_environment(4, incoming, "spin")