From 279dbdfd073bdbf677dbbc37fa56b47a023b1f25 Mon Sep 17 00:00:00 2001 From: Adam Date: Fri, 9 Oct 2026 05:46:20 +0000 Subject: [PATCH 1/6] refactor: adopt shared performance and changelog workflows Pin published research-repo-tools 0.1.8 and replace the copied changelog template with the supported dependency-body policy. Delegate common-harness measurement, complete run retention, worktrees and transactional publication while retaining la-stack scientific selection and legacy evidence readers. Refs #268, #273. --- .config/performance-report.toml | 4 + CONTRIBUTING.md | 4 +- cliff.toml | 150 -- docs/BENCHMARKING.md | 117 +- docs/RELEASING.md | 20 +- docs/code_organization.md | 29 +- justfile | 58 +- pyproject.toml | 13 +- scripts/README.md | 85 +- scripts/archive_performance.py | 2264 ++-------------- scripts/bench_compare.py | 44 +- scripts/benchmark_summaries.py | 127 +- scripts/criterion_dim_plot.py | 93 +- scripts/criterion_measurements.py | 44 - scripts/performance_phase.py | 52 + scripts/performance_runs.py | 308 +++ scripts/release_baseline.py | 21 +- scripts/tests/test_archive_performance.py | 2660 ------------------- scripts/tests/test_bench_compare.py | 16 +- scripts/tests/test_changelog_integration.py | 458 +--- scripts/tests/test_criterion_dim_plot.py | 58 - scripts/tests/test_performance_artifacts.py | 135 - scripts/tests/test_performance_workflow.py | 284 ++ scripts/tests/test_release_baseline.py | 52 +- scripts/tests/test_toolchain_integration.py | 2 +- uv.lock | 12 +- 26 files changed, 1221 insertions(+), 5889 deletions(-) create mode 100644 .config/performance-report.toml delete mode 100644 cliff.toml delete mode 100644 scripts/criterion_measurements.py create mode 100644 scripts/performance_phase.py create mode 100644 scripts/performance_runs.py delete mode 100644 scripts/tests/test_archive_performance.py create mode 100644 scripts/tests/test_performance_workflow.py diff --git a/.config/performance-report.toml b/.config/performance-report.toml new file mode 100644 index 0000000..abf807e --- /dev/null +++ b/.config/performance-report.toml @@ -0,0 +1,4 @@ +schema = 2 +current = "docs/performance-runs/current.md" +archive = "docs/performance-runs" +title = "la-stack complete benchmark measurements" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f8d95ce..09aafca 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -28,11 +28,11 @@ change files. Run bare `just` for the complete generated command list, including arguments and descriptions. The [command policy](docs/dev/docs.md#command-discovery-and-placement) owns recipe naming and documentation placement. -Setup, tool checks, updates, changelog, and opt-in CodeRabbit review use the published `research-repo-tools==0.1.7` package, +Setup, tool checks, updates, changelog, and opt-in CodeRabbit review use the published `research-repo-tools==0.1.8` package, locked in the `tooling` dependency group and included by `dev`. Normal setup and CI install it from PyPI through `uv sync --locked --group dev`; a sibling checkout is unnecessary. To upgrade it deliberately, review the exact -`tooling` requirement and refresh `uv.lock` together, then run the consumer +`tooling` and project runtime requirements and refresh `uv.lock` together, then run the consumer integration tests and `just ci`. See the [Scripts guide](scripts/README.md#changelog-and-release-tooling) for the retained changelog policy and ownership boundary. diff --git a/cliff.toml b/cliff.toml deleted file mode 100644 index c94b4a6..0000000 --- a/cliff.toml +++ /dev/null @@ -1,150 +0,0 @@ -# Shared git-cliff configuration -# https://git-cliff.org/docs/configuration -# Materialized from research-repo-tools 0.1.7; only the deps-dev body condition -# below differs from the packaged template. See scripts/README.md. - -[changelog] -# Keep a Changelog header -header = """ -# Changelog\n -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).\n -""" - -# Tera template for each release section. -# -# Release sections are generated from commit ranges. The normalizer escapes -# prose angle brackets after protecting Markdown code, so Rust generics remain -# literal inside code spans and fences. Breaking footers are separate from -# commit.body; render their full descriptions before grouping ordinary entries. -body = """ -{% if version -%} - ## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }} -{% else -%} - ## [Unreleased] -{% endif -%} -{% set breaking_commits = commits | filter(attribute="breaking", value=true) %} -{% if breaking_commits | length > 0 %} - ### ⚠️ Breaking Changes - {% for commit in breaking_commits %} - {% if commit.breaking_description and commit.breaking_description != commit.message %} - - {{ commit.breaking_description | trim | indent(prefix=" ", first=false) }} - {% else %} - - {{ commit.message | split(pat="\n") | first | upper_first | trim }} - {% endif %} - {% endfor %} -{% endif %} -{% for group, commits in commits | group_by(attribute="group") %} - ### {{ group | upper_first }} - {% for commit in commits %} - - {% if commit.breaking %}[**breaking**] {% endif %}\ - {{ commit.message | split(pat="\n") | first | upper_first | trim }} \ - [`{{ commit.id | truncate(length=7, end="") }}`]\ - (https://github.com/{{ remote.github.owner }}/{{ remote.github.repo }}\ - /commit/{{ commit.id }}) - {%- if commit.body and (group != "Dependencies" or commit.scope == "deps-dev") %} - - {{ commit.body | indent(prefix=" ", first=true) }} - {%- endif %} - {%- endfor %} -{% endfor %}\n - -""" - -# Compare-link footer (requires [remote.github] below) -footer = """ -{% for release in releases -%} - {% if release.version -%} - {% if release.previous.version -%} - [{{ release.version | trim_start_matches(pat="v") }}]: \ - https://github.com/{{ remote.github.owner }}/{{ remote.github.repo }}\ - /compare/{{ release.previous.version }}...{{ release.version }} - {% else -%} - [{{ release.version | trim_start_matches(pat="v") }}]: \ - https://github.com/{{ remote.github.owner }}/{{ remote.github.repo }}\ - /tree/{{ release.version }} - {% endif -%} - {% else -%} - {% if release.previous.version -%} - [Unreleased]: https://github.com/{{ remote.github.owner }}/{{ remote.github.repo }}\ - /compare/{{ release.previous.version }}...HEAD - {% else -%} - [Unreleased]: https://github.com/{{ remote.github.owner }}/{{ remote.github.repo }}\ - /commits/HEAD - {% endif -%} - {% endif -%} -{% endfor -%} -""" - -trim = true -# Collapse 3+ consecutive newlines into exactly 2 (one blank line). -# This prevents MD012 violations from Tera template whitespace artifacts. -postprocessors = [ - { pattern = '\n{3,}', replace = "\n\n" }, -] - -[git] -conventional_commits = true -# Include non-conventional commits so we capture the full history -# (mixed styles: "feat:", "Changed:", plain merge commits, dependency bumps). -filter_unconventional = false -filter_commits = false -protect_breaking_commits = true -# Expanded from the package's shared SemVer grammar when this template is read. -tag_pattern = '\Av(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)(?:-(?:(?:0|[1-9][0-9]*)|[0-9A-Za-z-]*[A-Za-z-][0-9A-Za-z-]*)(?:\.(?:(?:0|[1-9][0-9]*)|[0-9A-Za-z-]*[A-Za-z-][0-9A-Za-z-]*))*)?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?\z' -topo_order = false -sort_commits = "oldest" - -# Link (#N) references to GitHub PRs before rendering. -# NOTE: The repo path is hardcoded here because commit_preprocessors run before -# Tera template rendering, so {{ remote.github.owner }} variables are not -# available. The body/footer templates above do use the remote.github variables. -commit_preprocessors = [ - { pattern = '(?m)^Closes #\d+\n\n', replace = "" }, - { pattern = '\(#(\d+)\)', replace = '[#${1}](https://github.com/acgetchell/la-stack/pull/${1})' }, -] - -commit_parsers = [ - # Skip merge commits and release-prep noise from the changelog body. - { message = "^Merge pull request", skip = true }, - { message = "^Merge branch", skip = true }, - { message = '(?i)^(chore(:|\(release\):) |Changed: )?(prepare|release) v[0-9]', skip = true }, - - # Dependency bumps: keep all bump commits so archived release history - # preserves CI/action/tooling dependency updates as well as Rust crates. - { message = "^chore\\(deps(?:-[^)]+)?\\)!?: bump ", group = "Dependencies" }, - { message = "(?i)^bump\\s+(deps?\\b|dependencies\\b|[A-Za-z0-9_.-]+/[A-Za-z0-9_./-]+|[A-Za-z0-9_.-]+\\s+from\\s+)", group = "Dependencies" }, - - # Conventional commit types - { message = "^feat", group = "Added" }, - { message = "^fix", group = "Fixed" }, - { message = "^perf", group = "Performance" }, - { message = "^refactor", group = "Changed" }, - { message = "^docs?", group = "Documentation" }, - { message = "^test", group = "Changed" }, - { message = "^style", group = "Changed" }, - { message = "^build", group = "Maintenance" }, - - # Historical non-conventional commit categories - { message = "(?i)^add", group = "Added" }, - { message = "(?i)^support", group = "Added" }, - { message = "(?i)^implement", group = "Added" }, - { message = "(?i)^deprecate", group = "Deprecated" }, - { message = "(?i)^remove", group = "Removed" }, - { message = "(?i)^delete", group = "Removed" }, - { message = "(?i)^change", group = "Changed" }, - { message = "(?i)^update", group = "Changed" }, - { message = "(?i)^fix", group = "Fixed" }, - - # Chore / CI - { message = "^chore|^ci", group = "Maintenance" }, - - # Catch-all - { message = ".*", group = "Changed" }, -] - -[remote.github] -owner = "acgetchell" -repo = "la-stack" diff --git a/docs/BENCHMARKING.md b/docs/BENCHMARKING.md index 021251c..4319261 100644 --- a/docs/BENCHMARKING.md +++ b/docs/BENCHMARKING.md @@ -152,18 +152,18 @@ just performance-local This creates isolated temporary worktrees and runs both library revisions on the same machine with the current checkout's benchmark sources, manifests, lockfile, -benchmark-input tests, recipes, and Rust toolchain. Current example sources are +benchmark-input tests, phase driver, and Rust toolchain. Current example sources are also copied so Cargo can resolve every target declared by the shared manifest; the comparison does not build or time those examples. Staged and unstaged changes to tracked files are applied to the current worktree. Untracked files are excluded; stage a new file before running the command if it must participate in the comparison. Only the baseline library implementation comes from the release -tag. Before either timing run, the command runs `just test-bench-inputs` against -that revision under the shared current fixture harness. This is a prerequisite +tag. Both revisions must pass the same benchmark-input tests used by +`just test-bench-inputs`, via locked Cargo commands, before either timing phase. This is a prerequisite correctness gate over the deterministic fixtures and operations, not validation of each timed Criterion sample. It writes -`target/bench-reports/performance.md` plus retained `performance.csv` and -`performance.provenance.json` comparison inputs. The report and sidecar embed +`target/bench-reports/performance.md` plus retained `performance.run.json` and +`performance.evidence.json` complete-run inputs. The report and sidecar embed both commits, CPU, operating system, Rust toolchain, lockfile and harness digests, Criterion selection/commands, and both correctness-gate results. The report reader rejects malformed or mismatched provenance and incomplete @@ -179,7 +179,7 @@ For the default `all` suite, timing runs occur in this order: La-stack is measured once per revision; the nalgebra/faer measurements from the baseline phase are reused as peer context. The filtered current run disables Criterion HTML generation because skipped peer `new` samples are deliberately -removed before current measurements. The retained CSV/JSON inputs and Python +absent from the isolated current phase. The retained complete-run inputs and Python renderer provide the release report. Baseline-saving recipes also disable unused Criterion plots and HTML without changing sampling. @@ -285,9 +285,9 @@ just performance-readme ``` `performance-readme` does not run benchmarks. It loads the canonical -`target/bench-reports/performance.csv` and adjacent provenance JSON retained by +`target/bench-reports/performance.run.json` and its evidence envelope retained by `performance-release`, falling back to the latest committed local snapshot when -both scratch inputs are absent. It then uses the current la-stack result and the peer +all scratch inputs are absent. It then uses the current la-stack result and the peer nalgebra/faer results measured by that same shared current harness. It requires all three timings for every canonical dimension (D=2, 3, 4, 5, 8, 16, 32, and 64) before updating: @@ -297,16 +297,12 @@ all three timings for every canonical dimension (D=2, 3, 4, 5, 8, 16, 32, and - `docs/assets/bench/vs_linalg_lu_solve_median.svg` - `docs/assets/bench/vs_linalg_lu_solve_median.provenance.json` -The publisher verifies the retained artifact digest and schema, release version, -measured source state, commit, dependency lock, complete peer coverage, and -recorded measurement provenance before writing anything. When the retained -provenance includes a benchmark-contract digest, it also verifies the benchmark -code, inputs, dependencies, and toolchain against the current checkout. Legacy -retained artifacts without that field remain publishable but are labeled -`legacy-retained-artifact` rather than contract-matched. The derived provenance -sidecar preserves the measurement commands and environment and records the -retained CSV/JSON digests. Missing, stale, or inconsistent input aborts -publication atomically. +The publisher verifies the complete-run identity, release version, current +source and harness inventories/digests, dependency resolution, complete peer +coverage, and recorded phase provenance before writing. The derived sidecar +records the run identity and the actual current/baseline origins of each series. +Historical CSV/JSON evidence retains its original reader and provenance. +Missing, stale, or inconsistent input aborts publication atomically. For exploratory CSV/SVG output, run `bench-vs-linalg` and `plot-vs-linalg`. That path still reads raw Criterion output; `--allow-partial` remains @@ -327,40 +323,28 @@ With no arguments, `just performance-release` infers the current release tag from `Cargo.toml` and discovers the previous stable published release. During release preparation, passing both tags explicitly removes ambiguity. -This command creates temporary worktrees, validates the complete comparison, -and writes the exact report inputs to -`target/bench-reports/performance.csv` with adjacent -`performance.provenance.json`. The CSV records deterministic benchmark keys, -coverage status and notes, baseline/current median estimates, and -same-current-harness nalgebra/faer peer estimates with complete confidence -intervals in nanoseconds. Those peer fields are the source for -`performance-readme`. The JSON sidecar binds the CSV digest and row count to -the release pair, source states, commands, toolchain, Criterion version, -harness/configuration digests, host, and schema version. - -Before deleting the temporary worktree, the workflow also exports -`performance.full.csv` and `performance.full.provenance.json`. These preserve -every recorded case from the baseline and current phases, including diagnostics -and peer measurements outside the curated report selection. Means, medians, -95% confidence intervals, and sample counts retain full timing precision. -No additional benchmark executions are needed for this export. - -Before creating worktrees or running either benchmark revision, structured -local and release-report workflows require an identifiable CPU model. Raw -Criterion benchmark recipes may still record measurements when that metadata -is unavailable, but those measurements cannot be promoted as reproducible -release evidence. - -The pair is validated and published before the temporary worktree is removed. -All four inputs are preserved under `docs/performance///` -with `latest.json` pointing to that snapshot. `docs/performance.md` is rendered -from a validated reload of the selected pair, and the previous committed report is archived under -`docs/archive/performance/`. Archive filenames are release-pair names such as -`v0.4.2-vs-v0.4.1.md`. Serialization, validation, rendering, coverage, or -promotion failures preserve the previous valid report, inputs, and snapshot -pointer. Review and commit the complete snapshot along with the curated report. -See the [local summary index](performance/README.md) for the schema and -comparison limits. +This command uses the shared common-harness engine in isolated worktrees. +The native Criterion inventory declares every expected semantic ID, including +diagnostics outside the selected report. Each case requires 100 raw samples, +both mean and median estimates, and complete 95% intervals in nanoseconds. +The consumer declares canonical report rows and baseline-only nalgebra/faer +reference series. The pre-rational-input adapter marks unsupported baseline +rows explicitly; current-only measurements never imply cross-release speedups. + +Scratch evidence uses `performance.run.json` and `performance.evidence.json` +under `target/bench-reports/`. These preserve raw Criterion JSON, both source +identities, one captured harness identity, commands, environments, toolchain, +dependency versions, host identity, and independent gate results. + +Promotion publishes immutable `run.json`, `evidence.json`, and `report.md` +under `docs/performance-runs/runs//`, plus the shared index, latest +pointer, history index, and `docs/performance.md` in one transaction. +Repeated runs for one release pair coexist. The previous curated report is +retained under `docs/archive/performance/` when its archive path is new. +Serialization, validation, rendering, and promotion failures preserve existing +published reports and selection. Review and commit the whole new run and index. +Legacy snapshots under `docs/performance/` retain their original bytes, +hashes, framing, and reader; see the [legacy index](performance/README.md). To reproduce and promote the report without running Cargo or creating Git worktrees, use: @@ -370,12 +354,11 @@ just performance-doc ``` This command fails closed on a missing, partial, malformed, mismatched, or -unsupported artifact pair. It consumes the default CSV/JSON pair retained by a -successful `performance-local` or `performance-release` run, or follows -`docs/performance/latest.json` when both default scratch inputs are absent. -It rewrites the -scratch Markdown, promotes it to `docs/performance.md`, and archives the previous -committed report when the release pair changes. Promotion requires distinct +unsupported artifact pair. It consumes the default complete-run payload/envelope +or follows the validated `docs/performance-runs/latest.json` when all scratch +inputs are absent. A repository with only legacy evidence continues through +the historical reader. Report replay runs offline and updates the curated +document and shared history transactionally. Promotion requires distinct current and baseline package versions, so a same-version local comparison is retained and reproducible but cannot become release documentation. Use promotion for presentation-only report corrections; changes to benchmark inputs, code, @@ -399,9 +382,9 @@ published GitHub releases. Published artifacts preserve each release's original benchmark harness. Their historical timing environments may not have been recorded, so report provenance -labels those fields unavailable rather than reconstructing them. The workflow -still runs the current independent fixture gate against both source revisions -under the shared current fixture harness before reading the assets. Use a local +labels those fields, including historical correctness gates, unavailable. This +read-only comparison does not claim common-harness measurement or permit release +promotion. Use a local shared-harness workflow before attributing a difference solely to library code. ## Output Locations @@ -410,15 +393,13 @@ shared-harness workflow before attributing a difference solely to library code. |------|------------|----------|---------| | `target/criterion/` | No | `cargo bench`, `bench-save-*` | Local Criterion measurements and named baselines. | | `target/bench-reports/performance.md` | No | `bench-compare`, `performance-local`, `performance-release`, `performance-doc` | Canonical local comparison report. | -| `target/bench-reports/performance.csv` | No | `performance-local`, `performance-release` | Validated tabular inputs for the canonical comparison and README publisher. | -| `target/bench-reports/performance.provenance.json` | No | `performance-local`, `performance-release` | Schema, package identifiers, source, command, toolchain, host, digest, and harness provenance consumed by the README publisher. | -| `target/bench-reports/performance.full.*` | No | `performance-local`, `performance-release` | Every recorded local case, with timing summaries and provenance. | +| `target/bench-reports/performance.run.json` | No | `performance-local`, `performance-release` | Complete native cases, samples, and named phase series. | +| `target/bench-reports/performance.evidence.json` | No | `performance-local`, `performance-release` | Shared evidence envelope with source/harness and phase provenance. | | `target/bench-reports/performance-non-exact.*` | No | `performance-local-non-exact` | Narrowed non-exact report and retained peer-context comparison inputs. | | `target/bench-reports/github-assets-performance.md` | No | `performance-github-assets` | Local report from published release artifacts. | -| `target/bench-reports/github-assets-performance.csv` | No | `performance-github-assets` | Tabular inputs derived from published native archives. | -| `target/bench-reports/github-assets-performance.provenance.json` | No | `performance-github-assets` | Provenance for the published-asset report inputs. | | `docs/performance.md` | Yes | `performance-release`, `performance-doc` | Latest curated release-to-release comparison. | -| `docs/performance/` | Yes | `performance-release`, `performance-doc` | Complete local summary snapshots, selected report inputs, and latest pointer; survives `just clean`. | +| `docs/performance-runs/` | Yes | `performance-release`, `performance-doc` | Immutable complete runs, reports, index, and validated latest pointer. | +| `docs/performance/` | Yes | Legacy reader | Unchanged historical CSV/JSON evidence and hashes. | | `docs/archive/performance/` | Yes | `performance-release`, `performance-doc` | Older curated release-to-release comparisons. | | `docs/archive/performance/studies/` | Yes | Maintainer investigations | Completed optimization studies and decisions. | | `docs/assets/bench/` | Yes | `performance-readme` | README benchmark CSV/SVG assets and JSON provenance. | @@ -430,7 +411,7 @@ Published baseline assets use the filename Everything under `target/bench-reports/` is reproducible local scratch owned by the performance-report workflows. It survives temporary-worktree cleanup but may be removed by `just clean` or `cargo clean`. Promoted local snapshots remain -under `docs/performance/`, so report and README regeneration do not require +under `docs/performance-runs/`, so report and README regeneration do not require another measurement run. Unpromoted local experiments remain scratch data. The saved summaries do not replace the full native Criterion `.tar.gz` archive attached to each GitHub Release. Hosted archives retain raw samples from a diff --git a/docs/RELEASING.md b/docs/RELEASING.md index f72359f..63d4dd9 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -119,17 +119,15 @@ just performance-release The no-argument form compares the current package version with the previous stable published release. Review `docs/performance.md`, any archived comparison -under `docs/archive/performance/`, and the complete versioned local summary -snapshot and `latest.json` under `docs/performance/`. Include those CSV and JSON -files in the release commit; they preserve every recorded case, including the -measurements outside the readable report's selection. - -The temporary current worktree includes staged and unstaged changes to tracked -files, but excludes untracked files. Stage any new benchmark-relevant file before -running the comparison. Successful release promotion preserves the selected -report inputs and complete summaries under `docs/performance/`, so cleanup no -longer removes the data needed to regenerate the reports. Local experiments -from `performance-local` remain scratch until explicitly promoted. +under `docs/archive/performance/`, and the immutable complete-run evidence, +reports, index, and latest pointer under `docs/performance-runs/`. Include the +entire new run and index changes in the release PR. Shared complete evidence +retains both phases, mean and median estimates, 100 raw samples per case, and +95% intervals. It survives target cleanup and supports offline report/README +replay. Multiple runs for one release pair coexist without rewriting older +evidence. Historical CSV/JSON snapshots under `docs/performance/` remain +readable with unchanged hashes. Results from `performance-local` remain +scratch until explicitly promoted. ### 5. Refresh the README benchmark comparison diff --git a/docs/code_organization.md b/docs/code_organization.md index bca0b23..6d3bfd0 100644 --- a/docs/code_organization.md +++ b/docs/code_organization.md @@ -117,8 +117,9 @@ baselines, output locations, and report promotion. The [Scripts guide](../script owns the Python script inventory and entry points for comparisons and plotting. The pinned published `research-repo-tools` dependency owns changelog generation, normalization, minor-series archiving, note lookup, -and tag preparation through its CLI. Consumer policy stays in `cliff.toml`, -`changelog-rumdl.toml`, and `[tool.research-repo-tools]` in `pyproject.toml`; +and tag preparation through its CLI. Consumer identity and dependency-body policy +stay in `[tool.research-repo-tools.changelog]` in `pyproject.toml`, with +formatting in `changelog-rumdl.toml`; no common template is copied locally. focused integration checks live in `scripts/tests/test_changelog_integration.py`. The same dependency owns CodeRabbit review orchestration through thin Just wrappers. `scripts/tests/test_review_integration.py` owns consumer wiring checks @@ -155,23 +156,31 @@ Performance consumers use shared Criterion parsing and estimate/comparison validation, digest verification, archive extraction, byte-preserving document sections, and multi-file transactions. Local rendering produces complete candidate outputs before publication. Historical artifact schemas and fingerprint framing, -benchmark selection and eligibility, common-harness orchestration, and complete -run retention remain in the consumer pending the corresponding shared workflow -contract. Generic parsing, staging, and rollback tests belong upstream; local +benchmark selection and eligibility remain in the consumer. Published v0.1.8 +owns common-harness orchestration, completeness, worktrees, complete run retention, +validated latest selection, and transactional report publication. +Generic parsing, staging, and rollback tests belong upstream; local tests verify the scientific and retained-artifact integration boundaries. Hosted Dependabot approvals use the pinned shared GitHub workflow; the [rollout guide](dev/MANAGING_CHANGES.md#dependabot-approval-rollout) owns settings and deployment verification. -`scripts/release_baseline.py` owns release-suite inventory and complete raw -Criterion validation. The release workflow packages only datasets that pass +`scripts/release_baseline.py` owns release-suite inventory and delegates complete +raw Criterion validation to the shared policy. The release workflow packages only datasets that pass that gate; its regression and archive tests live in `scripts/tests/test_release_baseline.py`. -`scripts/criterion_measurements.py` owns raw sample validation shared by hosted -archives and local summaries. `scripts/benchmark_summaries.py` owns complete -local summary serialization, snapshot identity, and lookup after cleanup. +`scripts/performance_phase.py` declares native Cargo commands, features, and +input gates. `scripts/performance_runs.py` declares scientific coverage, +release compatibility, and named reference phases, and renders the local tables. +`scripts/archive_performance.py` composes the shared APIs for consumer commands. +`.config/performance-report.toml` owns the new retention paths. +`scripts/benchmark_summaries.py` only reads historical CSV/JSON snapshots; +`scripts/criterion_measurements.py` and the old generic measurement/retention +engines are removed. `scripts/tests/test_performance_workflow.py` verifies +installed-package integration and offline replay. The retained local +`criterion_dim_plot.py` preserves the multi-library figure format. `.github/actions/prepare-release-benchmarks/action.yml` groups tool installation, input validation, and inventory under the release workflow's shared setup timeout. diff --git a/justfile b/justfile index 9e3a4bf..9d88706 100644 --- a/justfile +++ b/justfile @@ -365,47 +365,23 @@ markdown-fix: tools-check echo "No markdown files found to format." fi -# Build release docs from retained scratch inputs or the latest docs/performance snapshot. +# Render release docs from complete runs, or read historical CSV/JSON evidence. performance-doc: python-sync - {{ _run }} uv run --locked archive-performance --promote-artifacts + {{ _run }} uv run --locked archive-performance report -# Compare stored GitHub Actions release benchmark assets without local cargo runs. +# Compare published release archives with their original per-release harnesses. performance-github-assets current_tag="" baseline_tag="": _ensure-gh python-sync - #!/usr/bin/env bash - set -euo pipefail - current_tag={{ quote(current_tag) }} - baseline_tag={{ quote(baseline_tag) }} - if [[ -n "$current_tag" || -n "$baseline_tag" ]]; then - if [[ -z "$current_tag" || -z "$baseline_tag" ]]; then - echo "current_tag and baseline_tag must be provided together" >&2 - exit 2 - fi - {{ _run }} uv run --locked archive-performance "$current_tag" "$baseline_tag" --github-assets --generate-in-temp-worktree --worktree-ref "$current_tag" --output-only --output target/bench-reports/github-assets-performance.md --artifact-csv target/bench-reports/github-assets-performance.csv --artifact-provenance target/bench-reports/github-assets-performance.provenance.json - else - {{ _run }} uv run --locked archive-performance --published-latest --github-assets --generate-in-temp-worktree --output-only --output target/bench-reports/github-assets-performance.md --artifact-csv target/bench-reports/github-assets-performance.csv --artifact-provenance target/bench-reports/github-assets-performance.provenance.json - fi + {{ _run }} uv run --locked archive-performance assets {{ quote(current_tag) }} {{ quote(baseline_tag) }} -# Compare the current tree against the latest release; untracked files are excluded. +# Measure tracked current inputs and the latest release with the captured current harness. performance-local: _ensure-gh python-sync - {{ _run }} uv run --locked archive-performance --current-vs-latest --generate-in-temp-worktree --output-only --local-report --output target/bench-reports/performance.md + {{ _run }} uv run --locked archive-performance local -# Compare current non-exact kernels locally without rerunning current peer crates. +# Measure the comparison suite while retaining baseline-phase nalgebra/faer context. performance-local-non-exact current_tag="" baseline_tag="": _ensure-gh python-sync - #!/usr/bin/env bash - set -euo pipefail - current_tag={{ quote(current_tag) }} - baseline_tag={{ quote(baseline_tag) }} - if [[ -n "$current_tag" || -n "$baseline_tag" ]]; then - if [[ -z "$current_tag" || -z "$baseline_tag" ]]; then - echo "current_tag and baseline_tag must be provided together" >&2 - exit 2 - fi - {{ _run }} uv run --locked archive-performance "$current_tag" "$baseline_tag" --suite vs_linalg --generate-in-temp-worktree --worktree-ref HEAD --output-only --local-report --output target/bench-reports/performance-non-exact.md --artifact-csv target/bench-reports/performance-non-exact.csv --artifact-provenance target/bench-reports/performance-non-exact.provenance.json - else - {{ _run }} uv run --locked archive-performance --current-vs-latest --suite vs_linalg --generate-in-temp-worktree --output-only --local-report --output target/bench-reports/performance-non-exact.md --artifact-csv target/bench-reports/performance-non-exact.csv --artifact-provenance target/bench-reports/performance-non-exact.provenance.json - fi + {{ _run }} uv run --locked archive-performance local {{ quote(current_tag) }} {{ quote(baseline_tag) }} --suite vs_linalg --stem target/bench-reports/performance-non-exact -# Publish README assets/table from retained measurements, including after target cleanup. +# Publish the existing README plot format from retained measurements after cleanup. performance-readme metric="lu_solve" stat="median" sample="new" log_y="true": python-sync #!/usr/bin/env bash set -euo pipefail @@ -415,21 +391,9 @@ performance-readme metric="lu_solve" stat="median" sample="new" log_y="true": py fi {{ _run }} uv run --locked criterion-dim-plot "${args[@]}" -# Measure locally, preserve complete summaries in docs/performance, and promote/archive docs. +# Measure, retain an immutable complete run, and promote distinct-release documentation. performance-release current_tag="" baseline_tag="": _ensure-gh python-sync - #!/usr/bin/env bash - set -euo pipefail - current_tag={{ quote(current_tag) }} - baseline_tag={{ quote(baseline_tag) }} - if [[ -n "$current_tag" || -n "$baseline_tag" ]]; then - if [[ -z "$current_tag" || -z "$baseline_tag" ]]; then - echo "current_tag and baseline_tag must be provided together" >&2 - exit 2 - fi - {{ _run }} uv run --locked archive-performance "$current_tag" "$baseline_tag" --generate-in-temp-worktree --worktree-ref HEAD - else - {{ _run }} uv run --locked archive-performance --infer-release --generate-in-temp-worktree --worktree-ref HEAD - fi + {{ _run }} uv run --locked archive-performance release {{ quote(current_tag) }} {{ quote(baseline_tag) }} # Plot: generate a single time-vs-dimension SVG from Criterion results. plot-vs-linalg metric="lu_solve" stat="median" sample="new" log_y="false" allow_partial="false": python-sync diff --git a/pyproject.toml b/pyproject.toml index 1691aa8..8a7d049 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ ] # Benchmark adapters use only the installed shared package and the standard library. -dependencies = [ "research-repo-tools==0.1.7" ] +dependencies = [ "research-repo-tools==0.1.8" ] [project.urls] "Homepage" = "https://github.com/acgetchell/la-stack" @@ -41,7 +41,7 @@ criterion-dim-plot = "criterion_dim_plot:main" # Configure setuptools to find modules in scripts/ directory. [tool.setuptools] package-dir = { "" = "scripts" } -py-modules = [ "archive_performance", "bench_compare", "benchmark_contract", "benchmark_process", "benchmark_summaries", "criterion_dim_plot", "criterion_measurements", "performance_artifacts" ] +py-modules = [ "archive_performance", "bench_compare", "benchmark_contract", "benchmark_process", "benchmark_summaries", "criterion_dim_plot", "performance_artifacts", "performance_phase", "performance_runs", "release_baseline" ] [tool.ruff] line-length = 160 @@ -91,8 +91,10 @@ known-first-party = [ "benchmark_process", "benchmark_summaries", "criterion_dim_plot", - "criterion_measurements", "performance_artifacts", + "performance_phase", + "performance_runs", + "release_baseline", ] force-single-line = false split-on-trailing-comma = true @@ -146,7 +148,7 @@ no-build-package = [ "semgrep" ] override-dependencies = [ "pyjwt[crypto]>=2.15.1,<3" ] [dependency-groups] -tooling = [ "research-repo-tools==0.1.7" ] +tooling = [ "research-repo-tools==0.1.8" ] dev = [ { include-group = "tooling" }, "actionlint-py==1.7.12.25", @@ -184,8 +186,7 @@ zizmor = "1.30.1" owner = "acgetchell" repository = "la-stack" formatter = "changelog-rumdl.toml" -# Packaged 0.1.7 template plus one deps-dev body-preservation exception. -cliff-config = "cliff.toml" +dependency-bodies = "preserve" [tool.research-repo-tools.zizmor] persona = "regular" diff --git a/scripts/README.md b/scripts/README.md index 48cd998..45a7582 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -90,21 +90,24 @@ The local release workflows run the independent benchmark-input correctness gate and then measure both library revisions with one hashed current benchmark harness. Reports record source-state, environment, toolchain, dependency, Criterion, harness, and validation provenance and fail on incomplete selected -coverage. `performance-local` writes Markdown plus schema-versioned -`performance.csv` and `performance.provenance.json` inputs under -`target/bench-reports/`, plus `performance.full.csv` and its provenance JSON -containing every recorded case from both measurement phases. -`performance-release` does the same measurement work, requires distinct releases, -and preserves all four files under `docs/performance///` -when promoting the validated result. `performance-doc` consumes the retained pair -from either workflow without Cargo or temporary worktrees, then promotes the -result into `docs/performance.md` and the archive. Same-version local artifacts -remain valid comparison evidence but cannot be promoted as a release report. -Scratch files may be removed with `target/`. After cleanup, `performance-doc` and -`performance-readme` resolve `docs/performance/latest.json` to the most recently -promoted complete snapshot. Existing partial scratch inputs fail validation. -The [local summary index](../docs/performance/README.md) owns the saved-data -schema and retention contract. Native Criterion release archives remain the +coverage. `performance-local` writes `performance.md`, `performance.run.json`, +and `performance.evidence.json` under `target/bench-reports/`. The shared +complete-run payload preserves every semantic Criterion case, 100 raw samples, +mean and median estimates, and 95% intervals from both phases. +`performance-release` requires distinct releases and publishes immutable +`run.json`, `evidence.json`, and `report.md` files under +`docs/performance-runs/runs//`, with a validated index and latest +pointer. Its scientific tables are published to `docs/performance.md` in the +same transaction. Repeated runs for one pair coexist; publication failures +preserve the previous reports and selection. `performance-doc` replays this +evidence without Cargo. Same-version local comparisons require distinct source +identities and cannot be promoted as release reports. + +After `target/` cleanup, report and README commands select validated shared +history. Any partial scratch bundle fails instead of selecting older evidence. +Legacy CSV/JSON bytes, hashes, and rendered reports remain readable through +their original schema under `docs/performance/`; new runs never overwrite or +reinterpret that history. Native Criterion release archives remain the durable raw baselines from the GitHub runner. Direct comparisons of separately published artifacts retain their original per-release harnesses and label unavailable historical measurement metadata explicitly. @@ -171,8 +174,8 @@ just performance-release just performance-readme lu_solve median new true ``` -The README recipe consumes `target/bench-reports/performance.csv` and its -adjacent provenance JSON, or the latest committed snapshot after cleanup; +The README recipe consumes the complete-run payload and evidence envelope, +or the validated latest committed run after cleanup; it does not run the benchmark-input gate or Criterion again. It uses the current la-stack timing and retained same-current-harness nalgebra/faer timings, requires all three at every canonical dimension, and then @@ -298,7 +301,7 @@ just changelog-check just release-notes vX.Y.Z ``` -The exact published `research-repo-tools==0.1.7` dependency is in `tooling`, +The exact published `research-repo-tools==0.1.8` dependency is in `tooling`, included by `dev`, and resolved from PyPI in `uv.lock`. Thin Just recipes use the documented `research-repo-tools changelog` CLI. Its implementation and common regressions belong to the shared package; this repository owns the @@ -321,15 +324,12 @@ Preview validates root and archive candidates without publishing them. Malformed versions/dates, duplicate releases, conflicting retained notes, and formatter failures stop publication with diagnostics. -`cliff.toml` is materialized from the installed package's common 0.1.7 template. -It adopts the shared categories, SemVer tag grammar, compare links, and -Markdown-aware code handling. Its only semantic exception retains bodies for -`deps-dev` commits when grouped as Dependencies, preserving historical Ruff, -Ty, and setuptools release-note, changelog, and comparison links. A focused -integration check compares the parsed configuration with the installed template -plus that single condition, preventing unrelated local policy drift. -The exception can retire after adopting a published shared solution to -[research-repo-tools#62](https://github.com/acgetchell/research-repo-tools/issues/62). +The shared changelog table in `pyproject.toml` declares the repository owner, +repository name, formatter, and `dependency-bodies = "preserve"`. The installed +v0.1.8 common template owns grouping, tag grammar, links, and Markdown handling; +there is no local `cliff.toml` or template exception. Installed-package tests +exercise Ruff, Ty, and setuptools authored text, links, and fenced code across +repeat generation, and check retained historical notes. The shared normalizer preserves complete breaking-change descriptions, Markdown links, and literal code in the input while adding merged-PR summaries. It retains embedded conventional headings that the old normalizer deduplicated. @@ -353,8 +353,9 @@ and recipe forwarding. `tests/test_cargo_update_integration.py` executes native Cargo upgrades against a disposable local registry; Python updates have a matching real-uv fixture in the toolchain tests. Common parser, transaction, Markdown, and fixture regressions belong to the shared package. Scientific -eligibility, benchmark orchestration, retained schemas, and report layouts remain -consumer-owned. Shared performance primitives are adopted below. +eligibility, Cargo commands/features, release adapters, selected rows, and report +layouts remain consumer-owned. The shared package owns common-harness phases, +completeness, worktrees, run identities, retention, and publication. Notebook tooling remains outside scope. The same pinned release owns opt-in CodeRabbit review orchestration through @@ -387,13 +388,14 @@ preview an annotation without creating a tag. | Script | Purpose | |---|---| -| `archive_performance.py` | Promote release performance docs and archive older comparisons | +| `archive_performance.py` | Select eligible releases and compose shared measurement/publication APIs | | `bench_compare.py` | Compare Criterion benchmark baselines and render Markdown reports | | `benchmark_contract.py` | Define and hash the consumer's benchmark inventory contract | | `benchmark_process.py` | Thin benchmark phase adapters over the published shared process API, plus consumer root selection | -| `benchmark_summaries.py` | Preserve every local case summary, bind provenance, and resolve saved report inputs after cleanup | +| `benchmark_summaries.py` | Read historical complete CSV/JSON summaries without changing framing | | `criterion_dim_plot.py` | Plot Criterion benchmark results (CSV + SVG + README table) | -| `criterion_measurements.py` | Validate full Criterion sampling and estimates for local summaries and hosted archives | +| `performance_phase.py` | Declare native Cargo inventory, gate, and timing commands | +| `performance_runs.py` | Declare scientific run policy and adapt retained series to local tables | | `performance_artifacts.py` | Validate and publish schema-versioned performance-comparison CSV/JSON inputs | | `release_baseline.py` | Inventory full Criterion suites and validate complete raw release baselines before packaging | @@ -403,6 +405,13 @@ and zizmor authentication belong to research-repo-tools. The former unit tests are removed. Consumer tests retain native adapter checks, benchmark contracts, caller file-policy coverage, and recipe forwarding. +The superseded `criterion_measurements.py`, archive/worktree measurement engine, +summary retention writer, and their duplicate generic tests are removed. +`test_performance_workflow.py` exercises the installed package through consumer +policies, failure preservation, historical golden bytes, and offline replay. +The shared multi-series plotting extension is deferred, so the existing local +CSV/SVG/README adapter preserves figure format and baseline-phase peer labels. + Performance scripts also use the published shared Criterion parser and estimate validation, comparison arithmetic, exact-byte digest verification, safe archive extraction, document marker replacement, and multi-file publication transaction. @@ -415,16 +424,12 @@ replaces the report, evidence, archive index, and retained summaries. Shared recovery errors identify preserved backups if rollback fails. The remaining modules own la-stack's historical CSV/JSON schemas and fingerprints, -scientific eligibility, benchmark inventories, current-harness installation, -release selection policy, complete run retention, and multi-library plot layout. +scientific eligibility, benchmark inventories, Cargo commands, release adapters, +release selection policy, and multi-library plot layout. Existing evidence keeps its schema and digest framing. Generic parser and transaction regressions belong upstream; consumer tests check retained-schema round trips, complete output groups, failure preservation, and scientific policy. -The remaining workflow migration requires a published shared contract for common -harnesses, configurable measurement completeness, and immutable per-run retention. -[research-repo-tools#64](https://github.com/acgetchell/research-repo-tools/issues/64) -tracks that contract; -[la-stack#268](https://github.com/acgetchell/la-stack/issues/268) tracks adoption -after publication. +The published v0.1.8 contract owns common-harness installation, independent +gates, measurement freshness, configurable completeness, and immutable retention. See `docs/RELEASING.md` for the full release workflow. diff --git a/scripts/archive_performance.py b/scripts/archive_performance.py index df9447d..2525c68 100644 --- a/scripts/archive_performance.py +++ b/scripts/archive_performance.py @@ -1,2109 +1,243 @@ #!/usr/bin/env -S uv run --locked -"""Generate, retain, render, and promote benchmark comparison reports. - -Release performance outputs have distinct lifetimes: - - - ``target/bench-reports/performance.md`` is local scratch output for the - current machine and branch. - - ``target/bench-reports/performance.csv`` and the adjacent provenance JSON - are validated, reproducible performance-comparison inputs. - - ``docs/performance.md`` is the latest curated release-to-release comparison. - - ``docs/performance///`` preserves complete local - summaries and the selected report inputs across target cleanup. - - ``docs/archive/performance/*.md`` stores older curated comparisons. - -This script renders from a validated artifact reload, copies the result into -``docs/performance.md``, and archives the previous committed report under a -filename derived from the report metadata, such as ``v0.4.2-vs-v0.4.1.md``. -""" +"""Consumer release selection and publication over shared performance workflows.""" import argparse -import hashlib import json -import os -import platform import re import shutil import subprocess import sys import tempfile import tomllib -from collections.abc import Generator, Mapping -from contextlib import contextmanager from dataclasses import dataclass, replace -from datetime import UTC, datetime from pathlib import Path -from typing import Any, Literal, cast from research_repo_tools.archives import extract_archive +from research_repo_tools.common_measurement import measure_prepared_pair +from research_repo_tools.complete_runs import run_identity +from research_repo_tools.evidence import Evidence, serialize_evidence from research_repo_tools.files import replace_many -from research_repo_tools.process import ExecutableNotFoundError, cpu_description, format_exception_diagnostics, run_git_bytes - -from bench_compare import HOW_TO_UPDATE_SECTION, render_release_artifacts -from benchmark_contract import benchmark_contract_digest -from benchmark_process import run_git_command, run_safe_command -from benchmark_summaries import report_input_paths, resolve_report_paths, retained_outputs, summary_outputs -from performance_artifacts import ( - NO_API_COMPATIBILITY, - ArtifactPaths, - bundle_outputs, - ensure_distinct_paths, - load_bundle, - resolve_shared_harness_compatibility, -) - -_VERSION_RE = re.compile(r"^\*\*la-stack\*\* v(?P[^\s`]+)", re.MULTILINE) -_BASELINE_RE = re.compile(r"^Comparison against baseline \*\*(?P[^*]+)\*\*:", re.MULTILINE) -_SEMVER_IDENTIFIER_RE = r"(?:0|[1-9][0-9]*|[0-9A-Za-z-]*[A-Za-z-][0-9A-Za-z-]*)" -_TAG_RE = re.compile( - rf"^v?(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)" - rf"(?:-{_SEMVER_IDENTIFIER_RE}(?:\.{_SEMVER_IDENTIFIER_RE})*)?" - r"(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?$" +from research_repo_tools.measurement import resolve_revision +from research_repo_tools.process import ExecutableNotFoundError, format_exception_diagnostics, run_git_command, run_safe_command +from research_repo_tools.publication import plan_outputs, publish_publication +from research_repo_tools.release_assets import download_release_asset +from research_repo_tools.release_discovery import normalize_tag, stable_published_releases +from research_repo_tools.release_pairs import PairMode, ReleasePair, resolve_pair +from research_repo_tools.run_reports import load_run_report_plan +from research_repo_tools.worktrees import apply_snapshot, capture_snapshot, temporary_worktree + +from bench_compare import render_historical_assets, render_release_artifacts +from benchmark_process import find_project_root +from benchmark_summaries import resolve_report_paths +from performance_artifacts import ArtifactPaths, load_bundle, resolve_shared_harness_compatibility +from performance_runs import ( + ARCHIVE, + REPORT_CONFIG, + measurement_plan, + render_scientific_report, + retained_run, + scratch_paths, + validate_scientific_run, ) -_SEMVER_PARTS_RE = re.compile(r"^v?(?P0|[1-9][0-9]*)\.(?P0|[1-9][0-9]*)\.(?P0|[1-9][0-9]*)$") - -_DEFAULT_SOURCE = "target/bench-reports/performance.md" -_DEFAULT_ARTIFACT_CSV = "target/bench-reports/performance.csv" -_DEFAULT_ARTIFACT_PROVENANCE = "target/bench-reports/performance.provenance.json" -_DEFAULT_CURRENT = "docs/performance.md" -_DEFAULT_ARCHIVE_DIR = "docs/archive/performance" -_DEFAULT_SUITE = "all" -_DEFAULT_SCOPE = "release-signal" -_SUPPORTED_SUITES = ("all", "exact", "vs_linalg") -_SUPPORTED_SCOPES = ("release-signal", "all-benches") -_BENCH_TIMEOUT_SECONDS = 7200 -_COMMAND_TIMEOUT_SECONDS = 600 -_HOW_TO_UPDATE_RE = re.compile(r"(?ms)^## How to Update\n.*\Z") -# Cargo validates declared example paths even when only benchmark tests are selected. -_BENCHMARK_HARNESS_DIRS = ("benches", "examples") -_BENCHMARK_HARNESS_FILES = ( - ".config/nextest.toml", - "Cargo.toml", - "Cargo.lock", - "rust-toolchain.toml", - "justfile", - "tests/exact_bench_config.rs", -) -_BENCHMARK_HARNESS_METADATA = ".la-stack-benchmark-harness.json" -_BENCHMARK_INPUT_GATE = ("just", "test-bench-inputs") -_COMPARISON_LINT_CAP = "--cap-lints=warn" -type BaselineSource = Literal["local", "github-assets"] -type BenchmarkSuite = Literal["all", "exact", "vs_linalg"] -type ComparisonScope = Literal["release-signal", "all-benches"] @dataclass(frozen=True) class ReportId: - """Release-pair identity parsed from a benchmark report.""" - - current_tag: str - baseline_tag: str - - @property - def archive_name(self) -> str: - """Return the canonical archive filename for this report.""" - return f"{self.current_tag}-vs-{self.baseline_tag}.md" - - -@dataclass(frozen=True) -class PromotionRequest: - """Destinations and expected identity for one report promotion.""" - - current: Path - archive_dir: Path - expected: ReportId - source_path: Path | None = None - output: Path | None = None - reserved_paths: Mapping[str, Path] | None = None - retained_outputs: Mapping[Path, str] | None = None - - -@dataclass(frozen=True) -class GenerationConfig: - """Configuration for benchmark report generation in a temp worktree.""" - - repo_root: Path - current_tag: str - baseline_tag: str - worktree_ref: str - suite: BenchmarkSuite = "all" - scope: ComparisonScope = "release-signal" - apply_current_diff: bool = True - baseline_source: BaselineSource = "local" - - def __post_init__(self) -> None: - """Reject unsupported benchmark selections at construction time.""" - if self.suite not in _SUPPORTED_SUITES: - msg = f"unsupported benchmark suite: {self.suite}" - raise ValueError(msg) - if self.scope not in _SUPPORTED_SCOPES: - msg = f"unsupported comparison scope: {self.scope}" - raise ValueError(msg) - resolve_shared_harness_compatibility( - current=self.current_tag, - baseline=self.baseline_tag, - shared_harness_rational_inputs=False, - ) - - -@dataclass(frozen=True, slots=True) -class ToolRunOptions: - """Execution controls for one repository support command.""" - - timeout: int = _COMMAND_TIMEOUT_SECONDS - env: dict[str, str] | None = None - stream_output: bool = False - - -@dataclass(frozen=True) -class ResolvedArchiveRequest: - """Release pair and worktree ref resolved from CLI arguments.""" + """Legacy release labels, retained without changing historical artifact framing.""" current_tag: str baseline_tag: str - worktree_ref: str - tags_to_fetch: tuple[str, ...] = () - - -@dataclass(frozen=True) -class ArchiveRequestOptions: - """CLI options used to resolve release tags.""" - - current_tag: str | None - baseline_tag: str | None - published_latest: bool - infer_release: bool - current_vs_latest: bool - worktree_ref: str - repo_root: Path - - -@dataclass(frozen=True) -class ArchivePaths: - """Filesystem paths used by the archive CLI.""" - - source: Path - current: Path - output: Path - archive_dir: Path - artifacts: ArtifactPaths - - -@dataclass(frozen=True) -class GeneratedReport: - """Validated report and candidate outputs available until temporary cleanup.""" - - text: str - artifacts: ArtifactPaths - outputs: Mapping[Path, bytes] - - -@dataclass(frozen=True) -class ArchiveResult: - """Result and destination metadata for a completed archive operation.""" - - report_id: ReportId - action: Literal["output", "promote-generated", "promote-source", "promote-artifacts"] - - -@dataclass(frozen=True) -class PublishedRelease: - """Stable GitHub release metadata used to infer release pairs.""" - - tag: str - published_at: datetime - - -@dataclass(frozen=True) -class BaselineRun: - """Validated baseline run details needed for final provenance.""" - - commit: str - command: tuple[str, ...] - harness_sha256: str - git_clean: bool - source_state_sha256: str - api_compatibility: str | None - shared_harness_rational_inputs: bool = False - - -def normalize_tag(tag: str) -> str: - """Return *tag* with a leading ``v`` and no surrounding whitespace.""" - normalized = tag.strip() - if not normalized: - msg = "tag must not be empty" - raise ValueError(msg) - if not normalized.startswith("v"): - normalized = f"v{normalized}" - if not _TAG_RE.fullmatch(normalized): - msg = f"expected a semver tag like v0.4.2, got {tag!r}" - raise ValueError(msg) - return normalized - - -def parse_report_id(text: str) -> ReportId: - """Parse the current version and baseline tag from a benchmark report.""" - version_matches = list(_VERSION_RE.finditer(text)) - if not version_matches: - msg = "could not find la-stack version line in benchmark report" - raise ValueError(msg) - if len(version_matches) != 1: - msg = f"expected exactly one la-stack version line in benchmark report, found {len(version_matches)}" - raise ValueError(msg) - - baseline_matches = list(_BASELINE_RE.finditer(text)) - if not baseline_matches: - msg = "could not find comparison baseline line in benchmark report" - raise ValueError(msg) - if len(baseline_matches) != 1: - msg = f"expected exactly one comparison baseline line in benchmark report, found {len(baseline_matches)}" - raise ValueError(msg) - - version_match = version_matches[0] - baseline_match = baseline_matches[0] - - return ReportId( - current_tag=normalize_tag(version_match.group("version")), - baseline_tag=normalize_tag(baseline_match.group("baseline")), - ) - - -def _semver_sort_key(tag: str) -> tuple[int, int, int]: - match = _SEMVER_PARTS_RE.fullmatch(normalize_tag(tag)) - if match is None: - msg = f"expected a stable semver tag like v0.4.2, got {tag!r}" - raise ValueError(msg) - return (int(match.group("major")), int(match.group("minor")), int(match.group("patch"))) - - -def _parse_published_at(value: object, *, release_index: int) -> datetime: - """Parse one GitHub publication timestamp as an aware UTC datetime.""" - if not isinstance(value, str) or not value: - msg = f"GitHub release entry {release_index} has invalid publishedAt: {value!r}" - raise TypeError(msg) - try: - parsed = datetime.fromisoformat(value) - except ValueError as exc: - msg = f"GitHub release entry {release_index} has invalid publishedAt: {value!r}" - raise ValueError(msg) from exc - if parsed.tzinfo is None or parsed.utcoffset() is None: - msg = f"GitHub release entry {release_index} publishedAt must include a UTC offset: {value!r}" - raise ValueError(msg) - return parsed.astimezone(UTC) - - -def _stable_published_releases(releases: object) -> list[PublishedRelease]: - if not isinstance(releases, list): - msg = "expected GitHub release list to be a JSON array" - raise TypeError(msg) - - stable_releases: dict[str, PublishedRelease] = {} - for index, release in enumerate(releases): - if not isinstance(release, Mapping): - msg = f"GitHub release entry {index} is not a JSON object" - raise TypeError(msg) - release = cast("Mapping[str, Any]", release) - is_draft = release.get("isDraft") - is_prerelease = release.get("isPrerelease") - if not isinstance(is_draft, bool) or not isinstance(is_prerelease, bool): - msg = f"GitHub release entry {index} must contain boolean isDraft and isPrerelease fields" - raise TypeError(msg) - if is_draft or is_prerelease: - continue - tag_name = release.get("tagName") - if not isinstance(tag_name, str) or not tag_name: - msg = f"GitHub release entry {index} has invalid tagName: {tag_name!r}" - raise TypeError(msg) - published_at = _parse_published_at(release.get("publishedAt"), release_index=index) - try: - normalized = normalize_tag(tag_name) - _semver_sort_key(normalized) - except ValueError: - continue - stable_releases[normalized] = PublishedRelease(tag=normalized, published_at=published_at) - - return list(stable_releases.values()) - - -def _github_release_list(repo_root: Path) -> object: - command = [ - "release", - "list", - "--json", - "tagName,isDraft,isPrerelease,publishedAt", - "--limit", - "100", - ] - result = _run_tool_output( - "gh", - command, - cwd=repo_root, - ) - try: - return json.loads(result.stdout) - except json.JSONDecodeError as exc: - msg = "could not parse GitHub release list JSON" - raise RuntimeError(msg) from exc - - -def _published_stable_releases(repo_root: Path) -> list[PublishedRelease]: - return _stable_published_releases(_github_release_list(repo_root)) - - -def published_stable_release_tags(repo_root: Path) -> list[str]: - """Return stable published GitHub release tags for release tooling.""" - return [release.tag for release in _published_stable_releases(repo_root)] - - -def _latest_published_release(repo_root: Path) -> PublishedRelease: - stable_releases = _published_stable_releases(repo_root) - if not stable_releases: - msg = "expected at least one published stable semver release" - raise RuntimeError(msg) - return max(stable_releases, key=lambda release: release.published_at) - - -def _previous_release_from_list(stable_releases: list[PublishedRelease], current_tag: str) -> PublishedRelease: - current_key = _semver_sort_key(current_tag) - previous_releases = sorted( - (release for release in stable_releases if _semver_sort_key(release.tag) < current_key), - key=lambda release: _semver_sort_key(release.tag), - ) - if not previous_releases: - msg = f"could not find a previous stable semver release before {current_tag}" - raise RuntimeError(msg) - return previous_releases[-1] - - -def _normalize_worktree_ref_for_tag(worktree_ref: str, current_tag: str) -> str: - try: - normalized_ref = normalize_tag(worktree_ref) - except ValueError: - return worktree_ref - return current_tag if normalized_ref == current_tag else worktree_ref - - -def _current_package_tag(repo_root: Path) -> str: - cargo_toml = repo_root / "Cargo.toml" - data = tomllib.loads(_read_text(cargo_toml)) - package = data.get("package") - if not isinstance(package, dict): - msg = f"could not find [package] in {cargo_toml}" - raise TypeError(msg) - version = package.get("version") - if not isinstance(version, str): - msg = f"could not find package.version in {cargo_toml}" - raise TypeError(msg) - return normalize_tag(version) - - -def _published_release_pair(repo_root: Path) -> ReportId: - stable_releases = _published_stable_releases(repo_root) - if len(stable_releases) < 2: - msg = "expected at least two published stable semver releases" - raise RuntimeError(msg) - - current = max(stable_releases, key=lambda release: release.published_at) - previous = _previous_release_from_list(stable_releases, current.tag) - return ReportId(current_tag=current.tag, baseline_tag=previous.tag) - - -def _unpublished_package_release_pair(repo_root: Path) -> ReportId: - """Pair an unpublished package version with its newest prior release.""" - current = _current_package_tag(repo_root) - stable_releases = _published_stable_releases(repo_root) - if not stable_releases: - msg = "expected at least one published stable semver release" - raise RuntimeError(msg) - latest_version = max(stable_releases, key=lambda release: _semver_sort_key(release.tag)) - if _semver_sort_key(current) <= _semver_sort_key(latest_version.tag): - msg = ( - f"current package tag {current} must be newer than latest published stable release " - f"{latest_version.tag}; update the package version before running --infer-release, " - "or pass explicit tags for a repair" - ) - raise ValueError(msg) - previous = _previous_release_from_list(stable_releases, current) - return ReportId(current_tag=current, baseline_tag=previous.tag) - - -def _read_text(path: Path) -> str: - return path.read_text(encoding="utf-8") - - -def _normalize_how_to_update(text: str) -> str: - if _HOW_TO_UPDATE_RE.search(text): - return _HOW_TO_UPDATE_RE.sub(HOW_TO_UPDATE_SECTION, text) - return f"{text.rstrip()}\n\n{HOW_TO_UPDATE_SECTION}" - - -def _write_text(path: Path, text: str) -> None: - replace_many({path: text.encode("utf-8")}) - - -def _snapshot_regular_file(path: Path, *, label: str) -> bytes | None: - """Snapshot a mutation target while rejecting symlinks and non-files.""" - if path.is_symlink(): - msg = f"{label} must not be a symlink: {path}" - raise ValueError(msg) - if not path.exists(): - return None - if not path.is_file(): - msg = f"{label} must be a regular file: {path}" - raise ValueError(msg) - return path.read_bytes() -def _decode_text_snapshot(payload: bytes) -> str: - """Decode UTF-8 bytes with the universal-newline behavior of ``Path.read_text``.""" - return payload.decode("utf-8").replace("\r\n", "\n").replace("\r", "\n") - - -def _archive_readme(archive_dir: Path, planned: Path | None = None) -> str: - reports = {path.name for path in archive_dir.glob("*.md") if path.name != "README.md"} - if planned is not None: - reports.add(planned.name) - lines = [ - "# Archived Performance Reports", - "", - "Older release-to-release benchmark comparisons are archived here.", - "`docs/performance.md` contains the latest curated comparison.", - "", - ] - if reports: - lines.extend(f"- [{name.removesuffix('.md')}]({name})" for name in sorted(reports)) - else: - lines.append("- No archived performance reports yet.") - if (archive_dir / "studies").is_dir(): - lines.extend(("", "Completed optimization investigations are in [archived studies](studies/README.md).")) - return "\n".join(lines) + "\n" - - -def _format_command_failure(command: list[str], exc: subprocess.CalledProcessError) -> str: - header = f"command failed ({exc.returncode}): {' '.join(command)}" - details = format_exception_diagnostics(exc).partition("\n")[2] - return f"{header}\n{details}" if details else header - - -def _format_command_timeout(command: list[str], exc: subprocess.TimeoutExpired) -> str: - header = f"command timed out after {exc.timeout} seconds: {' '.join(command)}" - details = format_exception_diagnostics(exc).partition("\n")[2] - return f"{header}\n{details}" if details else header - - -def _format_command_start_failure(command: list[str], exc: BaseException) -> str: - return f"command could not start: {' '.join(command)}: {exc}" - - -def _run_git(args: list[str], *, cwd: Path, timeout: int = _COMMAND_TIMEOUT_SECONDS) -> None: - _run_git_output(args, cwd=cwd, timeout=timeout) - - -@contextmanager -def _temporary_detached_worktree( - *, - repo_root: Path, - worktree: Path, - revision: str, - label: str, -) -> Generator[Path]: - """Create and remove a detached worktree without masking primary failures.""" - _run_git(["worktree", "add", "--detach", str(worktree), revision], cwd=repo_root) - primary_error: BaseException | None = None +def select_pair(root: Path, mode: PairMode, current: str | None, baseline: str | None) -> ReleasePair: + """Use publication chronology while retaining la-stack release eligibility.""" + package = tomllib.loads((root / "Cargo.toml").read_text(encoding="utf-8"))["package"]["version"] + releases = () + if mode != "explicit": + response = run_safe_command( + "gh", + ["release", "list", "--repo", "acgetchell/la-stack", "--limit", "1000", "--json", "tagName,isDraft,isPrerelease,publishedAt"], + cwd=root, + ) + releases = stable_published_releases(json.loads(response.stdout)) + pair = resolve_pair(mode, package_tag=package, releases=releases, current=current, baseline=baseline, order="published") + resolve_shared_harness_compatibility(current=pair.current, baseline=pair.baseline, shared_harness_rational_inputs=True) + return pair + + +def measure(root: Path, pair: ReleasePair, suite: str, scope: str) -> Evidence: + """Capture tracked changes and delegate isolated phases to the published engine.""" + version = tomllib.loads((root / "Cargo.toml").read_text(encoding="utf-8"))["package"]["version"] + if pair.current != normalize_tag(version): + msg = "current release label must match the measured checkout package" + raise ValueError(msg) + snapshot = replace(capture_snapshot(root), untracked=()) + # The existing consumer contract deliberately excludes untracked files. + run_git_command(["fetch", "origin", f"refs/tags/{pair.baseline}:refs/tags/{pair.baseline}"], cwd=root) + baseline_revision = resolve_revision(root, pair.baseline) + parent = Path(tempfile.mkdtemp(prefix="la-stack-performance-")) try: - yield worktree - except BaseException as exc: - primary_error = exc - raise - finally: - try: - _run_git(["worktree", "remove", "--force", str(worktree)], cwd=repo_root) - except RuntimeError as cleanup_error: - if primary_error is None: - msg = f"failed to remove {label}: {cleanup_error}" - raise RuntimeError(msg) from cleanup_error - msg = f"operation failed ({primary_error}); additionally failed to remove {label}: {cleanup_error}" - raise RuntimeError(msg) from primary_error - - -def _run_git_output( - args: list[str], - *, - cwd: Path, - timeout: int = _COMMAND_TIMEOUT_SECONDS, - env: dict[str, str] | None = None, -) -> str: - try: - return run_git_command(args, cwd=cwd, timeout=timeout, env=env).stdout - except subprocess.CalledProcessError as exc: - raise RuntimeError(_format_command_failure(["git", *args], exc)) from exc - except subprocess.TimeoutExpired as exc: - raise RuntimeError(_format_command_timeout(["git", *args], exc)) from exc - except (ExecutableNotFoundError, OSError) as exc: - raise RuntimeError(_format_command_start_failure(["git", *args], exc)) from exc - - -def _fetch_release_tags(*, repo_root: Path, tags: list[str]) -> None: - refspecs = [f"refs/tags/{tag}:refs/tags/{tag}" for tag in tags] - _run_git(["fetch", "origin", *refspecs], cwd=repo_root) - - -def _run_tool_output( - command: str, - args: list[str], - *, - cwd: Path, - options: ToolRunOptions | None = None, -) -> subprocess.CompletedProcess[str]: - """Run a support command and normalize all expected launch failures.""" - resolved_options = options or ToolRunOptions() - try: - return run_safe_command( - command, - args, - cwd=cwd, - timeout=resolved_options.timeout, - env=resolved_options.env, - capture_output=not resolved_options.stream_output, - ) - except subprocess.CalledProcessError as exc: - raise RuntimeError(_format_command_failure([command, *args], exc)) from exc - except subprocess.TimeoutExpired as exc: - raise RuntimeError(_format_command_timeout([command, *args], exc)) from exc - except (ExecutableNotFoundError, OSError) as exc: - raise RuntimeError(_format_command_start_failure([command, *args], exc)) from exc - - -def _run_tool( - command: str, - args: list[str], - *, - cwd: Path, - options: ToolRunOptions | None = None, -) -> None: - _run_tool_output( - command, - args, - cwd=cwd, - options=options, - ) - - -def _progress(message: str) -> None: - """Write one immediately visible workflow progress message.""" - print(f"[performance] {message}", file=sys.stderr, flush=True) - - -def _run_benchmark_input_gate(checkout: Path, *, env: dict[str, str] | None = None) -> None: - """Run the shared deterministic benchmark-fixture correctness gate.""" - _progress(f"validating benchmark inputs in {checkout.name}") - _run_tool( - _BENCHMARK_INPUT_GATE[0], - list(_BENCHMARK_INPUT_GATE[1:]), - cwd=checkout, - options=ToolRunOptions(env=env, stream_output=True), - ) - - -def _sha256_file(path: Path) -> str: - """Return a lowercase SHA-256 digest for a required file.""" - if not path.is_file(): - msg = f"required provenance file is missing: {path}" - raise FileNotFoundError(msg) - return hashlib.sha256(path.read_bytes()).hexdigest() - - -def _checkout_commit(checkout: Path) -> str: - """Return the full commit for a checkout, or an explicit unavailable label.""" - commit = _run_git_output(["--no-pager", "rev-parse", "HEAD"], cwd=checkout).strip() - return commit or "unavailable" - - -def _git_clean(checkout: Path) -> bool: - """Return whether Git reports a completely clean checkout.""" - status = _run_git_output( - ["--no-pager", "status", "--porcelain=v1", "--untracked-files=all"], - cwd=checkout, - ) - return not status.strip() - - -def _source_state_digest(checkout: Path) -> str: - """Hash measured library source content independent of commit cleanliness.""" - source_dir = checkout / "src" - if not source_dir.is_dir(): - msg = f"measured library source directory is missing: {source_dir}" - raise FileNotFoundError(msg) - files = sorted( - (path for path in source_dir.rglob("*") if path.is_file()), - key=lambda path: path.relative_to(checkout).as_posix(), - ) - if not files: - msg = f"measured library source directory contains no files: {source_dir}" - raise FileNotFoundError(msg) - digest = hashlib.sha256() - for path in files: - relative = path.relative_to(checkout).as_posix().encode() - payload = path.read_bytes() - digest.update(len(relative).to_bytes(8, "big")) - digest.update(relative) - digest.update(len(payload).to_bytes(8, "big")) - digest.update(payload) - return digest.hexdigest() - - -def _rustc_version(checkout: Path) -> str: - """Return one-line rustc version provenance for the active benchmark toolchain.""" - result = _run_tool_output( - "rustc", - ["--version"], - cwd=checkout, - options=ToolRunOptions(env=_benchmark_env(checkout)), - ) - version = result.stdout.strip() - return version or "unavailable" - - -def _environment_metadata(checkout: Path, *, harness_sha256: str) -> dict[str, object]: - """Capture deterministic machine, toolchain, revision, and lock provenance.""" - cpu = cpu_description() - os_description = " ".join(part for part in (platform.system(), platform.release(), platform.machine()) if part).strip() - return { - "benchmark_contract_sha256": benchmark_contract_digest(checkout), - "cargo_lock_sha256": _sha256_file(checkout / "Cargo.lock"), - "commit": _checkout_commit(checkout), - "correctness_gate": "passed", - "cpu": cpu, - "git_clean": _git_clean(checkout), - "harness_sha256": harness_sha256, - "os": os_description or "unavailable", - "rustc": _rustc_version(checkout), - "source_state_sha256": _source_state_digest(checkout), - } - - -def _require_recorded_measurement_cpu() -> str: - """Return the CPU model required for reproducible release measurements.""" - cpu = cpu_description() - if cpu.casefold() == "unavailable": - msg = ( - "cannot generate release performance measurements because the CPU model is unavailable; " - "raw local benchmarks may still be run with the ordinary Criterion recipes" - ) - raise RuntimeError(msg) - return cpu - - -def _criterion_dependency_version(checkout: Path) -> str: - """Return the resolved Criterion version, falling back to its manifest requirement.""" - lock_data = tomllib.loads(_read_text(checkout / "Cargo.lock")) - packages = lock_data.get("package") - if isinstance(packages, list): - for package in packages: - if isinstance(package, dict) and package.get("name") == "criterion": - version = package.get("version") - if isinstance(version, str) and version: - return version - - manifest = tomllib.loads(_read_text(checkout / "Cargo.toml")) - for section in ("dev-dependencies", "dependencies", "build-dependencies"): - dependencies = manifest.get(section) - if not isinstance(dependencies, dict): - continue - criterion = dependencies.get("criterion") - if isinstance(criterion, str) and criterion: - return f"manifest requirement {criterion}" - if isinstance(criterion, dict): - version = criterion.get("version") - if isinstance(version, str) and version: - return f"manifest requirement {version}" - msg = f"Criterion dependency version is unavailable in {checkout / 'Cargo.lock'} and {checkout / 'Cargo.toml'}" - raise ValueError(msg) - - -def _criterion_metadata( - *, - worktree: Path, - config: GenerationConfig, - baseline_command: tuple[str, ...], - current_command: tuple[str, ...], -) -> dict[str, object]: - """Record the exact Criterion selection and timing commands.""" - return { - "baseline_command": list(baseline_command), - "current_command": list(current_command), - "criterion_version": _criterion_dependency_version(worktree), - "sample": "new", - "scope": config.scope, - "statistic": "median", - "suite": config.suite, - } - - -def _write_local_run_provenance( - *, - worktree: Path, - config: GenerationConfig, - baseline_run: BaselineRun, - current_command: tuple[str, ...], -) -> None: - """Tie locally generated samples to their shared harness and environment.""" - publication = _environment_metadata(worktree, harness_sha256=baseline_run.harness_sha256) - measurement = { - "baseline_api_compatibility": baseline_run.api_compatibility or "none", - "baseline_commit": baseline_run.commit, - "benchmark_contract_sha256": publication["benchmark_contract_sha256"], - "cargo_lock_sha256": publication["cargo_lock_sha256"], - "cpu": publication["cpu"], - "current_commit": publication["commit"], - "current_git_clean": publication["git_clean"], - "current_source_state_sha256": publication["source_state_sha256"], - "harness_sha256": baseline_run.harness_sha256, - "os": publication["os"], - "rustc": publication["rustc"], - "baseline_git_clean": baseline_run.git_clean, - "baseline_source_state_sha256": baseline_run.source_state_sha256, - "status": "recorded", - } - metadata = { - "baseline": config.baseline_tag, - "current": config.current_tag, - "criterion": _criterion_metadata( - worktree=worktree, - config=config, - baseline_command=baseline_run.command, - current_command=current_command, - ), - "measurement": measurement, - "mode": "shared-current-harness", - "publication": publication, - "schema": 2, - "validation": { - "baseline_api_compatibility": baseline_run.api_compatibility or "none", - "baseline_commit": baseline_run.commit, - "baseline_git_clean": baseline_run.git_clean, - "baseline_revision": "passed", - "baseline_source_state_sha256": baseline_run.source_state_sha256, - "command": list(_BENCHMARK_INPUT_GATE), - "current_commit": publication["commit"], - "current_git_clean": publication["git_clean"], - "current_revision": "passed", - "current_source_state_sha256": publication["source_state_sha256"], - "harness": "shared-current", - "shared_harness_rational_inputs": baseline_run.shared_harness_rational_inputs, - }, - } - _write_text( - worktree / "target" / "criterion" / _BENCHMARK_HARNESS_METADATA, - json.dumps(metadata, indent=2, sort_keys=True) + "\n", - ) - - -def _write_historical_asset_provenance( - *, - worktree: Path, - config: GenerationConfig, - baseline_run: BaselineRun, -) -> None: - """Record validation while explicitly declining to invent historical timing metadata.""" - publication = _environment_metadata(worktree, harness_sha256=baseline_run.harness_sha256) - metadata = { - "baseline": config.baseline_tag, - "current": config.current_tag, - "criterion": _criterion_metadata( - worktree=worktree, - config=config, - baseline_command=("historical-release-asset", config.baseline_tag), - current_command=("historical-release-asset", config.current_tag), - ), - "measurement": { - "reason": "the downloaded release assets do not contain schema-2 measurement-environment provenance", - "status": "unavailable", - }, - "mode": "historical-assets", - "publication": publication, - "schema": 2, - "validation": { - "baseline_api_compatibility": baseline_run.api_compatibility or "none", - "baseline_commit": baseline_run.commit, - "baseline_git_clean": baseline_run.git_clean, - "baseline_revision": "passed", - "baseline_source_state_sha256": baseline_run.source_state_sha256, - "command": list(_BENCHMARK_INPUT_GATE), - "current_commit": publication["commit"], - "current_git_clean": publication["git_clean"], - "current_revision": "passed", - "current_source_state_sha256": publication["source_state_sha256"], - "harness": "shared-current", - "shared_harness_rational_inputs": baseline_run.shared_harness_rational_inputs, - }, - } - _write_text( - worktree / "target" / "criterion" / _BENCHMARK_HARNESS_METADATA, - json.dumps(metadata, indent=2, sort_keys=True) + "\n", - ) - - -def _current_rust_toolchain(checkout: Path) -> str | None: - rust_toolchain = checkout / "rust-toolchain.toml" - if not rust_toolchain.exists(): - return None - data = tomllib.loads(_read_text(rust_toolchain)) - toolchain = data.get("toolchain") - if not isinstance(toolchain, dict): - return None - channel = toolchain.get("channel") - return channel if isinstance(channel, str) else None - - -def _benchmark_env(checkout: Path) -> dict[str, str] | None: - if "RUSTUP_TOOLCHAIN" in os.environ: - return None - toolchain = _current_rust_toolchain(checkout) - if toolchain is None: - return None - env = os.environ.copy() - env["RUSTUP_TOOLCHAIN"] = toolchain - return env - - -def _append_rustflag(env: dict[str, str], flag: str) -> None: - """Append one rustc flag without discarding caller-selected codegen flags.""" - encoded = env.get("CARGO_ENCODED_RUSTFLAGS") - if encoded is not None: - env["CARGO_ENCODED_RUSTFLAGS"] = "\x1f".join(part for part in (encoded, flag) if part) - return - - rustflags = env.get("RUSTFLAGS", "").strip() - env["RUSTFLAGS"] = f"{rustflags} {flag}".strip() - - -def _shared_harness_rational_inputs(checkout: Path) -> bool: - """Return whether the installed exact benchmark harness emits rational-input groups.""" - exact_bench = checkout / "benches" / "exact.rs" - if not exact_bench.is_file(): - return False - source = _read_text(exact_bench) - return "fn bench_rational_input<" in source and "rational_input_d{D}" in source - - -def _baseline_api_compatibility( - *, - current_tag: str, - baseline_tag: str, - shared_harness_rational_inputs: bool, -) -> str | None: - """Return the baseline adapter selected by the shared compatibility resolver.""" - resolved = resolve_shared_harness_compatibility( - current=normalize_tag(current_tag), - baseline=normalize_tag(baseline_tag), - shared_harness_rational_inputs=shared_harness_rational_inputs, - ) - if resolved.baseline_api_compatibility == NO_API_COMPATIBILITY: - return None - return resolved.baseline_api_compatibility - - -def _comparison_benchmark_env(checkout: Path, *, api_compatibility: str | None = None) -> dict[str, str]: - """Build a comparable benchmark environment for current or historical code.""" - env = _benchmark_env(checkout) - if env is None: - env = os.environ.copy() - - # The shared current manifest can enable lints unknown to historical source. - # Cap diagnostics for both revisions so lint-policy drift cannot prevent a - # performance comparison; the cap changes diagnostics, not code generation. - _append_rustflag(env, _COMPARISON_LINT_CAP) - - if api_compatibility is not None: - _append_rustflag(env, f"--cfg={api_compatibility}") - return env - - -def _download_release_baseline(*, baseline_tag: str, download_dir: Path, repo_root: Path) -> Path: - artifact = download_dir / f"la-stack-{baseline_tag}-criterion-baseline.tar.gz" - _run_tool( - "gh", - [ - "release", - "download", - baseline_tag, - "--pattern", - artifact.name, - "--dir", - str(download_dir), - ], - cwd=repo_root, - ) - if not artifact.exists(): - msg = f"release baseline asset was not downloaded: {artifact}" - raise FileNotFoundError(msg) - return artifact - - -def _copy_criterion_sample(*, criterion_dir: Path, source_sample: str, target_sample: str) -> None: - copied = 0 - for source in list(criterion_dir.rglob(source_sample)): - if not source.is_dir() or not (source / "estimates.json").exists(): - continue - target = source.parent / target_sample - if target.exists(): - shutil.rmtree(target) - shutil.copytree(source, target) - copied += 1 - if copied == 0: - msg = f"could not find Criterion sample {source_sample!r} under {criterion_dir}" - raise FileNotFoundError(msg) - - -def _selected_criterion_groups(criterion_dir: Path, *, suite: str) -> list[Path]: - """Return selected Criterion group directories in deterministic order.""" - groups: list[Path] = [] - if not criterion_dir.is_dir(): - return groups - for child in criterion_dir.iterdir(): - if not child.is_dir(): - continue - is_exact = child.name.startswith(("exact_", "rational_input_", "canonical_conversion_", "det4_diagnostic_")) - is_vs_linalg = re.fullmatch(r"d[0-9]+", child.name) is not None or child.name.startswith("lu_solve_") - if suite == "all" or (suite == "exact" and is_exact) or (suite == "vs_linalg" and is_vs_linalg): - groups.append(child) - return sorted(groups, key=lambda path: path.name) - - -def _purge_criterion_new_samples(*, criterion_dir: Path, suite: str) -> list[Path]: - """Remove stale selected-suite `new` samples while preserving named baselines.""" - removed: list[Path] = [] - for group in _selected_criterion_groups(criterion_dir, suite=suite): - for sample in sorted(group.rglob("new")): - if sample.is_dir(): - shutil.rmtree(sample) - removed.append(sample) - return removed - - -def _benchmark_harness_files(checkout: Path) -> list[Path]: - """Return every file that defines the comparable benchmark harness.""" - files: list[Path] = [] - for relative in _BENCHMARK_HARNESS_FILES: - path = checkout / relative - if not path.is_file(): - msg = f"benchmark harness file is missing: {path}" - raise FileNotFoundError(msg) - files.append(path) - for relative in _BENCHMARK_HARNESS_DIRS: - directory = checkout / relative - if not directory.is_dir(): - msg = f"benchmark harness directory is missing: {directory}" - raise FileNotFoundError(msg) - files.extend(path for path in directory.rglob("*") if path.is_file()) - return sorted(files, key=lambda path: path.relative_to(checkout).as_posix()) - - -def _benchmark_harness_digest(checkout: Path) -> str: - """Hash benchmark sources, recipes, dependency resolution, and toolchain.""" - digest = hashlib.sha256() - for path in _benchmark_harness_files(checkout): - relative = path.relative_to(checkout).as_posix().encode() - payload = path.read_bytes() - digest.update(len(relative).to_bytes(8, "big")) - digest.update(relative) - digest.update(len(payload).to_bytes(8, "big")) - digest.update(payload) - return digest.hexdigest() - - -def _install_shared_benchmark_harness(*, source: Path, destination: Path) -> str: - """Replace a baseline checkout's harness with the current harness.""" - source_files = _benchmark_harness_files(source) - # The comparison test moved into its own package; historical copies would - # otherwise still compile as a library test without the peer dependencies. - if (source / "benches/comparison/Cargo.toml").is_file(): - (destination / "tests/vs_linalg_inputs.rs").unlink(missing_ok=True) - for relative in _BENCHMARK_HARNESS_DIRS: - source_dir = source / relative - destination_dir = destination / relative - if destination_dir.exists(): - shutil.rmtree(destination_dir) - shutil.copytree(source_dir, destination_dir) - for relative in _BENCHMARK_HARNESS_FILES: - destination_file = destination / relative - destination_file.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(source / relative, destination_file) - - expected = _benchmark_harness_digest(source) - actual = _benchmark_harness_digest(destination) - if actual != expected: - msg = f"shared benchmark harness copy changed content: expected {expected}, found {actual}" - raise RuntimeError(msg) - if not source_files: - msg = "benchmark harness unexpectedly contained no files" - raise RuntimeError(msg) - return expected - - -def _has_suite_aware_baseline_recipe(worktree: Path) -> bool: - justfile = worktree / "justfile" - return justfile.exists() and re.search(r'(?m)^bench-save-baseline\s+tag\s+suite(?:=|"|:|\s)', _read_text(justfile)) is not None - - -def _baseline_tool_args(*, baseline_tag: str, suite: str, baseline_worktree: Path) -> tuple[str, list[str]]: - if suite == _DEFAULT_SUITE: - return ("just", ["bench-save-baseline", baseline_tag]) - if _has_suite_aware_baseline_recipe(baseline_worktree): - return ("just", ["bench-save-baseline", baseline_tag, suite]) - match suite: - case "exact": - return ("cargo", ["bench", "--locked", "--features", "bench,exact", "--bench", "exact", "--", "--save-baseline", baseline_tag]) - case "vs_linalg": - return ("cargo", ["bench", "--locked", "--features", "bench", "--bench", "vs_linalg", "--", "--save-baseline", baseline_tag]) - case _: - msg = f"unsupported benchmark suite: {suite}" - raise ValueError(msg) - - -def _latest_recipe_args(*, suite: str) -> list[str]: - match suite: - case "all": - return ["bench-latest"] - case "exact": - return ["bench-exact"] - case "vs_linalg": - return ["bench-vs-linalg-la-stack"] - case _: - msg = f"unsupported benchmark suite: {suite}" - raise ValueError(msg) - - -def _fallback_current_command(*, suite: str) -> tuple[str, ...]: - """Return the Cargo command used when current benchmark recipes are unavailable.""" - match suite: - case "all": - return ("cargo", "bench", "--locked", "--features", "bench,exact") - case "exact": - return ("cargo", "bench", "--locked", "--features", "bench,exact", "--bench", "exact") - case "vs_linalg": - return ("cargo", "bench", "--locked", "--features", "bench", "--bench", "vs_linalg") - case _: - msg = f"unsupported benchmark suite: {suite}" - raise ValueError(msg) - - -def _generate_release_baseline( # noqa: PLR0913 - *, - current_tag: str, - baseline_tag: str, - suite: str, - repo_root: Path, - target_worktree: Path, - tmp_dir: Path, -) -> BaselineRun: - baseline_worktree = tmp_dir / "baseline-worktree" - with _temporary_detached_worktree( - repo_root=repo_root, - worktree=baseline_worktree, - revision=baseline_tag, - label="baseline worktree", - ): - harness_sha256 = _install_shared_benchmark_harness( - source=target_worktree, - destination=baseline_worktree, - ) - baseline_command, baseline_args = _baseline_tool_args( - baseline_tag=baseline_tag, - suite=suite, - baseline_worktree=baseline_worktree, - ) - shared_harness_rational_inputs = _shared_harness_rational_inputs(target_worktree) - api_compatibility = _baseline_api_compatibility( - current_tag=current_tag, - baseline_tag=baseline_tag, - shared_harness_rational_inputs=shared_harness_rational_inputs, - ) - benchmark_env = _comparison_benchmark_env(repo_root, api_compatibility=api_compatibility) - _run_benchmark_input_gate(baseline_worktree, env=benchmark_env) - peers = "; includes nalgebra/faer measurements" if suite != "exact" else "" - _progress(f"running {suite} baseline benchmarks for {baseline_tag}{peers}") - _run_tool( - baseline_command, - baseline_args, - cwd=baseline_worktree, - options=ToolRunOptions( - timeout=_BENCH_TIMEOUT_SECONDS, - env=benchmark_env, - stream_output=True, - ), - ) - _progress(f"completed {suite} baseline benchmarks for {baseline_tag}") - baseline_criterion = baseline_worktree / "target" / "criterion" - if not baseline_criterion.is_dir(): - msg = f"generated baseline Criterion results were not found: {baseline_criterion}" - raise FileNotFoundError(msg) - target_criterion = target_worktree / "target" / "criterion" - target_criterion.parent.mkdir(parents=True, exist_ok=True) - shutil.copytree(baseline_criterion, target_criterion, dirs_exist_ok=True) - return BaselineRun( - commit=_checkout_commit(baseline_worktree), - command=(baseline_command, *baseline_args), - harness_sha256=harness_sha256, - git_clean=_git_clean(baseline_worktree), - source_state_sha256=_source_state_digest(baseline_worktree), - api_compatibility=api_compatibility, - shared_harness_rational_inputs=shared_harness_rational_inputs, - ) - - -def _prepare_local_release_baseline( # noqa: PLR0913 - *, - current_tag: str, - baseline_tag: str, - suite: str, - repo_root: Path, - target_worktree: Path, - tmp_dir: Path, -) -> BaselineRun: - return _generate_release_baseline( - current_tag=current_tag, - baseline_tag=baseline_tag, - suite=suite, - repo_root=repo_root, - target_worktree=target_worktree, - tmp_dir=tmp_dir, - ) - - -def _validate_release_revision( - *, - current_tag: str, - revision: str, - repo_root: Path, - harness_source: Path, - tmp_dir: Path, -) -> BaselineRun: - """Validate one historical revision with the shared current fixture harness.""" - validation_worktree = tmp_dir / "baseline-validation-worktree" - with _temporary_detached_worktree( - repo_root=repo_root, - worktree=validation_worktree, - revision=revision, - label="baseline validation worktree", - ): - harness_sha256 = _install_shared_benchmark_harness( - source=harness_source, - destination=validation_worktree, - ) - shared_harness_rational_inputs = _shared_harness_rational_inputs(harness_source) - api_compatibility = _baseline_api_compatibility( - current_tag=current_tag, - baseline_tag=revision, - shared_harness_rational_inputs=shared_harness_rational_inputs, - ) - _run_benchmark_input_gate( - validation_worktree, - env=_comparison_benchmark_env(repo_root, api_compatibility=api_compatibility), - ) - return BaselineRun( - commit=_checkout_commit(validation_worktree), - command=("historical-release-asset", revision), - harness_sha256=harness_sha256, - git_clean=_git_clean(validation_worktree), - source_state_sha256=_source_state_digest(validation_worktree), - api_compatibility=api_compatibility, - shared_harness_rational_inputs=shared_harness_rational_inputs, - ) - - -def _prepare_github_release_assets(*, current_tag: str, baseline_tag: str, repo_root: Path, target_worktree: Path, tmp_dir: Path) -> None: - baseline_archive = _download_release_baseline( - baseline_tag=baseline_tag, - download_dir=tmp_dir, - repo_root=repo_root, - ) - current_archive = _download_release_baseline( - baseline_tag=current_tag, - download_dir=tmp_dir, - repo_root=repo_root, - ) - target_dir = target_worktree / "target" - for phase, archive in (("baseline", baseline_archive), ("current", current_archive)): - extracted = tmp_dir / f"{phase}-extracted" - extract_archive(archive, extracted) - # Only this disposable checkout can observe partial assembly. Keep the - # expanded archive off the heap while retaining shared path validation. - for path in sorted(extracted.rglob("*")): - if path.is_file(): - replace_many({target_dir / path.relative_to(extracted): path.read_bytes()}) - # Published artifacts retain their release-specific harnesses. Never let an - # embedded or stale local manifest claim that these samples shared one. - metadata_path = target_dir / "criterion" / _BENCHMARK_HARNESS_METADATA - if metadata_path.is_symlink() or metadata_path.is_file(): - metadata_path.unlink() - elif metadata_path.exists(): - msg = f"historical benchmark harness provenance path is not a file: {metadata_path}" - raise ValueError(msg) - _copy_criterion_sample(criterion_dir=target_dir / "criterion", source_sample=current_tag, target_sample="new") - - -def _apply_current_diff_to_worktree(*, repo_root: Path, worktree: Path) -> None: - # Diff HEAD directly so staged and unstaged tracked changes participate, - # while unrelated untracked files stay outside the benchmark worktree. - with tempfile.TemporaryDirectory(prefix="la-stack-current-tree-index-") as tmp: - temporary_dir = Path(tmp) - patch_path = temporary_dir / "current-tree.patch" - _run_git_output( - ["diff", "--binary", f"--output={patch_path}", "HEAD", "--", "."], - cwd=repo_root, - ) - diff = patch_path.read_bytes() - if diff.strip(): - try: - run_git_bytes(["apply", "--binary"], input=diff, cwd=worktree) - except subprocess.CalledProcessError as exc: - raise RuntimeError(_format_command_failure(["git", "apply", "--binary"], exc)) from exc - except subprocess.TimeoutExpired as exc: - raise RuntimeError(_format_command_timeout(["git", "apply", "--binary"], exc)) from exc - except (ExecutableNotFoundError, OSError) as exc: - raise RuntimeError(_format_command_start_failure(["git", "apply", "--binary"], exc)) from exc - - -def _has_current_release_signal_tooling(worktree: Path) -> bool: - justfile = worktree / "justfile" - bench_compare = worktree / "scripts" / "bench_compare.py" - if not justfile.exists() or not bench_compare.exists(): - return False - - justfile_text = _read_text(justfile) - bench_compare_text = _read_text(bench_compare) - return re.search(r"(?m)^bench-latest(?:[ :]|$)", justfile_text) is not None and '"--suite"' in bench_compare_text and '"--scope"' in bench_compare_text - - -def _render_report( - *, - worktree: Path, - report: Path, - artifacts: ArtifactPaths, - config: GenerationConfig, -) -> None: - """Render Markdown and export validated comparison artifacts.""" - command = [ - "run", - "--locked", - "--project", - str(config.repo_root), - "bench-compare", - config.baseline_tag, - "--repo-root", - str(worktree), - "--criterion-dir", - str(worktree / "target" / "criterion"), - "--suite", - config.suite, - "--scope", - config.scope, - ] - command.extend( - [ - "--csv-output", - str(artifacts.csv), - "--provenance-output", - str(artifacts.provenance), - ] - ) - command.extend(["--output", str(report)]) - _run_tool( - "uv", - command, - cwd=config.repo_root, - ) - - -def _run_benchmarks_and_render_report( - *, - worktree: Path, - report: Path, - artifacts: ArtifactPaths, - config: GenerationConfig, - baseline_run: BaselineRun, -) -> None: - benchmark_env = _comparison_benchmark_env(config.repo_root) - _run_benchmark_input_gate(worktree, env=benchmark_env) - _purge_criterion_new_samples( - criterion_dir=worktree / "target" / "criterion", - suite=config.suite, - ) - peers = "" - if _has_current_release_signal_tooling(worktree): - current_command = ("just", *_latest_recipe_args(suite=config.suite)) - if config.suite != "exact": - peers = "; la-stack only, reusing baseline nalgebra/faer measurements" - else: - current_command = _fallback_current_command(suite=config.suite) - _progress(f"running current {config.suite} benchmarks for {config.current_tag}{peers}") - _run_tool( - current_command[0], - list(current_command[1:]), - cwd=worktree, - options=ToolRunOptions( - timeout=_BENCH_TIMEOUT_SECONDS, - env=benchmark_env, - stream_output=True, - ), - ) - _progress(f"completed current {config.suite} benchmarks") - _write_local_run_provenance( - worktree=worktree, - config=config, - baseline_run=baseline_run, - current_command=current_command, - ) - _render_report(worktree=worktree, report=report, artifacts=artifacts, config=config) - - -def _default_artifact_paths(repo_root: Path) -> ArtifactPaths: - """Return the canonical retained comparison-artifact pair.""" - return ArtifactPaths( - csv=repo_root / _DEFAULT_ARTIFACT_CSV, - provenance=repo_root / _DEFAULT_ARTIFACT_PROVENANCE, - ) - - -@contextmanager -def _generated_report_in_temp_worktree( - *, - config: GenerationConfig, - published_artifacts: ArtifactPaths, -) -> Generator[GeneratedReport]: - """Generate and expose a report before its temporary worktree is removed.""" - if config.baseline_source == "local": - _require_recorded_measurement_cpu() - with tempfile.TemporaryDirectory(prefix="la-stack-performance-") as tmp: - tmp_dir = Path(tmp) - worktree = tmp_dir / "worktree" - report = tmp_dir / f"{config.current_tag}-vs-{config.baseline_tag}.md" - temporary_artifacts = ArtifactPaths( - csv=tmp_dir / "performance.csv", - provenance=tmp_dir / "performance.provenance.json", - ) - - with _temporary_detached_worktree( - repo_root=config.repo_root, - worktree=worktree, - revision=config.worktree_ref, - label="temporary worktree", + with ( + temporary_worktree(root, parent / "baseline", baseline_revision, allow_git_mutations=True) as baseline, + temporary_worktree(root, parent / "current", snapshot.revision, allow_git_mutations=True) as current, ): - if config.apply_current_diff: - _progress("applying staged and unstaged tracked changes; untracked files are excluded") - _apply_current_diff_to_worktree(repo_root=config.repo_root, worktree=worktree) - if config.baseline_source == "github-assets": - _prepare_github_release_assets( - current_tag=config.current_tag, - baseline_tag=config.baseline_tag, - repo_root=config.repo_root, - target_worktree=worktree, - tmp_dir=tmp_dir, - ) - baseline_run = _validate_release_revision( - current_tag=config.current_tag, - revision=config.baseline_tag, - repo_root=config.repo_root, - harness_source=worktree, - tmp_dir=tmp_dir, - ) - _run_benchmark_input_gate( - worktree, - env=_comparison_benchmark_env(config.repo_root), - ) - _write_historical_asset_provenance( - worktree=worktree, - config=config, - baseline_run=baseline_run, - ) - _render_report( - worktree=worktree, - report=report, - artifacts=temporary_artifacts, - config=config, - ) - else: - baseline_run = _prepare_local_release_baseline( - current_tag=config.current_tag, - baseline_tag=config.baseline_tag, - suite=config.suite, - repo_root=config.repo_root, - target_worktree=worktree, - tmp_dir=tmp_dir, - ) - _run_benchmarks_and_render_report( - worktree=worktree, - report=report, - artifacts=temporary_artifacts, - config=config, - baseline_run=baseline_run, - ) - report_text = _read_text(report) - if render_release_artifacts(temporary_artifacts) != report_text: - msg = "retained performance-comparison artifacts did not reproduce the generated Markdown" + apply_snapshot(current, snapshot) + # Consumer API-layout adapter: the comparison test moved to its own + # workspace member and the historical root copy must not compile. + (baseline / "tests/vs_linalg_inputs.rs").unlink(missing_ok=True) + plan = measurement_plan(current, pair, suite, scope, parent / "listing") + evidence = measure_prepared_pair(baseline, current, plan, pair, working_tree=True) + if replace(capture_snapshot(root), untracked=()) != snapshot: + msg = "source working tree changed while measurement ran" raise ValueError(msg) - bundle = load_bundle(temporary_artifacts) - summaries = {} - if config.baseline_source == "local": - temporary_summaries = summary_outputs(worktree / "target/criterion", config.baseline_tag, temporary_artifacts) - replace_many({path: text.encode("utf-8") for path, text in temporary_summaries.items()}) - summaries = { - published_artifacts.csv.with_suffix(".full.csv" if path.suffix == ".csv" else ".full.provenance.json"): text - for path, text in temporary_summaries.items() - } - outputs = bundle_outputs(published_artifacts, bundle) - outputs.update({path: text.encode("utf-8") for path, text in summaries.items()}) - yield GeneratedReport(text=report_text, artifacts=temporary_artifacts, outputs=outputs) - - -def _current_archive_state( - *, - current: Path, - source_id: ReportId, - archive_dir: Path, -) -> tuple[str | None, ReportId | None, Path | None]: - """Return normalized current-report state and its prospective archive path.""" - if not current.exists(): - return None, None, None - current_text = _normalize_how_to_update(_read_text(current)) - current_id = parse_report_id(current_text) - archive_path = archive_dir / current_id.archive_name if current_id != source_id else None - return current_text, current_id, archive_path - - -def _validate_promotion_paths(request: PromotionRequest, *, index_path: Path, archive_path: Path | None) -> None: - """Reject aliases and archive-contained scratch output before mutation.""" - named_paths: dict[str, Path] = { - "current report": request.current, - "archive index": index_path, - } - if request.source_path is not None: - named_paths["source report"] = request.source_path - if request.output is not None: - named_paths["rendered output"] = request.output - if request.output.resolve(strict=False).is_relative_to(request.archive_dir.resolve(strict=False)): - msg = f"rendered output must not be written inside the archive directory: {request.output}" - raise ValueError(msg) - if archive_path is not None: - named_paths["archived report"] = archive_path - ensure_distinct_paths({**named_paths, **(request.reserved_paths or {})}) - if request.retained_outputs is not None: - # Retained inputs may already occupy their own immutable destinations. - # Check destinations separately so this valid overlap cannot conceal an - # alias between a report output and either set of protected paths. - named_paths.update({f"retained summary {index}": path for index, path in enumerate(request.retained_outputs)}) - ensure_distinct_paths(named_paths) - - -def _existing_archive_matches(*, archive_path: Path | None, current_id: ReportId | None, current_text: str | None) -> bool: - """Validate a pre-existing canonical archive before trusting it.""" - if archive_path is None or (not archive_path.exists() and not archive_path.is_symlink()): - return False - if current_id is None or current_text is None: - msg = "existing archive validation invariant violated" - raise AssertionError(msg) - archived_payload = _snapshot_regular_file(archive_path, label="existing archived report") - if archived_payload is None: - msg = "existing archive snapshot invariant violated" - raise AssertionError(msg) - archived_text = _normalize_how_to_update(_decode_text_snapshot(archived_payload)) - try: - archived_id = parse_report_id(archived_text) - except (TypeError, ValueError) as exc: - msg = f"existing archived report does not match the current report being replaced: {archive_path}" - raise ValueError(msg) from exc - if archived_id != current_id or archived_text != current_text: - msg = f"existing archived report does not match the current report being replaced: {archive_path}" - raise ValueError(msg) - return True - - -def _promote_report_text(*, source_text: str, request: PromotionRequest, generated_outputs: Mapping[Path, bytes] | None = None) -> ReportId: - """Archive the old report and atomically promote validated Markdown text.""" - if request.expected.current_tag == request.expected.baseline_tag: - msg = "cannot promote a same-version local performance comparison as release documentation" - raise ValueError(msg) - source_text = _normalize_how_to_update(source_text) - source_id = parse_report_id(source_text) - if source_id != request.expected: - msg = ( - "benchmark report does not match requested release pair: " - f"found {source_id.current_tag} vs {source_id.baseline_tag}, " - f"expected {request.expected.current_tag} vs {request.expected.baseline_tag}" - ) - raise ValueError(msg) - - current_text, current_id, archive_path = _current_archive_state( - current=request.current, - source_id=source_id, - archive_dir=request.archive_dir, - ) - index_path = request.archive_dir / "README.md" - _validate_promotion_paths(request, index_path=index_path, archive_path=archive_path) - archive_exists = _existing_archive_matches( - archive_path=archive_path, - current_id=current_id, - current_text=current_text, - ) - outputs = dict(generated_outputs or {}) - outputs.update({path: text.encode("utf-8") for path, text in (request.retained_outputs or {}).items()}) - if archive_path is not None and not archive_exists: - if current_text is None: - msg = "archive promotion invariant violated" - raise AssertionError(msg) - outputs[archive_path] = current_text.encode("utf-8") - outputs[request.current] = source_text.encode("utf-8") - outputs[index_path] = _archive_readme(request.archive_dir, archive_path).encode("utf-8") - if request.output is not None: - outputs[request.output] = source_text.encode("utf-8") - replace_many(outputs) - return source_id - - -def promote_report( - *, - source: Path, - current: Path, - archive_dir: Path, - expected_current_tag: str, - expected_baseline_tag: str, -) -> ReportId: - """Archive the old committed report and promote *source* as the current one.""" - return _promote_report_text( - source_text=_read_text(source), - request=PromotionRequest( - current=current, - archive_dir=archive_dir, - expected=ReportId( - current_tag=normalize_tag(expected_current_tag), - baseline_tag=normalize_tag(expected_baseline_tag), - ), - source_path=source, - ), - ) + finally: + # A failed shared worktree cleanup leaves its path intact for recovery. + if not (parent / "baseline").exists() and not (parent / "current").exists(): + shutil.rmtree(parent) + validate_scientific_run(evidence) + return evidence + + +def save_scratch(root: Path, stem: Path, evidence: Evidence) -> None: + """Save new complete evidence and scientific prose without touching legacy files.""" + payload, manifest = scratch_paths(stem) + data, envelope = serialize_evidence(evidence) + report = render_scientific_report(evidence) + plan = plan_outputs( + root, + {path.relative_to(root).as_posix(): value for path, value in ((payload, data), (manifest, envelope), (stem.with_suffix(".md"), report))}, + inputs={REPORT_CONFIG: (root / REPORT_CONFIG).read_bytes()}, + ) + publish_publication(plan) + + +def promote(root: Path, stem: Path) -> None: + """Publish shared history, latest selection and scientific prose in one plan.""" + evidence = retained_run(root, stem) + validate_scientific_run(evidence, release=True) + payload, manifest = scratch_paths(stem) + kwargs = {} + if payload.exists() or manifest.exists(): + kwargs = {"payload": payload.relative_to(root).as_posix(), "manifest": manifest.relative_to(root).as_posix()} + history = load_run_report_plan(root, REPORT_CONFIG, **kwargs) + outputs = dict(history.outputs) + if json.loads(outputs[f"{ARCHIVE}/latest.json"])["run"]["id"] != run_identity(evidence): + msg = "selected evidence changed while composing scientific report" + raise ValueError(msg) + outputs["docs/performance.md"] = render_scientific_report(evidence) + # Retain the exact previous curated report at its established archive path. + previous = root / "docs/performance.md" + previous_bytes = previous.read_bytes() if previous.exists() else None + if previous_bytes is not None and previous_bytes != outputs["docs/performance.md"]: + identity = parse_report_id(previous_bytes.decode("utf-8")) + name = f"docs/archive/performance/{identity.current_tag}-vs-{identity.baseline_tag}.md" + if not (root / name).exists(): + outputs[name] = previous_bytes + originals = dict(history.originals) + inputs = {name: value for name, value in originals.items() if name not in outputs and value is not None} + immutable = tuple(name for name in outputs if "/runs/" in name or name.startswith("docs/archive/performance/")) + combined = plan_outputs(root, outputs, inputs=inputs, immutable=immutable) + combined = replace(combined, git_checks=history.git_checks, glob_checks=history.glob_checks) + if dict(combined.originals)["docs/performance.md"] != previous_bytes: + msg = "curated report changed during publication planning" + raise ValueError(msg) + if any(dict(combined.originals).get(name) != value for name, value in history.originals): + msg = "publication inputs changed while composing scientific report" + raise ValueError(msg) + publish_publication(combined) -def generate_and_promote_worktree_report( - *, - output: Path, - current: Path, - archive_dir: Path, - config: GenerationConfig, - artifacts: ArtifactPaths | None = None, -) -> ReportId: - """Generate a comparison in a temp worktree, then promote it.""" - current_tag = normalize_tag(config.current_tag) - baseline_tag = normalize_tag(config.baseline_tag) - if current_tag == baseline_tag: - msg = "release performance promotion requires distinct current and baseline releases" +def parse_report_id(text: str) -> ReportId: + """Read the established scientific report header without modifying its bytes.""" + current = re.search(r"^\*\*la-stack\*\* (v[^\s]+)", text, re.MULTILINE) + baseline = re.search(r"^Comparison against baseline \*\*([^*]+)\*\*:", text, re.MULTILINE) + if current is None or baseline is None: + msg = "performance report has no release identity" raise ValueError(msg) - config = GenerationConfig( - repo_root=config.repo_root, - current_tag=current_tag, - baseline_tag=baseline_tag, - worktree_ref=config.worktree_ref, - suite=config.suite, - scope=config.scope, - apply_current_diff=config.apply_current_diff, - baseline_source=config.baseline_source, - ) - published_artifacts = artifacts or _default_artifact_paths(config.repo_root) - expected = ReportId(current_tag=current_tag, baseline_tag=baseline_tag) - reserved_paths = report_input_paths(published_artifacts) - request = PromotionRequest( - current=current, - archive_dir=archive_dir, - expected=expected, - output=output, - reserved_paths=reserved_paths, - ) - _current_text, _current_id, archive_path = _current_archive_state( - current=current, - source_id=expected, - archive_dir=archive_dir, - ) - _validate_promotion_paths( - request, - index_path=archive_dir / "README.md", - archive_path=archive_path, - ) - with _generated_report_in_temp_worktree( - config=config, - published_artifacts=published_artifacts, - ) as generated: - return _promote_report_text( - source_text=generated.text, - request=replace( - request, - retained_outputs=retained_outputs(current.parent / "performance", generated.artifacts) if config.baseline_source == "local" else {}, - ), - generated_outputs=generated.outputs, - ) + return ReportId(normalize_tag(current[1]), normalize_tag(baseline[1])) -def generate_worktree_report( - *, - output: Path, - config: GenerationConfig, - artifacts: ArtifactPaths | None = None, -) -> ReportId: - """Generate a comparison in a temp worktree and write it to *output*.""" - current_tag = normalize_tag(config.current_tag) - baseline_tag = normalize_tag(config.baseline_tag) - config = GenerationConfig( - repo_root=config.repo_root, - current_tag=current_tag, - baseline_tag=baseline_tag, - worktree_ref=config.worktree_ref, - suite=config.suite, - scope=config.scope, - apply_current_diff=config.apply_current_diff, - baseline_source=config.baseline_source, - ) - published_artifacts = artifacts or ArtifactPaths( - csv=output.with_suffix(".csv"), - provenance=output.with_suffix(".provenance.json"), - ) - ensure_distinct_paths( - { - "rendered output": output, - **report_input_paths(published_artifacts), - } - ) - with _generated_report_in_temp_worktree( - config=config, - published_artifacts=published_artifacts, - ) as generated: - report_text = _normalize_how_to_update(generated.text) - report_id = parse_report_id(report_text) - expected = ReportId(current_tag=current_tag, baseline_tag=baseline_tag) - if report_id != expected: - msg = ( - "benchmark report does not match requested release pair: " - f"found {report_id.current_tag} vs {report_id.baseline_tag}, " - f"expected {expected.current_tag} vs {expected.baseline_tag}" - ) - raise ValueError(msg) - replace_many({**generated.outputs, output: report_text.encode("utf-8")}) - return report_id - - -def render_and_promote_artifacts( - *, - artifacts: ArtifactPaths, - output: Path, - current: Path, - archive_dir: Path, -) -> ReportId: - """Reproduce and promote Markdown using only retained CSV/JSON inputs.""" +def render_and_promote_artifacts(*, artifacts: ArtifactPaths, output: Path, current: Path, archive_dir: Path) -> ReportId: + """Read historical CSV/JSON through its original schema; never relabel hashes.""" bundle = load_bundle(artifacts) - expected = ReportId( - current_tag=normalize_tag(bundle.context.release.current), - baseline_tag=normalize_tag(bundle.context.release.baseline), - ) - if expected.current_tag == expected.baseline_tag: - msg = "cannot promote a same-version local performance comparison as release documentation" - raise ValueError(msg) - report_text = _normalize_how_to_update(render_release_artifacts(artifacts)) - observed = parse_report_id(report_text) - if observed != expected: - msg = ( - "rendered benchmark report does not match retained release pair: " - f"found {observed.current_tag} vs {observed.baseline_tag}, " - f"expected {expected.current_tag} vs {expected.baseline_tag}" - ) - raise ValueError(msg) - return _promote_report_text( - source_text=report_text, - request=PromotionRequest( - current=current, - archive_dir=archive_dir, - expected=expected, - output=output, - retained_outputs=retained_outputs(current.parent / "performance", artifacts), - reserved_paths=report_input_paths(artifacts), - ), - ) - - -def resolve_archive_request(options: ArchiveRequestOptions) -> ResolvedArchiveRequest: - """Resolve explicit, package-inferred, or latest-published release arguments.""" - current_tag = options.current_tag - baseline_tag = options.baseline_tag - worktree_ref = options.worktree_ref - repo_root = options.repo_root - published_latest = options.published_latest - infer_release = options.infer_release - current_vs_latest = options.current_vs_latest - requested_modes = sum((published_latest, infer_release, current_vs_latest)) - if requested_modes > 1: - msg = "choose only one of --published-latest, --infer-release, or --current-vs-latest" - raise ValueError(msg) - - if published_latest: - if current_tag is not None or baseline_tag is not None: - msg = "do not pass current_tag or baseline_tag with --published-latest" - raise ValueError(msg) - published_pair = _published_release_pair(repo_root) - resolved_worktree_ref = published_pair.current_tag if worktree_ref == "HEAD" else worktree_ref - return ResolvedArchiveRequest( - current_tag=published_pair.current_tag, - baseline_tag=published_pair.baseline_tag, - worktree_ref=resolved_worktree_ref, - tags_to_fetch=(published_pair.current_tag, published_pair.baseline_tag), - ) - - if infer_release: - if current_tag is not None or baseline_tag is not None: - msg = "do not pass current_tag or baseline_tag with --infer-release" - raise ValueError(msg) - inferred_pair = _unpublished_package_release_pair(repo_root) - return ResolvedArchiveRequest( - current_tag=inferred_pair.current_tag, - baseline_tag=inferred_pair.baseline_tag, - worktree_ref=worktree_ref, - tags_to_fetch=(inferred_pair.baseline_tag,), - ) - - if current_vs_latest: - if current_tag is not None or baseline_tag is not None: - msg = "do not pass current_tag or baseline_tag with --current-vs-latest" - raise ValueError(msg) - inferred_current = _current_package_tag(repo_root) - latest = _latest_published_release(repo_root).tag - return ResolvedArchiveRequest( - current_tag=inferred_current, - baseline_tag=latest, - worktree_ref=worktree_ref, - tags_to_fetch=(latest,), - ) - - if current_tag is None or baseline_tag is None: - msg = "current_tag and baseline_tag are required unless an inference mode is used" - raise ValueError(msg) - normalized_current = normalize_tag(current_tag) - normalized_baseline = normalize_tag(baseline_tag) - return ResolvedArchiveRequest( - current_tag=normalized_current, - baseline_tag=normalized_baseline, - worktree_ref=_normalize_worktree_ref_for_tag(worktree_ref, normalized_current), - tags_to_fetch=(normalized_baseline,), - ) - - -def build_parser() -> argparse.ArgumentParser: - """Build the CLI argument parser.""" - parser = argparse.ArgumentParser( - description="Generate local benchmark comparisons or promote validated comparison artifacts into release documentation.", - ) - parser.add_argument("current_tag", nargs="?", help="Current package-version label, e.g. v0.4.6; new comparisons require v0.4.4 or newer") - parser.add_argument( - "baseline_tag", nargs="?", help="Previous release tag used as the comparison baseline, e.g. v0.4.5; new comparisons require v0.4.4 or newer" - ) - parser.add_argument( - "--source", - default=_DEFAULT_SOURCE, - help=f"Generated benchmark report to promote (default: {_DEFAULT_SOURCE})", - ) - parser.add_argument( - "--current", - default=_DEFAULT_CURRENT, - help=f"Committed performance report path (default: {_DEFAULT_CURRENT})", - ) - parser.add_argument( - "--output", - default=_DEFAULT_SOURCE, - help=f"Rendered local comparison report path (default: {_DEFAULT_SOURCE})", - ) - parser.add_argument( - "--artifact-csv", - default=_DEFAULT_ARTIFACT_CSV, - help=f"Retained comparison CSV path (default: {_DEFAULT_ARTIFACT_CSV})", - ) - parser.add_argument( - "--artifact-provenance", - default=_DEFAULT_ARTIFACT_PROVENANCE, - help=f"Retained comparison JSON provenance path (default: {_DEFAULT_ARTIFACT_PROVENANCE})", - ) - parser.add_argument( - "--archive-dir", - default=_DEFAULT_ARCHIVE_DIR, - help=f"Archive directory for older reports (default: {_DEFAULT_ARCHIVE_DIR})", - ) - parser.add_argument( - "--generate-in-temp-worktree", - action="store_true", - help="Generate the comparison in a temporary detached worktree.", - ) - parser.add_argument( - "--published-latest", - action="store_true", - help="Infer the latest stable published GitHub release and its previous stable release.", - ) - parser.add_argument( - "--infer-release", - action="store_true", - help="Require an unpublished Cargo.toml version and compare it with the previous stable published release.", - ) - parser.add_argument( - "--current-vs-latest", - action="store_true", - help="Infer current_tag from Cargo.toml and baseline_tag from the latest stable published release.", - ) - parser.add_argument( - "--github-assets", - action="store_true", - help="Compare stored GitHub Release benchmark assets instead of generating the baseline locally.", - ) - parser.add_argument( - "--output-only", - action="store_true", - help="Write the generated report to --output without promoting docs/performance.md.", - ) - parser.add_argument( - "--local-report", - action="store_true", - help=( - "Generate a local report and retained comparison artifacts without promoting release documentation; " - "the current package version may match the baseline release." - ), - ) - parser.add_argument( - "--promote-artifacts", - action="store_true", - help="Render and promote from retained CSV/JSON artifacts without benchmarks or worktrees.", - ) - parser.add_argument( - "--worktree-ref", - default="HEAD", - help="Git ref to check out in the temporary worktree (default: HEAD).", - ) - parser.add_argument( - "--no-apply-current-diff", - action="store_true", - help="Do not apply the current checkout's tracked diff to the temporary worktree.", - ) - parser.add_argument( - "--suite", - default=None, - choices=_SUPPORTED_SUITES, - help=f"Benchmark suite for --generate-in-temp-worktree (default: {_DEFAULT_SUITE})", - ) - parser.add_argument( - "--scope", - default=None, - choices=_SUPPORTED_SCOPES, - help=f"Comparison scope for --generate-in-temp-worktree (default: {_DEFAULT_SCOPE})", - ) - return parser - - -def _resolve_cli_paths(root: Path, args: argparse.Namespace) -> ArchivePaths: - source = Path(args.source) - current = Path(args.current) - output = Path(args.output) - archive_dir = Path(args.archive_dir) - artifact_csv = Path(args.artifact_csv) - artifact_provenance = Path(args.artifact_provenance) - if not source.is_absolute(): - source = root / source - if not current.is_absolute(): - current = root / current - if not output.is_absolute(): - output = root / output - if not archive_dir.is_absolute(): - archive_dir = root / archive_dir - if not artifact_csv.is_absolute(): - artifact_csv = root / artifact_csv - if not artifact_provenance.is_absolute(): - artifact_provenance = root / artifact_provenance - return ArchivePaths( - source=source, - current=current, - output=output, - archive_dir=archive_dir, - artifacts=ArtifactPaths(csv=artifact_csv, provenance=artifact_provenance), - ) - - -def _fetch_required_tags(*, request: ResolvedArchiveRequest, repo_root: Path, include_current: bool) -> None: - tags_to_fetch = request.tags_to_fetch - if include_current and request.current_tag not in tags_to_fetch: - tags_to_fetch = (*tags_to_fetch, request.current_tag) - if tags_to_fetch: - _fetch_release_tags(repo_root=repo_root, tags=list(dict.fromkeys(tags_to_fetch))) - - -def _generation_config(*, args: argparse.Namespace, request: ResolvedArchiveRequest, repo_root: Path) -> GenerationConfig: - return GenerationConfig( - repo_root=repo_root, - current_tag=request.current_tag, - baseline_tag=request.baseline_tag, - worktree_ref=request.worktree_ref, - suite=cast("BenchmarkSuite", args.suite or _DEFAULT_SUITE), - scope=cast("ComparisonScope", args.scope or _DEFAULT_SCOPE), - apply_current_diff=not args.no_apply_current_diff and not args.github_assets, - baseline_source="github-assets" if args.github_assets else "local", - ) - - -def _validate_generation_request( - *, - args: argparse.Namespace, - request: ResolvedArchiveRequest, - repo_root: Path, -) -> None: - """Reject invalid release identities before fetching tags or benchmarking.""" - if not args.local_report and request.current_tag == request.baseline_tag: - msg = "release performance artifacts require distinct current and baseline releases" - raise ValueError(msg) - - if args.generate_in_temp_worktree and not args.github_assets and args.current_tag is not None and request.worktree_ref == "HEAD": - package_tag = _current_package_tag(repo_root) - if request.current_tag != package_tag: - msg = ( - f"explicit current tag {request.current_tag} does not match the current Cargo package tag " - f"{package_tag}; update the package version or pass the matching current tag" - ) - raise ValueError(msg) - - -def _run_archive_request(*, args: argparse.Namespace, paths: ArchivePaths, request: ResolvedArchiveRequest, repo_root: Path) -> ArchiveResult: - _validate_generation_request(args=args, request=request, repo_root=repo_root) - if args.generate_in_temp_worktree: - config = _generation_config(args=args, request=request, repo_root=repo_root) - _fetch_required_tags(request=request, repo_root=repo_root, include_current=args.github_assets) - if args.output_only: - return ArchiveResult( - report_id=generate_worktree_report( - output=paths.output, - config=config, - artifacts=paths.artifacts, - ), - action="output", - ) - return ArchiveResult( - report_id=generate_and_promote_worktree_report( - output=paths.output, - current=paths.current, - archive_dir=paths.archive_dir, - config=config, - artifacts=paths.artifacts, - ), - action="promote-generated", - ) - - if args.output_only: - msg = "--output-only requires --generate-in-temp-worktree" - raise ValueError(msg) - if args.github_assets: - msg = "--github-assets requires --generate-in-temp-worktree" - raise ValueError(msg) - _run_benchmark_input_gate(repo_root, env=_benchmark_env(repo_root)) - return ArchiveResult( - report_id=promote_report( - source=paths.source, - current=paths.current, - archive_dir=paths.archive_dir, - expected_current_tag=request.current_tag, - expected_baseline_tag=request.baseline_tag, - ), - action="promote-source", - ) - - -def _validate_cli_preflight(args: argparse.Namespace) -> None: - """Reject locally invalid options before release discovery or tag fetching.""" - if args.promote_artifacts and any( - ( - args.current_tag, - args.baseline_tag, - args.published_latest, - args.infer_release, - args.current_vs_latest, - args.github_assets, - args.generate_in_temp_worktree, - args.output_only, - args.local_report, - args.suite, - args.scope, - ) - ): - msg = "--promote-artifacts cannot be combined with release selection, generation, GitHub-asset, output-only, suite, or scope options" - raise ValueError(msg) - if args.output_only and not args.generate_in_temp_worktree: - msg = "--output-only requires --generate-in-temp-worktree" - raise ValueError(msg) - if args.local_report and not args.output_only: - msg = "--local-report requires --output-only" - raise ValueError(msg) - if args.local_report and args.github_assets: - msg = "--local-report cannot be combined with --github-assets" - raise ValueError(msg) - if args.github_assets and not args.generate_in_temp_worktree: - msg = "--github-assets requires --generate-in-temp-worktree" - raise ValueError(msg) + release = bundle.context.release + if release.current == release.baseline: + msg = "cannot promote a same-version local comparison as release documentation" + raise ValueError(msg) + text = render_release_artifacts(artifacts).encode() + root = current.parent.parent.resolve() + outputs = {current.relative_to(root).as_posix(): text, output.relative_to(root).as_posix(): text} + if current.exists() and current.read_bytes() != text: + identity = parse_report_id(current.read_text(encoding="utf-8")) + archived = archive_dir / f"{identity.current_tag}-vs-{identity.baseline_tag}.md" + outputs[archived.relative_to(root).as_posix()] = current.read_bytes() + inputs = {path.relative_to(root).as_posix(): path.read_bytes() for path in (artifacts.csv, artifacts.provenance)} + publish_publication(plan_outputs(root, outputs, inputs=inputs, immutable=tuple(name for name in outputs if name.startswith("docs/archive/")))) + return ReportId(release.current, release.baseline) + + +def compare_assets(root: Path, pair: ReleasePair, output: Path, suite: str, scope: str) -> None: + """Read published raw archives with shared bounded download/extraction APIs.""" + with tempfile.TemporaryDirectory(prefix="la-stack-assets-") as directory: + parent = Path(directory) + criterion = parent / "criterion" + for phase, tag in (("baseline", pair.baseline), ("current", pair.current)): + asset = parent / f"{tag}.tar.gz" + download_release_asset(root, "acgetchell/la-stack", tag, f"la-stack-{tag}-criterion-baseline.tar.gz", asset) + extracted = parent / phase + extract_archive(asset, extracted) + # Preserve native measurements, explicitly retaining their original + # per-release harnesses; these cannot become a complete common run. + sample = tag + destination_sample = "new" if phase == "current" else tag + for source in (extracted / "criterion").glob(f"**/{sample}/*.json"): + target = criterion / source.relative_to(extracted / "criterion").parent.parent / destination_sample / source.name + replace_many({target: source.read_bytes()}) + text = render_historical_assets(criterion, pair, suite, scope) + publish_publication(plan_outputs(root, {output.relative_to(root).as_posix(): text}, inputs={"Cargo.toml": (root / "Cargo.toml").read_bytes()})) def main(argv: list[str] | None = None) -> int: - """CLI entry point.""" - args = build_parser().parse_args(argv) - root = Path.cwd() - + """Keep routine performance commands thin and explicit.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=("local", "release", "report", "assets")) + parser.add_argument("current", nargs="?") + parser.add_argument("baseline", nargs="?") + parser.add_argument("--suite", choices=("all", "exact", "vs_linalg"), default="all") + parser.add_argument("--scope", choices=("release-signal", "all-benches"), default="release-signal") + parser.add_argument("--stem", type=Path, default=Path("target/bench-reports/performance")) + args = parser.parse_args(argv) + root = find_project_root(Path.cwd()).resolve() + stem = root / args.stem try: - _validate_cli_preflight(args) - paths = _resolve_cli_paths(root, args) - if args.promote_artifacts: - paths = replace(paths, artifacts=resolve_report_paths(root, paths.artifacts)) - result = ArchiveResult( - report_id=render_and_promote_artifacts( - artifacts=paths.artifacts, - output=paths.output, - current=paths.current, - archive_dir=paths.archive_dir, - ), - action="promote-artifacts", - ) - else: - request = resolve_archive_request( - ArchiveRequestOptions( - current_tag=args.current_tag, - baseline_tag=args.baseline_tag, - published_latest=args.published_latest, - infer_release=args.infer_release, - current_vs_latest=args.current_vs_latest, - worktree_ref=args.worktree_ref, - repo_root=root, + if args.command == "report": + if args.current or args.baseline: + parser.error("report reads retained evidence and takes no release labels") + if any(path.exists() or path.is_symlink() for path in (*scratch_paths(stem), root / ARCHIVE)): + promote(root, stem) + else: + paths = resolve_report_paths(root, ArtifactPaths(stem.with_suffix(".csv"), stem.with_suffix(".provenance.json"))) + render_and_promote_artifacts( + artifacts=paths, output=stem.with_suffix(".md"), current=root / "docs/performance.md", archive_dir=root / "docs/archive/performance" ) - ) - result = _run_archive_request(args=args, paths=paths, request=request, repo_root=root) - except ( - ExceptionGroup, - ExecutableNotFoundError, - OSError, - TypeError, - ValueError, - RuntimeError, - subprocess.CalledProcessError, - subprocess.TimeoutExpired, - ) as exc: - print(f"archive-performance: {format_exception_diagnostics(exc)}", file=sys.stderr) + return 0 + modes: dict[str, PairMode] = {"local": "current-vs-latest", "release": "infer-release", "assets": "published-latest"} + mode: PairMode = "explicit" if args.current or args.baseline else modes[args.command] + pair = select_pair(root, mode, args.current or None, args.baseline or None) + if args.command == "assets": + compare_assets(root, pair, root / "target/bench-reports/github-assets-performance.md", args.suite, args.scope) + else: + if args.command == "release" and pair.current == pair.baseline: + parser.error("release requires distinct release labels") + evidence = measure(root, pair, args.suite, args.scope) + save_scratch(root, stem, evidence) + if args.command == "release": + promote(root, stem) + return 0 + except (OSError, ValueError, TypeError, KeyError, ExecutableNotFoundError, subprocess.SubprocessError, ExceptionGroup) as error: + print(f"performance: {format_exception_diagnostics(error)}", file=sys.stderr) return 1 - if result.action == "output": - print(f"Generated benchmark report in a temporary worktree and wrote it to {paths.output}") - elif result.action == "promote-generated": - print(f"Generated benchmark report in a temporary worktree, wrote it to {paths.output}, and promoted it to {paths.current}") - elif result.action == "promote-artifacts": - print(f"Rendered {paths.output} from {paths.artifacts.csv} and promoted it to {paths.current}") - else: - print(f"Promoted {paths.source} to {paths.current}") - print(f"Current performance report: {result.report_id.current_tag} vs {result.report_id.baseline_tag}") - print(f"Archive directory: {paths.archive_dir}") - return 0 - if __name__ == "__main__": raise SystemExit(main()) diff --git a/scripts/bench_compare.py b/scripts/bench_compare.py index da94f7d..88af4e3 100644 --- a/scripts/bench_compare.py +++ b/scripts/bench_compare.py @@ -31,7 +31,7 @@ from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path -from typing import Literal, Protocol, cast +from typing import TYPE_CHECKING, Literal, Protocol, cast from research_repo_tools.criterion import Comparison as TimingComparison, Estimate, read_estimate from research_repo_tools.files import replace_many @@ -55,6 +55,9 @@ resolve_shared_harness_compatibility, ) +if TYPE_CHECKING: + from research_repo_tools.release_pairs import ReleasePair as SharedReleasePair + # --------------------------------------------------------------------------- # Benchmark group / bench discovery # --------------------------------------------------------------------------- @@ -797,7 +800,7 @@ def _validate_schema2_consistency( raise ValueError(msg) -def _assess_change(baseline: CriterionEstimate, current: CriterionEstimate) -> ChangeAssessment: +def assess_change(baseline: CriterionEstimate, current: CriterionEstimate) -> ChangeAssessment: """Classify a change conservatively from non-overlapping Criterion intervals.""" if not baseline.has_confidence_interval or not current.has_confidence_interval: return "unknown" @@ -947,7 +950,7 @@ def _collect_exact_comparisons( bench=bench, baseline=baseline, current=current, - assessment=_assess_change(baseline, current), + assessment=assess_change(baseline, current), ) ) @@ -1077,7 +1080,7 @@ def _collect_vs_linalg_comparisons( bench=bench, baseline=baseline, current=current, - assessment=_assess_change(baseline, current), + assessment=assess_change(baseline, current), baseline_nalgebra=baseline_nalgebra, baseline_faer=baseline_faer, ) @@ -1186,7 +1189,7 @@ def _collect_comparisons( # --------------------------------------------------------------------------- -def _format_time(ns: float) -> str: +def format_time(ns: float) -> str: """Format nanoseconds into a human-readable string.""" if ns < 1_000: return f"{ns:.1f} ns" @@ -1205,12 +1208,12 @@ def _format_confidence_interval(estimate: CriterionEstimate) -> str: """Format a Criterion confidence interval without inventing missing bounds.""" if estimate.ci_lo_ns is None or estimate.ci_hi_ns is None: return "unavailable" - return f"[{_format_time(estimate.ci_lo_ns)}, {_format_time(estimate.ci_hi_ns)}]" + return f"[{format_time(estimate.ci_lo_ns)}, {format_time(estimate.ci_hi_ns)}]" def _format_estimate(estimate: CriterionEstimate) -> str: """Format a point estimate together with its Criterion interval.""" - return f"{_format_time(estimate.point_ns)} {_format_confidence_interval(estimate)}" + return f"{format_time(estimate.point_ns)} {_format_confidence_interval(estimate)}" def _assessment_label(assessment: ChangeAssessment) -> str: @@ -1302,13 +1305,13 @@ def _snapshot_tables(results: list[BenchResult], stat: str) -> str: case = _group_heading_for_suite(suite, group) for r in items: ci_range = _format_confidence_interval(r.estimate) - lines.append(f"| {case} | {r.bench} | {_format_time(r.point_ns)} | {ci_range} |") + lines.append(f"| {case} | {r.bench} | {format_time(r.point_ns)} | {ci_range} |") sections.append("\n".join(lines)) return "\n\n".join(sections) -def _comparison_tables(comparisons: list[Comparison], baseline_name: str) -> str: +def comparison_tables(comparisons: list[Comparison], baseline_name: str) -> str: """Generate one Markdown comparison table per suite.""" sections: list[str] = [] @@ -1485,7 +1488,7 @@ def _artifact_comparisons(rows: tuple[PerformanceRow, ...]) -> list[Comparison]: bench=row.benchmark, baseline=baseline, current=current, - assessment=_assess_change(baseline, current), + assessment=assess_change(baseline, current), baseline_bench=(row.baseline_benchmark if row.baseline_benchmark != row.benchmark else None), baseline_nalgebra=(None if row.baseline_nalgebra is None else _criterion_timing(row.baseline_nalgebra)), baseline_faer=None if row.baseline_faer is None else _criterion_timing(row.baseline_faer), @@ -1519,7 +1522,7 @@ def _artifact_tables(bundle: PerformanceBundle) -> str: comparisons = _artifact_comparisons(bundle.sorted_rows) sections: list[str] = [] if comparisons: - sections.append(_comparison_tables(comparisons, bundle.context.release.baseline)) + sections.append(comparison_tables(comparisons, bundle.context.release.baseline)) coverage = _artifact_coverage_table(bundle.sorted_rows, bundle.context.release.baseline) if coverage: sections.append(coverage) @@ -1842,6 +1845,23 @@ def render_release_artifacts(paths: ArtifactPaths) -> str: return render_bundle(load_bundle(paths), paths) +def render_historical_assets(criterion: Path, pair: SharedReleasePair, suite: str, scope: str) -> bytes: + """Render native release assets without assigning local measurement provenance.""" + policy = ComparisonPolicy(scope=scope, shared_harness_rational_inputs=False) + collection = _collect_comparisons(criterion, pair.baseline, "median", suite, policy) + if collection.gaps or not collection.comparisons: + msg = "historical assets omit required la-stack report rows" + raise ValueError(msg) + return ( + f"# Published benchmark assets\n\n**la-stack** {pair.current}; baseline **{pair.baseline}**.\n\n" + "Each release retains its original harness. Historical host, toolchain, source fingerprints, and correctness gates are unknown.\n" + "These assets cannot be promoted as a common-harness run. Use `just performance-local` for controlled measurements.\n\n" + "Negative change means a smaller point estimate. Marginal interval separation is not a paired confidence interval or a significance claim.\n\n" + + comparison_tables(collection.comparisons, pair.baseline) + + "\n" + ).encode() + + def render_bundle(bundle: PerformanceBundle, paths: ArtifactPaths) -> str: """Render validated retained data before publishing any output.""" context = bundle.context @@ -2110,7 +2130,7 @@ def main(argv: list[str] | None = None) -> int: # noqa: C901, PLR0911, PLR0912, ) return 2 try: - table = _comparison_tables(collection.comparisons, baseline_name) + table = comparison_tables(collection.comparisons, baseline_name) except ValueError as err: print(f"Invalid Criterion comparison data: {err}", file=sys.stderr) return 2 diff --git a/scripts/benchmark_summaries.py b/scripts/benchmark_summaries.py index 5b4eb09..213e2d1 100644 --- a/scripts/benchmark_summaries.py +++ b/scripts/benchmark_summaries.py @@ -1,17 +1,15 @@ -"""Preserve every recorded local benchmark summary outside disposable targets.""" +"""Read historical la-stack summary schemas without rewriting their bytes or hashes.""" import csv import hashlib import io import json -import re from dataclasses import dataclass from pathlib import Path -from typing import Literal +from typing import Literal, cast -from research_repo_tools.criterion import read_estimate +from research_repo_tools.criterion import Estimate -from criterion_measurements import positive_number, read_object, validate_measurement from performance_artifacts import ArtifactPaths, TimingEstimate, load_bundle SCHEMA_VERSION = 1 @@ -45,6 +43,14 @@ def __post_init__(self) -> None: raise ValueError(msg) +def read_object(path: Path) -> dict[str, object]: + """Read the legacy JSON object without assigning new digest semantics.""" + value = json.loads(path.read_bytes()) + if not isinstance(value, dict): + raise TypeError(f"expected a legacy JSON object: {path}") + return cast("dict[str, object]", value) + + def full_summary_paths(report: ArtifactPaths) -> ArtifactPaths: """Keep the complete summary adjacent to the selected report inputs.""" return ArtifactPaths( @@ -64,33 +70,6 @@ def report_input_paths(report: ArtifactPaths) -> dict[str, Path]: } -def _estimate(directory: Path, statistic: Literal["mean", "median"]) -> TimingEstimate: - estimate = read_estimate(directory / "estimates.json", statistic=statistic) - if estimate.lower is None or estimate.upper is None: - raise ValueError(f"{directory}: complete summaries require confidence intervals") - return TimingEstimate(estimate.point, estimate.lower, estimate.upper) - - -def collect_measurements(criterion: Path, baseline: str) -> tuple[Measurement, ...]: - """Read all recorded cases, including peers and groups outside the report registry. - - The caller must remove baseline ``new`` directories before current timing. - Named baseline samples survive that cleanup. Their phase is never inferred - from the benchmark's library name or substituted for an absent current run. - """ - rows: list[Measurement] = [] - for phase, sample in (("baseline", baseline), ("current", "new")): - for directory in sorted(path for path in criterion.rglob(sample) if path.is_dir()): - metadata = read_object(directory / "benchmark.json") - benchmark_id = metadata.get("full_id") - if not isinstance(benchmark_id, str) or not benchmark_id.strip(): - raise ValueError(f"missing Criterion full_id in {directory}") - validate_measurement(directory) - rows.append(Measurement(phase, benchmark_id, _estimate(directory, "mean"), _estimate(directory, "median"))) - _validate_rows(rows) - return tuple(sorted(rows, key=lambda row: (row.phase, row.benchmark_id))) - - def _validate_rows(rows: list[Measurement] | tuple[Measurement, ...]) -> None: if {row.phase for row in rows} != {"baseline", "current"}: msg = "complete local summaries require both measurement phases" @@ -101,28 +80,6 @@ def _validate_rows(rows: list[Measurement] | tuple[Measurement, ...]) -> None: raise ValueError(msg) -def _serialize_rows(rows: tuple[Measurement, ...]) -> str: - output = io.StringIO(newline="") - writer = csv.writer(output, lineterminator="\n") - writer.writerow(COLUMNS) - for row in rows: - writer.writerow( - ( - row.phase, - row.benchmark_id, - 100, - row.mean.median_ns, - row.mean.ci_lower_ns, - row.mean.ci_upper_ns, - row.median.median_ns, - row.median.ci_lower_ns, - row.median.ci_upper_ns, - 0.95, - ) - ) - return output.getvalue() - - def _parse_rows(text: str) -> tuple[Measurement, ...]: reader = csv.DictReader(io.StringIO(text, newline="")) if reader.fieldnames != list(COLUMNS): @@ -140,7 +97,7 @@ def _parse_rows(text: str) -> tuple[Measurement, ...]: msg = "local summaries require full release sampling and 95% intervals" raise ValueError(msg) estimates = [ - TimingEstimate(*(positive_number(float(row[field])) for field in fields)) + TimingEstimate(*(Estimate(float(row[field])).point for field in fields)) for fields in ( ("mean_ns", "mean_ci_lower_ns", "mean_ci_upper_ns"), ("median_ns", "median_ci_lower_ns", "median_ci_upper_ns"), @@ -151,28 +108,6 @@ def _parse_rows(text: str) -> tuple[Measurement, ...]: return tuple(rows) -def summary_outputs(criterion: Path, baseline: str, report: ArtifactPaths) -> dict[Path, str]: - """Validate every measured case and bind its CSV to the selected report provenance.""" - bundle = load_bundle(report) - if bundle.context.benchmark_provenance["mode"] != "shared-current-harness": - msg = "local summaries require locally measured shared-harness results" - raise ValueError(msg) - rows = collect_measurements(criterion, baseline) - payload = _serialize_rows(rows) - if _parse_rows(payload) != rows: - msg = "local summary CSV did not preserve timing precision" - raise ValueError(msg) - provenance = read_object(report.provenance) - provenance["measurements"] = { - "schema_version": SCHEMA_VERSION, - "columns": COLUMNS, - "sha256": hashlib.sha256(payload.encode()).hexdigest(), - "counts": {phase: sum(row.phase == phase for row in rows) for phase in ("baseline", "current")}, - } - paths = full_summary_paths(report) - return {paths.csv: payload, paths.provenance: json.dumps(provenance, indent=2, sort_keys=True) + "\n"} - - def _validate_saved_summary(report: ArtifactPaths) -> None: full = full_summary_paths(report) payload = full.csv.read_bytes() @@ -194,49 +129,13 @@ def _validate_saved_summary(report: ArtifactPaths) -> None: load_bundle(report) -def retained_outputs(performance_dir: Path, report: ArtifactPaths) -> dict[Path, str]: - """Build an immutable run directory and latest pointer for a release promotion. - - Legacy selected-only artifacts remain renderable, but cannot be presented - as complete histories or replace the latest complete-summary pointer. - Include unchanged destinations so promotion can protect every retained file - before deciding which payloads need writing. - """ - full = full_summary_paths(report) - if not full.csv.exists() and not full.provenance.exists(): - return {} - _validate_saved_summary(report) - bundle = load_bundle(report) - release = bundle.context.release - if any(re.fullmatch(r"v[0-9A-Za-z.+-]+", tag) is None for tag in (release.current, release.baseline)): - msg = "invalid release identifier for persistent summaries" - raise ValueError(msg) - payloads = { - "performance.csv": report.csv.read_text(encoding="utf-8"), - "performance.provenance.json": report.provenance.read_text(encoding="utf-8"), - "performance.full.csv": full.csv.read_text(encoding="utf-8"), - "performance.full.provenance.json": full.provenance.read_text(encoding="utf-8"), - } - digest = hashlib.sha256(json.dumps(payloads, sort_keys=True).encode()).hexdigest() - relative = Path(f"{release.current}-vs-{release.baseline}") / digest - if not (performance_dir / relative).resolve().is_relative_to(performance_dir.resolve()): - msg = "benchmark snapshot escapes the performance directory" - raise ValueError(msg) - outputs = {performance_dir / relative / name: text for name, text in payloads.items()} - for path, text in outputs.items(): - if path.exists() and path.read_text(encoding="utf-8") != text: - raise ValueError(f"existing benchmark snapshot differs: {path}") - outputs[performance_dir / "latest.json"] = json.dumps({"schema_version": SCHEMA_VERSION, "run": relative.as_posix()}, indent=2) + "\n" - return outputs - - def resolve_report_paths(root: Path, report: ArtifactPaths) -> ArtifactPaths: """Use committed summaries after clean, without hiding partial scratch artifacts.""" default = root / "target/bench-reports/performance.csv" if ( report.csv.resolve() != default.resolve() or report.provenance.resolve() != default.with_suffix(".provenance.json").resolve() - or any(path.exists() for path in report_input_paths(report).values()) + or any(path.exists() or path.is_symlink() for path in report_input_paths(report).values()) ): return report directory = root / "docs/performance" diff --git a/scripts/criterion_dim_plot.py b/scripts/criterion_dim_plot.py index 5c8cfc6..a50f8e3 100644 --- a/scripts/criterion_dim_plot.py +++ b/scripts/criterion_dim_plot.py @@ -28,10 +28,12 @@ from pathlib import Path from typing import Final, TypeGuard, cast +from research_repo_tools.complete_runs import run_identity from research_repo_tools.criterion import Statistic, read_estimate +from research_repo_tools.evidence import serialize_evidence from research_repo_tools.files import replace_many from research_repo_tools.process import ExecutableNotFoundError, cpu_description, format_exception_diagnostics -from research_repo_tools.publication import MarkerPair, replace_section +from research_repo_tools.publication import MarkerPair, plan_outputs, publish_publication, replace_section from benchmark_contract import benchmark_contract_digest from benchmark_process import find_project_root, run_git_command, run_safe_command @@ -1131,7 +1133,7 @@ def _update_staged_readme_publication( staged_readme.write_bytes(updated_readme.encode("utf-8")) -def _stage_and_publish_outputs( # noqa: PLR0913 +def _stage_and_publish_outputs( # noqa: C901, PLR0913 *, root: Path, args: PlotCliArgs, @@ -1139,6 +1141,7 @@ def _stage_and_publish_outputs( # noqa: PLR0913 req: PlotRequest, provenance: dict[str, object], skipped: list[str], + validated_inputs: Mapping[Path, bytes] | None = None, ) -> int: """Render every output in isolation, then replace publication files together.""" final_provenance = req.csv_path.with_suffix(".provenance.json") @@ -1186,8 +1189,17 @@ def _stage_and_publish_outputs( # noqa: PLR0913 return 2 pairs.append((staged_readme, readme_path)) - if not _publish_staged_files(pairs): - return 2 + if validated_inputs is None: + if not _publish_staged_files(pairs): + return 2 + else: + outputs = {destination.relative_to(root).as_posix(): staged.read_bytes() for staged, destination in pairs} + inputs = {path.relative_to(root).as_posix(): data for path, data in validated_inputs.items() if path.relative_to(root).as_posix() not in outputs} + plan = plan_outputs(root, outputs, inputs=inputs) + if any(dict(plan.originals).get(path.relative_to(root).as_posix()) != data for path, data in validated_inputs.items()): + msg = "README publication inputs changed during rendering" + raise ValueError(msg) + publish_publication(plan) if skipped: print("Warning: some dimension groups were skipped:") @@ -1202,6 +1214,66 @@ def _stage_and_publish_outputs( # noqa: PLR0913 return 0 +def _complete_readme(root: Path, args: PlotCliArgs) -> int: + """Render the established figure format directly from shared complete runs.""" + # The report registry imports this module's metric definitions. + import performance_runs as complete # noqa: PLC0415 + + evidence = complete.retained_run(root, root / "target/bench-reports/performance") + run = complete.validate_scientific_run(evidence, release=True) + complete.verify_current_source(root, evidence) + values = complete.estimates_by_series(run, args.stat) + metric = METRICS[args.metric] + rows = [] + for dimension in CANONICAL_DIMS: + name = f"d{dimension}/{metric.la_bench}" + estimates = [values[series][name] for series in ("la-stack current", "nalgebra", "faer")] + numbers = [value for estimate in estimates for value in (estimate.point, estimate.lower, estimate.upper)] + if any(value is None for value in numbers): + msg = "README requires complete confidence intervals" + raise ValueError(msg) + rows.append(Row(dimension, *(float(value) for value in numbers if value is not None))) + sources = dict(evidence.sources) + context = dict(sources["current"].context) + baseline_context = dict(sources["baseline"].context) + inputs = {root / "README.md": (root / "README.md").read_bytes()} + for field in ("source-inventory", "harness-inventory"): + inputs.update({root / name: (root / name).read_bytes() for name in json.loads(context[field])}) + payload, manifest = complete.scratch_paths(root / "target/bench-reports/performance") + identity = run_identity(evidence) + if not payload.exists(): + payload = root / complete.ARCHIVE / "runs" / identity / "run.json" + manifest = payload.with_name("evidence.json") + for name in ("index.json", "latest.json"): + path = root / complete.ARCHIVE / name + inputs[path] = path.read_bytes() + inputs.update(zip((payload, manifest), serialize_evidence(evidence), strict=True)) + provenance: dict[str, object] = { + "schema": 3, + "artifact": "README vs_linalg dimension plot", + "run_id": identity, + "criterion": {"statistic": args.stat, "metric": args.metric, "dimensions": list(CANONICAL_DIMS), "log_y": args.log_y}, + "measurement": {"la_stack_phase": "current", "peer_phase": "baseline", "peer_release_context": baseline_context["release"]}, + "sources": { + phase: {"revision": source.revision, "source_sha256": source.source_sha256, "harness_sha256": source.harness_sha256} + for phase, source in evidence.sources + }, + } + out_svg, out_csv = _resolve_output_paths(root, args.metric, args.stat, args.out, args.csv) + request = PlotRequest( + csv_path=out_csv, + out_svg=out_svg, + title=f"{metric.title}: {args.stat} time vs dimension", + stat=args.stat, + dims=CANONICAL_DIMS, + la_label=_format_legend_label("la-stack", context["release"].removeprefix("v")), + na_label=_format_legend_label("nalgebra", baseline_context["dependency.nalgebra"]), + fa_label=_format_legend_label("faer", baseline_context["dependency.faer"]), + log_y=args.log_y, + ) + return _stage_and_publish_outputs(root=root, args=args, rows=rows, req=request, provenance=provenance, skipped=[], validated_inputs=inputs) + + def main(argv: list[str] | None = None) -> int: # noqa: C901, PLR0911, PLR0912, PLR0915 """Generate benchmark CSV and optional SVG or README output.""" args = _parse_args(sys.argv[1:] if argv is None else argv) @@ -1219,6 +1291,19 @@ def main(argv: list[str] | None = None) -> int: # noqa: C901, PLR0911, PLR0912, return 2 if rc != 0: return rc + if ( + args.update_readme + and args.performance_csv == _DEFAULT_PERFORMANCE_CSV + and any( + (root / path).exists() or (root / path).is_symlink() + for path in ("target/bench-reports/performance.run.json", "target/bench-reports/performance.evidence.json", "docs/performance-runs") + ) + ): + try: + return _complete_readme(root, args) + except (OSError, ValueError, TypeError, KeyError, ExceptionGroup) as error: + print(f"Invalid complete performance run: {format_exception_diagnostics(error)}", file=sys.stderr) + return 2 versions = _detect_versions(root) _print_versions(versions) diff --git a/scripts/criterion_measurements.py b/scripts/criterion_measurements.py deleted file mode 100644 index 205cdba..0000000 --- a/scripts/criterion_measurements.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Validate raw Criterion measurements shared by local and hosted retention.""" - -import json -from typing import TYPE_CHECKING, cast - -from research_repo_tools.criterion import Estimate, parse_estimate - -if TYPE_CHECKING: - from pathlib import Path - - -def read_object(path: Path) -> dict[str, object]: - """Require a JSON object at the raw artifact boundary.""" - data = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(data, dict): - raise TypeError(f"expected a JSON object in {path}") - return cast("dict[str, object]", data) - - -def positive_number(value: object) -> float: - """Reject booleans, nonnumeric values, and invalid timing numbers.""" - return Estimate(cast("float", value)).point - - -def validate_measurement(directory: Path) -> None: - """Check full default sampling and the estimates used by report consumers.""" - samples = read_object(directory / "sample.json") - for field in ("iters", "times"): - values = samples.get(field) - if not isinstance(values, list) or len(values) != 100: - raise ValueError(f"{directory}: {field} must contain 100 samples") - for index, value in enumerate(values): - try: - positive_number(value) - except ValueError as exc: - raise ValueError(f"{directory / 'sample.json'}: {field}[{index}]: {exc}") from exc - payload = (directory / "estimates.json").read_bytes() - for statistic in ("mean", "median"): - try: - estimate = parse_estimate(payload, statistic=statistic) - except ValueError as exc: - raise ValueError(f"{directory / 'estimates.json'}: {statistic}: {exc}") from exc - if estimate.lower is None or estimate.upper is None or estimate.confidence_level != 0.95: - raise ValueError(f"{directory}: expected a complete 95% confidence interval for {statistic}") diff --git a/scripts/performance_phase.py b/scripts/performance_phase.py new file mode 100644 index 0000000..b4b5c93 --- /dev/null +++ b/scripts/performance_phase.py @@ -0,0 +1,52 @@ +"""Consumer Cargo commands for the captured common benchmark harness.""" + +import argparse +import os +from pathlib import Path + +from research_repo_tools.process import run_command, run_command_live + +SUITES = {"vs_linalg": ("-p", "la-stack-comparison", "--features", "bench"), "exact": ("--features", "bench,exact")} + + +def commands(suite: str, phase: str) -> tuple[tuple[str, ...], ...]: + """Declare complete baseline timing and current la-stack-only timing.""" + selected = SUITES if suite == "all" else {suite: SUITES[suite]} + return tuple( + ("cargo", "bench", "--locked", *features, "--bench", name, "--", *(("la_stack",) if phase == "current" and name == "vs_linalg" else ()), "--noplot") + for name, features in selected.items() + ) + + +def discover(root: Path, suite: str, phase: str, env: dict[str, str]) -> tuple[str, ...]: + """Ask native Criterion binaries for semantic IDs, without timing samples.""" + expected: list[str] = [] + for command in commands(suite, phase): + result = run_command(command[0], [*command[1:], "--list"], cwd=root, env=env, timeout=7200) + ids = [line.removesuffix(": benchmark") for line in result.stdout.splitlines() if line.endswith(": benchmark")] + if not ids: + raise ValueError(f"empty Criterion inventory from {command}") + expected.extend(ids) + if len(expected) != len(set(expected)): + msg = "duplicate semantic benchmark IDs" + raise ValueError(msg) + return tuple(sorted(expected)) + + +def main() -> None: + """Execute one declared correctness or timing phase with inherited live output.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("gate", "measure")) + parser.add_argument("suite", choices=("all", *SUITES)) + parser.add_argument("phase", choices=("baseline", "current")) + args = parser.parse_args() + selected = commands(args.suite, args.phase) + if args.action == "gate": + selected = (("cargo", "test", "--locked", "--workspace", "--features", "bench,exact", "--test", "vs_linalg_inputs", "--test", "exact_bench_config"),) + for command in selected: + print(f"[performance] {args.phase} {args.action}: {' '.join(command)}", flush=True) + run_command_live(command[0], command[1:], cwd=Path.cwd(), env=os.environ, timeout=7200) + + +if __name__ == "__main__": + main() diff --git a/scripts/performance_runs.py b/scripts/performance_runs.py new file mode 100644 index 0000000..7abd89b --- /dev/null +++ b/scripts/performance_runs.py @@ -0,0 +1,308 @@ +"""la-stack scientific policy and rendering over shared complete-run contracts.""" + +import json +import os +import sys +import tomllib +from pathlib import Path +from typing import TYPE_CHECKING + +from research_repo_tools.common_measurement import CommonHarnessPlan, MeasurementPhase +from research_repo_tools.complete_runs import CompletePolicy, CompleteRun, RunSeries, run_identity, validate_run_evidence +from research_repo_tools.evidence import Evidence, fingerprint_files, load_evidence +from research_repo_tools.measurement import MeasurementConfig +from research_repo_tools.release_pairs import ReleasePair +from research_repo_tools.run_reports import load_latest_run + +import bench_compare as report +from performance_artifacts import NO_API_COMPATIBILITY, resolve_shared_harness_compatibility +from performance_phase import commands, discover + +if TYPE_CHECKING: + from research_repo_tools.criterion import Estimate + +ARCHIVE = "docs/performance-runs" +REPORT_CONFIG = ".config/performance-report.toml" +SOURCES = ("src/**/*.rs", "Cargo.toml") +HARNESS = ( + "benches/**/*.rs", + "benches/comparison/Cargo.toml", + "examples/**/*.rs", + "Cargo.toml", + "Cargo.lock", + "rust-toolchain.toml", + "tests/exact_bench_config.rs", + "scripts/performance_phase.py", +) +UNAVAILABLE_PREFIXES = ("rational_input_", "canonical_conversion_", "det4_diagnostic_") + + +def selected_ids(suite: str, scope: str, expected: tuple[str, ...]) -> tuple[str, ...]: + """Require the repository report registry, keeping diagnostic cases in the run.""" + if suite not in {"all", "exact", "vs_linalg"} or scope not in {"all-benches", "release-signal"}: + msg = "unknown la-stack suite or report scope" + raise ValueError(msg) + if scope == "all-benches": + return tuple(name for name in expected if not name.split("/")[0].removeprefix("d").isdigit() or "/la_stack_" in name) + required = { + *(f"{group}/{bench}" for group, benches in report.EXACT_GROUPS.items() for bench in benches if suite != "vs_linalg"), + *( + f"d{dimension}/{bench}" + for dimension in report.VS_LINALG_CANONICAL_DIMS + for bench in (*report.VS_LINALG_LA_STACK_BENCHES, *report.VS_LINALG_RELEASE_SIGNAL_BENCHES_BY_DIM.get(dimension, [])) + if suite != "exact" + ), + } + missing = required - set(expected) + if missing: + raise ValueError(f"inventory omits selected report rows: {sorted(missing)}") + return tuple(sorted(required)) + + +def phase_environment(root: Path, adapter: str = NO_API_COMPATIBILITY) -> tuple[tuple[str, str], ...]: + """Capture code-generation overrides and isolate each phase's output tree.""" + toolchain = tomllib.loads((root / "rust-toolchain.toml").read_text(encoding="utf-8"))["toolchain"]["channel"] + env = {key: os.environ[key] for key in ("RUSTFLAGS", "CARGO_ENCODED_RUSTFLAGS", "RUSTC", "RUSTC_WRAPPER", "CARGO_BUILD_TARGET") if key in os.environ} + env.update(RUSTUP_TOOLCHAIN=os.environ.get("RUSTUP_TOOLCHAIN", toolchain), CARGO_TARGET_DIR="target", CRITERION_HOME="target/criterion") + key, separator = ("CARGO_ENCODED_RUSTFLAGS", "\x1f") if "CARGO_ENCODED_RUSTFLAGS" in env else ("RUSTFLAGS", " ") + flags = [env.get(key, ""), "--cap-lints=warn"] + if adapter != NO_API_COMPATIBILITY: + flags.append(f"--cfg={adapter}") + env[key] = separator.join(flag for flag in flags if flag) + return tuple(sorted(env.items())) + + +def measurement_plan(root: Path, pair: ReleasePair, suite: str, scope: str, listing: Path) -> CommonHarnessPlan: + """Build the shared plan from native inventories and consumer API policy.""" + compatibility = resolve_shared_harness_compatibility(current=pair.current, baseline=pair.baseline, shared_harness_rational_inputs=True) + baseline_env = phase_environment(root, compatibility.baseline_api_compatibility) + current_env = phase_environment(root) + inventories = {} + for phase in ("baseline", "current"): + # Discover against the current API, then apply the exact groups excluded + # by the historical cfg. Old method names cannot compile on current code. + # The shared collector later checks this entire inventory on each source. + env = os.environ | dict(current_env) | {"CRITERION_HOME": str(listing / phase)} + ids = discover(root, suite, phase, env) + old_api = phase == "baseline" and compatibility.baseline_api_compatibility != NO_API_COMPATIBILITY + inventories[phase] = tuple(name for name in ids if not (old_api and name.startswith(UNAVAILABLE_PREFIXES))) + return declared_plan(pair, suite, scope, inventories, baseline_env=baseline_env, current_env=current_env) + + +def declared_plan( # noqa: PLR0913 + pair: ReleasePair, + suite: str, + scope: str, + inventories: dict[str, tuple[str, ...]], + *, + baseline_env: tuple[tuple[str, str], ...], + current_env: tuple[tuple[str, str], ...], +) -> CommonHarnessPlan: + """Bind exact case inventories, reference phases, commands and completeness.""" + compatibility = resolve_shared_harness_compatibility(current=pair.current, baseline=pair.baseline, shared_harness_rational_inputs=True) + selected = selected_ids(suite, scope, inventories["current"]) + unavailable = compatibility.baseline_api_compatibility != NO_API_COMPATIBILITY + baseline_rows = tuple(name for name in selected if not (unavailable and name.startswith(UNAVAILABLE_PREFIXES))) + missing = set(baseline_rows) - set(inventories["baseline"]) + if missing: + raise ValueError(f"baseline inventory omits selected rows: {sorted(missing)}") + series = [ + RunSeries("la-stack baseline", "baseline", tuple((name, name) for name in baseline_rows)), + RunSeries("la-stack current", "current", tuple((name, name) for name in selected)), + ] + for library, peer_index in (("nalgebra", 0), ("faer", 1)): + rows = tuple( + (name, f"{name.split('/')[0]}/{report.VS_LINALG_BASELINE_PEERS[name.split('/')[1]][peer_index]}") + for name in selected + if name.startswith("d") and name.split("/")[1] in report.VS_LINALG_BASELINE_PEERS + ) + if rows: + series.append(RunSeries(library, "baseline", rows)) + phases = { + phase: MeasurementPhase( + (sys.executable, "scripts/performance_phase.py", "measure", suite, phase), + (sys.executable, "scripts/performance_phase.py", "gate", suite, phase), + CompletePolicy(inventories[phase], sample_count=100, statistics=("mean", "median"), confidence_level=0.95), + environment, + ) + for phase, environment in (("baseline", baseline_env), ("current", current_env)) + } + dependencies = tuple((name, "Cargo.lock") for name in ("criterion", "nalgebra", "faer")) + probes = (("rustc", ("rustc", "-vV")), ("cargo", ("cargo", "--version"))) + config = MeasurementConfig( + phases["current"].command, + SOURCES, + HARNESS, + probes=probes, + dependencies=dependencies, + context=( + ("suite", suite), + ("scope", scope), + ("baseline-api-compatibility", compatibility.baseline_api_compatibility), + ("baseline-cargo-commands", json.dumps(commands(suite, "baseline"))), + ("current-cargo-commands", json.dumps(commands(suite, "current"))), + ), + compatible=( + "harness_sha256", + "context.os", + "context.architecture", + "context.cpu", + *(f"context.tool.{name}" for name, _ in probes), + *(f"context.dependency.{name}" for name, _ in dependencies), + ), + ) + return CommonHarnessPlan(config, phases["baseline"], phases["current"], tuple(series)) + + +def validate_scientific_run(evidence: Evidence, *, release: bool = False) -> CompleteRun: # noqa: C901 + """Require la-stack's sampling, selected coverage and original reference phases.""" + run = validate_run_evidence(evidence) + sources = dict(evidence.sources) + context = dict(sources["current"].context) + pair = ReleasePair(context["release"], dict(sources["baseline"].context)["release"]) + compatibility = resolve_shared_harness_compatibility(current=pair.current, baseline=pair.baseline, shared_harness_rational_inputs=True) + if release and pair.current == pair.baseline: + msg = "cannot promote a same-version local comparison as release documentation" + raise ValueError(msg) + if pair.current == pair.baseline and sources["current"].source_sha256 == sources["baseline"].source_sha256: + msg = "same-version local comparisons require distinct source identities" + raise ValueError(msg) + samples = dict(run.phases) + for sample in samples.values(): + if sample.policy != CompletePolicy(sample.policy.expected): + msg = "la-stack requires 100 samples, mean and median, and 95% intervals in ns" + raise ValueError(msg) + expected = declared_plan( + pair, context["suite"], context["scope"], {phase: sample.policy.expected for phase, sample in samples.items()}, baseline_env=(), current_env=() + ) + if run.series != expected.series or context["baseline-api-compatibility"] != compatibility.baseline_api_compatibility: + msg = "retained series or compatibility policy differs from la-stack declarations" + raise ValueError(msg) + if set(run.compatible) != set(expected.measurement.compatible): + msg = "retained run omits required host, toolchain, or dependency compatibility" + raise ValueError(msg) + for phase, source in evidence.sources: + actual = dict(source.context) + if any(actual.get(key) != value for key, value in expected.measurement.context): + msg = "retained phase differs from the declared consumer policy" + raise ValueError(msg) + for key, command in (("command", getattr(expected, phase).command), ("gate-command", getattr(expected, phase).gate)): + # Interpreter paths vary across hosts; retain them without executing + # evidence-supplied commands during offline rendering. + if tuple(json.loads(actual[key]))[1:] != command[1:]: + raise ValueError(f"retained {phase} {key} differs from the declared Cargo phase") + return run + + +def scratch_paths(stem: Path) -> tuple[Path, Path]: + """Use new paths and framing, independent of historical CSV/JSON evidence.""" + return stem.with_suffix(".run.json"), stem.with_suffix(".evidence.json") + + +def retained_run(root: Path, stem: Path) -> Evidence: + """Resolve validated shared history only when every scratch input is absent.""" + paths = scratch_paths(stem) + legacy = (stem.with_suffix(".csv"), stem.with_suffix(".provenance.json"), stem.with_suffix(".full.csv"), stem.with_suffix(".full.provenance.json")) + # Partial new or legacy scratch must never silently select older history. + present = any(path.exists() or path.is_symlink() for path in (*paths, *legacy)) + evidence = load_evidence(*paths) if present else load_latest_run(root, ARCHIVE) + validate_scientific_run(evidence) + return evidence + + +def verify_current_source(root: Path, evidence: Evidence) -> None: + """Bind README publication to the measured current library and captured harness.""" + source = dict(evidence.sources)["current"] + context = dict(source.context) + for key, expected in (("source-inventory", source.source_sha256), ("harness-inventory", source.harness_sha256)): + names = json.loads(context[key]) + patterns = SOURCES if key == "source-inventory" else HARNESS + actual = {path.relative_to(root).as_posix() for pattern in patterns for path in root.glob(pattern)} + if actual != set(names): + raise ValueError(f"current {key} has stale file membership") + if fingerprint_files(root, tuple(Path(name) for name in names)) != expected: + raise ValueError(f"current {key} is stale relative to retained measurements") + version = tomllib.loads((root / "Cargo.toml").read_text(encoding="utf-8"))["package"]["version"] + if f"v{version}" != context["release"]: + msg = "retained release does not match the current package" + raise ValueError(msg) + + +def estimates_by_series(run: CompleteRun, statistic: str = "median") -> dict[str, dict[str, Estimate]]: + """Select each named view from its actual measurement phase.""" + if statistic not in {"mean", "median"}: + raise ValueError(f"unsupported statistic: {statistic}") + phases = {phase: {case.full_id: case for case in sample.cases} for phase, sample in run.phases} + return {series.name: {label: phases[series.phase][full_id].estimate(statistic) for label, full_id in series.rows} for series in run.series} + + +def render_scientific_report(evidence: Evidence) -> bytes: + """Retain la-stack's comparison tables and cautious statistical interpretation.""" + run = validate_scientific_run(evidence) + sources = dict(evidence.sources) + current, baseline = (dict(sources[phase].context)["release"] for phase in ("current", "baseline")) + values = estimates_by_series(run) + comparisons = [] + current_only = [] + for name, estimate in values["la-stack current"].items(): + group, bench = name.split("/", 1) + prior = values["la-stack baseline"].get(name) + if prior is None: + current_only.append(f"- `{name}`: current {report.format_time(estimate.point)}; baseline API unavailable.") + continue + + def converted(value: Estimate | None) -> report.CriterionEstimate | None: + return report.CriterionEstimate(value.point, value.lower, value.upper) if value is not None else None + + comparisons.append( + report.Comparison( + suite="exact" if group.startswith(("exact_", *UNAVAILABLE_PREFIXES)) else "vs_linalg", + group=group, + bench=bench, + baseline_bench=bench, + baseline=report.CriterionEstimate(prior.point, prior.lower, prior.upper), + current=report.CriterionEstimate(estimate.point, estimate.lower, estimate.upper), + assessment=report.assess_change( + report.CriterionEstimate(prior.point, prior.lower, prior.upper), report.CriterionEstimate(estimate.point, estimate.lower, estimate.upper) + ), + baseline_nalgebra=converted(values.get("nalgebra", {}).get(name)), + baseline_faer=converted(values.get("faer", {}).get(name)), + ) + ) + lines = [ + "# Benchmark Performance", + "", + f"**la-stack** {current} · `{sources['current'].revision}`", + "", + f"Comparison against baseline **{baseline}**:", + "", + "**Statistic**: median; complete mean and median measurements are retained.", + "", + "Negative point-estimate change means a smaller current estimate; a baseline/current ratio above 1 has the same meaning.", + "Marginal Criterion interval separation is not a paired confidence interval or a statistical-significance claim.", + "", + "nalgebra/faer reference measurements originate in the baseline phase under the captured current harness; they were not rerun in the current phase.", + "", + f"**Complete run**: `{run_identity(evidence)}`", + "", + ] + for phase, source in evidence.sources: + context = dict(source.context) + lines.extend( + [ + f"- {phase}: revision `{source.revision}`, source `{source.source_sha256}`, harness `{source.harness_sha256}`.", + f" Gate `{context['gate-command']}`: {context['gate-status']}; command `{context['command']}`.", + ] + ) + lines.extend( + [ + "", + report.comparison_tables(comparisons, baseline), + "", + *current_only, + "", + "Regenerate from retained evidence with `just performance-doc`; update README with `just performance-readme`.", + "", + ] + ) + return "\n".join(lines).encode() diff --git a/scripts/release_baseline.py b/scripts/release_baseline.py index 14880e1..9f33a89 100644 --- a/scripts/release_baseline.py +++ b/scripts/release_baseline.py @@ -8,6 +8,7 @@ import sys from pathlib import Path +from research_repo_tools.complete_runs import CompletePolicy, collect_complete_sample from research_repo_tools.process import ExecutableNotFoundError, format_exception_diagnostics, run_command from bench_compare import ( @@ -16,7 +17,6 @@ VS_LINALG_RELEASE_SIGNAL_BENCHES_BY_DIM, VS_LINALG_STANDARD_BENCH_ORDER, ) -from criterion_measurements import read_object, validate_measurement SUITES = {"vs_linalg": "bench", "exact": "bench,exact"} RAW_FILES = ("benchmark.json", "estimates.json", "sample.json", "tukey.json") @@ -95,14 +95,12 @@ def validate(criterion: Path, manifest: Path, baseline: str) -> int: raise ValueError(msg) expected = inventory_ids(json.loads(manifest.read_text(encoding="utf-8"))) report_ids = required_report_ids() - observed: set[str] = set() - for metadata in sorted(criterion.glob("**/new/benchmark.json")): - benchmark = read_object(metadata).get("full_id") - if not isinstance(benchmark, str) or benchmark not in expected or benchmark in observed: - raise ValueError(f"unexpected or duplicate benchmark ID in {metadata}: {benchmark!r}") - directory = metadata.parent + complete = collect_complete_sample(criterion, "new", CompletePolicy(tuple(sorted(expected)))) + for case in complete.cases: + benchmark = case.full_id + directory = criterion / case.storage_path / "new" if benchmark in report_ids and directory.parent != criterion / benchmark: - raise ValueError(f"report benchmark is outside its consumer path: {metadata}") + raise ValueError(f"report benchmark is outside its consumer path: {directory}") saved = directory.parent / baseline for filename in RAW_FILES: # Criterion can log a write/copy error and still exit successfully. @@ -112,12 +110,7 @@ def validate(criterion: Path, manifest: Path, baseline: str) -> int: if source.read_bytes() != destination.read_bytes(): raise ValueError(f"saved baseline differs from new measurement: {destination}") json.loads(source.read_text(encoding="utf-8")) - validate_measurement(directory) - observed.add(benchmark) - missing = expected - observed - if missing: - raise ValueError(f"incomplete Criterion dataset: {', '.join(sorted(missing))}") - return len(observed) + return len(complete.cases) def main(argv: list[str] | None = None) -> int: diff --git a/scripts/tests/test_archive_performance.py b/scripts/tests/test_archive_performance.py deleted file mode 100644 index 27fe071..0000000 --- a/scripts/tests/test_archive_performance.py +++ /dev/null @@ -1,2660 +0,0 @@ -"""Tests for archive_performance.py.""" - -import csv -import io -import json -import shutil -import subprocess -import tarfile -from pathlib import Path -from types import SimpleNamespace -from typing import TYPE_CHECKING, Any, Never - -import pytest -from research_repo_tools.process import run_command - -import archive_performance -from archive_performance import ( - GenerationConfig, - generate_and_promote_worktree_report, - generate_worktree_report, - main, - normalize_tag, - parse_report_id, - promote_report, -) -from benchmark_summaries import collect_measurements, full_summary_paths, resolve_report_paths -from criterion_dim_plot import _resolve_performance_paths -from performance_artifacts import ( - ArtifactContext, - ArtifactPaths, - PerformanceBundle, - PerformanceRow, - ReleasePair, - ReportSource, - TimingEstimate, - write_bundle, -) - -if TYPE_CHECKING: - from collections.abc import Sequence - -type RunnerCall = tuple[str, tuple[str, ...], Path | None] - - -@pytest.fixture(autouse=True) -def _stable_cpu_description(monkeypatch: pytest.MonkeyPatch) -> None: - """Keep release-generation tests independent of the test host.""" - monkeypatch.setattr(archive_performance, "cpu_description", lambda: "Test CPU (x86_64)") - - -def _result(stdout: str = "") -> SimpleNamespace: - return SimpleNamespace(stdout=stdout) - - -def _git(repo_root: Path, *args: str) -> str: - return archive_performance.run_git_command(list(args), cwd=repo_root).stdout - - -def _report(version: str, baseline: str) -> str: - return ( - "# Benchmark Performance\n\n" - f"**la-stack** v{version} · `abc1234` (release/test) · 2026-06-08 12:00:00 UTC\n" - "**Statistic**: median\n" - "**Suite**: all\n" - "**Scope**: release-signal\n\n" - "## Benchmark Results\n\n" - f"Comparison against baseline **{baseline}**:\n\n" - "Negative change = faster. Speedup > 1.00x = improvement.\n\n" - "## Exact arithmetic\n\n" - "| Benchmark | Baseline | Latest | Change | Speedup |\n" - "|-----------|---------:|-------:|-------:|--------:|\n" - "| det_exact | 1.0 ns | 0.9 ns | -10.0% | 1.11x |\n" - ) - - -def _normalized_report(version: str, baseline: str) -> str: - return archive_performance._normalize_how_to_update(_report(version, baseline)) - - -def _write_fake_rendered_artifacts(args: Sequence[str], *, version: str, baseline: str) -> None: - output = Path(args[args.index("--output") + 1]) - if "--csv-output" not in args: - output.write_text(_report(version, baseline), encoding="utf-8") - return - - criterion_dir = Path(args[args.index("--criterion-dir") + 1]) - suite = args[args.index("--suite") + 1] - scope = args[args.index("--scope") + 1] - row_suite = "vs_linalg" if suite == "vs_linalg" else "exact" - group = "d2" if row_suite == "vs_linalg" else "exact_d2" - benchmark = "la_stack_lu" if row_suite == "vs_linalg" else "det_exact" - timing = TimingEstimate(median_ns=10.0, ci_lower_ns=9.0, ci_upper_ns=11.0) - paths = ArtifactPaths( - csv=Path(args[args.index("--csv-output") + 1]), - provenance=Path(args[args.index("--provenance-output") + 1]), - ) - benchmark_provenance = json.loads((criterion_dir / archive_performance._BENCHMARK_HARNESS_METADATA).read_text(encoding="utf-8")) - assert isinstance(benchmark_provenance, dict) - bundle = PerformanceBundle( - context=ArtifactContext( - release=ReleasePair(current=f"v{version}", baseline=baseline), - statistic="median", - suite=suite, - scope=scope, - source=ReportSource( - version=version, - commit="abc1234", - ref="release/test", - revision_timestamp="2026-06-08 12:00:00 UTC", - ), - benchmark_provenance=benchmark_provenance, - ), - rows=( - PerformanceRow( - suite=row_suite, - scope=scope, - benchmark_id=f"{group}/{benchmark}", - group=group, - benchmark=benchmark, - baseline_benchmark=benchmark, - coverage_status="comparable", - coverage_note="", - baseline=timing, - current=TimingEstimate(median_ns=9.0, ci_lower_ns=8.0, ci_upper_ns=10.0), - ), - ), - ) - write_bundle(paths, bundle) - output.write_text(archive_performance.render_release_artifacts(paths), encoding="utf-8") - # The fake renderer stands in for validated timing output. Supply complete - # raw samples so the independent summary exporter exercises real parsing. - for sample, value in ((baseline, 10.0), ("new", 9.0)): - _write_measurement(criterion_dir, f"{group}/{benchmark}", sample, value) - - -def _write_measurement(criterion: Path, benchmark: str, sample: str, value: float) -> None: - directory = criterion / benchmark / sample - directory.mkdir(parents=True, exist_ok=True) - (directory / "benchmark.json").write_text(json.dumps({"full_id": benchmark}), encoding="utf-8") - estimate = { - "point_estimate": value, - "confidence_interval": {"confidence_level": 0.95, "lower_bound": value * 0.9, "upper_bound": value * 1.1}, - } - (directory / "estimates.json").write_text(json.dumps({"mean": estimate, "median": estimate}), encoding="utf-8") - (directory / "sample.json").write_text(json.dumps({"iters": [1.0] * 100, "times": [value] * 100}), encoding="utf-8") - - -def _retained_report(root: Path, *, stem: str = "performance") -> str: - paths = ArtifactPaths( - csv=root / "target" / "bench-reports" / f"{stem}.csv", - provenance=root / "target" / "bench-reports" / f"{stem}.provenance.json", - ) - return archive_performance._normalize_how_to_update(archive_performance.render_release_artifacts(paths)) - - -def test_normalized_report_uses_shared_how_to_update_section() -> None: - text = _normalized_report("0.4.6", "v0.4.5") - - assert text.endswith(archive_performance.HOW_TO_UPDATE_SECTION) - - -def _legacy_report(version: str, baseline: str) -> str: - return ( - _report(version, baseline) - + "\n" - + "## How to Update\n\n" - + "```bash\n" - + "git checkout v0.2.0\n" - + "just bench-save-baseline v0.2.0\n" - + "git checkout main\n" - + "just bench-compare v0.2.0\n" - + "```\n" - ) - - -def _write_baseline_archive(path: Path, *, include_harness_metadata: bool = False) -> None: - tag = path.name.removeprefix("la-stack-").removesuffix("-criterion-baseline.tar.gz") - fixture_dir = path.parent / f"baseline-fixture-{tag}" - criterion_dir = fixture_dir / "criterion" - criterion_dir.mkdir(parents=True) - (criterion_dir / "placeholder.txt").write_text("baseline\n", encoding="utf-8") - sample_dir = criterion_dir / "exact_d2" / "det_exact" / tag - sample_dir.mkdir(parents=True) - (sample_dir / "estimates.json").write_text('{"median":{"point_estimate":1.0}}\n', encoding="utf-8") - if include_harness_metadata: - (criterion_dir / archive_performance._BENCHMARK_HARNESS_METADATA).write_text( - json.dumps( - { - "baseline": tag, - "mode": "shared-current-harness", - "schema": 1, - "sha256": "a" * 64, - } - ), - encoding="utf-8", - ) - with tarfile.open(path, "w:gz") as tar: - tar.add(criterion_dir, arcname="criterion") - - -def _write_unsafe_baseline_archive(path: Path) -> None: - payload = b"escape\n" - info = tarfile.TarInfo("../escape.txt") - info.size = len(payload) - with tarfile.open(path, "w:gz") as tar: - tar.addfile(info, io.BytesIO(payload)) - - -def _write_current_benchmark_tooling(worktree: Path) -> None: - (worktree / ".config").mkdir(parents=True, exist_ok=True) - (worktree / "scripts").mkdir(parents=True, exist_ok=True) - (worktree / "benches" / "common").mkdir(parents=True, exist_ok=True) - (worktree / "benches/comparison/tests").mkdir(parents=True, exist_ok=True) - (worktree / "examples").mkdir(parents=True, exist_ok=True) - (worktree / "src").mkdir(parents=True, exist_ok=True) - (worktree / "tests").mkdir(parents=True, exist_ok=True) - (worktree / "benches" / "vs_linalg.rs").write_text("fn main() {}\n", encoding="utf-8") - (worktree / "benches" / "common" / "inputs.rs").write_text("pub const INPUT: f64 = 1.0;\n", encoding="utf-8") - (worktree / "src" / "lib.rs").write_text("pub fn fixture() {}\n", encoding="utf-8") - (worktree / "tests" / "exact_bench_config.rs").write_text("// exact fixture\n", encoding="utf-8") - (worktree / "benches/comparison/tests/vs_linalg_inputs.rs").write_text("// linalg fixture\n", encoding="utf-8") - (worktree / "benches/comparison/Cargo.toml").write_text( - '[package]\nname = "fixture-comparison"\nversion = "0.1.0"\nedition = "2024"\n' - '[dev-dependencies]\nfixture = { path = "../.." }\n' - '[features]\nbench = []\nexact = ["fixture/exact"]\n', - encoding="utf-8", - ) - (worktree / ".config" / "nextest.toml").write_text("[profile.ci]\nretries = 1\n", encoding="utf-8") - (worktree / "Cargo.toml").write_text( - '[package]\nname = "fixture"\nversion = "0.1.0"\n[dev-dependencies]\ncriterion = "0.7.0"\n', - encoding="utf-8", - ) - (worktree / "Cargo.lock").write_text("version = 4\n", encoding="utf-8") - (worktree / "rust-toolchain.toml").write_text('[toolchain]\nchannel = "1.97.0"\n', encoding="utf-8") - (worktree / "justfile").write_text( - 'bench-save-baseline tag suite="all":\nbench-latest: bench-vs-linalg-la-stack bench-exact\n', - encoding="utf-8", - ) - (worktree / "scripts" / "bench_compare.py").write_text('parser.add_argument("--suite")\nparser.add_argument("--scope")\n', encoding="utf-8") - - -def _write_legacy_benchmark_tooling(worktree: Path) -> None: - _write_current_benchmark_tooling(worktree) - (worktree / "justfile").write_text("bench-exact:\n", encoding="utf-8") - (worktree / "scripts" / "bench_compare.py").write_text('parser.add_argument("--output")\n', encoding="utf-8") - - -def test_shared_benchmark_harness_replaces_baseline_content_and_has_stable_digest(tmp_path: Path) -> None: - current = tmp_path / "current" - baseline = tmp_path / "baseline" - current.mkdir() - baseline.mkdir() - _write_current_benchmark_tooling(current) - _write_current_benchmark_tooling(baseline) - (baseline / "tests/vs_linalg_inputs.rs").write_text('compile_error!("obsolete comparison test");\n', encoding="utf-8") - (baseline / "benches" / "vs_linalg.rs").write_text("fn obsolete() {}\n", encoding="utf-8") - (baseline / "Cargo.lock").write_text("version = 3\n", encoding="utf-8") - (baseline / "justfile").write_text("obsolete-benchmark-recipe:\n", encoding="utf-8") - (baseline / ".config" / "nextest.toml").write_text("[profile.default]\nretries = 0\n", encoding="utf-8") - - digest = archive_performance._install_shared_benchmark_harness( - source=current, - destination=baseline, - ) - - assert digest == archive_performance._benchmark_harness_digest(current) - assert digest == archive_performance._benchmark_harness_digest(baseline) - assert (baseline / "benches" / "vs_linalg.rs").read_text(encoding="utf-8") == "fn main() {}\n" - assert (baseline / "justfile").read_text(encoding="utf-8") == (current / "justfile").read_text(encoding="utf-8") - assert (baseline / ".config" / "nextest.toml").read_text(encoding="utf-8") == (current / ".config" / "nextest.toml").read_text(encoding="utf-8") - assert not (baseline / "tests/vs_linalg_inputs.rs").exists() - - -@pytest.mark.parametrize("example_path", ["examples/rational_input_5x5.rs", "examples/rational_input_5x5/main.rs"]) -def test_shared_harness_preserves_manifest_example_targets(tmp_path: Path, example_path: str) -> None: - """Cargo must discover new examples before it can run historical benchmark tests.""" - current = tmp_path / "current" - baseline = tmp_path / "baseline" - _write_current_benchmark_tooling(current) - _write_current_benchmark_tooling(baseline) - baseline_source = baseline / "src" / "lib.rs" - baseline_source.write_text("pub fn historical_api() {}\n", encoding="utf-8") - (current / "src" / "lib.rs").write_text("pub fn current_api() {}\n", encoding="utf-8") - example = current / example_path - example.parent.mkdir(parents=True, exist_ok=True) - example.write_text("fn main() { fixture::current_api(); }\n", encoding="utf-8") - manifest = current / "Cargo.toml" - manifest.write_text( - '[package]\nname = "fixture"\nversion = "0.1.0"\nedition = "2024"\n' - "[features]\nbench = []\nexact = []\n" - '[[example]]\nname = "rational_input_5x5"\nrequired-features = ["exact"]\n' - '[workspace]\nmembers = ["benches/comparison"]\nresolver = "3"\n', - encoding="utf-8", - ) - - digest = archive_performance._install_shared_benchmark_harness(source=current, destination=baseline) - run_command( - "cargo", - [ - "test", - "--offline", - "--workspace", - "--no-run", - "--target-dir", - str(tmp_path / "target"), - "--features", - "bench,exact", - "--test", - "vs_linalg_inputs", - "--test", - "exact_bench_config", - "--manifest-path", - str(baseline / "Cargo.toml"), - ], - cwd=Path(__file__).resolve().parents[2], - ) - metadata = json.loads( - run_command( - "cargo", - ["metadata", "--offline", "--no-deps", "--format-version", "1", "--manifest-path", str(baseline / "Cargo.toml")], - cwd=Path(__file__).resolve().parents[2], - ).stdout - ) - - library = next(package for package in metadata["packages"] if package["name"] == "fixture") - examples = [target for target in library["targets"] if target["kind"] == ["example"]] - assert [target["name"] for target in examples] == ["rational_input_5x5"] - assert Path(examples[0]["src_path"]) == baseline / example_path - assert (baseline / example_path).read_bytes() == example.read_bytes() - assert baseline_source.read_text(encoding="utf-8") == "pub fn historical_api() {}\n" - assert digest == archive_performance._benchmark_harness_digest(current) - - -def test_github_release_assets_discard_embedded_shared_harness_metadata(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - for tag in ("v0.4.5", "v0.4.6"): - _write_baseline_archive( - tmp_path / f"la-stack-{tag}-criterion-baseline.tar.gz", - include_harness_metadata=True, - ) - - def fake_download(*, baseline_tag: str, download_dir: Path, repo_root: Path) -> Path: - del download_dir, repo_root - return tmp_path / f"la-stack-{baseline_tag}-criterion-baseline.tar.gz" - - target_worktree = tmp_path / "worktree" - target_worktree.mkdir() - monkeypatch.setattr(archive_performance, "_download_release_baseline", fake_download) - - archive_performance._prepare_github_release_assets( - current_tag="v0.4.6", - baseline_tag="v0.4.5", - repo_root=tmp_path, - target_worktree=target_worktree, - tmp_dir=tmp_path, - ) - - criterion_dir = target_worktree / "target" / "criterion" - assert not (criterion_dir / archive_performance._BENCHMARK_HARNESS_METADATA).exists() - assert (criterion_dir / "exact_d2" / "det_exact" / "new" / "estimates.json").is_file() - - -def test_purge_selected_new_samples_preserves_named_baselines_and_other_suites(tmp_path: Path) -> None: - criterion_dir = tmp_path / "criterion" - exact_new = criterion_dir / "exact_d2" / "det_exact" / "new" - exact_baseline = criterion_dir / "exact_d2" / "det_exact" / "v0.4.5" - rational_new = criterion_dir / "rational_input_d8" / "det_row_cleared_bareiss" / "new" - linalg_new = criterion_dir / "d2" / "la_stack_lu" / "new" - for directory in (exact_new, exact_baseline, rational_new, linalg_new): - directory.mkdir(parents=True) - (directory / "estimates.json").write_text("{}\n", encoding="utf-8") - - removed = archive_performance._purge_criterion_new_samples( - criterion_dir=criterion_dir, - suite="exact", - ) - - assert removed == [exact_new, rational_new] - assert not exact_new.exists() - assert not rational_new.exists() - assert exact_baseline.is_dir() - assert linalg_new.is_dir() - - -def test_complete_summaries_preserve_diagnostic_phases_and_full_precision(tmp_path: Path) -> None: - baseline = "v0.4.5" - peer = "lu_solve_pivoting_d2/faer_lu_solve" - changed = "lu_solve_pivoting_d2/la_stack_lu_solve" - added = "canonical_conversion_non_dyadic_d2/convert" - precise = 1.2345678901234567 - for sample in (baseline, "new"): - _write_measurement(tmp_path, peer, sample, 20.0) - _write_measurement(tmp_path, changed, sample, 10.0) - archive_performance._purge_criterion_new_samples(criterion_dir=tmp_path, suite="all") - _write_measurement(tmp_path, changed, "new", precise) - _write_measurement(tmp_path, added, "new", 3.0) - - rows = collect_measurements(tmp_path, baseline) - - assert {(row.phase, row.benchmark_id) for row in rows} == { - ("baseline", peer), - ("baseline", changed), - ("current", changed), - ("current", added), - } - observed = next(row for row in rows if row.phase == "current" and row.benchmark_id == changed) - assert observed.mean.median_ns == precise - assert observed.median.median_ns == precise - - -@pytest.mark.parametrize("missing", ["benchmark.json", "estimates.json", "sample.json"]) -def test_complete_summaries_reject_incomplete_diagnostics(tmp_path: Path, missing: str) -> None: - case = "det4_diagnostic_dense/det" - for sample in ("v0.4.5", "new"): - _write_measurement(tmp_path, case, sample, 1.0) - (tmp_path / case / "new" / missing).unlink() - - with pytest.raises(FileNotFoundError): - collect_measurements(tmp_path, "v0.4.5") - - -def test_complete_summaries_reject_duplicate_criterion_identities(tmp_path: Path) -> None: - for sample in ("v0.4.5", "new"): - _write_measurement(tmp_path, "diagnostic/first", sample, 1.0) - _write_measurement(tmp_path, "diagnostic/second", "new", 2.0) - (tmp_path / "diagnostic/second/new/benchmark.json").write_text('{"full_id":"diagnostic/first"}', encoding="utf-8") - - with pytest.raises(ValueError, match="duplicate phase/benchmark"): - collect_measurements(tmp_path, "v0.4.5") - - -@pytest.mark.parametrize("line_ending", ["\n", "\r\n"], ids=["lf", "crlf"]) -def test_apply_current_diff_includes_only_tracked_changes_without_mutating_index(tmp_path: Path, line_ending: str) -> None: - repo_root = tmp_path / "repo" - worktree = tmp_path / "worktree" - repo_root.mkdir() - _git(repo_root, "init", "--quiet") - _git(repo_root, "config", "user.name", "Test User") - _git(repo_root, "config", "user.email", "test@example.com") - _git(repo_root, "config", "commit.gpgsign", "false") - _git(repo_root, "config", "core.autocrlf", "false") - - (repo_root / ".gitignore").write_text("ignored.bin\n", encoding="utf-8") - tracked = repo_root / "tracked.txt" - tracked.write_text("committed\n", encoding="utf-8", newline=line_ending) - _git(repo_root, "add", "--", ".gitignore", "tracked.txt") - _git(repo_root, "commit", "--quiet", "-m", "initial") - _git(repo_root, "worktree", "add", "--quiet", "--detach", str(worktree), "HEAD") - - tracked.write_text("staged\n", encoding="utf-8", newline=line_ending) - _git(repo_root, "add", "--", "tracked.txt") - tracked.write_text("working tree\n", encoding="utf-8", newline=line_ending) - binary_payload = bytes(range(256)) * 2 - binary = repo_root / "--untracked.bin" - binary.write_bytes(binary_payload) - link = repo_root / "untracked-link" - try: - link.symlink_to(binary.name) - except OSError as exc: - pytest.skip(f"symlinks unavailable: {exc}") - (repo_root / "ignored.bin").write_bytes(b"ignored\x00payload") - - status_before = _git(repo_root, "status", "--porcelain=v1", "--untracked-files=all") - archive_performance._apply_current_diff_to_worktree(repo_root=repo_root, worktree=worktree) - - assert (worktree / "tracked.txt").read_bytes() == f"working tree{line_ending}".encode() - assert not (worktree / binary.name).exists() - assert not (worktree / link.name).exists() - assert not (worktree / "ignored.bin").exists() - assert _git(repo_root, "show", ":tracked.txt") == f"staged{line_ending}" - assert _git(repo_root, "status", "--porcelain=v1", "--untracked-files=all") == status_before - - -def test_apply_current_diff_preserves_crlf_patch_bytes(tmp_path: Path) -> None: - repo_root = tmp_path / "repo" - worktree = tmp_path / "worktree" - repo_root.mkdir() - _git(repo_root, "init", "--quiet") - _git(repo_root, "config", "user.name", "Test User") - _git(repo_root, "config", "user.email", "test@example.com") - _git(repo_root, "config", "commit.gpgsign", "false") - _git(repo_root, "config", "core.autocrlf", "false") - - (repo_root / ".gitattributes").write_text("tracked.txt -text\n", encoding="utf-8") - tracked = repo_root / "tracked.txt" - tracked.write_bytes(b"committed\r\n") - _git(repo_root, "add", "--", ".gitattributes", tracked.name) - _git(repo_root, "commit", "--quiet", "-m", "initial") - _git(repo_root, "worktree", "add", "--quiet", "--detach", str(worktree), "HEAD") - - tracked.write_bytes(b"staged\r\n") - _git(repo_root, "add", "--", tracked.name) - tracked.write_bytes(b"working tree\r\n") - index_before = _git(repo_root, "rev-parse", f":{tracked.name}") - - archive_performance._apply_current_diff_to_worktree(repo_root=repo_root, worktree=worktree) - - assert (worktree / tracked.name).read_bytes() == b"working tree\r\n" - assert _git(repo_root, "rev-parse", f":{tracked.name}") == index_before - - -def test_apply_current_diff_fails_loudly_and_cleans_temporary_directory( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - repo_root = tmp_path / "repo" - worktree = tmp_path / "worktree" - repo_root.mkdir() - worktree.mkdir() - patch_path: Path | None = None - - def fake_run_git(args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - nonlocal patch_path - assert cwd == repo_root - assert kwargs.get("env") is None - if args[:2] == ["diff", "--binary"]: - patch_path = Path(next(part.removeprefix("--output=") for part in args if part.startswith("--output="))) - assert patch_path.parent.is_dir() - raise subprocess.CalledProcessError( - 128, - ["git", *args], - output="snapshot stdout", - stderr="cannot diff current tree", - ) - return _result() - - monkeypatch.setattr(archive_performance, "run_git_command", fake_run_git) - - with pytest.raises(RuntimeError) as exc_info: - archive_performance._apply_current_diff_to_worktree(repo_root=repo_root, worktree=worktree) - - error = str(exc_info.value) - assert "command failed (128): git diff --binary" in error - assert "snapshot stdout" in error - assert "cannot diff current tree" in error - assert patch_path is not None - assert not patch_path.parent.exists() - - -@pytest.mark.parametrize( - ("failure", "message"), - [ - (archive_performance.ExecutableNotFoundError("missing tool"), "command could not start: tool --flag: missing tool"), - ( - subprocess.TimeoutExpired(["tool", "--flag"], 17, stderr="stalled"), - "command timed out after 17 seconds: tool --flag\nstderr:\nstalled", - ), - ( - subprocess.CalledProcessError(1, ["tool", "--flag"], output=b"partial output\n", stderr=b"failed\n"), - "command failed (1): tool --flag\nstdout:\npartial output\nstderr:\nfailed", - ), - ( - subprocess.TimeoutExpired(["tool", "--flag"], 17, output=b"partial output\n", stderr=b"stalled\n"), - "command timed out after 17 seconds: tool --flag\nstdout:\npartial output\nstderr:\nstalled", - ), - (OSError("working directory unavailable"), "command could not start: tool --flag: working directory unavailable"), - ], -) -def test_run_tool_normalizes_command_failures( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - failure: Exception, - message: str, -) -> None: - def fail_run(*_args: object, **_kwargs: object) -> SimpleNamespace: - raise failure - - monkeypatch.setattr(archive_performance, "run_safe_command", fail_run) - - with pytest.raises(RuntimeError) as exc_info: - archive_performance._run_tool("tool", ["--flag"], cwd=tmp_path) - - assert str(exc_info.value) == message - assert exc_info.value.__cause__ is failure - - -def test_run_tool_can_stream_long_running_command_output( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - observed_kwargs: dict[str, object] = {} - - def fake_run(*_args: object, **kwargs: object) -> SimpleNamespace: - observed_kwargs.update(kwargs) - return _result() - - monkeypatch.setattr(archive_performance, "run_safe_command", fake_run) - - archive_performance._run_tool( - "tool", - ["--flag"], - cwd=tmp_path, - options=archive_performance.ToolRunOptions(stream_output=True), - ) - - assert observed_kwargs["capture_output"] is False - - -def test_benchmark_input_gate_streams_progress( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - calls: list[tuple[str, tuple[str, ...], bool]] = [] - - def fake_run_tool( - command: str, - args: list[str], - *, - cwd: Path, - options: archive_performance.ToolRunOptions, - ) -> None: - del cwd - calls.append((command, tuple(args), options.stream_output)) - - monkeypatch.setattr(archive_performance, "_run_tool", fake_run_tool) - - archive_performance._run_benchmark_input_gate(tmp_path) - - assert calls == [("just", ("test-bench-inputs",), True)] - assert "[performance] validating benchmark inputs" in capsys.readouterr().err - - -def test_temporary_worktree_cleanup_failure_fails_successful_operation( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - def fake_run_git(args: list[str], *, cwd: Path, timeout: int = 600) -> None: - del cwd, timeout - if args[:3] == ["worktree", "remove", "--force"]: - msg = "cleanup failed" - raise RuntimeError(msg) - - monkeypatch.setattr(archive_performance, "_run_git", fake_run_git) - - def complete_operation() -> None: - with archive_performance._temporary_detached_worktree( - repo_root=tmp_path, - worktree=tmp_path / "worktree", - revision="HEAD", - label="test worktree", - ): - pass - - with pytest.raises(RuntimeError, match="failed to remove test worktree"): - complete_operation() - - -def test_temporary_worktree_cleanup_does_not_mask_primary_error( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - def fake_run_git(args: list[str], *, cwd: Path, timeout: int = 600) -> None: - del cwd, timeout - if args[:3] == ["worktree", "remove", "--force"]: - msg = "cleanup failed" - raise RuntimeError(msg) - - monkeypatch.setattr(archive_performance, "_run_git", fake_run_git) - - def fail_operation() -> None: - with archive_performance._temporary_detached_worktree( - repo_root=tmp_path, - worktree=tmp_path / "worktree", - revision="HEAD", - label="test worktree", - ): - msg = "primary failed" - raise ValueError(msg) - - with pytest.raises(RuntimeError, match="operation failed") as exc_info: - fail_operation() - - assert "additionally failed to remove test worktree: cleanup failed" in str(exc_info.value) - assert isinstance(exc_info.value.__cause__, ValueError) - - -def test_normalize_tag_adds_leading_v() -> None: - assert normalize_tag("0.4.5") == "v0.4.5" - assert normalize_tag("v0.4.5") == "v0.4.5" - assert normalize_tag("v1.2.3-rc.1+build.7") == "v1.2.3-rc.1+build.7" - - -def test_normalize_tag_rejects_non_semver_baseline_names() -> None: - with pytest.raises(ValueError, match="semver tag"): - normalize_tag("last") - - -def test_parse_report_id_reads_current_and_baseline_tags() -> None: - report_id = parse_report_id(_report("0.4.5", "v0.4.4")) - - assert report_id.current_tag == "v0.4.5" - assert report_id.baseline_tag == "v0.4.4" - assert report_id.archive_name == "v0.4.5-vs-v0.4.4.md" - - -@pytest.mark.parametrize( - ("extra", "message"), - [ - ("\n**la-stack** v0.4.6 · `def5678` (release/test) · 2026-06-09 12:00:00 UTC\n", "la-stack version line"), - ("\nComparison against baseline **v0.4.0**:\n", "comparison baseline line"), - ], -) -def test_parse_report_id_rejects_duplicate_identity_lines(extra: str, message: str) -> None: - with pytest.raises(ValueError, match=rf"exactly one {message}.*found 2"): - parse_report_id(_report("0.4.5", "v0.4.4") + extra) - - -def test_main_preserves_exception_group_diagnostics( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - def fail_request(_options: object) -> Never: - message = "publication and rollback failed" - raise ExceptionGroup( - message, - [OSError("could not publish docs/performance.md"), OSError("could not restore prior report")], - ) - - monkeypatch.setattr(archive_performance, "resolve_archive_request", fail_request) - - assert main(["v0.4.6", "v0.4.5"]) == 1 - captured = capsys.readouterr() - assert "publication and rollback failed (2 sub-exceptions)" in captured.err - assert "could not publish docs/performance.md" in captured.err - assert "could not restore prior report" in captured.err - - -def test_published_release_pair_discovers_latest_stable_semver_pair(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - assert command == "gh" - assert args == [ - "release", - "list", - "--json", - "tagName,isDraft,isPrerelease,publishedAt", - "--limit", - "100", - ] - assert cwd == tmp_path - return _result( - "[" - '{"tagName":"v0.4.5","isDraft":false,"isPrerelease":false,"publishedAt":"2026-01-01T00:00:00Z"},' - '{"tagName":"v0.4.10","isDraft":false,"isPrerelease":false,"publishedAt":"2026-04-01T00:00:00Z"},' - '{"tagName":"v0.4.11-rc.1","isDraft":false,"isPrerelease":true,"publishedAt":"2026-06-01T00:00:00Z"},' - '{"tagName":"v0.4.11","isDraft":true,"isPrerelease":false,"publishedAt":"2026-06-02T00:00:00Z"},' - '{"tagName":"not-semver","isDraft":false,"isPrerelease":false,"publishedAt":"2026-06-03T00:00:00Z"},' - '{"tagName":"v0.4.6","isDraft":false,"isPrerelease":false,"publishedAt":"2026-03-01T00:00:00Z"}' - "]" - ) - - monkeypatch.setattr(archive_performance, "run_safe_command", fake_run_safe) - - report_id = archive_performance._published_release_pair(tmp_path) - - assert report_id.current_tag == "v0.4.10" - assert report_id.baseline_tag == "v0.4.6" - - -def test_published_release_pair_uses_latest_published_release_not_highest_semver(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - assert command == "gh" - assert cwd == tmp_path - return _result( - "[" - '{"tagName":"v0.5.0","isDraft":false,"isPrerelease":false,"publishedAt":"2026-01-01T00:00:00Z"},' - '{"tagName":"v0.4.9","isDraft":false,"isPrerelease":false,"publishedAt":"2026-02-01T00:00:00Z"},' - '{"tagName":"v0.4.8","isDraft":false,"isPrerelease":false,"publishedAt":"2025-12-01T00:00:00Z"}' - "]" - ) - - monkeypatch.setattr(archive_performance, "run_safe_command", fake_run_safe) - - report_id = archive_performance._published_release_pair(tmp_path) - - assert report_id.current_tag == "v0.4.9" - assert report_id.baseline_tag == "v0.4.8" - - -@pytest.mark.parametrize( - ("field", "value"), - [ - ("isDraft", "false"), - ("isDraft", 0), - ("isPrerelease", None), - ], -) -def test_stable_published_releases_requires_boolean_flags(field: str, value: object) -> None: - release: dict[str, object] = { - "tagName": "v0.4.6", - "isDraft": False, - "isPrerelease": False, - "publishedAt": "2026-02-01T00:00:00Z", - } - release[field] = value - - with pytest.raises(TypeError, match="boolean isDraft and isPrerelease"): - archive_performance._stable_published_releases([release]) - - -@pytest.mark.parametrize("published_at", ["2026-02-01T00:00:00", "not-a-timestamp", ""]) -def test_stable_published_releases_requires_aware_timestamp(published_at: str) -> None: - release = { - "tagName": "v0.4.6", - "isDraft": False, - "isPrerelease": False, - "publishedAt": published_at, - } - - with pytest.raises((TypeError, ValueError), match="publishedAt"): - archive_performance._stable_published_releases([release]) - - -def test_stable_published_releases_normalizes_timestamp_to_utc() -> None: - releases = archive_performance._stable_published_releases( - [ - { - "tagName": "v0.4.6", - "isDraft": False, - "isPrerelease": False, - "publishedAt": "2026-02-01T01:00:00+01:00", - } - ] - ) - - assert releases[0].published_at.isoformat() == "2026-02-01T00:00:00+00:00" - - -def test_resolve_archive_request_infer_release_uses_package_version_and_previous_release(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - (tmp_path / "Cargo.toml").write_text('[package]\nversion = "0.4.6"\n', encoding="utf-8") - - def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - assert command == "gh" - assert args[:2] == ["release", "list"] - assert cwd == tmp_path - return _result( - "[" - '{"tagName":"v0.4.4","isDraft":false,"isPrerelease":false,"publishedAt":"2026-01-01T00:00:00Z"},' - '{"tagName":"v0.4.5","isDraft":false,"isPrerelease":false,"publishedAt":"2026-02-01T00:00:00Z"}' - "]" - ) - - monkeypatch.setattr(archive_performance, "run_safe_command", fake_run_safe) - - request = archive_performance.resolve_archive_request( - archive_performance.ArchiveRequestOptions( - current_tag=None, - baseline_tag=None, - published_latest=False, - infer_release=True, - current_vs_latest=False, - worktree_ref="HEAD", - repo_root=tmp_path, - ) - ) - - assert request.current_tag == "v0.4.6" - assert request.baseline_tag == "v0.4.5" - assert request.worktree_ref == "HEAD" - assert request.tags_to_fetch == ("v0.4.5",) - - -@pytest.mark.parametrize("package_version", ["0.4.4", "0.4.6"]) -def test_resolve_archive_request_infer_release_requires_unpublished_package_version( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - package_version: str, -) -> None: - (tmp_path / "Cargo.toml").write_text(f'[package]\nversion = "{package_version}"\n', encoding="utf-8") - - def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - assert command == "gh" - assert args[:2] == ["release", "list"] - assert cwd == tmp_path - return _result( - "[" - '{"tagName":"v0.4.6","isDraft":false,"isPrerelease":false,"publishedAt":"2026-02-01T00:00:00Z"},' - '{"tagName":"v0.4.4","isDraft":false,"isPrerelease":false,"publishedAt":"2026-03-01T00:00:00Z"}' - "]" - ) - - monkeypatch.setattr(archive_performance, "run_safe_command", fake_run_safe) - - with pytest.raises(ValueError, match=r"must be newer than latest published stable release v0\.4\.6"): - archive_performance.resolve_archive_request( - archive_performance.ArchiveRequestOptions( - current_tag=None, - baseline_tag=None, - published_latest=False, - infer_release=True, - current_vs_latest=False, - worktree_ref="HEAD", - repo_root=tmp_path, - ) - ) - - -def test_resolve_archive_request_current_vs_latest_uses_package_version_and_latest_release(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - (tmp_path / "Cargo.toml").write_text('[package]\nversion = "0.4.6"\n', encoding="utf-8") - - def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - assert command == "gh" - assert args[:2] == ["release", "list"] - assert cwd == tmp_path - return _result( - "[" - '{"tagName":"v0.4.4","isDraft":false,"isPrerelease":false,"publishedAt":"2026-01-01T00:00:00Z"},' - '{"tagName":"v0.4.5","isDraft":false,"isPrerelease":false,"publishedAt":"2026-02-01T00:00:00Z"}' - "]" - ) - - monkeypatch.setattr(archive_performance, "run_safe_command", fake_run_safe) - - request = archive_performance.resolve_archive_request( - archive_performance.ArchiveRequestOptions( - current_tag=None, - baseline_tag=None, - published_latest=False, - infer_release=False, - current_vs_latest=True, - worktree_ref="HEAD", - repo_root=tmp_path, - ) - ) - - assert request.current_tag == "v0.4.6" - assert request.baseline_tag == "v0.4.5" - assert request.worktree_ref == "HEAD" - assert request.tags_to_fetch == ("v0.4.5",) - - -def test_resolve_archive_request_current_vs_latest_allows_equal_release_tags_for_local_reports( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - (tmp_path / "Cargo.toml").write_text('[package]\nversion = "0.4.6"\n', encoding="utf-8") - - def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - assert command == "gh" - assert args[:2] == ["release", "list"] - assert cwd == tmp_path - return _result('[{"tagName":"v0.4.6","isDraft":false,"isPrerelease":false,"publishedAt":"2026-03-01T00:00:00Z"}]') - - monkeypatch.setattr(archive_performance, "run_safe_command", fake_run_safe) - - request = archive_performance.resolve_archive_request( - archive_performance.ArchiveRequestOptions( - current_tag=None, - baseline_tag=None, - published_latest=False, - infer_release=False, - current_vs_latest=True, - worktree_ref="HEAD", - repo_root=tmp_path, - ) - ) - - assert request.current_tag == "v0.4.6" - assert request.baseline_tag == "v0.4.6" - assert request.worktree_ref == "HEAD" - assert request.tags_to_fetch == ("v0.4.6",) - - -def test_benchmark_env_uses_current_repo_toolchain(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("RUSTUP_TOOLCHAIN", raising=False) - (tmp_path / "rust-toolchain.toml").write_text('[toolchain]\nchannel = "1.97.0"\n', encoding="utf-8") - - env = archive_performance._benchmark_env(tmp_path) - - assert env is not None - assert env["RUSTUP_TOOLCHAIN"] == "1.97.0" - - -def test_benchmark_env_respects_existing_toolchain_override(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("RUSTUP_TOOLCHAIN", "nightly") - (tmp_path / "rust-toolchain.toml").write_text('[toolchain]\nchannel = "1.97.0"\n', encoding="utf-8") - - assert archive_performance._benchmark_env(tmp_path) is None - - -def test_parser_rejects_unsupported_scope() -> None: - with pytest.raises(SystemExit): - archive_performance.build_parser().parse_args(["v0.4.6", "v0.4.5", "--scope", "quick"]) - - -@pytest.mark.parametrize( - ("field", "value", "message"), - [ - ("suite", "other", "unsupported benchmark suite"), - ("scope", "quick", "unsupported comparison scope"), - ], -) -def test_generation_config_rejects_unsupported_benchmark_selection( - tmp_path: Path, - field: str, - value: str, - message: str, -) -> None: - kwargs: dict[str, Any] = { - "repo_root": tmp_path, - "current_tag": "v0.4.6", - "baseline_tag": "v0.4.5", - "worktree_ref": "HEAD", - } - kwargs[field] = value - - with pytest.raises(ValueError, match=message): - GenerationConfig(**kwargs) - - -def test_comparison_benchmark_env_preserves_flags_and_selects_pre_rational_adapter( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.delenv("CARGO_ENCODED_RUSTFLAGS", raising=False) - monkeypatch.delenv("RUSTUP_TOOLCHAIN", raising=False) - monkeypatch.setenv("RUSTFLAGS", "-C target-cpu=native") - (tmp_path / "rust-toolchain.toml").write_text( - '[toolchain]\nchannel = "1.97.0"\n', - encoding="utf-8", - ) - - current = archive_performance._comparison_benchmark_env(tmp_path) - baseline = archive_performance._comparison_benchmark_env( - tmp_path, - api_compatibility="la_stack_pre_rational_input_api", - ) - - assert current["RUSTFLAGS"] == "-C target-cpu=native --cap-lints=warn" - assert baseline["RUSTFLAGS"] == ("-C target-cpu=native --cap-lints=warn --cfg=la_stack_pre_rational_input_api") - assert current["RUSTUP_TOOLCHAIN"] == "1.97.0" - - -@pytest.mark.parametrize("baseline_tag", ["v0.4.4", "0.4.5"]) -def test_comparison_benchmark_env_selects_pre_rational_adapter( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - baseline_tag: str, -) -> None: - monkeypatch.delenv("CARGO_ENCODED_RUSTFLAGS", raising=False) - monkeypatch.delenv("RUSTFLAGS", raising=False) - monkeypatch.delenv("RUSTUP_TOOLCHAIN", raising=False) - (tmp_path / "rust-toolchain.toml").write_text( - '[toolchain]\nchannel = "1.98.0"\n', - encoding="utf-8", - ) - - baseline = archive_performance._comparison_benchmark_env( - tmp_path, - api_compatibility=archive_performance._baseline_api_compatibility( - current_tag="v0.4.6", - baseline_tag=baseline_tag, - shared_harness_rational_inputs=True, - ), - ) - - assert baseline["RUSTFLAGS"] == ("--cap-lints=warn --cfg=la_stack_pre_rational_input_api") - - -def test_comparison_benchmark_env_extends_encoded_rustflags( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("RUSTUP_TOOLCHAIN", "nightly") - monkeypatch.setenv("CARGO_ENCODED_RUSTFLAGS", "-C\x1ftarget-cpu=native") - - env = archive_performance._comparison_benchmark_env( - tmp_path, - api_compatibility="la_stack_pre_rational_input_api", - ) - - assert env["CARGO_ENCODED_RUSTFLAGS"] == ("-C\x1ftarget-cpu=native\x1f--cap-lints=warn\x1f--cfg=la_stack_pre_rational_input_api") - assert env["RUSTUP_TOOLCHAIN"] == "nightly" - - -@pytest.mark.parametrize("suite", ["exact", "vs_linalg"]) -def test_fallback_baseline_cargo_commands_enforce_lockfile( - tmp_path: Path, - suite: str, -) -> None: - worktree = tmp_path / "legacy" - worktree.mkdir() - _write_legacy_benchmark_tooling(worktree) - - command, args = archive_performance._baseline_tool_args( - baseline_tag="v0.4.5", - suite=suite, - baseline_worktree=worktree, - ) - - assert command == "cargo" - assert args[:2] == ["bench", "--locked"] - - -@pytest.mark.parametrize( - ("suite", "expected"), - [ - ("all", ("cargo", "bench", "--locked", "--features", "bench,exact")), - ("exact", ("cargo", "bench", "--locked", "--features", "bench,exact", "--bench", "exact")), - ("vs_linalg", ("cargo", "bench", "--locked", "--features", "bench", "--bench", "vs_linalg")), - ], -) -def test_fallback_current_cargo_command_matches_suite( - suite: str, - expected: tuple[str, ...], -) -> None: - assert archive_performance._fallback_current_command(suite=suite) == expected - - -@pytest.mark.parametrize("github_assets", [False, True]) -@pytest.mark.parametrize("baseline", ["v0.4.3", "v0.4.2"]) -def test_unsupported_baseline_fails_before_fetch_or_worktree( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], - github_assets: bool, - baseline: str, -) -> None: - (tmp_path / "Cargo.toml").write_text('[package]\nversion = "0.4.6"\n', encoding="utf-8") - monkeypatch.chdir(tmp_path) - - def fail_external_work(*_args: object, **_kwargs: object) -> None: - pytest.fail("unsupported baseline reached external work") - - monkeypatch.setattr(archive_performance, "run_git_command", fail_external_work) - monkeypatch.setattr(archive_performance, "run_safe_command", fail_external_work) - args = ["v0.4.6", baseline, "--generate-in-temp-worktree"] - if github_assets: - args.append("--github-assets") - - assert main(args) == 1 - assert "comparisons require v0.4.4 or newer" in capsys.readouterr().err - assert not (tmp_path / "target").exists() - - -@pytest.mark.parametrize("retained_outputs", [None, {}], ids=["absent", "empty"]) -def test_validate_promotion_paths_without_retained_outputs_preserves_reports( - tmp_path: Path, - retained_outputs: dict[Path, str] | None, -) -> None: - current = tmp_path / "performance.md" - archive_dir = tmp_path / "archive" - index = archive_dir / "README.md" - current.write_text("current report\n", encoding="utf-8") - archive_dir.mkdir() - index.write_text("archive index\n", encoding="utf-8") - request = archive_performance.PromotionRequest( - current=current, - archive_dir=archive_dir, - expected=archive_performance.ReportId(current_tag="v0.4.6", baseline_tag="v0.4.5"), - retained_outputs=retained_outputs, - ) - - assert archive_performance._validate_promotion_paths(request, index_path=index, archive_path=None) is None - - assert current.read_text(encoding="utf-8") == "current report\n" - assert index.read_text(encoding="utf-8") == "archive index\n" - assert set(tmp_path.iterdir()) == {current, archive_dir} - assert list(archive_dir.iterdir()) == [index] - - -def test_validate_promotion_paths_rejects_retained_summary_alias_without_mutation(tmp_path: Path) -> None: - current = tmp_path / "performance.md" - archive_dir = tmp_path / "archive" - current.write_text("current report\n", encoding="utf-8") - request = archive_performance.PromotionRequest( - current=current, - archive_dir=archive_dir, - expected=archive_performance.ReportId(current_tag="v0.4.6", baseline_tag="v0.4.5"), - retained_outputs={current: "retained summary\n"}, - ) - - with pytest.raises(ValueError, match="current report and retained summary 0 must use distinct paths") as error: - archive_performance._validate_promotion_paths(request, index_path=archive_dir / "README.md", archive_path=None) - - assert str(error.value) == f"current report and retained summary 0 must use distinct paths: {current}" - assert current.read_text(encoding="utf-8") == "current report\n" - assert not archive_dir.exists() - - -def test_promote_report_archives_previous_and_updates_sorted_index(tmp_path: Path) -> None: - source = tmp_path / "target" / "bench-reports" / "performance.md" - current = tmp_path / "docs" / "performance.md" - archive_dir = tmp_path / "docs" / "archive" / "performance" - - source.parent.mkdir(parents=True) - current.parent.mkdir(parents=True) - archive_dir.mkdir(parents=True) - source.write_text(_report("0.4.5", "v0.4.4"), encoding="utf-8") - current.write_text(_report("0.4.4", "v0.4.0"), encoding="utf-8") - (archive_dir / "v0.3.1-vs-v0.3.0.md").write_text(_report("0.3.1", "v0.3.0"), encoding="utf-8") - - promoted = promote_report( - source=source, - current=current, - archive_dir=archive_dir, - expected_current_tag="v0.4.5", - expected_baseline_tag="v0.4.4", - ) - - assert promoted.archive_name == "v0.4.5-vs-v0.4.4.md" - assert current.read_text(encoding="utf-8") == _normalized_report("0.4.5", "v0.4.4") - assert (archive_dir / "v0.4.4-vs-v0.4.0.md").read_text(encoding="utf-8") == _normalized_report("0.4.4", "v0.4.0") - assert (archive_dir / "README.md").read_text(encoding="utf-8") == ( - "# Archived Performance Reports\n\n" - "Older release-to-release benchmark comparisons are archived here.\n" - "`docs/performance.md` contains the latest curated comparison.\n\n" - "- [v0.3.1-vs-v0.3.0](v0.3.1-vs-v0.3.0.md)\n" - "- [v0.4.4-vs-v0.4.0](v0.4.4-vs-v0.4.0.md)\n" - ) - - -def test_promote_report_is_idempotent_for_same_release_pair(tmp_path: Path) -> None: - source = tmp_path / "performance-new.md" - current = tmp_path / "docs" / "performance.md" - archive_dir = tmp_path / "docs" / "archive" / "performance" - - source.write_text(_report("0.4.5", "v0.4.4"), encoding="utf-8") - current.parent.mkdir(parents=True) - current.write_text(_report("0.4.5", "v0.4.4"), encoding="utf-8") - - promote_report( - source=source, - current=current, - archive_dir=archive_dir, - expected_current_tag="v0.4.5", - expected_baseline_tag="v0.4.4", - ) - - assert not (archive_dir / "v0.4.5-vs-v0.4.4.md").exists() - assert "- No archived performance reports yet." in (archive_dir / "README.md").read_text(encoding="utf-8") - - -def test_promote_report_does_not_overwrite_existing_archive(tmp_path: Path) -> None: - source = tmp_path / "performance-new.md" - current = tmp_path / "docs" / "performance.md" - archive_dir = tmp_path / "docs" / "archive" / "performance" - archived = archive_dir / "v0.4.4-vs-v0.4.0.md" - - source.write_text(_report("0.4.5", "v0.4.4"), encoding="utf-8") - current.parent.mkdir(parents=True) - current.write_text(_report("0.4.4", "v0.4.0"), encoding="utf-8") - archive_dir.mkdir(parents=True) - archived_payload = _normalized_report("0.4.4", "v0.4.0").replace("\n", "\r\n").encode() - archived.write_bytes(archived_payload) - - promote_report( - source=source, - current=current, - archive_dir=archive_dir, - expected_current_tag="v0.4.5", - expected_baseline_tag="v0.4.4", - ) - - assert archived.read_bytes() == archived_payload - - -def test_promote_report_rejects_mismatched_existing_archive_without_mutation(tmp_path: Path) -> None: - source = tmp_path / "performance-new.md" - current = tmp_path / "docs" / "performance.md" - archive_dir = tmp_path / "docs" / "archive" / "performance" - archived = archive_dir / "v0.4.4-vs-v0.4.0.md" - index = archive_dir / "README.md" - original = _report("0.4.4", "v0.4.0") - - source.write_text(_report("0.4.5", "v0.4.4"), encoding="utf-8") - current.parent.mkdir(parents=True) - current.write_text(original, encoding="utf-8") - archive_dir.mkdir(parents=True) - archived.write_text("unrelated archive contents\n", encoding="utf-8") - index.write_text("original index\n", encoding="utf-8") - - with pytest.raises(ValueError, match="does not match the current report"): - promote_report( - source=source, - current=current, - archive_dir=archive_dir, - expected_current_tag="v0.4.5", - expected_baseline_tag="v0.4.4", - ) - - assert current.read_text(encoding="utf-8") == original - assert archived.read_text(encoding="utf-8") == "unrelated archive contents\n" - assert index.read_text(encoding="utf-8") == "original index\n" - - -@pytest.mark.parametrize("collision", ["directory", "symlink"]) -def test_promote_report_rejects_non_file_archive_collision_without_mutation(tmp_path: Path, collision: str) -> None: - source = tmp_path / "performance-new.md" - current = tmp_path / "docs" / "performance.md" - archive_dir = tmp_path / "docs" / "archive" / "performance" - archived = archive_dir / "v0.4.4-vs-v0.4.0.md" - original = _report("0.4.4", "v0.4.0") - - source.write_text(_report("0.4.5", "v0.4.4"), encoding="utf-8") - current.parent.mkdir(parents=True) - current.write_text(original, encoding="utf-8") - archive_dir.mkdir(parents=True) - if collision == "directory": - archived.mkdir() - else: - target = tmp_path / "unrelated.md" - target.write_text("unrelated target\n", encoding="utf-8") - archived.symlink_to(target) - - with pytest.raises(ValueError, match=r"must (?:not be a symlink|be a regular file)"): - promote_report( - source=source, - current=current, - archive_dir=archive_dir, - expected_current_tag="v0.4.5", - expected_baseline_tag="v0.4.4", - ) - - assert current.read_text(encoding="utf-8") == original - assert not (archive_dir / "README.md").exists() - - -def test_promote_report_rejects_unexpected_release_pair(tmp_path: Path) -> None: - source = tmp_path / "performance-new.md" - current = tmp_path / "docs" / "performance.md" - archive_dir = tmp_path / "docs" / "archive" / "performance" - source.write_text(_report("0.4.5", "v0.4.4"), encoding="utf-8") - - with pytest.raises(ValueError, match="does not match requested release pair"): - promote_report( - source=source, - current=current, - archive_dir=archive_dir, - expected_current_tag="v0.4.6", - expected_baseline_tag="v0.4.5", - ) - - -def test_promote_report_rewrites_legacy_update_instructions(tmp_path: Path) -> None: - source = tmp_path / "performance-new.md" - current = tmp_path / "docs" / "performance.md" - archive_dir = tmp_path / "docs" / "archive" / "performance" - source.write_text(_legacy_report("0.4.6", "v0.4.5"), encoding="utf-8") - current.parent.mkdir(parents=True) - current.write_text(_legacy_report("0.4.5", "v0.4.4"), encoding="utf-8") - - promote_report( - source=source, - current=current, - archive_dir=archive_dir, - expected_current_tag="v0.4.6", - expected_baseline_tag="v0.4.5", - ) - - current_text = current.read_text(encoding="utf-8") - archived_text = (archive_dir / "v0.4.5-vs-v0.4.4.md").read_text(encoding="utf-8") - assert "just performance-local" in current_text - assert "just performance-release" in current_text - assert "just performance-github-assets" in current_text - assert "just performance-release " in current_text - assert "git checkout" not in current_text - assert "just performance-local" in archived_text - assert "just performance-github-assets" in archived_text - assert "git checkout" not in archived_text - - -def test_main_promotes_generated_report_to_docs_performance( - tmp_path: Path, - capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, -) -> None: - source = tmp_path / "target" / "bench-reports" / "performance.md" - current = tmp_path / "docs" / "performance.md" - archive_dir = tmp_path / "docs" / "archive" / "performance" - generated = _report("0.4.6", "v0.4.5") - - source.parent.mkdir(parents=True) - source.write_text(generated, encoding="utf-8") - current.parent.mkdir(parents=True) - current.write_text(_report("0.4.5", "v0.4.4"), encoding="utf-8") - gates: list[Path] = [] - monkeypatch.chdir(tmp_path) - monkeypatch.setattr( - archive_performance, - "_run_benchmark_input_gate", - lambda checkout, **_kwargs: gates.append(checkout), - ) - - rc = main( - [ - "v0.4.6", - "v0.4.5", - "--source", - str(source), - "--current", - str(current), - "--archive-dir", - str(archive_dir), - ] - ) - - assert rc == 0 - assert current.read_text(encoding="utf-8") == archive_performance._normalize_how_to_update(generated) - assert (archive_dir / "v0.4.5-vs-v0.4.4.md").exists() - assert gates == [tmp_path] - assert "Current performance report: v0.4.6 vs v0.4.5" in capsys.readouterr().out - - -def test_main_reports_release_pair_mismatch_to_stderr( - tmp_path: Path, - capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, -) -> None: - source = tmp_path / "target" / "bench-reports" / "performance.md" - current = tmp_path / "docs" / "performance.md" - archive_dir = tmp_path / "docs" / "archive" / "performance" - source.parent.mkdir(parents=True) - source.write_text(_report("0.4.6", "v0.4.5"), encoding="utf-8") - monkeypatch.setattr(archive_performance, "_run_benchmark_input_gate", lambda *_args, **_kwargs: None) - - rc = main( - [ - "v0.4.4", - "v0.4.6", - "--source", - str(source), - "--current", - str(current), - "--archive-dir", - str(archive_dir), - ] - ) - - captured = capsys.readouterr() - assert rc == 1 - assert "does not match requested release pair" in captured.err - assert not current.exists() - - -def test_main_rejects_local_option_conflict_before_release_discovery( - tmp_path: Path, - capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.chdir(tmp_path) - - def fail_discovery(_options: object) -> None: - msg = "release discovery must not run" - raise AssertionError(msg) - - monkeypatch.setattr(archive_performance, "resolve_archive_request", fail_discovery) - - rc = main(["--published-latest", "--output-only"]) - - assert rc == 1 - assert "--output-only requires --generate-in-temp-worktree" in capsys.readouterr().err - - -@pytest.mark.parametrize( - ("argv", "message"), - [ - (["--local-report"], "--local-report requires --output-only"), - ( - ["--local-report", "--output-only", "--generate-in-temp-worktree", "--github-assets"], - "--local-report cannot be combined with --github-assets", - ), - ], -) -def test_main_rejects_invalid_local_report_options_before_release_discovery( - tmp_path: Path, - capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, - argv: list[str], - message: str, -) -> None: - monkeypatch.chdir(tmp_path) - - def fail_discovery(_options: object) -> None: - msg = "release discovery must not run" - raise AssertionError(msg) - - monkeypatch.setattr(archive_performance, "resolve_archive_request", fail_discovery) - - assert main(argv) == 1 - assert message in capsys.readouterr().err - - -@pytest.mark.parametrize("selection", [["--suite", "exact"], ["--scope", "all-benches"]]) -def test_main_rejects_artifact_promotion_benchmark_selectors_before_loading_artifacts( - tmp_path: Path, - capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, - selection: list[str], -) -> None: - monkeypatch.chdir(tmp_path) - - def fail_load(_paths: object) -> None: - msg = "artifact loading must not start" - raise AssertionError(msg) - - monkeypatch.setattr(archive_performance, "load_bundle", fail_load) - - assert main(["--promote-artifacts", *selection]) == 1 - assert "--promote-artifacts cannot be combined" in capsys.readouterr().err - - -def test_archive_performance_help_describes_local_and_release_workflows() -> None: - help_text = archive_performance.build_parser().format_help() - - assert "Generate local benchmark comparisons or promote validated comparison artifacts" in help_text - assert "Rendered local comparison report path" in help_text - assert "Retained comparison CSV path" in help_text - assert "before promoting it" not in help_text - - -def test_main_rejects_explicit_current_tag_mismatch_before_external_work( - tmp_path: Path, - capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, -) -> None: - (tmp_path / "Cargo.toml").write_text('[package]\nversion = "0.4.4"\n', encoding="utf-8") - monkeypatch.chdir(tmp_path) - - def fail_external_work(*_args: object, **_kwargs: object) -> None: - msg = "external work must not start" - raise AssertionError(msg) - - monkeypatch.setattr(archive_performance, "_fetch_required_tags", fail_external_work) - - rc = main( - [ - "v0.4.5", - "v0.4.4", - "--generate-in-temp-worktree", - "--output-only", - "--local-report", - ] - ) - - assert rc == 1 - assert "explicit current tag v0.4.5 does not match the current Cargo package tag v0.4.4" in capsys.readouterr().err - - -def test_main_rejects_equal_release_artifact_tags_before_external_work( - tmp_path: Path, - capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, -) -> None: - (tmp_path / "Cargo.toml").write_text('[package]\nversion = "0.4.4"\n', encoding="utf-8") - monkeypatch.chdir(tmp_path) - - def fail_external_work(*_args: object, **_kwargs: object) -> None: - msg = "external work must not start" - raise AssertionError(msg) - - monkeypatch.setattr(archive_performance, "_fetch_required_tags", fail_external_work) - - rc = main( - [ - "v0.4.4", - "v0.4.4", - "--generate-in-temp-worktree", - "--output-only", - ] - ) - - assert rc == 1 - assert "release performance artifacts require distinct current and baseline releases" in capsys.readouterr().err - - -def test_local_release_generation_rejects_unavailable_cpu_before_external_work( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - output = tmp_path / "performance.md" - - def fail_external_work(*_args: object, **_kwargs: object) -> None: - msg = "external work must not start" - raise AssertionError(msg) - - monkeypatch.setattr(archive_performance, "cpu_description", lambda: "unavailable") - monkeypatch.setattr(archive_performance, "run_git_command", fail_external_work) - monkeypatch.setattr(archive_performance, "run_safe_command", fail_external_work) - - with pytest.raises(RuntimeError, match="CPU model is unavailable"): - generate_worktree_report( - output=output, - config=GenerationConfig( - repo_root=tmp_path, - current_tag="v0.4.4", - baseline_tag="v0.4.6", - worktree_ref="HEAD", - ), - ) - - assert not output.exists() - - -@pytest.mark.parametrize("alias", ["artifact-csv", "full-csv", "full-json", "current-report"]) -def test_main_promote_artifacts_rejects_output_alias_without_mutation( - tmp_path: Path, - capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, - alias: str, -) -> None: - artifacts = ArtifactPaths( - csv=tmp_path / "target" / "bench-reports" / "performance.csv", - provenance=tmp_path / "target" / "bench-reports" / "performance.provenance.json", - ) - artifacts.csv.parent.mkdir(parents=True) - artifacts.csv.write_bytes(b"original csv\n") - artifacts.provenance.write_bytes(b"original provenance\n") - current = tmp_path / "docs" / "performance.md" - current.parent.mkdir(parents=True) - current.write_text(_report("0.4.6", "v0.4.5"), encoding="utf-8") - archive_dir = tmp_path / "docs" / "archive" / "performance" - output = { - "artifact-csv": artifacts.csv, - "full-csv": full_summary_paths(artifacts).csv, - "full-json": full_summary_paths(artifacts).provenance, - "current-report": current, - }[alias] - bundle = SimpleNamespace( - context=SimpleNamespace( - release=SimpleNamespace(current="v0.4.4", baseline="v0.4.6"), - ) - ) - monkeypatch.chdir(tmp_path) - monkeypatch.setattr(archive_performance, "load_bundle", lambda _paths: bundle) - monkeypatch.setattr( - archive_performance, - "render_release_artifacts", - lambda _paths: _report("0.4.4", "v0.4.6"), - ) - - rc = main( - [ - "--promote-artifacts", - "--artifact-csv", - str(artifacts.csv), - "--artifact-provenance", - str(artifacts.provenance), - "--output", - str(output), - "--current", - str(current), - "--archive-dir", - str(archive_dir), - ] - ) - - captured = capsys.readouterr() - assert rc == 1 - assert "must use distinct paths" in captured.err - assert artifacts.csv.read_bytes() == b"original csv\n" - assert artifacts.provenance.read_bytes() == b"original provenance\n" - assert current.read_text(encoding="utf-8") == _report("0.4.6", "v0.4.5") - assert not archive_dir.exists() - - -def test_main_reraises_unexpected_errors(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - def fail_unexpected(*, args: object, paths: object, request: object, repo_root: Path) -> archive_performance.ArchiveResult: - msg = "unexpected test failure" - raise AssertionError(msg) - - monkeypatch.chdir(tmp_path) - monkeypatch.setattr(archive_performance, "_run_archive_request", fail_unexpected) - - with pytest.raises(AssertionError, match="unexpected test failure"): - main(["v0.4.6", "v0.4.5"]) - - -def test_main_generates_report_in_temp_worktree(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: - current = tmp_path / "docs" / "performance.md" - archive_dir = tmp_path / "docs" / "archive" / "performance" - calls: list[RunnerCall] = [] - - def fake_run_git(args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append(("git", tuple(args), cwd)) - if args[:3] == ["worktree", "add", "--detach"]: - worktree = Path(args[3]) - worktree.mkdir(parents=True) - _write_current_benchmark_tooling(worktree) - return _result() - - def fake_run_git_with_input(args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append(("git-stdin", tuple(args), cwd)) - return _result() - - def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append((command, tuple(args), cwd)) - if command == "just" and args == ["bench-save-baseline", "v0.4.5", "exact"]: - assert cwd is not None - criterion_dir = cwd / "target" / "criterion" - criterion_dir.mkdir(parents=True) - (criterion_dir / "baseline.txt").write_text("baseline\n", encoding="utf-8") - if command == "uv": - _write_fake_rendered_artifacts(args, version="0.4.6", baseline="v0.4.5") - return _result() - - monkeypatch.chdir(tmp_path) - monkeypatch.setattr(archive_performance, "run_git_command", fake_run_git) - monkeypatch.setattr(archive_performance, "run_git_bytes", fake_run_git_with_input) - monkeypatch.setattr(archive_performance, "run_safe_command", fake_run_safe) - - rc = main( - [ - "v0.4.6", - "v0.4.5", - "--current", - str(current), - "--archive-dir", - str(archive_dir), - "--generate-in-temp-worktree", - "--worktree-ref", - "v0.4.6", - "--no-apply-current-diff", - "--suite", - "exact", - "--scope", - "release-signal", - ] - ) - - captured = capsys.readouterr() - assert rc == 0 - assert current.read_text(encoding="utf-8") == _retained_report(tmp_path) - output = tmp_path / "target" / "bench-reports" / "performance.md" - assert output.read_text(encoding="utf-8") == current.read_text(encoding="utf-8") - assert "Generated benchmark report in a temporary worktree" in captured.out - assert str(output) in captured.out - assert any(kind == "git" and args[:3] == ("worktree", "add", "--detach") and args[4] == "v0.4.6" for kind, args, _ in calls) - assert any(kind == "just" and args == ("bench-exact",) for kind, args, _ in calls) - assert any(kind == "uv" and "--suite" in args and args[args.index("--suite") + 1] == "exact" for kind, args, _ in calls) - assert any(kind == "uv" and args[:2] == ("run", "--locked") for kind, args, _ in calls) - assert not any(kind == "git" and args[:1] == ("read-tree",) for kind, args, _ in calls) - assert not any(kind == "git-stdin" for kind, _, _ in calls) - - external_call_count = len(calls) - promote_rc = main( - [ - "--promote-artifacts", - "--current", - str(current), - "--archive-dir", - str(archive_dir), - ] - ) - promoted = capsys.readouterr() - assert promote_rc == 0 - assert "Rendered" in promoted.out - assert len(calls) == external_call_count - assert output.read_text(encoding="utf-8") == current.read_text(encoding="utf-8") - - -def test_temp_worktree_is_removed_when_benchmark_command_fails(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: - (tmp_path / "Cargo.toml").write_text('[package]\nversion = "0.4.6"\n', encoding="utf-8") - current = tmp_path / "docs" / "performance.md" - archive_dir = tmp_path / "docs" / "archive" / "performance" - calls: list[RunnerCall] = [] - - def fake_run_git(args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append(("git", tuple(args), cwd)) - if args[:2] == ["fetch", "origin"]: - return _result() - if args[:3] == ["worktree", "add", "--detach"]: - worktree = Path(args[3]) - worktree.mkdir(parents=True) - if worktree.name == "baseline-worktree": - _write_legacy_benchmark_tooling(worktree) - else: - _write_current_benchmark_tooling(worktree) - return _result() - - def fake_run_git_with_input(args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append(("git-stdin", tuple(args), cwd)) - return _result() - - def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append((command, tuple(args), cwd)) - if command == "just" and args == ["bench-save-baseline", "v0.4.5"]: - assert cwd is not None - criterion_dir = cwd / "target" / "criterion" - criterion_dir.mkdir(parents=True) - (criterion_dir / "baseline.txt").write_text("baseline\n", encoding="utf-8") - if command == "just" and args == ["bench-latest"]: - raise subprocess.CalledProcessError(42, [command, *args], output="bench stdout", stderr="bench stderr") - return _result() - - monkeypatch.chdir(tmp_path) - monkeypatch.setattr(archive_performance, "run_git_command", fake_run_git) - monkeypatch.setattr(archive_performance, "run_git_bytes", fake_run_git_with_input) - monkeypatch.setattr(archive_performance, "run_safe_command", fake_run_safe) - - rc = main( - [ - "v0.4.6", - "v0.4.5", - "--current", - str(current), - "--archive-dir", - str(archive_dir), - "--generate-in-temp-worktree", - "--worktree-ref", - "HEAD", - "--no-apply-current-diff", - ] - ) - - captured = capsys.readouterr() - assert rc == 1 - assert "command failed (42): just bench-latest" in captured.err - assert "bench stderr" in captured.err - assert not current.exists() - assert any(kind == "git" and args[:3] == ("worktree", "remove", "--force") for kind, args, _ in calls) - - -def test_generate_report_rejects_unsafe_baseline_archive(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: - current = tmp_path / "docs" / "performance.md" - archive_dir = tmp_path / "docs" / "archive" / "performance" - calls: list[RunnerCall] = [] - - def fake_run_git(args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append(("git", tuple(args), cwd)) - if args[:3] == ["worktree", "add", "--detach"]: - worktree = Path(args[3]) - worktree.mkdir(parents=True) - _write_current_benchmark_tooling(worktree) - return _result() - - def fake_run_git_with_input(args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append(("git-stdin", tuple(args), cwd)) - return _result() - - def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append((command, tuple(args), cwd)) - if command == "gh": - download_dir = Path(args[args.index("--dir") + 1]) - tag = args[2] - _write_unsafe_baseline_archive(download_dir / f"la-stack-{tag}-criterion-baseline.tar.gz") - return _result() - - monkeypatch.chdir(tmp_path) - monkeypatch.setattr(archive_performance, "run_git_command", fake_run_git) - monkeypatch.setattr(archive_performance, "run_git_bytes", fake_run_git_with_input) - monkeypatch.setattr(archive_performance, "run_safe_command", fake_run_safe) - - rc = main( - [ - "v0.4.6", - "v0.4.5", - "--current", - str(current), - "--archive-dir", - str(archive_dir), - "--generate-in-temp-worktree", - "--worktree-ref", - "HEAD", - "--no-apply-current-diff", - "--github-assets", - ] - ) - - captured = capsys.readouterr() - assert rc == 1 - assert "unsafe or nonportable archive path: '../escape.txt'" in captured.err - assert not (tmp_path / "escape.txt").exists() - assert not current.exists() - assert not any(kind in {"just", "uv"} for kind, _, _ in calls) - assert any(kind == "git" and args[:3] == ("worktree", "remove", "--force") for kind, args, _ in calls) - - -def test_generate_report_generates_release_baseline_locally( # noqa: PLR0915 - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - monkeypatch.delenv("RUSTUP_TOOLCHAIN", raising=False) - (tmp_path / "Cargo.toml").write_text('[package]\nversion = "0.4.6"\n', encoding="utf-8") - (tmp_path / "rust-toolchain.toml").write_text('[toolchain]\nchannel = "1.97.0"\n', encoding="utf-8") - current = tmp_path / "docs" / "performance.md" - archive_dir = tmp_path / "docs" / "archive" / "performance" - calls: list[RunnerCall] = [] - - def fake_run_git(args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append(("git", tuple(args), cwd)) - if args[:3] == ["worktree", "add", "--detach"]: - worktree = Path(args[3]) - worktree.mkdir(parents=True) - if worktree.name == "baseline-worktree": - _write_legacy_benchmark_tooling(worktree) - else: - _write_current_benchmark_tooling(worktree) - return _result() - - def fake_run_git_with_input(args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append(("git-stdin", tuple(args), cwd)) - return _result() - - def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append((command, tuple(args), cwd)) - if command == "just" and args == ["bench-save-baseline", "v0.4.5"]: - assert kwargs["env"]["RUSTUP_TOOLCHAIN"] == "1.97.0" - assert cwd is not None - assert "bench-latest" in (cwd / "justfile").read_text(encoding="utf-8") - criterion_dir = cwd / "target" / "criterion" - criterion_dir.mkdir(parents=True) - (criterion_dir / "baseline.txt").write_text("baseline\n", encoding="utf-8") - for sample in ("new", "v0.4.5"): - estimates = criterion_dir / "exact_d2" / "det_exact" / sample / "estimates.json" - estimates.parent.mkdir(parents=True, exist_ok=True) - estimates.write_text("{}\n", encoding="utf-8") - if command == "just" and args == ["bench-latest"]: - assert kwargs["env"]["RUSTUP_TOOLCHAIN"] == "1.97.0" - assert cwd is not None - criterion_dir = cwd / "target" / "criterion" / "exact_d2" / "det_exact" - assert not (criterion_dir / "new").exists() - assert (criterion_dir / "v0.4.5" / "estimates.json").is_file() - if command == "uv": - report_root = Path(args[args.index("--repo-root") + 1]) - criterion_root = Path(args[args.index("--criterion-dir") + 1]) - metadata = json.loads((criterion_root / archive_performance._BENCHMARK_HARNESS_METADATA).read_text(encoding="utf-8")) - assert metadata["baseline"] == "v0.4.5" - assert metadata["mode"] == "shared-current-harness" - assert metadata["schema"] == 2 - assert metadata["measurement"]["harness_sha256"] == archive_performance._benchmark_harness_digest(report_root) - assert metadata["measurement"]["benchmark_contract_sha256"] == archive_performance.benchmark_contract_digest(report_root) - assert metadata["measurement"]["current_source_state_sha256"] == archive_performance._source_state_digest(report_root) - assert metadata["criterion"]["criterion_version"] == "manifest requirement 0.7.0" - assert metadata["validation"]["baseline_revision"] == "passed" - assert metadata["validation"]["current_revision"] == "passed" - _write_fake_rendered_artifacts(args, version="0.4.6", baseline="v0.4.5") - return _result() - - monkeypatch.chdir(tmp_path) - monkeypatch.setattr(archive_performance, "run_git_command", fake_run_git) - monkeypatch.setattr(archive_performance, "run_git_bytes", fake_run_git_with_input) - monkeypatch.setattr(archive_performance, "run_safe_command", fake_run_safe) - - rc = main( - [ - "v0.4.6", - "v0.4.5", - "--current", - str(current), - "--archive-dir", - str(archive_dir), - "--generate-in-temp-worktree", - "--worktree-ref", - "HEAD", - "--no-apply-current-diff", - ] - ) - - captured = capsys.readouterr() - assert rc == 0 - assert captured.err.splitlines() == [ - "[performance] validating benchmark inputs in baseline-worktree", - "[performance] running all baseline benchmarks for v0.4.5; includes nalgebra/faer measurements", - "[performance] completed all baseline benchmarks for v0.4.5", - "[performance] validating benchmark inputs in worktree", - "[performance] running current all benchmarks for v0.4.6; la-stack only, reusing baseline nalgebra/faer measurements", - "[performance] completed current all benchmarks", - ] - assert current.read_text(encoding="utf-8") == _retained_report(tmp_path) - assert not any(kind == "gh" for kind, _, _ in calls) - assert any(kind == "just" and args == ("bench-save-baseline", "v0.4.5") for kind, args, _ in calls) - assert any(kind == "just" and args == ("bench-latest",) for kind, args, _ in calls) - assert any(kind == "uv" and "--suite" in args for kind, args, _ in calls) - baseline_timing_index = next(index for index, (kind, args, _) in enumerate(calls) if kind == "just" and args == ("bench-save-baseline", "v0.4.5")) - current_timing_index = next(index for index, (kind, args, _) in enumerate(calls) if kind == "just" and args == ("bench-latest",)) - baseline_gate_index = next( - index - for index, (kind, args, cwd) in enumerate(calls) - if kind == "just" and args == ("test-bench-inputs",) and cwd is not None and cwd.name == "baseline-worktree" - ) - current_gate_index = next( - index - for index, (kind, args, cwd) in enumerate(calls) - if kind == "just" and args == ("test-bench-inputs",) and cwd is not None and cwd.name == "worktree" - ) - assert baseline_gate_index < baseline_timing_index - assert current_gate_index < current_timing_index - assert sum(1 for kind, args, _ in calls if kind == "git" and args[:3] == ("worktree", "remove", "--force")) == 2 - - -def test_generate_local_non_exact_report_retains_same_version_comparison_artifacts( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.delenv("RUSTUP_TOOLCHAIN", raising=False) - (tmp_path / "rust-toolchain.toml").write_text('[toolchain]\nchannel = "1.97.0"\n', encoding="utf-8") - output = tmp_path / "target" / "bench-reports" / "performance.md" - calls: list[RunnerCall] = [] - - def fake_run_git(args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append(("git", tuple(args), cwd)) - if args[:3] == ["worktree", "add", "--detach"]: - worktree = Path(args[3]) - worktree.mkdir(parents=True) - if worktree.name == "baseline-worktree": - _write_legacy_benchmark_tooling(worktree) - else: - _write_current_benchmark_tooling(worktree) - return _result() - - def fake_run_git_with_input(args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append(("git-stdin", tuple(args), cwd)) - return _result() - - def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append((command, tuple(args), cwd)) - if command == "just" and args == ["bench-save-baseline", "v0.4.6", "vs_linalg"]: - assert kwargs["env"]["RUSTUP_TOOLCHAIN"] == "1.97.0" - assert cwd is not None - assert "bench-latest" in (cwd / "justfile").read_text(encoding="utf-8") - criterion_dir = cwd / "target" / "criterion" - criterion_dir.mkdir(parents=True) - (criterion_dir / "baseline.txt").write_text("baseline\n", encoding="utf-8") - if command == "just" and args == ["bench-vs-linalg-la-stack"]: - assert kwargs["env"]["RUSTUP_TOOLCHAIN"] == "1.97.0" - if command == "uv": - _write_fake_rendered_artifacts(args, version="0.4.6", baseline="v0.4.6") - return _result() - - monkeypatch.setattr(archive_performance, "run_git_command", fake_run_git) - monkeypatch.setattr(archive_performance, "run_git_bytes", fake_run_git_with_input) - monkeypatch.setattr(archive_performance, "run_safe_command", fake_run_safe) - - report_id = generate_worktree_report( - output=output, - config=GenerationConfig( - repo_root=tmp_path, - current_tag="v0.4.6", - baseline_tag="v0.4.6", - worktree_ref="HEAD", - suite="vs_linalg", - apply_current_diff=False, - ), - ) - - assert report_id.archive_name == "v0.4.6-vs-v0.4.6.md" - assert output.read_text(encoding="utf-8") == _retained_report(tmp_path) - assert output.with_suffix(".csv").is_file() - assert output.with_suffix(".provenance.json").is_file() - assert any(kind == "just" and args == ("bench-save-baseline", "v0.4.6", "vs_linalg") for kind, args, _ in calls) - assert any(kind == "just" and args == ("bench-vs-linalg-la-stack",) for kind, args, _ in calls) - assert any(kind == "uv" and "--csv-output" in args and "--provenance-output" in args for kind, args, _ in calls) - assert not any(kind == "cargo" for kind, _, _ in calls) - assert not any(kind == "just" and args == ("bench-exact",) for kind, args, _ in calls) - assert not any(kind == "just" and args == ("bench-latest",) for kind, args, _ in calls) - - current = tmp_path / "docs" / "performance.md" - with pytest.raises(ValueError, match="cannot promote a same-version local performance comparison"): - archive_performance.render_and_promote_artifacts( - artifacts=ArtifactPaths( - csv=output.with_suffix(".csv"), - provenance=output.with_suffix(".provenance.json"), - ), - output=output, - current=current, - archive_dir=tmp_path / "docs" / "archive" / "performance", - ) - assert not current.exists() - - -def _fail_publication_once(monkeypatch: pytest.MonkeyPatch, target: Path) -> None: - real_replace = Path.replace - failed = False - - def fail(source: Path, destination: Path) -> Path: - nonlocal failed - if destination == target and not failed: - failed = True - msg = "simulated retained summary promotion failure" - raise OSError(msg) - return real_replace(source, destination) - - monkeypatch.setattr(Path, "replace", fail) - - -@pytest.mark.parametrize("fail_promotion", [False, True]) -def test_generate_local_report_artifacts_can_be_rendered_and_promoted( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - *, - fail_promotion: bool, -) -> None: - output = tmp_path / "target" / "bench-reports" / "performance.md" - calls: list[RunnerCall] = [] - - def fake_run_git(args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append(("git", tuple(args), cwd)) - if args[:3] == ["worktree", "add", "--detach"]: - worktree = Path(args[3]) - worktree.mkdir(parents=True) - _write_current_benchmark_tooling(worktree) - return _result() - - def fake_run_git_with_input(args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append(("git-stdin", tuple(args), cwd)) - return _result() - - def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append((command, tuple(args), cwd)) - if command == "just" and args == ["bench-save-baseline", "v0.4.5", "vs_linalg"]: - assert cwd is not None - criterion_dir = cwd / "target" / "criterion" - criterion_dir.mkdir(parents=True) - (criterion_dir / "baseline.txt").write_text("baseline\n", encoding="utf-8") - for sample in ("v0.4.5", "new"): - _write_measurement(criterion_dir, "lu_solve_pivoting_d2/faer_lu_solve", sample, 20.0) - _write_measurement(criterion_dir, "lu_solve_pivoting_d2/la_stack_lu_solve", sample, 10.0) - if command == "just" and args == ["bench-vs-linalg-la-stack"]: - assert cwd is not None - _write_measurement(cwd / "target/criterion", "lu_solve_pivoting_d2/la_stack_lu_solve", "new", 8.123456789012345) - if command == "uv": - _write_fake_rendered_artifacts(args, version="0.4.6", baseline="v0.4.5") - return _result() - - monkeypatch.setattr(archive_performance, "run_git_command", fake_run_git) - monkeypatch.setattr(archive_performance, "run_git_bytes", fake_run_git_with_input) - monkeypatch.setattr(archive_performance, "run_safe_command", fake_run_safe) - - report_id = generate_worktree_report( - output=output, - config=GenerationConfig( - repo_root=tmp_path, - current_tag="v0.4.6", - baseline_tag="v0.4.5", - worktree_ref="HEAD", - suite="vs_linalg", - apply_current_diff=False, - ), - ) - - assert report_id.archive_name == "v0.4.6-vs-v0.4.5.md" - assert any(kind == "just" and args == ("bench-save-baseline", "v0.4.5", "vs_linalg") for kind, args, _ in calls) - assert any(kind == "just" and args == ("bench-vs-linalg-la-stack",) for kind, args, _ in calls) - assert not any(kind == "cargo" for kind, _, _ in calls) - assert not any(kind == "just" and args == ("bench-exact",) for kind, args, _ in calls) - - current = tmp_path / "docs" / "performance.md" - if fail_promotion: - _fail_publication_once(monkeypatch, tmp_path / "docs/archive/performance/README.md") - with pytest.raises(OSError, match="simulated retained summary"): - archive_performance.render_and_promote_artifacts( - artifacts=ArtifactPaths(csv=output.with_suffix(".csv"), provenance=output.with_suffix(".provenance.json")), - output=output, - current=current, - archive_dir=tmp_path / "docs/archive/performance", - ) - assert not current.exists() - assert not list((tmp_path / "docs/performance").rglob("*.csv")) - assert not (tmp_path / "docs/performance/latest.json").exists() - return - promoted = archive_performance.render_and_promote_artifacts( - artifacts=ArtifactPaths( - csv=output.with_suffix(".csv"), - provenance=output.with_suffix(".provenance.json"), - ), - output=output, - current=current, - archive_dir=tmp_path / "docs" / "archive" / "performance", - ) - assert promoted == report_id - assert current.read_text(encoding="utf-8") == output.read_text(encoding="utf-8") - _assert_complete_retention_after_clean(tmp_path, calls, monkeypatch) - - -def _assert_complete_retention_after_clean(tmp_path: Path, calls: list[RunnerCall], monkeypatch: pytest.MonkeyPatch) -> None: - output = tmp_path / "target/bench-reports/performance.md" - current = tmp_path / "docs/performance.md" - original = current.read_bytes() - scratch = ArtifactPaths(csv=output.with_suffix(".csv"), provenance=output.with_suffix(".provenance.json")) - full_text = full_summary_paths(scratch).csv.read_text(encoding="utf-8") - rows = list(csv.DictReader(io.StringIO(full_text))) - assert len(rows) == 5 - assert {(row["phase"], row["benchmark_id"]) for row in rows if "faer" in row["benchmark_id"]} == { - ("baseline", "lu_solve_pivoting_d2/faer_lu_solve"), - } - precise = next(row for row in rows if row["phase"] == "current" and row["benchmark_id"].startswith("lu_solve_pivoting")) - assert float(precise["mean_ns"]) == 8.123456789012345 - shutil.rmtree(tmp_path / "target") - retained = resolve_report_paths(tmp_path, scratch) - assert full_summary_paths(retained).csv.read_text(encoding="utf-8") == full_text - assert _resolve_performance_paths(tmp_path, "target/bench-reports/performance.csv") == retained - calls.clear() - monkeypatch.chdir(tmp_path) - assert main(["--promote-artifacts"]) == 0 - assert not calls - assert current.read_bytes() == original - assert full_summary_paths(retained).csv.read_text(encoding="utf-8") == full_text - - -def test_main_generates_latest_published_report_from_github_releases(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - current = tmp_path / "docs" / "performance.md" - archive_dir = tmp_path / "docs" / "archive" / "performance" - calls: list[RunnerCall] = [] - - def fake_run_git(args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append(("git", tuple(args), cwd)) - if args[:3] == ["worktree", "add", "--detach"]: - worktree = Path(args[3]) - worktree.mkdir(parents=True) - _write_current_benchmark_tooling(worktree) - return _result() - - def fake_run_git_with_input(args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append(("git-stdin", tuple(args), cwd)) - return _result() - - def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append((command, tuple(args), cwd)) - if command == "gh" and args[:2] == ["release", "list"]: - return _result( - "[" - '{"tagName":"v0.4.5","isDraft":false,"isPrerelease":false,"publishedAt":"2026-01-01T00:00:00Z"},' - '{"tagName":"v0.4.6","isDraft":false,"isPrerelease":false,"publishedAt":"2026-02-01T00:00:00Z"}' - "]" - ) - if command == "gh": - download_dir = Path(args[args.index("--dir") + 1]) - tag = args[2] - _write_baseline_archive(download_dir / f"la-stack-{tag}-criterion-baseline.tar.gz") - if command == "uv": - _write_fake_rendered_artifacts(args, version="0.4.6", baseline="v0.4.5") - return _result() - - monkeypatch.chdir(tmp_path) - monkeypatch.setattr(archive_performance, "run_git_command", fake_run_git) - monkeypatch.setattr(archive_performance, "run_git_bytes", fake_run_git_with_input) - monkeypatch.setattr(archive_performance, "run_safe_command", fake_run_safe) - - rc = main( - [ - "--current", - str(current), - "--archive-dir", - str(archive_dir), - "--published-latest", - "--github-assets", - "--generate-in-temp-worktree", - "--no-apply-current-diff", - ] - ) - - assert rc == 0 - assert current.read_text(encoding="utf-8") == _retained_report(tmp_path) - assert any( - kind == "git" - and args - == ( - "fetch", - "origin", - "refs/tags/v0.4.6:refs/tags/v0.4.6", - "refs/tags/v0.4.5:refs/tags/v0.4.5", - ) - for kind, args, _ in calls - ) - fetch_index = next(index for index, (kind, args, _) in enumerate(calls) if kind == "git" and args[:2] == ("fetch", "origin")) - worktree_index = next(index for index, (kind, args, _) in enumerate(calls) if kind == "git" and args[:3] == ("worktree", "add", "--detach")) - assert fetch_index < worktree_index - assert any(kind == "git" and args[:3] == ("worktree", "add", "--detach") and args[4] == "v0.4.6" for kind, args, _ in calls) - - -def test_main_normalizes_explicit_bare_tags_before_fetching_and_checkout(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - output = tmp_path / "target" / "bench-reports" / "github-assets-performance.md" - current = tmp_path / "docs" / "performance.md" - archive_dir = tmp_path / "docs" / "archive" / "performance" - calls: list[RunnerCall] = [] - - def fake_run_git(args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append(("git", tuple(args), cwd)) - if args[:3] == ["worktree", "add", "--detach"]: - worktree = Path(args[3]) - worktree.mkdir(parents=True) - _write_current_benchmark_tooling(worktree) - return _result() - - def fake_run_git_with_input(args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append(("git-stdin", tuple(args), cwd)) - return _result() - - def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append((command, tuple(args), cwd)) - if command == "gh": - download_dir = Path(args[args.index("--dir") + 1]) - tag = args[2] - _write_baseline_archive(download_dir / f"la-stack-{tag}-criterion-baseline.tar.gz") - if command == "uv": - _write_fake_rendered_artifacts(args, version="0.4.6", baseline="v0.4.5") - return _result() - - monkeypatch.chdir(tmp_path) - monkeypatch.setattr(archive_performance, "run_git_command", fake_run_git) - monkeypatch.setattr(archive_performance, "run_git_bytes", fake_run_git_with_input) - monkeypatch.setattr(archive_performance, "run_safe_command", fake_run_safe) - - rc = main( - [ - "0.4.6", - "0.4.5", - "--current", - str(current), - "--archive-dir", - str(archive_dir), - "--github-assets", - "--generate-in-temp-worktree", - "--worktree-ref", - "0.4.6", - "--output-only", - "--output", - str(output), - ] - ) - - assert rc == 0 - assert output.read_text(encoding="utf-8") == _retained_report(tmp_path) - assert not current.exists() - assert any( - kind == "git" - and args - == ( - "fetch", - "origin", - "refs/tags/v0.4.5:refs/tags/v0.4.5", - "refs/tags/v0.4.6:refs/tags/v0.4.6", - ) - for kind, args, _ in calls - ) - fetch_index = next(index for index, (kind, args, _) in enumerate(calls) if kind == "git" and args[:2] == ("fetch", "origin")) - worktree_index = next(index for index, (kind, args, _) in enumerate(calls) if kind == "git" and args[:3] == ("worktree", "add", "--detach")) - assert fetch_index < worktree_index - assert any(kind == "git" and args[:3] == ("worktree", "add", "--detach") and args[4] == "v0.4.6" for kind, args, _ in calls) - assert any(kind == "gh" and args[:3] == ("release", "download", "v0.4.5") for kind, args, _ in calls) - assert any(kind == "gh" and args[:3] == ("release", "download", "v0.4.6") for kind, args, _ in calls) - - -def test_main_published_latest_fetch_failure_stops_before_worktree(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: - current = tmp_path / "docs" / "performance.md" - archive_dir = tmp_path / "docs" / "archive" / "performance" - calls: list[RunnerCall] = [] - - def fake_run_git(args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append(("git", tuple(args), cwd)) - if args[:2] == ["fetch", "origin"]: - raise subprocess.CalledProcessError(128, ["git", *args], output="fetch stdout", stderr="missing tag") - if args[:3] == ["worktree", "add", "--detach"]: - worktree = Path(args[3]) - worktree.mkdir(parents=True) - _write_current_benchmark_tooling(worktree) - return _result() - - def fake_run_git_with_input(args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append(("git-stdin", tuple(args), cwd)) - return _result() - - def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append((command, tuple(args), cwd)) - if command == "gh" and args[:2] == ["release", "list"]: - return _result( - "[" - '{"tagName":"v0.4.5","isDraft":false,"isPrerelease":false,"publishedAt":"2026-01-01T00:00:00Z"},' - '{"tagName":"v0.4.6","isDraft":false,"isPrerelease":false,"publishedAt":"2026-02-01T00:00:00Z"}' - "]" - ) - return _result() - - monkeypatch.chdir(tmp_path) - monkeypatch.setattr(archive_performance, "run_git_command", fake_run_git) - monkeypatch.setattr(archive_performance, "run_git_bytes", fake_run_git_with_input) - monkeypatch.setattr(archive_performance, "run_safe_command", fake_run_safe) - - rc = main( - [ - "--current", - str(current), - "--archive-dir", - str(archive_dir), - "--published-latest", - "--generate-in-temp-worktree", - "--no-apply-current-diff", - ] - ) - - captured = capsys.readouterr() - assert rc == 1 - assert "command failed (128): git fetch origin" in captured.err - assert "missing tag" in captured.err - assert not current.exists() - assert not any(kind == "git" and args[:3] == ("worktree", "add", "--detach") for kind, args, _ in calls) - assert not any(kind in {"just", "uv"} for kind, _, _ in calls) - assert not any(kind == "git-stdin" for kind, _, _ in calls) - - -def test_failed_archive_index_update_rolls_back_report_and_new_archive( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - source = tmp_path / "performance-new.md" - current = tmp_path / "docs" / "performance.md" - archive_dir = tmp_path / "docs" / "archive" / "performance" - original = _normalized_report("0.4.5", "v0.4.4") - source.write_text(_report("0.4.6", "v0.4.5"), encoding="utf-8") - current.parent.mkdir(parents=True) - current.write_text(original, encoding="utf-8") - - real_replace = Path.replace - failed = False - - def fail_index(source_path: Path, destination: Path) -> Path: - nonlocal failed - if destination == archive_dir / "README.md" and not failed: - failed = True - msg = "simulated archive index failure" - raise OSError(msg) - return real_replace(source_path, destination) - - monkeypatch.setattr(Path, "replace", fail_index) - - with pytest.raises(OSError, match="simulated archive index failure"): - promote_report( - source=source, - current=current, - archive_dir=archive_dir, - expected_current_tag="v0.4.6", - expected_baseline_tag="v0.4.5", - ) - - assert current.read_text(encoding="utf-8") == original - assert not (archive_dir / "v0.4.5-vs-v0.4.4.md").exists() - assert not (archive_dir / "README.md").exists() - - -def test_failed_artifact_promotion_output_write_rolls_back_report_archive_and_index( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - current = tmp_path / "docs" / "performance.md" - archive_dir = tmp_path / "docs" / "archive" / "performance" - index = archive_dir / "README.md" - output = tmp_path / "target" / "bench-reports" / "performance.md" - original_current = _report("0.4.5", "v0.4.4") - original_index = "original index\n" - original_output = "original output\n" - current.parent.mkdir(parents=True) - current.write_text(original_current, encoding="utf-8") - archive_dir.mkdir(parents=True) - index.write_text(original_index, encoding="utf-8") - output.parent.mkdir(parents=True) - output.write_text(original_output, encoding="utf-8") - real_replace = Path.replace - failed = False - - def fail_output_write(source: Path, destination: Path) -> Path: - nonlocal failed - if destination == output and not failed: - failed = True - msg = "simulated artifact promotion output failure" - raise OSError(msg) - return real_replace(source, destination) - - monkeypatch.setattr(Path, "replace", fail_output_write) - - with pytest.raises(OSError, match="simulated artifact promotion output failure"): - archive_performance._promote_report_text( - source_text=_report("0.4.6", "v0.4.5"), - request=archive_performance.PromotionRequest( - current=current, - archive_dir=archive_dir, - expected=archive_performance.ReportId(current_tag="v0.4.6", baseline_tag="v0.4.5"), - output=output, - ), - ) - - assert current.read_text(encoding="utf-8") == original_current - assert index.read_text(encoding="utf-8") == original_index - assert output.read_text(encoding="utf-8") == original_output - assert not (archive_dir / "v0.4.5-vs-v0.4.4.md").exists() - - -def test_generate_and_promote_rejects_artifact_alias_before_worktree(tmp_path: Path) -> None: - current = tmp_path / "docs" / "performance.md" - archive_dir = tmp_path / "docs" / "archive" / "performance" - current.parent.mkdir(parents=True) - current.write_text(_report("0.4.5", "v0.4.4"), encoding="utf-8") - artifacts = ArtifactPaths( - csv=current, - provenance=current.with_suffix(".provenance.json"), - ) - - with pytest.raises(ValueError, match="must use distinct paths"): - generate_and_promote_worktree_report( - output=tmp_path / "target" / "bench-reports" / "performance.md", - current=current, - archive_dir=archive_dir, - config=GenerationConfig( - repo_root=tmp_path, - current_tag="v0.4.6", - baseline_tag="v0.4.5", - worktree_ref="HEAD", - ), - artifacts=artifacts, - ) - - assert current.read_text(encoding="utf-8") == _report("0.4.5", "v0.4.4") - assert not archive_dir.exists() - - -@pytest.mark.parametrize("alias", ["selected", "full-csv", "full-json"]) -def test_generate_output_only_rejects_artifact_alias_before_worktree(tmp_path: Path, alias: str) -> None: - report = tmp_path / "target" / "bench-reports" / "performance.csv" - artifacts = ArtifactPaths( - csv=report, - provenance=report.with_suffix(".provenance.json"), - ) - output = {"selected": report, "full-csv": full_summary_paths(artifacts).csv, "full-json": full_summary_paths(artifacts).provenance}[alias] - - with pytest.raises(ValueError, match="must use distinct paths"): - generate_worktree_report( - output=output, - config=GenerationConfig( - repo_root=tmp_path, - current_tag="v0.4.6", - baseline_tag="v0.4.5", - worktree_ref="HEAD", - ), - artifacts=artifacts, - ) - - assert not output.exists() - - -def test_generate_and_promote_uses_temp_worktree_and_current_diff( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - monkeypatch.delenv("RUSTUP_TOOLCHAIN", raising=False) - (tmp_path / "rust-toolchain.toml").write_text('[toolchain]\nchannel = "1.97.0"\n', encoding="utf-8") - current = tmp_path / "docs" / "performance.md" - output = tmp_path / "target" / "bench-reports" / "performance.md" - archive_dir = tmp_path / "docs" / "archive" / "performance" - current.parent.mkdir(parents=True) - current.write_text(_report("0.4.5", "v0.4.4"), encoding="utf-8") - calls: list[RunnerCall] = [] - - def fake_run_git(args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append(("git", tuple(args), cwd)) - if args[:3] == ["worktree", "add", "--detach"]: - worktree = Path(args[3]) - worktree.mkdir(parents=True) - _write_current_benchmark_tooling(worktree) - if args[:2] == ["diff", "--binary"]: - output_arg = next(arg for arg in args if arg.startswith("--output=")) - Path(output_arg.removeprefix("--output=")).write_bytes(b"diff --git a/README.md b/README.md\n") - return _result() - - def fake_run_git_with_input(args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append(("git-stdin", tuple(args), cwd)) - payload = kwargs["input"] - assert isinstance(payload, bytes) - assert b"diff --git" in payload - return _result() - - def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append((command, tuple(args), cwd)) - if command == "just" and args == ["bench-save-baseline", "v0.4.5"]: - assert kwargs["env"]["RUSTUP_TOOLCHAIN"] == "1.97.0" - assert cwd is not None - criterion_dir = cwd / "target" / "criterion" - criterion_dir.mkdir(parents=True) - (criterion_dir / "baseline.txt").write_text("baseline\n", encoding="utf-8") - if command == "just" and args == ["bench-latest"]: - assert kwargs["env"]["RUSTUP_TOOLCHAIN"] == "1.97.0" - if command == "uv": - _write_fake_rendered_artifacts(args, version="0.4.6", baseline="v0.4.5") - return _result() - - monkeypatch.setattr(archive_performance, "run_git_command", fake_run_git) - monkeypatch.setattr(archive_performance, "run_git_bytes", fake_run_git_with_input) - monkeypatch.setattr(archive_performance, "run_safe_command", fake_run_safe) - - report_id = generate_and_promote_worktree_report( - output=output, - current=current, - archive_dir=archive_dir, - config=GenerationConfig( - repo_root=tmp_path, - current_tag="v0.4.6", - baseline_tag="v0.4.5", - worktree_ref="HEAD", - apply_current_diff=True, - ), - ) - - assert report_id.archive_name == "v0.4.6-vs-v0.4.5.md" - assert current.read_text(encoding="utf-8") == _retained_report(tmp_path) - assert output.read_text(encoding="utf-8") == current.read_text(encoding="utf-8") - assert (archive_dir / "v0.4.5-vs-v0.4.4.md").exists() - assert any(kind == "git" and args[:3] == ("worktree", "add", "--detach") and args[4] == "HEAD" for kind, args, _ in calls) - assert any(kind == "git-stdin" and args == ("apply", "--binary") for kind, args, _ in calls) - assert any(kind == "just" and args == ("bench-latest",) for kind, args, _ in calls) - assert any(kind == "git" and args[:3] == ("worktree", "remove", "--force") for kind, args, _ in calls) - assert "applying staged and unstaged tracked changes; untracked files are excluded" in capsys.readouterr().err - - -def test_generate_and_promote_legacy_published_tag_uses_legacy_commands(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - current = tmp_path / "docs" / "performance.md" - output = tmp_path / "target" / "bench-reports" / "performance.md" - archive_dir = tmp_path / "docs" / "archive" / "performance" - calls: list[RunnerCall] = [] - - def fake_run_git(args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append(("git", tuple(args), cwd)) - if args[:3] == ["worktree", "add", "--detach"]: - worktree = Path(args[3]) - worktree.mkdir(parents=True) - _write_legacy_benchmark_tooling(worktree) - return _result() - - def fake_run_git_with_input(args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append(("git-stdin", tuple(args), cwd)) - return _result() - - def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: - calls.append((command, tuple(args), cwd)) - if command == "just" and args == ["bench-save-baseline", "v0.4.4"]: - assert cwd is not None - criterion_dir = cwd / "target" / "criterion" - criterion_dir.mkdir(parents=True) - (criterion_dir / "baseline.txt").write_text("baseline\n", encoding="utf-8") - if command == "uv": - _write_fake_rendered_artifacts(args, version="0.4.5", baseline="v0.4.4") - return _result() - - monkeypatch.setattr(archive_performance, "run_git_command", fake_run_git) - monkeypatch.setattr(archive_performance, "run_git_bytes", fake_run_git_with_input) - monkeypatch.setattr(archive_performance, "run_safe_command", fake_run_safe) - - report_id = generate_and_promote_worktree_report( - output=output, - current=current, - archive_dir=archive_dir, - config=GenerationConfig( - repo_root=tmp_path, - current_tag="v0.4.5", - baseline_tag="v0.4.4", - worktree_ref="v0.4.5", - apply_current_diff=False, - ), - ) - - assert report_id.archive_name == "v0.4.5-vs-v0.4.4.md" - assert current.read_text(encoding="utf-8") == _retained_report(tmp_path) - assert output.read_text(encoding="utf-8") == current.read_text(encoding="utf-8") - assert any(kind == "git" and args[:3] == ("worktree", "add", "--detach") and args[4] == "v0.4.5" for kind, args, _ in calls) - assert any(kind == "cargo" and args == ("bench", "--locked", "--features", "bench,exact") for kind, args, _ in calls) - assert not any(kind == "just" and args == ("bench-exact",) for kind, args, _ in calls) - assert not any(kind == "just" and args == ("bench-latest",) for kind, args, _ in calls) - assert any(kind == "uv" and "--suite" in args for kind, args, _ in calls) - assert any(kind == "uv" and "--scope" in args for kind, args, _ in calls) - assert not any(kind == "git" and args[:1] == ("read-tree",) for kind, args, _ in calls) - assert not any(kind == "git-stdin" for kind, _, _ in calls) diff --git a/scripts/tests/test_bench_compare.py b/scripts/tests/test_bench_compare.py index 1a3c94e..4411bf3 100644 --- a/scripts/tests/test_bench_compare.py +++ b/scripts/tests/test_bench_compare.py @@ -191,13 +191,13 @@ def _build_vs_linalg_tree(criterion_dir: Path, stat: str = "median") -> None: class TestFormatTime: def test_nanoseconds(self) -> None: - assert bench_compare._format_time(42.5) == "42.5 ns" + assert bench_compare.format_time(42.5) == "42.5 ns" def test_microseconds(self) -> None: - assert bench_compare._format_time(1_500.0) == "1.50 µs" + assert bench_compare.format_time(1_500.0) == "1.50 µs" def test_milliseconds(self) -> None: - assert bench_compare._format_time(2_500_000.0) == "2.50 ms" + assert bench_compare.format_time(2_500_000.0) == "2.50 ms" class TestFormatPct: @@ -586,7 +586,7 @@ def test_comparison_table_preserves_signed_change(current_ns: float, change: str assessment="unknown", ) - table = bench_compare._comparison_tables([comparison], "last") + table = bench_compare.comparison_tables([comparison], "last") assert f"| {change} |" in table @@ -623,7 +623,7 @@ def test_comparison_uses_one_table_per_suite_with_case_column(tmp_path: Path) -> _write_estimates(group / bench / "v0.4.4" / "estimates.json", "median", det_exact * 1.3) comparisons = bench_compare._collect_comparisons(tmp_path, "v0.4.4", "median").comparisons - tables = bench_compare._comparison_tables(comparisons, "v0.4.4") + tables = bench_compare.comparison_tables(comparisons, "v0.4.4") assert tables.count("| Case | Benchmark | v0.4.4 (point + CI)") == 1 assert "| D=2 |" in tables assert "| D=3 |" in tables @@ -631,10 +631,10 @@ def test_comparison_uses_one_table_per_suite_with_case_column(tmp_path: Path) -> assert "| solve_exact_f64_result |" in tables -def test_comparison_tables_include_vs_linalg_peer_context(tmp_path: Path) -> None: +def testcomparison_tables_include_vs_linalg_peer_context(tmp_path: Path) -> None: _build_vs_linalg_tree(tmp_path) comparisons = bench_compare._collect_comparisons(tmp_path, "last", "median", suite="vs_linalg").comparisons - tables = bench_compare._comparison_tables(comparisons, "last") + tables = bench_compare.comparison_tables(comparisons, "last") assert ( "| Case | Benchmark | last (point + CI) | Latest (point + CI) | Point-estimate change | CI relation | " @@ -1151,7 +1151,7 @@ def test_main_preserves_artifact_exception_group_diagnostics( "_collect_comparisons", lambda *_args, **_kwargs: SimpleNamespace(comparisons=[object()], gaps=[]), ) - monkeypatch.setattr(bench_compare, "_comparison_tables", lambda *_args, **_kwargs: "table") + monkeypatch.setattr(bench_compare, "comparison_tables", lambda *_args, **_kwargs: "table") def fail_publication(*_args: object, **_kwargs: object) -> None: message = "artifact publication and rollback failed" diff --git a/scripts/tests/test_changelog_integration.py b/scripts/tests/test_changelog_integration.py index d1f774a..ea4f765 100644 --- a/scripts/tests/test_changelog_integration.py +++ b/scripts/tests/test_changelog_integration.py @@ -1,405 +1,93 @@ -"""Consumer integration with the pinned, published changelog CLI.""" +"""Consumer policy and authored history against the installed published CLI.""" import json -import os -import re import shutil -import sys import tomllib -from datetime import UTC, datetime, timedelta from importlib.metadata import version from pathlib import Path +from typing import TYPE_CHECKING -import pytest from research_repo_tools.cli import main from benchmark_process import run_git_command, run_safe_command -REPO_ROOT = Path(__file__).resolve().parents[2] +if TYPE_CHECKING: + import pytest +ROOT = Path(__file__).resolve().parents[2] -def create_consumer(root: Path, source: Path) -> Path: - """Use real configuration and policy with a stable synthetic package identity.""" - for name in ("pyproject.toml", "cliff.toml", "changelog-rumdl.toml", "README.md", ".python-version", "rust-toolchain.toml"): - shutil.copyfile(source / name, root / name) - (root / "Cargo.toml").write_text('[package]\nname = "la-stack"\nversion = "0.4.6"\n', encoding="utf-8") - run_git_command(["init", "--quiet"], cwd=root) - run_git_command(["config", "user.name", "Changelog fixture"], cwd=root) - run_git_command(["config", "user.email", "fixture@example.invalid"], cwd=root) - run_git_command(["config", "commit.gpgsign", "false"], cwd=root) - run_git_command(["config", "tag.gpgsign", "false"], cwd=root) - return root - -@pytest.fixture -def consumer(tmp_path: Path) -> Path: - """Create one disposable consumer independent of live release and signing state.""" - return create_consumer(tmp_path, REPO_ROOT) - - -def cli(root: Path, *args: str) -> int: - """Call only the documented installed-package entry point.""" - return main(["--root", str(root), "changelog", *args]) - - -def markdown_bytes(root: Path) -> dict[Path, bytes]: - """Capture changelog outputs for read-only and failed-publication assertions.""" - paths = [root / "CHANGELOG.md", *(root / "docs/archives/changelog").glob("*.md")] - return {path.relative_to(root): path.read_bytes() for path in paths if path.is_file()} - - -def commit(root: Path, message: str) -> None: - """Append one real commit to the fixture history.""" - run_git_command(["commit", "--quiet", "--allow-empty", "-m", message], cwd=root) - - -def test_published_package_is_exactly_pinned_and_included_in_dev() -> None: - """Normal sync installs a PyPI release, with no sibling or editable override.""" - manifest = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8")) - groups = manifest["dependency-groups"] - assert groups["tooling"] == ["research-repo-tools==0.1.7"] - assert {"include-group": "tooling"} in groups["dev"] - assert version("research-repo-tools") == "0.1.7" - lock = tomllib.loads((REPO_ROOT / "uv.lock").read_text(encoding="utf-8")) - package = next(package for package in lock["package"] if package["name"] == "research-repo-tools") - assert package["version"] == "0.1.7" +def test_published_package_and_shared_policy() -> None: + """Both dependency owners and the registry lock select the published package.""" + manifest = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8")) + assert manifest["project"]["dependencies"] == ["research-repo-tools==0.1.8"] + assert manifest["dependency-groups"]["tooling"] == ["research-repo-tools==0.1.8"] + assert {"include-group": "tooling"} in manifest["dependency-groups"]["dev"] + assert version("research-repo-tools") == "0.1.8" + lock = tomllib.loads((ROOT / "uv.lock").read_text(encoding="utf-8")) + package = next(item for item in lock["package"] if item["name"] == "research-repo-tools") + assert package["version"] == "0.1.8" assert package["source"] == {"registry": "https://pypi.org/simple"} - - -def test_consumer_template_has_only_the_documented_shared_policy_exception() -> None: - """Keep the installed common format, with one bounded historical-body exception.""" - shared = run_safe_command( - "uv", - ["run", "--locked", "--group", "dev", "research-repo-tools", "templates", "cliff.toml", "--owner", "acgetchell", "--repository", "la-stack"], - cwd=REPO_ROOT, - ).stdout - expected = shared.replace( - '{%- if commit.body and group != "Dependencies" %}', - '{%- if commit.body and (group != "Dependencies" or commit.scope == "deps-dev") %}', - ) - assert tomllib.loads((REPO_ROOT / "cliff.toml").read_text(encoding="utf-8")) == tomllib.loads(expected) - - -def test_archive_rotation_preserves_notes_references_and_relative_links(consumer: Path, capsys: pytest.CaptureFixture[str]) -> None: - """Root and archived notes remain discoverable after a minor-series rotation.""" - changelog = consumer / "CHANGELOG.md" - changelog.write_text( - "# Changelog\n\n## [Unreleased]\n\n- Pending.\n\n" - "## [0.5.0] - 2026-09-30\n\n- New minor.\n\n" - "## [0.4.6] - 2026-09-08\n\n- Keep `Vector` and [guide][guide].\n\n" - "## [0.4.5] - 2026-08-21\n\n- Prior patch.\n\n" - "[guide]: README.md\n" - "[0.5.0]: https://github.com/acgetchell/la-stack/compare/v0.4.6...v0.5.0\n" - "[0.4.6]: https://github.com/acgetchell/la-stack/compare/v0.4.5...v0.4.6\n", - encoding="utf-8", - ) - assert cli(consumer, "archive") == 0 - root = changelog.read_text(encoding="utf-8") - archive = consumer / "docs/archives/changelog/0.4.md" - archived = archive.read_text(encoding="utf-8") - assert "## [Unreleased]" in root - assert "## [0.5.0] - 2026-09-30" in root - assert "## [0.4.6]" not in root - assert "[0.4.x](docs/archives/changelog/0.4.md)" in root - assert "## [0.4.6] - 2026-09-08" in archived - assert "## [0.4.5] - 2026-08-21" in archived - assert "`Vector`" in archived - assert "[guide]: ../../../README.md" in archived - before = markdown_bytes(consumer) - assert cli(consumer, "archive") == 0 - assert cli(consumer, "check") == 0 - assert markdown_bytes(consumer) == before - capsys.readouterr() - assert cli(consumer, "notes", "v0.4.6") == 0 - notes = capsys.readouterr().out - assert "Keep `Vector`" in notes - assert "[guide]: ../../../README.md" in notes - assert "Prior patch" not in notes - assert "[0.5.0]:" not in notes - assert cli(consumer, "notes", "v0.5.0") == 0 - assert "New minor" in capsys.readouterr().out - assert markdown_bytes(consumer) == before - - -def test_generation_uses_consumer_policy_dates_and_transactional_preview(consumer: Path, capsys: pytest.CaptureFixture[str]) -> None: - """Real Git/git-cliff generation retains authored dependency links and explicit dates.""" - commit(consumer, "feat: add exact arithmetic") - run_git_command(["tag", "v0.4.5"], cwd=consumer) - commit( - consumer, - "chore(deps-dev): bump ruff from 0.16.1 to 0.16.2\n\nRead [release notes](https://github.com/astral-sh/ruff/compare/0.16.1...0.16.2).", - ) - commit( - consumer, - "fix!: preserve `Vector` invariants (#42)\n\n" - "Keep `Matrix` and literal `<D>`.\n\n" - "```rust\nfn value() -> Vector { todo!() }\n```\n\n" - "BREAKING CHANGE: preserve exact errors and migration instructions.", + assert package["wheels"][0]["hash"] == "sha256:9db45f09a33cac4ff32ddf010d30523e295f607e24c680bcfc3d3d9786991293" + assert package["sdist"]["hash"] == "sha256:bc855aa930ee04301dceebc8340a22576bb25712e322b51d7afaf20f5fab1ba0" + assert manifest["tool"]["research-repo-tools"]["changelog"] == { + "owner": "acgetchell", + "repository": "la-stack", + "formatter": "changelog-rumdl.toml", + "dependency-bodies": "preserve", + } + assert not (ROOT / "cliff.toml").exists() + + +def test_authored_dependency_notes_survive_generation(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + """Exercise the consumer policy with Ruff, Ty and setuptools notes and code.""" + for name in ("pyproject.toml", "changelog-rumdl.toml", ".python-version", "rust-toolchain.toml"): + shutil.copyfile(ROOT / name, tmp_path / name) + run_git_command(["init", "--quiet"], cwd=tmp_path) + for key, value in (("user.name", "Fixture"), ("user.email", "fixture@example.invalid"), ("commit.gpgsign", "false"), ("tag.gpgsign", "false")): + run_git_command(["config", key, value], cwd=tmp_path) + run_git_command(["commit", "--quiet", "--allow-empty", "-m", "feat: initial history"], cwd=tmp_path) + run_git_command(["tag", "v0.4.5"], cwd=tmp_path) + notes = ( + ("ruff", "deps-dev", "Read [release notes](https://github.com/astral-sh/ruff/compare/0.16.1...0.16.2) before changing `Vector` checks."), + ("ty", "deps-dev", "Keep [changelog](https://github.com/astral-sh/ty/releases) context and `--error all`."), + ( + "setuptools", + "deps", + 'See [comparison](https://github.com/pypa/setuptools/compare/v83.0.0...v84.0.0).\n\n```toml\nrequires = ["setuptools>=84.0.0"]\n```', + ), ) - (consumer / "CHANGELOG.md").write_text("# Changelog\n\n## [0.4.5] - 2000-01-02\n\n- Authored date.\n", encoding="utf-8") - metadata_before = {name: (consumer / name).read_bytes() for name in ("Cargo.toml", "pyproject.toml")} - before = markdown_bytes(consumer) - assert cli(consumer, "generate", "--tag", "v0.5.0", "--date", "2026-09-30", "--dry-run") == 0 + for dependency, scope, body in notes: + run_git_command(["commit", "--quiet", "--allow-empty", "-m", f"chore({scope}): bump {dependency}\n\n{body}"], cwd=tmp_path) + (tmp_path / "CHANGELOG.md").write_text("# Changelog\n\n## [0.4.5] - 2000-01-02\n\n- Historical date.\n", encoding="utf-8") + args = ["--root", str(tmp_path), "changelog", "generate", "--tag", "v0.4.6", "--date", "2026-10-09"] + assert main([*args, "--dry-run"]) == 0 preview = capsys.readouterr().out - assert "## [0.5.0] - 2026-09-30" in preview - assert "https://github.com/astral-sh/ruff/compare/0.16.1...0.16.2" in preview - assert "https://github.com/acgetchell/la-stack/pull/42" in preview - assert "preserve exact errors and migration instructions" in preview - assert "`Vector`" in preview - assert "`Matrix`" in preview - assert "`<D>`" in preview - assert "fn value() -> Vector" in preview - assert markdown_bytes(consumer) == before - assert not (consumer / "docs/archives/changelog").exists() - assert cli(consumer, "generate", "--tag", "v0.5.0", "--date", "2026-09-30") == 0 - assert (consumer / "CHANGELOG.md").read_text(encoding="utf-8") == preview - archive = consumer / "docs/archives/changelog/0.4.md" - assert "## [0.4.5] - 2000-01-02" in archive.read_text(encoding="utf-8") - assert {name: (consumer / name).read_bytes() for name in metadata_before} == metadata_before - generated = markdown_bytes(consumer) - assert cli(consumer, "generate", "--tag", "v0.5.0", "--date", "2026-09-30") == 0 - assert markdown_bytes(consumer) == generated - - -@pytest.mark.parametrize( - ("args", "diagnostic"), - [ - (("generate", "--tag", "v0.5.0"), "both --tag and --date"), - (("generate", "--date", "2026-09-30"), "both --tag and --date"), - (("generate", "--tag", "v0.5.0", "--date", "2026-02-30"), "day"), - (("generate", "--tag", "v01.5.0", "--date", "2026-09-30"), "SemVer format"), - (("notes", "0.4.6"), "SemVer format"), - (("notes", "v9.9.9"), "not found"), - ], -) -def test_cli_errors_are_actionable_and_preserve_outputs(consumer: Path, capsys: pytest.CaptureFixture[str], args: tuple[str, ...], diagnostic: str) -> None: - (consumer / "CHANGELOG.md").write_text("# Changelog\n\n## [0.4.6] - 2026-09-08\n\n- Retained.\n", encoding="utf-8") - before = markdown_bytes(consumer) - assert cli(consumer, *args) == 1 - error = capsys.readouterr().err - assert "research-repo-tools:" in error - assert diagnostic.casefold() in error.casefold() - assert "Traceback" not in error - assert markdown_bytes(consumer) == before - - -@pytest.mark.parametrize( - ("headings", "diagnostic"), - [ - ("## [0.4.5]\n\n- Older.\n\n## [0.4.6]\n\n- Newer.\n", "out of order"), - ("## [0.4.6]\n\n- One.\n\n## [0.4.6]\n\n- Duplicate.\n", "duplicate"), - ("## [0.4.6] - 2026-02-30\n\n- Invalid date.\n", "date"), - ], -) -def test_whole_history_errors_do_not_rotate_files(consumer: Path, capsys: pytest.CaptureFixture[str], headings: str, diagnostic: str) -> None: - (consumer / "CHANGELOG.md").write_text("# Changelog\n\n" + headings, encoding="utf-8") - before = markdown_bytes(consumer) - for command in ("check", "archive"): - assert cli(consumer, command) == 1 - assert diagnostic in capsys.readouterr().err.casefold() - assert markdown_bytes(consumer) == before - - -def test_archive_conflicts_preserve_retained_history(consumer: Path, capsys: pytest.CaptureFixture[str]) -> None: - """Changed notes for an already archived release fail before any replacement.""" - (consumer / "CHANGELOG.md").write_text( - "# Changelog\n\n## [0.5.0] - 2026-09-30\n\n- Current.\n\n## [0.4.6] - 2026-09-08\n\n- Conflicting notes.\n", - encoding="utf-8", - ) - archive = consumer / "docs/archives/changelog/0.4.md" - archive.parent.mkdir(parents=True) - archive.write_text("# Changelog - 0.4.x\n\n## [0.4.6] - 2026-09-08\n\n- Authored retained notes.\n", encoding="utf-8") - before = markdown_bytes(consumer) - assert cli(consumer, "archive") == 1 - assert "conflicting retained release" in capsys.readouterr().err.casefold() - assert markdown_bytes(consumer) == before - - -def test_tag_previews_and_force_preserve_existing_ref_on_error(consumer: Path, capsys: pytest.CaptureFixture[str]) -> None: - """Exercise tag and force through the public CLI in a disposable repository only.""" - commit(consumer, "feat: initial fixture") - release_date = datetime.now(UTC).date().isoformat() - (consumer / "CHANGELOG.md").write_text(f"# Changelog\n\n## [0.4.6] - {release_date}\n\n- Exact release notes.\n", encoding="utf-8") - before = markdown_bytes(consumer) - assert cli(consumer, "tag", "v0.4.6", "--dry-run") == 0 - assert "Exact release notes" in capsys.readouterr().out - assert run_git_command(["tag", "--list"], cwd=consumer).stdout == "" - assert cli(consumer, "tag", "v0.4.6") == 0 - ref = run_git_command(["rev-parse", "refs/tags/v0.4.6"], cwd=consumer).stdout - assert run_git_command(["cat-file", "-t", "refs/tags/v0.4.6"], cwd=consumer).stdout.strip() == "tag" - assert cli(consumer, "tag", "v0.4.6") == 1 - assert "already exists" in capsys.readouterr().err - assert cli(consumer, "tag", "v0.5.0", "--force") == 1 - assert "does not match package version" in capsys.readouterr().err - (consumer / "CHANGELOG.md").write_text("# Changelog\n\n## [0.4.6] - 2026-02-30\n\n- Invalid.\n", encoding="utf-8") - assert cli(consumer, "tag", "v0.4.6", "--force") == 1 - assert run_git_command(["rev-parse", "refs/tags/v0.4.6"], cwd=consumer).stdout == ref - (consumer / "CHANGELOG.md").write_bytes(before[Path("CHANGELOG.md")]) - assert cli(consumer, "tag", "v0.4.6", "--force") == 0 - assert markdown_bytes(consumer) == before - - -def test_consumer_recipes_forward_cli_arguments_and_keep_metadata_separate() -> None: - """Check the actual merged recipes and alias, including shell-safe argument forwarding.""" - recipes = json.loads(run_safe_command("just", ["--dump", "--dump-format", "json"], cwd=REPO_ROOT).stdout) + assert "### Dependencies" in preview + assert "## [0.4.5] - 2000-01-02" in preview + for expected in ( + "https://github.com/astral-sh/ruff/compare/0.16.1...0.16.2", + "https://github.com/astral-sh/ty/releases", + "https://github.com/pypa/setuptools/compare/v83.0.0...v84.0.0", + "before changing `Vector` checks.", + "context and `--error all`.", + 'requires = ["setuptools>=84.0.0"]', + ): + assert expected in preview + assert main(args) == 0 + generated = (tmp_path / "CHANGELOG.md").read_bytes() + assert generated == preview.encode() + assert main(args) == 0 + assert (tmp_path / "CHANGELOG.md").read_bytes() == generated + + +def test_consumer_recipes_and_retained_history(capsys: pytest.CaptureFixture[str]) -> None: + """Native recipes delegate generation and all historical notes remain readable.""" + recipes = json.loads(run_safe_command("just", ["--dump", "--dump-format", "json"], cwd=ROOT).stdout) assert recipes["aliases"]["changelog-unreleased"]["target"] == "changelog-release" - commands = { - ("changelog",): "changelog generate", - ("changelog-preview", "--tag", "v0.5.0", "--date", "2026-09-30"): 'generate --dry-run "$@"', - ("changelog-release", "v0.5.0", "2026-09-30"): "generate --tag 'v0.5.0' --date '2026-09-30'", - ("changelog-unreleased", "v0.5.0", "2026-09-30"): "generate --tag 'v0.5.0' --date '2026-09-30'", - ("changelog-archive",): "changelog archive", - ("changelog-check",): "changelog check", - ("release-notes", "v0.4.6"): "changelog notes 'v0.4.6'", - ("tag", "v0.4.6"): "changelog tag 'v0.4.6'", - ("tag-force", "v0.4.6"): "changelog tag 'v0.4.6' --force", - } - for args, expected in commands.items(): - result = run_safe_command("just", ["--dry-run", *args], cwd=REPO_ROOT) - assert expected in result.stderr - assert "research-repo-tools" in result.stderr - if "generate" in expected: - assert "research-repo-tools toolchain run -- research-repo-tools changelog generate" in result.stderr - assert "update-release-version" not in result.stderr - - -def test_preview_recipe_executes_explicit_tag_and_date_without_publishing(consumer: Path) -> None: - """Run the imported Just adapter and installed CLI, skipping installation only.""" - commit(consumer, "feat: initial fixture") - run_git_command(["tag", "v0.4.5"], cwd=consumer) - commit(consumer, "fix: preview target date") - (consumer / "CHANGELOG.md").write_text("# Changelog\n\n## [0.4.5] - 2000-01-02\n\n- Retained.\n", encoding="utf-8") - wrapper = consumer / "recipes.just" - wrapper.write_text( - f"set allow-duplicate-recipes\nimport '{(REPO_ROOT / 'justfile').as_posix()}'\ntools-check:\n\npython-sync:\n", - encoding="utf-8", - ) - before = markdown_bytes(consumer) - metadata = {name: (consumer / name).read_bytes() for name in ("Cargo.toml", "pyproject.toml")} - result = run_safe_command( - "just", - ["--justfile", str(wrapper), "--working-directory", str(consumer), "changelog-preview", "--tag", "v0.5.0", "--date", "2026-09-30"], - cwd=consumer, - env=os.environ | {"UV_NO_SYNC": "1", "UV_PROJECT_ENVIRONMENT": sys.prefix}, - ) - assert "## [0.5.0] - 2026-09-30" in result.stdout - assert markdown_bytes(consumer) == before - assert {name: (consumer / name).read_bytes() for name in metadata} == metadata - assert not (consumer / "docs/archives/changelog").exists() - - -def test_final_repository_links_and_archived_notes_remain_valid(capsys: pytest.CaptureFixture[str]) -> None: - """Check consumer history, including references in every completed series.""" - assert cli(REPO_ROOT, "check") == 0 - root = (REPO_ROOT / "CHANGELOG.md").read_text(encoding="utf-8") - for relative in re.findall(r"\]\((docs/archives/changelog/[^)]+)\)", root): - assert (REPO_ROOT / relative).is_file() + result = run_safe_command("just", ["--dry-run", "changelog-preview", "--tag", "v0.4.7", "--date", "2026-10-09"], cwd=ROOT) + assert 'research-repo-tools changelog generate --dry-run "$@"' in result.stderr + assert main(["--root", str(ROOT), "changelog", "check"]) == 0 for tag in ("v0.1.0", "v0.2.0", "v0.3.0", "v0.4.6"): - assert cli(REPO_ROOT, "notes", tag) == 0 + assert main(["--root", str(ROOT), "changelog", "notes", tag]) == 0 assert capsys.readouterr().out.strip() - policy = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8"))["tool"]["rumdl"] - assert "MD057" not in policy["disable"] - formatter = tomllib.loads((REPO_ROOT / "changelog-rumdl.toml").read_text(encoding="utf-8")) - assert formatter["extends"] == "pyproject.toml" - assert formatter["global"]["extend-disable"] == ["MD057"] - - -def test_tag_preview_checks_citation_date_and_handles_oversized_notes(consumer: Path, capsys: pytest.CaptureFixture[str]) -> None: - """Tag annotations validate consumer release dates and retain a full-notes link.""" - commit(consumer, "feat: initial fixture") - released = datetime.now(UTC).date().isoformat() - (consumer / "CHANGELOG.md").write_text(f"# Changelog\n\n## [0.4.6] - {released}\n\n- {'Long notes. ' * 12000}\n", encoding="utf-8") - (consumer / "CITATION.cff").write_text("version: 0.4.6\ndate-released: 2000-01-01\n", encoding="utf-8") - assert cli(consumer, "tag", "v0.4.6", "--dry-run") == 1 - assert "citation.cff date and changelog release date differ" in capsys.readouterr().err.casefold() - (consumer / "CITATION.cff").write_text(f"version: 0.4.6\ndate-released: {released}\n", encoding="utf-8") - before = markdown_bytes(consumer) - assert cli(consumer, "tag", "v0.4.6", "--dry-run") == 0 - preview = capsys.readouterr().out - assert "https://github.com/acgetchell/la-stack/blob/v0.4.6/CHANGELOG.md#" in preview - assert len(preview.encode("utf-8")) < 125000 - assert markdown_bytes(consumer) == before - assert run_git_command(["tag", "--list"], cwd=consumer).stdout == "" - - -def test_tag_accepts_a_declared_release_date_before_the_tagging_day(consumer: Path) -> None: - """A release prepared before merge remains taggable with matching metadata.""" - commit(consumer, "feat: prior-day release fixture") - released = (datetime.now(UTC).date() - timedelta(days=1)).isoformat() - (consumer / "CHANGELOG.md").write_text(f"# Changelog\n\n## [0.4.6] - {released}\n\n- Prior-day release.\n", encoding="utf-8") - (consumer / "CITATION.cff").write_text(f"version: 0.4.6\ndate-released: {released}\n", encoding="utf-8") - before = markdown_bytes(consumer) - assert cli(consumer, "tag", "v0.4.6") == 0 - assert run_git_command(["cat-file", "-t", "refs/tags/v0.4.6"], cwd=consumer).stdout.strip() == "tag" - assert "Prior-day release" in run_git_command(["cat-file", "-p", "refs/tags/v0.4.6"], cwd=consumer).stdout - assert markdown_bytes(consumer) == before - - -def test_fixture_isolates_commits_and_tags_from_global_signing(consumer: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """Synthetic Git operations need no key when ambient signing is enabled.""" - signing_config = consumer / "global-signing.gitconfig" - signing_config.write_text("[commit]\ngpgsign = true\n[tag]\ngpgsign = true\n[gpg]\nprogram = fixture-signing-must-not-run\n", encoding="utf-8") - monkeypatch.setenv("GIT_CONFIG_GLOBAL", str(signing_config)) - commit(consumer, "feat: signing isolation fixture") - released = datetime.now(UTC).date().isoformat() - (consumer / "CHANGELOG.md").write_text(f"# Changelog\n\n## [0.4.6] - {released}\n\n- Unsigned fixture tag.\n", encoding="utf-8") - assert cli(consumer, "tag", "v0.4.6") == 0 - assert run_git_command(["cat-file", "-t", "refs/tags/v0.4.6"], cwd=consumer).stdout.strip() == "tag" - - -def test_fixture_tags_remain_valid_after_the_source_release_version_advances(tmp_path: Path) -> None: - """A release update in the checkout cannot invalidate synthetic tag scenarios.""" - source = tmp_path / "advanced-source" - source.mkdir() - for name in ("pyproject.toml", "cliff.toml", "changelog-rumdl.toml", "README.md", ".python-version", "rust-toolchain.toml"): - shutil.copyfile(REPO_ROOT / name, source / name) - manifest = source / "pyproject.toml" - text = manifest.read_text(encoding="utf-8") - version_before = tomllib.loads(text)["project"]["version"] - manifest.write_text(text.replace(f'version = "{version_before}"', 'version = "0.4.7"', 1), encoding="utf-8") - (source / "Cargo.toml").write_text('[package]\nname = "la-stack"\nversion = "0.4.7"\n', encoding="utf-8") - root = tmp_path / "consumer" - root.mkdir() - create_consumer(root, source) - commit(root, "feat: next source release fixture") - released = datetime.now(UTC).date().isoformat() - (root / "CHANGELOG.md").write_text(f"# Changelog\n\n## [0.4.6] - {released}\n\n- Stable fixture identity.\n", encoding="utf-8") - (root / "CITATION.cff").write_text(f"version: 0.4.6\ndate-released: {released}\n", encoding="utf-8") - assert tomllib.loads((source / "Cargo.toml").read_text(encoding="utf-8"))["package"]["version"] == "0.4.7" - assert tomllib.loads(manifest.read_text(encoding="utf-8"))["project"]["version"] == "0.4.7" - assert cli(root, "tag", "v0.4.6") == 0 - assert run_git_command(["cat-file", "-t", "refs/tags/v0.4.6"], cwd=root).stdout.strip() == "tag" - - -def test_normalization_preserves_fenced_examples_and_prerelease_notes(consumer: Path, capsys: pytest.CaptureFixture[str]) -> None: - """Rust code, Markdown links, and fenced release examples remain literal.""" - (consumer / "CHANGELOG.md").write_text( - "# Changelog\n\n## [0.4.6-rc.1+build.7] - 2026-09-30\n\n" - "- Keep `Vector` and [PR](https://github.com/acgetchell/la-stack/pull/42).\n\n" - "```rust\nfn value() -> Vector { todo!() }\n```\n\n" - "~~~text\n## [99.9.9]\n~~~\n", - encoding="utf-8", - ) - assert cli(consumer, "normalize") == 0 - assert cli(consumer, "notes", "v0.4.6-rc.1+build.7") == 0 - notes = capsys.readouterr().out - assert "`Vector`" in notes - assert "fn value() -> Vector" in notes - assert "## [99.9.9]" in notes - assert "https://github.com/acgetchell/la-stack/pull/42" in notes - assert cli(consumer, "check") == 0 - - -def test_formatter_failure_preserves_every_candidate(consumer: Path, capsys: pytest.CaptureFixture[str]) -> None: - """A consumer configuration error cannot publish a partial root or archive.""" - commit(consumer, "feat: initial fixture") - run_git_command(["tag", "v0.4.5"], cwd=consumer) - commit(consumer, "fix: add an unreleased change") - (consumer / "CHANGELOG.md").write_text("# Changelog\n\n## [0.4.5] - 2026-08-21\n\n- Retained.\n", encoding="utf-8") - manifest = consumer / "pyproject.toml" - manifest.write_text(manifest.read_text(encoding="utf-8").replace('formatter = "changelog-rumdl.toml"', 'formatter = "missing.toml"'), encoding="utf-8") - before = markdown_bytes(consumer) - assert cli(consumer, "generate", "--tag", "v0.5.0", "--date", "2026-09-30") == 1 - assert "formatter configuration not found" in capsys.readouterr().err.casefold() - assert markdown_bytes(consumer) == before - assert not (consumer / "docs/archives/changelog").exists() diff --git a/scripts/tests/test_criterion_dim_plot.py b/scripts/tests/test_criterion_dim_plot.py index 1989525..69fa91a 100644 --- a/scripts/tests/test_criterion_dim_plot.py +++ b/scripts/tests/test_criterion_dim_plot.py @@ -3,7 +3,6 @@ import hashlib import json import re -import shutil import subprocess import tomllib from dataclasses import replace @@ -14,7 +13,6 @@ import pytest import criterion_dim_plot -from benchmark_summaries import retained_outputs, summary_outputs from performance_artifacts import ( ArtifactContext, ArtifactPaths, @@ -23,7 +21,6 @@ ReleasePair, ReportSource, TimingEstimate, - load_bundle, write_bundle, ) @@ -475,34 +472,6 @@ def _write_performance_bundle( return paths -def _retain_complete_plot_inputs(root: Path, paths: ArtifactPaths) -> None: - bundle = load_bundle(paths) - baseline = bundle.context.release.baseline - criterion = root / "target/criterion" - for row in bundle.rows: - for sample, name, timing in ( - (baseline, row.benchmark, row.baseline), - ("new", row.benchmark, row.current), - (baseline, "nalgebra_lu_solve", row.baseline_nalgebra), - (baseline, "faer_lu_solve", row.baseline_faer), - ): - assert timing is not None - directory = criterion / row.group / name / sample - directory.mkdir(parents=True) - estimate = { - "point_estimate": timing.median_ns, - "confidence_interval": {"confidence_level": 0.95, "lower_bound": timing.ci_lower_ns, "upper_bound": timing.ci_upper_ns}, - } - (directory / "benchmark.json").write_text(json.dumps({"full_id": f"{row.group}/{name}"}), encoding="utf-8") - (directory / "estimates.json").write_text(json.dumps({"mean": estimate, "median": estimate}), encoding="utf-8") - (directory / "sample.json").write_text(json.dumps({"iters": [1.0] * 100, "times": [timing.median_ns] * 100}), encoding="utf-8") - for path, payload in summary_outputs(criterion, baseline, paths).items(): - path.write_text(payload, encoding="utf-8", newline="\n") - for path, payload in retained_outputs(root / "docs/performance", paths).items(): - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(payload, encoding="utf-8", newline="\n") - - def test_main_update_readme_happy_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: _write_benchmark_checkout(tmp_path) performance_paths = _write_performance_bundle(tmp_path) @@ -571,33 +540,6 @@ def fake_render(request: criterion_dim_plot.PlotRequest) -> None: assert {path: (path.read_bytes(), path.stat().st_mtime_ns) for path in published} == first -def test_main_readme_replays_saved_measurements_after_artifact_commit_and_cleanup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - _write_benchmark_checkout(tmp_path) - performance_paths = _write_performance_bundle(tmp_path) - readme = tmp_path / "README.md" - readme.write_text(_canonical_benchmark_readme("0.0.8"), encoding="utf-8") - _mock_publication_environment(tmp_path, monkeypatch) - monkeypatch.setattr(criterion_dim_plot, "_render_svg_with_gnuplot", lambda request: request.out_svg.write_text("\n", encoding="utf-8")) - assert criterion_dim_plot.main(["--update-readme"]) == 0 - out_csv = tmp_path / "docs/assets/bench/vs_linalg_lu_solve_median.csv" - out_svg = out_csv.with_suffix(".svg") - first = {path: path.read_bytes() for path in (readme, out_csv, out_svg)} - provenance = json.loads(out_csv.with_suffix(".provenance.json").read_text(encoding="utf-8")) - _retain_complete_plot_inputs(tmp_path, performance_paths) - shutil.rmtree(tmp_path / "target") - # Committing only the saved results advances HEAD without changing the - # measured source, lockfile, release, or benchmark contract. - publication_commit = "b" * 40 - monkeypatch.setattr(criterion_dim_plot, "run_git_command", lambda *_args, **_kwargs: SimpleNamespace(stdout=f"{publication_commit}\n")) - assert criterion_dim_plot.main(["--update-readme"]) == 0 - assert {path: path.read_bytes() for path in first} == first - saved_provenance = json.loads(out_csv.with_suffix(".provenance.json").read_text(encoding="utf-8")) - assert saved_provenance["performance_artifact"]["csv"].startswith("docs/performance/v0.4.6-vs-v0.4.5/") - assert saved_provenance["performance_artifact"]["csv_sha256"] == provenance["performance_artifact"]["csv_sha256"] - assert saved_provenance["measurement"]["current_commit"] == _TEST_COMMIT - assert saved_provenance["publication"]["commit"] == publication_commit - - def test_dim_parsing_and_discovery(tmp_path: Path) -> None: assert criterion_dim_plot._dim_from_group_dir("d2") == 2 assert criterion_dim_plot._dim_from_group_dir("d10") == 10 diff --git a/scripts/tests/test_performance_artifacts.py b/scripts/tests/test_performance_artifacts.py index 5e511b8..25cb867 100644 --- a/scripts/tests/test_performance_artifacts.py +++ b/scripts/tests/test_performance_artifacts.py @@ -14,8 +14,6 @@ if TYPE_CHECKING: from pathlib import Path -from archive_performance import render_and_promote_artifacts -from benchmark_summaries import full_summary_paths, resolve_report_paths, retained_outputs, summary_outputs from performance_artifacts import ( ArtifactContext, ArtifactPaths, @@ -238,139 +236,6 @@ def _replace_csv_and_digest(csv_payload: bytes, provenance_payload: bytes) -> tu return csv_payload, (json.dumps(provenance, indent=2, sort_keys=True) + "\n").encode() -def _write_outputs(outputs: dict[Path, str]) -> None: - for path, payload in outputs.items(): - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(payload, encoding="utf-8", newline="\n") - - -def _complete_local_artifacts(root: Path, *, current_value: float = 8.0) -> ArtifactPaths: - bundle = _bundle(current_value=current_value) - paths = ArtifactPaths(csv=root / "target/bench-reports/performance.csv", provenance=root / "target/bench-reports/performance.provenance.json") - write_bundle(paths, bundle) - criterion = root / "raw" - for row in bundle.rows: - for sample, timing in ((bundle.context.release.baseline, row.baseline), ("new", row.current)): - if timing is None: - continue - directory = criterion / row.benchmark_id / sample - estimate = { - "point_estimate": timing.median_ns, - "confidence_interval": {"confidence_level": 0.95, "lower_bound": timing.ci_lower_ns, "upper_bound": timing.ci_upper_ns}, - } - _write_outputs( - { - directory / "benchmark.json": json.dumps({"full_id": row.benchmark_id}), - directory / "estimates.json": json.dumps({"mean": estimate, "median": estimate}), - directory / "sample.json": json.dumps({"iters": [1.0] * 100, "times": [timing.median_ns] * 100}), - } - ) - _write_outputs(summary_outputs(criterion, bundle.context.release.baseline, paths)) - return paths - - -def test_complete_local_snapshots_are_idempotent_and_preserve_previous_runs(tmp_path: Path) -> None: - report = _complete_local_artifacts(tmp_path) - directory = tmp_path / "docs/performance" - first = retained_outputs(directory, report) - _write_outputs(first) - assert retained_outputs(directory, report) == first - - _complete_local_artifacts(tmp_path, current_value=7.123456789012345) - second = retained_outputs(directory, report) - _write_outputs(second) - - assert len(list(directory.rglob("performance.full.csv"))) == 2 - for path, payload in first.items(): - if path.name != "latest.json": - assert path.read_text(encoding="utf-8") == payload - assert json.loads(first[directory / "latest.json"])["run"] != json.loads(second[directory / "latest.json"])["run"] - assert retained_outputs(directory, report) == second - - -@pytest.mark.parametrize("corruption", ["digest", "columns", "samples", "counts", "schema", "provenance", "missing"]) -def test_complete_local_snapshots_reject_corrupt_or_mismatched_inputs(tmp_path: Path, corruption: str) -> None: - report = _complete_local_artifacts(tmp_path) - full = full_summary_paths(report) - if corruption in {"digest", "columns", "samples"}: - old, new = {"digest": ("8.0", "8.01"), "columns": ("mean_ns", "unknown_ns"), "samples": (",100,", ",10,")}[corruption] - full.csv.write_text(full.csv.read_text(encoding="utf-8").replace(old, new), encoding="utf-8") - elif corruption == "missing": - full.provenance.unlink() - else: - metadata = json.loads(full.provenance.read_text(encoding="utf-8")) - if corruption == "counts": - metadata["measurements"]["counts"]["current"] += 1 - elif corruption == "schema": - metadata["measurements"]["schema_version"] = True - else: - metadata["benchmark_provenance"]["measurement"]["cpu"] = "Different CPU" - full.provenance.write_text(json.dumps(metadata), encoding="utf-8") - - with pytest.raises((ValueError, FileNotFoundError)): - retained_outputs(tmp_path / "docs/performance", report) - assert not (tmp_path / "docs/performance").exists() - - -@pytest.mark.parametrize( - "destination", - ["latest.json", "performance.csv", "performance.provenance.json", "performance.full.csv", "performance.full.provenance.json"], -) -def test_repeat_promotion_protects_unchanged_retained_destinations(tmp_path: Path, destination: str) -> None: - report = _complete_local_artifacts(tmp_path) - directory = tmp_path / "docs/performance" - output = tmp_path / "target/bench-reports/performance.md" - current = tmp_path / "docs/performance.md" - archive = tmp_path / "docs/archive/performance" - render_and_promote_artifacts(artifacts=report, output=output, current=current, archive_dir=archive) - retained = retained_outputs(directory, report) - protected = {path: path.read_bytes() for path in (*retained, output, current, archive / "README.md")} - alias = next(path for path in retained if path.name == destination) - - with pytest.raises(ValueError, match="distinct"): - render_and_promote_artifacts(artifacts=report, output=alias, current=current, archive_dir=archive) - - assert {path: path.read_bytes() for path in protected} == protected - - -@pytest.mark.parametrize("requested", ["partial-csv", "partial-json", "full-csv", "full-json", "full-pair", "custom-csv", "custom-json"]) -def test_saved_summary_fallback_does_not_hide_explicit_or_partial_inputs(tmp_path: Path, requested: str) -> None: - report = _complete_local_artifacts(tmp_path) - _write_outputs(retained_outputs(tmp_path / "docs/performance", report)) - if requested != "partial-csv": - report.csv.unlink() - if requested != "partial-json": - report.provenance.unlink() - full = full_summary_paths(report) - if requested not in {"full-csv", "full-pair"}: - full.csv.unlink() - if requested not in {"full-json", "full-pair"}: - full.provenance.unlink() - if requested == "custom-csv": - report = ArtifactPaths(csv=report.csv.with_name("custom.csv"), provenance=report.provenance) - elif requested == "custom-json": - report = ArtifactPaths(csv=report.csv, provenance=report.provenance.with_name("custom.json")) - - assert resolve_report_paths(tmp_path, report) == report - with pytest.raises(FileNotFoundError, match="incomplete"): - render_and_promote_artifacts( - artifacts=resolve_report_paths(tmp_path, report), - output=tmp_path / "target/bench-reports/performance.md", - current=tmp_path / "docs/performance.md", - archive_dir=tmp_path / "docs/archive/performance", - ) - assert not (tmp_path / "docs/performance.md").exists() - - -@pytest.mark.parametrize("run", ["../outside", "/outside", "v0.4.4-vs-v0.4.3/../../outside"]) -def test_saved_summary_pointer_cannot_escape_its_directory(tmp_path: Path, run: str) -> None: - report = ArtifactPaths(csv=tmp_path / "target/bench-reports/performance.csv", provenance=tmp_path / "target/bench-reports/performance.provenance.json") - _write_outputs({tmp_path / "docs/performance/latest.json": json.dumps({"schema_version": 1, "run": run})}) - - with pytest.raises(ValueError, match="escapes"): - resolve_report_paths(tmp_path, report) - - def test_artifact_round_trip_preserves_comparable_and_one_sided_rows() -> None: bundle = _bundle() diff --git a/scripts/tests/test_performance_workflow.py b/scripts/tests/test_performance_workflow.py new file mode 100644 index 0000000..b2cb0c3 --- /dev/null +++ b/scripts/tests/test_performance_workflow.py @@ -0,0 +1,284 @@ +"""Installed-package acceptance of la-stack's complete performance workflow.""" + +import hashlib +import io +import json +import shutil +import tarfile +from dataclasses import replace +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest +from research_repo_tools.common_measurement import CommonHarnessPlan, measure_prepared_pair +from research_repo_tools.complete_runs import RunSeries, run_identity, serialize_run +from research_repo_tools.process import run_git_command +from research_repo_tools.release_pairs import ReleasePair + +import archive_performance as workflow +import criterion_dim_plot as plot +import performance_runs as policy +from bench_compare import render_release_artifacts +from performance_artifacts import ArtifactPaths +from performance_phase import commands +from release_baseline import required_report_ids + +if TYPE_CHECKING: + from research_repo_tools.evidence import Evidence + +ROOT = Path(__file__).resolve().parents[2] +PAIR = ReleasePair("v0.4.6", "v0.4.5") +DRIVER = """import json, os, sys +from pathlib import Path +action, suite, phase = sys.argv[1:] +fault = os.environ.get("FIXTURE_FAULT", "") +if action == "gate": + if fault == "gate": + raise SystemExit(1) + if fault == "stale": + Path("src/lib.rs").write_text("changed during gate") +else: + ids = INVENTORIES[phase] + for name in ids[:-1] if fault == "missing-row" else ids: + path = Path("target/criterion") / name / "new" + path.mkdir(parents=True) + value = 10.0 if phase == "baseline" else 8.0 + estimate = {"point_estimate": value, "confidence_interval": {"lower_bound": value * .9, "upper_bound": value * 1.1, "confidence_level": .95}} + estimates = {"median": estimate} if fault == "missing-statistic" else {"mean": estimate, "median": estimate} + count = 99 if fault == "missing-sample" else 100 + sample = {"iters": [1] * count, "times": [value] * count} + for filename, data in (("benchmark.json", {"full_id": name}), ("estimates.json", estimates), ("sample.json", sample)): + (path / filename).write_text(json.dumps(data)) +""" + + +def prepared(tmp_path: Path, fault: str = "") -> tuple[Path, Path, CommonHarnessPlan]: + """Use the real consumer inventory and policies with a bounded timing fixture.""" + ids = tuple(sorted(required_report_ids())) + inventories = { + "baseline": tuple(name for name in ids if not name.startswith(policy.UNAVAILABLE_PREFIXES)), + "current": tuple(name for name in ids if not name.startswith("d") or "/la_stack_" in name), + } + for phase in ("baseline", "current"): + root = tmp_path / phase + root.mkdir(parents=True) + (root / "src").mkdir() + (root / "src/lib.rs").write_text(f"// {phase}\n", encoding="utf-8") + for pattern in policy.HARNESS: + for source in ROOT.glob(pattern): + destination = root / source.relative_to(ROOT) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + (root / "scripts/performance_phase.py").write_text(f"INVENTORIES = {inventories!r}\n" + DRIVER, encoding="utf-8") + run_git_command(["init", "--quiet"], cwd=root) + run_git_command( + [ + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.invalid", + "-c", + "commit.gpgsign=false", + "commit", + "--allow-empty", + "--quiet", + "-m", + phase, + ], + cwd=root, + ) + baseline, current = tmp_path / "baseline", tmp_path / "current" + plan = policy.declared_plan(PAIR, "all", "release-signal", inventories, baseline_env=(("FIXTURE_FAULT", fault),), current_env=()) + return baseline, current, plan + + +def measured(tmp_path: Path) -> tuple[Path, Evidence]: + baseline, current, plan = prepared(tmp_path) + evidence = measure_prepared_pair(baseline, current, plan, PAIR, working_tree=True) + (current / ".config").mkdir(exist_ok=True) + shutil.copyfile(ROOT / policy.REPORT_CONFIG, current / policy.REPORT_CONFIG) + return current, evidence + + +def test_common_harness_and_reference_phase(tmp_path: Path) -> None: + baseline, current, plan = prepared(tmp_path) + (baseline / "benches/exact.rs").write_text("obsolete harness\n", encoding="utf-8") + evidence = measure_prepared_pair(baseline, current, plan, PAIR, working_tree=True) + run = policy.validate_scientific_run(evidence) + sources = dict(evidence.sources) + assert sources["baseline"].harness_sha256 == sources["current"].harness_sha256 + assert sources["baseline"].source_sha256 != sources["current"].source_sha256 + assert (baseline / "benches/exact.rs").read_bytes() == (current / "benches/exact.rs").read_bytes() + assert {series.phase for series in run.series if series.name in {"nalgebra", "faer"}} == {"baseline"} + assert all(sample.policy.statistics == ("mean", "median") for _, sample in run.phases) + report = policy.render_scientific_report(evidence) + assert b"not rerun in the current phase" in report + assert b"baseline API unavailable" in report + assert b"paired confidence interval" in report + + +@pytest.mark.parametrize("fault", ["gate", "stale", "missing-row", "missing-statistic", "missing-sample"]) +def test_declared_gates_and_completeness_fail_closed(tmp_path: Path, fault: str) -> None: + baseline, current, plan = prepared(tmp_path, fault) + with pytest.raises(ValueError, match=r"gate|changed|incomplete|sample|statistic|expected|missing|mean"): + measure_prepared_pair(baseline, current, plan, PAIR, working_tree=True) + assert not (current / "target/criterion").exists() + if fault in {"gate", "stale"}: + assert not (baseline / "target/criterion").exists() + + +def test_shared_retention_repeated_runs_and_offline_report(tmp_path: Path) -> None: + root, first = measured(tmp_path) + stem = root / "target/bench-reports/performance" + workflow.save_scratch(root, stem, first) + workflow.promote(root, stem) + first_report = (root / "docs/performance.md").read_bytes() + retained = root / policy.ARCHIVE / "runs" / run_identity(first) + original = {path: path.read_bytes() for path in retained.iterdir()} + # A second valid measurement with the same release pair has new provenance. + sources = tuple((phase, replace(source, context=(*source.context, ("experiment", "second")))) for phase, source in first.sources) + second = replace(first, sources=sources) + workflow.save_scratch(root, stem, second) + workflow.promote(root, stem) + assert len(list((root / policy.ARCHIVE / "runs").iterdir())) == 2 + assert {path: path.read_bytes() for path in original} == original + assert (root / "docs/archive/performance/v0.4.6-vs-v0.4.5.md").read_bytes() == first_report + shutil.rmtree(root / "target") + assert run_identity(policy.retained_run(root, stem)) == run_identity(second) + before = (root / "docs/performance.md").read_bytes() + workflow.promote(root, stem) + assert (root / "docs/performance.md").read_bytes() == before + (root / policy.ARCHIVE / "latest.json").write_text("{}", encoding="utf-8") + with pytest.raises(ValueError, match=r"latest|pointer|fields|schema"): + policy.retained_run(root, stem) + + +def test_partial_scratch_and_stale_source_cannot_select_old_history(tmp_path: Path) -> None: + root, evidence = measured(tmp_path) + stem = root / "target/bench-reports/performance" + workflow.save_scratch(root, stem, evidence) + workflow.promote(root, stem) + policy.scratch_paths(stem)[1].unlink() + with pytest.raises(FileNotFoundError): + policy.retained_run(root, stem) + (root / "src/lib.rs").write_text("// stale\n", encoding="utf-8") + with pytest.raises(ValueError, match="stale"): + policy.verify_current_source(root, evidence) + + +def test_readme_uses_retained_reference_estimates_after_cleanup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + root, evidence = measured(tmp_path) + stem = root / "target/bench-reports/performance" + workflow.save_scratch(root, stem, evidence) + workflow.promote(root, stem) + shutil.copyfile(ROOT / "README.md", root / "README.md") + shutil.rmtree(root / "target") + monkeypatch.setattr(plot, "_repo_root", lambda: root) + monkeypatch.setattr(plot, "_render_svg_with_gnuplot", lambda request: request.out_svg.write_bytes(b"\n")) + assert plot.main(["--update-readme", "--log-y"]) == 0 + assets = root / "docs/assets/bench" + csv = assets / "vs_linalg_lu_solve_median.csv" + assert "2,8.0,7.2,8.8,10.0,9.0,11.0,10.0,9.0,11.0" in csv.read_text(encoding="utf-8") + provenance = json.loads(csv.with_suffix(".provenance.json").read_bytes()) + assert provenance["measurement"]["peer_phase"] == "baseline" + assert provenance["run_id"] == run_identity(evidence) + before = {path: path.read_bytes() for path in (*assets.iterdir(), root / "README.md")} + assert plot.main(["--update-readme", "--log-y"]) == 0 + assert {path: path.read_bytes() for path in before} == before + + +def test_legacy_bytes_hashes_and_report_are_unchanged() -> None: + directory = ROOT / "docs/performance/v0.4.6-vs-v0.4.5/39719a1eb599d294dbbc1e46dc6fb0de2028ad2e59aab5f94fb48c7bf3d7f841" + paths = ArtifactPaths(directory / "performance.csv", directory / "performance.provenance.json") + hashes = { + "performance.csv": "b263bb34bcfd43b8fc8109110db9f2e908516aef6c01ce16f70f5ac18dc7ef2d", + "performance.full.csv": "4fa5cc13159f5dc91e711ae67554845bcbd1ace2cfc3e31487fc85f75d3dc459", + "performance.full.provenance.json": "a43f66e52035aecd8d235dfded916b23c3240529d0c07f793d27322521cfb0d0", + "performance.provenance.json": "89144a2899ba73f55d009da4fa20b4e6477600902b93cde469383f719c497508", + } + assert {name: hashlib.sha256((paths.csv.parent / name).read_bytes()).hexdigest() for name in hashes} == hashes + assert hashlib.sha256(render_release_artifacts(paths).encode()).hexdigest() == "4ea0ff9bca4c7bb92cd57bd9163b3cacabe78f502abdd2663162cf9562ec6582" + + +def test_native_command_policy_and_compatibility_adapter() -> None: + assert all("--locked" in command and "--noplot" in command for command in commands("all", "baseline")) + assert "la_stack" in commands("vs_linalg", "current")[0] + assert "la_stack" not in commands("vs_linalg", "baseline")[0] + environment = dict(policy.phase_environment(ROOT, "la_stack_pre_rational_input_api")) + assert "--cfg=la_stack_pre_rational_input_api" in environment.get("CARGO_ENCODED_RUSTFLAGS", environment.get("RUSTFLAGS", "")) + assert environment["CARGO_TARGET_DIR"] == "target" + assert environment["CRITERION_HOME"] == "target/criterion" + + +def test_reference_phase_and_same_version_policy(tmp_path: Path) -> None: + _, evidence = measured(tmp_path) + run = policy.validate_scientific_run(evidence) + # Same semantic IDs exist in both phases, so only consumer policy detects + # an otherwise well-formed la-stack series assigned to the wrong phase. + series = tuple(RunSeries(item.name, "current", item.rows) if item.name == "la-stack baseline" else item for item in run.series) + with pytest.raises(ValueError, match="retained series"): + policy.validate_scientific_run(replace(evidence, payload=serialize_run(replace(run, series=series)))) + # Use a current/current release pair with both sources under the same API. + baseline, current, plan = prepared(tmp_path / "same") + pair = ReleasePair("v0.4.6", "v0.4.6") + inventories = {phase: tuple(case.full_id for case in sample.cases) for phase, sample in run.phases} + # The older-API fixture deliberately lacks rows; removing them from the + # current selection allows a narrow all-benches same-version comparison. + inventories["current"] = tuple(name for name in inventories["current"] if name in inventories["baseline"]) + driver = f"INVENTORIES = {inventories!r}\n" + DRIVER + (current / "scripts/performance_phase.py").write_text(driver, encoding="utf-8") + plan = policy.declared_plan(pair, "all", "all-benches", inventories, baseline_env=(), current_env=()) + same = measure_prepared_pair(baseline, current, plan, pair, working_tree=True) + policy.validate_scientific_run(same) + with pytest.raises(ValueError, match="same-version"): + policy.validate_scientific_run(same, release=True) + sources = dict(same.sources) + identical = replace( + same, sources=(("baseline", replace(sources["baseline"], source_sha256=sources["current"].source_sha256)), ("current", sources["current"])) + ) + with pytest.raises(ValueError, match="distinct source"): + policy.validate_scientific_run(identical) + + +def test_publication_io_failure_preserves_all_published_outputs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + root, evidence = measured(tmp_path) + stem = root / "target/bench-reports/performance" + workflow.save_scratch(root, stem, evidence) + workflow.promote(root, stem) + originals = {path: path.read_bytes() for path in (root / "docs").rglob("*") if path.is_file()} + second = replace( + evidence, sources=tuple((phase, replace(source, context=(*source.context, ("experiment", "second")))) for phase, source in evidence.sources) + ) + workflow.save_scratch(root, stem, second) + native_replace = Path.replace + failed = False + + def fail_once(source: Path, destination: Path) -> Path: + nonlocal failed + if destination == root / policy.ARCHIVE / "latest.json" and not failed: + failed = True + msg = "injected publication failure" + raise OSError(msg) + return native_replace(source, destination) + + monkeypatch.setattr(Path, "replace", fail_once) + with pytest.raises(OSError, match="injected publication failure"): + workflow.promote(root, stem) + assert failed + assert {path: path.read_bytes() for path in (root / "docs").rglob("*") if path.is_file()} == originals + + +def test_unsafe_release_asset_cannot_replace_report(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + output = tmp_path / "report.md" + output.write_bytes(b"previous report\n") + + def unsafe_asset(_root: Path, _repo: str, _tag: str, _name: str, destination: Path) -> None: + with tarfile.open(destination, "w:gz") as archive: + member = tarfile.TarInfo("../../report.md") + member.size = 4 + archive.addfile(member, io.BytesIO(b"evil")) + + monkeypatch.setattr(workflow, "download_release_asset", unsafe_asset) + with pytest.raises(ValueError, match=r"unsafe|traversal|relative|member"): + workflow.compare_assets(tmp_path, PAIR, output, "all", "release-signal") + assert output.read_bytes() == b"previous report\n" diff --git a/scripts/tests/test_release_baseline.py b/scripts/tests/test_release_baseline.py index 999a860..a5fe6c3 100644 --- a/scripts/tests/test_release_baseline.py +++ b/scripts/tests/test_release_baseline.py @@ -13,7 +13,6 @@ import pytest import release_baseline -from criterion_measurements import validate_measurement REPO_ROOT = Path(__file__).resolve().parents[2] WORKFLOW = REPO_ROOT / ".github/workflows/release-benchmarks.yml" @@ -104,8 +103,8 @@ def test_missing_raw_file_blocks_publication(dataset: tuple[Path, Path], sample: criterion, manifest = dataset missing = criterion / "diagnostic/peer" / sample / filename missing.unlink() - if sample == "new" and filename == "benchmark.json": - with pytest.raises(ValueError, match="incomplete Criterion dataset: diagnostic/peer"): + if sample == "new" and filename != "tukey.json": + with pytest.raises(ValueError, match="incomplete Criterion sample"): release_baseline.validate(criterion, manifest, BASELINE) else: with pytest.raises(FileNotFoundError) as error: @@ -116,7 +115,7 @@ def test_missing_raw_file_blocks_publication(dataset: tuple[Path, Path], sample: def test_missing_entire_suite_blocks_publication(dataset: tuple[Path, Path]) -> None: criterion, manifest = dataset shutil.rmtree(criterion / "exact_d2") - with pytest.raises(ValueError, match="incomplete Criterion dataset: exact_d2/det_exact"): + with pytest.raises(ValueError, match=r"inventory differs: missing=.*exact_d2/det_exact"): release_baseline.validate(criterion, manifest, BASELINE) @@ -134,51 +133,6 @@ def test_report_consumers_require_canonical_paths(dataset: tuple[Path, Path]) -> release_baseline.validate(criterion, manifest, BASELINE) -@pytest.mark.parametrize("corruption", ["short-samples", "nonfinite", "reversed-interval", "wrong-confidence", "malformed"]) -def test_invalid_measurement_blocks_publication(dataset: tuple[Path, Path], corruption: str) -> None: - criterion, manifest = dataset - for sample in ("new", BASELINE): - directory = criterion / "d2/la_stack_dot" / sample - if corruption == "malformed": - (directory / "estimates.json").write_text("{broken", encoding="utf-8", newline="\n") - elif corruption in {"short-samples", "nonfinite"}: - times = [2.0] * 99 if corruption == "short-samples" else [float("inf")] * 100 - write_json(directory / "sample.json", {"iters": [1.0] * 100, "times": times}) - else: - estimates = json.loads((directory / "estimates.json").read_text(encoding="utf-8")) - interval = estimates["median"]["confidence_interval"] - interval["lower_bound" if corruption == "reversed-interval" else "confidence_level"] = 4.0 - write_json(directory / "estimates.json", estimates) - errors = { - "short-samples": "times must contain 100 samples", - "nonfinite": "finite positive number", - "reversed-interval": "lower bound exceeds upper bound", - "wrong-confidence": "confidence_level requires an interval", - "malformed": "Expecting property name", - } - with pytest.raises(ValueError, match=errors[corruption]): - release_baseline.validate(criterion, manifest, BASELINE) - - -@pytest.mark.parametrize(("filename", "location"), [("sample.json", "times[9]"), ("estimates.json", "mean")]) -def test_measurement_error_identifies_file_and_field(dataset: tuple[Path, Path], filename: str, location: str) -> None: - criterion, _manifest = dataset - directory = criterion / "d2/la_stack_dot/new" - path = directory / filename - data = json.loads(path.read_text(encoding="utf-8")) - if filename == "sample.json": - data["times"][9] = -1.0 - else: - data["mean"]["point_estimate"] = -1.0 - write_json(path, data) - - with pytest.raises(ValueError, match="finite positive number") as error: - validate_measurement(directory) - - assert str(path) in str(error.value) - assert location in str(error.value) - - @pytest.mark.parametrize("baseline", ["../escape", "", "new", "base", "change", "report", "/absolute"]) def test_invalid_baseline_is_rejected(dataset: tuple[Path, Path], baseline: str) -> None: with pytest.raises(ValueError, match="safe, nonreserved"): diff --git a/scripts/tests/test_toolchain_integration.py b/scripts/tests/test_toolchain_integration.py index ebb2ee1..1cd8f9b 100644 --- a/scripts/tests/test_toolchain_integration.py +++ b/scripts/tests/test_toolchain_integration.py @@ -171,7 +171,7 @@ def test_python_update_preserves_shared_pin_and_tools_with_explicit_dev_sync(tmp run_safe_command("uv", ["lock", "--upgrade"], cwd=root) lock = tomllib.loads((root / "uv.lock").read_text(encoding="utf-8")) packages = {package["name"]: package["version"] for package in lock["package"]} - assert packages["research-repo-tools"] == "0.1.7" + assert packages["research-repo-tools"] == "0.1.8" assert packages["ruff"] == "99.0.0" run_safe_command("uv", ["sync", "--locked", "--group", "dev"], cwd=root) installed = run_safe_command( diff --git a/uv.lock b/uv.lock index c6d1c36..24f9901 100644 --- a/uv.lock +++ b/uv.lock @@ -493,13 +493,13 @@ tooling = [ ] [package.metadata] -requires-dist = [{ name = "research-repo-tools", specifier = "==0.1.7" }] +requires-dist = [{ name = "research-repo-tools", specifier = "==0.1.8" }] [package.metadata.requires-dev] dev = [ { name = "actionlint-py", specifier = "==1.7.12.25" }, { name = "pytest", specifier = "==9.1.1" }, - { name = "research-repo-tools", specifier = "==0.1.7" }, + { name = "research-repo-tools", specifier = "==0.1.8" }, { name = "ruff", specifier = "==0.16.10" }, { name = "semgrep", specifier = "==1.180.0" }, { name = "shellcheck-py", specifier = "==0.11.0.1" }, @@ -507,7 +507,7 @@ dev = [ { name = "ty", specifier = "==0.0.85" }, { name = "yamllint", specifier = "==1.38.0" }, ] -tooling = [{ name = "research-repo-tools", specifier = "==0.1.7" }] +tooling = [{ name = "research-repo-tools", specifier = "==0.1.8" }] [[package]] name = "markdown-it-py" @@ -974,16 +974,16 @@ wheels = [ [[package]] name = "research-repo-tools" -version = "0.1.7" +version = "0.1.8" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "packaging" }, { name = "pyyaml" }, { name = "rust-just" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/28/bf/ee8d41dab3fe80da99147b7caebae7eb72a4001606fef588273bd369312b/research_repo_tools-0.1.7.tar.gz", hash = "sha256:2a8aa607a6ffccf2c5e2a48565e1eceb4ab3f361ebbf2136d6fa5a197ce68a19", size = 540024, upload-time = "2026-09-26T20:01:09.301Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/fd/f804affa973b85f455b20d5f129527fc2176f78bbc05ab710906f4242034/research_repo_tools-0.1.8.tar.gz", hash = "sha256:bc855aa930ee04301dceebc8340a22576bb25712e322b51d7afaf20f5fab1ba0", size = 765360, upload-time = "2026-10-09T04:38:42.035Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/91/e7/58faeb7238ea1276471670ca5e746834da07b67e201eb9ef4c8a5a24f8d3/research_repo_tools-0.1.7-py3-none-any.whl", hash = "sha256:ff60397c563d5a7a89d73ddba219e741dd3f4936bfa001ea498c4607aea05cb6", size = 232872, upload-time = "2026-09-26T20:01:07.556Z" }, + { url = "https://files.pythonhosted.org/packages/4f/2c/f37ffb902034866dd03c3f939c580f2d151e51955669c7d9bc4d00cf4858/research_repo_tools-0.1.8-py3-none-any.whl", hash = "sha256:9db45f09a33cac4ff32ddf010d30523e295f607e24c680bcfc3d3d9786991293", size = 339758, upload-time = "2026-10-09T04:38:39.806Z" }, ] [[package]] From 25cd362ea19e47e711d3e96c4997f64dc894bc8c Mon Sep 17 00:00:00 2001 From: Adam Date: Fri, 9 Oct 2026 05:48:12 +0000 Subject: [PATCH 2/6] fix: preserve explicit same-version local comparisons Keep the consumer-only local comparison exception while release promotion still requires distinct labels and sources remain independently identified. Cover explicit selection and current-phase gate failure before either timing phase. --- scripts/archive_performance.py | 11 ++++++++--- scripts/tests/test_performance_workflow.py | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/scripts/archive_performance.py b/scripts/archive_performance.py index 2525c68..03adb1a 100644 --- a/scripts/archive_performance.py +++ b/scripts/archive_performance.py @@ -49,7 +49,7 @@ class ReportId: baseline_tag: str -def select_pair(root: Path, mode: PairMode, current: str | None, baseline: str | None) -> ReleasePair: +def select_pair(root: Path, mode: PairMode, current: str | None, baseline: str | None, *, local: bool = False) -> ReleasePair: """Use publication chronology while retaining la-stack release eligibility.""" package = tomllib.loads((root / "Cargo.toml").read_text(encoding="utf-8"))["package"]["version"] releases = () @@ -60,7 +60,12 @@ def select_pair(root: Path, mode: PairMode, current: str | None, baseline: str | cwd=root, ) releases = stable_published_releases(json.loads(response.stdout)) - pair = resolve_pair(mode, package_tag=package, releases=releases, current=current, baseline=baseline, order="published") + # Shared explicit selection describes release pairs. Consumer local audits + # additionally permit two different sources carrying the same package label. + if local and current is not None and baseline is not None and normalize_tag(current) == normalize_tag(baseline): + pair = ReleasePair(current, baseline) + else: + pair = resolve_pair(mode, package_tag=package, releases=releases, current=current, baseline=baseline, order="published") resolve_shared_harness_compatibility(current=pair.current, baseline=pair.baseline, shared_harness_rational_inputs=True) return pair @@ -223,7 +228,7 @@ def main(argv: list[str] | None = None) -> int: return 0 modes: dict[str, PairMode] = {"local": "current-vs-latest", "release": "infer-release", "assets": "published-latest"} mode: PairMode = "explicit" if args.current or args.baseline else modes[args.command] - pair = select_pair(root, mode, args.current or None, args.baseline or None) + pair = select_pair(root, mode, args.current or None, args.baseline or None, local=args.command == "local") if args.command == "assets": compare_assets(root, pair, root / "target/bench-reports/github-assets-performance.md", args.suite, args.scope) else: diff --git a/scripts/tests/test_performance_workflow.py b/scripts/tests/test_performance_workflow.py index b2cb0c3..24f246e 100644 --- a/scripts/tests/test_performance_workflow.py +++ b/scripts/tests/test_performance_workflow.py @@ -210,6 +210,23 @@ def test_native_command_policy_and_compatibility_adapter() -> None: assert environment["CRITERION_HOME"] == "target/criterion" +def test_local_explicit_labels_and_release_eligibility() -> None: + assert workflow.select_pair(ROOT, "explicit", "v0.4.6", "v0.4.6", local=True) == ReleasePair("v0.4.6", "v0.4.6") + with pytest.raises(ValueError, match="must differ"): + workflow.select_pair(ROOT, "explicit", "v0.4.6", "v0.4.6") + with pytest.raises(ValueError, match=r"v0.4.4|unsupported"): + workflow.select_pair(ROOT, "explicit", "v0.4.6", "v0.4.3", local=True) + + +def test_current_gate_failure_prevents_both_timing_phases(tmp_path: Path) -> None: + baseline, current, plan = prepared(tmp_path) + plan = replace(plan, current=replace(plan.current, environment=(("FIXTURE_FAULT", "gate"),))) + with pytest.raises(ValueError, match="current preflight gate failed"): + measure_prepared_pair(baseline, current, plan, PAIR, working_tree=True) + assert not (baseline / "target/criterion").exists() + assert not (current / "target/criterion").exists() + + def test_reference_phase_and_same_version_policy(tmp_path: Path) -> None: _, evidence = measured(tmp_path) run = policy.validate_scientific_run(evidence) From deaecbc81ba0acb575f73b4b93a0bdcdb3aba44b Mon Sep 17 00:00:00 2001 From: Adam Date: Fri, 9 Oct 2026 05:55:51 +0000 Subject: [PATCH 3/6] fix: retain report test collection and release guidance Restore the peer-context test prefix and describe complete-run README inputs, staged retention files, and the supported changelog dependency-body policy. --- docs/RELEASING.md | 11 +++++++---- scripts/tests/test_bench_compare.py | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 63d4dd9..b459226 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -107,7 +107,8 @@ Use `just changelog-preview --tag "$TAG" --date "$RELEASE_DATE"` for a read-only preview, and `just changelog-check` to validate the root and every archive. Review the generated changelog and any archive changes. The [Scripts guide](../scripts/README.md#changelog-and-release-tooling) explains -the consumer policy exception that preserves historical dependency-note links. +the supported `dependency-bodies = "preserve"` policy in `pyproject.toml`, which +preserves authored dependency-note links, text, and code without a local template. ### 4. Generate the release performance comparison @@ -135,12 +136,13 @@ scratch until explicitly promoted. just performance-readme ``` -This consumes the validated CSV and provenance JSON retained by +This consumes the complete-run payload and evidence envelope retained by `just performance-release`; it does not run benchmarks again. It atomically updates the table and tag-pinned benchmark links in `README.md` with the CSV, SVG, and provenance JSON under `docs/assets/bench/`. Until it succeeds, those README links continue to reference the previous published artifacts. -If scratch inputs were cleaned, it uses the committed local snapshot. Existing +If scratch inputs were cleaned, it uses the validated +`docs/performance-runs/latest.json` run. Existing partial or corrupt scratch inputs remain errors and never trigger a fallback. See `docs/BENCHMARKING.md` for repair commands, local comparison modes, artifact @@ -167,7 +169,8 @@ git --no-pager diff Expected release artifacts include package metadata and lockfiles, `CITATION.cff`, `CHANGELOG.md`, `README.md`, `docs/performance.md`, and generated -files under `docs/archive/`, `docs/archives/changelog/`, and `docs/assets/bench/`. Stage only the reviewed +files under `docs/archive/`, `docs/archives/changelog/`, `docs/assets/bench/`, and +`docs/performance-runs/`. Stage only the reviewed paths that were intentionally changed; do not stage the entire `docs/` tree. Then inspect the staged diff and commit it: diff --git a/scripts/tests/test_bench_compare.py b/scripts/tests/test_bench_compare.py index 4411bf3..10f4df3 100644 --- a/scripts/tests/test_bench_compare.py +++ b/scripts/tests/test_bench_compare.py @@ -631,7 +631,7 @@ def test_comparison_uses_one_table_per_suite_with_case_column(tmp_path: Path) -> assert "| solve_exact_f64_result |" in tables -def testcomparison_tables_include_vs_linalg_peer_context(tmp_path: Path) -> None: +def test_comparison_tables_include_vs_linalg_peer_context(tmp_path: Path) -> None: _build_vs_linalg_tree(tmp_path) comparisons = bench_compare._collect_comparisons(tmp_path, "last", "median", suite="vs_linalg").comparisons tables = bench_compare.comparison_tables(comparisons, "last") From 6decbf8e30c5e151738ea148caa8d10797a3a285 Mon Sep 17 00:00:00 2001 From: Adam Getchell Date: Fri, 9 Oct 2026 07:58:52 -0700 Subject: [PATCH 4/6] refactor: replace Python adapters with shared tooling - Move benchmark commands, measurement inputs, provenance settings, and report paths into tooling configuration. - Use shared process, Criterion, and Just inspection APIs and remove duplicate local adapters and generic tests. - Preserve scientific report contracts, historical evidence replay, and configurable retained-history selection. - Refresh the uv pin and locked tooling dependencies with just update. --- docs/BENCHMARKING.md | 5 +- docs/code_organization.md | 19 ++- justfile | 1 + pyproject.toml | 6 +- scripts/README.md | 37 +++-- scripts/archive_performance.py | 14 +- scripts/bench_compare.py | 137 ++++++------------ scripts/benchmark_contract.py | 17 ++- scripts/benchmark_process.py | 68 --------- scripts/criterion_dim_plot.py | 57 ++------ scripts/performance_phase.py | 52 ------- scripts/performance_runs.py | 135 ++++++++++------- scripts/release_baseline.py | 9 +- scripts/tests/test_bench_compare.py | 34 +---- .../tests/test_cargo_update_integration.py | 5 +- scripts/tests/test_changelog_integration.py | 21 ++- scripts/tests/test_criterion_dim_plot.py | 93 ++---------- .../tests/test_justfile_discoverability.py | 32 ++-- scripts/tests/test_maintenance_integration.py | 25 ++-- scripts/tests/test_performance_artifacts.py | 17 --- scripts/tests/test_performance_workflow.py | 132 ++++++++++++++--- scripts/tests/test_release_baseline.py | 8 +- scripts/tests/test_review_integration.py | 70 ++------- scripts/tests/test_security_integration.py | 5 +- scripts/tests/test_toolchain_integration.py | 19 +-- semgrep.yaml | 4 +- tests/semgrep/scripts/python_portability.py | 11 +- .../scripts/tests/python_exceptions.py | 4 - {.config => tooling}/performance-report.toml | 0 tooling/performance.just | 20 +++ tooling/performance.toml | 41 ++++++ uv.lock | 6 +- 32 files changed, 463 insertions(+), 641 deletions(-) delete mode 100644 scripts/benchmark_process.py delete mode 100644 scripts/performance_phase.py rename {.config => tooling}/performance-report.toml (100%) create mode 100644 tooling/performance.just create mode 100644 tooling/performance.toml diff --git a/docs/BENCHMARKING.md b/docs/BENCHMARKING.md index 4319261..c0018eb 100644 --- a/docs/BENCHMARKING.md +++ b/docs/BENCHMARKING.md @@ -152,7 +152,10 @@ just performance-local This creates isolated temporary worktrees and runs both library revisions on the same machine with the current checkout's benchmark sources, manifests, lockfile, -benchmark-input tests, phase driver, and Rust toolchain. Current example sources are +benchmark-input tests, native Just recipes, measurement configuration, and Rust +toolchain. The declarations live in `tooling/performance.just` and +`tooling/performance.toml`; `tooling/performance-report.toml` owns shared report +and retention paths. Current example sources are also copied so Cargo can resolve every target declared by the shared manifest; the comparison does not build or time those examples. Staged and unstaged changes to tracked files are applied to the current worktree. Untracked files are diff --git a/docs/code_organization.md b/docs/code_organization.md index 6d3bfd0..2703b17 100644 --- a/docs/code_organization.md +++ b/docs/code_organization.md @@ -146,11 +146,11 @@ actual release selectors and preservation of scientific evidence. upgrades and coupled exclusions against a disposable local registry. The shared package also owns process discovery, captured/live execution, exact -byte transport, CPU metadata, diagnostics, and zizmor authentication. The thin -`scripts/benchmark_process.py` adapter retains benchmark phase signatures and -consumer root selection. Generic `subprocess_utils.py` and `run_zizmor.sh` -implementations and their duplicate tests are retired; scientific schemas and -benchmark policy remain here. +byte transport, CPU metadata, diagnostics, and zizmor authentication. Consumers +call its public process API directly; `scripts/benchmark_contract.py` retains +Cargo checkout selection and historical digest framing. Shared Criterion +estimates replace local timing wrappers. Tests use shared Just inspection and +retain scientific schemas and consumer policy checks. Performance consumers use shared Criterion parsing and estimate/comparison validation, digest verification, archive extraction, byte-preserving document @@ -171,11 +171,14 @@ raw Criterion validation to the shared policy. The release workflow packages onl that gate; its regression and archive tests live in `scripts/tests/test_release_baseline.py`. -`scripts/performance_phase.py` declares native Cargo commands, features, and -input gates. `scripts/performance_runs.py` declares scientific coverage, +`tooling/performance.just` declares native Cargo commands, features, and input +gates for both local measurements and release inventories. +`tooling/performance.toml` uses the shared measurement schema for source/harness +inputs, probes, timeouts, and compatibility fields. +`scripts/performance_runs.py` declares scientific coverage, release compatibility, and named reference phases, and renders the local tables. `scripts/archive_performance.py` composes the shared APIs for consumer commands. -`.config/performance-report.toml` owns the new retention paths. +`tooling/performance-report.toml` owns the new retention paths. `scripts/benchmark_summaries.py` only reads historical CSV/JSON snapshots; `scripts/criterion_measurements.py` and the old generic measurement/retention engines are removed. `scripts/tests/test_performance_workflow.py` verifies diff --git a/justfile b/justfile index 9d88706..d39ddbb 100644 --- a/justfile +++ b/justfile @@ -308,6 +308,7 @@ github-actions-check: action-lint zizmor # Keep the command-memory layer itself canonically formatted. justfile-fmt-check: just --fmt --check + just --justfile tooling/performance.just --fmt --check # Check code, documentation, and configuration without changing sources. lint: lint-code lint-docs lint-config diff --git a/pyproject.toml b/pyproject.toml index 8a7d049..61e218a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ criterion-dim-plot = "criterion_dim_plot:main" # Configure setuptools to find modules in scripts/ directory. [tool.setuptools] package-dir = { "" = "scripts" } -py-modules = [ "archive_performance", "bench_compare", "benchmark_contract", "benchmark_process", "benchmark_summaries", "criterion_dim_plot", "performance_artifacts", "performance_phase", "performance_runs", "release_baseline" ] +py-modules = [ "archive_performance", "bench_compare", "benchmark_contract", "benchmark_summaries", "criterion_dim_plot", "performance_artifacts", "performance_runs", "release_baseline" ] [tool.ruff] line-length = 160 @@ -88,11 +88,9 @@ known-first-party = [ "archive_performance", "bench_compare", "benchmark_contract", - "benchmark_process", "benchmark_summaries", "criterion_dim_plot", "performance_artifacts", - "performance_phase", "performance_runs", "release_baseline", ] @@ -140,7 +138,7 @@ line-length = 160 [tool.uv] package = true -required-version = "==0.12.23" +required-version = "==0.12.24" # Semgrep wheels bundle the native engine; building its sdist does not. no-build-package = [ "semgrep" ] # Semgrep 1.178.0 restricts PyJWT to 2.13.x; 2.15.1 contains the OSV fixes. diff --git a/scripts/README.md b/scripts/README.md index 45a7582..4499fa2 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -78,9 +78,8 @@ just performance-doc just performance-github-assets ``` -Local benchmark generation streams Cargo and Criterion progress while retaining -the existing fail-closed report and provenance checks. Lines prefixed with -`[performance]` identify the active validation or timing phase. Staged and +Local benchmark generation streams native Just, Cargo, and Criterion progress +while retaining the existing fail-closed report and provenance checks. Staged and unstaged changes to tracked files participate. Untracked files are excluded; stage a new file before running the command if it must participate. A local current-vs-latest report may use the same package and release identifier because @@ -361,8 +360,8 @@ Notebook tooling remains outside scope. The same pinned release owns opt-in CodeRabbit review orchestration through `research-repo-tools review branch --base=REF` and `review uncommitted`. Thin Just wrappers retain the common implementation upstream; consumer checks -in `tests/test_review_integration.py` exercise recipe forwarding, instruction -discovery, freshness diagnostics, and failure propagation with local stubs. +in `tests/test_review_integration.py` exercise recipe forwarding, argument +quoting, and failure propagation with local stubs. CodeRabbit remains externally installed and authenticated. See the [contributor review workflow](../CONTRIBUTING.md#coderabbit-review) for scopes and invocation policy. @@ -390,20 +389,34 @@ preview an annotation without creating a tag. |---|---| | `archive_performance.py` | Select eligible releases and compose shared measurement/publication APIs | | `bench_compare.py` | Compare Criterion benchmark baselines and render Markdown reports | -| `benchmark_contract.py` | Define and hash the consumer's benchmark inventory contract | -| `benchmark_process.py` | Thin benchmark phase adapters over the published shared process API, plus consumer root selection | +| `benchmark_contract.py` | Select the Cargo checkout and hash the historical benchmark inventory contract | | `benchmark_summaries.py` | Read historical complete CSV/JSON summaries without changing framing | | `criterion_dim_plot.py` | Plot Criterion benchmark results (CSV + SVG + README table) | -| `performance_phase.py` | Declare native Cargo inventory, gate, and timing commands | | `performance_runs.py` | Declare scientific run policy and adapt retained series to local tables | | `performance_artifacts.py` | Validate and publish schema-versioned performance-comparison CSV/JSON inputs | | `release_baseline.py` | Inventory full Criterion suites and validate complete raw release baselines before packaging | Shared process discovery, execution, byte transport, CPU detection, diagnostics, -and zizmor authentication belong to research-repo-tools. The former -`subprocess_utils.py` and `run_zizmor.sh` implementations and their duplicated -unit tests are removed. Consumer tests retain native adapter checks, benchmark -contracts, caller file-policy coverage, and recipe forwarding. +and zizmor authentication belong to research-repo-tools. Callers use its public +process API directly. Shared Criterion estimates and comparisons also replace +local timing wrappers and numerical validators. Consumer tests use the shared +Just inspection API and retain benchmark contracts, caller file-policy coverage, +and recipe forwarding; common parser and review regressions belong upstream. + +The `tooling/` directory owns declarative performance configuration: + +- `performance.toml`: shared measurement inputs, harness files, tool/dependency + probes, timeout, and provenance compatibility. +- `performance.just`: native Cargo inventory, independent gates, and timing + recipes, including baseline-only peer measurements. +- `performance-report.toml`: shared report and immutable-history paths. + +Both local measurement and release inventory use the native recipes. The shared +runner streams them in the measured checkout, with the recipes and measurement +configuration included in the harness fingerprint. Scientific row selection, +sampling requirements, historical API adapters, and figure layout remain Python. +Old evidence naming the retired Python phase driver can still be validated and +rendered offline; its recorded commands are never executed. The superseded `criterion_measurements.py`, archive/worktree measurement engine, summary retention writer, and their duplicate generic tests are removed. diff --git a/scripts/archive_performance.py b/scripts/archive_performance.py index 03adb1a..c73d714 100644 --- a/scripts/archive_performance.py +++ b/scripts/archive_performance.py @@ -18,7 +18,7 @@ from research_repo_tools.evidence import Evidence, serialize_evidence from research_repo_tools.files import replace_many from research_repo_tools.measurement import resolve_revision -from research_repo_tools.process import ExecutableNotFoundError, format_exception_diagnostics, run_git_command, run_safe_command +from research_repo_tools.process import ExecutableNotFoundError, format_exception_diagnostics, run_command from research_repo_tools.publication import plan_outputs, publish_publication from research_repo_tools.release_assets import download_release_asset from research_repo_tools.release_discovery import normalize_tag, stable_published_releases @@ -27,12 +27,12 @@ from research_repo_tools.worktrees import apply_snapshot, capture_snapshot, temporary_worktree from bench_compare import render_historical_assets, render_release_artifacts -from benchmark_process import find_project_root +from benchmark_contract import find_project_root from benchmark_summaries import resolve_report_paths from performance_artifacts import ArtifactPaths, load_bundle, resolve_shared_harness_compatibility from performance_runs import ( - ARCHIVE, REPORT_CONFIG, + archive_directory, measurement_plan, render_scientific_report, retained_run, @@ -54,7 +54,7 @@ def select_pair(root: Path, mode: PairMode, current: str | None, baseline: str | package = tomllib.loads((root / "Cargo.toml").read_text(encoding="utf-8"))["package"]["version"] releases = () if mode != "explicit": - response = run_safe_command( + response = run_command( "gh", ["release", "list", "--repo", "acgetchell/la-stack", "--limit", "1000", "--json", "tagName,isDraft,isPrerelease,publishedAt"], cwd=root, @@ -78,7 +78,7 @@ def measure(root: Path, pair: ReleasePair, suite: str, scope: str) -> Evidence: raise ValueError(msg) snapshot = replace(capture_snapshot(root), untracked=()) # The existing consumer contract deliberately excludes untracked files. - run_git_command(["fetch", "origin", f"refs/tags/{pair.baseline}:refs/tags/{pair.baseline}"], cwd=root) + run_command("git", ["--no-pager", "fetch", "origin", f"refs/tags/{pair.baseline}:refs/tags/{pair.baseline}"], cwd=root) baseline_revision = resolve_revision(root, pair.baseline) parent = Path(tempfile.mkdtemp(prefix="la-stack-performance-")) try: @@ -126,7 +126,7 @@ def promote(root: Path, stem: Path) -> None: kwargs = {"payload": payload.relative_to(root).as_posix(), "manifest": manifest.relative_to(root).as_posix()} history = load_run_report_plan(root, REPORT_CONFIG, **kwargs) outputs = dict(history.outputs) - if json.loads(outputs[f"{ARCHIVE}/latest.json"])["run"]["id"] != run_identity(evidence): + if json.loads(outputs[f"{archive_directory(root)}/latest.json"])["run"]["id"] != run_identity(evidence): msg = "selected evidence changed while composing scientific report" raise ValueError(msg) outputs["docs/performance.md"] = render_scientific_report(evidence) @@ -218,7 +218,7 @@ def main(argv: list[str] | None = None) -> int: if args.command == "report": if args.current or args.baseline: parser.error("report reads retained evidence and takes no release labels") - if any(path.exists() or path.is_symlink() for path in (*scratch_paths(stem), root / ARCHIVE)): + if any(path.exists() or path.is_symlink() for path in (*scratch_paths(stem), root / archive_directory(root))): promote(root, stem) else: paths = resolve_report_paths(root, ArtifactPaths(stem.with_suffix(".csv"), stem.with_suffix(".provenance.json"))) diff --git a/scripts/bench_compare.py b/scripts/bench_compare.py index 88af4e3..690b85e 100644 --- a/scripts/bench_compare.py +++ b/scripts/bench_compare.py @@ -35,9 +35,9 @@ from research_repo_tools.criterion import Comparison as TimingComparison, Estimate, read_estimate from research_repo_tools.files import replace_many -from research_repo_tools.process import ExecutableNotFoundError, format_exception_diagnostics +from research_repo_tools.process import ExecutableNotFoundError, format_exception_diagnostics, run_command -from benchmark_process import find_project_root, run_git_command +from benchmark_contract import find_project_root from criterion_dim_plot import METRICS from performance_artifacts import ( PRE_RATIONAL_INPUT_API_COMPATIBILITY, @@ -223,24 +223,6 @@ type Statistic = Literal["mean", "median"] -@dataclass(frozen=True, slots=True) -class CriterionEstimate: - """A Criterion point estimate and its optional confidence interval.""" - - point_ns: float - ci_lo_ns: float | None - ci_hi_ns: float | None - - def __post_init__(self) -> None: - """Keep every stored timing finite, positive, and interval-complete.""" - Estimate(self.point_ns, self.ci_lo_ns, self.ci_hi_ns) - - @property - def has_confidence_interval(self) -> bool: - """Return whether both confidence bounds were present.""" - return self.ci_lo_ns is not None and self.ci_hi_ns is not None - - @dataclass(frozen=True, slots=True) class BenchResult: """A single benchmark measurement (point estimate + confidence interval).""" @@ -248,22 +230,7 @@ class BenchResult: suite: str group: str bench: str - estimate: CriterionEstimate - - @property - def point_ns(self) -> float: - """Return the point estimate in nanoseconds.""" - return self.estimate.point_ns - - @property - def ci_lo_ns(self) -> float | None: - """Return the lower confidence bound, when Criterion recorded it.""" - return self.estimate.ci_lo_ns - - @property - def ci_hi_ns(self) -> float | None: - """Return the upper confidence bound, when Criterion recorded it.""" - return self.estimate.ci_hi_ns + estimate: Estimate @dataclass(frozen=True, slots=True) @@ -273,42 +240,42 @@ class Comparison: suite: str group: str bench: str - baseline: CriterionEstimate - current: CriterionEstimate + baseline: Estimate + current: Estimate assessment: ChangeAssessment baseline_bench: str | None = None - baseline_nalgebra: CriterionEstimate | None = None - baseline_faer: CriterionEstimate | None = None + baseline_nalgebra: Estimate | None = None + baseline_faer: Estimate | None = None @property def baseline_ns(self) -> float: """Return the baseline point estimate.""" - return self.baseline.point_ns + return self.baseline.point @property def current_ns(self) -> float: """Return the current point estimate.""" - return self.current.point_ns + return self.current.point @property def speedup(self) -> float: """Return baseline/current, where values above one are faster.""" - return TimingComparison(self.bench, Estimate(self.baseline_ns), Estimate(self.current_ns)).speedup + return TimingComparison(self.bench, self.baseline, self.current).speedup @property def pct_change(self) -> float: """Return signed point-estimate change, where negative is faster.""" - return 0.0 - TimingComparison(self.bench, Estimate(self.baseline_ns), Estimate(self.current_ns)).percent_reduction + return 0.0 - TimingComparison(self.bench, self.baseline, self.current).percent_reduction @property def baseline_nalgebra_ns(self) -> float | None: """Return the baseline nalgebra point estimate, when available.""" - return None if self.baseline_nalgebra is None else self.baseline_nalgebra.point_ns + return None if self.baseline_nalgebra is None else self.baseline_nalgebra.point @property def baseline_faer_ns(self) -> float | None: """Return the baseline faer point estimate, when available.""" - return None if self.baseline_faer is None else self.baseline_faer.point_ns + return None if self.baseline_faer is None else self.baseline_faer.point @dataclass(frozen=True, slots=True) @@ -449,12 +416,6 @@ def _is_selected_comparison_row( return dim is not None and (bench in VS_LINALG_LA_STACK_BENCHES or bench in VS_LINALG_RELEASE_SIGNAL_BENCHES_BY_DIM.get(dim, [])) -def _read_estimate(estimates_json: Path, stat: str = "median") -> CriterionEstimate: - """Adapt shared Criterion parsing to the retained report schema.""" - estimate = read_estimate(estimates_json, statistic=cast("Statistic", stat)) - return CriterionEstimate(estimate.point, estimate.lower, estimate.upper) - - def _read_harness_provenance( criterion_dir: Path, *, @@ -800,17 +761,13 @@ def _validate_schema2_consistency( raise ValueError(msg) -def assess_change(baseline: CriterionEstimate, current: CriterionEstimate) -> ChangeAssessment: +def assess_change(baseline: Estimate, current: Estimate) -> ChangeAssessment: """Classify a change conservatively from non-overlapping Criterion intervals.""" - if not baseline.has_confidence_interval or not current.has_confidence_interval: + if baseline.lower is None or baseline.upper is None or current.lower is None or current.upper is None: return "unknown" - - if baseline.ci_lo_ns is None or baseline.ci_hi_ns is None or current.ci_lo_ns is None or current.ci_hi_ns is None: - msg = "confidence-interval presence invariant violated" - raise AssertionError(msg) - if current.ci_hi_ns < baseline.ci_lo_ns: + if current.upper < baseline.lower: return "improvement" - if current.ci_lo_ns > baseline.ci_hi_ns: + if current.lower > baseline.upper: return "regression" return "inconclusive" @@ -829,7 +786,7 @@ def _collect_exact_results(criterion_dir: Path, sample: str, stat: str) -> list[ if not est_path.exists(): continue - estimate = _read_estimate(est_path, stat) + estimate = read_estimate(est_path, statistic=cast("Statistic", stat)) results.append(BenchResult(suite="exact", group=group, bench=bench, estimate=estimate)) return results @@ -859,7 +816,7 @@ def _collect_vs_linalg_results(criterion_dir: Path, sample: str, stat: str) -> l for _dim, group_dir in sorted(dim_groups, key=lambda item: item[0]): for bench in _ordered_vs_linalg_benches(group_dir, sample): est_path = group_dir / bench / sample / "estimates.json" - estimate = _read_estimate(est_path, stat) + estimate = read_estimate(est_path, statistic=cast("Statistic", stat)) results.append(BenchResult(suite="vs_linalg", group=group_dir.name, bench=bench, estimate=estimate)) return results @@ -924,7 +881,7 @@ def _collect_exact_comparisons( ) ) else: - _read_estimate(new_path, stat) + read_estimate(new_path, statistic=cast("Statistic", stat)) continue if missing_current or missing_baseline: @@ -940,8 +897,8 @@ def _collect_exact_comparisons( ) continue - current = _read_estimate(new_path, stat) - baseline = _read_estimate(base_path, stat) + current = read_estimate(new_path, statistic=cast("Statistic", stat)) + baseline = read_estimate(base_path, statistic=cast("Statistic", stat)) comparisons.append( Comparison( @@ -1001,11 +958,11 @@ def _vs_linalg_dimension_groups(criterion_dir: Path, scope: str) -> list[tuple[i return groups -def _read_optional_estimate(estimates_json: Path, stat: str) -> CriterionEstimate | None: +def _read_optional_estimate(estimates_json: Path, stat: str) -> Estimate | None: """Read an optional Criterion estimate.""" if not estimates_json.exists(): return None - return _read_estimate(estimates_json, stat) + return read_estimate(estimates_json, statistic=cast("Statistic", stat)) def _baseline_peer_estimates( @@ -1013,7 +970,7 @@ def _baseline_peer_estimates( bench: str, baseline_name: str, stat: str, -) -> tuple[CriterionEstimate | None, CriterionEstimate | None]: +) -> tuple[Estimate | None, Estimate | None]: """Return last-release nalgebra/faer context for a la-stack vs_linalg bench.""" peers = VS_LINALG_BASELINE_PEERS.get(bench) if peers is None: @@ -1069,8 +1026,8 @@ def _collect_vs_linalg_comparisons( ) continue - current = _read_estimate(new_path, stat) - baseline = _read_estimate(base_path, stat) + current = read_estimate(new_path, statistic=cast("Statistic", stat)) + baseline = read_estimate(base_path, statistic=cast("Statistic", stat)) baseline_nalgebra, baseline_faer = _baseline_peer_estimates(group_dir, bench, baseline_name, stat) comparisons.append( @@ -1204,16 +1161,16 @@ def _format_pct(pct: float, assessment: ChangeAssessment) -> str: return f"{pct:+.1f}%" -def _format_confidence_interval(estimate: CriterionEstimate) -> str: +def _format_confidence_interval(estimate: Estimate) -> str: """Format a Criterion confidence interval without inventing missing bounds.""" - if estimate.ci_lo_ns is None or estimate.ci_hi_ns is None: + if estimate.lower is None or estimate.upper is None: return "unavailable" - return f"[{format_time(estimate.ci_lo_ns)}, {format_time(estimate.ci_hi_ns)}]" + return f"[{format_time(estimate.lower)}, {format_time(estimate.upper)}]" -def _format_estimate(estimate: CriterionEstimate) -> str: +def _format_estimate(estimate: Estimate) -> str: """Format a point estimate together with its Criterion interval.""" - return f"{format_time(estimate.point_ns)} {_format_confidence_interval(estimate)}" + return f"{format_time(estimate.point)} {_format_confidence_interval(estimate)}" def _assessment_label(assessment: ChangeAssessment) -> str: @@ -1305,7 +1262,7 @@ def _snapshot_tables(results: list[BenchResult], stat: str) -> str: case = _group_heading_for_suite(suite, group) for r in items: ci_range = _format_confidence_interval(r.estimate) - lines.append(f"| {case} | {r.bench} | {format_time(r.point_ns)} | {ci_range} |") + lines.append(f"| {case} | {r.bench} | {format_time(r.estimate.point)} | {ci_range} |") sections.append("\n".join(lines)) return "\n\n".join(sections) @@ -1384,24 +1341,24 @@ def _coverage_table(gaps: list[CoverageGap], baseline_name: str) -> str: return "\n".join(lines) -def _artifact_timing(estimate: CriterionEstimate) -> TimingEstimate: +def _artifact_timing(estimate: Estimate) -> TimingEstimate: """Convert a Criterion estimate into the stricter persisted timing model.""" - if estimate.ci_lo_ns is None or estimate.ci_hi_ns is None: + if estimate.lower is None or estimate.upper is None: msg = "release-performance artifacts require complete Criterion confidence intervals" raise ValueError(msg) return TimingEstimate( - median_ns=estimate.point_ns, - ci_lower_ns=estimate.ci_lo_ns, - ci_upper_ns=estimate.ci_hi_ns, + median_ns=estimate.point, + ci_lower_ns=estimate.lower, + ci_upper_ns=estimate.upper, ) -def _criterion_timing(estimate: TimingEstimate) -> CriterionEstimate: +def _criterion_timing(estimate: TimingEstimate) -> Estimate: """Convert a validated persisted timing back into the report model.""" - return CriterionEstimate( - point_ns=estimate.median_ns, - ci_lo_ns=estimate.ci_lower_ns, - ci_hi_ns=estimate.ci_upper_ns, + return Estimate( + point=estimate.median_ns, + lower=estimate.ci_lower_ns, + upper=estimate.ci_upper_ns, ) @@ -1450,7 +1407,7 @@ def _unavailable_artifact_rows( # noqa: PLR0913 current_path = criterion_dir / group / bench / "new" / "estimates.json" if not current_path.is_file(): continue - current = _read_estimate(current_path, stat) + current = read_estimate(current_path, statistic=cast("Statistic", stat)) rows.append( PerformanceRow( suite=row_suite, @@ -1550,7 +1507,7 @@ def _get_git_info(root: Path) -> tuple[str, str]: short_hash = "unknown" branch = "unknown" try: - result = run_git_command(["--no-pager", "rev-parse", "--short", "HEAD"], cwd=root) + result = run_command("git", ["--no-pager", "rev-parse", "--short", "HEAD"], cwd=root) short_hash = result.stdout.strip() except ( ExecutableNotFoundError, @@ -1560,7 +1517,7 @@ def _get_git_info(root: Path) -> tuple[str, str]: ): pass try: - result = run_git_command(["--no-pager", "rev-parse", "--abbrev-ref", "HEAD"], cwd=root) + result = run_command("git", ["--no-pager", "rev-parse", "--abbrev-ref", "HEAD"], cwd=root) branch = result.stdout.strip() except ( ExecutableNotFoundError, @@ -1575,7 +1532,7 @@ def _get_git_info(root: Path) -> tuple[str, str]: def _get_git_source_date(root: Path) -> str: """Return the reproducible source revision timestamp normalized to UTC.""" try: - result = run_git_command(["--no-pager", "show", "-s", "--format=%cI", "HEAD"], cwd=root) + result = run_command("git", ["--no-pager", "show", "-s", "--format=%cI", "HEAD"], cwd=root) except ( ExecutableNotFoundError, OSError, diff --git a/scripts/benchmark_contract.py b/scripts/benchmark_contract.py index 2caaabc..59be4af 100644 --- a/scripts/benchmark_contract.py +++ b/scripts/benchmark_contract.py @@ -1,10 +1,21 @@ """Stable content identity for release benchmark measurements.""" import hashlib -from typing import TYPE_CHECKING, Final +from pathlib import Path +from typing import Final + + +def find_project_root(start: Path | None = None) -> Path: + """Find the nearest Cargo manifest from the caller's working directory.""" + directory = (start or Path.cwd()).resolve() + if directory.is_file(): + directory = directory.parent + for candidate in (directory, *directory.parents): + if (candidate / "Cargo.toml").is_file(): + return candidate + msg = "Could not locate Cargo.toml to determine benchmark root" + raise FileNotFoundError(msg) -if TYPE_CHECKING: - from pathlib import Path BENCHMARK_CONTRACT_DIRS: Final[tuple[str, ...]] = ("benches",) BENCHMARK_CONTRACT_FILES: Final[tuple[str, ...]] = ( diff --git a/scripts/benchmark_process.py b/scripts/benchmark_process.py deleted file mode 100644 index e27120f..0000000 --- a/scripts/benchmark_process.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Benchmark command adapters around the published shared process API. - -The shared package owns discovery, execution, exact byte transport, CPU metadata, -and diagnostics. These adapters retain existing benchmark call signatures and -the distinction between captured commands and live measurement progress. -""" - -import subprocess -from pathlib import Path -from typing import TYPE_CHECKING, TypedDict, Unpack - -from research_repo_tools.process import run_command, run_command_live - -if TYPE_CHECKING: - from collections.abc import Mapping - - -class CommandOptions(TypedDict, total=False): - """Options needed by captured plotting and live measurement phases.""" - - env: Mapping[str, str] | None - check: bool - timeout: float | None - capture_output: bool - input: str - - -def run_safe_command( - command: str, - args: list[str], - cwd: Path | None = None, - **options: Unpack[CommandOptions], -) -> subprocess.CompletedProcess[str]: - """Select the shared captured or live runner for a benchmark phase.""" - env = options.get("env") - check = options.get("check", True) - timeout = options.get("timeout", 300.0) - if options.get("capture_output", True): - return run_command(command, args, cwd=cwd, env=env, check=check, timeout=timeout, input=options.get("input")) - if "input" in options: - msg = "Live benchmark commands require inherited stdin" - raise ValueError(msg) - result = run_command_live(command, args, cwd=cwd, env=env, check=check, timeout=timeout) - return subprocess.CompletedProcess(result.args, result.returncode, "", "") - - -def run_git_command( - args: list[str], - cwd: Path | None = None, - *, - check: bool = True, - timeout: float | None = 300.0, - env: Mapping[str, str] | None = None, -) -> subprocess.CompletedProcess[str]: - """Capture benchmark Git metadata without opening a pager.""" - return run_command("git", ["--no-pager", *args], cwd=cwd, check=check, timeout=timeout, env=env) - - -def find_project_root(start: Path | None = None) -> Path: - """Find the benchmark consumer's nearest Cargo manifest from its working directory.""" - directory = (start or Path.cwd()).resolve() - if directory.is_file(): - directory = directory.parent - for candidate in (directory, *directory.parents): - if (candidate / "Cargo.toml").is_file(): - return candidate - msg = "Could not locate Cargo.toml to determine benchmark root" - raise FileNotFoundError(msg) diff --git a/scripts/criterion_dim_plot.py b/scripts/criterion_dim_plot.py index a50f8e3..94a0954 100644 --- a/scripts/criterion_dim_plot.py +++ b/scripts/criterion_dim_plot.py @@ -15,7 +15,6 @@ import argparse import hashlib import json -import math import platform import re import shutil @@ -29,14 +28,13 @@ from typing import Final, TypeGuard, cast from research_repo_tools.complete_runs import run_identity -from research_repo_tools.criterion import Statistic, read_estimate +from research_repo_tools.criterion import Comparison, Estimate, Statistic, read_estimate from research_repo_tools.evidence import serialize_evidence from research_repo_tools.files import replace_many -from research_repo_tools.process import ExecutableNotFoundError, cpu_description, format_exception_diagnostics +from research_repo_tools.process import ExecutableNotFoundError, cpu_description, format_exception_diagnostics, run_command from research_repo_tools.publication import MarkerPair, plan_outputs, publish_publication, replace_section -from benchmark_contract import benchmark_contract_digest -from benchmark_process import find_project_root, run_git_command, run_safe_command +from benchmark_contract import benchmark_contract_digest, find_project_root from benchmark_summaries import report_input_paths, resolve_report_paths from performance_artifacts import ArtifactPaths, PerformanceBundle, TimingEstimate, ensure_distinct_paths, load_bundle @@ -113,21 +111,9 @@ def __post_init__(self) -> None: if self.dim <= 0: msg = f"dimension must be positive: {self.dim}" raise ValueError(msg) - for field, value in ( - ("la_time", self.la_time), - ("la_lo", self.la_lo), - ("la_hi", self.la_hi), - ("na_time", self.na_time), - ("na_lo", self.na_lo), - ("na_hi", self.na_hi), - ("fa_time", self.fa_time), - ("fa_lo", self.fa_lo), - ("fa_hi", self.fa_hi), - ): - _require_positive_finite_time(value, field) - _require_confidence_interval(self.la_lo, self.la_hi, "la_stack row") - _require_confidence_interval(self.na_lo, self.na_hi, "nalgebra row") - _require_confidence_interval(self.fa_lo, self.fa_hi, "faer row") + Estimate(self.la_time, self.la_lo, self.la_hi) + Estimate(self.na_time, self.na_lo, self.na_hi) + Estimate(self.fa_time, self.fa_lo, self.fa_hi) class ReadmeBenchmarkLinkError(ValueError): @@ -310,19 +296,6 @@ def _read_estimate(estimates_json: Path, stat: str) -> tuple[float, float, float return estimate.point, estimate.lower, estimate.upper -def _require_positive_finite_time(value: float, context: str) -> float: - if not math.isfinite(value) or value <= 0.0: - msg = f"{context} must be finite and positive: {value!r}" - raise ValueError(msg) - return value - - -def _require_confidence_interval(lo: float, hi: float, context: str) -> None: - if lo > hi: - msg = f"{context} lower bound must be <= upper bound: {lo!r} > {hi!r}" - raise ValueError(msg) - - def _write_csv(out_csv: Path, rows: list[Row]) -> None: out_csv.parent.mkdir(parents=True, exist_ok=True) with out_csv.open("w", encoding="utf-8", newline="\n") as f: @@ -333,9 +306,7 @@ def _write_csv(out_csv: Path, rows: list[Row]) -> None: def _pct_reduction(baseline: float, value: float) -> str: """Percent time reduction relative to baseline (positive = value is faster).""" - if baseline == 0.0: - return "n/a" - pct = ((baseline - value) / baseline) * 100.0 + pct = Comparison("dimension", Estimate(baseline), Estimate(value)).percent_reduction return f"{pct:+.1f}%" @@ -430,7 +401,7 @@ def _render_svg_with_gnuplot(req: PlotRequest) -> None: ) try: - run_safe_command("gnuplot", [], input="\n".join(gp_lines)) + run_command("gnuplot", [], input="\n".join(gp_lines)) except ExecutableNotFoundError as exc: msg = "gnuplot not found. Install it (macOS: `brew install gnuplot`) or re-run with --no-plot." raise FileNotFoundError(msg) from exc @@ -616,7 +587,7 @@ def _collect_rows(criterion_dir: Path, dims: list[int], metric: Metric, stat: st def _git_value(root: Path, args: list[str]) -> str: """Return deterministic Git provenance or an explicit unavailable label.""" try: - value = run_git_command(args, cwd=root).stdout.strip() + value = run_command("git", ["--no-pager", *args], cwd=root).stdout.strip() except ( ExecutableNotFoundError, OSError, @@ -630,7 +601,8 @@ def _git_value(root: Path, args: list[str]) -> str: def _git_status_metadata(root: Path) -> tuple[bool | None, str]: """Return checkout cleanliness and a deterministic digest of porcelain status.""" try: - status = run_git_command( + status = run_command( + "git", ["--no-pager", "status", "--porcelain=v1", "--untracked-files=all"], cwd=root, ).stdout @@ -708,7 +680,7 @@ def _provenance_harness_digest(root: Path) -> tuple[str, list[str]]: def _rustc_version(root: Path) -> str: """Return the rustc version used by the publication workflow.""" try: - value = run_safe_command( + value = run_command( "rustc", ["--version"], cwd=root, @@ -1242,10 +1214,11 @@ def _complete_readme(root: Path, args: PlotCliArgs) -> int: payload, manifest = complete.scratch_paths(root / "target/bench-reports/performance") identity = run_identity(evidence) if not payload.exists(): - payload = root / complete.ARCHIVE / "runs" / identity / "run.json" + archive = root / complete.archive_directory(root) + payload = archive / "runs" / identity / "run.json" manifest = payload.with_name("evidence.json") for name in ("index.json", "latest.json"): - path = root / complete.ARCHIVE / name + path = archive / name inputs[path] = path.read_bytes() inputs.update(zip((payload, manifest), serialize_evidence(evidence), strict=True)) provenance: dict[str, object] = { diff --git a/scripts/performance_phase.py b/scripts/performance_phase.py deleted file mode 100644 index b4b5c93..0000000 --- a/scripts/performance_phase.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Consumer Cargo commands for the captured common benchmark harness.""" - -import argparse -import os -from pathlib import Path - -from research_repo_tools.process import run_command, run_command_live - -SUITES = {"vs_linalg": ("-p", "la-stack-comparison", "--features", "bench"), "exact": ("--features", "bench,exact")} - - -def commands(suite: str, phase: str) -> tuple[tuple[str, ...], ...]: - """Declare complete baseline timing and current la-stack-only timing.""" - selected = SUITES if suite == "all" else {suite: SUITES[suite]} - return tuple( - ("cargo", "bench", "--locked", *features, "--bench", name, "--", *(("la_stack",) if phase == "current" and name == "vs_linalg" else ()), "--noplot") - for name, features in selected.items() - ) - - -def discover(root: Path, suite: str, phase: str, env: dict[str, str]) -> tuple[str, ...]: - """Ask native Criterion binaries for semantic IDs, without timing samples.""" - expected: list[str] = [] - for command in commands(suite, phase): - result = run_command(command[0], [*command[1:], "--list"], cwd=root, env=env, timeout=7200) - ids = [line.removesuffix(": benchmark") for line in result.stdout.splitlines() if line.endswith(": benchmark")] - if not ids: - raise ValueError(f"empty Criterion inventory from {command}") - expected.extend(ids) - if len(expected) != len(set(expected)): - msg = "duplicate semantic benchmark IDs" - raise ValueError(msg) - return tuple(sorted(expected)) - - -def main() -> None: - """Execute one declared correctness or timing phase with inherited live output.""" - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("action", choices=("gate", "measure")) - parser.add_argument("suite", choices=("all", *SUITES)) - parser.add_argument("phase", choices=("baseline", "current")) - args = parser.parse_args() - selected = commands(args.suite, args.phase) - if args.action == "gate": - selected = (("cargo", "test", "--locked", "--workspace", "--features", "bench,exact", "--test", "vs_linalg_inputs", "--test", "exact_bench_config"),) - for command in selected: - print(f"[performance] {args.phase} {args.action}: {' '.join(command)}", flush=True) - run_command_live(command[0], command[1:], cwd=Path.cwd(), env=os.environ, timeout=7200) - - -if __name__ == "__main__": - main() diff --git a/scripts/performance_runs.py b/scripts/performance_runs.py index 7abd89b..7e39a90 100644 --- a/scripts/performance_runs.py +++ b/scripts/performance_runs.py @@ -2,41 +2,50 @@ import json import os -import sys import tomllib +from dataclasses import replace from pathlib import Path from typing import TYPE_CHECKING from research_repo_tools.common_measurement import CommonHarnessPlan, MeasurementPhase from research_repo_tools.complete_runs import CompletePolicy, CompleteRun, RunSeries, run_identity, validate_run_evidence from research_repo_tools.evidence import Evidence, fingerprint_files, load_evidence -from research_repo_tools.measurement import MeasurementConfig +from research_repo_tools.measurement import MeasurementConfig, load_measurement +from research_repo_tools.process import run_command from research_repo_tools.release_pairs import ReleasePair from research_repo_tools.run_reports import load_latest_run import bench_compare as report +from benchmark_contract import find_project_root from performance_artifacts import NO_API_COMPATIBILITY, resolve_shared_harness_compatibility -from performance_phase import commands, discover +from release_baseline import parse_benchmark_list if TYPE_CHECKING: from research_repo_tools.criterion import Estimate -ARCHIVE = "docs/performance-runs" -REPORT_CONFIG = ".config/performance-report.toml" -SOURCES = ("src/**/*.rs", "Cargo.toml") -HARNESS = ( - "benches/**/*.rs", - "benches/comparison/Cargo.toml", - "examples/**/*.rs", - "Cargo.toml", - "Cargo.lock", - "rust-toolchain.toml", - "tests/exact_bench_config.rs", - "scripts/performance_phase.py", -) +REPORT_CONFIG = "tooling/performance-report.toml" +MEASUREMENT_CONFIG = "tooling/performance.toml" UNAVAILABLE_PREFIXES = ("rational_input_", "canonical_conversion_", "det4_diagnostic_") +def configuration(root: Path | None = None) -> MeasurementConfig: + """Load the shared measurement schema from the consumer checkout.""" + config = load_measurement(root or find_project_root(), MEASUREMENT_CONFIG) + if not isinstance(config, MeasurementConfig): + msg = "la-stack requires schema 1 with dynamically discovered Criterion inventories" + raise ValueError(msg) # noqa: TRY004 - wrong configuration schema, not a caller type error + return config + + +def archive_directory(root: Path) -> str: + """Read the history path; shared report APIs validate its path and contents.""" + archive = tomllib.loads((root / REPORT_CONFIG).read_text(encoding="utf-8")).get("archive") + if not isinstance(archive, str) or not archive: + msg = "performance report configuration requires an archive path" + raise ValueError(msg) + return archive + + def selected_ids(suite: str, scope: str, expected: tuple[str, ...]) -> tuple[str, ...]: """Require the repository report registry, keeping diagnostic cases in the run.""" if suite not in {"all", "exact", "vs_linalg"} or scope not in {"all-benches", "release-signal"}: @@ -77,16 +86,24 @@ def measurement_plan(root: Path, pair: ReleasePair, suite: str, scope: str, list compatibility = resolve_shared_harness_compatibility(current=pair.current, baseline=pair.baseline, shared_harness_rational_inputs=True) baseline_env = phase_environment(root, compatibility.baseline_api_compatibility) current_env = phase_environment(root) + config = configuration(root) inventories = {} for phase in ("baseline", "current"): # Discover against the current API, then apply the exact groups excluded # by the historical cfg. Old method names cannot compile on current code. # The shared collector later checks this entire inventory on each source. env = os.environ | dict(current_env) | {"CRITERION_HOME": str(listing / phase)} - ids = discover(root, suite, phase, env) + ids = [] + for selected in ("vs_linalg", "exact") if suite == "all" else (suite,): + command = (*config.command[:-1], f"{phase}-{selected}", "list") + result = run_command(command[0], command[1:], cwd=root, env=env, timeout=config.timeout) + ids.extend(parse_benchmark_list(result.stdout)) + if len(ids) != len(set(ids)): + msg = "duplicate semantic benchmark IDs" + raise ValueError(msg) old_api = phase == "baseline" and compatibility.baseline_api_compatibility != NO_API_COMPATIBILITY - inventories[phase] = tuple(name for name in ids if not (old_api and name.startswith(UNAVAILABLE_PREFIXES))) - return declared_plan(pair, suite, scope, inventories, baseline_env=baseline_env, current_env=current_env) + inventories[phase] = tuple(sorted(name for name in ids if not (old_api and name.startswith(UNAVAILABLE_PREFIXES)))) + return declared_plan(pair, suite, scope, inventories, baseline_env=baseline_env, current_env=current_env, config=config) def declared_plan( # noqa: PLR0913 @@ -97,8 +114,10 @@ def declared_plan( # noqa: PLR0913 *, baseline_env: tuple[tuple[str, str], ...], current_env: tuple[tuple[str, str], ...], + config: MeasurementConfig | None = None, ) -> CommonHarnessPlan: """Bind exact case inventories, reference phases, commands and completeness.""" + config = config or configuration() compatibility = resolve_shared_harness_compatibility(current=pair.current, baseline=pair.baseline, shared_harness_rational_inputs=True) selected = selected_ids(suite, scope, inventories["current"]) unavailable = compatibility.baseline_api_compatibility != NO_API_COMPATIBILITY @@ -120,40 +139,38 @@ def declared_plan( # noqa: PLR0913 series.append(RunSeries(library, "baseline", rows)) phases = { phase: MeasurementPhase( - (sys.executable, "scripts/performance_phase.py", "measure", suite, phase), - (sys.executable, "scripts/performance_phase.py", "gate", suite, phase), + (*config.command[:-1], f"{phase}-{suite}"), + (*config.command[:-1], "gate"), CompletePolicy(inventories[phase], sample_count=100, statistics=("mean", "median"), confidence_level=0.95), environment, ) for phase, environment in (("baseline", baseline_env), ("current", current_env)) } - dependencies = tuple((name, "Cargo.lock") for name in ("criterion", "nalgebra", "faer")) - probes = (("rustc", ("rustc", "-vV")), ("cargo", ("cargo", "--version"))) - config = MeasurementConfig( - phases["current"].command, - SOURCES, - HARNESS, - probes=probes, - dependencies=dependencies, + config = replace( + config, + command=phases["current"].command, context=( + *config.context, ("suite", suite), ("scope", scope), ("baseline-api-compatibility", compatibility.baseline_api_compatibility), - ("baseline-cargo-commands", json.dumps(commands(suite, "baseline"))), - ("current-cargo-commands", json.dumps(commands(suite, "current"))), - ), - compatible=( - "harness_sha256", - "context.os", - "context.architecture", - "context.cpu", - *(f"context.tool.{name}" for name, _ in probes), - *(f"context.dependency.{name}" for name, _ in dependencies), ), ) return CommonHarnessPlan(config, phases["baseline"], phases["current"], tuple(series)) +def _legacy_cargo_commands(suite: str, phase: str) -> str: + """Validate the frozen pre-Just evidence contract without running its driver.""" + suites = {"vs_linalg": ("-p", "la-stack-comparison", "--features", "bench"), "exact": ("--features", "bench,exact")} + return json.dumps( + [ + ("cargo", "bench", "--locked", *features, "--bench", name, "--", *(("la_stack",) if phase == "current" and name == "vs_linalg" else ()), "--noplot") + for name, features in suites.items() + if suite in {"all", name} + ] + ) + + def validate_scientific_run(evidence: Evidence, *, release: bool = False) -> CompleteRun: # noqa: C901 """Require la-stack's sampling, selected coverage and original reference phases.""" run = validate_run_evidence(evidence) @@ -178,18 +195,28 @@ def validate_scientific_run(evidence: Evidence, *, release: bool = False) -> Com if run.series != expected.series or context["baseline-api-compatibility"] != compatibility.baseline_api_compatibility: msg = "retained series or compatibility policy differs from la-stack declarations" raise ValueError(msg) - if set(run.compatible) != set(expected.measurement.compatible): + legacy = "phase-driver" not in context + compatible = set(expected.measurement.compatible) - ({"context.tool.just"} if legacy else set()) + if set(run.compatible) != compatible: msg = "retained run omits required host, toolchain, or dependency compatibility" raise ValueError(msg) + required = dict(expected.measurement.context) + if legacy: + del required["phase-driver"] + required.update((f"{phase}-cargo-commands", _legacy_cargo_commands(context["suite"], phase)) for phase in ("baseline", "current")) for phase, source in evidence.sources: actual = dict(source.context) - if any(actual.get(key) != value for key, value in expected.measurement.context): + if actual.get("phase-driver") != context.get("phase-driver") or any(actual.get(key) != value for key, value in required.items()): msg = "retained phase differs from the declared consumer policy" raise ValueError(msg) for key, command in (("command", getattr(expected, phase).command), ("gate-command", getattr(expected, phase).gate)): - # Interpreter paths vary across hosts; retain them without executing - # evidence-supplied commands during offline rendering. - if tuple(json.loads(actual[key]))[1:] != command[1:]: + recorded = tuple(json.loads(actual[key])) + if legacy: + # Interpreter paths varied across hosts. Old evidence remains + # readable without retaining or executing the Python driver. + command = ("scripts/performance_phase.py", "measure" if key == "command" else "gate", context["suite"], phase) + recorded = recorded[1:] + if recorded != command: raise ValueError(f"retained {phase} {key} differs from the declared Cargo phase") return run @@ -205,7 +232,7 @@ def retained_run(root: Path, stem: Path) -> Evidence: legacy = (stem.with_suffix(".csv"), stem.with_suffix(".provenance.json"), stem.with_suffix(".full.csv"), stem.with_suffix(".full.provenance.json")) # Partial new or legacy scratch must never silently select older history. present = any(path.exists() or path.is_symlink() for path in (*paths, *legacy)) - evidence = load_evidence(*paths) if present else load_latest_run(root, ARCHIVE) + evidence = load_evidence(*paths) if present else load_latest_run(root, archive_directory(root)) validate_scientific_run(evidence) return evidence @@ -214,9 +241,10 @@ def verify_current_source(root: Path, evidence: Evidence) -> None: """Bind README publication to the measured current library and captured harness.""" source = dict(evidence.sources)["current"] context = dict(source.context) + config = configuration(root) for key, expected in (("source-inventory", source.source_sha256), ("harness-inventory", source.harness_sha256)): names = json.loads(context[key]) - patterns = SOURCES if key == "source-inventory" else HARNESS + patterns = config.sources if key == "source-inventory" else config.harness actual = {path.relative_to(root).as_posix() for pattern in patterns for path in root.glob(pattern)} if actual != set(names): raise ValueError(f"current {key} has stale file membership") @@ -251,22 +279,17 @@ def render_scientific_report(evidence: Evidence) -> bytes: current_only.append(f"- `{name}`: current {report.format_time(estimate.point)}; baseline API unavailable.") continue - def converted(value: Estimate | None) -> report.CriterionEstimate | None: - return report.CriterionEstimate(value.point, value.lower, value.upper) if value is not None else None - comparisons.append( report.Comparison( suite="exact" if group.startswith(("exact_", *UNAVAILABLE_PREFIXES)) else "vs_linalg", group=group, bench=bench, baseline_bench=bench, - baseline=report.CriterionEstimate(prior.point, prior.lower, prior.upper), - current=report.CriterionEstimate(estimate.point, estimate.lower, estimate.upper), - assessment=report.assess_change( - report.CriterionEstimate(prior.point, prior.lower, prior.upper), report.CriterionEstimate(estimate.point, estimate.lower, estimate.upper) - ), - baseline_nalgebra=converted(values.get("nalgebra", {}).get(name)), - baseline_faer=converted(values.get("faer", {}).get(name)), + baseline=prior, + current=estimate, + assessment=report.assess_change(prior, estimate), + baseline_nalgebra=values.get("nalgebra", {}).get(name), + baseline_faer=values.get("faer", {}).get(name), ) ) lines = [ diff --git a/scripts/release_baseline.py b/scripts/release_baseline.py index 9f33a89..f3227fc 100644 --- a/scripts/release_baseline.py +++ b/scripts/release_baseline.py @@ -18,7 +18,7 @@ VS_LINALG_STANDARD_BENCH_ORDER, ) -SUITES = {"vs_linalg": "bench", "exact": "bench,exact"} +SUITES = ("vs_linalg", "exact") RAW_FILES = ("benchmark.json", "estimates.json", "sample.json", "tukey.json") @@ -66,12 +66,11 @@ def discover(root: Path, manifest: Path, criterion: Path) -> None: if criterion.exists() and any(criterion.iterdir()): raise ValueError(f"release inventory requires a fresh Criterion directory: {criterion}") inventory: dict[str, list[str]] = {} - for suite, features in SUITES.items(): + for suite in SUITES: print(f"[release-baseline] Discovering {suite}", flush=True) - package = ["-p", "la-stack-comparison"] if suite == "vs_linalg" else [] result = run_command( - "cargo", - ["bench", "--locked", *package, "--features", features, "--bench", suite, "--", "--list"], + "just", + ["--justfile", "tooling/performance.just", "--working-directory", ".", f"baseline-{suite}", "list"], cwd=root, timeout=None, # The workflow bounds compilation and discovery together. ) diff --git a/scripts/tests/test_bench_compare.py b/scripts/tests/test_bench_compare.py index 10f4df3..dee1e79 100644 --- a/scripts/tests/test_bench_compare.py +++ b/scripts/tests/test_bench_compare.py @@ -270,34 +270,6 @@ def test_exact_registry_tracks_every_rational_input_release_dimension() -> None: # --------------------------------------------------------------------------- -def test_read_estimate_success(tmp_path: Path) -> None: - est = tmp_path / "estimates.json" - _write_estimates(est, "median", 42.0) - estimate = bench_compare._read_estimate(est, "median") - assert estimate.point_ns == 42.0 - assert estimate.ci_lo_ns == pytest.approx(42.0 * 0.9) - assert estimate.ci_hi_ns == pytest.approx(42.0 * 1.1) - - -def test_read_estimate_no_ci(tmp_path: Path) -> None: - """Missing confidence bounds remain unavailable rather than implying certainty.""" - est = tmp_path / "estimates.json" - est.parent.mkdir(parents=True, exist_ok=True) - est.write_text( - json.dumps({"median": {"point_estimate": 99.0}}), - encoding="utf-8", - ) - estimate = bench_compare._read_estimate(est, "median") - assert estimate.point_ns == 99.0 - assert estimate.ci_lo_ns is None - assert estimate.ci_hi_ns is None - - -# --------------------------------------------------------------------------- -# collect_results / collect_comparisons -# --------------------------------------------------------------------------- - - def test_collect_results(tmp_path: Path) -> None: _build_criterion_tree(tmp_path) results = bench_compare._collect_results(tmp_path, "new", "median") @@ -581,8 +553,8 @@ def test_comparison_table_preserves_signed_change(current_ns: float, change: str suite="exact", group="exact_d2", bench="det", - baseline=bench_compare.CriterionEstimate(10.0, None, None), - current=bench_compare.CriterionEstimate(current_ns, None, None), + baseline=bench_compare.Estimate(10.0, None, None), + current=bench_compare.Estimate(current_ns, None, None), assessment="unknown", ) @@ -1355,7 +1327,7 @@ def test_generate_markdown_labels_absent_provenance_unavailable(tmp_path: Path, def test_git_source_date_is_normalized_to_unambiguous_utc(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( bench_compare, - "run_git_command", + "run_command", lambda *_args, **_kwargs: subprocess.CompletedProcess([], 0, stdout="2026-06-01T05:34:56-07:00\n"), ) diff --git a/scripts/tests/test_cargo_update_integration.py b/scripts/tests/test_cargo_update_integration.py index 40ff3f0..a0403d3 100644 --- a/scripts/tests/test_cargo_update_integration.py +++ b/scripts/tests/test_cargo_update_integration.py @@ -14,8 +14,7 @@ from typing import TYPE_CHECKING import pytest - -from benchmark_process import run_safe_command +from research_repo_tools.process import run_command if TYPE_CHECKING: from collections.abc import Iterator @@ -93,7 +92,7 @@ def consumer(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]: def test_native_updates_advance_eligible_dependencies_and_preserve_coupled_requirements(consumer: Path, recipe: str) -> None: """Cargo upgrade and cargo update run through the installed checked toolchain.""" before = tomllib.loads((consumer / "Cargo.toml").read_text(encoding="utf-8"))["dependencies"] - result = run_safe_command( + result = run_command( "just", ["--justfile", str(consumer / "recipes.just"), "--working-directory", str(consumer), recipe], cwd=consumer, diff --git a/scripts/tests/test_changelog_integration.py b/scripts/tests/test_changelog_integration.py index ea4f765..1c5cd9c 100644 --- a/scripts/tests/test_changelog_integration.py +++ b/scripts/tests/test_changelog_integration.py @@ -1,6 +1,5 @@ """Consumer policy and authored history against the installed published CLI.""" -import json import shutil import tomllib from importlib.metadata import version @@ -8,8 +7,8 @@ from typing import TYPE_CHECKING from research_repo_tools.cli import main - -from benchmark_process import run_git_command, run_safe_command +from research_repo_tools.just_inspect import dry_run, inspect_justfile +from research_repo_tools.process import run_command if TYPE_CHECKING: import pytest @@ -43,11 +42,11 @@ def test_authored_dependency_notes_survive_generation(tmp_path: Path, capsys: py """Exercise the consumer policy with Ruff, Ty and setuptools notes and code.""" for name in ("pyproject.toml", "changelog-rumdl.toml", ".python-version", "rust-toolchain.toml"): shutil.copyfile(ROOT / name, tmp_path / name) - run_git_command(["init", "--quiet"], cwd=tmp_path) + run_command("git", ["--no-pager", "init", "--quiet"], cwd=tmp_path) for key, value in (("user.name", "Fixture"), ("user.email", "fixture@example.invalid"), ("commit.gpgsign", "false"), ("tag.gpgsign", "false")): - run_git_command(["config", key, value], cwd=tmp_path) - run_git_command(["commit", "--quiet", "--allow-empty", "-m", "feat: initial history"], cwd=tmp_path) - run_git_command(["tag", "v0.4.5"], cwd=tmp_path) + run_command("git", ["--no-pager", "config", key, value], cwd=tmp_path) + run_command("git", ["--no-pager", "commit", "--quiet", "--allow-empty", "-m", "feat: initial history"], cwd=tmp_path) + run_command("git", ["--no-pager", "tag", "v0.4.5"], cwd=tmp_path) notes = ( ("ruff", "deps-dev", "Read [release notes](https://github.com/astral-sh/ruff/compare/0.16.1...0.16.2) before changing `Vector` checks."), ("ty", "deps-dev", "Keep [changelog](https://github.com/astral-sh/ty/releases) context and `--error all`."), @@ -58,7 +57,7 @@ def test_authored_dependency_notes_survive_generation(tmp_path: Path, capsys: py ), ) for dependency, scope, body in notes: - run_git_command(["commit", "--quiet", "--allow-empty", "-m", f"chore({scope}): bump {dependency}\n\n{body}"], cwd=tmp_path) + run_command("git", ["--no-pager", "commit", "--quiet", "--allow-empty", "-m", f"chore({scope}): bump {dependency}\n\n{body}"], cwd=tmp_path) (tmp_path / "CHANGELOG.md").write_text("# Changelog\n\n## [0.4.5] - 2000-01-02\n\n- Historical date.\n", encoding="utf-8") args = ["--root", str(tmp_path), "changelog", "generate", "--tag", "v0.4.6", "--date", "2026-10-09"] assert main([*args, "--dry-run"]) == 0 @@ -83,9 +82,9 @@ def test_authored_dependency_notes_survive_generation(tmp_path: Path, capsys: py def test_consumer_recipes_and_retained_history(capsys: pytest.CaptureFixture[str]) -> None: """Native recipes delegate generation and all historical notes remain readable.""" - recipes = json.loads(run_safe_command("just", ["--dump", "--dump-format", "json"], cwd=ROOT).stdout) - assert recipes["aliases"]["changelog-unreleased"]["target"] == "changelog-release" - result = run_safe_command("just", ["--dry-run", "changelog-preview", "--tag", "v0.4.7", "--date", "2026-10-09"], cwd=ROOT) + recipes = inspect_justfile(ROOT) + assert recipes.aliases["changelog-unreleased"] == "changelog-release" + result = dry_run(ROOT, "changelog-preview", ["--tag", "v0.4.7", "--date", "2026-10-09"]) assert 'research-repo-tools changelog generate --dry-run "$@"' in result.stderr assert main(["--root", str(ROOT), "changelog", "check"]) == 0 for tag in ("v0.1.0", "v0.2.0", "v0.3.0", "v0.4.6"): diff --git a/scripts/tests/test_criterion_dim_plot.py b/scripts/tests/test_criterion_dim_plot.py index 69fa91a..f96aa06 100644 --- a/scripts/tests/test_criterion_dim_plot.py +++ b/scripts/tests/test_criterion_dim_plot.py @@ -116,7 +116,7 @@ def test_markdown_table_formats_values_and_pct() -> None: def test_row_rejects_zero_peer_time_before_markdown_rendering() -> None: - with pytest.raises(ValueError, match="na_time must be finite and positive"): + with pytest.raises(ValueError, match="finite positive number"): criterion_dim_plot.Row( dim=2, la_time=10.0, @@ -164,42 +164,6 @@ def test_update_readme_table_replaces_only_between_markers(tmp_path: Path) -> No assert changed_again is False -def test_update_readme_table_errors_on_missing_markers(tmp_path: Path) -> None: - readme = tmp_path / "README.md" - readme.write_text("# Title\n", encoding="utf-8") - - with pytest.raises(ValueError, match="exactly one ordered publication marker pair"): - criterion_dim_plot._update_readme_table( - readme, - "", - "", - "| x |", - ) - - -def test_update_readme_table_errors_on_out_of_order_markers(tmp_path: Path) -> None: - marker_begin, marker_end = criterion_dim_plot._readme_table_markers("lu_solve", "median", "new") - - readme = tmp_path / "README.md" - readme.write_text(f"{marker_end}\n{marker_begin}\n", encoding="utf-8") - - with pytest.raises(ValueError, match="exactly one ordered publication marker pair"): - criterion_dim_plot._update_readme_table(readme, marker_begin, marker_end, "| x |") - - -def test_update_readme_table_errors_on_non_unique_markers(tmp_path: Path) -> None: - marker_begin, marker_end = criterion_dim_plot._readme_table_markers("lu_solve", "median", "new") - - readme = tmp_path / "README.md" - readme.write_text( - f"{marker_begin}\n{marker_begin}\n{marker_end}\n", - encoding="utf-8", - ) - - with pytest.raises(ValueError, match="exactly one ordered publication marker pair"): - criterion_dim_plot._update_readme_table(readme, marker_begin, marker_end, "| x |") - - def _canonical_benchmark_readme(version: str) -> str: begin, end = criterion_dim_plot._readme_table_markers("lu_solve", "median", "new") return ( @@ -478,11 +442,11 @@ def test_main_update_readme_happy_path(tmp_path: Path, monkeypatch: pytest.Monke readme = tmp_path / "README.md" readme.write_text(_canonical_benchmark_readme("0.0.8"), encoding="utf-8") - def fake_run_git(args: list[str], **_kwargs: object) -> SimpleNamespace: + def fake_run_git(_command: str, args: list[str], **_kwargs: object) -> SimpleNamespace: return SimpleNamespace(stdout=f"{_TEST_COMMIT}\n") monkeypatch.setattr(criterion_dim_plot, "_repo_root", lambda: tmp_path) - monkeypatch.setattr(criterion_dim_plot, "run_git_command", fake_run_git) + monkeypatch.setattr(criterion_dim_plot, "run_command", fake_run_git) def fake_render(request: criterion_dim_plot.PlotRequest) -> None: request.out_svg.write_text("\n", encoding="utf-8") @@ -582,26 +546,6 @@ def test_format_legend_label() -> None: assert criterion_dim_plot._format_legend_label("faer", "unknown") == "faer" -def test_read_estimate_errors_and_success(tmp_path: Path) -> None: - estimates = tmp_path / "estimates.json" - estimates.write_text( - json.dumps( - { - "median": { - "point_estimate": 5.0, - "confidence_interval": {"lower_bound": 4.0, "upper_bound": 6.0}, - } - } - ), - encoding="utf-8", - ) - point, lo, hi = criterion_dim_plot._read_estimate(estimates, "median") - assert (point, lo, hi) == (5.0, 4.0, 6.0) - - with pytest.raises(ValueError, match="Criterion mean must be a JSON object"): - criterion_dim_plot._read_estimate(estimates, "mean") - - @pytest.mark.parametrize("interval", [None, {"lower_bound": 4.0}]) def test_plot_adapter_requires_complete_confidence_interval(tmp_path: Path, interval: dict[str, float] | None) -> None: estimates = tmp_path / "estimates.json" @@ -611,23 +555,6 @@ def test_plot_adapter_requires_complete_confidence_interval(tmp_path: Path, inte criterion_dim_plot._read_estimate(estimates, "median") -def test_read_estimate_allows_point_outside_percentile_confidence_interval(tmp_path: Path) -> None: - estimates = tmp_path / "estimates.json" - estimates.write_text( - json.dumps( - { - "median": { - "point_estimate": 5.0, - "confidence_interval": {"lower_bound": 1.0, "upper_bound": 4.0}, - } - } - ), - encoding="utf-8", - ) - - assert criterion_dim_plot._read_estimate(estimates, "median") == (5.0, 1.0, 4.0) - - def test_row_rejects_invalid_dimension_and_times() -> None: with pytest.raises(ValueError, match="dimension must be positive"): criterion_dim_plot.Row( @@ -643,7 +570,7 @@ def test_row_rejects_invalid_dimension_and_times() -> None: fa_hi=1.0, ) - with pytest.raises(ValueError, match="la_time must be finite and positive"): + with pytest.raises(ValueError, match="finite positive number"): criterion_dim_plot.Row( dim=2, la_time=float("inf"), @@ -890,21 +817,21 @@ def test_provenance_helpers_treat_timeout_and_os_error_as_unavailable( def fail_command(*_args: object, **_kwargs: object) -> SimpleNamespace: raise failure - monkeypatch.setattr(criterion_dim_plot, "run_git_command", fail_command) + monkeypatch.setattr(criterion_dim_plot, "run_command", fail_command) assert criterion_dim_plot._git_value(tmp_path, ["rev-parse", "HEAD"]) == "unavailable" git_clean, _status_digest = criterion_dim_plot._git_status_metadata(tmp_path) assert git_clean is None - monkeypatch.setattr(criterion_dim_plot, "run_safe_command", fail_command) + monkeypatch.setattr(criterion_dim_plot, "run_command", fail_command) assert criterion_dim_plot._rustc_version(tmp_path) == "unavailable" def _mock_publication_environment(root: Path, monkeypatch: pytest.MonkeyPatch) -> None: - def fake_run_git(args: list[str], **_kwargs: object) -> SimpleNamespace: + def fake_run_git(_command: str, args: list[str], **_kwargs: object) -> SimpleNamespace: return SimpleNamespace(stdout=f"{_TEST_COMMIT}\n") monkeypatch.setattr(criterion_dim_plot, "_repo_root", lambda: root) - monkeypatch.setattr(criterion_dim_plot, "run_git_command", fake_run_git) + monkeypatch.setattr(criterion_dim_plot, "run_command", fake_run_git) def test_main_publication_fails_closed_when_git_commit_is_unavailable( @@ -922,7 +849,7 @@ def unavailable_git(*_args: object, **_kwargs: object) -> SimpleNamespace: message = "git unavailable" raise OSError(message) - monkeypatch.setattr(criterion_dim_plot, "run_git_command", unavailable_git) + monkeypatch.setattr(criterion_dim_plot, "run_command", unavailable_git) assert criterion_dim_plot.main(["--update-readme"]) == 2 assert "current checkout commit 'unavailable'" in capsys.readouterr().err @@ -1045,7 +972,7 @@ def test_main_publication_rejects_stale_measurement_identity( manifest = tmp_path / "Cargo.toml" manifest.write_text(manifest.read_text(encoding="utf-8").replace("0.4.6", "0.4.7"), encoding="utf-8") else: - monkeypatch.setattr(criterion_dim_plot, "run_git_command", lambda *_args, **_kwargs: SimpleNamespace(stdout="b" * 40)) + monkeypatch.setattr(criterion_dim_plot, "run_command", lambda *_args, **_kwargs: SimpleNamespace(stdout="b" * 40)) assert criterion_dim_plot.main(["--update-readme"]) == 2 assert readme.read_text(encoding="utf-8") == original diff --git a/scripts/tests/test_justfile_discoverability.py b/scripts/tests/test_justfile_discoverability.py index 94f5c41..f29e1e3 100644 --- a/scripts/tests/test_justfile_discoverability.py +++ b/scripts/tests/test_justfile_discoverability.py @@ -3,18 +3,18 @@ import json import re import shlex -import shutil -import subprocess from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any import yaml +from research_repo_tools.just_inspect import dry_run, inspect_justfile from research_repo_tools.process import run_command -from benchmark_process import run_safe_command - REPO_ROOT = Path(__file__).resolve().parents[2] +if TYPE_CHECKING: + import subprocess + def run_just( *args: str, @@ -22,24 +22,12 @@ def run_just( env: dict[str, str] | None = None, ) -> subprocess.CompletedProcess[str]: """Run the repository's installed Just executable without a shell.""" - executable = shutil.which("just") - assert executable is not None - return subprocess.run( # noqa: S603 - executable is resolved; arguments are fixed by tests. - [executable, *args], - cwd=REPO_ROOT, - check=check, - capture_output=True, - encoding="utf-8", - env=env, - ) + return run_command("just", args, cwd=REPO_ROOT, check=check, env=env) def just_recipes() -> dict[str, dict[str, Any]]: """Return parsed recipe metadata from the pinned Just executable.""" - result = run_just("--dump", "--dump-format", "json") - recipes = json.loads(result.stdout)["recipes"] - assert isinstance(recipes, dict) - return recipes + return inspect_justfile(REPO_ROOT).recipes def test_exact_benchmark_package_excludes_peer_libraries() -> None: @@ -66,11 +54,11 @@ def test_exact_benchmark_package_excludes_peer_libraries() -> None: def test_release_runs_each_suite_once_and_reuses_peer_measurements() -> None: - baseline = run_just("--dry-run", "bench-save-baseline", "v0.4.5", "all") + baseline = dry_run(REPO_ROOT, "bench-save-baseline", ["v0.4.5", "all"]) # Execute the real recipe's shell branching while replacing only Cargo. runner = run_just("--evaluate", "_run").stdout.strip() script = 'cargo() { printf "%s\\n" "$*"; }\n' + baseline.stderr.replace(runner + " ", "") - baseline_run = run_safe_command("bash", ["--noprofile", "--norc", "-euc", script], cwd=REPO_ROOT) + baseline_run = run_command("bash", ["--noprofile", "--norc", "-euc", script], cwd=REPO_ROOT) baseline_commands = [shlex.split(line) for line in baseline_run.stdout.splitlines()] recipes = just_recipes() current = recipes["bench-latest"] @@ -79,7 +67,7 @@ def test_release_runs_each_suite_once_and_reuses_peer_measurements() -> None: for dependency in current["dependencies"]: recipe = recipes[dependency["recipe"]] assert not recipe["dependencies"] - commands = run_just("--dry-run", dependency["recipe"]).stderr + commands = dry_run(REPO_ROOT, dependency["recipe"]).stderr current_commands.extend(shlex.split(line.removeprefix(runner + " ")) for line in commands.splitlines()) assert len(baseline_commands) == len(current_commands) == 2 diff --git a/scripts/tests/test_maintenance_integration.py b/scripts/tests/test_maintenance_integration.py index e1af1fd..2f6e261 100644 --- a/scripts/tests/test_maintenance_integration.py +++ b/scripts/tests/test_maintenance_integration.py @@ -9,8 +9,8 @@ import pytest from research_repo_tools.cli import main - -from benchmark_process import run_git_command, run_safe_command +from research_repo_tools.just_inspect import dry_run +from research_repo_tools.process import run_command REPO_ROOT = Path(__file__).resolve().parents[2] @@ -18,7 +18,7 @@ @pytest.fixture def consumer(tmp_path: Path) -> Path: """Copy release inputs and real documentation, without live repository state.""" - files = run_git_command(["ls-files", "-co", "--exclude-standard", "-z"], cwd=REPO_ROOT).stdout.split("\0") + files = run_command("git", ["--no-pager", "ls-files", "-co", "--exclude-standard", "-z"], cwd=REPO_ROOT).stdout.split("\0") metadata = {"Cargo.toml", "Cargo.lock", "pyproject.toml", "uv.lock", "CITATION.cff", ".python-version", "rust-toolchain.toml"} for name in files: source = REPO_ROOT / name @@ -103,7 +103,7 @@ def test_update_recipe_runs_the_installed_cli_with_preview_arguments(consumer: P ) before = snapshot(consumer) args = release_args(consumer, "--dry-run")[4:] - result = run_safe_command( + result = run_command( "just", ["--justfile", str(wrapper), "--working-directory", str(consumer), "update-version", *args], cwd=consumer, @@ -113,16 +113,11 @@ def test_update_recipe_runs_the_installed_cli_with_preview_arguments(consumer: P assert snapshot(consumer) == before -def test_markdown_recipe_uses_the_published_checker_and_retains_character_limit(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - """Verify the consumer boundary with Unicode, tables, spaces, and path quoting.""" - path = tmp_path / "document with spaces.md" - path.write_text("é" * 160 + "\n|" + "x" * 200 + "\n", encoding="utf-8") - assert main(["--root", str(tmp_path), "docs", "check-lines", str(path)]) == 0 - path.write_text("é" * 161 + "\n", encoding="utf-8") - assert main(["--root", str(tmp_path), "docs", "check-lines", str(path)]) == 1 - assert "line length 161 exceeds 160" in capsys.readouterr().err - dry_run = run_safe_command("just", ["--dry-run", "markdown-check"], cwd=REPO_ROOT) - assert 'research-repo-tools docs check-lines "${files[@]}"' in dry_run.stderr +def test_markdown_recipe_uses_the_published_checker() -> None: + """Retain the consumer's tracked-file selection and quoted path forwarding.""" + result = dry_run(REPO_ROOT, "markdown-check") + assert 'research-repo-tools docs check-lines "${files[@]}"' in result.stderr + assert "git ls-files -co --exclude-standard -z" in result.stderr def test_semgrep_adapter_uses_real_consumer_rules_and_fixtures() -> None: @@ -131,5 +126,5 @@ def test_semgrep_adapter_uses_real_consumer_rules_and_fixtures() -> None: assert config["config"] == "semgrep.yaml" assert config["fixtures"] == "tests/semgrep" assert config["namespace"] == "la-stack." - result = run_safe_command("just", ["--dry-run", "semgrep-test"], cwd=REPO_ROOT) + result = dry_run(REPO_ROOT, "semgrep-test") assert "research-repo-tools semgrep check-fixtures" in result.stderr diff --git a/scripts/tests/test_performance_artifacts.py b/scripts/tests/test_performance_artifacts.py index 25cb867..f3f8683 100644 --- a/scripts/tests/test_performance_artifacts.py +++ b/scripts/tests/test_performance_artifacts.py @@ -478,23 +478,6 @@ def test_bundle_rejects_duplicate_benchmark_keys() -> None: PerformanceBundle(context=bundle.context, rows=(bundle.rows[0], bundle.rows[0])) -@pytest.mark.parametrize("value", [0.0, -1.0, float("inf"), float("nan")]) -def test_timing_rejects_non_positive_or_non_finite_values(value: float) -> None: - with pytest.raises(ValueError, match="finite positive number"): - TimingEstimate(median_ns=value, ci_lower_ns=1.0, ci_upper_ns=2.0) - - -def test_timing_accepts_ordered_bootstrap_interval_that_excludes_point_estimate() -> None: - estimate = TimingEstimate(median_ns=12.0, ci_lower_ns=9.0, ci_upper_ns=11.0) - - assert estimate.median_ns == 12.0 - - -def test_timing_rejects_reversed_interval() -> None: - with pytest.raises(ValueError, match="lower bound exceeds upper bound"): - TimingEstimate(median_ns=10.0, ci_lower_ns=11.0, ci_upper_ns=9.0) - - def test_artifact_context_freezes_nested_provenance() -> None: context = _context() diff --git a/scripts/tests/test_performance_workflow.py b/scripts/tests/test_performance_workflow.py index 24f246e..3f17277 100644 --- a/scripts/tests/test_performance_workflow.py +++ b/scripts/tests/test_performance_workflow.py @@ -3,7 +3,10 @@ import hashlib import io import json +import os +import shlex import shutil +import sys import tarfile from dataclasses import replace from pathlib import Path @@ -12,7 +15,8 @@ import pytest from research_repo_tools.common_measurement import CommonHarnessPlan, measure_prepared_pair from research_repo_tools.complete_runs import RunSeries, run_identity, serialize_run -from research_repo_tools.process import run_git_command +from research_repo_tools.just_inspect import dry_run +from research_repo_tools.process import run_command from research_repo_tools.release_pairs import ReleasePair import archive_performance as workflow @@ -20,7 +24,6 @@ import performance_runs as policy from bench_compare import render_release_artifacts from performance_artifacts import ArtifactPaths -from performance_phase import commands from release_baseline import required_report_ids if TYPE_CHECKING: @@ -30,7 +33,7 @@ PAIR = ReleasePair("v0.4.6", "v0.4.5") DRIVER = """import json, os, sys from pathlib import Path -action, suite, phase = sys.argv[1:] +action, phase = sys.argv[1:] fault = os.environ.get("FIXTURE_FAULT", "") if action == "gate": if fault == "gate": @@ -64,15 +67,28 @@ def prepared(tmp_path: Path, fault: str = "") -> tuple[Path, Path, CommonHarness root.mkdir(parents=True) (root / "src").mkdir() (root / "src/lib.rs").write_text(f"// {phase}\n", encoding="utf-8") - for pattern in policy.HARNESS: + for pattern in policy.configuration(ROOT).harness: for source in ROOT.glob(pattern): destination = root / source.relative_to(ROOT) destination.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(source, destination) - (root / "scripts/performance_phase.py").write_text(f"INVENTORIES = {inventories!r}\n" + DRIVER, encoding="utf-8") - run_git_command(["init", "--quiet"], cwd=root) - run_git_command( + (root / "tooling/fixture.py").write_text(f"INVENTORIES = {inventories!r}\n" + DRIVER, encoding="utf-8") + (root / "tooling/performance.just").write_text( + "\n".join( + f"{recipe}:\n {shlex.quote(sys.executable)} tooling/fixture.py {action} {phase}\n" + for recipe, action, phase in (("gate", "gate", "current"), ("baseline-all", "measure", "baseline"), ("current-all", "measure", "current")) + ), + encoding="utf-8", + ) + config = (root / policy.MEASUREMENT_CONFIG).read_text(encoding="utf-8") + (root / policy.MEASUREMENT_CONFIG).write_text( + config.replace(' "tooling/performance.just",', ' "tooling/performance.just", "tooling/fixture.py",'), encoding="utf-8" + ) + run_command("git", ["--no-pager", "init", "--quiet"], cwd=root) + run_command( + "git", [ + "--no-pager", "-c", "user.name=Fixture", "-c", @@ -88,14 +104,16 @@ def prepared(tmp_path: Path, fault: str = "") -> tuple[Path, Path, CommonHarness cwd=root, ) baseline, current = tmp_path / "baseline", tmp_path / "current" - plan = policy.declared_plan(PAIR, "all", "release-signal", inventories, baseline_env=(("FIXTURE_FAULT", fault),), current_env=()) + plan = policy.declared_plan( + PAIR, "all", "release-signal", inventories, baseline_env=(("FIXTURE_FAULT", fault),), current_env=(), config=policy.configuration(current) + ) return baseline, current, plan def measured(tmp_path: Path) -> tuple[Path, Evidence]: baseline, current, plan = prepared(tmp_path) evidence = measure_prepared_pair(baseline, current, plan, PAIR, working_tree=True) - (current / ".config").mkdir(exist_ok=True) + (current / "tooling").mkdir(exist_ok=True) shutil.copyfile(ROOT / policy.REPORT_CONFIG, current / policy.REPORT_CONFIG) return current, evidence @@ -129,18 +147,20 @@ def test_declared_gates_and_completeness_fail_closed(tmp_path: Path, fault: str) def test_shared_retention_repeated_runs_and_offline_report(tmp_path: Path) -> None: root, first = measured(tmp_path) + config = root / policy.REPORT_CONFIG + config.write_text(config.read_text(encoding="utf-8").replace("docs/performance-runs", "docs/performance-history"), encoding="utf-8") stem = root / "target/bench-reports/performance" workflow.save_scratch(root, stem, first) workflow.promote(root, stem) first_report = (root / "docs/performance.md").read_bytes() - retained = root / policy.ARCHIVE / "runs" / run_identity(first) + retained = root / policy.archive_directory(root) / "runs" / run_identity(first) original = {path: path.read_bytes() for path in retained.iterdir()} # A second valid measurement with the same release pair has new provenance. sources = tuple((phase, replace(source, context=(*source.context, ("experiment", "second")))) for phase, source in first.sources) second = replace(first, sources=sources) workflow.save_scratch(root, stem, second) workflow.promote(root, stem) - assert len(list((root / policy.ARCHIVE / "runs").iterdir())) == 2 + assert len(list((root / policy.archive_directory(root) / "runs").iterdir())) == 2 assert {path: path.read_bytes() for path in original} == original assert (root / "docs/archive/performance/v0.4.6-vs-v0.4.5.md").read_bytes() == first_report shutil.rmtree(root / "target") @@ -148,7 +168,7 @@ def test_shared_retention_repeated_runs_and_offline_report(tmp_path: Path) -> No before = (root / "docs/performance.md").read_bytes() workflow.promote(root, stem) assert (root / "docs/performance.md").read_bytes() == before - (root / policy.ARCHIVE / "latest.json").write_text("{}", encoding="utf-8") + (root / policy.archive_directory(root) / "latest.json").write_text("{}", encoding="utf-8") with pytest.raises(ValueError, match=r"latest|pointer|fields|schema"): policy.retained_run(root, stem) @@ -201,15 +221,91 @@ def test_legacy_bytes_hashes_and_report_are_unchanged() -> None: def test_native_command_policy_and_compatibility_adapter() -> None: - assert all("--locked" in command and "--noplot" in command for command in commands("all", "baseline")) - assert "la_stack" in commands("vs_linalg", "current")[0] - assert "la_stack" not in commands("vs_linalg", "baseline")[0] + justfile = Path("tooling/performance.just") + baseline = dry_run(ROOT, "baseline-all", justfile=justfile).stderr + current = dry_run(ROOT, "current-all", justfile=justfile).stderr + assert baseline.count("cargo bench --locked") == current.count("cargo bench --locked") == 2 + assert baseline.count("--noplot") == current.count("--noplot") == 2 + assert "la_stack" in current + assert "la_stack" not in baseline + assert "--list" not in baseline + assert dry_run(ROOT, "baseline-all", ["list"], justfile=justfile).stderr.count("--list") == 2 + gate = dry_run(ROOT, "gate", justfile=justfile).stderr + assert "--test vs_linalg_inputs --test exact_bench_config" in gate environment = dict(policy.phase_environment(ROOT, "la_stack_pre_rational_input_api")) assert "--cfg=la_stack_pre_rational_input_api" in environment.get("CARGO_ENCODED_RUSTFLAGS", environment.get("RUSTFLAGS", "")) assert environment["CARGO_TARGET_DIR"] == "target" assert environment["CRITERION_HOME"] == "target/criterion" +def test_native_inventory_and_phase_commands_use_the_measured_checkout(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Exercise the configured Justfile, stubbing only the expensive Cargo boundary.""" + root = tmp_path / "checkout with spaces" + shutil.copytree(ROOT / "tooling", root / "tooling") + shutil.copyfile(ROOT / "rust-toolchain.toml", root / "rust-toolchain.toml") + binary = tmp_path / "bin" + binary.mkdir() + cargo = binary / "cargo" + cargo.write_text( + "#!/bin/sh\n" + 'printf "%s|%s\\n" "$PWD" "$*" >> "$CARGO_RECORD"\n' + 'case "$*" in\n' + ' *"--bench exact"*) printf "exact_d2/det: benchmark\\n" ;;\n' + ' *"la_stack --noplot"*) printf "d2/la_stack_lu_solve: benchmark\\n" ;;\n' + ' *"--bench vs_linalg"*) printf "%s: benchmark\\n" d2/la_stack_lu_solve d2/nalgebra_lu_solve d2/faer_lu_solve ;;\n' + "esac\n", + encoding="utf-8", + ) + cargo.chmod(0o755) + record = tmp_path / "commands.txt" + monkeypatch.setenv("PATH", f"{binary}{os.pathsep}{os.environ['PATH']}") + monkeypatch.setenv("CARGO_RECORD", str(record)) + plan = policy.measurement_plan(root, PAIR, "all", "all-benches", tmp_path / "inventory") + assert plan.current.policy.expected == ("d2/la_stack_lu_solve", "exact_d2/det") + assert set(plan.baseline.policy.expected) == {"d2/la_stack_lu_solve", "d2/nalgebra_lu_solve", "d2/faer_lu_solve", "exact_d2/det"} + for command in (plan.baseline.gate, plan.baseline.command, plan.current.command): + run_command(command[0], command[1:], cwd=root) + lines = record.read_text(encoding="utf-8").splitlines() + assert len(lines) == 9 + assert all(line.startswith(f"{root}|") and "--locked" in line for line in lines) + assert all("--list" in line for line in lines[:4]) + assert all("--list" not in line for line in lines[4:]) + assert "test --locked --workspace --features bench,exact --test vs_linalg_inputs --test exact_bench_config" in lines[4] + + +def test_pre_just_run_remains_readable_without_its_python_driver(tmp_path: Path) -> None: + _, evidence = measured(tmp_path) + run = policy.validate_scientific_run(evidence) + cargo = { + "baseline": [ + ["cargo", "bench", "--locked", "-p", "la-stack-comparison", "--features", "bench", "--bench", "vs_linalg", "--", "--noplot"], + ["cargo", "bench", "--locked", "--features", "bench,exact", "--bench", "exact", "--", "--noplot"], + ], + "current": [ + ["cargo", "bench", "--locked", "-p", "la-stack-comparison", "--features", "bench", "--bench", "vs_linalg", "--", "la_stack", "--noplot"], + ["cargo", "bench", "--locked", "--features", "bench,exact", "--bench", "exact", "--", "--noplot"], + ], + } + sources = [] + for phase, source in evidence.sources: + context = dict(source.context) + del context["phase-driver"] + del context["tool.just"] + context.update((f"{name}-cargo-commands", json.dumps(commands)) for name, commands in cargo.items()) + for key, action in (("command", "measure"), ("gate-command", "gate")): + context[key] = json.dumps(["/retired/venv/python", "scripts/performance_phase.py", action, "all", phase]) + sources.append((phase, replace(source, context=tuple(context.items())))) + legacy = replace( + evidence, + sources=tuple(sources), + payload=serialize_run(replace(run, compatible=tuple(field for field in run.compatible if field != "context.tool.just"))), + ) + assert b"not rerun in the current phase" in policy.render_scientific_report(legacy) + bad = replace(sources[0][1], context=tuple((key, "[]" if key == "baseline-cargo-commands" else value) for key, value in sources[0][1].context)) + with pytest.raises(ValueError, match="consumer policy"): + policy.validate_scientific_run(replace(legacy, sources=(("baseline", bad), sources[1]))) + + def test_local_explicit_labels_and_release_eligibility() -> None: assert workflow.select_pair(ROOT, "explicit", "v0.4.6", "v0.4.6", local=True) == ReleasePair("v0.4.6", "v0.4.6") with pytest.raises(ValueError, match="must differ"): @@ -243,8 +339,8 @@ def test_reference_phase_and_same_version_policy(tmp_path: Path) -> None: # current selection allows a narrow all-benches same-version comparison. inventories["current"] = tuple(name for name in inventories["current"] if name in inventories["baseline"]) driver = f"INVENTORIES = {inventories!r}\n" + DRIVER - (current / "scripts/performance_phase.py").write_text(driver, encoding="utf-8") - plan = policy.declared_plan(pair, "all", "all-benches", inventories, baseline_env=(), current_env=()) + (current / "tooling/fixture.py").write_text(driver, encoding="utf-8") + plan = policy.declared_plan(pair, "all", "all-benches", inventories, baseline_env=(), current_env=(), config=policy.configuration(current)) same = measure_prepared_pair(baseline, current, plan, pair, working_tree=True) policy.validate_scientific_run(same) with pytest.raises(ValueError, match="same-version"): @@ -272,7 +368,7 @@ def test_publication_io_failure_preserves_all_published_outputs(tmp_path: Path, def fail_once(source: Path, destination: Path) -> Path: nonlocal failed - if destination == root / policy.ARCHIVE / "latest.json" and not failed: + if destination == root / policy.archive_directory(root) / "latest.json" and not failed: failed = True msg = "injected publication failure" raise OSError(msg) diff --git a/scripts/tests/test_release_baseline.py b/scripts/tests/test_release_baseline.py index a5fe6c3..7410eb4 100644 --- a/scripts/tests/test_release_baseline.py +++ b/scripts/tests/test_release_baseline.py @@ -165,9 +165,9 @@ def host_write_text(path: Path, data: str, encoding: str | None = None, errors: return write_text(path, data, encoding=encoding, errors=errors, newline=host_newline if newline is None else newline) def run(command: str, args: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: - assert command == "cargo" + assert command == "just" calls.append(args) - suite = args[args.index("--bench") + 1] + suite = args[-2].removeprefix("baseline-") return subprocess.CompletedProcess(args, 0, stdout="\n".join(f"{name}: benchmark" for name in IDS[suite])) monkeypatch.setattr(release_baseline, "run_command", run) @@ -177,8 +177,8 @@ def run(command: str, args: list[str], **kwargs: object) -> subprocess.Completed release_baseline.discover(tmp_path, manifest, tmp_path / "criterion") assert manifest.read_bytes() == (json.dumps(IDS, indent=2) + "\n").encode("utf-8") assert calls == [ - ["bench", "--locked", "-p", "la-stack-comparison", "--features", "bench", "--bench", "vs_linalg", "--", "--list"], - ["bench", "--locked", "--features", "bench,exact", "--bench", "exact", "--", "--list"], + ["--justfile", "tooling/performance.just", "--working-directory", ".", "baseline-vs_linalg", "list"], + ["--justfile", "tooling/performance.just", "--working-directory", ".", "baseline-exact", "list"], ] diff --git a/scripts/tests/test_review_integration.py b/scripts/tests/test_review_integration.py index 68d9265..edfd669 100644 --- a/scripts/tests/test_review_integration.py +++ b/scripts/tests/test_review_integration.py @@ -9,9 +9,8 @@ from pathlib import Path import pytest -from research_repo_tools.cli import main - -from benchmark_process import run_git_command, run_safe_command +from research_repo_tools.just_inspect import inspect_justfile +from research_repo_tools.process import run_command REPO_ROOT = Path(__file__).resolve().parents[2] @@ -26,13 +25,15 @@ def consumer(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: (root / "recipes.just").write_text(f"import '{(REPO_ROOT / 'justfile').as_posix()}'\n", encoding="utf-8") monkeypatch.setenv("GIT_CONFIG_GLOBAL", os.devnull) monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1") - run_git_command(["init", "--quiet", "--initial-branch=main"], cwd=root) - run_git_command( - ["-c", "user.name=Review fixture", "-c", "user.email=fixture@example.invalid", "commit", "--quiet", "--allow-empty", "-m", "fixture"], cwd=root + run_command("git", ["--no-pager", "init", "--quiet", "--initial-branch=main"], cwd=root) + run_command( + "git", + ["--no-pager", "-c", "user.name=Review fixture", "-c", "user.email=fixture@example.invalid", "commit", "--quiet", "--allow-empty", "-m", "fixture"], + cwd=root, ) - head = run_git_command(["rev-parse", "HEAD"], cwd=root).stdout.strip() - run_git_command(["update-ref", "refs/remotes/origin/main", head], cwd=root) - run_git_command(["remote", "add", "origin", str(root)], cwd=root) + head = run_command("git", ["--no-pager", "rev-parse", "HEAD"], cwd=root).stdout.strip() + run_command("git", ["--no-pager", "update-ref", "refs/remotes/origin/main", head], cwd=root) + run_command("git", ["--no-pager", "remote", "add", "origin", str(root)], cwd=root) return root @@ -62,7 +63,7 @@ def stub(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: def recipe(root: Path, *args: str) -> subprocess.CompletedProcess[str]: """Execute the imported adapter with the installed locked package.""" - return run_safe_command( + return run_command( "just", ["--justfile", str(root / "recipes.just"), "--working-directory", str(root), *args], cwd=root, @@ -77,7 +78,7 @@ def recipe(root: Path, *args: str) -> subprocess.CompletedProcess[str]: ) def test_actual_recipes_forward_scopes_and_consumer_instructions(consumer: Path, stub: Path, args: tuple[str, ...], scope: str) -> None: """Both scopes use structured output, untracked inputs, and consumer instructions.""" - before = run_git_command(["show-ref"], cwd=consumer).stdout + before = run_command("git", ["--no-pager", "show-ref"], cwd=consumer).stdout result = recipe(consumer, *args) assert result.returncode == 0, result.stderr assert json.loads(result.stdout) == [ @@ -90,13 +91,7 @@ def test_actual_recipes_forward_scopes_and_consumer_instructions(consumer: Path, str(consumer / ".coderabbit.yaml"), ] assert "fixture diagnostic" in result.stderr - assert run_git_command(["show-ref"], cwd=consumer).stdout == before - - -@pytest.mark.parametrize("args", [("review", "HEAD"), ("review-uncommitted",)]) -def test_local_scopes_do_not_require_a_remote(consumer: Path, stub: Path, args: tuple[str, ...]) -> None: - run_git_command(["remote", "remove", "origin"], cwd=consumer) - assert recipe(consumer, *args).returncode == 0 + assert run_command("git", ["--no-pager", "show-ref"], cwd=consumer).stdout == before @pytest.mark.parametrize("status", [7, 130]) @@ -108,23 +103,6 @@ def test_service_failures_and_interruption_status_reach_just(consumer: Path, stu assert "fixture diagnostic" in result.stderr -@pytest.mark.parametrize("problem", ["missing", "stale", "remote"]) -def test_default_base_fails_closed_before_service_invocation(consumer: Path, stub: Path, problem: str) -> None: - if problem == "missing": - run_git_command(["update-ref", "-d", "refs/remotes/origin/main"], cwd=consumer) - elif problem == "stale": - run_git_command( - ["-c", "user.name=Review fixture", "-c", "user.email=fixture@example.invalid", "commit", "--quiet", "--allow-empty", "-m", "advance"], cwd=consumer - ) - else: - run_git_command(["remote", "set-url", "origin", str(consumer / "missing-remote")], cwd=consumer) - result = recipe(consumer, "review") - assert result.returncode != 0 - assert result.stdout == "" - assert "fixture diagnostic" not in result.stderr - assert ("Cannot verify origin/main" if problem == "remote" else "git fetch origin") in result.stderr - - def test_shell_metacharacters_are_passed_as_one_rejected_base(consumer: Path, stub: Path) -> None: result = recipe(consumer, "review", "HEAD'; echo INJECTION_EXECUTED; #") assert result.returncode != 0 @@ -132,26 +110,10 @@ def test_shell_metacharacters_are_passed_as_one_rejected_base(consumer: Path, st assert result.stdout == "" -@pytest.mark.parametrize("name", ["AGENTS.md", ".coderabbit.yaml"]) -def test_missing_consumer_instructions_stop_review(consumer: Path, stub: Path, name: str) -> None: - (consumer / name).unlink() - result = recipe(consumer, "review-uncommitted") - assert result.returncode != 0 - assert "review requires" in result.stderr - assert result.stdout == "" - - -def test_missing_cli_reports_explicit_installation(consumer: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: - original = shutil.which - monkeypatch.setattr(shutil, "which", lambda command, *args, **kwargs: None if Path(command).name == "coderabbit" else original(command, *args, **kwargs)) - assert main(["--root", str(consumer), "review", "uncommitted"]) == 1 - assert "Install and authenticate it explicitly" in capsys.readouterr().err - - def test_reviews_are_discoverable_and_outside_routine_gates() -> None: - surface = json.loads(run_safe_command("just", ["--dump", "--dump-format", "json"], cwd=REPO_ROOT).stdout)["recipes"] - help_text = run_safe_command("just", [], cwd=REPO_ROOT).stdout - listed = run_safe_command("just", ["--list"], cwd=REPO_ROOT).stdout + surface = inspect_justfile(REPO_ROOT).recipes + help_text = run_command("just", [], cwd=REPO_ROOT).stdout + listed = run_command("just", ["--list"], cwd=REPO_ROOT).stdout for name in ("review", "review-uncommitted"): assert name in help_text assert name in listed diff --git a/scripts/tests/test_security_integration.py b/scripts/tests/test_security_integration.py index e622beb..58eabba 100644 --- a/scripts/tests/test_security_integration.py +++ b/scripts/tests/test_security_integration.py @@ -4,8 +4,7 @@ from pathlib import Path import pytest - -from benchmark_process import run_safe_command +from research_repo_tools.process import run_command REPO_ROOT = Path(__file__).resolve().parents[2] @@ -28,7 +27,7 @@ def test_historical_alias_exception_is_limited_to_its_assignment_and_path(tmp_pa token = hashlib.sha256(b"public Gitleaks regression fixture").hexdigest() contents += "api_" + f'key = "{token}"\n' source.write_text(contents, encoding="utf-8") - result = run_safe_command( + result = run_command( "uv", [ "run", diff --git a/scripts/tests/test_toolchain_integration.py b/scripts/tests/test_toolchain_integration.py index 1cd8f9b..fc1ea04 100644 --- a/scripts/tests/test_toolchain_integration.py +++ b/scripts/tests/test_toolchain_integration.py @@ -12,8 +12,7 @@ import pytest from research_repo_tools.cli import main - -from benchmark_process import run_safe_command +from research_repo_tools.process import run_command if TYPE_CHECKING: import subprocess @@ -76,7 +75,7 @@ def consumer(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: def invoke(root: Path, recipe: str) -> tuple[subprocess.CompletedProcess[str], list[list[str]]]: """Run Just's actual dependency sequencing without live upgrades.""" - result = run_safe_command("just", ["--justfile", str(root / "recipes.just"), "--working-directory", str(root), recipe], cwd=root, check=False) + result = run_command("just", ["--justfile", str(root / "recipes.just"), "--working-directory", str(root), recipe], cwd=root, check=False) calls = [cast("list[str]", json.loads(line)) for line in (root / "calls.jsonl").read_text(encoding="utf-8").splitlines()] return result, calls @@ -103,7 +102,7 @@ def test_declared_tools_are_verified_and_cargo_edit_runs_from_managed_store(caps assert main(["--root", str(REPO_ROOT), "toolchain", "check", "--json"]) == 0 report = json.loads(capsys.readouterr().out) assert all(status["ok"] for status in report) - result = run_safe_command("uv", [*CHECKED, "cargo", "upgrade", "--version"], cwd=REPO_ROOT) + result = run_command("uv", [*CHECKED, "cargo", "upgrade", "--version"], cwd=REPO_ROOT) declared = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8"))["tool"]["research-repo-tools"]["toolchain"]["cargo"] names = {"cargo-edit": "cargo-edit-upgrade", "taplo-cli": "taplo"} for package, expected in declared.items(): @@ -115,9 +114,7 @@ def test_declared_tools_are_verified_and_cargo_edit_runs_from_managed_store(caps assert directory.parent.parent.name == package assert result.stdout.strip() == f"cargo-edit-upgrade {declared['cargo-edit']}" # The checked runner supplies the same managed Rust to subprocesses, including native Python builds. - result = run_safe_command( - "uv", [*CHECKED, "uv", "run", "--locked", "--no-sync", "python", "-c", "import shutil; print(shutil.which('cargo'))"], cwd=REPO_ROOT - ) + result = run_command("uv", [*CHECKED, "uv", "run", "--locked", "--no-sync", "python", "-c", "import shutil; print(shutil.which('cargo'))"], cwd=REPO_ROOT) rustup = next(status for status in report if status["name"] == "rustup") assert Path(result.stdout.strip()).parent == Path(rustup["path"]).parent @@ -168,13 +165,11 @@ def test_python_update_preserves_shared_pin_and_tools_with_explicit_dev_sync(tmp assert after["dependency-groups"]["tooling"] == groups["tooling"] assert after["tool"]["research-repo-tools"] == before["tool"]["research-repo-tools"] assert after["tool"]["uv"] == before["tool"]["uv"] - run_safe_command("uv", ["lock", "--upgrade"], cwd=root) + run_command("uv", ["lock", "--upgrade"], cwd=root) lock = tomllib.loads((root / "uv.lock").read_text(encoding="utf-8")) packages = {package["name"]: package["version"] for package in lock["package"]} assert packages["research-repo-tools"] == "0.1.8" assert packages["ruff"] == "99.0.0" - run_safe_command("uv", ["sync", "--locked", "--group", "dev"], cwd=root) - installed = run_safe_command( - "uv", ["run", "--locked", "--no-sync", "python", "-c", "from importlib.metadata import version; print(version('ruff'))"], cwd=root - ) + run_command("uv", ["sync", "--locked", "--group", "dev"], cwd=root) + installed = run_command("uv", ["run", "--locked", "--no-sync", "python", "-c", "from importlib.metadata import version; print(version('ruff'))"], cwd=root) assert installed.stdout.strip() == "99.0.0" diff --git a/semgrep.yaml b/semgrep.yaml index b4233d9..4014927 100644 --- a/semgrep.yaml +++ b/semgrep.yaml @@ -1042,8 +1042,6 @@ rules: - "/tests/semgrep/scripts/tests/**" patterns: - pattern-either: - - pattern: run_safe_command("git", ..., input=$INPUT, ...) - - pattern: benchmark_process.run_safe_command("git", ..., input=$INPUT, ...) - pattern: research_repo_tools.process.run_command("git", ..., input=$INPUT, ...) - pattern: subprocess.run(["git", ...], ..., input=$INPUT, ...) - pattern: subprocess.Popen(["git", ...], ..., stdin=subprocess.PIPE, ...) @@ -1247,7 +1245,7 @@ rules: languages: - python severity: WARNING - message: "Use research_repo_tools.process runners or benchmark_process adapters instead of calling subprocess.run directly." + message: "Use research_repo_tools.process runners instead of calling subprocess.run directly." metadata: category: security rationale: "Support scripts should inherit the repository's subprocess hardening." diff --git a/tests/semgrep/scripts/python_portability.py b/tests/semgrep/scripts/python_portability.py index 3f152d9..db7cb4d 100644 --- a/tests/semgrep/scripts/python_portability.py +++ b/tests/semgrep/scripts/python_portability.py @@ -11,18 +11,11 @@ from research_repo_tools import process as shared_process from research_repo_tools.process import run_command, run_git_bytes as git_input -import benchmark_process as utils -from benchmark_process import run_git_command, run_safe_command - if TYPE_CHECKING: from pathlib import Path def git_input_routing(payload: str, argv: list[str]) -> None: - # ruleid: la-stack.python.git-stdin-use-shared-helper - run_safe_command("git", argv, input=payload) - # ruleid: la-stack.python.git-stdin-use-shared-helper - utils.run_safe_command("git", argv, input=payload) # ruleid: la-stack.python.git-stdin-use-shared-helper run_command("git", argv, input=payload) # ruleid: la-stack.python.git-stdin-use-shared-helper @@ -44,13 +37,11 @@ def git_input_routing(payload: str, argv: list[str]) -> None: # ok: la-stack.python.git-stdin-use-shared-helper shared_process.run_git_bytes(argv, input=payload.encode("utf-8")) # ok: la-stack.python.git-stdin-use-shared-helper - run_git_command(argv) + run_command("git", ["--no-pager", *argv]) # ok: la-stack.python.git-stdin-use-shared-helper shared_process.run_command("git", argv, input=None) # Ordinary text input to non-Git commands is outside this rule's scope. # ok: la-stack.python.git-stdin-use-shared-helper - run_safe_command("ruff", ["check", "-"], input=payload) - # ok: la-stack.python.git-stdin-use-shared-helper run_command("ruff", ["check", "-"], input=payload) # ok: la-stack.python.git-stdin-use-shared-helper subprocess.run(["formatter"], input=payload, text=True, encoding="utf-8") diff --git a/tests/semgrep/scripts/tests/python_exceptions.py b/tests/semgrep/scripts/tests/python_exceptions.py index 92e456c..c3c58af 100644 --- a/tests/semgrep/scripts/tests/python_exceptions.py +++ b/tests/semgrep/scripts/tests/python_exceptions.py @@ -4,8 +4,6 @@ from research_repo_tools.process import run_command -from benchmark_process import run_safe_command - if TYPE_CHECKING: from pathlib import Path @@ -86,5 +84,3 @@ def direct_subprocess_run() -> None: def shared_process_runners() -> None: # ok: la-stack.python.no-direct-subprocess-run-outside-wrapper run_command("git", ["status"], check=False) - # ok: la-stack.python.no-direct-subprocess-run-outside-wrapper - run_safe_command("git", ["status"], check=False) diff --git a/.config/performance-report.toml b/tooling/performance-report.toml similarity index 100% rename from .config/performance-report.toml rename to tooling/performance-report.toml diff --git a/tooling/performance.just b/tooling/performance.just new file mode 100644 index 0000000..6e8e04a --- /dev/null +++ b/tooling/performance.just @@ -0,0 +1,20 @@ +# Captured native commands run in the measured checkout with its phase environment. +# Pass "list" to discover Criterion IDs without collecting samples. + +baseline-all mode="measure": (baseline-vs_linalg mode) (baseline-exact mode) + +baseline-exact mode="measure": + cargo bench --locked --features bench,exact --bench exact -- --noplot {{ if mode == "list" { "--list" } else { "" } }} + +baseline-vs_linalg mode="measure": + cargo bench --locked -p la-stack-comparison --features bench --bench vs_linalg -- --noplot {{ if mode == "list" { "--list" } else { "" } }} + +current-all mode="measure": (current-vs_linalg mode) (current-exact mode) + +current-exact mode="measure": (baseline-exact mode) + +current-vs_linalg mode="measure": + cargo bench --locked -p la-stack-comparison --features bench --bench vs_linalg -- la_stack --noplot {{ if mode == "list" { "--list" } else { "" } }} + +gate: + cargo test --locked --workspace --features bench,exact --test vs_linalg_inputs --test exact_bench_config diff --git a/tooling/performance.toml b/tooling/performance.toml new file mode 100644 index 0000000..d498b39 --- /dev/null +++ b/tooling/performance.toml @@ -0,0 +1,41 @@ +# Shared measurement inputs; scientific row selection stays in performance_runs. +schema = 1 +command = [ "just", "--justfile", "tooling/performance.just", "--working-directory", ".", "current-all" ] +sources = [ "src/**/*.rs", "Cargo.toml" ] +harness = [ + "benches/**/*.rs", + "benches/comparison/Cargo.toml", + "examples/**/*.rs", + "Cargo.toml", + "Cargo.lock", + "rust-toolchain.toml", + "tests/exact_bench_config.rs", + "tooling/performance.toml", + "tooling/performance.just", +] +timeout = 7200 +compatible = [ + "harness_sha256", + "context.os", + "context.architecture", + "context.cpu", + "context.tool.rustc", + "context.tool.cargo", + "context.tool.just", + "context.dependency.criterion", + "context.dependency.nalgebra", + "context.dependency.faer", +] + +[probes] +rustc = [ "rustc", "-vV" ] +cargo = [ "cargo", "--version" ] +just = [ "just", "--version" ] + +[dependencies] +criterion = "Cargo.lock" +nalgebra = "Cargo.lock" +faer = "Cargo.lock" + +[context] +phase-driver = "just-v1" diff --git a/uv.lock b/uv.lock index 24f9901..624fad7 100644 --- a/uv.lock +++ b/uv.lock @@ -743,11 +743,11 @@ wheels = [ [[package]] name = "pycparser" -version = "3.1" +version = "3.11" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ac/d3/eb1d3bc30dda12f7e69640ae2ac8cb10240b71fb73024ad528b7d2ae73da/pycparser-3.1.tar.gz", hash = "sha256:b3fc6dec06a8b2fefa0ed4ff92285306a5e3be9987bc5603c9edbdc4e492418f", size = 105946, upload-time = "2026-10-08T12:26:34.349Z" } +sdist = { url = "https://files.pythonhosted.org/packages/da/a8/c5fdbeee588bb8ada9458774f43adf1bdd30bd59157055142183e769a024/pycparser-3.11.tar.gz", hash = "sha256:d875f09c3507d00e1aba0eecc6dcadc1352f30fff09dc6bff2f1c2935e97c2bc", size = 113796, upload-time = "2026-10-09T12:56:59.539Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/99/ce/b3ae9ee0324d991c860187be2a6ee436d27a3d02397eaddbb101ec901f3d/pycparser-3.1-py3-none-any.whl", hash = "sha256:f09d358c840bd147b79e55f2bc494f18ea869dc897f5852a8f5766b74f787882", size = 48709, upload-time = "2026-10-08T12:26:32.97Z" }, + { url = "https://files.pythonhosted.org/packages/90/11/0e6f11117525ff0eec40ebac3d313376f102df93ca44ad9e893ee85e4f89/pycparser-3.11-py3-none-any.whl", hash = "sha256:51d5a8ba2be0bbe440b99d2112604c95bbbc3c2748a64260186c541e1729cd80", size = 51178, upload-time = "2026-10-09T12:56:58.131Z" }, ] [[package]] From 0ef7b782b9f8fa4f65efc8a4e51797beab93b301 Mon Sep 17 00:00:00 2001 From: Adam Getchell Date: Fri, 9 Oct 2026 12:23:50 -0700 Subject: [PATCH 5/6] fix: harden shared performance publication and Windows fixtures - Exclude unsupported norm cases from historical baseline inventories. - Preserve prior reports on archive conflicts and reject ambiguous identities. - Protect complete-run inputs during README publication and honor configured archive locations and equivalent input paths. - Keep immutable generated reports outside Markdown formatting and render curated provenance with valid list structure. - Compare command fixture directories by filesystem identity across Windows and POSIX shells, and align the managed rumdl pin to 0.2.79. --- docs/BENCHMARKING.md | 6 + justfile | 4 +- pyproject.toml | 2 +- scripts/archive_performance.py | 22 +- scripts/criterion_dim_plot.py | 36 ++-- scripts/performance_runs.py | 23 ++- scripts/tests/test_performance_workflow.py | 225 +++++++++++++++++++-- tooling/performance-report.toml | 1 + 8 files changed, 272 insertions(+), 47 deletions(-) diff --git a/docs/BENCHMARKING.md b/docs/BENCHMARKING.md index c0018eb..9bb5605 100644 --- a/docs/BENCHMARKING.md +++ b/docs/BENCHMARKING.md @@ -349,6 +349,12 @@ published reports and selection. Review and commit the whole new run and index. Legacy snapshots under `docs/performance/` retain their original bytes, hashes, framing, and reader; see the [legacy index](performance/README.md). +The Markdown check and format recipes exclude the generated +`docs/performance-runs/` archive because shared validation requires exact report +bytes. Keep both recipe exclusions aligned with `archive` in +`tooling/performance-report.toml` when relocating it. The curated +`docs/performance.md` remains subject to active Markdown checks. + To reproduce and promote the report without running Cargo or creating Git worktrees, use: diff --git a/justfile b/justfile index d39ddbb..4dcc328 100644 --- a/justfile +++ b/justfile @@ -329,7 +329,7 @@ markdown-check: tools-check _ensure-uv files=() while IFS= read -r -d '' file; do case "$file" in - CHANGELOG.md|docs/archive/*|docs/archives/changelog/*) continue ;; + CHANGELOG.md|docs/archive/*|docs/archives/changelog/*|docs/performance-runs/*) continue ;; esac if [ -f "$file" ]; then files+=("$file") @@ -353,7 +353,7 @@ markdown-fix: tools-check files=() while IFS= read -r -d '' file; do case "$file" in - CHANGELOG.md|docs/archive/*|docs/archives/changelog/*) continue ;; + CHANGELOG.md|docs/archive/*|docs/archives/changelog/*|docs/performance-runs/*) continue ;; esac if [ -f "$file" ]; then files+=("$file") diff --git a/pyproject.toml b/pyproject.toml index 61e218a..bfe792e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -174,7 +174,7 @@ cargo-nextest = "0.9.148" clippy-sarif = "0.8.0" dprint = "0.61.1" git-cliff = "2.14.2" -rumdl = "0.2.78" +rumdl = "0.2.79" sarif-fmt = "0.8.0" taplo-cli = "0.10.0" typos-cli = "1.51.1" diff --git a/scripts/archive_performance.py b/scripts/archive_performance.py index c73d714..4a0ecb5 100644 --- a/scripts/archive_performance.py +++ b/scripts/archive_performance.py @@ -130,13 +130,14 @@ def promote(root: Path, stem: Path) -> None: msg = "selected evidence changed while composing scientific report" raise ValueError(msg) outputs["docs/performance.md"] = render_scientific_report(evidence) - # Retain the exact previous curated report at its established archive path. + # Same-pair reruns are retained in shared history. Archive the previous + # curated pair when advancing releases, validating even existing archives. previous = root / "docs/performance.md" previous_bytes = previous.read_bytes() if previous.exists() else None if previous_bytes is not None and previous_bytes != outputs["docs/performance.md"]: identity = parse_report_id(previous_bytes.decode("utf-8")) - name = f"docs/archive/performance/{identity.current_tag}-vs-{identity.baseline_tag}.md" - if not (root / name).exists(): + if identity != parse_report_id(outputs["docs/performance.md"].decode("utf-8")): + name = f"docs/archive/performance/{identity.current_tag}-vs-{identity.baseline_tag}.md" outputs[name] = previous_bytes originals = dict(history.originals) inputs = {name: value for name, value in originals.items() if name not in outputs and value is not None} @@ -154,12 +155,12 @@ def promote(root: Path, stem: Path) -> None: def parse_report_id(text: str) -> ReportId: """Read the established scientific report header without modifying its bytes.""" - current = re.search(r"^\*\*la-stack\*\* (v[^\s]+)", text, re.MULTILINE) - baseline = re.search(r"^Comparison against baseline \*\*([^*]+)\*\*:", text, re.MULTILINE) - if current is None or baseline is None: - msg = "performance report has no release identity" + current = re.findall(r"^\*\*la-stack\*\* (v[^\s]+)", text, re.MULTILINE) + baseline = re.findall(r"^Comparison against baseline \*\*([^*]+)\*\*:", text, re.MULTILINE) + if len(current) != 1 or len(baseline) != 1: + msg = "performance report must contain exactly one current and baseline release identity" raise ValueError(msg) - return ReportId(normalize_tag(current[1]), normalize_tag(baseline[1])) + return ReportId(normalize_tag(current[0]), normalize_tag(baseline[0])) def render_and_promote_artifacts(*, artifacts: ArtifactPaths, output: Path, current: Path, archive_dir: Path) -> ReportId: @@ -174,8 +175,9 @@ def render_and_promote_artifacts(*, artifacts: ArtifactPaths, output: Path, curr outputs = {current.relative_to(root).as_posix(): text, output.relative_to(root).as_posix(): text} if current.exists() and current.read_bytes() != text: identity = parse_report_id(current.read_text(encoding="utf-8")) - archived = archive_dir / f"{identity.current_tag}-vs-{identity.baseline_tag}.md" - outputs[archived.relative_to(root).as_posix()] = current.read_bytes() + if identity != ReportId(release.current, release.baseline): + archived = archive_dir / f"{identity.current_tag}-vs-{identity.baseline_tag}.md" + outputs[archived.relative_to(root).as_posix()] = current.read_bytes() inputs = {path.relative_to(root).as_posix(): path.read_bytes() for path in (artifacts.csv, artifacts.provenance)} publish_publication(plan_outputs(root, outputs, inputs=inputs, immutable=tuple(name for name in outputs if name.startswith("docs/archive/")))) return ReportId(release.current, release.baseline) diff --git a/scripts/criterion_dim_plot.py b/scripts/criterion_dim_plot.py index 94a0954..be47218 100644 --- a/scripts/criterion_dim_plot.py +++ b/scripts/criterion_dim_plot.py @@ -17,7 +17,6 @@ import json import platform import re -import shutil import subprocess import sys import tempfile @@ -1117,6 +1116,8 @@ def _stage_and_publish_outputs( # noqa: C901, PLR0913 ) -> int: """Render every output in isolation, then replace publication files together.""" final_provenance = req.csv_path.with_suffix(".provenance.json") + readme_path = _resolve_under_root(root, args.readme) + readme_original = readme_path.read_bytes() if args.update_readme else b"" with tempfile.TemporaryDirectory(prefix=".criterion-dim-plot-", dir=root) as tmp: stage_dir = Path(tmp) staged_csv = stage_dir / "benchmark.csv" @@ -1150,9 +1151,8 @@ def _stage_and_publish_outputs( # noqa: C901, PLR0913 pairs.append((staged_svg, req.out_svg)) if args.update_readme: - readme_path = _resolve_under_root(root, args.readme) staged_readme = stage_dir / "README.md" - shutil.copy2(readme_path, staged_readme) + staged_readme.write_bytes(readme_original) try: _update_staged_readme_publication(root=root, args=args, rows=rows, staged_readme=staged_readme) except (OSError, ValueError) as exc: @@ -1166,9 +1166,12 @@ def _stage_and_publish_outputs( # noqa: C901, PLR0913 return 2 else: outputs = {destination.relative_to(root).as_posix(): staged.read_bytes() for staged, destination in pairs} - inputs = {path.relative_to(root).as_posix(): data for path, data in validated_inputs.items() if path.relative_to(root).as_posix() not in outputs} + ensure_distinct_paths( + {**{f"input {path}": path for path in validated_inputs}, **{f"output {destination}": destination for _, destination in pairs}} + ) + inputs = {path.relative_to(root).as_posix(): data for path, data in validated_inputs.items()} plan = plan_outputs(root, outputs, inputs=inputs) - if any(dict(plan.originals).get(path.relative_to(root).as_posix()) != data for path, data in validated_inputs.items()): + if args.update_readme and dict(plan.originals)[readme_path.relative_to(root).as_posix()] != readme_original: msg = "README publication inputs changed during rendering" raise ValueError(msg) publish_publication(plan) @@ -1208,7 +1211,8 @@ def _complete_readme(root: Path, args: PlotCliArgs) -> int: sources = dict(evidence.sources) context = dict(sources["current"].context) baseline_context = dict(sources["baseline"].context) - inputs = {root / "README.md": (root / "README.md").read_bytes()} + report_config = root / complete.REPORT_CONFIG + inputs = {report_config: report_config.read_bytes()} for field in ("source-inventory", "harness-inventory"): inputs.update({root / name: (root / name).read_bytes() for name in json.loads(context[field])}) payload, manifest = complete.scratch_paths(root / "target/bench-reports/performance") @@ -1217,7 +1221,7 @@ def _complete_readme(root: Path, args: PlotCliArgs) -> int: archive = root / complete.archive_directory(root) payload = archive / "runs" / identity / "run.json" manifest = payload.with_name("evidence.json") - for name in ("index.json", "latest.json"): + for name in ("index.json", "latest.json", f"runs/{identity}/report.md"): path = archive / name inputs[path] = path.read_bytes() inputs.update(zip((payload, manifest), serialize_evidence(evidence), strict=True)) @@ -1264,16 +1268,16 @@ def main(argv: list[str] | None = None) -> int: # noqa: C901, PLR0911, PLR0912, return 2 if rc != 0: return rc - if ( - args.update_readme - and args.performance_csv == _DEFAULT_PERFORMANCE_CSV - and any( - (root / path).exists() or (root / path).is_symlink() - for path in ("target/bench-reports/performance.run.json", "target/bench-reports/performance.evidence.json", "docs/performance-runs") - ) - ): + if args.update_readme and _resolve_under_root(root, args.performance_csv).resolve() == (root / _DEFAULT_PERFORMANCE_CSV).resolve(): try: - return _complete_readme(root, args) + # The report registry imports this module's metric definitions. + import performance_runs as complete # noqa: PLC0415 + + candidates = list(complete.scratch_paths(root / "target/bench-reports/performance")) + if (root / complete.REPORT_CONFIG).exists(): + candidates.append(root / complete.archive_directory(root)) + if any(path.exists() or path.is_symlink() for path in candidates): + return _complete_readme(root, args) except (OSError, ValueError, TypeError, KeyError, ExceptionGroup) as error: print(f"Invalid complete performance run: {format_exception_diagnostics(error)}", file=sys.stderr) return 2 diff --git a/scripts/performance_runs.py b/scripts/performance_runs.py index 7e39a90..53eeb71 100644 --- a/scripts/performance_runs.py +++ b/scripts/performance_runs.py @@ -81,6 +81,16 @@ def phase_environment(root: Path, adapter: str = NO_API_COMPATIBILITY) -> tuple[ return tuple(sorted(env.items())) +def _historical_case_available(full_id: str) -> bool: + """Mirror the groups and norm cases disabled by the historical harness cfg.""" + bench = full_id.partition("/")[2] + return ( + not full_id.startswith(UNAVAILABLE_PREFIXES) + and bench != "la_stack_norm2" + and not bench.startswith(("la_stack_norm2_scenario_", "iterative_f64_hypot_norm2_scenario_", "delaunay_scaled_norm_norm2_scenario_")) + ) + + def measurement_plan(root: Path, pair: ReleasePair, suite: str, scope: str, listing: Path) -> CommonHarnessPlan: """Build the shared plan from native inventories and consumer API policy.""" compatibility = resolve_shared_harness_compatibility(current=pair.current, baseline=pair.baseline, shared_harness_rational_inputs=True) @@ -102,7 +112,7 @@ def measurement_plan(root: Path, pair: ReleasePair, suite: str, scope: str, list msg = "duplicate semantic benchmark IDs" raise ValueError(msg) old_api = phase == "baseline" and compatibility.baseline_api_compatibility != NO_API_COMPATIBILITY - inventories[phase] = tuple(sorted(name for name in ids if not (old_api and name.startswith(UNAVAILABLE_PREFIXES)))) + inventories[phase] = tuple(sorted(name for name in ids if not old_api or _historical_case_available(name))) return declared_plan(pair, suite, scope, inventories, baseline_env=baseline_env, current_env=current_env, config=config) @@ -121,7 +131,7 @@ def declared_plan( # noqa: PLR0913 compatibility = resolve_shared_harness_compatibility(current=pair.current, baseline=pair.baseline, shared_harness_rational_inputs=True) selected = selected_ids(suite, scope, inventories["current"]) unavailable = compatibility.baseline_api_compatibility != NO_API_COMPATIBILITY - baseline_rows = tuple(name for name in selected if not (unavailable and name.startswith(UNAVAILABLE_PREFIXES))) + baseline_rows = tuple(name for name in selected if not unavailable or _historical_case_available(name)) missing = set(baseline_rows) - set(inventories["baseline"]) if missing: raise ValueError(f"baseline inventory omits selected rows: {sorted(missing)}") @@ -313,8 +323,13 @@ def render_scientific_report(evidence: Evidence) -> bytes: context = dict(source.context) lines.extend( [ - f"- {phase}: revision `{source.revision}`, source `{source.source_sha256}`, harness `{source.harness_sha256}`.", - f" Gate `{context['gate-command']}`: {context['gate-status']}; command `{context['command']}`.", + f"- {phase}:", + f" - Revision: `{source.revision}`.", + f" - Source: `{source.source_sha256}`.", + f" - Harness: `{source.harness_sha256}`.", + f" - Gate status: {context['gate-status']}.", + f" - Gate: `{context['gate-command']}`.", + f" - Command: `{context['command']}`.", ] ) lines.extend( diff --git a/scripts/tests/test_performance_workflow.py b/scripts/tests/test_performance_workflow.py index 3f17277..0bb09a3 100644 --- a/scripts/tests/test_performance_workflow.py +++ b/scripts/tests/test_performance_workflow.py @@ -118,6 +118,20 @@ def measured(tmp_path: Path) -> tuple[Path, Evidence]: return current, evidence +@pytest.fixture(scope="module") +def measured_run(tmp_path_factory: pytest.TempPathFactory) -> tuple[Path, Evidence]: + """Measure once; publication regressions operate on independent copies.""" + return measured(tmp_path_factory.mktemp("measured-run")) + + +@pytest.fixture +def complete_run(tmp_path: Path, measured_run: tuple[Path, Evidence]) -> tuple[Path, Evidence]: + root, evidence = measured_run + destination = tmp_path / "current" + shutil.copytree(root, destination) + return destination, evidence + + def test_common_harness_and_reference_phase(tmp_path: Path) -> None: baseline, current, plan = prepared(tmp_path) (baseline / "benches/exact.rs").write_text("obsolete harness\n", encoding="utf-8") @@ -162,7 +176,11 @@ def test_shared_retention_repeated_runs_and_offline_report(tmp_path: Path) -> No workflow.promote(root, stem) assert len(list((root / policy.archive_directory(root) / "runs").iterdir())) == 2 assert {path: path.read_bytes() for path in original} == original - assert (root / "docs/archive/performance/v0.4.6-vs-v0.4.5.md").read_bytes() == first_report + assert (retained / "run.json").is_file() + assert first_report != (root / "docs/performance.md").read_bytes() + # Same-pair reruns live in complete-run history, leaving the canonical + # release-pair archive available for a later release transition. + assert not (root / "docs/archive/performance/v0.4.6-vs-v0.4.5.md").exists() shutil.rmtree(root / "target") assert run_identity(policy.retained_run(root, stem)) == run_identity(second) before = (root / "docs/performance.md").read_bytes() @@ -173,6 +191,60 @@ def test_shared_retention_repeated_runs_and_offline_report(tmp_path: Path) -> No policy.retained_run(root, stem) +@pytest.mark.parametrize("destination", ["missing", "identical", "different", "directory", "symlink"]) +def test_promotion_preserves_prior_report_on_archive_collision(complete_run: tuple[Path, Evidence], destination: str) -> None: + root, evidence = complete_run + stem = root / "target/bench-reports/performance" + workflow.save_scratch(root, stem, evidence) + current = root / "docs/performance.md" + current.parent.mkdir(parents=True, exist_ok=True) + previous = b"**la-stack** v0.4.5\n\nComparison against baseline **v0.4.4**:\n\nPrior curated report.\n" + current.write_bytes(previous) + archived = root / "docs/archive/performance/v0.4.5-vs-v0.4.4.md" + archived.parent.mkdir(parents=True) + if destination in {"identical", "different"}: + archived.write_bytes(previous if destination == "identical" else b"Conflicting archive.\n") + elif destination == "directory": + archived.mkdir() + elif destination == "symlink": + archived.symlink_to(current) + before = {path: path.read_bytes() for path in root.rglob("*") if path.is_file() and ".git" not in path.parts} + if destination in {"missing", "identical"}: + workflow.promote(root, stem) + assert archived.read_bytes() == previous + assert current.read_bytes() == policy.render_scientific_report(evidence) + assert run_identity(policy.retained_run(root, stem)) == run_identity(evidence) + else: + errors = {"different": "immutable output differs", "directory": "regular file", "symlink": "symlink"} + with pytest.raises(ValueError, match=errors[destination]): + workflow.promote(root, stem) + assert {path: path.read_bytes() for path in root.rglob("*") if path.is_file() and ".git" not in path.parts} == before + assert not (root / policy.archive_directory(root)).exists() + if destination == "directory": + assert archived.is_dir() + elif destination == "symlink": + assert archived.is_symlink() + assert archived.readlink() == current + + +@pytest.mark.parametrize("duplicate", ["**la-stack** v0.4.4", "Comparison against baseline **v0.4.3**:"]) +def test_duplicate_report_identity_prevents_publication(complete_run: tuple[Path, Evidence], duplicate: str) -> None: + root, evidence = complete_run + stem = root / "target/bench-reports/performance" + workflow.save_scratch(root, stem, evidence) + current = root / "docs/performance.md" + current.parent.mkdir(parents=True, exist_ok=True) + text = f"**la-stack** v0.4.5\n\nComparison against baseline **v0.4.4**:\n\n{duplicate}\n" + current.write_text(text, encoding="utf-8") + before = {path: path.read_bytes() for path in root.rglob("*") if path.is_file() and ".git" not in path.parts} + with pytest.raises(ValueError, match="exactly one"): + workflow.parse_report_id(text) + with pytest.raises(ValueError, match="exactly one"): + workflow.promote(root, stem) + assert {path: path.read_bytes() for path in root.rglob("*") if path.is_file() and ".git" not in path.parts} == before + assert not (root / policy.archive_directory(root)).exists() + + def test_partial_scratch_and_stale_source_cannot_select_old_history(tmp_path: Path) -> None: root, evidence = measured(tmp_path) stem = root / "target/bench-reports/performance" @@ -186,8 +258,14 @@ def test_partial_scratch_and_stale_source_cannot_select_old_history(tmp_path: Pa policy.verify_current_source(root, evidence) -def test_readme_uses_retained_reference_estimates_after_cleanup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - root, evidence = measured(tmp_path) +@pytest.mark.parametrize("archive", ["docs/performance-runs", "docs/performance-history"]) +@pytest.mark.parametrize("csv_path", ["relative", "dot-relative", "absolute"]) +def test_readme_uses_retained_reference_estimates_after_cleanup( + complete_run: tuple[Path, Evidence], monkeypatch: pytest.MonkeyPatch, archive: str, csv_path: str +) -> None: + root, evidence = complete_run + config = root / policy.REPORT_CONFIG + config.write_text(config.read_text(encoding="utf-8").replace("docs/performance-runs", archive), encoding="utf-8") stem = root / "target/bench-reports/performance" workflow.save_scratch(root, stem, evidence) workflow.promote(root, stem) @@ -195,7 +273,13 @@ def test_readme_uses_retained_reference_estimates_after_cleanup(tmp_path: Path, shutil.rmtree(root / "target") monkeypatch.setattr(plot, "_repo_root", lambda: root) monkeypatch.setattr(plot, "_render_svg_with_gnuplot", lambda request: request.out_svg.write_bytes(b"\n")) - assert plot.main(["--update-readme", "--log-y"]) == 0 + performance_csv = "target/bench-reports/performance.csv" + if csv_path == "absolute": + performance_csv = str(root / performance_csv) + elif csv_path == "dot-relative": + performance_csv = f"./{performance_csv}" + arguments = ["--update-readme", "--log-y", "--performance-csv", performance_csv] + assert plot.main(arguments) == 0 assets = root / "docs/assets/bench" csv = assets / "vs_linalg_lu_solve_median.csv" assert "2,8.0,7.2,8.8,10.0,9.0,11.0,10.0,9.0,11.0" in csv.read_text(encoding="utf-8") @@ -203,10 +287,86 @@ def test_readme_uses_retained_reference_estimates_after_cleanup(tmp_path: Path, assert provenance["measurement"]["peer_phase"] == "baseline" assert provenance["run_id"] == run_identity(evidence) before = {path: path.read_bytes() for path in (*assets.iterdir(), root / "README.md")} - assert plot.main(["--update-readme", "--log-y"]) == 0 + assert plot.main(arguments) == 0 assert {path: path.read_bytes() for path in before} == before +@pytest.mark.parametrize( + ("cleanup", "destination"), + [ + (False, "run"), + (False, "evidence"), + (False, "alias"), + (False, "source"), + (False, "harness"), + (False, "configuration"), + (True, "run"), + (True, "evidence"), + (True, "report"), + (True, "index"), + (True, "latest"), + ], +) +def test_readme_outputs_cannot_replace_complete_run_inputs( + complete_run: tuple[Path, Evidence], monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], cleanup: bool, destination: str +) -> None: + root, evidence = complete_run + stem = root / "target/bench-reports/performance" + workflow.save_scratch(root, stem, evidence) + workflow.promote(root, stem) + archive = root / policy.archive_directory(root) + retained = archive / "runs" / run_identity(evidence) + payload, manifest = policy.scratch_paths(stem) + if cleanup: + shutil.rmtree(root / "target") + payload, manifest = retained / "run.json", retained / "evidence.json" + alias = root / "docs/alias.csv" + if destination == "alias": + alias.hardlink_to(payload) + targets = { + "run": payload, + "evidence": manifest, + "alias": alias, + "source": root / "src/lib.rs", + "harness": root / "tooling/performance.just", + "configuration": root / policy.REPORT_CONFIG, + "report": retained / "report.md", + "index": archive / "index.json", + "latest": archive / "latest.json", + } + shutil.copyfile(ROOT / "README.md", root / "docs/custom.md") + before = {path: path.read_bytes() for path in root.rglob("*") if path.is_file() and ".git" not in path.parts} + monkeypatch.setattr(plot, "_repo_root", lambda: root) + monkeypatch.setattr(plot, "_render_svg_with_gnuplot", lambda request: request.out_svg.write_bytes(b"\n")) + assert plot.main(["--update-readme", "--readme", "docs/custom.md", "--csv", str(targets[destination])]) == 2 + assert "must use distinct paths" in capsys.readouterr().err + assert {path: path.read_bytes() for path in root.rglob("*") if path.is_file() and ".git" not in path.parts} == before + + +def test_complete_report_passes_markdown_recipes_without_rewriting_history(complete_run: tuple[Path, Evidence]) -> None: + root, evidence = complete_run + stem = root / "target/bench-reports/performance" + workflow.save_scratch(root, stem, evidence) + workflow.promote(root, stem) + # Exercise the real recipes against generated, nonignored Markdown in a + # disposable checkout, reusing the already locked test environment. + for name in ("pyproject.toml", "uv.lock", ".python-version"): + shutil.copyfile(ROOT / name, root / name) + history = root / policy.archive_directory(root) + originals = {path: path.read_bytes() for path in history.rglob("*") if path.is_file()} + for recipe in ("markdown-check", "markdown-fix", "markdown-check"): + result = run_command( + "just", + ["--no-deps", "--justfile", str(ROOT / "justfile"), "--working-directory", str(root), recipe], + env={**os.environ, "UV_NO_SYNC": "1", "UV_PROJECT_ENVIRONMENT": sys.prefix}, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert {path: path.read_bytes() for path in originals} == originals + shutil.rmtree(root / "target") + assert run_identity(policy.retained_run(root, stem)) == run_identity(evidence) + + def test_legacy_bytes_hashes_and_report_are_unchanged() -> None: directory = ROOT / "docs/performance/v0.4.6-vs-v0.4.5/39719a1eb599d294dbbc1e46dc6fb0de2028ad2e59aab5f94fb48c7bf3d7f841" paths = ArtifactPaths(directory / "performance.csv", directory / "performance.provenance.json") @@ -238,7 +398,9 @@ def test_native_command_policy_and_compatibility_adapter() -> None: assert environment["CRITERION_HOME"] == "target/criterion" -def test_native_inventory_and_phase_commands_use_the_measured_checkout(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +@pytest.mark.parametrize("baseline_tag", ["v0.4.4", "v0.4.5", "v0.4.6"]) +@pytest.mark.parametrize("scope", ["release-signal", "all-benches"]) +def test_native_inventory_and_phase_commands_use_the_measured_checkout(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, baseline_tag: str, scope: str) -> None: """Exercise the configured Justfile, stubbing only the expensive Cargo boundary.""" root = tmp_path / "checkout with spaces" shutil.copytree(ROOT / "tooling", root / "tooling") @@ -246,13 +408,35 @@ def test_native_inventory_and_phase_commands_use_the_measured_checkout(tmp_path: binary = tmp_path / "bin" binary.mkdir() cargo = binary / "cargo" + # These are the registrations guarded by the historical API cfg in + # benches/vs_linalg.rs, including the two reference scenario kernels. + norm_rows = { + f"d{dimension}/{bench}" + for dimension in (2, 3, 4, 5) + for bench in ( + "la_stack_norm2", + *( + f"{kernel}_norm2_scenario_{scenario}" + for kernel in ("la_stack", "iterative_f64_hypot", "delaunay_scaled_norm") + for scenario in ("descending", "repeated_scale", "sparse", "wide_dynamic_range") + ), + ) + } + ordinary_norm_peers = { + f"d{dimension}/{bench}" for dimension in (2, 3, 4, 5) for bench in ("iterative_f64_hypot", "delaunay_scaled_norm", "nalgebra_norm", "faer_norm_l2") + } + ids = required_report_ids() | norm_rows | ordinary_norm_peers + exact = sorted(name for name in ids if not name.startswith("d")) + baseline_ids = sorted(name for name in ids if name.startswith("d")) + current_ids = sorted(name for name in baseline_ids if "/la_stack_" in name) cargo.write_text( "#!/bin/sh\n" - 'printf "%s|%s\\n" "$PWD" "$*" >> "$CARGO_RECORD"\n' + "directory=$(pwd -W 2>/dev/null || pwd -P)\n" + 'printf "%s|%s\\n" "$directory" "$*" >> "$CARGO_RECORD"\n' 'case "$*" in\n' - ' *"--bench exact"*) printf "exact_d2/det: benchmark\\n" ;;\n' - ' *"la_stack --noplot"*) printf "d2/la_stack_lu_solve: benchmark\\n" ;;\n' - ' *"--bench vs_linalg"*) printf "%s: benchmark\\n" d2/la_stack_lu_solve d2/nalgebra_lu_solve d2/faer_lu_solve ;;\n' + f' *"--bench exact"*) printf "%s: benchmark\\n" {shlex.join(exact)} ;;\n' + f' *"la_stack --noplot"*) printf "%s: benchmark\\n" {shlex.join(current_ids)} ;;\n' + f' *"--bench vs_linalg"*) printf "%s: benchmark\\n" {shlex.join(baseline_ids)} ;;\n' "esac\n", encoding="utf-8", ) @@ -260,14 +444,27 @@ def test_native_inventory_and_phase_commands_use_the_measured_checkout(tmp_path: record = tmp_path / "commands.txt" monkeypatch.setenv("PATH", f"{binary}{os.pathsep}{os.environ['PATH']}") monkeypatch.setenv("CARGO_RECORD", str(record)) - plan = policy.measurement_plan(root, PAIR, "all", "all-benches", tmp_path / "inventory") - assert plan.current.policy.expected == ("d2/la_stack_lu_solve", "exact_d2/det") - assert set(plan.baseline.policy.expected) == {"d2/la_stack_lu_solve", "d2/nalgebra_lu_solve", "d2/faer_lu_solve", "exact_d2/det"} + pair = ReleasePair("v0.4.6", baseline_tag) + plan = policy.measurement_plan(root, pair, "all", scope, tmp_path / "inventory") + assert set(plan.current.policy.expected) == set(current_ids) | set(exact) + unavailable = norm_rows | {name for name in exact if name.startswith(("rational_input_", "canonical_conversion_", "det4_diagnostic_"))} + assert set(plan.baseline.policy.expected) == (ids if baseline_tag == "v0.4.6" else ids - unavailable) + assert ordinary_norm_peers <= set(plan.baseline.policy.expected) + assert "d2/la_stack_norm2_sq" in plan.baseline.policy.expected + series = {item.name: {name for name, _ in item.rows} for item in plan.series} + assert series["la-stack baseline"] == (series["la-stack current"] if baseline_tag == "v0.4.6" else series["la-stack current"] - unavailable) + if scope == "all-benches": + assert "d2/la_stack_norm2" in series["la-stack current"] + assert "d2/la_stack_norm2_scenario_sparse" in series["la-stack current"] for command in (plan.baseline.gate, plan.baseline.command, plan.current.command): run_command(command[0], command[1:], cwd=root) lines = record.read_text(encoding="utf-8").splitlines() assert len(lines) == 9 - assert all(line.startswith(f"{root}|") and "--locked" in line for line in lines) + for line in lines: + directory, separator, arguments = line.partition("|") + assert separator == "|" + assert Path(directory).samefile(root) + assert "--locked" in arguments.split() assert all("--list" in line for line in lines[:4]) assert all("--list" not in line for line in lines[4:]) assert "test --locked --workspace --features bench,exact --test vs_linalg_inputs --test exact_bench_config" in lines[4] diff --git a/tooling/performance-report.toml b/tooling/performance-report.toml index abf807e..a2ca6e8 100644 --- a/tooling/performance-report.toml +++ b/tooling/performance-report.toml @@ -1,4 +1,5 @@ schema = 2 current = "docs/performance-runs/current.md" +# Keep the generated-archive exclusions in both Markdown recipes aligned. archive = "docs/performance-runs" title = "la-stack complete benchmark measurements" From 0d3532d16bc308fd32cc64619bfd0455b448e743 Mon Sep 17 00:00:00 2001 From: Adam Getchell Date: Fri, 9 Oct 2026 12:56:59 -0700 Subject: [PATCH 6/6] test: compare archive symlink targets by filesystem identity - Preserve the observed link target across rejected report publication. - Cover absolute and relative symlinks without assuming Windows readlink returns the same path representation used when creating the link. --- scripts/tests/test_performance_workflow.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/tests/test_performance_workflow.py b/scripts/tests/test_performance_workflow.py index 0bb09a3..4627a83 100644 --- a/scripts/tests/test_performance_workflow.py +++ b/scripts/tests/test_performance_workflow.py @@ -191,7 +191,7 @@ def test_shared_retention_repeated_runs_and_offline_report(tmp_path: Path) -> No policy.retained_run(root, stem) -@pytest.mark.parametrize("destination", ["missing", "identical", "different", "directory", "symlink"]) +@pytest.mark.parametrize("destination", ["missing", "identical", "different", "directory", "symlink", "relative-symlink"]) def test_promotion_preserves_prior_report_on_archive_collision(complete_run: tuple[Path, Evidence], destination: str) -> None: root, evidence = complete_run stem = root / "target/bench-reports/performance" @@ -208,6 +208,9 @@ def test_promotion_preserves_prior_report_on_archive_collision(complete_run: tup archived.mkdir() elif destination == "symlink": archived.symlink_to(current) + elif destination == "relative-symlink": + archived.symlink_to(os.path.relpath(current, archived.parent)) + original_link_target = archived.readlink() if archived.is_symlink() else None before = {path: path.read_bytes() for path in root.rglob("*") if path.is_file() and ".git" not in path.parts} if destination in {"missing", "identical"}: workflow.promote(root, stem) @@ -215,16 +218,17 @@ def test_promotion_preserves_prior_report_on_archive_collision(complete_run: tup assert current.read_bytes() == policy.render_scientific_report(evidence) assert run_identity(policy.retained_run(root, stem)) == run_identity(evidence) else: - errors = {"different": "immutable output differs", "directory": "regular file", "symlink": "symlink"} + errors = {"different": "immutable output differs", "directory": "regular file", "symlink": "symlink", "relative-symlink": "symlink"} with pytest.raises(ValueError, match=errors[destination]): workflow.promote(root, stem) assert {path: path.read_bytes() for path in root.rglob("*") if path.is_file() and ".git" not in path.parts} == before assert not (root / policy.archive_directory(root)).exists() if destination == "directory": assert archived.is_dir() - elif destination == "symlink": + elif destination in {"symlink", "relative-symlink"}: assert archived.is_symlink() - assert archived.readlink() == current + assert archived.readlink() == original_link_target + assert archived.samefile(current) @pytest.mark.parametrize("duplicate", ["**la-stack** v0.4.4", "Comparison against baseline **v0.4.3**:"])