diff --git a/.github/workflows/release-benchmarks.yml b/.github/workflows/release-benchmarks.yml index 8be4182..dafa48a 100644 --- a/.github/workflows/release-benchmarks.yml +++ b/.github/workflows/release-benchmarks.yml @@ -62,9 +62,9 @@ jobs: release-baseline: needs: validate-release runs-on: ubuntu-latest - # 2 min checkout + 28 min shared setup + 150 min comparative + 90 min exact - # + 15 min for the tail and runner overhead. - timeout-minutes: 285 + # 2 min checkout + 90 min shared setup + 150 min comparative + 90 min exact + # + 18 min for the tail and runner overhead; below the hosted 360 min limit. + timeout-minutes: 350 env: RELEASE_TAG: ${{ inputs.tag }} CRITERION_HOME: ${{ github.workspace }}/target/criterion @@ -82,7 +82,7 @@ jobs: - name: Prepare release benchmarks # One parent timeout bounds all nested installs, validation, and inventory. - timeout-minutes: 28 + timeout-minutes: 90 uses: ./.github/actions/prepare-release-benchmarks # zizmor: ignore[self-repository] actionlint 1.7.12 does not accept $/... with: validated-attempt: ${{ needs.validate-release.outputs.validated-attempt }} diff --git a/docs/BENCHMARKING.md b/docs/BENCHMARKING.md index f06e4d2..8c3fd36 100644 --- a/docs/BENCHMARKING.md +++ b/docs/BENCHMARKING.md @@ -811,15 +811,22 @@ benchmarking. Publication makes the attached evidence immutable. The producer runs full `vs_linalg` and `exact` suites sequentially on one `ubuntu-latest` runner. Separate steps allow 150 and 90 minutes respectively; -the outer job allows 285 minutes. Checkout has a 2-minute timeout, followed -by a composite preparation step with one shared 28-minute timeout covering -all tool installation, input validation, and inventory. These two limits bound -setup execution to 30 minutes; exceeding either fails the producer before -benchmarking or publication. The remaining 15 minutes provide headroom for +the outer job allows 350 minutes. Checkout has a 2-minute timeout, followed +by a composite preparation step with one shared 90-minute timeout covering +all tool installation, input validation, and inventory. Preparation reserves +35 minutes for uncached tools, the two declared 25-minute discovery deadlines, +and 5 minutes for input validation. Exceeding either setup deadline fails the +producer before benchmarking or publication. The remaining 18 minutes provide headroom for dataset validation, packaging, upload, diagnostics, and runner overhead. Discovery compiles both suites before measurement and shares the preparation deadline with the preceding work. Dependency caches remain disabled. +The [first v0.5.0 run](https://github.com/acgetchell/la-stack/actions/runs/38998792985) +spent about 27 minutes installing uncached tools, then exhausted its old +28-minute preparation limit during inventory compilation. Neither measurement +suite started. The expanded setup budget accommodates compilation separately +from tool installation without reducing measurement time or sampling. + The [v0.4.5 run](https://github.com/acgetchell/la-stack/actions/runs/32444040827) measured 304 comparative benchmarks in about 55m 34s, plus 2m 9s compilation (57m 43s total). Its exact suite was cancelled during `exact_d2/det_exact`; diff --git a/docs/RELEASING.md b/docs/RELEASING.md index b47e324..210be94 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -403,7 +403,9 @@ for draft visibility; the separate asset writer receives it to attach the archiv Neither privileged benchmark job checks out or executes repository code. The producer budgets 150 minutes for `vs_linalg` and 90 minutes for `exact` -within a 285-minute job. Both suites retain full release sampling. Inventory +within a 350-minute job, including a 90-minute preparation budget for uncached +tool installation, compilation, inventory and input validation. +Both suites retain full release sampling. Inventory and raw-data validation must succeed before the single complete archive is packaged and uploaded; inspect the suite timing summary when a run fails. See the [hosted runtime budget](BENCHMARKING.md#hosted-release-runtime-budget) @@ -418,6 +420,11 @@ or dispatch again. The producer requires a preflight from the current attempt; rerunning failed benchmark jobs alone stops before tool setup or measurement because the earlier draft check is stale. +A rerun uses the original workflow at the release tag; it does not pick up a +workflow fix merged later. If the tagged workflow needs repair, keep the release +as a draft and have the maintainer review the corrected release commit and tag +before dispatching again. Never move a published tag. + If the producer succeeded and only the publisher failed, preserve the existing 30-day Actions artifact and rerun failed jobs: diff --git a/scripts/tests/test_release_baseline.py b/scripts/tests/test_release_baseline.py index 6bbbfb0..b549cd3 100644 --- a/scripts/tests/test_release_baseline.py +++ b/scripts/tests/test_release_baseline.py @@ -211,7 +211,11 @@ def test_setup_limits_preserve_benchmark_and_tail_budgets() -> None: assert limit is not None, f"unbounded setup step: {step}" setup_limits.append(int(limit[1])) assert all(limit > 0 for limit in setup_limits) - assert sum(setup_limits) <= 30 + discovery = load_discovery(REPO_ROOT, "tooling/criterion-inventory.toml") + discovery_minutes = sum(suite.timeout for suite in discovery.suites) / 60 + # The failed hosted run spent 27 minutes on uncached tool installation. + # Reserve 35 for tools, each declared cold discovery deadline, and 5 for inputs. + assert setup_limits[-1] >= 35 + discovery_minutes + 5 assert "uses: ./.github/actions/prepare-release-benchmarks" in setup assert "using: composite" in PREPARATION.read_text(encoding="utf-8") benchmark_limits = [ @@ -219,6 +223,7 @@ def test_setup_limits_preserve_benchmark_and_tail_budgets() -> None: ] assert benchmark_limits == [150, 90] assert int(job_limit[1]) - sum(setup_limits) - sum(benchmark_limits) >= 15 + assert int(job_limit[1]) <= 360 @pytest.mark.parametrize(("suite", "step"), [("vs_linalg", "Save comparative Criterion baseline"), ("exact", "Save exact Criterion baseline")])