diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 987da5080..5003cc4dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,6 +146,10 @@ jobs: run: python3 -m unittest discover -s tests/python -p test_native_compose_benchmark.py - name: Check native frontend acceptance controls run: python3 -m unittest discover -s tests/python -p test_native_frontend_acceptance.py + - name: Check application benchmark protocol controls + run: | + python3 scripts/check-application-benchmark-protocol.py + python3 -m unittest discover -s tests/python -p test_application_benchmark_protocol.py runtime-core: name: Candidate core (${{ matrix.os }}) diff --git a/docs/performance.md b/docs/performance.md index be295998a..4953ff2f5 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -707,5 +707,46 @@ bound to its incarnation, so it still acquires and validates every image. Preloa make bases project-specific and require the verifier to check layer contents, so bases stay image-free. The larger remaining costs are image acquisition and per-incarnation revalidation. +## Matched application benchmark protocol + +Revision 1 of the v5 matched application benchmark is frozen in the +[application benchmark protocol](reference/application-benchmark-protocol.md) +(HACK-1169). It fixes the exact candidate and stable artifacts, the host and engine +versions, the Event Agent and small-fixture workloads, capacities, cache classes, a +10-round rotated order, sample counts, observer accounting and the program budgets. +It records no results. + +## Known v5 limit: storage-witness startup cost + +On 2026-10-19 the user accepted the current storage-witness cost as a known v5 limit +for now (HACK-1123). It is part of every native warm `up` and is not removed from +candidate timings under the benchmark protocol. + +**Current cost.** Since #313, a warm, image-only, unrouted native `up` or `restart` +without hooks runs 4 storage proofs instead of 6. Each proof starts 3 helper +containers. Estimated from a traced hosted docker-e2e run, the warm `up` went from +about 22 s to about 17.7 s, and the witness share went from about 13 s to about 8.7 s. +The target is at most 5 s of witness overhead per warm `up`, so the target is not met. +These are hosted-engine estimates, not measurements from the frozen benchmark host. +The proof sequence is described in the +[storage witness reference](reference/native-compose-storage-witness.md). + +**Why it is accepted.** The remaining proofs are the witness safety contract. Cutting +more changes that contract, not just its cost: + +- Reusing a proof receipt across commands was analyzed and found unsafe. The root + identity is only device, inode, uid and gid, so a recreated volume can match it. The + marker is a `user.*` extended attribute that any workload container can remove. + Docker exposes no boot ID or start time, and the engine ID survives restarts. + Checking a stored receipt would itself need a helper container. +- A third in-command cut was left out because it changes when a refusal surfaces. + +Every command therefore still opens with a full proof. + +**Revisit trigger.** Revisit this limit after the matched benchmark protocol +(HACK-1169) is frozen and its first complete revision-1 cohort has run, using that +cohort's measured warm startup and restart results. No proof-policy change may be made +without a separate safety review and approval. + Native recovery investigations can use the explicit, value-free [process census phase diagnostics](guides/native-census-diagnostics.md). diff --git a/docs/reference.md b/docs/reference.md index ee490977b..edca74747 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -21,6 +21,7 @@ - [Native configuration validation](./reference/native-config-compiler.md) — the authored `.hack/hack.project.json` compiler contract - [Native configuration with Compose](./reference/native-compose-commands.md) — the Compose command owner for authored projects - [Authored configuration with the native runtime](./reference/native-authored-commands.md) — the explicit native backend command slices and their support matrix +- [Application benchmark protocol](./reference/application-benchmark-protocol.md) — the frozen matched benchmark artifacts, workloads, order, sample counts and budgets ## Extensions diff --git a/docs/reference/application-benchmark-protocol.md b/docs/reference/application-benchmark-protocol.md new file mode 100644 index 000000000..98bafa25e --- /dev/null +++ b/docs/reference/application-benchmark-protocol.md @@ -0,0 +1,364 @@ +# Matched application benchmark protocol + +This page freezes revision 1 of the protocol for the v5 matched application benchmark +(Linear HACK-1169, under WU10 HACK-1123). It fixes the artifacts, host, workloads, +capacities, cache classes, run order, sample counts, observer accounting and budgets +**before** any trial runs. It records no results. No performance or resource claim +follows from this page. + +The machine-readable copy is +[`scripts/application-benchmark-protocol.json`](../../scripts/application-benchmark-protocol.json). +[`scripts/check-application-benchmark-protocol.py`](../../scripts/check-application-benchmark-protocol.py) +validates it and judges retained summaries offline. It never starts a runtime or calls +Docker. + +A change to anything on this page needs a new protocol revision, made before the +trials it governs. Results from different revisions are never pooled. A cohort that +cannot meet this protocol is reported as inconclusive, not adjusted after the fact. + +## Artifacts + +Every lane runs an exact released artifact, installed into its own isolated home and +state paths. A lane never uses whatever `hack` is on `PATH`; the M3 host currently has +4.2.0 installed, and that installation is not a lane. Before each cohort, the harness +must verify the SHA-256 digest of each downloaded asset and record the executable's +reported version. A mismatch refuses the cohort. + +| Lane | Tag | Commit | Asset | Asset SHA-256 | +| --- | --- | --- | --- | --- | +| `candidate` | `v5.0.0-next.5` (prerelease) | `28de2f8902d3776c3da9326e7ae6c6aa6e7f69bc` | `hack-5.0.0-next.5-darwin-arm64-native.tar.gz` | `ba1da4d349ab70069a1b0775ba671838440b5479cffd1fc2fe41c59816c3df11` | +| `hack-4.1.1` | `v4.1.1` | `74bf0672f41bd3f68a4ce6cba28b9ba15467c2b2` | `hack-4.1.1-darwin-arm64.tar.gz` | `968f75fe66cd84fdf93b0c5b64ca5df02f98c74281b5a9238125b3afd2f3d72b` | +| `hack-4.2.0` | `v4.2.0` | `575847486963db94623e1e7f8cd49acca286c517` | `hack-4.2.0-darwin-arm64.tar.gz` | `c62b533aaddbeaafa594cab682cf10041c9bcaea69e2fe25e33e85df49654a85` | +| `hack-4.2.1` | `v4.2.1` | `5a4c51babb7ef725a30f2f3fd3677081a64bc247` | `hack-4.2.1-darwin-arm64.tar.gz` | `557583d946abcd0dd0a3fa12b2f7b81f18310d594b0699d663ebf239d0d4d729` | +| `compose` | none | none | Docker CLI and Compose on the host | see below | + +The candidate is built from `28de2f89`, the `v5.0.0-next.5` tag commit. A later +candidate needs a new revision; it does not inherit this one's results. + +## Host and engine + +Native stateful benchmarks are host-specific, so one host is frozen. These values were +read from the host on 2026-10-19: + +| Item | Value | +| --- | --- | +| Host | `m3-max`: Apple M3 Max, 16 logical CPUs, 128 GiB memory | +| OS | macOS 27.0 | +| Docker CLI | 29.4.0 | +| Docker Compose | v5.1.2 | +| OrbStack | 2.2.3 (build 20963), configured with 16 CPUs and 71680 MiB memory | + +Before each cohort, the harness records these values again. If any of them differs, +the cohort is refused and a new revision is needed. Results from another host, +including the M5 Max development host, are not revision-1 results. + +Only one stateful benchmark owner may use the host and its runtimes at a time. Other +agents' runtime tests, builds and Docker work must not overlap a cohort. Independent +source-only checks may still run elsewhere. + +## Capacities + +Each lane runs with the capacity a user gets by default. Capacities are recorded and +reported side by side; they are never used to scale results. + +| Lane | Capacity | +| --- | --- | +| `candidate` | native `development` profile: 4 vCPUs, 6144 MiB guest memory, 32 GiB storage, 10 GiB overlay | +| `compose`, `hack-4.1.1`, `hack-4.2.0`, `hack-4.2.1` | the shared OrbStack VM above: 16 CPUs, 71680 MiB memory limit | + +The capacities differ. The compared footprint metric is the measured physical footprint +of the whole runtime, not the configured limit. A lower configured limit is not a +footprint result. Any capacity-matched control is a separate experiment with its own +revision. + +**Why `development`.** The candidate's own source describes the `development` profile +as an experimental allocation that cannot supply benchmark qualification, and its +lifecycle records carry the label `experimental-development-not-benchmark-qualified`. +That label belongs to the earlier WU02 provider comparison, whose frozen envelope is the +`research` profile (2 vCPUs, 2048 MiB, 4 GiB storage). It does not apply to this +protocol, for two reasons: + +- Graph allocation refuses every profile except `development` with `graph_profile`, + and both workloads here are service graphs. The `research` profile cannot run either + workload. +- A native `hack up` or `restart` always requests `development`. It is the capacity a + user gets by default, which is what this table freezes. + +So revision 1 freezes `development` on purpose. A candidate record that carries that +label is expected and recorded, not a reason to refuse the cohort. This protocol does +not produce WU02 research-envelope results, and WU02 results are not revision-1 +results. Changing the candidate's label or comment is separate runtime work; it cannot +change the frozen `28de2f89` artifact. + +## Workloads + +Two workloads run in every lane, and each workload is compared against every required +baseline. A cohort records each workload's pin, and every lane in that cohort uses the +same pin. + +**Event Agent.** The actual application configuration, not a reduced copy. The cohort +records the application commit and a digest of its configuration. Its expected shape is +14 services, 9 HTTP health checks and 2 initializers. A different shape refuses the +cohort until the protocol is revised. Credentials come only through authorized managed +injection; candidate endpoints and data stay isolated from v4, global state and QA. +A lifecycle counts as correct only when: + +- every declared long-running service is running and every HTTP health check is healthy; +- both initializers exited 0; +- the routed sign-in page returns HTTP 200; +- a per-run data marker written before the restart reads back after it. + +**Small fixture.** A pinned two-service graph: an initializer and a web service, both +on `oven/bun:1.4.2-slim` with `pull never`, the image ID recorded per cohort. The +initializer creates a SQLite schema in a named volume and exits 0. The web service +serves a health endpoint and the current source token, and reloads on a source edit. +A lifecycle counts as correct only when: + +- the initializer exits 0 and the web health endpoint returns HTTP 200; +- the check task inserts a per-run row and reads back the expected count; +- that row reads back after the restart; +- a source edit is served within the watcher acknowledgement window. + +A lane that cannot run a workload correctly records a failure for that lifecycle. It +is not dropped and its timings are not used. A baseline lane is `not_applicable` for a +metric only when it cannot express the workload at all, and that reason is recorded. +The candidate is never `not_applicable`: it is the subject of every budget. + +## Cache classes + +| Class | Meaning | Use | +| --- | --- | --- | +| `warm` | Engine or VM running; images, dependency and build caches filled by an earlier correct lifecycle of the same lane; application volumes kept | Budget class | +| `cold-capacity` | Engine or VM stopped before the trial; images and caches kept | Reported separately | +| `cold-cache` | Engine or VM running; images present; dependency and build caches empty | Reported separately | + +No trial pulls an image, so image download time never enters a lifecycle. The host page +cache is not flushed, and each receipt says so. Every receipt names its cache class. +Classes are never mixed in one statistic. + +## Lifecycle and phases + +One lifecycle is: warm startup to full readiness and correctness, the workload's check +and build/test task, an exec, a source reload, a restart to full readiness and data +readback, a steady idle window, sustained application load, a retaining shutdown, a +restore (an ordinary start after that shutdown, to full readiness with the per-run data +marker or row read back), and a final shutdown with cleanup readback. Each phase is +timed and reported on its own: + +`cold-startup`, `warm-startup`, `restart`, `restore`, `reload`, `build-test`, `exec`, +`steady-idle`, `sustained-load`, `shutdown`, `cleanup`. + +Readiness means the workload's correctness checks pass, not that a process started or +a port opened. Cleanup readback inventories owned containers, processes, sockets and +allocated disk, and keeps persistent data. + +## Scale points + +Each lane is also measured at 1, 8 and 32 branch instances of the small fixture, and +with 1 and 32 MCP clients connected, using the same accounting, order and sample counts. +A lane that has no MCP server records `not_applicable` with its reason. Revision 1 +judges budgets only at the single-instance matched lifecycle (`{"branch_instances": 1}`). +It sets no separate numerical budget for the other scale points: their results are +reported per lane, and every regression at any scale point is reported and blocks +acceptance under [Admission and failures](#admission-and-failures). The evaluator +refuses a summary for another scale point rather than judge it. + +## Order and repetitions + +Each workload runs 10 rounds. Each round runs every lane once, in the order below. This +is a Williams design for five lanes: every lane takes every position twice, and every +lane runs directly after every other lane exactly twice. It balances position and +first-order carryover, so a lane does not benefit from always running after the same +neighbour. The program floor is five rotated lifecycle runs per lane; this protocol uses +10. + +| Round | Order | +| --- | --- | +| 1 | candidate, compose, hack-4.2.1, hack-4.1.1, hack-4.2.0 | +| 2 | compose, hack-4.1.1, candidate, hack-4.2.0, hack-4.2.1 | +| 3 | hack-4.1.1, hack-4.2.0, compose, hack-4.2.1, candidate | +| 4 | hack-4.2.0, hack-4.2.1, hack-4.1.1, candidate, compose | +| 5 | hack-4.2.1, candidate, hack-4.2.0, compose, hack-4.1.1 | +| 6 | hack-4.2.0, hack-4.1.1, hack-4.2.1, compose, candidate | +| 7 | hack-4.2.1, hack-4.2.0, candidate, hack-4.1.1, compose | +| 8 | candidate, hack-4.2.1, compose, hack-4.2.0, hack-4.1.1 | +| 9 | compose, candidate, hack-4.1.1, hack-4.2.1, hack-4.2.0 | +| 10 | hack-4.1.1, compose, hack-4.2.0, candidate, hack-4.2.1 | + +Print the same schedule with +`python3 scripts/check-application-benchmark-protocol.py --schedule`. + +## Sample counts + +- **Lifecycle metrics** (startup, restart, footprint, CPU per task) use the median of + at least 10 lifecycles per lane, with minimum, maximum and interquartile range. +- **Tail metrics** (info, warm admission, cached status, watcher acknowledgement and + request latency under load) use the nearest-rank p95 of at least 60 samples per lane, + per workload. The sample maximum is reported as an upper bound. For any distribution, + the maximum of n samples is at or above the true p95 with probability 1 − 0.95ⁿ; + 59 samples is the smallest count that reaches 95%. +- A statistic with fewer samples than planned, or fewer than these floors, is + inconclusive. It is never a pass. +- A statistic is computed only from admitted samples. If any planned sample was + flagged, or its lifecycle failed, or the summary does not account for flags and + failures, the statistic is inconclusive; a failed candidate lifecycle is a failure. + Samples are never rerun or replaced to fill a gap: a summary that accounts for more + samples than were planned is refused. + +## Measurement boundary + +Each lifecycle records: + +- wall time per phase and to full readiness; +- CPU seconds for the CLI process tree, every runtime helper, the whole VM (OrbStack or + the native provider) and the guest, with the sampling window for any CPU percentage; +- CPU per completed task, using a count of completed correct operations; +- physical footprint and peak for the whole runtime, including the VM and every helper. + RSS sums include shared pages and are not reported as physical memory savings; +- logical and allocated disk, before and after cleanup; +- host load, memory pressure and background CPU at the start, end and every second; +- request latency under sustained load; +- every failure, refusal and retry. + +Near-zero CPU percentages are not reported as wins. Unobserved values stay null, and +each summary states its coverage. + +## Observer overhead + +The sampler runs with the same settings in every lane. Before each cohort, it runs +alone for a control window of at least 600 seconds with no workload, to measure its +own CPU and footprint. During the cohort, its own process-tree CPU is recorded for +each lifecycle and reported as its own line. + +Observer cost is never subtracted from a lane. When the candidate and a baseline differ +by less than 2 times the observer's cost for that metric, the comparison is +inconclusive. A summary that does not report observer cost for a compared metric is +inconclusive for that metric. + +## Admission and failures + +A sample is flagged, not dropped, when build tools run, host load or memory pressure is +above the idle baseline, a security scan or indexing burst is attributed to the run, or +any of these could not be observed. Flagged samples are summarized separately. A +background-contention, low-headroom, failed-correctness or stalled-subscriber control +must produce a flag or failure, not a clean sample. + +Every reproducible regression is reported, even where the aggregate looks better. A +regression in a required workflow blocks acceptance until it is fixed or the goal is +explicitly reviewed. + +## Budgets + +**Comparative budgets.** These are judged in the `warm` class, separately against each +applicable baseline: `compose`, `hack-4.1.1`, `hack-4.2.0` and `hack-4.2.1`. There is no +aggregate score. Ratio is candidate divided by baseline; a ratio at the ceiling passes. + +| Metric | Statistic | Ceiling ratio | Program target | +| --- | --- | --- | --- | +| `steady_physical_footprint_bytes` | median | 0.70 | at least 30% lower steady total physical footprint | +| `cpu_seconds_per_task` | median | 0.75 | at least 25% lower CPU per completed matched task | +| `warm_startup_seconds` | median | 0.80 | at least 20% faster median warm startup | +| `warm_restart_seconds` | median | 0.80 | at least 20% faster median warm restart | + +**Absolute budgets.** These are candidate-only. They come from the program acceptance +document. + +| Metric | Statistic | Budget | Workload | +| --- | --- | --- | --- | +| `info_latency_ms` | p95 | ≤ 100 ms | both | +| `warm_admission_ms` | p95 | ≤ 250 ms | both | +| `cached_status_1000_objects_ms` | p95 | ≤ 150 ms, with 1,000 cached objects | both | +| `idle_control_plane_core_fraction` | mean over 600 s | < 1% of one core | both | +| `watcher_ack_ms` | p95 | ≤ 200 ms | small fixture | + +Only the watcher budget is limited to the small fixture, as in the program acceptance +document. A later revision may apply a budget to more workloads, but never fewer. The +checker also refuses a changed statistic, a raised ceiling, a lower observer factor, a +changed p95 method or confidence, a workload that drops a required baseline, a changed +workload shape, a missing phase, a missing scale point and a Compose lane that differs +from the frozen host engine. + +The identities on this page are exact, not floors. The checker pins each lane's tag, +version, commit, asset name and digest, each lane's kind and backend, the candidate's +capacity, the host and engine values, each workload's pin and the rotated order above. +A different or missing value is refused (`artifact_identity_mismatch`, +`lane_runtime_mismatch`, `candidate_capacity_mismatch`, `host_mismatch`, +`workload_pin_mismatch`, `schedule_order_mismatch`), and so is a workload that drops a +frozen correctness check (`workload_correctness_weakened`). A revision may add +correctness checks. The sample counts are floors at their frozen values: a protocol +that plans fewer than 10 lifecycle runs or 60 tail samples per lane is refused, even +where the program floor or the p95 bound would allow fewer. Every number in the protocol or a summary must be +within ±(2⁵³ − 1), the interoperable JSON range; a larger value is refused as malformed. + +These are acceptance thresholds, not measured achievements. + +## Judging a summary + +`--evaluate SUMMARY` reads one workload's retained summary for the `warm` class at the +single-instance scale point. It returns one verdict per budget and baseline: + +| Verdict | Meaning | +| --- | --- | +| `pass` | the statistic meets the budget | +| `fail` | the statistic misses the budget, or a candidate lifecycle failed | +| `not_applicable` | a baseline cannot express the workload, with a recorded reason | +| `inconclusive_coverage` | fewer admitted samples than planned | +| `inconclusive_samples` | fewer samples than the protocol's floor | +| `inconclusive_admission` | a sample was flagged, a baseline lifecycle failed, or flags and failures were not accounted | +| `inconclusive_observer` | the difference is within 2 times the observer's cost, or that cost is not reported | +| `inconclusive_missing` | the lane, metric or planned count is missing | +| `inconclusive_no_baseline` | every baseline is `not_applicable`, so there is no comparison | + +It exits 0 only when every verdict is `pass` or `not_applicable`, and a comparative +budget always has at least one applicable baseline or an `inconclusive_no_baseline` +record. It exits 1 when any verdict fails or is inconclusive. It exits 2, without +judging, when the protocol or summary is invalid: for example a summary for another +cache class or scale point, a candidate marked `not_applicable`, a `not_applicable` +record that carries samples, a negative sample, a planned count that is not a positive +integer, or a value of the wrong type. Diagnostics are closed codes and never repeat +input values. + +A summary has this shape. `samples` lists only admitted samples; `flagged` and `failed` +count the planned samples left out, so `samples` plus both counts never exceeds +`expected`: + +```json +{ + "protocol_revision": 1, + "workload": "small-fixture", + "cache_class": "warm", + "scale": { "branch_instances": 1 }, + "observer_cost": { "cpu_seconds_per_task": 0.02 }, + "lanes": { + "candidate": { + "cpu_seconds_per_task": { "samples": [1.2], "expected": 10, "flagged": 0, "failed": 0 } + }, + "compose": { + "cpu_seconds_per_task": { "not_applicable": true, "reason": "no matched task" } + } + } +} +``` + +## Known limits + +The accepted storage-witness startup cost is recorded in +[performance diagnostics](../performance.md#known-v5-limit-storage-witness-startup-cost). +It is part of every candidate warm `up` measured under this protocol, and is not +removed from candidate timings. + +## Checks + +```sh +python3 scripts/check-application-benchmark-protocol.py +python3 -m unittest discover -s tests/python -p test_application_benchmark_protocol.py +``` + +The unit tests check the frozen file, the rotation balance, the p95 sample floor and +that this page carries every frozen artifact, budget, phase, round and verdict. They +also prove that each weakening listed under [Budgets](#budgets), a bad digest, a +changed artifact, host, capacity, workload pin or rotated order, an unbalanced order, +too few samples and a missing observer policy is refused; that missing, flagged, +`not_applicable`-only or cold-cache data is never accepted; and that any malformed +protocol or summary value, including an integer too large for a float, yields a closed +code rather than a traceback. They run offline, locally and in CI. diff --git a/docs/reference/native-compose-storage-witness.md b/docs/reference/native-compose-storage-witness.md index 8146831af..a4ad83673 100644 --- a/docs/reference/native-compose-storage-witness.md +++ b/docs/reference/native-compose-storage-witness.md @@ -137,6 +137,8 @@ proves four times (opening, prepublication, effect-entry and post-workload) rather than six. With `HACK_NATIVE_COMPOSE_PHASE_TRACE=1`, each performed verification proof emits one `storage.proof` span; a reused proof emits none. These counts establish call ordering and helper invocations, not wall time. +The remaining per-command cost is an accepted known v5 limit; see +[performance diagnostics](../performance.md#known-v5-limit-storage-witness-startup-cost). The one-off `run` path separately omits the enrollment-tail read that would immediately repeat its mandatory pre-spawn witness read. `up` and `restart` diff --git a/scripts/application-benchmark-protocol.json b/scripts/application-benchmark-protocol.json new file mode 100644 index 000000000..34e1e7c83 --- /dev/null +++ b/scripts/application-benchmark-protocol.json @@ -0,0 +1,238 @@ +{ + "protocol": "hack-application-benchmark", + "revision": 1, + "frozen_on": "2026-10-19", + "linear_issue": "HACK-1169", + "document": "docs/reference/application-benchmark-protocol.md", + "host": { + "id": "m3-max", + "cpu": "Apple M3 Max", + "logical_cpus": 16, + "memory_bytes": 137438953472, + "os": "macOS 27.0", + "docker_cli": "29.4.0", + "compose": "5.1.2", + "orbstack": { + "version": "2.2.3", + "build": "20963", + "cpu": 16, + "memory_mib": 71680 + } + }, + "lanes": [ + { + "id": "candidate", + "kind": "hack", + "version": "5.0.0-next.5", + "tag": "v5.0.0-next.5", + "commit": "28de2f8902d3776c3da9326e7ae6c6aa6e7f69bc", + "asset": "hack-5.0.0-next.5-darwin-arm64-native.tar.gz", + "asset_sha256": "ba1da4d349ab70069a1b0775ba671838440b5479cffd1fc2fe41c59816c3df11", + "backend": "native", + "capacity": { + "profile": "development", + "vcpus": 4, + "memory_mib": 6144, + "storage_gib": 32, + "overlay_gib": 10 + } + }, + { + "id": "compose", + "kind": "compose", + "engine": "orbstack", + "compose_version": "5.1.2", + "backend": "docker-compose" + }, + { + "id": "hack-4.1.1", + "kind": "hack", + "version": "4.1.1", + "tag": "v4.1.1", + "commit": "74bf0672f41bd3f68a4ce6cba28b9ba15467c2b2", + "asset": "hack-4.1.1-darwin-arm64.tar.gz", + "asset_sha256": "968f75fe66cd84fdf93b0c5b64ca5df02f98c74281b5a9238125b3afd2f3d72b", + "backend": "compose" + }, + { + "id": "hack-4.2.0", + "kind": "hack", + "version": "4.2.0", + "tag": "v4.2.0", + "commit": "575847486963db94623e1e7f8cd49acca286c517", + "asset": "hack-4.2.0-darwin-arm64.tar.gz", + "asset_sha256": "c62b533aaddbeaafa594cab682cf10041c9bcaea69e2fe25e33e85df49654a85", + "backend": "compose" + }, + { + "id": "hack-4.2.1", + "kind": "hack", + "version": "4.2.1", + "tag": "v4.2.1", + "commit": "5a4c51babb7ef725a30f2f3fd3677081a64bc247", + "asset": "hack-4.2.1-darwin-arm64.tar.gz", + "asset_sha256": "557583d946abcd0dd0a3fa12b2f7b81f18310d594b0699d663ebf239d0d4d729", + "backend": "compose" + } + ], + "workloads": [ + { + "id": "event-agent", + "lanes": [ + "candidate", + "compose", + "hack-4.1.1", + "hack-4.2.0", + "hack-4.2.1" + ], + "pin": "application commit and configuration digest recorded per cohort; identical across lanes", + "shape": { "services": 14, "http_health_checks": 9, "initializers": 2 }, + "correctness": [ + "all declared long-running services running and every HTTP health check healthy", + "both initializers exited 0", + "routed sign-in page returns HTTP 200", + "a per-run data marker written before restart reads back after restart" + ] + }, + { + "id": "small-fixture", + "lanes": [ + "candidate", + "compose", + "hack-4.1.1", + "hack-4.2.0", + "hack-4.2.1" + ], + "pin": "image oven/bun:1.4.2-slim with its image ID recorded per cohort, pull never", + "shape": { "services": 2, "http_health_checks": 1, "initializers": 1 }, + "correctness": [ + "initializer creates the SQLite schema in a named volume and exits 0", + "web health endpoint returns HTTP 200", + "check task inserts a per-run row and reads the expected count", + "the row reads back after restart", + "a source edit is served within the watcher acknowledgement window" + ] + } + ], + "phases": [ + "cold-startup", + "warm-startup", + "restart", + "restore", + "reload", + "build-test", + "exec", + "steady-idle", + "sustained-load", + "shutdown", + "cleanup" + ], + "scale_points": { + "branch_instances": [1, 8, 32], + "mcp_clients": [1, 32], + "judged": { "branch_instances": 1 } + }, + "cache_classes": { + "budget_class": "warm", + "warm": "engine or VM running; images, dependency and build caches populated by a prior correct lifecycle of the same lane; application volumes retained", + "cold-capacity": "engine or VM stopped before the trial; images and caches retained", + "cold-cache": "engine or VM running; images present; dependency and build caches empty" + }, + "schedule": { + "design": "williams", + "rounds_per_workload": 10, + "orders": [ + ["candidate", "compose", "hack-4.2.1", "hack-4.1.1", "hack-4.2.0"], + ["compose", "hack-4.1.1", "candidate", "hack-4.2.0", "hack-4.2.1"], + ["hack-4.1.1", "hack-4.2.0", "compose", "hack-4.2.1", "candidate"], + ["hack-4.2.0", "hack-4.2.1", "hack-4.1.1", "candidate", "compose"], + ["hack-4.2.1", "candidate", "hack-4.2.0", "compose", "hack-4.1.1"], + ["hack-4.2.0", "hack-4.1.1", "hack-4.2.1", "compose", "candidate"], + ["hack-4.2.1", "hack-4.2.0", "candidate", "hack-4.1.1", "compose"], + ["candidate", "hack-4.2.1", "compose", "hack-4.2.0", "hack-4.1.1"], + ["compose", "candidate", "hack-4.1.1", "hack-4.2.1", "hack-4.2.0"], + ["hack-4.1.1", "compose", "hack-4.2.0", "candidate", "hack-4.2.1"] + ] + }, + "samples": { + "min_lifecycle_runs_per_lane": 10, + "p95_quantile": 0.95, + "p95_method": "nearest-rank", + "p95_bound_confidence": 0.95, + "p95_min_samples": 60 + }, + "budgets": { + "comparative": [ + { + "metric": "steady_physical_footprint_bytes", + "statistic": "median", + "max_ratio": 0.7 + }, + { + "metric": "cpu_seconds_per_task", + "statistic": "median", + "max_ratio": 0.75 + }, + { + "metric": "warm_startup_seconds", + "statistic": "median", + "max_ratio": 0.8 + }, + { + "metric": "warm_restart_seconds", + "statistic": "median", + "max_ratio": 0.8 + } + ], + "absolute": [ + { + "metric": "info_latency_ms", + "statistic": "p95", + "max": 100, + "inclusive": true + }, + { + "metric": "warm_admission_ms", + "statistic": "p95", + "max": 250, + "inclusive": true + }, + { + "metric": "cached_status_1000_objects_ms", + "statistic": "p95", + "max": 150, + "inclusive": true + }, + { + "metric": "idle_control_plane_core_fraction", + "statistic": "window_mean", + "max": 0.01, + "inclusive": false + }, + { + "metric": "watcher_ack_ms", + "statistic": "p95", + "max": 200, + "inclusive": true, + "workload": "small-fixture" + } + ] + }, + "observer": { + "control_window_seconds": 600, + "inconclusive_factor": 2 + }, + "known_limits": [ + { + "id": "storage-witness-warm-up", + "issue": "HACK-1123", + "after": "hack-dance/hack#313", + "proofs_per_warm_up": 4, + "warm_up_seconds": 17.7, + "witness_seconds": 8.7, + "witness_target_seconds": 5, + "basis": "estimate from a hosted docker-e2e phase trace, not a revision-1 cohort measurement", + "revisit": "after the first complete revision-1 cohort, or before any proof-policy change" + } + ] +} diff --git a/scripts/check-application-benchmark-protocol.py b/scripts/check-application-benchmark-protocol.py new file mode 100644 index 000000000..bc20e4336 --- /dev/null +++ b/scripts/check-application-benchmark-protocol.py @@ -0,0 +1,835 @@ +#!/usr/bin/env python3 +"""Validate the frozen matched application benchmark protocol and judge retained summaries. + +This helper never runs a benchmark, starts a runtime, calls Docker or reads host state. It: + + (default) validates scripts/application-benchmark-protocol.json against the + protocol's invariants and prints a value-free verdict; + --schedule prints the frozen rotated lane order for each round; + --evaluate SUMMARY judges a retained cohort summary against the frozen budgets. + +Revision-1 identities (artifacts, host, lane kinds and backends, candidate capacity, +workload pins, rotated lane order) are exact; budgets and sample counts are floors at +their frozen values, which a revision may tighten. +Every number must lie within MAX_JSON_NUMBER. +Diagnostics are closed codes from DIAGNOSTICS; they never echo input values or paths. +A budget verdict is one of VERDICTS. A statistic is judged only from complete, +sufficient, admitted samples: missing, short, flagged or unaccounted samples are +inconclusive, never a pass. The candidate is never `not_applicable`, and a comparative +budget with no applicable baseline is inconclusive. Malformed input is refused with a +closed code (exit 2), never a traceback. See docs/reference/application-benchmark-protocol.md. +""" +import argparse +import json +import math +from pathlib import Path +import re +import sys + +DEFAULT_PROTOCOL = Path(__file__).resolve().parent / "application-benchmark-protocol.json" +PROTOCOL_ID = "hack-application-benchmark" +PROTOCOL_REVISION = 1 + +COMMIT = re.compile(r"^[0-9a-f]{40}$") +SHA256 = re.compile(r"^[0-9a-f]{64}$") +VERSION = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.]+)?$") +LANE_ID = re.compile(r"^[a-z0-9][a-z0-9.-]{0,31}$") + +# Program floors (Linear program acceptance, Gate C) and the frozen revision-1 choices. +# A protocol revision may be stricter than these, never weaker. +MIN_ROTATED_LIFECYCLE_RUNS = 5 +# Frozen revision-1 sample counts. These are floors: a revision may plan more, never fewer. +FROZEN_LIFECYCLE_RUNS = 10 +FROZEN_P95_MIN_SAMPLES = 60 +REQUIRED_BASELINES = ("compose", "hack-4.1.1", "hack-4.2.0", "hack-4.2.1") +# metric: (ceiling ratio, frozen statistic). A ratio at the ceiling passes. +COMPARATIVE_CEILINGS = { + "steady_physical_footprint_bytes": (0.70, "median"), + "cpu_seconds_per_task": (0.75, "median"), + "warm_startup_seconds": (0.80, "median"), + "warm_restart_seconds": (0.80, "median"), +} +# metric: (ceiling, whether the ceiling itself passes, frozen statistic, frozen scope). +# "p95 <= 100 ms" versus "< 1% of one core". A scope of None means every workload. A +# scoped budget may be broadened to every workload, but no budget may be narrowed. +ABSOLUTE_CEILINGS = { + "info_latency_ms": (100, True, "p95", None), + "cached_status_1000_objects_ms": (150, True, "p95", None), + "warm_admission_ms": (250, True, "p95", None), + "idle_control_plane_core_fraction": (0.01, False, "window_mean", None), + "watcher_ack_ms": (200, True, "p95", "small-fixture"), +} +STATISTICS = ("median", "p95", "window_mean") +P95_QUANTILE = 0.95 +P95_MIN_CONFIDENCE = 0.95 +P95_METHOD = "nearest-rank" +SCHEDULE_DESIGN = "williams" +MIN_OBSERVER_FACTOR = 2 +MIN_OBSERVER_WINDOW_SECONDS = 600 +WORKLOAD_SHAPES = { + "event-agent": {"services": 14, "http_health_checks": 9, "initializers": 2}, + "small-fixture": {"services": 2, "http_health_checks": 1, "initializers": 1}, +} +# Gate C phases. "restore" is an ordinary up after a retaining down, with data readback. +REQUIRED_PHASES = ( + "cold-startup", "warm-startup", "restart", "restore", "reload", "build-test", "exec", + "steady-idle", "sustained-load", "shutdown", "cleanup", +) +# Gate C scale points. Revision 1 judges budgets at the single-instance lifecycle only. +SCALE_POINTS = {"branch_instances": (1, 8, 32), "mcp_clients": (1, 32)} +JUDGED_SCALE = {"branch_instances": 1} + +# Revision-1 identities. These are exact, not floors: a different artifact, host, lane +# kind or backend, candidate capacity or workload pin is a different experiment and +# needs a new revision (and new constants here), never an edit to revision 1. +FROZEN_HOST = { + "id": "m3-max", + "cpu": "Apple M3 Max", + "logical_cpus": 16, + "memory_bytes": 137438953472, + "os": "macOS 27.0", + "docker_cli": "29.4.0", + "compose": "5.1.2", + "orbstack": {"version": "2.2.3", "build": "20963", "cpu": 16, "memory_mib": 71680}, +} +# Fields that identify the exact released artifact a lane installs. +ARTIFACT_FIELDS = ("version", "tag", "commit", "asset", "asset_sha256") +# Fields that identify how a lane runs. +LANE_RUNTIME_FIELDS = ("kind", "backend", "engine", "compose_version") +FROZEN_LANES = { + "candidate": { + "kind": "hack", "backend": "native", "version": "5.0.0-next.5", "tag": "v5.0.0-next.5", + "commit": "28de2f8902d3776c3da9326e7ae6c6aa6e7f69bc", + "asset": "hack-5.0.0-next.5-darwin-arm64-native.tar.gz", + "asset_sha256": "ba1da4d349ab70069a1b0775ba671838440b5479cffd1fc2fe41c59816c3df11", + }, + "compose": {"kind": "compose", "backend": "docker-compose", "engine": "orbstack", "compose_version": "5.1.2"}, + "hack-4.1.1": { + "kind": "hack", "backend": "compose", "version": "4.1.1", "tag": "v4.1.1", + "commit": "74bf0672f41bd3f68a4ce6cba28b9ba15467c2b2", + "asset": "hack-4.1.1-darwin-arm64.tar.gz", + "asset_sha256": "968f75fe66cd84fdf93b0c5b64ca5df02f98c74281b5a9238125b3afd2f3d72b", + }, + "hack-4.2.0": { + "kind": "hack", "backend": "compose", "version": "4.2.0", "tag": "v4.2.0", + "commit": "575847486963db94623e1e7f8cd49acca286c517", + "asset": "hack-4.2.0-darwin-arm64.tar.gz", + "asset_sha256": "c62b533aaddbeaafa594cab682cf10041c9bcaea69e2fe25e33e85df49654a85", + }, + "hack-4.2.1": { + "kind": "hack", "backend": "compose", "version": "4.2.1", "tag": "v4.2.1", + "commit": "5a4c51babb7ef725a30f2f3fd3677081a64bc247", + "asset": "hack-4.2.1-darwin-arm64.tar.gz", + "asset_sha256": "557583d946abcd0dd0a3fa12b2f7b81f18310d594b0699d663ebf239d0d4d729", + }, +} +# The native `development` profile: graph execution, and so every workload here, +# refuses any other profile (see the protocol page, "Capacities"). +FROZEN_CANDIDATE_CAPACITY = {"profile": "development", "vcpus": 4, "memory_mib": 6144, "storage_gib": 32, + "overlay_gib": 10} +FROZEN_WORKLOAD_PINS = { + "event-agent": "application commit and configuration digest recorded per cohort; identical across lanes", + "small-fixture": "image oven/bun:1.4.2-slim with its image ID recorded per cohort, pull never", +} +# Frozen correctness checks per workload. A revision may add checks, never drop one. +FROZEN_CORRECTNESS = { + "event-agent": ( + "all declared long-running services running and every HTTP health check healthy", + "both initializers exited 0", + "routed sign-in page returns HTTP 200", + "a per-run data marker written before restart reads back after restart", + ), + "small-fixture": ( + "initializer creates the SQLite schema in a named volume and exits 0", + "web health endpoint returns HTTP 200", + "check task inserts a per-run row and reads the expected count", + "the row reads back after restart", + "a source edit is served within the watcher acknowledgement window", + ), +} +# Largest JSON number magnitude accepted anywhere: the I-JSON (RFC 7493) interoperable +# integer range. Bounding every number keeps each statistic and ratio finite and keeps +# Python from converting an unbounded JSON integer to a float. +MAX_JSON_NUMBER = 2 ** 53 - 1 + +DIAGNOSTICS = frozenset({ + "protocol_unreadable", + "protocol_not_object", + "protocol_id_mismatch", + "protocol_revision_mismatch", + "host_incomplete", + "host_mismatch", + "lanes_malformed", + "lane_id_duplicate", + "compose_lane_host_mismatch", + "candidate_lane_missing", + "baseline_lane_missing", + "artifact_commit_malformed", + "artifact_digest_malformed", + "artifact_version_malformed", + "artifact_identity_mismatch", + "lane_runtime_mismatch", + "candidate_capacity_mismatch", + "workloads_malformed", + "workload_lane_unknown", + "workload_baseline_missing", + "workload_shape_mismatch", + "workload_pin_mismatch", + "workload_correctness_weakened", + "phases_mismatch", + "scale_points_mismatch", + "cache_class_missing", + "schedule_design_mismatch", + "schedule_lane_set_mismatch", + "schedule_rounds_below_minimum", + "schedule_rounds_mismatch", + "schedule_position_unbalanced", + "schedule_carryover_unbalanced", + "schedule_order_mismatch", + "p95_samples_insufficient", + "p95_method_mismatch", + "comparative_budget_missing", + "comparative_budget_weakened", + "absolute_budget_missing", + "absolute_budget_weakened", + "budget_statistic_mismatch", + "budget_scope_narrowed", + "statistic_unknown", + "observer_policy_missing", + "known_limit_malformed", + "summary_unreadable", + "summary_not_object", + "summary_revision_mismatch", + "summary_workload_unknown", + "summary_cache_class_mismatch", + "summary_scale_unknown", + "summary_scale_not_judged", + "summary_lane_unknown", + "summary_lane_malformed", + "summary_entry_malformed", + "summary_observer_malformed", + "summary_candidate_not_applicable", +}) + +VERDICTS = frozenset({ + "pass", + "fail", + "not_applicable", + "inconclusive_coverage", + "inconclusive_samples", + "inconclusive_admission", + "inconclusive_observer", + "inconclusive_missing", + "inconclusive_no_baseline", +}) +ACCEPTING_VERDICTS = frozenset({"pass", "not_applicable"}) + +SAMPLE_ENTRY_KEYS = frozenset({"samples", "expected", "flagged", "failed"}) +NOT_APPLICABLE_KEYS = frozenset({"not_applicable", "reason"}) + + +def williams_orders(lanes): + """Williams design: each lane takes each position, and follows each other lane, equally. + + An even lane count needs one square (n rounds); an odd count needs the square plus + its mirror (2n rounds) to balance first-order carryover. + """ + n = len(lanes) + base, low, high = [0], 1, n - 1 + while len(base) < n: + base.append(low) + low += 1 + if len(base) < n: + base.append(high) + high -= 1 + rows = [[(index + shift) % n for index in base] for shift in range(n)] + if n % 2: + rows += [list(reversed(row)) for row in rows] + return [[lanes[index] for index in row] for row in rows] + + +# The frozen revision-1 rotation: the Williams square for this lane sequence. Another +# balanced order is a different schedule, so it needs a new revision. +FROZEN_LANE_SEQUENCE = ("candidate", "compose", "hack-4.1.1", "hack-4.2.0", "hack-4.2.1") +FROZEN_ORDERS = williams_orders(list(FROZEN_LANE_SEQUENCE)) + + +def position_balanced(orders): + lanes = sorted(orders[0]) + for position in range(len(lanes)): + counts = {lane: 0 for lane in lanes} + for order in orders: + counts[order[position]] += 1 + if len(set(counts.values())) != 1: + return False + return True + + +def carryover_balanced(orders): + """Every ordered (predecessor, successor) pair of distinct lanes occurs equally often.""" + lanes = sorted(orders[0]) + pairs = {(a, b): 0 for a in lanes for b in lanes if a != b} + for order in orders: + for a, b in zip(order, order[1:]): + pairs[(a, b)] += 1 + return len(set(pairs.values())) == 1 + + +def minimum_order_statistic_samples(quantile, confidence): + """Smallest n whose sample maximum bounds the true quantile with this confidence. + + P(max of n samples >= true q-quantile) = 1 - q**n, distribution free. + """ + if not (0 < quantile < 1 and 0 < confidence < 1): + raise ValueError("quantile and confidence must be in (0, 1)") + n = math.ceil(math.log(1 - confidence) / math.log(quantile)) + while 1 - quantile ** n < confidence: + n += 1 + while n > 1 and 1 - quantile ** (n - 1) >= confidence: + n -= 1 + return n + + +def nearest_rank(samples, quantile): + ordered = sorted(samples) + return ordered[max(1, math.ceil(quantile * len(ordered))) - 1] + + +def median(samples): + ordered = sorted(samples) + middle = len(ordered) // 2 + if len(ordered) % 2: + return ordered[middle] + return (ordered[middle - 1] + ordered[middle]) / 2 + + +def _number(value): + """A finite JSON number within MAX_JSON_NUMBER; a boolean is not a number. + + The range comparison never converts an int to a float, so an oversized integer is + refused rather than raising OverflowError, and NaN and infinities compare false. + """ + return isinstance(value, (int, float)) and not isinstance(value, bool) \ + and -MAX_JSON_NUMBER <= value <= MAX_JSON_NUMBER + + +def _same(value, frozen): + """Exact JSON equality: unlike ==, 4.0 is not 4 and true is not 1.""" + return json.dumps(value, sort_keys=True) == json.dumps(frozen, sort_keys=True) + + +def _count(value): + """A non-negative JSON integer; a boolean is not a count.""" + return isinstance(value, int) and not isinstance(value, bool) and value >= 0 + + +def _strings(value): + return isinstance(value, list) and all(isinstance(item, str) for item in value) + + +def _check_host(host, found): + fields = ("id", "cpu", "logical_cpus", "memory_bytes", "os", "docker_cli", "compose", "orbstack") + if not isinstance(host, dict) or any(not host.get(field) for field in fields): + found.add("host_incomplete") + elif not isinstance(host["orbstack"], dict) or any( + not host["orbstack"].get(field) for field in ("version", "build", "cpu", "memory_mib")): + found.add("host_incomplete") + # Native stateful results are host-specific: any other host or engine is a new revision. + if not _same(host, FROZEN_HOST): + found.add("host_mismatch") + + +def _check_lane_identity(lane_id, lane, found): + """Refuse a frozen lane whose artifact, runtime or capacity differs from revision 1. + + A missing field counts as different, so removing an identity cannot weaken it. + """ + frozen = FROZEN_LANES.get(lane_id) + if frozen is None: + return + for field in ARTIFACT_FIELDS + LANE_RUNTIME_FIELDS: + if (field in frozen or field in lane) and not _same(lane.get(field), frozen.get(field)): + found.add("artifact_identity_mismatch" if field in ARTIFACT_FIELDS else "lane_runtime_mismatch") + if lane_id == "candidate": + if not _same(lane.get("capacity"), FROZEN_CANDIDATE_CAPACITY): + found.add("candidate_capacity_mismatch") + elif "capacity" in lane: + # Baselines run with the shared engine's default capacity; a per-lane capacity + # would make them a different experiment. + found.add("lane_runtime_mismatch") + + +def _check_lanes(lanes, host, found): + """Return the well-formed lane ids, in order.""" + lane_ids = [] + if not isinstance(lanes, list) or not lanes or not all(isinstance(lane, dict) for lane in lanes): + found.add("lanes_malformed") + lanes = [] + for lane in lanes: + lane_id = lane.get("id") + if not isinstance(lane_id, str) or not LANE_ID.fullmatch(lane_id): + found.add("lanes_malformed") + continue + if lane_id in lane_ids: + found.add("lane_id_duplicate") + lane_ids.append(lane_id) + if lane.get("kind") == "hack": + if not isinstance(lane.get("commit"), str) or not COMMIT.fullmatch(lane["commit"]): + found.add("artifact_commit_malformed") + if not isinstance(lane.get("asset_sha256"), str) or not SHA256.fullmatch(lane["asset_sha256"]): + found.add("artifact_digest_malformed") + if not isinstance(lane.get("version"), str) or not VERSION.fullmatch(lane["version"]) \ + or lane.get("tag") != "v" + lane["version"]: + found.add("artifact_version_malformed") + elif lane.get("kind") == "compose": + if not lane.get("engine") or not lane.get("compose_version"): + found.add("lanes_malformed") + # The direct Compose lane is the frozen host's own engine and Compose. + elif not isinstance(host, dict) or lane["engine"] != "orbstack" \ + or lane["compose_version"] != host.get("compose"): + found.add("compose_lane_host_mismatch") + else: + found.add("lanes_malformed") + _check_lane_identity(lane_id, lane, found) + if "candidate" not in lane_ids: + found.add("candidate_lane_missing") + if any(baseline not in lane_ids for baseline in REQUIRED_BASELINES): + found.add("baseline_lane_missing") + return lane_ids + + +def _check_workloads(workloads, lane_ids, found): + """Return the workload ids when every workload has a distinct string id; otherwise [].""" + if not isinstance(workloads, list) or not all(isinstance(w, dict) for w in workloads): + found.add("workloads_malformed") + return [] + ids = [w.get("id") for w in workloads] + if not _strings(ids) or len(set(ids)) != len(ids): + found.add("workloads_malformed") + ids = [] + if not set(WORKLOAD_SHAPES) <= set(ids): + found.add("workloads_malformed") + for workload in workloads: + applicable = workload.get("lanes") + if not _strings(applicable) or len(set(applicable)) != len(applicable) or "candidate" not in applicable: + found.add("workloads_malformed") + else: + if any(lane not in lane_ids for lane in applicable): + found.add("workload_lane_unknown") + # Every workload is compared against every required baseline; dropping one + # from a workload would silently drop that workload's comparison. + if any(baseline not in applicable for baseline in REQUIRED_BASELINES): + found.add("workload_baseline_missing") + if not _strings(workload.get("correctness")) or not workload["correctness"]: + found.add("workloads_malformed") + workload_id = workload.get("id") + shape = WORKLOAD_SHAPES.get(workload_id) if isinstance(workload_id, str) else None + if shape is not None and not _same(workload.get("shape"), shape): + found.add("workload_shape_mismatch") + if isinstance(workload_id, str) and workload_id in FROZEN_WORKLOAD_PINS: + if not _same(workload.get("pin"), FROZEN_WORKLOAD_PINS[workload_id]): + found.add("workload_pin_mismatch") + checks = workload.get("correctness") + if not _strings(checks) or not set(FROZEN_CORRECTNESS[workload_id]) <= set(checks): + found.add("workload_correctness_weakened") + return ids + + +def _check_phases_and_scale(doc, found): + phases = doc.get("phases") + if not _strings(phases) or len(set(phases)) != len(phases) or not set(REQUIRED_PHASES) <= set(phases): + found.add("phases_mismatch") + scale = doc.get("scale_points") + if not isinstance(scale, dict) or scale.get("judged") != JUDGED_SCALE: + found.add("scale_points_mismatch") + return + for kind, required in SCALE_POINTS.items(): + points = scale.get(kind) + if not isinstance(points, list) or not all(_count(point) and point > 0 for point in points) \ + or not set(required) <= set(points): + found.add("scale_points_mismatch") + + +def _check_schedule(doc, lane_ids, found): + schedule = doc.get("schedule") if isinstance(doc.get("schedule"), dict) else {} + if schedule.get("design") != SCHEDULE_DESIGN: + found.add("schedule_design_mismatch") + orders = schedule.get("orders") + if not isinstance(orders, list) or not orders or not all(_strings(order) for order in orders) \ + or any(sorted(order) != sorted(lane_ids) or len(set(order)) != len(order) for order in orders): + found.add("schedule_lane_set_mismatch") + return + rounds = schedule.get("rounds_per_workload") + if not _count(rounds) or rounds != len(orders): + found.add("schedule_rounds_mismatch") + samples = doc.get("samples") if isinstance(doc.get("samples"), dict) else {} + planned = samples.get("min_lifecycle_runs_per_lane") + if not _count(planned) or planned < max(MIN_ROTATED_LIFECYCLE_RUNS, FROZEN_LIFECYCLE_RUNS) \ + or len(orders) < planned: + found.add("schedule_rounds_below_minimum") + if not position_balanced(orders): + found.add("schedule_position_unbalanced") + if not carryover_balanced(orders): + found.add("schedule_carryover_unbalanced") + if orders != FROZEN_ORDERS: + found.add("schedule_order_mismatch") + + +def _check_samples(doc, found): + samples = doc.get("samples") if isinstance(doc.get("samples"), dict) else {} + quantile = samples.get("p95_quantile") + confidence = samples.get("p95_bound_confidence") + floor = samples.get("p95_min_samples") + if not _number(quantile) or quantile != P95_QUANTILE \ + or not _number(confidence) or not P95_MIN_CONFIDENCE <= confidence < 1 \ + or not _count(floor) or floor < FROZEN_P95_MIN_SAMPLES \ + or floor < minimum_order_statistic_samples(P95_QUANTILE, confidence): + found.add("p95_samples_insufficient") + if samples.get("p95_method") != P95_METHOD: + found.add("p95_method_mismatch") + + +def _budget_index(items, kind, found): + """Index a budget list by metric; malformed or duplicate entries weaken the list.""" + if not isinstance(items, list): + found.add(f"{kind}_budget_missing") + return {} + index = {} + for item in items: + if not isinstance(item, dict) or not isinstance(item.get("metric"), str) or item["metric"] in index: + found.add(f"{kind}_budget_weakened") + continue + index[item["metric"]] = item + return index + + +def _check_budgets(doc, workload_ids, found): + budgets = doc.get("budgets") if isinstance(doc.get("budgets"), dict) else {} + comparative = _budget_index(budgets.get("comparative"), "comparative", found) + absolute = _budget_index(budgets.get("absolute"), "absolute", found) + # Structural checks for every listed budget, frozen or additional: the evaluator + # judges each listed budget, so each must be judgeable. + for budget in list(comparative.values()) + list(absolute.values()): + if budget.get("statistic") not in STATISTICS: + found.add("statistic_unknown") + for budget in comparative.values(): + if not _number(budget.get("max_ratio")): + found.add("comparative_budget_weakened") + for budget in absolute.values(): + if not _number(budget.get("max")) or not isinstance(budget.get("inclusive"), bool): + found.add("absolute_budget_weakened") + scope = budget.get("workload") + if scope is not None and scope not in workload_ids: + found.add("budget_scope_narrowed") + + for metric, (ceiling, statistic) in COMPARATIVE_CEILINGS.items(): + budget = comparative.get(metric) + if budget is None: + found.add("comparative_budget_missing") + continue + if _number(budget.get("max_ratio")) and budget["max_ratio"] > ceiling: + found.add("comparative_budget_weakened") + if budget.get("statistic") in STATISTICS and budget["statistic"] != statistic: + found.add("budget_statistic_mismatch") + for metric, (ceiling, ceiling_inclusive, statistic, scope) in ABSOLUTE_CEILINGS.items(): + budget = absolute.get(metric) + if budget is None: + found.add("absolute_budget_missing") + continue + if _number(budget.get("max")) and isinstance(budget.get("inclusive"), bool) and ( + budget["max"] > ceiling + or (budget["max"] == ceiling and budget["inclusive"] and not ceiling_inclusive)): + found.add("absolute_budget_weakened") + if budget.get("statistic") in STATISTICS and budget["statistic"] != statistic: + found.add("budget_statistic_mismatch") + if budget.get("workload") not in (None, scope): + found.add("budget_scope_narrowed") + + +def validate_protocol(doc): + """Return sorted closed diagnostic codes; empty means the protocol is valid. + + Never raises on any JSON value: every malformed node maps to a closed code. + """ + if not isinstance(doc, dict): + return ["protocol_not_object"] + found = set() + if doc.get("protocol") != PROTOCOL_ID: + found.add("protocol_id_mismatch") + if not _count(doc.get("revision")) or doc["revision"] != PROTOCOL_REVISION: + found.add("protocol_revision_mismatch") + _check_host(doc.get("host"), found) + lane_ids = _check_lanes(doc.get("lanes"), doc.get("host"), found) + workload_ids = _check_workloads(doc.get("workloads"), lane_ids, found) + _check_phases_and_scale(doc, found) + + cache = doc.get("cache_classes") + if not isinstance(cache, dict) or not {"warm", "cold-capacity", "cold-cache"} <= set(cache) \ + or cache.get("budget_class") != "warm": + found.add("cache_class_missing") + + _check_schedule(doc, lane_ids, found) + _check_samples(doc, found) + _check_budgets(doc, workload_ids, found) + + observer = doc.get("observer") + if not isinstance(observer, dict) or not _number(observer.get("inconclusive_factor")) \ + or observer["inconclusive_factor"] < MIN_OBSERVER_FACTOR \ + or not _number(observer.get("control_window_seconds")) \ + or observer["control_window_seconds"] < MIN_OBSERVER_WINDOW_SECONDS: + found.add("observer_policy_missing") + + limits = doc.get("known_limits", []) + if not isinstance(limits, list): + found.add("known_limit_malformed") + limits = [] + for limit in limits: + if not isinstance(limit, dict) or not all( + _number(limit.get(field)) for field in ("warm_up_seconds", "witness_seconds", "witness_target_seconds")) \ + or not limit.get("issue") or not limit.get("revisit"): + found.add("known_limit_malformed") + return sorted(found) + + +def _entry_malformed(entry): + """True when a budget-metric entry is neither a sample record nor a not-applicable record. + + A sample record is {"samples": [non-negative numbers], "expected": positive int, + "flagged": count, "failed": count}; samples hold only admitted samples, and flagged + or failed lifecycles are counted, not listed. A not-applicable record is exactly + {"not_applicable": true, "reason": non-empty string}: it can never carry samples. + """ + if not isinstance(entry, dict): + return True + if "not_applicable" in entry: + return set(entry) != NOT_APPLICABLE_KEYS or entry["not_applicable"] is not True \ + or not isinstance(entry["reason"], str) or not entry["reason"].strip() + if not set(entry) <= SAMPLE_ENTRY_KEYS: + return True + samples = entry.get("samples") + if not isinstance(samples, list) or not all(_number(value) and value >= 0 for value in samples): + return True + if any(field in entry and not _count(entry[field]) for field in ("flagged", "failed")): + return True + expected = entry.get("expected") + if expected is None: + return False + if not _count(expected) or expected == 0: + return True + # More accounted samples than planned means unplanned reruns, which could select results. + return len(samples) + entry.get("flagged", 0) + entry.get("failed", 0) > expected + + +def _scale_known(protocol, scale): + if not isinstance(scale, dict) or len(scale) != 1: + return False + ((kind, count),) = scale.items() + points = protocol["scale_points"].get(kind) if kind in SCALE_POINTS else None + return isinstance(points, list) and _count(count) and count in points + + +def _budget_metrics(protocol): + return {budget["metric"] for kind in ("comparative", "absolute") for budget in protocol["budgets"][kind]} + + +def validate_summary(protocol, summary): + """Return (sorted closed diagnostics, workload); a summary with diagnostics is never judged. + + `protocol` must already be valid. Never raises on any JSON value. + """ + if not isinstance(summary, dict): + return ["summary_not_object"], None + if not _count(summary.get("protocol_revision")) or summary["protocol_revision"] != protocol["revision"]: + return ["summary_revision_mismatch"], None + workload_id = summary.get("workload") + workloads = {w["id"]: w for w in protocol["workloads"]} + workload = workloads.get(workload_id) if isinstance(workload_id, str) else None + if workload is None: + return ["summary_workload_unknown"], None + found = set() + # Budgets are frozen for the warm class only; other classes are reported separately. + if summary.get("cache_class") != protocol["cache_classes"]["budget_class"]: + found.add("summary_cache_class_mismatch") + if not _scale_known(protocol, summary.get("scale")): + found.add("summary_scale_unknown") + elif summary["scale"] != protocol["scale_points"]["judged"]: + found.add("summary_scale_not_judged") + + observer = summary.get("observer_cost", {}) + if not isinstance(observer, dict) or not all(_number(cost) and cost >= 0 for cost in observer.values()): + found.add("summary_observer_malformed") + + lanes = summary.get("lanes") + if not isinstance(lanes, dict): + found.add("summary_lane_malformed") + return sorted(found), workload + metrics = _budget_metrics(protocol) + for lane_id, lane in lanes.items(): + if lane_id not in workload["lanes"]: + found.add("summary_lane_unknown") + if not isinstance(lane, dict): + found.add("summary_lane_malformed") + continue + for metric in metrics & set(lane): + if _entry_malformed(lane[metric]): + found.add("summary_entry_malformed") + elif lane_id == "candidate" and "not_applicable" in lane[metric]: + # The candidate is the subject of every budget: it cannot opt out of one. + found.add("summary_candidate_not_applicable") + return sorted(found), workload + + +def _statistic(samples, statistic): + if statistic == "median": + return median(samples) + if statistic == "p95": + return nearest_rank(samples, P95_QUANTILE) + return sum(samples) / len(samples) + + +def _sample_floor(protocol, statistic): + if statistic == "p95": + return protocol["samples"]["p95_min_samples"] + return protocol["samples"]["min_lifecycle_runs_per_lane"] + + +def _judge(protocol, entry, statistic, candidate): + """Return (statistic value or None, verdict or None) for a validated entry. + + A None verdict means the value is judgeable. A failed candidate lifecycle is a + failure; a flagged sample, an unaccounted admission or a failed baseline lifecycle + leaves the statistic inconclusive. + """ + if entry is None: + return None, "inconclusive_missing" + if "not_applicable" in entry: + return None, "not_applicable" + if candidate and entry.get("failed"): + return None, "fail" + expected = entry.get("expected") + if expected is None: + return None, "inconclusive_missing" + if not _count(entry.get("flagged")) or not _count(entry.get("failed")) or entry["flagged"] or entry["failed"]: + return None, "inconclusive_admission" + samples = entry["samples"] + if len(samples) < expected: + return None, "inconclusive_coverage" + if len(samples) < _sample_floor(protocol, statistic): + return None, "inconclusive_samples" + return _statistic(samples, statistic), None + + +def evaluate(protocol, summary): + """Judge one workload's retained summary. Returns (diagnostics, verdict records). + + Comparative budgets are judged against each applicable baseline separately; there is no + aggregate. A candidate/baseline difference smaller than inconclusive_factor times the + observer's own cost for that metric is inconclusive rather than a pass or a fail. A + comparative budget whose baselines are all not_applicable gets one extra + inconclusive_no_baseline record, so it can never be accepted without a comparison. + """ + diagnostics, workload = validate_summary(protocol, summary) + if diagnostics: + return diagnostics, [] + lanes = summary["lanes"] + observer = summary.get("observer_cost", {}) + factor = protocol["observer"]["inconclusive_factor"] + candidate = lanes.get("candidate", {}) + records = [] + + for budget in protocol["budgets"]["comparative"]: + metric, statistic = budget["metric"], budget["statistic"] + cand_value, cand_problem = _judge(protocol, candidate.get(metric), statistic, candidate=True) + applicable = 0 + for baseline in (lane for lane in workload["lanes"] if lane != "candidate"): + record = {"workload": workload["id"], "metric": metric, "statistic": statistic, + "baseline": baseline, "max_ratio": budget["max_ratio"]} + base_value, base_problem = _judge(protocol, lanes.get(baseline, {}).get(metric), statistic, + candidate=False) + if base_problem == "not_applicable": + record["verdict"] = "not_applicable" + records.append(record) + continue + applicable += 1 + problem = cand_problem or base_problem + cost = observer.get(metric) + if problem is None and cost is None: + problem = "inconclusive_observer" + if problem is None and base_value <= 0: + problem = "inconclusive_missing" + ratio = None + if problem is None: + ratio = cand_value / base_value + # A tiny positive baseline can overflow the ratio to infinity: that is a + # failure, but only a finite ratio is printed, so the report stays strict JSON. + if math.isfinite(ratio): + record["ratio"] = ratio + if abs(cand_value - base_value) < factor * cost: + problem = "inconclusive_observer" + record["verdict"] = problem or ("pass" if ratio <= budget["max_ratio"] else "fail") + records.append(record) + if applicable == 0: + records.append({"workload": workload["id"], "metric": metric, "statistic": statistic, + "max_ratio": budget["max_ratio"], "applicable_baselines": 0, + "verdict": "fail" if cand_problem == "fail" else "inconclusive_no_baseline"}) + + for budget in protocol["budgets"]["absolute"]: + if budget.get("workload") not in (None, workload["id"]): + continue + metric, statistic = budget["metric"], budget["statistic"] + record = {"workload": workload["id"], "metric": metric, "statistic": statistic, "max": budget["max"]} + value, problem = _judge(protocol, candidate.get(metric), statistic, candidate=True) + if problem is None: + record["value"] = value + within = value <= budget["max"] if budget["inclusive"] else value < budget["max"] + problem = "pass" if within else "fail" + record["verdict"] = problem + records.append(record) + return [], records + + +def accepted(records): + """A summary is accepted only when it has verdicts and every one is pass or not_applicable.""" + return bool(records) and all(record["verdict"] in ACCEPTING_VERDICTS for record in records) + + +def _load(path, unreadable): + try: + return json.loads(Path(path).read_text()), None + except (OSError, ValueError, UnicodeDecodeError, RecursionError): + return None, unreadable + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--protocol", default=str(DEFAULT_PROTOCOL)) + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--schedule", action="store_true") + mode.add_argument("--evaluate", metavar="SUMMARY") + args = parser.parse_args(argv) + + protocol, problem = _load(args.protocol, "protocol_unreadable") + diagnostics = [problem] if problem else validate_protocol(protocol) + if diagnostics: + print(json.dumps({"status": "invalid", "diagnostics": diagnostics})) + return 2 + if args.schedule: + print(json.dumps({"protocol": PROTOCOL_ID, "revision": PROTOCOL_REVISION, + "orders": protocol["schedule"]["orders"]}, indent=2)) + return 0 + if args.evaluate: + summary, problem = _load(args.evaluate, "summary_unreadable") + diagnostics, records = ([problem], []) if problem else evaluate(protocol, summary) + if diagnostics: + print(json.dumps({"status": "invalid", "diagnostics": diagnostics})) + return 2 + ok = accepted(records) + print(json.dumps({"status": "judged", "accepted": ok, "verdicts": records}, indent=2)) + return 0 if ok else 1 + print(json.dumps({"status": "valid", "protocol": PROTOCOL_ID, "revision": PROTOCOL_REVISION, + "lanes": len(protocol["lanes"]), "rounds": len(protocol["schedule"]["orders"])})) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/python/test_application_benchmark_protocol.py b/tests/python/test_application_benchmark_protocol.py new file mode 100644 index 000000000..4ac0b0af9 --- /dev/null +++ b/tests/python/test_application_benchmark_protocol.py @@ -0,0 +1,706 @@ +"""Offline controls for the frozen matched application benchmark protocol; no live runs.""" +import contextlib +import copy +import importlib.util +import io +import json +from pathlib import Path +import re +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[2] +SOURCE = ROOT / "scripts/check-application-benchmark-protocol.py" +SPEC = importlib.util.spec_from_file_location("application_benchmark_protocol", SOURCE) +protocol_check = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(protocol_check) + +PROTOCOL = json.loads((ROOT / "scripts/application-benchmark-protocol.json").read_text()) +PROTOCOL_DOC = (ROOT / "docs/reference/application-benchmark-protocol.md").read_text() +PERFORMANCE_DOC = (ROOT / "docs/performance.md").read_text() +MARKER = "PRIVATE-MARKER-9f2c" +BASELINES = ("compose", "hack-4.1.1", "hack-4.2.0", "hack-4.2.1") +COMPARATIVE = ("steady_physical_footprint_bytes", "cpu_seconds_per_task", "warm_startup_seconds", + "warm_restart_seconds") +ABSOLUTE = ("info_latency_ms", "warm_admission_ms", "cached_status_1000_objects_ms", + "idle_control_plane_core_fraction", "watcher_ack_ms") +# Every JSON value type, including the non-standard numbers Python's json module accepts +# and integers too large for a float, which Python's json module also parses. +HUGE = 10 ** 400 +FUZZ_VALUES = (None, True, 0, -1, 1.5, float("nan"), float("inf"), float("-inf"), HUGE, -HUGE, 2 ** 53, + "x", [], ["x"], [["x"]], {}, {"x": []}) +# Another well-formed 40-hex commit and 64-hex digest: only the identity pin refuses them. +OTHER_COMMIT = "0123456789abcdef0123456789abcdef01234567" +OTHER_DIGEST = "0123456789abcdef" * 4 + + +def mutated(change, doc=None): + doc = copy.deepcopy(PROTOCOL if doc is None else doc) + change(doc) + return doc + + +def lane(doc, lane_id): + return next(item for item in doc["lanes"] if item["id"] == lane_id) + + +def workload(doc, workload_id): + return next(item for item in doc["workloads"] if item["id"] == workload_id) + + +def budget(doc, kind, metric): + return next(item for item in doc["budgets"][kind] if item["metric"] == metric) + + +def entry(values, expected=None, flagged=0, failed=0): + return {"samples": list(values), "expected": len(values) if expected is None else expected, + "flagged": flagged, "failed": failed} + + +def not_applicable(): + return {"not_applicable": True, "reason": "lane cannot express this workload"} + + +def small_summary(candidate_scale=0.5, runs=10, tail=60): + """A complete small-fixture summary where the candidate is `candidate_scale` of each baseline.""" + base = {"steady_physical_footprint_bytes": 1000.0, "cpu_seconds_per_task": 10.0, + "warm_startup_seconds": 20.0, "warm_restart_seconds": 10.0} + lanes = {item: {metric: entry([value] * runs) for metric, value in base.items()} for item in BASELINES} + lanes["candidate"] = {metric: entry([value * candidate_scale] * runs) for metric, value in base.items()} + lanes["candidate"].update({ + "info_latency_ms": entry([50] * tail), + "warm_admission_ms": entry([100] * tail), + "cached_status_1000_objects_ms": entry([100] * tail), + "idle_control_plane_core_fraction": entry([0.005] * runs), + "watcher_ack_ms": entry([100] * tail), + }) + return {"protocol_revision": 1, "workload": "small-fixture", "cache_class": "warm", + "scale": {"branch_instances": 1}, + "observer_cost": {metric: 0.0 for metric in base}, "lanes": lanes} + + +def verdicts(records, metric, baseline=None): + return [record["verdict"] for record in records + if record["metric"] == metric and record.get("baseline", baseline) == baseline] + + +def run_cli(argv): + out = io.StringIO() + with contextlib.redirect_stdout(out): + code = protocol_check.main(argv) + return code, json.loads(out.getvalue()) + + +def evaluate_file(summary): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "summary.json" + path.write_text(json.dumps(summary)) + return run_cli(["--evaluate", str(path)]) + + +def node_paths(node, prefix=(), list_limit=None): + """Yield the path of every node below `node`; `list_limit` caps indices per list.""" + if isinstance(node, dict): + children = node.items() + elif isinstance(node, list): + children = list(enumerate(node))[:list_limit] + else: + return + for key, child in children: + yield prefix + (key,) + yield from node_paths(child, prefix + (key,), list_limit) + + +def replace_at(doc, path, value): + doc = copy.deepcopy(doc) + parent = doc + for key in path[:-1]: + parent = parent[key] + if value is DELETE: + del parent[path[-1]] + else: + parent[path[-1]] = copy.deepcopy(value) + return doc + + +DELETE = object() + + +class FrozenProtocolTest(unittest.TestCase): + def test_frozen_protocol_is_valid(self): + self.assertEqual(protocol_check.validate_protocol(PROTOCOL), []) + + def test_frozen_schedule_is_the_williams_design_for_the_frozen_lanes(self): + lanes = [item["id"] for item in PROTOCOL["lanes"]] + self.assertEqual(PROTOCOL["schedule"]["orders"], protocol_check.williams_orders(lanes)) + self.assertEqual(PROTOCOL["schedule"]["orders"], protocol_check.FROZEN_ORDERS) + self.assertEqual(PROTOCOL["samples"]["min_lifecycle_runs_per_lane"], protocol_check.FROZEN_LIFECYCLE_RUNS) + self.assertEqual(PROTOCOL["samples"]["p95_min_samples"], protocol_check.FROZEN_P95_MIN_SAMPLES) + self.assertEqual(PROTOCOL["schedule"]["rounds_per_workload"], len(PROTOCOL["schedule"]["orders"])) + self.assertGreaterEqual(len(PROTOCOL["schedule"]["orders"]), protocol_check.MIN_ROTATED_LIFECYCLE_RUNS) + self.assertEqual(PROTOCOL["samples"]["min_lifecycle_runs_per_lane"], len(PROTOCOL["schedule"]["orders"])) + + def test_williams_design_balances_position_and_carryover(self): + for count in range(2, 9): + orders = protocol_check.williams_orders([f"lane{index}" for index in range(count)]) + self.assertEqual(len(orders), count * (2 if count % 2 else 1)) + self.assertTrue(protocol_check.position_balanced(orders), count) + self.assertTrue(protocol_check.carryover_balanced(orders), count) + + def test_p95_sample_floor_is_the_distribution_free_bound(self): + required = protocol_check.minimum_order_statistic_samples(0.95, 0.95) + self.assertEqual(required, 59) + self.assertLess(1 - 0.95 ** (required - 1), 0.95) + self.assertGreaterEqual(1 - 0.95 ** required, 0.95) + self.assertGreaterEqual(PROTOCOL["samples"]["p95_min_samples"], required) + + def test_nearest_rank_and_median(self): + self.assertEqual(protocol_check.nearest_rank(list(range(1, 11)), 0.95), 10) + self.assertEqual(protocol_check.nearest_rank(list(range(61, 0, -1)), 0.95), 58) + self.assertEqual(protocol_check.nearest_rank(list(range(1, 61)), 0.95), 57) + self.assertEqual(protocol_check.median([3, 1, 2]), 2) + self.assertEqual(protocol_check.median([4, 1, 3, 2]), 2.5) + + def test_frozen_gate_c_coverage(self): + self.assertIn("restore", PROTOCOL["phases"]) + self.assertEqual(PROTOCOL["scale_points"]["branch_instances"], [1, 8, 32]) + self.assertEqual(PROTOCOL["scale_points"]["mcp_clients"], [1, 32]) + scopes = {item["metric"]: item.get("workload") for item in PROTOCOL["budgets"]["absolute"]} + # Gate C limits only the watcher budget to the small fixture. + self.assertEqual(scopes, {metric: None for metric in ABSOLUTE} | {"watcher_ack_ms": "small-fixture"}) + for item in PROTOCOL["workloads"]: + self.assertEqual(sorted(item["lanes"]), sorted(("candidate",) + BASELINES)) + + def test_frozen_file_matches_the_pinned_identities(self): + for lane_id, frozen in protocol_check.FROZEN_LANES.items(): + item = lane(PROTOCOL, lane_id) + for field, value in frozen.items(): + self.assertEqual(item[field], value, (lane_id, field)) + self.assertEqual(lane(PROTOCOL, "candidate")["capacity"], protocol_check.FROZEN_CANDIDATE_CAPACITY) + self.assertEqual(PROTOCOL["host"], protocol_check.FROZEN_HOST) + for workload_id, pin in protocol_check.FROZEN_WORKLOAD_PINS.items(): + self.assertEqual(workload(PROTOCOL, workload_id)["pin"], pin) + self.assertEqual(workload(PROTOCOL, workload_id)["correctness"], + list(protocol_check.FROZEN_CORRECTNESS[workload_id])) + + def test_documents_carry_every_frozen_artifact_and_budget(self): + # Driven by the validator's pins, not the JSON, so a removed field is still required. + for lane_id, frozen in protocol_check.FROZEN_LANES.items(): + if frozen["kind"] != "hack": + continue + for field in ("commit", "asset", "asset_sha256", "tag"): + self.assertIn(f"`{frozen[field]}`", PROTOCOL_DOC, (lane_id, field)) + capacity = protocol_check.FROZEN_CANDIDATE_CAPACITY + self.assertIn(f"native `{capacity['profile']}` profile: {capacity['vcpus']} vCPUs, " + f"{capacity['memory_mib']} MiB guest memory, {capacity['storage_gib']} GiB storage, " + f"{capacity['overlay_gib']} GiB overlay", PROTOCOL_DOC) + # The candidate's own source labels this profile non-qualifying; the page must say why it is used. + for phrase in ("experimental-development-not-benchmark-qualified", "graph_profile", "`research`"): + self.assertIn(phrase, PROTOCOL_DOC) + host = PROTOCOL["host"] + for value in (host["cpu"], host["os"], host["docker_cli"], host["compose"], + host["orbstack"]["version"], host["orbstack"]["build"]): + self.assertIn(str(value), PROTOCOL_DOC) + for item in PROTOCOL["budgets"]["comparative"]: + self.assertIn(f"`{item['metric']}`", PROTOCOL_DOC) + self.assertIn(f"{item['max_ratio']:.2f}", PROTOCOL_DOC) + for item in PROTOCOL["budgets"]["absolute"]: + self.assertIn(f"`{item['metric']}`", PROTOCOL_DOC) + for phase in PROTOCOL["phases"]: + self.assertIn(f"`{phase}`", PROTOCOL_DOC) + self.assertIn("1, 8 and 32 branch instances", PROTOCOL_DOC) + self.assertIn("1 and 32 MCP clients", PROTOCOL_DOC) + for code in sorted(protocol_check.VERDICTS): + self.assertIn(f"`{code}`", PROTOCOL_DOC) + self.assertIn(f"at least {protocol_check.FROZEN_P95_MIN_SAMPLES} samples per lane", PROTOCOL_DOC) + self.assertIn(f"at least {protocol_check.FROZEN_LIFECYCLE_RUNS} lifecycles per lane", PROTOCOL_DOC) + # The published order table is the frozen rotation, round for round. + rows = re.findall(r"^\| (\d+) \| ([a-z0-9., -]+) \|$", PROTOCOL_DOC, re.MULTILINE) + self.assertEqual([int(number) for number, _ in rows], + list(range(1, len(protocol_check.FROZEN_ORDERS) + 1))) + self.assertEqual([order.split(", ") for _, order in rows], protocol_check.FROZEN_ORDERS) + limit = PROTOCOL["known_limits"][0] + for value in (limit["issue"], "#313", f"{limit['warm_up_seconds']} s", f"{limit['witness_seconds']} s", + f"{limit['witness_target_seconds']} s", "HACK-1169"): + self.assertIn(value, PERFORMANCE_DOC) + self.assertIn("application-benchmark-protocol.md", PERFORMANCE_DOC) + + +def _recast_as_compose(item): + for field in ("version", "commit", "asset_sha256", "tag", "asset", "capacity"): + item.pop(field, None) + item.update(kind="compose", engine="orbstack", compose_version="5.1.2") + + +def _reorder_schedule(doc): + doc["lanes"][1], doc["lanes"][2] = doc["lanes"][2], doc["lanes"][1] + doc["schedule"]["orders"] = protocol_check.williams_orders([item["id"] for item in doc["lanes"]]) + + +def _set_scope(metric, scope): + def change(doc): + budget(doc, "absolute", metric)["workload"] = scope + return change + + +class NegativeProtocolControls(unittest.TestCase): + CASES = { + "comparative_budget_weakened": lambda d: budget(d, "comparative", "steady_physical_footprint_bytes") + .update(max_ratio=0.71), + # A weaker duplicate listed before the frozen entry must not hide behind it. + "comparative_budget_weakened:duplicate": lambda d: d["budgets"]["comparative"].insert( + 0, {"metric": "cpu_seconds_per_task", "statistic": "median", "max_ratio": 0.9}), + "absolute_budget_weakened": lambda d: budget(d, "absolute", "idle_control_plane_core_fraction") + .update(inclusive=True), + "absolute_budget_weakened:ceiling": lambda d: budget(d, "absolute", "info_latency_ms").update(max=1000), + "absolute_budget_weakened:ceiling-exclusive": lambda d: budget(d, "absolute", "watcher_ack_ms") + .update(max=201, inclusive=False), + "comparative_budget_missing": lambda d: d["budgets"]["comparative"].pop(), + "absolute_budget_missing": lambda d: d["budgets"]["absolute"].pop(), + "statistic_unknown": lambda d: budget(d, "absolute", "info_latency_ms").update(statistic="mean"), + "budget_statistic_mismatch:absolute": lambda d: budget(d, "absolute", "info_latency_ms") + .update(statistic="median"), + "budget_statistic_mismatch:comparative": lambda d: budget(d, "comparative", "cpu_seconds_per_task") + .update(statistic="window_mean"), + "budget_statistic_mismatch:idle": lambda d: budget(d, "absolute", "idle_control_plane_core_fraction") + .update(statistic="median"), + "budget_scope_narrowed:watcher": _set_scope("watcher_ack_ms", "event-agent"), + "budget_scope_narrowed:info": _set_scope("info_latency_ms", "small-fixture"), + "budget_scope_narrowed:idle": _set_scope("idle_control_plane_core_fraction", "small-fixture"), + "budget_scope_narrowed:unknown": _set_scope("watcher_ack_ms", "nowhere"), + "budget_scope_narrowed:additional": lambda d: d["budgets"]["absolute"].append( + {"metric": "exec_ms", "statistic": "p95", "max": 50, "inclusive": True, "workload": "nowhere"}), + "workload_baseline_missing": lambda d: workload(d, "event-agent")["lanes"].remove("compose"), + "workload_shape_mismatch": lambda d: workload(d, "event-agent")["shape"].update(services=13), + "phases_mismatch": lambda d: d["phases"].remove("restore"), + "scale_points_mismatch:instances": lambda d: d["scale_points"].update(branch_instances=[1, 8]), + "scale_points_mismatch:clients": lambda d: d["scale_points"].update(mcp_clients=[1]), + "scale_points_mismatch:judged": lambda d: d["scale_points"].update(judged={"branch_instances": 8}), + "artifact_commit_malformed": lambda d: lane(d, "candidate").update(commit="28de2f89"), + "artifact_digest_malformed": lambda d: lane(d, "hack-4.2.1").update(asset_sha256="0" * 63), + "artifact_version_malformed": lambda d: lane(d, "hack-4.1.1").update(tag="v4.1.2"), + "p95_samples_insufficient": lambda d: d["samples"].update(p95_min_samples=58), + # 59 clears the distribution-free bound; only the frozen revision-1 floor of 60 refuses it. + "p95_samples_insufficient:frozen-floor": lambda d: d["samples"].update(p95_min_samples=59), + # 0.90 needs only 29 samples, so 60 still clears the bound: only the quantile check catches it. + "p95_samples_insufficient:quantile": lambda d: d["samples"].update(p95_quantile=0.9), + # 0.90 confidence needs only 45 samples, so 60 still clears it: only the confidence floor catches it. + "p95_samples_insufficient:confidence": lambda d: d["samples"].update(p95_bound_confidence=0.9), + "p95_method_mismatch": lambda d: d["samples"].update(p95_method="linear-interpolation"), + "schedule_rounds_mismatch": lambda d: d["schedule"].update(rounds_per_workload=9), + "schedule_design_mismatch": lambda d: d["schedule"].update(design="latin-square"), + # Above the program floor of 5 but below the frozen 10 runs per lane. + "schedule_rounds_below_minimum:frozen-floor": lambda d: d["samples"].update(min_lifecycle_runs_per_lane=5), + # Another balanced Williams square, from a reordered lane list: only the order pin refuses it. + "schedule_order_mismatch": _reorder_schedule, + "observer_policy_missing": lambda d: d["observer"].update(inconclusive_factor=0.5), + "observer_policy_missing:factor": lambda d: d["observer"].update(inconclusive_factor=1), + "observer_policy_missing:window": lambda d: d["observer"].update(control_window_seconds=599), + "cache_class_missing": lambda d: d["cache_classes"].update(budget_class="cold-cache"), + "workload_lane_unknown": lambda d: d["workloads"][0]["lanes"].append("hack-3.0.0"), + "workloads_malformed:id": lambda d: d["workloads"][0].update(id=["event-agent"]), + "host_incomplete": lambda d: d["host"]["orbstack"].pop("build"), + "protocol_revision_mismatch": lambda d: d.update(revision=2), + "protocol_revision_mismatch:bool": lambda d: d.update(revision=True), + "known_limit_malformed": lambda d: d["known_limits"][0].pop("revisit"), + "known_limit_malformed:number": lambda d: d.update(known_limits=7), + "lane_id_duplicate": lambda d: d["lanes"].append(copy.deepcopy(d["lanes"][0])), + "compose_lane_host_mismatch": lambda d: lane(d, "compose").update(compose_version="2.0.0"), + "compose_lane_host_mismatch:engine": lambda d: lane(d, "compose").update(engine="docker-desktop"), + # Well-formed but different identities: only the revision-1 pins refuse these. + "artifact_identity_mismatch:candidate-commit": lambda d: lane(d, "candidate").update(commit=OTHER_COMMIT), + "artifact_identity_mismatch:baseline-commit": lambda d: lane(d, "hack-4.2.0").update(commit=OTHER_COMMIT), + "artifact_identity_mismatch:digest": lambda d: lane(d, "hack-4.1.1").update(asset_sha256=OTHER_DIGEST), + "artifact_identity_mismatch:version": lambda d: lane(d, "candidate").update( + version="5.0.0-next.6", tag="v5.0.0-next.6"), + "artifact_identity_mismatch:asset": lambda d: lane(d, "hack-4.2.1").update( + asset="hack-4.2.1-darwin-x64.tar.gz"), + "artifact_identity_mismatch:asset-removed": lambda d: lane(d, "candidate").pop("asset"), + # A hack lane recast as a Compose lane on the frozen engine, with its artifact removed. + "lane_runtime_mismatch:candidate-kind": lambda d: _recast_as_compose(lane(d, "candidate")), + "lane_runtime_mismatch:baseline-kind": lambda d: _recast_as_compose(lane(d, "hack-4.1.1")), + "lane_runtime_mismatch:backend": lambda d: lane(d, "candidate").update(backend="compose"), + "lane_runtime_mismatch:backend-removed": lambda d: lane(d, "candidate").pop("backend"), + "lane_runtime_mismatch:compose-backend": lambda d: lane(d, "compose").update(backend="hack"), + "lane_runtime_mismatch:baseline-capacity": lambda d: lane(d, "compose").update(capacity={"cpus": 4}), + "candidate_capacity_mismatch": lambda d: lane(d, "candidate")["capacity"].update(memory_mib=12288), + "candidate_capacity_mismatch:profile": lambda d: lane(d, "candidate")["capacity"].update( + profile="research"), + "candidate_capacity_mismatch:removed": lambda d: lane(d, "candidate").pop("capacity"), + "candidate_capacity_mismatch:float": lambda d: lane(d, "candidate")["capacity"].update(vcpus=4.0), + "host_mismatch:os": lambda d: d["host"].update(os="macOS 26.5.1"), + "host_mismatch:cpu": lambda d: d["host"].update(cpu="Apple M5 Max", logical_cpus=18), + "host_mismatch:orbstack": lambda d: d["host"]["orbstack"].update(memory_mib=32768), + "workload_pin_mismatch": lambda d: workload(d, "small-fixture").update(pin="image oven/bun:latest"), + "workload_correctness_weakened": lambda d: workload(d, "event-agent")["correctness"].pop(), + } + + def test_each_weakening_is_refused_with_its_closed_code(self): + for code, change in self.CASES.items(): + with self.subTest(code=code): + found = protocol_check.validate_protocol(mutated(change)) + self.assertIn(code.split(":")[0], found) + self.assertTrue(set(found) <= protocol_check.DIAGNOSTICS) + + def test_stricter_revisions_stay_valid(self): + stricter = mutated(lambda d: ( + workload(d, "small-fixture")["correctness"].append("cleanup leaves no owned container"), + budget(d, "absolute", "watcher_ack_ms").pop("workload"), + budget(d, "absolute", "info_latency_ms").update(max=90), + budget(d, "comparative", "cpu_seconds_per_task").update(max_ratio=0.6), + d["observer"].update(inconclusive_factor=3), + d["scale_points"].update(branch_instances=[1, 8, 16, 32]), + )) + self.assertEqual(protocol_check.validate_protocol(stricter), []) + + def test_dropping_a_required_baseline_is_refused(self): + def drop(doc): + doc["lanes"] = [item for item in doc["lanes"] if item["id"] != "hack-4.2.0"] + doc["schedule"]["orders"] = protocol_check.williams_orders([item["id"] for item in doc["lanes"]]) + doc["schedule"]["rounds_per_workload"] = len(doc["schedule"]["orders"]) + for item in doc["workloads"]: + item["lanes"].remove("hack-4.2.0") + self.assertIn("baseline_lane_missing", protocol_check.validate_protocol(mutated(drop))) + + def test_unbalanced_or_short_rotation_is_refused(self): + def swap(doc): + order = doc["schedule"]["orders"][0] + order[0], order[1] = order[1], order[0] + self.assertIn("schedule_position_unbalanced", protocol_check.validate_protocol(mutated(swap))) + repeated = mutated(lambda d: d["schedule"].update(orders=[d["schedule"]["orders"][0]] * 10)) + found = protocol_check.validate_protocol(repeated) + self.assertIn("schedule_position_unbalanced", found) + self.assertIn("schedule_carryover_unbalanced", found) + short = mutated(lambda d: d["schedule"].update(orders=d["schedule"]["orders"][:4])) + self.assertIn("schedule_rounds_below_minimum", protocol_check.validate_protocol(short)) + floor = mutated(lambda d: d["samples"].update(min_lifecycle_runs_per_lane=4)) + self.assertIn("schedule_rounds_below_minimum", protocol_check.validate_protocol(floor)) + more_than_scheduled = mutated(lambda d: d["samples"].update(min_lifecycle_runs_per_lane=11)) + self.assertIn("schedule_rounds_below_minimum", protocol_check.validate_protocol(more_than_scheduled)) + missing_lane = mutated(lambda d: d["schedule"]["orders"][3].pop()) + self.assertIn("schedule_lane_set_mismatch", protocol_check.validate_protocol(missing_lane)) + + def test_any_malformed_node_maps_to_closed_codes_without_raising(self): + for path in node_paths(PROTOCOL): + for value in FUZZ_VALUES + (DELETE,): + with self.subTest(path=path, value=repr(value)): + found = protocol_check.validate_protocol(replace_at(PROTOCOL, path, value)) + self.assertTrue(set(found) <= protocol_check.DIAGNOSTICS, found) + for value in FUZZ_VALUES: + found = protocol_check.validate_protocol(value) + if not isinstance(value, dict): + self.assertEqual(found, ["protocol_not_object"]) + self.assertTrue(found and set(found) <= protocol_check.DIAGNOSTICS) + + def test_diagnostics_never_echo_input_values(self): + doc = mutated(lambda d: lane(d, "candidate").update(commit=MARKER, asset_sha256=MARKER)) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / f"{MARKER}.json" + path.write_text(json.dumps(doc)) + out = io.StringIO() + with contextlib.redirect_stdout(out): + code = protocol_check.main(["--protocol", str(path)]) + self.assertEqual(code, 2) + self.assertNotIn(MARKER, out.getvalue()) + self.assertEqual(json.loads(out.getvalue())["diagnostics"], + ["artifact_commit_malformed", "artifact_digest_malformed", + "artifact_identity_mismatch"]) + path.write_text("{not json " + MARKER) + out = io.StringIO() + with contextlib.redirect_stdout(out): + self.assertEqual(protocol_check.main(["--protocol", str(path)]), 2) + self.assertEqual(json.loads(out.getvalue()), {"status": "invalid", "diagnostics": ["protocol_unreadable"]}) + + +class BudgetEvaluationTest(unittest.TestCase): + def judge(self, summary): + diagnostics, records = protocol_check.evaluate(PROTOCOL, summary) + self.assertEqual(diagnostics, []) + self.assertTrue({record["verdict"] for record in records} <= protocol_check.VERDICTS) + return records + + def refused(self, summary): + diagnostics, records = protocol_check.evaluate(PROTOCOL, summary) + self.assertEqual(records, []) + self.assertTrue(diagnostics and set(diagnostics) <= protocol_check.DIAGNOSTICS, diagnostics) + return diagnostics + + def test_complete_improved_cohort_passes_every_budget_against_every_baseline(self): + records = self.judge(small_summary()) + self.assertEqual({record["verdict"] for record in records}, {"pass"}) + comparative = [record for record in records if "baseline" in record] + self.assertEqual(len(comparative), 4 * 4) + self.assertEqual(len(records) - len(comparative), 5) + self.assertTrue(protocol_check.accepted(records)) + self.assertFalse(protocol_check.accepted([])) + + def test_boundary_ratios_follow_the_frozen_thresholds(self): + records = self.judge(small_summary(candidate_scale=0.75)) + self.assertEqual(set(verdicts(records, "cpu_seconds_per_task", "compose")), {"pass"}) + self.assertEqual(set(verdicts(records, "steady_physical_footprint_bytes", "compose")), {"fail"}) + self.assertEqual(set(verdicts(records, "warm_startup_seconds", "compose")), {"pass"}) + + def test_each_baseline_is_judged_separately_without_aggregation(self): + summary = small_summary() + summary["lanes"]["hack-4.2.1"]["warm_startup_seconds"] = entry([5.0] * 10) + records = self.judge(summary) + self.assertEqual(verdicts(records, "warm_startup_seconds", "hack-4.2.1"), ["fail"]) + self.assertEqual(verdicts(records, "warm_startup_seconds", "compose"), ["pass"]) + + def test_too_few_rotated_runs_or_tail_samples_are_inconclusive(self): + records = self.judge(small_summary(runs=9)) + self.assertEqual(set(verdicts(records, "warm_restart_seconds", "compose")), {"inconclusive_samples"}) + records = self.judge(small_summary(tail=59)) + self.assertEqual(verdicts(records, "info_latency_ms"), ["inconclusive_samples"]) + self.assertEqual(verdicts(records, "watcher_ack_ms"), ["inconclusive_samples"]) + + def test_missing_planned_samples_are_inconclusive_coverage(self): + summary = small_summary() + summary["lanes"]["compose"]["cpu_seconds_per_task"]["expected"] = 11 + records = self.judge(summary) + self.assertEqual(verdicts(records, "cpu_seconds_per_task", "compose"), ["inconclusive_coverage"]) + summary["lanes"]["compose"]["cpu_seconds_per_task"]["expected"] = None + records = self.judge(summary) + self.assertEqual(verdicts(records, "cpu_seconds_per_task", "compose"), ["inconclusive_missing"]) + + def test_flagged_failed_or_unaccounted_samples_are_never_passes(self): + summary = small_summary() + summary["lanes"]["compose"]["cpu_seconds_per_task"] = entry([10.0] * 10, expected=11, flagged=1) + summary["lanes"]["hack-4.1.1"]["cpu_seconds_per_task"] = entry([10.0] * 10, expected=11, failed=1) + summary["lanes"]["hack-4.2.0"]["cpu_seconds_per_task"].pop("flagged") + summary["lanes"]["candidate"]["info_latency_ms"].pop("failed") + records = self.judge(summary) + self.assertEqual(verdicts(records, "cpu_seconds_per_task", "compose"), ["inconclusive_admission"]) + self.assertEqual(verdicts(records, "cpu_seconds_per_task", "hack-4.1.1"), ["inconclusive_admission"]) + self.assertEqual(verdicts(records, "cpu_seconds_per_task", "hack-4.2.0"), ["inconclusive_admission"]) + self.assertEqual(verdicts(records, "info_latency_ms"), ["inconclusive_admission"]) + self.assertEqual(verdicts(records, "cpu_seconds_per_task", "hack-4.2.1"), ["pass"]) + self.assertFalse(protocol_check.accepted(records)) + + def test_failed_candidate_lifecycle_is_a_failure(self): + summary = small_summary() + summary["lanes"]["candidate"]["warm_startup_seconds"] = entry([10.0] * 10, expected=11, failed=1) + summary["lanes"]["candidate"]["watcher_ack_ms"] = entry([100] * 60, expected=61, failed=1) + records = self.judge(summary) + self.assertEqual(set(verdicts(records, "warm_startup_seconds", "compose")), {"fail"}) + self.assertEqual(verdicts(records, "watcher_ack_ms"), ["fail"]) + + def test_difference_within_observer_cost_is_inconclusive(self): + summary = small_summary() + summary["observer_cost"]["cpu_seconds_per_task"] = 2.6 + records = self.judge(summary) + self.assertEqual(set(verdicts(records, "cpu_seconds_per_task", "compose")), {"inconclusive_observer"}) + summary["observer_cost"].pop("warm_startup_seconds") + records = self.judge(summary) + self.assertEqual(set(verdicts(records, "warm_startup_seconds", "compose")), {"inconclusive_observer"}) + + def test_frozen_observer_factor_is_two(self): + # A 5.0 difference against a 2.6 cost is inside 2 x cost; a factor of 1 would pass it. + summary = small_summary() + summary["observer_cost"]["cpu_seconds_per_task"] = 2.6 + weak = mutated(lambda d: d["observer"].update(inconclusive_factor=1)) + _, records = protocol_check.evaluate(weak, summary) + self.assertEqual(set(verdicts(records, "cpu_seconds_per_task", "compose")), {"pass"}) + self.assertIn("observer_policy_missing", protocol_check.validate_protocol(weak)) + + def test_absolute_ceilings_respect_inclusivity(self): + summary = small_summary() + summary["lanes"]["candidate"]["info_latency_ms"] = entry([100] * 60) + summary["lanes"]["candidate"]["idle_control_plane_core_fraction"] = entry([0.01] * 10) + records = self.judge(summary) + self.assertEqual(verdicts(records, "info_latency_ms"), ["pass"]) + self.assertEqual(verdicts(records, "idle_control_plane_core_fraction"), ["fail"]) + summary["lanes"]["candidate"]["info_latency_ms"] = entry([50] * 57 + [101] * 3) + self.assertEqual(verdicts(self.judge(summary), "info_latency_ms"), ["pass"]) + summary["lanes"]["candidate"]["info_latency_ms"] = entry([50] * 56 + [101] * 4) + self.assertEqual(verdicts(self.judge(summary), "info_latency_ms"), ["fail"]) + + def test_a_not_applicable_baseline_with_a_reason_is_skipped_and_missing_lanes_are_inconclusive(self): + summary = small_summary() + summary["lanes"]["hack-4.1.1"]["warm_startup_seconds"] = not_applicable() + records = self.judge(summary) + self.assertEqual(verdicts(records, "warm_startup_seconds", "hack-4.1.1"), ["not_applicable"]) + self.assertEqual(verdicts(records, "warm_startup_seconds", "compose"), ["pass"]) + self.assertTrue(protocol_check.accepted(records)) + summary["lanes"].pop("hack-4.2.0") + records = self.judge(summary) + self.assertEqual(set(verdicts(records, "cpu_seconds_per_task", "hack-4.2.0")), {"inconclusive_missing"}) + self.assertFalse(protocol_check.accepted(records)) + + def test_no_applicable_baseline_is_inconclusive_never_accepted(self): + summary = small_summary() + for baseline in BASELINES: + summary["lanes"][baseline] = {metric: not_applicable() for metric in COMPARATIVE} + records = self.judge(summary) + for metric in COMPARATIVE: + self.assertEqual(verdicts(records, metric), ["inconclusive_no_baseline"]) + self.assertFalse(protocol_check.accepted(records)) + code, report = evaluate_file(summary) + self.assertEqual((code, report["accepted"]), (1, False)) + + def test_the_candidate_can_never_be_not_applicable(self): + summary = small_summary() + summary["lanes"]["candidate"] = {metric: not_applicable() for metric in COMPARATIVE + ABSOLUTE} + self.assertEqual(self.refused(summary), ["summary_candidate_not_applicable"]) + self.assertEqual(evaluate_file(summary), + (2, {"status": "invalid", "diagnostics": ["summary_candidate_not_applicable"]})) + for metric in COMPARATIVE + ABSOLUTE: + with self.subTest(metric=metric): + one = small_summary() + one["lanes"]["candidate"][metric] = not_applicable() + self.assertEqual(self.refused(one), ["summary_candidate_not_applicable"]) + + def test_not_applicable_never_overrides_samples(self): + for lane_id in ("candidate", "compose"): + with self.subTest(lane=lane_id): + summary = small_summary(candidate_scale=5) + summary["lanes"][lane_id]["warm_startup_seconds"]["not_applicable"] = True + self.assertEqual(self.refused(summary), ["summary_entry_malformed"]) + for bad in ({"not_applicable": True}, {"not_applicable": True, "reason": " "}, + {"not_applicable": False, "reason": "x"}, {"not_applicable": 1, "reason": "x"}): + with self.subTest(entry=bad): + summary = small_summary() + summary["lanes"]["compose"]["warm_startup_seconds"] = bad + self.assertEqual(self.refused(summary), ["summary_entry_malformed"]) + + def test_only_warm_single_instance_summaries_are_judged(self): + cases = { + "summary_cache_class_mismatch": [("cache_class", "cold-cache"), ("cache_class", None)], + "summary_scale_unknown": [("scale", None), ("scale", {"branch_instances": 2}), + ("scale", {"branch_instances": True}), + ("scale", {"branch_instances": 1, "mcp_clients": 1}), ("scale", [1])], + "summary_scale_not_judged": [("scale", {"branch_instances": 32}), ("scale", {"mcp_clients": 1})], + } + for code, changes in cases.items(): + for key, value in changes: + with self.subTest(code=code, value=value): + self.assertEqual(self.refused({**small_summary(), key: value}), [code]) + summary = small_summary() + summary.pop("cache_class") + self.assertEqual(self.refused(summary), ["summary_cache_class_mismatch"]) + self.assertEqual(evaluate_file({**small_summary(), "cache_class": "cold-cache"})[0], 2) + + def test_malformed_values_are_refused_with_closed_codes(self): + def candidate(metric, value): + def change(summary): + summary["lanes"]["candidate"][metric] = value + return change + + def baseline(metric, field, value): + def change(summary): + summary["lanes"]["compose"][metric][field] = value + return change + + cases = [ + ("summary_entry_malformed", candidate("info_latency_ms", entry([-5] * 60))), + ("summary_entry_malformed", candidate("warm_startup_seconds", entry([-1.0] * 10))), + ("summary_entry_malformed", baseline("cpu_seconds_per_task", "expected", 0)), + ("summary_entry_malformed", candidate("info_latency_ms", entry([], expected=0))), + ("summary_entry_malformed", baseline("cpu_seconds_per_task", "expected", -10)), + ("summary_entry_malformed", baseline("cpu_seconds_per_task", "expected", True)), + ("summary_entry_malformed", baseline("cpu_seconds_per_task", "expected", 10.0)), + ("summary_entry_malformed", baseline("cpu_seconds_per_task", "expected", 9)), + ("summary_entry_malformed", baseline("cpu_seconds_per_task", "flagged", -1)), + ("summary_entry_malformed", baseline("cpu_seconds_per_task", "failed", "0")), + ("summary_entry_malformed", baseline("cpu_seconds_per_task", "samples", [1, "2"])), + ("summary_entry_malformed", baseline("cpu_seconds_per_task", "samples", [float("nan")] * 10)), + ("summary_entry_malformed", baseline("cpu_seconds_per_task", "median", 1.0)), + ("summary_entry_malformed", candidate("watcher_ack_ms", [100] * 60)), + ("summary_lane_malformed", lambda s: s["lanes"].update(compose=[MARKER])), + ("summary_lane_malformed", lambda s: s.update(lanes=[MARKER])), + ("summary_observer_malformed", lambda s: s["observer_cost"].update(cpu_seconds_per_task=-1)), + ("summary_observer_malformed", lambda s: s.update(observer_cost=[0])), + ("summary_workload_unknown", lambda s: s.update(workload=["x"])), + ("summary_workload_unknown", lambda s: s.update(workload={"id": "small-fixture"})), + ("summary_revision_mismatch", lambda s: s.update(protocol_revision=True)), + ] + for code, change in cases: + summary = small_summary() + change(summary) + with self.subTest(code=code, summary=summary.get("workload")): + self.assertEqual(self.refused(summary), [code]) + + def test_oversized_integers_are_refused_not_a_traceback(self): + """Python parses 10**400 exactly; converting it to a float would raise OverflowError.""" + cases = { + "candidate": lambda s: s["lanes"]["candidate"].update(info_latency_ms=entry([HUGE] * 60)), + "baseline": lambda s: s["lanes"]["compose"].update(cpu_seconds_per_task=entry([HUGE] * 10)), + "beyond-interoperable": lambda s: s["lanes"]["compose"].update( + cpu_seconds_per_task=entry([2 ** 53] * 10)), + } + for name, change in cases.items(): + with self.subTest(case=name): + summary = small_summary() + change(summary) + self.assertEqual(evaluate_file(summary), + (2, {"status": "invalid", "diagnostics": ["summary_entry_malformed"]})) + summary = small_summary() + summary["observer_cost"]["cpu_seconds_per_task"] = HUGE + self.assertEqual(evaluate_file(summary), + (2, {"status": "invalid", "diagnostics": ["summary_observer_malformed"]})) + protocol = mutated(lambda d: budget(d, "comparative", "cpu_seconds_per_task").update(max_ratio=HUGE)) + self.assertIn("comparative_budget_weakened", protocol_check.validate_protocol(protocol)) + + def test_an_overflowing_ratio_fails_and_the_report_stays_strict_json(self): + summary = small_summary() + summary["lanes"]["compose"]["warm_startup_seconds"] = entry([5e-324] * 10) + records = self.judge(summary) + [record] = [item for item in records + if item["metric"] == "warm_startup_seconds" and item.get("baseline") == "compose"] + self.assertEqual(record["verdict"], "fail") + self.assertNotIn("ratio", record) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "summary.json" + path.write_text(json.dumps(summary)) + out = io.StringIO() + with contextlib.redirect_stdout(out): + self.assertEqual(protocol_check.main(["--evaluate", str(path)]), 1) + + def refuse_constant(name): + raise AssertionError(f"non-standard JSON constant {name}") + json.loads(out.getvalue(), parse_constant=refuse_constant) + + def test_list_valued_workload_exits_invalid_not_with_a_traceback(self): + self.assertEqual(evaluate_file({**small_summary(), "workload": ["x"]}), + (2, {"status": "invalid", "diagnostics": ["summary_workload_unknown"]})) + + def test_any_malformed_summary_node_maps_to_closed_codes_without_raising(self): + summary = small_summary() + for path in node_paths(summary, list_limit=1): + for value in FUZZ_VALUES + (DELETE,): + with self.subTest(path=path, value=repr(value)): + diagnostics, records = protocol_check.evaluate(PROTOCOL, replace_at(summary, path, value)) + self.assertTrue(set(diagnostics) <= protocol_check.DIAGNOSTICS, diagnostics) + self.assertTrue({record["verdict"] for record in records} <= protocol_check.VERDICTS) + self.assertTrue(bool(diagnostics) != bool(records)) + + def test_workload_scoped_budgets_only_apply_to_their_workload(self): + summary = small_summary() + summary["workload"] = "event-agent" + records = self.judge(summary) + metrics = {record["metric"] for record in records if "baseline" not in record} + self.assertEqual(metrics, set(ABSOLUTE) - {"watcher_ack_ms"}) + + def test_malformed_summaries_are_refused_with_closed_codes(self): + cases = { + "summary_not_object": [], + "summary_revision_mismatch": {**small_summary(), "protocol_revision": 2}, + "summary_workload_unknown": {**small_summary(), "workload": MARKER}, + "summary_lane_unknown": {**small_summary(), "lanes": {MARKER: {}}}, + } + for code, summary in cases.items(): + with self.subTest(code=code): + self.assertEqual(protocol_check.evaluate(PROTOCOL, summary), ([code], [])) + + def test_cli_exit_codes(self): + for scale, expected in ((0.5, 0), (0.9, 1)): + code, report = evaluate_file(small_summary(candidate_scale=scale)) + self.assertEqual(code, expected) + self.assertEqual(report["accepted"], expected == 0) + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(protocol_check.main([]), 0) + self.assertEqual(protocol_check.main(["--schedule"]), 0) + + +if __name__ == "__main__": + unittest.main()