fix(perf): fsum window means and bound every protocol number

docs/reference/application-benchmark-protocol.md

@@ -287,8 +287,13 @@ A different or missing value is refused (`artifact_identity_mismatch`,
 frozen correctness check (`workload_correctness_weakened`). A revision may add
 correctness checks. The sample counts are floors at their frozen values: a protocol
 that plans fewer than 10 lifecycle runs or 60 tail samples per lane is refused, even
-where the program floor or the p95 bound would allow fewer. Every number in the protocol or a summary must be
-within ±(2⁵³ − 1), the interoperable JSON range; a larger value is refused as malformed.
+where the program floor or the p95 bound would allow fewer. Every number anywhere in
+the protocol or a summary, including a field the checker does not otherwise read, must
+be finite and within ±(2⁵³ − 1), the interoperable JSON range. Any other number,
+including the `NaN` and `Infinity` that Python's JSON parser accepts, is refused as
+`protocol_number_out_of_range` or `summary_number_out_of_range`, alongside any code
+for the field that holds it. A mean is the correctly rounded sum divided by the
+count, so a mean at an exclusive ceiling gets the same verdict on every Python version.
 
 These are acceptance thresholds, not measured achievements.
 
@@ -315,7 +320,7 @@ record. It exits 1 when any verdict fails or is inconclusive. It exits 2, withou
 judging, when the protocol or summary is invalid: for example a summary for another
 cache class or scale point, a candidate marked `not_applicable`, a `not_applicable`
 record that carries samples, a negative sample, a planned count that is not a positive
-integer, or a value of the wrong type. Diagnostics are closed codes and never repeat
+integer, a number outside ±(2⁵³ − 1), or a value of the wrong type. Diagnostics are closed codes and never repeat
 input values.
 
 A summary has this shape. `samples` lists only admitted samples; `flagged` and `failed`

scripts/check-application-benchmark-protocol.py

@@ -11,7 +11,8 @@
 Revision-1 identities (artifacts, host, lane kinds and backends, candidate capacity,
 workload pins, rotated lane order) are exact; budgets and sample counts are floors at
 their frozen values, which a revision may tighten.
-Every number must lie within MAX_JSON_NUMBER.
+Every number anywhere in the protocol or a summary must be finite and lie within
+MAX_JSON_NUMBER, including fields the checker does not otherwise read.
 Diagnostics are closed codes from DIAGNOSTICS; they never echo input values or paths.
 A budget verdict is one of VERDICTS. A statistic is judged only from complete,
 sufficient, admitted samples: missing, short, flagged or unaccounted samples are
@@ -197,8 +198,10 @@
     "statistic_unknown",
     "observer_policy_missing",
     "known_limit_malformed",
+    "protocol_number_out_of_range",
     "summary_unreadable",
     "summary_not_object",
+    "summary_number_out_of_range",
     "summary_revision_mismatch",
     "summary_workload_unknown",
     "summary_cache_class_mismatch",
@@ -313,6 +316,24 @@ def _number(value):
         and -MAX_JSON_NUMBER <= value <= MAX_JSON_NUMBER
 
 
+def _numbers_in_range(value):
+    """True when every number anywhere in a JSON value satisfies _number.
+
+    Booleans are not numbers and object keys are strings, so neither is checked. The
+    walk is iterative, so a deeply nested value cannot raise RecursionError.
+    """
+    pending = [value]
+    while pending:
+        node = pending.pop()
+        if isinstance(node, dict):
+            pending.extend(node.values())
+        elif isinstance(node, list):
+            pending.extend(node)
+        elif isinstance(node, (int, float)) and not isinstance(node, bool) and not _number(node):
+            return False
+    return True
+
+
 def _same(value, frozen):
     """Exact JSON equality: unlike ==, 4.0 is not 4 and true is not 1."""
     return json.dumps(value, sort_keys=True) == json.dumps(frozen, sort_keys=True)
@@ -554,6 +575,8 @@ def validate_protocol(doc):
     if not isinstance(doc, dict):
         return ["protocol_not_object"]
     found = set()
+    if not _numbers_in_range(doc):
+        found.add("protocol_number_out_of_range")
     if doc.get("protocol") != PROTOCOL_ID:
         found.add("protocol_id_mismatch")
     if not _count(doc.get("revision")) or doc["revision"] != PROTOCOL_REVISION:
@@ -647,6 +670,8 @@ def validate_summary(protocol, summary):
     if workload is None:
         return ["summary_workload_unknown"], None
     found = set()
+    if not _numbers_in_range(summary):
+        found.add("summary_number_out_of_range")
     # Budgets are frozen for the warm class only; other classes are reported separately.
     if summary.get("cache_class") != protocol["cache_classes"]["budget_class"]:
         found.add("summary_cache_class_mismatch")
@@ -679,12 +704,22 @@ def validate_summary(protocol, summary):
     return sorted(found), workload
 
 
+def window_mean(samples):
+    """The correctly rounded sum divided by the count.
+
+    Built-in sum() rounds after every addition, and Python 3.12 changed how, so a mean
+    at an exclusive ceiling (ten samples of 0.01) could land just below it on one
+    interpreter and on it on another. math.fsum gives the same verdict everywhere.
+    """
+    return math.fsum(samples) / len(samples)
+
+
 def _statistic(samples, statistic):
     if statistic == "median":
         return median(samples)
     if statistic == "p95":
         return nearest_rank(samples, P95_QUANTILE)
-    return sum(samples) / len(samples)
+    return window_mean(samples)
 
 
 def _sample_floor(protocol, statistic):

tests/python/test_application_benchmark_protocol.py

@@ -4,6 +4,7 @@
 import importlib.util
 import io
 import json
+import math
 from pathlib import Path
 import re
 import tempfile
@@ -515,6 +516,23 @@ def test_absolute_ceilings_respect_inclusivity(self):
         summary["lanes"]["candidate"]["info_latency_ms"] = entry([50] * 56 + [101] * 4)
         self.assertEqual(verdicts(self.judge(summary), "info_latency_ms"), ["fail"])
 
+    def test_window_mean_is_correctly_rounded_on_every_interpreter(self):
+        # Built-in sum() gives 0.009999999999999998 here before Python 3.12, which would
+        # pass the exclusive 1% idle ceiling; on 3.12+ it differs from the exact mean on
+        # the second list. Both must judge from the correctly rounded mean.
+        self.assertEqual(protocol_check.window_mean([0.01] * 10), 0.01)
+        large = [1000000000000000.2, 0.001, 0.007, 0.1, 0.01, 3.0, 1.0, 0.0, 0.007, 1000000000000000.2]
+        exact = math.fsum(large) / len(large)
+        self.assertEqual(protocol_check.window_mean(large), exact)
+        # A stricter extra budget whose exclusive ceiling is that exact mean must fail.
+        strict = mutated(lambda d: d["budgets"]["absolute"].append(
+            {"metric": "probe_window_mean", "statistic": "window_mean", "max": exact, "inclusive": False}))
+        self.assertEqual(protocol_check.validate_protocol(strict), [])
+        summary = small_summary()
+        summary["lanes"]["candidate"]["probe_window_mean"] = entry(large)
+        _, records = protocol_check.evaluate(strict, summary)
+        self.assertEqual(verdicts(records, "probe_window_mean"), ["fail"])
+
     def test_a_not_applicable_baseline_with_a_reason_is_skipped_and_missing_lanes_are_inconclusive(self):
         summary = small_summary()
         summary["lanes"]["hack-4.1.1"]["warm_startup_seconds"] = not_applicable()
@@ -603,7 +621,8 @@ def change(summary):
             ("summary_entry_malformed", baseline("cpu_seconds_per_task", "flagged", -1)),
             ("summary_entry_malformed", baseline("cpu_seconds_per_task", "failed", "0")),
             ("summary_entry_malformed", baseline("cpu_seconds_per_task", "samples", [1, "2"])),
-            ("summary_entry_malformed", baseline("cpu_seconds_per_task", "samples", [float("nan")] * 10)),
+            (("summary_entry_malformed", "summary_number_out_of_range"),
+             baseline("cpu_seconds_per_task", "samples", [float("nan")] * 10)),
             ("summary_entry_malformed", baseline("cpu_seconds_per_task", "median", 1.0)),
             ("summary_entry_malformed", candidate("watcher_ack_ms", [100] * 60)),
             ("summary_lane_malformed", lambda s: s["lanes"].update(compose=[MARKER])),
@@ -618,7 +637,7 @@ def change(summary):
             summary = small_summary()
             change(summary)
             with self.subTest(code=code, summary=summary.get("workload")):
-                self.assertEqual(self.refused(summary), [code])
+                self.assertEqual(self.refused(summary), [code] if isinstance(code, str) else list(code))
 
     def test_oversized_integers_are_refused_not_a_traceback(self):
         """Python parses 10**400 exactly; converting it to a float would raise OverflowError."""
@@ -632,15 +651,57 @@ def test_oversized_integers_are_refused_not_a_traceback(self):
             with self.subTest(case=name):
                 summary = small_summary()
                 change(summary)
-                self.assertEqual(evaluate_file(summary),
-                                 (2, {"status": "invalid", "diagnostics": ["summary_entry_malformed"]}))
+                self.assertEqual(evaluate_file(summary), (2, {"status": "invalid", "diagnostics": [
+                    "summary_entry_malformed", "summary_number_out_of_range"]}))
         summary = small_summary()
         summary["observer_cost"]["cpu_seconds_per_task"] = HUGE
-        self.assertEqual(evaluate_file(summary),
-                         (2, {"status": "invalid", "diagnostics": ["summary_observer_malformed"]}))
+        self.assertEqual(evaluate_file(summary), (2, {"status": "invalid", "diagnostics": [
+            "summary_number_out_of_range", "summary_observer_malformed"]}))
         protocol = mutated(lambda d: budget(d, "comparative", "cpu_seconds_per_task").update(max_ratio=HUGE))
         self.assertIn("comparative_budget_weakened", protocol_check.validate_protocol(protocol))
 
+    def test_every_number_anywhere_must_be_finite_and_interoperable(self):
+        # The page promises every number in a protocol or summary is within +/-(2**53 - 1).
+        # These sit where no other check would refuse them: an unread field, or a count
+        # that is only compared as a floor or against the sample list.
+        limit = 2 ** 53 - 1
+        for value in (limit + 1, -limit - 1, HUGE, float("nan"), float("inf"), -float("inf"), 1e300):
+            with self.subTest(value=repr(value)):
+                unread = mutated(lambda d: d.update(notes=[{"bytes": value}]))
+                self.assertEqual(protocol_check.validate_protocol(unread), ["protocol_number_out_of_range"])
+                floor = mutated(lambda d: d["samples"].update(p95_min_samples=value))
+                self.assertIn("protocol_number_out_of_range", protocol_check.validate_protocol(floor))
+                summary = small_summary()
+                summary["lanes"]["candidate"]["unbudgeted_metric"] = {"samples": [value]}
+                summary["notes"] = value
+                self.assertEqual(self.refused(summary), ["summary_number_out_of_range"])
+                summary = small_summary()
+                summary["lanes"]["candidate"]["info_latency_ms"]["expected"] = value
+                self.assertIn("summary_number_out_of_range", self.refused(summary))
+        self.assertEqual(protocol_check.validate_protocol(mutated(lambda d: d["samples"].update(
+            p95_min_samples=limit))), [])
+        self.assertEqual(protocol_check.validate_protocol(mutated(lambda d: d.update(
+            notes=[limit, -limit, 1e-300, True]))), [])
+        summary = small_summary()
+        summary["lanes"]["candidate"]["unbudgeted_metric"] = {"samples": [limit, -limit]}
+        self.judge(summary)
+        summary["lanes"]["candidate"]["unbudgeted_metric"]["samples"].append(limit + 1)
+        self.assertEqual(evaluate_file(summary),
+                         (2, {"status": "invalid", "diagnostics": ["summary_number_out_of_range"]}))
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "protocol.json"
+            path.write_text(json.dumps(mutated(lambda d: d.update(notes=float("nan")))))
+            self.assertEqual(run_cli(["--protocol", str(path)]),
+                             (2, {"status": "invalid", "diagnostics": ["protocol_number_out_of_range"]}))
+
+    def test_a_deeply_nested_unread_field_is_scanned_without_raising(self):
+        # Far deeper than the recursion limit: a recursive walk would raise RecursionError.
+        deep = [2 ** 53]
+        for _ in range(100000):
+            deep = [deep]
+        self.assertEqual(protocol_check.validate_protocol(mutated(lambda d: d.update(notes=deep))),
+                         ["protocol_number_out_of_range"])
+
     def test_an_overflowing_ratio_fails_and_the_report_stays_strict_json(self):
         summary = small_summary()
         summary["lanes"]["compose"]["warm_startup_seconds"] = entry([5e-324] * 10)