test(od): run the OpenDecision fixtures on od-base, od-suite job in heavy

.github/workflows/heavy.yml

@@ -160,4 +160,92 @@ jobs:
           chmod 755 "$RUNNER_TEMP/kind-bin/kind"
           "$RUNNER_TEMP/kind-bin/kind" version | grep -q "^kind $v "
           echo "$RUNNER_TEMP/kind-bin" >> "$GITHUB_PATH"
-      - run: bash scripts/check.sh -rs --ignore=tests/sim --ignore=tests/e2e
+      # The real od-base island files run in od-suite below. The 14 tests of
+      # test_authority_snapshot_v2.py are deselected by name: their 48-trial fixture writes
+      # ~190 GB on od-base and the exported 1.48 GB start_state exceeds
+      # scripts/network_authority_snapshot.py MAX_FILE (16 MiB) / MAX_BYTES (1 GiB).
+      # Founder decision 2026-10-21 (option A): documented residue, tracked in an issue.
+      # Deselected by name, not --ignore, so a new test added to that file still runs.
+      - name: Full suite
+        shell: bash
+        run: |
+          set -euo pipefail
+          SNAP=tests/challenge/test_authority_snapshot_v2.py
+          DESELECT=()
+          for t in \
+            'test_original_export_validates_and_bootstraps_actual_service' \
+            'test_modified_package_rejects_before_restore[object]' \
+            'test_modified_package_rejects_before_restore[reference]' \
+            'test_modified_package_rejects_before_restore[foreign]' \
+            'test_modified_package_rejects_before_restore[traversal]' \
+            'test_modified_package_rejects_before_restore[digest]' \
+            'test_modified_package_rejects_before_restore[screen]' \
+            'test_modified_package_rejects_before_restore[strike]' \
+            'test_modified_package_rejects_before_restore[pending]' \
+            'test_modified_package_rejects_before_restore[funding]' \
+            'test_modified_package_rejects_before_restore[rotation]' \
+            'test_modified_package_rejects_before_restore[signature]' \
+            'test_modified_package_rejects_before_restore[finality]' \
+            'test_modified_package_rejects_before_restore[assignment]'
+          do DESELECT+=("--deselect=$SNAP::$t"); done
+          bash scripts/check.sh -rs --ignore=tests/sim --ignore=tests/e2e \
+            --ignore=tests/layout/test_od_island_v2.py \
+            --ignore=tests/auditor/test_island_bisect_v2.py \
+            --ignore=tests/auditor/test_island_replay_v2.py \
+            --ignore=tests/miner/test_island_launch_v2.py \
+            --ignore=tests/challenge/test_carry_genesis_v2.py \
+            "${DESELECT[@]}"
+
+  # Real od-base (123.8 M / 143.2 M params) island tests at the manifest's 1 CPU thread.
+  # Serial on a 32-core host, 2026-10-21: test_od_island_v2.py 62.5 min (peak 53 GB RSS),
+  # bisect 10.5, replay 10.4, launch 7.0, carry 2.5 min. Three ~31 min shards.
+  od-suite:
+    runs-on: blacksmith-16vcpu-ubuntu-2404
+    timeout-minutes: 75
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [layout-n4, layout-rest, island]
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
+        with:
+          version: "0.12.17"
+          python-version: "3.12"
+          enable-cache: true
+          cache-dependency-glob: "uv.lock"
+      - run: uv sync --locked --all-extras --group dev
+      - name: od-base shard
+        env:
+          SHARD: ${{ matrix.shard }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          N4=(
+            "tests/layout/test_od_island_v2.py::test_od_objectives_layout_reference_and_hooks[False-4-mlm]"
+            "tests/layout/test_od_island_v2.py::test_od_objectives_layout_reference_and_hooks[False-4-decision]"
+            "tests/layout/test_od_island_v2.py::test_od_objectives_layout_reference_and_hooks[False-4-distill]"
+            "tests/layout/test_od_island_v2.py::test_od_objectives_layout_reference_and_hooks[True-4-mlm]"
+            "tests/layout/test_od_island_v2.py::test_od_objectives_layout_reference_and_hooks[True-4-decision]"
+            "tests/layout/test_od_island_v2.py::test_od_objectives_layout_reference_and_hooks[True-4-distill]"
+          )
+          case "$SHARD" in
+            layout-n4) SEL=("${N4[@]}") ;;
+            layout-rest)
+              SEL=(tests/layout/test_od_island_v2.py)
+              for id in "${N4[@]}"; do SEL+=("--deselect=$id"); done ;;
+            island)
+              SEL=(tests/auditor/test_island_bisect_v2.py tests/auditor/test_island_replay_v2.py
+                tests/miner/test_island_launch_v2.py tests/challenge/test_carry_genesis_v2.py) ;;
+            *) echo "unknown shard $SHARD" >&2; exit 1 ;;
+          esac
+          # Peak host memory for this shard: sample `free` every 5 s, print the peak.
+          ( while sleep 5; do free -m | awk '/^Mem:/{print $3}'; done ) > "$RUNNER_TEMP/mem.log" &
+          SAMPLER=$!
+          rc=0
+          uv run --frozen pytest -q -rs --durations=25 "${SEL[@]}" || rc=$?
+          kill "$SAMPLER" 2>/dev/null || true
+          echo "shard=$SHARD peak_used_MiB=$(sort -n "$RUNNER_TEMP/mem.log" | tail -1) total_MiB=$(free -m | awk '/^Mem:/{print $2}')" | tee -a "$GITHUB_STEP_SUMMARY"
+          exit $rc

scripts/network_authority_snapshot.py

@@ -144,7 +144,7 @@ def _authority(root: Path) -> Bundle:
             m.arch != "od-encoder"
             or m.od is None
             or m.od.objective != "mlm"
-            or m.param_count != 116288
+            or m.param_count != 123_753_984
             or layout.n_gpus != 2
             or layout.dp_size != 2
             or layout.ep_size != 1

tests/layout/test_od_island_v2.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import hashlib
 import importlib.util
 from pathlib import Path
 
@@ -33,8 +34,8 @@ def od_manifest(objective: str, n: int, zero1: bool, dtype: str = "fp32") -> Run
     b = mod.od_body()
     rec = dict(state_len=8, n_questions=1, n_options=2, opt_len=2, instr_len=2)
     b["model"].update(
-        seq_len=8 if objective == "mlm" else 18,
-        param_count=116288 if objective == "mlm" else 187140,
+        seq_len=1024 if objective == "mlm" else 18,
+        param_count=123_753_984 if objective == "mlm" else 143_241_220,
         compute_dtype=dtype,
     )
     b["model"]["od"].update(objective=objective, record=None if objective == "mlm" else rec)
@@ -103,7 +104,9 @@ def test_od_objectives_layout_reference_and_hooks(objective: str, n: int, zero1:
     seen = [[] for _ in range(n)]
 
     def hook(ctx: TraceContext, x: torch.Tensor) -> torch.Tensor:
-        raw = x.detach().cpu().contiguous().reshape(-1).view(torch.uint8).numpy().tobytes()
+        raw = hashlib.sha256(
+            x.detach().cpu().contiguous().reshape(-1).view(torch.uint8).numpy()
+        ).digest()
         seen[ctx.rank].append((ctx, raw))
         return x
 
@@ -121,7 +124,9 @@ def hook(ctx: TraceContext, x: torch.Tensor) -> torch.Tensor:
     repeated = [[] for _ in range(n)]
 
     def capture(ctx: TraceContext, x: torch.Tensor) -> torch.Tensor:
-        raw = x.detach().cpu().contiguous().reshape(-1).view(torch.uint8).numpy().tobytes()
+        raw = hashlib.sha256(
+            x.detach().cpu().contiguous().reshape(-1).view(torch.uint8).numpy()
+        ).digest()
         repeated[ctx.rank].append((ctx, raw))
         return x
 
@@ -133,9 +138,9 @@ def capture(ctx: TraceContext, x: torch.Tensor) -> torch.Tensor:
     if objective != "mlm":
         mlm = TrainConfig.from_manifest_v2(od_manifest("mlm", 1, False))
         extended = extend_params(init_params(mlm.model), cfg.model)
-        assert all(
-            torch.equal(extended[k], init_params(mlm.model)[k]) for k in init_params(mlm.model)
-        )
+        # One independent re-init, not one per key: od-base init is ~3 s.
+        stage_a = init_params(mlm.model)
+        assert all(torch.equal(extended[k], stage_a[k]) for k in stage_a)
     pots = emulate(
         n,
         lambda c: honeypot_island(Honeypot(w.training.coord_pubkey, "honest"), w, c, theta, a, get),

tests/miner/test_island_launch_v2.py

@@ -1071,7 +1071,7 @@ def test_real_od_two_rank_worker_paths(tmp_path: Path, objective: str) -> None:
         ef_in_sha256=hashlib.sha256(blobs["ef_in"]).hexdigest(),
         v0_sha256=hashlib.sha256(blobs["v0"]).hexdigest(),
         object_paths={k: k for k in blobs},
-        deadline=int(time.time()) + 120,
+        deadline=int(time.time()) + 900,
     )
     result = launch_island(job, tmp_path, backend="cpu", trace=True)
     refs = emulate(

tests/models/od_fixtures.py

@@ -17,10 +17,11 @@
 from hypertrain.trainer.rng import rng_ctr, uniform_f64
 
 RUN_ID = "cd" * 32
-SEQ = 32
+SEQ = 32  # v1 training path; check_manifest (A6) pins od-base mlm to N = 1024
+N = 1024
 H, J, MB = 3, 1, 2
 REC = {"state_len": 16, "n_questions": 2, "n_options": 3, "opt_len": 4, "instr_len": 4}
-STAGE_A_COUNT, FULL_COUNT = 116_288, 187_140
+STAGE_A_COUNT, FULL_COUNT = 123_753_984, 143_241_220
 
 
 def _l2_od_body() -> dict[str, Any]:
@@ -32,23 +33,27 @@ def _l2_od_body() -> dict[str, Any]:
     return mod.od_body()  # type: ignore[no-any-return]
 
 
-def od_body(objective: str = "mlm", profile: str = "od-fp32-ref-v1") -> dict[str, Any]:
+def od_body(
+    objective: str = "mlm", profile: str = "od-fp32-ref-v1", seq_len: int = SEQ
+) -> dict[str, Any]:
     b = copy.deepcopy(_l2_od_body())
     b["reference_spec"]["profile"] = profile
     b["model"]["compute_dtype"] = "fp32" if profile == "od-fp32-ref-v1" else "bf16"
     b["inner"] |= {"H": H, "J": J, "micro_batch": MB, "grad_accum": 1}
     b["inner"]["lr_schedule"] |= {"peak_lr": f32hex(1e-3), "warmup": 1}
     if objective == "mlm":
-        b["model"] |= {"seq_len": SEQ, "param_count": STAGE_A_COUNT}
+        b["model"] |= {"seq_len": seq_len, "param_count": STAGE_A_COUNT}
     else:
         shape = RecordShape(**REC)
         b["model"] |= {"seq_len": shape.length - 1, "param_count": FULL_COUNT}
         b["model"]["od"] |= {"objective": objective, "record": dict(REC)}
     return b
 
 
-def cfg_of(objective: str = "mlm", profile: str = "od-fp32-ref-v1") -> TrainConfig:
-    return TrainConfig.from_manifest(od_body(objective, profile))
+def cfg_of(
+    objective: str = "mlm", profile: str = "od-fp32-ref-v1", seq_len: int = SEQ
+) -> TrainConfig:
+    return TrainConfig.from_manifest(od_body(objective, profile, seq_len))
 
 
 def text_sample(cfg: TrainConfig) -> SampleFn:

tests/models/test_opendecision.py

@@ -14,6 +14,7 @@
 from od_fixtures import (
     FULL_COUNT,
     STAGE_A_COUNT,
+    N,
     assignment,
     cfg_of,
     fresh_carry,
@@ -201,7 +202,8 @@ def test_stage_a_param_set_excludes_head() -> None:
     assert sf == {k: tuple(v.shape) for k, v in sd.items()}
     assert sorted(od.init_params(a)) == sorted(sa)
     assert od.stage_of("tok.weight", a, 2) == 0
-    assert od.stage_of("encoder.blocks.1.qkv.weight", a, 2) == 1
+    assert od.stage_of("encoder.blocks.5.qkv.weight", a, 2) == 0
+    assert od.stage_of("encoder.blocks.6.qkv.weight", a, 2) == 1
     assert od.stage_of("scorer.3.bias", full, 2) == 1
 
 
@@ -262,16 +264,16 @@ def test_extend_params_and_warmstart_cli(tmp_path: Path) -> None:
 
 
 def test_check_manifest_param_count() -> None:
-    od.check_manifest(RunManifest.model_validate(od_body()))
+    od.check_manifest(RunManifest.model_validate(od_body(seq_len=N)))
     od.check_manifest(RunManifest.model_validate(od_body("decision")))
-    b = od_body()
+    b = od_body(seq_len=N)
     b["model"]["param_count"] = FULL_COUNT
     with pytest.raises(ValueError, match="param_count"):
         od.check_manifest(RunManifest.model_validate(b))
 
 
 def test_eval_holdout_and_eval_cli(tmp_path: Path) -> None:
-    cfg = cfg_of()
+    cfg = cfg_of(seq_len=N)
     get = text_sample(cfg)
     theta = od.init_params(cfg.model)
     v = od.eval_holdout(cfg.model, theta, get, [0, 1, 2], batch=2)
@@ -285,10 +287,10 @@ def test_eval_holdout_and_eval_cli(tmp_path: Path) -> None:
     write_shards16((get(i) for i in range(8)), hold, cfg.model.seq_len, unit=4, samples_per_shard=8)
     state, man, metrics = tmp_path / "s", tmp_path / "run.json", tmp_path / "public" / "m.jsonl"
     state.write_bytes(pack_state(theta))
-    man.write_text(json.dumps(od_body()))
+    man.write_text(json.dumps(od_body(seq_len=N)))
     args = ["eval", "--manifest", str(man), "--state", str(state), "--holdout", str(hold)]
     assert main([*args, "--w", "4", "--metrics", str(metrics)]) == 0
     line = json.loads(metrics.read_text())
-    run_id = RunManifest.model_validate(od_body()).run_id()
+    run_id = RunManifest.model_validate(od_body(seq_len=N)).run_id()
     want = od.eval_holdout(cfg.model, theta, get, range(8))
     assert line == {"run_id": run_id, "w": 4, "value": want}

tests/protocol/test_network_v2.py

@@ -131,13 +131,13 @@ def commit(tokens: int = 7680) -> CommitV2:
 def test_od_general_n_when_v2(n: int, objective: str, zero1: bool) -> None:
     # Given
     body = od_body()
-    body["model"]["param_count"] = 116_288
+    body["model"]["param_count"] = 123_753_984
     body["inner"].update(H=1, J=1, micro_batch=1, grad_accum=1)
     body["reference_spec"]["layout"].update(n_gpus=n, dp_size=n, ep_size=1, zero1=zero1)
     if objective != "mlm":
         rec = {"state_len": 8, "n_questions": 1, "n_options": 2, "opt_len": 2, "instr_len": 2}
         body["model"]["od"].update(objective=objective, record=rec)
-        body["model"].update(seq_len=18, param_count=187_140)
+        body["model"].update(seq_len=18, param_count=143_241_220)
     # When
     model = wrapper(body)
     # Then
@@ -152,7 +152,7 @@ def test_od_general_n_when_v2(n: int, objective: str, zero1: bool) -> None:
 def test_assign_unit_when_only_island_batch_is_divisible() -> None:
     # Given
     body = od_body()
-    body["model"]["param_count"] = 116_288
+    body["model"]["param_count"] = 123_753_984
     body["inner"].update(H=1, J=1, micro_batch=1, grad_accum=1)
     body["reference_spec"]["layout"].update(n_gpus=8, dp_size=8)
     body["dataset"].update(assign_unit=8, unit_sha256_root=H, n_samples=4096)
@@ -1026,7 +1026,7 @@ def test_aggregation_when_computational_f32_is_nonfinite(field: str) -> None:
 @pytest.mark.parametrize("field", ["mask_ratio", "rps_weight", "distill_temp"])
 def test_od_when_objective_f32_is_nonfinite(field: str) -> None:
     body = od_body()  # Given
-    body["model"]["param_count"] = 116_288
+    body["model"]["param_count"] = 123_753_984
     body["model"]["od"][field] = "0000c07f"
     with pytest.raises(ValidationError):  # When / Then
         wrapper(body)