From 0f7f691fbe177c36a0958d6b83e87991b80096da Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Wed, 21 Oct 2026 06:09:28 +0000 Subject: [PATCH 1/5] test(od): pin OpenDecision fixtures to the od-base counts PR #7 removed the od-tiny preset from src, but the od fixtures still built manifests with its dimensions (seq_len 8/18, param_count 116 288 / 187 140). The validators now reject them, so 80 od tests in heavy fail before they test anything. Use the od-base values the validators pin: 123 753 984 trainable parameters for MLM, 143 241 220 for the decision/distill heads, MLM seq_len 1024. The v1 training helpers in od_fixtures keep seq_len 32, which RunManifest accepts; only check_manifest (A6) needs N = 1024, so od_body/cfg_of take a seq_len argument for those two tests. The stage split assertion moves to the od-base layer boundary (12 layers, 2 stages: blocks 0-5 and 6-11). The authority snapshot script checks the same od-base MLM count. od-tiny is not reintroduced (founder decision D1 = A). Co-Authored-By: Claude Sonnet 5.5 --- scripts/network_authority_snapshot.py | 2 +- tests/layout/test_od_island_v2.py | 4 ++-- tests/models/od_fixtures.py | 17 +++++++++++------ tests/models/test_opendecision.py | 14 ++++++++------ tests/protocol/test_network_v2.py | 8 ++++---- 5 files changed, 26 insertions(+), 19 deletions(-) diff --git a/scripts/network_authority_snapshot.py b/scripts/network_authority_snapshot.py index ff3e0fe..e533c65 100644 --- a/scripts/network_authority_snapshot.py +++ b/scripts/network_authority_snapshot.py @@ -144,7 +144,7 @@ def _authority(root: Path) -> Bundle: m.arch != "od-encoder" or m.od is None or m.od.objective != "mlm" - or m.param_count != 116288 + or m.param_count != 123_753_984 or layout.n_gpus != 2 or layout.dp_size != 2 or layout.ep_size != 1 diff --git a/tests/layout/test_od_island_v2.py b/tests/layout/test_od_island_v2.py index 1797cb5..be0517d 100644 --- a/tests/layout/test_od_island_v2.py +++ b/tests/layout/test_od_island_v2.py @@ -33,8 +33,8 @@ def od_manifest(objective: str, n: int, zero1: bool, dtype: str = "fp32") -> Run b = mod.od_body() rec = dict(state_len=8, n_questions=1, n_options=2, opt_len=2, instr_len=2) b["model"].update( - seq_len=8 if objective == "mlm" else 18, - param_count=116288 if objective == "mlm" else 187140, + seq_len=1024 if objective == "mlm" else 18, + param_count=123_753_984 if objective == "mlm" else 143_241_220, compute_dtype=dtype, ) b["model"]["od"].update(objective=objective, record=None if objective == "mlm" else rec) diff --git a/tests/models/od_fixtures.py b/tests/models/od_fixtures.py index dd74bd8..00a93c2 100644 --- a/tests/models/od_fixtures.py +++ b/tests/models/od_fixtures.py @@ -17,10 +17,11 @@ from hypertrain.trainer.rng import rng_ctr, uniform_f64 RUN_ID = "cd" * 32 -SEQ = 32 +SEQ = 32 # v1 training path; check_manifest (A6) pins od-base mlm to N = 1024 +N = 1024 H, J, MB = 3, 1, 2 REC = {"state_len": 16, "n_questions": 2, "n_options": 3, "opt_len": 4, "instr_len": 4} -STAGE_A_COUNT, FULL_COUNT = 116_288, 187_140 +STAGE_A_COUNT, FULL_COUNT = 123_753_984, 143_241_220 def _l2_od_body() -> dict[str, Any]: @@ -32,14 +33,16 @@ def _l2_od_body() -> dict[str, Any]: return mod.od_body() # type: ignore[no-any-return] -def od_body(objective: str = "mlm", profile: str = "od-fp32-ref-v1") -> dict[str, Any]: +def od_body( + objective: str = "mlm", profile: str = "od-fp32-ref-v1", seq_len: int = SEQ +) -> dict[str, Any]: b = copy.deepcopy(_l2_od_body()) b["reference_spec"]["profile"] = profile b["model"]["compute_dtype"] = "fp32" if profile == "od-fp32-ref-v1" else "bf16" b["inner"] |= {"H": H, "J": J, "micro_batch": MB, "grad_accum": 1} b["inner"]["lr_schedule"] |= {"peak_lr": f32hex(1e-3), "warmup": 1} if objective == "mlm": - b["model"] |= {"seq_len": SEQ, "param_count": STAGE_A_COUNT} + b["model"] |= {"seq_len": seq_len, "param_count": STAGE_A_COUNT} else: shape = RecordShape(**REC) b["model"] |= {"seq_len": shape.length - 1, "param_count": FULL_COUNT} @@ -47,8 +50,10 @@ def od_body(objective: str = "mlm", profile: str = "od-fp32-ref-v1") -> dict[str return b -def cfg_of(objective: str = "mlm", profile: str = "od-fp32-ref-v1") -> TrainConfig: - return TrainConfig.from_manifest(od_body(objective, profile)) +def cfg_of( + objective: str = "mlm", profile: str = "od-fp32-ref-v1", seq_len: int = SEQ +) -> TrainConfig: + return TrainConfig.from_manifest(od_body(objective, profile, seq_len)) def text_sample(cfg: TrainConfig) -> SampleFn: diff --git a/tests/models/test_opendecision.py b/tests/models/test_opendecision.py index 06627e2..97d781c 100644 --- a/tests/models/test_opendecision.py +++ b/tests/models/test_opendecision.py @@ -14,6 +14,7 @@ from od_fixtures import ( FULL_COUNT, STAGE_A_COUNT, + N, assignment, cfg_of, fresh_carry, @@ -201,7 +202,8 @@ def test_stage_a_param_set_excludes_head() -> None: assert sf == {k: tuple(v.shape) for k, v in sd.items()} assert sorted(od.init_params(a)) == sorted(sa) assert od.stage_of("tok.weight", a, 2) == 0 - assert od.stage_of("encoder.blocks.1.qkv.weight", a, 2) == 1 + assert od.stage_of("encoder.blocks.5.qkv.weight", a, 2) == 0 + assert od.stage_of("encoder.blocks.6.qkv.weight", a, 2) == 1 assert od.stage_of("scorer.3.bias", full, 2) == 1 @@ -262,16 +264,16 @@ def test_extend_params_and_warmstart_cli(tmp_path: Path) -> None: def test_check_manifest_param_count() -> None: - od.check_manifest(RunManifest.model_validate(od_body())) + od.check_manifest(RunManifest.model_validate(od_body(seq_len=N))) od.check_manifest(RunManifest.model_validate(od_body("decision"))) - b = od_body() + b = od_body(seq_len=N) b["model"]["param_count"] = FULL_COUNT with pytest.raises(ValueError, match="param_count"): od.check_manifest(RunManifest.model_validate(b)) def test_eval_holdout_and_eval_cli(tmp_path: Path) -> None: - cfg = cfg_of() + cfg = cfg_of(seq_len=N) get = text_sample(cfg) theta = od.init_params(cfg.model) v = od.eval_holdout(cfg.model, theta, get, [0, 1, 2], batch=2) @@ -285,10 +287,10 @@ def test_eval_holdout_and_eval_cli(tmp_path: Path) -> None: write_shards16((get(i) for i in range(8)), hold, cfg.model.seq_len, unit=4, samples_per_shard=8) state, man, metrics = tmp_path / "s", tmp_path / "run.json", tmp_path / "public" / "m.jsonl" state.write_bytes(pack_state(theta)) - man.write_text(json.dumps(od_body())) + man.write_text(json.dumps(od_body(seq_len=N))) args = ["eval", "--manifest", str(man), "--state", str(state), "--holdout", str(hold)] assert main([*args, "--w", "4", "--metrics", str(metrics)]) == 0 line = json.loads(metrics.read_text()) - run_id = RunManifest.model_validate(od_body()).run_id() + run_id = RunManifest.model_validate(od_body(seq_len=N)).run_id() want = od.eval_holdout(cfg.model, theta, get, range(8)) assert line == {"run_id": run_id, "w": 4, "value": want} diff --git a/tests/protocol/test_network_v2.py b/tests/protocol/test_network_v2.py index 1ef100e..340e80b 100644 --- a/tests/protocol/test_network_v2.py +++ b/tests/protocol/test_network_v2.py @@ -131,13 +131,13 @@ def commit(tokens: int = 7680) -> CommitV2: def test_od_general_n_when_v2(n: int, objective: str, zero1: bool) -> None: # Given body = od_body() - body["model"]["param_count"] = 116_288 + body["model"]["param_count"] = 123_753_984 body["inner"].update(H=1, J=1, micro_batch=1, grad_accum=1) body["reference_spec"]["layout"].update(n_gpus=n, dp_size=n, ep_size=1, zero1=zero1) if objective != "mlm": rec = {"state_len": 8, "n_questions": 1, "n_options": 2, "opt_len": 2, "instr_len": 2} body["model"]["od"].update(objective=objective, record=rec) - body["model"].update(seq_len=18, param_count=187_140) + body["model"].update(seq_len=18, param_count=143_241_220) # When model = wrapper(body) # Then @@ -152,7 +152,7 @@ def test_od_general_n_when_v2(n: int, objective: str, zero1: bool) -> None: def test_assign_unit_when_only_island_batch_is_divisible() -> None: # Given body = od_body() - body["model"]["param_count"] = 116_288 + body["model"]["param_count"] = 123_753_984 body["inner"].update(H=1, J=1, micro_batch=1, grad_accum=1) body["reference_spec"]["layout"].update(n_gpus=8, dp_size=8) body["dataset"].update(assign_unit=8, unit_sha256_root=H, n_samples=4096) @@ -1026,7 +1026,7 @@ def test_aggregation_when_computational_f32_is_nonfinite(field: str) -> None: @pytest.mark.parametrize("field", ["mask_ratio", "rps_weight", "distill_temp"]) def test_od_when_objective_f32_is_nonfinite(field: str) -> None: body = od_body() # Given - body["model"]["param_count"] = 116_288 + body["model"]["param_count"] = 123_753_984 body["model"]["od"][field] = "0000c07f" with pytest.raises(ValidationError): # When / Then wrapper(body) From 47d843f3317b46a2f51c419a727135cc0ac77ef0 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Wed, 21 Oct 2026 06:55:47 +0000 Subject: [PATCH 2/5] test(od): keep the od-base island hook test within memory and time On od-base the hook test kept the raw bytes of every traced tensor: 5 726 tensors and 14.9 GB for one n=1 decision round, twice over, and the n=2 and n=4 cases ran a worker out of memory on a 186 GB host. It now keeps a SHA-256 of the same bytes, which is still compared between the traced and the repeated run. Equal digests mean equal bytes, so the "same ops, same values" check keeps its strength. The warm-start check called init_params(mlm) twice per key, about 400 od-base initialisations at ~3 s each (~10 min per decision/distill case). It now builds one independent initialisation and compares every key against it, the same assertion. Co-Authored-By: Claude Sonnet 5.5 --- tests/layout/test_od_island_v2.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/layout/test_od_island_v2.py b/tests/layout/test_od_island_v2.py index be0517d..1e0b23f 100644 --- a/tests/layout/test_od_island_v2.py +++ b/tests/layout/test_od_island_v2.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib import importlib.util from pathlib import Path @@ -103,7 +104,9 @@ def test_od_objectives_layout_reference_and_hooks(objective: str, n: int, zero1: seen = [[] for _ in range(n)] def hook(ctx: TraceContext, x: torch.Tensor) -> torch.Tensor: - raw = x.detach().cpu().contiguous().reshape(-1).view(torch.uint8).numpy().tobytes() + raw = hashlib.sha256( + x.detach().cpu().contiguous().reshape(-1).view(torch.uint8).numpy() + ).digest() seen[ctx.rank].append((ctx, raw)) return x @@ -121,7 +124,9 @@ def hook(ctx: TraceContext, x: torch.Tensor) -> torch.Tensor: repeated = [[] for _ in range(n)] def capture(ctx: TraceContext, x: torch.Tensor) -> torch.Tensor: - raw = x.detach().cpu().contiguous().reshape(-1).view(torch.uint8).numpy().tobytes() + raw = hashlib.sha256( + x.detach().cpu().contiguous().reshape(-1).view(torch.uint8).numpy() + ).digest() repeated[ctx.rank].append((ctx, raw)) return x @@ -133,9 +138,9 @@ def capture(ctx: TraceContext, x: torch.Tensor) -> torch.Tensor: if objective != "mlm": mlm = TrainConfig.from_manifest_v2(od_manifest("mlm", 1, False)) extended = extend_params(init_params(mlm.model), cfg.model) - assert all( - torch.equal(extended[k], init_params(mlm.model)[k]) for k in init_params(mlm.model) - ) + # One independent re-init, not one per key: od-base init is ~3 s. + stage_a = init_params(mlm.model) + assert all(torch.equal(extended[k], stage_a[k]) for k in stage_a) pots = emulate( n, lambda c: honeypot_island(Honeypot(w.training.coord_pubkey, "honest"), w, c, theta, a, get), From bc33e53489852b6a5a6ea630c6f60f1a9be08ce3 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Wed, 21 Oct 2026 06:55:47 +0000 Subject: [PATCH 3/5] test(miner): give the real od-base two-rank launch a 900 s deadline test_real_od_two_rank_worker_paths set the island job deadline to now + 120 s, sized for od-tiny. A real od-base round on two torchrun CPU ranks (1 thread each, as the manifest pins) takes 150-660 s on the test host, so launch_island hit "island deadline expired" before the workers finished. The deadline is test input, not the behaviour under test: the assertions (delta, state and trace equal to the emulated reference) are unchanged. 900 s leaves room on a loaded 16 vCPU runner. Co-Authored-By: Claude Sonnet 5.5 --- tests/miner/test_island_launch_v2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/miner/test_island_launch_v2.py b/tests/miner/test_island_launch_v2.py index dcd0fea..e3a3ebf 100644 --- a/tests/miner/test_island_launch_v2.py +++ b/tests/miner/test_island_launch_v2.py @@ -1071,7 +1071,7 @@ def test_real_od_two_rank_worker_paths(tmp_path: Path, objective: str) -> None: ef_in_sha256=hashlib.sha256(blobs["ef_in"]).hexdigest(), v0_sha256=hashlib.sha256(blobs["v0"]).hexdigest(), object_paths={k: k for k in blobs}, - deadline=int(time.time()) + 120, + deadline=int(time.time()) + 900, ) result = launch_island(job, tmp_path, backend="cpu", trace=True) refs = emulate( From d362de139531a02620cc990620adb6c166546c95 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Wed, 21 Oct 2026 08:49:38 +0000 Subject: [PATCH 4/5] ci(heavy): run the real od-base island tests in a sharded od-suite job With the fixtures on od-base, the island files are real 123.8 M / 143.2 M parameter rounds at the manifest's 1 CPU thread. Measured serially on a 32-core host: test_od_island_v2.py 62.5 min (peak 53 GB RSS), test_island_bisect_v2.py 10.5, test_island_replay_v2.py 10.4, test_island_launch_v2.py 7.0, test_carry_genesis_v2.py 2.5. Adding them to full-suite (54.5 min today) would far exceed its 90 min limit. full-suite now ignores these files. A new od-suite job runs them in three parallel shards of about 31 min each on the same Blacksmith 16 vCPU runner, with a 75 min timeout: the six n=4 layout cases, the rest of the layout file, and the four island files. Every test still runs and none is reduced (founder decision, 2026-10-21, option A). Collection check: full-suite 2731 + od-suite 90 + the 14 below = 2835, the same as before. test_authority_snapshot_v2.py is ignored everywhere and documented as a residue (founder decision, option A). Its 48-trial fixture writes ~190 GB on od-base, and a 1.48 GB od-base start_state exceeds the snapshot script's MAX_FILE (16 MiB) and MAX_BYTES (1 GiB), so export fails with "snapshot file size/type" after 65 min. Rejected: lifting those bounds in the test, which would no longer prove the real bounds and does not fit the runner disk. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/heavy.yml | 61 ++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/.github/workflows/heavy.yml b/.github/workflows/heavy.yml index b05869a..6a10371 100644 --- a/.github/workflows/heavy.yml +++ b/.github/workflows/heavy.yml @@ -160,4 +160,63 @@ jobs: chmod 755 "$RUNNER_TEMP/kind-bin/kind" "$RUNNER_TEMP/kind-bin/kind" version | grep -q "^kind $v " echo "$RUNNER_TEMP/kind-bin" >> "$GITHUB_PATH" - - run: bash scripts/check.sh -rs --ignore=tests/sim --ignore=tests/e2e + # The real od-base island files run in od-suite below. test_authority_snapshot_v2.py + # cannot run on od-base: its 48-trial fixture writes ~190 GB and the exported + # 1.48 GB start_state exceeds scripts/network_authority_snapshot.py MAX_FILE + # (16 MiB) / MAX_BYTES (1 GiB). Founder decision 2026-10-21: documented residue, + # see livrables/forge/2026-10-21_heavy-od-fixtures.md. + - run: >- + bash scripts/check.sh -rs --ignore=tests/sim --ignore=tests/e2e + --ignore=tests/layout/test_od_island_v2.py + --ignore=tests/auditor/test_island_bisect_v2.py + --ignore=tests/auditor/test_island_replay_v2.py + --ignore=tests/miner/test_island_launch_v2.py + --ignore=tests/challenge/test_carry_genesis_v2.py + --ignore=tests/challenge/test_authority_snapshot_v2.py + + # Real od-base (123.8 M / 143.2 M params) island tests at the manifest's 1 CPU thread. + # Serial on a 32-core host, 2026-10-21: test_od_island_v2.py 62.5 min (peak 53 GB RSS), + # bisect 10.5, replay 10.4, launch 7.0, carry 2.5 min. Three ~31 min shards. + od-suite: + runs-on: blacksmith-16vcpu-ubuntu-2404 + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + shard: [layout-n4, layout-rest, island] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + version: "0.12.17" + python-version: "3.12" + enable-cache: true + cache-dependency-glob: "uv.lock" + - run: uv sync --locked --all-extras --group dev + - name: od-base shard + env: + SHARD: ${{ matrix.shard }} + shell: bash + run: | + set -euo pipefail + N4=( + "tests/layout/test_od_island_v2.py::test_od_objectives_layout_reference_and_hooks[False-4-mlm]" + "tests/layout/test_od_island_v2.py::test_od_objectives_layout_reference_and_hooks[False-4-decision]" + "tests/layout/test_od_island_v2.py::test_od_objectives_layout_reference_and_hooks[False-4-distill]" + "tests/layout/test_od_island_v2.py::test_od_objectives_layout_reference_and_hooks[True-4-mlm]" + "tests/layout/test_od_island_v2.py::test_od_objectives_layout_reference_and_hooks[True-4-decision]" + "tests/layout/test_od_island_v2.py::test_od_objectives_layout_reference_and_hooks[True-4-distill]" + ) + case "$SHARD" in + layout-n4) SEL=("${N4[@]}") ;; + layout-rest) + SEL=(tests/layout/test_od_island_v2.py) + for id in "${N4[@]}"; do SEL+=("--deselect=$id"); done ;; + island) + SEL=(tests/auditor/test_island_bisect_v2.py tests/auditor/test_island_replay_v2.py + tests/miner/test_island_launch_v2.py tests/challenge/test_carry_genesis_v2.py) ;; + *) echo "unknown shard $SHARD" >&2; exit 1 ;; + esac + uv run --frozen pytest -q -rs --durations=25 "${SEL[@]}" From 39de077c17aea5d7ae45c6c205d4bd49a446732f Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Wed, 21 Oct 2026 20:04:11 +0000 Subject: [PATCH 5/5] ci(heavy): deselect the 14 authority-snapshot tests by name, log shard RAM full-suite ignored the whole test_authority_snapshot_v2.py file. It now deselects its 14 test ids by name, so a test added to that file later still runs. The 14 ids cannot run on od-base: the 48-trial fixture writes ~190 GB and the 1.48 GB start_state exceeds the snapshot script's MAX_FILE (16 MiB) and MAX_BYTES (1 GiB). Each od-suite shard samples host memory every 5 s and writes its peak to the job summary, so the runner size can be set from a measured number instead of the 53 GB local peak. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/heavy.yml | 57 ++++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/.github/workflows/heavy.yml b/.github/workflows/heavy.yml index 6a10371..47de9d0 100644 --- a/.github/workflows/heavy.yml +++ b/.github/workflows/heavy.yml @@ -160,19 +160,41 @@ jobs: chmod 755 "$RUNNER_TEMP/kind-bin/kind" "$RUNNER_TEMP/kind-bin/kind" version | grep -q "^kind $v " echo "$RUNNER_TEMP/kind-bin" >> "$GITHUB_PATH" - # The real od-base island files run in od-suite below. test_authority_snapshot_v2.py - # cannot run on od-base: its 48-trial fixture writes ~190 GB and the exported - # 1.48 GB start_state exceeds scripts/network_authority_snapshot.py MAX_FILE - # (16 MiB) / MAX_BYTES (1 GiB). Founder decision 2026-10-21: documented residue, - # see livrables/forge/2026-10-21_heavy-od-fixtures.md. - - run: >- - bash scripts/check.sh -rs --ignore=tests/sim --ignore=tests/e2e - --ignore=tests/layout/test_od_island_v2.py - --ignore=tests/auditor/test_island_bisect_v2.py - --ignore=tests/auditor/test_island_replay_v2.py - --ignore=tests/miner/test_island_launch_v2.py - --ignore=tests/challenge/test_carry_genesis_v2.py - --ignore=tests/challenge/test_authority_snapshot_v2.py + # The real od-base island files run in od-suite below. The 14 tests of + # test_authority_snapshot_v2.py are deselected by name: their 48-trial fixture writes + # ~190 GB on od-base and the exported 1.48 GB start_state exceeds + # scripts/network_authority_snapshot.py MAX_FILE (16 MiB) / MAX_BYTES (1 GiB). + # Founder decision 2026-10-21 (option A): documented residue, tracked in an issue. + # Deselected by name, not --ignore, so a new test added to that file still runs. + - name: Full suite + shell: bash + run: | + set -euo pipefail + SNAP=tests/challenge/test_authority_snapshot_v2.py + DESELECT=() + for t in \ + 'test_original_export_validates_and_bootstraps_actual_service' \ + 'test_modified_package_rejects_before_restore[object]' \ + 'test_modified_package_rejects_before_restore[reference]' \ + 'test_modified_package_rejects_before_restore[foreign]' \ + 'test_modified_package_rejects_before_restore[traversal]' \ + 'test_modified_package_rejects_before_restore[digest]' \ + 'test_modified_package_rejects_before_restore[screen]' \ + 'test_modified_package_rejects_before_restore[strike]' \ + 'test_modified_package_rejects_before_restore[pending]' \ + 'test_modified_package_rejects_before_restore[funding]' \ + 'test_modified_package_rejects_before_restore[rotation]' \ + 'test_modified_package_rejects_before_restore[signature]' \ + 'test_modified_package_rejects_before_restore[finality]' \ + 'test_modified_package_rejects_before_restore[assignment]' + do DESELECT+=("--deselect=$SNAP::$t"); done + bash scripts/check.sh -rs --ignore=tests/sim --ignore=tests/e2e \ + --ignore=tests/layout/test_od_island_v2.py \ + --ignore=tests/auditor/test_island_bisect_v2.py \ + --ignore=tests/auditor/test_island_replay_v2.py \ + --ignore=tests/miner/test_island_launch_v2.py \ + --ignore=tests/challenge/test_carry_genesis_v2.py \ + "${DESELECT[@]}" # Real od-base (123.8 M / 143.2 M params) island tests at the manifest's 1 CPU thread. # Serial on a 32-core host, 2026-10-21: test_od_island_v2.py 62.5 min (peak 53 GB RSS), @@ -219,4 +241,11 @@ jobs: tests/miner/test_island_launch_v2.py tests/challenge/test_carry_genesis_v2.py) ;; *) echo "unknown shard $SHARD" >&2; exit 1 ;; esac - uv run --frozen pytest -q -rs --durations=25 "${SEL[@]}" + # Peak host memory for this shard: sample `free` every 5 s, print the peak. + ( while sleep 5; do free -m | awk '/^Mem:/{print $3}'; done ) > "$RUNNER_TEMP/mem.log" & + SAMPLER=$! + rc=0 + uv run --frozen pytest -q -rs --durations=25 "${SEL[@]}" || rc=$? + kill "$SAMPLER" 2>/dev/null || true + echo "shard=$SHARD peak_used_MiB=$(sort -n "$RUNNER_TEMP/mem.log" | tail -1) total_MiB=$(free -m | awk '/^Mem:/{print $2}')" | tee -a "$GITHUB_STEP_SUMMARY" + exit $rc