diff --git a/.github/workflows/heavy.yml b/.github/workflows/heavy.yml index b05869a..47de9d0 100644 --- a/.github/workflows/heavy.yml +++ b/.github/workflows/heavy.yml @@ -160,4 +160,92 @@ jobs: chmod 755 "$RUNNER_TEMP/kind-bin/kind" "$RUNNER_TEMP/kind-bin/kind" version | grep -q "^kind $v " echo "$RUNNER_TEMP/kind-bin" >> "$GITHUB_PATH" - - run: bash scripts/check.sh -rs --ignore=tests/sim --ignore=tests/e2e + # The real od-base island files run in od-suite below. The 14 tests of + # test_authority_snapshot_v2.py are deselected by name: their 48-trial fixture writes + # ~190 GB on od-base and the exported 1.48 GB start_state exceeds + # scripts/network_authority_snapshot.py MAX_FILE (16 MiB) / MAX_BYTES (1 GiB). + # Founder decision 2026-10-21 (option A): documented residue, tracked in an issue. + # Deselected by name, not --ignore, so a new test added to that file still runs. + - name: Full suite + shell: bash + run: | + set -euo pipefail + SNAP=tests/challenge/test_authority_snapshot_v2.py + DESELECT=() + for t in \ + 'test_original_export_validates_and_bootstraps_actual_service' \ + 'test_modified_package_rejects_before_restore[object]' \ + 'test_modified_package_rejects_before_restore[reference]' \ + 'test_modified_package_rejects_before_restore[foreign]' \ + 'test_modified_package_rejects_before_restore[traversal]' \ + 'test_modified_package_rejects_before_restore[digest]' \ + 'test_modified_package_rejects_before_restore[screen]' \ + 'test_modified_package_rejects_before_restore[strike]' \ + 'test_modified_package_rejects_before_restore[pending]' \ + 'test_modified_package_rejects_before_restore[funding]' \ + 'test_modified_package_rejects_before_restore[rotation]' \ + 'test_modified_package_rejects_before_restore[signature]' \ + 'test_modified_package_rejects_before_restore[finality]' \ + 'test_modified_package_rejects_before_restore[assignment]' + do DESELECT+=("--deselect=$SNAP::$t"); done + bash scripts/check.sh -rs --ignore=tests/sim --ignore=tests/e2e \ + --ignore=tests/layout/test_od_island_v2.py \ + --ignore=tests/auditor/test_island_bisect_v2.py \ + --ignore=tests/auditor/test_island_replay_v2.py \ + --ignore=tests/miner/test_island_launch_v2.py \ + --ignore=tests/challenge/test_carry_genesis_v2.py \ + "${DESELECT[@]}" + + # Real od-base (123.8 M / 143.2 M params) island tests at the manifest's 1 CPU thread. + # Serial on a 32-core host, 2026-10-21: test_od_island_v2.py 62.5 min (peak 53 GB RSS), + # bisect 10.5, replay 10.4, launch 7.0, carry 2.5 min. Three ~31 min shards. + od-suite: + runs-on: blacksmith-16vcpu-ubuntu-2404 + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + shard: [layout-n4, layout-rest, island] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + version: "0.12.17" + python-version: "3.12" + enable-cache: true + cache-dependency-glob: "uv.lock" + - run: uv sync --locked --all-extras --group dev + - name: od-base shard + env: + SHARD: ${{ matrix.shard }} + shell: bash + run: | + set -euo pipefail + N4=( + "tests/layout/test_od_island_v2.py::test_od_objectives_layout_reference_and_hooks[False-4-mlm]" + "tests/layout/test_od_island_v2.py::test_od_objectives_layout_reference_and_hooks[False-4-decision]" + "tests/layout/test_od_island_v2.py::test_od_objectives_layout_reference_and_hooks[False-4-distill]" + "tests/layout/test_od_island_v2.py::test_od_objectives_layout_reference_and_hooks[True-4-mlm]" + "tests/layout/test_od_island_v2.py::test_od_objectives_layout_reference_and_hooks[True-4-decision]" + "tests/layout/test_od_island_v2.py::test_od_objectives_layout_reference_and_hooks[True-4-distill]" + ) + case "$SHARD" in + layout-n4) SEL=("${N4[@]}") ;; + layout-rest) + SEL=(tests/layout/test_od_island_v2.py) + for id in "${N4[@]}"; do SEL+=("--deselect=$id"); done ;; + island) + SEL=(tests/auditor/test_island_bisect_v2.py tests/auditor/test_island_replay_v2.py + tests/miner/test_island_launch_v2.py tests/challenge/test_carry_genesis_v2.py) ;; + *) echo "unknown shard $SHARD" >&2; exit 1 ;; + esac + # Peak host memory for this shard: sample `free` every 5 s, print the peak. + ( while sleep 5; do free -m | awk '/^Mem:/{print $3}'; done ) > "$RUNNER_TEMP/mem.log" & + SAMPLER=$! + rc=0 + uv run --frozen pytest -q -rs --durations=25 "${SEL[@]}" || rc=$? + kill "$SAMPLER" 2>/dev/null || true + echo "shard=$SHARD peak_used_MiB=$(sort -n "$RUNNER_TEMP/mem.log" | tail -1) total_MiB=$(free -m | awk '/^Mem:/{print $2}')" | tee -a "$GITHUB_STEP_SUMMARY" + exit $rc diff --git a/scripts/network_authority_snapshot.py b/scripts/network_authority_snapshot.py index ff3e0fe..e533c65 100644 --- a/scripts/network_authority_snapshot.py +++ b/scripts/network_authority_snapshot.py @@ -144,7 +144,7 @@ def _authority(root: Path) -> Bundle: m.arch != "od-encoder" or m.od is None or m.od.objective != "mlm" - or m.param_count != 116288 + or m.param_count != 123_753_984 or layout.n_gpus != 2 or layout.dp_size != 2 or layout.ep_size != 1 diff --git a/tests/layout/test_od_island_v2.py b/tests/layout/test_od_island_v2.py index 1797cb5..1e0b23f 100644 --- a/tests/layout/test_od_island_v2.py +++ b/tests/layout/test_od_island_v2.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib import importlib.util from pathlib import Path @@ -33,8 +34,8 @@ def od_manifest(objective: str, n: int, zero1: bool, dtype: str = "fp32") -> Run b = mod.od_body() rec = dict(state_len=8, n_questions=1, n_options=2, opt_len=2, instr_len=2) b["model"].update( - seq_len=8 if objective == "mlm" else 18, - param_count=116288 if objective == "mlm" else 187140, + seq_len=1024 if objective == "mlm" else 18, + param_count=123_753_984 if objective == "mlm" else 143_241_220, compute_dtype=dtype, ) b["model"]["od"].update(objective=objective, record=None if objective == "mlm" else rec) @@ -103,7 +104,9 @@ def test_od_objectives_layout_reference_and_hooks(objective: str, n: int, zero1: seen = [[] for _ in range(n)] def hook(ctx: TraceContext, x: torch.Tensor) -> torch.Tensor: - raw = x.detach().cpu().contiguous().reshape(-1).view(torch.uint8).numpy().tobytes() + raw = hashlib.sha256( + x.detach().cpu().contiguous().reshape(-1).view(torch.uint8).numpy() + ).digest() seen[ctx.rank].append((ctx, raw)) return x @@ -121,7 +124,9 @@ def hook(ctx: TraceContext, x: torch.Tensor) -> torch.Tensor: repeated = [[] for _ in range(n)] def capture(ctx: TraceContext, x: torch.Tensor) -> torch.Tensor: - raw = x.detach().cpu().contiguous().reshape(-1).view(torch.uint8).numpy().tobytes() + raw = hashlib.sha256( + x.detach().cpu().contiguous().reshape(-1).view(torch.uint8).numpy() + ).digest() repeated[ctx.rank].append((ctx, raw)) return x @@ -133,9 +138,9 @@ def capture(ctx: TraceContext, x: torch.Tensor) -> torch.Tensor: if objective != "mlm": mlm = TrainConfig.from_manifest_v2(od_manifest("mlm", 1, False)) extended = extend_params(init_params(mlm.model), cfg.model) - assert all( - torch.equal(extended[k], init_params(mlm.model)[k]) for k in init_params(mlm.model) - ) + # One independent re-init, not one per key: od-base init is ~3 s. + stage_a = init_params(mlm.model) + assert all(torch.equal(extended[k], stage_a[k]) for k in stage_a) pots = emulate( n, lambda c: honeypot_island(Honeypot(w.training.coord_pubkey, "honest"), w, c, theta, a, get), diff --git a/tests/miner/test_island_launch_v2.py b/tests/miner/test_island_launch_v2.py index dcd0fea..e3a3ebf 100644 --- a/tests/miner/test_island_launch_v2.py +++ b/tests/miner/test_island_launch_v2.py @@ -1071,7 +1071,7 @@ def test_real_od_two_rank_worker_paths(tmp_path: Path, objective: str) -> None: ef_in_sha256=hashlib.sha256(blobs["ef_in"]).hexdigest(), v0_sha256=hashlib.sha256(blobs["v0"]).hexdigest(), object_paths={k: k for k in blobs}, - deadline=int(time.time()) + 120, + deadline=int(time.time()) + 900, ) result = launch_island(job, tmp_path, backend="cpu", trace=True) refs = emulate( diff --git a/tests/models/od_fixtures.py b/tests/models/od_fixtures.py index dd74bd8..00a93c2 100644 --- a/tests/models/od_fixtures.py +++ b/tests/models/od_fixtures.py @@ -17,10 +17,11 @@ from hypertrain.trainer.rng import rng_ctr, uniform_f64 RUN_ID = "cd" * 32 -SEQ = 32 +SEQ = 32 # v1 training path; check_manifest (A6) pins od-base mlm to N = 1024 +N = 1024 H, J, MB = 3, 1, 2 REC = {"state_len": 16, "n_questions": 2, "n_options": 3, "opt_len": 4, "instr_len": 4} -STAGE_A_COUNT, FULL_COUNT = 116_288, 187_140 +STAGE_A_COUNT, FULL_COUNT = 123_753_984, 143_241_220 def _l2_od_body() -> dict[str, Any]: @@ -32,14 +33,16 @@ def _l2_od_body() -> dict[str, Any]: return mod.od_body() # type: ignore[no-any-return] -def od_body(objective: str = "mlm", profile: str = "od-fp32-ref-v1") -> dict[str, Any]: +def od_body( + objective: str = "mlm", profile: str = "od-fp32-ref-v1", seq_len: int = SEQ +) -> dict[str, Any]: b = copy.deepcopy(_l2_od_body()) b["reference_spec"]["profile"] = profile b["model"]["compute_dtype"] = "fp32" if profile == "od-fp32-ref-v1" else "bf16" b["inner"] |= {"H": H, "J": J, "micro_batch": MB, "grad_accum": 1} b["inner"]["lr_schedule"] |= {"peak_lr": f32hex(1e-3), "warmup": 1} if objective == "mlm": - b["model"] |= {"seq_len": SEQ, "param_count": STAGE_A_COUNT} + b["model"] |= {"seq_len": seq_len, "param_count": STAGE_A_COUNT} else: shape = RecordShape(**REC) b["model"] |= {"seq_len": shape.length - 1, "param_count": FULL_COUNT} @@ -47,8 +50,10 @@ def od_body(objective: str = "mlm", profile: str = "od-fp32-ref-v1") -> dict[str return b -def cfg_of(objective: str = "mlm", profile: str = "od-fp32-ref-v1") -> TrainConfig: - return TrainConfig.from_manifest(od_body(objective, profile)) +def cfg_of( + objective: str = "mlm", profile: str = "od-fp32-ref-v1", seq_len: int = SEQ +) -> TrainConfig: + return TrainConfig.from_manifest(od_body(objective, profile, seq_len)) def text_sample(cfg: TrainConfig) -> SampleFn: diff --git a/tests/models/test_opendecision.py b/tests/models/test_opendecision.py index 06627e2..97d781c 100644 --- a/tests/models/test_opendecision.py +++ b/tests/models/test_opendecision.py @@ -14,6 +14,7 @@ from od_fixtures import ( FULL_COUNT, STAGE_A_COUNT, + N, assignment, cfg_of, fresh_carry, @@ -201,7 +202,8 @@ def test_stage_a_param_set_excludes_head() -> None: assert sf == {k: tuple(v.shape) for k, v in sd.items()} assert sorted(od.init_params(a)) == sorted(sa) assert od.stage_of("tok.weight", a, 2) == 0 - assert od.stage_of("encoder.blocks.1.qkv.weight", a, 2) == 1 + assert od.stage_of("encoder.blocks.5.qkv.weight", a, 2) == 0 + assert od.stage_of("encoder.blocks.6.qkv.weight", a, 2) == 1 assert od.stage_of("scorer.3.bias", full, 2) == 1 @@ -262,16 +264,16 @@ def test_extend_params_and_warmstart_cli(tmp_path: Path) -> None: def test_check_manifest_param_count() -> None: - od.check_manifest(RunManifest.model_validate(od_body())) + od.check_manifest(RunManifest.model_validate(od_body(seq_len=N))) od.check_manifest(RunManifest.model_validate(od_body("decision"))) - b = od_body() + b = od_body(seq_len=N) b["model"]["param_count"] = FULL_COUNT with pytest.raises(ValueError, match="param_count"): od.check_manifest(RunManifest.model_validate(b)) def test_eval_holdout_and_eval_cli(tmp_path: Path) -> None: - cfg = cfg_of() + cfg = cfg_of(seq_len=N) get = text_sample(cfg) theta = od.init_params(cfg.model) v = od.eval_holdout(cfg.model, theta, get, [0, 1, 2], batch=2) @@ -285,10 +287,10 @@ def test_eval_holdout_and_eval_cli(tmp_path: Path) -> None: write_shards16((get(i) for i in range(8)), hold, cfg.model.seq_len, unit=4, samples_per_shard=8) state, man, metrics = tmp_path / "s", tmp_path / "run.json", tmp_path / "public" / "m.jsonl" state.write_bytes(pack_state(theta)) - man.write_text(json.dumps(od_body())) + man.write_text(json.dumps(od_body(seq_len=N))) args = ["eval", "--manifest", str(man), "--state", str(state), "--holdout", str(hold)] assert main([*args, "--w", "4", "--metrics", str(metrics)]) == 0 line = json.loads(metrics.read_text()) - run_id = RunManifest.model_validate(od_body()).run_id() + run_id = RunManifest.model_validate(od_body(seq_len=N)).run_id() want = od.eval_holdout(cfg.model, theta, get, range(8)) assert line == {"run_id": run_id, "w": 4, "value": want} diff --git a/tests/protocol/test_network_v2.py b/tests/protocol/test_network_v2.py index 1ef100e..340e80b 100644 --- a/tests/protocol/test_network_v2.py +++ b/tests/protocol/test_network_v2.py @@ -131,13 +131,13 @@ def commit(tokens: int = 7680) -> CommitV2: def test_od_general_n_when_v2(n: int, objective: str, zero1: bool) -> None: # Given body = od_body() - body["model"]["param_count"] = 116_288 + body["model"]["param_count"] = 123_753_984 body["inner"].update(H=1, J=1, micro_batch=1, grad_accum=1) body["reference_spec"]["layout"].update(n_gpus=n, dp_size=n, ep_size=1, zero1=zero1) if objective != "mlm": rec = {"state_len": 8, "n_questions": 1, "n_options": 2, "opt_len": 2, "instr_len": 2} body["model"]["od"].update(objective=objective, record=rec) - body["model"].update(seq_len=18, param_count=187_140) + body["model"].update(seq_len=18, param_count=143_241_220) # When model = wrapper(body) # Then @@ -152,7 +152,7 @@ def test_od_general_n_when_v2(n: int, objective: str, zero1: bool) -> None: def test_assign_unit_when_only_island_batch_is_divisible() -> None: # Given body = od_body() - body["model"]["param_count"] = 116_288 + body["model"]["param_count"] = 123_753_984 body["inner"].update(H=1, J=1, micro_batch=1, grad_accum=1) body["reference_spec"]["layout"].update(n_gpus=8, dp_size=8) body["dataset"].update(assign_unit=8, unit_sha256_root=H, n_samples=4096) @@ -1026,7 +1026,7 @@ def test_aggregation_when_computational_f32_is_nonfinite(field: str) -> None: @pytest.mark.parametrize("field", ["mask_ratio", "rps_weight", "distill_temp"]) def test_od_when_objective_f32_is_nonfinite(field: str) -> None: body = od_body() # Given - body["model"]["param_count"] = 116_288 + body["model"]["param_count"] = 123_753_984 body["model"]["od"][field] = "0000c07f" with pytest.raises(ValidationError): # When / Then wrapper(body)